diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index 8b8bb440d..8fbdc654d 100755 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -301,18 +301,6 @@ EOF fi echo "$conf" > $HOME/seekdb/tools/deploy/config.yaml - if [[ $WITH_SHARED_STORAGE == "1" ]] - then - # shared_storage 需要在global修改相关的mode - share_storage_workdir="$(date +%s)-$(uuidgen)" - first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` - set +x - sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ - shared_storage: \"oss://oss-436751-0701-all-test/$FARM2_RUN_USER/$share_storage_workdir?host=oss-cn-hangzhou.aliyuncs.com&access_id=${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}&access_key=${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}&max_iops=0&max_bandwidth=0B&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml - sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/seekdb/tools/deploy/config.yaml - set -x - fi - if [[ $SHARED_STORAGE_MODE == "1" ]] then # shared_storage 需要在global修改相关的mode @@ -327,10 +315,7 @@ EOF shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml fi - if [[ ! $WITH_SHARED_STORAGE == "1" ]] - then - cat $HOME/seekdb/tools/deploy/config.yaml - fi + cat $HOME/seekdb/tools/deploy/config.yaml } function obd_prepare_bin { @@ -650,26 +635,6 @@ function obd_run_mysqltest { mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX echo "finish!" return $ret - elif [[ $WITH_SHARED_STORAGE == "1" ]] - then - # 把sensitive_test目录下的测试集合移动到tools/deploy/mysql_test - # 并且创建一个新的目录用来表示单独的测试 - if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] - then - return 0 - fi - need_mv_dirs=`ls -1 $HOME/seekdb/sensitive_test/mysql_test/test_suite|xargs` - run_suites="" - for need_mv_dir in $need_mv_dirs - do - new_dir=$HOME/seekdb/tools/deploy/mysql_test/test_suite/shared_storage__${need_mv_dir} - run_suites="${run_suites}shared_storage__${need_mv_dir}," - mkdir -p $new_dir - cp -r $HOME/seekdb/sensitive_test/mysql_test/test_suite/$need_mv_dir/* $new_dir - done - run_suites=`echo $run_suites | awk '{print substr($0, 1, length($0)-1)}'` - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=$run_suites" - $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) else if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] then @@ -709,12 +674,6 @@ function obd_collect_log { mkdir -p collected_log/mysqltest_rec_log find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ - set +x - if [[ $WITH_SHARED_STORAGE == "1" ]] - then - sed -i "s/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}//" collected_log/obd_log/obd - fi - set -x mv seekdb/tools/deploy/var/log/* collected_log/mysqltest_log/ mv seekdb/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ mv collected_log collected_log_$SLICE_IDX @@ -751,14 +710,6 @@ function run { ([[ -f $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || ([[ -f $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) then - if [[ $WITH_SHARED_STORAGE == "1" ]] - then - # 单独判断下是否有对应的case集合 - if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] - then - return 0 - fi - fi timeout=18000 [[ "$SPECIAL_RUN" == "1" ]] && timeout=72000 obd_prepare_env && @@ -796,4 +747,4 @@ function run { } -run "$@" \ No newline at end of file +run "$@" diff --git a/deps/oblib/src/common/meta_programming/ob_type_traits.h b/deps/oblib/src/common/meta_programming/ob_type_traits.h index 637229aa4..e419f2e7e 100644 --- a/deps/oblib/src/common/meta_programming/ob_type_traits.h +++ b/deps/oblib/src/common/meta_programming/ob_type_traits.h @@ -36,10 +36,6 @@ namespace mds class BufferCtx; } } -namespace transaction -{ - class ObMulSourceDataNotifyArg; -} namespace common { namespace meta @@ -139,7 +135,6 @@ REGISTER_FUNCTION_TRAIT(compare) REGISTER_FUNCTION_TRAIT(swap) REGISTER_FUNCTION_TRAIT(check_can_do_tx_end) REGISTER_FUNCTION_TRAIT(check_can_replay_commit) -REGISTER_FUNCTION_TRAIT(on_commit_for_old_mds) REGISTER_FUNCTION_TRAIT(mds_serialize) REGISTER_FUNCTION_TRAIT(mds_deserialize) REGISTER_FUNCTION_TRAIT(mds_get_serialize_size) @@ -333,13 +328,6 @@ typename std::enable_if::type #define ENABLE_IF_NOT_HAS_CHECK_CAN_REPLAY_COMMIT(CLASS) \ typename std::enable_if::type = true -#define OB_TRAIT_MDS_COMMIT_FOR_OLD_MDS(CLASS) \ -::oceanbase::common::meta::has_on_commit_for_old_mds::value -#define ENABLE_IF_MDS_COMMIT_FOR_OLD_MDS(CLASS) \ -typename std::enable_if::type = true -#define ENABLE_IF_NOT_MDS_COMMIT_FOR_OLD_MDS(CLASS) \ -typename std::enable_if::type = true - #define OB_TRAIT_IS_SAME_CLASS(CLASS, DEST_CLASS) \ std::is_same::value #define ENABLE_IF_IS_SAME_CLASS(CLASS, DEST_CLASS) \ @@ -383,18 +371,6 @@ struct MdsCheckCanReplayWrapper { } }; -template -struct MdsCommitForOldMdsWrapper { - template - static int on_commit_for_old_mds(const char *buf, const int64_t buf_len, const transaction::ObMulSourceDataNotifyArg &arg) { - return T::on_commit_for_old_mds(buf, buf_len, arg); - } - template - static int on_commit_for_old_mds(const char *, const int64_t, const transaction::ObMulSourceDataNotifyArg &) { - return OB_SUCCESS; - } -}; - } } } diff --git a/deps/oblib/src/common/ob_common_types.h b/deps/oblib/src/common/ob_common_types.h index 37e120796..d439dba65 100644 --- a/deps/oblib/src/common/ob_common_types.h +++ b/deps/oblib/src/common/ob_common_types.h @@ -62,10 +62,9 @@ struct ObQueryFlag #define OBSF_BIT_IS_SELECT_FOLLOWER 1 #define OBSF_BIT_ENABLE_LOB_PREFETCH 1 #define OBSF_BIT_IS_BARE_ROW_SCAN 1 -#define OBSF_BIT_MR_MV_SCAN 2 -#define OBSF_BIT_IS_PLAIN_INSERT 1 +#define OBSF_BIT_SNAPSHOT_OPT 1 #define OBSF_BIT_SKIP_RUNNING_TX 1 -#define OBSF_BIT_RESERVED 19 +#define OBSF_BIT_RESERVED 21 static const uint64_t OBSF_MASK_SCAN_ORDER = (0x1UL << OBSF_BIT_SCAN_ORDER) - 1; static const uint64_t OBSF_MASK_DAILY_MERGE = (0x1UL << OBSF_BIT_DAILY_MERGE) - 1; @@ -96,7 +95,7 @@ struct ObQueryFlag static const uint64_t OBSF_MASK_IS_SELECT_FOLLOWER = (0x1UL << OBSF_BIT_IS_SELECT_FOLLOWER) - 1; static const uint64_t OBSF_MASK_ENABLE_LOB_PREFETCH = (0x1UL << OBSF_BIT_ENABLE_LOB_PREFETCH) - 1; static const uint64_t OBSF_MASK_IS_DIRECT_SCAN = (0x1UL << OBSF_BIT_IS_BARE_ROW_SCAN) - 1; - static const uint64_t OBSF_MASK_IS_PLAIN_INSERT = (0x1UL << OBSF_BIT_IS_PLAIN_INSERT) - 1; + static const uint64_t OBSF_MASK_SNAPSHOT_OPT = (0x1UL << OBSF_BIT_SNAPSHOT_OPT) - 1; static const uint64_t OBSF_MASK_SKIP_RUNNING_TX = (0x1UL << OBSF_BIT_SKIP_RUNNING_TX) - 1; @@ -127,12 +126,6 @@ struct ObQueryFlag ReservedMode = 2, }; - enum MRMVScanMode - { - NormalMode = 0, - RefreshMode = 1, - RealTimeMode = 2, - }; union { uint64_t flag_; @@ -173,8 +166,7 @@ struct ObQueryFlag uint64_t is_mds_query_ : OBSF_BIT_IS_MDS_QUERY; uint64_t enable_lob_prefetch_ : OBSF_BIT_ENABLE_LOB_PREFETCH; uint64_t is_bare_row_scan_ : OBSF_BIT_IS_BARE_ROW_SCAN; // 1: to scan mult version row directly without compact. - uint64_t mr_mv_scan_ : OBSF_BIT_MR_MV_SCAN; // 0: normal table scan. 1. major refresh mview base table scan in refresh 2. major refresh rt-mview base table scan - uint64_t is_plain_insert_gts_opt_ : OBSF_BIT_IS_PLAIN_INSERT; + uint64_t use_snapshot_opt_ : OBSF_BIT_SNAPSHOT_OPT; uint64_t skip_running_tx_ : OBSF_BIT_SKIP_RUNNING_TX; // 1: skip RUNNING transactions (e.g., for fork operations) uint64_t reserved_ : OBSF_BIT_RESERVED; }; @@ -274,8 +266,8 @@ struct ObQueryFlag inline bool is_select_follower() const { return is_select_follower_; } inline void set_enable_lob_prefetch() { enable_lob_prefetch_ = true; } inline bool enable_lob_prefetch() const { return enable_lob_prefetch_; } - inline void set_plain_insert_gts_opt() { is_plain_insert_gts_opt_ = true; } - inline bool is_plain_insert_gts_opt() const { return is_plain_insert_gts_opt_; } + inline void set_snapshot_opt() { use_snapshot_opt_ = true; } + inline bool is_snapshot_opt() const { return use_snapshot_opt_; } inline void disable_cache() { set_not_use_row_cache(); @@ -285,10 +277,6 @@ struct ObQueryFlag } inline void set_is_new_query_range() { is_new_query_range_ = true; } inline bool is_new_query_range() const { return is_new_query_range_; } - inline bool is_mr_mview_refresh_base_scan() const { return RefreshMode == mr_mv_scan_; } - inline bool is_mr_rt_mview_base_scan() const { return RealTimeMode == mr_mv_scan_; } - inline bool is_mr_mview_query() const { return is_mr_mview_refresh_base_scan() || is_mr_rt_mview_base_scan(); } - TO_STRING_KV("scan_order", scan_order_, "daily_merge", daily_merge_, "rmmb_optimize", rmmb_optimize_, @@ -321,8 +309,7 @@ struct ObQueryFlag "is_select_follower", is_select_follower_, "enable_lob_prefetch", enable_lob_prefetch_, "is_bare_row_scan", is_bare_row_scan_, - "mr_mv_scan", mr_mv_scan_, - "is_plain_insert_gts_opt", is_plain_insert_gts_opt_, + "is_snapshot_opt", use_snapshot_opt_, "skip_running_tx", skip_running_tx_, "reserved", reserved_); OB_UNIS_VERSION(1); diff --git a/deps/oblib/src/common/object/ob_object.h b/deps/oblib/src/common/object/ob_object.h index fcc1faadc..99621d26e 100644 --- a/deps/oblib/src/common/object/ob_object.h +++ b/deps/oblib/src/common/object/ob_object.h @@ -922,13 +922,12 @@ struct ObMemLobReadSnapshot struct ObMemLobLocationInfo { ObMemLobLocationInfo(){} - ObMemLobLocationInfo(int64_t table_id, int64_t ls_id, ObCollationType collation_id) : - tablet_id_(table_id), ls_id_(ls_id), cid_(collation_id) + ObMemLobLocationInfo(int64_t table_id, ObCollationType collation_id) : + tablet_id_(table_id), cid_(collation_id) {} - TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(cid)); + TO_STRING_KV(K_(tablet_id), K_(cid)); int64_t tablet_id_; - int64_t ls_id_; ObCollationType cid_; // charset for dbmslob char data_[0]; }; diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index be36ce2e2..e2051e641 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -571,12 +571,6 @@ const int64_t OB_MAX_SERVER_SESSION_CNT = 32767; const int64_t OB_MAX_SERVER_TENANT_CNT = 5; // for sys,500,508,509 const int64_t OB_RECYCLE_MACRO_BLOCK_DURATION = 10 * 60 * 1000 * 1000LL; // 10 minutes const int64_t OB_MINOR_FREEZE_TEAM_UP_INTERVAL = 2LL * 60 * 60 * 1000 * 1000; // 2 hours -// for define -const int64_t OB_MAX_SPECIAL_LS_NUM = 1; // 1 for sys ls -const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET = 1024; // the maximum of _max_ls_cnt_per_server -const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER = (10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER) > OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET ? - OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET : 10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER)); // magnification is 10x -const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT = 1; // the tenant that smaller than 4G will be limit to 1 const int64_t OB_MAX_TIME = 3020399000000; // Max add partition member timeout. // Used to make sure no member added after lease expired + %OB_MAX_ADD_MEMBER_TIMEOUT @@ -714,28 +708,10 @@ const uint64_t OB_ACTION_FLAG_COLUMN_ID = OB_ALL_MAX_COLUMN_ID // internal compaction trans seq const int64_t DELETE_INSERT_TRANS_SEQUENCE = ((__INT64_C(1LL << 62)) + 1); -// materialized view log -const uint64_t OB_MLOG_SEQ_NO_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - - OB_END_RESERVED_COLUMN_ID_NUM + 2; /* 65521 */ -const uint64_t OB_MLOG_DML_TYPE_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - - OB_END_RESERVED_COLUMN_ID_NUM + 3; /* 65522 */ -const uint64_t OB_MLOG_OLD_NEW_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - - OB_END_RESERVED_COLUMN_ID_NUM + 4; /* 65523 */ -const uint64_t OB_MLOG_ROWID_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - - OB_END_RESERVED_COLUMN_ID_NUM + 5; /* 65524 */ -const uint64_t OB_MIN_MLOG_SPECIAL_COLUMN_ID = OB_MLOG_SEQ_NO_COLUMN_ID; -const uint64_t OB_MAX_MLOG_SPECIAL_COLUMN_ID = OB_MLOG_ROWID_COLUMN_ID; - -const char *const OB_MLOG_SEQ_NO_COLUMN_NAME = "SEQUENCE$$"; -const char *const OB_MLOG_DML_TYPE_COLUMN_NAME = "DMLTYPE$$"; -const char *const OB_MLOG_OLD_NEW_COLUMN_NAME = "OLD_NEW$$"; -const char *const OB_MLOG_ROWID_COLUMN_NAME = "M_ROW$$"; - const uint64_t OB_MAX_TMP_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - OB_END_RESERVED_COLUMN_ID_NUM; // pseudo column id used in special table scan const uint64_t OB_COUNT_AGG_PD_COLUMN_ID = INT32_MAX - 1; -const uint64_t OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID = INT32_MAX - 2; const int64_t OB_MAX_AUTOINC_SEQ_VALUE = (1LL << 40) - 1; // max value for 40bit @@ -753,10 +729,6 @@ const char *const OB_HIDDEN_LINE_NUMBER_COLUMN_NAME = "__line_number"; // used f // internal index prefix const char *const OB_INDEX_PREFIX = "__idx_"; -// internal materialized view log prefix -const char *const OB_MLOG_PREFIX_MYSQL = "mlog$_"; -const char *const OB_TMP_MLOG_PREFIX_MYSQL = "tmlog$_"; - // internal user const char *const OB_INTERNAL_USER = "__ob_server"; @@ -978,7 +950,6 @@ const int64_t MAX_ENABLED_ROLES = 148; // table id range definition // //////////////////////////////////////////////////////////// const uint64_t OB_MIN_GENERATED_COLUMN_ID = 2000; -const uint64_t OB_MIN_MV_COLUMN_ID = 10000; const uint64_t OB_MIN_SHADOW_COLUMN_ID = 32767; const uint64_t OB_MAX_SYS_POOL_ID = 100; @@ -1639,21 +1610,9 @@ OB_INLINE uint64_t combine_two_ids(uint64_t high_id, uint64_t low_id) const char *const OB_RANDOM_PRIMARY_ZONE = "RANDOM"; -OB_INLINE bool is_mlog_reference_column(const uint64_t column_id) -{ - return (common::OB_MLOG_ROWID_COLUMN_ID == column_id); -} - -OB_INLINE bool is_mlog_special_column(const uint64_t column_id) -{ - return (column_id >= common::OB_MIN_MLOG_SPECIAL_COLUMN_ID - && column_id <= common::OB_MAX_MLOG_SPECIAL_COLUMN_ID); -} - OB_INLINE bool is_shadow_column(const uint64_t column_id) { - return (column_id > common::OB_MIN_SHADOW_COLUMN_ID) - && !is_mlog_special_column(column_id); + return column_id > common::OB_MIN_SHADOW_COLUMN_ID; } OB_INLINE bool is_bootstrap_resource_pool(const uint64_t resource_pool_id) diff --git a/deps/oblib/src/lib/ob_name_id_def.h b/deps/oblib/src/lib/ob_name_id_def.h index 15335fd86..31984ee19 100644 --- a/deps/oblib/src/lib/ob_name_id_def.h +++ b/deps/oblib/src/lib/ob_name_id_def.h @@ -135,7 +135,7 @@ DEF_NAME(id, "id") DEF_NAME(start_trans, "start transaction") DEF_NAME(register_timeout_task, "register timeout task") DEF_NAME(unregister_timeout_task, "unregister timeout task") - DEF_NAME(start_part, "start participant") + DEF_NAME(start_part, "start write participant") DEF_NAME(trans_id, "transaction id") DEF_NAME(local_task_completed, "local task completed") DEF_NAME(remote_task_completed, "remote task completed") @@ -280,7 +280,6 @@ DEF_NAME(id, "id") DEF_NAME(handle_tx_commit_result, "handle tx commit result") DEF_NAME(exec_commit_cb, "exec commit callback") DEF_NAME(get_read_snapshot, "get read snapshot") - DEF_NAME(get_ls_read_snapshot, "get ls read snapshot") DEF_NAME(register_snapshot, "register snapshot to txn") DEF_NAME(unregister_snapshot, "register snapshot to txn") DEF_NAME(release_snapshot, "release snapshot") @@ -479,7 +478,7 @@ DEF_NAME(id, "id") DEF_NAME(logic_clock, "the logical clock") DEF_NAME(tx_free_route_update_static, "txn free route update static state") DEF_NAME(tx_free_route_update_dynamic, "txn free route update dynamic state") - DEF_NAME(tx_free_route_update_participants, "txn free route update participants state") + DEF_NAME(tx_free_route_update_write_participant, "txn free route update write participant state") DEF_NAME(tx_free_route_update_extra, "txn free route update extra state") DEF_NAME(tx_free_route_send_state, "txn free route send state") DEF_NAME(tx_free_route_recv_state, "txn free route receive state") @@ -722,7 +721,7 @@ DEF_NAME(id, "id") DEF_NAME_PAIR(parse_job, "parse job") DEF_NAME_PAIR(do_open_plan, "do open plan") DEF_NAME_PAIR(sql_start_stmt, "sql start stmt") - DEF_NAME_PAIR(sql_start_participant, "sql start participant") + DEF_NAME_PAIR(sql_start_participant, "sql start write participant") DEF_NAME_PAIR(kv_get, "kv get") DEF_NAME_PAIR(plc_get_from_cache, "plc get from cache") DEF_NAME_PAIR(plc_serialize, "plc serialize") @@ -731,7 +730,7 @@ DEF_NAME(id, "id") DEF_NAME_PAIR(exec_remote_plan, "exec remote plan") DEF_NAME_PAIR(exec_mini_plan, "exec mini plan") - DEF_NAME_PAIR(end_participant, "start_end_participant") + DEF_NAME_PAIR(end_participant, "start_end_write_participant") DEF_NAME_PAIR(close_plan, "start_close_plan") DEF_NAME_PAIR(auto_end_plan, "start_auto_end_plan") DEF_NAME_PAIR(storage_table_scan, "table scan") @@ -763,8 +762,8 @@ DEF_NAME(id, "id") DEF_NAME_PAIR(T_end_trans, "transaction: end transaction") DEF_NAME_PAIR(T_start_stmt, "transaction: start statement") DEF_NAME_PAIR(T_end_stmt, "transaction: end statement") - DEF_NAME_PAIR(T_start_part, "transaction: start participant") - DEF_NAME_PAIR(T_end_part, "transaction: end participant") + DEF_NAME_PAIR(T_start_part, "transaction: start write participant") + DEF_NAME_PAIR(T_end_part, "transaction: end write participant") DEF_NAME_PAIR(T_part_destroy, "transaction: part_trans_ctx destroy") DEF_NAME_PAIR(T_destroy_callback, "transaction: destroy callback") DEF_NAME_PAIR(E_analysis_plan, "executor: analysis plan") diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index f02f6b362..9343c3a25 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -71,7 +71,7 @@ STAT_EVENT_ADD_DEF(TRANS_ROLLBACK_TIME, "trans rollback time", ObStatClassIds::T STAT_EVENT_ADD_DEF(TRANS_TIMEOUT_COUNT, "trans timeout count", ObStatClassIds::TRANS, 30011, false, true, true) STAT_EVENT_ADD_DEF(TRANS_LOCAL_COUNT, "trans local trans count", ObStatClassIds::TRANS, 30012, false, true, true) STAT_EVENT_ADD_DEF(TRANS_DIST_COUNT, "trans distribute trans count", ObStatClassIds::TRANS, 30013, false, true, true) -STAT_EVENT_ADD_DEF(TRANS_READONLY_COUNT, "trans without participant count", ObStatClassIds::TRANS, 30017, false, true, true) +STAT_EVENT_ADD_DEF(TRANS_READONLY_COUNT, "trans without write participant count", ObStatClassIds::TRANS, 30017, false, true, true) STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_COUNT, "redo log replay count", ObStatClassIds::TRANS, 30018, false, true, true) STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_TIME, "redo log replay time", ObStatClassIds::TRANS, 30019, false, true, true) STAT_EVENT_ADD_DEF(PREPARE_LOG_REPLAY_COUNT, "prepare log replay count", ObStatClassIds::TRANS, 30020, false, true, true) @@ -223,8 +223,6 @@ STAT_EVENT_ADD_DEF(FUSE_ROW_CACHE_HIT, "fuse row cache hit", ObStatClassIds::CAC STAT_EVENT_ADD_DEF(FUSE_ROW_CACHE_MISS, "fuse row cache miss", ObStatClassIds::CACHE, 50034, true, true, true) STAT_EVENT_ADD_DEF(SCHEMA_CACHE_HIT, "schema cache hit", ObStatClassIds::CACHE, 50035, false, true, true) STAT_EVENT_ADD_DEF(SCHEMA_CACHE_MISS, "schema cache miss", ObStatClassIds::CACHE, 50036, false, true, true) -STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_HIT, "tablet ls cache hit", ObStatClassIds::CACHE, 50037, false, true, true) -STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_MISS, "tablet ls cache miss", ObStatClassIds::CACHE, 50038, false, true, true) // obsoleted part from INDEX_CLOG_CACHE_HIT to USER_TABLE_STAT_CACHE_MISS STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_HIT, "index clog cache hit", ObStatClassIds::CACHE, 50039, false, true, true) STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_MISS, "index clog cache miss", ObStatClassIds::CACHE, 50040, false, true, true) @@ -257,8 +255,6 @@ STAT_EVENT_ADD_DEF(OPT_SYSTEM_STAT_CACHE_MISS, "opt system stat cache miss", ObS STAT_EVENT_ADD_DEF(DATA_BLOCK_CACHE_MISS, "data block cache miss", ObStatClassIds::CACHE, 50067, true, true, true) STAT_EVENT_ADD_DEF(INDEX_BLOCK_CACHE_MISS, "index block cache miss", ObStatClassIds::CACHE, 50068, true, true, true) -STAT_EVENT_ADD_DEF(MULTI_VERSION_FUSE_ROW_CACHE_HIT, "multi version fuse row cache hit", ObStatClassIds::CACHE, 50069, true, true, true) -STAT_EVENT_ADD_DEF(MULTI_VERSION_FUSE_ROW_CACHE_MISS, "multi version fuse row cache miss", ObStatClassIds::CACHE, 50070, true, true, true) STAT_EVENT_ADD_DEF(BACKUP_INDEX_CACHE_HIT, "backup index cache hit", ObStatClassIds::CACHE, 50071, true, true, true) STAT_EVENT_ADD_DEF(BACKUP_INDEX_CACHE_MISS, "backup index cache miss", ObStatClassIds::CACHE, 50072, true, true, true) STAT_EVENT_ADD_DEF(BACKUP_META_CACHE_HIT, "backup meta cache hit", ObStatClassIds::CACHE, 50073, true, true, true) @@ -507,7 +503,6 @@ STAT_EVENT_SET_DEF(STANDBY_FETCH_LOG_BANDWIDTH_LIMIT, "standby fetch log bandwid // SQL // CACHE -STAT_EVENT_SET_DEF(TABLET_LS_CACHE_SIZE, "tablet ls cache size", ObStatClassIds::CACHE, 120001, false, true, true) STAT_EVENT_SET_DEF(OPT_TAB_STAT_CACHE_SIZE, "table stat cache size", ObStatClassIds::CACHE, 120002, false, true, true) STAT_EVENT_SET_DEF(OPT_TAB_COL_STAT_CACHE_SIZE, "table col stat cache size", ObStatClassIds::CACHE, 120003, false, true, true) STAT_EVENT_SET_DEF(INDEX_BLOCK_CACHE_SIZE, "index block cache size", ObStatClassIds::CACHE, 120004, false, true, true) diff --git a/deps/oblib/src/lib/trace/ob_trace_def.h b/deps/oblib/src/lib/trace/ob_trace_def.h index 008959975..b6d6e759d 100644 --- a/deps/oblib/src/lib/trace/ob_trace_def.h +++ b/deps/oblib/src/lib/trace/ob_trace_def.h @@ -187,7 +187,7 @@ FLT_DEF_TAG(sqc_resp_finish_ts, "sqc finish response qc timestamp") // transaction TODO shengle SET_TAG FLT_DEF_TAG(trans_id, "transaction identifier") FLT_DEF_TAG(trans_type, "transaction type : 1 is for sp trans and 2 is for dist trans") -FLT_DEF_TAG(participant_num, "total participant number") +FLT_DEF_TAG(write_participant_num, "total write participant number") FLT_DEF_TAG(trans_expired_time, "transaction expired time") FLT_DEF_TAG(trans_start_ts, "transaction start ts") FLT_DEF_TAG(session_id, "session id") diff --git a/deps/oblib/src/lib/utility/ob_tracepoint_def.h b/deps/oblib/src/lib/utility/ob_tracepoint_def.h index 0bf7587ce..ec21d8555 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint_def.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint_def.h @@ -243,7 +243,7 @@ GLOBAL_ERRSIM_POINT_DEF(233, EN_DML_DISABLE_RANDOM_RESHUFFLE, ""); GLOBAL_ERRSIM_POINT_DEF(234, EN_RESIZE_PHYSICAL_FILE_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(235, EN_ALLOCATE_RESIZE_MEMORY_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(236, EN_WRITE_SUPER_BLOCK_FAILED, ""); -GLOBAL_ERRSIM_POINT_DEF(237, EN_GC_FAILED_PARTICIPANTS, ""); +GLOBAL_ERRSIM_POINT_DEF(237, EN_GC_FAILED_WRITE_PARTICIPANT, ""); GLOBAL_ERRSIM_POINT_DEF(238, EN_SSL_INVITE_NODES_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(239, EN_ADD_TRIGGER_SKIP_MAP, ""); GLOBAL_ERRSIM_POINT_DEF(240, EN_DEL_TRIGGER_SKIP_MAP, ""); @@ -284,7 +284,7 @@ GLOBAL_ERRSIM_POINT_DEF(272, EN_CLOG_SW_OUT_OF_RANGE, ""); GLOBAL_ERRSIM_POINT_DEF(273, EN_DFC_FACTOR, ""); GLOBAL_ERRSIM_POINT_DEF(274, EN_LOGSERVICE_IO_TIMEOUT, ""); -GLOBAL_ERRSIM_POINT_DEF(275, EN_PARTICIPANTS_SIZE_OVERFLOW, ""); +GLOBAL_ERRSIM_POINT_DEF(275, EN_WRITE_PARTICIPANT_SIZE_OVERFLOW, ""); GLOBAL_ERRSIM_POINT_DEF(276, EN_UNDO_ACTIONS_SIZE_OVERFLOW, ""); GLOBAL_ERRSIM_POINT_DEF(277, EN_PART_PLUS_UNDO_OVERFLOW, ""); GLOBAL_ERRSIM_POINT_DEF(278, EN_HANDLE_PREPARE_MESSAGE_EAGAIN, ""); @@ -480,7 +480,6 @@ GLOBAL_ERRSIM_POINT_DEF(751, EN_SESSION_LEAK_COUNT_THRESHOLD, "used to control t GLOBAL_ERRSIM_POINT_DEF(752, EN_PL_MEMORY_ALLOCA_SWITCH, "used to control the strage of obplallocator memory alloc"); GLOBAL_ERRSIM_POINT_DEF(753, EN_PL_PACKAGE_ENCODE_SWITCH, "used to control the strage of package state encode"); GLOBAL_ERRSIM_POINT_DEF(761, EN_FAST_RECLAIM_THREAD, "used to speed up reclaiming thread"); -GLOBAL_ERRSIM_POINT_DEF(762, EN_MV_LARGE_SSTABLE_THRESHOLD, "used to check mv sstable minor merge"); GLOBAL_ERRSIM_POINT_DEF(763, EN_SHARED_STORAGE_MICRO_CACHE_WRITE_DISK_ERR, "ss_micro_cache write disk error"); GLOBAL_ERRSIM_POINT_DEF(800, EN_END_PARTICIPANT, ""); diff --git a/deps/oblib/src/rpc/ob_request.h b/deps/oblib/src/rpc/ob_request.h index ffc15809d..d0e827d19 100644 --- a/deps/oblib/src/rpc/ob_request.h +++ b/deps/oblib/src/rpc/ob_request.h @@ -49,7 +49,7 @@ class ObRequest: public common::ObLink { public: friend class ObSqlRequestOperator; - enum Type { OB_RPC, OB_MYSQL, OB_TASK, OB_TS_TASK, OB_SQL_TASK, OB_SQL_SOCK_TASK, OB_DAS_PARALLEL_TASK }; + enum Type { OB_RPC, OB_MYSQL, OB_TASK, OB_SQL_TASK, OB_SQL_SOCK_TASK, OB_DAS_PARALLEL_TASK }; enum TransportProto { TRANSPORT_PROTO_EASY = 0, TRANSPORT_PROTO_POC = 1, diff --git a/src/logservice/CMakeLists.txt b/src/logservice/CMakeLists.txt index b561718ae..bece57383 100644 --- a/src/logservice/CMakeLists.txt +++ b/src/logservice/CMakeLists.txt @@ -1,12 +1,9 @@ ob_set_subtarget(ob_logservice common - logrpc/ob_log_rpc_arg.cpp ob_tenant_mutil_allocator.cpp ob_tenant_mutil_allocator_mgr.cpp ob_append_callback.cpp - ob_location_adapter.cpp ob_log_base_header.cpp ob_log_handler.cpp - ob_log_handler_base.cpp ob_log_service.cpp ob_ls_adapter.cpp ob_server_log_block_mgr.cpp @@ -18,10 +15,7 @@ ob_set_subtarget(ob_logservice common ob_set_subtarget(ob_logservice common_mixed applyservice/ob_log_apply_service.cpp - logrpc/ob_log_request_handler.cpp - logrpc/ob_log_rpc_req.cpp - rcservice/ob_role_change_handler.cpp - rcservice/ob_role_change_service.cpp + localservice/ob_local_log_handler_set.cpp replayservice/ob_log_replay_service.cpp replayservice/ob_replay_handler.cpp replayservice/ob_replay_status.cpp @@ -30,13 +24,10 @@ ob_set_subtarget(ob_logservice common_mixed ob_set_subtarget(ob_logservice palf palf/block_gc_timer_task.cpp - palf/election_self.cpp - palf/fetch_log_engine.cpp palf/log_block_handler.cpp palf/log_block_header.cpp palf/log_block_mgr.cpp palf/log_checksum.cpp - palf/log_config_mgr.cpp palf/log_define.cpp palf/log_engine.cpp palf/log_entry.cpp @@ -51,20 +42,13 @@ ob_set_subtarget(ob_logservice palf palf/log_shared_task.cpp palf/log_io_worker.cpp palf/log_iterator_storage.cpp - palf/log_learner.cpp palf/log_loop_thread.cpp palf/log_meta.cpp palf/log_meta_entry.cpp palf/log_meta_entry_header.cpp palf/log_meta_info.cpp - palf/log_net_service.cpp palf/log_reader.cpp palf/log_reader_utils.cpp - palf/log_reconfirm.cpp - palf/log_req.cpp - palf/log_request_handler.cpp - palf/log_rpc.cpp - palf/log_simple_member_list.cpp palf/log_sliding_window.cpp palf/log_state_mgr.cpp palf/log_storage.cpp @@ -83,7 +67,6 @@ ob_set_subtarget(ob_logservice palf palf/log_block_pool_interface.cpp palf/palf_options.cpp palf/log_mode_mgr.cpp - palf/log_updater.cpp palf/log_io_utils.cpp palf/log_io_worker_wrapper.cpp palf/log_throttle.cpp @@ -93,7 +76,6 @@ ob_set_subtarget(ob_logservice palf ob_set_subtarget(ob_logservice common_util common_util/ob_log_time_utils.cpp - common_util/ob_log_ls_define.cpp ) diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp index f6a547c79..c340e0eb0 100644 --- a/src/logservice/applyservice/ob_log_apply_service.cpp +++ b/src/logservice/applyservice/ob_log_apply_service.cpp @@ -15,9 +15,10 @@ */ #include "ob_log_apply_service.h" +#include "logservice/ob_append_callback.h" #include "logservice/ob_ls_adapter.h" -#include "src/storage/tx_storage/ob_ls_map.h" - +#include "logservice/palf/palf_env.h" +#include "lib/stat/ob_diagnostic_info_guard.h" namespace oceanbase { using namespace common; @@ -47,13 +48,10 @@ void ObApplyFsCb::destroy() apply_status_ = NULL; } -int ObApplyFsCb::update_end_lsn(int64_t id, - const LSN &end_lsn, - const SCN &end_scn, - const int64_t proposal_id) +int ObApplyFsCb::update_end_lsn(const LSN &end_lsn, + const SCN &end_scn) { - UNUSED(id); - return apply_status_->update_palf_committed_end_lsn(end_lsn, end_scn, proposal_id); + return apply_status_->update_palf_committed_end_lsn(end_lsn, end_scn); } //---------------ObApplyServiceTask---------------// @@ -250,9 +248,7 @@ ObApplyStatus::ObApplyStatus() : is_inited_(false), is_in_stop_state_(true), ref_cnt_(0), - ls_id_(), - role_(FOLLOWER), - proposal_id_(-1), + accepting_append_(false), ap_sv_(NULL), palf_committed_end_lsn_(0), palf_committed_end_scn_(), @@ -279,31 +275,28 @@ ObApplyStatus::~ObApplyStatus() destroy(); } -int ObApplyStatus::init(const share::ObLSID &id, - palf::PalfEnv *palf_env, +int ObApplyStatus::init(palf::PalfEnv *palf_env, ObLogApplyService *ap_sv) { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; CLOG_LOG(WARN, "apply status has already been inited", K(ret)); - } else if (!id.is_valid() || OB_ISNULL(ap_sv)) { + } else if (OB_ISNULL(palf_env) || OB_ISNULL(ap_sv)) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(id), K(ap_sv), K(ret)); - } else if (OB_FAIL(palf_env->open(id.id(), palf_handle_))) { - CLOG_LOG(ERROR, "failed to open palf handle", K(palf_env), K(id)); + CLOG_LOG(WARN, "invalid argument", KP(palf_env), K(ap_sv), K(ret)); + } else if (OB_FAIL(palf_env->open(palf_handle_))) { + CLOG_LOG(ERROR, "failed to open palf handle", K(palf_env)); } else if (OB_FAIL(submit_task_.init(this))) { - CLOG_LOG(WARN, "failed to init submit_task", K(ret), K(id)); + CLOG_LOG(WARN, "failed to init submit_task", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < APPLY_TASK_QUEUE_SIZE; ++i) { if (OB_FAIL(cb_queues_[i].init(this, i))) { - CLOG_LOG(WARN, "failed to init cb_queues", K(ret), K(id)); + CLOG_LOG(WARN, "failed to init cb_queues", K(ret)); } } if (OB_SUCCESS == ret) { - ls_id_ = id; - role_ = FOLLOWER; - proposal_id_ = -1; + accepting_append_ = false; ap_sv_ = ap_sv; palf_env_ = palf_env; last_check_scn_.reset(); @@ -313,7 +306,7 @@ int ObApplyStatus::init(const share::ObLSID &id, IGNORE_RETURN new (&fs_cb_) ObApplyFsCb(this); is_in_stop_state_ = false; if (OB_FAIL(palf_handle_.register_file_size_cb(&fs_cb_))) { - CLOG_LOG(ERROR, "failed to register cb", K(ret), K(id)); + CLOG_LOG(ERROR, "failed to register cb", K(ret)); } else { is_inited_ = true; CLOG_LOG(INFO, "apply status init success", K(ret), KPC(this), KP(&cb_append_stat_), @@ -333,8 +326,7 @@ void ObApplyStatus::destroy() close_palf_handle(); is_inited_ = false; is_in_stop_state_ = true; - proposal_id_ = -1; - role_ = FOLLOWER; + accepting_append_ = false; fs_cb_.destroy(); palf_committed_end_scn_.reset(); palf_committed_end_lsn_.reset(); @@ -344,7 +336,6 @@ void ObApplyStatus::destroy() try_wrlock_debug_time_ = OB_INVALID_TIMESTAMP; palf_env_ = NULL; ap_sv_ = NULL; - ls_id_.reset(); submit_task_.reset(); for (int64_t i = 0; i < APPLY_TASK_QUEUE_SIZE; ++i) { cb_queues_[i].reset(); @@ -361,7 +352,7 @@ int ObApplyStatus::stop() if (OB_SUCC(lock_.wrlock(abs_timeout_us))) { is_in_stop_state_ = true; close_palf_handle(); - switch_to_follower_(); + stop_local_append_(); lock_.unlock(); } else { ret = OB_EAGAIN; @@ -493,8 +484,8 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, cb_first_handle_time, cb_start_time, idx); cb_queue->inc_total_apply_cb_cnt(); } - } else if (FOLLOWER == role_) { - // Callbacks greater than the confirmed log position should be called back with on_failure when applystatus switches to follower + } else if (!accepting_append_) { + // Fail callbacks that remain beyond the confirmed position after local append stops. if (OB_FAIL(cb_queue->pop())) { CLOG_LOG(ERROR, "cb_queue pop failed", KPC(cb_queue), KPC(this), K(ret)); } else { @@ -512,7 +503,7 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, } else { cb->set_cb_first_handle_ts(ObTimeUtility::fast_current_time()); CLOG_LOG(TRACE, "cb on_wait", K(lsn), K(cb->__get_scn()), KPC(cb_queue), KPC(this)); - // Wait for log position advancement or role switch + // Wait for log position advancement or local append shutdown. ret = OB_EAGAIN; } if (OB_SUCC(ret) && !is_queue_empty) { @@ -542,7 +533,7 @@ int ObApplyStatus::is_apply_done(bool &is_done, is_done = true; for (int64_t i = 0; OB_SUCC(ret) && is_done && i < APPLY_TASK_QUEUE_SIZE; ++i) { if (OB_FAIL(cb_queues_[i].is_apply_done(is_done))) { - CLOG_LOG(WARN, "check is_apply_done failed", K(ret), K(ls_id_), K(i)); + CLOG_LOG(WARN, "check is_apply_done failed", K(ret), K(i)); } } if (is_done) { @@ -552,7 +543,7 @@ int ObApplyStatus::is_apply_done(bool &is_done, return ret; } -int ObApplyStatus::switch_to_leader(const int64_t new_proposal_id) +int ObApplyStatus::start_local_append() { int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(lock_, WRLOCK_RETRY_INTERVAL_US, WRLOCK_RETRY_INTERVAL_US); @@ -562,55 +553,49 @@ int ObApplyStatus::switch_to_leader(const int64_t new_proposal_id) } else if (is_in_stop_state_) { ret = OB_NOT_RUNNING; CLOG_LOG(INFO, "apply status has been stopped"); - } else if (new_proposal_id < ATOMIC_LOAD(&proposal_id_)) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid proposal id", K(ret), K(new_proposal_id), KPC(this)); - } else if (LEADER == role_) { + } else if (accepting_append_) { ret = OB_STATE_NOT_MATCH; - CLOG_LOG(WARN, "apply status has already been leader", KPC(this)); + CLOG_LOG(WARN, "local append has already started", KPC(this)); } else { - ATOMIC_STORE(&proposal_id_, new_proposal_id); - role_ = LEADER; - CLOG_LOG(INFO, "apply status switch_to_leader success", KPC(this)); + accepting_append_ = true; + CLOG_LOG(INFO, "apply status start_local_append success", KPC(this)); } return ret; } -int ObApplyStatus::switch_to_follower() +int ObApplyStatus::stop_local_append() { int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(lock_, WRLOCK_RETRY_INTERVAL_US, WRLOCK_RETRY_INTERVAL_US); - if (OB_FAIL(switch_to_follower_())) { - CLOG_LOG(WARN, "ObApplyStatus switch_to_follower_ failed", K(ret)); + if (OB_FAIL(stop_local_append_())) { + CLOG_LOG(WARN, "ObApplyStatus stop_local_append_ failed", K(ret)); } return ret; } //Needs lock protection -int ObApplyStatus::switch_to_follower_() +int ObApplyStatus::stop_local_append_() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply status has not been inited"); - } else if (FOLLOWER == role_) { - CLOG_LOG(INFO, "apply status has already been follower", KPC(this), K(ret)); + } else if (!accepting_append_) { + CLOG_LOG(INFO, "local append has already stopped", KPC(this), K(ret)); } else { lib::ObMutexGuard guard(mutex_); - // truncate scenario the old leader's max_scn may roll back last_check_scn_.reset(); - role_ = FOLLOWER; + accepting_append_ = false; if (OB_FAIL(submit_task_to_apply_service_(submit_task_))) { CLOG_LOG(ERROR, "submit_task_to_apply_service_ failed", KPC(this), K(ret)); } else { - CLOG_LOG(INFO, "switch_to_follower submit_task_to_apply_service_ success", KPC(this), K(ret)); + CLOG_LOG(INFO, "stop_local_append submit_task_to_apply_service_ success", KPC(this), K(ret)); } } return ret; } //Single-threaded call int ObApplyStatus::update_palf_committed_end_lsn(const palf::LSN &end_lsn, - const SCN &end_scn, - const int64_t proposal_id) + const SCN &end_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(IS_NOT_INIT)) { @@ -619,31 +604,25 @@ int ObApplyStatus::update_palf_committed_end_lsn(const palf::LSN &end_lsn, } else { { RLockGuard guard(lock_); - int64_t curr_proposal_id = ATOMIC_LOAD(&proposal_id_); if (is_in_stop_state_) { //skip CLOG_LOG(WARN, "apply status has been stopped", K(ret), KPC(this)); - } else if (proposal_id == curr_proposal_id && LEADER == role_) { + } else if (accepting_append_) { if (palf_committed_end_lsn_ >= end_lsn) { - CLOG_LOG(ERROR, "invalid new end_lsn", KPC(this), K(proposal_id), K(end_lsn)); + CLOG_LOG(ERROR, "invalid new end_lsn", KPC(this), K(end_lsn)); } else { palf_committed_end_scn_.atomic_store(end_scn); palf_committed_end_lsn_ = end_lsn; if (OB_FAIL(submit_task_to_apply_service_(submit_task_))) { - CLOG_LOG(ERROR, "submit_task_to_apply_service_ failed", KPC(this), K(ret), K(proposal_id), K(end_lsn)); + CLOG_LOG(ERROR, "submit_task_to_apply_service_ failed", KPC(this), K(ret), K(end_lsn)); } } - } else if ((proposal_id == curr_proposal_id && FOLLOWER == role_) - || proposal_id < curr_proposal_id) { - // After apply switches to follower, logs with the same proposal_id should not be able to slide out - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "invalid new end_lsn", KPC(this), K(proposal_id), K(end_lsn)); } else { - CLOG_LOG(TRACE, "update_palf_committed_end_lsn skip", KPC(this), K(proposal_id), K(end_lsn)); - // palf has already done a leader-follower switch, skip + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "invalid new end_lsn", KPC(this), K(end_lsn)); } } - CLOG_LOG(TRACE, "update_palf_committed_end_lsn", KPC(this), K(proposal_id), K(end_lsn), KR(ret)); + CLOG_LOG(TRACE, "update_palf_committed_end_lsn", KPC(this), K(end_lsn), KR(ret)); } return ret; } @@ -659,7 +638,7 @@ int ObApplyStatus::unregister_file_size_cb() int ret = OB_SUCCESS; if (palf_handle_.is_valid()) { if (OB_FAIL(palf_handle_.unregister_file_size_cb())) { - CLOG_LOG(ERROR, "failed to unregister cb", K(ret), K(ls_id_)); + CLOG_LOG(ERROR, "failed to unregister cb", K(ret)); } else { // do nothing } @@ -687,13 +666,8 @@ int ObApplyStatus::get_max_applied_scn(SCN &scn) ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply status has not been inited", K(ret)); } else if (OB_UNLIKELY(is_in_stop_state_)) { - // After stopping, it will no longer take office, and will always return the cached value from the last round as leader - } else if (FOLLOWER == role_) { - //The max_applied_cb_scn_ undergoes asynchronous updating, and under circumstances where a - //transiting to a follower role, there exists a possibility that its recorded value might underestimate the actual one. - //Upon a log stream replica's shift from the leader role to a follower role, it guarantees the - //application of every log entry that has been confirmed. Thus, while in the follower phase, the - //value of max_applied_cb_scn_ can be securely incremented to match palf_committed_end_scn_. + // After stopping, return the last cached callback point. + } else if (!accepting_append_) { palf::LSN apply_end_lsn; bool is_done = false; const SCN cur_palf_committed_end_scn = palf_committed_end_scn_.atomic_load(); @@ -702,8 +676,7 @@ int ObApplyStatus::get_max_applied_scn(SCN &scn) } else if (OB_FAIL(is_apply_done(is_done, apply_end_lsn))) { CLOG_LOG(WARN, "check is_apply_done failed", K(ret), KPC(this)); } else if (!is_done) { - // During the follower period, do not make any updates until cb is fully called back - // Always return the value cached from the last round when it was the leader + // Do not update until all callbacks are complete. // All cb callbacks completed, attempt to advance the maximum consecutive callback point once } else if (max_applied_cb_scn_ < cur_palf_committed_end_scn) { max_applied_cb_scn_ = cur_palf_committed_end_scn; @@ -720,7 +693,7 @@ int ObApplyStatus::get_max_applied_scn(SCN &scn) bool is_done = true; for (int64_t i = 0; OB_SUCC(ret) && is_done && i < APPLY_TASK_QUEUE_SIZE; ++i) { if (OB_FAIL(cb_queues_[i].is_snapshot_apply_done(is_done))) { - CLOG_LOG(WARN, "check is_snapshot_apply_done failed", K(ret), K(ls_id_), K(i)); + CLOG_LOG(WARN, "check is_snapshot_apply_done failed", K(ret), K(i)); } } if (OB_SUCC(ret) && is_done) { @@ -751,9 +724,6 @@ int ObApplyStatus::stat(LSApplyStat &stat) const if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - stat.ls_id_ = ls_id_.id(); - stat.role_ = role_; - stat.proposal_id_ = proposal_id_; stat.end_lsn_ = palf_committed_end_lsn_; stat.pending_cnt_ = 0; for (int64_t i = 0; OB_SUCC(ret) && i < APPLY_TASK_QUEUE_SIZE; ++i) { @@ -787,12 +757,6 @@ int ObApplyStatus::diagnose(ApplyDiagnoseInfo &diagnose_info) return ret; } -void ObApplyStatus::reset_proposal_id() -{ - ATOMIC_STORE(&proposal_id_, -1); - CLOG_LOG(INFO, "reset_proposal_id success"); -} - void ObApplyStatus::reset_meta() { RLockGuard rlock(lock_); @@ -827,29 +791,15 @@ int ObApplyStatus::submit_task_to_apply_service_(ObApplyServiceTask &task) int ObApplyStatus::update_last_check_scn_() { int ret = OB_SUCCESS; - ObRole palf_role = FOLLOWER; - int64_t curr_proposal_id = ATOMIC_LOAD(&proposal_id_); - int64_t palf_proposal_id = -1; - bool is_pending_state = true; SCN palf_max_scn; if (OB_FAIL(palf_handle_.get_max_scn(palf_max_scn))) { - CLOG_LOG(WARN, "get_max_scn failed", K(ret), K(ls_id_)); - } else if (OB_FAIL(palf_handle_.get_role(palf_role, palf_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "palf get_role failed", K(ret), K(ls_id_)); + CLOG_LOG(WARN, "get_max_scn failed", K(ret)); } else if (max_applied_cb_scn_.is_valid() && palf_max_scn < max_applied_cb_scn_) { //Defensive check, palf's max_scn should not regress to before the agreed max_applied_cb_scn_ ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "invalid palf_max_scn", K(ret), K(ls_id_), K(palf_max_scn), KPC(this)); - } else if ((palf_proposal_id != curr_proposal_id) || (FOLLOWER == palf_role)) { - if (palf_proposal_id < proposal_id_) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "invalid palf_proposal_id", K(ret), K(ls_id_), K(palf_proposal_id), KPC(this)); - } else { - // palf has switched leader, max_scn may contain logs from the new leader, do nothing - CLOG_LOG(TRACE, "skip update_last_check_scn", K(ret), K(ls_id_), K(palf_max_scn), K(palf_proposal_id), KPC(this)); - } - } else if (OB_FAIL(ap_sv_->wait_append_sync(ls_id_))) { - CLOG_LOG(WARN, "wait_append_sync failed", K(ret), K(ls_id_), KPC(this)); + CLOG_LOG(ERROR, "invalid palf_max_scn", K(ret), K(palf_max_scn), KPC(this)); + } else if (OB_FAIL(ap_sv_->wait_append_sync())) { + CLOG_LOG(WARN, "wait_append_sync failed", K(ret), KPC(this)); } else { for (int64_t i = 0; i < APPLY_TASK_QUEUE_SIZE; ++i) { cb_queues_[i].set_snapshot_check_submit_cb_cnt(); @@ -976,64 +926,13 @@ void ObApplyStatusGuard::set_apply_status_(ObApplyStatus *apply_status) } //---------------ObLogApplyService---------------// -bool ObLogApplyService::GetApplyStatusFunctor::operator()(const share::ObLSID &id, - ObApplyStatus *apply_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(apply_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is NULL", K(id), KR(ret)); - } else { - guard_.set_apply_status_(apply_status); - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - -bool ObLogApplyService::RemoveApplyStatusFunctor::operator()(const share::ObLSID &id, - ObApplyStatus *apply_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(apply_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is NULL", K(id), KR(ret)); - } else if (OB_FAIL(apply_status->unregister_file_size_cb())) { - CLOG_LOG(ERROR, "apply_status unregister_file_size_cb failed", K(ret), K(id)); - } else if (0 == apply_status->dec_ref()) { - CLOG_LOG(INFO, "free apply status", KPC(apply_status)); - apply_status->~ObApplyStatus(); - mtl_free(apply_status); - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - -bool ObLogApplyService::ResetApplyStatusFunctor::operator()(const share::ObLSID &id, - ObApplyStatus *apply_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(apply_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is NULL", K(id), KR(ret)); - } else if (OB_FAIL(apply_status->handle_drop_cb())) { - CLOG_LOG(ERROR, "apply_status handle_drop_cb failed", K(ret), K(id)); - } else if (OB_FAIL(apply_status->unregister_file_size_cb())) { - CLOG_LOG(ERROR, "apply_status unregister_file_size_cb failed", K(ret), K(id)); - } else if (0 == apply_status->dec_ref()) { - CLOG_LOG(INFO, "free apply status", KPC(apply_status)); - apply_status->~ObApplyStatus(); - mtl_free(apply_status); - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - ObLogApplyService::ObLogApplyService() : is_inited_(false), is_running_(false), palf_env_(NULL), ls_adapter_(NULL), - apply_status_map_() + apply_status_(NULL), + lock_() {} ObLogApplyService::~ObLogApplyService() @@ -1054,13 +953,13 @@ int ObLogApplyService::init(PalfEnv *palf_env, CLOG_LOG(WARN, "invalid argument", K(ret), KP(palf_env), K(ls_adapter)); } else if (OB_FAIL(common::ObLinkQueueThreadPool::init( 1, - (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET, + common::APPLY_TASK_QUEUE_SIZE + 1, "ApplySrv"))) { CLOG_LOG(WARN, "fail to init apply service thread pool", K(ret)); } else if (OB_FAIL(common::ObLinkQueueThreadPool::set_adaptive_thread(1, 1))) { CLOG_LOG(WARN, "fail to set apply service thread count", K(ret)); - } else if (OB_FAIL(apply_status_map_.init("APPLY_STATUS"))) { - CLOG_LOG(WARN, "apply_status_map_ init error", K(ret)); + } else if (OB_FAIL(lock_.init(ObMemAttr("ApplyStatus")))) { + CLOG_LOG(WARN, "apply status lock init error", K(ret)); } else { is_inited_ = true; CLOG_LOG(INFO, "ObLogApplyService init success", K(is_inited_)); @@ -1074,6 +973,9 @@ int ObLogApplyService::init(PalfEnv *palf_env, void ObLogApplyService::destroy() { + if (is_inited_) { + (void)remove_status(); + } is_inited_ = false; is_running_ = false; CLOG_LOG(INFO, "apply service destroy"); @@ -1082,7 +984,7 @@ void ObLogApplyService::destroy() common::ObLinkQueueThreadPool::destroy(); palf_env_ = NULL; ls_adapter_ = NULL; - apply_status_map_.destroy(); + lock_.destroy(); } int ObLogApplyService::start() @@ -1122,11 +1024,11 @@ void ObLogApplyService::wait() //At this point, it can be guaranteed that no new queue tasks will enter the thread pool, //At the same time, all other modules have called stop, so no new cb will enter the queue //It is safe to clean up all remaining cbs - remove_all_ls_(); + (void)remove_status(); CLOG_LOG(INFO, "ObLogApplyService wait finish"); } -int ObLogApplyService::add_ls(const share::ObLSID &id) +int ObLogApplyService::create_status() { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1139,48 +1041,58 @@ int ObLogApplyService::add_ls(const share::ObLSID &id) CLOG_LOG(ERROR, "apply service has been stopped", K(ret)); } else if (NULL == (apply_status = static_cast(mtl_malloc(sizeof(ObApplyStatus), attr)))){ ret = OB_ALLOCATE_MEMORY_FAILED; - CLOG_LOG(WARN, "failed to alloc apply status", K(ret), K(id)); + CLOG_LOG(WARN, "failed to alloc apply status", K(ret)); } else { new (apply_status) ObApplyStatus(); - if (OB_FAIL(apply_status->init(id, palf_env_, this))) { + if (OB_FAIL(apply_status->init(palf_env_, this))) { mtl_free(apply_status); apply_status = NULL; - CLOG_LOG(WARN, "failed to init apply status", K(ret), K(id), K(palf_env_), K(this)); + CLOG_LOG(WARN, "failed to init apply status", K(ret), K(palf_env_), K(this)); } else { apply_status->inc_ref(); - if (OB_FAIL(apply_status_map_.insert(id, apply_status))) { - CLOG_LOG(ERROR, "insert apply status failed", K(ret), K(id), KPC(apply_status)); + { + ObQSyncLockWriteGuard guard(lock_); + if (OB_NOT_NULL(apply_status_)) { + ret = OB_ENTRY_EXIST; + } else { + apply_status_ = apply_status; + apply_status = NULL; + } + } + if (OB_NOT_NULL(apply_status)) { revert_apply_status(apply_status); - } else { - CLOG_LOG(TRACE, "add_ls success", K(ret), K(id), KPC(apply_status)); } } } return ret; } -int ObLogApplyService::remove_ls(const share::ObLSID &id) +int ObLogApplyService::remove_status() { int ret = OB_SUCCESS; - RemoveApplyStatusFunctor functor; + ObApplyStatus *apply_status = NULL; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(apply_status_map_.erase_if(id, functor))) { - if (OB_ENTRY_NOT_EXIST == ret) { - CLOG_LOG(WARN, "apply service remove ls failed", K(ret), K(id)); - ret = OB_SUCCESS; - } else { - CLOG_LOG(ERROR, "apply service remove ls failed", K(ret), K(id)); - } } else { - CLOG_LOG(INFO, "apply service remove ls", K(id)); + { + ObQSyncLockWriteGuard guard(lock_); + apply_status = apply_status_; + apply_status_ = NULL; + } + if (OB_NOT_NULL(apply_status)) { + if (OB_FAIL(apply_status->handle_drop_cb())) { + CLOG_LOG(ERROR, "apply status handle drop cb failed", K(ret)); + } else if (OB_FAIL(apply_status->unregister_file_size_cb())) { + CLOG_LOG(ERROR, "apply status unregister file size cb failed", K(ret)); + } + revert_apply_status(apply_status); + } } return ret; } -int ObLogApplyService::is_apply_done(const share::ObLSID &id, - bool &is_done, +int ObLogApplyService::is_apply_done(bool &is_done, palf::LSN &end_lsn) { int ret = OB_SUCCESS; @@ -1189,21 +1101,20 @@ int ObLogApplyService::is_apply_done(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); } else if (OB_FAIL(apply_status->is_apply_done(is_done, end_lsn))) { - CLOG_LOG(WARN, "apply status check is_apply_done failed", K(ret), K(id), K(is_done)); + CLOG_LOG(WARN, "apply status check is_apply_done failed", K(ret), K(is_done)); } else { - CLOG_LOG(TRACE, "apply service check is_apply_done", K(id), K(is_done), K(end_lsn)); + CLOG_LOG(TRACE, "apply service check is_apply_done", K(is_done), K(end_lsn)); } return ret; } -int ObLogApplyService::switch_to_leader(const share::ObLSID &id, - const int64_t proposal_id) +int ObLogApplyService::start_local_append() { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1211,20 +1122,20 @@ int ObLogApplyService::switch_to_leader(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); - } else if (OB_FAIL(apply_status->switch_to_leader(proposal_id))) { - CLOG_LOG(WARN, "apply status switch_to_leader failed", K(ret), K(id), K(proposal_id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); + } else if (OB_FAIL(apply_status->start_local_append())) { + CLOG_LOG(WARN, "apply status start_local_append failed", K(ret)); } else { - CLOG_LOG(INFO, "apply service switch_to_leader success", K(id), K(proposal_id)); + CLOG_LOG(INFO, "apply service start_local_append success"); } return ret; } -int ObLogApplyService::switch_to_follower(const share::ObLSID &id) +int ObLogApplyService::stop_local_append() { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1232,20 +1143,20 @@ int ObLogApplyService::switch_to_follower(const share::ObLSID &id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); - } else if (OB_FAIL(apply_status->switch_to_follower())) { - CLOG_LOG(WARN, "apply status switch_to_follower failed", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); + } else if (OB_FAIL(apply_status->stop_local_append())) { + CLOG_LOG(WARN, "apply status stop_local_append failed", K(ret)); } else { - CLOG_LOG(INFO, "apply service switch_to_follower success", K(id)); + CLOG_LOG(INFO, "apply service stop_local_append success"); } return ret; } -int ObLogApplyService::get_max_applied_scn(const share::ObLSID &id, SCN &scn) +int ObLogApplyService::get_max_applied_scn(SCN &scn) { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1253,20 +1164,20 @@ int ObLogApplyService::get_max_applied_scn(const share::ObLSID &id, SCN &scn) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); } else if (OB_FAIL(apply_status->get_max_applied_scn(scn))) { - CLOG_LOG(WARN, "apply status get_max_applied_scn failed", K(ret), K(id)); + CLOG_LOG(WARN, "apply status get_max_applied_scn failed", K(ret)); } else { - CLOG_LOG(TRACE, "apply service get_max_applied_scn success", K(id)); + CLOG_LOG(TRACE, "apply service get_max_applied_scn success"); } return ret; } -int ObLogApplyService::get_palf_committed_end_scn(const share::ObLSID &id, share::SCN &scn) +int ObLogApplyService::get_palf_committed_end_scn(share::SCN &scn) { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1274,14 +1185,14 @@ int ObLogApplyService::get_palf_committed_end_scn(const share::ObLSID &id, share if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); } else { scn = apply_status->get_palf_committed_end_scn(); - CLOG_LOG(TRACE, "apply service get palf_committed_end_lsn success", K(id), K(scn)); + CLOG_LOG(TRACE, "apply service get palf_committed_end_lsn success", K(scn)); } return ret; } @@ -1305,18 +1216,19 @@ int ObLogApplyService::push_task(ObApplyServiceTask *task) return ret; } -int ObLogApplyService::get_apply_status(const share::ObLSID &id, - ObApplyStatusGuard &guard) +int ObLogApplyService::get_apply_status(ObApplyStatusGuard &guard) { int ret = OB_SUCCESS; - GetApplyStatusFunctor functor(guard); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "ObLogApplyServicenot init", K(ret)); - } else if (OB_FAIL(apply_status_map_.operate(id, functor))) { - CLOG_LOG(WARN, "get_apply_statusfailed", K(ret), K(id)); + CLOG_LOG(WARN, "ObLogApplyService not init", K(ret)); } else { - CLOG_LOG(TRACE, "get_apply_status success", K(ret), K(id)); + ObQSyncLockReadGuard lock_guard(lock_); + if (OB_ISNULL(apply_status_)) { + ret = OB_ENTRY_NOT_EXIST; + } else { + guard.set_apply_status_(apply_status_); + } } return ret; } @@ -1411,48 +1323,42 @@ void ObLogApplyService::handle_drop(common::LinkTask *task) } } -int ObLogApplyService::wait_append_sync(const share::ObLSID &ls_id) +int ObLogApplyService::wait_append_sync() { int ret = OB_SUCCESS; int64_t start_ts = ObTimeUtility::fast_current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "ObLogApplyService not init", K(ret)); - } else if (OB_FAIL(ls_adapter_->wait_append_sync(ls_id))) { - CLOG_LOG(WARN, "wait_append_sync failed", K(ret), K(ls_id)); + } else if (OB_FAIL(ls_adapter_->wait_append_sync())) { + CLOG_LOG(WARN, "wait_append_sync failed", K(ret)); // TODO:@keqing.llt Remove before release } else { int64_t cost_time = ObTimeUtility::fast_current_time() - start_ts; if (cost_time > 10 * 1000) { //10ms - CLOG_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "wait_append_sync cost too much time", K(ret), K(ls_id), K(cost_time)); + CLOG_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "wait_append_sync cost too much time", K(ret), K(cost_time)); } } return ret; } -int ObLogApplyService::stat_for_each(const common::ObFunction &func) +int ObLogApplyService::stat(LSApplyStat &apply_stat) { - auto stat_func = [&func](const share::ObLSID &id, ObApplyStatus *apply_status) -> bool { - int ret = OB_SUCCESS; - bool bret = true; - if (OB_FAIL(func(*apply_status))) { - bret = false; - CLOG_LOG(WARN, "iter apply stat failed", K(ret)); - } - return bret; - }; int ret = OB_SUCCESS; - if (!func.is_valid()) { - // ObFunction will be invalid when allocating memory failed. - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - ret = apply_status_map_.for_each(stat_func); + ObApplyStatusGuard guard; + ObApplyStatus *apply_status = NULL; + if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "failed to get apply status", K(ret)); + } else if (OB_ISNULL(apply_status = guard.get_apply_status())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "apply status is null", K(ret)); + } else if (OB_FAIL(apply_status->stat(apply_stat))) { + CLOG_LOG(WARN, "stat apply failed", K(ret), KPC(apply_status)); } return ret; } -int ObLogApplyService::diagnose(const share::ObLSID &id, - ApplyDiagnoseInfo &diagnose_info) +int ObLogApplyService::diagnose(ApplyDiagnoseInfo &diagnose_info) { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; @@ -1460,15 +1366,15 @@ int ObLogApplyService::diagnose(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "apply service not init", K(ret)); - } else if (OB_FAIL(get_apply_status(id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + } else if (OB_FAIL(get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); } else if (OB_FAIL(apply_status->diagnose(diagnose_info))) { - CLOG_LOG(WARN, "apply status diagnose failed", K(ret), K(id)); + CLOG_LOG(WARN, "apply status diagnose failed", K(ret)); } else { - CLOG_LOG(TRACE, "apply service diagnose success", K(id)); + CLOG_LOG(TRACE, "apply service diagnose success"); } return ret; } @@ -1494,22 +1400,5 @@ int ObLogApplyService::handle_submit_task_(ObApplyStatus *apply_status) } return ret; } -// Called during destruction, return the count of all log streams -int ObLogApplyService::remove_all_ls_() -{ - int ret = OB_SUCCESS; - ObApplyStatus *apply_status = NULL; - ResetApplyStatusFunctor functor; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "apply service not init", K(ret)); - } else if (OB_FAIL(apply_status_map_.for_each(functor))) { - CLOG_LOG(ERROR, "apply service remove all ls failed", K(ret)); - } else { - CLOG_LOG(INFO, "apply service remove all ls"); - } - return ret; -} - } // namespace logservice } // namespace oceanbase diff --git a/src/logservice/applyservice/ob_log_apply_service.h b/src/logservice/applyservice/ob_log_apply_service.h index 0a9d214ae..f6701e8f4 100644 --- a/src/logservice/applyservice/ob_log_apply_service.h +++ b/src/logservice/applyservice/ob_log_apply_service.h @@ -17,14 +17,13 @@ #ifndef OCEANBASE_LOGSERVICE_LOG_APPLY_SERVICE_ #define OCEANBASE_LOGSERVICE_LOG_APPLY_SERVICE_ #include "common/ob_role.h" -#include "lib/hash/ob_link_hashmap.h" +#include "lib/lock/ob_qsync_lock.h" #include "lib/queue/ob_link_queue.h" #include "lib/thread/ob_simple_thread_pool.h" #include "lib/thread/ob_thread_lease.h" #include "logservice/palf/palf_callback.h" #include "logservice/palf/palf_handle.h" #include "share/scn.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -54,16 +53,10 @@ enum class ObApplyServiceTaskType //Virtual table statistics struct LSApplyStat { - int64_t ls_id_; - common::ObRole role_; - int64_t proposal_id_; palf::LSN end_lsn_; int64_t pending_cnt_; - TO_STRING_KV(K(ls_id_), - K(role_), - K(proposal_id_), - K(end_lsn_), + TO_STRING_KV(K(end_lsn_), K(pending_cnt_)); }; @@ -85,10 +78,8 @@ class ObApplyFsCb : public palf::PalfFSCb ObApplyFsCb(ObApplyStatus *apply_status); ~ObApplyFsCb(); void destroy(); - int update_end_lsn(int64_t id, - const palf::LSN &end_lsn, - const share::SCN &end_scn, - const int64_t proposal_id); + int update_end_lsn(const palf::LSN &end_lsn, + const share::SCN &end_scn); private: ObApplyStatus *apply_status_; }; @@ -160,8 +151,7 @@ class ObApplyStatus ObApplyStatus(); ~ObApplyStatus(); public: - int init(const share::ObLSID &id, - palf::PalfEnv *palf_env, + int init(palf::PalfEnv *palf_env, ObLogApplyService *ap_sv); void destroy(); int stop(); @@ -173,12 +163,10 @@ class ObApplyStatus int try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, bool &is_timeslice_run_out); int is_apply_done(bool &is_done, palf::LSN &end_lsn); - //leader-follower switch related - //int can_switch_to_follower(bool &can_revoke); //Non-maximum protection mode does not require - int switch_to_leader(const int64_t new_proposal_id); - int switch_to_follower(); + int start_local_append(); + int stop_local_append(); //palf related - int update_palf_committed_end_lsn(const palf::LSN &end_lsn, const share::SCN &end_scn, const int64_t proposal_id); + int update_palf_committed_end_lsn(const palf::LSN &end_lsn, const share::SCN &end_scn); share::SCN get_palf_committed_end_scn() const; int unregister_file_size_cb(); void close_palf_handle(); @@ -187,33 +175,8 @@ class ObApplyStatus int stat(LSApplyStat &stat) const; int handle_drop_cb(); int diagnose(ApplyDiagnoseInfo &diagnose_info); - // offline related - // - // The constraint between palf and apply: - // - // Palf guarantee that switch apply to follower only when there is not - // any uncommitted logs in previous LEADER, therefore, apply only update - // 'palf_committed_end_lsn_' when 'proposal_id_' is as same as current - // proposal_id of palf. - // - // To increase robustness, apply assums that update 'palf_committed_end_lsn_' - // when the role of apply is LEADER execpet above constraints. otherwise, - // apply consider it as unexpected error. - // - // However, in rebuild scenario, apply will be reset to FOLLOWER even if there - // are logs to be committed when 'proposal_id_' is as same as current proposal_id - // of palf. - // - // To solve above problem, add an interface which used to reset 'proposal_id_' of - // apply. - // - // NB: this interface only can be used in 'ObLogHandler::offline'. - void reset_proposal_id(); - // NB: this interface only can be used in 'ObLogHandler::online'. void reset_meta(); - TO_STRING_KV(K(ls_id_), - K(role_), - K(proposal_id_), + TO_STRING_KV(K(accepting_append_), K(palf_committed_end_lsn_), K(palf_committed_end_scn_), K(last_check_scn_), @@ -223,7 +186,7 @@ class ObApplyStatus int check_and_update_max_applied_scn_(const share::SCN scn); int update_last_check_scn_(); int handle_drop_cb_queue_(ObApplyServiceQueueTask &cb_queue); - int switch_to_follower_(); + int stop_local_append_(); //Get profiling information from cb void get_cb_trace_(AppendCb *cb, int64_t &append_start_time, @@ -248,9 +211,7 @@ class ObApplyStatus bool is_inited_; bool is_in_stop_state_; // after stop, it cannot take over, remaining cb will continue to be processed int64_t ref_cnt_; // guarantee the effectiveness of self memory - share::ObLSID ls_id_; - common::ObRole role_; - int64_t proposal_id_; + bool accepting_append_; ObLogApplyService *ap_sv_; palf::LSN palf_committed_end_lsn_; share::SCN palf_committed_end_scn_; @@ -263,7 +224,7 @@ class ObApplyStatus palf::PalfEnv *palf_env_; palf::PalfHandle palf_handle_; ObApplyFsCb fs_cb_; - mutable RWLock lock_; // protect role_, proposal_id_ and is_in_stop_state_ + mutable RWLock lock_; mutable lib::ObMutex mutex_; // Mutex for acquiring the maximum consecutive checkpoint will not be called concurrently mutable int64_t get_info_debug_time_; mutable int64_t try_wrlock_debug_time_; @@ -286,78 +247,37 @@ class ObLogApplyService : public common::ObLinkQueueThreadPool int start(); void stop(); void wait(); - int add_ls(const share::ObLSID &id); - int remove_ls(const share::ObLSID &id); - int get_apply_status(const share::ObLSID &id, ObApplyStatusGuard &guard); + int create_status(); + int remove_status(); + int get_apply_status(ObApplyStatusGuard &guard); void revert_apply_status(ObApplyStatus *apply_status); void handle(common::LinkTask *task) override; void handle_drop(common::LinkTask *task) override; - int is_apply_done(const share::ObLSID &id, - bool &is_done, + int is_apply_done(bool &is_done, palf::LSN &end_lsn); - int switch_to_leader(const share::ObLSID &id, const int64_t proposal_id); - int switch_to_follower(const share::ObLSID &id); - int get_max_applied_scn(const share::ObLSID &id, share::SCN &scn); - int get_palf_committed_end_scn(const share::ObLSID &id, share::SCN &scn); + int start_local_append(); + int stop_local_append(); + int get_max_applied_scn(share::SCN &scn); + int get_palf_committed_end_scn(share::SCN &scn); int push_task(ObApplyServiceTask *task); - int wait_append_sync(const share::ObLSID &ls_id); - int stat_for_each(const common::ObFunction &func); - int diagnose(const share::ObLSID &id, ApplyDiagnoseInfo &diagnose_info); -public: - class GetApplyStatusFunctor - { - public: - GetApplyStatusFunctor(ObApplyStatusGuard &guard) - : ret_code_(common::OB_SUCCESS), guard_(guard){} - ~GetApplyStatusFunctor(){} - bool operator()(const share::ObLSID &id, ObApplyStatus *apply_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - ObApplyStatusGuard &guard_; - }; - class RemoveApplyStatusFunctor - { - public: - explicit RemoveApplyStatusFunctor() - : ret_code_(common::OB_SUCCESS) {} - ~RemoveApplyStatusFunctor(){} - bool operator()(const share::ObLSID &id, ObApplyStatus *replay_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - }; - //Delete and clean up all cb - class ResetApplyStatusFunctor - { - public: - explicit ResetApplyStatusFunctor() - : ret_code_(common::OB_SUCCESS) {} - ~ResetApplyStatusFunctor(){} - bool operator()(const share::ObLSID &id, ObApplyStatus *replay_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - }; + int wait_append_sync(); + int stat(LSApplyStat &apply_stat); + int diagnose(ApplyDiagnoseInfo &diagnose_info); private: int handle_cb_queue_(ObApplyStatus *apply_status, ObApplyServiceQueueTask *cb_queue, bool &is_timeslice_run_out); int handle_submit_task_(ObApplyStatus *apply_status); - int remove_all_ls_(); private: bool is_inited_; bool is_running_; palf::PalfEnv *palf_env_; ObLSAdapter *ls_adapter_; - common::ObLinearHashMap apply_status_map_; + ObApplyStatus *apply_status_; + mutable common::ObQSyncLock lock_; DISALLOW_COPY_AND_ASSIGN(ObLogApplyService); }; -// for apply_status_map in apply service class ObApplyStatusGuard { public: @@ -367,7 +287,7 @@ class ObApplyStatusGuard private: void set_apply_status_(ObApplyStatus *apply_status); private: - friend class ObLogApplyService::GetApplyStatusFunctor; + friend class ObLogApplyService; ObApplyStatus *apply_status_; DISALLOW_COPY_AND_ASSIGN(ObApplyStatusGuard); }; diff --git a/src/logservice/common_util/ob_log_ls_define.cpp b/src/logservice/common_util/ob_log_ls_define.cpp deleted file mode 100644 index d62f09820..000000000 --- a/src/logservice/common_util/ob_log_ls_define.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_log_ls_define.h" - -namespace oceanbase -{ -namespace logservice -{ -int TenantLSID::compare(const TenantLSID &other) const -{ - int cmp_ret = 0; - - if (ls_id_ > other.ls_id_) { - cmp_ret = 1; - } else if (ls_id_ < other.ls_id_) { - cmp_ret = -1; - } else { - cmp_ret = 0; - } - - return cmp_ret; -} - -TenantLSID &TenantLSID::operator=(const TenantLSID &other) -{ - - this->ls_id_ = other.get_ls_id(); - return *this; -} - -} // namespace logservice -} // namespace oceanbase diff --git a/src/logservice/common_util/ob_log_ls_define.h b/src/logservice/common_util/ob_log_ls_define.h deleted file mode 100644 index a2c43df4d..000000000 --- a/src/logservice/common_util/ob_log_ls_define.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_COMMON_LS_DEFINE_H_ -#define OCEANBASE_LOGSERVICE_COMMON_LS_DEFINE_H_ - -#include "share/ob_ls_id.h" - -namespace oceanbase -{ -namespace logservice -{ -class TenantLSID -{ -public: - TenantLSID() { reset(); } - TenantLSID(const share::ObLSID &ls_id) - : ls_id_(ls_id) - {} - ~TenantLSID() { reset(); } - - void reset() - { - - ls_id_.reset(); - } - - bool is_valid() const { - return ls_id_.is_valid_with_tenant(); - } - - uint64_t hash() const - { - uint64_t hash_val = 0; - const uint64_t hash_seed = 1; - hash_val = common::murmurhash(&hash_seed, sizeof(hash_seed), hash_val); - hash_val = common::murmurhash(&ls_id_, sizeof(ls_id_), hash_val); - - return hash_val; - } - - int hash(uint64_t &hash_val) const - { - hash_val = hash(); - return OB_SUCCESS; - } - - bool is_sys_log_stream() const { return ls_id_.is_sys_ls(); } - - int compare(const TenantLSID &other) const; - bool operator==(const TenantLSID &other) const - { - return (true) &&(ls_id_ == other.ls_id_); - } - bool operator!=(const TenantLSID &other) const { return !operator==(other); } - bool operator<(const TenantLSID &other) const { return -1 == compare(other); } - TenantLSID &operator=(const TenantLSID &other); - - - const share::ObLSID &get_ls_id() const { return ls_id_; } - - TO_STRING_KV(K_(ls_id)); - -private: - - share::ObLSID ls_id_; -}; - -} // namespace logservice -} // namespace oceanbase - -#endif diff --git a/src/logservice/data_dictionary/ob_data_dict_service.cpp b/src/logservice/data_dictionary/ob_data_dict_service.cpp index f3a38006d..c56dd2093 100644 --- a/src/logservice/data_dictionary/ob_data_dict_service.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_service.cpp @@ -17,6 +17,7 @@ #include "ob_data_dict_service.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" // ObLSService #include "storage/tx/ob_ts_mgr.h" // OB_TS_MGR @@ -45,7 +46,7 @@ const int64_t ObDataDictService::DEFAULT_REPORT_TIMEOUT = 10 * _MIN_; ObDataDictService::ObDataDictService() : is_inited_(false), - is_leader_(false), + is_active_(false), stop_flag_(true), allocator_("ObDataDictSvc"), sql_client_(), @@ -140,7 +141,7 @@ void ObDataDictService::destroy() sql_client_.destroy(); allocator_.reset(); stop_flag_ = true; - is_leader_ = false; + is_active_ = false; is_inited_ = false; DDLOG(INFO, "destroy datadict_service"); } @@ -152,12 +153,12 @@ void ObDataDictService::runTimerTask() if (IS_INIT) { refresh_config_(); - bool is_leader = ATOMIC_LOAD(&is_leader_); + bool is_active = ATOMIC_LOAD(&is_active_); const int64_t start_time = ObClockGenerator::getClock(); const bool is_reach_time_interval = (start_time >= ATOMIC_LOAD(&last_dump_succ_time_) + ATOMIC_LOAD(&dump_interval_)); const bool force_need_dump = ATOMIC_LOAD(&force_need_dump_); - if (is_leader && (is_reach_time_interval || force_need_dump)) { + if (is_active && (is_reach_time_interval || force_need_dump)) { int ret = OB_SUCCESS; uint64_t data_version = 0; @@ -165,7 +166,7 @@ void ObDataDictService::runTimerTask() DDLOG(WARN, "get_min_data_version failed", KR(ret), K(force_need_dump)); } else if (OB_FAIL(do_dump_data_dict_())) { if (OB_STATE_NOT_MATCH == ret) { - DDLOG(WARN, "dump_data_dict_, maybe not ls_leader or lsn not valid, ignore.", KR(ret), K(force_need_dump)); + DDLOG(WARN, "dump_data_dict_, maybe not local_log or lsn not valid, ignore.", KR(ret), K(force_need_dump)); } else if (OB_IN_STOP_STATE != ret) { DDLOG(WARN, "dump_data_dict_ failed", KR(ret), K(force_need_dump)); } @@ -184,33 +185,17 @@ void ObDataDictService::runTimerTask() } } -void ObDataDictService::switch_to_follower_forcedly() +void ObDataDictService::deactivate() { - bool is_leader = false; - switch_role_to_(is_leader); + bool is_active = false; + set_active_(is_active); } -int ObDataDictService::switch_to_leader() +int ObDataDictService::activate() { int ret = OB_SUCCESS; - bool is_leader = true; - switch_role_to_(is_leader); - return ret; -} - -int ObDataDictService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - bool is_leader = false; - switch_role_to_(is_leader); - return ret; -} - -int ObDataDictService::resume_leader() -{ - int ret = OB_SUCCESS; - bool is_leader = true; - switch_role_to_(is_leader); + bool is_active = true; + set_active_(is_active); return ret; } @@ -223,26 +208,22 @@ void ObDataDictService::refresh_config_() } } -OB_INLINE void ObDataDictService::switch_role_to_(bool is_leader) +OB_INLINE void ObDataDictService::set_active_(bool is_active) { - ATOMIC_SET(&is_leader_, is_leader); - ATOMIC_SET(&stop_flag_, ! is_leader); - DDLOG(INFO, "switch_role", K(is_leader)); + ATOMIC_SET(&is_active_, is_active); + ATOMIC_SET(&stop_flag_, ! is_active); + DDLOG(INFO, "set data dictionary active state", K(is_active)); } int ObDataDictService::do_dump_data_dict_() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSHandle ls_handle; // NOTICE: ls_handle is a guard for usage of log_handler. - ObLS *ls = NULL; + ObLS *ls = nullptr; ObLogHandler *log_handler = NULL; - bool is_leader = false; share::SCN snapshot_scn; palf::LSN start_lsn; palf::LSN end_lsn; - int64_t start_proposal_id = 0; - int64_t end_proposal_id = 0; bool is_cluster_status_normal = false; bool is_data_dict_dump_success = false; bool is_any_log_callback_fail = false; @@ -262,30 +243,19 @@ int ObDataDictService::do_dump_data_dict_() DDLOG(TRACE, "check_cluster_status_normal_ failed", KR(ret), K(is_cluster_status_normal)); } else if (OB_UNLIKELY(! is_cluster_status_normal)) { DDLOG(TRACE, "cluster_status not normal, won't dump_data_dict", K(is_cluster_status_normal)); - } else if (OB_FAIL(ls_service_->get_ls(share::SYS_LS, ls_handle, ObLSGetMod::DATA_DICT_MOD))) { - if (OB_LS_NOT_EXIST != ret || REACH_TIME_INTERVAL_THREAD_LOCAL(PRINT_DETAIL_INTERVAL)) { - DDLOG(WARN, "get_ls for data_dict_service from ls_service failed", KR(ret)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - DDLOG(WARN, "invalid ls get from ls_handle", KR(ret)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - DDLOG(WARN, "invalid log_handler_ get from OBLS", KR(ret)); - } else if (OB_FAIL(check_ls_leader(log_handler, is_leader, start_proposal_id))) { - DDLOG(WARN, "check_is_sys_ls_leader failed", KR(ret), K(start_proposal_id)); - } else if (! is_leader) { - ret = OB_STATE_NOT_MATCH; - DDLOG(WARN, "won't do_dump_data_dict_ cause not ls_leader", KR(ret), K(is_leader), K(start_proposal_id)); - // do nothing if not ls leader. - } else if (OB_FAIL(get_snapshot_scn_(snapshot_scn))) { - DDLOG(WARN, "get_snapshot_scn failed", KR(ret), K(snapshot_scn)); - } else if (OB_FAIL(storage_.prepare(snapshot_scn, log_handler))) { - DDLOG(WARN, "storage prepare for data_dict_dump failed", KR(ret), K(snapshot_scn)); - } else if (OB_FAIL(generate_dict_and_dump_(snapshot_scn))) { - DDLOG(WARN, "generate_dict_and_dump_", KR(ret), K(snapshot_scn), K(start_proposal_id)); + } else if (OB_FAIL(ls_service_->get_ls(ls))) { + DDLOG(WARN, "get log stream for data dictionary failed", KR(ret)); } else { - is_data_dict_dump_success = true; + log_handler = ls->get_log_handler(); + if (OB_FAIL(get_snapshot_scn_(snapshot_scn))) { + DDLOG(WARN, "get_snapshot_scn failed", KR(ret), K(snapshot_scn)); + } else if (OB_FAIL(storage_.prepare(snapshot_scn, log_handler))) { + DDLOG(WARN, "storage prepare for data_dict_dump failed", KR(ret), K(snapshot_scn)); + } else if (OB_FAIL(generate_dict_and_dump_(snapshot_scn))) { + DDLOG(WARN, "generate_dict_and_dump_", KR(ret), K(snapshot_scn)); + } else { + is_data_dict_dump_success = true; + } } if (OB_SUCCESS != (tmp_ret = storage_.finish( @@ -303,12 +273,6 @@ int ObDataDictService::do_dump_data_dict_() ret = OB_STATE_NOT_MATCH; DDLOG(INFO, "won't report data_dict persist info cause data_dict dump failed or log_callback failed", KR(ret), K(is_data_dict_dump_success), K(is_any_log_callback_fail)); - } else if (OB_FAIL(check_ls_leader(log_handler, is_leader, end_proposal_id))) { - DDLOG(WARN, "check_is_sys_ls_leader failed", KR(ret), K(start_proposal_id), K(end_proposal_id)); - } else if (OB_UNLIKELY(! is_leader || start_proposal_id != end_proposal_id)) { - ret = OB_STATE_NOT_MATCH; - DDLOG(INFO, "won't report data_dict persist info cause currently not ls_leader or not the same election term", - KR(ret), K(is_leader), K(start_proposal_id), K(end_proposal_id)); } else { // only report when dict dump success and all log_callback success. const int64_t half_dump_interval = ATOMIC_LOAD(&dump_interval_) / 2; @@ -371,7 +335,7 @@ int ObDataDictService::get_snapshot_scn_(share::SCN &snapshot_scn) const int64_t expire_ts_ns = get_timestamp_ns() + gts_get_timeout_ns; do{ - if (OB_FAIL(OB_TS_MGR.get_gts(stc, NULL, gts_scn, tmp_receive_gts_ts))) { + if (OB_FAIL(OB_TS_MGR.get_gts(stc, gts_scn, tmp_receive_gts_ts))) { if (OB_EAGAIN == ret) { if (expire_ts_ns < get_timestamp_ns()) { ret = OB_TIMEOUT; @@ -477,7 +441,6 @@ int ObDataDictService::handle_tenant_meta_( ObIArray &table_ids) { int ret = OB_SUCCESS; - ObLSArray ls_array; database_ids.reset(); table_ids.reset(); const ObTenantSchema *tenant_schema = NULL; @@ -486,9 +449,7 @@ int ObDataDictService::handle_tenant_meta_( ObDictTenantMeta tenant_meta(&allocator_); ObDictMetaHeader header(ObDictMetaType::TENANT_META); - if (OB_FAIL(sql_client_.get_ls_info(snapshot_scn, ls_array))) { - DDLOG(WARN, "get_ls_info failed", KR(ret), K(snapshot_scn), K(ls_array)); - } else if (OB_FAIL(get_tenant_schema_guard_(schema_version, tenant_schema_guard, true/*is_force_fallback*/))) { + if (OB_FAIL(get_tenant_schema_guard_(schema_version, tenant_schema_guard, true/*is_force_fallback*/))) { DDLOG(WARN, "get_tenant_schema_guard failed", KR(ret), K(snapshot_scn), K(schema_version)); ret = OB_SCHEMA_EAGAIN; } else if (OB_FAIL(check_tenant_status_normal_(tenant_schema_guard, is_normal))) { @@ -500,8 +461,8 @@ int ObDataDictService::handle_tenant_meta_( } else if (OB_ISNULL(tenant_schema)) { ret = OB_ERR_UNEXPECTED; DDLOG(WARN, "invalid tenant_schema", KR(ret), K(snapshot_scn)); - } else if (OB_FAIL(tenant_meta.init_with_ls_info(*tenant_schema, ls_array))) { - DDLOG(WARN, "init tenant_meta failed", KR(ret), K(ls_array), K(tenant_meta)); + } else if (OB_FAIL(tenant_meta.init(*tenant_schema))) { + DDLOG(WARN, "init tenant_meta failed", KR(ret), K(tenant_meta)); } else if (OB_FAIL(get_database_ids_(tenant_schema_guard, database_ids))) { DDLOG(WARN, "get_database_ids_in_tenant failed", KR(ret), K(snapshot_scn), K(schema_version)); } else if (OB_FAIL(tenant_schema_guard.get_table_ids_in_tenant(table_ids))) { @@ -646,7 +607,7 @@ int ObDataDictService::handle_table_metas_( if (OB_SUCC(ret) && stop_flag_) { ret = OB_STATE_NOT_MATCH; - DDLOG(WARN, "data_dict service marked stop_flag, may already switch to follower", KR(ret), K_(stop_flag)); + DDLOG(WARN, "data_dict service marked stop_flag, may already stop", KR(ret), K_(stop_flag)); } } diff --git a/src/logservice/data_dictionary/ob_data_dict_service.h b/src/logservice/data_dictionary/ob_data_dict_service.h index ad8eb6ba7..ae988f9dc 100644 --- a/src/logservice/data_dictionary/ob_data_dict_service.h +++ b/src/logservice/data_dictionary/ob_data_dict_service.h @@ -43,7 +43,7 @@ namespace datadict class ObDataDictService : public common::ObTimerTask, public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -68,10 +68,8 @@ class ObDataDictService const palf::LSN &lsn, const share::SCN &scn) override final { return OB_SUCCESS; } // for ReplaySubHandler virtual int flush(share::SCN &rec_scn) override final { return OB_SUCCESS; } // for CheckpointSubHandler - virtual void switch_to_follower_forcedly() override final; // for RoleChangeSubHandler - virtual int switch_to_leader() override final; - virtual int switch_to_follower_gracefully() override final; - virtual int resume_leader() override final; + void deactivate() override final; + int activate() override final; virtual share::SCN get_rec_scn() override final { return share::SCN::max_scn(); } public: OB_INLINE int64_t get_last_dump_succ_time() const { return last_dump_succ_time_; } @@ -80,7 +78,7 @@ class ObDataDictService { ATOMIC_SET(&force_need_dump_, need_dump); } private: void refresh_config_(); - void switch_role_to_(bool is_leader); + void set_active_(bool is_active); int do_dump_data_dict_(); int check_cluster_status_normal_(bool &is_normal); int get_snapshot_scn_(share::SCN &snapshot_scn); @@ -115,7 +113,7 @@ class ObDataDictService static const int64_t DEFAULT_REPORT_TIMEOUT; private: bool is_inited_; - bool is_leader_; + bool is_active_; volatile bool stop_flag_; ObArenaAllocator allocator_; ObDataDictSqlClient sql_client_; diff --git a/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp b/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp index 42d4f04da..925d4d041 100644 --- a/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp @@ -70,18 +70,6 @@ void ObDataDictSqlClient::destroy() sql_proxy_ = NULL; } -int ObDataDictSqlClient::get_ls_info( - const share::SCN &snapshot_scn, - share::ObLSArray &ls_array) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ls_array.push_back(SYS_LS))) { - DDLOG(WARN, "push_back normal ls into ls_array failed", KR(ret), - K(snapshot_scn)); - } - return ret; -} - int ObDataDictSqlClient::get_schema_version( const share::SCN &snapshot_scn, int64_t &schema_version) diff --git a/src/logservice/data_dictionary/ob_data_dict_sql_client.h b/src/logservice/data_dictionary/ob_data_dict_sql_client.h index b412885b9..56a2cf259 100644 --- a/src/logservice/data_dictionary/ob_data_dict_sql_client.h +++ b/src/logservice/data_dictionary/ob_data_dict_sql_client.h @@ -20,7 +20,6 @@ #include "common/mysqlclient/ob_mysql_result.h" // ObMySQLResult #include "logservice/palf/lsn.h" // LSN #include "share/scn.h" // SCN -#include "share/ob_ls_id.h" // ObLSArray namespace oceanbase { @@ -40,9 +39,6 @@ class ObDataDictSqlClient int init(common::ObMySQLProxy *sql_client); void destroy(); public: - int get_ls_info( - const share::SCN &snapshot_scn, - share::ObLSArray &ls_array); int get_schema_version( const share::SCN &snapshot_scn, int64_t &schema_version); diff --git a/src/logservice/data_dictionary/ob_data_dict_storager.cpp b/src/logservice/data_dictionary/ob_data_dict_storager.cpp index b3fed5af7..634e3dcf7 100644 --- a/src/logservice/data_dictionary/ob_data_dict_storager.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_storager.cpp @@ -539,14 +539,12 @@ int ObDataDictStorage::segment_dict_buf_to_palf_(ObDictMetaHeader &header) int ObDataDictStorage::submit_to_palf_() { int ret = OB_SUCCESS; - bool is_leader = false; ObDataDictPersistCallback *callback = NULL; const SCN &ref_scn = snapshot_scn_; // ns const bool need_nonblock = false; // TODO Is non-block needed? const bool allow_compression = true; palf::LSN lsn; SCN submit_scn; - int64_t proposal_id = 0; if (OB_ISNULL(palf_buf_) || OB_ISNULL(log_handler_) @@ -559,11 +557,6 @@ int ObDataDictStorage::submit_to_palf_() DDLOG(WARN, "log_handler_ is not valid", KR(ret)); } else if (OB_UNLIKELY(palf_pos_ == 0)) { DDLOG(INFO, "empty palf_buf, do nothing", K_(palf_buf_len), K_(palf_pos)); - } else if (OB_FAIL(check_ls_leader(log_handler_, is_leader, proposal_id))) { - DDLOG(WARN, "check_ls_leader failed", KR(ret), K(is_leader), K(proposal_id)); - } else if (OB_UNLIKELY(! is_leader)) { - ret = OB_STATE_NOT_MATCH; - DDLOG(INFO, "do-nothing on non-leader logstream.", KR(ret), K(is_leader), K(proposal_id)); } else if (OB_FAIL(alloc_palf_cb_(callback))) { DDLOG(WARN, "alloc_palf_cb_ failed", KR(ret)); } else if (OB_FAIL(log_handler_->append( @@ -694,7 +687,7 @@ int ObDataDictStorage::check_callback_list_( } if (is_all_invoked || need_print_cb_status) { - // log callback status, NOTICE: stop_flag may set if ls role change or tenant stop. + // log callback status, NOTICE: stop_flag may set if ls local lifecycle or tenant stop. DDLOG(INFO, "[STAT] callbacks_status", KR(ret), K(total_cb_count), K(not_invoked_cb_count), K(failed_cb_count), K(is_all_invoked), K(need_print_cb_status), K(stop_flag)); } diff --git a/src/logservice/data_dictionary/ob_data_dict_struct.cpp b/src/logservice/data_dictionary/ob_data_dict_struct.cpp index debcdb51a..270b9a80d 100644 --- a/src/logservice/data_dictionary/ob_data_dict_struct.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_struct.cpp @@ -363,21 +363,6 @@ int ObDictTenantMeta::incremental_data_update(const ObDictTenantMeta &new_tenant return ret; } -int ObDictTenantMeta::init_with_ls_info( - const schema::ObTenantSchema &tenant_schema, - const share::ObLSArray &ls_array) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(init(tenant_schema))) { - DDLOG(WARN, "init tenant_meta by schema failed", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(ls_arr_.assign(ls_array))) { - DDLOG(WARN, "assign ls_info_arr failed", KR(ret), K(ls_array), K(tenant_schema), KPC(this)); - } - - return ret; -} - ObDictDatabaseMeta::ObDictDatabaseMeta(ObIAllocator *allocator) : allocator_(allocator), database_name_() { reset(); @@ -1348,7 +1333,7 @@ int ObDictTableMeta::build_column_id_arr_(const share::schema::ObTableSchema &ta { int ret = OB_SUCCESS; column_id_arr_order_by_table_def_.reset(); - if (table_schema.is_view_table() && !table_schema.is_materialized_view()) { + if (table_schema.is_view_table()) { DDLOG(DEBUG, "build_column_id_arr_ skip view", KPC(this)); } else { ObColumnIterByPrevNextID pre_next_id_iter(table_schema); diff --git a/src/logservice/data_dictionary/ob_data_dict_struct.h b/src/logservice/data_dictionary/ob_data_dict_struct.h index 712838675..dc0bbf0b4 100644 --- a/src/logservice/data_dictionary/ob_data_dict_struct.h +++ b/src/logservice/data_dictionary/ob_data_dict_struct.h @@ -18,7 +18,6 @@ #define OCEANBASE_DICT_SERVICE_META_DICT_STRUCT_ #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" // share::ObLSID #include "share/schema/ob_schema_struct.h" #include "share/schema/ob_table_schema.h" #include "share/scn.h" @@ -123,12 +122,7 @@ class ObDictTenantMeta ObDictTenantMeta &operator=(const ObDictTenantMeta &src_schema) = delete; public: - // For schema_service (without ls_info) int init(const share::schema::ObTenantSchema &tenant_schema); - // For data_dict_service - int init_with_ls_info( - const share::schema::ObTenantSchema &tenant_schema, - const share::ObLSArray &ls_array); // For incremental data update int incremental_data_update(const ObDictTenantMeta &new_tenant_meta); @@ -153,16 +147,13 @@ class ObDictTenantMeta OB_INLINE common::ObCharsetType get_charset_type() const { return charset_type_; } OB_INLINE common::ObCollationType get_collation_type() const { return collation_type_; } OB_INLINE bool is_in_recyclebin() const { return in_recyclebin_; } - OB_INLINE const share::ObLSArray &get_ls_array() const { return ls_arr_; } - NEED_SERIALIZE_AND_DESERIALIZE_DICT; TO_STRING_KV( K_(schema_version), K_(tenant_name), K_(tenant_status), - K_(in_recyclebin), - K_(ls_arr)); + K_(in_recyclebin)); private: ObIAllocator *allocator_; @@ -178,7 +169,6 @@ class ObDictTenantMeta common::ObCharsetType charset_type_; common::ObCollationType collation_type_; bool in_recyclebin_; - share::ObLSArray ls_arr_; }; // end of ObDictTenantMeta class ObDictDatabaseMeta @@ -289,8 +279,7 @@ class ObDictColumnMeta OB_INLINE bool is_virtual_generated_column() const { return column_flags_ & VIRTUAL_GENERATED_COLUMN_FLAG; } OB_INLINE bool is_stored_generated_column() const { return column_flags_ & STORED_GENERATED_COLUMN_FLAG; } OB_INLINE bool is_generated_column() const { return is_virtual_generated_column() || is_stored_generated_column(); } - OB_INLINE bool is_shadow_column() const { return (column_id_ > common::OB_MIN_SHADOW_COLUMN_ID) - && !is_mlog_special_column(column_id_); } + OB_INLINE bool is_shadow_column() const { return column_id_ > common::OB_MIN_SHADOW_COLUMN_ID; } OB_INLINE bool has_generated_column_deps() const { return column_flags_ & GENERATED_DEPS_CASCADE_FLAG; } int get_cascaded_column_ids(ObIArray &column_ids) const; @@ -422,12 +411,10 @@ class ObDictTableMeta OB_INLINE bool is_view_table() const { return share::schema::ObTableType::USER_VIEW == table_type_ - || share::schema::ObTableType::SYSTEM_VIEW == table_type_ - || share::schema::ObTableType::MATERIALIZED_VIEW == table_type_; + || share::schema::ObTableType::SYSTEM_VIEW == table_type_; } OB_INLINE share::schema::ObIndexType get_index_type() const { return index_type_; } OB_INLINE bool is_index_table() const { return share::schema::is_index_table(table_type_); } - OB_INLINE bool is_mlog_table() const { return share::schema::is_mlog_table(table_type_); } OB_INLINE bool is_normal_index() const { return share::schema::INDEX_TYPE_NORMAL_LOCAL == index_type_ diff --git a/src/logservice/data_dictionary/ob_data_dict_utils.cpp b/src/logservice/data_dictionary/ob_data_dict_utils.cpp index 22d15c0d1..69ae6a977 100644 --- a/src/logservice/data_dictionary/ob_data_dict_utils.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_utils.cpp @@ -128,23 +128,5 @@ int deep_copy_str_array( return ret; } -int check_ls_leader(logservice::ObLogHandler *log_handler, bool &is_leader, int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - common::ObRole role = common::ObRole::INVALID_ROLE; - proposal_id = 0; - - if (OB_ISNULL(log_handler)) { - ret = OB_ERR_UNEXPECTED; - DDLOG(WARN, "log_handler_ is invalid", KR(ret)); - } else if (OB_FAIL(log_handler->get_role(role, proposal_id))) { - DDLOG(WARN, "get ls role fail", K(ret), K(proposal_id)); - } else { - is_leader = is_strong_leader(role); - } - - return ret; -} - } // namespace datadict } // namespace oceanbase diff --git a/src/logservice/data_dictionary/ob_data_dict_utils.h b/src/logservice/data_dictionary/ob_data_dict_utils.h index f50a5e442..d1fc9a539 100644 --- a/src/logservice/data_dictionary/ob_data_dict_utils.h +++ b/src/logservice/data_dictionary/ob_data_dict_utils.h @@ -125,8 +125,6 @@ OB_INLINE const char *extract_str(const ObString &str) return str.empty() ? "" : str.ptr(); } -int check_ls_leader(logservice::ObLogHandler *handler, bool &is_leader, int64_t &proposal_id); - } // namespace datadict } // namespace oceanbase diff --git a/src/logservice/localservice/ob_local_log_handler_set.cpp b/src/logservice/localservice/ob_local_log_handler_set.cpp new file mode 100644 index 000000000..c0909d5f7 --- /dev/null +++ b/src/logservice/localservice/ob_local_log_handler_set.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ob_local_log_handler_set.h" +#include "share/rc/ob_tenant_base.h" +namespace oceanbase +{ +using namespace common; +namespace logservice +{ +ObLocalLogHandlerSet::ObLocalLogHandlerSet(): lock_(common::ObLatchIds::RCS_LOCK), + local_log_handlers_() +{ + reset(); +} + +ObLocalLogHandlerSet::~ObLocalLogHandlerSet() +{ + reset(); +} + +void ObLocalLogHandlerSet::reset() +{ + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { + local_log_handlers_[i] = NULL; + } +} + +int ObLocalLogHandlerSet::register_handler(const ObLogBaseType &type, + ObILocalLogHandler *handler) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + if (false == is_valid_log_base_type(type)) { + ret = OB_INVALID_ARGUMENT; + } else { + local_log_handlers_[type] = handler; + CLOG_LOG(INFO, "register local log handler success", K(ret), K(type), KP(handler)); + } + return ret; +} + +void ObLocalLogHandlerSet::unregister_handler(const ObLogBaseType &type) +{ + ObSpinLockGuard guard(lock_); + if (true == is_valid_log_base_type(type)) { + local_log_handlers_[type] = NULL; + CLOG_LOG(INFO, "unregister_handler success", K(type)); + } +} + +void ObLocalLogHandlerSet::deactivate() +{ + ObSpinLockGuard guard(lock_); + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { + ObILocalLogHandler *handler = local_log_handlers_[i]; + char local_log_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + ObLogBaseType base_type = static_cast(i); + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, local_log_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + if (NULL != handler) { + handler->deactivate(); + CLOG_LOG(INFO, "deactivate local log handler", + "cursor", i, "name", has_defined_to_string ? local_log_handler_str : "hasn't define to string"); + } + } +} + +int ObLocalLogHandlerSet::activate() +{ + int ret = OB_SUCCESS; + CLOG_LOG(INFO, "ObLocalLogHandlerSet::activate called"); + ObSpinLockGuard guard(lock_); + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE && OB_SUCC(ret); i++) { + ObILocalLogHandler *handler = local_log_handlers_[i]; + char local_log_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + ObLogBaseType base_type = static_cast(i); + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, local_log_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + if (NULL == handler) { + if (i == static_cast(ObLogBaseType::TIMESTAMP_LOG_BASE_TYPE)) { + CLOG_LOG(WARN, "TIMESTAMP_LOG_BASE_TYPE handler is NULL", K(i)); + } + } else if (OB_FAIL(handler->activate())) { + CLOG_LOG(WARN, "activate failed", K(ret), KP(handler), K(i), + "cursor", i, "name", has_defined_to_string ? local_log_handler_str : "hasn't define to string"); + } else { + CLOG_LOG(INFO, "activate local log handler", + "cursor", i, "name", has_defined_to_string ? local_log_handler_str : "hasn't define to string"); + } + } + CLOG_LOG(INFO, "ObLocalLogHandlerSet::activate finished", K(ret)); + return ret; +} + +} // end namespace logservice +} // end namespace oceanbase diff --git a/src/logservice/localservice/ob_local_log_handler_set.h b/src/logservice/localservice/ob_local_log_handler_set.h new file mode 100644 index 000000000..5049fc4d5 --- /dev/null +++ b/src/logservice/localservice/ob_local_log_handler_set.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_LOGSERVICE_OB_LOCAL_LOG_HANDLER_SET_ +#define OCEANBASE_LOGSERVICE_OB_LOCAL_LOG_HANDLER_SET_ +#include "lib/container/ob_fixed_array.h" +#include "lib/container/ob_se_array.h" +#include "lib/lock/ob_spin_lock.h" +#include "lib/utility/ob_print_utils.h" +#include "logservice/ob_log_base_header.h" +#include "logservice/ob_log_base_type.h" +#include "share/ob_errno.h" +namespace oceanbase +{ +namespace logservice +{ +class ObLocalLogHandlerSet { +public: + ObLocalLogHandlerSet(); + ~ObLocalLogHandlerSet(); + void reset(); + int register_handler(const ObLogBaseType &type, ObILocalLogHandler *handler); + void unregister_handler(const ObLogBaseType &type); + + void deactivate(); + int activate(); +private: + ObSpinLock lock_; + ObILocalLogHandler* local_log_handlers_[ObLogBaseType::MAX_LOG_BASE_TYPE]; +}; +} +} +#endif // OCEANBASE_LOGSERVICE_OB_LOCAL_LOG_HANDLER_SET_ diff --git a/src/logservice/logrouteservice/ob_log_route_service.cpp b/src/logservice/logrouteservice/ob_log_route_service.cpp deleted file mode 100644 index c636f5e7c..000000000 --- a/src/logservice/logrouteservice/ob_log_route_service.cpp +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#define USING_LOG_PREFIX OBLOG -#include "ob_log_route_service.h" -#include "share/ob_server_struct.h" // GCTX -#include "share/config/ob_server_config.h" // GCONF -#include "lib/string/ob_sql_string.h" // ObSqlString - -using namespace oceanbase::share; -using namespace oceanbase::common; - -namespace oceanbase -{ -namespace logservice -{ -ObLogRouteService::ObLogRouteService() : - is_inited_(false), - cluster_id_(OB_INVALID_CLUSTER_ID), - is_stopped_(true), - ls_route_timer_task_(*this), - next_server_called_(false) -{ -} - -ObLogRouteService::~ObLogRouteService() -{ - destroy(); -} - -#ifdef ERRSIM -ERRSIM_POINT_DEF(LOG_ROUTE_TIMER_INIT_FAIL); -ERRSIM_POINT_DEF(LOG_ROUTE_HANDLER_INIT_FAIL); -ERRSIM_POINT_DEF(LOG_ROUTE_HANDLER_START_FAIL); -#endif -int ObLogRouteService::init(ObISQLClient *proxy, - const int64_t cluster_id, - const bool is_across_cluster, - void *err_handler, - const char *external_server_blacklist, - const int64_t background_refresh_time_sec, - const int64_t all_server_cache_update_interval_sec, - const int64_t all_zone_cache_update_interval_sec, - const int64_t blacklist_survival_time_sec, - const int64_t blacklist_survival_time_upper_limit_min, - const int64_t blacklist_survival_time_penalty_period_min, - const int64_t blacklist_history_overdue_time_min, - const int64_t blacklist_history_clear_interval_min) -{ - int ret = OB_SUCCESS; - UNUSED(proxy); - UNUSED(is_across_cluster); - UNUSED(err_handler); - UNUSED(external_server_blacklist); - UNUSED(background_refresh_time_sec); - UNUSED(all_server_cache_update_interval_sec); - UNUSED(all_zone_cache_update_interval_sec); - UNUSED(blacklist_survival_time_sec); - UNUSED(blacklist_survival_time_upper_limit_min); - UNUSED(blacklist_survival_time_penalty_period_min); - UNUSED(blacklist_history_overdue_time_min); - UNUSED(blacklist_history_clear_interval_min); - - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObLogRouteService has been inited twice", KR(ret)); - } else if (OB_UNLIKELY(OB_INVALID_CLUSTER_ID == cluster_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid cluster_id", KR(ret), K(cluster_id)); - } else { - cluster_id_ = cluster_id; - is_stopped_ = true; - is_inited_ = true; - LOG_INFO("ObLogRouteService init succ (simplified for single machine)", K(cluster_id)); - } - - return ret; -} - -int ObLogRouteService::start() -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLogRouteService has not been inited", KR(ret)); - } else { - is_stopped_ = false; - LOG_INFO("ObLogRouteService start succ (simplified for single machine)"); - } - - return ret; -} - -void ObLogRouteService::stop() -{ - LOG_INFO("ObLogRouteService stop begin"); - is_stopped_ = true; - LOG_INFO("ObLogRouteService stop finish"); -} - -void ObLogRouteService::wait() -{ - LOG_INFO("ObLogRouteService wait begin"); - LOG_INFO("ObLogRouteService wait finish"); -} - -void ObLogRouteService::destroy() -{ - LOG_INFO("ObLogRouteService destroy begin"); - cluster_id_ = OB_INVALID_CLUSTER_ID; - is_inited_ = false; - LOG_INFO("ObLogRouteService destroy finish"); -} - -void ObLogRouteService::handle(void *task) -{ - UNUSED(task); - // For single machine: no async task to handle - LOG_TRACE("handle ignored for single machine"); -} - -int ObLogRouteService::next_server( - const share::ObLSID &ls_id, - const palf::LSN &next_lsn, - common::ObAddr &svr) -{ - int ret = OB_SUCCESS; - UNUSED(ls_id); - UNUSED(next_lsn); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); - } else { - // Read restore source from config parameter - common::ObAddr restore_source; - if (OB_FAIL(get_restore_source_addr_(restore_source))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_ITER_END; - LOG_WARN("restore_source is not configured, return OB_ITER_END", KR(ret)); - } - } else if (next_server_called_) { - // Already returned once, return OB_ITER_END to indicate iteration complete - ret = OB_ITER_END; - next_server_called_ = false; // Reset for next iteration round - LOG_TRACE("next_server return OB_ITER_END (iteration complete)"); - } else { - svr = restore_source; - next_server_called_ = true; - LOG_TRACE("next_server return restore_source", K(svr)); - } - } - - return ret; -} - -int ObLogRouteService::get_restore_source_addr_(common::ObAddr &addr) const -{ - // backup/restore (log_restore_source) removed: no restore source address available - UNUSED(addr); - return OB_ENTRY_NOT_EXIST; -} - -int ObLogRouteService::get_server_array_for_locate_start_lsn( - const share::ObLSID &ls_id, - ObIArray &svr_array) -{ - int ret = OB_SUCCESS; - UNUSED(ls_id); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); - } else { - svr_array.reset(); - common::ObAddr restore_source; - if (OB_FAIL(get_restore_source_addr_(restore_source))) { - LOG_WARN("restore_source is not configured", KR(ret)); - } else if (OB_FAIL(svr_array.push_back(restore_source))) { - LOG_WARN("push_back restore_source failed", KR(ret), K(restore_source)); - } else { - LOG_TRACE("get_server_array_for_locate_start_lsn return restore_source", K(restore_source)); - } - } - - return ret; -} - -bool ObLogRouteService::need_switch_server( - const share::ObLSID &ls_id, - const palf::LSN &next_lsn, - const common::ObAddr &cur_svr) -{ - UNUSED(ls_id); - UNUSED(next_lsn); - - // For single machine, single tenant, single LS: never need to switch server - // Only return true if current server is not restore_source - common::ObAddr restore_source; - bool bool_ret = false; - if (OB_SUCCESS == get_restore_source_addr_(restore_source)) { - bool_ret = (restore_source.is_valid() && cur_svr != restore_source); - } - LOG_TRACE("need_switch_server", K(bool_ret), K(cur_svr), K(restore_source)); - return bool_ret; -} - -int ObLogRouteService::get_server_count( - const share::ObLSID &ls_id, - int64_t &avail_svr_count) const -{ - int ret = OB_SUCCESS; - UNUSED(ls_id); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); - } else { - common::ObAddr restore_source; - if (OB_SUCCESS == get_restore_source_addr_(restore_source)) { - avail_svr_count = 1; - } else { - avail_svr_count = 0; - } - LOG_TRACE("get_server_count return", K(avail_svr_count)); - } - - return ret; -} - -int ObLogRouteService::async_server_query_req( - const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - UNUSED(ls_id); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); - } else { - // For single machine, single tenant, single LS: no need to query server - LOG_TRACE("async_server_query_req ignored for single machine"); - } - - return ret; -} - -ObLogRouteService::ObLSRouteTimerTask::ObLSRouteTimerTask(ObLogRouteService &log_route_service) : - is_inited_(false), - log_route_service_(log_route_service) -{} - -int ObLogRouteService::ObLSRouteTimerTask::init(int thread_id) -{ - int ret = OB_SUCCESS; - - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_ERROR("ObLSRouteTimerTask has already been inited", KR(ret), K(thread_id)); - } else { - is_inited_ = true; - LOG_INFO("ls_route_timer_task inited", K(thread_id)); - } - - return ret; -} - -void ObLogRouteService::ObLSRouteTimerTask::destroy() -{ - is_inited_ = false; -} - -void ObLogRouteService::ObLSRouteTimerTask::runTimerTask() -{ - // For single machine: no timer task needed - LOG_TRACE("ls_route_timer_task ignored for single machine"); -} - -} // namespace logservice -} // namespace oceanbase diff --git a/src/logservice/logrouteservice/ob_log_route_service.h b/src/logservice/logrouteservice/ob_log_route_service.h deleted file mode 100755 index a53c39668..000000000 --- a/src/logservice/logrouteservice/ob_log_route_service.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef OCEANBASE_LOG_ROUTE_SERVICE_H_ -#define OCEANBASE_LOG_ROUTE_SERVICE_H_ - -#include "logservice/palf/lsn.h" // LSN -#include "share/ob_server_struct.h" // GCTX - -namespace oceanbase -{ -namespace logservice -{ -/* - * LogRouteService is used for OB-CDC and Standby cluster etc to fetch log. Supports multiple policies, such as: - * - * 1. Select machines based on multiple replicated type policies; - * 3. Fetch log to the standby replica preferentially to minimize the impact on the leader. - * If logs cannot be obtained from all replica, logs can be obtained from the leader; - * 4. Complete blacklist mechanism, including internal automatic maintenance of Server blacklist and whitewashing mechanism; - * Supports the list of servers for filtering external configurations; - * 5. Perfect switch server mechanism, detection of serviceable machines, automatic fault switching, - * to ensure availability and real-time; - * 6. Supports filtering encrypts Zone to avoid unnecessary access; - * 7. Supports filtering inactive servers. - * ...... - * - */ -class ObLogRouteService -{ -public: - ObLogRouteService(); - virtual ~ObLogRouteService(); - - // @param [in] proxy ObMySQLProxy - // @param [in] cluster_id ClusterID - // @param [in] is_across_cluster Whether SQL queries cross cluster - // For Standby Cluster, need to set is_across_cluster=true. So SQL Query will contain cluster_id. - // For CDC, need to set is_across_cluster=false. So SQL Query will not contain cluster_id. - // @param [in] external_server_blacklist External server blacklist - // @param [in] background_refresh_time_sec Background periodic refresh time, in second - // @param [in] all_server_cache_update_interval_sec AllServer periodic refresh time, in second - // @param [in] all_zone_cache_update_interval_sec AllZone periodic refresh time, in second - // @param [in] blacklist_survival_time_sec The survival time of the Server in the blacklist, in seconds - // @param [in] blacklist_survival_time_upper_limit_min Upper threshold of the Server blacklist duration (in minute) - // @param [in] blacklist_survival_time_penalty_period_min - // The server is blacklisted, based on the time of the current server service LS - to decide whether to penalize - // the survival time. - // When the service time is less than a certain interval, a doubling-live-time policy is adopted (in minutes) - // @param [in] blacklist_history_overdue_time_min Blacklist history expiration time, used to delete history(in minutes) - // @param [in] blacklist_history_clear_interval_min Clear blacklist history period(in minutes) - // - // @retval OB_SUCCESS Success - // @retval Other return values Failed - int init(ObISQLClient *proxy, - const int64_t cluster_id, - const bool is_across_cluster, - void *err_handler, - const char *external_server_blacklist = "|", - const int64_t background_refresh_time_sec = 1200, - const int64_t all_server_cache_update_interval_sec = 5, - const int64_t all_zone_cache_update_interval_sec = 5, - const int64_t blacklist_survival_time_sec = 30, - const int64_t blacklist_survival_time_upper_limit_min = 4, - const int64_t blacklist_survival_time_penalty_period_min = 1, - const int64_t blacklist_history_overdue_time_min = 30, - const int64_t blacklist_history_clear_interval_min = 20); - int start(); - void stop(); - void wait(); - void destroy(); - void handle(void *task); - -public: - // Get the server synchronously, iterate over the server for the next service log. - // If the local cache does not exist, the SQL query is triggered and construct struct to insert. - // - // 1. If the server has completed one round of iteration (all servers have been iterated over), then OB_ITER_END is returned - // 2. After returning OB_ITER_END, the list of servers will be iterated over from the beginning next time - // 3. If no servers are available, return OB_ITER_END - // - // @param [in] ls_id LS ID - // @param [in] next_lsn next LSN - // @param [out] svr return fetch log server - // - // @retval OB_SUCCESS Success - // @retval OB_ITER_END Server list iterations complete one round, need to query server - // @retval Other return values Failed, may be network anomaly and so on. - int next_server( - const share::ObLSID &ls_id, - const palf::LSN &next_lsn, - common::ObAddr &svr); - - // @param [in] ls_id LS ID - // @param [out] svr_array return fetch log server - // - // @retval OB_SUCCESS Success - // @retval Other return values Failed - int get_server_array_for_locate_start_lsn( - const share::ObLSID &ls_id, - ObIArray &svr_array); - - // Determine if the server needs to be switched - // - // @param [in] ls_id LS ID - // @param [in] next_lsn next LSN - // @param [in] cur_svr current fetch log server - // - // @retval true Need to switch server - // @retval false Don't need to switch server - bool need_switch_server( - const share::ObLSID &ls_id, - const palf::LSN &next_lsn, - const common::ObAddr &cur_svr); - - // Get available server count - // - // @param [in] ls_id LS ID - // @param [out] avail_svr_count available server count - // - // @retval OB_SUCCESS Success - // @retval Other error codes Fail - int get_server_count( - const share::ObLSID &ls_id, - int64_t &avail_svr_count) const; - - // Launch an asynchronous update task for the server list of LS - // - // @param [in] ls_id LS ID - // - // @retval OB_SUCCESS Success - // @retval OB_EAGAIN thread queue is already full - // @retval Other return values Failed - int async_server_query_req( - const share::ObLSID &ls_id); - -private: - // Read the configured restore source (log_restore_source = $IP:$RPC_PORT) and return its address. - int get_restore_source_addr_(common::ObAddr &addr) const; - - class ObLSRouteTimerTask : public common::ObTimerTask - { - public: - explicit ObLSRouteTimerTask(ObLogRouteService &log_route_service); - virtual ~ObLSRouteTimerTask() {} - int init(int thread_id); - void destroy(); - virtual void runTimerTask() override; - private: - bool is_inited_; - ObLogRouteService &log_route_service_; - }; - -private: - bool is_inited_; - int64_t cluster_id_; - volatile bool is_stopped_ CACHE_ALIGNED; - ObLSRouteTimerTask ls_route_timer_task_; - // For single machine: track if next_server has been called to return OB_ITER_END on second call - bool next_server_called_; - - DISALLOW_COPY_AND_ASSIGN(ObLogRouteService); -}; - -} // namespace logservice -} // namespace oceanbase - -#endif diff --git a/src/logservice/logrpc/ob_log_request_handler.cpp b/src/logservice/logrpc/ob_log_request_handler.cpp deleted file mode 100755 index e582bcfab..000000000 --- a/src/logservice/logrpc/ob_log_request_handler.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "ob_log_request_handler.h" -#include "share/rc/ob_module_provider.h" -#include "logservice/ob_log_service.h" - -namespace oceanbase -{ -namespace logservice -{ -LogRequestHandler::LogRequestHandler() -{ -} - -LogRequestHandler::~LogRequestHandler() -{ -} - -int LogRequestHandler::get_palf_handle_guard_(const int64_t palf_id, - palf::PalfHandleGuard &palf_handle_guard) const -{ - int ret = OB_SUCCESS; - logservice::ObLogService *log_service = nullptr; - share::ObLSID ls_id(palf_id); - if (OB_ISNULL(log_service = share::g_mp->log_service())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "get_log_service failed", K(ret)); - } else if (OB_FAIL(log_service->open_palf(ls_id, palf_handle_guard))) { - CLOG_LOG(WARN, "open palf failed", K(ret), K(palf_id)); - } - return ret; -} - -int LogRequestHandler::get_log_handler_( - const int64_t palf_id, - storage::ObLSHandle &ls_handle, - logservice::ObLogHandler *&log_handler) const -{ - log_handler = nullptr; - storage::ObLSService *ls_svr = share::g_mp->ls_service(); - storage::ObLS *ls = nullptr; - share::ObLSID ls_id(palf_id); - int ret = OB_SUCCESS; - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::LOG_MOD))) { - CLOG_LOG(WARN, "get ls failed", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "ls should not be null", KR(ret)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "log_handler is null", KR(ret), K(ls_id)); - } - return ret; -} - -int LogRequestHandler::get_self_addr_(common::ObAddr &self) const -{ - int ret = OB_SUCCESS; - const common::ObAddr self_addr = GCTX.self_addr(); - if (false == self_addr.is_valid()) { - ret = OB_ERR_UNEXPECTED; - } else { - self = self_addr; - } - return ret; -} - -template <> -int LogRequestHandler::handle_sync_request( - const LogConfigChangeCmd &req, - LogConfigChangeCmdResp &resp) -{ - int ret = common::OB_SUCCESS; - if (false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); - } else { - const int64_t palf_id = req.palf_id_; - const common::ObAddr &server = req.src_; - palf::PalfHandleGuard palf_handle_guard; - if (OB_FAIL(get_palf_handle_guard_(palf_id, palf_handle_guard))) { - CLOG_LOG(WARN, "get_palf_handle_guard_ failed", K(ret), K(palf_id)); - } else { - palf::PalfHandle *palf_handle = palf_handle_guard.get_palf_handle(); - ConfigChangeCmdHandler cmd_handler(palf_handle); - if (OB_FAIL(cmd_handler.handle_config_change_cmd(req, resp))) { - CLOG_LOG(WARN, "handle_config_change_cmd failed", K(ret), K(palf_id), K(server), K(req)); - } else { - CLOG_LOG(INFO, "handle_config_change_cmd success", K(ret), K(palf_id), K(server), K(req), K(resp)); - } - resp.ret_ = ret; - ret = OB_SUCCESS; - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_sync_request( - const LogGetPalfStatReq &req, - LogGetPalfStatResp &resp) -{ - int ret = common::OB_SUCCESS; - if (false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); - } else { - palf::PalfHandleGuard palf_handle_guard; - const int64_t palf_id = req.palf_id_; - const common::ObAddr &server = req.src_; - common::ObRole role = FOLLOWER; - int64_t unused_pid; - bool is_pending_state = true; - int64_t paxos_replica_num = 0; - if (OB_FAIL(get_palf_handle_guard_(palf_id, palf_handle_guard))) { - CLOG_LOG(WARN, "get_palf_handle_guard_ failed", K(ret), K(palf_id)); - } else if (req.is_to_leader_ && OB_FAIL(palf_handle_guard.get_role(role, unused_pid, is_pending_state))) { - CLOG_LOG(WARN, "palf_handle get_role failed", K(ret), K(palf_id), K(server)); - } else if (req.is_to_leader_ && (role != LEADER || true == is_pending_state)) { - ret = OB_NOT_MASTER; - CLOG_LOG(WARN, "get_palf_stat failed", K(ret), K(req), K(role), K(is_pending_state)); - } else if (OB_FAIL(palf_handle_guard.stat(resp.palf_stat_))) { - CLOG_LOG(WARN, "palf stat failed", K(ret), K(palf_id), K(server)); - } else { - CLOG_LOG(TRACE, "get_palf_stat success", K(ret), K(palf_id), K(server), K(req), K(resp)); - } - } - return ret; -} - -int ConfigChangeCmdHandler::handle_config_change_cmd(const LogConfigChangeCmd &req, - LogConfigChangeCmdResp &resp) const -{ - int ret = OB_SUCCESS; - if (NULL == palf_handle_) { - ret = OB_NOT_INIT; - } else if (false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else { - switch (req.cmd_type_) { - case FORCE_SINGLE_MEMBER_CMD: - ret = palf_handle_->force_set_as_single_replica(); - break; - case FORCE_SET_MEMBER_LIST_CMD: - ret = palf_handle_->force_set_member_list(req.new_member_list_, req.new_replica_num_); - break; - case CHANGE_REPLICA_NUM_CMD: - ret = palf_handle_->change_replica_num(req.curr_member_list_, req.curr_replica_num_, - req.new_replica_num_, req.timeout_us_); - break; - case ADD_MEMBER_CMD: - ret = palf_handle_->add_member(req.added_member_, req.new_replica_num_, req.config_version_, req.timeout_us_); - break; - case REMOVE_MEMBER_CMD: - ret = palf_handle_->remove_member(req.removed_member_, req.new_replica_num_, req.timeout_us_); - break; - case REPLACE_MEMBER_CMD: - ret = palf_handle_->replace_member(req.added_member_, req.removed_member_, req.config_version_, req.timeout_us_); - break; - case ADD_LEARNER_CMD: - ret = palf_handle_->add_learner(req.added_member_, req.timeout_us_); - break; - case REMOVE_LEARNER_CMD: - ret = palf_handle_->remove_learner(req.removed_member_, req.timeout_us_); - break; - case SWITCH_TO_ACCEPTOR_CMD: - ret = palf_handle_->switch_learner_to_acceptor(req.added_member_, req.new_replica_num_, req.config_version_, req.timeout_us_); - break; - case SWITCH_TO_LEARNER_CMD: - ret = palf_handle_->switch_acceptor_to_learner(req.removed_member_, req.new_replica_num_, req.timeout_us_); - break; - case TRY_LOCK_CONFIG_CHANGE_CMD: - ret = palf_handle_->try_lock_config_change(req.lock_owner_, req.timeout_us_); - break; - case UNLOCK_CONFIG_CHANGE_CMD: - ret = palf_handle_->unlock_config_change(req.lock_owner_, req.timeout_us_); - break; - case GET_CONFIG_CHANGE_LOCK_STAT_CMD: - ret = palf_handle_->get_config_change_lock_stat(resp.lock_owner_, resp.is_locked_); - break; - case REPLACE_LEARNERS_CMD: - ret = palf_handle_->replace_learners(req.added_list_, req.removed_list_, req.timeout_us_); - break; - case REPLACE_MEMBER_WITH_LEARNER_CMD: - ret = palf_handle_->replace_member_with_learner(req.added_member_, req.removed_member_, req.config_version_, req.timeout_us_); - break; - default: - break; - } - } - resp.ret_ = ret; - return ret; -} - -template <> -int LogRequestHandler::handle_request(const LogChangeAccessModeCmd &req) -{ - int ret = common::OB_SUCCESS; - if (false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); - } else if (OB_FAIL(change_access_mode_(req))) { - CLOG_LOG(WARN, "handle_request fail", K(ret), K(req)); - } else { - CLOG_LOG(TRACE, "handle_request success", K(ret), K(req)); - } - return ret; -} - -int LogRequestHandler::change_access_mode_(const LogChangeAccessModeCmd &req) -{ - int ret = OB_SUCCESS; - const int64_t palf_id = req.ls_id_; - const common::ObAddr &server = req.src_; - storage::ObLSHandle ls_handle; - logservice::ObLogHandler *log_handler = nullptr; - if (OB_FAIL(get_log_handler_(palf_id, ls_handle, log_handler))) { - CLOG_LOG(WARN, "get_log_handler_ failed", K(ret), K(palf_id), K(server)); - } else if (OB_FAIL(log_handler->change_access_mode(req.mode_version_, req.access_mode_, req.ref_scn_))) { - CLOG_LOG(WARN, "change_access_mode failed", K(ret), K(palf_id), K(server)); - } else { - CLOG_LOG(INFO, "change_access_mode success", K(ret), K(req)); - } - return ret; -} - -template <> -int LogRequestHandler::handle_sync_request( - const LogGetCkptReq &req, - LogGetCkptResp &resp) -{ - int ret = common::OB_SUCCESS; - storage::ObLSService *ls_svr = share::g_mp->ls_service(); - storage::ObLS *ls = nullptr; - const share::ObLSID &ls_id = req.ls_id_; - storage::ObLSHandle handle; - - if (false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); - } else if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::LOG_MOD))) { - CLOG_LOG(WARN, "get ls failed", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "ls should not be null", KR(ret)); - } else { - resp.ckpt_scn_ = ls->get_ls_meta().get_clog_checkpoint_scn(); - resp.ckpt_lsn_ = ls->get_ls_meta().get_clog_base_lsn(); - } - return ret; -} - -} // end namespace logservice -} // end namespace oceanbase diff --git a/src/logservice/logrpc/ob_log_request_handler.h b/src/logservice/logrpc/ob_log_request_handler.h deleted file mode 100644 index 4f1f0b9e4..000000000 --- a/src/logservice/logrpc/ob_log_request_handler.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_LOG_REQUEST_HANDLER_ -#define OCEANBASE_LOGSERVICE_OB_LOG_REQUEST_HANDLER_ - -#include "lib/ob_errno.h" // OB_SUCCESS... -#include "logservice/palf_handle_guard.h" // palf::PalfHandleGuard -#include "ob_log_rpc_req.h" // Req... - -namespace oceanbase -{ - -namespace palf -{ -class PalfEnv; -} - -namespace obcall -{ -class ObLogServiceRpcProxy; -} - -namespace logservice -{ -class ObLogHandler; - -class LogRequestHandler -{ -public: - LogRequestHandler(); - ~LogRequestHandler(); - template - int handle_sync_request(const ReqType &req, RespType &resp); - template - int handle_request(const ReqType &req); -private: - int get_palf_handle_guard_(const int64_t palf_id, palf::PalfHandleGuard &palf_handle_guard) const; - int get_self_addr_(common::ObAddr &self) const; - int get_log_handler_(const int64_t palf_id, - storage::ObLSHandle &ls_handle, - logservice::ObLogHandler *&log_handler) const; - int change_access_mode_(const LogChangeAccessModeCmd &req); -}; - -class ConfigChangeCmdHandler{ -public: - explicit ConfigChangeCmdHandler(palf::PalfHandle *palf_handle) - { - if (NULL != palf_handle) { - palf_handle_ = palf_handle; - } - } - ~ConfigChangeCmdHandler() - { - palf_handle_ = NULL; - } - int handle_config_change_cmd(const LogConfigChangeCmd &req, - LogConfigChangeCmdResp &resp) const; -private: - palf::PalfHandle *palf_handle_; -}; -} // end namespace logservice -} // end namespace oceanbase - -#endif diff --git a/src/logservice/logrpc/ob_log_rpc_arg.cpp b/src/logservice/logrpc/ob_log_rpc_arg.cpp deleted file mode 100644 index da9d60a46..000000000 --- a/src/logservice/logrpc/ob_log_rpc_arg.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#define USING_LOG_PREFIX RPC -#include "logservice/logrpc/ob_log_rpc_arg.h" -namespace oceanbase -{ -namespace obcall -{ -OB_SERIALIZE_MEMBER(ObFetchStableMemberListArg, ls_id_); -OB_SERIALIZE_MEMBER(ObFetchStableMemberListInfo, member_list_, config_version_); -OB_SERIALIZE_MEMBER(ObLSAccessModeInfo, ls_id_, mode_version_, - access_mode_, ref_scn_, addr_, sys_ls_end_scn_); -} // namespace obcall -} // namespace oceanbase diff --git a/src/logservice/logrpc/ob_log_rpc_arg.h b/src/logservice/logrpc/ob_log_rpc_arg.h deleted file mode 100644 index b6154984a..000000000 --- a/src/logservice/logrpc/ob_log_rpc_arg.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef OCEANBASE_LOGSERVICE_LOGRPC_OB_LOG_RPC_ARG_H_ -#define OCEANBASE_LOGSERVICE_LOGRPC_OB_LOG_RPC_ARG_H_ -// moved from share/ob_rpc_struct.h:palf member-list/access-mode RPC arguments -// (owner=logservice;ns obrpc unchanged,no semantic change for consumers;only a lightweight header,does not pull in the palf implementation) -#include "lib/utility/ob_unify_serialize.h" -#include "share/ob_ls_id.h" -#include "common/ob_member_list.h" -#include "logservice/palf/palf_options.h" -#include "logservice/palf/log_meta_info.h" -namespace oceanbase -{ -namespace obcall -{ -struct ObLSAccessModeInfo -{ - OB_UNIS_VERSION(1); -public: - ObLSAccessModeInfo(): ls_id_(), - mode_version_(palf::INVALID_PROPOSAL_ID), - access_mode_(palf::AccessMode::INVALID_ACCESS_MODE), - ref_scn_(), - addr_(), - sys_ls_end_scn_() {} - ~ObLSAccessModeInfo() {} - bool is_valid() const; - int init(const share::ObLSID &ls_idd, - const int64_t mode_version, - const palf::AccessMode &access_mode, - const share::SCN &ref_scn, - const share::SCN &sys_ls_end_scn); - int assign(const ObLSAccessModeInfo &other); - TO_STRING_KV(K_(ls_id), K_(mode_version), - K_(access_mode), K_(ref_scn), K_(sys_ls_end_scn)); - share::ObLSID get_ls_id() const - { - return ls_id_; - } - palf::AccessMode get_access_mode() const - { - return access_mode_; - } - int64_t get_mode_version() const - { - return mode_version_; - } - const share::SCN &get_ref_scn() const - { - return ref_scn_; - } - const share::SCN &get_sys_ls_end_scn() const - { - return sys_ls_end_scn_; - } -private: - DISALLOW_COPY_AND_ASSIGN(ObLSAccessModeInfo); -private: - share::ObLSID ls_id_; - int64_t mode_version_; - palf::AccessMode access_mode_; - share::SCN ref_scn_; - ObAddr addr_;//no used, add in 4200 RC1 - share::SCN sys_ls_end_scn_; // new arg in V4.2.0 -}; - -struct ObFetchStableMemberListArg final -{ - OB_UNIS_VERSION(1); -public: - ObFetchStableMemberListArg(): ls_id_() {} - ~ObFetchStableMemberListArg() {} - bool is_valid() const { return ls_id_.is_valid() && 1UL != OB_INVALID_TENANT_ID; } - void reset() { ls_id_.reset(); } - const share::ObLSID &get_ls_id() const { return ls_id_; } - TO_STRING_KV(K_(ls_id)); -private: - share::ObLSID ls_id_; -}; - -struct ObFetchStableMemberListInfo final -{ - OB_UNIS_VERSION(1); -public: - ObFetchStableMemberListInfo() : member_list_(), config_version_() {} - ~ObFetchStableMemberListInfo() {} - bool is_valid() const { return member_list_.is_valid() && config_version_.is_valid(); } - void reset() { member_list_.reset(); } - int init(const common::ObMemberList &member_list, const palf::LogConfigVersion &config_version); - const common::ObMemberList &get_member_list() const { return member_list_; } - const palf::LogConfigVersion &get_config_version() const { return config_version_; } - TO_STRING_KV(K_(member_list)); -private: - common::ObMemberList member_list_; - palf::LogConfigVersion config_version_; -}; -} // namespace obcall -} // namespace oceanbase -#endif diff --git a/src/logservice/logrpc/ob_log_rpc_req.cpp b/src/logservice/logrpc/ob_log_rpc_req.cpp deleted file mode 100644 index 7830d3921..000000000 --- a/src/logservice/logrpc/ob_log_rpc_req.cpp +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_log_rpc_req.h" - -namespace oceanbase -{ -using namespace share; -namespace logservice -{ - -// ============= LogConfigChangeCmd begin =========== -LogConfigChangeCmd::LogConfigChangeCmd() - : src_(), - palf_id_(-1), - added_member_(), - removed_member_(), - curr_member_list_(), - curr_replica_num_(0), - new_replica_num_(0), - cmd_type_(INVALID_CONFIG_CHANGE_CMD), - timeout_us_(0), - lock_owner_(palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - config_version_(), - added_list_(), - removed_list_(), - new_member_list_() { } - -LogConfigChangeCmd::LogConfigChangeCmd( - const common::ObAddr &src, - const int64_t palf_id, - const common::ObMember &added_member, - const common::ObMember &removed_member, - const int64_t new_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us) - : src_(src), - palf_id_(palf_id), - added_member_(added_member), - removed_member_(removed_member), - curr_member_list_(), - curr_replica_num_(), - new_replica_num_(new_replica_num), - cmd_type_(cmd_type), - timeout_us_(timeout_us), - lock_owner_(palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - config_version_(), - added_list_(), - removed_list_(), - new_member_list_() { } - -LogConfigChangeCmd::LogConfigChangeCmd( - const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us) - : src_(src), - palf_id_(palf_id), - added_member_(), - removed_member_(), - curr_member_list_(member_list), - curr_replica_num_(curr_replica_num), - new_replica_num_(new_replica_num), - cmd_type_(cmd_type), - timeout_us_(timeout_us), - lock_owner_(palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - config_version_(), - added_list_(), - removed_list_(), - new_member_list_() { } - -LogConfigChangeCmd::LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const int64_t lock_owner, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us) - : src_(src), - palf_id_(palf_id), - added_member_(), - removed_member_(), - curr_member_list_(), - curr_replica_num_(0), - new_replica_num_(0), - cmd_type_(cmd_type), - timeout_us_(timeout_us), - lock_owner_(lock_owner), - config_version_(), - added_list_(), - removed_list_(), - new_member_list_() { } - -LogConfigChangeCmd::LogConfigChangeCmd( - const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &added_list, - const common::ObMemberList &removed_list, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us) - : src_(src), - palf_id_(palf_id), - added_member_(), - removed_member_(), - curr_member_list_(), - curr_replica_num_(0), - new_replica_num_(0), - cmd_type_(cmd_type), - timeout_us_(timeout_us), - lock_owner_(palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - config_version_(), - added_list_(added_list), - removed_list_(removed_list), - new_member_list_() { } - -LogConfigChangeCmd::LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &new_member_list, - const int64_t new_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us) - : src_(src), - palf_id_(palf_id), - added_member_(), - removed_member_(), - curr_member_list_(), - curr_replica_num_(0), - new_replica_num_(new_replica_num), - cmd_type_(cmd_type), - timeout_us_(timeout_us), - lock_owner_(palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - config_version_(), - added_list_(), - removed_list_(), - new_member_list_(new_member_list) { } - -LogConfigChangeCmd::~LogConfigChangeCmd() -{ - reset(); -} - -void LogConfigChangeCmd::in_leader(const palf::LogConfigVersion &config_version) -{ - config_version_ = config_version; -} - -bool LogConfigChangeCmd::is_valid() const -{ - bool bool_ret = false; - bool_ret = (src_.is_valid() && palf_id_ >= 0 && timeout_us_ > 0 && \ - cmd_type_ != INVALID_CONFIG_CHANGE_CMD)? true: false; - bool_ret = bool_ret && ((is_add_member_list() || ADD_LEARNER_CMD == cmd_type_ || \ - SWITCH_TO_LEARNER_CMD == cmd_type_)? added_member_.is_valid(): true); - bool_ret = bool_ret && ((is_remove_member_list() || REMOVE_LEARNER_CMD == cmd_type_ || \ - SWITCH_TO_ACCEPTOR_CMD == cmd_type_)? removed_member_.is_valid(): true); - bool_ret = bool_ret && ((is_set_new_replica_num())? is_valid_replica_num(new_replica_num_): true); - bool_ret = bool_ret && ((CHANGE_REPLICA_NUM_CMD == cmd_type_)? curr_member_list_.is_valid() \ - && is_valid_replica_num(curr_replica_num_) && is_valid_replica_num(new_replica_num_): true);\ - bool_ret = bool_ret && ((TRY_LOCK_CONFIG_CHANGE_CMD == cmd_type_ || UNLOCK_CONFIG_CHANGE_CMD == cmd_type_) ? \ - (palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER != lock_owner_) : true); - bool_ret = bool_ret && ((REPLACE_LEARNERS_CMD == cmd_type_)? (added_list_.is_valid() \ - && removed_list_.is_valid()): true); - bool_ret = bool_ret && ((FORCE_SET_MEMBER_LIST_CMD == cmd_type_) ? (new_member_list_.is_valid() \ - && new_replica_num_ == new_member_list_.get_member_number()) : true); - return bool_ret; -} - -bool LogConfigChangeCmd::is_remove_member_list() const -{ - return REMOVE_MEMBER_CMD == cmd_type_ - || REPLACE_MEMBER_CMD == cmd_type_ - || SWITCH_TO_LEARNER_CMD == cmd_type_ - || REPLACE_MEMBER_WITH_LEARNER_CMD == cmd_type_; -} - -bool LogConfigChangeCmd::is_add_member_list() const -{ - return ADD_MEMBER_CMD == cmd_type_ - || REPLACE_MEMBER_CMD == cmd_type_ - || SWITCH_TO_ACCEPTOR_CMD == cmd_type_ - || REPLACE_MEMBER_WITH_LEARNER_CMD == cmd_type_; -} - -bool LogConfigChangeCmd::is_set_new_replica_num() const -{ - return ADD_MEMBER_CMD == cmd_type_ - || REMOVE_MEMBER_CMD == cmd_type_ - || SWITCH_TO_LEARNER_CMD == cmd_type_ - || SWITCH_TO_ACCEPTOR_CMD == cmd_type_ - || FORCE_SET_MEMBER_LIST_CMD == cmd_type_; -} - -void LogConfigChangeCmd::reset() -{ - src_.reset(); - palf_id_ = -1; - added_member_.reset(); - removed_member_.reset(); - curr_member_list_.reset(); - curr_replica_num_ = 0; - new_replica_num_ = 0; - cmd_type_ = INVALID_CONFIG_CHANGE_CMD; - timeout_us_ = 0; - lock_owner_ = palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - config_version_.reset(); - added_list_.reset(); - removed_list_.reset(); - new_member_list_.reset(); -} - -OB_SERIALIZE_MEMBER(LogConfigChangeCmd, src_, palf_id_, added_member_, removed_member_, -curr_member_list_, curr_replica_num_, new_replica_num_, cmd_type_, timeout_us_, lock_owner_, -config_version_, added_list_, removed_list_, new_member_list_); -// ============= LogConfigChangeCmd end ============= - -// ============= LogConfigChangeCmdResp begin =========== -LogConfigChangeCmdResp::LogConfigChangeCmdResp() -{ - reset(); -} - -LogConfigChangeCmdResp::~LogConfigChangeCmdResp() -{ - reset(); -} - -void LogConfigChangeCmdResp::reset() -{ - ret_ = OB_MAX_ERROR_CODE; - lock_owner_ = palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - is_locked_ = false; -} - -OB_SERIALIZE_MEMBER(LogConfigChangeCmdResp, ret_, lock_owner_, is_locked_); -// ============= LogConfigChangeCmdResp end ============= - -// ============= LogGetPalfStatReq begin =========== -LogGetPalfStatReq::LogGetPalfStatReq( - const common::ObAddr &src_addr, - const int64_t palf_id, - const bool is_to_leader) - : src_(src_addr), - palf_id_(palf_id), - is_to_leader_(is_to_leader) { } - -LogGetPalfStatReq::~LogGetPalfStatReq() -{ - reset(); -} - -bool LogGetPalfStatReq::is_valid() const -{ - return src_.is_valid() && palf_id_ >= 0; -} - -void LogGetPalfStatReq::reset() -{ - src_.reset(); - palf_id_ = -1; -} -OB_SERIALIZE_MEMBER(LogGetPalfStatReq, src_, palf_id_, is_to_leader_); -// ============= LogGetPalfStatReq end ============= - -// ============= LogGetPalfStatResp begin =========== -LogGetPalfStatResp::LogGetPalfStatResp( - const palf::PalfStat &palf_stat) - : palf_stat_(palf_stat) { } - -LogGetPalfStatResp::~LogGetPalfStatResp() -{ - reset(); -} - -void LogGetPalfStatResp::reset() -{ - palf_stat_.reset(); -} - -OB_SERIALIZE_MEMBER(LogGetPalfStatResp, palf_stat_); -// ============= LogGetPalfStatResp end ============= - -// ============= LogChangeAccessModeCmd start ============= -LogChangeAccessModeCmd::LogChangeAccessModeCmd() - : src_(), - ls_id_(-1), - mode_version_(palf::INVALID_PROPOSAL_ID), - access_mode_(palf::AccessMode::INVALID_ACCESS_MODE), - ref_scn_() { } - -LogChangeAccessModeCmd::LogChangeAccessModeCmd( - const common::ObAddr &src, - const int64_t ls_id, - const int64_t mode_version, - const palf::AccessMode &access_mode, - const SCN &ref_scn) - : src_(src), - ls_id_(ls_id), - mode_version_(mode_version), - access_mode_(access_mode), - ref_scn_(ref_scn) { } - -bool LogChangeAccessModeCmd::is_valid() const -{ - return src_.is_valid() && - -1 != ls_id_ && - palf::INVALID_PROPOSAL_ID != mode_version_ && - palf::AccessMode::INVALID_ACCESS_MODE != access_mode_ && - ref_scn_.is_valid(); -} - -void LogChangeAccessModeCmd::reset() -{ - src_.reset(); - ls_id_ = -1; - mode_version_ = palf::INVALID_PROPOSAL_ID; - access_mode_ = palf::AccessMode::INVALID_ACCESS_MODE; - ref_scn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogChangeAccessModeCmd, src_, ls_id_, mode_version_, access_mode_, ref_scn_); -// ============= LogChangeAccessModeCmd end ============= - -// ============= LogGetCkptReq begin =========== -LogGetCkptReq::LogGetCkptReq( - const common::ObAddr &src, - const share::ObLSID &ls_id) - : src_(src), - ls_id_(ls_id) { } - -LogGetCkptReq::~LogGetCkptReq() -{ - reset(); -} - -bool LogGetCkptReq::is_valid() const -{ - return src_.is_valid() && ls_id_.is_valid(); -} - -void LogGetCkptReq::reset() -{ - src_.reset(); - ls_id_.reset(); -} -OB_SERIALIZE_MEMBER(LogGetCkptReq, src_, ls_id_); -// ============= LogGetCkptReq end ============= - -// ============= LogGetCkptResp begin =========== -LogGetCkptResp::LogGetCkptResp( - const share::SCN &scn, - const palf::LSN &lsn) - : ckpt_scn_(scn), - ckpt_lsn_(lsn) { } - -LogGetCkptResp::~LogGetCkptResp() -{ - reset(); -} - -void LogGetCkptResp::reset() -{ - ckpt_scn_.reset(); - ckpt_lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogGetCkptResp, ckpt_scn_, ckpt_lsn_); -// ============= LogGetCkptResp end ============= - -// ================= LogSyncBaseLSNReq start ================ -LogSyncBaseLSNReq::LogSyncBaseLSNReq() -{ - reset(); -} - -LogSyncBaseLSNReq::~LogSyncBaseLSNReq() -{ - reset(); -} - -LogSyncBaseLSNReq::LogSyncBaseLSNReq(const common::ObAddr &src, - const share::ObLSID &id, - const palf::LSN &base_lsn) - : src_(src), ls_id_(id), base_lsn_(base_lsn) -{ -} - -bool LogSyncBaseLSNReq::is_valid() const -{ - return (src_.is_valid() &&ls_id_.is_valid() && base_lsn_.is_valid()); -} - -void LogSyncBaseLSNReq::reset() -{ - src_.reset(); - ls_id_.reset(); - base_lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogSyncBaseLSNReq, src_, ls_id_, base_lsn_); - -// ================= LogSyncBaseLSNReq end ================ -} // end namespace logservice -}// end namespace oceanbase diff --git a/src/logservice/logrpc/ob_log_rpc_req.h b/src/logservice/logrpc/ob_log_rpc_req.h deleted file mode 100644 index c8640beed..000000000 --- a/src/logservice/logrpc/ob_log_rpc_req.h +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_LOG_RPC_REQ_H_ -#define OCEANBASE_LOGSERVICE_OB_LOG_RPC_REQ_H_ - -#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION -#include "lib/utility/ob_print_utils.h" // TO_STRING_KV -#include "common/ob_member_list.h" // ObMemberList -#include "logservice/palf/palf_options.h" // AccessMode -#include "logservice/palf/palf_handle_impl.h" // PalfStat -#include "share/scn.h" - -namespace oceanbase -{ -namespace logservice -{ - -enum LogConfigChangeCmdType { - INVALID_CONFIG_CHANGE_CMD = 0, - CHANGE_REPLICA_NUM_CMD, - ADD_MEMBER_CMD, - REMOVE_MEMBER_CMD, - REPLACE_MEMBER_CMD, - ADD_LEARNER_CMD, - REMOVE_LEARNER_CMD, - SWITCH_TO_ACCEPTOR_CMD,//discarded - SWITCH_TO_LEARNER_CMD, - FORCE_SINGLE_MEMBER_CMD, - TRY_LOCK_CONFIG_CHANGE_CMD, - UNLOCK_CONFIG_CHANGE_CMD, - GET_CONFIG_CHANGE_LOCK_STAT_CMD, - REPLACE_LEARNERS_CMD, - REPLACE_MEMBER_WITH_LEARNER_CMD, - FORCE_SET_MEMBER_LIST_CMD, -}; - -inline const char *log_config_change_cmd2str(const LogConfigChangeCmdType state) -{ - #define CHECK_CMD_TYPE_STR(x) case(LogConfigChangeCmdType::x): return #x - switch(state) - { - CHECK_CMD_TYPE_STR(ADD_MEMBER_CMD); - CHECK_CMD_TYPE_STR(REMOVE_MEMBER_CMD); - CHECK_CMD_TYPE_STR(REPLACE_MEMBER_CMD); - CHECK_CMD_TYPE_STR(ADD_LEARNER_CMD); - CHECK_CMD_TYPE_STR(REMOVE_LEARNER_CMD); - CHECK_CMD_TYPE_STR(SWITCH_TO_ACCEPTOR_CMD); - CHECK_CMD_TYPE_STR(SWITCH_TO_LEARNER_CMD); - CHECK_CMD_TYPE_STR(CHANGE_REPLICA_NUM_CMD); - CHECK_CMD_TYPE_STR(TRY_LOCK_CONFIG_CHANGE_CMD); - CHECK_CMD_TYPE_STR(UNLOCK_CONFIG_CHANGE_CMD); - CHECK_CMD_TYPE_STR(GET_CONFIG_CHANGE_LOCK_STAT_CMD); - CHECK_CMD_TYPE_STR(REPLACE_LEARNERS_CMD); - CHECK_CMD_TYPE_STR(REPLACE_MEMBER_WITH_LEARNER_CMD); - CHECK_CMD_TYPE_STR(FORCE_SET_MEMBER_LIST_CMD); - default: - return "Invalid"; - } - #undef CHECK_CMD_TYPE_STR -} - -struct LogConfigChangeCmd { - OB_UNIS_VERSION(1); -public: - LogConfigChangeCmd(); - LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const common::ObMember &added_member, - const common::ObMember &removed_member, - const int64_t paxos_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us); - LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us); - LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const int64_t lock_owner, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us); - LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &added_list, - const common::ObMemberList &removed_list, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us); - LogConfigChangeCmd(const common::ObAddr &src, - const int64_t palf_id, - const common::ObMemberList &new_member_list, - const int64_t new_replica_num, - const LogConfigChangeCmdType cmd_type, - const int64_t timeout_us); - ~LogConfigChangeCmd(); - bool is_valid() const; - void reset(); - bool is_remove_member_list() const; - bool is_add_member_list() const; - void in_leader(const palf::LogConfigVersion &config_version); - bool is_set_new_replica_num() const; - TO_STRING_KV("cmd_type", log_config_change_cmd2str(cmd_type_), K_(src), K_(palf_id), \ - K_(added_member), K_(removed_member), K_(curr_member_list), K_(curr_replica_num), \ - K_(new_replica_num), K_(timeout_us), K_(lock_owner), K_(config_version), \ - K_(added_list), K_(removed_list), K_(new_member_list)); - common::ObAddr src_; - int64_t palf_id_; - common::ObMember added_member_; - common::ObMember removed_member_; - common::ObMemberList curr_member_list_; - int64_t curr_replica_num_; - int64_t new_replica_num_; - LogConfigChangeCmdType cmd_type_; - int64_t timeout_us_; - int64_t lock_owner_; - palf::LogConfigVersion config_version_; - common::ObMemberList added_list_; - common::ObMemberList removed_list_; - common::ObMemberList new_member_list_; -}; - -struct LogConfigChangeCmdResp { - OB_UNIS_VERSION(1); -public: - LogConfigChangeCmdResp(); - ~LogConfigChangeCmdResp(); - void reset(); - TO_STRING_KV(K_(ret), K_(lock_owner), K_(is_locked)); -public: - int ret_; - int64_t lock_owner_; - bool is_locked_; -}; - -struct LogGetPalfStatReq { - OB_UNIS_VERSION(1); -public: - LogGetPalfStatReq(): src_(), palf_id_(-1), is_to_leader_(false) { } - LogGetPalfStatReq(const common::ObAddr &src, - const int64_t palf_id, - const bool is_to_leader); - ~LogGetPalfStatReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(src), K_(palf_id), K_(is_to_leader)); - common::ObAddr src_; - int64_t palf_id_; - bool is_to_leader_; -}; - -struct LogGetPalfStatResp { - OB_UNIS_VERSION(1); -public: - LogGetPalfStatResp() : palf_stat_() { } - LogGetPalfStatResp(const palf::PalfStat &palf_stat); - ~LogGetPalfStatResp(); - void reset(); - TO_STRING_KV(K_(palf_stat)); - palf::PalfStat palf_stat_; -}; - -enum LogServerProbeType -{ - PROBE_REQ = 0, - PROBE_RESP, -}; - -struct LogChangeAccessModeCmd { - OB_UNIS_VERSION(1); -public: - LogChangeAccessModeCmd(); - LogChangeAccessModeCmd(const common::ObAddr &src, - const int64_t ls_id, - const int64_t mode_version, - const palf::AccessMode &access_mode, - const share::SCN &ref_scn); - ~LogChangeAccessModeCmd() - { - reset(); - } - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(src), K_(ls_id), K_(mode_version), \ - K_(access_mode), K_(ref_scn)); - common::ObAddr src_; - int64_t ls_id_; - int64_t mode_version_; - palf::AccessMode access_mode_; - share::SCN ref_scn_; -}; - -struct LogGetCkptReq { - OB_UNIS_VERSION(1); -public: - LogGetCkptReq(): src_(), ls_id_() { } - LogGetCkptReq(const common::ObAddr &src, - const share::ObLSID &ls_id); - ~LogGetCkptReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(src), K_(ls_id)); - common::ObAddr src_; - share::ObLSID ls_id_; -}; - -struct LogGetCkptResp { - OB_UNIS_VERSION(1); -public: - LogGetCkptResp() : ckpt_scn_(), ckpt_lsn_() { } - LogGetCkptResp(const share::SCN &scn, const palf::LSN &lsn); - ~LogGetCkptResp(); - void reset(); - TO_STRING_KV(K_(ckpt_scn), K_(ckpt_lsn)); - share::SCN ckpt_scn_; - palf::LSN ckpt_lsn_; -}; - -struct LogSyncBaseLSNReq -{ -public: - OB_UNIS_VERSION(1); -public: - LogSyncBaseLSNReq(); - LogSyncBaseLSNReq(const common::ObAddr &src, const share::ObLSID &id, - const palf::LSN &base_lsn); - ~LogSyncBaseLSNReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(src), K_(ls_id), K_(base_lsn)); -public: - common::ObAddr src_; - share::ObLSID ls_id_; - palf::LSN base_lsn_; -}; - -} // end namespace logservice -}// end namespace oceanbase - -#endif diff --git a/src/logservice/ob_location_adapter.cpp b/src/logservice/ob_location_adapter.cpp deleted file mode 100644 index 9e01c8f36..000000000 --- a/src/logservice/ob_location_adapter.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#define USING_LOG_PREFIX CLOG -#include "ob_location_adapter.h" -#include "share/location_cache/ob_location_service.h" - -namespace oceanbase -{ -using namespace share; -namespace logservice -{ - -ObLocationAdapter::ObLocationAdapter() : - is_inited_(false), - location_service_(NULL) - {} - -ObLocationAdapter::~ObLocationAdapter() -{ - destroy(); -} - -int ObLocationAdapter::init(ObLocationService *location_service) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - CLOG_LOG(WARN, "ObLocationAdapter init twice", K(ret)); - } else if (OB_ISNULL(location_service)) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid arguments", K(location_service), K(ret)); - } else { - location_service_ = location_service; - is_inited_ = true; - PALF_LOG(INFO, "ObLocationAdapter init success", K(ret), K(location_service_)); - } - return ret; -} - -void ObLocationAdapter::destroy() -{ - is_inited_ = false; - location_service_ = NULL; -} - -int ObLocationAdapter::get_leader(const int64_t id, common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - int64_t cluster_id = 1L; // obcall CLUSTER_ID inlined - - ObLSID ls_id(id); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "location adapter is not inited", K(ret)); - } else if (OB_FAIL(location_service_->get_leader( - cluster_id, ls_id, false, leader))) { - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - CLOG_LOG(WARN, "location adapter get_leader failed", K(ret), K(id), K(leader)); - } - } else { - // do nothing - } - return ret; -} -int ObLocationAdapter::nonblock_get_leader(int64_t id, - common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - int64_t cluster_id = 1L; // obcall CLUSTER_ID inlined - ObLSID ls_id(id); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "location adapter is not inited", K(ret)); - } else if (OB_FAIL(location_service_->nonblock_get_leader( - cluster_id, ls_id, leader))) { - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - CLOG_LOG(WARN, "location adapter nonblock_get_leader failed", K(ret), K(id), K(leader)); - } - } else { - // do nothing - } - return ret; -} - -bool ObLocationAdapter::is_location_service_renew_error(const int err) const -{ - return err == OB_LS_LOCATION_NOT_EXIST - || err == OB_LS_LOCATION_LEADER_NOT_EXIST - || err == OB_GET_LOCATION_TIME_OUT; -} -} -} diff --git a/src/logservice/ob_location_adapter.h b/src/logservice/ob_location_adapter.h deleted file mode 100644 index b8a599323..000000000 --- a/src/logservice/ob_location_adapter.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_LOCATION_ADAPTER_H_ -#define OCEANBASE_LOGSERVICE_OB_LOCATION_ADAPTER_H_ - -#include -#include "logservice/palf/palf_callback.h" -#include "lib/net/ob_addr.h" - -namespace oceanbase -{ -namespace share -{ -class ObLocationService; -} -namespace logservice -{ -class ObLocationAdapter : public palf::PalfLocationCacheCb -{ -public: - ObLocationAdapter(); - virtual ~ObLocationAdapter(); - int init(share::ObLocationService *location_service); - void destroy(); -public: - // Gets leader address of a log stream synchronously. - int get_leader(const int64_t id, common::ObAddr &leader) override final; - // Nonblock way to get leader address of the log stream. - int nonblock_get_leader(int64_t id, common::ObAddr &leader) override final; - bool is_location_service_renew_error(const int err) const; -private: - bool is_inited_; - share::ObLocationService *location_service_; -}; - -} // logservice -} // oceanbase - -#endif diff --git a/src/logservice/ob_log_base_header.h b/src/logservice/ob_log_base_header.h index d63012452..cb9ac44cc 100644 --- a/src/logservice/ob_log_base_header.h +++ b/src/logservice/ob_log_base_header.h @@ -26,7 +26,7 @@ namespace oceanbase { namespace logservice { -// ObReplayBarrierType is the barrier type for follower log replay, divided into the following three categories +// ObReplayBarrierType defines ordering barriers for local disk replay. // 1. STRICT_BARRIER: // The prerequisite condition for replaying this log is that all logs with a log ts smaller than this log have already been replayed, // And before the replay of this log is complete, logs with a log ts greater than this log will not be replayed. diff --git a/src/logservice/ob_log_base_type.h b/src/logservice/ob_log_base_type.h index 3ab5416e4..8931cb397 100644 --- a/src/logservice/ob_log_base_type.h +++ b/src/logservice/ob_log_base_type.h @@ -60,11 +60,6 @@ enum ObLogBaseType //for standby timestamp service STANDBY_TIMESTAMP_LOG_BASE_TYPE = 13, - // for global auto increment service - GAIS_LOG_BASE_TYPE = 14, - - // for das id service - DAS_ID_LOG_BASE_TYPE = 15, //for recovery_ls_service RESTORE_SERVICE_LOG_BASE_TYPE = 16, @@ -101,23 +96,17 @@ enum ObLogBaseType COMMON_LS_SERVICE_LOG_BASE_TYPE = 36, - // only use role change service, do not write clog + // local lifecycle callback only; does not write clog LS_BLOCK_TX_SERVICE_LOG_BASE_TYPE = 37, TTL_LOG_BASE_TYPE = 39, - - // for table load resource manager - TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE = 40, // for tenant snapshot SNAPSHOT_SCHEDULER_LOG_BASE_TYPE = 41, //for tenant clone CLONE_SCHEDULER_LOG_BASE_TYPE = 42, - // for mview maintenance service - MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE = 43, - // for DBMS_SCHEDULER DBMS_SCHEDULER_LOG_BASE_TYPE = 45, @@ -143,7 +132,7 @@ enum ObLogBaseType DDL_SERVICE_LAUNCHER_LOG_BASE_TYPE = 54, // for vector index scheduler VEC_INDEX_SERVICE_LOG_BASE_TYPE = 60, - // for sys tenant load sys package service on sys ls leader + // for the local sys tenant package service SYS_TENANT_LOAD_SYS_PACKAGE_SERVICE_LOG_BASE_TYPE = 61, // for internal table change notifier (timezone, SRS, etc.) INTERNAL_TABLE_NOTIFIER_LOG_BASE_TYPE = 62, @@ -189,10 +178,6 @@ int log_base_type_to_string(const ObLogBaseType log_type, strncpy(str ,"RECOVERY_LS_SERVICE", str_len); } else if (log_type == STANDBY_TIMESTAMP_LOG_BASE_TYPE) { strncpy(str ,"STANDBY_TIMESTAMP", str_len); - } else if (log_type == GAIS_LOG_BASE_TYPE) { - strncpy(str ,"GAIS", str_len); - } else if (log_type == DAS_ID_LOG_BASE_TYPE) { - strncpy(str ,"DAS_ID", str_len); } else if (log_type == RESTORE_SERVICE_LOG_BASE_TYPE) { strncpy(str ,"RESTORE_SERVICE", str_len); } else if (log_type == RESERVED_SNAPSHOT_LOG_BASE_TYPE) { @@ -229,10 +214,6 @@ int log_base_type_to_string(const ObLogBaseType log_type, strncpy(str ,"SNAPSHOT_SCHEDULER", str_len); } else if (log_type == CLONE_SCHEDULER_LOG_BASE_TYPE) { strncpy(str ,"CLONE_SCHEDULER", str_len); - } else if (log_type == TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE) { - strncpy(str ,"TABLE_LOAD_RESOURCE_SERVICE", str_len); - } else if (log_type == MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE) { - strncpy(str, "MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE", str_len); } else if (log_type == TABLE_LOCK_LOG_BASE_TYPE) { strncpy(str ,"TABLE_LOCK", str_len); } else if (log_type == SHARED_STORAGE_PRE_WARM_LOG_BASE_TYPE) { @@ -279,29 +260,12 @@ class ObIReplaySubHandler const share::SCN &scn) = 0; }; -class ObIRoleChangeSubHandler +class ObILocalLogHandler { public: - virtual void switch_to_follower_forcedly() = 0; - virtual int switch_to_leader() = 0; - // @retval - // OB_SUCCESS, role switch execution succeeded - // OB_LS_NEED_REVOKE, role switch execution failed, need to step down proactively - // OTHERS , role switch execution failed, no need to resign proactively - // - // The reason for adding OB_LS_NEED_REVOKE is that switch_to_follower_gracefully includes two steps: - // a. Execute leader->follower; - // b. Execute follower->leader switch. - // - // a. Execution successful, indicates that the subsequent leader switch work can be performed - // a. Execution failed, need to execute 1.b - // b. Execution successful, indicates that this leader switch operation failed, but does not require an active resignation - // b. Execution failed, indicates that this leader switch operation failed, but needs to step down proactively - // - // After negotiation, it was decided to use error codes to distinguish the above various exceptions. - virtual int switch_to_follower_gracefully() = 0; - virtual int resume_leader() = 0; - VIRTUAL_TO_STRING_KV("ObIRoleChangeSubHandler", "Dummy"); + virtual void deactivate() = 0; + virtual int activate() = 0; + VIRTUAL_TO_STRING_KV("ObILocalLogHandler", "Dummy"); }; // services inherit from ObICheckpointSubHandler @@ -316,33 +280,48 @@ class ObICheckpointSubHandler #define REGISTER_TO_LOGSERVICE(type, subhandler) \ if (OB_SUCC(ret)) { \ if (OB_FAIL(replay_handler_.register_handler(type, subhandler))) { \ - LOG_WARN("replay_handler_ register failed", K(ret), K(type), K(ls_meta_.ls_id_)); \ - } else if (OB_FAIL(role_change_handler_.register_handler(type, subhandler))) { \ - LOG_WARN("role_change_handler_ register failed", K(ret), K(type), K(ls_meta_.ls_id_));\ + LOG_WARN("replay_handler_ register failed", K(ret), K(type)); \ + } else if (OB_FAIL(local_log_handler_set_.register_handler(type, subhandler))) { \ + LOG_WARN("local_log_handler_set_ register failed", K(ret), K(type)); \ } else if (OB_FAIL(checkpoint_executor_.register_handler(type, subhandler))) { \ - LOG_WARN("checkpoint_executor_ register failed", K(ret), K(type), K(ls_meta_.ls_id_));\ + LOG_WARN("checkpoint_executor_ register failed", K(ret), K(type)); \ } else { \ - LOG_INFO("register to logservice success", K(type), K(ls_meta_.ls_id_)); \ + LOG_INFO("register to logservice success", K(type)); \ } \ } #define UNREGISTER_FROM_LOGSERVICE(type, subhandler) \ (void)replay_handler_.unregister_handler(type); \ - (void)role_change_handler_.unregister_handler(type); \ + (void)local_log_handler_set_.unregister_handler(type); \ (void)checkpoint_executor_.unregister_handler(type); \ +#define REGISTER_REPLAY_CHECKPOINT_HANDLER(type, subhandler) \ + if (OB_SUCC(ret)) { \ + if (OB_FAIL(replay_handler_.register_handler(type, subhandler))) { \ + LOG_WARN("replay_handler_ register failed", K(ret), K(type)); \ + } else if (OB_FAIL(checkpoint_executor_.register_handler(type, subhandler))) { \ + LOG_WARN("checkpoint_executor_ register failed", K(ret), K(type)); \ + } else { \ + LOG_INFO("register replay and checkpoint handler success", K(type)); \ + } \ + } + +#define UNREGISTER_REPLAY_CHECKPOINT_HANDLER(type) \ + (void)replay_handler_.unregister_handler(type); \ + (void)checkpoint_executor_.unregister_handler(type); \ + #define REGISTER_TO_RESTORESERVICE(type, subhandler) \ if (OB_SUCC(ret)) { \ - if (OB_FAIL(restore_role_change_handler_.register_handler(type, subhandler))) { \ - LOG_WARN("restore_role_change_handler_ register failed", \ - K(ret), K(type), K(ls_meta_.ls_id_)); \ + if (OB_FAIL(restore_local_log_handler_set_.register_handler(type, subhandler))) { \ + LOG_WARN("restore_local_log_handler_set_ register failed", \ + K(ret), K(type)); \ } else { \ - LOG_INFO("register to restoreservice success", K(type), K(ls_meta_.ls_id_)); \ + LOG_INFO("register to restoreservice success", K(type)); \ } \ } #define UNREGISTER_FROM_RESTORESERVICE(type, subhandler) \ - (void)restore_role_change_handler_.unregister_handler(type); + (void)restore_local_log_handler_set_.unregister_handler(type); } // namespace logservice } // namespace oceanbase diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp index d79f8391a..154f95764 100755 --- a/src/logservice/ob_log_handler.cpp +++ b/src/logservice/ob_log_handler.cpp @@ -17,6 +17,8 @@ #define USING_LOG_PREFIX PALF #include "lib/stat/ob_diagnostic_info_guard.h" #include "ob_log_handler.h" +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_service.h" #include "share/rc/ob_module_provider.h" #ifdef OB_BUILD_LOG_STORAGE_COMPRESS #include "logservice/ob_log_compression.h" @@ -33,13 +35,15 @@ namespace logservice using namespace palf; ObLogHandler::ObLogHandler() : self_(), + lock_(), + palf_handle_(), + palf_env_(NULL), + is_in_stop_state_(true), + is_inited_(false), apply_status_(NULL), apply_service_(NULL), replay_service_(NULL), - rc_service_(NULL), deps_lock_(), - lc_cb_(NULL), - rpc_proxy_(NULL), append_cost_stat_("[PALF STAT APPEND COST TIME]", 1 * 1000 * 1000), is_offline_(false), #ifdef OB_BUILD_LOG_STORAGE_COMPRESS @@ -54,58 +58,45 @@ ObLogHandler::~ObLogHandler() destroy(); } -int ObLogHandler::init(const int64_t id, - const common::ObAddr &self, +int ObLogHandler::init(const common::ObAddr &self, ObLogApplyService *apply_service, ObLogReplayService *replay_service, - ObRoleChangeService *rc_service, PalfEnv *palf_env, - PalfLocationCacheCb *lc_cb, - obcall::ObLogServiceRpcProxy *rpc_proxy, common::ObILogAllocator *alloc_mgr) { int ret = OB_SUCCESS; ObApplyStatus *apply_status = NULL; ObApplyStatusGuard guard; - share::ObLSID ls_id(id); if (IS_INIT) { ret = OB_INIT_TWICE; } else if (OB_ISNULL(palf_env) || OB_ISNULL(apply_service) || - OB_ISNULL(lc_cb) || - OB_ISNULL(rpc_proxy) || OB_ISNULL(alloc_mgr)) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid arguments", K(id), KP(palf_env), KP(lc_cb), KP(rpc_proxy), KP(alloc_mgr)); - } else if (OB_FAIL(apply_service->get_apply_status(ls_id, guard))) { - CLOG_LOG(WARN, "guard get apply status failed", K(ret), K(id)); + CLOG_LOG(WARN, "invalid arguments", KP(palf_env), KP(alloc_mgr)); + } else if (OB_FAIL(apply_service->get_apply_status(guard))) { + CLOG_LOG(WARN, "guard get apply status failed", K(ret)); } else if (NULL == (apply_status_ = guard.get_apply_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "apply status is not exist", K(ret), K(id)); - } else if (OB_FAIL(palf_env->open(id, palf_handle_))) { - CLOG_LOG(WARN, "open palf failed", K(ret), K(id)); + CLOG_LOG(WARN, "apply status is not exist", K(ret)); + } else if (OB_FAIL(palf_env->open(palf_handle_))) { + CLOG_LOG(WARN, "open palf failed", K(ret)); #ifdef OB_BUILD_LOG_STORAGE_COMPRESS - } else if (OB_FAIL(compressor_wrapper_.init(id, alloc_mgr))) { - CLOG_LOG(WARN, "failed to init compressor_wrapper_", K(id)); + } else if (OB_FAIL(compressor_wrapper_.init(1, alloc_mgr))) { + CLOG_LOG(WARN, "failed to init compressor_wrapper_"); #endif } else { get_max_decided_scn_debug_time_ = OB_INVALID_TIMESTAMP; apply_service_ = apply_service; replay_service_ = replay_service; - rc_service_ = rc_service; apply_status_->inc_ref(); - PALF_REPORT_INFO_KV(K(id)); - append_cost_stat_.set_extra_info(EXTRA_INFOS); - id_ = id; + append_cost_stat_.set_extra_info(""); self_ = self; palf_env_ = palf_env; - role_ = FOLLOWER; - lc_cb_ = lc_cb; - rpc_proxy_ = rpc_proxy; is_in_stop_state_ = false; is_offline_ = true; // offline at default. is_inited_ = true; - FLOG_INFO("ObLogHandler init success", K(id)); + FLOG_INFO("ObLogHandler init success"); } if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { destroy(); @@ -122,11 +113,10 @@ bool ObLogHandler::is_valid() const NULL != palf_env_ && NULL != apply_status_ && NULL != apply_service_ && - NULL != lc_cb_ && #ifdef OB_BUILD_LOG_STORAGE_COMPRESS compressor_wrapper_.is_valid() && #endif - NULL != rpc_proxy_; + NULL != replay_service_; } int ObLogHandler::stop() @@ -144,10 +134,6 @@ int ObLogHandler::stop() if (OB_FAIL(apply_status_->stop())) { CLOG_LOG(INFO, "apply_status stop failed", KPC(this), KPC(apply_status_), KR(ret)); } else if (false == palf_handle_.is_valid()) { - // Note: we disable log sync in here, therefore executing ObLogHander::offline() - // is safe after ObLogHandler::stop() has been executed - } else if (OB_FAIL(palf_handle_.disable_sync())) { - CLOG_LOG(WARN, "disable_sync failed", KPC(this), KR(ret)); } else { tg.click("apply stop end"); palf_env_->close(palf_handle_); @@ -157,33 +143,6 @@ int ObLogHandler::stop() } return ret; } -//Determine if is_apply_done depends on log handler, no further append can be made -//So need to set is_in_stop_state_ to true to indicate that no more logs can be submitted in the stop phase -int ObLogHandler::safe_to_destroy(bool &is_safe_destroy) -{ - int ret = OB_SUCCESS; - bool is_done = false; - LSN end_lsn; - is_safe_destroy = true; - WLockGuard guard(lock_); - if (IS_INIT) { - if (palf_handle_.is_valid() || !is_in_stop_state_) { - ret = OB_STATE_NOT_MATCH; - } else if (OB_FAIL(apply_status_->is_apply_done(is_done, end_lsn))) { - CLOG_LOG(ERROR, "check apply status is_apply_done failed", K(ret), K(is_done), K(end_lsn), KPC(apply_status_)); - } else if (false == is_done) { - ret = OB_EAGAIN; - CLOG_LOG(INFO, "wait apply done false", K(ret), K(is_done), K(end_lsn), KPC(apply_status_)); - } else { - CLOG_LOG(INFO, "wait apply done finish", K(ret), K(is_done), K(end_lsn), KPC(apply_status_)); - } - } - if (OB_FAIL(ret)) { - is_safe_destroy = false; - } - return ret; -} - void ObLogHandler::destroy() { WLockGuard guard(lock_); @@ -200,11 +159,7 @@ void ObLogHandler::destroy() if (NULL != palf_env_ && true == palf_handle_.is_valid()) { palf_env_->close(palf_handle_); } - rc_service_ = NULL; - lc_cb_ = NULL; - rpc_proxy_ = NULL; palf_env_ = NULL; - id_ = -1; get_max_decided_scn_debug_time_ = OB_INVALID_TIMESTAMP; #ifdef OB_BUILD_LOG_STORAGE_COMPRESS compressor_wrapper_.reset(); @@ -251,37 +206,9 @@ int ObLogHandler::append_big_log(const void *buffer, return ret; } -void ObLogHandler::switch_role(const common::ObRole &role, const int64_t proposal_id) -{ - WLockGuard guard(lock_); - role_ = role; - proposal_id_ = proposal_id; -} - -int ObLogHandler::get_role(common::ObRole &role, int64_t &proposal_id) const -{ - return ObLogHandlerBase::get_role(role, proposal_id); -} - -int ObLogHandler::get_access_mode(int64_t &mode_version, palf::AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.get_access_mode(mode_version, access_mode))) { - CLOG_LOG(WARN, "palf get_access_mode failed", K(ret), K_(id)); - } else { - } - return ret; -} - int ObLogHandler::get_append_mode_initial_scn(share::SCN &ref_scn) const { int ret = OB_SUCCESS; - int64_t mode_version = INVALID_PROPOSAL_ID; AccessMode access_mode = AccessMode::INVALID_ACCESS_MODE; share::SCN curr_ref_scn; ref_scn.reset(); @@ -290,8 +217,8 @@ int ObLogHandler::get_append_mode_initial_scn(share::SCN &ref_scn) const ret = OB_NOT_INIT; } else if (is_in_stop_state_) { ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.get_access_mode_ref_scn(mode_version, access_mode, curr_ref_scn))) { - CLOG_LOG(WARN, "get_access_mode_ref_scn failed", K(ret), K_(id)); + } else if (OB_FAIL(palf_handle_.get_access_mode_ref_scn(access_mode, curr_ref_scn))) { + CLOG_LOG(WARN, "get_access_mode_ref_scn failed", K(ret)); } else if (AccessMode::APPEND == access_mode) { ref_scn = curr_ref_scn; } else { @@ -300,29 +227,6 @@ int ObLogHandler::get_append_mode_initial_scn(share::SCN &ref_scn) const return ret; } -int ObLogHandler::change_access_mode(const int64_t mode_version, - const palf::AccessMode &access_mode, - const SCN &ref_scn) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - // do not check role of LogHander with PALF, check proposal_id is enough. - // If you want change_access_mode from RAW_WRITE to APPEND, - // in this time, ObLogHandler is FOLLOWER and ObRestoreHandler is LEADER. - const int64_t proposal_id = ATOMIC_LOAD(&proposal_id_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.change_access_mode(proposal_id, mode_version, access_mode, ref_scn))) { - CLOG_LOG(WARN, "palf change_access_mode failed", K(ret), K_(id), K(proposal_id), K(mode_version), - K(access_mode), K(ref_scn)); - } else { - FLOG_INFO("change_access_mode success", K(ret), K_(id), K(proposal_id), K(mode_version), K(access_mode), K(ref_scn)); - } - return ret; -} - int ObLogHandler::seek(const LSN &lsn, PalfBufferIterator &iter) { int ret = OB_SUCCESS; @@ -330,9 +234,9 @@ int ObLogHandler::seek(const LSN &lsn, PalfBufferIterator &iter) if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (OB_FAIL(seek_log_iterator_dispatch_(lsn, default_suggested_max_read_buf_size, iter))) { - CLOG_LOG(WARN, "seek_log_iterator_dispatch failed", KP(palf_env_), K(id_), K(lsn)); + CLOG_LOG(WARN, "seek_log_iterator_dispatch failed", KP(palf_env_), K(lsn)); } else { - CLOG_LOG(TRACE, "seek success", KP(palf_env_), K(id_), K(lsn)); + CLOG_LOG(TRACE, "seek success", KP(palf_env_), K(lsn)); } return ret; } @@ -344,31 +248,21 @@ int ObLogHandler::seek(const LSN &lsn, PalfGroupBufferIterator &iter) if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (OB_FAIL(seek_log_iterator_dispatch_(lsn, default_suggested_max_read_buf_size, iter))) { - CLOG_LOG(WARN, "seek_log_iterator_dispatch failed", KP(palf_env_), K(id_), K(lsn)); + CLOG_LOG(WARN, "seek_log_iterator_dispatch failed", KP(palf_env_), K(lsn)); } else { - CLOG_LOG(TRACE, "seek success", KP(palf_env_), K(id_), K(lsn)); + CLOG_LOG(TRACE, "seek success", KP(palf_env_), K(lsn)); } return ret; } -int ObLogHandler::set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) +int ObLogHandler::bootstrap() { RLockGuard guard(lock_); - return palf_handle_.set_initial_member_list(member_list, paxos_replica_num, learner_list); -} - -int ObLogHandler::set_election_priority(palf::election::ElectionPriority *priority) -{ - RLockGuard guard(lock_); - return palf_handle_.set_election_priority(priority); -} - -int ObLogHandler::reset_election_priority() -{ - RLockGuard guard(lock_); - return palf_handle_.reset_election_priority(); + int ret = palf_handle_.bootstrap(); + if (OB_FAIL(ret)) { + CLOG_LOG(WARN, "failed to bootstrap local palf", K(ret), K_(self)); + } + return ret; } int ObLogHandler::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) @@ -376,7 +270,7 @@ int ObLogHandler::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) int ret = OB_SUCCESS; RLockGuard guard(lock_); if (OB_FAIL(palf_handle_.locate_by_scn_coarsely(scn, result_lsn))) { - CLOG_LOG(WARN, "locate_by_scn_coarsely from palf failed", K(id_), K(scn)); + CLOG_LOG(WARN, "locate_by_scn_coarsely from palf failed", K(scn)); } return ret; @@ -391,23 +285,7 @@ int ObLogHandler::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) } else if (is_in_stop_state_) { ret = OB_NOT_RUNNING; } else if (OB_FAIL(palf_handle_.locate_by_lsn_coarsely(lsn, result_scn))) { - CLOG_LOG(WARN, "locate_by_scn_coarsely from palf failed", KR(ret), K(id_), K(lsn)); - } - return ret; -} - -int ObLogHandler::get_max_decided_scn_as_leader(share::SCN &scn) const -{ - int ret = OB_SUCCESS; - share::ObLSID ls_id; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_ISNULL(apply_service_)) { - ret = OB_ERR_UNEXPECTED; - } else if (FALSE_IT(ls_id = id_)) { - } else if (OB_FAIL(apply_service_->get_palf_committed_end_scn(ls_id, scn))) { - CLOG_LOG(WARN, "get palf_committed_end_lsn fail", K(ret), K(id_)); + CLOG_LOG(WARN, "locate_by_scn_coarsely from palf failed", KR(ret), K(lsn)); } return ret; } @@ -449,103 +327,6 @@ int ObLogHandler::get_end_scn(SCN &scn) const return palf_handle_.get_end_scn(scn); } -int ObLogHandler::get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const -{ - RLockGuard guard(lock_); - return palf_handle_.get_paxos_member_list(member_list, paxos_replica_num); -} - -int ObLogHandler::get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const -{ - RLockGuard guard(lock_); - return palf_handle_.get_paxos_member_list_and_learner_list(member_list, paxos_replica_num, learner_list); -} - -int ObLogHandler::get_global_learner_list(common::GlobalLearnerList &learner_list) const -{ - RLockGuard guard(lock_); - return palf_handle_.get_global_learner_list(learner_list); -} - -int ObLogHandler::get_stable_membership( - palf::LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "loghandler is not inited or maybe destroyed", K(ret), K(id_)); - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - CLOG_LOG(INFO, "loghandler is stopped", K(ret), K_(id)); - } else if (OB_FAIL(palf_handle_.get_stable_membership(config_version, - member_list, paxos_replica_num, learner_list))) { - CLOG_LOG(WARN, "get_stable_membership failed", K(ret), KPC(this)); - } else {/*do nothing*/} - return ret; -} - -int ObLogHandler::get_election_leader(common::ObAddr &addr) const -{ - RLockGuard guard(lock_); - return palf_handle_.get_election_leader(addr); -} - -int ObLogHandler::get_parent(common::ObAddr &parent) const -{ - RLockGuard guard(lock_); - return palf_handle_.get_parent(parent); -} - -int ObLogHandler::enable_sync() -{ - RLockGuard guard(lock_); - return palf_handle_.enable_sync(); -} - -int ObLogHandler::disable_sync() -{ - RLockGuard guard(lock_); - return palf_handle_.disable_sync(); -} - -bool ObLogHandler::is_sync_enabled() const -{ - RLockGuard guard(lock_); - return palf_handle_.is_sync_enabled(); -} - -int ObLogHandler::advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) -{ - int ret = OB_SUCCESS; - bool is_replay_enabled = false; - WLockGuard guard(lock_); - share::ObLSID ls_id; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (FALSE_IT(ls_id = id_)) { - } else if (OB_FAIL(replay_service_->is_enabled(ls_id, is_replay_enabled))) { - CLOG_LOG(WARN, "check replay status failed", K(ret), K(ls_id)); - } else if (OB_UNLIKELY(is_replay_enabled)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay is not disabled", K(ret), K(ls_id)); - } else { - if (OB_FAIL(palf_handle_.advance_base_info(palf_base_info, is_rebuild))) { - CLOG_LOG(WARN, "advance_base_info failed", K(ret), K_(id), K(palf_base_info)); - } else { - CLOG_LOG(INFO, "advance_base_info success", K(ret), K_(id), K(palf_base_info)); - } - } - return ret; -} - int ObLogHandler::get_palf_base_info(const LSN &base_lsn, PalfBaseInfo &palf_base_info) { int ret = OB_SUCCESS; @@ -561,585 +342,7 @@ int ObLogHandler::get_palf_base_info(const LSN &base_lsn, PalfBaseInfo &palf_bas } else if (FALSE_IT(new_base_lsn.val_ = lsn_2_block(base_lsn, PALF_BLOCK_SIZE) * PALF_BLOCK_SIZE)) { } else { ret = palf_handle_.get_base_info(new_base_lsn, palf_base_info); - CLOG_LOG(INFO, "get_palf_base_info finish", KR(ret), K_(id), K(base_lsn), K(new_base_lsn), K(palf_base_info)); - } - return ret; -} - -int ObLogHandler::is_in_sync(bool &is_log_sync, - bool &is_need_rebuild) const -{ - int ret = OB_SUCCESS; - is_log_sync = false; - is_need_rebuild = false; - palf::PalfStat palf_stat; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.stat(palf_stat))) { - CLOG_LOG(WARN, "palf stat failed", K(ret), K_(id)); - } else { - is_log_sync = palf_stat.is_in_sync_; - is_need_rebuild = palf_stat.is_need_rebuild_; - } - if (REACH_TIME_INTERVAL(2 * 1000 * 1000)) { - CLOG_LOG(INFO, "is_in_sync", K(ret), K_(id), K(is_log_sync), K(is_need_rebuild)); - } - return ret; -} - -int ObLogHandler::get_leader_config_version(LogConfigVersion &config_version) const -{ - int ret = OB_SUCCESS; - common::ObRole role; - common::ObRole new_role; - int64_t proposal_id; - int64_t new_proposal_id; - bool is_pending_state = false; - config_version.reset(); - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "loghandler is not inited or maybe destroyed", K(ret), K(id_)); - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - CLOG_LOG(INFO, "loghandler is stopped", K(ret), K_(id)); - } else if (OB_FAIL(palf_handle_.get_role(role, proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "get_role failed", K(ret), KPC(this)); - } else if (LEADER != role || true == is_pending_state) { - ret = OB_NOT_MASTER; - } else if (OB_FAIL(palf_handle_.get_config_version(config_version))) { - CLOG_LOG(WARN, "get_config_version failed", K(ret), KPC(this)); - } else if (OB_FAIL(palf_handle_.get_role(new_role, new_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "get_role failed", K(ret), KPC(this)); - } else if (role != new_role || proposal_id != new_proposal_id) { - ret = OB_NOT_MASTER; - CLOG_LOG(INFO, "role changed during getting config version", K(ret), KPC(this), K(role), - K(new_role), K(proposal_id), K(new_proposal_id)); - } else {/*do nothing*/} - return ret; -} - -// @desc: change_replica_num interface -// | 1.change_replica_num() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. (CHANGE_REPLICA_NUM) -int ObLogHandler::change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!member_list.is_valid() || - !is_valid_replica_num(curr_replica_num) || - !is_valid_replica_num(new_replica_num) || - curr_replica_num <= new_replica_num || - timeout_us <= 0) { - // NB: do not allow to change replica_num to bigger number now - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(member_list), K(curr_replica_num), - K(new_replica_num), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, member_list, curr_replica_num, new_replica_num, - CHANGE_REPLICA_NUM_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "change_replica_num success", KR(ret), K_(id), K(member_list), - K(curr_replica_num), K(new_replica_num)); - } - } - return ret; -} - -// @desc: force_set_as_single_replica interface -// | 1.force_set_as_single_replica() -// V -// [any_member] ----- 2.one_stage_config_change_(FORCE_SINGLE_MEMBER) -int ObLogHandler::force_set_as_single_replica() -{ - int ret = OB_SUCCESS; - common::TCWLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else { - common::ObMemberList dummy_member_list; - int64_t dummy_replica_num = -1, new_replica_num = 1; - const int64_t timeout_us = 10 * 1000 * 1000L; - LogConfigChangeCmd req(self_, id_, dummy_member_list, dummy_replica_num, new_replica_num, - FORCE_SINGLE_MEMBER_CMD, timeout_us); - ConfigChangeCmdHandler cmd_handler(&palf_handle_); - LogConfigChangeCmdResp resp; - if (OB_FAIL(cmd_handler.handle_config_change_cmd(req, resp))) { - CLOG_LOG(WARN, "handle_config_change_cmd failed", KR(ret), K_(id)); - } else { - CLOG_LOG(INFO, "force_set_as_single_replica success", KR(ret), K_(id), K_(self)); - } - } - return ret; -} - -int ObLogHandler::force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num) -{ - int ret = OB_SUCCESS; - common::TCWLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "ObLogHandler is not inited", KR(ret)); - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - CLOG_LOG(WARN, "ObLogHandler is in stop state", KR(ret)); - } else { - const int64_t timeout_us = 10 * 1000L * 1000L; - LogConfigChangeCmd req(self_, id_, new_member_list, new_replica_num, - FORCE_SET_MEMBER_LIST_CMD, timeout_us); - ConfigChangeCmdHandler cmd_handler(&palf_handle_); - LogConfigChangeCmdResp resp; - if (OB_FAIL(cmd_handler.handle_config_change_cmd(req, resp))) { - CLOG_LOG(WARN, "handle force_set_member_list_cmd failed", KR(ret), K_(id), K(req)); - } else { - CLOG_LOG(INFO, "force_set_member_list success", K_(id), K_(self), K(new_member_list), K(new_replica_num), K(resp)); - } - } - - return ret; -} - -// @desc: add_member interface -// | 1.add_member() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(ADD_MEMBER) -int ObLogHandler::add_member(const common::ObMember &added_member, - const int64_t new_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!added_member.is_valid() || - !is_valid_replica_num(new_replica_num) || - !config_version.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_member), - K(new_replica_num), K(config_version), K(timeout_us)); - } else { - common::ObMember dummy_member; - LogConfigChangeCmd req(self_, id_, added_member, dummy_member, new_replica_num, ADD_MEMBER_CMD, timeout_us); - req.in_leader(config_version); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "add_member failed", KR(ret), K_(id), K(added_member), K(new_replica_num), K(config_version)); - } else { - CLOG_LOG(INFO, "add_member success", KR(ret), K_(id), K(added_member), K(new_replica_num), K(config_version)); - } - } - return ret; -} - -// @desc: remove_member interface -// | 1. remove_member() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]----> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(REMOVE_MEMBER) -int ObLogHandler::remove_member(const common::ObMember &removed_member, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!removed_member.is_valid() || - !is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(removed_member), K(new_replica_num), K(timeout_us)); - } else { - common::ObMember dummy_member; - LogConfigChangeCmd req(self_, id_, dummy_member, removed_member, new_replica_num, REMOVE_MEMBER_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "remove_member success", KR(ret), K_(id), K(removed_member), K(new_replica_num)); - } - } - return ret; -} - -// @desc: replace_member interface -// | 1.replace_member() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]----->[leader] -// | -// V 3. one_stage_config_change_(ADD_MEMBER_AND_NUM) -// V 4. one_stage_config_change_(REMOVE_MEMBER_AND_NUM) -// [any_member] <----[5. Sync LogConfigChangeCmdResp]----- -int ObLogHandler::replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) { - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!added_member.is_valid() || - !removed_member.is_valid() || - !config_version.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_member), K(removed_member), K(config_version), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, added_member, removed_member, 0, REPLACE_MEMBER_CMD, timeout_us); - req.in_leader(config_version); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "replace_member failed", KR(ret), K_(id), K(added_member), K(removed_member), K(config_version)); - } else { - CLOG_LOG(INFO, "replace_member success", KR(ret), K_(id), K(added_member), K(removed_member), K(config_version)); - } - } - return ret; -} - -int ObLogHandler::replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!added_member.is_valid() || - !removed_member.is_valid() || - false == config_version.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_member), - K(removed_member), K(config_version), K(timeout_us)); - } else { - LogConfigChangeCmd replace_req(self_, id_, added_member, removed_member, 0, - REPLACE_MEMBER_WITH_LEARNER_CMD, timeout_us); - replace_req.in_leader(config_version); - if (OB_FAIL(submit_config_change_cmd_(replace_req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(replace_req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "replace_member_with_learner success", KR(ret), K_(id), K(added_member), - K(removed_member), K(config_version), K(timeout_us)); - } - } - return ret; -} - -// @desc: add_learner interface -// | 1.add_learner() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(ADD_LEARNER) -int ObLogHandler::add_learner(const common::ObMember &added_learner, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!added_learner.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_learner), K(timeout_us)); - } else { - common::ObMember dummy_member; - LogConfigChangeCmd req(self_, id_, added_learner, dummy_member, 0, ADD_LEARNER_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "add_learner success", KR(ret), K_(id), K(added_learner)); - } - } - return ret; -} - -// @desc: remove_learner interface -// | 1.remove_learner() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(REMOVE_LEARNER) -int ObLogHandler::remove_learner(const common::ObMember &removed_learner, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!removed_learner.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(removed_learner), K(timeout_us)); - } else { - common::ObMember dummy_member; - LogConfigChangeCmd req(self_, id_, dummy_member, removed_learner, 0, REMOVE_LEARNER_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "remove_learner success", KR(ret), K_(id), K(removed_learner)); - } - - } - return ret; -} - -int ObLogHandler::replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!added_learners.is_valid() || - !removed_learners.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_learners), K(removed_learners), K(timeout_us)); - } else { - LogConfigChangeCmd replace_req(self_, id_, added_learners, removed_learners, REPLACE_LEARNERS_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(replace_req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(replace_req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "replace_learners success", KR(ret), K_(id), K(added_learners), K(removed_learners), K(timeout_us)); - } - } - return ret; -} - -// @desc: switch_learner_to_acceptor interface -// | 1.switch_learner_to_accetpor() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(SWITCH_LEARNER_TO_ACCEPTOR) -int ObLogHandler::switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!learner.is_valid() || - !is_valid_replica_num(new_replica_num) || - !config_version.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(learner), K(new_replica_num), K(config_version), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, learner, learner, new_replica_num, SWITCH_TO_ACCEPTOR_CMD, timeout_us); - req.in_leader(config_version); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "switch_learner_to_acceptor failed", KR(ret), K_(id), K(learner), K(new_replica_num), K(config_version)); - } else { - CLOG_LOG(INFO, "switch_learner_to_acceptor success", KR(ret), K_(id), K(learner), K(new_replica_num), K(config_version)); - } - } - return ret; -} - -// @desc: switch_acceptor_to_learner interface -// | 1.switch_acceptor_to_learner() -// V -// [any_member] -----[2. Sync LogConfigChangeCmd]---> [leader] -// | -// [any_member] <----[4. Sync LogConfigChangeCmdResp]--- | 3. one_stage_config_change_(SWITCH_ACCEPTOR_TO_LEARNER) -int ObLogHandler::switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (!member.is_valid() || - !is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(member), K(new_replica_num), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, member, member, new_replica_num, SWITCH_TO_LEARNER_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us)); - } else { - CLOG_LOG(INFO, "switch_acceptor_to_learner success", KR(ret), K_(id), K(member), K(new_replica_num)); - } - } - return ret; -} - -int ObLogHandler::try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us) - -{ - int ret = OB_SUCCESS; - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER == lock_owner || timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(lock_owner), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, lock_owner, TRY_LOCK_CONFIG_CHANGE_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "try_lock_config_change failed", KR(ret), K_(id), K(lock_owner), K(timeout_us)); - } else { - CLOG_LOG(INFO, "try_lock_config_change success", KR(ret), K_(id), K(lock_owner)); - } - } - return ret; -} - -int ObLogHandler::unlock_config_change(const int64_t lock_owner, const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - // Note: rlock is safe, the deps_lock_ is used to protect states of ObLogHandler - // such as is_in_stop_state_. - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER == lock_owner || timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(lock_owner), K(timeout_us)); - } else { - LogConfigChangeCmd req(self_, id_, lock_owner, UNLOCK_CONFIG_CHANGE_CMD, timeout_us); - if (OB_FAIL(submit_config_change_cmd_(req))) { - CLOG_LOG(WARN, "unlock_config_change failed", KR(ret), K_(id), K(lock_owner), K(timeout_us)); - } else { - CLOG_LOG(INFO, "unlock_config_change success", KR(ret), K_(id), K(lock_owner)); - } - } - return ret; -} - -int ObLogHandler::get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) -{ - int ret = OB_SUCCESS; - const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s - RLockGuard deps_guard(deps_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else { - LogConfigChangeCmd req(self_, id_, palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER/*unused*/, - GET_CONFIG_CHANGE_LOCK_STAT_CMD, CONFIG_CHANGE_TIMEOUT/*timeout_us*/); - LogConfigChangeCmdResp resp; - if (OB_FAIL(submit_config_change_cmd_(req, resp))) { - CLOG_LOG(WARN, "get_config_change_lock_stat failed", KR(ret), K_(id)); - } else { - lock_owner = resp.lock_owner_; - is_locked = resp.is_locked_; - CLOG_LOG(INFO, "get_config_change_lock_stat success", KR(ret), K_(id), K(lock_owner), K(is_locked)); - } - } - return ret; -} - -int ObLogHandler::submit_config_change_cmd_(const LogConfigChangeCmd &req) -{ - LogConfigChangeCmdResp unused_resp; - return submit_config_change_cmd_(req, unused_resp); -} - -int ObLogHandler::submit_config_change_cmd_(const LogConfigChangeCmd &req, - LogConfigChangeCmdResp &resp) -{ - int ret = OB_SUCCESS; - if (!req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(req)); - } else { - constexpr int64_t RENEW_LEADER_INTERVAL_US = 500 * 1000L; // 500ms - const int64_t start_time_us = common::ObClockGenerator::getClock(); - int64_t last_renew_leader_time_us = OB_INVALID_TIMESTAMP; - FLOG_INFO("config_change start", K_(id), K(req)); - - while(OB_SUCCESS == ret || OB_NOT_MASTER == ret) { - // judge init status to avoiding log_handler destoring gets stuck - if (IS_NOT_INIT || OB_ISNULL(lc_cb_)) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "PalfHandleImpl not init", KR(ret), K_(id)); - break; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - CLOG_LOG(WARN, "ObLogHandler is not running", KR(ret), K_(id)); - break; - } else if (common::ObTimeUtility::current_time() - start_time_us >= req.timeout_us_) { - ret = OB_TIMEOUT; - FLOG_WARN("config_change timeout", KR(ret), KPC(this), K(req), K(start_time_us)); - break; - } - - common::ObAddr leader; - ConfigChangeCmdHandler cmd_handler(&palf_handle_); - bool need_renew_leader = false; - if (OB_FAIL(lc_cb_->get_leader(id_, leader))) { - need_renew_leader = true; - ret = OB_SUCCESS; - } else if (FALSE_IT(cmd_handler.handle_config_change_cmd(req, resp))) { - CLOG_LOG(WARN, "failed to handle_config_change_cmd", KR(ret), K_(id), K(req)); - } else if (OB_SUCC(resp.ret_)) { - FLOG_INFO("config_change finish", KR(ret), KPC(this), K(req), - "cost time(ns)", common::ObTimeUtility::current_time() - start_time_us); - break; - } else if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - ob_usleep(50 * 1000); - } else if (OB_NOT_MASTER == ret) { - need_renew_leader = true; - } else if (OB_NOT_ALLOW_REMOVING_LEADER == ret && - req.is_remove_member_list() && - (req.removed_member_.get_server() == leader)) { - } else { - CLOG_LOG(WARN, "handle_config_change_cmd failed", KR(ret), KPC(this), K(req), K(leader)); - } - } + CLOG_LOG(INFO, "get_palf_base_info finish", KR(ret), K(base_lsn), K(new_base_lsn), K(palf_base_info)); } return ret; } @@ -1157,7 +360,6 @@ int ObLogHandler::append_(const void *buffer, int64_t wait_times = 0; PalfAppendOptions opts; opts.need_nonblock = need_nonblock; - opts.need_check_proposal_id = true; ObTimeGuard tg("ObLogHandler::append", 100000); const void *final_buf = buffer; int64_t final_nbytes = nbytes; @@ -1173,8 +375,6 @@ int ObLogHandler::append_(const void *buffer, #endif const int64_t begin_ts = ObClockGenerator::getClock(); while (true) { - // generate opts - opts.proposal_id = ATOMIC_LOAD(&proposal_id_); do { RLockGuard guard(lock_); CriticalGuard(ls_qs_); @@ -1183,8 +383,6 @@ int ObLogHandler::append_(const void *buffer, ret = OB_NOT_INIT; } else if (is_in_stop_state_ || is_offline_) { ret = OB_NOT_RUNNING; - } else if (LEADER != ATOMIC_LOAD(&role_)) { - ret = OB_NOT_MASTER; } else if (OB_FAIL(palf_handle_.append(opts, final_buf, final_nbytes, ref_scn, lsn, scn))) { if (REACH_TIME_INTERVAL(1*1000*1000)) { CLOG_LOG(WARN, "palf_handle_ append failed", K(ret), KPC(this)); @@ -1195,7 +393,7 @@ int ObLogHandler::append_(const void *buffer, cb->__set_scn(scn); ret = apply_status_->push_append_cb(cb); CLOG_LOG(TRACE, "palf_handle_ push_append_cb success", K(lsn), K(scn), K(log_compressed), - K(nbytes), K(final_nbytes), K(id_)); + K(nbytes), K(final_nbytes)); #ifdef OB_BUILD_LOG_STORAGE_COMPRESS //add stat event EVENT_ADD(LOG_STORAGE_COMPRESS_ORIGINAL_SIZE, nbytes); @@ -1241,64 +439,6 @@ int ObLogHandler::append_(const void *buffer, return ret; } -int ObLogHandler::get_member_gc_stat(const common::ObAddr &addr, - bool &is_valid_member, - LogMemberGCStat &stat) const -{ - int ret = OB_SUCCESS; - common::ObRole role; - common::ObRole new_role; - int64_t proposal_id; - int64_t new_proposal_id; - common::ObMemberList member_list; - int64_t paxos_replica_num = 0; - GlobalLearnerList learner_list; - bool is_pending_state = false; - is_valid_member = true; - stat = LogMemberGCStat::LOG_MEMBER_NORMAL_GC_STAT; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "loghandler not inited or maybe destroyed", K(ret), K(addr)); - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - CLOG_LOG(INFO, "loghandler is stopped", K(ret), K(addr)); - } else if (!addr.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "invalid arguments", K(ret), K(addr), K(id_)); - } else if (OB_FAIL(palf_handle_.get_role(role, proposal_id, is_pending_state))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "get_role failed", K(ret), KPC(this)); - } else if (LEADER != role) { - ret = OB_NOT_MASTER; - } else if (OB_FAIL(palf_handle_.get_paxos_member_list_and_learner_list(member_list, - paxos_replica_num, learner_list))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "get_paxos_member_list_and_learner_list failed", K(ret), KPC(this)); - } else if (OB_FAIL(palf_handle_.get_role(new_role, new_proposal_id, is_pending_state))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "get_role failed", K(ret), KPC(this)); - } else { - if (role == new_role && proposal_id == new_proposal_id) { - is_valid_member = member_list.contains(addr) || learner_list.contains(addr); - if (learner_list.contains(addr)) { - ObMember member; - if (OB_FAIL(learner_list.get_learner_by_addr(addr, member))) { - CLOG_LOG(ERROR, "failed to get_learner_by_addr", K(learner_list), K(addr), KPC(this)); - } else { - stat = member.is_migrating() ? LogMemberGCStat::LOG_LEARNER_IN_MIGRATING : LogMemberGCStat::LOG_MEMBER_NORMAL_GC_STAT; - } - } - } else { - ret = OB_NOT_MASTER; - CLOG_LOG(INFO, "role changed during is_valid_member", K(ret), KPC(this), K(role), - K(new_role), K(proposal_id), K(new_proposal_id)); - } - CLOG_LOG(INFO, "get_member_gc_stat", K(is_valid_member), K(stat), K(member_list), K(learner_list), K(addr), KPC(this)); - } - return ret; -} - void ObLogHandler::wait_append_sync() { WaitQuiescent(ls_qs_); } @@ -1308,17 +448,15 @@ int ObLogHandler::enable_replay(const palf::LSN &lsn, { int ret = OB_SUCCESS; RLockGuard guard(lock_); - share::ObLSID id; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (FALSE_IT(id = id_)) { } else if (!lsn.is_valid() || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(ret), K(id), K(lsn), K(scn)); - } else if (OB_FAIL(replay_service_->enable(id, lsn, scn))) { - CLOG_LOG(WARN, "failed to enable replay", K(ret), K(id), K(lsn), K(scn)); + CLOG_LOG(WARN, "invalid argument", K(ret), K(lsn), K(scn)); + } else if (OB_FAIL(replay_service_->enable(lsn, scn))) { + CLOG_LOG(WARN, "failed to enable replay", K(ret), K(lsn), K(scn)); } else { - CLOG_LOG(INFO, "enable replay success", K(ret), K(id), K(lsn), K(scn)); + CLOG_LOG(INFO, "enable replay success", K(ret), K(lsn), K(scn)); } return ret; } @@ -1327,14 +465,12 @@ int ObLogHandler::disable_replay() { int ret = OB_SUCCESS; RLockGuard guard(lock_); - share::ObLSID id; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (FALSE_IT(id = id_)) { - } else if (OB_FAIL(replay_service_->disable(id))) { - CLOG_LOG(WARN, "failed to disable replay", K(ret), K(id)); + } else if (OB_FAIL(replay_service_->disable())) { + CLOG_LOG(WARN, "failed to disable replay", K(ret)); } else { - CLOG_LOG(INFO, "disable replay success", K(ret), K(id)); + CLOG_LOG(INFO, "disable replay success", K(ret)); } return ret; } @@ -1343,14 +479,12 @@ int ObLogHandler::pend_submit_replay_log() { int ret = OB_SUCCESS; RLockGuard guard(lock_); - share::ObLSID id; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (FALSE_IT(id = id_)) { - } else if (OB_FAIL(replay_service_->block_submit_log(id))) { - CLOG_LOG(WARN, "failed to block_submit_log", K(ret), K(id)); + } else if (OB_FAIL(replay_service_->block_submit_log())) { + CLOG_LOG(WARN, "failed to block_submit_log", K(ret)); } else { - CLOG_LOG(INFO, "block_submit_log success", K(ret), K(id)); + CLOG_LOG(INFO, "block_submit_log success", K(ret)); } return ret; } @@ -1359,14 +493,12 @@ int ObLogHandler::restore_submit_replay_log() { int ret = OB_SUCCESS; RLockGuard guard(lock_); - share::ObLSID id; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (FALSE_IT(id = id_)) { - } else if (OB_FAIL(replay_service_->unblock_submit_log(id))) { - CLOG_LOG(WARN, "failed to unblock_submit_log", K(ret), K(id)); + } else if (OB_FAIL(replay_service_->unblock_submit_log())) { + CLOG_LOG(WARN, "failed to unblock_submit_log", K(ret)); } else { - CLOG_LOG(INFO, "unblock_submit_log success", K(ret), K(id)); + CLOG_LOG(INFO, "unblock_submit_log success", K(ret)); } return ret; } @@ -1376,11 +508,9 @@ bool ObLogHandler::is_replay_enabled() const bool bool_ret = false; int tmp_ret = OB_SUCCESS; RLockGuard guard(lock_); - share::ObLSID id; if (IS_NOT_INIT) { - } else if (FALSE_IT(id = id_)) { - } else if (OB_SUCCESS != (tmp_ret = replay_service_->is_enabled(id, bool_ret))) { - CLOG_LOG_RET(WARN, tmp_ret, "check replay service is enabled failed", K(tmp_ret), K(id)); + } else if (OB_SUCCESS != (tmp_ret = replay_service_->is_enabled(bool_ret))) { + CLOG_LOG_RET(WARN, tmp_ret, "check replay service is enabled failed", K(tmp_ret)); } else { // do nothing } @@ -1392,7 +522,6 @@ int ObLogHandler::get_max_decided_scn(SCN &scn) int ret = OB_SUCCESS; SCN max_replayed_scn; SCN max_applied_scn; - share::ObLSID id; RLockGuard guard(lock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -1401,22 +530,21 @@ int ObLogHandler::get_max_decided_scn(SCN &scn) ret = OB_STATE_NOT_MATCH; } else if (is_offline()) { ret = OB_STATE_NOT_MATCH; - CLOG_LOG(WARN, "log handle is offline", K(id_)); - } else if (FALSE_IT(id = id_)) { - } else if (OB_FAIL(apply_service_->get_max_applied_scn(id, max_applied_scn))) { - CLOG_LOG(WARN, "failed to get_max_applied_scn", K(ret), K(id)); - } else if (OB_FAIL(replay_service_->get_max_replayed_scn(id, max_replayed_scn))) { + CLOG_LOG(WARN, "log handle is offline"); + } else if (OB_FAIL(apply_service_->get_max_applied_scn(max_applied_scn))) { + CLOG_LOG(WARN, "failed to get_max_applied_scn", K(ret)); + } else if (OB_FAIL(replay_service_->get_max_replayed_scn(max_replayed_scn))) { if (OB_STATE_NOT_MATCH != ret) { - CLOG_LOG(WARN, "failed to get_max_replayed_scn", K(ret), K(id)); + CLOG_LOG(WARN, "failed to get_max_replayed_scn", K(ret)); } else if (palf_reach_time_interval(1000 * 1000, get_max_decided_scn_debug_time_)) { - CLOG_LOG(ERROR, "failed to get_max_replayed_scn, replay status is not enabled", K(ret), K(id)); + CLOG_LOG(ERROR, "failed to get_max_replayed_scn, replay status is not enabled", K(ret)); } if (OB_STATE_NOT_MATCH == ret && max_applied_scn.is_valid()) { //Replay is not enabled, but the maximum consecutive callback point obtained in the apply service is valid ret = OB_SUCCESS; scn = max_applied_scn > SCN::min_scn() ? max_applied_scn : SCN::min_scn(); if (palf_reach_time_interval(1000 * 1000, get_max_decided_scn_debug_time_)) { - CLOG_LOG(INFO, "replay is not enabled, get_max_decided_scn from apply", K(ret), K(id), + CLOG_LOG(INFO, "replay is not enabled, get_max_decided_scn from apply", K(ret), K(max_replayed_scn), K(max_applied_scn), K(scn)); } } @@ -1427,78 +555,6 @@ int ObLogHandler::get_max_decided_scn(SCN &scn) return ret; } -int ObLogHandler::disable_vote(const bool need_check_log_missing) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else { - ret = palf_handle_.disable_vote(need_check_log_missing); - } - return ret; -} - -int ObLogHandler::enable_vote() -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else { - ret = palf_handle_.enable_vote(); - } - return ret; -} - -int ObLogHandler::register_rebuild_cb(palf::PalfRebuildCb *rebuild_cb) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.register_rebuild_cb(rebuild_cb))) { - CLOG_LOG(WARN, "register_rebuild_cb failed", K(ret), KPC(this)); - } - return ret; -} - -int ObLogHandler::unregister_rebuild_cb() -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(palf_handle_.unregister_rebuild_cb())) { - CLOG_LOG(WARN, "unregister_rebuild_cb failed", K(ret), KPC(this)); - } else { - } - return ret; -} - -int ObLogHandler::diagnose(LogHandlerDiagnoseInfo &diagnose_info) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else { - diagnose_info.log_handler_role_ = ATOMIC_LOAD(&role_); - diagnose_info.log_handler_proposal_id_ = ATOMIC_LOAD(&proposal_id_); - } - return ret; -} - // reentrant int ObLogHandler::offline() { @@ -1507,39 +563,10 @@ int ObLogHandler::offline() PALF_LOG(INFO, "ObLogHandler has already been destroyed", K(ret), KPC(this)); } else if (OB_FAIL(disable_replay())) { CLOG_LOG(WARN, "disable_replay failed", K(ret), KPC(this)); - } else if (OB_FAIL(disable_sync()) && OB_NOT_INIT != ret) { - CLOG_LOG(WARN, "disable_sync failed", K(ret), KPC(this)); } else { WLockGuard guard(lock_); - // NB: make proposal_id_ to be invalid: - // 1. avoid append success. - // 2. make role change success(role change service require proposal_id of log_handler is not same as palf) - // 3. don't make role to follower at here, otherwise, role change thread will execute follower to follower. - proposal_id_ = INVALID_PROPOSAL_ID; - - // NB: - // 1. After set 'is_offline_' to true, we must prohibit apply log, otherwise, - // log handler may be come LEADER after offline, and the proposal id of apply - // is -1, update committed end ls of appy will print ERROR logs. - // - // 2. Must reset proposal_id of apply_status_ before set 'is_offline', otherwise, - // concurrent 'switch to follower' event may set apply status to FOLLOWER, however, - // there are some uncommitted logs in PALF. and before reset_proposal_id, these - // uncommitted logs has been committed. and then update committed end ls of apply - // will print ERROR logs, because the role of apply is FOLLOWER, and the proposal_id - // of apply is as same as PALF. - apply_status_->reset_proposal_id(); - // - // 3. Must keep the order of set 'is_offline_' between reset the proposal id of apply. - // - MEM_BARRIER(); is_offline_ = true; - // NB: must ensure on_role_change not fail. - if (OB_FAIL(rc_service_->on_role_change(id_))) { - CLOG_LOG(WARN, "on_role_change failed", K(ret), KPC(this)); - } else { - CLOG_LOG(INFO, "LogHandler offline success", K(ret), KPC(this)); - } + CLOG_LOG(INFO, "LogHandler offline success", K(ret), KPC(this)); } return ret; } @@ -1567,24 +594,14 @@ int ObLogHandler::online(const LSN &lsn, const SCN &scn) ret = OB_NOT_RUNNING; } else if (OB_FAIL(enable_replay(lsn, scn))) { CLOG_LOG(WARN, "enable_replay failed", K(ret), KPC(this), K(lsn), K(scn)); - } else if (OB_FAIL(enable_sync())) { - CLOG_LOG(WARN, "enable_sync failed", K(ret), KPC(this)); } else { WLockGuard guard(lock_); - proposal_id_ = INVALID_PROPOSAL_ID; //reset_meta to avoid contributing excessively large max_decided_scn //reset_meta is placed here rather than offline() because after offline, callbacks will be //handled after offline which may refer to palf_committed_end_lsn_ apply_status_->reset_meta(); is_offline_ = false; - // NB: before notify role change service, we need set role to FOLLOWER, - // otherwise, role change service may need switch leader to leader. - role_ = common::FOLLOWER; - if (OB_FAIL(rc_service_->on_role_change(id_))) { - CLOG_LOG(WARN, "on_role_change failed", K(ret), KPC(this)); - } else { - CLOG_LOG(INFO, "LogHander online success", K(ret), KPC(this), K(lsn), K(scn)); - } + CLOG_LOG(INFO, "LogHander online success", K(ret), KPC(this), K(lsn), K(scn)); } return ret; } @@ -1603,9 +620,8 @@ int ObLogHandler::is_replay_fatal_error(bool &has_fatal_error) ret = OB_NOT_RUNNING; } else { RLockGuard guard(lock_); - ObLSID ls_id(id_); - if (OB_FAIL(replay_service_->has_fatal_error(ls_id, has_fatal_error))) { - CLOG_LOG(WARN, "has_fatal_error failed", KR(ret), K(ls_id)); + if (OB_FAIL(replay_service_->has_fatal_error(has_fatal_error))) { + CLOG_LOG(WARN, "has_fatal_error failed", KR(ret)); } } return ret; @@ -1617,33 +633,22 @@ int ObLogHandler::advance_base_lsn_impl_(const LSN &lsn) RLockGuard guard(lock_); if (is_in_stop_state_) { ret = OB_NOT_RUNNING; - CLOG_LOG(WARN, "ObLogHandler is not running", KR(ret), K_(id)); + CLOG_LOG(WARN, "ObLogHandler is not running", KR(ret)); } else if (OB_FAIL(palf_handle_.advance_base_lsn(lsn))) { CLOG_LOG(WARN, "advance_base_lsn failed", KR(ret), K(lsn)); } else {} return ret; } -int __get_log_handler(const ObLSID &ls_id, - ObLogHandler *&log_handler, - ObLSHandle &ls_handle) +int __get_log_handler(ObLogHandler *&log_handler, + storage::ObLS *&ls) { int ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLS *ls = NULL; - if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::LOG_MOD))) { - CLOG_LOG(WARN, "get_ls failed", K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "unexpected error!!! ls must not nullptr", K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "unexpected error!!! ls must not nullptr", K(ls_id)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - } else {} + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + CLOG_LOG(WARN, "get_ls failed"); + } else { + log_handler = ls->get_log_handler(); + } return ret; } diff --git a/src/logservice/ob_log_handler.h b/src/logservice/ob_log_handler.h index 7f0894774..b9facb46c 100755 --- a/src/logservice/ob_log_handler.h +++ b/src/logservice/ob_log_handler.h @@ -20,19 +20,15 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/lock/ob_tc_rwlock.h" #include "common/ob_role.h" -#include "common/ob_member_list.h" #include "share/ob_delegate.h" +#include "palf/palf_env.h" #include "palf/palf_handle.h" #include "palf/palf_base_info.h" #include "palf/palf_iterator.h" -#include "logrpc/ob_log_service_rpc_shell.h" -#include "logrpc/ob_log_request_handler.h" -#include "ob_log_handler_base.h" #ifdef OB_BUILD_LOG_STORAGE_COMPRESS #include "logservice/ob_log_compression.h" #endif -#include "storage/tx_storage/ob_ls_handle.h" // ObLSHandle complete type, previously hidden behind a transitive include namespace oceanbase { @@ -48,9 +44,12 @@ namespace transaction { class ObTsMgr; } +namespace storage +{ +class ObLS; +} namespace palf { -class PalfEnv; class LSN; } namespace logservice @@ -59,20 +58,6 @@ class ObLogApplyService; class ObApplyStatus; class ObLogReplayService; class AppendCb; -struct LogHandlerDiagnoseInfo { - LogHandlerDiagnoseInfo() { reset(); } - ~LogHandlerDiagnoseInfo() { reset(); } - common::ObRole log_handler_role_; - int64_t log_handler_proposal_id_; - void reset() { - log_handler_role_ = FOLLOWER; - log_handler_proposal_id_ = palf::INVALID_PROPOSAL_ID; - } - TO_STRING_KV(K(log_handler_role_), - K(log_handler_proposal_id_)); -}; - -class ObRoleChangeService; class ObILogHandler { public: @@ -96,24 +81,13 @@ class ObILogHandler palf::LSN &lsn, share::SCN &scn) = 0; - virtual int get_role(common::ObRole &role, int64_t &proposal_id) const = 0; - - virtual int change_access_mode(const int64_t mode_version, - const palf::AccessMode &access_mode, - const share::SCN &ref_scn) = 0; - virtual int get_access_mode(int64_t &mode_version, palf::AccessMode &access_mode) const = 0; virtual int get_append_mode_initial_scn(SCN &ref_scn) const = 0; virtual int seek(const palf::LSN &lsn, palf::PalfBufferIterator &iter) = 0; virtual int seek(const palf::LSN &lsn, palf::PalfGroupBufferIterator &iter) = 0; - virtual int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) = 0; - virtual int set_election_priority(palf::election::ElectionPriority *priority) = 0; - virtual int reset_election_priority() = 0; + virtual int bootstrap() = 0; virtual int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &result_lsn) = 0; virtual int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) = 0; - virtual int get_max_decided_scn_as_leader(share::SCN &scn) const = 0; virtual int advance_base_lsn(const palf::LSN &lsn) = 0; virtual int get_begin_lsn(palf::LSN &lsn) const = 0; virtual int get_end_lsn(palf::LSN &lsn) const = 0; @@ -121,62 +95,7 @@ class ObILogHandler virtual int get_max_scn(share::SCN &scn) const = 0; virtual int get_end_scn(share::SCN &scn) const = 0; - virtual int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const = 0; - virtual int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const = 0; - virtual int get_global_learner_list(common::GlobalLearnerList &learner_list) const = 0; - virtual int get_leader_config_version(palf::LogConfigVersion &config_version) const = 0; - virtual int get_stable_membership(palf::LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const = 0; - // get leader from election, used only for non_palf_leader rebuilding. - virtual int get_election_leader(common::ObAddr &addr) const = 0; - virtual int get_parent(common::ObAddr &parent) const = 0; - virtual int change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) = 0; - virtual int force_set_as_single_replica() = 0; - virtual int force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num) = 0; - virtual int add_member(const common::ObMember &member, - const int64_t paxos_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - virtual int remove_member(const common::ObMember &member, - const int64_t paxos_replica_num, - const int64_t timeout_us) = 0; - virtual int replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - virtual int add_learner(const common::ObMember &added_learner, const int64_t timeout_us) = 0; - virtual int remove_learner(const common::ObMember &removed_learner, const int64_t timeout_us) = 0; - virtual int replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) = 0; - virtual int replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - virtual int switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t paxos_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - virtual int switch_acceptor_to_learner(const common::ObMember &member, - const int64_t paxos_replica_num, - const int64_t timeout_us) = 0; - virtual int try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us) = 0; - virtual int unlock_config_change(const int64_t lock_owner, const int64_t timeout_us) = 0; - virtual int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) = 0; virtual int get_palf_base_info(const palf::LSN &base_lsn, palf::PalfBaseInfo &palf_base_info) = 0; - virtual int is_in_sync(bool &is_log_sync, bool &is_need_rebuild) const = 0; - virtual int enable_sync() = 0; - virtual int disable_sync() = 0; - virtual bool is_sync_enabled() const = 0; - virtual int advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild) = 0; - virtual int get_member_gc_stat(const common::ObAddr &addr, bool &is_valid_member, obcall::LogMemberGCStat &stat) const = 0; virtual void wait_append_sync() = 0; virtual int enable_replay(const palf::LSN &initial_lsn, const share::SCN &initial_scn) = 0; virtual int disable_replay() = 0; @@ -184,38 +103,26 @@ class ObILogHandler virtual int pend_submit_replay_log() = 0; virtual int restore_submit_replay_log() = 0; virtual bool is_replay_enabled() const = 0; - virtual int disable_vote(const bool need_check_log_missing) = 0; - virtual int enable_vote() = 0; - virtual int register_rebuild_cb(palf::PalfRebuildCb *rebuild_cb) = 0; - virtual int unregister_rebuild_cb() = 0; virtual int offline() = 0; virtual int online(const palf::LSN &lsn, const share::SCN &scn) = 0; virtual bool is_offline() const = 0; virtual int is_replay_fatal_error(bool &has_fatal_error) = 0; }; -class ObLogHandler : public ObILogHandler, public ObLogHandlerBase +class ObLogHandler : public ObILogHandler { public: ObLogHandler(); virtual ~ObLogHandler(); - int init(const int64_t id, - const common::ObAddr &self, + int init(const common::ObAddr &self, ObLogApplyService *apply_service, ObLogReplayService *replay_service, - ObRoleChangeService *rc_service, palf::PalfEnv *palf_env, - palf::PalfLocationCacheCb *lc_cb, - obcall::ObLogServiceRpcProxy *rpc_proxy, common::ObILogAllocator *alloc_mgr); bool is_valid() const; int stop(); void destroy(); - // @breif, wait cb append onto apply service all be called - // is reentrant and should be called before destroy(), - // protect cb will not be used when log handler destroyed - int safe_to_destroy(bool &is_safe_destroy); // @brief append count bytes from the buffer starting at buf to the palf handle, return the LSN and timestamp // @param[in] const void *, the data buffer. // @param[in] const uint64_t, the length of data buffer. @@ -262,51 +169,11 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase palf::LSN &lsn, share::SCN &scn) override final; - // @brief switch log_handle role, to LEADER or FOLLOWER - // @param[in], role, LEADER or FOLLOWER - // @param[in], proposal_id, global monotonically increasing id - virtual void switch_role(const common::ObRole &role, const int64_t proposal_id) override; - // @brief query role and proposal_id from ObLogHandler. - // @param[out], role: - // LEADER, if 'role_' of ObLogHandler is LEADER and 'proposal_id' is same with PalfHandle. - // FOLLOWER, otherwise. - // @param[out], proposal_id, global monotonically increasing. - // @retval - // OB_SUCCESS - // NB: for standby, is always FOLLOWER - int get_role(common::ObRole &role, int64_t &proposal_id) const override final; - // @brief: query the access_mode of palf and it's corresponding mode_version - // @param[out] palf::AccessMode &access_mode: current access_mode - // @param[out] int64_t &mode_version: mode_version corresponding to AccessMode - // @retval - // OB_SUCCESS - int get_access_mode(int64_t &mode_version, palf::AccessMode &access_mode) const override final; // @description: get ref_scn of APPEND mode // @return // - OB_SUCCESS // - OB_STATE_NOT_MATCH: current access mode is not APPEND int get_append_mode_initial_scn(share::SCN &initial_scn) const override final; - // @brief change AccessMode of palf. - // @param[in] const int64_t &mode_version: mode_version corresponding to AccessMode, - // can be gotted by get_access_mode - // @param[in] const palf::AccessMode access_mode: access_mode will be changed to - // @param[in] const int64_t ref_scn: scn of all submitted logs after changing access mode - // are bigger than ref_scn - // NB: ref_scn will take effect only when: - // a. ref_scn is bigger than/equal to max_ts(get_max_scn()) - // b. AccessMode is set to APPEND - // @retval - // OB_SUCCESS - // OB_NOT_MASTER: self is not active leader - // OB_EAGAIN: another change_acess_mode is running, try again later - // OB_STATE_NOT_MATCH: the param 'mode_version' don't match with current mode_version - // NB: 1. if return OB_EAGAIN, caller need execute 'change_access_mode' again. - // 2. before execute 'change_access_mode', caller need execute 'get_access_mode' to - // get 'mode_version' and pass it to 'change_access_mode' - int change_access_mode(const int64_t mode_version, - const palf::AccessMode &access_mode, - const share::SCN &ref_scn) override final; - // @desc: seek a log buffer iterator by lsn, the first log A in iterator must meet // the start lsn of log A must equal to 'start_lsn'. // @params [in] start_lsn: @@ -370,18 +237,7 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase // - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled // - others: bug - // @brief set the initial member list of paxos group - // @param[in] ObMemberList, the initial member list - // @param[in] int64_t, the paxos relica num - // @param[in] GlobalLearnerList, the initial learner list - // @retval - // return OB_SUCCESS if success - // else return other errno - int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) override final; - int set_election_priority(palf::election::ElectionPriority *priority) override final; - int reset_election_priority() override final; + int bootstrap() override final; // @desc: query coarse lsn by ts(ns), that means there is a LogGroupEntry in disk, // its lsn and scn are result_lsn and result_scn, and result_scn <= scn. // Note that this function may be time-consuming @@ -408,19 +264,6 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase // - OB_NEED_RETRY: the block is being recycled or switched, need retry. // - others: bug int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) override final; - // @brief, get max committed scn from applyservice, which is the max scn of log committed by oneself as leader; - // Example: - // At time T1, the replica of the log stream is the leader and the maximum SCN of the logs - // confirmed by this replica is 100. At time T2, the replica switches to being a follower and the - // maximum SCN of the logs synchronized and replayed is 200. At time T3, the log stream replica - // switches back to being the leader and writes logs with SCNs ranging from 201 to 300, all of - // which are not confirmed. In this case, the returned value of the interface would be 100. - // @param[out] max scn of logs confirmed by this replica as being leader - // @return - // OB_NOT_INIT : ls is not inited - // OB_ERR_UNEXPECTED: unexpected error such as apply_service_ is NULL - // OB_SUCCESS - int get_max_decided_scn_as_leader(share::SCN &scn) const override final; // @brief, set the recycable lsn, palf will ensure that the data before recycable lsn readable. // @param[in] const LSN&, recycable lsn. int advance_base_lsn(const palf::LSN &lsn) override final; @@ -437,48 +280,12 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase // @brief, get timestamp of end lsn. // @param[out] int64_t, timestamp. int get_end_scn(share::SCN &scn) const override final; - // @brief, get paxos member list of this paxos group - // @param[out] common::ObMemberList& - // @param[out] int64_t& - int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const override final; - // @brief, get paxos member list and global list of this paxos group atomically - // @param[out] common::ObMemberList& - // @param[out] int64_t& - // @param[out] common::GlobalLearnerList& - int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const override final; - // @brief, get global learner list of this paxos group - // @param[out] common::GlobalLearnerList& - int get_global_learner_list(common::GlobalLearnerList &learner_list) const override final; - // @brief, get stable membership info from the leader - // @param[out] palf::LogConfigVersion& - // @param[out] common::ObMemberList& - // @param[out] int64_t& - // @param[out] common::GlobalLearnerList& - // retval: - // OB_SUCCESS - // OB_NOT_INIT - // OB_EAGAIN: the leader is changing memberlist, try again - // OB_NOT_RUNNING: the replica is offlined - int get_stable_membership(palf::LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const override final; - // @brief, get leader from election, used only for non_palf_leader rebuilding - // @param[out] addr: address of leader - // retval: - // OB_SUCCESS - // OB_NOT_INIT - // OB_LEADER_NOT_EXIST - int get_election_leader(common::ObAddr &addr) const override final; // @brief, get parent // @param[out] addr: address of parent // retval: // OB_SUCCESS // OB_NOT_INIT // OB_ENTRY_NOT_EXIST: parent is invalid - int get_parent(common::ObAddr &parent) const override final; // PalfBaseInfo include the 'base_lsn' and the 'prev_log_info' of sliding window. // @param[in] const LSN&, base_lsn of ls. // @param[out] PalfBaseInfo&, palf_base_info @@ -486,228 +293,6 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase // OB_SUCCESS // OB_ERR_OUT_OF_LOWER_BOUND, the block of 'base_lsn' has been recycled int get_palf_base_info(const palf::LSN &base_lsn, palf::PalfBaseInfo &palf_base_info) override final; - // @brief, advance base_log_info for migrate/rebuild scenes. - // @param[in] const PalfBaseInfo&, palf_base_info. - // NB: caller should call disable_sync() before calling advance_base_info. after calling advance_base_info, replay_status will be disabled. - // So if advance_base_info returns OB_SUCCESS, that means log sync and replay_status have been disabled. - int advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild) override final; - // check if palf is in sync state or need rebuild - // @param [out] is_log_sync: if the log of this replica is sync with leader's, - // - false: leader's max_scn - local max_scn > 3s + keepalive_interval - // - true: leader's max_scn - local max_scn < 3s + keepalive_interval - // @param [out] is_need_rebuild, if the log of this replica is far behind than leader's - // and need do rebuild - int is_in_sync(bool &is_log_sync, bool &is_need_rebuild) const override final; - - // @brief, enable log sync - int enable_sync() override final; - // @brief, disable log sync - int disable_sync() override final; - // @brief, check if log sync is enabled - bool is_sync_enabled() const override final; - - // @brief: get config_version - // @return - // - OB_SUCCESS: get config_version successfully - // - OB_NOT_INIT: is not inited - // - OB_NOT_RUNNING: is in stopped state - // - OB_NOT_MASTER: this replica is not master - // - other: bug - int get_leader_config_version(palf::LogConfigVersion &config_version) const; - - // @brief: a special config change interface, change replica number of paxos group - // @param[in] common::ObMemberList: current memberlist, for pre-check - // @param[in] const int64_t curr_replica_num: current replica num, for pre-check - // @param[in] const int64_t new_replica_num: new replica num - // @param[in] const int64_t timeout_us: timeout, us - // @return - // - OB_SUCCESS: change_replica_num successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: change_replica_num timeout - // - other: bug - int change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) override final; - - // @brief: force set self as single replica. - // @return - // - OB_SUCCESS: change_replica_num successfully - // - OB_TIMEOUT: change_replica_num timeout - // - other: bug - virtual int force_set_as_single_replica() override final; - // @brief: force set member list. - // @param[in] const common::ObMemberList &new_member_list: members which will be set forcely - // @param[in] const int64_t new_replica_num: replica number of paxos group after forcing to set member list - // @return - // - OB_SUCCESS: force_set_member_list successfully - // - OB_TIMEOUT: force_set_member_list timeout - // - OB_NOT_RUNNING: log stream is stopped - // - OB_INVALID_ARGUMENT: invalid argument - // - other: bug - virtual int force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num) override final; - // @brief, add a member to paxos group, can be called in any member - // @param[in] common::ObMember &member: member which will be added - // @param[in] const int64_t paxos_replica_num: replica number of paxos group after adding 'member' - // @param[in] const palf::LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us: add member timeout, us - // @return - // - OB_SUCCESS: add member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_NOT_MASTER: not master - // - OB_STATE_NOT_MATCH: leader has switched - // - other: bug - int add_member(const common::ObMember &member, - const int64_t paxos_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) override final; - - // @brief, remove a member from paxos group, can be called in any member - // @param[in] common::ObMember &member: member which will be removed - // @param[in] const int64_t paxos_replica_num: replica number of paxos group after removing 'member' - // @param[in] const int64_t timeout_us: remove member timeout, us - // @return - // - OB_SUCCESS: remove member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: remove member timeout - // - other: bug - int remove_member(const common::ObMember &member, - const int64_t paxos_replica_num, - const int64_t timeout_us) override final; - - // @brief, replace old_member with new_member, can be called in any member - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const palf::LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - other: bug - int replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) override final; - - // @brief: add a learner(read only replica) in this cluster - // @param[in] const common::ObMember &added_learner: learner will be added - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: add_learner timeout - int add_learner(const common::ObMember &added_learner, - const int64_t timeout_us) override final; - - // @brief, replace removed_learners with added_learners, can be called in any member - // @param[in] const common::ObMemberList &added_learners: learners will be added - // @param[in] const common::ObMemberList &removed_learners: learners will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace learner successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace learner timeout - // - other: bug - int replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) override final; - - // @brief, replace removed_member with learner, can be called in any member - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const palf::LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - other: bug - int replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) override final; - - // @brief: remove a learner(read only replica) in this cluster - // @param[in] const common::ObMember &removed_learner: learner will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: remove_learner timeout - int remove_learner(const common::ObMember &removed_learner, - const int64_t timeout_us) override final; - - // @brief: switch a learner(read only replica) to acceptor(full replica) in this cluster - // @param[in] const common::ObMember &learner: learner will be switched to acceptor - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // learner to acceptor (similar to add_member) - // @param[in] const palf::LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_learner_to_acceptor timeout - int switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) override final; - - // @brief: switch an acceptor(full replica) to learner(read only replica) in this cluster - // @param[in] const common::ObMember &member: acceptor will be switched to learner - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // acceptor to learner (similar to remove_member) - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_acceptor_to_learner timeout - int switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) override final; - - //---------config change lock related--------// - //@brief: try lock config change which will forbidden changing on memberlist - // @param[in] const int64_ lock_owner: owner of locking_config_change operation - // @param[in] const int64_t timeout_us - //@return - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS successfull lock - // -- OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT failed to lock because of locked by others - // -- OB_TIMEOUT timeout, may lock successfully or not - // -- OB_EAGAIN other config change operation is going on,need retry later - // -- OB_NOT_MASTER this replica is not leader, not refresh location and retry with actual leader - // -- OB_STATE_NOT_MATCH lock_owner is smaller than previous lock_owner - int try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us); - - //@brief: unlock config change which will allow changing on memberlist - // @param[in] const int64_ lock_owner: expected owner of config_change - // @param[in] const int64_t timeout_us - //@return - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS successfull unlock - // -- OB_TIMEOUT timeout, may unlock successfully or not - // -- OB_EAGAIN other config change operation is going on,need retry later - // -- OB_NOT_MASTER this replica is not leader, need refresh location and retry with actual leader - // -- OB_STATE_NOT_MATCH lock_owner is smaller than previous lock_owner,or lock_owner is bigger than previous lock_owner - int unlock_config_change(const int64_t lock_owner, const int64_t timeout_us); - - //@brief: get config change lock stat - //@return - //ret: - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS success - // -- OB_NOT_MASTER this replica is not leader, not refresh location and retry with actual leader - // -- OB_EAGAIN is_locking or unlocking - // lock_owner: owner of config_change_lock - // is_locked: whether config_change_lock is locked - int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked); - - // @breif, check request server is in self member list - // @param[in] const common::ObAddr, request server. - // @param[out] is_valid_member&, whether in member list or learner list. - // @param[out] obcall::LogMemberGCStat&, gc stat like learner in migrating. - int get_member_gc_stat(const common::ObAddr &addr, bool &is_valid_member, obcall::LogMemberGCStat &stat) const override final; // @breif, wait cb append onto apply service Qsync // protect submit log and push cb in Qsync guard void wait_append_sync() override final; @@ -732,25 +317,8 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase // OB_STATE_NOT_MATCH: ls is offline or stopped // OB_SUCCESS int get_max_decided_scn(share::SCN &scn) override final; - // @brief: store a persistent flag which means this paxos replica can not reply ack when receiving logs. - // By default, paxos replica can reply ack. - // @param[in] need_check_log_missing: for rebuildinng caused by log missing, need check whether log - // is actually missing - // @return: - // OB_NOT_INIT: not inited - // OB_NOT_RUNNING: in stop state - // OB_OP_NOT_ALLOW: no need to rebuilds, log is available. rebuilding should be abandoned. - // OB_LEADER_NOT_EXIST: no leader when double checking. rebuilding should retry. - int disable_vote(const bool need_check_log_missing) override final; - // @brief: store a persistent flag which means this paxos replica - // can reply ack when receiving logs. - // By default, paxos replica can reply ack. - int enable_vote() override final; - int register_rebuild_cb(palf::PalfRebuildCb *rebuild_cb) override final; - int unregister_rebuild_cb() override final; - int diagnose(LogHandlerDiagnoseInfo &diagnose_info) const; int diagnose_palf(palf::PalfDiagnoseInfo &diagnose_info) const; - TO_STRING_KV(K_(role), K_(proposal_id), KP(palf_env_), K(is_in_stop_state_), K(is_inited_), K(id_)); + TO_STRING_KV(KP(palf_env_), K(is_in_stop_state_), K(is_inited_)); int offline() override final; int online(const palf::LSN &lsn, const share::SCN &scn) override final; bool is_offline() const override final; @@ -770,14 +338,10 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase private: static constexpr int64_t MIN_CONN_TIMEOUT_US = 5 * 1000 * 1000; // 5s const int64_t MAX_APPEND_RETRY_INTERNAL = 500 * 1000L; - typedef common::TCRWLock::RLockGuardWithTimeout RLockGuardWithTimeout; - typedef common::TCRWLock::RLockGuard RLockGuard; - typedef common::TCRWLock::WLockGuardWithTimeout WLockGuardWithTimeout; + typedef common::RWLock RWLock; + typedef RWLock::RLockGuard RLockGuard; + typedef RWLock::WLockGuard WLockGuard; private: - int submit_config_change_cmd_(const LogConfigChangeCmd &req); - int submit_config_change_cmd_(const LogConfigChangeCmd &req, - LogConfigChangeCmdResp &resp); - int append_(const void *buffer, const int64_t nbytes, const share::SCN &ref_scn, @@ -796,15 +360,16 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase DISALLOW_COPY_AND_ASSIGN(ObLogHandler); private: common::ObAddr self_; + mutable RWLock lock_; + palf::PalfHandle palf_handle_; + palf::PalfEnv *palf_env_; + bool is_in_stop_state_; + bool is_inited_; //log_handler will frequently call apply_status, reducing the overhead of hashing through applyservice ObApplyStatus *apply_status_; ObLogApplyService *apply_service_; ObLogReplayService *replay_service_; - ObRoleChangeService *rc_service_; - // Note: using TCRWLock for using WLockGuardWithTimeout common::TCRWLock deps_lock_; - mutable palf::PalfLocationCacheCb *lc_cb_; - mutable obcall::ObLogServiceRpcProxy *rpc_proxy_; common::ObQSync ls_qs_; ObMiniStat::ObStatItem append_cost_stat_; bool is_offline_; @@ -817,14 +382,6 @@ class ObLogHandler : public ObILogHandler, public ObLogHandlerBase struct ObLogStat { public: - // @desc: role of ls - // if ((log_handler is leader or restore_handler is leader) && palf is leader) - // role_ = LEADER; - // else - // role_ = FOLLOWER - common::ObRole role_; - // @desc: proposal_id of ls(log_handler) - int64_t proposal_id_; palf::PalfStat palf_stat_; bool in_sync_; TO_STRING_KV(K_(palf_stat), K_(in_sync)) @@ -869,7 +426,6 @@ struct LogDestroyIteratorStorageFunctor { template int seek_log_iterator_no_shared_storage(palf::PalfEnv *palf_env, - const int64_t palf_id, const StartPoint &start_point, IteratorType &iterator) { @@ -877,13 +433,13 @@ int seek_log_iterator_no_shared_storage(palf::PalfEnv *palf_env, palf::PalfHandle palf_handle; const bool first_inited = !iterator.is_inited(); bool need_release_palf_handle = true; - if (NULL == palf_env || !palf::is_valid_palf_id(palf_id) || !start_point.is_valid()) { + if (NULL == palf_env || !start_point.is_valid()) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", KP(palf_env), K(palf_id), K(start_point)); - } else if (OB_FAIL(palf_env->open(palf_id, palf_handle))) { - CLOG_LOG(WARN, "failed to open palf_handle", K(ret), K(palf_id)); + CLOG_LOG(WARN, "invalid argument", KP(palf_env), K(start_point)); + } else if (OB_FAIL(palf_env->open(palf_handle))) { + CLOG_LOG(WARN, "failed to open palf_handle", K(ret)); } else if (OB_FAIL(palf_handle.seek(start_point, iterator))) { - CLOG_LOG(WARN, "seek iterator from palf_handle failed", KR(ret), K(palf_id), K(start_point)); + CLOG_LOG(WARN, "seek iterator from palf_handle failed", KR(ret), K(start_point)); // only set destroy functor when iterator is initialized for the first time. } else if (first_inited) { // NB: the ownership of palf_handle has transfered to iterator after set_destroy_iterator_storage_functor successfully, @@ -892,8 +448,7 @@ int seek_log_iterator_no_shared_storage(palf::PalfEnv *palf_env, // To make code readable, add 'need_release_palf_handle' instead of using std::move and move constructor LogDestroyIteratorStorageFunctor functor(palf_env, palf_handle); if (OB_FAIL(iterator.set_destroy_iterator_storage_functor(functor))) { - CLOG_LOG(WARN, "set_destroy_iterator_storage_functor failed", KR(ret), KP(palf_env), K(palf_id), - K(start_point)); + CLOG_LOG(WARN, "set_destroy_iterator_storage_functor failed", KR(ret), KP(palf_env), K(start_point)); iterator.destroy(); } else { need_release_palf_handle = false; @@ -913,7 +468,7 @@ int ObLogHandler::seek_log_iterator_dispatch_(const StartPoint &start_point, { int ret = OB_SUCCESS; RLockGuard guard(lock_); - ret = seek_log_iterator_no_shared_storage(palf_env_, id_, start_point, iterator); + ret = seek_log_iterator_no_shared_storage(palf_env_, start_point, iterator); return ret; } @@ -934,44 +489,39 @@ int init_log_iterator( return ret; } -int __get_log_handler(const ObLSID &ls_id, - ObLogHandler *&log_handler, - storage::ObLSHandle &ls_handle); +int __get_log_handler(ObLogHandler *&log_handler, + storage::ObLS *&ls); template -int init_log_iterator_( - const ObLSID &ls_id, - const StartPoint &start_point, +int init_log_iterator_(const StartPoint &start_point, const int64_t suggested_read_buf_size, palf::PalfIterator &iterator) { int ret = OB_SUCCESS; ObLogHandler *log_handler = NULL; - storage::ObLSHandle ls_handle; - if (OB_FAIL(__get_log_handler(ls_id, log_handler, ls_handle))) { - CLOG_LOG(WARN, "__get_log_handler failed", K(ls_id)); + storage::ObLS *ls = nullptr; + if (OB_FAIL(__get_log_handler(log_handler, ls))) { + CLOG_LOG(WARN, "__get_log_handler failed"); } else if (OB_FAIL(init_log_iterator(log_handler, start_point, suggested_read_buf_size, iterator))) { - CLOG_LOG(WARN, "seek iterator from log_handler failed", K(ls_id), K(start_point)); + CLOG_LOG(WARN, "seek iterator from log_handler failed", K(start_point)); } else {} return ret; } template -int seek_log_iterator(const share::ObLSID &ls_id, - const StartPoint &start_point, +int seek_log_iterator(const StartPoint &start_point, palf::PalfIterator &iterator) { constexpr int64_t suggested_read_buf_size = palf::PALF_BLOCK_SIZE; - return init_log_iterator_(ls_id, start_point, suggested_read_buf_size, iterator); + return init_log_iterator_(start_point, suggested_read_buf_size, iterator); } template -int seek_log_iterator_for_cdc(const share::ObLSID &ls_id, - const StartPoint &start_point, +int seek_log_iterator_for_cdc(const StartPoint &start_point, const int64_t suggested_read_buf_size, palf::PalfIterator &iterator) { - return init_log_iterator_(ls_id, start_point, suggested_read_buf_size, iterator); + return init_log_iterator_(start_point, suggested_read_buf_size, iterator); } // =============================== Iterator end=========================== } // end namespace logservice diff --git a/src/logservice/ob_log_handler_base.cpp b/src/logservice/ob_log_handler_base.cpp deleted file mode 100644 index e67bd2726..000000000 --- a/src/logservice/ob_log_handler_base.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_log_handler_base.h" - -namespace oceanbase -{ -namespace logservice -{ -ObLogHandlerBase::ObLogHandlerBase() : - lock_(), - role_(common::FOLLOWER), - proposal_id_(palf::INVALID_PROPOSAL_ID), - id_(-1), - palf_handle_(), - palf_env_(NULL), - is_in_stop_state_(true), - is_inited_(false) -{} - -int ObLogHandlerBase::prepare_switch_role(common::ObRole &curr_role, - int64_t &curr_proposal_id, - common::ObRole &new_role, - int64_t &new_proposal_id, - bool &is_pending_state) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (OB_FAIL(palf_handle_.get_role(new_role, new_proposal_id, is_pending_state))) { - PALF_LOG(WARN, "PalfHandle get_role failed", K(ret), K_(id), K(curr_role), K(curr_proposal_id)); - } else { - curr_role = role_; - curr_proposal_id = proposal_id_; - PALF_LOG(TRACE, "prepare_switch_role success", K(ret), K_(id), K(curr_role), K(curr_proposal_id), - K(new_role), K(new_proposal_id)); - } - return ret; -} - -int ObLogHandlerBase::advance_election_epoch_and_downgrade_priority(const int64_t downgrade_priority_time_us, - const char *reason) -{ - RLockGuard guard(lock_); - return palf_handle_.advance_election_epoch_and_downgrade_priority(proposal_id_, downgrade_priority_time_us, reason); -} - -int ObLogHandlerBase::change_leader_to(const common::ObAddr &dst_addr) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (OB_FAIL(palf_handle_.change_leader_to(dst_addr))) { - PALF_LOG(WARN, "palf change_leader failed", K(ret), K(dst_addr), K(palf_handle_)); - } else { - PALF_LOG(INFO, "ObLogHandler change_laeder success", K(ret), K(dst_addr), K(palf_handle_)); - } - return ret; -} - -int ObLogHandlerBase::get_role(common::ObRole &role, int64_t &proposal_id) const -{ - int ret = OB_SUCCESS; - bool is_pending_state = false; - int64_t curr_palf_proposal_id; - ObRole curr_palf_role; - // Get the current proposal_id - RLockGuard guard(lock_); - const int64_t saved_proposal_id = ATOMIC_LOAD(&proposal_id_); - const ObRole saved_role = ATOMIC_LOAD(&role_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_in_stop_state_) { - ret = OB_NOT_RUNNING; - } else if (FOLLOWER == saved_role) { - role = FOLLOWER; - proposal_id = saved_proposal_id; - } else if (OB_FAIL(palf_handle_.get_role(curr_palf_role, curr_palf_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "get_role failed", K(ret)); - } else if (curr_palf_proposal_id != saved_proposal_id) { - // palf's proposal_id has changed, return FOLLOWER - role = FOLLOWER; - proposal_id = saved_proposal_id; - } else { - role = curr_palf_role; - proposal_id = saved_proposal_id; - } - return ret; -} - -// Note: do not acquire any lock in the function -int ObLogHandlerBase::get_role_atomically(common::ObRole &role) const -{ - int ret = OB_SUCCESS; - role = ATOMIC_LOAD(&role_); - return ret; -} -} // end namespace logservice -} // end namespace oceanbase diff --git a/src/logservice/ob_log_handler_base.h b/src/logservice/ob_log_handler_base.h deleted file mode 100644 index b2e8140e8..000000000 --- a/src/logservice/ob_log_handler_base.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_LOG_HANDLER_BASE_ -#define OCEANBASE_LOGSERVICE_OB_LOG_HANDLER_BASE_ -#include -#include "lib/lock/ob_tc_rwlock.h" -#include "common/ob_role.h" -#include "palf/palf_handle.h" - -namespace oceanbase -{ -namespace common -{ -class ObAddr; -} -namespace palf -{ -class PalfEnv; -} -namespace logservice -{ -class ObLogHandlerBase -{ -public: - ObLogHandlerBase(); - // @breif query role and proposal_id from ObLogHandlerBase and palf. - // @param[out], curr_role, role of ObLogHandler. - // @param[out], curr_proposal_id, proposal_id of ObLogHandler. - // @param[out], new_role, role of palf. - // @param[out], new_proposal_id, proposal_id of palf. - int prepare_switch_role(common::ObRole &curr_role, - int64_t &curr_proposal_id, - common::ObRole &new_role, - int64_t &new_proposal_id, - bool &is_pending_state) const; - // NB: only called by ObRoleChangeService - virtual void switch_role(const common::ObRole &role, const int64_t proposal_id) = 0; - int advance_election_epoch_and_downgrade_priority(const int64_t downgrade_priority_time_us, - const char *reason); - int change_leader_to(const common::ObAddr &dst_addr); - int get_role_atomically(common::ObRole &role) const; - -protected: - // @brief query role and proposal_id from ObLogHandler or ObLogRestoreHandler - // @param[out], role: - // LEADER, if 'role_' of ObLogHandler or ObLogRestoreHandler is LEADER and 'proposal_id' is same with PalfHandle. - // FOLLOWER, otherwise. - // @param[out], proposal_id, global monotonically increasing. - // @retval - // OB_SUCCESS - // NB: for standby, ObLogHandler is always FOLLOWER and for primary, ObLogRestoreHandler is always FOLLOWER - int get_role(common::ObRole &role, int64_t &proposal_id) const; - -public: - typedef common::RWLock RWLock; - typedef RWLock::RLockGuard RLockGuard; - typedef RWLock::WLockGuard WLockGuard; - mutable RWLock lock_; - common::ObRole role_; - int64_t proposal_id_; - int64_t id_; - palf::PalfHandle palf_handle_; - palf::PalfEnv *palf_env_; - bool is_in_stop_state_; - bool is_inited_; -}; -} // end namespace logservice -} // end namespace oceanbase -#endif diff --git a/src/logservice/ob_log_monitor.cpp b/src/logservice/ob_log_monitor.cpp index 6c31ee996..60b7d7a75 100644 --- a/src/logservice/ob_log_monitor.cpp +++ b/src/logservice/ob_log_monitor.cpp @@ -22,130 +22,10 @@ namespace oceanbase { namespace logservice { -#define LOG_MONITOR_EVENT_FMT_PREFIX "LOG", type_to_string_(event), "LS_ID", palf_id -#define LOG_MONITOR_EVENT_STR_FMT_PREFIX "LOG", event_str, "LS_ID", palf_id +#define LOG_MONITOR_EVENT_FMT_PREFIX "LOG", type_to_string_(event) // =========== PALF Event Reporting =========== -int ObLogMonitor::record_set_initial_member_list_event(const int64_t palf_id, - const int64_t replica_num, - const char *member_list, - const char *extra_info) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::SET_INITIAL_MEMBER_LIST; - if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "MEMBER_LIST", member_list, - "REPLICA_NUM", replica_num, - "", NULL, - "", NULL, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "MEMBER_LIST", member_list, - "REPLICA_NUM", replica_num); - } - return ret; -} - -int ObLogMonitor::record_election_leader_change_event(const int64_t palf_id, const common::ObAddr &dest_addr) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::ELECTION_LEADER_CHANGE; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, "LEADER", dest_addr); - return ret; -} - -int ObLogMonitor::record_reconfiguration_event(const char *sub_event, - const int64_t palf_id, - const palf::LogConfigVersion& config_version, - const int64_t prev_replica_num, - const int64_t curr_replica_num, - const char* extra_info) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::RECONFIGURATION; - const int64_t MAX_BUF_LEN = 50; - char event_str[MAX_BUF_LEN] = {'\0'}; - if (0 >= snprintf(event_str, MAX_BUF_LEN, "%s:%s", type_to_string_(event), sub_event)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(event), K(sub_event)); - } else if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY("LOG", event_str, - "LS_ID", palf_id, - "CONFIG_VERSION", config_version, - "PREV_REPLICA_NUM", prev_replica_num, - "CURR_REPLICA_NUM", curr_replica_num, - "", NULL, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY("LOG", event_str, - "LS_ID", palf_id, - "CONFIG_VERSION", config_version, - "PREV_REPLICA_NUM", prev_replica_num, - "CURR_REPLICA_NUM", curr_replica_num); - } - return ret; -} -int ObLogMonitor::record_replica_type_change_event(const int64_t palf_id, - const palf::LogConfigVersion& config_version, - const char *prev_replica_type, - const char *curr_replica_type, - const char *extra_info) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::REPLICA_TYPE_TRANSITION; - if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "CONFIG_VERSION", config_version, - "PREV_REPLICA_TYPE", prev_replica_type, - "CURR_REPLICA_TYPE", curr_replica_type, - "", NULL, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "CONFIG_VERSION", config_version, - "PREV_REPLICA_TYPE", prev_replica_type, - "CURR_REPLICA_TYPE", curr_replica_type); - } - - return ret; -} - -int ObLogMonitor::record_access_mode_change_event(const int64_t palf_id, - const int64_t prev_mode_version, - const int64_t curr_mode_verion, - const palf::AccessMode& prev_access_mode, - const palf::AccessMode& curr_access_mode, - const char *extra_info) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::ACCESS_MODE_TRANSITION; - const int64_t MAX_BUF_LEN = 32; - char prev_access_mode_str[MAX_BUF_LEN] = {'\0'}; - char curr_access_mode_str[MAX_BUF_LEN] = {'\0'}; - if (OB_FAIL(palf::access_mode_to_string(prev_access_mode, prev_access_mode_str, sizeof(prev_access_mode_str)))) { - PALF_LOG(WARN, "access_mode_to_string failed", K(prev_access_mode)); - } else if (OB_FAIL(palf::access_mode_to_string(curr_access_mode, curr_access_mode_str, sizeof(curr_access_mode_str)))) { - PALF_LOG(WARN, "access_mode_to_string failed", K(prev_access_mode)); - } else if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "PREV_MODE_VERSION", prev_mode_version, - "CURR_MODE_VERSION", curr_mode_verion, - "PREV_ACCESS_MODE", prev_access_mode_str, - "CURR_ACCESS_MODE", curr_access_mode_str, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "PREV_MODE_VERSION", prev_mode_version, - "CURR_MODE_VERSION", curr_mode_verion, - "PREV_ACCESS_MODE", prev_access_mode_str, - "CURR_ACCESS_MODE", curr_access_mode_str); - } - return ret; -} - -int ObLogMonitor::record_set_base_lsn_event(const int64_t palf_id, const palf::LSN &new_base_lsn) +int ObLogMonitor::record_set_base_lsn_event(const palf::LSN &new_base_lsn) { int ret = OB_SUCCESS; const EventType event = EventType::SET_BASE_LSN; @@ -154,39 +34,7 @@ int ObLogMonitor::record_set_base_lsn_event(const int64_t palf_id, const palf::L return ret; } -int ObLogMonitor::record_enable_sync_event(const int64_t palf_id) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::ENABLE_SYNC; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX); - return ret; -} - -int ObLogMonitor::record_disable_sync_event(const int64_t palf_id) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::DISABLE_SYNC; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX); - return ret; -} - -int ObLogMonitor::record_enable_vote_event(const int64_t palf_id) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::ENABLE_VOTE; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX); - return ret; -} - -int ObLogMonitor::record_disable_vote_event(const int64_t palf_id) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::DISABLE_VOTE; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX); - return ret; -} - -int ObLogMonitor::record_advance_base_info_event(const int64_t palf_id, const palf::PalfBaseInfo &palf_base_info) +int ObLogMonitor::record_advance_base_info_event(const palf::PalfBaseInfo &palf_base_info) { int ret = OB_SUCCESS; const EventType event = EventType::ADVANCE_BASE_INFO; @@ -195,20 +43,7 @@ int ObLogMonitor::record_advance_base_info_event(const int64_t palf_id, const pa return ret; } -int ObLogMonitor::record_rebuild_event(const int64_t palf_id, - const common::ObAddr &server, - const palf::LSN &base_lsn) -{ - int ret = OB_SUCCESS; - const EventType event = EventType::REBUILD; - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "SOURCE_SERVER", server, - "BASE_LSN", base_lsn); - return ret; -} - -int ObLogMonitor::record_truncate_event(const int64_t palf_id, - const palf::LSN &lsn, +int ObLogMonitor::record_truncate_event(const palf::LSN &lsn, const int64_t min_block_id, const int64_t max_block_id, const int64_t truncate_end_block_id) @@ -223,92 +58,8 @@ int ObLogMonitor::record_truncate_event(const int64_t palf_id, return ret; } -int ObLogMonitor::record_role_change_event(const int64_t palf_id, - const common::ObRole &prev_role, - const palf::ObReplicaState &prev_state, - const common::ObRole &curr_role, - const palf::ObReplicaState &curr_state, - const char *extra_info) -{ - int ret = OB_SUCCESS; - int pret = OB_SUCCESS; - const EventType event = EventType::ROLE_TRANSITION; - const int64_t MAX_BUF_LEN = 50; - char prev_str[MAX_BUF_LEN] = {'\0'}; - char curr_str[MAX_BUF_LEN] = {'\0'}; - TIMEGUARD_INIT(LOG_MONITOR, 100_ms); \ - if (0 >= (pret = snprintf(prev_str, MAX_BUF_LEN, "%s %s", role_to_string(prev_role), - replica_state_to_string(prev_state)))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(prev_role), K(prev_state)); - } else if (0 >= (pret = snprintf(curr_str, MAX_BUF_LEN, "%s %s", role_to_string(curr_role), - replica_state_to_string(curr_state)))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(curr_role), K(curr_state)); - } else { - CLICK(); - if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "PREVIOUS", prev_str, - "CURRENT", curr_str, - "", NULL, - "", NULL, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_FMT_PREFIX, - "PREVIOUS", prev_str, - "CURRENT", curr_str); - } - CLICK(); - } - return ret; -} - -int ObLogMonitor::record_parent_child_change_event(const int64_t palf_id, - const bool is_register, /* true: register; false; retire; */ - const bool is_parent, /* true: parent; false: child; */ - const common::ObAddr &server, - const int64_t register_time_us, - const char *extra_info) -{ - int ret = OB_SUCCESS; - int pret = OB_SUCCESS; - const char *action_str = (is_register)? "REGISTER": "RETIRE"; - const char *object_str = (is_parent)? "PARENT": "CHILD"; - const int64_t MAX_BUF_LEN = 50; - char event_str[MAX_BUF_LEN] = {'\0'}; - TIMEGUARD_INIT(LOG_MONITOR, 100_ms); \ - if (0 >= (pret = snprintf(event_str, MAX_BUF_LEN, "%s %s", action_str, object_str))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(action_str), K(object_str)); - } else { - CLICK(); - if (OB_NOT_NULL(extra_info)) { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_STR_FMT_PREFIX, - object_str, server, - "REGISTER_TIME_US", register_time_us, - "", NULL, - "info", extra_info); - } else { - SERVER_EVENT_ADD_WITH_RETRY(LOG_MONITOR_EVENT_STR_FMT_PREFIX, - object_str, server, - "REGISTER_TIME_US", register_time_us, - "", NULL); - } - CLICK(); - } - return ret; -} // =========== PALF Event Reporting =========== -// =========== PALF Performance Statistic =========== -int ObLogMonitor::add_log_write_stat(const int64_t palf_id, const int64_t log_write_size) -{ - int ret = OB_SUCCESS; - // TODO - return ret; -} -// =========== PALF Performance Statistic =========== #undef LOG_MONITOR_EVENT_FMT_PREFIX } // end namespace logservice diff --git a/src/logservice/ob_log_monitor.h b/src/logservice/ob_log_monitor.h index 11e7b91f1..dbb20f8e9 100644 --- a/src/logservice/ob_log_monitor.h +++ b/src/logservice/ob_log_monitor.h @@ -31,81 +31,20 @@ class ObLogMonitor : public palf::PalfMonitorCb virtual ~ObLogMonitor() { } public: // =========== PALF Event Reporting =========== - int record_set_initial_member_list_event(const int64_t palf_id, - const int64_t replica_num, - const char *member_list = NULL, - const char *extra_info = NULL) override final; - int record_election_leader_change_event(const int64_t palf_id, const common::ObAddr &dest_addr) override final; - int record_reconfiguration_event(const char *sub_event, - const int64_t palf_id, - const palf::LogConfigVersion& config_version, - const int64_t prev_replica_num, - const int64_t curr_replica_num, - const char *extra_info = NULL) override final; - int record_replica_type_change_event(const int64_t palf_id, - const palf::LogConfigVersion& config_version, - const char *prev_replica_type, - const char *curr_replica_type, - const char *extra_info = NULL) override final; - int record_access_mode_change_event(const int64_t palf_id, - const int64_t prev_mode_version, - const int64_t curr_mode_verion, - const palf::AccessMode& prev_access_mode, - const palf::AccessMode& curr_access_mode, - const char *extra_info = NULL) override final; - int record_set_base_lsn_event(const int64_t palf_id, const palf::LSN &new_base_lsn) override final; - int record_enable_sync_event(const int64_t palf_id) override final; - int record_disable_sync_event(const int64_t palf_id) override final; - int record_enable_vote_event(const int64_t palf_id) override final; - int record_disable_vote_event(const int64_t palf_id) override final; - int record_advance_base_info_event(const int64_t palf_id, const palf::PalfBaseInfo &palf_base_info) override final; - int record_rebuild_event(const int64_t palf_id, - const common::ObAddr &server, - const palf::LSN &base_lsn) override final; - int record_truncate_event(const int64_t palf_id, - const palf::LSN &lsn, + int record_set_base_lsn_event(const palf::LSN &new_base_lsn) override final; + int record_advance_base_info_event(const palf::PalfBaseInfo &palf_base_info) override final; + int record_truncate_event(const palf::LSN &lsn, const int64_t min_block_id, const int64_t max_block_id, const int64_t truncate_end_block_id) override final; - int record_role_change_event(const int64_t palf_id, - const common::ObRole &prev_role, - const palf::ObReplicaState &prev_state, - const common::ObRole &curr_role, - const palf::ObReplicaState &curr_state, - const char *extra_info = NULL) override final; - int record_parent_child_change_event(const int64_t palf_id, - const bool is_register, /* true: register; false; retire; */ - const bool is_parent, /* true: parent; false: child; */ - const common::ObAddr &server, - const int64_t register_time_us, - const char *extra_info = NULL) override final; // =========== PALF Event Reporting =========== -public: - // =========== PALF Performance Statistic =========== - int add_log_write_stat(const int64_t palf_id, const int64_t log_write_size) override final; - // =========== PALF Performance Statistic =========== private: enum EventType { UNKNOWN = 0, - DEGRADE, - UPGRADE, - SET_INITIAL_MEMBER_LIST, - ELECTION_LEADER_CHANGE, - ROLE_TRANSITION, - RECONFIGURATION, - REPLICA_TYPE_TRANSITION, - ACCESS_MODE_TRANSITION, SET_BASE_LSN, - ENABLE_SYNC, - DISABLE_SYNC, - ENABLE_VOTE, - DISABLE_VOTE, ADVANCE_BASE_INFO, - REBUILD, - TRUNCATE, - ENTER_ELECTION_SILENT, - EXIT_ELECTION_SILENT + TRUNCATE }; const char *type_to_string_(const EventType &event) const @@ -113,38 +52,11 @@ class ObLogMonitor : public palf::PalfMonitorCb #define CHECK_LOG_EVENT_TYPE_STR(x) case(EventType::x): return #x switch (event) { - CHECK_LOG_EVENT_TYPE_STR(DEGRADE); - CHECK_LOG_EVENT_TYPE_STR(UPGRADE); - case (EventType::SET_INITIAL_MEMBER_LIST): - return "SET INITIAL MEMBER LIST"; - case (EventType::ELECTION_LEADER_CHANGE): - return "ELECTION LEADER CHANGE"; - case (EventType::ROLE_TRANSITION): - return "ROLE TRANSITION"; - case (EventType::RECONFIGURATION): - return "RECONFIGURATION"; - case (EventType::REPLICA_TYPE_TRANSITION): - return "REPLICA TYPE TRANSITION"; - case (EventType::ACCESS_MODE_TRANSITION): - return "ACCESS MODE TRANSITION"; case (EventType::SET_BASE_LSN): return "SET BASE LSN"; - case (EventType::ENABLE_SYNC): - return "ENABLE SYNC"; - case (EventType::DISABLE_SYNC): - return "DISABLE SYNC"; - case (EventType::ENABLE_VOTE): - return "ENABLE VOTE"; - case (EventType::DISABLE_VOTE): - return "DISABLE VOTE"; case (EventType::ADVANCE_BASE_INFO): return "ADVANCE BASE INFO"; - CHECK_LOG_EVENT_TYPE_STR(REBUILD); CHECK_LOG_EVENT_TYPE_STR(TRUNCATE); - case (EventType::ENTER_ELECTION_SILENT): - return "ENTER ELECTION SILENT"; - case (EventType::EXIT_ELECTION_SILENT): - return "EXIT ELECTION SILENT"; default: return "UNKNOWN"; } diff --git a/src/logservice/ob_log_service.cpp b/src/logservice/ob_log_service.cpp index 1edd5bc24..1f862dacd 100644 --- a/src/logservice/ob_log_service.cpp +++ b/src/logservice/ob_log_service.cpp @@ -18,6 +18,7 @@ #include "ob_log_service.h" #include "share/rc/ob_module_provider.h" #include "ob_server_log_block_mgr.h" +#include "logservice/palf_handle_guard.h" #include "logservice/ob_tenant_mutil_allocator_mgr.h" #include "share/rc/ob_tenant_module_init_ctx.h" #include "observer/ob_srv_network_frame.h" @@ -25,6 +26,8 @@ #include "logservice/ob_net_keepalive_adapter.h" // ObNetKeepAliveAdapter #include "share/ob_io_device_helper.h" #include "lib/ob_running_mode.h" +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_service.h" #include "share/ob_share_util.h" // relocated-definition owner namespace oceanbase @@ -47,8 +50,6 @@ ObLogService::ObLogService() : alloc_mgr_(NULL), apply_service_(), replay_service_(), - role_change_service_(), - location_adapter_(), ls_adapter_(), rpc_proxy_(), monitor_(), @@ -70,10 +71,8 @@ int ObLogService::mtl_init(ObLogService* &logservice) const palf::PalfOptions &palf_options = MTL_INIT_CTX()->palf_options_; const char *tenant_clog_dir = MTL_INIT_CTX()->tenant_clog_dir_; const char *clog_dir = OB_FILE_SYSTEM_ROUTER.get_clog_dir(); - ObLocationService *location_service = GCTX.location_service_; ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_; common::ObILogAllocator *alloc_mgr = NULL; - common::ObMySQLProxy *mysql_proxy = GCTX.sql_proxy_; ObNetKeepAliveAdapter *net_keepalive_adapter = NULL; if (OB_FAIL(TMA_MGR_INSTANCE.get_tenant_log_allocator(alloc_mgr))) { CLOG_LOG(WARN, "get_tenant_log_allocator failed", K(ret)); @@ -85,9 +84,7 @@ int ObLogService::mtl_init(ObLogService* &logservice) self, alloc_mgr, share::g_mp->ls_service(), - location_service, log_block_mgr, - mysql_proxy, net_keepalive_adapter))) { CLOG_LOG(ERROR, "init ObLogService failed", K(ret), K(tenant_clog_dir)); } else if (OB_FAIL(FileDirectoryUtils::fsync_dir(clog_dir))) { @@ -122,8 +119,6 @@ int ObLogService::start() CLOG_LOG(WARN, "failed to start apply_service_", K(ret)); } else if (OB_FAIL(replay_service_.start())) { CLOG_LOG(WARN, "failed to start replay_service_", K(ret)); - } else if (OB_FAIL(role_change_service_.start())) { - CLOG_LOG(WARN, "failed to start role_change_service_", K(ret)); } else { is_running_ = true; FLOG_INFO("ObLogService is started"); @@ -137,7 +132,6 @@ void ObLogService::stop() CLOG_LOG(INFO, "begin to stop ObLogService"); (void)apply_service_.stop(); (void)replay_service_.stop(); - (void)role_change_service_.stop(); FLOG_INFO("ObLogService is stopped"); } @@ -145,7 +139,6 @@ void ObLogService::wait() { apply_service_.wait(); replay_service_.wait(); - role_change_service_.wait(); } void ObLogService::destroy() @@ -154,8 +147,6 @@ void ObLogService::destroy() self_.reset(); apply_service_.destroy(); replay_service_.destroy(); - role_change_service_.destroy(); - location_adapter_.destroy(); ls_adapter_.destroy(); rpc_proxy_.destroy(); if (NULL != palf_env_) { @@ -193,9 +184,7 @@ int ObLogService::init(const PalfOptions &options, const common::ObAddr &self, common::ObILogAllocator *alloc_mgr, ObLSService *ls_service, - ObLocationService *location_service, palf::ILogBlockPool *log_block_pool, - common::ObMySQLProxy *sql_proxy, IObNetKeepAliveAdapter *net_keepalive_adapter) { int ret = OB_SUCCESS; @@ -208,12 +197,10 @@ int ObLogService::init(const PalfOptions &options, CLOG_LOG(WARN, "ObLogService init twice", K(ret)); } else if (false == options.is_valid() || OB_ISNULL(base_dir) || OB_UNLIKELY(!self.is_valid()) || OB_ISNULL(alloc_mgr) || OB_ISNULL(ls_service) - || OB_ISNULL(location_service) || OB_ISNULL(log_block_pool) - || OB_ISNULL(sql_proxy) || OB_ISNULL(net_keepalive_adapter)) { + || OB_ISNULL(log_block_pool) || OB_ISNULL(net_keepalive_adapter)) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "invalid arguments", K(ret), K(options), KP(base_dir), K(self), - KP(alloc_mgr), KP(ls_service), KP(location_service), - KP(log_block_pool), KP(sql_proxy), KP(net_keepalive_adapter)); + KP(alloc_mgr), KP(ls_service), KP(log_block_pool), KP(net_keepalive_adapter)); } else if (OB_FAIL(PalfEnv::create_palf_env(options, base_dir, self, alloc_mgr, log_block_pool, &monitor_, &LOCAL_DEVICE_INSTANCE, &OB_IO_MANAGER, palf_env_))) { @@ -227,10 +214,6 @@ int ObLogService::init(const PalfOptions &options, CLOG_LOG(WARN, "failed to init apply_service", K(ret)); } else if (OB_FAIL(replay_service_.init(palf_env_, &ls_adapter_, alloc_mgr))) { CLOG_LOG(WARN, "failed to init replay_service", K(ret)); - } else if (OB_FAIL(role_change_service_.init(ls_service, &apply_service_, &replay_service_))) { - CLOG_LOG(WARN, "failed to init role_change_service_", K(ret)); - } else if (OB_FAIL(location_adapter_.init(location_service))) { - CLOG_LOG(WARN, "failed to init location_adapter_", K(ret)); } else if (OB_FAIL(rpc_proxy_.init())) { CLOG_LOG(WARN, "LogServiceRpcProxy init failed", K(ret)); } else { @@ -248,64 +231,34 @@ int ObLogService::init(const PalfOptions &options, return ret; } -palf::AccessMode ObLogService::get_palf_access_mode(const share::ObTenantRole &tenant_role) -{ - palf::AccessMode mode = palf::AccessMode::INVALID_ACCESS_MODE; - switch (tenant_role.value()) { - case share::ObTenantRole::INVALID_TENANT: - mode = palf::AccessMode::INVALID_ACCESS_MODE; - break; - case share::ObTenantRole::PRIMARY_TENANT: - mode = palf::AccessMode::APPEND; - break; - case share::ObTenantRole::STANDBY_TENANT: - case share::ObTenantRole::RESTORE_TENANT: - mode = palf::AccessMode::RAW_WRITE; - break; - default: - mode = palf::AccessMode::INVALID_ACCESS_MODE; - break; - } - return mode; -} - -int ObLogService::create_ls(const share::ObLSID &id, - const common::ObReplicaType &replica_type, - const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync, +int ObLogService::create_ls(const palf::PalfBaseInfo &palf_base_info, ObLogHandler &log_handler) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + CLOG_LOG(WARN, "log_service is not inited", K(ret)); } else if (!palf_base_info.is_valid()) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid arguments", K(ret), K(id), K(replica_type), - K(tenant_role), K(palf_base_info)); - } else if (OB_FAIL(create_ls_(id, replica_type, tenant_role, palf_base_info, - allow_log_sync, log_handler))) { - CLOG_LOG(WARN, "create ls failed", K(ret), K(id), K(replica_type), - K(tenant_role), K(palf_base_info)); + CLOG_LOG(WARN, "invalid palf base info", K(ret), K(palf_base_info)); + } else if (OB_FAIL(create_ls_(palf_base_info, log_handler))) { + CLOG_LOG(WARN, "create ls failed", K(ret), K(palf_base_info)); } else { - FLOG_INFO("ObLogService create_ls success", K(ret), K(id), K(replica_type), K(tenant_role), K(palf_base_info), - K(log_handler)); + FLOG_INFO("ObLogService create_ls success", K(ret), K(palf_base_info), K(log_handler)); } return ret; } -int ObLogService::remove_ls(const ObLSID &id, - ObLogHandler &log_handler) +int ObLogService::remove_ls(ObLogHandler &log_handler) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); - } else if (OB_FAIL(apply_service_.remove_ls(id))) { - CLOG_LOG(WARN, "failed to remove from apply_service", K(ret), K(id)); - } else if (OB_FAIL(replay_service_.remove_ls(id))) { - CLOG_LOG(WARN, "failed to remove from replay_service", K(ret), K(id)); + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else if (OB_FAIL(apply_service_.remove_status())) { + CLOG_LOG(WARN, "failed to remove from apply_service", K(ret)); + } else if (OB_FAIL(replay_service_.remove_status())) { + CLOG_LOG(WARN, "failed to remove from replay_service", K(ret)); } else { // NB: can not execute destroy, otherwise, each interface in log_handler or restore_handler // may return OB_NOT_INIT. @@ -314,30 +267,30 @@ int ObLogService::remove_ls(const ObLSID &id, // In normal case(for gc), stop has been executed, this stop has no effect. // In abnormal case(create ls failed, need remove ls directlly), there is no possibility for dead lock. log_handler.stop(); - if (OB_FAIL(palf_env_->remove(id.id()))) { - CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret), K(id)); + if (OB_FAIL(palf_env_->remove())) { + CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret)); } else { - FLOG_INFO("ObLogService remove_ls success", K(ret), K(id)); + FLOG_INFO("ObLogService remove_ls success", K(ret)); } } return ret; } -int ObLogService::check_palf_exist(const ObLSID &id, bool &exist) const +int ObLogService::check_palf_exist(bool &exist) const { int ret = OB_SUCCESS; PalfHandle handle; exist = true; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "ObLogService is not inited", K(ret), K(id)); - } else if (OB_FAIL(palf_env_->open(id.id(), handle))) { + CLOG_LOG(WARN, "ObLogService is not inited", K(ret)); + } else if (OB_FAIL(palf_env_->open(handle))) { if (OB_ENTRY_NOT_EXIST == ret ) { ret = OB_SUCCESS; exist = false; } else { - CLOG_LOG(WARN, "open palf failed", K(ret), K(id)); + CLOG_LOG(WARN, "open palf failed", K(ret)); } } @@ -347,51 +300,38 @@ int ObLogService::check_palf_exist(const ObLSID &id, bool &exist) const return ret; } -int ObLogService::add_ls(const ObLSID &id, - ObLogHandler &log_handler) +int ObLogService::add_ls(ObLogHandler &log_handler) { int ret = OB_SUCCESS; - PalfHandle &log_handler_palf_handle = log_handler.palf_handle_; - PalfRoleChangeCb *rc_cb = &role_change_service_; - PalfLocationCacheCb *loc_cache_cb = &location_adapter_; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); - } else if (OB_FAIL(apply_service_.add_ls(id))) { - CLOG_LOG(WARN, "failed to add_ls for apply_service", K(ret), K(id)); - } else if (OB_FAIL(replay_service_.add_ls(id))) { - CLOG_LOG(WARN, "failed to add_ls for replay_service", K(ret), K(id)); - } else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_, - &role_change_service_, palf_env_, loc_cache_cb, &rpc_proxy_, alloc_mgr_))) { - CLOG_LOG(WARN, "ObLogHandler init failed", K(ret), K(id), KP(palf_env_)); - } else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) { - CLOG_LOG(WARN, "register_role_change_cb failed", K(ret)); - } else if (OB_FAIL(log_handler_palf_handle.set_location_cache_cb(loc_cache_cb))) { - CLOG_LOG(WARN, "set_location_cache_cb failed", K(ret), K(id)); + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else if (OB_FAIL(apply_service_.create_status())) { + CLOG_LOG(WARN, "failed to create apply status", K(ret)); + } else if (OB_FAIL(replay_service_.create_status())) { + CLOG_LOG(WARN, "failed to create replay status", K(ret)); + } else if (OB_FAIL(log_handler.init(self_, &apply_service_, &replay_service_, + palf_env_, alloc_mgr_))) { + CLOG_LOG(WARN, "ObLogHandler init failed", K(ret), KP(palf_env_)); } else { - FLOG_INFO("add_ls success", K(ret), K(id), KP(this)); - } - - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_LS_NOT_EXIST; + FLOG_INFO("add_ls success", K(ret), KP(this)); } return ret; } -int ObLogService::open_palf(const share::ObLSID &id, - palf::PalfHandleGuard &palf_handle_guard) +int ObLogService::open_palf(palf::PalfHandleGuard &palf_handle_guard) { int ret = OB_SUCCESS; palf::PalfHandle palf_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); - } else if (OB_FAIL(palf_env_->open(id.id(), palf_handle))) { - CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id)); + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else if (OB_FAIL(palf_env_->open(palf_handle))) { + CLOG_LOG(WARN, "failed to get palf_handle", K(ret)); } else if (FALSE_IT(palf_handle_guard.set(palf_handle, palf_env_))) { } else { - CLOG_LOG(TRACE, "ObLogService open_palf success", K(ret), K(id)); + CLOG_LOG(TRACE, "ObLogService open_palf success", K(ret)); } if (OB_FAIL(ret)) { @@ -400,9 +340,6 @@ int ObLogService::open_palf(const share::ObLSID &id, } } - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_LS_NOT_EXIST; - } return ret; } @@ -414,9 +351,6 @@ int ObLogService::update_replayable_point(const SCN &replayable_point) CLOG_LOG(WARN, "log_service is not inited", K(ret)); } else if (OB_FAIL(replay_service_.update_replayable_point(replayable_point))) { CLOG_LOG(WARN, "update_replayable_point failed", K(ret), K(replayable_point)); - // should be removed in version 4.2.0.0 - } else if (OB_FAIL(palf_env_->update_replayable_point(replayable_point))) { - CLOG_LOG(WARN, "update_replayable_point failed", K(replayable_point)); } return ret; } @@ -433,23 +367,6 @@ int ObLogService::get_replayable_point(SCN &replayable_point) return ret; } -int ObLogService::get_palf_role(const share::ObLSID &id, - common::ObRole &role, - int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - palf::PalfHandleGuard palf_handle_guard; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); - } else if (OB_FAIL(open_palf(id, palf_handle_guard))) { - CLOG_LOG(WARN, "failed to open palf", K(ret), K(id)); - } else if (OB_FAIL(palf_handle_guard.get_role(role, proposal_id))) { - CLOG_LOG(WARN, "failed to get role", K(ret), K(id)); - } - return ret; -} - int ObLogService::get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte) { int ret = OB_SUCCESS; @@ -478,27 +395,15 @@ int ObLogService::update_palf_options_except_disk_usage_limit_size() int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!true) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "tenant_config is not valid", K(ret)); } else { PalfOptions palf_opts; - common::ObCompressorType compressor_type = ZSTD_1_3_8_COMPRESSOR; - if (OB_FAIL(common::ObCompressorPool::get_instance().get_compressor_type( - GCONF.log_transport_compress_func, compressor_type))) { - CLOG_LOG(ERROR, "log_transport_compress_func invalid.", K(ret)); - //Need to get log_disk_usage_limit_size - } else if (OB_FAIL(palf_env_->get_options(palf_opts))) { + if (OB_FAIL(palf_env_->get_options(palf_opts))) { CLOG_LOG(WARN, "palf get_options failed", K(ret)); } else { palf_opts.disk_options_.log_disk_utilization_threshold_ = GCONF.log_disk_utilization_threshold; palf_opts.disk_options_.log_disk_utilization_limit_threshold_ = GCONF.log_disk_utilization_limit_threshold; palf_opts.disk_options_.log_disk_throttling_percentage_ = GCONF.log_disk_throttling_percentage; palf_opts.disk_options_.log_disk_throttling_maximum_duration_ = GCONF.log_disk_throttling_maximum_duration; - palf_opts.compress_options_.enable_transport_compress_ = GCONF.log_transport_compress_all; - palf_opts.compress_options_.transport_compress_func_ = compressor_type; - palf_opts.rebuild_replica_log_lag_threshold_ = GCONF._rebuild_replica_log_lag_threshold; - palf_opts.disk_options_.log_writer_parallelism_ = GCONF._log_writer_parallelism; palf_opts.enable_log_cache_ = GCONF._enable_log_cache; if (OB_FAIL(palf_env_->update_options(palf_opts))) { CLOG_LOG(WARN, "palf update_options failed", K(ret), K(palf_opts)); @@ -539,136 +444,106 @@ int ObLogService::get_palf_options(palf::PalfOptions &opts) return ret; } -int ObLogService::iterate_palf(const ObFunction &func) +int ObLogService::stat_palf(PalfStat &palf_stat) { int ret = OB_SUCCESS; + PalfHandleGuard guard; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else { - ret = palf_env_->for_each(func); + } else if (OB_FAIL(open_palf(guard))) { + CLOG_LOG(WARN, "failed to open palf", K(ret)); + } else if (OB_FAIL(guard.get_palf_handle()->stat(palf_stat))) { + CLOG_LOG(WARN, "failed to stat palf", K(ret)); } return ret; } -int ObLogService::iterate_apply(const ObFunction &func) +int ObLogService::stat_apply(LSApplyStat &apply_stat) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - ret = apply_service_.stat_for_each(func); + ret = apply_service_.stat(apply_stat); } return ret; } -int ObLogService::iterate_replay(const ObFunction &func) +int ObLogService::stat_replay(LSReplayStat &replay_stat) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - ret = replay_service_.stat_for_each(func); + ret = replay_service_.stat(replay_stat); } return ret; } -int ObLogService::create_ls_(const share::ObLSID &id, - const common::ObReplicaType &replica_type, - const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync, +int ObLogService::create_ls_(const palf::PalfBaseInfo &palf_base_info, ObLogHandler &log_handler) { int ret = OB_SUCCESS; PalfHandle palf_handle; - PalfRoleChangeCb *rc_cb = &role_change_service_; - PalfLocationCacheCb *loc_cache_cb = &location_adapter_; - const bool is_arb_replica = (replica_type == REPLICA_TYPE_ARBITRATION); - PalfHandle &log_handler_palf_handle = log_handler.palf_handle_; bool palf_exist = true; - if (false == id.is_valid() || - INVALID_TENANT_ROLE == tenant_role || - false == palf_base_info.is_valid()) { + if (false == palf_base_info.is_valid()) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(ret), K(id), K(tenant_role), K(palf_base_info)); - } else if (OB_FAIL(check_palf_exist(id, palf_exist))) { - CLOG_LOG(WARN, "check_palf_exist failed", K(ret), K(id), K(tenant_role), K(palf_base_info)); + CLOG_LOG(WARN, "invalid argument", K(ret), K(palf_base_info)); + } else if (OB_FAIL(check_palf_exist(palf_exist))) { + CLOG_LOG(WARN, "check_palf_exist failed", K(ret), K(palf_base_info)); } else if (palf_exist) { ret = OB_ENTRY_EXIST; - CLOG_LOG(WARN, "palf has eixst", K(ret), K(id), K(tenant_role), K(palf_base_info)); + CLOG_LOG(WARN, "palf has eixst", K(ret), K(palf_base_info)); } else { - if (!is_arb_replica && - OB_FAIL(palf_env_->create(id.id(), get_palf_access_mode(tenant_role), palf_base_info, palf_handle))) { - CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id), K(replica_type)); - } else if (false == allow_log_sync && OB_FAIL(palf_handle.disable_sync())) { - CLOG_LOG(WARN, "failed to disable_sync", K(ret), K(id)); - } else if (OB_FAIL(apply_service_.add_ls(id))) { - CLOG_LOG(WARN, "failed to add_ls for apply engine", K(ret), K(id)); - } else if (OB_FAIL(replay_service_.add_ls(id))) { - CLOG_LOG(WARN, "failed to add_ls", K(ret), K(id)); - } else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_, - &role_change_service_, palf_env_, loc_cache_cb, &rpc_proxy_, alloc_mgr_))) { + if (OB_FAIL(palf_env_->create(palf::AccessMode::APPEND, palf_base_info, palf_handle))) { + CLOG_LOG(WARN, "failed to get palf_handle", K(ret)); + } else if (OB_FAIL(apply_service_.create_status())) { + CLOG_LOG(WARN, "failed to create apply status", K(ret)); + } else if (OB_FAIL(replay_service_.create_status())) { + CLOG_LOG(WARN, "failed to create replay status", K(ret)); + } else if (OB_FAIL(log_handler.init(self_, &apply_service_, &replay_service_, + palf_env_, alloc_mgr_))) { CLOG_LOG(WARN, "ObLogHandler init failed", K(ret), KP(palf_env_), K(palf_handle)); - } else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) { - CLOG_LOG(WARN, "register_role_change_cb failed", K(ret), K(id)); - } else if (OB_FAIL(log_handler_palf_handle.set_location_cache_cb(loc_cache_cb))) { - CLOG_LOG(WARN, "set_location_cache_cb failed", K(ret), K(id)); } else { - CLOG_LOG(INFO, "ObLogService create_ls success", K(ret), K(id), K(log_handler)); + CLOG_LOG(INFO, "ObLogService create_ls success", K(ret), K(log_handler)); } if (palf_handle.is_valid() && nullptr != palf_env_) { palf_env_->close(palf_handle); } if (OB_FAIL(ret)) { - CLOG_LOG(ERROR, "create_ls failed!!!", KR(ret), K(id)); - replay_service_.remove_ls(id); - apply_service_.remove_ls(id); + CLOG_LOG(ERROR, "create_ls failed!!!", KR(ret)); + replay_service_.remove_status(); + apply_service_.remove_status(); log_handler.destroy(); palf_env_->close(palf_handle); - palf_env_->remove(id.id()); + palf_env_->remove(); } } return ret; } -int ObLogService::diagnose_role_change(RCDiagnoseInfo &diagnose_info) +int ObLogService::diagnose_replay(ReplayDiagnoseInfo &diagnose_info) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "log_service is not inited", K(ret)); - } else if (OB_FAIL(role_change_service_.diagnose(diagnose_info))) { - CLOG_LOG(WARN, "role_change_service diagnose failed", K(ret)); + } else if (OB_FAIL(replay_service_.diagnose(diagnose_info))) { + CLOG_LOG(WARN, "replay_service diagnose failed", K(ret)); } else { // do nothing } return ret; } -int ObLogService::diagnose_replay(const share::ObLSID &id, - ReplayDiagnoseInfo &diagnose_info) +int ObLogService::diagnose_apply(ApplyDiagnoseInfo &diagnose_info) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "log_service is not inited", K(ret)); - } else if (OB_FAIL(replay_service_.diagnose(id, diagnose_info))) { - CLOG_LOG(WARN, "replay_service diagnose failed", K(ret), K(id)); - } else { - // do nothing - } - return ret; -} - -int ObLogService::diagnose_apply(const share::ObLSID &id, - ApplyDiagnoseInfo &diagnose_info) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "log_service is not inited", K(ret)); - } else if (OB_FAIL(apply_service_.diagnose(id, diagnose_info))) { - CLOG_LOG(WARN, "apply_service diagnose failed", K(ret), K(id)); + } else if (OB_FAIL(apply_service_.diagnose(diagnose_info))) { + CLOG_LOG(WARN, "apply_service diagnose failed", K(ret)); } else { // do nothing } @@ -713,55 +588,41 @@ int ObLogService::check_need_do_checkpoint(bool &need_do_checkpoint) } else if (OB_FAIL(palf_env_->get_disk_usage(used_size, total_size))) { CLOG_LOG(WARN, "get_disk_usage failed", K(ret)); } else { + int64_t unrecyclable_log_disk_size = 0; const int64_t CHECKPOINT_PERCENTAGE = GCTX.is_shared_storage_mode() ? 60 : 30; - ObLSService *ls_service = share::g_mp->ls_service(); - ObSharedGuard iterator; - if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "ObLSService is nullptr", KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls_iter(iterator, ObLSGetMod::LOG_MOD))) { - CLOG_LOG(WARN, "get_ls_iter failed", KP(ls_service)); + if (OB_FAIL(get_unrecyclable_log_disk_size(unrecyclable_log_disk_size))) { + CLOG_LOG(WARN, "get unrecyclable log disk size failed", K(ret)); } else { - ObLS *ls = NULL; - GetUnrecycableLogDiskSizeFunctor functor; - const int64_t &unrecycable_log_disk_size = functor.unrecycable_log_disk_size_; - while (OB_SUCC(iterator->get_next(ls))) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(functor(ls))) { - CLOG_LOG(WARN, "get unrecycable_log_disk_size failed", KR(tmp_ret)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - need_do_checkpoint = (unrecycable_log_disk_size * 100 >= total_size * CHECKPOINT_PERCENTAGE); - CLOG_LOG(TRACE, "check_need_do_checkpoint", K(unrecycable_log_disk_size), K(total_size), K(need_do_checkpoint)); - } + need_do_checkpoint = (unrecyclable_log_disk_size * 100 >= total_size * CHECKPOINT_PERCENTAGE); + CLOG_LOG(TRACE, "check_need_do_checkpoint", K(unrecyclable_log_disk_size), K(total_size), K(need_do_checkpoint)); } } return ret; } -int ObLogService::GetUnrecycableLogDiskSizeFunctor::operator()(ObLS *ls) +int ObLogService::get_unrecyclable_log_disk_size(int64_t &unrecyclable_log_disk_size) { int ret = OB_SUCCESS; - ObLogHandler *log_handler = NULL; - LSN end_lsn; - LSN base_lsn; - if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "unexpected error, ObLS is nullptr", KP(ls)); - } else if (FALSE_IT(log_handler = ls->get_log_handler())) { - } else if (OB_ISNULL(log_handler)) { + ObLSService *ls_service = share::g_mp->ls_service(); + ObLS *ls = nullptr; + unrecyclable_log_disk_size = 0; + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "unexpected error, ObLogHandler is nullptr", KP(ls)); - } else if (FALSE_IT(base_lsn = ls->get_clog_base_lsn())) { - } else if (OB_FAIL(log_handler->get_end_lsn(end_lsn))) { - CLOG_LOG(WARN, "get_end_lsn failed", KP(ls), K(base_lsn)); - } else if (end_lsn < base_lsn) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "end_lsn is smaller than base_lsn", K(lbt()), K(end_lsn), K(base_lsn)); + CLOG_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + CLOG_LOG(WARN, "get log stream failed", KP(ls_service), K(ret)); } else { - unrecycable_log_disk_size_ += (end_lsn - base_lsn); + ObLogHandler *log_handler = ls->get_log_handler(); + LSN end_lsn; + LSN base_lsn = ls->get_clog_base_lsn(); + if (OB_FAIL(log_handler->get_end_lsn(end_lsn))) { + CLOG_LOG(WARN, "get_end_lsn failed", KP(ls), K(base_lsn)); + } else if (end_lsn < base_lsn) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "end_lsn is smaller than base_lsn", K(lbt()), K(end_lsn), K(base_lsn)); + } else { + unrecyclable_log_disk_size = end_lsn - base_lsn; + } } return ret; } diff --git a/src/logservice/ob_log_service.h b/src/logservice/ob_log_service.h index c548ee3d0..ff22e034a 100644 --- a/src/logservice/ob_log_service.h +++ b/src/logservice/ob_log_service.h @@ -20,15 +20,13 @@ #include "common/ob_role.h" #include "lib/ob_define.h" #include "applyservice/ob_log_apply_service.h" -#include "logrpc/ob_log_rpc_req.h" #include "logrpc/ob_log_service_rpc_shell.h" #include "palf/log_block_pool_interface.h" // ILogBlockPool #include "palf/log_define.h" -#include "rcservice/ob_role_change_service.h" +#include "localservice/ob_local_log_handler_set.h" #include "replayservice/ob_log_replay_service.h" #include "ob_net_keepalive_adapter.h" #include "ob_ls_adapter.h" -#include "ob_location_adapter.h" #include "ob_log_handler.h" #include "ob_log_monitor.h" @@ -38,7 +36,6 @@ namespace common { class ObAddr; class ObILogAllocator; -class ObMySQLProxy; } namespace rpc @@ -51,22 +48,20 @@ class ObReqTransport; namespace share { -class ObLSID; -class ObLocationService; class SCN; } +namespace storage +{ +class ObLSService; +} + namespace palf { class PalfHandleGuard; -class PalfRoleChangeCb; class PalfDiskOptions; class PalfEnv; } -namespace storage -{ -class ObLSService; -} namespace logservice { @@ -83,51 +78,24 @@ class ObLogService void wait(); void destroy(); public: - static palf::AccessMode get_palf_access_mode(const share::ObTenantRole &tenant_role); int init(const palf::PalfOptions &options, const char *base_dir, const common::ObAddr &self, common::ObILogAllocator *alloc_mgr, storage::ObLSService *ls_service, - share::ObLocationService *location_service, palf::ILogBlockPool *log_block_pool, - common::ObMySQLProxy *sql_proxy, IObNetKeepAliveAdapter *net_keepalive_adapter); - //--Log stream related interfaces-- - //New log stream interface, this interface will create the corresponding directory for the log stream and create a new log stream with PalfBaseInfo as the log base point. - //This includes generating and initializing the corresponding ObReplayStatus structure - // @param [in] id, log stream identifier - // @param [in] replica_type, the replica type of the log stream - // @param [in] tenant_role, tenant role, this decides the Palf usage mode (APPEND/RAW_WRITE) - // @param [in] palf_base_info, log synchronization base point information - // @param [out] log_handler, new log stream returned in the form of ObLogHandler, ensuring the lifecycle of the log stream when used by upper layers - int create_ls(const share::ObLSID &id, - const common::ObReplicaType &replica_type, - const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync, + // Create the unique log stream from its persisted base point. + int create_ls(const palf::PalfBaseInfo &palf_base_info, ObLogHandler &log_handler); //Delete log stream interface: After the outer call to create_ls(), if subsequent processes fail, remove_ls() needs to be called - int remove_ls(const share::ObLSID &id, - ObLogHandler &log_handler); - - int check_palf_exist(const share::ObLSID &id, bool &exist) const; - //Downtime restart recovery log stream interface, including generating and initializing the corresponding ObReplayStatus structure - // @param [in] id, log stream identifier - // @param [out] log_handler, new log stream returned in the form of ObLogHandler, ensuring the lifecycle of the log stream when used by upper layers - int add_ls(const share::ObLSID &id, - ObLogHandler &log_handler); + int remove_ls(ObLogHandler &log_handler); - int open_palf(const share::ObLSID &id, - palf::PalfHandleGuard &palf_handle); + int check_palf_exist(bool &exist) const; + // Restart recovery interface for the single log stream. + int add_ls(ObLogHandler &log_handler); - // get role of current palf replica. - // NB: distinguish the difference from get_role of log_handler - // In general, get the replica role to do migration/blance/report, use this interface, - // to write log, use get_role of log_handler - int get_palf_role(const share::ObLSID &id, - common::ObRole &role, - int64_t &proposal_id); + int open_palf(palf::PalfHandleGuard &palf_handle); int update_replayable_point(const share::SCN &replayable_point); int get_replayable_point(share::SCN &replayable_point); @@ -169,13 +137,12 @@ class ObLogService int update_palf_options_except_disk_usage_limit_size(); int update_log_disk_usage_limit_size(const int64_t log_disk_usage_limit_size); int get_palf_options(palf::PalfOptions &options); - int iterate_palf(const ObFunction &func); - int iterate_apply(const ObFunction &func); - int iterate_replay(const ObFunction &func); + int stat_palf(palf::PalfStat &palf_stat); + int stat_apply(LSApplyStat &apply_stat); + int stat_replay(LSReplayStat &replay_stat); - int diagnose_role_change(RCDiagnoseInfo &diagnose_info); - int diagnose_replay(const share::ObLSID &id, ReplayDiagnoseInfo &diagnose_info); - int diagnose_apply(const share::ObLSID &id, ApplyDiagnoseInfo &diagnose_info); + int diagnose_replay(ReplayDiagnoseInfo &diagnose_info); + int diagnose_apply(ApplyDiagnoseInfo &diagnose_info); int get_io_start_time(int64_t &last_working_time); int check_disk_space_enough(bool &is_disk_enough); @@ -191,18 +158,9 @@ class ObLogService int check_need_do_checkpoint(bool &need_do_checkpoint); private: - int create_ls_(const share::ObLSID &id, - const common::ObReplicaType &replica_type, - const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync, + int create_ls_(const palf::PalfBaseInfo &palf_base_info, ObLogHandler &log_handler); - struct GetUnrecycableLogDiskSizeFunctor { - GetUnrecycableLogDiskSizeFunctor() : unrecycable_log_disk_size_(0) {} - ~GetUnrecycableLogDiskSizeFunctor() { unrecycable_log_disk_size_ = 0; } - int operator()(ObLS *ls); - int64_t unrecycable_log_disk_size_; - }; + int get_unrecyclable_log_disk_size(int64_t &unrecyclable_log_disk_size); private: bool is_inited_; bool is_running_; @@ -215,8 +173,6 @@ class ObLogService ObLogApplyService apply_service_; ObLogReplayService replay_service_; - ObRoleChangeService role_change_service_; - ObLocationAdapter location_adapter_; ObLSAdapter ls_adapter_; obcall::ObLogServiceRpcProxy rpc_proxy_; ObLogMonitor monitor_; diff --git a/src/logservice/ob_ls_adapter.cpp b/src/logservice/ob_ls_adapter.cpp index 4e307b68b..c45573a77 100644 --- a/src/logservice/ob_ls_adapter.cpp +++ b/src/logservice/ob_ls_adapter.cpp @@ -15,6 +15,7 @@ */ #include "ob_ls_adapter.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -58,17 +59,13 @@ void ObLSAdapter::destroy() int ObLSAdapter::replay(ObLogReplayTask *replay_task) { int ret = OB_SUCCESS; - ObLS *ls = NULL; - ObLSHandle ls_handle; + ObLS *ls = nullptr; int64_t start_ts = ObTimeUtility::fast_current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "ObLSAdapter not inited", K(ret)); - } else if (OB_FAIL(ls_service_->get_ls(replay_task->ls_id_, ls_handle, ObLSGetMod::ADAPTER_MOD))) { + } else if (OB_FAIL(ls_service_->get_ls(ls))) { CLOG_LOG(ERROR, "get log stream failed", KPC(replay_task), K(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, " log stream not exist", KPC(replay_task), K(ret)); } else if (ObLogBaseType::PADDING_LOG_BASE_TYPE == replay_task->log_type_) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "padding log entry can't be replayed, unexpected error", KPC(replay_task)); @@ -110,22 +107,14 @@ int ObLSAdapter::replay(ObLogReplayTask *replay_task) return ret; } -int ObLSAdapter::wait_append_sync(const share::ObLSID &ls_id) +int ObLSAdapter::wait_append_sync() { int ret = OB_SUCCESS; - ObLS *ls = NULL; - ObLSHandle ls_handle; - ObLogHandler *log_handler = NULL; - if (OB_FAIL(ls_service_->get_ls(ls_id, ls_handle, ObLSGetMod::ADAPTER_MOD))) { - CLOG_LOG(WARN, "get log stream failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "log stream not exist", K(ret), K(ls_id)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "log_handler is NULL", K(ret), K(ls_id)); + ObLS *ls = nullptr; + if (OB_FAIL(ls_service_->get_ls(ls))) { + CLOG_LOG(WARN, "get log stream failed", K(ret)); } else { - log_handler->wait_append_sync(); + ls->get_log_handler()->wait_append_sync(); } return ret; } diff --git a/src/logservice/ob_ls_adapter.h b/src/logservice/ob_ls_adapter.h index 3781191b1..dc3c162c7 100644 --- a/src/logservice/ob_ls_adapter.h +++ b/src/logservice/ob_ls_adapter.h @@ -18,8 +18,6 @@ #define OCEANBASE_LOGSERVICE_OB_LS_ADAPTER_H_ #include -#include "share/ob_ls_id.h" - namespace oceanbase { namespace storage @@ -38,7 +36,7 @@ class ObLSAdapter void destroy(); public: virtual int replay(ObLogReplayTask *replay_task); - virtual int wait_append_sync(const share::ObLSID &ls_id); + virtual int wait_append_sync(); private: const int64_t MAX_SINGLE_REPLAY_WARNING_TIME_THRESOLD = 100 * 1000; //100ms const int64_t MAX_SINGLE_REPLAY_ERROR_TIME_THRESOLD = 2 * 1000 * 1000; // 2s Single log replay execution time exceeding this value reports error diff --git a/src/logservice/ob_server_log_block_mgr.cpp b/src/logservice/ob_server_log_block_mgr.cpp index 062862d0f..e87ad2a5c 100644 --- a/src/logservice/ob_server_log_block_mgr.cpp +++ b/src/logservice/ob_server_log_block_mgr.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX CLOG #include "ob_server_log_block_mgr.h" #include "share/rc/ob_module_provider.h" -#include // std::regex #ifdef __APPLE__ #include // For fcntl, F_PREALLOCATE on macOS #include // For ftruncate @@ -289,19 +288,7 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, return ret; } -// step1. scan the directory of 'log_disk_path'(ie: like **/store/clog), get the total -// block count. -// -// step2. scan the directory of 'log_pool_path_'(ie: like **/log_pool), get -// the total block count, and then trim the directory. -// -// step3. load the meta. -// -// step4. if the 'status_' of meta is EXPANDING_STATUS or SHRINKING_STATUS, continous to -// finish it. -// -// step5. check the total block count of 'log_disk_path' and 'log_pool_path_' whether is -// same as the 'curr_total_size_' of 'log_pool_meta_'. +// Clean temporary files, then restore the allocated block count of the single log stream. int ObServerLogBlockMgr::do_load_(const char *log_disk_path) { int ret = OB_SUCCESS; @@ -418,8 +405,6 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_( int ret = OB_SUCCESS; DIR *dir = NULL; struct dirent *entry = NULL; - std::regex pattern_tenant(".*/sys"); - std::regex pattern_log_pool(".*/log_pool/*"); if (NULL == (dir = opendir(log_disk_path))) { ret = OB_ERR_SYS; CLOG_LOG(WARN, "opendir failed", K(log_disk_path)); @@ -442,10 +427,8 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_( LOG_DBA_ERROR_V2(OB_LOG_EXTERNAL_FILE_EXIST, ret, "Attention!!!", "There are several files in the log directory that are not generated by " "OceanBase.", "[suggestion] Please confirm whether manual deletion is required", ", unexpected file path is ", current_file_path); - } else if (true == std::regex_match(current_file_path, pattern_tenant)) { + } else if (0 == strcmp(entry->d_name, "sys")) { ret = scan_tenant_dir_(current_file_path, has_allocated_block_cnt); - } else if (true == std::regex_match(current_file_path, pattern_log_pool)) { - CLOG_LOG(INFO, "ignore log_pool path", K(current_file_path), KPC(this)); } else { ret = OB_ERR_UNEXPECTED; LOG_DBA_ERROR_V2(OB_LOG_EXTERNAL_FILE_EXIST, ret, "Attention!!!", "There are several files in the log directory that are not generated by " @@ -464,7 +447,6 @@ int ObServerLogBlockMgr::remove_tmp_file_or_directory_for_tenant_(const char *lo { int ret = OB_SUCCESS; DIR *dir = NULL; - std::regex pattern_tenant(".*/sys"); struct dirent *entry = NULL; if (NULL == (dir = opendir(log_disk_path))) { ret = OB_ERR_SYS; @@ -485,7 +467,7 @@ int ObServerLogBlockMgr::remove_tmp_file_or_directory_for_tenant_(const char *lo CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name)); } else if (false == is_dir) { CLOG_LOG(ERROR, "is not diectory, unexpected", K(ret), K(log_disk_path), K(current_file_path)); - } else if (true == std::regex_match(current_file_path, pattern_tenant)) { + } else if (0 == strcmp(entry->d_name, "sys")) { if (OB_FAIL(remove_tmp_file_or_directory_at(current_file_path, this))) { CLOG_LOG(ERROR, "this dir is tenant, remove_tmp_file_or_directory_at failed", K(ret), K(current_file_path)); } else { @@ -527,14 +509,11 @@ int ObServerLogBlockMgr::fsync_until_success_(const FileDesc &dest_dir_fd) } return ret; } -// the prefix is tenant_xxx int ObServerLogBlockMgr::scan_tenant_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt) { int ret = OB_SUCCESS; DIR *dir = NULL; - std::regex pattern_log_stream(".*/sys/[1-9]\\d*"); - std::regex pattern_tmp_dir(".*/sys/tmp_dir"); struct dirent *entry = NULL; if (NULL == (dir = opendir(tenant_dir))) { ret = OB_ERR_SYS; @@ -558,9 +537,9 @@ int ObServerLogBlockMgr::scan_tenant_dir_(const char *tenant_dir, LOG_DBA_ERROR_V2(OB_LOG_EXTERNAL_FILE_EXIST, ret, "Attention!!!", "There are several files in the log directory that are not generated by " "OceanBase.", "[suggestion] Please confirm whether manual deletion is required", ", unexpected file is ", current_file_path); - } else if (true == std::regex_match(current_file_path, pattern_log_stream)) { - ret = scan_ls_dir_(current_file_path, has_allocated_block_cnt); - } else if (true == std::regex_match(current_file_path, pattern_tmp_dir)) { + } else if (0 == strcmp(entry->d_name, "log_stream")) { + ret = scan_log_stream_dir_(current_file_path, has_allocated_block_cnt); + } else if (0 == strcmp(entry->d_name, "tmp_dir")) { CLOG_LOG(INFO, "ignore tmp_dir", K(current_file_path), K(has_allocated_block_cnt), KPC(this)); } else { ret = OB_ERR_UNEXPECTED; @@ -576,18 +555,15 @@ int ObServerLogBlockMgr::scan_tenant_dir_(const char *tenant_dir, return ret; } -// the prefix of ls_dir tenant_xxx/xxxx -int ObServerLogBlockMgr::scan_ls_dir_(const char *ls_dir, - int64_t &has_allocated_block_cnt) +int ObServerLogBlockMgr::scan_log_stream_dir_(const char *log_stream_dir, + int64_t &has_allocated_block_cnt) { int ret = OB_SUCCESS; DIR *dir = NULL; - std::regex pattern_log(".*/sys/[1-9]\\d*/log"); - std::regex pattern_meta(".*/sys/[1-9]\\d*/meta"); struct dirent *entry = NULL; - if (NULL == (dir = opendir(ls_dir))) { + if (NULL == (dir = opendir(log_stream_dir))) { ret = OB_ERR_SYS; - CLOG_LOG(WARN, "opendir failed", K(ls_dir)); + CLOG_LOG(WARN, "opendir failed", K(log_stream_dir)); } else { char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; while ((entry = readdir(dir)) != NULL && OB_SUCC(ret)) { @@ -596,9 +572,9 @@ int ObServerLogBlockMgr::scan_ls_dir_(const char *ls_dir, if (0 == strcmp(entry->d_name, ".") || 0 == strcmp(entry->d_name, "..")) { // do nothing } else if (0 >= snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", - ls_dir, entry->d_name)) { + log_stream_dir, entry->d_name)) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(ls_dir), + CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(log_stream_dir), K(entry->d_name)); } else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) { CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name)); @@ -607,8 +583,7 @@ int ObServerLogBlockMgr::scan_ls_dir_(const char *ls_dir, LOG_DBA_ERROR_V2(OB_LOG_EXTERNAL_FILE_EXIST, ret, "Attention!!!", "There are several files in the log directory that are not generated by " "OceanBase.", "[suggestion] Please confirm whether manual deletion is required", ", unexpected file is ", current_file_path); - } else if (true == std::regex_match(current_file_path, pattern_log) - || true == std::regex_match(current_file_path, pattern_meta)) { + } else if (0 == strcmp(entry->d_name, "log") || 0 == strcmp(entry->d_name, "meta")) { GetBlockCountFunctor functor(current_file_path); if (OB_FAIL(palf::scan_dir(current_file_path, functor))) { LOG_DBA_ERROR_V2(OB_LOG_EXTERNAL_FILE_EXIST, ret, "Attention!!!", "There are several files in the log directory that are not generated by " diff --git a/src/logservice/ob_server_log_block_mgr.h b/src/logservice/ob_server_log_block_mgr.h index 4a1df3abf..c07149964 100644 --- a/src/logservice/ob_server_log_block_mgr.h +++ b/src/logservice/ob_server_log_block_mgr.h @@ -128,7 +128,7 @@ class ObServerLogBlockMgr : public palf::ILogBlockPool const int flag); int fsync_until_success_(const palf::FileDesc &src_fd); int scan_tenant_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt); - int scan_ls_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt); + int scan_log_stream_dir_(const char *log_stream_dir, int64_t &has_allocated_block_cnt); private: typedef common::ObFunction GetTenantsLogDiskSize; GetTenantsLogDiskSize get_tenants_log_disk_size_func_; diff --git a/src/logservice/ob_tenant_mutil_allocator.cpp b/src/logservice/ob_tenant_mutil_allocator.cpp index cef7b174b..764e7770f 100644 --- a/src/logservice/ob_tenant_mutil_allocator.cpp +++ b/src/logservice/ob_tenant_mutil_allocator.cpp @@ -15,7 +15,9 @@ */ #include "ob_tenant_mutil_allocator.h" -#include "observer/omt/ob_multi_tenant.h" // previously hidden behind the server_struct include chain, make the dependency explicit +#include "share/ob_server_struct.h" +#include "observer/omt/ob_multi_tenant.h" +#include "logservice/palf/log_io_task.h" #include "logservice/palf/log_shared_task.h" #include "logservice/replayservice/ob_replay_status.h" @@ -31,10 +33,8 @@ ObTenantMutilAllocator::ObTenantMutilAllocator() : total_limit_(INT64_MAX), pending_replay_mutator_size_(0), LOG_HANDLE_SUBMIT_TASK_SIZE(sizeof(palf::LogHandleSubmitTask)), LOG_IO_FLUSH_LOG_TASK_SIZE(sizeof(palf::LogIOFlushLogTask)), - LOG_IO_TRUNCATE_LOG_TASK_SIZE(sizeof(palf::LogIOTruncateLogTask)), LOG_IO_FLUSH_META_TASK_SIZE(sizeof(palf::LogIOFlushMetaTask)), LOG_IO_TRUNCATE_PREFIX_BLOCKS_TASK_SIZE(sizeof(palf::LogIOTruncatePrefixBlocksTask)), - PALF_FETCH_LOG_TASK_SIZE(sizeof(palf::FetchLogTask)), LOG_IO_PURGE_THROTTLING_TASK_SIZE(sizeof(palf::LogIOPurgeThrottlingTask)), clog_blk_alloc_(), replay_log_task_blk_alloc_(REPLAY_MEM_LIMIT_THRESHOLD), @@ -42,10 +42,8 @@ ObTenantMutilAllocator::ObTenantMutilAllocator() clog_ge_alloc_(ObMemAttr(ObModIds::OB_CLOG_GE), ObVSliceAlloc::DEFAULT_BLOCK_SIZE, clog_blk_alloc_), log_handle_submit_task_alloc_(LOG_HANDLE_SUBMIT_TASK_SIZE, ObMemAttr("HandleSubmit"), choose_blk_size(LOG_HANDLE_SUBMIT_TASK_SIZE), clog_blk_alloc_, this), log_io_flush_log_task_alloc_(LOG_IO_FLUSH_LOG_TASK_SIZE, ObMemAttr("FlushLog"), choose_blk_size(LOG_IO_FLUSH_LOG_TASK_SIZE), clog_blk_alloc_, this), - log_io_truncate_log_task_alloc_(LOG_IO_TRUNCATE_LOG_TASK_SIZE, ObMemAttr("TruncateLog"), choose_blk_size(LOG_IO_TRUNCATE_LOG_TASK_SIZE), clog_blk_alloc_, this), log_io_flush_meta_task_alloc_(LOG_IO_FLUSH_META_TASK_SIZE, ObMemAttr("FlushMeta"), choose_blk_size(LOG_IO_FLUSH_META_TASK_SIZE), clog_blk_alloc_, this), log_io_truncate_prefix_blocks_task_alloc_(LOG_IO_TRUNCATE_PREFIX_BLOCKS_TASK_SIZE, ObMemAttr("FlushMeta"), choose_blk_size(LOG_IO_TRUNCATE_PREFIX_BLOCKS_TASK_SIZE), clog_blk_alloc_, this), - palf_fetch_log_task_alloc_(PALF_FETCH_LOG_TASK_SIZE, ObMemAttr(ObModIds::OB_FETCH_LOG_TASK), choose_blk_size(PALF_FETCH_LOG_TASK_SIZE), clog_blk_alloc_, this), replay_log_task_alloc_(ObMemAttr(ObModIds::OB_LOG_REPLAY_TASK), common::OB_MALLOC_BIG_BLOCK_SIZE, replay_log_task_blk_alloc_), log_io_purge_throttling_task_alloc_(LOG_IO_PURGE_THROTTLING_TASK_SIZE, ObMemAttr("PurgeThrottle"), choose_blk_size(LOG_IO_PURGE_THROTTLING_TASK_SIZE), clog_blk_alloc_, this), clog_compression_buf_alloc_(ObMemAttr("LogComBuf"), common::OB_MALLOC_BIG_BLOCK_SIZE, clog_compressing_blk_alloc_) @@ -74,11 +72,9 @@ void ObTenantMutilAllocator::destroy() clog_ge_alloc_.destroy(); log_handle_submit_task_alloc_.destroy(); log_io_flush_log_task_alloc_.destroy(); - log_io_truncate_log_task_alloc_.destroy(); log_io_flush_meta_task_alloc_.destroy(); log_io_truncate_prefix_blocks_task_alloc_.destroy(); log_io_purge_throttling_task_alloc_.destroy(); - palf_fetch_log_task_alloc_.destroy(); replay_log_task_alloc_.destroy(); clog_compression_buf_alloc_.destroy(); } @@ -130,12 +126,12 @@ const ObBlockAllocMgr &ObTenantMutilAllocator::get_clog_blk_alloc_mgr() const } LogIOFlushLogTask *ObTenantMutilAllocator::alloc_log_io_flush_log_task( - const int64_t palf_id, const int64_t palf_epoch) + const int64_t palf_epoch) { LogIOFlushLogTask *ret_ptr = NULL; void *ptr = log_io_flush_log_task_alloc_.alloc(); if (NULL != ptr) { - ret_ptr = new(ptr)LogIOFlushLogTask(palf_id, palf_epoch); + ret_ptr = new(ptr)LogIOFlushLogTask(palf_epoch); ATOMIC_INC(&flying_log_task_); } return ret_ptr; @@ -151,12 +147,12 @@ void ObTenantMutilAllocator::free_log_io_flush_log_task(LogIOFlushLogTask *ptr) } LogHandleSubmitTask *ObTenantMutilAllocator::alloc_log_handle_submit_task( - const int64_t palf_id, const int64_t palf_epoch) + const int64_t palf_epoch) { LogHandleSubmitTask *ret_ptr = NULL; void *ptr = log_handle_submit_task_alloc_.alloc(); if (NULL != ptr) { - ret_ptr = new(ptr)LogHandleSubmitTask(palf_id, palf_epoch); + ret_ptr = new(ptr)LogHandleSubmitTask(palf_epoch); ATOMIC_INC(&flying_log_handle_submit_task_); } return ret_ptr; @@ -171,32 +167,13 @@ void ObTenantMutilAllocator::free_log_handle_submit_task(LogHandleSubmitTask *pt } } -LogIOTruncateLogTask *ObTenantMutilAllocator::alloc_log_io_truncate_log_task( - const int64_t palf_id, const int64_t palf_epoch) -{ - LogIOTruncateLogTask *ret_ptr = NULL; - void *ptr = log_io_truncate_log_task_alloc_.alloc(); - if (NULL != ptr) { - ret_ptr = new(ptr) LogIOTruncateLogTask(palf_id, palf_epoch); - } - return ret_ptr; -} - -void ObTenantMutilAllocator::free_log_io_truncate_log_task(LogIOTruncateLogTask *ptr) -{ - if (OB_LIKELY(NULL != ptr)) { - ptr->~LogIOTruncateLogTask(); - log_io_truncate_log_task_alloc_.free(ptr); - } -} - LogIOFlushMetaTask *ObTenantMutilAllocator::alloc_log_io_flush_meta_task( - const int64_t palf_id, const int64_t palf_epoch) + const int64_t palf_epoch) { LogIOFlushMetaTask *ret_ptr = NULL; void *ptr = log_io_flush_meta_task_alloc_.alloc(); if (NULL != ptr) { - ret_ptr = new(ptr)LogIOFlushMetaTask(palf_id, palf_epoch); + ret_ptr = new(ptr)LogIOFlushMetaTask(palf_epoch); ATOMIC_INC(&flying_meta_task_); } return ret_ptr; @@ -212,12 +189,12 @@ void ObTenantMutilAllocator::free_log_io_flush_meta_task(LogIOFlushMetaTask *ptr } palf::LogIOTruncatePrefixBlocksTask *ObTenantMutilAllocator::alloc_log_io_truncate_prefix_blocks_task( - const int64_t palf_id, const int64_t palf_epoch) + const int64_t palf_epoch) { LogIOTruncatePrefixBlocksTask *ret_ptr = NULL; void *ptr = log_io_truncate_prefix_blocks_task_alloc_.alloc(); if (NULL != ptr) { - ret_ptr = new(ptr)LogIOTruncatePrefixBlocksTask(palf_id ,palf_epoch); + ret_ptr = new(ptr)LogIOTruncatePrefixBlocksTask(palf_epoch); } return ret_ptr; } @@ -230,24 +207,6 @@ void ObTenantMutilAllocator::free_log_io_truncate_prefix_blocks_task(palf::LogIO } } -palf::FetchLogTask *ObTenantMutilAllocator::alloc_palf_fetch_log_task() -{ - FetchLogTask *ret_ptr = NULL; - void *ptr = palf_fetch_log_task_alloc_.alloc(); - if (NULL != ptr) { - ret_ptr = new(ptr)FetchLogTask(); - } - return ret_ptr; -} - -void ObTenantMutilAllocator::free_palf_fetch_log_task(palf::FetchLogTask *ptr) -{ - if (OB_LIKELY(NULL != ptr)) { - ptr->~FetchLogTask(); - palf_fetch_log_task_alloc_.free(ptr); - } -} - void *ObTenantMutilAllocator::alloc_replay_task(const int64_t size) { return replay_log_task_alloc_.alloc(size); @@ -273,12 +232,12 @@ void ObTenantMutilAllocator::free_replay_log_buf(void *ptr) } } -LogIOPurgeThrottlingTask *ObTenantMutilAllocator::alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch) +LogIOPurgeThrottlingTask *ObTenantMutilAllocator::alloc_log_io_purge_throttling_task(const int64_t palf_epoch) { LogIOPurgeThrottlingTask *ret_ptr = NULL; void *ptr = log_io_purge_throttling_task_alloc_.alloc(); if (NULL != ptr) { - ret_ptr = new(ptr)LogIOPurgeThrottlingTask(palf_id, palf_epoch); + ret_ptr = new(ptr)LogIOPurgeThrottlingTask(palf_epoch); } return ret_ptr; } diff --git a/src/logservice/ob_tenant_mutil_allocator.h b/src/logservice/ob_tenant_mutil_allocator.h index c2395e08b..b43969724 100644 --- a/src/logservice/ob_tenant_mutil_allocator.h +++ b/src/logservice/ob_tenant_mutil_allocator.h @@ -30,12 +30,10 @@ namespace palf { class LogIOFlushLogTask; class LogHandleSubmitTask; -class LogIOTruncateLogTask; class LogIOFlushMetaTask; class LogIOTruncatePrefixBlocksTask; class LogIOFlushMetaTask; class LogIOPurgeThrottlingTask; -class FetchLogTask; } namespace logservice { @@ -58,23 +56,19 @@ class ObILogAllocator : public ObIAllocator virtual void *ge_alloc(const int64_t size) = 0; virtual void ge_free(void *ptr) = 0; virtual const ObBlockAllocMgr &get_clog_blk_alloc_mgr() const = 0; - virtual palf::LogHandleSubmitTask *alloc_log_handle_submit_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual palf::LogHandleSubmitTask *alloc_log_handle_submit_task(const int64_t palf_epoch) = 0; virtual void free_log_handle_submit_task(palf::LogHandleSubmitTask *ptr) = 0; - virtual palf::LogIOFlushLogTask *alloc_log_io_flush_log_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual palf::LogIOFlushLogTask *alloc_log_io_flush_log_task(const int64_t palf_epoch) = 0; virtual void free_log_io_flush_log_task(palf::LogIOFlushLogTask *ptr) = 0; - virtual palf::LogIOTruncateLogTask *alloc_log_io_truncate_log_task(const int64_t palf_id, const int64_t palf_epoch) = 0; - virtual void free_log_io_truncate_log_task(palf::LogIOTruncateLogTask *ptr) = 0; - virtual palf::LogIOFlushMetaTask *alloc_log_io_flush_meta_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual palf::LogIOFlushMetaTask *alloc_log_io_flush_meta_task(const int64_t palf_epoch) = 0; virtual void free_log_io_flush_meta_task(palf::LogIOFlushMetaTask *ptr) = 0; - virtual palf::LogIOTruncatePrefixBlocksTask *alloc_log_io_truncate_prefix_blocks_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual palf::LogIOTruncatePrefixBlocksTask *alloc_log_io_truncate_prefix_blocks_task(const int64_t palf_epoch) = 0; virtual void free_log_io_truncate_prefix_blocks_task(palf::LogIOTruncatePrefixBlocksTask *ptr) = 0; - virtual palf::FetchLogTask *alloc_palf_fetch_log_task() = 0; - virtual void free_palf_fetch_log_task(palf::FetchLogTask *ptr) = 0; virtual void *alloc_replay_task(const int64_t size) = 0; virtual void *alloc_replay_log_buf(const int64_t size) = 0; virtual void free_replay_task(logservice::ObLogReplayTask *ptr) = 0; virtual void free_replay_log_buf(void *ptr) = 0; - virtual palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_epoch) = 0; virtual void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr) = 0; virtual void *alloc_append_compression_buf(const int64_t size) = 0; virtual void free_append_compression_buf(void *ptr) = 0; @@ -125,23 +119,19 @@ class ObTenantMutilAllocator void ge_free(void *ptr); const ObBlockAllocMgr &get_clog_blk_alloc_mgr() const; // V4.0 - palf::LogIOFlushLogTask *alloc_log_io_flush_log_task(const int64_t palf_id, const int64_t palf_epoch); + palf::LogIOFlushLogTask *alloc_log_io_flush_log_task(const int64_t palf_epoch); void free_log_io_flush_log_task(palf::LogIOFlushLogTask *ptr); - palf::LogHandleSubmitTask *alloc_log_handle_submit_task(const int64_t palf_id, const int64_t palf_epoch); + palf::LogHandleSubmitTask *alloc_log_handle_submit_task(const int64_t palf_epoch); void free_log_handle_submit_task(palf::LogHandleSubmitTask *ptr); - palf::LogIOTruncateLogTask *alloc_log_io_truncate_log_task(const int64_t palf_id, const int64_t palf_epoch); - void free_log_io_truncate_log_task(palf::LogIOTruncateLogTask *ptr); - palf::LogIOFlushMetaTask *alloc_log_io_flush_meta_task(const int64_t palf_id, const int64_t palf_epoch); + palf::LogIOFlushMetaTask *alloc_log_io_flush_meta_task(const int64_t palf_epoch); void free_log_io_flush_meta_task(palf::LogIOFlushMetaTask *ptr); - palf::LogIOTruncatePrefixBlocksTask *alloc_log_io_truncate_prefix_blocks_task(const int64_t palf_id, const int64_t palf_epoch); + palf::LogIOTruncatePrefixBlocksTask *alloc_log_io_truncate_prefix_blocks_task(const int64_t palf_epoch); void free_log_io_truncate_prefix_blocks_task(palf::LogIOTruncatePrefixBlocksTask *ptr); - palf::FetchLogTask *alloc_palf_fetch_log_task(); - void free_palf_fetch_log_task(palf::FetchLogTask *ptr); void *alloc_replay_task(const int64_t size); void *alloc_replay_log_buf(const int64_t size); void free_replay_task(logservice::ObLogReplayTask *ptr); void free_replay_log_buf(void *ptr); - palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch); + palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_epoch); void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr); void *alloc_append_compression_buf(const int64_t size); @@ -156,10 +146,8 @@ class ObTenantMutilAllocator int64_t pending_replay_mutator_size_; const int LOG_HANDLE_SUBMIT_TASK_SIZE; const int LOG_IO_FLUSH_LOG_TASK_SIZE; - const int LOG_IO_TRUNCATE_LOG_TASK_SIZE; const int LOG_IO_FLUSH_META_TASK_SIZE; const int LOG_IO_TRUNCATE_PREFIX_BLOCKS_TASK_SIZE; - const int PALF_FETCH_LOG_TASK_SIZE; const int LOG_IO_PURGE_THROTTLING_TASK_SIZE; ObBlockAllocMgr clog_blk_alloc_; ObBlockAllocMgr replay_log_task_blk_alloc_; @@ -167,10 +155,8 @@ class ObTenantMutilAllocator ObVSliceAlloc clog_ge_alloc_; ObSliceAlloc log_handle_submit_task_alloc_; ObSliceAlloc log_io_flush_log_task_alloc_; - ObSliceAlloc log_io_truncate_log_task_alloc_; ObSliceAlloc log_io_flush_meta_task_alloc_; ObSliceAlloc log_io_truncate_prefix_blocks_task_alloc_; - ObSliceAlloc palf_fetch_log_task_alloc_; ObVSliceAlloc replay_log_task_alloc_; ObSliceAlloc log_io_purge_throttling_task_alloc_; ObVSliceAlloc clog_compression_buf_alloc_; diff --git a/src/logservice/palf/election_self.cpp b/src/logservice/palf/election_self.cpp deleted file mode 100644 index fc755a22e..000000000 --- a/src/logservice/palf/election_self.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX PALF -#include "election_self.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -namespace election -{ - -ElectionImpl::ElectionImpl() - : is_inited_(false), - is_running_(false), - lock_(common::ObLatchIds::ELECTION_LOCK), - id_(0), - self_addr_() -{} - -ElectionImpl::~ElectionImpl() -{ - if (is_running_) { - stop(); - } - is_inited_ = false; -} - -int ElectionImpl::init_and_start(const int64_t id, - const common::ObAddr &self_addr, - const uint64_t inner_priority_seed, - const int64_t restart_counter, - const common::ObFunction &prepare_change_leader_cb, - const common::ObFunction &role_change_cb) -{ - UNUSED(inner_priority_seed); - UNUSED(restart_counter); - UNUSED(prepare_change_leader_cb); - int ret = OB_SUCCESS; - ObSpinLockGuard lock_guard(lock_); - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - PALF_LOG(ERROR, "init election impl twice", K(ret), K(id)); - } else { - id_ = id; - self_addr_ = self_addr; - is_inited_ = true; - is_running_ = true; - PALF_LOG(INFO, "election init and start (single-replica self-leader)", K(id), K(self_addr)); - } - // NB: palf passes only prepare_change_leader_cb; role_change_cb is the default - // no-op. Real leader takeover is synthesized by LogStateMgr (get_role()==LEADER). - role_change_cb(this, common::ObRole::FOLLOWER, common::ObRole::LEADER, RoleChangeReason::DevoteToBeLeader); - return ret; -} - -void ElectionImpl::stop() -{ - ObSpinLockGuard lock_guard(lock_); - if (OB_UNLIKELY(!is_inited_ || !is_running_)) { - PALF_LOG_RET(WARN, OB_NOT_RUNNING, "election is not running or not inited", K_(self_addr)); - } else { - is_running_ = false; - PALF_LOG(INFO, "election stopped", K_(self_addr)); - } -} - -int ElectionImpl::can_set_memberlist(const palf::LogConfigVersion &new_config_version) const -{ - int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(!new_config_version.is_valid())) { - ret = OB_INVALID_ARGUMENT; - } else { - ObSpinLockGuard lock_guard(lock_); - CHECK_ELECTION_INIT(); - } - return ret; -} - -int ElectionImpl::set_memberlist(const MemberList &new_memberlist) -{ - int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(!new_memberlist.is_valid())) { - ret = OB_INVALID_ARGUMENT; - } else { - ObSpinLockGuard lock_guard(lock_); - CHECK_ELECTION_INIT(); - bool self_in_memberlist = false; - const ObArray &addr_list = new_memberlist.get_addr_list(); - for (int64_t i = 0; !self_in_memberlist && i < addr_list.count(); ++i) { - self_in_memberlist = addr_list.at(i) == self_addr_; - } - if (!self_in_memberlist) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "self addr not in memberlist", K(ret), K(new_memberlist), K_(self_addr)); - } - } - return ret; -} - -int ElectionImpl::change_leader_to(const common::ObAddr &dest_addr) -{ - UNUSED(dest_addr); - return OB_NOT_SUPPORTED; -} - -int ElectionImpl::set_priority(ElectionPriority *priority) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(priority)) { - ret = OB_INVALID_ARGUMENT; - } else { - ObSpinLockGuard lock_guard(lock_); - CHECK_ELECTION_INIT(); - } - return ret; -} - -int ElectionImpl::reset_priority() -{ - ObSpinLockGuard lock_guard(lock_); - CHECK_ELECTION_INIT(); - return OB_SUCCESS; -} - -int ElectionImpl::temporarily_downgrade_protocol_priority(const int64_t time_us, const char *reason) -{ - UNUSED(time_us); - UNUSED(reason); - return OB_NOT_SUPPORTED; -} - -int ElectionImpl::add_inner_priority_seed_bit(const PRIORITY_SEED_BIT new_bit) -{ - UNUSED(new_bit); - return OB_NOT_SUPPORTED; -} - -int ElectionImpl::clear_inner_priority_seed_bit(const PRIORITY_SEED_BIT old_bit) -{ - UNUSED(old_bit); - return OB_NOT_SUPPORTED; -} - -int ElectionImpl::set_inner_priority_seed(const uint64_t seed) -{ - UNUSED(seed); - return OB_NOT_SUPPORTED; -} - -}// namespace election -}// namespace palf -}// namespace oceanbase diff --git a/src/logservice/palf/election_self.h b/src/logservice/palf/election_self.h deleted file mode 100644 index 3fcdad4b1..000000000 --- a/src/logservice/palf/election_self.h +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// election_self.h -// -// Single-replica (seekdb-lite) degenerate "election". The real palf election -// algorithm/messages/transport (formerly src/logservice/palf/election/) have -// been deleted: in a single-node single-replica deployment there is no peer to -// vote with, so this replica is leader forever. This header relocates the small -// self-leader stub plus the minimal interface surface still referenced by the -// rest of palf (LogStateMgr / LogConfigMgr query get_role()/get_current_leader_likely() -// and deterministically take over via the existing follower->reconfirm->leader path). - -#ifndef OCEANBASE_LOGSERVICE_PALF_ELECTION_SELF_H_ -#define OCEANBASE_LOGSERVICE_PALF_ELECTION_SELF_H_ - -#include "lib/container/ob_array.h" -#include "lib/net/ob_addr.h" -#include "lib/lock/ob_spin_lock.h" -#include "lib/ob_errno.h" -#include "lib/oblog/ob_log_module.h" -#include "lib/function/ob_function.h" -#include "common/ob_role.h" -#include "logservice/palf/log_meta_info.h" // LogConfigVersion -#include "palf_callback_wrapper.h" // LogPlugins, PalfRoleChangeCbWrapper (formerly via election.h) - -namespace oceanbase -{ -namespace palf -{ -namespace election -{ - -// retained constant: ObTimestampService preallocated range is derived from this. -constexpr int64_t MAX_LEASE_TIME = 10L * 1000L * 1000L; // 10s - -// retained for palf_handle_impl priority-seed bookkeeping (all no-ops now). -enum class PRIORITY_SEED_BIT : uint64_t -{ - DEFAULT_SEED = (1ULL << 12), - TEST_BIT = (1ULL << 13), - SEED_TEMORARILY_DOWNGRADE_PRIORIY_BIT = (1ULL << 22), - SEED_IN_REBUILD_PHASE_BIT = (1ULL << 32), - SEED_NOT_NORMOL_REPLICA_BIT = (1ULL << 48), -}; - -enum class RoleChangeReason -{ - DevoteToBeLeader = 1, - ChangeLeaderToBeLeader = 2, - LeaseExpiredToRevoke = 3, - ChangeLeaderToRevoke = 4, - StopToRevoke = 5, -}; - -// ---- minimal arg-checker macros (relocated from election_args_checker.h) ---- -#define CHECK_ELECTION_INIT() \ -do {\ - if (OB_UNLIKELY(!is_inited_)) {\ - PALF_LOG_RET(WARN, common::OB_NOT_INIT, "election not init yet", K(*this));\ - return common::OB_NOT_INIT;\ - }\ -} while(0) - -// election member list: a degenerate single-member view used only to validate -// that self is contained in the configured member list. -class MemberList -{ -public: - MemberList() : addr_list_(), membership_version_(), replica_num_(0) {} - int set_new_member_list(const common::ObArray &addr_list, - const LogConfigVersion membership_version, - const int64_t replica_num) - { - int ret = common::OB_SUCCESS; - if (OB_FAIL(addr_list_.assign(addr_list))) { - } else { - membership_version_ = membership_version; - replica_num_ = static_cast(replica_num); - } - return ret; - } - const common::ObArray &get_addr_list() const { return addr_list_; } - bool is_valid() const { return membership_version_.is_valid() && replica_num_ > 0; } - TO_STRING_KV(K_(addr_list), K_(membership_version), K_(replica_num)); -private: - common::ObArray addr_list_; - LogConfigVersion membership_version_; - uint8_t replica_num_; -}; - -// abstract priority base: only referenced as a pointer type in now-no-op -// set_election_priority()/reset_election_priority() signatures. The concrete -// impl (coordinator::ElectionPriorityImpl) was deleted with leader_coordinator. -class ElectionPriority -{ -public: - virtual ~ElectionPriority() {} - virtual int64_t to_string(char *buf, const int64_t buf_len) const = 0; -}; - -// abstract election interface still queried by LogStateMgr / LogConfigMgr. -class Election -{ -public: - virtual ~Election() {} - virtual void stop() = 0; - virtual int can_set_memberlist(const palf::LogConfigVersion &new_config_version) const = 0; - virtual int set_memberlist(const MemberList &new_member_list) = 0; - virtual int get_role(common::ObRole &role, int64_t &epoch) const = 0; - virtual int get_current_leader_likely(common::ObAddr &addr, int64_t &cur_leader_epoch) const = 0; - virtual int change_leader_to(const common::ObAddr &dest_addr) = 0; - virtual int temporarily_downgrade_protocol_priority(const int64_t time_us, const char *reason) = 0; - virtual const common::ObAddr &get_self_addr() const = 0; - virtual int64_t to_string(char *buf, const int64_t buf_len) const = 0; - virtual int set_priority(ElectionPriority *priority) = 0; - virtual int reset_priority() = 0; -}; - -struct ElectionImpl; -struct DefaultRoleChangeCallBack -{ - void operator()(ElectionImpl *, common::ObRole before, common::ObRole after, RoleChangeReason reason) - { - UNUSED(before); UNUSED(after); UNUSED(reason); - } -}; - -// constant self-leader election. init_and_start() makes this replica leader and -// it stays leader forever; the real leader-takeover event is synthesized by the -// unchanged LogStateMgr state machine, which observes get_role()==LEADER on the -// follower-active->reconfirm->leader-active path and fires -// PalfRoleChangeCbWrapper::on_role_change exactly once per LS per boot. -class ElectionImpl : public Election -{ - friend struct DefaultRoleChangeCallBack; -public: - ElectionImpl(); - ~ElectionImpl(); - int init_and_start(const int64_t id, - const common::ObAddr &self_addr, - const uint64_t inner_priority_seed, - const int64_t restart_counter, - const common::ObFunction &prepare_change_leader_cb, - const common::ObFunction &cb = DefaultRoleChangeCallBack()); - virtual void stop() override final; - virtual int can_set_memberlist(const palf::LogConfigVersion &new_config_version) const override final; - virtual int set_memberlist(const MemberList &new_memberlist) override final; - virtual int change_leader_to(const common::ObAddr &dest_addr) override final; - virtual int temporarily_downgrade_protocol_priority(const int64_t time_us, const char *reason) override final; - // single-replica: self is leader forever. - virtual int get_role(common::ObRole &role, int64_t &epoch) const override final - { - int ret = common::OB_SUCCESS; - CHECK_ELECTION_INIT(); - role = common::ObRole::LEADER; - epoch = 1; - return ret; - } - virtual int get_current_leader_likely(common::ObAddr &addr, int64_t &cur_leader_epoch) const override final - { - int ret = common::OB_SUCCESS; - CHECK_ELECTION_INIT(); - addr = self_addr_; - cur_leader_epoch = 1; - return ret; - } - virtual const common::ObAddr &get_self_addr() const override final { return self_addr_; } - virtual int set_priority(ElectionPriority *priority) override final; - virtual int reset_priority() override final; - int add_inner_priority_seed_bit(const PRIORITY_SEED_BIT new_bit); - int clear_inner_priority_seed_bit(const PRIORITY_SEED_BIT old_bit); - int set_inner_priority_seed(const uint64_t seed); - TO_STRING_KV(K_(is_inited), K_(is_running), K_(self_addr)); -private: - bool is_inited_; - bool is_running_; - mutable common::ObSpinLock lock_; - int64_t id_; - common::ObAddr self_addr_; -}; - -}// namespace election -}// namespace palf -}// namespace oceanbase -#endif // OCEANBASE_LOGSERVICE_PALF_ELECTION_SELF_H_ diff --git a/src/logservice/palf/fetch_log_engine.cpp b/src/logservice/palf/fetch_log_engine.cpp deleted file mode 100644 index fe27765ed..000000000 --- a/src/logservice/palf/fetch_log_engine.cpp +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fetch_log_engine.h" -#include "palf_handle_impl_guard.h" -#include "palf_env_impl.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -int FetchLogTask::set(const int64_t id, - const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &proposal_id, - const LSN &prev_lsn, - const LSN &start_lsn, - const int64_t log_size, - const int64_t log_count, - const int64_t accepted_mode_pid) -{ - int ret = OB_SUCCESS; - - if (id < 0 - || !server.is_valid() - || INVALID_PROPOSAL_ID == proposal_id - || !start_lsn.is_valid() - || log_size < 0 - || log_count < 0 - || INVALID_PROPOSAL_ID == accepted_mode_pid) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(id), K(server), K(proposal_id), K(prev_lsn), - K(start_lsn), K(log_size), K(log_count), K(accepted_mode_pid)); - } else { - timestamp_us_ = ObTimeUtility::current_time(); - id_ = id; - server_ = server; - fetch_type_ = fetch_type; - proposal_id_ = proposal_id; - prev_lsn_ = prev_lsn; - start_lsn_ = start_lsn; - log_size_ = log_size; - log_count_ = log_count; - accepted_mode_pid_ = accepted_mode_pid; - } - return ret; -} - -void FetchLogTask::reset() -{ - timestamp_us_ = common::OB_INVALID_TIMESTAMP; - id_ = -1; - server_.reset(); - fetch_type_ = FETCH_LOG_FOLLOWER; - proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - start_lsn_.reset(); - log_size_ = 0; - log_count_ = 0; - accepted_mode_pid_ = INVALID_PROPOSAL_ID; -} - -bool FetchLogTask::is_valid() const -{ - return (id_ >= 0 && server_.is_valid() && start_lsn_.is_valid()); -} - -FetchLogTask& FetchLogTask::operator=(const FetchLogTask &task) -{ - if (&task != this) { - this->timestamp_us_ = task.get_timestamp_us(); - this->id_ = task.get_id(); - this->server_ = task.get_server(); - this->fetch_type_ = task.get_fetch_type(); - this->proposal_id_ = task.get_proposal_id(); - this->prev_lsn_ = task.get_prev_lsn(); - this->start_lsn_ = task.get_start_lsn(); - this->log_size_ = task.get_log_size(); - this->log_count_ = task.get_log_count(); - this->accepted_mode_pid_ = task.get_accepted_mode_pid(); - } - return *this; -} - -FetchLogEngine::FetchLogEngine() - : common::ObLinkQueueThreadPool(), - is_inited_(false), - enabled_(true), - palf_env_impl_(NULL), - allocator_(NULL), - replayable_point_(), - cache_lock_(), - fetch_task_cache_(), - fetch_wait_cost_stat_("[PALF STAT FETCH LOG TASK IN QUEUE TIME]", PALF_STAT_PRINT_INTERVAL_US), - fetch_log_cost_stat_("[PALF STAT FETCH LOG EXECUTE COST TIME]", PALF_STAT_PRINT_INTERVAL_US) -{} - - -int FetchLogEngine::init(IPalfEnvImpl *palf_env_impl, - common::ObILogAllocator *alloc_mgr, - const bool enabled) -{ - int ret = OB_SUCCESS; - - if (is_inited_) { - PALF_LOG(WARN, "FetchLogEngine init twice"); - ret = OB_INIT_TWICE; - } else if (OB_ISNULL(palf_env_impl) - || OB_ISNULL(alloc_mgr)) { - PALF_LOG(WARN, "invalid argument", KP(palf_env_impl), K(alloc_mgr)); - ret = OB_INVALID_ARGUMENT; - } else { - palf_env_impl_ = palf_env_impl; - allocator_ = alloc_mgr; - enabled_ = enabled; - if (enabled_ && OB_FAIL(common::ObLinkQueueThreadPool::init( - FETCH_LOG_THREAD_COUNT, - FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET, - "FetchLog"))) { - PALF_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret)); - } else { - is_inited_ = true; - } - } - if ((OB_FAIL(ret)) && (OB_INIT_TWICE != ret)) { - PALF_LOG(WARN, "FetchLogEngine init failed", K(ret), KP(palf_env_impl)); - destroy(); - } - - return ret; -} - -int FetchLogEngine::start() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "FetchLogEngine not inited!!!", K(ret)); - } else if (!enabled_) { - PALF_LOG(INFO, "FetchLogEngine is disabled"); - } else { - PALF_LOG(INFO, "start FetchLogEngine success", K(ret)); - } - return ret; -} - -int FetchLogEngine::stop() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "FetchLogEngine not inited!!!", K(ret)); - } else if (!enabled_) { - PALF_LOG(INFO, "FetchLogEngine is disabled"); - } else { - common::ObLinkQueueThreadPool::stop(); - PALF_LOG(INFO, "stop FetchLogEngine success"); - } - return ret; -} - -int FetchLogEngine::wait() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "FetchLogEngine not inited!!!", K(ret)); - } else if (!enabled_) { - PALF_LOG(INFO, "FetchLogEngine is disabled"); - } else { - common::ObLinkQueueThreadPool::wait(); - PALF_LOG(INFO, "wait FetchLogEngine success"); - } - return ret; -} - -void FetchLogEngine::destroy() -{ - if (is_inited_ && enabled_) { - common::ObLinkQueueThreadPool::stop(); - common::ObLinkQueueThreadPool::wait(); - common::ObLinkQueueThreadPool::destroy(); - } - is_inited_ = false; - enabled_ = true; - palf_env_impl_ = NULL; - allocator_ = NULL; - fetch_task_cache_.destroy(); - PALF_LOG(INFO, "destroy FetchLogEngine success"); -} - -int FetchLogEngine::submit_fetch_log_task(FetchLogTask *fetch_log_task) -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "FetchLogEngine not init", K(ret)); - } else if (!enabled_) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "FetchLogEngine is disabled", K(ret)); - } else if (OB_ISNULL(fetch_log_task)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KP(fetch_log_task)); - } else if (OB_FAIL(push_task_into_cache_(fetch_log_task))) { - PALF_LOG(WARN, "push_task_into_cache_ failed", K(ret), KPC(fetch_log_task)); - } else if (OB_FAIL(common::ObLinkQueueThreadPool::push(fetch_log_task))) { - PALF_LOG(WARN, "push failed", K(ret), KPC(fetch_log_task)); - } else { - //do nothing - } - - return ret; -} - -int FetchLogEngine::push_task_into_cache_(FetchLogTask *fetch_log_task) -{ - // If this task exists in cache, it returns OB_ENTRY_EXIST. - // If this task destn't exist in cache, and cache is full, just ignore it - // and return OB_SUCCESS. - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - } else if (!enabled_) { - ret = OB_NOT_SUPPORTED; - } else if (OB_ISNULL(fetch_log_task)) { - ret = OB_INVALID_ARGUMENT; - } - if (OB_SUCC(ret)) { - SpinLockGuard lock_guard(cache_lock_); - int64_t count = fetch_task_cache_.count(); - if (count >= MAX_CACHED_FETCH_TASK_NUM) { - if (REACH_THREAD_TIME_INTERVAL(5 * 1000 * 1000)) { - PALF_LOG(INFO, "fetch_task_cache_ is full", K(ret), K_(fetch_task_cache)); - } - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { - if (fetch_task_cache_[i].get_id() == fetch_log_task->get_id() - && fetch_task_cache_[i].get_server() == fetch_log_task->get_server()) { - // found existed task for this - ret = OB_ENTRY_EXIST; - break; - } - } - if (OB_SUCCESS == ret) { - fetch_task_cache_.push_back(*fetch_log_task); - } - } - } - return ret; -} - -int FetchLogEngine::try_remove_task_from_cache_(FetchLogTask *fetch_log_task) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - } else if (!enabled_) { - ret = OB_NOT_SUPPORTED; - } else if (OB_ISNULL(fetch_log_task)) { - ret = OB_INVALID_ARGUMENT; - } - if (OB_SUCC(ret)) { - SpinLockGuard lock_guard(cache_lock_); - int64_t count = fetch_task_cache_.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { - if (fetch_task_cache_[i].get_id() == fetch_log_task->get_id() - && fetch_task_cache_[i].get_server() == fetch_log_task->get_server()) { - // found existed task for this - if (OB_FAIL(fetch_task_cache_.remove(i))) { - PALF_LOG(WARN, "fetch_task_cache_.remove failed", K(ret), K(i)); - } - break; - } - } - } - return ret; -} - -void FetchLogEngine::handle(common::LinkTask *task) -{ - int ret = OB_SUCCESS; - IPalfHandleImpl *palf_handle_impl = NULL; - if (!is_inited_) { - PALF_LOG(WARN, "FetchLogEngine not init"); - } else if (!enabled_) { - PALF_LOG(WARN, "FetchLogEngine is disabled"); - } else if (OB_ISNULL(task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "invalid argument", KR(ret), KP(task)); - } else if (OB_ISNULL(palf_env_impl_) || OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "FetchLogEngine dependency is null", K(ret), KP(palf_env_impl_), KP(allocator_)); - } else { - int64_t handle_start_time_us = ObTimeUtility::current_time(); - FetchLogTask *fetch_log_task = static_cast(task); - int64_t palf_id = -1; - FetchLogStat fetch_stat; - if (OB_ISNULL(fetch_log_task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "fetch_log_task is NULL", KR(ret)); - } else { - palf_id = fetch_log_task->get_id(); - PALF_LOG(INFO, "handle fetch_log_task", KPC(fetch_log_task)); - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - if (OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(ERROR, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else { - PALF_LOG(WARN, "ObLogService has not existed!!!", K(ret), K(palf_id)); - } - } else if (OB_ISNULL(palf_handle_impl = guard.get_palf_handle_impl())) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "invalid log service", K(ret), K(palf_id)); - } else if (OB_FAIL(palf_handle_impl->fetch_log_from_storage(fetch_log_task->get_server(), - fetch_log_task->get_fetch_type(), - fetch_log_task->get_proposal_id(), - fetch_log_task->get_prev_lsn(), - fetch_log_task->get_start_lsn(), - fetch_log_task->get_log_size(), - fetch_log_task->get_log_count(), - fetch_log_task->get_accepted_mode_pid(), - replayable_point_.atomic_load(), - fetch_stat))) { - PALF_LOG(WARN, "fetch_log_from_storage failed", K(ret), K(palf_id), KPC(fetch_log_task)); - } else { - // do nothing - } - } - // remove it from cache - if (OB_NOT_NULL(fetch_log_task)) { - (void) try_remove_task_from_cache_(fetch_log_task); - } - int64_t fetch_log_task_submit_time_us = fetch_log_task->get_timestamp_us(); - int64_t handle_finish_time_us = ObTimeUtility::current_time(); - int64_t fetch_wait_cost_time_us = handle_start_time_us - fetch_log_task_submit_time_us; - int64_t handle_cost_time_us = handle_finish_time_us - handle_start_time_us; - fetch_wait_cost_stat_.stat(fetch_wait_cost_time_us); - fetch_log_cost_stat_.stat(handle_cost_time_us); - if (REACH_TIME_INTERVAL(100 * 1000L)) { - PALF_LOG(INFO, "handle fetch log task", K(ret), K(palf_id), K(handle_cost_time_us), KPC(fetch_log_task), K(fetch_stat)); - } else if (handle_cost_time_us > 200 * 1000L) { - PALF_LOG(INFO, "handle fetch log task cost too much time", K(ret), K(palf_id), K(handle_cost_time_us), - KPC(fetch_log_task), K(fetch_stat)); - } - free_fetch_log_task(fetch_log_task); - } -} - -void FetchLogEngine::handle_drop(common::LinkTask *task) -{ - int ret = OB_SUCCESS; - IPalfHandleImpl *palf_handle_impl = NULL; - if (!is_inited_) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "FetchLogEngine not init"); - } else if (!enabled_) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "FetchLogEngine is disabled", K(ret)); - } else if (OB_ISNULL(task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "invalid argument", KP(task)); - } else if (OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "FetchLogEngine allocator is null", K(ret)); - } else { - FetchLogTask *fetch_log_task = static_cast(task); - free_fetch_log_task(fetch_log_task); - } -} - -FetchLogTask *FetchLogEngine::alloc_fetch_log_task() -{ - return is_inited_ && enabled_ && OB_NOT_NULL(allocator_) - ? allocator_->alloc_palf_fetch_log_task() - : NULL; -} - -void FetchLogEngine::free_fetch_log_task(FetchLogTask *task) -{ - if (is_inited_ && enabled_ && OB_NOT_NULL(allocator_) && OB_NOT_NULL(task)) { - allocator_->free_palf_fetch_log_task(task); - } -} - -int FetchLogEngine::update_replayable_point(const share::SCN &replayable_scn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - replayable_point_.atomic_store(replayable_scn); - } - return ret; -} - -} // namespace palf -} // namespace oceanbase diff --git a/src/logservice/palf/fetch_log_engine.h b/src/logservice/palf/fetch_log_engine.h deleted file mode 100644 index cd64aa19b..000000000 --- a/src/logservice/palf/fetch_log_engine.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_FETCH_LOG_ENGINE_ -#define OCEANBASE_LOGSERVICE_OB_FETCH_LOG_ENGINE_ -#include -#include "lib/thread/ob_simple_thread_pool.h" -#include "log_define.h" -#include "log_req.h" -#include "lsn.h" - -namespace oceanbase -{ -namespace common -{ -class ObILogAllocator; -} -namespace palf -{ -class IPalfEnvImpl; -class FetchLogTask : public common::LinkTask -{ -public: - FetchLogTask() - { reset(); } - ~FetchLogTask() { reset(); } - void reset(); -public: - bool is_valid() const; - int set(const int64_t id, - const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &proposal_id, - const LSN &prev_lsn, - const LSN &start_lsn, - const int64_t log_size, - const int64_t log_count, - const int64_t accepted_mode_pid); - int64_t get_timestamp_us() const { return timestamp_us_; } - int64_t get_id() const { return id_; } - const common::ObAddr &get_server() const { return server_; } - FetchLogType get_fetch_type() const { return fetch_type_; } - const int64_t &get_proposal_id() const { return proposal_id_; } - const LSN &get_prev_lsn() const { return prev_lsn_; } - const LSN &get_start_lsn() const { return start_lsn_; } - int64_t get_log_size() const { return log_size_; } - int64_t get_log_count() const { return log_count_; } - int64_t get_accepted_mode_pid() const { return accepted_mode_pid_; } - FetchLogTask& operator=(const FetchLogTask &task); - - TO_STRING_KV(K_(timestamp_us), K_(id), K_(server), "fetch_type", fetch_type_2_str(fetch_type_), K_(proposal_id), - K_(prev_lsn), K_(start_lsn), K_(log_size), K_(log_count), K_(accepted_mode_pid)); -private: - int64_t timestamp_us_; - int64_t id_; - common::ObAddr server_; - FetchLogType fetch_type_; - int64_t proposal_id_; - LSN prev_lsn_; - LSN start_lsn_; - int64_t log_size_; - int64_t log_count_; - int64_t accepted_mode_pid_; -}; - -class FetchLogEngine : public common::ObLinkQueueThreadPool -{ -public: - // dynamic with tenant unit - static const int64_t FETCH_LOG_THREAD_COUNT = 1; - static const int64_t MINI_MODE_FETCH_LOG_THREAD_COUNT = 1; - static const int64_t FETCH_LOG_TASK_MAX_COUNT_PER_LS = 64; - static const int64_t DEFAULT_CACHED_FETCH_TASK_NUM = 8; - static const int64_t MAX_CACHED_FETCH_TASK_NUM = 1024; -public: - FetchLogEngine(); - ~FetchLogEngine() { destroy(); } -public: - int init(IPalfEnvImpl *palf_env_impl, ObILogAllocator *alloc_mgr, const bool enabled = true); - void destroy(); - int submit_fetch_log_task(FetchLogTask *fetch_log_task); - bool is_enabled() const { return is_inited_ && enabled_; } -public: - int start(); - int stop(); - int wait(); - void handle(common::LinkTask *task) override; - void handle_drop(common::LinkTask *task) override; - FetchLogTask *alloc_fetch_log_task(); - void free_fetch_log_task(FetchLogTask *task); - int update_replayable_point(const share::SCN &replayable_scn); -private: - int try_remove_task_from_cache_(FetchLogTask *fetch_log_task); - int push_task_into_cache_(FetchLogTask *fetch_log_task); -private: - typedef common::ObSpinLock SpinLock; - typedef common::ObSpinLockGuard SpinLockGuard; -private: - bool is_inited_; - bool enabled_; - IPalfEnvImpl *palf_env_impl_; - common::ObILogAllocator *allocator_; - share::SCN replayable_point_; - mutable SpinLock cache_lock_; - ObSEArray fetch_task_cache_; - ObMiniStat::ObStatItem fetch_wait_cost_stat_; - ObMiniStat::ObStatItem fetch_log_cost_stat_; - DISALLOW_COPY_AND_ASSIGN(FetchLogEngine); -}; -} // namespace logservice -} // namespace oceanbase - -#endif // OCEANBASE_LOGSERVICE_OB_FETCH_LOG_ENGINE_ diff --git a/src/logservice/palf/log_ack_info.h b/src/logservice/palf/log_ack_info.h deleted file mode 100644 index 8471f3e97..000000000 --- a/src/logservice/palf/log_ack_info.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef OCEANBASE_LOGSERVICE_LOG_ACK_INFO_ -#define OCEANBASE_LOGSERVICE_LOG_ACK_INFO_ - -#include "common/ob_member.h" -#include "common/ob_learner_list.h" -#include "lib/container/ob_se_array.h" -#include "log_define.h" -#include "lsn.h" - -namespace oceanbase -{ -namespace palf -{ - -struct LsnTsInfo -{ - LSN lsn_; - int64_t last_ack_time_us_; - int64_t last_advance_time_us_; - LsnTsInfo() - : lsn_(), last_ack_time_us_(OB_INVALID_TIMESTAMP), last_advance_time_us_(OB_INVALID_TIMESTAMP) - {} - LsnTsInfo(const LSN &lsn, const int64_t ack_time_us) - : lsn_(lsn), last_ack_time_us_(ack_time_us), last_advance_time_us_(ack_time_us) - {} - bool is_valid() const { - return (lsn_.is_valid() && OB_INVALID_TIMESTAMP != last_ack_time_us_); - } - void reset() - { - lsn_.reset(); - last_ack_time_us_ = OB_INVALID_TIMESTAMP; - last_advance_time_us_ = OB_INVALID_TIMESTAMP; - } - void operator=(const LsnTsInfo &val) - { - lsn_ = val.lsn_; - last_ack_time_us_ = val.last_ack_time_us_; - last_advance_time_us_ = val.last_advance_time_us_; - } - TO_STRING_KV(K_(lsn), K_(last_ack_time_us), K_(last_advance_time_us)); -}; - -struct LogMemberAckInfo -{ - LogMemberAckInfo () - : member_(), - last_ack_time_us_(OB_INVALID_TIMESTAMP), - last_flushed_end_lsn_() - { } - LogMemberAckInfo (const common::ObMember &member, - const int64_t last_ack_time_us, - const LSN &last_flushed_end_lsn) - : member_(member), - last_ack_time_us_(last_ack_time_us), - last_flushed_end_lsn_(last_flushed_end_lsn) - { } - common::ObMember member_; - // Downgrade when double check - int64_t last_ack_time_us_; - // Upgrade time double check - LSN last_flushed_end_lsn_; - - void operator=(const LogMemberAckInfo &ack_info) - { - member_ = ack_info.member_; - last_ack_time_us_ = ack_info.last_ack_time_us_; - last_flushed_end_lsn_ = ack_info.last_flushed_end_lsn_; - } - - TO_STRING_KV(K_(member), K_(last_ack_time_us), K_(last_flushed_end_lsn)); -}; - -typedef common::ObSEArray LogMemberAckInfoList; - -template -inline int64_t ack_info_list_get_index(const common::ObSEArray &list_a, - const common::ObAddr &addr) -{ - int64_t index = -1; - for (int64_t i = 0; i < list_a.count(); ++i) { - if (list_a.at(i).member_.get_server() == addr) { - index = i; - break; - } - } - return index; -} - -template -inline bool ack_info_list_addr_equal(const common::GlobalLearnerList &list_a, - const common::ObSEArray &list_b) -{ - bool bool_ret = true; - if (list_a.get_member_number() != list_b.count()) { - bool_ret = false; - } else { - common::ObAddr addr; - for (int64_t i = 0; i < list_b.count(); ++i) { - if (!list_a.contains(list_b.at(i).member_.get_server())) { - bool_ret = false; - } - } - } - return bool_ret; -} - -template -inline bool ack_info_list_addr_equal(const common::ObSEArray &list_a, - const common::ObSEArray &list_b) -{ - bool bool_ret = true; - if (list_a.count() != list_b.count()) { - bool_ret = false; - } else { - common::ObAddr addr; - for (int64_t i = 0; i < list_b.count(); ++i) { - if (-1 == ack_info_list_get_index(list_a, list_b.at(i).member_.get_server())) { - bool_ret = false; - break; - } - } - } - return bool_ret; -} - -} // namespace palf -} // namespace oceanbase -#endif // OCEANBASE_LOGSERVICE_LOG_ACK_INFO_ diff --git a/src/logservice/palf/log_block_header.cpp b/src/logservice/palf/log_block_header.cpp index 9f78847fa..07ff4771e 100644 --- a/src/logservice/palf/log_block_header.cpp +++ b/src/logservice/palf/log_block_header.cpp @@ -39,8 +39,7 @@ bool LogBlockHeader::is_valid() const && LOG_INFO_BLOCK_VERSION == version_ && true == min_lsn_.is_valid() && true == min_scn_.is_valid() - && true == is_valid_block_id(curr_block_id_) - && true == is_valid_palf_id(palf_id_); + && true == is_valid_block_id(curr_block_id_); } void LogBlockHeader::reset() @@ -52,7 +51,6 @@ void LogBlockHeader::reset() min_scn_.reset(); max_scn_.reset(); curr_block_id_ = LOG_INVALID_BLOCK_ID; - palf_id_ = INVALID_PALF_ID; checksum_ = 0; } @@ -63,34 +61,10 @@ void LogBlockHeader::update_lsn_and_scn(const LSN &lsn, min_scn_ = scn; } -void LogBlockHeader::update_palf_id_and_curr_block_id(const int64_t palf_id, - const block_id_t curr_block_id) +void LogBlockHeader::update_curr_block_id(const block_id_t curr_block_id) { - palf_id_ = palf_id; curr_block_id_ = curr_block_id; } - -int LogBlockHeader::generate(const int64_t palf_id, - const block_id_t block_id, - const LSN &min_lsn, - const share::SCN &min_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY((!is_valid_palf_id(palf_id)) || (!is_valid_block_id(block_id)) - || (!min_lsn.is_valid()) || (!min_scn.is_valid()))) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(palf_id), K(block_id), K(min_lsn), K(min_scn)); - } else { - magic_ = MAGIC; - version_ = LOG_INFO_BLOCK_VERSION; - palf_id_ = palf_id; - curr_block_id_ = block_id; - min_lsn_ = min_lsn; - min_scn_ = min_scn; - checksum_ = calc_checksum_(); - } - return ret; -} void LogBlockHeader::mark_block_can_be_reused(const SCN &max_scn) { flag_ |= REUSED_BLOCK_MASK; @@ -151,7 +125,6 @@ DEFINE_SERIALIZE(LogBlockHeader) || OB_FAIL(min_scn_.fixed_serialize(buf, buf_len, new_pos)) || OB_FAIL(max_scn_.fixed_serialize(buf, buf_len, new_pos)) || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, curr_block_id_)) - || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, palf_id_)) || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, checksum_))) { PALF_LOG(ERROR, "serialize failed", K(ret), K(pos), K(buf_len)); } else { @@ -173,7 +146,6 @@ DEFINE_DESERIALIZE(LogBlockHeader) || OB_FAIL(min_scn_.fixed_deserialize(buf, data_len, new_pos)) || OB_FAIL(max_scn_.fixed_deserialize_without_transform(buf, data_len, new_pos)) || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&curr_block_id_))) - || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &palf_id_)) || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &checksum_))) { PALF_LOG(ERROR, "serialize failed", K(ret), K(pos), K(data_len)); } else { @@ -192,7 +164,6 @@ DEFINE_GET_SERIALIZE_SIZE(LogBlockHeader) size += min_scn_.get_fixed_serialize_size(); size += max_scn_.get_fixed_serialize_size(); size += serialization::encoded_length_i64(curr_block_id_); - size += serialization::encoded_length_i64(palf_id_); size += serialization::encoded_length_i64(checksum_); return size; } diff --git a/src/logservice/palf/log_block_header.h b/src/logservice/palf/log_block_header.h index 607d45c97..7ce210ccc 100644 --- a/src/logservice/palf/log_block_header.h +++ b/src/logservice/palf/log_block_header.h @@ -34,11 +34,9 @@ struct LogBlockHeader ~LogBlockHeader(); bool is_valid() const; void reset(); - int generate(const int64_t palf_id, const block_id_t curr_block_id, - const LSN &min_lsn, const share::SCN &min_scn); // NB: not thread safe void update_lsn_and_scn(const LSN &lsn, const share::SCN &scn); - void update_palf_id_and_curr_block_id(const int64_t palf_id, const block_id_t curr_block_id); + void update_curr_block_id(const block_id_t curr_block_id); void mark_block_can_be_reused(const share::SCN &max_scn); block_id_t get_curr_block_id() const; const share::SCN &get_min_scn() const; @@ -55,7 +53,7 @@ struct LogBlockHeader void calc_checksum(); bool check_integrity() const; NEED_SERIALIZE_AND_DESERIALIZE; - TO_STRING_KV(K_(magic), K_(version), K_(min_lsn), K_(min_scn), K_(curr_block_id), K_(palf_id)); + TO_STRING_KV(K_(magic), K_(version), K_(min_lsn), K_(min_scn), K_(curr_block_id)); // 0x4942 means InfoBlock static constexpr int16_t MAGIC= 0x4942; static constexpr int16_t LOG_INFO_BLOCK_VERSION = 1; @@ -80,7 +78,6 @@ struct LogBlockHeader // palf instance, even if switch block when write failed. // NB: to locate logs by LSN, we need keep the pair(physical block name, InfoBlock). block_id_t curr_block_id_; - int64_t palf_id_; int64_t checksum_; }; } // end namespace palf diff --git a/src/logservice/palf/log_block_mgr.cpp b/src/logservice/palf/log_block_mgr.cpp index 64f235f4d..45bf4b6f9 100644 --- a/src/logservice/palf/log_block_mgr.cpp +++ b/src/logservice/palf/log_block_mgr.cpp @@ -294,34 +294,6 @@ int LogBlockMgr::load_block_handler(const block_id_t block_id, const offset_t of return ret; } -int LogBlockMgr::create_tmp_block_handler(const block_id_t block_id) -{ - int ret = OB_SUCCESS; - char tmp_block_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; - if (OB_FAIL(block_id_to_tmp_string(block_id, tmp_block_path, OB_MAX_FILE_NAME_LENGTH))) { - PALF_LOG(ERROR, "block_id_to_tmp_string failed", K(ret), KPC(this), K(block_id)); - } else if (OB_FAIL(curr_writable_handler_.close())) { - PALF_LOG(ERROR, "curr_writable_handler_ close success"); - } else if (FALSE_IT(curr_writable_handler_.destroy())) { - } else if (OB_FAIL(curr_writable_handler_.init(log_block_size_, align_size_, align_buf_size_, io_adapter_))) { - PALF_LOG(ERROR, "curr_writable_handler_ init failed", K(ret), KPC(this)); - } else if (OB_FAIL(log_block_pool_->create_block_at(dir_fd_, tmp_block_path, log_block_size_))) { - } else if (OB_FAIL(construct_absolute_tmp_block_path(log_dir_, block_id, OB_MAX_FILE_NAME_LENGTH, tmp_block_path))) { - PALF_LOG(ERROR, "failed to construct absolute tmp block path", K(ret), KPC(this), K(block_id)); - } else if (OB_FAIL(curr_writable_handler_.open(tmp_block_path))) { - PALF_LOG(ERROR, "create_tmp_block failed", K(ret), KPC(this), K(block_id)); - } else { - curr_writable_block_id_ = block_id; - PALF_LOG(INFO, "create_tmp_block_handler success", K(ret), KPC(this), K(block_id)); - } - return ret; -} - -int LogBlockMgr::delete_block_from_back_to_front_until(const block_id_t block_id) -{ - return delete_block_from_back_to_front_until_(block_id); -} - // step1: firstly, delete each block after lsn.block_id_; // step2: secondly, truncate data in curr_lsn_; // step3: keep last dio_aligned_buf_. diff --git a/src/logservice/palf/log_block_mgr.h b/src/logservice/palf/log_block_mgr.h index ca0661d28..1215e5534 100644 --- a/src/logservice/palf/log_block_mgr.h +++ b/src/logservice/palf/log_block_mgr.h @@ -77,11 +77,6 @@ class LogBlockMgr { int load_block_handler(const block_id_t block_id, const offset_t offset); - int create_tmp_block_handler(const block_id_t block_id); - - // ======================= delete ===================== - int delete_block_from_back_to_front_until(const block_id_t block_id); - // ======================================================= TO_STRING_KV(K_(log_dir), K_(dir_fd), K_(min_block_id), K_(max_block_id), K_(curr_writable_block_id)); private: // @brief this function used to rebuild 'blocks_' diff --git a/src/logservice/palf/log_cache.cpp b/src/logservice/palf/log_cache.cpp index 3d6fbf277..20365eaac 100644 --- a/src/logservice/palf/log_cache.cpp +++ b/src/logservice/palf/log_cache.cpp @@ -25,8 +25,7 @@ using namespace share; namespace palf { LogHotCache::LogHotCache() - : palf_id_(INVALID_PALF_ID), - palf_handle_impl_(NULL), + : palf_handle_impl_(NULL), read_size_(0), hit_count_(0), read_count_(0), @@ -48,21 +47,19 @@ void LogHotCache::reset() { is_inited_ = false; palf_handle_impl_ = NULL; - palf_id_ = INVALID_PALF_ID; } -int LogHotCache::init(const int64_t palf_id, IPalfHandleImpl *palf_handle_impl) +int LogHotCache::init(IPalfHandleImpl *palf_handle_impl) { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; - } else if (false == is_valid_palf_id(palf_id) || OB_ISNULL(palf_handle_impl)) { + } else if (OB_ISNULL(palf_handle_impl)) { ret = OB_INVALID_ARGUMENT; } else { - palf_id_ = palf_id; palf_handle_impl_ = palf_handle_impl; is_inited_ = true; - PALF_LOG(TRACE, "init hot cache successfully", K(palf_id_)); + PALF_LOG(TRACE, "init hot cache successfully"); } return ret; } @@ -78,15 +75,15 @@ int LogHotCache::read(const LSN &read_begin_lsn, int64_t start_ts = ObTimeUtility::fast_current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "hot cache is not inited", K(ret), K(palf_id_)); + PALF_LOG(WARN, "hot cache is not inited", K(ret)); } else if (!read_begin_lsn.is_valid() || in_read_size <= 0 || OB_ISNULL(buf)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size), + PALF_LOG(WARN, "invalid arguments", K(ret), K(read_begin_lsn), K(in_read_size), KP(buf)); } else if (OB_FAIL(palf_handle_impl_->read_data_from_buffer(read_begin_lsn, in_read_size, \ buf, out_read_size))) { if (OB_ERR_OUT_OF_LOWER_BOUND != ret) { - PALF_LOG(WARN, "read_data_from_buffer failed", K(ret), K_(palf_id), K(read_begin_lsn), + PALF_LOG(WARN, "read_data_from_buffer failed", K(ret), K(read_begin_lsn), K(in_read_size)); } } else { @@ -96,13 +93,13 @@ int LogHotCache::read(const LSN &read_begin_lsn, EVENT_TENANT_INC(ObStatEventIds::PALF_READ_COUNT_FROM_HOT_CACHE); EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_HOT_CACHE, out_read_size); EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_HOT_CACHE, cost_ts); - PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K_(palf_id), K(read_begin_lsn), + PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K(read_begin_lsn), K(in_read_size), K(out_read_size)); } read_cnt = ATOMIC_AAF(&read_count_, 1); if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_print_time_)) { read_cnt = read_cnt == 0 ? 1 : read_cnt; - PALF_LOG(INFO, "[PALF STAT HOT CACHE HIT RATE]", K_(palf_id), K(read_size), K(hit_cnt), K(read_cnt), "hit rate", hit_cnt * 1.0 / read_cnt); + PALF_LOG(INFO, "[PALF STAT HOT CACHE HIT RATE]", K(read_size), K(hit_cnt), K(read_cnt), "hit rate", hit_cnt * 1.0 / read_cnt); hit_count_ = 0; read_size_ = 0; read_count_ = 0; @@ -120,24 +117,21 @@ LogCache::~LogCache() void LogCache::destroy() { - palf_id_ = INVALID_PALF_ID; hot_cache_.destroy(); is_inited_ = false; } -int LogCache::init(const int64_t palf_id, - IPalfHandleImpl *palf_handle_impl) +int LogCache::init(IPalfHandleImpl *palf_handle_impl) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; PALF_LOG(WARN, "LogCache init failed", K(ret)); - } else if (OB_FAIL(hot_cache_.init(palf_id, palf_handle_impl))){ - PALF_LOG(WARN, "hot cache init failed", K(ret), K(palf_id)); + } else if (OB_FAIL(hot_cache_.init(palf_handle_impl))){ + PALF_LOG(WARN, "hot cache init failed", K(ret)); } else { - palf_id_ = palf_id; is_inited_ = true; - PALF_LOG(INFO, "LogCache init successfully", K(palf_id)); + PALF_LOG(INFO, "LogCache init successfully"); } return ret; diff --git a/src/logservice/palf/log_cache.h b/src/logservice/palf/log_cache.h index 6e30ffa50..1653c7458 100644 --- a/src/logservice/palf/log_cache.h +++ b/src/logservice/palf/log_cache.h @@ -36,13 +36,12 @@ class LogHotCache ~LogHotCache(); void destroy(); void reset(); - int init(const int64_t palf_id, IPalfHandleImpl *palf_handle_impl); + int init(IPalfHandleImpl *palf_handle_impl); int read(const LSN &read_begin_lsn, const int64_t in_read_size, char *buf, int64_t &out_read_size) const; private: - int64_t palf_id_; IPalfHandleImpl *palf_handle_impl_; mutable int64_t read_size_; mutable int64_t hit_count_; @@ -57,22 +56,20 @@ class LogCache LogCache(); ~LogCache(); void destroy(); - int init(const int64_t palf_id, - IPalfHandleImpl *palf_handle_impl); + int init(IPalfHandleImpl *palf_handle_impl); bool is_inited() const; int read(const LSN &lsn, const int64_t in_read_size, ReadBuf &read_buf, int64_t &out_read_size, LogIOContext &io_ctx); - TO_STRING_KV(K(is_inited_), K(palf_id_)); + TO_STRING_KV(K(is_inited_)); private: int read_hot_cache_(const LSN &read_begin_lsn, const int64_t in_read_size, char *buf, int64_t &out_read_size); private: - int64_t palf_id_; LogHotCache hot_cache_; bool is_inited_; }; diff --git a/src/logservice/palf/log_checksum.cpp b/src/logservice/palf/log_checksum.cpp index 618f64715..47d8101b1 100644 --- a/src/logservice/palf/log_checksum.cpp +++ b/src/logservice/palf/log_checksum.cpp @@ -30,17 +30,16 @@ LogChecksum::LogChecksum() verify_checksum_(0) {} -int LogChecksum::init(const int64_t palf_id, const int64_t accum_checksum) +int LogChecksum::init(const int64_t accum_checksum) { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; } else { - palf_id_ = palf_id; accum_checksum_ = accum_checksum; verify_checksum_ = accum_checksum; is_inited_ = true; - PALF_LOG(INFO, "LogChecksum init success", K_(palf_id), K_(verify_checksum), K_(accum_checksum)); + PALF_LOG(INFO, "LogChecksum init success", K_(verify_checksum), K_(accum_checksum)); } return ret; } @@ -48,7 +47,6 @@ int LogChecksum::init(const int64_t palf_id, const int64_t accum_checksum) void LogChecksum::destroy() { is_inited_ = false; - palf_id_ = INVALID_PALF_ID; accum_checksum_ = 0; verify_checksum_ = 0; } @@ -64,7 +62,7 @@ int LogChecksum::acquire_accum_checksum(const int64_t data_checksum, accum_checksum_ = common::ob_crc64(accum_checksum_, const_cast(&data_checksum), sizeof(data_checksum)); accum_checksum = accum_checksum_; - PALF_LOG(TRACE, "acquire_accum_checksum success", K_(palf_id), K(data_checksum), K_(verify_checksum), + PALF_LOG(TRACE, "acquire_accum_checksum success", K(data_checksum), K_(verify_checksum), K_(prev_accum_checksum), K_(accum_checksum)); } return ret; @@ -85,7 +83,7 @@ int LogChecksum::verify_accum_checksum(const int64_t data_checksum, } else { // Update verify_checksum_ when checking succeeds. verify_checksum_ = new_verify_checksum; - PALF_LOG(TRACE, "verify_accum_checksum success", K(ret), K_(palf_id), K(data_checksum), K(accum_checksum), + PALF_LOG(TRACE, "verify_accum_checksum success", K(ret), K(data_checksum), K(accum_checksum), K_(verify_checksum), K_(accum_checksum)); } @@ -119,12 +117,12 @@ int LogChecksum::rollback_accum_checksum(const int64_t curr_accum_checksum) int ret = common::OB_SUCCESS; if (curr_accum_checksum != accum_checksum_) { ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "accum_checksum does not match", K_(palf_id), K(curr_accum_checksum), K_(prev_accum_checksum), + PALF_LOG(WARN, "accum_checksum does not match", K(curr_accum_checksum), K_(prev_accum_checksum), K_(accum_checksum)); } else { accum_checksum_ = prev_accum_checksum_; prev_accum_checksum_ = 0; - PALF_LOG(INFO, "rollback_accum_checksum", K_(palf_id), K(curr_accum_checksum), K_(prev_accum_checksum), + PALF_LOG(INFO, "rollback_accum_checksum", K(curr_accum_checksum), K_(prev_accum_checksum), K_(accum_checksum)); } return ret; @@ -133,13 +131,13 @@ int LogChecksum::rollback_accum_checksum(const int64_t curr_accum_checksum) void LogChecksum::set_accum_checksum(const int64_t accum_checksum) { accum_checksum_ = accum_checksum; - PALF_LOG(INFO, "set_accum_checksum", K_(palf_id), K(accum_checksum)); + PALF_LOG(INFO, "set_accum_checksum", K(accum_checksum)); } void LogChecksum::set_verify_checksum(const int64_t verify_checksum) { verify_checksum_ = verify_checksum; - PALF_LOG(INFO, "set_verify_checksum", K_(palf_id), K(verify_checksum)); + PALF_LOG(INFO, "set_verify_checksum", K(verify_checksum)); } } // namespace palf diff --git a/src/logservice/palf/log_checksum.h b/src/logservice/palf/log_checksum.h index b11c2212b..e994da559 100644 --- a/src/logservice/palf/log_checksum.h +++ b/src/logservice/palf/log_checksum.h @@ -30,7 +30,7 @@ class LogChecksum LogChecksum(); virtual ~LogChecksum() {} public: - int init(const int64_t id, const int64_t accum_checksum); + int init(const int64_t accum_checksum); void destroy(); virtual int acquire_accum_checksum(const int64_t data_checksum, int64_t &accum_checksum); @@ -45,7 +45,6 @@ class LogChecksum virtual int rollback_accum_checksum(const int64_t curr_accum_checksum); private: bool is_inited_; - int64_t palf_id_; int64_t prev_accum_checksum_; int64_t accum_checksum_; int64_t verify_checksum_; diff --git a/src/logservice/palf/log_config_mgr.cpp b/src/logservice/palf/log_config_mgr.cpp deleted file mode 100755 index fbb5a3731..000000000 --- a/src/logservice/palf/log_config_mgr.cpp +++ /dev/null @@ -1,3135 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX PALF -#include "log_config_mgr.h" -#include "log_engine.h" -#include "log_io_task_cb_utils.h" // FlushMetaCbCtx -#include "log_sliding_window.h" -#include "log_mode_mgr.h" -#include "log_reconfirm.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -LogConfigMgr::LogConfigMgr() - : log_ms_meta_(), - alive_paxos_memberlist_(), - alive_paxos_replica_num_(0), - all_learnerlist_(), - running_args_(), - lock_(common::ObLatchIds::PALF_CM_CONFIG_LOCK), - palf_id_(), - self_(), - reconfig_barrier_(), - checking_barrier_(), - state_(INIT), - last_submit_config_log_time_us_(OB_INVALID_TIMESTAMP), - ms_ack_list_(), - need_change_config_bkgd_(false), - bkgd_config_version_(), - is_sw_interrupted_by_degrade_(false), - will_upgrade_(false), - last_start_upgrade_time_us_(OB_INVALID_TIMESTAMP), - resend_config_version_(), - resend_log_list_(), - election_leader_epoch_(OB_INVALID_TIMESTAMP), - last_broadcast_leader_info_time_us_(OB_INVALID_TIMESTAMP), - persistent_config_version_(), - barrier_print_log_time_(OB_INVALID_TIMESTAMP), - last_check_init_state_time_us_(OB_INVALID_TIMESTAMP), - check_config_print_time_(OB_INVALID_TIMESTAMP), - start_wait_barrier_time_us_(OB_INVALID_TIMESTAMP), - last_wait_barrier_time_us_(OB_INVALID_TIMESTAMP), - last_wait_committed_end_lsn_(), - last_sync_meta_for_arb_election_leader_time_us_(OB_INVALID_TIMESTAMP), - forwarding_config_proposal_id_(INVALID_PROPOSAL_ID), - parent_lock_(common::ObLatchIds::PALF_CM_PARENT_LOCK), - register_time_us_(OB_INVALID_TIMESTAMP), - register_parent_reason_(RegisterParentReason::INVALID), - parent_(), - parent_keepalive_time_us_(OB_INVALID_TIMESTAMP), - last_submit_register_req_time_us_(OB_INVALID_TIMESTAMP), - last_first_register_time_us_(OB_INVALID_TIMESTAMP), - child_lock_(common::ObLatchIds::PALF_CM_CHILD_LOCK), - children_(), - log_sync_children_(), - last_submit_keepalive_time_us_(OB_INVALID_TIMESTAMP), - log_engine_(NULL), - sw_(NULL), - state_mgr_(NULL), - election_(NULL), - mode_mgr_(NULL), - reconfirm_(NULL), - plugins_(NULL), - is_inited_(false) -{} - -LogConfigMgr::~LogConfigMgr() -{ - destroy(); -} - - -int LogConfigMgr::init(const int64_t palf_id, - const ObAddr &self, - const LogConfigMeta &log_ms_meta, - LogEngine *log_engine, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - election::Election* election, - LogModeMgr *mode_mgr, - LogReconfirm *reconfirm, - LogPlugins *plugins) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - PALF_LOG(WARN, "LogConfigMgr init twice", KR(ret), K_(palf_id), K_(self)); - } else if (false== is_valid_palf_id(palf_id) || - false == self.is_valid() || - false == log_ms_meta.is_valid() || - OB_ISNULL(log_engine) || - OB_ISNULL(sw) || - OB_ISNULL(state_mgr) || - OB_ISNULL(election) || - OB_ISNULL(mode_mgr) || - OB_ISNULL(reconfirm) || - OB_ISNULL(plugins)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(palf_id), K(self), - K(log_ms_meta), KP(log_engine), KP(sw), KP(state_mgr), KP(election), KP(mode_mgr), - KP(reconfirm), KP(plugins)); - } else { - palf_id_ = palf_id; - self_ = self; - log_ms_meta_ = log_ms_meta; - log_engine_ = log_engine; - sw_ = sw; - state_mgr_ = state_mgr; - election_ = election; - mode_mgr_ = mode_mgr; - reconfirm_ = reconfirm; - plugins_ = plugins; - if (true == log_ms_meta.curr_.is_valid()) { - if (OB_FAIL(append_config_info_(log_ms_meta.curr_))) { - PALF_LOG(WARN, "append_config_info_ failed", K(ret), K(palf_id), K(log_ms_meta)); - } else if (OB_FAIL(update_election_meta_(log_ms_meta_.curr_))) { - PALF_LOG(WARN, "update_election_meta_ failed", K(ret), K(palf_id), K(log_ms_meta)); - } - } - if (OB_SUCC(ret)) { - persistent_config_version_ = log_ms_meta_.curr_.config_.config_version_; - is_inited_ = true; - PALF_LOG(INFO, "LogConfigMgr init success", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), KP(this)); - } - } - return ret; -} - -void LogConfigMgr::destroy() -{ - SpinLockGuard guard(lock_); - if (IS_INIT) { - PALF_LOG(INFO, "LogConfigMgr destory", K_(palf_id), K_(self)); - is_inited_ = false; - election_ = NULL; - mode_mgr_ = NULL; - state_mgr_ = NULL; - sw_ = NULL; - log_engine_ = NULL; - reconfirm_ = NULL; - plugins_ = NULL; - register_time_us_ = OB_INVALID_TIMESTAMP; - register_parent_reason_ = RegisterParentReason::INVALID; - parent_.reset(); - parent_keepalive_time_us_ = OB_INVALID_TIMESTAMP; - reset_registering_state_(); - children_.reset(); - log_sync_children_.reset(); - last_submit_keepalive_time_us_ = OB_INVALID_TIMESTAMP; - ms_ack_list_.reset(); - resend_config_version_.reset(); - resend_log_list_.reset(); - running_args_.reset(); - last_submit_config_log_time_us_ = OB_INVALID_TIMESTAMP; - log_ms_meta_.reset(); - alive_paxos_memberlist_.reset(); - alive_paxos_replica_num_ = 0; - all_learnerlist_.reset(); - last_sync_meta_for_arb_election_leader_time_us_ = OB_INVALID_TIMESTAMP; - forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID; - state_ = ConfigChangeState::INIT; - reconfig_barrier_.reset(); - checking_barrier_.reset(); - persistent_config_version_.reset(); - barrier_print_log_time_ = OB_INVALID_TIMESTAMP; - self_.reset(); - palf_id_ = 0; - } -} - -int LogConfigMgr::set_initial_member_list(const ObMemberList &member_list, - const int64_t replica_num, - const common::GlobalLearnerList &learner_list, - const int64_t proposal_id, - LogConfigVersion &init_config_version) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (!member_list.is_valid() || - replica_num <= 0 || - replica_num > OB_MAX_MEMBER_NUMBER || - INVALID_PROPOSAL_ID == proposal_id || - false == can_memberlist_majority_(member_list.get_member_number(), replica_num)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K(member_list), K(replica_num)); - } else { - LogConfigInfoV2 config_info = log_ms_meta_.curr_; - config_info.config_.log_sync_memberlist_ = member_list; - config_info.config_.log_sync_replica_num_ = replica_num; - config_info.config_.learnerlist_ = learner_list; - if (OB_FAIL(set_initial_config_info_(config_info, proposal_id, init_config_version))) { - PALF_LOG(WARN, "set_initial_config_info failed", K(ret), K_(palf_id), K_(self), K(config_info), K(proposal_id)); - } else { - PALF_LOG(INFO, "set_initial_member_list success", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member_list), K(replica_num), K(proposal_id)); - } - } - return ret; -} - -int LogConfigMgr::set_initial_config_info_(const LogConfigInfoV2 &config_info, - const int64_t proposal_id, - LogConfigVersion &init_config_version) -{ - int ret = OB_SUCCESS; - const int64_t initial_config_seq = 1; - LogReplicaType replica_type = state_mgr_->get_replica_type(); - const bool valid_replica_type = (config_info.config_.arbitration_member_.get_server() == self_) ? \ - (replica_type == ARBITRATION_REPLICA) : true; - if (false == valid_replica_type) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "set_initial_member_list don't match with replica_type", KR(ret), K_(palf_id), K_(self), K(replica_type), K(config_info)); - } else if (OB_FAIL(init_config_version.generate(proposal_id, initial_config_seq))) { - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K(proposal_id), K(initial_config_seq)); - } else { - LogConfigInfoV2 init_config_info = config_info; - init_config_info.config_.config_version_ = init_config_version; - if (false == init_config_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "initial config info is invalid", K_(palf_id), K(config_info), K(proposal_id)); - } else if (log_ms_meta_.curr_.config_.config_version_ > init_config_version || - false == check_need_update_memberlist_without_lock_(init_config_version)) { - PALF_LOG(INFO, "persistent_config_version_ has been greater than or equal to config_version, \ - no need set_initial_config_info_", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K_(persistent_config_version), K(init_config_version)); - } else if (OB_FAIL(log_ms_meta_.generate(proposal_id, init_config_info, init_config_info, - proposal_id, LSN(PALF_INITIAL_LSN_VAL), proposal_id))) { - PALF_LOG(WARN, "generate LogConfigMeta failed", KR(ret), K_(palf_id), K(proposal_id), K(initial_config_seq)); - } else { - FlushMetaCbCtx cb_ctx; - cb_ctx.type_ = MetaType::CHANGE_CONFIG_META; - cb_ctx.proposal_id_ = proposal_id; - cb_ctx.config_version_ = log_ms_meta_.curr_.config_.config_version_; - if (OB_FAIL(append_config_info_(log_ms_meta_.curr_))) { - PALF_LOG(WARN, "append_config_info_ failed", K(ret), K_(palf_id), K_(log_ms_meta)); - } else if (OB_FAIL(update_election_meta_(log_ms_meta_.curr_))) { - PALF_LOG(WARN, "update_election_memberlist_ failed", K(ret)); - } else if (OB_FAIL(log_engine_->submit_flush_change_config_meta_task(cb_ctx, log_ms_meta_))) { - PALF_LOG(WARN, "LogEngine submit_flush_change_config_meta_task failed", K(ret), K_(log_ms_meta)); - } else { - PALF_LOG(INFO, "set_initial_config_info_ success", K_(palf_id), K_(self), K_(log_ms_meta), K(config_info), \ - K_(alive_paxos_memberlist), K_(alive_paxos_replica_num), KP(this)); - } - } - } - return ret; -} - -int LogConfigMgr::get_prev_member_list(ObMemberList &member_list) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(member_list.deep_copy(log_ms_meta_.prev_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta)); - } else if (log_ms_meta_.prev_.config_.arbitration_member_.is_valid() && - OB_FAIL(member_list.add_member(log_ms_meta_.prev_.config_.arbitration_member_))) { - PALF_LOG(WARN, "add_member failed", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta)); - // no nothing - } - return ret; -} - -int LogConfigMgr::get_global_learner_list(common::GlobalLearnerList &learner_list) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(learner_list.deep_copy(log_ms_meta_.curr_.config_.learnerlist_))) { - PALF_LOG(WARN, "deep_copy learner_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - // pass - } - return ret; -} - -int LogConfigMgr::get_degraded_learner_list(common::GlobalLearnerList °raded_learner_list) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(degraded_learner_list.deep_copy(log_ms_meta_.curr_.config_.degraded_learnerlist_))) { - PALF_LOG(WARN, "deep_copy degraded_learnerlist_ failed", KR(ret), K_(palf_id), K_(self)); - } else { - // pass - } - return ret; -} - -int LogConfigMgr::get_curr_member_list(ObMemberList &member_list, int64_t &replica_num) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(log_ms_meta_.curr_.config_.get_expected_paxos_memberlist(member_list, replica_num))) { - PALF_LOG(WARN, "get_expected_paxos_memberlist failed", KR(ret), K_(palf_id), K_(self)); - } else { - // no nothing - } - return ret; -} - -int LogConfigMgr::get_alive_member_list_with_arb( - common::ObMemberList &member_list, - int64_t &replica_num) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(member_list.deep_copy(alive_paxos_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - replica_num = alive_paxos_replica_num_; - } - return ret; -} - -// require rlock of PalfHandleImpl -int LogConfigMgr::get_log_sync_member_list_for_generate_committed_lsn( - ObMemberList &prev_member_list, - int64_t &prev_replica_num, - ObMemberList &curr_member_list, - int64_t &curr_replica_num, - bool &is_before_barrier, - LSN &barrier_lsn) const -{ - int ret = OB_SUCCESS; - LSN prev_committed_end_lsn; - sw_->get_committed_end_lsn(prev_committed_end_lsn); - const int64_t prev_mode_pid = mode_mgr_->get_last_submit_mode_meta().proposal_id_; - is_before_barrier = false; - barrier_lsn = LSN(PALF_INITIAL_LSN_VAL); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(curr_member_list.deep_copy(log_ms_meta_.curr_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(curr_replica_num = log_ms_meta_.curr_.config_.log_sync_replica_num_)) { - } else if (OB_UNLIKELY(prev_committed_end_lsn < reconfig_barrier_.prev_end_lsn_ && - reconfig_barrier_.prev_end_lsn_.is_valid() && - prev_mode_pid == reconfig_barrier_.prev_mode_pid_)) { - is_before_barrier = true; - barrier_lsn = reconfig_barrier_.prev_end_lsn_; - // Scenario: 2F1A - // 1. A reconfiguration (upgrade B) has been executed successfully with log_barrier 100 - // 2. the palf group is flashed back to 50, but reconfig_barrier in LogConfigMgr is still 100 - // 3. change to APPEND mode - // 4. the leader may commit logs in (50, 100) by prev_member_list(A) - // Note: to address above issue, we check mode_proposal_id. The previous memberlist will - // be used only when the reconfir_barrier_.prev_mode_pid_ is equal to current mode - // proposal_id. That means access mode hasn’t been changed (PALF hasn’t been flashed back) - // since last reconfiguration. - if (OB_FAIL(prev_member_list.deep_copy(log_ms_meta_.prev_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - prev_replica_num = log_ms_meta_.prev_.config_.log_sync_replica_num_; - } - } else { } - return ret; -} - -int LogConfigMgr::get_log_sync_member_list(ObMemberList &member_list, - int64_t &replica_num) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(member_list.deep_copy(log_ms_meta_.curr_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - replica_num = log_ms_meta_.curr_.config_.log_sync_replica_num_; - } - return ret; -} - -int LogConfigMgr::get_children_list(LogLearnerList &children) const -{ - int ret = OB_SUCCESS; - // TODO by yunlong: this lock may be hotspot, need check further - SpinLockGuard gaurd(child_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(children.deep_copy(children_))) { - PALF_LOG(WARN, "deep_copy children_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - //pass - } - return ret; -} - -// considering log replication performance, we update log_sync_children_ in the background -int LogConfigMgr::get_log_sync_children_list(LogLearnerList &children) const -{ - int ret = OB_SUCCESS; - SpinLockGuard gaurd(child_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(children.deep_copy(log_sync_children_))) { - PALF_LOG(WARN, "deep_copy children_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - //pass - } - return ret; -} - -int LogConfigMgr::get_config_version(LogConfigVersion &config_version) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - config_version = log_ms_meta_.curr_.config_.config_version_; - } - return ret; -} - -int LogConfigMgr::get_replica_num(int64_t &replica_num) const -{ - int ret = OB_SUCCESS; - common::ObMemberList member_list; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(log_ms_meta_.curr_.config_.get_expected_paxos_memberlist(member_list, replica_num))) { - PALF_LOG(WARN, "get_expected_paxos_memberlist failed", KR(ret), K_(palf_id), K_(self)); - } else {/*do nothing*/} - return ret; -} - -int LogConfigMgr::get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) -{ - SpinLockGuard guard(lock_); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (!state_mgr_->is_leader_active()) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "need to be leader active", KR(ret), K_(palf_id), K_(self), - "role", state_mgr_->get_role(), "state", state_mgr_->get_state()); - } else { - const bool is_curr_locked = log_ms_meta_.curr_.lock_meta_.is_locked(); - const bool is_prev_locked = log_ms_meta_.prev_.lock_meta_.is_locked(); - lock_owner = log_ms_meta_.curr_.lock_meta_.lock_owner_; - if (CHANGING == state_) { - if (is_curr_locked != is_prev_locked) { - ret = OB_EAGAIN; - } else { - is_locked = is_curr_locked; - } - } else { - is_locked = is_curr_locked; - } - } - return ret; -} - -int64_t LogConfigMgr::get_accept_proposal_id() const -{ - return log_ms_meta_.proposal_id_; -} - -void LogConfigMgr::set_sync_to_degraded_learners() -{ - will_upgrade_ = true; - last_start_upgrade_time_us_ = common::ObTimeUtility::current_time(); - PALF_LOG_RET(INFO, OB_SUCCESS, "set_sync_to_degraded_learners to true", K_(palf_id), K_(self)); -} - -bool LogConfigMgr::is_sync_to_degraded_learners() const -{ - // Note: do not need acquire lock - return will_upgrade_; -} - -int LogConfigMgr::submit_broadcast_leader_info(const int64_t proposal_id) const -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(submit_broadcast_leader_info_(proposal_id))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", KR(ret), K_(palf_id), K_(self), K(proposal_id)); - } else { - PALF_LOG(INFO, "submit_prepare_meta_req success", KR(ret), K_(palf_id), K_(self), K(proposal_id)); - } - return ret; -} - -int LogConfigMgr::submit_broadcast_leader_info_(const int64_t proposal_id) const -{ - int ret = OB_SUCCESS; - const common::GlobalLearnerList &learner_list = all_learnerlist_; - if (INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - } else if (false == learner_list.is_valid()) { - } else if (OB_FAIL(log_engine_->submit_prepare_meta_req(learner_list, proposal_id))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", KR(ret), K_(palf_id), K_(self), K(proposal_id)); - } else { - PALF_LOG(INFO, "submit_prepare_meta_req success", K(ret), K_(palf_id), K_(self), K(proposal_id)); - } - return ret; -} - -const common::ObAddr &LogConfigMgr::get_parent() const -{ - SpinLockGuard guard(parent_lock_); - return parent_; -} - -//================================ Config Change ================================ - -// 1. switch config change state machine -// 2. broadcasting leader info periodically -// 3. resend config log to members and followers who haven't responsed ack -// after config change finished. -int LogConfigMgr::leader_do_loop_work(bool &need_change_config) -{ - int ret = OB_SUCCESS; - need_change_config = false; - SpinLockGuard guard(lock_); - const int64_t proposal_id = state_mgr_->get_proposal_id(); - const bool is_leader = (self_ == state_mgr_->get_leader()); - const bool is_leader_active = state_mgr_->is_leader_active(); - if (is_leader) { - bool need_rlock = false; - bool need_wlock = false; - if ((ConfigChangeState::INIT != state_) && - true == need_change_config_bkgd_ && - !FALSE_IT(is_state_changed_(need_rlock, need_wlock))) { - if (true == need_rlock) { - (void) change_config_(running_args_, proposal_id, election_leader_epoch_, bkgd_config_version_); - } else if (true == need_wlock) { - need_change_config = true; - } - } - if (OB_FAIL(try_resend_config_log_(proposal_id))) { - PALF_LOG(WARN, "try_resend_config_log failed", KR(ret), K_(palf_id), K_(self)); - } - - // reset will_upgrade_ flag if it have not been updated for 5s - if (will_upgrade_ == true && palf_reach_time_interval(5 * 1000 * 1000, last_start_upgrade_time_us_)) { - will_upgrade_ = false; - PALF_LOG_RET(INFO, OB_SUCCESS, "set_sync_to_degraded_learners to false", K_(palf_id), K_(self)); - } - } - if (is_leader_active && palf_reach_time_interval(PALF_BROADCAST_LEADER_INFO_INTERVAL_US, last_broadcast_leader_info_time_us_) && - OB_FAIL(submit_broadcast_leader_info_(proposal_id))) { - PALF_LOG(WARN, "submit_broadcast_leader_info failed", KR(ret), K_(self), K_(palf_id)); - } - return ret; -} - -int LogConfigMgr::switch_state() -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - const int64_t proposal_id = state_mgr_->get_proposal_id(); - const bool is_leader = (self_ == state_mgr_->get_leader()); - if (is_leader) { - bool need_rlock = false; - bool need_wlock = false; - if ((ConfigChangeState::INIT != state_) && - true == need_change_config_bkgd_ && - !FALSE_IT(is_state_changed_(need_rlock, need_wlock)) && - true == need_wlock) { - (void) change_config_(running_args_, proposal_id, election_leader_epoch_, bkgd_config_version_); - } - } - return ret; -} - -int LogConfigMgr::after_flush_config_log(const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - LogLearnerList removed_children; - do { - SpinLockGuard guard(lock_); - persistent_config_version_ = (persistent_config_version_.is_valid())? MAX(persistent_config_version_, config_version): config_version; - } while (0); - // if self is in memberlist, then retire parent, otherwise paxos member may has a parent. - if (alive_paxos_memberlist_.contains(self_)) { - SpinLockGuard guard_parent(parent_lock_); - if (parent_.is_valid() && OB_FAIL(retire_parent_(RetireParentReason::IS_FULL_MEMBER))) { - PALF_LOG(WARN, "retire_parent failed", KR(ret), K_(palf_id), K_(self)); - } - } - // if child is not in learnerlist, then retire child - if (OB_FAIL(remove_child_is_not_learner_(removed_children))) { - PALF_LOG(WARN, "remove_child_is_not_learner failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(submit_retire_children_req_(removed_children, - RetireChildReason::CHILD_NOT_IN_LEARNER_LIST))) { - PALF_LOG(WARN, "submit_retire_children_req failed", KR(ret), K_(palf_id), K_(self), K(removed_children)); - } - PALF_LOG(INFO, "after_flush_config_log success", K_(palf_id), K_(self), K(config_version), K_(persistent_config_version)); - return ret; -} - -int LogConfigMgr::is_state_changed(bool &need_rlock, bool &need_wlock) const -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = is_state_changed_(need_rlock, need_wlock); - } - return ret; -} - -int LogConfigMgr::is_state_changed_(bool &need_rlock, bool &need_wlock) const -{ - int ret = OB_SUCCESS; - need_rlock = false; - need_wlock = false; - switch(state_) { - case (ConfigChangeState::INIT): { - need_wlock = need_recheck_init_state_(); - break; - } - case (ConfigChangeState::CHANGING): { - if (is_reach_majority_()) { - need_wlock = true; - } else if (need_resend_config_log_()) { - need_rlock = true; - } - break; - } - default: - { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "invalid ConfigChangeState", K_(palf_id), K_(self), K(state_)); - break; - } - } - return ret; -} - -int LogConfigMgr::check_config_version_matches_state_(const LogConfigChangeType &type, - const LogConfigVersion &config_version) const -{ - int ret = OB_SUCCESS; - if (ConfigChangeState::INIT == state_) { - if (config_version.is_valid()) { - if (FORCE_SINGLE_MEMBER != type) { - // For force set single member case, this may occur. After updating election's member_list, - // Self may be elected and finish reconfirm quickly, which will reset config_mgr state and - // write start_working log successfully. - ret = OB_ERR_UNEXPECTED; - } else { - PALF_LOG(INFO, "Another config change(maybe self reconfirm) has finished during force set single member", K_(palf_id), K_(self), K_(state), K(type), K(config_version), K_(log_ms_meta)); - } - } - } else { - ret = (config_version != log_ms_meta_.curr_.config_.config_version_)? OB_EAGAIN: OB_SUCCESS; - } - return ret; -} - -int LogConfigMgr::start_change_config(int64_t &proposal_id, - int64_t &election_epoch, - const LogConfigChangeType &type) -{ - int ret = OB_SUCCESS; - common::ObRole ele_role; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (CHANGING == state_) { - ret = OB_EAGAIN; - } else if (OB_FAIL(election_->get_role(ele_role, election_epoch))) { - PALF_LOG(ERROR, "election get_role failed", K(ret), K_(self), K_(palf_id)); - } else { - proposal_id = state_mgr_->get_proposal_id(); - start_wait_barrier_time_us_ = OB_INVALID_TIMESTAMP; - last_wait_barrier_time_us_ = OB_INVALID_TIMESTAMP; - is_sw_interrupted_by_degrade_ = (type == DEGRADE_ACCEPTOR_TO_LEARNER); - will_upgrade_ = (type == LogConfigChangeType::UPGRADE_LEARNER_TO_ACCEPTOR); - last_start_upgrade_time_us_ = (will_upgrade_)? common::ObTimeUtility::current_time(): last_start_upgrade_time_us_; - } - return ret; -} - -int LogConfigMgr::end_degrade() -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - is_sw_interrupted_by_degrade_ = false; - } - return ret; -} - -int LogConfigMgr::change_config(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = change_config_(args, proposal_id, election_epoch, config_version); - PALF_LOG(INFO, "config_change stat", K_(palf_id), K_(self), K(args), K(proposal_id), - K_(checking_barrier), K_(reconfig_barrier), K_(state), K(config_version), - K_(persistent_config_version), K_(ms_ack_list), K_(resend_config_version), - K_(resend_log_list), K_(log_ms_meta), K_(last_submit_config_log_time_us)); - } - return ret; -} - -// state machine for config change -// ┌──────────────────────┐ -// | Config Change | -// └──────────────────────┘ -// │ INIT | -// │ │ │ -// │ ▼ │ -// │ CHANGING │ -// └──────────────────────┘ -int LogConfigMgr::change_config_(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - // args may be invalid when background retry config_change, so don't check it here - if (need_exec_on_leader_(args.type_) - && false == is_leader_for_config_change_(args.type_, proposal_id, election_epoch)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "not leader, can't change member", KR(ret), K_(palf_id), K_(self), - "role", state_mgr_->get_role(), "state", state_mgr_->get_state()); - } else if (false == mode_mgr_->can_do_paxos_accept()) { - // do not allow chagne_config when ModeMgr is in prepare state - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "is changing access_mode, try again", KR(ret), K_(palf_id), K_(self), - "role", state_mgr_->get_role(), "state", state_mgr_->get_state()); - } else if (OB_FAIL(check_config_version_matches_state_(args.type_, config_version))) { - PALF_LOG(WARN, "config_version does not match with state, try again", KR(ret), K_(palf_id), K_(self), - K(config_version), K_(state), K_(log_ms_meta)); - } else { - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - switch(state_) { - case (ConfigChangeState::INIT): { - bool is_already_finished = false; - last_check_init_state_time_us_ = common::ObTimeUtility::current_time(); - if (OB_FAIL(append_config_meta_(curr_proposal_id, args, is_already_finished))) { - if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "append_config_meta failed", KR(ret), K_(palf_id), K_(self), K(curr_proposal_id), K(args)); - } - } else if (true == is_already_finished) { - ret = OB_SUCCESS; - } else { - PALF_LOG(INFO, "append_config_meta success, start config change", K(ret), K_(palf_id), K_(self), - K(curr_proposal_id), K(args)); - ms_ack_list_.reset(); - (void) set_resend_log_info_(); - state_ = ConfigChangeState::CHANGING; - config_version = log_ms_meta_.curr_.config_.config_version_; - running_args_ = args; - need_change_config_bkgd_ = false; - bkgd_config_version_.reset(); - election_leader_epoch_ = election_epoch; - last_submit_config_log_time_us_ = OB_INVALID_TIMESTAMP; - ret = OB_EAGAIN; - } - break; - } - case (ConfigChangeState::CHANGING): { - if (is_reach_majority_()) { - (void) after_config_log_majority_(proposal_id, config_version); - state_ = INIT; - ms_ack_list_.reset(); - resend_config_version_ = log_ms_meta_.curr_.config_.config_version_; - last_submit_config_log_time_us_ = OB_INVALID_TIMESTAMP; - ret = OB_SUCCESS; - } else if (need_resend_config_log_()) { - if (OB_SUCC(submit_config_log_(alive_paxos_memberlist_, curr_proposal_id, - reconfig_barrier_.prev_log_proposal_id_, reconfig_barrier_.prev_lsn_, - reconfig_barrier_.prev_mode_pid_, log_ms_meta_))) { - PALF_LOG(INFO, "submit_config_log success", KR(ret), K_(palf_id), K_(self)); - ret = OB_EAGAIN; - } else if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "submit_config_log failed", KR(ret), K_(palf_id), K_(self)); - } - } else { - ret = OB_EAGAIN; - } - break; - } - default: - { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "invalid ConfigChangeState", K_(palf_id), K_(self), K(state_)); - break; - } - } - } - return ret; -} - -void LogConfigMgr::after_config_change_timeout(const LogConfigVersion &config_version) -{ - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - } else { - // need config change background - need_change_config_bkgd_ = true; - bkgd_config_version_ = config_version; - } -} - -bool LogConfigMgr::is_leader_for_config_change_(const LogConfigChangeType &type, - const int64_t proposal_id, - const int64_t election_epoch) const -{ - bool bool_ret = false; - bool is_leader = false; - // 1. PALF leader state check - // UPGRADE should be executed in leader active state - // DEGRADE could be executed in leader active/reconfirm state - if (LogConfigChangeType::DEGRADE_ACCEPTOR_TO_LEARNER == type) { - is_leader = state_mgr_->is_leader_active() || - (state_mgr_->is_leader_reconfirm() && reconfirm_->can_do_degrade()); - } else if (LogConfigChangeType::STARTWORKING == type) { - is_leader = state_mgr_->is_leader_reconfirm(); - } else { - is_leader = state_mgr_->is_leader_active(); - } - // 2. palf proposal_id check - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - bool is_pid_changed = (proposal_id != curr_proposal_id); - // 3. election leader epoch check - int ret = OB_SUCCESS; - common::ObRole ele_role; - int64_t ele_epoch = OB_INVALID_TIMESTAMP; - bool is_epoch_changed = true; - if (OB_FAIL(election_->get_role(ele_role, ele_epoch))) { - PALF_LOG(ERROR, "election get_role failed", K(ret), K_(self), K_(palf_id)); - } else if (LEADER != ele_role) { - is_leader = false; - } else { - is_epoch_changed = (election_epoch != ele_epoch); - } - bool_ret = (is_leader && !is_pid_changed && !is_epoch_changed); - if (false == bool_ret) { - PALF_LOG(WARN, "is_leader_for_config_change return false", K(ret), K(is_leader), K(is_pid_changed), - K(is_epoch_changed), K_(self), K_(palf_id), K(type), K(proposal_id), K(curr_proposal_id), - K(election_epoch), K(ele_role), K(ele_epoch)); - } - return bool_ret; -} - -bool LogConfigMgr::is_reach_majority_() const -{ - int64_t curr_replica_num = alive_paxos_replica_num_; - return (ms_ack_list_.get_count() > (curr_replica_num / 2)); -} - -bool LogConfigMgr::need_resend_config_log_() const -{ - const int64_t RESEND_INTERVAL_US = (state_mgr_->is_changing_config_with_arb())? \ - PALF_RESEND_CONFIG_LOG_FOR_ARB_INTERVAL_US: PALF_RESEND_CONFIG_LOG_INTERVAL_US; - int64_t curr_time_us = common::ObTimeUtility::current_time(); - return (last_submit_config_log_time_us_ == OB_INVALID_TIMESTAMP || - curr_time_us - last_submit_config_log_time_us_ > RESEND_INTERVAL_US); -} - -bool LogConfigMgr::need_recheck_init_state_() const -{ - int64_t curr_time_us = common::ObTimeUtility::current_time(); - return (last_check_init_state_time_us_ == OB_INVALID_TIMESTAMP || - curr_time_us - last_check_init_state_time_us_ > 10 * 1000); -} - -// require wlock of PalfHandleImpl -int LogConfigMgr::renew_config_change_barrier() -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = renew_config_change_barrier_(); - } - return ret; -} - -int LogConfigMgr::renew_config_change_barrier_() -{ - int ret = OB_SUCCESS; - int64_t prev_log_proposal_id = INVALID_PROPOSAL_ID; - int64_t prev_mode_pid = INVALID_PROPOSAL_ID; - LSN prev_log_lsn, prev_log_end_lsn; - int64_t unused_log_id; - if (OB_FAIL(sw_->get_last_submit_log_info(prev_log_lsn, prev_log_end_lsn, - unused_log_id, prev_log_proposal_id))) { - PALF_LOG(WARN, "get_last_submit_log_info failed", KR(ret), K_(palf_id), K_(self)); - } else { - checking_barrier_.prev_mode_pid_ = mode_mgr_->get_last_submit_mode_meta().proposal_id_; - checking_barrier_.prev_log_proposal_id_ = prev_log_proposal_id; - checking_barrier_.prev_lsn_ = prev_log_lsn; - checking_barrier_.prev_end_lsn_ = prev_log_end_lsn; - PALF_LOG(INFO, "renew_config_change_barrier_ success", KR(ret), K_(palf_id), K_(self), - K_(checking_barrier)); - } - return ret; -} - -int LogConfigMgr::check_config_change_args_(const LogConfigChangeArgs &args, bool &is_already_finished) const -{ - int ret = OB_SUCCESS; - is_already_finished = false; - const LogConfigVersion &config_version = log_ms_meta_.curr_.config_.config_version_; - if (!args.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (is_remove_member_list(args.type_) && self_ == args.server_.get_server()) { - // can not remove leader, return OB_NOT_ALLOW_REMOVING_LEADER, - // proposer of remove_member cmd will retry later and notify election to switch leader. - ret = OB_NOT_ALLOW_REMOVING_LEADER; - PALF_LOG(WARN, "leader can not remove itself", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (log_ms_meta_.curr_.is_config_change_locked() && is_paxos_member_list_change(args.type_)) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "paxos_member_change is locked, can't do change config now", - KR(ret), K_(palf_id), K_(self), K(args), K_(log_ms_meta), K_(state)); - } else if (need_check_config_version(args.type_) && (args.config_version_ > config_version)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "config version not match", KR(ret), K_(palf_id), K_(self), K(args), K(config_version)); - } else if (OB_FAIL(check_config_change_args_by_type_(args, is_already_finished))) { - PALF_LOG(WARN, "check_config_change_args_by_type_ failed", KR(ret), K_(palf_id), K_(self), - K(args), K(is_already_finished)); - } else if (is_already_finished) { - PALF_LOG(INFO, "reconfiguration is already finished", KR(ret), K_(palf_id), K_(self), - K(args), K_(log_ms_meta)); - // Note: order if vital. we need check is the reconfiguration is already finished, - // then check if the config version equals to current config version. - } else if (need_check_config_version(args.type_) && (args.config_version_ != config_version)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "config version not match", KR(ret), K_(palf_id), K_(self), K(args), K(config_version)); - } else { - // check if reaches majority - LogConfigInfoV2 new_config_info; - common::ObMemberList new_paxos_memberlist; - int64_t new_paxos_replica_num; - GlobalLearnerList unused_list; - // defensive check - // 1. check if reaches majority - // 2. new memberlist contains self - // 1F1A, if command is add_member(C, 5), if add C, length of log_sync_member_list is 2, log_sync_replica_num is 4, - // so reject add_member(C, 5) - // 2F, if command is add_member(C, 5), if add C, length of log_sync_member_list is 3, log_sync_replica_num is 5, - // so allow add_member(C, 5) - // 2F2F1A(ABCDE), remove(D, 5) success, then remove(C, 5). if remove C, length of log_sync_member_list is 2, log_sync_replica_num is 4, - // so reject remove(C, 5) - if (OB_FAIL(generate_new_config_info_(state_mgr_->get_proposal_id(), args, new_config_info))) { - PALF_LOG(WARN, "generate_new_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (OB_FAIL(new_config_info.convert_to_complete_config(new_paxos_memberlist, new_paxos_replica_num, unused_list))) { - } else if (false == new_paxos_memberlist.contains(self_) || false == new_config_info.config_.log_sync_memberlist_.contains(self_)) { - PALF_LOG(ERROR, "new memberlist doesn't contain self", KR(ret), K_(palf_id), K_(self), K(new_config_info), K(new_paxos_memberlist)); - } else if (false == can_memberlist_majority_(new_config_info.config_.log_sync_memberlist_.get_member_number(), - new_config_info.config_.log_sync_replica_num_) || - false == can_memberlist_majority_(new_paxos_memberlist.get_member_number(), new_paxos_replica_num)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "can't change config, memberlist don't reach majority", KR(ret), K_(palf_id), K_(self), K(new_config_info), - K(new_paxos_memberlist), K(new_paxos_replica_num), K(args)); - } else { - // ensure that serialization size of the LogMetaEntry is smaller than MAX_META_ENTRY_SIZE - LogConfigMeta config_meta = log_ms_meta_; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (OB_FAIL(config_meta.generate(curr_proposal_id, config_meta.curr_, new_config_info, - checking_barrier_.prev_log_proposal_id_, checking_barrier_.prev_lsn_, - checking_barrier_.prev_mode_pid_))) { - PALF_LOG(WARN, "generate LogConfigMeta failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (FALSE_IT(config_meta.prev_.config_.learnerlist_.reset())) { - } else if (OB_FAIL(log_engine_->check_config_meta_size(config_meta))) { - PALF_LOG(WARN, "check_config_meta_size failed, too many learners", KR(ret), K_(palf_id), - K_(self), K(args), K(config_meta)); - } - } - } - return ret; -} - -int LogConfigMgr::check_config_change_args_by_type_(const LogConfigChangeArgs &args, - bool &is_already_finished) const -{ - int ret = OB_SUCCESS; - common::ObMemberList curr_member_list; - int64_t curr_replica_num = -1; - is_already_finished = false; - if (OB_FAIL(get_curr_member_list(curr_member_list, curr_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else { - const ObMemberList &log_sync_member_list = log_ms_meta_.curr_.config_.log_sync_memberlist_; - const common::GlobalLearnerList &curr_learner_list = log_ms_meta_.curr_.config_.learnerlist_; - const common::GlobalLearnerList °raded_learnerlist = log_ms_meta_.curr_.config_.degraded_learnerlist_; - const common::ObMember &member = args.server_; - const common::ObMember member_wo_flag = common::ObMember(args.server_.get_server(), \ - args.server_.get_timestamp()); - const int64_t new_replica_num = args.new_replica_num_; - // Note: for reentrancy of SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM, we check if the member - // without the flag is in the log_sync_memberlist - const bool is_in_log_sync_memberlist = log_sync_member_list.contains(member_wo_flag); - const bool is_in_degraded_learnerlist = degraded_learnerlist.contains(member); - const bool is_in_learnerlist = curr_learner_list.contains(member); - const bool is_arb_replica = (log_ms_meta_.curr_.config_.arbitration_member_ == member); - const bool has_arb_replica = (log_ms_meta_.curr_.config_.arbitration_member_.is_valid()); - const bool is_curr_lock_owner_valid = log_ms_meta_.curr_.lock_meta_.is_lock_owner_valid(); - const int64_t curr_lock_owner = log_ms_meta_.curr_.lock_meta_.lock_owner_; - const int64_t curr_lock_type = log_ms_meta_.curr_.lock_meta_.lock_type_; - switch (args.type_) { - case CHANGE_REPLICA_NUM: - { - if (curr_replica_num == new_replica_num && - true == curr_member_list.member_addr_equal(args.curr_member_list_)) { - is_already_finished = true; - PALF_LOG(INFO, "change_replica_num has finished, donot need do again", KR(ret), K_(palf_id), - K_(self), K(args), K_(alive_paxos_memberlist), K_(alive_paxos_replica_num)); - } else if (false == curr_member_list.member_addr_equal(args.curr_member_list_) - || curr_replica_num != args.curr_replica_num_) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "pre-check failed, cannot change_replica_num", KR(ret), K_(palf_id), K_(self), - K(args), K(curr_member_list), K(curr_replica_num)); - } - break; - } - case ADD_MEMBER: - case ADD_MEMBER_AND_NUM: - { - if (is_in_learnerlist || is_in_degraded_learnerlist || is_arb_replica) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is learner/arbitration member, can not add_member/replace_member", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), - K(is_in_learnerlist), K(is_in_degraded_learnerlist), K(is_arb_replica), K(member)); - } else if (is_in_log_sync_memberlist) { - if (args.type_ == ADD_MEMBER_AND_NUM || new_replica_num == curr_replica_num) { - // config change has finished successfully, do not need change again - is_already_finished = true; - if (palf_reach_time_interval(100 * 1000, check_config_print_time_)) { - PALF_LOG(INFO, "member already exists, don't need add_member/replace_member", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(INFO, "member already exists, but new_replica_num not equal to curr val", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } - } - break; - } - case REMOVE_MEMBER: - case REMOVE_MEMBER_AND_NUM: - { - if (is_in_learnerlist || is_arb_replica) { - // Members in degraded_learnerlist are allowed to be removed. - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is learner/arbitration member, can not remove_member/replace_member", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), - K(is_in_learnerlist), K(is_arb_replica), K(member)); - } else if (!is_in_log_sync_memberlist) { - if (is_in_degraded_learnerlist) { - PALF_LOG(INFO, "member to be removed is a degraded learner", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num)); - } else if (args.type_ == REMOVE_MEMBER_AND_NUM || - new_replica_num == curr_replica_num || - (true == has_arb_replica && - 0 == degraded_learnerlist.get_member_number() && - log_sync_member_list.get_member_number() * 2 == new_replica_num)) { - // config change has finished successfully, do not need change again - // Note: 2F1A, B has been degraded, permanent offline B, i.e. remove(B, 2). - // Previous remove(B, 2) has been executed successfully, another remove(B, 2) will - // be identified as invalid, because curr_replica_num is 1. For reentrany of the - // remove_member interface, we just return OB_SUCCESS; - is_already_finished = true; - PALF_LOG(INFO, "member is already removed, don't need remove_member", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(INFO, "member is already removed, but new_replica_num not equal to curr val", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } - } - break; - } - case ADD_LEARNER: - { - if (is_in_log_sync_memberlist || is_in_degraded_learnerlist || is_arb_replica) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is already in memberlist, can not add_learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if (is_in_learnerlist) { - // config change has finished successfully, do not need change again - is_already_finished = true; - PALF_LOG(INFO, "learner already exists, don't need add_learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } - break; - } - case REMOVE_LEARNER: - { - if (is_in_log_sync_memberlist || is_in_degraded_learnerlist || is_arb_replica) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is already in memberlist, can not remove_learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if (!is_in_learnerlist) { - // config change has finished successfully, do not need change again - is_already_finished = true; - PALF_LOG(INFO, "learner don't exist, don't need remove_learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } - break; - } - case SWITCH_LEARNER_TO_ACCEPTOR: - case SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM: - case UPGRADE_LEARNER_TO_ACCEPTOR: - { - if ((is_in_degraded_learnerlist || is_in_learnerlist) && is_in_log_sync_memberlist) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "server is both in memberlist and in learnerlist", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if ((is_in_learnerlist || is_in_degraded_learnerlist) && !is_in_log_sync_memberlist) { - if (UPGRADE_LEARNER_TO_ACCEPTOR == args.type_ && true == is_in_learnerlist) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "can not upgrade a normal learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } else if (UPGRADE_LEARNER_TO_ACCEPTOR != args.type_ && true == is_in_degraded_learnerlist) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "can not switch a degraded learner to member", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } - } else if (!is_in_learnerlist && !is_in_degraded_learnerlist && is_in_log_sync_memberlist) { - if (args.type_ != SWITCH_LEARNER_TO_ACCEPTOR || new_replica_num == curr_replica_num) { - is_already_finished = true; - PALF_LOG(INFO, "learner_to_acceptor is already finished", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(INFO, "member already exists, but new_replica_num not equal to curr val", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is neither in memberlist nor in learnerlist", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } - break; - } - case SWITCH_ACCEPTOR_TO_LEARNER: - case DEGRADE_ACCEPTOR_TO_LEARNER: - { - if (false == has_arb_replica && args.type_ == DEGRADE_ACCEPTOR_TO_LEARNER) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "do not allow to degrade member without arbitration member", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if ((is_in_learnerlist || is_in_degraded_learnerlist) && is_in_log_sync_memberlist) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "server is both in memberlist and in learnerlist", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if ((is_in_learnerlist || is_in_degraded_learnerlist) && false == is_in_log_sync_memberlist) { - if (args.type_ == DEGRADE_ACCEPTOR_TO_LEARNER && true == is_in_learnerlist) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is a learner, can't degrade", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else if (args.type_ == SWITCH_ACCEPTOR_TO_LEARNER && true == is_in_degraded_learnerlist) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server has been degraded, can't switch to learner", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else { - if (args.type_ == DEGRADE_ACCEPTOR_TO_LEARNER || new_replica_num == curr_replica_num) { - is_already_finished = true; - PALF_LOG(INFO, "acceptor_to_learner is already finished", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(INFO, "member is already removed, but new_replica_num not equal to curr val", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(alive_paxos_replica_num)); - } - } - } else if (!is_in_degraded_learnerlist && is_in_log_sync_memberlist) { - // degrade operation can only be done when there is arbitration replica in paxos group - if (args.type_ == DEGRADE_ACCEPTOR_TO_LEARNER && false == has_arb_replica) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "arb member is invalid, can't degrade", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is neither in memberlist nor in learnerlist", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member)); - } - break; - } - case STARTWORKING: - { - break; - } - case FORCE_SINGLE_MEMBER: - { - break; - } - case FORCE_SET_MEMBER_LIST: - { - break; - } - case TRY_LOCK_CONFIG_CHANGE: - { - if (args.lock_owner_ == curr_lock_owner) { - if (ConfigChangeLockType::LOCK_PAXOS_MEMBER_CHANGE == curr_lock_type) { - is_already_finished = true; - } else { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "lock state not match", KR(ret), K_(self), K_(log_ms_meta), K(args), K(curr_lock_type)); - } - } else if (args.lock_owner_ > curr_lock_owner) { - if (ConfigChangeLockType::LOCK_NOTHING == curr_lock_type) { - //go on locking - } else { - ret = OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT; - PALF_LOG(WARN, "config change lock conflict", KR(ret), K_(self), K_(log_ms_meta), K(args)); - } - } else if (args.lock_owner_ < curr_lock_owner) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "config change lock state not match", KR(ret), K_(self), K_(log_ms_meta), K(args)); - } else {/*do nothing*/} - break; - } - case UNLOCK_CONFIG_CHANGE: - { - if (args.lock_owner_ == curr_lock_owner) { - if (ConfigChangeLockType::LOCK_PAXOS_MEMBER_CHANGE == curr_lock_type) { - //go on unlock - } else if (ConfigChangeLockType::LOCK_NOTHING == curr_lock_type) { - is_already_finished = true; - } else { - ret = OB_NOT_SUPPORTED; - PALF_LOG(ERROR, "not supported lock type", KR(ret), K(curr_lock_type), K_(self), K_(log_ms_meta), K(args)); - } - } else { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "config change lock state not match", KR(ret), K_(self), K_(log_ms_meta), K(args)); - } - break; - } - case REPLACE_LEARNERS: - { - bool all_added_in_learnerlist = true; - bool all_removed_not_in_learnerlist = true; - for (int i = 0; OB_SUCC(ret) && i < args.added_list_.get_member_number(); i++) { - common::ObMember member; - if (OB_FAIL(args.added_list_.get_member_by_index(i, member))) { - PALF_LOG(WARN, "get_member_by_index failed", KR(ret), K_(palf_id), K_(self), K(member), K(args)); - } else if (true == curr_member_list.contains(member.get_server())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "server is already in memberlist, can not replace_learners", KR(ret), - K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } else if (false == curr_learner_list.contains(member)) { - all_added_in_learnerlist = false; - break; - } - } - for (int i = 0; OB_SUCC(ret) && i < args.removed_list_.get_member_number(); i++) { - common::ObMember member; - if (OB_FAIL(args.removed_list_.get_member_by_index(i, member))) { - PALF_LOG(WARN, "get_member_by_index failed", KR(ret), K_(palf_id), K_(self), K(member), K(args)); - } else if (true == curr_learner_list.contains(member)) { - all_removed_not_in_learnerlist = false; - break; - } - } - is_already_finished = OB_SUCC(ret) && all_added_in_learnerlist && all_removed_not_in_learnerlist; - if (is_already_finished) { - PALF_LOG(INFO, "replace_learners is already finished", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } - break; - } - default: - { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "unknown LogConfigChangeType", KR(ret), K_(palf_id), K_(self), K(args.type_)); - break; - } - } - } - return ret; -} - -bool LogConfigMgr::can_memberlist_majority_(const int64_t new_member_list_len, const int64_t new_replica_num) const -{ - // NB: new_replica_num is not the number of paxos member after config changing, - // it means that after config changing, availability of this paxos group should be like - // 'new_replica_num' member group, even if paxos member number is smaller than 'new_replica_num'. - // For example, 'member_list' is (A, B, C, D) and 'replica_num' is 4, then request remove(D, 4) arrives, - // after removing D, 'member_list' is (A, B, C) and 'replica_num' is still 4 (rather than 3). A new log will - // be committed only when it has been flushed by 3 replicas at least. Even if there are only 3 replicas in - // this paxos group, its availibility is equal to 4 replicas paxos group. - // constraints: - // 1. replica_num >= len(member_list) - // 2. len(member_list) >= replica_num / 2 + 1 - bool bool_ret = false; - if (new_member_list_len > new_replica_num) { - PALF_LOG_RET(WARN, OB_INVALID_ARGUMENT, "replica_num too small", K_(palf_id), K_(self), K(new_replica_num), K(new_member_list_len)); - } else if (new_member_list_len < (new_replica_num / 2 + 1)) { - PALF_LOG_RET(WARN, OB_INVALID_ARGUMENT, "replica_num too large", K_(palf_id), K_(self), K(new_replica_num), K(new_member_list_len)); - } else { - bool_ret = true; - } - return bool_ret; -} - -int LogConfigMgr::check_args_and_generate_config(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - bool &is_already_finished, - LogConfigInfoV2 &new_config_info) const -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (need_exec_on_leader_(args.type_) - && false == is_leader_for_config_change_(args.type_, proposal_id, election_epoch)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "is_leader_for_config_change_ return false", K(ret), K_(palf_id), K_(self), - K(args.type_), K(proposal_id), K(election_epoch)); - } else if (OB_FAIL(check_config_change_args_(args, is_already_finished))) { - PALF_LOG(WARN, "check_config_change_args_ failed", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } else if (is_already_finished) { - } else if (OB_FAIL(generate_new_config_info_(proposal_id, args, new_config_info))) { - PALF_LOG(WARN, "generate_new_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else { - PALF_LOG(INFO, "check_args_and_generate_config success", K(ret), K_(palf_id), K_(self), K(args), K(is_already_finished), K(new_config_info)); - } - return ret; -} - -int LogConfigMgr::sync_meta_for_arb_election_leader() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - const bool is_arb_replica = state_mgr_->is_arb_replica(); - common::ObAddr ele_leader; - int64_t leader_epoch; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == is_arb_replica) { - } else if (false == palf_reach_time_interval(500 * 1000, last_sync_meta_for_arb_election_leader_time_us_)) { - // skip - } else if (OB_SUCCESS != (tmp_ret = election_->get_current_leader_likely(ele_leader, leader_epoch))) { - // skip - } else if (self_ == ele_leader) { - const int64_t proposal_id = state_mgr_->get_proposal_id(); - for (int i = 0; OB_SUCC(ret) && i < alive_paxos_memberlist_.get_member_number(); i++) { - common::ObMember member; - if (OB_FAIL(alive_paxos_memberlist_.get_member_by_index(i, member))) { - PALF_LOG(WARN, "get_member_by_index failed", KR(ret), K_(palf_id), K_(self), K(i), K(alive_paxos_memberlist_)); - } else if (self_ == member.get_server()) { - // skip - } else if (OB_FAIL(pre_sync_config_log_and_mode_meta_(member, proposal_id, is_arb_replica))) { - PALF_LOG(WARN, "pre_sync_config_log_and_mode_meta_ failed", KR(ret), K_(palf_id), K_(self), K(member), K(proposal_id)); - } - } - if (OB_SUCC(ret)) { - PALF_LOG(INFO, "sync_meta_for_arb_election_leader success", KR(ret), K_(palf_id), K_(self), - K_(alive_paxos_memberlist), K(proposal_id), K_(log_ms_meta)); - } - } - return ret; -} - -int LogConfigMgr::pre_sync_config_log_and_mode_meta(const common::ObMember &server, - const int64_t proposal_id) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(pre_sync_config_log_and_mode_meta_(server, proposal_id, false))) { - PALF_LOG(WARN, "pre_sync_config_log_and_mode_meta_ failed", KR(ret), K_(palf_id), K_(self), K(server), K(proposal_id)); - } - return ret; -} - -int LogConfigMgr::pre_sync_config_log_and_mode_meta_(const common::ObMember &server, - const int64_t proposal_id, - const bool is_arb_replica) -{ - int ret = OB_SUCCESS; - common::ObMemberList member_list; - // the log barrier must not be used in normal replica because of compatibility - const int64_t prev_log_proposal_id = (is_arb_replica)? log_ms_meta_.prev_log_proposal_id_: \ - reconfig_barrier_.prev_log_proposal_id_; - const LSN prev_lsn = (is_arb_replica)? log_ms_meta_.prev_lsn_: reconfig_barrier_.prev_lsn_; - const int64_t prev_mode_pid = (is_arb_replica)? log_ms_meta_.prev_mode_pid_: \ - reconfig_barrier_.prev_mode_pid_; - if (false == server.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (CHANGING == state_ && false == is_arb_replica) { - ret = OB_EAGAIN; - } else if (CHANGING == state_ && true == is_arb_replica) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "arbitration replica is not allowed to change config", K(ret), K_(palf_id), K_(self), K_(state)); - } else if (OB_FAIL(mode_mgr_->submit_fetch_mode_meta_resp(server.get_server(), proposal_id, prev_mode_pid))) { - PALF_LOG(WARN, "submit_fetch_mode_meta_resp failed", K(ret), K_(palf_id), K_(self), K(proposal_id)); - } else if (FALSE_IT(member_list.add_member(server))) { - } else if (OB_FAIL(log_engine_->submit_change_config_meta_req(member_list, proposal_id, - prev_log_proposal_id, prev_lsn, prev_mode_pid, log_ms_meta_))) { - PALF_LOG(WARN, "submit_change_config_meta_req failed", KR(ret), K_(palf_id), K_(self), K(proposal_id), K(server)); - } - return ret; -} - -// Require caller hold wlock of ObLogService -int LogConfigMgr::append_config_meta_(const int64_t curr_proposal_id, - const LogConfigChangeArgs &args, - bool &is_already_finished) -{ - int ret = OB_SUCCESS; - LogConfigInfoV2 new_config_info; - bool has_arb_member = false; - bool unused_bool; - if (INVALID_PROPOSAL_ID == curr_proposal_id || !args.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(args), K(curr_proposal_id)); - } else if (need_exec_on_leader_(args.type_) && - !(state_mgr_->get_leader() == self_ && state_mgr_->get_proposal_id() == curr_proposal_id)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "leader has switched during config changing", KR(ret), K_(palf_id), K_(self), - "role", state_mgr_->get_role(), K(curr_proposal_id), "proposal_id", state_mgr_->get_proposal_id(), - "leader", state_mgr_->get_leader()); - } else if (false == need_exec_on_leader_(args.type_) && !(state_mgr_->get_proposal_id() == curr_proposal_id)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "proposal_id has switched during config changing", KR(ret), K_(palf_id), K_(self), - "role", state_mgr_->get_role(), K(curr_proposal_id), "proposal_id", state_mgr_->get_proposal_id(), - "leader", state_mgr_->get_leader()); - } else if (OB_FAIL(check_config_change_args_(args, is_already_finished))) { - PALF_LOG(WARN, "check_config_change_args_ failed", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } else if (is_already_finished) { - PALF_LOG(INFO, "config_change is already success", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(args)); - } else if (OB_FAIL(generate_new_config_info_(curr_proposal_id, args, new_config_info))) { - PALF_LOG(WARN, "generate_new_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - // new_member_list contains arb member, stop appending logs and check log barrier - } else if (FALSE_IT(has_arb_member = new_config_info.config_.arbitration_member_.is_valid())) { - } else if (OB_FAIL(update_election_meta_(new_config_info))) { - if (OB_OP_NOT_ALLOW == ret) { - ret = OB_EAGAIN; - } else { - PALF_LOG(WARN, "update_election_meta_ failed", KR(ret), K_(palf_id), K_(self), K(new_config_info), K_(log_ms_meta)); - } - } else if (false == has_arb_member && OB_FAIL(renew_config_change_barrier_())) { - PALF_LOG(WARN, "renew_config_change_barrier_ failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(log_ms_meta_.generate(curr_proposal_id, log_ms_meta_.curr_, new_config_info, - checking_barrier_.prev_log_proposal_id_, checking_barrier_.prev_lsn_, - checking_barrier_.prev_mode_pid_))) { - PALF_LOG(WARN, "generate LogConfigMeta failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (OB_FAIL(append_config_info_(new_config_info))) { - PALF_LOG(WARN, "append_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(new_config_info)); - } else { - // To reduce the serialized size of LogMeta to 4KB, we reset the previous learner_list. - // It's safe because the previous learner_list is useless in our design. - log_ms_meta_.prev_.config_.learnerlist_.reset(); - // log_ms_meta_ and reconfig_barrier_ must be updated atomically - reconfig_barrier_ = checking_barrier_; - // Note: can not generate committed_end_lsn while changing configs with arb. - // The reason is described in LogSlidingWindow::gen_committed_end_lsn_. - if (false == has_arb_member) { - (void) update_match_lsn_map_(args, new_config_info); - } - PALF_LOG(INFO, "append_config_meta_ success", KR(ret), K_(palf_id), K_(self), K(curr_proposal_id), - K(args), K(new_config_info), K_(log_ms_meta)); - } - return ret; -} - -int LogConfigMgr::update_match_lsn_map_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info) -{ - int ret = OB_SUCCESS; - ObMemberList added_memberlist; - ObMemberList removed_memberlist; - if (is_add_log_sync_member_list(args.type_) && OB_FAIL(added_memberlist.add_member(args.server_))) { - PALF_LOG(WARN, "add_member failed", K(ret), K_(palf_id), K_(self), K(added_memberlist), K(args)); - } else if (is_remove_log_sync_member_list(args.type_) && OB_FAIL(removed_memberlist.add_member(args.server_))) { - PALF_LOG(WARN, "add_member failed", K(ret), K_(palf_id), K_(self), K(removed_memberlist), K(args)); - } else if (FORCE_SET_MEMBER_LIST == args.type_ && OB_FAIL(removed_memberlist.deep_copy(args.removed_list_))) { - PALF_LOG(WARN, "failed to get removed members", K(ret), K_(palf_id), K_(self), K(removed_memberlist), K(args)); - } - if (OB_SUCC(ret) && OB_FAIL(sw_->config_change_update_match_lsn_map(added_memberlist, - removed_memberlist, new_config_info.config_.log_sync_memberlist_, - new_config_info.config_.log_sync_replica_num_))) { - PALF_LOG(WARN, "config_change_update_match_lsn_map failed", K(ret), K_(palf_id), K_(self), K(added_memberlist), K(removed_memberlist)); - } - return ret; -} - -// caller hold lock_ -int LogConfigMgr::append_config_info_(const LogConfigInfoV2 &config_info) -{ - int ret = OB_SUCCESS; - common::ObMemberList alive_paxos_memberlist; - int64_t alive_paxos_replica_num; - GlobalLearnerList all_learners; - if (OB_FAIL(config_info.convert_to_complete_config(alive_paxos_memberlist, \ - alive_paxos_replica_num, all_learners))) { - PALF_LOG(WARN, "convert_to_complete_config failed", K(ret), K_(palf_id), K(config_info)); - } else if (OB_FAIL(all_learnerlist_.deep_copy(all_learners))) { - PALF_LOG(WARN, "deep_copy failed", K(ret), K_(palf_id), K(all_learners)); - } else { - alive_paxos_memberlist_ = alive_paxos_memberlist; - alive_paxos_replica_num_ = alive_paxos_replica_num; - } - return ret; -} - -int LogConfigMgr::set_resend_log_info_() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - resend_config_version_.reset(); - resend_log_list_.reset(); - (void) all_learnerlist_.deep_copy_to(resend_log_list_); - const int64_t member_number = alive_paxos_memberlist_.get_member_number(); - for (int64_t idx = 0; idx < member_number && OB_SUCCESS == tmp_ret; ++idx) { - common::ObAddr server; - if (OB_SUCCESS != (tmp_ret = (alive_paxos_memberlist_.get_server_by_index(idx, server)))) { - PALF_LOG(WARN, "get_server_by_index failed", K(tmp_ret), K(idx)); - } else if (server == self_) { - } else if (OB_SUCCESS != (tmp_ret = (resend_log_list_.add_learner(ObMember(server, 1))))) { - PALF_LOG(WARN, "add_learner failed", K(ret), K(server)); - } - } - return ret; -} - -// require caller hold rlock -// Before calling this func, must call check_config_change_args to ensure LogConfigChangeArgs can -// be applied to current ConfigMeta safely. -int LogConfigMgr::generate_new_config_info_(const int64_t proposal_id, - const LogConfigChangeArgs &args, - LogConfigInfoV2 &new_config_info) const -{ - int ret = OB_SUCCESS; - const LogConfigChangeType cc_type = args.type_; - const common::ObMember member = args.server_; - new_config_info = log_ms_meta_.curr_; - if (INVALID_PROPOSAL_ID == proposal_id || !args.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(new_config_info.config_.config_version_.inc_update_version(proposal_id))) { - PALF_LOG(WARN, "generate config_version failed", KR(ret), K_(palf_id), K_(self), K(new_config_info), K(proposal_id)); - } else if (STARTWORKING == cc_type) { - // pass - } else { - // change replcia num - int64_t new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_; - if (is_may_change_replica_num(cc_type)) { - const bool is_remove_degraded_learner = is_remove_log_sync_member_list(args.type_) && - new_config_info.config_.degraded_learnerlist_.contains(member); - if (is_use_replica_num_args(cc_type)) { - // Note: consider the number of degraded learners when setting replica_num, - // and note that current number of degraded learners is not the eventual value - // if the request is going to remove a degraded learner. - int64_t degraded_cnt = new_config_info.config_.degraded_learnerlist_.get_member_number(); - degraded_cnt = (is_remove_degraded_learner)? degraded_cnt - 1: degraded_cnt; - new_log_sync_replica_num = args.new_replica_num_ - degraded_cnt; - } else if (is_add_log_sync_member_list(cc_type)) { - new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_ + 1; - } else if (is_remove_log_sync_member_list(cc_type) && true == is_remove_degraded_learner) { - new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_; - } else if (is_remove_log_sync_member_list(cc_type) && false == is_remove_degraded_learner) { - new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_ - 1; - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected config change type", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - if (OB_SUCC(ret) && is_remove_degraded_learner && - (new_config_info.config_.log_sync_memberlist_.get_member_number() * 2 == new_log_sync_replica_num)) { - // Note: 2F1A, B has been degraded, permanent offline B, i.e. remove(B, 2). - // Remaining F members (A) in member_list do not reach majority of replica_num(2), - // so we just remove it and do not update log_sync_replica_num_. - new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_; - } - } - // memberlist add, update replica number - if (OB_SUCC(ret) && is_add_member_list(cc_type)) { - // update log_sync_member_list or arb_member - if (is_add_log_sync_member_list(args.type_)) { - // Note: all members in log_sync_member_list must not be migrating status - common::ObMember added_log_sync_member = member; - added_log_sync_member.reset_migrating(); - if (OB_FAIL(new_config_info.config_.log_sync_memberlist_.add_member(added_log_sync_member))) { - PALF_LOG(WARN, "new_member_list add_member failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else { - new_config_info.config_.arbitration_member_ = member; - } - } - // memberlist remove, update replica number - if (OB_SUCC(ret) && is_remove_member_list(cc_type)) { - // update log_sync_member_list or arb_member - if (is_remove_log_sync_member_list(args.type_)) { - if (new_config_info.config_.log_sync_memberlist_.contains(member)) { - if (OB_FAIL(new_config_info.config_.log_sync_memberlist_.remove_server(member.get_server()))) { - PALF_LOG(WARN, "remove member failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else if (new_config_info.config_.degraded_learnerlist_.contains(member)) { - if (OB_FAIL(new_config_info.config_.degraded_learnerlist_.remove_learner(member))) { - PALF_LOG(WARN, "new_member_list remove member failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "member to be removed does not exist", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else { - new_config_info.config_.arbitration_member_.reset(); - } - } - // learnerlist add - if (OB_SUCC(ret) && is_add_learner_list(cc_type)) { - if (DEGRADE_ACCEPTOR_TO_LEARNER == cc_type) { - if (OB_FAIL(new_config_info.config_.degraded_learnerlist_.add_learner(member))) { - PALF_LOG(WARN, "new_learner_list add_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else if (is_use_added_list(cc_type)) { - for (int i = 0; OB_SUCC(ret) && i < args.added_list_.get_member_number(); i++) { - common::ObMember added_learner; - if (OB_FAIL(args.added_list_.get_member_by_index(i, added_learner))) { - } else if (OB_FAIL(new_config_info.config_.learnerlist_.add_learner(added_learner)) && OB_ENTRY_EXIST != ret) { - PALF_LOG(WARN, "new_learner_list add_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } else { - ret = OB_SUCCESS; - } - } - } else { - if (OB_FAIL(new_config_info.config_.learnerlist_.add_learner(member))) { - PALF_LOG(WARN, "new_learner_list add_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } - } - // learnerlist remove - if (OB_SUCC(ret) && is_remove_learner_list(cc_type)) { - if (UPGRADE_LEARNER_TO_ACCEPTOR == cc_type) { - if (OB_FAIL(new_config_info.config_.degraded_learnerlist_.remove_learner(member))) { - PALF_LOG(WARN, "new_learner_list add_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } else if (is_use_removed_list(cc_type)) { - for (int i = 0; OB_SUCC(ret) && i < args.removed_list_.get_member_number(); i++) { - common::ObMember removed_learner; - if (OB_FAIL(args.removed_list_.get_member_by_index(i, removed_learner))) { - } else if (OB_FAIL(new_config_info.config_.learnerlist_.remove_learner(removed_learner)) && - OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(WARN, "new_learner_list remove_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } else { - ret = OB_SUCCESS; - } - } - } else { - if (OB_FAIL(new_config_info.config_.learnerlist_.remove_learner(member))) { - PALF_LOG(WARN, "new_learner_list add_learner failed", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } - } - - // try lock config_change - if (OB_SUCC(ret) && is_try_lock_config_change(cc_type)) { - if (OB_FAIL(new_config_info.lock_meta_.generate(args.lock_owner_, args.lock_type_))) { - PALF_LOG(WARN, "failed to generate lock_meta", KR(ret), K_(palf_id), K_(self), K(args), K(new_config_info)); - } - } - - //unlock config_change - if (OB_SUCC(ret) && is_unlock_config_change(cc_type)) { - new_config_info.lock_meta_.unlock(); - } - - // generate log_sync_replica_num_ - if (OB_SUCC(ret)) { - new_config_info.config_.log_sync_replica_num_ = new_log_sync_replica_num; - } - // Note: order is vital - if (OB_SUCC(ret) && FORCE_SINGLE_MEMBER == cc_type) { - // force set single member - new_config_info.config_.log_sync_memberlist_.reset(); - new_config_info.config_.degraded_learnerlist_.reset(); - new_config_info.config_.arbitration_member_.reset(); - new_config_info.config_.log_sync_memberlist_.add_member(member); - new_config_info.config_.log_sync_replica_num_ = args.new_replica_num_; - } - if (OB_SUCC(ret) && FORCE_SET_MEMBER_LIST == cc_type) { - // force set member list - new_config_info.config_.log_sync_memberlist_.reset(); - new_config_info.config_.degraded_learnerlist_.reset(); - new_config_info.config_.arbitration_member_.reset(); - new_config_info.config_.log_sync_memberlist_.deep_copy(args.new_member_list_); - new_config_info.config_.log_sync_replica_num_ = args.new_replica_num_; - } - // check if the new_config_info is valid - if (OB_SUCC(ret) && false == new_config_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "generate_new_config_info_ failed", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), - K(proposal_id), K(args), K(new_config_info)); - } - } - return ret; -} - -bool LogConfigMgr::check_need_update_memberlist_without_lock_(const LogConfigVersion &config_version) const -{ - return persistent_config_version_ < config_version; -} - -int LogConfigMgr::update_election_meta_(const ObMemberList &member_list, - const LogConfigVersion &config_version, - const int64_t new_replica_num) -{ - int ret = OB_SUCCESS; - election::MemberList new_election_member_list; - ObArray addr_list; - if (OB_FAIL(member_list.get_addr_array(addr_list))) { - PALF_LOG(WARN, "get addr list from member list failed", KR(ret), K_(palf_id), K_(self), K(member_list)); - } else if (OB_FAIL(new_election_member_list.set_new_member_list(addr_list, config_version, new_replica_num))) { - PALF_LOG(WARN, "create new memberlist failed", - KR(ret), K_(palf_id), K_(self), K(member_list), K(config_version), K(new_replica_num)); - } else { - ret = election_->set_memberlist(new_election_member_list); - if (OB_SUCC(ret)) { - PALF_LOG(INFO, "update_election_meta_ success", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(new_election_member_list)); - } - } - return ret; -} - -int LogConfigMgr::update_election_meta_(const LogConfigInfoV2 &info) -{ - int ret = OB_SUCCESS; - common::ObMemberList memberlist; - int64_t replica_num = 0; - GlobalLearnerList unused_list; - if (OB_FAIL(info.convert_to_complete_config(memberlist, replica_num, unused_list))) { - PALF_LOG(WARN, "convert_to_complete_config failed", K(ret), K_(palf_id), K(info)); - } else { - ret = update_election_meta_(memberlist, info.config_.config_version_, replica_num); - } - return ret; -} - -int LogConfigMgr::confirm_start_working_log(const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - const bool has_finished = (ConfigChangeState::INIT == state_ && config_version.is_valid()) || - (ConfigChangeState::CHANGING == state_ && config_version.is_valid() && log_ms_meta_.curr_.config_.config_version_ > config_version); - common::ObMember dummy_member; - const LogConfigChangeArgs args(dummy_member, 1, STARTWORKING); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret), K_(palf_id), K_(self)); - } else if (is_sw_interrupted_by_degrade_) { - ret = OB_EAGAIN; - PALF_LOG(INFO, "is degrading, skip this startworking", KR(ret), K_(palf_id), K_(self), - K(config_version), K_(state), K(is_sw_interrupted_by_degrade_)); - } else if (has_finished) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "start_working log has reached majority, pass", KR(ret), K_(palf_id), K_(self), - K(config_version), K_(log_ms_meta), K_(state)); - } else if (ConfigChangeState::INIT == state_ && - OB_FAIL(wait_log_barrier_before_start_working_(args))) { - PALF_LOG(WARN, "wait_log_barrier_before_start_working_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (OB_FAIL(change_config_(args, proposal_id, election_epoch, config_version))) { - if (OB_EAGAIN != ret) { - PALF_LOG(INFO, "confirm_start_working_log failed", KR(ret), K_(palf_id), K_(self), K(args)); - } - } else { - PALF_LOG(INFO, "confirm_start_working_log success", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta)); - } - return ret; -} - -int LogConfigMgr::wait_config_log_persistence(const LogConfigVersion &config_version) const -{ - int ret = OB_SUCCESS; - const int64_t timeout_ts = 1 * 1000 * 1000; - const int64_t start_ts = ObTimeUtility::current_time(); - const int64_t SLEEP_INTERVAL_TS = 1 * 1000; - int64_t cost_ts = OB_INVALID_TIMESTAMP; - while (OB_SUCC(ret)) { - cost_ts = ObTimeUtility::current_time() - start_ts; - bool bool_ret = false; - do { - SpinLockGuard guard(lock_); - bool_ret = check_need_update_memberlist_without_lock_(config_version); - } while(0); - if (false == bool_ret){ - break; - } else if (cost_ts >= timeout_ts) { - ret = OB_TIMEOUT; - } else { - ob_usleep(SLEEP_INTERVAL_TS); - } - } - return ret; -} - -void LogConfigMgr::reset_status() -{ - SpinLockGuard guard(lock_); - state_ = ConfigChangeState::INIT; - ms_ack_list_.reset(); - need_change_config_bkgd_ = false; - bkgd_config_version_.reset(); - resend_config_version_.reset(); - resend_log_list_.reset(); - running_args_.reset(); - election_leader_epoch_ = OB_INVALID_TIMESTAMP; - last_submit_config_log_time_us_ = OB_INVALID_TIMESTAMP; - last_check_init_state_time_us_ = OB_INVALID_TIMESTAMP; - start_wait_barrier_time_us_ = OB_INVALID_TIMESTAMP; - last_wait_barrier_time_us_ = OB_INVALID_TIMESTAMP; - last_wait_committed_end_lsn_.reset(); - will_upgrade_ = false; - last_start_upgrade_time_us_ = OB_INVALID_TIMESTAMP; -} - -// leader check barrier condition when config changing -int LogConfigMgr::check_barrier_condition_(const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid) const -{ - int ret = OB_SUCCESS; - LSN unused_lsn; - LSN max_flushed_lsn; - int64_t max_flushed_log_pid = INVALID_PROPOSAL_ID; - int64_t max_flushed_mode_pid = INVALID_PROPOSAL_ID; - if (false == prev_lsn.is_valid() || INVALID_PROPOSAL_ID == prev_mode_pid) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(prev_log_proposal_id), - K(prev_lsn), K(prev_mode_pid)); - } else if (OB_FAIL(sw_->get_max_flushed_log_info(max_flushed_lsn, unused_lsn, max_flushed_log_pid))) { - PALF_LOG(WARN, "get_max_flushed_log_info failed", KR(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(max_flushed_mode_pid = mode_mgr_->get_accepted_mode_meta().proposal_id_)) { - } else { - ret = ((INVALID_PROPOSAL_ID == prev_log_proposal_id || max_flushed_log_pid >= prev_log_proposal_id) && - max_flushed_lsn >= prev_lsn && - max_flushed_mode_pid != INVALID_PROPOSAL_ID && - max_flushed_mode_pid >= prev_mode_pid)? OB_SUCCESS: OB_EAGAIN; - if (OB_EAGAIN == ret && palf_reach_time_interval(500 * 1000, barrier_print_log_time_)) { - PALF_LOG(INFO, "check_barrier_condition_ eagain", KR(ret), K_(palf_id), K_(self), - K(max_flushed_log_pid), K(max_flushed_lsn), K(prev_log_proposal_id), K(prev_lsn)); - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = log_engine_->submit_purge_throttling_task(PurgeThrottlingType::PURGE_BY_CHECK_BARRIER_CONDITION))) { - LOG_WARN_RET(tmp_ret, "submit_purge_throttling_task", K_(palf_id), K_(self)); - } - } - } - return ret; -} - -// require caller hold rlock -int LogConfigMgr::submit_config_log_(const common::ObMemberList &paxos_member_list, - const int64_t proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid, - const LogConfigMeta &config_meta) -{ - int ret = OB_SUCCESS; - ObMemberList dst_member_list = paxos_member_list; - const common::GlobalLearnerList &learner_list = all_learnerlist_; - const bool need_skip_log_barrier = mode_mgr_->need_skip_log_barrier(); - if (false == paxos_member_list.is_valid() || INVALID_PROPOSAL_ID == proposal_id || false == prev_lsn.is_valid() || - INVALID_PROPOSAL_ID == prev_mode_pid || false == config_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(prev_mode_pid), K(config_meta)); - } else if (need_skip_log_barrier == false && - OB_FAIL(check_barrier_condition_(prev_log_proposal_id, prev_lsn, prev_mode_pid))) { - // barrier condition may don't match, need retry - } else { - (void) dst_member_list.remove_server(self_); - const LogConfigVersion config_version = config_meta.curr_.config_.config_version_; - FlushMetaCbCtx cb_ctx; - cb_ctx.type_ = MetaType::CHANGE_CONFIG_META; - cb_ctx.proposal_id_ = proposal_id; - cb_ctx.config_version_ = config_version; - if (OB_FAIL(log_engine_->submit_flush_change_config_meta_task(cb_ctx, config_meta))) { - PALF_LOG(WARN, "submit_flush_change_config_meta_task failed", KR(ret), K_(palf_id), K_(self), K(proposal_id), - K(prev_log_proposal_id), K(prev_lsn), K(config_meta)); - } else if (dst_member_list.is_valid() && - OB_FAIL(log_engine_->submit_change_config_meta_req(dst_member_list, proposal_id, prev_log_proposal_id, - prev_lsn, prev_mode_pid, config_meta))) { - PALF_LOG(WARN, "submit_change_config_meta_req failed, to member", KR(ret), K_(palf_id), K_(self), K(proposal_id), - K(prev_log_proposal_id), K(prev_lsn), K(prev_mode_pid), K(config_meta)); - } else if (learner_list.is_valid() && - OB_FAIL(log_engine_->submit_change_config_meta_req(learner_list, proposal_id, prev_log_proposal_id, - prev_lsn, prev_mode_pid, config_meta))) { - PALF_LOG(WARN, "submit_change_config_meta_req failed, to learner", KR(ret), K_(palf_id), K_(self), K(proposal_id), - K(prev_log_proposal_id), K(prev_lsn), K(prev_mode_pid), K(config_meta)); - } else { - last_submit_config_log_time_us_ = common::ObTimeUtility::current_time(); - PALF_LOG(INFO, "submit_config_log success", KR(ret), K_(palf_id), K_(self), K(dst_member_list), K(proposal_id), - K(prev_log_proposal_id), K(prev_lsn), K(prev_mode_pid), K(config_meta)); - } - } - return ret; -} - -bool LogConfigMgr::can_receive_config_log(const common::ObAddr &leader, const LogConfigMeta &meta) const -{ - bool bool_ret = false; - const LogConfigVersion &config_version = meta.curr_.config_.config_version_; - int ret = OB_SUCCESS; - if (IS_NOT_INIT || !config_version.is_valid()) { - } else { - bool_ret = config_version > log_ms_meta_.curr_.config_.config_version_; - if ((persistent_config_version_ == config_version) && - OB_FAIL(log_engine_->submit_change_config_meta_resp(leader, meta.proposal_id_, config_version))) { - PALF_LOG(WARN, "submit_change_config_meta_resp fail", K(ret), K_(palf_id), K_(self), K(leader), K(meta)); - } - } - PALF_LOG(INFO, "can_receive_config_log", K(bool_ret), K_(palf_id), K_(self), K(leader), K(meta), K_(log_ms_meta)); - return bool_ret; -} - -int LogConfigMgr::receive_config_log(const common::ObAddr &leader, const LogConfigMeta &meta) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (false == meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(meta)); - } else { - FlushMetaCbCtx cb_ctx; - cb_ctx.type_ = MetaType::CHANGE_CONFIG_META; - cb_ctx.proposal_id_ = meta.proposal_id_; - cb_ctx.config_version_ = meta.curr_.config_.config_version_; - // Note: order is vital, submit flush task may fail - if (OB_FAIL(log_engine_->submit_flush_change_config_meta_task(cb_ctx, meta))) { - PALF_LOG(WARN, "LogEngine submit_flush_change_config_meta_task failed", KR(ret), K_(palf_id), K_(self), K(meta)); - } else if (OB_FAIL(update_election_meta_(meta.curr_))) { - PALF_LOG(ERROR, "update_election_meta_ failed", KR(ret), K_(palf_id), K_(self), K(meta)); - } else if (OB_FAIL(append_config_info_(meta.curr_))) { - PALF_LOG(ERROR, "append_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(meta)); - } else { - log_ms_meta_ = meta; - } - } - return ret; -} - -int LogConfigMgr::ack_config_log(const common::ObAddr &sender, - const int64_t proposal_id, - const LogConfigVersion &config_version, - bool &is_majority) -{ - int ret = OB_SUCCESS; - is_majority = false; - SpinLockGuard guard(lock_); - const bool is_in_memberlist = alive_paxos_memberlist_.contains(sender); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (false == sender.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(sender)); - } else if (proposal_id != log_ms_meta_.proposal_id_ || - config_version != log_ms_meta_.curr_.config_.config_version_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "config_version has been changed", KR(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(proposal_id), - K_(state), K(config_version), K(sender)); - } else if (FALSE_IT(resend_log_list_.remove_learner(common::ObMember(sender, 1)))) { - } else if (is_in_memberlist && ConfigChangeState::INIT != state_) { - if (OB_FAIL(ms_ack_list_.add_server(sender)) && ret != OB_ENTRY_EXIST) { - PALF_LOG(ERROR, "add server to ack list failed", KR(ret), K_(palf_id), K_(self), K(sender), K_(ms_ack_list)); - } else { - ret = OB_SUCCESS; - // NB: can set majority repeatedly. - is_majority = is_reach_majority_(); - PALF_LOG(INFO, "ack_config_log success", KR(ret), K_(palf_id), K_(self), K(config_version), K(sender), - K(is_majority), K_(ms_ack_list), K(alive_paxos_replica_num_)); - } - } - return ret; -} - -int LogConfigMgr::after_config_log_majority(const int64_t proposal_id, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else { - ret = after_config_log_majority_(proposal_id, config_version); - } - return ret; -} - -// do not change LogConfigMgr::state_ -int LogConfigMgr::after_config_log_majority_(const int64_t proposal_id, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (proposal_id != log_ms_meta_.proposal_id_ || - config_version != log_ms_meta_.curr_.config_.config_version_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "config_version has been changed", KR(ret), K_(palf_id), - K_(self), K_(log_ms_meta), K(proposal_id), K_(state), K(config_version)); - } else if (is_reach_majority_()) { - (void) state_mgr_->reset_changing_config_with_arb(); - (void) update_match_lsn_map_(running_args_, log_ms_meta_.curr_); - } - return ret; -} - -int LogConfigMgr::try_resend_config_log_(const int64_t proposal_id) -{ - int ret = OB_SUCCESS; - if (INIT == state_ && - resend_log_list_.get_member_number() != 0 && - resend_config_version_.is_valid() && - palf_reach_time_interval(PALF_RESEND_CONFIG_LOG_INTERVAL_US, last_submit_config_log_time_us_) && - OB_FAIL(log_engine_->submit_change_config_meta_req(resend_log_list_, proposal_id, - reconfig_barrier_.prev_log_proposal_id_, reconfig_barrier_.prev_lsn_, - reconfig_barrier_.prev_mode_pid_, log_ms_meta_))) { - PALF_LOG(WARN, "resend config log failed", KR(ret), K(proposal_id), K(resend_config_version_), K_(log_ms_meta)); - } - return ret; -} - -bool LogConfigChangeArgs::is_valid() const -{ - bool bool_ret = true; - bool_ret = bool_ret && (type_ != INVALID_LOG_CONFIG_CHANGE_TYPE); - bool_ret = bool_ret && ((is_add_member_list(type_) || is_remove_member_list(type_) || - is_add_learner_list(type_) || is_remove_learner_list(type_))? \ - (server_.is_valid() || (added_list_.is_valid() && removed_list_.is_valid())): true); - bool_ret = bool_ret && ((is_use_replica_num_args(type_))? is_valid_replica_num(new_replica_num_): true); - bool_ret = bool_ret && ((type_ == CHANGE_REPLICA_NUM)? \ - (curr_member_list_.is_valid() && is_valid_replica_num(curr_replica_num_) && is_valid_replica_num(new_replica_num_)): true); - const bool is_lock_meta_valid = (OB_INVALID_CONFIG_CHANGE_LOCK_OWNER != lock_owner_); - bool_ret = bool_ret && ((TRY_LOCK_CONFIG_CHANGE == type_ || UNLOCK_CONFIG_CHANGE == type_) ? is_lock_meta_valid : true); - // Note: We do not check the validity of config_version_, because it's new variable in version 4.2. - // If a OBServer v4.1 sends a LogConfigChangeCmd to the leader v4.2, config_version_ may be invalid - return bool_ret; -} - -void LogConfigChangeArgs::reset() -{ - server_.reset(); - curr_member_list_.reset(); - curr_replica_num_ = 0; - new_replica_num_ = 0; - config_version_.reset(); - ref_scn_.reset() ; - lock_owner_ = OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - lock_type_ = ConfigChangeLockType::LOCK_NOTHING; - type_ = INVALID_LOG_CONFIG_CHANGE_TYPE; - added_list_.reset(); - removed_list_.reset(); - new_member_list_.reset(); -} - -int LogConfigMgr::check_follower_sync_status(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - bool &added_member_has_new_version) const -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(lock_); - return check_follower_sync_status_(args, new_config_info, added_member_has_new_version); -} - -int LogConfigMgr::wait_log_barrier(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info) const -{ - SpinLockGuard guard(lock_); - return wait_log_barrier_(args, new_config_info); -} - -int LogConfigMgr::wait_log_barrier_before_start_working_(const LogConfigChangeArgs &args) -{ - int ret = OB_SUCCESS; - LogConfigInfoV2 config_info; - if (OB_FAIL(generate_new_config_info_(state_mgr_->get_proposal_id(), args, config_info))) { - PALF_LOG(WARN, "generate_new_config_info_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - } else if (config_info.config_.arbitration_member_.is_valid() == false) { - } else if (OB_FAIL(state_mgr_->set_changing_config_with_arb())) { - PALF_LOG(WARN, "set_changing_config_with_arb failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(renew_config_change_barrier_())) { - PALF_LOG(WARN, "renew_config_change_barrier failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(wait_log_barrier_(args, config_info)) && OB_EAGAIN != ret) { - PALF_LOG(WARN, "wait_log_barrier_ failed", KR(ret), K_(palf_id), K_(self), K(args)); - ret = (OB_LOG_NOT_SYNC == ret)? OB_EAGAIN: ret; - } - return ret; -} - -int LogConfigMgr::wait_log_barrier_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info) const -{ - int ret = OB_SUCCESS; - LSN first_committed_end_lsn; - LSN unused_lsn; - int64_t unused_id = INT64_MAX; - bool unused_bool = false; - - int64_t curr_ts_us = common::ObTimeUtility::current_time(); - constexpr int64_t conn_timeout_us = 3 * 1000 * 1000L; // 3s - constexpr bool need_purge_throttling = true; - constexpr bool need_remote_check = false; - const bool need_skip_log_barrier = mode_mgr_->need_skip_log_barrier(); - LSN prev_log_end_lsn = checking_barrier_.prev_end_lsn_; - start_wait_barrier_time_us_ = (OB_INVALID_TIMESTAMP == start_wait_barrier_time_us_)? \ - curr_ts_us: start_wait_barrier_time_us_; - if (new_config_info.config_.log_sync_memberlist_.get_member_number() == 0) { - ret = OB_INVALID_ARGUMENT; - } else if (curr_ts_us - start_wait_barrier_time_us_ > MAX_WAIT_BARRIER_TIME_US_FOR_RECONFIGURATION && - args.type_ != LogConfigChangeType::STARTWORKING) { - ret = OB_LOG_NOT_SYNC; - PALF_LOG(WARN, "waiting for log barrier timeout, skip", KR(ret), K_(palf_id), K_(self), - K_(start_wait_barrier_time_us), K(first_committed_end_lsn), K(prev_log_end_lsn)); - start_wait_barrier_time_us_ = curr_ts_us; - } else if (OB_FAIL(sync_get_committed_end_lsn_(args, new_config_info, need_purge_throttling, - need_remote_check, conn_timeout_us, first_committed_end_lsn, unused_bool, unused_lsn, unused_id))) { - PALF_LOG(WARN, "sync_get_committed_end_lsn failed", K(ret), K_(palf_id), K_(self), K(new_config_info)); - } else if (need_skip_log_barrier) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "skip log barrier for current access mode", K(ret), K_(palf_id), K_(self), \ - "accepted_mode_meta", mode_mgr_->get_accepted_mode_meta()); - } else if (FALSE_IT(ret = (first_committed_end_lsn >= prev_log_end_lsn)? OB_SUCCESS: OB_EAGAIN)) { - } else if (OB_EAGAIN == ret) { - // skip the reconfiguration if: - // 1. committed_end_lsn do not change during 1s, or - // 2. majority of members do not reach the log barrier during 2s - curr_ts_us = common::ObTimeUtility::current_time(); - if (OB_INVALID_TIMESTAMP == last_wait_barrier_time_us_) { - last_wait_committed_end_lsn_ = first_committed_end_lsn; - last_wait_barrier_time_us_ = curr_ts_us; - } else if (curr_ts_us - last_wait_barrier_time_us_ > MAX_WAIT_BARRIER_TIME_US_FOR_STABLE_LOG) { - if (last_wait_committed_end_lsn_ == first_committed_end_lsn) { - ret = OB_LOG_NOT_SYNC; - PALF_LOG(ERROR, "waiting for log barrier failed, committed_end_lsn havn't been advanced", KR(ret), - K_(palf_id), K_(self), K_(last_wait_barrier_time_us), K_(last_wait_committed_end_lsn)); - last_wait_barrier_time_us_ = OB_INVALID_TIMESTAMP; - last_wait_committed_end_lsn_.reset(); - } else { - last_wait_committed_end_lsn_ = first_committed_end_lsn; - last_wait_barrier_time_us_ = curr_ts_us; - } - } - } - PALF_LOG(INFO, "waiting for log barrier", K(ret), K_(palf_id), K_(self), - K(first_committed_end_lsn), K(prev_log_end_lsn), K(new_config_info)); - return ret; -} - -int LogConfigMgr::check_follower_sync_status_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - bool &added_member_has_new_version) const -{ - int ret = OB_SUCCESS; - LSN first_leader_committed_end_lsn, second_leader_committed_end_lsn; - LSN first_committed_end_lsn, second_committed_end_lsn; - constexpr int64_t conn_timeout_us = 3 * 1000 * 1000L; // 3s - const int64_t max_log_gap_time = PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US / 4; - added_member_has_new_version = is_add_member_list(args.type_)? false: true; - LSN added_member_flushed_end_lsn; - int64_t added_member_last_slide_log_id = INT64_MAX; - int64_t leader_last_slide_log_id = sw_->get_last_slide_log_id(); - const bool need_purge_throttling = (DEGRADE_ACCEPTOR_TO_LEARNER == args.type_); - const bool need_remote_check = true; - - (void) sw_->get_committed_end_lsn(first_leader_committed_end_lsn); - const bool need_skip_log_barrier = mode_mgr_->need_skip_log_barrier(); - if (new_config_info.config_.log_sync_memberlist_.get_member_number() == 0) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(sync_get_committed_end_lsn_(args, new_config_info, need_purge_throttling, - need_remote_check, conn_timeout_us, first_committed_end_lsn, added_member_has_new_version, - added_member_flushed_end_lsn, added_member_last_slide_log_id))) { - PALF_LOG(WARN, "sync_get_committed_end_lsn failed", K(ret), K_(palf_id), K_(self), K(new_config_info), - K(added_member_has_new_version)); - } else if (need_skip_log_barrier) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "skip log barrier for current access mode", K(ret), K_(palf_id), K_(self), \ - "accepted_mode_meta", mode_mgr_->get_accepted_mode_meta()); - } else if (first_committed_end_lsn >= first_leader_committed_end_lsn) { - // if committed lsn of new majority do not retreat, then start config change - PALF_LOG(INFO, "majority of new_member_list are sync with leader, start config change", K(ret), K_(palf_id), K_(self), - K(first_committed_end_lsn), K(first_leader_committed_end_lsn), K(new_config_info), K(conn_timeout_us)); - // when quorum has been changed (e.g., 1 -> 2), committed_end_lsn of new memberlist may always be behind the committed_end_lsn of - // leader, so we relax the condition for adding members which has changed quorum - } else if (is_add_log_sync_member_list(args.type_) && - (new_config_info.config_.log_sync_replica_num_ / 2) > (log_ms_meta_.curr_.config_.log_sync_replica_num_ / 2) && - log_ms_meta_.curr_.config_.arbitration_member_.is_valid()) { - if (added_member_flushed_end_lsn.is_valid() && - first_leader_committed_end_lsn - added_member_flushed_end_lsn < LEADER_DEFAULT_GROUP_BUFFER_SIZE && - (added_member_last_slide_log_id != INT64_MAX && - leader_last_slide_log_id - added_member_last_slide_log_id < PALF_SLIDING_WINDOW_SIZE)) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "the gap between the leader and added member is smaller than the group_buffer_size", - K(ret), K_(palf_id), K_(self), K(args), K(new_config_info), K(first_leader_committed_end_lsn), - K(added_member_flushed_end_lsn), K(leader_last_slide_log_id), K(added_member_last_slide_log_id)); - } else { - ret = OB_EAGAIN; - PALF_LOG(INFO, "the gap between the leader and added member is larger than the group_buffer_size, skip", - K(ret), K_(palf_id), K_(self), K(args), K(new_config_info), K(first_leader_committed_end_lsn), - K(added_member_flushed_end_lsn), K(leader_last_slide_log_id), K(added_member_last_slide_log_id)); - } - } else { - PALF_LOG(INFO, "majority of new_member_list aren't sync with leader", K_(palf_id), K_(self), K(first_committed_end_lsn), - K(first_leader_committed_end_lsn), K(new_config_info), K(conn_timeout_us)); - // committed_lsn of new majority is behind than old majority's, we want to know if - // they can catch up with leader during config change timeout. If they can, start config change - ob_usleep(500 * 1000); - sw_->get_committed_end_lsn(second_leader_committed_end_lsn); - int64_t expected_sync_time_s; - int64_t sync_speed_gap; - added_member_has_new_version = is_add_member_list(args.type_)? false: true; - if (OB_FAIL(sync_get_committed_end_lsn_(args, new_config_info, false/*no need purge throttling*/, - need_remote_check, conn_timeout_us, second_committed_end_lsn, added_member_has_new_version, - added_member_flushed_end_lsn, added_member_last_slide_log_id))) { - PALF_LOG(WARN, "sync_get_committed_end_lsn failed", K(ret), K_(palf_id), K_(self), - K(new_config_info), K(added_member_has_new_version)); - } else if (second_committed_end_lsn >= second_leader_committed_end_lsn) { - // if committed lsn of new majority do not retreat, then start config change - PALF_LOG(INFO, "majority of new_member_list are sync with leader, start config change", K_(palf_id), K_(self), - K(second_committed_end_lsn), K(second_leader_committed_end_lsn), K(new_config_info), K(conn_timeout_us)); - } else if (FALSE_IT(sync_speed_gap = ((second_committed_end_lsn - first_committed_end_lsn) * 2) - \ - ((second_leader_committed_end_lsn - first_leader_committed_end_lsn) * 2) )) { - } else if (sync_speed_gap <= 0) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "follwer is not sync with leader after waiting 500 ms", K_(palf_id), K_(self), K(sync_speed_gap), - K(ret), K(second_committed_end_lsn), K(second_leader_committed_end_lsn)); - } else if (FALSE_IT(expected_sync_time_s = (second_leader_committed_end_lsn - second_committed_end_lsn) / sync_speed_gap)) { - } else if ((expected_sync_time_s * 1E6) <= max_log_gap_time) { - PALF_LOG(INFO, "majority of new_member_list are sync with leader, start config change", - K_(palf_id), K_(self), K(ret), K(second_committed_end_lsn), K(first_committed_end_lsn), K(sync_speed_gap), - K(second_leader_committed_end_lsn), K(first_leader_committed_end_lsn), K(expected_sync_time_s), K(max_log_gap_time)); - } else { - ret = OB_EAGAIN; - PALF_LOG(INFO, "majority of new_member_list are far behind, can not change member", - K_(palf_id), K_(self), K(ret), K(second_committed_end_lsn), K(first_committed_end_lsn), K(sync_speed_gap), - K(second_leader_committed_end_lsn), K(first_leader_committed_end_lsn), K(expected_sync_time_s)); - } - } - return ret; -} - -int LogConfigMgr::check_servers_lsn_and_version_(const common::ObAddr &server, - const LogConfigVersion &config_version, - const int64_t conn_timeout_us, - const bool force_remote_check, - const bool need_purge_throttling, - LSN &max_flushed_end_lsn, - bool &has_same_version, - int64_t &last_slide_log_id) const -{ - int ret = OB_SUCCESS; - LogGetMCStResp resp; - LsnTsInfo ack_info; - bool get_from_local = false; - if (self_ == server) { - get_from_local = true; - sw_->get_max_flushed_end_lsn(max_flushed_end_lsn); - if (need_purge_throttling) { - int tmp_ret = log_engine_->submit_purge_throttling_task(PurgeThrottlingType::PURGE_BY_CHECK_SERVERS_LSN_AND_VERSION); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN_RET(tmp_ret, "submit_purge_throttling_task", K_(palf_id), K_(self)); - } - } - } else if (false == force_remote_check && - (OB_FAIL(sw_->get_server_ack_info(server, ack_info)) && - OB_ENTRY_NOT_EXIST != ret)) { - PALF_LOG(WARN, "get_server_ack_info failed", KR(ret), K_(palf_id), K_(self), K(server)); - } else if (false == force_remote_check && - OB_SUCC(ret) && - common::ObTimeUtility::current_time() - ack_info.last_ack_time_us_ < conn_timeout_us) { - // ABC remove C, but B is far behind from leader(A), we should not remove C. - // so we check ack_ts to ensure B's ack_info is fresh. - max_flushed_end_lsn = ack_info.lsn_; - get_from_local = true; - } else if (OB_FAIL(log_engine_->submit_config_change_pre_check_req(server, config_version, need_purge_throttling, - conn_timeout_us, resp))) { - // PALF_LOG(WARN, "submit_config_change_pre_check_req failed", KR(ret), K_(palf_id), K_(self), - // K(server), K(need_purge_throttling), K(config_version), K(conn_timeout_us), K(resp)); - has_same_version = false; - } else if (false == resp.is_normal_replica_) { - has_same_version = false; - ret = OB_EAGAIN; - } else { - has_same_version = !resp.need_update_config_meta_; - max_flushed_end_lsn = resp.max_flushed_end_lsn_; - last_slide_log_id = resp.last_slide_log_id_; - get_from_local = false; - } - PALF_LOG(INFO, "check_servers_lsn_and_version_ finish", K(ret), K_(palf_id), K_(self), K(server), K(config_version), - K(conn_timeout_us), K(force_remote_check), K(need_purge_throttling), K(get_from_local), K(resp), - K(max_flushed_end_lsn), K(has_same_version)); - return ret; -} - -// 1. get committed_end_lsn of new_member_list -// 2. check if the config_version of added member are same to current config_version. -// if the config change don't add member to list, return true -int LogConfigMgr::sync_get_committed_end_lsn_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - const bool need_purge_throttling, - const bool need_remote_check, - const int64_t conn_timeout_us, - LSN &committed_end_lsn, - bool &added_member_has_new_version, - LSN &added_member_flushed_end_lsn, - int64_t &added_member_last_slide_log_id) const -{ - int ret = OB_SUCCESS, tmp_ret = OB_SUCCESS; - int64_t log_sync_resp_cnt = 0, paxos_resp_cnt = 0; - const LogConfigVersion config_version = log_ms_meta_.curr_.config_.config_version_; - LSN lsn_array[OB_MAX_MEMBER_NUMBER]; - const common::ObMemberList new_log_sync_memberlist = new_config_info.config_.log_sync_memberlist_; - const int64_t new_log_sync_replica_num = new_config_info.config_.log_sync_replica_num_; - common::ObMemberList new_paxos_memberlist; - int64_t new_paxos_replica_num = 0; - GlobalLearnerList unused_list; - - added_member_has_new_version = is_add_member_list(args.type_)? false: true; - added_member_flushed_end_lsn.reset(); - added_member_last_slide_log_id = 0; - - if (OB_FAIL(new_config_info.convert_to_complete_config(new_paxos_memberlist, new_paxos_replica_num, unused_list))) { - PALF_LOG(WARN, "convert_to_complete_config failed", K(ret), K_(palf_id), K_(self), K(new_config_info)); - } - - for (int64_t i = 0; OB_SUCC(ret) && i < new_paxos_memberlist.get_member_number(); ++i) { - common::ObAddr server; - bool is_added_member = false; - bool force_remote_check = false; - LSN max_flushed_end_lsn; - bool has_same_version = false; - bool is_arb_member = false; - int64_t last_slide_log_id = 0; - if (OB_SUCCESS != (tmp_ret = new_paxos_memberlist.get_server_by_index(i, server))) { - PALF_LOG(ERROR, "get_server_by_index failed", KR(ret), K_(palf_id), K_(self), K(i), K(new_paxos_memberlist)); - } else if (FALSE_IT(is_added_member = (is_add_member_list(args.type_) && (args.server_.get_server() == server)))) { - } else if (FALSE_IT(is_arb_member = (server == new_config_info.config_.arbitration_member_.get_server()))) { - } else if (FALSE_IT(force_remote_check = is_added_member || need_purge_throttling || need_remote_check)) { - } else if (OB_SUCCESS != (tmp_ret = check_servers_lsn_and_version_(server, config_version, - conn_timeout_us, force_remote_check, need_purge_throttling, max_flushed_end_lsn, has_same_version, - last_slide_log_id))) { - // PALF_LOG(WARN, "check_servers_lsn_and_version_ failed", K(ret), K(tmp_ret), K_(palf_id), K_(self), K(server), - // K(config_version), K(conn_timeout_us), K(force_remote_check), K(max_flushed_end_lsn), K(has_same_version)); - } else if (false == is_arb_member && max_flushed_end_lsn.is_valid()) { - lsn_array[log_sync_resp_cnt++] = max_flushed_end_lsn; - paxos_resp_cnt++; - } else if (true == is_arb_member) { - paxos_resp_cnt++; - } else { } - added_member_has_new_version = (is_added_member)? has_same_version: added_member_has_new_version; - added_member_flushed_end_lsn = (is_added_member)? max_flushed_end_lsn: added_member_flushed_end_lsn; - added_member_last_slide_log_id = (is_added_member)? last_slide_log_id: added_member_last_slide_log_id; - } - - if (false == added_member_has_new_version) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "added member don't have new version, eagain", K(ret), K_(palf_id), - K_(self), K(args), K(config_version)); - } else if ((paxos_resp_cnt < new_paxos_replica_num / 2 + 1) || - (log_sync_resp_cnt < new_log_sync_replica_num / 2 + 1)) { - // do not recv majority resp, can not change member - ret = OB_EAGAIN; - PALF_LOG(WARN, "connection timeout with majority of new_member_list, can't change member!", - K_(palf_id), K_(self), K(new_paxos_replica_num), K(paxos_resp_cnt), - K(new_log_sync_replica_num), K(log_sync_resp_cnt), K(conn_timeout_us)); - } else { - lib::ob_sort(lsn_array, lsn_array + log_sync_resp_cnt, LSNCompare()); - committed_end_lsn = lsn_array[new_log_sync_replica_num / 2]; - } - PALF_LOG(INFO, "sync_get_committed_end_lsn_ finish", K(ret), K_(palf_id), K_(self), K(args), - K(new_config_info), K(need_purge_throttling), K(need_remote_check), - K(conn_timeout_us), K(committed_end_lsn), K(added_member_has_new_version), - K(added_member_flushed_end_lsn), K(added_member_last_slide_log_id), - K(paxos_resp_cnt), K(new_paxos_replica_num), K(log_sync_resp_cnt), K(new_log_sync_replica_num), - "lsn_array:", common::ObArrayWrap(lsn_array, log_sync_resp_cnt)); - return ret; -} - -// need rlock of PalfHandleImpl -// The arb server don't support set_initial_member_list, -// so we need to forward LogConfigMeta to arb member. otherwise, -// if only 1F1A are created successfully when creating PALF group, -// A will not vote for the F because of empty config meta. -int LogConfigMgr::forward_initial_config_meta_to_arb() -{ - int ret = OB_SUCCESS; - const common::ObMember &arb_member = log_ms_meta_.curr_.config_.arbitration_member_; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (INVALID_PROPOSAL_ID == forwarding_config_proposal_id_ || - false == arb_member.is_valid()) { - // skip - } else { - common::ObMemberList forward_list; - if (forwarding_config_proposal_id_ != state_mgr_->get_proposal_id() || - forwarding_config_proposal_id_ != log_ms_meta_.proposal_id_) { - forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID; - PALF_LOG(INFO, "stop forward_initial_config_meta_to_arb", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(forward_list.add_member(arb_member))) { - PALF_LOG(WARN, "add_member failed", KR(ret), K_(palf_id), K_(self), K(forward_list), K(arb_member)); - } else if (OB_FAIL(log_engine_->submit_change_config_meta_req(forward_list, - log_ms_meta_.proposal_id_, log_ms_meta_.prev_log_proposal_id_, - log_ms_meta_.prev_lsn_, log_ms_meta_.prev_mode_pid_, log_ms_meta_))) { - PALF_LOG(WARN, "submit_change_config_meta_req failed", KR(ret), K_(palf_id), K_(self), - K(arb_member), K_(log_ms_meta)); - } - } - return ret; -} - -int LogConfigMgr::force_set_member_list(const LogConfigChangeArgs &args, const int64_t proposal_id) -{ - int ret = OB_SUCCESS; - bool unused = false; - SpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr is not inited!", K(ret)); - } else if (!args.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(args), K(proposal_id)); - } else if (OB_FAIL(append_config_meta_(proposal_id, args, unused))) { - PALF_LOG(WARN, "append_config_meta_ failed", K(ret), K(args), K(proposal_id)); - } else { - FlushMetaCbCtx cb_ctx; - cb_ctx.type_ = MetaType::CHANGE_CONFIG_META; - cb_ctx.proposal_id_ = proposal_id; - cb_ctx.config_version_ = log_ms_meta_.curr_.config_.config_version_; - if (OB_FAIL(log_engine_->submit_flush_change_config_meta_task(cb_ctx, log_ms_meta_))) { - PALF_LOG(WARN, "LogEngine failed to submit flush_change_config_meta_task", K(ret), K(cb_ctx), K(log_ms_meta_)); - } else { - PALF_LOG(INFO, "force to set member list successfully", K_(palf_id), K(args), K(proposal_id), K(log_ms_meta_)); - } - } - - return ret; -} - -//================================ Config Change ================================ - -//================================ Child ================================ -void LogConfigMgr::reset_registering_state_() -{ - last_submit_register_req_time_us_ = OB_INVALID_TIMESTAMP; -} - -bool LogConfigMgr::is_registering_() const -{ - return (last_submit_register_req_time_us_ != OB_INVALID_TIMESTAMP); -} - -int LogConfigMgr::get_register_leader_(common::ObAddr &leader) const -{ - int ret = OB_SUCCESS; - // TODO by yunlong, get leader from location cache temporarily, need remove - sw_->get_leader_from_cache(leader); - if (!leader.is_valid()) { - ret = OB_EAGAIN; - } - return ret; -} - -int LogConfigMgr::after_register_parent_done_(const LogLearner &parent, - const RegisterParentReason &reason) const -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int pret = OB_SUCCESS; - const int64_t MAX_BUF_LEN = 50; - char reason_str[MAX_BUF_LEN] = {'\0'}; - if (OB_FAIL(sw_->try_fetch_log(FetchTriggerType::LEARNER_REGISTER))){ - PALF_LOG(WARN, "try_fetch_log failed", KR(ret), K_(palf_id), K_(self), K_(parent), K_(register_time_us)); - } - if (0 >= (pret = snprintf(reason_str, MAX_BUF_LEN, "REASON:%s", register_parent_reason_2_str_(reason)))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(reason_str), "reason", register_parent_reason_2_str_(reason)); - } else if (OB_TMP_FAIL(plugins_->record_parent_child_change_event(palf_id_, true /*is_register*/, - true /* is_parent*/, parent.server_, parent.register_time_us_, reason_str))) { - PALF_LOG(WARN, "record_parent_child_change_event failed", KR(tmp_ret), K_(palf_id), K_(self), K(parent)); - } - PALF_EVENT("register_parent", palf_id_, K(ret), K_(self), K(parent), - "reason", register_parent_reason_2_str_(reason)); - return ret; -} - -int LogConfigMgr::after_retire_parent_done_(const LogLearner &parent, - const RetireParentReason &reason) const -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int pret = OB_SUCCESS; - const int64_t MAX_BUF_LEN = 50; - char reason_str[MAX_BUF_LEN] = {'\0'}; - if (0 >= (pret = snprintf(reason_str, MAX_BUF_LEN, "REASON:%s", retire_parent_reason_2_str_(reason)))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(reason_str), K(reason)); - } else if (OB_TMP_FAIL(plugins_->record_parent_child_change_event(palf_id_, false /*is_register*/, - true /* is_parent*/, parent.server_, parent.register_time_us_, reason_str))) { - PALF_LOG(WARN, "record_parent_child_change_event failed", KR(tmp_ret), K_(palf_id), K_(self), K(parent), K(reason)); - } - PALF_EVENT("retire_parent", palf_id_, K(ret), K_(self), K(parent), "reason", retire_parent_reason_2_str_(reason)); - return ret; -} - -int LogConfigMgr::register_parent() -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(parent_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!parent_.is_valid() && OB_FAIL(register_parent_(RegisterParentReason::FIRST_REGISTER))) { - PALF_LOG(WARN, "register_parent failed", KR(ret), K_(palf_id), K_(self), K_(parent)); - } else { - } - return ret; -} - -// @return -// - OB_EAGAIN: leader is invalid -int LogConfigMgr::register_parent_(const RegisterParentReason &reason) -{ - int ret = OB_SUCCESS; - ObAddr leader; - if (OB_FAIL(get_register_leader_(leader))) { - PALF_LOG(TRACE, "get_register_leader_ failed", KR(ret), K_(palf_id), K_(self)); - } else { - const bool is_to_leader = true; - const int64_t curr_time_us = ObTimeUtility::current_time(); - LogLearner child_self(self_, curr_time_us); - if (OB_FAIL(log_engine_->submit_register_parent_req(leader, child_self, is_to_leader))) { - // NB: register_req sends my addr_ and register_time_us_ - PALF_LOG(WARN, "submit_register_parent_req failed", KR(ret), K_(palf_id), K_(self), K(leader), K(child_self), K(is_to_leader)); - } else { - last_submit_register_req_time_us_ = curr_time_us; - register_time_us_ = curr_time_us; - register_parent_reason_ = (RegisterParentReason::INVALID != reason)? \ - reason: register_parent_reason_; - parent_keepalive_time_us_ = OB_INVALID_TIMESTAMP; - PALF_LOG(INFO, "register_parent_", KR(ret), K_(palf_id), K_(self), K(reason), K(leader), - K_(register_time_us), K_(last_submit_register_req_time_us), K_(register_parent_reason)); - } - } - return ret; -} - -int LogConfigMgr::handle_register_parent_resp(const LogLearner &server, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) -{ - int ret = OB_SUCCESS; - bool do_after_register_parent_done = false; - RegisterParentReason reason = RegisterParentReason::INVALID; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (!server.is_valid() || INVALID_REG_RET == reg_ret) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(server)); - } else { - SpinLockGuard guard(parent_lock_); - if (!is_registering_() || register_time_us_ != server.register_time_us_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "receive wrong register resp", K_(palf_id), K_(self), K(server), K_(register_time_us), "registering", is_registering_()); - } else if (REGISTER_DONE == reg_ret) { - // register done, just set parent_ and clean registering state - parent_ = server.server_; - parent_keepalive_time_us_ = common::ObTimeUtility::current_time(); - reason = register_parent_reason_; - reset_registering_state_(); - do_after_register_parent_done = true; - } else if (REGISTER_CONTINUE == reg_ret && candidate_list.get_member_number() > 0) { - common::ObAddr reg_dst; - const int64_t reg_dst_idx = ObRandom::rand(0, candidate_list.get_member_number() - 1); - LogLearner child_self(self_, register_time_us_); - if (OB_FAIL(candidate_list.get_server_by_index(reg_dst_idx, reg_dst))) { - PALF_LOG(WARN, "get_server_by_index failed", KR(ret), K_(palf_id), K_(self), K(candidate_list), K(reg_dst)); - } else if (OB_FAIL(log_engine_->submit_register_parent_req(reg_dst, child_self, false))) { - PALF_LOG(WARN, "submit_register_parent_req failed", KR(ret), K_(palf_id), K_(self), K(reg_dst)); - } else { - last_submit_register_req_time_us_ = common::ObTimeUtility::current_time(); - } - } else if (REGISTER_NOT_MASTER == reg_ret) { - // skip, wait retry - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected Register_Return", K_(palf_id), K_(self), K(server), K(candidate_list), K(reg_ret)); - } - } - if (do_after_register_parent_done && OB_FAIL(after_register_parent_done_(server, reason))) { - PALF_LOG(WARN, "after_register_parent_done failed", KR(ret), K_(palf_id), K_(self)); - } - PALF_LOG(INFO, "handle_register_parent_resp finished", KR(ret), K_(palf_id), K_(self), K(server), K(candidate_list), K(reg_ret)); - return ret; -} - -int LogConfigMgr::retire_parent() -{ - int ret = OB_SUCCESS; - const RetireParentReason reason = RetireParentReason::IS_FULL_MEMBER; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(retire_parent_(reason))) { - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else { - PALF_LOG(INFO, "retire_parent success", KR(ret), "reason", retire_parent_reason_2_str_(reason)); - } - return ret; -} - -int LogConfigMgr::retire_parent_(const RetireParentReason &reason) -{ - int ret = OB_SUCCESS; - if (!parent_.is_valid()) { - // parent is already invalid, skip - } else { - LogLearner child_self(self_, register_time_us_); - if (OB_FAIL(log_engine_->submit_retire_parent_req(parent_, child_self))) { - PALF_LOG(WARN, "submit_retire_parent_req failed", KR(ret), K_(palf_id), K_(self), K_(parent), K_(register_time_us)); - } else { - PALF_LOG(INFO, "submit_retire_parent_req success", K_(palf_id), K_(self), K_(parent), K_(register_time_us)); - after_retire_parent_done_(LogLearner(parent_, register_time_us_), reason); - reset_parent_info_(); - } - } - return ret; -} - -int LogConfigMgr::handle_retire_child(const LogLearner &parent) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(parent_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (is_registering_() || parent.server_ != parent_ || parent.register_time_us_ != register_time_us_) { - PALF_LOG(WARN, "handle_retire_child failed, invalid msg", KR(ret), K(parent), K(parent_), K_(self)); - } else { - reset_parent_info_(); - PALF_LOG(INFO, "re_register_parent reason: handle_retire_child", K_(palf_id), K_(self), K(parent)); - if (OB_FAIL(register_parent_(RegisterParentReason::RETIRED_BY_PARENT))) { - PALF_LOG(WARN, "register_parent failed when recving retire child", KR(ret), K_(self), K(parent)); - } else { - PALF_LOG(INFO, "handle_retire_child success", KR(ret), K_(self), K(parent)); - } - } - return ret; -} - -int LogConfigMgr::handle_learner_keepalive_req(const LogLearner &parent) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(parent_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (!parent.is_valid() || parent.register_time_us_ <= 0) { - ret = OB_INVALID_ARGUMENT; - } else if (true == is_registering_() || parent.server_ != parent_ || parent.register_time_us_ != register_time_us_) { - PALF_LOG(WARN, "handle_keepalive failed", KR(ret), K_(palf_id), K_(self), K(parent), K(parent_), K_(register_time_us)); - } else { - parent_keepalive_time_us_ = common::ObTimeUtility::current_time(); - LogLearner child_itself(self_, register_time_us_); - if (OB_FAIL(log_engine_->submit_learner_keepalive_resp(parent.server_, child_itself))) { - PALF_LOG(WARN, "submit_learner_keepalive_resp failed", KR(ret), K_(palf_id), K_(self), K(parent)); - } else { - PALF_LOG(INFO, "handle_learner_keepalive_req success", KR(ret), K_(palf_id), K_(self), K(parent), K_(parent_keepalive_time_us)); - } - } - return ret; -} - -//loop thread call this periodically -// 1. check parent keepalive_ts timeout -// 2. push registering state -int LogConfigMgr::check_parent_health() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - { - SpinLockGuard parent_guard(parent_lock_); - SpinLockGuard child_guard(child_lock_); - // break learner loop - if (parent_.is_valid() && - children_.contains(parent_) && - OB_FAIL(retire_parent_(RetireParentReason::PARENT_CHILD_LOOP))) { - PALF_LOG(WARN, "retire_parent_ failed", KR(ret), K_(palf_id), K_(self)); - } - } - SpinLockGuard guard(parent_lock_); - const int64_t curr_time_us = common::ObTimeUtility::current_time(); - const bool is_registering_timeout = (is_registering_() && - curr_time_us - last_submit_register_req_time_us_ > PALF_CHILD_RESEND_REGISTER_INTERVAL_US); - const bool first_registration = (!parent_.is_valid() && !is_registering_() && - palf_reach_time_interval(PALF_CHILD_RESEND_REGISTER_INTERVAL_US, last_first_register_time_us_)); - const bool parent_timeout = (parent_.is_valid() && curr_time_us - parent_keepalive_time_us_ > PALF_PARENT_CHILD_TIMEOUT_US); - if (is_registering_timeout || first_registration || parent_timeout) { - PALF_LOG(INFO, "re_register_parent reason", K_(palf_id), K_(self), K(is_registering_timeout), K(first_registration), K(parent_timeout), - K_(parent_keepalive_time_us), K_(last_submit_register_req_time_us), K_(last_first_register_time_us), K_(register_time_us), K(curr_time_us)); - RegisterParentReason reason = RegisterParentReason::INVALID; - reason = (first_registration)? RegisterParentReason::FIRST_REGISTER: reason; - reason = (parent_timeout)? RegisterParentReason::PARENT_NOT_ALIVE: reason; - reason = (is_registering_timeout)? register_parent_reason_: reason; - reset_parent_info_(); - if (OB_FAIL(register_parent_(reason))) { - PALF_LOG(WARN, "register request timeout, re_register_parent failed", KR(ret), K_(palf_id), K_(self)); - } else { - PALF_LOG(INFO, "re register_parent success", KR(ret), K_(palf_id), K_(self)); - } - } - } - return ret; -} - -void LogConfigMgr::reset_parent_info_() -{ - register_time_us_ = OB_INVALID_TIMESTAMP; - register_parent_reason_ = RegisterParentReason::INVALID; - parent_.reset(); - parent_keepalive_time_us_ = OB_INVALID_TIMESTAMP; - last_submit_register_req_time_us_ = OB_INVALID_TIMESTAMP; -} -//================================ Child ================================ - -//================================ Parent ================================ -int LogConfigMgr::handle_register_parent_req(const LogLearner &child, const bool is_to_leader) -{ - int ret = OB_SUCCESS; - LogCandidateList candidate_list; - RegisterReturn reg_ret = INVALID_REG_RET; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!child.is_valid() || child.register_time_us_ <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K_(self), K(child)); - } else if (is_to_leader && !all_learnerlist_.contains(child.get_server())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "registering child is not in learner list", K_(palf_id), K_(self), K(child)); - } else { - SpinLockGuard guard(child_lock_); - int64_t idx = -1; - const bool is_in_children = (-1 != (idx = children_.get_index_by_addr(child.server_))); - if (is_in_children) { - // if child is already in the children list, - // replace old child in children_list - children_.get_learner(idx).register_time_us_ = child.register_time_us_; - children_.get_learner(idx).update_keepalive_ts(); - reg_ret = REGISTER_DONE; - } else if (is_to_leader) { - if (OB_FAIL(generate_candidate_list_(child, candidate_list))) { - PALF_LOG(WARN, "generate_candidate_list failed", KR(ret), K_(palf_id), K_(self), K(child)); - } else if (candidate_list.get_member_number() > 0) { - // register continue - reg_ret = REGISTER_CONTINUE; - } else if (children_.is_full()) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(ERROR, "leader's children is full", KR(ret), K_(palf_id), K_(self), K(child), K_(children)); - } else { - // register to self - LogLearner dst_child(child); - dst_child.keepalive_ts_ = common::ObTimeUtility::current_time(); - dst_child.register_time_us_ = child.register_time_us_; - if (OB_FAIL(children_.add_learner(dst_child))) { - PALF_LOG(WARN, "handle_register_parent_req failed", KR(ret), K_(palf_id), K_(self), K(is_to_leader), K(dst_child)); - } else if (OB_FAIL(log_sync_children_.add_learner(dst_child))) { - PALF_LOG(WARN, "add_learner failed", KR(ret), K_(palf_id), K_(self), K_(log_sync_children), K(dst_child)); - } else { - reg_ret = REGISTER_DONE; - } - } - } else if (children_.get_member_number() < OB_MAX_CHILD_MEMBER_NUMBER_IN_FOLLOWER) { - // register to self - LogLearner dst_child(child); - dst_child.keepalive_ts_ = common::ObTimeUtility::current_time(); - dst_child.register_time_us_ = child.register_time_us_; - if (OB_FAIL(children_.add_learner(dst_child))) { - PALF_LOG(WARN, "handle_register_parent_req failed", KR(ret), K_(palf_id), K_(self), K(is_to_leader), K(dst_child)); - } else if (OB_FAIL(log_sync_children_.add_learner(dst_child))) { - PALF_LOG(WARN, "add_learner failed", KR(ret), K_(palf_id), K_(self), K_(log_sync_children), K(dst_child)); - } else { - reg_ret = REGISTER_DONE; - } - } else if (OB_FAIL(generate_candidate_list_from_children_(child, candidate_list))) { - PALF_LOG(WARN, "generate_candidate_list failed", KR(ret), K_(palf_id), K_(self), K(child)); - } else if (candidate_list.get_member_number() <= 0) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "candidate_list is empty", KR(ret), K_(palf_id), K_(self), K(candidate_list), K_(children)); - } else { - // register continue - reg_ret = REGISTER_CONTINUE; - } - } - if (OB_SUCC(ret)){ - SpinLockGuard guard(child_lock_); - LogLearner parent(self_, child.register_time_us_); - if (reg_ret == REGISTER_DONE || - reg_ret == REGISTER_CONTINUE) { - if (OB_FAIL(log_engine_->submit_register_parent_resp(child.server_, parent, candidate_list, reg_ret))) { - PALF_LOG(WARN, "submit_register_parent_resp failed", KR(ret), K_(palf_id), K_(self), K(child)); - } - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "invalid register ret", K(ret), K_(palf_id), K_(self), K(reg_ret)); - } - PALF_LOG(INFO, "handle_register_parent_req success", K(ret), K(child), K(is_to_leader), K(candidate_list), - K(reg_ret), K_(children), "member_list", log_ms_meta_.curr_.config_.log_sync_memberlist_); - } - return ret; -} - -int LogConfigMgr::handle_retire_parent(const LogLearner &child) -{ - int ret = OB_SUCCESS; - int64_t idx = -1; - SpinLockGuard guard(child_lock_); - LogLearner learner; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!child.is_valid() || child.register_time_us_ <= 0) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_SUCC(children_.get_learner_by_addr(child.server_, learner)) && - learner.register_time_us_ == child.register_time_us_) { - if (OB_FAIL(children_.remove_learner(child))) { - PALF_LOG(WARN, "children_ remove_learner failed", KR(ret), K_(palf_id), K_(self), K_(children), K(child)); - } else { - PALF_LOG(INFO, "handle_retire_parent success", KR(ret), K_(palf_id), K_(self), K(child)); - } - } else { - PALF_LOG(INFO, "handle_retire_parent failed, invalid req", KR(ret), K_(palf_id), K_(self), K(child), K_(children)); - // skip - } - return ret; -} - -int LogConfigMgr::handle_learner_keepalive_resp(const LogLearner &child) -{ - int ret = OB_SUCCESS; - bool need_resp = false; - int64_t idx = -1; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!child.is_valid() || child.register_time_us_ <= 0) { - ret = OB_INVALID_ARGUMENT; - } else { - SpinLockGuard guard(child_lock_); - int64_t idx = -1; - if (-1 != (idx = children_.get_index_by_addr(child.server_)) && - children_.get_learner(idx).register_time_us_ == child.register_time_us_) { - children_.get_learner(idx).update_keepalive_ts(); - PALF_LOG(INFO, "handle_learner_keepalive_resp success", K_(palf_id), K_(self), K_(children)); - } - } - return ret; -} - -// caller guarantees role do not change -// common:: -// 1. check if children are timeout -void LogConfigMgr::check_children_health() -{ - int ret = OB_SUCCESS; - LogLearnerList dead_children; - LogLearnerList parent_disable_sync_retire_children; - const bool is_leader = state_mgr_->is_leader_active(); - const bool enable_vote = state_mgr_->is_allow_vote(); - const bool enable_sync = state_mgr_->is_sync_enabled(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - if (children_.is_valid()) { - SpinLockGuard guard(child_lock_); - // 1. remove child that is timeout(follower, leader) - if (OB_FAIL(remove_timeout_child_(dead_children))) { - PALF_LOG(WARN, "remove_timeout_child failed", KR(ret), K_(palf_id), K_(self)); - } - // 4. parent is disable_sync or disable_vote, retire all children. - if (!enable_sync || !enable_vote) { - parent_disable_sync_retire_children = children_; - children_.reset(); - } - } - // 4. send keepalive msg to children - if (children_.is_valid() && palf_reach_time_interval(PALF_PARENT_KEEPALIVE_INTERVAL_US, last_submit_keepalive_time_us_)) { - // reach keepalive interval, submit keepalive req - SpinLockGuard guard(child_lock_); - LogLearner parent_self(self_, OB_INVALID_TIMESTAMP); - for (int64_t i = 0; i < children_.get_member_number(); ++i) { - LogLearner child; - if (OB_FAIL(children_.get_learner(i, child))) { - PALF_LOG(WARN, "children_.get_learner failed", KR(ret), K_(palf_id), K_(self), K(i)); - } else if (FALSE_IT(parent_self.register_time_us_ = child.register_time_us_)) { - } else if (OB_FAIL(log_engine_->submit_learner_keepalive_req(child.get_server(), parent_self))) { - PALF_LOG(WARN, "submit_learner_keepalive_req failed", KR(ret), K_(palf_id), K_(self), K(child)); - } else { - } - } - } - // 5. retire removed children - if (OB_FAIL(submit_retire_children_req_(dead_children, RetireChildReason::CHILD_NOT_ALIVE))) { - // overwrite ret - } else if (OB_FAIL(submit_retire_children_req_(parent_disable_sync_retire_children, RetireChildReason::PARENT_DISABLE_SYNC))) { - // overwrite ret - PALF_LOG(WARN, "submit_retire_children_req failed", KR(ret), K_(palf_id), K_(self), K(dead_children)); - } - // 6. update log_sync_children_ - { - SpinLockGuard guard(child_lock_); - log_sync_children_.reset(); - for (int i = 0; i < children_.get_member_number(); i++) { - const LogLearner &learner = children_.get_learner(i); - common::ObMember learner_in_list; - const int in_list_ret = all_learnerlist_.get_learner_by_addr(learner.get_server(), learner_in_list); - if (OB_SUCCESS != in_list_ret || learner_in_list.is_migrating()) { - // skip - } else if (OB_FAIL(log_sync_children_.add_learner(learner))) { - PALF_LOG(WARN, "add_learner failed", KR(ret), K_(palf_id), K_(self), K(learner), K_(log_sync_children)); - } - } - } - } -} - -int LogConfigMgr::remove_timeout_child_(LogLearnerList &dead_children) -{ - int ret = OB_SUCCESS; - LogLearnerCond child_timeout_cond; - LogLearnerAction child_timeout_action; - // if child is timeout, then add it to dead_children - if (OB_FAIL(child_timeout_cond.assign([](const LogLearner &child) { return child.is_timeout(PALF_PARENT_CHILD_TIMEOUT_US);}))) { - PALF_LOG(WARN, "child_timeout_cond assign failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(child_timeout_action.assign([&dead_children](const LogLearner &child) {return dead_children.add_learner(child);}))) { - PALF_LOG(WARN, "child_timeout_action assign failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(children_if_cond_then_action_(child_timeout_cond, child_timeout_action))) { - PALF_LOG(WARN, "children_if_cond_then_action failed", KR(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(remove_children_(children_, dead_children))) { - PALF_LOG(WARN, "remove_children failed", KR(ret), K(dead_children), K_(children)); - } else if (dead_children.get_member_number() > 0) { - PALF_LOG(INFO, "remove_timeout_child success", K(ret), K_(palf_id), K_(self), K_(children), K(dead_children)); - } - return ret; -} - -int LogConfigMgr::remove_child_is_not_learner_(LogLearnerList &removed_children) -{ - int ret = OB_SUCCESS; - SpinLockGuard guard(child_lock_); - LogLearnerCond cond; - LogLearnerAction action; - const GlobalLearnerList &learnerlist = all_learnerlist_; - if (OB_FAIL(cond.assign([&learnerlist](const LogLearner &child)->bool { return !learnerlist.contains(child.get_server()); }))) { - PALF_LOG(WARN, "learnerlist cond assign failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(action.assign([&removed_children](const LogLearner &child)->int { return removed_children.add_learner(child); }))) { - PALF_LOG(WARN, "learnerlist action assign failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(children_if_cond_then_action_(cond, action))) { - PALF_LOG(WARN, "children_if_cond_then_action failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(remove_children_(children_, removed_children))) { - PALF_LOG(WARN, "remove_children failed", KR(ret), K(removed_children), K_(children)); - } else if (removed_children.get_member_number() > 0) { - PALF_LOG(INFO, "remove_child_is_not_learner success", K(ret), K_(palf_id), K_(self), K_(children), K(removed_children)); - } - return ret; -} - -int LogConfigMgr::children_if_cond_then_action_(const LogLearnerCond &cond, const LogLearnerAction &action) -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; i < children_.get_member_number(); ++i) { - LogLearner child; - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = children_.get_learner(i, child))) { - PALF_LOG(WARN, "get_server_by_addr failed", K(tmp_ret), K_(palf_id), K_(self), K(i)); - } else if (cond(child)) { - if (OB_SUCCESS != (tmp_ret = action(child))) { - PALF_LOG(WARN, "add_learner failed", K(tmp_ret), K(child)); - } - } - } - return ret; -} - -int LogConfigMgr::remove_children_(LogLearnerList &this_children, const LogLearnerList &removed_children) -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; i < removed_children.get_member_number(); ++i) { - LogLearner removed_learner; - if (OB_FAIL(removed_children.get_learner(i, removed_learner))) { - PALF_LOG(WARN, "get_learner failed", KR(ret)); - } else if (OB_FAIL(this_children.remove_learner(removed_learner))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } - } - } - return ret; -} - -int LogConfigMgr::generate_candidate_list_(const LogLearner &child, LogCandidateList &candidate_list) -{ - int ret = OB_SUCCESS; - if (!child.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(generate_candidate_list_from_member_(child, candidate_list))){ - PALF_LOG(WARN, "generate_candidate_list_from_emmber_ failed", KR(ret), K(child), K(candidate_list)); - } else if (OB_FAIL(generate_candidate_list_from_children_(child, candidate_list))) { - PALF_LOG(WARN, "generate_candidate_list_from_children_ failed", KR(ret), K(child), K(candidate_list)); - } else { - } - return ret; -} - -int LogConfigMgr::generate_candidate_list_from_member_(const LogLearner &child, LogCandidateList &candidate_list) -{ - int ret = OB_SUCCESS; - UNUSED(child); - const ObMemberList &curr_member_list = log_ms_meta_.curr_.config_.log_sync_memberlist_; - for (int64_t i = 0; i < curr_member_list.get_member_number(); ++i) { - ObAddr addr; - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = curr_member_list.get_server_by_index(i, addr))) { - PALF_LOG(WARN, "get_server_by_index failed", KR(ret), K(curr_member_list), K(i)); - } else if (addr == self_) { - // skip - } else if (OB_SUCCESS == (tmp_ret = candidate_list.add_learner(common::ObMember(addr, 1)))) { - } else if (OB_ENTRY_EXIST == tmp_ret) { - continue; - } else if (OB_SIZE_OVERFLOW == tmp_ret) { - break; - } else { - ret = tmp_ret; - PALF_LOG(WARN, "add_learner failed", KR(ret)); - break; - } - } - return ret; -} - -int LogConfigMgr::generate_candidate_list_from_children_(const LogLearner &child, LogCandidateList &candidate_list) -{ - int ret = OB_SUCCESS; - UNUSED(child); - for (int64_t i = 0; i < children_.get_member_number(); ++i) { - LogLearner learner; - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = children_.get_learner(i, learner))) { - PALF_LOG(WARN, "get_learner failed", KR(tmp_ret), K_(children)); - } else if (OB_SUCCESS == (tmp_ret = candidate_list.add_learner(common::ObMember(learner.server_, 1)))) { - } else if (OB_ENTRY_EXIST == tmp_ret) { - continue; - } else if (OB_SIZE_OVERFLOW == tmp_ret) { - break; - } else { - ret = tmp_ret; - PALF_LOG(WARN, "add_learner failed", KR(ret)); - break; - } - } - return ret; -} - -int LogConfigMgr::submit_retire_children_req_(const LogLearnerList &retired_children, - const RetireChildReason &reason) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - LogLearner parent(self_, OB_INVALID_ARGUMENT); - for (int64_t i = 0; i < retired_children.get_member_number(); ++i) { - LogLearner retired_child; - if (OB_SUCCESS != (tmp_ret = retired_children.get_learner(i, retired_child))) { - PALF_LOG(WARN, "get_learner failed", K(retired_children)); - } else if (FALSE_IT(parent.register_time_us_ = retired_child.register_time_us_)) { - } else if (OB_SUCCESS != (tmp_ret = log_engine_->submit_retire_child_req(retired_child.server_, parent))) { - PALF_LOG(WARN, "submit_retire_child_req failed", KR(ret), K(retired_child), K(parent)); - } else { - (void) after_retire_child_done_(retired_child, reason); - } - } - return ret; -} - -int LogConfigMgr::after_register_child_done_(const LogLearner &child) const -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - const int64_t MAX_BUF_LEN = 50; - char reason_str[MAX_BUF_LEN] = {'\0'}; - if (OB_FAIL(plugins_->record_parent_child_change_event(palf_id_, true /*is_register*/, - false /* is_parent*/, child.server_, child.register_time_us_, reason_str))) { - PALF_LOG(WARN, "record_parent_child_change_event failed", KR(tmp_ret), K_(palf_id), K_(self), K(child)); - } - PALF_EVENT("register_child", palf_id_, K(ret), K_(self), K(child)); - return ret; -} - -int LogConfigMgr::after_retire_child_done_(const LogLearner &child, - const RetireChildReason &reason) const -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int pret = OB_SUCCESS; - const int64_t MAX_BUF_LEN = 50; - char reason_str[MAX_BUF_LEN] = {'\0'}; - if (0 >= (pret = snprintf(reason_str, MAX_BUF_LEN, "REASON:%s", retire_child_reason_2_str_(reason)))) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "snprintf failed", KR(ret), K(reason_str), K(reason)); - } else if (OB_TMP_FAIL(plugins_->record_parent_child_change_event(palf_id_, false /*is_register*/, - false /* is_parent*/, child.server_, child.register_time_us_, reason_str))) { - PALF_LOG(WARN, "record_parent_child_change_event failed", KR(tmp_ret), K_(palf_id), K_(self), - K(child), "reason", retire_child_reason_2_str_(reason)); - } - PALF_EVENT("retire_child", palf_id_, K(ret), K_(self), K(child), "reason", retire_child_reason_2_str_(reason)); - return ret; -} -//================================ Parent ================================ -} // namespace palf -} // namespace oceanbase diff --git a/src/logservice/palf/log_config_mgr.h b/src/logservice/palf/log_config_mgr.h deleted file mode 100755 index 5dbd4c4a9..000000000 --- a/src/logservice/palf/log_config_mgr.h +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_CONFIG_MGR_ -#define OCEANBASE_LOGSERVICE_LOG_CONFIG_MGR_ - -#include "common/ob_member_list.h" // ObMemberList -#include "lib/lock/ob_spin_lock.h" // SpinRWLock -#include "lib/random/ob_random.h" // ObRandom -#include "lib/hash/ob_hashmap.h" // ObHashMap -#include "lib/function/ob_function.h" // ObFunction -#include "share/scn.h" //SCN -#include "log_define.h" // utils -#include "log_meta_info.h" // LogMembershipMeta -#include "log_req.h" // LogLearnerReqType -#include "log_simple_member_list.h" // LogSimpleMemberList -#include "log_state_mgr.h" // LogStateMgr -#include "palf_callback.h" // PalfLocationCacheCb - -namespace oceanbase -{ -namespace common -{ -class ObILogAllocator; -class ObAddr; -} - -namespace palf -{ -class LogSlidingWindow; -class LogStateMgr; -class LogConfigInfoV2; -class LogConfigMeta; -class LSN; -class LogEngine; -class LogConfigVersion; -class LogLearner; -class LogModeMgr; -namespace election -{ -class Election; -} - -enum LogConfigChangeType -{ - INVALID_LOG_CONFIG_CHANGE_TYPE = 0, - CHANGE_REPLICA_NUM, - ADD_MEMBER, - REMOVE_MEMBER, - ADD_MEMBER_AND_NUM, - REMOVE_MEMBER_AND_NUM, - ADD_LEARNER, - REMOVE_LEARNER, - SWITCH_LEARNER_TO_ACCEPTOR, - SWITCH_ACCEPTOR_TO_LEARNER, - DEGRADE_ACCEPTOR_TO_LEARNER, - UPGRADE_LEARNER_TO_ACCEPTOR, - STARTWORKING, - FORCE_SINGLE_MEMBER, - TRY_LOCK_CONFIG_CHANGE, - UNLOCK_CONFIG_CHANGE, - REPLACE_LEARNERS, - SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM, - FORCE_SET_MEMBER_LIST, -}; - -inline const char *LogConfigChangeType2Str(const LogConfigChangeType state) -{ - #define CHECK_LOG_CONFIG_TYPE_STR(x) case(LogConfigChangeType::x): return #x - switch(state) - { - CHECK_LOG_CONFIG_TYPE_STR(CHANGE_REPLICA_NUM); - CHECK_LOG_CONFIG_TYPE_STR(ADD_MEMBER); - CHECK_LOG_CONFIG_TYPE_STR(REMOVE_MEMBER); - CHECK_LOG_CONFIG_TYPE_STR(ADD_MEMBER_AND_NUM); - CHECK_LOG_CONFIG_TYPE_STR(REMOVE_MEMBER_AND_NUM); - CHECK_LOG_CONFIG_TYPE_STR(ADD_LEARNER); - CHECK_LOG_CONFIG_TYPE_STR(REMOVE_LEARNER); - CHECK_LOG_CONFIG_TYPE_STR(SWITCH_LEARNER_TO_ACCEPTOR); - CHECK_LOG_CONFIG_TYPE_STR(SWITCH_ACCEPTOR_TO_LEARNER); - CHECK_LOG_CONFIG_TYPE_STR(DEGRADE_ACCEPTOR_TO_LEARNER); - CHECK_LOG_CONFIG_TYPE_STR(UPGRADE_LEARNER_TO_ACCEPTOR); - CHECK_LOG_CONFIG_TYPE_STR(STARTWORKING); - CHECK_LOG_CONFIG_TYPE_STR(FORCE_SINGLE_MEMBER); - CHECK_LOG_CONFIG_TYPE_STR(TRY_LOCK_CONFIG_CHANGE); - CHECK_LOG_CONFIG_TYPE_STR(UNLOCK_CONFIG_CHANGE); - CHECK_LOG_CONFIG_TYPE_STR(REPLACE_LEARNERS); - CHECK_LOG_CONFIG_TYPE_STR(SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM); - CHECK_LOG_CONFIG_TYPE_STR(FORCE_SET_MEMBER_LIST); - default: - return "Invalid"; - } - #undef CHECK_LOG_CONFIG_TYPE_STR -} - -// Note: We need to check if the cluster has been upgraded to version 4.2. -// If not, invalid config_version is allowed because OBServer v4.1 -// may send a LogConfigChangeCmd (with invalid config_version) to -// the leader v4.2, we need to allow the reconfiguration. -inline bool need_check_config_version(const LogConfigChangeType type) -{ - return (ADD_MEMBER == type || ADD_MEMBER_AND_NUM == type || - SWITCH_LEARNER_TO_ACCEPTOR == type || SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM == type); -} - -inline bool is_add_log_sync_member_list(const LogConfigChangeType type) -{ - return ADD_MEMBER == type || ADD_MEMBER_AND_NUM == type || - SWITCH_LEARNER_TO_ACCEPTOR == type || UPGRADE_LEARNER_TO_ACCEPTOR == type || - SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM == type; -} - -inline bool is_remove_log_sync_member_list(const LogConfigChangeType type) -{ - return REMOVE_MEMBER == type || REMOVE_MEMBER_AND_NUM == type || - SWITCH_ACCEPTOR_TO_LEARNER == type || DEGRADE_ACCEPTOR_TO_LEARNER == type; -} - -inline bool is_add_member_list(const LogConfigChangeType type) -{ - return is_add_log_sync_member_list(type); -} - -inline bool is_remove_member_list(const LogConfigChangeType type) -{ - return is_remove_log_sync_member_list(type); -} - -inline bool is_add_learner_list(const LogConfigChangeType type) -{ - return ADD_LEARNER == type || SWITCH_ACCEPTOR_TO_LEARNER == type || - DEGRADE_ACCEPTOR_TO_LEARNER == type || REPLACE_LEARNERS == type; -} - -inline bool is_remove_learner_list(const LogConfigChangeType type) -{ - return REMOVE_LEARNER == type || SWITCH_LEARNER_TO_ACCEPTOR == type || - UPGRADE_LEARNER_TO_ACCEPTOR == type || SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM == type || - REPLACE_LEARNERS == type; -} - -inline bool is_upgrade_or_degrade(const LogConfigChangeType type) -{ - return UPGRADE_LEARNER_TO_ACCEPTOR == type || DEGRADE_ACCEPTOR_TO_LEARNER == type; -} - -inline bool is_use_replica_num_args(const LogConfigChangeType type) -{ - return ADD_MEMBER == type || REMOVE_MEMBER == type || - CHANGE_REPLICA_NUM == type || FORCE_SINGLE_MEMBER == type || - SWITCH_LEARNER_TO_ACCEPTOR == type || - SWITCH_ACCEPTOR_TO_LEARNER == type || FORCE_SET_MEMBER_LIST == type; -} - -inline bool need_exec_on_leader_(const LogConfigChangeType type) -{ - return (FORCE_SINGLE_MEMBER != type && FORCE_SET_MEMBER_LIST != type); -} - -inline bool is_may_change_replica_num(const LogConfigChangeType type) -{ - return is_add_member_list(type) || is_remove_member_list(type) || CHANGE_REPLICA_NUM == type || FORCE_SINGLE_MEMBER == type || FORCE_SET_MEMBER_LIST == type; -} - -inline bool is_must_not_change_replica_num(const LogConfigChangeType type) -{ - return ADD_LEARNER == type || REMOVE_LEARNER == type || REPLACE_LEARNERS == type || - TRY_LOCK_CONFIG_CHANGE == type || UNLOCK_CONFIG_CHANGE == type; -} - -inline bool is_paxos_member_list_change(const LogConfigChangeType type) -{ - return (ADD_MEMBER == type || REMOVE_MEMBER == type - || ADD_MEMBER_AND_NUM == type || REMOVE_MEMBER_AND_NUM == type - || SWITCH_LEARNER_TO_ACCEPTOR == type || SWITCH_ACCEPTOR_TO_LEARNER == type - || CHANGE_REPLICA_NUM == type || SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM == type); -} - -inline bool is_try_lock_config_change(const LogConfigChangeType type) -{ - return TRY_LOCK_CONFIG_CHANGE == type; -} - -inline bool is_unlock_config_change(const LogConfigChangeType type) -{ - return UNLOCK_CONFIG_CHANGE == type; -} - -inline bool is_use_added_list(const LogConfigChangeType type) -{ - return REPLACE_LEARNERS == type; -} - -inline bool is_use_removed_list(const LogConfigChangeType type) -{ - return REPLACE_LEARNERS == type; -} - -struct LogConfigChangeArgs -{ -public: - LogConfigChangeArgs(): - server_(), - curr_member_list_(), - curr_replica_num_(0), - new_replica_num_(0), - config_version_(), - ref_scn_(), - lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), - type_(INVALID_LOG_CONFIG_CHANGE_TYPE), - added_list_(), - removed_list_(), - new_member_list_() { } - - LogConfigChangeArgs(const common::ObMember &server, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const LogConfigChangeType type) - : server_(server), curr_member_list_(), curr_replica_num_(0), new_replica_num_(new_replica_num), - config_version_(config_version), ref_scn_(), lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), type_(type), - added_list_(), removed_list_(), new_member_list_() { } - - LogConfigChangeArgs(const common::ObMember &server, - const int64_t new_replica_num, - const LogConfigChangeType type) - : server_(server), curr_member_list_(), curr_replica_num_(0), new_replica_num_(new_replica_num), - config_version_(), ref_scn_(), lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), type_(type), added_list_(), removed_list_(), new_member_list_() { } - - LogConfigChangeArgs(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const LogConfigChangeType type) - : server_(), curr_member_list_(member_list), curr_replica_num_(curr_replica_num), new_replica_num_(new_replica_num), - config_version_(), ref_scn_(), lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), type_(type), added_list_(), removed_list_(), new_member_list_() { } - - LogConfigChangeArgs(const int64_t lock_owner, - const int64_t lock_type, - const LogConfigChangeType type) - : server_(), curr_member_list_(), curr_replica_num_(0), new_replica_num_(), - config_version_(), ref_scn_(), lock_owner_(lock_owner), - lock_type_(lock_type), type_(type), added_list_(), removed_list_(), new_member_list_() { } - - LogConfigChangeArgs(const common::ObMemberList &added_list, - const common::ObMemberList &removed_list, - const LogConfigChangeType type) - : server_(), curr_member_list_(), curr_replica_num_(0), - new_replica_num_(0), config_version_(), ref_scn_(), - lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), type_(type), - added_list_(added_list), removed_list_(removed_list), new_member_list_() { } - - LogConfigChangeArgs(const common::ObMemberList &new_member_list, - const int64_t new_replica_num, - const LogConfigChangeType type) - : server_(), curr_member_list_(), curr_replica_num_(0), - new_replica_num_(new_replica_num), config_version_(), ref_scn_(), - lock_owner_(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER), - lock_type_(ConfigChangeLockType::LOCK_NOTHING), type_(type), - added_list_(), removed_list_(), new_member_list_(new_member_list) { } - ~LogConfigChangeArgs() - { - reset(); - } - bool is_valid() const; - void reset(); - - TO_STRING_KV(K_(server), K_(curr_member_list), K_(curr_replica_num), K_(new_replica_num), - K_(config_version), K_(ref_scn), K_(lock_owner), K_(lock_type), K_(added_list), - K_(removed_list), K_(new_member_list), "type", LogConfigChangeType2Str(type_)); - common::ObMember server_; - common::ObMemberList curr_member_list_; - int64_t curr_replica_num_; - int64_t new_replica_num_; - LogConfigVersion config_version_; - share::SCN ref_scn_; - int64_t lock_owner_; - int64_t lock_type_; - LogConfigChangeType type_; - common::ObMemberList added_list_; - common::ObMemberList removed_list_; - common::ObMemberList new_member_list_; -}; - -struct LogReconfigBarrier -{ - LogReconfigBarrier() - : prev_log_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(PALF_INITIAL_LSN_VAL), - prev_end_lsn_(PALF_INITIAL_LSN_VAL), - prev_mode_pid_(INVALID_PROPOSAL_ID) { } - LogReconfigBarrier(const int64_t prev_log_pid, - const LSN &prev_lsn, - const LSN &prev_end_lsn, - const int64_t prev_mode_pid) - : prev_log_proposal_id_(prev_log_pid), - prev_lsn_(prev_lsn), - prev_end_lsn_(prev_end_lsn), - prev_mode_pid_(prev_mode_pid) { } - ~LogReconfigBarrier() { reset(); } - - void reset() - { - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - prev_end_lsn_.reset(); - prev_mode_pid_ = INVALID_PROPOSAL_ID; - } - TO_STRING_KV(K_(prev_log_proposal_id), K_(prev_lsn), K_(prev_end_lsn), K_(prev_mode_pid)); - // previous log proposal_id for barrier - int64_t prev_log_proposal_id_; - // previous lsn for barrier - LSN prev_lsn_; - LSN prev_end_lsn_; - // previous mode proposal_id for barrier - int64_t prev_mode_pid_; -}; - -class LogConfigMgr -{ -public: - LogConfigMgr(); - virtual ~LogConfigMgr(); -public: - virtual int init(const int64_t palf_id, - const common::ObAddr &self, - const LogConfigMeta &log_ms_meta, - LogEngine *log_engine, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - election::Election *election, - LogModeMgr *mode_mgr, - LogReconfirm *reconfirm, - LogPlugins *plugins); - virtual void destroy(); - - // require caller holds WLock in PalfHandleImpl - virtual int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t replica_num, - const common::GlobalLearnerList &learner_list, - const int64_t proposal_id, - LogConfigVersion &config_version); - // following get ops need caller holds RLock in PalfHandleImpl - virtual int64_t get_accept_proposal_id() const; - int get_global_learner_list(common::GlobalLearnerList &learner_list) const; - virtual int get_degraded_learner_list(common::GlobalLearnerList °raded_learner_list) const; - // @brief get the expected paxos member list without arbitraion member, - // including degraded members. - // This interface is generally used by outer modules, such as reporting replica logic. - // @param[in/out] ObMemberList, the output member list - // @param[in/out] int64_t, the output replica_num - // @retval - // return OB_SUCCESS if success - // else return other errno - virtual int get_curr_member_list(common::ObMemberList &member_list, - int64_t &replica_num) const; - // @brief get the paxos member list which are all alive, including arbitraion member, - // and excluding degraded members. - // This list can be used for config change/paxos prepare phase. - // This interface is only used by palf. - // @param[in/out] ObMemberList, the output member list - // @param[in/out] int64_t, the output replica_num - // @retval - // return OB_SUCCESS if success - // else return other errno - virtual int get_alive_member_list_with_arb(common::ObMemberList &member_list, - int64_t &replica_num) const; - // @brief get the paxos member list which is responsible for sync group log, excluding arbitraion member, - // and excluding degraded paxos members. - // This interface is only used by palf. - // @param[in/out] ObMemberList, the output member list - // @param[in/out] int64_t, the output replica_num - // @retval - // return OB_SUCCESS if success - // else return other errno - virtual int get_log_sync_member_list(common::ObMemberList &member_list, - int64_t &replica_num) const; - // @brief get the paxos member list which is responsible for generating - // committed_end_lsn in the leader, excluding arbitraion member - // and excluding degraded paxos members. - // This interface is only used by palf. - // @param[in/out] ObMemberList, the output member list - // @param[in/out] int64_t, the output replica_num - // @param[in/out] bool, whehter the current committed_end_lsn is smaller than - // log barrier of a reconfiguration. If the current committed_end_lsn is - // smaller(larger) than barrier, the output member_list and replica_num will be - // the configuration before(after) the reconfiguration. - // @param[in/out] barrier_lsn, log barrier of a reconfiguration (last_submit_end_lsn) - // @retval - // return OB_SUCCESS if success - // else return other errno - virtual int get_log_sync_member_list_for_generate_committed_lsn( - ObMemberList &prev_member_list, - int64_t &prev_replica_num, - ObMemberList &curr_member_list, - int64_t &curr_replica_num, - bool &is_before_barrier, - LSN &barrier_lsn) const; - virtual int get_prev_member_list(common::ObMemberList &member_list) const; - virtual int get_children_list(LogLearnerList &children) const; - virtual int get_log_sync_children_list(LogLearnerList &children) const; - virtual int get_config_version(LogConfigVersion &config_version) const; - // @brief get replica_num of expected paxos member list, excluding arbitraion member, - // and including degraded members. - // @param[in/out] int64_t, the output replica_num - // @retval - // return OB_SUCCESS if success - // else return other errno - virtual int get_replica_num(int64_t &replica_num) const; - const common::ObAddr &get_parent() const; - int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked); - virtual int leader_do_loop_work(bool &need_change_config); - virtual int switch_state(); - virtual int wait_log_barrier(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info) const; - virtual int renew_config_change_barrier(); - // ================= Config Change ================= - - int check_args_and_generate_config(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - bool &is_already_finished, - LogConfigInfoV2 &new_config_info) const; - int pre_sync_config_log_and_mode_meta(const common::ObMember &server, const int64_t proposal_id); - int start_change_config(int64_t &proposal_id, - int64_t &election_epoch, - const LogConfigChangeType &type); - int end_degrade(); - int is_state_changed(bool &need_rlock, bool &need_wlock) const; - int change_config(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version); - void after_config_change_timeout(const LogConfigVersion &config_version); - - // for reconfirm - virtual int confirm_start_working_log(const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version); - // for PalfHandleImpl::receive_config_log - virtual bool can_receive_config_log(const common::ObAddr &leader, const LogConfigMeta &meta) const; - virtual int after_flush_config_log(const LogConfigVersion &config_version); - // follower receives member change log and needs to perform forward validation - virtual int receive_config_log(const common::ObAddr &leader, const LogConfigMeta &meta); - - // for PalfHandleImpl::ack_config_log - virtual int ack_config_log(const common::ObAddr &sender, - const int64_t proposal_id, - const LogConfigVersion &config_version, - bool &is_majority); - virtual int after_config_log_majority(const int64_t proposal_id, - const LogConfigVersion &config_version); - int wait_config_log_persistence(const LogConfigVersion &config_version) const; - // broadcast leader info to global learners, only called in leader active - virtual int submit_broadcast_leader_info(const int64_t proposal_id) const; - virtual void reset_status(); - int check_follower_sync_status(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - bool &added_member_has_new_version) const; - int wait_log_barrier_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info) const; - int wait_log_barrier_before_start_working_(const LogConfigChangeArgs &args); - int sync_meta_for_arb_election_leader(); - void set_sync_to_degraded_learners(); - bool is_sync_to_degraded_learners() const; - int forward_initial_config_meta_to_arb(); - int force_set_member_list(const LogConfigChangeArgs &args, const int64_t proposal_id); - // ================ Config Change ================== - // ==================== Child ======================== - virtual int register_parent(); - virtual int handle_register_parent_resp(const LogLearner &server, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret); - // NB: no handle_retire_parent and retire_children - virtual int handle_retire_child(const LogLearner &parent); - virtual int handle_learner_keepalive_req(const LogLearner &parent); - int retire_parent(); - // failure detector - int check_parent_health(); - // ==================== Child ======================== - // ==================== Parent ======================== - int handle_register_parent_req(const LogLearner &child, const bool is_to_leader); - int handle_retire_parent(const LogLearner &child); - int handle_learner_keepalive_resp(const LogLearner &child); - void check_children_health(); - // ==================== Parent ======================== - int64_t to_string(char* buf, const int64_t buf_len) const - { - SpinLockGuard guard(lock_); - int64_t pos = 0; - J_OBJ_START(); - J_KV(K_(palf_id), K_(self), K_(alive_paxos_memberlist), K_(alive_paxos_replica_num), \ - K_(log_ms_meta), K_(running_args), K_(state), K_(checking_barrier), K_(reconfig_barrier), \ - K_(persistent_config_version), K_(ms_ack_list), K_(resend_config_version), K_(resend_log_list), \ - K_(last_submit_config_log_time_us), K_(need_change_config_bkgd), K_(bkgd_config_version), \ - K_(register_time_us), K_(parent), K_(parent_keepalive_time_us), \ - K_(last_submit_register_req_time_us), K_(children), K_(last_submit_keepalive_time_us), KP(this)); - J_OBJ_END(); - return pos; - } -private: - enum ConfigChangeState - { - INIT = 0, - CHANGING = 1, - }; - typedef common::ObSpinLockGuard SpinLockGuard; - typedef common::ObFunction LogLearnerCond; - typedef common::ObFunction LogLearnerAction; - static constexpr int64_t MAX_WAIT_BARRIER_TIME_US_FOR_RECONFIGURATION = 2 * 1000 * 1000; - static constexpr int64_t MAX_WAIT_BARRIER_TIME_US_FOR_STABLE_LOG = 1 * 1000 * 1000; -private: - int set_initial_config_info_(const LogConfigInfoV2 &config_info, - const int64_t proposal_id, - LogConfigVersion &init_config_version); - bool can_memberlist_majority_(const int64_t new_member_list_len, const int64_t new_replica_num) const; - int check_config_change_args_(const LogConfigChangeArgs &args, bool &is_already_finished) const; - int check_config_change_args_by_type_(const LogConfigChangeArgs &args, bool &is_already_finished) const; - int check_config_version_matches_state_(const LogConfigChangeType &type, const LogConfigVersion &config_version) const; - int generate_new_config_info_(const int64_t proposal_id, - const LogConfigChangeArgs &args, - LogConfigInfoV2 &new_config_info) const; - - - int append_config_info_(const LogConfigInfoV2 &config_info); - int apply_config_info_(const LogConfigInfoV2 &config_info); - int update_match_lsn_map_(const LogConfigChangeArgs &args, const LogConfigInfoV2 &new_config_info); - int update_election_meta_(const LogConfigInfoV2 &info); - int update_election_meta_(const ObMemberList &member_list, - const LogConfigVersion &config_version, - const int64_t new_replica_num); - int is_state_changed_(bool &need_rlock, bool &need_wlock) const; - int renew_config_change_barrier_(); - bool is_reach_majority_() const; - bool need_resend_config_log_() const; - bool need_recheck_init_state_() const; - bool is_leader_for_config_change_(const LogConfigChangeType &type, - const int64_t proposal_id, - const int64_t election_epoch) const; - int change_config_(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - LogConfigVersion &config_version); - int append_config_meta_(const int64_t curr_proposal_id, - const LogConfigChangeArgs &args, - bool &is_already_finished); - int set_resend_log_info_(); - int submit_config_log_(const common::ObMemberList &paxos_member_list, - const int64_t proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid, - const LogConfigMeta &config_meta); - int check_barrier_condition_(const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid) const; - bool check_need_update_memberlist_without_lock_(const LogConfigVersion &config_version) const; - int update_election_memberlist_(const LogConfigMeta &log_ms_meta); - // int check_ms_log_committed_(const int64_t proposal_id, const LogConfigVersion &config_version); - int try_resend_config_log_(const int64_t proposal_id); - // broadcast leader info to global learners, only called in leader active - int submit_broadcast_leader_info_(const int64_t proposal_id) const; - int check_servers_lsn_and_version_(const common::ObAddr &server, - const LogConfigVersion &config_version, - const int64_t conn_timeout_us, - const bool force_remote_check, - const bool need_purge_throttling, - LSN &max_flushed_end_lsn, - bool &has_same_version, - int64_t &last_slide_log_id) const; - int sync_get_committed_end_lsn_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - const bool need_purge_throttling, - const bool need_remote_check, - const int64_t conn_timeout_us, - LSN &committed_end_lsn, - bool &added_member_has_new_version, - LSN &added_member_flushed_end_lsn, - int64_t &added_member_last_slide_log_id) const; - int check_follower_sync_status_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - bool &added_member_has_new_version) const; - int pre_sync_config_log_and_mode_meta_(const common::ObMember &server, - const int64_t proposal_id, - const bool is_arb_replica); - int after_config_log_majority_(const int64_t proposal_id, - const LogConfigVersion &config_version); -private: -enum class RegisterParentReason -{ - INVALID = 0, - FIRST_REGISTER = 1, - PARENT_NOT_ALIVE = 2, - RETIRED_BY_PARENT = 4, -}; - -inline const char *register_parent_reason_2_str_(const RegisterParentReason &reason) const -{ - #define CHECK_REASON_STR(x) case(RegisterParentReason::x): return #x - switch(reason) - { - CHECK_REASON_STR(FIRST_REGISTER); - CHECK_REASON_STR(PARENT_NOT_ALIVE); - CHECK_REASON_STR(RETIRED_BY_PARENT); - default: - return "Invalid"; - } - #undef CHECK_REASON_STR -} - -enum class RetireParentReason -{ - INVALID = 0, - IS_FULL_MEMBER = 1, - PARENT_CHILD_LOOP = 3, -}; - -inline const char *retire_parent_reason_2_str_(const RetireParentReason &reason) const -{ - #define CHECK_REASON_STR(x) case(RetireParentReason::x): return #x - switch(reason) - { - CHECK_REASON_STR(IS_FULL_MEMBER); - CHECK_REASON_STR(PARENT_CHILD_LOOP); - default: - return "Invalid"; - } - #undef CHECK_REASON_STR -} - -enum class RetireChildReason -{ - INVALID = 0, - CHILDREN_LIST_FULL = 1, - CHILD_NOT_IN_LEARNER_LIST = 2, - CHILD_NOT_ALIVE = 3, - PARENT_DISABLE_SYNC = 6, -}; - -inline const char *retire_child_reason_2_str_(const RetireChildReason reason) const -{ - #define CHECK_REASON_STR(x) case(RetireChildReason::x): return #x - switch(reason) - { - CHECK_REASON_STR(CHILDREN_LIST_FULL); - CHECK_REASON_STR(CHILD_NOT_IN_LEARNER_LIST); - CHECK_REASON_STR(CHILD_NOT_ALIVE); - CHECK_REASON_STR(PARENT_DISABLE_SYNC); - default: - return "Invalid"; - } - #undef CHECK_REASON_STR -} - -private: - // inner_config_meta_ is protected by RWLock in PalfHandleImpl, - // any read/write ops to inner_config_meta_ should acquire RLock/WLock in PalfHandleImpl. - // ================= Config Change ================= - // ==================== Child ======================== - bool is_registering_() const; - void reset_registering_state_(); - int register_parent_(const RegisterParentReason &reason); - int after_register_parent_done_(const LogLearner &parent, const RegisterParentReason &reason) const; - int after_retire_parent_done_(const LogLearner &parent, const RetireParentReason &reason) const; - int get_register_leader_(common::ObAddr &leader) const; - int retire_parent_(const RetireParentReason &reason); - void reset_parent_info_(); - // ==================== Child ======================== - // ==================== Parent ======================== - int generate_candidate_list_(const LogLearner &child, LogCandidateList &candidate_list); - int generate_candidate_list_from_member_(const LogLearner &child, LogCandidateList &candidate_list); - int generate_candidate_list_from_children_(const LogLearner &child, LogCandidateList &candidate_list); - int remove_timeout_child_(LogLearnerList &dead_children); - int remove_child_is_not_learner_(LogLearnerList &removed_children); - int remove_children_(LogLearnerList &this_children, const LogLearnerList &removed_children); - int submit_retire_children_req_(const LogLearnerList &retired_children, const RetireChildReason &reason); - int children_if_cond_then_action_(const LogLearnerCond &cond, const LogLearnerAction &action); - int after_register_child_done_(const LogLearner &child) const; - int after_retire_child_done_(const LogLearner &child, const RetireChildReason &reason) const; - // ==================== Parent ======================== -private: - // inner_config_meta_ is protected by RWLock in PalfHandleImpl, - // any read/write ops should acquire RLock/WLock in PalfHandleImpl. - // inner_config_meta_, inner_alive_paxos_memberlist_, inner_alive_paxos_replica_num_, - // and inner_all_learnerlist_ take effect as long as they are accepted by the replica, - // they are used within LogConfigMgr - LogConfigMeta log_ms_meta_; - // list of all alive paxos members, including arbitration member - common::ObMemberList alive_paxos_memberlist_; - // number of all alive paxos members, including arbitration member - int64_t alive_paxos_replica_num_; - // list of all learners, including learners which has been degraded from acceptors - GlobalLearnerList all_learnerlist_; - LogConfigChangeArgs running_args_; - // this lock protects all states related to config change, except for inner_config_meta_ - mutable common::ObSpinLock lock_; - int64_t palf_id_; - common::ObAddr self_; - // ================= Config Change ================= - // barrier for reconfiguration - LogReconfigBarrier reconfig_barrier_; - // barrier for checking log before reconfiguration - LogReconfigBarrier checking_barrier_; - ConfigChangeState state_; - int64_t last_submit_config_log_time_us_; - // record ack to membership log - LogSimpleMemberList ms_ack_list_; - // need change_config with background thread - bool need_change_config_bkgd_; - LogConfigVersion bkgd_config_version_; - bool is_sw_interrupted_by_degrade_; - bool will_upgrade_; - int64_t last_start_upgrade_time_us_; - // In our current implement, leader won't send config change log to followers and learners - // after config change log has committed. Considering following scenario: - // Paxos group (A, B, C), their config version are all 1, user switches leader from A to B. - // B sends startworking log (config version 2) and this log is writed in B and C, not in A. - // In this time user wants to switch leader from B to A, but A's config version is smaller than - // majority's, so A can't be leader. - // To solve the above problem, leader should guarantee config change log should be writed in all - // paxos members and all learners. - // for resend config log - LogConfigVersion resend_config_version_; - ResendConfigLogList resend_log_list_; - // the epoch of leader who is executing config changing request - int64_t election_leader_epoch_; - int64_t last_broadcast_leader_info_time_us_; - LogConfigVersion persistent_config_version_; - mutable int64_t barrier_print_log_time_; - mutable int64_t last_check_init_state_time_us_; - mutable int64_t check_config_print_time_; - mutable int64_t start_wait_barrier_time_us_; - mutable int64_t last_wait_barrier_time_us_; - mutable LSN last_wait_committed_end_lsn_; - int64_t last_sync_meta_for_arb_election_leader_time_us_; - int64_t forwarding_config_proposal_id_; - // ================= Config Change ================= - // ==================== Child ======================== - mutable common::ObSpinLock parent_lock_; - int64_t register_time_us_; - RegisterParentReason register_parent_reason_; - common::ObAddr parent_; - int64_t parent_keepalive_time_us_; - // registering state - int64_t last_submit_register_req_time_us_; - // control register req frequency - int64_t last_first_register_time_us_; - // ==================== Child ======================== - // ==================== Parent ======================== - mutable common::ObSpinLock child_lock_; - LogLearnerList children_; - // cached children_ for pushing logs - // log_sync_children_ = children_ - migrating learners - learners not in learnerlist_ - LogLearnerList log_sync_children_; - int64_t last_submit_keepalive_time_us_; - // ==================== Parent ======================== - LogEngine *log_engine_; - LogSlidingWindow *sw_; - LogStateMgr *state_mgr_; - election::Election* election_; - LogModeMgr *mode_mgr_; - LogReconfirm *reconfirm_; - LogPlugins *plugins_; - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(LogConfigMgr); -}; -} // namespace palf -} // namespace oceanbase - -#endif // OCEANBASE_LOGSERVICE_LOG_CONFIG_MGR_ diff --git a/src/logservice/palf/log_define.cpp b/src/logservice/palf/log_define.cpp index 543dfa515..549d6e95c 100644 --- a/src/logservice/palf/log_define.cpp +++ b/src/logservice/palf/log_define.cpp @@ -128,10 +128,7 @@ const char *get_purge_throttling_type_str(PurgeThrottlingType type) const char *result= "UNKNOWN_TYPE"; switch (type) { case INVALID_PURGE_TYPE : - result = "PURGE_BY_RECONFIRM"; - break; - case PURGE_BY_RECONFIRM: - result = "PURGE_BY_RECONFIRM"; + result = "INVALID_PURGE_TYPE"; break; case PURGE_BY_CHECK_BARRIER_CONDITION: result = "PURGE_BY_CHECK_BARRIER_CONDITION"; diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h index 635364369..16f14b6e9 100644 --- a/src/logservice/palf/log_define.h +++ b/src/logservice/palf/log_define.h @@ -56,7 +56,7 @@ namespace palf { #define TMP_SUFFIX ".tmp" -#define PALF_EVENT(info_string, palf_id, args...) FLOG_INFO("[PALF_EVENT] "info_string, "palf_id", palf_id, args) +#define PALF_EVENT(info_string, args...) FLOG_INFO("[PALF_EVENT] " info_string, args) #define PALF_REPORT_INFO_KV(args...) \ const int64_t MAX_INFO_LENGTH = 512; \ @@ -67,7 +67,6 @@ ::oceanbase::common::databuff_print_kv(EXTRA_INFOS, MAX_INFO_LENGTH, pos, ##args typedef int FileDesc; typedef uint64_t block_id_t ; typedef uint64_t offset_t; -constexpr int64_t INVALID_PALF_ID = -1; class LSN; class LogWriteBuf; class ILogBlockPool; @@ -107,57 +106,23 @@ typedef common::ObFixedArray LSNArray; typedef common::ObFixedArray LogWriteBufArray; // ==================== block and log end =========================== -// ====================== Consensus begin =========================== -// follower's group buffer size is as same as leader's. -constexpr int64_t LEADER_DEFAULT_GROUP_BUFFER_SIZE = 1 << 22; -constexpr int64_t FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE = LEADER_DEFAULT_GROUP_BUFFER_SIZE + 0L; +// ====================== Local log state begin ===================== +constexpr int64_t DEFAULT_GROUP_BUFFER_SIZE = 1 << 22; const int64_t PALF_STAT_PRINT_INTERVAL_US = 1 * 1000 * 1000L; // The advance delay threshold for match lsn is 1s. const int64_t PALF_IO_STAT_PRINT_INTERVAL_US = 10 * 1000 * 1000L; const int64_t MATCH_LSN_ADVANCE_DELAY_THRESHOLD_US = 1 * 1000 * 1000L; -const int64_t PALF_RECONFIRM_FETCH_MAX_LSN_INTERVAL = 1 * 1000 * 1000; -const int64_t PALF_FETCH_LOG_INTERVAL_US = 2 * 1000 * 1000L; // 2s -// Control the fetch interval trigger by outer(eg. config change pre check) by 500ms. -const int64_t PALF_FETCH_LOG_OUTER_TRIGGER_INTERVAL_US = 500 * 1000; // 500 ms -const int64_t PALF_FETCH_LOG_RENEW_LEADER_INTERVAL_US = 5 * 1000 * 1000; // 5s -const int64_t PALF_LEADER_RECONFIRM_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // 10s -const int64_t PREPARE_LOG_BUFFER_SIZE = 2048; -const int64_t PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // 10s const int32_t PALF_MAX_REPLAY_TIMEOUT = 500 * 1000; const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 100 * 1000; // 100ms const int32_t LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US = 1 * 1000; // 1ms const int64_t PALF_SLIDING_WINDOW_SIZE = 1 << 11; // must be 2^n(n>0), default 2^11 = 2048 -const int64_t PALF_MAX_LEADER_SUBMIT_LOG_COUNT = PALF_SLIDING_WINDOW_SIZE / 2; // max number of concurrent submitting group log in leader -const int64_t PALF_RESEND_CONFIG_LOG_INTERVAL_US = 500 * 1000L; // 500 ms -const int64_t PALF_RESEND_CONFIG_LOG_FOR_ARB_INTERVAL_US = 10 * 1000L; // 10 ms -const int64_t PALF_BROADCAST_LEADER_INFO_INTERVAL_US = 5 * 1000 * 1000L; // 5s +const int64_t PALF_MAX_SUBMIT_LOG_COUNT = PALF_SLIDING_WINDOW_SIZE / 2; const int64_t FIRST_VALID_LOG_ID = 1; // The first valid log_id is 1. -const int64_t PALF_PARENT_CHILD_TIMEOUT_US = 4 * 1000 * 1000L; // 4000ms, 4s -const int64_t PALF_PARENT_KEEPALIVE_INTERVAL_US = 1 * 1000 * 1000L; // 1000ms, 1s -const int64_t PALF_CHILD_RESEND_REGISTER_INTERVAL_US = 4 * 1000 * 1000L; // 4000ms -const int64_t PALF_CHECK_PARENT_CHILD_INTERVAL_US = 1 * 1000 * 1000; // 1000ms const int64_t PALF_DUMP_DEBUG_INFO_INTERVAL_US = 10 * 1000 * 1000; // 10s -const int64_t PALF_UPDATE_CACHED_STAT_INTERVAL_US = 500 * 1000; // 500 ms -const int64_t PALF_SYNC_RPC_TIMEOUT_US = 2 * 1000 * 1000; // 2 s -const int64_t PALF_LOG_SYNC_DELAY_THRESHOLD_US = 3 * 1000 * 1000L; // 3 s -constexpr int64_t INVALID_PROPOSAL_ID = INT64_MAX; -constexpr int64_t PALF_MAX_PROPOSAL_ID = INT64_MAX - 1; -constexpr int64_t PALF_INITIAL_PROPOSAL_ID = 1; constexpr char PADDING_LOG_CONTENT_CHAR = '\0'; const int64_t MIN_WRITING_THTOTTLING_TRIGGER_PERCENTAGE = 40; constexpr int64_t PALF_IO_WAIT_EVENT_TIMEOUT_MS = 100; -inline int64_t max_proposal_id(const int64_t a, const int64_t b) -{ - if ((INVALID_PROPOSAL_ID == a && INVALID_PROPOSAL_ID == b) || - (INVALID_PROPOSAL_ID != a && INVALID_PROPOSAL_ID == b)) { - return a; - } else if (INVALID_PROPOSAL_ID == a && INVALID_PROPOSAL_ID != b) { - return b; - } else { - return MAX(a, b); - } -} // ====================== Consensus end ============================== // =========== LSN begin ============== @@ -183,33 +148,24 @@ constexpr int64_t LOG_BATCH_PUSH_LOG_RESP = 2; constexpr offset_t LOG_CACHE_ALIGN_SIZE = 64 * 1024; // ========== LogCache end ================= -const int64_t OB_INVALID_CONFIG_CHANGE_LOCK_OWNER = -1; - -enum ObReplicaState { +enum LogState { INVALID_STATE = 0, INIT = 1, ACTIVE = 2, - RECONFIRM = 3, - PENDING = 4, + RECOVERING = 3, }; -const int64_t SYS_PALF_ID = 1; -inline bool is_sys_palf_id(int64_t palf_id) -{ - return SYS_PALF_ID == palf_id; -} -inline const char *replica_state_to_string(const ObReplicaState &state) +inline const char *log_state_to_string(const LogState state) { - #define CHECK_OB_REPLICA_STATE(x) case(ObReplicaState::x): return #x + #define CHECK_LOG_STATE(x) case(LogState::x): return #x switch (state) { - CHECK_OB_REPLICA_STATE(INIT); - CHECK_OB_REPLICA_STATE(ACTIVE); - CHECK_OB_REPLICA_STATE(RECONFIRM); - CHECK_OB_REPLICA_STATE(PENDING); + CHECK_LOG_STATE(INIT); + CHECK_LOG_STATE(ACTIVE); + CHECK_LOG_STATE(RECOVERING); default: return "INVALID_STATE"; } - #undef CHECK_OB_REPLICA_STATE + #undef CHECK_LOG_STATE } enum LogType @@ -221,40 +177,6 @@ enum LogType LOG_TYPE_MAX = 1000 }; -enum LogReplicaType -{ - INVALID_REPLICA = 0, - NORMAL_REPLICA, // full replica - ARBITRATION_REPLICA, // arbitration replica -}; - -inline const char *replica_type_2_str(const LogReplicaType state) -{ - #define CHECK_REPLICA_TYPE_STR(x) case(LogReplicaType::x): return #x - switch(state) - { - CHECK_REPLICA_TYPE_STR(NORMAL_REPLICA); - CHECK_REPLICA_TYPE_STR(ARBITRATION_REPLICA); - default: - return "InvalidReplicaType"; - } - #undef CHECK_REPLICA_TYPE_STR -} - - -inline int log_replica_type_to_string(const LogReplicaType replica_type, char *str_buf_, const int64_t str_len) -{ - int ret = OB_SUCCESS; - if (LogReplicaType::NORMAL_REPLICA == replica_type) { - strncpy(str_buf_, "NORMAL_REPLICA", str_len); - } else if (LogReplicaType::ARBITRATION_REPLICA == replica_type) { - strncpy(str_buf_, "ARBITRATION_REPLICA", str_len); - } else { - ret = OB_INVALID_ARGUMENT; - } - return ret; -} - inline bool is_valid_log_id(const int64_t log_id) { return (log_id > 0); @@ -327,11 +249,6 @@ inline bool palf_reach_time_interval(const int64_t interval, int64_t &warn_time) return bool_ret; } -inline bool is_valid_palf_id(const int64_t id) -{ - return 0 <= id; -} - inline bool is_valid_file_desc(const FileDesc &fd) { return 0 <= fd; @@ -350,61 +267,14 @@ int convert_sys_errno(); bool is_number(const char *); -struct LSKey { - LSKey() : id_(-1) {} - explicit LSKey(const int64_t id) : id_(id) {} - ~LSKey() {id_ = -1;} - LSKey(const LSKey &key) { this->id_ = key.id_; } - LSKey &operator=(const LSKey &other) - { - this->id_ = other.id_; - return *this; - } - - bool operator==(const LSKey &palf_id) const - { - return this->compare(palf_id) == 0; - } - bool operator!=(const LSKey &palf_id) const - { - return this->compare(palf_id) != 0; - } - uint64_t hash() const - { - uint64_t hash_val = 0; - hash_val = common::murmurhash(&hash_val, sizeof(id_), id_); - return hash_val; - } - int hash(uint64_t &hash_val) const - { - hash_val = hash(); - return OB_SUCCESS; - } - int compare(const LSKey &palf_id) const - { - if (palf_id.id_ < id_) { - return 1; - } else if (palf_id.id_ == id_) { - return 0; - } else { - return -1; - } - } - void reset() {id_ = -1;} - bool is_valid() const {return -1 != id_;} - int64_t id_; - TO_STRING_KV(K_(id)); -}; - enum PurgeThrottlingType { INVALID_PURGE_TYPE = 0, - PURGE_BY_RECONFIRM = 1, - PURGE_BY_CHECK_BARRIER_CONDITION = 2, - PURGE_BY_PRE_CHECK_FOR_CONFIG = 3, - PURGE_BY_CHECK_SERVERS_LSN_AND_VERSION = 4, - PURGE_BY_GET_MC_REQ = 5, - PURGE_BY_NOTIFY_FETCH_LOG = 6, + PURGE_BY_CHECK_BARRIER_CONDITION = 1, + PURGE_BY_PRE_CHECK_FOR_CONFIG = 2, + PURGE_BY_CHECK_SERVERS_LSN_AND_VERSION = 3, + PURGE_BY_GET_MC_REQ = 4, + PURGE_BY_NOTIFY_FETCH_LOG = 5, MAX_PURGE_TYPE }; @@ -414,7 +284,6 @@ inline const char *purge_throttling_type_2_str(const PurgeThrottlingType type) switch(type) { EXTRACT_PURGE_TYPE(INVALID_PURGE_TYPE); - EXTRACT_PURGE_TYPE(PURGE_BY_RECONFIRM); EXTRACT_PURGE_TYPE(PURGE_BY_CHECK_BARRIER_CONDITION); EXTRACT_PURGE_TYPE(PURGE_BY_PRE_CHECK_FOR_CONFIG); EXTRACT_PURGE_TYPE(PURGE_BY_CHECK_SERVERS_LSN_AND_VERSION); diff --git a/src/logservice/palf/log_engine.cpp b/src/logservice/palf/log_engine.cpp index b5d7e2e83..e866dc49d 100644 --- a/src/logservice/palf/log_engine.cpp +++ b/src/logservice/palf/log_engine.cpp @@ -39,12 +39,10 @@ LogEngine::LogEngine() : log_meta_(), log_meta_storage_(), log_storage_(), - log_net_service_(), alloc_mgr_(NULL), log_io_worker_(NULL), log_shared_queue_th_(NULL), plugins_(NULL), - palf_id_(INVALID_PALF_ID), palf_epoch_(-1), last_purge_throttling_ts_(OB_INVALID_TIMESTAMP), is_inited_(false) @@ -62,11 +60,11 @@ int LogEngine::append_log_meta_(const LogMeta &log_meta) if (NULL == (buf = reinterpret_cast(mtl_malloc(buf_len, "LogEngine")))) { ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(ERROR, "allocate memory failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "allocate memory failed", K(ret), K_(is_inited)); } else if (OB_FAIL(serialize_log_meta_(log_meta, buf, buf_len))) { - PALF_LOG(ERROR, "serialize_log_meta_ failed", K(ret), K_(palf_id), K_(is_inited), K(log_meta)); + PALF_LOG(ERROR, "serialize_log_meta_ failed", K(ret), K_(is_inited), K(log_meta)); } else if (OB_FAIL(log_meta_storage_.append_meta(buf, buf_len))) { - PALF_LOG(ERROR, "log_meta_storage_ append failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "log_meta_storage_ append failed", K(ret), K_(is_inited)); } if (NULL != buf) { mtl_free(buf); @@ -74,13 +72,11 @@ int LogEngine::append_log_meta_(const LogMeta &log_meta) return ret; } -int LogEngine::init(const int64_t palf_id, - const char *base_dir, +int LogEngine::init(const char *base_dir, const LogMeta &log_meta, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, LogCache *log_cache, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, LogPlugins *plugins, @@ -99,16 +95,15 @@ int LogEngine::init(const int64_t palf_id, }; if (IS_INIT) { ret = OB_INIT_TWICE; - PALF_LOG(ERROR, "LogEngine has inited!!!", K(ret), K(palf_id)); - } else if (false == is_valid_palf_id(palf_id) || OB_ISNULL(base_dir) || OB_ISNULL(alloc_mgr) - || OB_ISNULL(log_rpc) || OB_ISNULL(log_io_worker) + PALF_LOG(ERROR, "LogEngine has inited!!!", K(ret)); + } else if (OB_ISNULL(base_dir) || OB_ISNULL(alloc_mgr) + || OB_ISNULL(log_io_worker) || OB_ISNULL(plugins) || OB_ISNULL(io_adapter)) { ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid argument!!!", K(ret), - K(palf_id), - K(base_dir), + K(base_dir), K(log_meta), K(log_cache), K(alloc_mgr), @@ -120,7 +115,6 @@ int LogEngine::init(const int64_t palf_id, } else if (OB_FAIL(log_meta_storage_.init(base_dir, "meta", LSN(PALF_INITIAL_LSN_VAL), - palf_id, log_meta_storage_block_ize, LOG_DIO_ALIGN_SIZE, LOG_DIO_ALIGNED_BUF_SIZE_META, @@ -129,12 +123,11 @@ int LogEngine::init(const int64_t palf_id, plugins, NULL /*set log_cache to NULL for meta storage*/, io_adapter))) { - PALF_LOG(ERROR, "LogMetaStorage init failed", K(ret), K(palf_id), K(base_dir)); + PALF_LOG(ERROR, "LogMetaStorage init failed", K(ret), K(base_dir)); } else if(0 != log_storage_block_size && OB_FAIL(log_storage_.init(base_dir, "log", log_meta.get_log_snapshot_meta().base_lsn_, - palf_id, log_storage_block_size, LOG_DIO_ALIGN_SIZE, LOG_DIO_ALIGNED_BUF_SIZE_REDO, @@ -143,13 +136,10 @@ int LogEngine::init(const int64_t palf_id, plugins, log_cache, io_adapter))) { - PALF_LOG(ERROR, "LogStorage init failed!!!", K(ret), K(palf_id), K(base_dir), K(log_meta)); - } else if (OB_FAIL(log_net_service_.init(palf_id, log_rpc))) { - PALF_LOG(ERROR, "LogNetService init failed", K(ret), K(palf_id)); + PALF_LOG(ERROR, "LogStorage init failed!!!", K(ret), K(base_dir), K(log_meta)); } else if (OB_FAIL(append_log_meta_(log_meta))) { PALF_LOG(ERROR, "append_log_meta_ failed", K(ret)); } else { - palf_id_ = palf_id; log_meta_ = log_meta; alloc_mgr_ = alloc_mgr; log_io_worker_ = log_io_worker; @@ -158,7 +148,7 @@ int LogEngine::init(const int64_t palf_id, palf_epoch_ = palf_epoch; base_lsn_for_block_gc_ = log_meta.get_log_snapshot_meta().base_lsn_; is_inited_ = true; - PALF_LOG(INFO, "LogEngine init success", K(ret), K(palf_id), K(base_dir), K(palf_epoch)); + PALF_LOG(INFO, "LogEngine init success", K(ret), K(base_dir), K(palf_epoch)); } if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { destroy(); @@ -169,13 +159,11 @@ int LogEngine::init(const int64_t palf_id, void LogEngine::destroy() { if (IS_INIT) { - PALF_LOG(INFO, "LogEngine destroy", K_(palf_id), K_(is_inited)); + PALF_LOG(INFO, "LogEngine destroy", K_(is_inited)); is_inited_ = false; - palf_id_ = INVALID_PALF_ID; log_io_worker_ = NULL; log_shared_queue_th_ = NULL; alloc_mgr_ = NULL; - log_net_service_.destroy(); log_meta_storage_.destroy(); log_meta_.reset(); log_storage_.destroy(); @@ -188,12 +176,10 @@ void LogEngine::destroy() } } -int LogEngine::load(const int64_t palf_id, - const char *base_dir, +int LogEngine::load(const char *base_dir, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, LogCache *log_cache, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, LogPlugins *plugins, @@ -230,16 +216,15 @@ int LogEngine::load(const int64_t palf_id, LogMetaEntryHeader unused_meta_entry_header; if (IS_INIT) { ret = OB_INIT_TWICE; - PALF_LOG(ERROR, "LogEngine has initted!!!", K(ret), K(palf_id)); - } else if (false == is_valid_palf_id(palf_id) || OB_ISNULL(base_dir) - || OB_ISNULL(alloc_mgr) || OB_ISNULL(log_rpc) || OB_ISNULL(log_io_worker) || OB_ISNULL(io_adapter)) { + PALF_LOG(ERROR, "LogEngine has initted!!!", K(ret)); + } else if (OB_ISNULL(base_dir) + || OB_ISNULL(alloc_mgr) || OB_ISNULL(log_io_worker) || OB_ISNULL(io_adapter)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(base_dir), K(log_cache), K(alloc_mgr), + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(base_dir), K(log_cache), K(alloc_mgr), K(log_io_worker), KP(io_adapter)); } else if (OB_FAIL(log_meta_storage_.load(base_dir, "meta", LSN(PALF_INITIAL_LSN_VAL), - palf_id, log_meta_storage_block_ize, LOG_DIO_ALIGN_SIZE, LOG_DIO_ALIGNED_BUF_SIZE_META, @@ -250,30 +235,27 @@ int LogEngine::load(const int64_t palf_id, io_adapter, unused_meta_entry_header, last_meta_entry_start_lsn))) { - PALF_LOG(ERROR, "LogMetaStorage load failed", K(ret), K(palf_id)); + PALF_LOG(ERROR, "LogMetaStorage load failed", K(ret)); } else if (OB_FAIL(construct_log_meta_(last_meta_entry_start_lsn, expected_next_block_id))) { PALF_LOG(ERROR, "construct_log_meta_ failed", K(ret)); } else if (FALSE_IT(guard.click("load log_meta_storage")) || (0 != log_storage_block_size && OB_FAIL(log_storage_.load(base_dir, "log", - log_meta_.get_log_snapshot_meta().base_lsn_, palf_id, + log_meta_.get_log_snapshot_meta().base_lsn_, log_storage_block_size, LOG_DIO_ALIGN_SIZE, LOG_DIO_ALIGNED_BUF_SIZE_REDO, log_storage_update_manifest_cb, log_block_pool, plugins, log_cache, io_adapter, entry_header, last_group_entry_header_lsn)))) { - PALF_LOG(ERROR, "LogStorage load failed", K(ret), K(palf_id), K(base_dir)); + PALF_LOG(ERROR, "LogStorage load failed", K(ret), K(base_dir)); } else if (FALSE_IT(guard.click("load log_storage")) || (0 != log_storage_block_size && OB_FAIL(try_clear_up_holes_and_check_storage_integrity_( last_group_entry_header_lsn, expected_next_block_id, entry_header)))) { PALF_LOG(ERROR, "the last block may be deleted by human, restart failed!!!", K(ret), - K_(palf_id), K_(is_inited)); + K_(is_inited)); } else if (OB_FAIL(integrity_verify_(last_meta_entry_start_lsn, last_group_entry_header_lsn, is_integrity))) { PALF_LOG(ERROR, "integrity_verify_ failed, unexpected error", K(ret), KPC(this)); - } else if (OB_FAIL(log_net_service_.init(palf_id, log_rpc))) { - PALF_LOG(ERROR, "LogNetService init failed", K(ret), K(palf_id)); } else { - palf_id_ = palf_id; palf_epoch_ = palf_epoch; alloc_mgr_ = alloc_mgr; log_io_worker_ = log_io_worker; @@ -282,7 +264,7 @@ int LogEngine::load(const int64_t palf_id, is_inited_ = true; PALF_LOG(INFO, "LogEngine load success", - K_(palf_id), K_(is_inited), + K_(is_inited), K(entry_header), K(log_storage_), K(log_meta_storage_), @@ -375,68 +357,7 @@ int LogEngine::submit_handle_submit_task() return ret; } -int LogEngine::submit_flush_prepare_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogPrepareMeta &prepare_meta) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret)); - } else if (false == flush_meta_cb_ctx.is_valid() || false == prepare_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(flush_meta_cb_ctx), K(prepare_meta)); - } else if (OB_FAIL(log_meta_.update_log_prepare_meta(prepare_meta))) { - PALF_LOG(ERROR, "LogMeta update_log_prepare_meta failed", K(ret), K_(palf_id), K_(is_inited)); - } else if (OB_FAIL(submit_flush_meta_task_(flush_meta_cb_ctx, log_meta_))) { - PALF_LOG(WARN, "submit_flush_meta_task_ failed", K(ret)); - } else { - PALF_LOG(INFO, "submit_flush_prepare_meta_task success", K(ret), K(flush_meta_cb_ctx), K(prepare_meta)); - } - return ret; -} -int LogEngine::submit_flush_change_config_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogConfigMeta &config_meta) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEnginenot inited!!!", K(ret)); - } else if (false == flush_meta_cb_ctx.is_valid() || false == config_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(flush_meta_cb_ctx), K(config_meta)); - } else if (OB_FAIL(log_meta_.update_log_config_meta(config_meta))) { - PALF_LOG(ERROR, "LogMeta update_log_config_meta failed", K(ret), K_(palf_id), K_(is_inited)); - } else if (OB_FAIL(submit_flush_meta_task_(flush_meta_cb_ctx, log_meta_))) { - PALF_LOG(WARN, "submit_flush_meta_task_ failed", K(ret)); - } else { - PALF_LOG(INFO, "submit_flush_change_config_meta_task success", K(ret), K(flush_meta_cb_ctx), K(config_meta)); - } - return ret; -} - -int LogEngine::submit_flush_mode_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEnginenot inited!!!", K(ret)); - } else if (false == flush_meta_cb_ctx.is_valid() || false == mode_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(flush_meta_cb_ctx), K(mode_meta)); - } else if (OB_FAIL(log_meta_.update_log_mode_meta(mode_meta))) { - PALF_LOG(ERROR, "LogMeta update_log_mode_meta failed", K(ret), K_(palf_id), K_(is_inited)); - } else if (OB_FAIL(submit_flush_meta_task_(flush_meta_cb_ctx, log_meta_))) { - PALF_LOG(WARN, "submit_flush_meta_task_ failed", K(ret)); - } else { - PALF_LOG(INFO, "submit_flush_mode_meta_task success", K(ret), K(flush_meta_cb_ctx), K(mode_meta)); - } - return ret; -} int LogEngine::submit_flush_snapshot_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogSnapshotMeta &log_snapshot_meta) @@ -457,61 +378,13 @@ int LogEngine::submit_flush_snapshot_meta_task(const FlushMetaCbCtx &flush_meta_ } else if (OB_FAIL(submit_flush_meta_task_(flush_meta_cb_ctx, log_meta_))) { // revert snapshot meta, otherwise next flush request will be ignored log_meta_.update_log_snapshot_meta(prev_log_snapshot_meta); - PALF_LOG(WARN, "submit_flush_snapshot_meta_task_ failed", K(ret), K(flush_meta_cb_ctx), K_(palf_id), K_(is_inited)); + PALF_LOG(WARN, "submit_flush_snapshot_meta_task_ failed", K(ret), K(flush_meta_cb_ctx), K_(is_inited)); } else { PALF_LOG(TRACE, "submit_flush_snapshot_meta_task success", K(ret), K(flush_meta_cb_ctx)); } return ret; } -int LogEngine::submit_flush_replica_property_meta_task( - const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogReplicaPropertyMeta &log_replica_property_meta) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret)); - } else if (false == flush_meta_cb_ctx.is_valid() - || false == log_replica_property_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(flush_meta_cb_ctx), - K(log_replica_property_meta)); - } else if (OB_FAIL(log_meta_.update_log_replica_property_meta(log_replica_property_meta))) { - PALF_LOG(WARN, "update_log_replica_property_meta failed", K(log_replica_property_meta)); - } else if (OB_FAIL(submit_flush_meta_task_(flush_meta_cb_ctx, log_meta_))) { - PALF_LOG(WARN, "submit_flush_replica_property_meta_task_ failed", K(ret), K(flush_meta_cb_ctx), K_(palf_id), K_(is_inited)); - } else { - PALF_LOG(TRACE, "submit_flush_replica_property_meta_task_ success", K(ret), K(flush_meta_cb_ctx)); - } - return ret; -} - -int LogEngine::submit_truncate_log_task(const TruncateLogCbCtx &truncate_log_cb_ctx) -{ - int ret = OB_SUCCESS; - LogIOTruncateLogTask *truncate_log_task = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret)); - } else if (false == truncate_log_cb_ctx.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(truncate_log_cb_ctx)); - } else if (OB_FAIL(generate_truncate_log_task_(truncate_log_cb_ctx, truncate_log_task))) { - PALF_LOG(ERROR, "generate_truncate_log_task_ failed", K(ret), K(truncate_log_cb_ctx)); - } else if (OB_FAIL(log_io_worker_->submit_io_task(truncate_log_task))) { - PALF_LOG(WARN, "submit_io_task failed", K(ret)); - } else { - PALF_LOG(INFO, "submit_truncate_log_task success", K(ret), K(truncate_log_cb_ctx)); - } - if (OB_FAIL(ret) && OB_NOT_NULL(truncate_log_task)) { - alloc_mgr_->free_log_io_truncate_log_task(truncate_log_task); - truncate_log_task = NULL; - } - return ret; -} - int LogEngine::submit_truncate_prefix_blocks_task( const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx) { @@ -577,15 +450,15 @@ int LogEngine::append_log(const LSN &lsn, const LogWriteBuf &write_buf, const SC int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == lsn.is_valid() || false == write_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn), K(write_buf)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(lsn), K(write_buf)); } else if (OB_FAIL(log_storage_.writev(lsn, write_buf, scn))) { - PALF_LOG(ERROR, "LogStorage append_log failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogStorage append_log failed", K(ret), K_(is_inited)); } else { PALF_LOG( - TRACE, "LogEngine append_log success", K(ret), K_(palf_id), K_(is_inited), K(lsn), K(write_buf), K(scn)); + TRACE, "LogEngine append_log success", K(ret), K_(is_inited), K(lsn), K(write_buf), K(scn)); } return ret; } @@ -596,9 +469,9 @@ int LogEngine::append_log(const LSNArray &lsn_array, const LogWriteBufArray &wri int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (OB_FAIL(log_storage_.writev(lsn_array, write_buf_array, scn_array))) { - PALF_LOG(ERROR, "LogStorage writev failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogStorage writev failed", K(ret), K_(is_inited)); } else { } return ret; @@ -613,10 +486,10 @@ int LogEngine::read_log(const LSN &lsn, LogIOContext io_ctx(LogIOUser::DEFAULT); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == lsn.is_valid() || 0 >= in_read_size || false == read_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn), K(in_read_size), K(read_buf)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(lsn), K(in_read_size), K(read_buf)); } else if (OB_FAIL(log_storage_.pread(lsn, in_read_size, read_buf, out_read_size, io_ctx))) { PALF_LOG(ERROR, "LogEngine read_log failed", K(ret), K(lsn), K(in_read_size), K(read_buf)); } else { @@ -634,7 +507,7 @@ int LogEngine::read_group_entry_header(const LSN &lsn, LogGroupEntryHeader &log_ ReadBuf &read_buf = read_buf_guard.read_buf_; int64_t out_read_size = 0; int64_t pos = 0; - LogIOContext io_ctx(palf_id_, LogIOUser::META_INFO); + LogIOContext io_ctx(LogIOUser::META_INFO); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -668,14 +541,14 @@ int LogEngine::truncate(const LSN &lsn) LogGroupEntryHeader log_group_entry_header; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(lsn)); } else if (OB_FAIL(read_group_entry_header(lsn, log_group_entry_header)) && OB_ERR_OUT_OF_UPPER_BOUND != ret) { PALF_LOG(ERROR, "read_group_entry_header failed, unexpected error, lsn must be the start position" - "of one LogGroupEntry", K(ret), K(lsn), K_(palf_id), K_(is_inited)); + "of one LogGroupEntry", K(ret), K(lsn), K_(is_inited)); } else if (OB_FAIL(log_storage_.truncate(lsn))) { PALF_LOG(ERROR, "LogStorage truncate failed", K(ret), K(lsn)); } else { @@ -690,12 +563,12 @@ int LogEngine::truncate_prefix_blocks(const LSN &lsn) int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(lsn)); } else if (OB_FAIL(log_storage_.truncate_prefix_blocks(lsn))) { - PALF_LOG(WARN, "truncate_prefix_blocks failed", K(ret), K_(palf_id), K_(is_inited), K(lsn)); + PALF_LOG(WARN, "truncate_prefix_blocks failed", K(ret), K_(is_inited), K(lsn)); } else { (void)reset_min_block_info_(); PALF_LOG(INFO, "truncate_prefix_blocks success", KPC(this), K(lsn)); @@ -715,14 +588,14 @@ int LogEngine::delete_block(const block_id_t &block_id) SCN next_block_max_scn; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == is_valid_block_id(block_id)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(block_id)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(block_id)); } else if (OB_FAIL(log_storage_.delete_block(block_id))) { PALF_LOG(WARN, "LogStorage delete block failed", K(ret), K(block_id)); } else { - PALF_LOG(INFO, "delete success", K(block_id), K_(palf_id), K_(is_inited)); + PALF_LOG(INFO, "delete success", K(block_id), K_(is_inited)); } // the block whose names with 'block_id' may be concurrently delete by rebuild, @@ -749,7 +622,7 @@ int LogEngine::delete_block(const block_id_t &block_id) // min_block_info_cache_version is important, otherwise, the cache of min block info // (i.e. min_block_min_scn_, min_block_max_scn_) is incorrect, consider following case: // T1 timestamp, thread A call delete_block, and get_block_min_scn successfully(e.g. next_block_id + 1 is 12); - // T2 timestamp, thread B calls truncate, the content of block whose names with 12 is overwritten; + // T2 timestamp, thread B calls truncate, the content of block 12 is overwritten; // T3 timestamp, thread A call set_min_block_info_for_gc_, and the min_block_info(i.e. min_block_max_scn_, min_block_min_scn_) // will reset to incorrect scn. set_min_block_info_for_gc_(min_block_info_cache_version, next_block_id, next_block_max_scn); @@ -799,11 +672,11 @@ int LogEngine::get_min_block_info_for_gc(block_id_t &block_id, SCN &max_scn) } else if (OB_FAIL(get_block_min_scn(min_block_id+1, min_block_max_scn))) { PALF_LOG(TRACE, "get_block_min_scn failed", K(ret)); } else { - // after the first call, 'min_block_max_scn_' is always valid.(except after rebuild or truncate) + // after the first call, 'min_block_max_scn_' is always valid except after rebuild or truncate. // it's important to get 'min_block_info_cache_version' before 'get_block_min_scn', otherwise, the cache of // min block info(i.e. min_block_max_scn_, min_block_min_scn_) is incorrect, consider following case: // T1 timestamp, thread A call get_min_block_info_for_gc, and get_block_min_scn successfully(e.g. min_block_id is 10); - // T2 timestamp, thread B calls truncate, the content of block whose names with 11 is overwritten; + // T2 timestamp, thread B calls truncate, the content of block 11 is overwritten; // T3 timestamp, thread A call set_min_block_info_for_gc_, and the min_block_info(i.e. min_block_max_scn_, min_block_min_scn_) // will reset to incorrect scn. set_min_block_info_for_gc_(min_block_info_cache_version, min_block_id, min_block_max_scn); @@ -841,12 +714,12 @@ int LogEngine::get_min_block_info(block_id_t &block_id, SCN &min_scn) } else if (OB_FAIL(get_block_min_scn(min_block_id, min_block_min_scn))) { PALF_LOG(TRACE, "get_block_min_scn failed", K(ret)); } else { - // after the first call, 'min_block_min_scn_' is always valid.(except after rebuild or truncate) + // after the first call, 'min_block_min_scn_' is always valid except after rebuild or truncate. // it's important to get 'min_block_info_cache_version' before 'get_block_min_scn', otherwise, the cache of // min block info(i.e. min_block_min_scn_) is incorrect, consider following case: // // T1 timestamp, thread A call get_min_block_info, and get_block_min_scn successfully(e.g. min_block_id is 10); - // T2 timestamp, thread B calls truncate, the content of block whose names with 10 is overwritten; + // T2 timestamp, thread B calls truncate, the content of block 10 is overwritten; // T3 timestamp, thread A call set_min_block_info_, and the min_block_info(i.e. min_block_min_scn_) will reset // to incorrect scn. set_min_block_info_(min_block_info_cache_version, min_block_id, min_block_min_scn); @@ -857,6 +730,18 @@ int LogEngine::get_min_block_info(block_id_t &block_id, SCN &min_scn) return ret; } +LogMeta LogEngine::get_log_meta() const +{ + ObSpinLockGuard guard(log_meta_lock_); + return log_meta_; +} + +const LSN &LogEngine::get_base_lsn_used_for_block_gc() const +{ + ObSpinLockGuard guard(log_meta_lock_); + return base_lsn_for_block_gc_; +} + int LogEngine::get_total_used_disk_space(int64_t &total_used_size_byte, int64_t &unrecyclable_disk_space) const { @@ -909,10 +794,10 @@ int LogEngine::raw_read(const LSN &lsn, int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (false == lsn.is_valid() || 0 >= in_read_size || false == read_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn), K(in_read_size), K(read_buf)); + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(is_inited), K(lsn), K(in_read_size), K(read_buf)); } else if (need_read_block_header) { ret = log_storage_.pread_with_block_header(lsn, in_read_size, read_buf, out_read_size, io_ctx); } else if (!need_read_block_header) { @@ -938,7 +823,7 @@ int LogEngine::update_base_lsn_used_for_gc(const LSN &lsn) K(base_lsn_for_block_gc_)); } else { base_lsn_for_block_gc_ = lsn; - PALF_LOG(INFO, "update_base_lsn_used_for_gc success", K(ret), K(lsn), K_(palf_id), K_(is_inited)); + PALF_LOG(INFO, "update_base_lsn_used_for_gc success", K(ret), K(lsn), K_(is_inited)); } return ret; } @@ -950,12 +835,12 @@ int LogEngine::update_manifest(const block_id_t block_id) ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "invalid argument!!!", KPC(this), K(block_id)); } else if (OB_FAIL(log_meta_storage_.update_manifest_used_for_meta_storage(block_id))) { - PALF_LOG(WARN, "update_manifest_used_for_meta_storage failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(WARN, "update_manifest_used_for_meta_storage failed", K(ret), K_(is_inited)); } else { PALF_LOG(INFO, "update_max_block_id_for_switch_block_cb success", K(ret), - K_(palf_id), K_(is_inited), + K_(is_inited), K(block_id)); } return ret; @@ -966,10 +851,10 @@ int LogEngine::append_meta(const char *buf, const int64_t buf_len) int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(is_inited)); } else if (OB_UNLIKELY(NULL == buf || 0 >= buf_len)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argumnt!!!", K(ret), K_(palf_id), K_(is_inited), K(buf), K(buf_len)); + PALF_LOG(ERROR, "Invalid argumnt!!!", K(ret), K_(is_inited), K(buf), K(buf_len)); } else if (OB_FAIL(log_meta_storage_.append_meta(buf, buf_len))) { PALF_LOG(ERROR, "LogMetaStorage pwrite failed", K(ret), K(buf), K(buf_len)); } else { @@ -978,313 +863,6 @@ int LogEngine::append_meta(const char *buf, const int64_t buf_len) return ret; } -// ===================== MetaStorage end ======================= - -// ===================== NetService start ====================== -int LogEngine::submit_push_log_req(const common::ObAddr &server, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not init", K(ret), K_(palf_id), K_(is_inited)); - } else if (OB_FAIL(log_net_service_.submit_push_log_req(server, - push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf))) { - PALF_LOG(ERROR, - "LogNetService submit_group_entry_to_server failed", - K(ret), - K_(palf_id), K_(is_inited), - K(server), - K(prev_log_proposal_id), - K(prev_lsn), - K(curr_lsn), - K(write_buf)); - } else { - PALF_LOG(TRACE, - "submit_group_entry_to_server success", - K(ret), - K_(palf_id), K_(is_inited), - K(server), - K(prev_log_proposal_id), - K(prev_lsn), - K(curr_lsn)); - } - return ret; -} - -int LogEngine::submit_push_log_resp(const ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool is_batch) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_push_log_resp(server, msg_proposal_id, lsn, is_batch); - PALF_LOG(TRACE, "submit_push_log_resp success", K(ret), K(server)); - } - return ret; -} - -int LogEngine::submit_prepare_meta_resp(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_prepare_meta_resp( - server, msg_proposal_id, vote_granted, log_proposal_id, max_flushed_lsn, committed_end_lsn, mode_meta); - } - return ret; -} - -int LogEngine::submit_change_config_meta_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_change_config_meta_resp(server, msg_proposal_id, config_version); - } - return ret; -} - -int LogEngine::submit_change_mode_meta_resp(const common::ObAddr &server, - const int64_t &msg_proposal_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_change_mode_meta_resp(server, msg_proposal_id); - } - return ret; -} - -int LogEngine::submit_config_change_pre_check_req(const common::ObAddr &server, - const LogConfigVersion &config_version, - const bool need_purge_throttling, - const int64_t timeout_us, - LogGetMCStResp &resp) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_config_change_pre_check_req( - server, config_version, need_purge_throttling, timeout_us, resp); - } - return ret; -} - -int LogEngine::submit_fetch_log_req(const ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_fetch_log_req( - server, fetch_type, msg_proposal_id, prev_lsn, lsn, - fetch_log_size, fetch_log_count, accepted_mode_pid); - } - return ret; -} - -int LogEngine::submit_batch_fetch_log_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_batch_fetch_log_resp( - server, msg_proposal_id, prev_log_proposal_id, - prev_lsn, curr_lsn, write_buf); - } - return ret; -} - -int LogEngine::submit_notify_rebuild_req(const ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_notify_rebuild_req(server, base_lsn, base_prev_log_info); - } - return ret; -} - -int LogEngine::submit_notify_fetch_log_req(const ObMemberList &dst_list) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_notify_fetch_log_req(dst_list); - } - return ret; -} - -int LogEngine::submit_register_parent_req(const common::ObAddr &server, - const LogLearner &child_itself, - const bool is_to_leader) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_register_parent_req(server, child_itself, is_to_leader); - } - return ret; -} - -int LogEngine::submit_register_parent_resp(const common::ObAddr &server, - const LogLearner &parent_itself, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_register_parent_resp( - server, parent_itself, candidate_list, reg_ret); - } - return ret; -} - -int LogEngine::submit_retire_parent_req(const common::ObAddr &server, - const LogLearner &child_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_retire_parent_req(server, child_itself); - } - return ret; -} - -int LogEngine::submit_retire_child_req(const common::ObAddr &server, - const LogLearner &parent_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_retire_child_req(server, parent_itself); - } - return ret; -} - -int LogEngine::submit_learner_keepalive_req(const common::ObAddr &server, - const LogLearner &sender_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_learner_keepalive_req(server, sender_itself); - } - return ret; -} - -int LogEngine::submit_learner_keepalive_resp(const common::ObAddr &server, - const LogLearner &sender_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_learner_keepalive_resp(server, sender_itself); - } - return ret; -} - -int LogEngine::submit_committed_info_req( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not init", K(ret), KPC(this)); - } else if (OB_FAIL(log_net_service_.submit_committed_info_req( - server, msg_proposal_id, - prev_log_id, prev_log_proposal_id, committed_end_lsn))) { - PALF_LOG(ERROR, "LogNetService submit_committed_info_req failed", K(ret), - KPC(this), K(server), - K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn)); - } else { - PALF_LOG(TRACE, "submit_committed_info_req success", K(ret), KPC(this), - K(server), K(msg_proposal_id), K(prev_log_id), - K(prev_log_proposal_id), K(committed_end_lsn)); - } - return ret; -} - -int LogEngine::submit_get_stat_req(const common::ObAddr &server, - const int64_t timeout_us, - const LogGetStatReq &req, - LogGetStatResp &resp) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_get_stat_req(server, timeout_us, req, resp); - } - return ret; -} - -LogMeta LogEngine::get_log_meta() const -{ - ObSpinLockGuard guard(log_meta_lock_); - return log_meta_; -} - -const LSN &LogEngine::get_base_lsn_used_for_block_gc() const -{ - ObSpinLockGuard guard(log_meta_lock_); - return base_lsn_for_block_gc_; -} - int LogEngine::submit_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogMeta &log_meta) { @@ -1313,15 +891,15 @@ int LogEngine::construct_log_meta_(const LSN &lsn, block_id_t &expected_next_blo ReadBufGuard guard("LogEngine", buf_len); ReadBuf &read_buf = guard.read_buf_; LogMetaEntry meta_entry; - LogIOContext io_ctx(palf_id_, LogIOUser::RESTART); + LogIOContext io_ctx(LogIOUser::RESTART); if (false == lsn.is_valid()) { - PALF_LOG(INFO, "there is no meta entry, maybe create palf failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(INFO, "there is no meta entry, maybe create palf failed", K(ret), K_(is_inited)); } else if (!read_buf.is_valid()) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "allocate memory failed", KPC(this), K(lsn)); } else if (OB_FAIL(log_meta_storage_.pread(lsn, buf_len, read_buf, out_read_size, io_ctx))) { - PALF_LOG(WARN, "ObLogMetaStorage pread failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(WARN, "ObLogMetaStorage pread failed", K(ret), K_(is_inited)); // NB: when lsn is invalid, means there is no data on disk. } else if (OB_FAIL(meta_entry.deserialize(read_buf.buf_, buf_len, pos))) { PALF_LOG(WARN, "LogMetaEntry deserialize failed", K(ret), K(pos)); @@ -1331,7 +909,7 @@ int LogEngine::construct_log_meta_(const LSN &lsn, block_id_t &expected_next_blo "the data of LogMeta has been corrupted, unexpected error!!!", K(ret), K(meta_entry), - K_(palf_id), K_(is_inited)); + K_(is_inited)); } else if (FALSE_IT(pos = 0)) { } else if (OB_FAIL(log_meta_.deserialize(meta_entry.get_buf(), meta_entry.get_data_len(), pos))) { PALF_LOG(WARN, "LogMeta deserialize failed"); @@ -1352,7 +930,7 @@ int LogEngine::generate_flush_log_task_(const FlushLogCbCtx &flush_log_cb_ctx, flush_log_task = NULL; if (false == flush_log_cb_ctx.is_valid() || false == write_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; - } else if (NULL == (flush_log_task = alloc_mgr_->alloc_log_io_flush_log_task(palf_id_, palf_epoch_))) { + } else if (NULL == (flush_log_task = alloc_mgr_->alloc_log_io_flush_log_task(palf_epoch_))) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(ERROR, "alloc_log_io_flush_log_task failed", K(ret)); } else if (OB_FAIL(flush_log_task->init(flush_log_cb_ctx, write_buf))) { @@ -1369,7 +947,7 @@ int LogEngine::generate_handle_submit_task_(LogHandleSubmitTask *&handle_submit_ { int ret = OB_SUCCESS; handle_submit_task = NULL; - if (NULL == (handle_submit_task = alloc_mgr_->alloc_log_handle_submit_task(palf_id_, palf_epoch_))) { + if (NULL == (handle_submit_task = alloc_mgr_->alloc_log_handle_submit_task(palf_epoch_))) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(ERROR, "alloc_log_handle_submit_task failed", K(ret)); } @@ -1381,37 +959,17 @@ int LogEngine::generate_handle_submit_task_(LogHandleSubmitTask *&handle_submit_ return ret; } -int LogEngine::generate_truncate_log_task_(const TruncateLogCbCtx &truncate_log_cb_ctx, - LogIOTruncateLogTask *&truncate_log_task) -{ - int ret = OB_SUCCESS; - truncate_log_task = NULL; - if (NULL == (truncate_log_task = alloc_mgr_->alloc_log_io_truncate_log_task(palf_id_, palf_epoch_))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(ERROR, "alloc_log_io_truncate_log_task failed", K(ret)); - } else if (OB_FAIL(truncate_log_task->init(truncate_log_cb_ctx))) { - PALF_LOG(ERROR, "init LogIOTruncateLogTask failed", K(ret), K_(palf_id), K_(is_inited)); - } else { - PALF_LOG(TRACE, "generate_truncate_log_task_ success", K(ret), K_(palf_id), K_(is_inited)); - } - if (OB_FAIL(ret) && NULL != truncate_log_task) { - alloc_mgr_->free_log_io_truncate_log_task(truncate_log_task); - truncate_log_task = NULL; - } - return ret; -} - int LogEngine::generate_truncate_prefix_blocks_task_( const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx, LogIOTruncatePrefixBlocksTask *&truncate_prefix_blocks_task) { int ret = OB_SUCCESS; if (NULL - == (truncate_prefix_blocks_task = alloc_mgr_->alloc_log_io_truncate_prefix_blocks_task(palf_id_, palf_epoch_))) { + == (truncate_prefix_blocks_task = alloc_mgr_->alloc_log_io_truncate_prefix_blocks_task(palf_epoch_))) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(ERROR, "alloc_log_io_truncate_prefix_blocks_task failed", K(ret)); } else if (OB_FAIL(truncate_prefix_blocks_task->init(truncate_prefix_blocks_ctx))) { - PALF_LOG(ERROR, "init LogIOTruncatePrefixBlocksTask failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "init LogIOTruncatePrefixBlocksTask failed", K(ret), K_(is_inited)); } else {/*do nothing*/} if (OB_FAIL(ret) && NULL != truncate_prefix_blocks_task) { alloc_mgr_->free_log_io_truncate_prefix_blocks_task(truncate_prefix_blocks_task); @@ -1431,19 +989,19 @@ int LogEngine::generate_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx flush_meta_task = NULL; if (false == flush_meta_cb_ctx.is_valid()) { ret = OB_INVALID_ARGUMENT; - } else if (NULL == (flush_meta_task = alloc_mgr_->alloc_log_io_flush_meta_task(palf_id_, palf_epoch_))) { + } else if (NULL == (flush_meta_task = alloc_mgr_->alloc_log_io_flush_meta_task(palf_epoch_))) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(ERROR, "alloc_log_io_flush_meta_task failed", K(ret)); } else if (NULL == (buf = reinterpret_cast(mtl_malloc(buf_len, "LogEngine")))) { ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(ERROR, "allocate memory failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "allocate memory failed", K(ret), K_(is_inited)); } else if (OB_FAIL(serialize_log_meta_(log_meta, buf, buf_len))) { - PALF_LOG(ERROR, "serialize_log_meta_ failed", K(ret), K_(palf_id), K_(is_inited), K(log_meta)); + PALF_LOG(ERROR, "serialize_log_meta_ failed", K(ret), K_(is_inited), K(log_meta)); } else if (OB_FAIL(flush_meta_task->init(flush_meta_cb_ctx, buf, buf_len))) { PALF_LOG(ERROR, "init LogIOFlushMetaTask failed", K(ret)); } else { - PALF_LOG(TRACE, "generate_flush_meta_task_ success", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(TRACE, "generate_flush_meta_task_ success", K(ret), K_(is_inited)); } if (OB_FAIL(ret)) { @@ -1467,7 +1025,7 @@ int LogEngine::generate_purge_throttling_task_(const PurgeThrottlingCbCtx &purge purge_task = NULL; if (!purge_cb_ctx.is_valid()) { ret = OB_INVALID_ARGUMENT; - } else if (NULL == (purge_task = alloc_mgr_->alloc_log_io_purge_throttling_task(palf_id_, palf_epoch_))) { + } else if (NULL == (purge_task = alloc_mgr_->alloc_log_io_purge_throttling_task(palf_epoch_))) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(ERROR, "alloc_log_io_purge_throttling_task failed", KPC(this)); } else if (OB_FAIL(purge_task->init(purge_cb_ctx))) { @@ -1497,12 +1055,12 @@ int LogEngine::serialize_log_meta_(const LogMeta& log_meta, char *buf, int64_t b if (MAX_META_ENTRY_SIZE != buf_len) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "log_meta_entry_len must be smaller than or equal to LOG_DIO_ALIGN_SIZE", - K(ret), K_(palf_id), K_(is_inited), K(log_meta_entry_header_len), K(log_meta_entry_body_len), K(log_meta_entry_len), + K(ret), K_(is_inited), K(log_meta_entry_header_len), K(log_meta_entry_body_len), K(log_meta_entry_len), K(buf_len)); } else if (FALSE_IT(memset(buf, '\0', buf_len))) { // serialize log_meta_entry_body } else if (OB_FAIL(log_meta.serialize(log_meta_body_serialize_buf, log_meta_body_serialize_buf_len, pos))) { - PALF_LOG(ERROR, "log_meta serialize failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "log_meta serialize failed", K(ret), K_(is_inited)); } else if (OB_FAIL(log_meta_entry_header.generate(log_meta_body_serialize_buf, log_meta_entry_body_len))) { PALF_LOG(ERROR, "generate LogMetaEntryHeader failed", K(ret), K(buf), K(log_meta_entry_body_len)); } else if (OB_FAIL(log_meta_entry.generate(log_meta_entry_header, log_meta_body_serialize_buf))) { @@ -1518,18 +1076,6 @@ int LogEngine::serialize_log_meta_(const LogMeta& log_meta, char *buf, int64_t b return ret; } -int LogEngine::update_config_meta_guarded_by_lock_(const LogConfigMeta &config_meta, - LogMeta &log_meta) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - if (OB_FAIL(log_meta_.update_log_config_meta(config_meta))) { - PALF_LOG(WARN, "update_log_config_meta failed", K(ret), K_(palf_id), K_(is_inited), K(config_meta)); - } else { - log_meta = log_meta_; - } - return ret; -} // Background: int LogEngine::try_clear_up_holes_and_check_storage_integrity_( @@ -1551,11 +1097,11 @@ int LogEngine::try_clear_up_holes_and_check_storage_integrity_( int64_t logical_block_size = 0; const LSN log_storage_tail = log_storage_.get_end_lsn(); if (OB_FAIL(log_storage_.get_logical_block_size(logical_block_size))) { - PALF_LOG(WARN, "get_logical_block_size failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(WARN, "get_logical_block_size failed", K(ret), K_(is_inited)); } else if (FALSE_IT(base_block_id = lsn_2_block(base_lsn, logical_block_size))) { } else if (OB_FAIL(log_storage_.get_block_id_range(min_block_id, max_block_id)) && OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(ERROR, "get_block_id_range failed", K(ret), K_(palf_id), K_(is_inited)); + PALF_LOG(ERROR, "get_block_id_range failed", K(ret), K_(is_inited)); } else if (OB_ENTRY_NOT_EXIST == ret) { // For empty directory, ethier 'base_lsn' is initial lsn or 'prev_log_info(rebuild)' is valid, // meanwhile, 'expected_next_block_id' must be smaller than or eaul to 'base_block_id'. @@ -1565,7 +1111,7 @@ int LogEngine::try_clear_up_holes_and_check_storage_integrity_( PALF_LOG(ERROR, "the directory of log is empty, but expected_next_block_id is greater than " "base_block_id, unexpected error!!!", - K(ret), K_(palf_id), K_(is_inited)); + K(ret), K_(is_inited)); } else { ret = OB_SUCCESS; } @@ -1626,17 +1172,17 @@ int LogEngine::try_clear_up_holes_and_check_storage_integrity_( if (OB_SUCC(ret) && true == prev_log_info_is_valid) { if (OB_FAIL(log_storage_.truncate_prefix_blocks(base_lsn))) { PALF_LOG(ERROR, "clear_up_holes_ failed", K(ret), K(min_block_id), K(max_block_id), - K(base_block_id), K_(palf_id), K_(is_inited)); + K(base_block_id), K_(is_inited)); } else if (base_lsn >= last_group_entry_header.get_committed_end_lsn()) { PALF_LOG(WARN, "the max committed end lsn is smaller than or equal to base_lsn," " there is a rebuild operation before restart, and we will use prev_log_info" - " to construct PalfBaseInfo", K_(palf_id), K(base_lsn), K(last_group_entry_header), + " to construct PalfBaseInfo", K(base_lsn), K(last_group_entry_header), K(log_snapshot_meta)); last_group_entry_header.reset(); } } } - PALF_LOG(INFO, "try_clear_up_holes_and_check_storage_integrity_ finish", K(ret), K_(palf_id), K(min_block_id), + PALF_LOG(INFO, "try_clear_up_holes_and_check_storage_integrity_ finish", K(ret), K(min_block_id), K(max_block_id), K(base_block_id), K(expected_next_block_id), K(prev_log_info), K(log_snapshot_meta), K(prev_log_info_is_valid)); return ret; @@ -1649,7 +1195,7 @@ bool LogEngine::check_last_block_whether_is_integrity_(const block_id_t expected // NB: // 1. 'expected_next_block_id' == 'max_block_id' + 1, normal case // 2. 'expected_next_block_id' <= 'max_block_id', means: - // 1. a 'truncate' opt before stop palf, we need update manifest first, + // 1. a truncate before stopping palf updates the manifest first, // and stop palf before delete blocks on disk, 'expected_next_block_id' is smaller // than or equal to 'max_block_id'. // 2. a switch block opt before stop palf, and just create new block on disk success, @@ -1699,32 +1245,6 @@ void LogEngine::reset_min_block_info_() min_block_info_cache_version_++; } -LogNetService& LogEngine::get_net_service() -{ - return log_net_service_; -} -int LogEngine::check_config_meta_size(const LogConfigMeta &config_meta) const -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(log_meta_lock_); - LogMeta log_meta = log_meta_; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEnginenot inited!!!", K(ret)); - } else if (OB_FAIL(log_meta.update_log_config_meta(config_meta))) { - PALF_LOG(ERROR, "LogMeta update_log_config_meta failed", K(ret), K_(palf_id), K_(is_inited)); - } else { - LogMetaEntryHeader log_meta_entry_header; - const int64_t log_meta_entry_header_len = log_meta_entry_header.get_serialize_size(); - const int64_t log_meta_body_len = log_meta.get_serialize_size(); - if (log_meta_entry_header_len + log_meta_body_len > MAX_META_ENTRY_SIZE) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "check_config_meta_size failed", K(ret), K_(palf_id), - K(log_meta_entry_header_len), K(log_meta_body_len), K(config_meta)); - } - } - return ret; -} } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_engine.h b/src/logservice/palf/log_engine.h index 2c22c7133..56baf414d 100644 --- a/src/logservice/palf/log_engine.h +++ b/src/logservice/palf/log_engine.h @@ -19,9 +19,7 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/utility/ob_print_utils.h" // TO_STRING_KV -#include "common/ob_member_list.h" // ObMemberList #include "log_storage.h" // LogStorage -#include "log_net_service.h" // LogNetService #include "log_meta.h" // LogMeta #include "log_define.h" #include "log_shared_queue_thread.h" @@ -35,7 +33,6 @@ class ObILogAllocator; } // namespace common namespace palf { -class LogRpc; class LogGroupEntry; class LSN; class LogIOWorker; @@ -44,11 +41,9 @@ class PalfHandleImpl; class LogIOTask; class LogHandleSubmitTask; class LogIOFlushLogTask; -class LogIOTruncateLogTask; class LogIOFlushMetaTask; class LogIOTruncatePrefixBlocksTask; class FlushLogCbCtx; -class TruncateLogCbCtx; class FlushMetaCbCtx; class TruncatePrefixBlocksCbCtx; class LogWriteBuf; @@ -58,55 +53,20 @@ class LogIOTruncatePrefixBlocksTask; class LogIOPurgeThrottlingTask; class PurgeThrottlingCbCtx; -#define OVERLOAD_SUBMIT_CHANGE_CONFIG_META_REQ(type) \ - virtual int submit_change_config_meta_req(const type &member_list, \ - const int64_t &msg_proposal_id, \ - const int64_t &prev_log_proposal_id, \ - const LSN &prev_lsn, \ - const int64_t &prev_mode_pid, \ - const LogConfigMeta &config_meta) \ - { \ - return submit_change_config_meta_req_( \ - member_list, msg_proposal_id, prev_log_proposal_id, prev_lsn, \ - prev_mode_pid, config_meta); \ - } - -#define OVERLOAD_SUBMIT_PREPARE_META_REQ(type) \ - virtual int submit_prepare_meta_req(const type &member_list, \ - const int64_t &log_proposal_id) \ - { \ - return submit_prepare_meta_req_(member_list, log_proposal_id); \ - } - -#define OVERLOAD_SUBMIT_CHANGE_MODE_META_REQ(type) \ - virtual int submit_change_mode_meta_req(const type &member_list, \ - const int64_t &msg_proposal_id, \ - const bool is_applied_mode_meta, \ - const LogModeMeta &mode_meta) \ - { \ - return submit_change_mode_meta_req_(member_list, msg_proposal_id, \ - is_applied_mode_meta, mode_meta); \ - } class LogEngine { - friend class PalfHandleImpl; // need get net_service to init election + friend class PalfHandleImpl; public: LogEngine(); virtual ~LogEngine(); public: - // int64_t is as the unique id for LogNetService - // - // NB: Except LogNetService, other components should just - // only use int64_t to debug - int init(const int64_t palf_id, - const char *base_dir, + int init(const char *base_dir, const LogMeta &log_meta, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, LogCache *log_cache, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, LogPlugins *plugins, @@ -116,12 +76,10 @@ class LogEngine LogIOAdapter *io_adapter); void destroy(); - int load(const int64_t palf_id, - const char *base_dir, + int load(const char *base_dir, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, LogCache *log_cache, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, LogPlugins *plugins, @@ -142,28 +100,13 @@ class LogEngine virtual int submit_flush_log_task(const FlushLogCbCtx &flush_log_cb_ctx, const LogWriteBuf &write_buf); virtual int submit_handle_submit_task(); - int submit_flush_prepare_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogPrepareMeta &prepare_meta); - - virtual int submit_flush_change_config_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogConfigMeta &config_meta); - - virtual int submit_flush_mode_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogModeMeta &mode_meta); - int submit_flush_snapshot_meta_task(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogSnapshotMeta &log_snapshot_meta); - virtual int submit_truncate_log_task(const TruncateLogCbCtx &truncate_log_cb_ctx); - virtual int submit_flush_replica_property_meta_task( - const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogReplicaPropertyMeta &log_replica_property_meta); - int submit_truncate_prefix_blocks_task( const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx); int submit_purge_throttling_task(const PurgeThrottlingType purge_type); - virtual int check_config_meta_size(const LogConfigMeta &config_meta) const; // ==================== Submit aysnc task end ================== // ====================== LogStorage start ===================== @@ -198,240 +141,6 @@ class LogEngine // // ===================== MetaStorage end ======================= - // ===================== NetService start ====================== - LogNetService& get_net_service(); - // @brief: this function used to transfer log to remote node - // @param[in] member_list: remote member list - // @param[in] msg_proposal_id: the current proposal_id - // @param[in] prev_lsn: the offset of prev log - // @param[in] prev_log_proposal_id: the proposal_id of prev log - // @param[in] curr_lsn: the offset of curr log - // - // NB: this function just only send log to the remote member list - template - int submit_push_log_req(const List &member_list, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf, - const bool need_batch_rpc) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not init", K(ret), KPC(this)); - } else if (!need_batch_rpc - && OB_FAIL(log_net_service_.submit_push_log_req(member_list, - push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf))) { - } else if (need_batch_rpc - && OB_FAIL(log_net_service_.submit_batch_push_log_req(member_list, - push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf))) { - } else { - PALF_LOG(TRACE, - "submit_group_entry_to_memberlist success", - K(ret), - KPC(this), - K(member_list), - K(msg_proposal_id), - K(prev_log_proposal_id), - K(prev_lsn), - K(curr_lsn), - K(need_batch_rpc)); - } - return ret; - } - virtual int submit_push_log_req(const common::ObAddr &addr, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - - // @brief: this function used to submit acknowledgement to specified server - // @param[in] server: the address of remote server(leader) - // @param[in] proposal_id: the proposal id of this log, used to filter stale log - // @param[in] lsn: the offset of log - virtual int submit_push_log_resp(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool is_batch); - - template - int submit_prepare_meta_req_(const List &member_list, const int64_t &log_proposal_id) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_prepare_meta_req(member_list, log_proposal_id); - } - return ret; - } - - virtual int submit_prepare_meta_resp(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &mode_meta); - - template - int submit_change_config_meta_req_(const List &member_list, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &config_meta) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_change_config_meta_req( - member_list, msg_proposal_id, prev_log_proposal_id, prev_lsn, prev_mode_pid, config_meta); - PALF_LOG(INFO, "submit_change_config_meta_req_ success", K(ret), K(member_list), K(msg_proposal_id), - K(prev_log_proposal_id), K(prev_lsn), K(prev_mode_pid), K(config_meta)); - } - return ret; - } - - template - int submit_change_mode_meta_req_( - const List &member_list, - const int64_t &msg_proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = log_net_service_.submit_change_mode_meta_req(member_list, msg_proposal_id, - is_applied_mode_meta, mode_meta); - } - return ret; - } - - OVERLOAD_SUBMIT_CHANGE_CONFIG_META_REQ(common::ObMemberList); - OVERLOAD_SUBMIT_CHANGE_CONFIG_META_REQ(common::GlobalLearnerList); - OVERLOAD_SUBMIT_CHANGE_CONFIG_META_REQ(common::ResendConfigLogList); - OVERLOAD_SUBMIT_PREPARE_META_REQ(common::ObMemberList); - OVERLOAD_SUBMIT_PREPARE_META_REQ(common::GlobalLearnerList); - OVERLOAD_SUBMIT_CHANGE_MODE_META_REQ(common::ObMemberList); - OVERLOAD_SUBMIT_CHANGE_MODE_META_REQ(common::ResendConfigLogList); - - virtual int submit_change_config_meta_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version); - - virtual int submit_change_mode_meta_resp(const common::ObAddr &server, - const int64_t &msg_proposal_id); - - virtual int submit_config_change_pre_check_req(const common::ObAddr &server, - const LogConfigVersion &config_version, - const bool need_purge_throttling, - const int64_t timeout_us, - LogGetMCStResp &resp); - - // @brief: this function used to submit fetch log request to sepcified server - // @param[in] server: the address of remote server(data source) - // @param[in] fetch_type: follower fetching or leader reconfirm fetching - // @param[in] prev_lsn: the lsn of prev log, used to forward check - // @param[in] lsn: the lsn of log - // @param[in] fetch_log_size: the totoal size - // @param[in] fetch_log_count: the totoal log count by log_id - // @param[in] accepted_mode_pid: proposal_id of accepted_mode_meta - virtual int submit_fetch_log_req(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid); - virtual int submit_batch_fetch_log_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - virtual int submit_register_parent_req(const common::ObAddr &server, - const LogLearner &child_itself, - const bool is_to_leader); - virtual int submit_register_parent_resp(const common::ObAddr &server, - const LogLearner &parent_itself, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret); - virtual int submit_retire_parent_req(const common::ObAddr &server, - const LogLearner &child_itself); - virtual int submit_retire_child_req(const common::ObAddr &server, - const LogLearner &parent_itself); - virtual int submit_learner_keepalive_req(const common::ObAddr &server, - const LogLearner &sender_itself); - virtual int submit_learner_keepalive_resp(const common::ObAddr &server, - const LogLearner &sender_itself); - - virtual int submit_notify_rebuild_req(const ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info); - int submit_notify_fetch_log_req(const ObMemberList &dst_list); - int submit_committed_info_req( - const ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn); - // @brief: this function used to send committed_info to child replica - // @param[in] member_list: current paxos member list - // @param[in] msg_proposal_id: the current proposal_id - // @param[in] prev_log_id: the log_id of prev log - // @param[in] prev_log_proposal_id: the proposal_id of prev log - // @param[in] committed_end_lsn: the current committed_end_lsn of leader - // - template - int submit_committed_info_req( - const List &member_list, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogEngine not init", K(ret), KPC(this)); - } else if (OB_FAIL(log_net_service_.submit_committed_info_req( - member_list, msg_proposal_id, - prev_log_id, prev_log_proposal_id, committed_end_lsn))) { - PALF_LOG(ERROR, "LogNetService submit_committed_info_req failed", K(ret), - KPC(this), K(member_list), - K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn)); - } else { - PALF_LOG(TRACE, "submit_committed_info_req success", K(ret), KPC(this), - K(member_list), K(msg_proposal_id), K(prev_log_id), - K(prev_log_proposal_id), K(committed_end_lsn)); - } - return ret; - } - - int submit_get_stat_req(const common::ObAddr &server, - const int64_t timeout_us, - const LogGetStatReq &req, - LogGetStatResp &resp); LogMeta get_log_meta() const; const LSN &get_base_lsn_used_for_block_gc() const; @@ -443,7 +152,7 @@ class LogEngine int get_total_used_disk_space(int64_t &total_used_size_byte, int64_t &unrecyclable_disk_space) const; virtual int64_t get_palf_epoch() const { return palf_epoch_; } - TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_max_scn), K_(min_block_id), K_(min_block_min_scn), K_(base_lsn_for_block_gc), + TO_STRING_KV(K_(is_inited), K_(min_block_max_scn), K_(min_block_id), K_(min_block_min_scn), K_(base_lsn_for_block_gc), K_(log_meta), K_(log_meta_storage), K_(log_storage), K_(palf_epoch), K_(last_purge_throttling_ts), KP(this)); private: int submit_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogMeta &log_meta); @@ -454,8 +163,6 @@ class LogEngine const LogWriteBuf &write_buf, LogIOFlushLogTask *&flush_log_task); int generate_handle_submit_task_(LogHandleSubmitTask *&handle_submit_task); - int generate_truncate_log_task_(const TruncateLogCbCtx &truncate_log_cb_ctx, - LogIOTruncateLogTask *&truncate_log_task); int generate_truncate_prefix_blocks_task_( const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx, LogIOTruncatePrefixBlocksTask *&truncate_prefix_blocks_task); @@ -465,7 +172,6 @@ class LogEngine LogIOFlushMetaTask *&flush_meta_task); int generate_purge_throttling_task_(const PurgeThrottlingCbCtx &purge_cb_ctx, LogIOPurgeThrottlingTask *&purge_task); - int update_config_meta_guarded_by_lock_(const LogConfigMeta &meta, LogMeta &log_meta); int try_clear_up_holes_and_check_storage_integrity_( const LSN &last_entry_begin_lsn, const block_id_t &expected_next_block_id, @@ -520,13 +226,10 @@ class LogEngine LogMeta log_meta_; LogStorage log_meta_storage_; LogStorage log_storage_; - LogNetService log_net_service_; common::ObILogAllocator *alloc_mgr_; LogIOWorker *log_io_worker_; LogSharedQueueTh *log_shared_queue_th_; LogPlugins *plugins_; - // Except for LogNetService, this field is just only used for debug - int64_t palf_id_; // palf_epoch_ is used for identifying an uniq palf instance. int64_t palf_epoch_; //used to control frequency of purging throttling diff --git a/src/logservice/palf/log_entry_header.cpp b/src/logservice/palf/log_entry_header.cpp index b4278f288..c3a8103c4 100644 --- a/src/logservice/palf/log_entry_header.cpp +++ b/src/logservice/palf/log_entry_header.cpp @@ -31,13 +31,9 @@ const int64_t LogEntryHeader::HEADER_SER_SIZE = sizeof(LogEntryHeader); const int64_t LogEntryHeader::PADDING_LOG_ENTRY_SIZE = sizeof(LogEntryHeader) + sizeof(logservice::ObLogBaseHeader); const int16_t LogEntryHeader::MAGIC = 0x4C48; -const int16_t LogEntryHeader::LOG_ENTRY_HEADER_VERSION = 1; -const int64_t LogEntryHeader::PADDING_TYPE_MASK = 1 << 1; - -const int16_t LogEntryHeader::LOG_ENTRY_HEADER_VERSION2 = 2; -const int64_t LogEntryHeader::PADDING_TYPE_MASK_VERSION2 = 1ll << 62; +const int16_t LogEntryHeader::LOG_ENTRY_HEADER_VERSION = 2; +const int64_t LogEntryHeader::PADDING_TYPE_MASK = 1ll << 62; const int64_t LogEntryHeader::CRC16_MASK = 0xffff; -const int64_t LogEntryHeader::PARITY_MASK = 0x01; LogEntryHeader::LogEntryHeader() : magic_(0), @@ -76,37 +72,26 @@ void LogEntryHeader::reset() bool LogEntryHeader::is_valid() const { - return (magic_ == LogEntryHeader::MAGIC && log_size_ > 0 && scn_.is_valid()); + return magic_ == LogEntryHeader::MAGIC + && version_ == LOG_ENTRY_HEADER_VERSION + && log_size_ > 0 + && scn_.is_valid(); } uint16_t LogEntryHeader::calculate_header_checksum_() const { uint16_t checksum = 0; - if (LOG_ENTRY_HEADER_VERSION == version_) { - bool bool_ret = parity_check(reinterpret_cast(magic_)); - bool_ret ^= parity_check(reinterpret_cast(version_)); - bool_ret ^= parity_check(reinterpret_cast(log_size_)); - bool_ret ^= parity_check((scn_.get_val_for_logservice())); - bool_ret ^= parity_check(reinterpret_cast(data_checksum_)); - int64_t tmp_flag = (flag_ & ~PARITY_MASK); - bool_ret ^= parity_check(reinterpret_cast(tmp_flag)); - checksum = bool_ret ? 1 : 0; - } else if (LOG_ENTRY_HEADER_VERSION2 == version_) { - // NB: To avoid dealing with endianness issue, make the last two bytes of flag_ with zero. - int64_t ori_flag = flag_; - this->flag_ = (ori_flag & ~CRC16_MASK); - checksum = xxhash_16(checksum, reinterpret_cast(this), sizeof(LogEntryHeader)); - this->flag_ = ori_flag; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version", KPC(this)); - } + int64_t ori_flag = flag_; + this->flag_ = (ori_flag & ~CRC16_MASK); + checksum = xxhash_16(checksum, reinterpret_cast(this), sizeof(LogEntryHeader)); + this->flag_ = ori_flag; PALF_LOG(TRACE, "update_header_checksum_", KPC(this), K(checksum)); return checksum; } void LogEntryHeader::update_header_checksum_() { - reset_header_checksum_(); + flag_ &= ~CRC16_MASK; flag_ = (flag_ | calculate_header_checksum_()); } @@ -119,7 +104,7 @@ int LogEntryHeader::generate_header(const char *log_data, ret = OB_INVALID_ARGUMENT; } else { magic_ = LogEntryHeader::MAGIC; - version_ = get_version_(); + version_ = LOG_ENTRY_HEADER_VERSION; log_size_ = data_len; scn_ = scn; data_checksum_ = common::ob_crc64(log_data, data_len); @@ -133,11 +118,10 @@ bool LogEntryHeader::check_header_checksum_() const { bool bool_ret = false; const uint16_t header_checksum = calculate_header_checksum_(); - if (LOG_ENTRY_HEADER_VERSION2 != version_ && LOG_ENTRY_HEADER_VERSION != version_) { + if (LOG_ENTRY_HEADER_VERSION != version_) { PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "check_header_checksum_ failed, invalid version_", KPC(this)); } else { - int64_t mask = get_header_checksum_mask_(); - const uint16_t saved_header_checksum = (flag_ & mask); + const uint16_t saved_header_checksum = (flag_ & CRC16_MASK); bool_ret = (header_checksum == saved_header_checksum); if (false == bool_ret) { PALF_LOG_RET(ERROR, OB_INVALID_DATA, "check_header_checksum_ failed", K(saved_header_checksum), K(header_checksum), KPC(this)); @@ -148,7 +132,7 @@ bool LogEntryHeader::check_header_checksum_() const bool LogEntryHeader::is_padding_log_() const { - return (flag_ & get_padding_mask_()) > 0; + return (flag_ & PADDING_TYPE_MASK) > 0; } // static member function @@ -233,11 +217,11 @@ int LogEntryHeader::generate_padding_header_(const char *log_data, ret = OB_INVALID_ARGUMENT; } else { magic_ = LogEntryHeader::MAGIC; - version_ = get_version_(); + version_ = LOG_ENTRY_HEADER_VERSION; log_size_ = padding_data_len; scn_ = scn; data_checksum_ = common::ob_crc64(log_data, base_header_len); - flag_ = (flag_ | get_padding_mask_()); + flag_ = (flag_ | PADDING_TYPE_MASK); // update header checksum after all member vars assigned (void) update_header_checksum_(); PALF_LOG(INFO, "generate_padding_header_ success", KPC(this), K(log_data), K(base_header_len), K(padding_data_len)); @@ -245,48 +229,6 @@ int LogEntryHeader::generate_padding_header_(const char *log_data, return ret; } -int16_t LogEntryHeader::get_version_() const -{ - return LOG_ENTRY_HEADER_VERSION2; -} - -int64_t LogEntryHeader::get_padding_mask_() const -{ - int64_t padding_mask = 0; - if (LOG_ENTRY_HEADER_VERSION2 == version_) { - padding_mask = PADDING_TYPE_MASK_VERSION2; - } else if (LOG_ENTRY_HEADER_VERSION == version_) { - padding_mask = PADDING_TYPE_MASK; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version!!!", KPC(this)); - } - return padding_mask; -} - -int64_t LogEntryHeader::get_header_checksum_mask_() const -{ - int64_t header_checksum_mask = 0; - if (LOG_ENTRY_HEADER_VERSION2 == version_) { - header_checksum_mask = CRC16_MASK; - } else if (LOG_ENTRY_HEADER_VERSION == version_) { - header_checksum_mask = PARITY_MASK; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version!!!", KPC(this)); - } - return header_checksum_mask; -} - -void LogEntryHeader::reset_header_checksum_() -{ - if (LOG_ENTRY_HEADER_VERSION2 == version_) { - flag_ &= (~CRC16_MASK); - } else if (LOG_ENTRY_HEADER_VERSION == version_) { - flag_ &= (~PARITY_MASK); - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version_", KPC(this)); - } -} - DEFINE_SERIALIZE(LogEntryHeader) { int ret = OB_SUCCESS; diff --git a/src/logservice/palf/log_entry_header.h b/src/logservice/palf/log_entry_header.h index 8fed2b5eb..47517fe1f 100644 --- a/src/logservice/palf/log_entry_header.h +++ b/src/logservice/palf/log_entry_header.h @@ -75,28 +75,16 @@ class LogEntryHeader const int64_t base_header_len, const int64_t padding_data_len, const share::SCN &scn); - int16_t get_version_() const; - int64_t get_padding_mask_() const; - int64_t get_header_checksum_mask_() const; - void reset_header_checksum_(); private: static const int16_t LOG_ENTRY_HEADER_VERSION; static const int64_t PADDING_TYPE_MASK; - static const int16_t LOG_ENTRY_HEADER_VERSION2; - static const int64_t PADDING_TYPE_MASK_VERSION2; static const int64_t CRC16_MASK; - static const int64_t PARITY_MASK; private: int16_t magic_; int16_t version_; int32_t log_size_; share::SCN scn_; int64_t data_checksum_; - // The lowest bit is used for parity check. - // LOG_ENTRY_HEADER_VERSION - // | sign bit | 61 unused bit | PADDING bit | PARITY CHECKSUM bit | - // - // LOG_ENTRY_HEADER_VERSION2 // | sign bit | PADDING bit | 46 unused bit | 16 crc16 bit | mutable int64_t flag_; }; diff --git a/src/logservice/palf/log_group_buffer.cpp b/src/logservice/palf/log_group_buffer.cpp index 7bec96ac0..bde7e196c 100644 --- a/src/logservice/palf/log_group_buffer.cpp +++ b/src/logservice/palf/log_group_buffer.cpp @@ -52,7 +52,7 @@ int LogGroupBuffer::init(const LSN &start_lsn) ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid arguments", K(ret), K(start_lsn)); } else { - int64_t group_buffer_size = FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE; + int64_t group_buffer_size = DEFAULT_GROUP_BUFFER_SIZE; // omt::ObTenantConfigGuard tenant_config(&GCONF); // if (!tenant_config.is_valid()) { // PALF_LOG(WARN, "get tenant config failed", K(ret)); @@ -341,40 +341,17 @@ int LogGroupBuffer::check_log_buf_wrapped(const LSN &lsn, const int64_t log_len, return ret; } // Depend on the write lock of palf_handle_impl to ensure no concurrent updates to group_buffer during the call to this interface -int LogGroupBuffer::to_leader() +int LogGroupBuffer::activate() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; -#if LEADER_DEFAULT_GROUP_BUFFER_SIZE != FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE - } else if (LEADER_DEFAULT_GROUP_BUFFER_SIZE == get_available_buffer_size()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "available_buffer_size_ is already for leader", K(ret), K_(available_buffer_size)); -#endif } else { - ATOMIC_STORE(&available_buffer_size_, LEADER_DEFAULT_GROUP_BUFFER_SIZE); + ATOMIC_STORE(&available_buffer_size_, DEFAULT_GROUP_BUFFER_SIZE); } - PALF_LOG(INFO, "to_leader finished", K(ret), K_(available_buffer_size), K_(reserved_buffer_size)); + PALF_LOG(INFO, "activate group buffer finished", K(ret), K_(available_buffer_size), K_(reserved_buffer_size)); return ret; } -// Depend on the write lock of palf_handle_impl to ensure no concurrent updates to group_buffer during the call to this interface -int LogGroupBuffer::to_follower() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE == get_available_buffer_size()) { - // The case is maybe : pending -> reconfirm fail -> pending. - PALF_LOG(INFO, "current buffer_size is already for follower, no need execute again", K(ret), - K_(available_buffer_size)); - } else { - // Here we cannot reset buffer, because some data may be waiting to flush. - ATOMIC_STORE(&available_buffer_size_, FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE); - } - PALF_LOG(INFO, "to_follower finished", K(ret), K_(available_buffer_size), K_(reserved_buffer_size)); - return ret; -} - int64_t LogGroupBuffer::get_available_buffer_size() const { // This available_buffer_size_ will change according to role. diff --git a/src/logservice/palf/log_group_buffer.h b/src/logservice/palf/log_group_buffer.h index 5b2a45e1f..78d881ae6 100644 --- a/src/logservice/palf/log_group_buffer.h +++ b/src/logservice/palf/log_group_buffer.h @@ -62,8 +62,7 @@ class LogGroupBuffer int check_log_buf_wrapped(const LSN &lsn, const int64_t log_len, bool &is_buf_wrapped) const; int64_t get_available_buffer_size() const; int64_t get_reserved_buffer_size() const; - int to_leader(); - int to_follower(); + int activate(); // inc update readable_begin_lsn, used by append_disk_log(). int inc_update_readable_begin_lsn(const LSN &new_lsn); // inc update reuse_lsn, used for flush log cb case. diff --git a/src/logservice/palf/log_group_entry.cpp b/src/logservice/palf/log_group_entry.cpp index 1bd2027b7..00ef67fee 100644 --- a/src/logservice/palf/log_group_entry.cpp +++ b/src/logservice/palf/log_group_entry.cpp @@ -83,11 +83,6 @@ bool LogGroupEntry::check_integrity(int64_t &data_checksum) const } -int LogGroupEntry::truncate(const SCN &upper_limit_scn, const int64_t pre_accum_checksum) -{ - return header_.truncate(buf_, get_data_len(), upper_limit_scn, pre_accum_checksum); -} - DEFINE_SERIALIZE(LogGroupEntry) { int ret = OB_SUCCESS; @@ -156,9 +151,5 @@ int LogGroupEntry::get_log_min_scn(SCN &min_scn) const return ret; } -bool LogGroupEntry::check_compatibility() const -{ - return header_.check_compatibility(); -} } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_group_entry.h b/src/logservice/palf/log_group_entry.h index 4f625a7c5..2d0a1b92c 100644 --- a/src/logservice/palf/log_group_entry.h +++ b/src/logservice/palf/log_group_entry.h @@ -54,12 +54,6 @@ class LogGroupEntry LSN get_committed_end_lsn() const { return header_.get_committed_end_lsn(); } const LogGroupEntryHeader &get_header() const { return header_; } const char *get_data_buf() const { return buf_; } - // @brief truncate log group entry the upper_limit_scn, only log entries with scn not bigger than which can reserve - // param[in] upper_limit_scn the upper bound to determain which log entries can reserve - // param[in] pre_accum_checksum, the accum_checksum of the pre log - int truncate(const share::SCN &upper_limit_scn, const int64_t pre_accum_checksum); - bool check_compatibility() const; - TO_STRING_KV("LogGroupEntryHeader", header_); NEED_SERIALIZE_AND_DESERIALIZE; static const int64_t BLOCK_SIZE = PALF_BLOCK_SIZE; diff --git a/src/logservice/palf/log_group_entry_header.cpp b/src/logservice/palf/log_group_entry_header.cpp index 61e9f4910..37d7eb2a1 100644 --- a/src/logservice/palf/log_group_entry_header.cpp +++ b/src/logservice/palf/log_group_entry_header.cpp @@ -15,7 +15,6 @@ */ #include "log_group_entry_header.h" // LogGroupEntryHeader -#include "lib/checksum/ob_parity_check.h" // parity_check #include "lib/checksum/ob_crc64.h" // ob_crc64 #include "share/ob_cluster_version.h" // GET_MIN_DATA_VERSION #include "share/rc/ob_tenant_base.h" // MTL_ID @@ -32,16 +31,11 @@ using namespace share; const int64_t LogGroupEntryHeader::HEADER_SER_SIZE = sizeof(LogGroupEntryHeader); const int16_t LogGroupEntryHeader::MAGIC = 0x4752; -const int16_t LogGroupEntryHeader::LOG_GROUP_ENTRY_HEADER_VERSION = 1; -const int64_t LogGroupEntryHeader::PADDING_TYPE_MASK = 1 << 1; -const int64_t LogGroupEntryHeader::RAW_WRITE_MASK = 1 << 2; +const int16_t LogGroupEntryHeader::LOG_GROUP_ENTRY_HEADER_VERSION = 3; +const int64_t LogGroupEntryHeader::PADDING_TYPE_MASK = 1ll << 62; const int64_t LogGroupEntryHeader::PADDING_LOG_DATA_CHECKSUM = 0; -const int16_t LogGroupEntryHeader::LOG_GROUP_ENTRY_HEADER_VERSION2 = 2; -const int64_t LogGroupEntryHeader::PADDING_TYPE_MASK_VERSION2 = 1ll << 62; -const int64_t LogGroupEntryHeader::RAW_WRITE_MASK_VERSION2 = 1ll << 61; const int64_t LogGroupEntryHeader::CRC16_MASK = 0xffff; -const int64_t LogGroupEntryHeader::PARITY_MASK = 0x01; LogGroupEntryHeader::LogGroupEntryHeader() { @@ -56,8 +50,7 @@ LogGroupEntryHeader::~LogGroupEntryHeader() bool LogGroupEntryHeader::is_valid() const { return LogGroupEntryHeader::MAGIC == magic_ - && (LOG_GROUP_ENTRY_HEADER_VERSION == version_ || LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) - && INVALID_PROPOSAL_ID != proposal_id_ + && LOG_GROUP_ENTRY_HEADER_VERSION == version_ && true == committed_end_lsn_.is_valid() && true == max_scn_.is_valid() && true == is_valid_log_id(log_id_); @@ -68,7 +61,6 @@ void LogGroupEntryHeader::reset() magic_ = 0; version_ = 0; group_size_ = 0; - proposal_id_ = INVALID_PROPOSAL_ID; committed_end_lsn_.reset(); max_scn_.reset(); accumulated_checksum_ = 0; @@ -76,38 +68,31 @@ void LogGroupEntryHeader::reset() flag_ = 0; } -int LogGroupEntryHeader::generate(const bool is_raw_write, - const bool is_padding_log, +int LogGroupEntryHeader::generate(const bool is_padding_log, const LogWriteBuf &log_write_buf, const int64_t data_len, const SCN &max_scn, const int64_t log_id, const LSN &committed_end_lsn, - const int64_t &log_proposal_id, int64_t &data_checksum) { int ret = OB_SUCCESS; if (false == max_scn.is_valid() || false == is_valid_log_id(log_id) - || false == committed_end_lsn.is_valid() - || INVALID_PROPOSAL_ID == log_proposal_id) { + || false == committed_end_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid arguments", K(ret), - K(max_scn), K(log_id), K(committed_end_lsn), K(log_proposal_id)); + K(max_scn), K(log_id), K(committed_end_lsn)); } else { magic_ = LogGroupEntryHeader::MAGIC; - version_ = get_version_(); + version_ = LOG_GROUP_ENTRY_HEADER_VERSION; group_size_ = static_cast(data_len); max_scn_ = max_scn; log_id_ = log_id; committed_end_lsn_ = committed_end_lsn; - proposal_id_ = log_proposal_id; if (is_padding_log) { - flag_ = (flag_ | get_padding_mask_()); + flag_ = (flag_ | PADDING_TYPE_MASK); } - if (is_raw_write) { - flag_ = (flag_ | get_raw_write_mask_()); - } if (OB_FAIL(calculate_log_checksum_(is_padding_log, log_write_buf, data_len, data_checksum))) { PALF_LOG(ERROR, "calculate_log_checksum_ failed", K(ret), KPC(this)); } @@ -226,27 +211,10 @@ int LogGroupEntryHeader::calculate_log_checksum_(const bool is_padding_log, uint16_t LogGroupEntryHeader::calculate_header_checksum_() const { uint16_t checksum = 0; - if (LOG_GROUP_ENTRY_HEADER_VERSION == version_) { - bool bool_ret = parity_check(reinterpret_cast(magic_)); - bool_ret ^= parity_check(reinterpret_cast(version_)); - bool_ret ^= parity_check(reinterpret_cast(group_size_)); - bool_ret ^= parity_check(reinterpret_cast(proposal_id_)); - bool_ret ^= parity_check(committed_end_lsn_.val_); - bool_ret ^= parity_check(max_scn_.get_val_for_logservice()); - bool_ret ^= parity_check(reinterpret_cast(accumulated_checksum_)); - bool_ret ^= parity_check(reinterpret_cast(log_id_)); - int64_t tmp_flag = (flag_ & ~PARITY_MASK); - bool_ret ^= parity_check(reinterpret_cast(tmp_flag)); - checksum = (bool_ret ? 1 : 0); - } else if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) { - // NB: To avoid dealing with endianness issue, make the last two bytes of flag_ with zero. - int64_t ori_flag = flag_; - this->flag_ = (ori_flag & ~CRC16_MASK); - checksum = xxhash_16(checksum, reinterpret_cast(this), sizeof(LogGroupEntryHeader)); - this->flag_ = ori_flag; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version", KPC(this)); - } + int64_t ori_flag = flag_; + this->flag_ = (ori_flag & ~CRC16_MASK); + checksum = xxhash_16(checksum, reinterpret_cast(this), sizeof(LogGroupEntryHeader)); + this->flag_ = ori_flag; return checksum; } @@ -257,7 +225,7 @@ void LogGroupEntryHeader::update_header_checksum() void LogGroupEntryHeader::update_header_checksum_() { - reset_header_checksum_(); + flag_ &= ~CRC16_MASK; flag_ = (flag_ | calculate_header_checksum_()); PALF_LOG(TRACE, "update_header_checksum_ finished", KPC(this)); } @@ -267,7 +235,6 @@ LogGroupEntryHeader& LogGroupEntryHeader::operator=(const LogGroupEntryHeader &h magic_ = header.magic_; version_ = header.version_; group_size_ = header.group_size_; - proposal_id_ = header.proposal_id_; committed_end_lsn_ = header.committed_end_lsn_; max_scn_ = header.max_scn_; accumulated_checksum_ = header.accumulated_checksum_; @@ -281,7 +248,6 @@ bool LogGroupEntryHeader::operator==(const LogGroupEntryHeader &header) const return (magic_ == header.magic_ && version_ == header.version_ && group_size_ == header.group_size_ - && proposal_id_ == header.proposal_id_ && committed_end_lsn_ == header.committed_end_lsn_ && max_scn_ == header.max_scn_ && accumulated_checksum_ == header.accumulated_checksum_ @@ -320,41 +286,6 @@ bool LogGroupEntryHeader::check_integrity(const char *buf, return bool_ret; } -int LogGroupEntryHeader::update_log_proposal_id( - const int64_t &log_proposal_id) -{ - int ret = OB_SUCCESS; - - if (INVALID_PROPOSAL_ID == log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument", K(ret), K(log_proposal_id)); - } else { - proposal_id_ = log_proposal_id; - update_header_checksum_(); - } - return ret; -} - -int LogGroupEntryHeader::update_committed_end_lsn(const LSN &lsn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(! lsn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "invalid argument", K(ret), K(lsn)); - } else { - committed_end_lsn_ = lsn; - update_header_checksum_(); - } - return ret; -} - -void LogGroupEntryHeader::update_write_mode(const bool is_raw_write) -{ - if (true == is_raw_write) { - flag_ = (flag_ | get_raw_write_mask_()); - } -} - DEFINE_SERIALIZE(LogGroupEntryHeader) { int ret = OB_SUCCESS; @@ -364,7 +295,6 @@ DEFINE_SERIALIZE(LogGroupEntryHeader) } else if (OB_FAIL(serialization::encode_i16(buf, buf_len, new_pos, magic_)) || OB_FAIL(serialization::encode_i16(buf, buf_len, new_pos, version_)) || OB_FAIL(serialization::encode_i32(buf, buf_len, new_pos, group_size_)) - || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, proposal_id_)) || OB_FAIL(committed_end_lsn_.serialize(buf, buf_len, new_pos)) || OB_FAIL(max_scn_.fixed_serialize(buf, buf_len, new_pos)) || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, accumulated_checksum_)) @@ -388,7 +318,6 @@ DEFINE_DESERIALIZE(LogGroupEntryHeader) } else if ((OB_FAIL(serialization::decode_i16(buf, data_len, new_pos, &magic_))) || OB_FAIL(serialization::decode_i16(buf, data_len, new_pos, &version_)) || OB_FAIL(serialization::decode_i32(buf, data_len, new_pos, &group_size_)) - || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&proposal_id_))) || OB_FAIL(committed_end_lsn_.deserialize(buf, data_len, new_pos)) || OB_FAIL(max_scn_.fixed_deserialize(buf, data_len, new_pos)) || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &accumulated_checksum_)) @@ -410,7 +339,6 @@ DEFINE_GET_SERIALIZE_SIZE(LogGroupEntryHeader) size += serialization::encoded_length_i16(magic_); size += serialization::encoded_length_i16(version_); size += serialization::encoded_length_i32(group_size_); - size += serialization::encoded_length_i64(proposal_id_); size += committed_end_lsn_.get_serialize_size(); size += max_scn_.get_fixed_serialize_size(); size += serialization::encoded_length_i64(accumulated_checksum_); @@ -428,11 +356,10 @@ bool LogGroupEntryHeader::check_header_checksum_() const { bool bool_ret = false; const uint16_t header_checksum = calculate_header_checksum_(); - if (LOG_GROUP_ENTRY_HEADER_VERSION2 != version_ && LOG_GROUP_ENTRY_HEADER_VERSION != version_) { + if (LOG_GROUP_ENTRY_HEADER_VERSION != version_) { PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "check_header_checksum_ failed, invalid version_", KPC(this)); } else { - int64_t mask = get_header_checksum_mask_(); - const uint16_t saved_header_checksum = (flag_ & mask); + const uint16_t saved_header_checksum = (flag_ & CRC16_MASK); bool_ret = (header_checksum == saved_header_checksum); } return bool_ret; @@ -478,130 +405,8 @@ bool LogGroupEntryHeader::check_log_checksum_(const char *buf, bool LogGroupEntryHeader::is_padding_log() const { - return (flag_ & get_padding_mask_()) > 0; -} - -bool LogGroupEntryHeader::is_raw_write() const -{ - return (flag_ & get_raw_write_mask_()) > 0; + return (flag_ & PADDING_TYPE_MASK) > 0; } -int LogGroupEntryHeader::truncate(const char *buf, - const int64_t data_len, - const SCN &cut_scn, - const int64_t pre_accum_checksum) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(NULL == buf || data_len <= 0 || (!cut_scn.is_valid()))) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid arguments", K(ret), KP(buf), K(data_len), K(cut_scn)); - } else if (is_padding_log()) { - if (max_scn_ > cut_scn) { - group_size_ = 0; - update_header_checksum(); - } - PALF_LOG(INFO, "This is a padding log", K(data_len), K(cut_scn), K(pre_accum_checksum), KPC(this)); - } else { - SCN tmp_max_scn; - int64_t pos = 0; - int64_t cut_pos = 0; - LogEntryHeader log_entry_header; - int64_t log_entry_data_checksum = 0; - int64_t tmp_log_checksum = 0; - while (OB_SUCC(ret) && pos < data_len) { - if (OB_FAIL(log_entry_header.deserialize(buf, data_len, pos))) { - PALF_LOG(ERROR, "log_entry_header deserialize failed", K(ret), KP(buf), K(data_len)); - } else if (log_entry_header.get_scn() > cut_scn) { - break; - } else { - log_entry_data_checksum = log_entry_header.get_data_checksum(); - tmp_log_checksum = common::ob_crc64(tmp_log_checksum, &log_entry_data_checksum, sizeof(log_entry_data_checksum)); - pos += log_entry_header.get_data_len(); - cut_pos = pos; - if (!tmp_max_scn.is_valid() || log_entry_header.get_scn() > tmp_max_scn) { - tmp_max_scn = log_entry_header.get_scn(); - } - PALF_LOG(INFO, "each log in truncate", K(log_entry_header), K(buf)); - } - } - if (OB_SUCC(ret)) { - group_size_ = cut_pos; - max_scn_ = tmp_max_scn; - update_accumulated_checksum(common::ob_crc64(pre_accum_checksum, const_cast(&tmp_log_checksum), - sizeof(tmp_log_checksum))); - update_header_checksum(); - } - } - PALF_LOG(INFO, "truncate finished", K(ret), K(*this)); - return ret; -} - -int16_t LogGroupEntryHeader::get_version_() const -{ - return LOG_GROUP_ENTRY_HEADER_VERSION2; -} - -int64_t LogGroupEntryHeader::get_padding_mask_() const -{ - int64_t mask = PADDING_TYPE_MASK; - if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) { - mask = PADDING_TYPE_MASK_VERSION2; - } else if (LOG_GROUP_ENTRY_HEADER_VERSION == version_) { - mask = PADDING_TYPE_MASK; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version_", KPC(this)); - } - return mask; -} - -int64_t LogGroupEntryHeader::get_header_checksum_mask_() const -{ - int64_t header_checksum_mask = 0; - if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) { - header_checksum_mask = CRC16_MASK; - } else if (LOG_GROUP_ENTRY_HEADER_VERSION == version_) { - header_checksum_mask = PARITY_MASK; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version!!!", KPC(this)); - } - return header_checksum_mask; -} - -int64_t LogGroupEntryHeader::get_raw_write_mask_() const -{ - int64_t mask = RAW_WRITE_MASK; - if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) { - mask = RAW_WRITE_MASK_VERSION2; - } else if (LOG_GROUP_ENTRY_HEADER_VERSION == version_) { - mask = RAW_WRITE_MASK; - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version_", KPC(this)); - } - return mask; -} - -void LogGroupEntryHeader::reset_header_checksum_() -{ - if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_) { - flag_ &= (~CRC16_MASK); - } else if (LOG_GROUP_ENTRY_HEADER_VERSION == version_) { - flag_ &= (~PARITY_MASK); - } else { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected version_", KPC(this)); - } -} - -bool LogGroupEntryHeader::check_compatibility() const -{ - bool bool_ret = false; - if (!is_valid()) { - PALF_LOG_RET(WARN, OB_EAGAIN, "invalid LogGroupEntryHeader", KPC(this)); - } else if (LOG_GROUP_ENTRY_HEADER_VERSION2 == version_ && LOG_GROUP_ENTRY_HEADER_VERSION2 != get_version_()) { - PALF_LOG_RET(WARN, OB_EAGAIN, "data version not match!!!", KPC(this)); - } else { - bool_ret = true; - } - return bool_ret; -} } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_group_entry_header.h b/src/logservice/palf/log_group_entry_header.h index eb305d63f..dc603621e 100644 --- a/src/logservice/palf/log_group_entry_header.h +++ b/src/logservice/palf/log_group_entry_header.h @@ -36,22 +36,18 @@ class LogGroupEntryHeader public: using ENTRYTYPE = LogGroupEntry; // @brief generate an object used to serialize and deserialize - // @param[in] is_raw_write: whether this log generaete in RAW_WRITE mode // @param[in] is_padding_log: whether this log is a padding log // @param[in] buf: the data pointer of this log // @param[in] data_len: the data len of this log // @param[in] max_timestamp: the max timestamp of this log(group buffer) // @param[in] log_id: the log id of this log, just only used for sliding window // @param[in] committed_end_lsn: the last committed log lsn before this log - // @param[in] log_proposal_id: the proposal id of this log, used for consensus - int generate(const bool is_raw_write, - const bool is_padding_log, + int generate(const bool is_padding_log, const LogWriteBuf &log_write_buf, const int64_t data_len, const share::SCN &max_scn, const int64_t log_id, const LSN &committed_end_lsn, - const int64_t &log_proposal_id, int64_t &data_checksum); bool is_valid() const; void reset(); @@ -60,10 +56,8 @@ class LogGroupEntryHeader int64_t get_accum_checksum() const { return accumulated_checksum_; } const share::SCN &get_max_scn() const { return max_scn_; } int64_t get_log_id() const { return log_id_; } - const int64_t &get_log_proposal_id() const { return proposal_id_; } const LSN &get_committed_end_lsn() const { return committed_end_lsn_; } bool is_padding_log() const; - bool is_raw_write() const; bool operator==(const LogGroupEntryHeader &header) const; // This function used to check the checksum of buf is as same as @@ -76,29 +70,15 @@ class LogGroupEntryHeader // NB: The data integrity is the responsibility of // the caller. // - // Used to update the proposal id of this log, for standby cluster - int update_log_proposal_id(const int64_t &log_proposal_id); - // Used to update the committed end lsn, for standby cluster - int update_committed_end_lsn(const LSN &lsn); - // Used to update write mode of this log, for standby cluster - void update_write_mode(const bool is_raw_write); - // Used to update header checksum void update_header_checksum(); void update_accumulated_checksum(int64_t accumulated_checksum); - int truncate(const char *buf, - const int64_t data_len, - const share::SCN &cut_scn, - const int64_t pre_accum_checksum); - - bool check_compatibility() const; NEED_SERIALIZE_AND_DESERIALIZE; TO_STRING_KV("magic", magic_, "version", version_, "group_size", group_size_, - "proposal_id", proposal_id_, "committed_lsn", committed_end_lsn_, "max_scn", max_scn_, "accumulated_checksum", accumulated_checksum_, @@ -114,11 +94,6 @@ class LogGroupEntryHeader bool check_header_checksum_() const; bool check_log_checksum_(const char *buf, const int64_t data_len, int64_t &group_log_checksum) const; uint16_t calculate_header_checksum_() const; - int16_t get_version_() const; - int64_t get_padding_mask_() const; - int64_t get_raw_write_mask_() const; - int64_t get_header_checksum_mask_() const; - void reset_header_checksum_(); public: // Update this variable when modifying header's member static const int64_t HEADER_SER_SIZE; @@ -127,24 +102,16 @@ class LogGroupEntryHeader private: static const int16_t LOG_GROUP_ENTRY_HEADER_VERSION; static const int64_t PADDING_TYPE_MASK; - static const int64_t RAW_WRITE_MASK; static const int64_t PADDING_LOG_DATA_CHECKSUM; - static const int16_t LOG_GROUP_ENTRY_HEADER_VERSION2; - static const int64_t PADDING_TYPE_MASK_VERSION2; - static const int64_t RAW_WRITE_MASK_VERSION2; static const int64_t CRC16_MASK; - static const int64_t PARITY_MASK; private: // Binary visualization, for LogGroupEntryHeader, its' magic number // is 0x4752, means GR(group header) int16_t magic_; - // Upgrade compatible int16_t version_; // The length of data, not including the header int32_t group_size_; - // The proposal id of this log - int64_t proposal_id_; // The max committed log offset before this log LSN committed_end_lsn_; // The max scn of this log @@ -156,14 +123,7 @@ class LogGroupEntryHeader // sliding window int64_t log_id_; - // LOG_GROUP_ENTRY_HEADER_VERSION - // | sign bit | 60 unused bit | RAW WRITE bit | PADDING bit | PARITY CHECKSUM bit | - // The lowest bit is used for parity check. - // The second bit from last is used for padding type flag. - // The third bit from last is used for checking whether is RAW_WRITE - // - // LOG_GROUP_ENTRY_HEADER_VERSION2 - // | sign bit | PADDING bit | RAW WRITE BIT | 45 unused bit | 16 crc16 bit| + // | sign bit | PADDING bit | 46 unused bits | 16 crc16 bits | mutable int64_t flag_; }; diff --git a/src/logservice/palf/log_io_context.cpp b/src/logservice/palf/log_io_context.cpp index 6003c39f6..fdc2511b1 100644 --- a/src/logservice/palf/log_io_context.cpp +++ b/src/logservice/palf/log_io_context.cpp @@ -22,20 +22,12 @@ namespace oceanbase namespace palf { LogIOContext::LogIOContext() - : palf_id_(0), - user_() + : user_() { } LogIOContext::LogIOContext(const LogIOUser &user) - : palf_id_(0), - user_(user) -{ -} - -LogIOContext::LogIOContext(const int64_t palf_id, const LogIOUser &user) - : palf_id_(palf_id), - user_(user) + : user_(user) { } diff --git a/src/logservice/palf/log_io_context.h b/src/logservice/palf/log_io_context.h index 475f35c87..6e68c4a12 100644 --- a/src/logservice/palf/log_io_context.h +++ b/src/logservice/palf/log_io_context.h @@ -66,7 +66,6 @@ class LogIOContext LogIOContext(); // do not get group_id LogIOContext(const LogIOUser &user); - LogIOContext(const int64_t palf_id, const LogIOUser &user); ~LogIOContext() { destroy(); } bool is_valid() const { @@ -74,14 +73,12 @@ class LogIOContext } void destroy() { - palf_id_ = 0; user_ = LogIOUser::DEFAULT; iterator_info_.reset(); } LogIOContext &operator=(const LogIOContext &io_ctx) { if (&io_ctx != this) { - this->palf_id_ = io_ctx.palf_id_; this->user_ = io_ctx.user_; this->iterator_info_ = io_ctx.iterator_info_; } @@ -113,9 +110,8 @@ class LogIOContext { iterator_info_.inc_cache_read_size(read_size); } - TO_STRING_KV("user", log_io_user_str(user_), K_(palf_id), K_(iterator_info)); + TO_STRING_KV("user", log_io_user_str(user_), K_(iterator_info)); private: - int64_t palf_id_; LogIOUser user_; LogIteratorInfo iterator_info_; }; diff --git a/src/logservice/palf/log_io_task.cpp b/src/logservice/palf/log_io_task.cpp index 327aa175e..f71d3706f 100644 --- a/src/logservice/palf/log_io_task.cpp +++ b/src/logservice/palf/log_io_task.cpp @@ -47,10 +47,9 @@ int push_task_into_cb_thread_pool(LogIOTaskCbThreadPool *cb_thread_pool, LogIOTa return ret; } -LogIOTask::LogIOTask(const int64_t palf_id, const int64_t palf_epoch) +LogIOTask::LogIOTask(const int64_t palf_epoch) { reset(); - palf_id_ = palf_id; palf_epoch_ = palf_epoch; init_task_ts_ = ObTimeUtility::current_time(); submit_seq_ = 0; @@ -63,7 +62,6 @@ LogIOTask::~LogIOTask() void LogIOTask::reset() { - palf_id_ = INVALID_PALF_ID; palf_epoch_ = -1; init_task_ts_ = OB_INVALID_TIMESTAMP; push_cb_into_cb_pool_ts_ = OB_INVALID_TIMESTAMP; @@ -82,7 +80,7 @@ int LogIOTask::do_task(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf if (delay_ts >= MAX_DELAY_TIME) { PALF_LOG(INFO, "[io delay]", K(do_task_ts), K(delay_ts)); } - if (OB_FAIL(palf_env_impl->get_palf_handle_impl(palf_id_, guard))) { + if (OB_FAIL(palf_env_impl->get_palf_handle_impl(guard))) { PALF_LOG(WARN, "get_palf_handle_impl failed", KPC(this)); } else if (OB_FAIL(guard.get_palf_handle_impl()->get_palf_epoch(palf_epoch))) { PALF_LOG(WARN, "get_palf_epoch failed", KPC(this)); @@ -107,10 +105,10 @@ int LogIOTask::after_consume(IPalfEnvImpl *palf_env_impl) if (delay_ts >= MAX_DELAY_TIME) { PALF_LOG(INFO, "[io delay]", K(after_consume_ts), K(delay_ts)); } - if (OB_FAIL(palf_env_impl->get_palf_handle_impl(palf_id_, guard))) { - PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id_)); + if (OB_FAIL(palf_env_impl->get_palf_handle_impl(guard))) { + PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret)); } else if (OB_FAIL(guard.get_palf_handle_impl()->get_palf_epoch(palf_epoch))) { - PALF_LOG(WARN, "IPalfEnvImpl get_palf_epoch failed", K(ret), K(palf_id_)); + PALF_LOG(WARN, "IPalfEnvImpl get_palf_epoch failed", K(ret)); } else if (palf_epoch != palf_epoch_) { ret = OB_STATE_NOT_MATCH; PALF_LOG(WARN, "palf_epoch has changed, drop task", K(ret), KPC(this)); @@ -150,8 +148,8 @@ int LogIOTask::push_task_into_cb_thread_pool_(LogIOTaskCbThreadPool *cb_thread_p return push_task_into_cb_thread_pool(cb_thread_pool, io_task); } -LogIOFlushLogTask::LogIOFlushLogTask(const int64_t palf_id, const int64_t palf_epoch) - : LogIOTask(palf_id, palf_epoch), flush_log_cb_ctx_(), write_buf_(), is_inited_(false) +LogIOFlushLogTask::LogIOFlushLogTask(const int64_t palf_epoch) + : LogIOTask(palf_epoch), flush_log_cb_ctx_(), write_buf_(), is_inited_(false) {} LogIOFlushLogTask::~LogIOFlushLogTask() @@ -168,7 +166,7 @@ int LogIOFlushLogTask::init(const FlushLogCbCtx &flush_log_cb_ctx, PALF_LOG(ERROR, "LogIOFlushLogTask has inited", K(ret)); } else if (false == flush_log_cb_ctx.is_valid() || false == write_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invaild arguments!!!", K(ret), K(write_buf), K(palf_id_), K(palf_epoch_)); + PALF_LOG(ERROR, "Invaild arguments!!!", K(ret), K(write_buf), K(palf_epoch_)); } else { flush_log_cb_ctx_ = flush_log_cb_ctx; write_buf_ = write_buf; @@ -233,73 +231,8 @@ int64_t LogIOFlushLogTask::get_io_size_() const return write_buf_.get_total_size(); } -LogIOTruncateLogTask::LogIOTruncateLogTask(const int64_t palf_id, const int64_t palf_epoch) - : LogIOTask(palf_id, palf_epoch), truncate_log_cb_ctx_(), is_inited_(false) -{} - -LogIOTruncateLogTask::~LogIOTruncateLogTask() -{ - destroy(); -} - -int LogIOTruncateLogTask::init(const TruncateLogCbCtx &truncate_log_cb_ctx) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - } else if (false == truncate_log_cb_ctx.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else { - truncate_log_cb_ctx_ = truncate_log_cb_ctx; - is_inited_ = true; - } - return ret; -} - -void LogIOTruncateLogTask::destroy() -{ - if (IS_INIT) { - is_inited_ = false; - } -} - -int LogIOTruncateLogTask::do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPalfHandleImplGuard &guard) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(guard.get_palf_handle_impl()->inner_truncate_log(truncate_log_cb_ctx_.lsn_))) { - PALF_LOG(WARN, "PalfHandleImpl inner_truncate_log failed", K(ret), K(palf_id_), - K_(truncate_log_cb_ctx)); - } else if (OB_FAIL(push_task_into_cb_thread_pool_(cb_thread_pool, this))) { - PALF_LOG(WARN, "push_task_into_cb_thread_pool failed", K(ret), KP(cb_thread_pool), KP(this)); - } else { - } - return ret; -} - -int LogIOTruncateLogTask::after_consume_(IPalfHandleImplGuard &guard) -{ - int ret = OB_SUCCESS; - int64_t palf_epoch = -1; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogITruncateLogTask not inited!!!", K(ret)); - } else if (OB_FAIL( - guard.get_palf_handle_impl()->inner_after_truncate_log(truncate_log_cb_ctx_))) { - PALF_LOG(WARN, "PalfHandleImpl inner_after_truncate_log failed", K(ret)); - } else { - } - return ret; -} - -void LogIOTruncateLogTask::free_this_(IPalfEnvImpl *palf_env_impl) -{ - palf_env_impl->get_log_allocator()->free_log_io_truncate_log_task(this); -} - -LogIOFlushMetaTask::LogIOFlushMetaTask(const int64_t palf_id, const int64_t palf_epoch) : - LogIOTask(palf_id, palf_epoch), +LogIOFlushMetaTask::LogIOFlushMetaTask(const int64_t palf_epoch) : + LogIOTask(palf_epoch), flush_meta_cb_ctx_(), buf_(NULL), buf_len_(0), @@ -354,7 +287,7 @@ int LogIOFlushMetaTask::do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPalfHan ret = OB_NOT_INIT; PALF_LOG(ERROR, "LogIOFlushMetaTask not inited!!!", K(ret)); } else if (OB_FAIL(guard.get_palf_handle_impl()->inner_append_meta(buf_, buf_len_))) { - PALF_LOG(ERROR, "PalfHandleImpl inner_append_meta failed", K(ret), K(palf_id_)); + PALF_LOG(ERROR, "PalfHandleImpl inner_append_meta failed", K(ret)); } else if (OB_FAIL(push_task_into_cb_thread_pool_(cb_thread_pool, this))) { PALF_LOG(WARN, "push_task_into_cb_thread_pool failed", K(ret), KP(cb_thread_pool), KP(this)); } else { @@ -380,8 +313,8 @@ void LogIOFlushMetaTask::free_this_(IPalfEnvImpl *palf_env_impl) palf_env_impl->get_log_allocator()->free_log_io_flush_meta_task(this); } -LogIOTruncatePrefixBlocksTask::LogIOTruncatePrefixBlocksTask(const int64_t palf_id, const int64_t palf_epoch) - : LogIOTask(palf_id, palf_epoch), truncate_prefix_blocks_ctx_(), is_inited_(false) +LogIOTruncatePrefixBlocksTask::LogIOTruncatePrefixBlocksTask(const int64_t palf_epoch) + : LogIOTask(palf_epoch), truncate_prefix_blocks_ctx_(), is_inited_(false) {} LogIOTruncatePrefixBlocksTask::~LogIOTruncatePrefixBlocksTask() @@ -410,7 +343,6 @@ void LogIOTruncatePrefixBlocksTask::destroy() if (IS_INIT) { is_inited_ = false; truncate_prefix_blocks_ctx_.reset(); - palf_id_ = -1; } } @@ -422,7 +354,7 @@ int LogIOTruncatePrefixBlocksTask::do_task_(LogIOTaskCbThreadPool *cb_thread_poo PALF_LOG(ERROR, "LogIOTruncatePrefixBlocksTask not inited!!!", K(ret)); } else if (OB_FAIL(guard.get_palf_handle_impl()->inner_truncate_prefix_blocks( truncate_prefix_blocks_ctx_.lsn_))) { - PALF_LOG(ERROR, "PalfHandleImpl inner_truncate_prefix_blocks failed", K(ret), K(palf_id_)); + PALF_LOG(ERROR, "PalfHandleImpl inner_truncate_prefix_blocks failed", K(ret)); } else if (OB_FAIL(push_task_into_cb_thread_pool_(cb_thread_pool, this))) { PALF_LOG(ERROR, "push_task_into_cb_thread_pool_ failed", K(ret), KP(cb_thread_pool), KP(this)); } else { @@ -456,7 +388,6 @@ BatchLogIOFlushLogTask::BatchLogIOFlushLogTask() log_write_buf_array_(), scn_array_(), lsn_array_(), - palf_id_(INVALID_PALF_ID), accum_in_queue_time_(0), accum_size_(0), is_inited_(false) @@ -501,7 +432,6 @@ void BatchLogIOFlushLogTask::reuse() log_write_buf_array_.clear(); scn_array_.clear(); lsn_array_.clear(); - palf_id_ = INVALID_PALF_ID; accum_size_ = 0; } @@ -512,28 +442,22 @@ void BatchLogIOFlushLogTask::destroy() log_write_buf_array_.destroy(); scn_array_.destroy(); lsn_array_.destroy(); - palf_id_ = INVALID_PALF_ID; } int BatchLogIOFlushLogTask::push_back(LogIOFlushLogTask *task) { int ret = OB_SUCCESS; - const int64_t palf_id = task->get_palf_id(); const int64_t task_size = task->get_io_size_(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(ERROR, "LogIOFlushMetaTask not inited!!!", K(ret), KPC(this)); - } else if (palf_id_ != INVALID_PALF_ID && palf_id != palf_id_) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "only same palf id can be batched", K(ret), KPC(task), K(palf_id)); - } else if (accum_size_ > SINGLE_TASK_MAX_SIZE) { + } else if (accum_size_ > SINGLE_TASK_MAX_SIZE) { ret = OB_SIZE_OVERFLOW; } else if (OB_FAIL(io_task_array_.push_back(task))) { PALF_LOG(WARN, "push failed", K(ret)); } else { - palf_id_ = palf_id; accum_size_ += task_size; - PALF_LOG(TRACE, "push_back success", K(palf_id_), KPC(this)); + PALF_LOG(TRACE, "push_back success", KPC(this)); } return ret; } @@ -545,7 +469,7 @@ int BatchLogIOFlushLogTask::do_task(LogIOTaskCbThreadPool *cb_thread_pool, IPalf if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(ERROR, "LogIOFlushMetaTask not inited!!!", K(ret), KPC(this)); - } else if (INVALID_PALF_ID == palf_id_ && true == io_task_array_.empty()) { + } else if (io_task_array_.empty()) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "BatchLogIOFlushLogTask is empty", K(ret), KPC(this)); } else if (OB_FAIL(do_task_(cb_thread_pool, palf_env_impl))) { @@ -593,10 +517,10 @@ int BatchLogIOFlushLogTask::do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPal IPalfHandleImplGuard guard; LSN flushed_log_end_lsn; const int64_t first_handle_ts = ObTimeUtility::fast_current_time(); - if (OB_FAIL(palf_env_impl->get_palf_handle_impl(palf_id_, guard))) { - PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id_)); + if (OB_FAIL(palf_env_impl->get_palf_handle_impl(guard))) { + PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret)); } else if (OB_FAIL(guard.get_palf_handle_impl()->get_palf_epoch(palf_epoch))) { - PALF_LOG(WARN, "IPalfEnvImpl get_palf_epoch failed", K(ret), K(palf_id_)); + PALF_LOG(WARN, "IPalfEnvImpl get_palf_epoch failed", K(ret)); } else { const int64_t count = io_task_array_.count(); bool has_valid_data = false; @@ -606,7 +530,7 @@ int BatchLogIOFlushLogTask::do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPal ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "io_task is nullptr, unpexected error", K(ret), KP(io_task), KPC(this)); } else if (palf_epoch != io_task->get_palf_epoch()) { - PALF_LOG(WARN, "palf_epoch has changed, drop task", K(ret), K(palf_id_), K(palf_epoch), + PALF_LOG(WARN, "palf_epoch has changed, drop task", K(ret), K(palf_epoch), KPC(io_task), KPC(io_task)); io_task->free_this(palf_env_impl); io_task_array_[i] = NULL; @@ -652,8 +576,8 @@ void BatchLogIOFlushLogTask::clear_memory_(IPalfEnvImpl *palf_env_impl) } } -LogIOPurgeThrottlingTask::LogIOPurgeThrottlingTask(const int64_t palf_id, const int64_t palf_epoch) : - LogIOTask(palf_id, palf_epoch), purge_ctx_(), is_inited_(false) +LogIOPurgeThrottlingTask::LogIOPurgeThrottlingTask(const int64_t palf_epoch) : + LogIOTask(palf_epoch), purge_ctx_(), is_inited_(false) {} LogIOPurgeThrottlingTask::~LogIOPurgeThrottlingTask() diff --git a/src/logservice/palf/log_io_task.h b/src/logservice/palf/log_io_task.h index d5862a9db..6b8a2f220 100644 --- a/src/logservice/palf/log_io_task.h +++ b/src/logservice/palf/log_io_task.h @@ -37,7 +37,6 @@ enum class LogIOTaskType FLUSH_LOG_TYPE = 1, FLUSH_META_TYPE = 2, TRUNCATE_PREFIX_TYPE = 3, - TRUNCATE_LOG_TYPE = 4, PURGE_THROTTLING_TYPE = 6, }; @@ -49,7 +48,6 @@ OB_INLINE const char *log_io_task_type_str(const LogIOTaskType type) IO_TASK_TYPE_NAME(FLUSH_LOG_TYPE); IO_TASK_TYPE_NAME(FLUSH_META_TYPE); IO_TASK_TYPE_NAME(TRUNCATE_PREFIX_TYPE); - IO_TASK_TYPE_NAME(TRUNCATE_LOG_TYPE); IO_TASK_TYPE_NAME(PURGE_THROTTLING_TYPE); default: str = "UNKNOWN_TYPE"; } @@ -65,7 +63,7 @@ int push_task_into_cb_thread_pool(LogIOTaskCbThreadPool *cb_thread_pool, LogIOTa class LogIOTask : public common::LinkTask { public: - LogIOTask(const int64_t palf_id, const int64_t palf_epoch); + explicit LogIOTask(const int64_t palf_epoch); virtual ~LogIOTask(); void reset(); public: @@ -75,13 +73,11 @@ class LogIOTask : public common::LinkTask int64_t get_init_task_ts(); void free_this(IPalfEnvImpl *palf_env_impl); int64_t get_io_size(); - int64_t get_palf_id() const { return palf_id_; } int64_t get_submit_seq() const {return submit_seq_;} void set_submit_seq(const int64_t submit_seq) {submit_seq_ = submit_seq;} bool need_purge_throttling(); int64_t get_palf_epoch() const { return palf_epoch_; } VIRTUAL_TO_STRING_KV("BaseClass", "LogIOTask", - "palf_id", palf_id_, "palf_epoch", palf_epoch_, "create_task_ts", init_task_ts_, "push_cb_into_cb_pool_ts", push_cb_into_cb_pool_ts_, @@ -97,7 +93,6 @@ class LogIOTask : public common::LinkTask int push_task_into_cb_thread_pool_(LogIOTaskCbThreadPool *cb_thread_pool, LogIOTask *io_task); protected: - int64_t palf_id_; int64_t palf_epoch_; int64_t init_task_ts_; int64_t push_cb_into_cb_pool_ts_; @@ -111,7 +106,7 @@ class LogIOTask : public common::LinkTask class LogIOFlushLogTask : public LogIOTask { friend class BatchLogIOFlushLogTask; public: - LogIOFlushLogTask(const int64_t palf_id,const int64_t palf_epoch); + explicit LogIOFlushLogTask(const int64_t palf_epoch); ~LogIOFlushLogTask() override; int init(const FlushLogCbCtx &flush_log_cb_ctx, @@ -134,30 +129,9 @@ class LogIOFlushLogTask : public LogIOTask { bool is_inited_; }; -class LogIOTruncateLogTask : public LogIOTask { -public: - LogIOTruncateLogTask(const int64_t palf_id,const int64_t palf_epoch); - ~LogIOTruncateLogTask() override; - - int init(const TruncateLogCbCtx &truncate_log_cb_ctx); - void destroy(); - - INHERIT_TO_STRING_KV("LogIOTask", LogIOTask, K_(truncate_log_cb_ctx)); -private: - int do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPalfHandleImplGuard &guard) override final; - int after_consume_(IPalfHandleImplGuard &guard) override final; - LogIOTaskType get_io_task_type_() const override final { return LogIOTaskType::TRUNCATE_LOG_TYPE; } - void free_this_(IPalfEnvImpl *palf_env_impl) override final; - int64_t get_io_size_() const override final {return 0;} - bool need_purge_throttling_() const override final {return false;} -private: - TruncateLogCbCtx truncate_log_cb_ctx_; - bool is_inited_; -}; - class LogIOFlushMetaTask : public LogIOTask { public: - LogIOFlushMetaTask(const int64_t palf_id,const int64_t palf_epoch); + explicit LogIOFlushMetaTask(const int64_t palf_epoch); ~LogIOFlushMetaTask() override; public: @@ -184,7 +158,7 @@ class LogIOFlushMetaTask : public LogIOTask { class LogIOTruncatePrefixBlocksTask : public LogIOTask { public: - LogIOTruncatePrefixBlocksTask(const int64_t palf_id,const int64_t palf_epoch); + explicit LogIOTruncatePrefixBlocksTask(const int64_t palf_epoch); ~LogIOTruncatePrefixBlocksTask(); int init(const TruncatePrefixBlocksCbCtx& truncate_prefix_blocks_ctx); @@ -217,11 +191,10 @@ class BatchLogIOFlushLogTask { void get_io_task_array(BatchIOTaskArray& io_task_array) { io_task_array = io_task_array_; } int do_task(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf_env_impl); void reset_accum_in_queue_time() { accum_in_queue_time_ = 0; } - int64_t get_palf_id() const { return palf_id_; } int64_t get_count() const { return io_task_array_.count(); } int64_t get_accum_in_queue_time() const { return accum_in_queue_time_; } - TO_STRING_KV(K_(palf_id), "count", io_task_array_.count(), K_(lsn_array), K_(accum_size)); + TO_STRING_KV("count", io_task_array_.count(), K_(lsn_array), K_(accum_size)); private: int push_flush_cb_to_thread_pool_(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf_env_impl); int do_task_(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf_env_impl); @@ -232,7 +205,6 @@ class BatchLogIOFlushLogTask { LogWriteBufArray log_write_buf_array_; SCNArray scn_array_; LSNArray lsn_array_; - int64_t palf_id_; int64_t accum_in_queue_time_; int64_t accum_size_; bool is_inited_; @@ -240,11 +212,11 @@ class BatchLogIOFlushLogTask { class LogIOPurgeThrottlingTask : public LogIOTask { public: - LogIOPurgeThrottlingTask(const int64_t palf_id, const int64_t palf_epoch); + explicit LogIOPurgeThrottlingTask(const int64_t palf_epoch); ~LogIOPurgeThrottlingTask(); void reset(); public: - int init(const PurgeThrottlingCbCtx & purge_ctx); + int init(const PurgeThrottlingCbCtx &purge_ctx); void destroy(); INHERIT_TO_STRING_KV("LogIOTask", LogIOTask, K_(purge_ctx), K_(is_inited)); private: diff --git a/src/logservice/palf/log_io_task_cb_utils.cpp b/src/logservice/palf/log_io_task_cb_utils.cpp index 9510af1a4..c0e585bfe 100644 --- a/src/logservice/palf/log_io_task_cb_utils.cpp +++ b/src/logservice/palf/log_io_task_cb_utils.cpp @@ -25,26 +25,11 @@ FlushLogCbCtx::FlushLogCbCtx() : log_id_(OB_INVALID_LOG_ID), scn_(), lsn_(), - log_proposal_id_(INVALID_PROPOSAL_ID), total_len_(0), - curr_proposal_id_(INVALID_PROPOSAL_ID), begin_ts_(OB_INVALID_TIMESTAMP) { } -FlushLogCbCtx::FlushLogCbCtx(const int64_t log_id, const SCN &scn, const LSN &lsn, - const int64_t &log_proposal_id, const int64_t total_len, - const int64_t &curr_proposal_id, const int64_t begin_ts) - : log_id_(log_id), - lsn_(lsn), - log_proposal_id_(log_proposal_id), - total_len_(total_len), - curr_proposal_id_(curr_proposal_id), - begin_ts_(begin_ts) -{ - scn_ = scn; -} - FlushLogCbCtx::~FlushLogCbCtx() { reset(); @@ -55,9 +40,7 @@ void FlushLogCbCtx::reset() log_id_ = OB_INVALID_LOG_ID; scn_.reset(); lsn_.reset(); - log_proposal_id_ = INVALID_PROPOSAL_ID; total_len_ = 0; - curr_proposal_id_ = INVALID_PROPOSAL_ID; begin_ts_ = OB_INVALID_TIMESTAMP; } @@ -66,47 +49,14 @@ FlushLogCbCtx& FlushLogCbCtx::operator=(const FlushLogCbCtx &arg) log_id_ = arg.log_id_; scn_ = arg.scn_; lsn_ = arg.lsn_; - log_proposal_id_ = arg.log_proposal_id_; total_len_ = arg.total_len_; - curr_proposal_id_ = arg.curr_proposal_id_; begin_ts_ = arg.begin_ts_; return *this; } -TruncateLogCbCtx::TruncateLogCbCtx() - : lsn_() -{ -} - -TruncateLogCbCtx::TruncateLogCbCtx(const LSN &lsn) - : lsn_(lsn) -{ -} - -TruncateLogCbCtx::~TruncateLogCbCtx() -{ - reset(); -} - -void TruncateLogCbCtx::reset() -{ - lsn_.reset(); -} - -TruncateLogCbCtx& TruncateLogCbCtx::operator=(const TruncateLogCbCtx &arg) -{ - lsn_ = arg.lsn_; - return *this; -} - FlushMetaCbCtx::FlushMetaCbCtx() : type_ (INVALID_META_TYPE), - proposal_id_(INVALID_PROPOSAL_ID), - config_version_(), - base_lsn_(), - allow_vote_(true), - is_applied_mode_meta_(false), - log_mode_meta_() + base_lsn_() { } @@ -118,23 +68,13 @@ FlushMetaCbCtx::~FlushMetaCbCtx() void FlushMetaCbCtx::reset() { type_ = INVALID_META_TYPE; - proposal_id_ = INVALID_PROPOSAL_ID; - config_version_.reset(); base_lsn_.reset(); - allow_vote_ = true; - is_applied_mode_meta_ = false; - log_mode_meta_.reset(); } FlushMetaCbCtx &FlushMetaCbCtx::operator=(const FlushMetaCbCtx &arg) { this->type_ = arg.type_; - this->proposal_id_ = arg.proposal_id_; - this->config_version_ = arg.config_version_; this->base_lsn_ = arg.base_lsn_; - this->allow_vote_ = arg.allow_vote_; - this->is_applied_mode_meta_ = arg.is_applied_mode_meta_; - this->log_mode_meta_ = arg.log_mode_meta_; return *this; } diff --git a/src/logservice/palf/log_io_task_cb_utils.h b/src/logservice/palf/log_io_task_cb_utils.h index 030314408..c24422ab1 100644 --- a/src/logservice/palf/log_io_task_cb_utils.h +++ b/src/logservice/palf/log_io_task_cb_utils.h @@ -33,34 +33,18 @@ namespace palf struct FlushLogCbCtx { FlushLogCbCtx(); - FlushLogCbCtx(const int64_t log_id, const share::SCN &scn, const LSN &lsn, - const int64_t &log_proposal_id, const int64_t total_len, - const int64_t &curr_log_proposal_id, const int64_t begine_ts); ~FlushLogCbCtx(); bool is_valid() const { return true == lsn_.is_valid() && true == scn_.is_valid(); } void reset(); FlushLogCbCtx &operator=(const FlushLogCbCtx &flush_log_cb_ctx); - TO_STRING_KV(K_(log_id), K_(scn), K_(lsn), K_(log_proposal_id), K_(total_len), K_(curr_proposal_id), K_(begin_ts)); + TO_STRING_KV(K_(log_id), K_(scn), K_(lsn), K_(total_len), K_(begin_ts)); int64_t log_id_; share::SCN scn_; LSN lsn_; - int64_t log_proposal_id_; int64_t total_len_; - int64_t curr_proposal_id_; int64_t begin_ts_; }; -struct TruncateLogCbCtx { - TruncateLogCbCtx(const LSN &lsn); - TruncateLogCbCtx(); - ~TruncateLogCbCtx(); - bool is_valid() const { return true == lsn_.is_valid();} - void reset(); - TruncateLogCbCtx &operator=(const TruncateLogCbCtx &truncate_log_cb_ctx); - TO_STRING_KV(K_(lsn)); - LSN lsn_; -}; - struct TruncatePrefixBlocksCbCtx { TruncatePrefixBlocksCbCtx(const LSN &lsn); TruncatePrefixBlocksCbCtx(); @@ -73,11 +57,7 @@ struct TruncatePrefixBlocksCbCtx { }; enum MetaType { - PREPARE_META = 0, - CHANGE_CONFIG_META = 1, - MODE_META = 2, - SNAPSHOT_META = 3, - REPLICA_PROPERTY_META = 4, + SNAPSHOT_META = 0, INVALID_META_TYPE }; @@ -86,11 +66,7 @@ inline const char *meta_type_2_str(const MetaType type) #define EXTRACT_META_TYPE(type_var) case(type_var): return #type_var switch(type) { - EXTRACT_META_TYPE(PREPARE_META); - EXTRACT_META_TYPE(CHANGE_CONFIG_META); - EXTRACT_META_TYPE(MODE_META); EXTRACT_META_TYPE(SNAPSHOT_META); - EXTRACT_META_TYPE(REPLICA_PROPERTY_META); default: return "Invalid Type"; @@ -104,16 +80,9 @@ struct FlushMetaCbCtx { bool is_valid() const { return INVALID_META_TYPE != type_; } void reset(); FlushMetaCbCtx &operator=(const FlushMetaCbCtx &flush_meta_cb_ctx); - TO_STRING_KV("type", meta_type_2_str(type_), K_(proposal_id), K_(config_version), K_(base_lsn), K_(allow_vote), - K_(log_mode_meta)); + TO_STRING_KV("type", meta_type_2_str(type_), K_(base_lsn)); MetaType type_; - int64_t proposal_id_; - LogConfigVersion config_version_; LSN base_lsn_; - bool allow_vote_; - // log_mode_meta_ is apply-effective, so need record log_mode_meta in FlushCtx - bool is_applied_mode_meta_; - LogModeMeta log_mode_meta_; }; diff --git a/src/logservice/palf/log_io_utils.cpp b/src/logservice/palf/log_io_utils.cpp index cc35adb04..5320aa0fd 100644 --- a/src/logservice/palf/log_io_utils.cpp +++ b/src/logservice/palf/log_io_utils.cpp @@ -422,7 +422,6 @@ int GetBlockCountFunctor::func(const dirent *entry) if (false == is_number(entry_name)) { ret = OB_ERR_UNEXPECTED; PALF_LOG(WARN, "this is block is not used for palf!!!", K(ret), K(entry_name)); - // do nothing, skip invalid block like tmp } else { count_ ++; } @@ -438,11 +437,9 @@ int TrimLogDirectoryFunctor::func(const dirent *entry) PALF_LOG(WARN, "invalid args", K(ret), KP(entry)); } else { const char *entry_name = entry->d_name; - bool str_is_number = is_number(entry_name); - if (false == str_is_number) { + if (false == is_number(entry_name)) { ret = OB_ERR_UNEXPECTED; PALF_LOG(WARN, "this is block is not used for palf!!!", K(ret), K(entry_name)); - // do nothing, skip invalid block like tmp } else { uint32_t block_id = static_cast(strtol(entry->d_name, nullptr, 10)); if (LOG_INVALID_BLOCK_ID == min_block_id_ || block_id < min_block_id_) { @@ -456,31 +453,6 @@ int TrimLogDirectoryFunctor::func(const dirent *entry) return ret; } -int TrimLogDirectoryFunctor::try_to_remove_block_(const int dir_fd, const char *file_name) -{ - int ret = OB_SUCCESS; - int fd = -1; - if (-1 == (fd = ::openat(dir_fd, file_name, LOG_READ_FLAG))) { - ret = convert_sys_errno(); - } - // if file not exist, return OB_SUCCESS; - if (OB_FAIL(ret)) { - if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "block file does not exist during restart trim", K(file_name)); - } else { - PALF_LOG(ERROR, "open file failed", K(file_name)); - } - } else if (OB_FAIL(log_block_pool_->remove_block_at(dir_fd, file_name))) { - PALF_LOG(ERROR, "remove_block_at failed", K(dir_fd), K(file_name)); - } - if (-1 != fd && -1 == ::close(fd)) { - ret = convert_sys_errno(); - PALF_LOG(ERROR, "close fd failed", K(file_name)); - } - return ret; -} - int reuse_block_at(const int dir_fd, const char *block_path) { int ret = OB_SUCCESS; diff --git a/src/logservice/palf/log_io_utils.h b/src/logservice/palf/log_io_utils.h index 66c222712..77175ddac 100644 --- a/src/logservice/palf/log_io_utils.h +++ b/src/logservice/palf/log_io_utils.h @@ -81,7 +81,6 @@ class TrimLogDirectoryFunctor : public ObBaseDirFunctor block_id_t get_min_block_id() const { return min_block_id_; } block_id_t get_max_block_id() const { return max_block_id_; } private: - int try_to_remove_block_(const int dir_fd, const char *file_name); const char *dir_; block_id_t min_block_id_; block_id_t max_block_id_; diff --git a/src/logservice/palf/log_io_worker.cpp b/src/logservice/palf/log_io_worker.cpp index 08867b76e..5ef7642af 100644 --- a/src/logservice/palf/log_io_worker.cpp +++ b/src/logservice/palf/log_io_worker.cpp @@ -26,8 +26,7 @@ using namespace share; namespace palf { LogIOWorker::LogIOWorker() - : log_io_worker_num_(-1), - cb_thread_pool_(NULL), + : cb_thread_pool_(NULL), palf_env_impl_(NULL), do_task_used_ts_(0), do_task_count_(0), @@ -66,17 +65,15 @@ int LogIOWorker::init(const LogIOWorkerConfig &config, KP(throttle), KP(palf_env_impl)); } else if (OB_FAIL(queue_.init(config.io_queue_capcity_, "IOWorkerLQ"))) { PALF_LOG(ERROR, "io task queue init failed", K(ret), K(config)); - } else if (OB_FAIL(batch_io_task_mgr_.init(config.batch_width_, - config.batch_depth_, + } else if (OB_FAIL(batch_io_task_mgr_.init(config.batch_depth_, allocator, &wait_cost_stat_))) { PALF_LOG(ERROR, "BatchLogIOFlushLogTaskMgr init failed", K(ret), K(config)); } else { share::ObThreadPool::set_run_wrapper(MTL_CTX()); - log_io_worker_num_ = config.io_worker_num_; cb_thread_pool_ = cb_thread_pool; palf_env_impl_ = palf_env_impl; - PALF_REPORT_INFO_KV(K_(log_io_worker_num), KP_(cb_thread_pool)); + PALF_REPORT_INFO_KV(KP_(cb_thread_pool)); throttle_ = throttle; log_io_worker_queue_size_stat_.set_extra_info(EXTRA_INFOS); purge_throttling_task_submitted_seq_ = 0; @@ -112,7 +109,6 @@ void LogIOWorker::destroy() throttle_ = NULL; cb_thread_pool_ = NULL; palf_env_impl_ = NULL; - log_io_worker_num_ = -1; queue_.destroy(); batch_io_task_mgr_.destroy(); } @@ -271,7 +267,6 @@ bool LogIOWorker::need_reduce_(LogIOTask *io_task) break; case LogIOTaskType::FLUSH_META_TYPE: case LogIOTaskType::TRUNCATE_PREFIX_TYPE: - case LogIOTaskType::TRUNCATE_LOG_TYPE: default: break; } @@ -339,8 +334,7 @@ int LogIOWorker::update_throttling_options_() } LogIOWorker::BatchLogIOFlushLogTaskMgr::BatchLogIOFlushLogTaskMgr() - : handle_count_(0), usable_count_(0), batch_width_(0), - wait_cost_stat_(NULL) + : batch_io_task_(), handle_count_(0), wait_cost_stat_(NULL) {} LogIOWorker::BatchLogIOFlushLogTaskMgr::~BatchLogIOFlushLogTaskMgr() @@ -348,41 +342,14 @@ LogIOWorker::BatchLogIOFlushLogTaskMgr::~BatchLogIOFlushLogTaskMgr() destroy(); } -int LogIOWorker::BatchLogIOFlushLogTaskMgr::init(int64_t batch_width, - int64_t batch_depth, +int LogIOWorker::BatchLogIOFlushLogTaskMgr::init(int64_t batch_depth, ObIAllocator *allocator, ObMiniStat::ObStatItem *wait_cost_stat) { int ret = OB_SUCCESS; - batch_io_task_array_.set_allocator(allocator); - if (OB_FAIL(batch_io_task_array_.init(batch_width))) { - PALF_LOG(ERROR, "batch_io_task_array_ init failed", K(ret)); + if (OB_FAIL(batch_io_task_.init(batch_depth, allocator))) { + PALF_LOG(ERROR, "BatchLogIOFlushLogTask init failed", K(ret)); } else { - for (int i = 0; i < batch_width && OB_SUCC(ret); i++) { - bool last_io_task_push_success = false; - char *ptr = reinterpret_cast(mtl_malloc(sizeof(BatchLogIOFlushLogTask), "LogIOTask")); - BatchLogIOFlushLogTask *io_task = NULL; - if (NULL == ptr) { - ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(ERROR, "allocate memory failed", K(ret)); - } else if (FALSE_IT(io_task = new(ptr)(BatchLogIOFlushLogTask))) { - } else if (OB_FAIL(io_task->init(batch_depth, allocator))) { - PALF_LOG(ERROR, "BatchLogIOFlushLogTask init failed", K(ret)); - // NB: push batch will not failed becaue batch_io_task_array_ has reserved. - } else if (OB_FAIL(batch_io_task_array_.push_back(io_task))) { - PALF_LOG(ERROR, "batch_io_task_array_ push_back failed", K(ret), KP(io_task)); - } else { - last_io_task_push_success = true; - PALF_LOG(INFO, "BatchLogIOFlushLogTask init success", K(ret), K(i), - KP(io_task)); - } - if (!last_io_task_push_success && NULL != io_task) { - io_task->destroy(); - mtl_free(io_task); - io_task = NULL; - } - } - batch_width_ = usable_count_ = batch_width; wait_cost_stat_ = wait_cost_stat; } if (OB_FAIL(ret)) { @@ -393,29 +360,16 @@ int LogIOWorker::BatchLogIOFlushLogTaskMgr::init(int64_t batch_width, void LogIOWorker::BatchLogIOFlushLogTaskMgr::destroy() { - handle_count_ = batch_width_ = usable_count_ = 0; - for (int i = 0; i < batch_io_task_array_.count(); i++) { - BatchLogIOFlushLogTask *&io_task = batch_io_task_array_[i]; - if (NULL != io_task) { - io_task->~BatchLogIOFlushLogTask(); - mtl_free(io_task); - io_task = NULL; - } - } + handle_count_ = 0; + batch_io_task_.destroy(); wait_cost_stat_ = NULL; - batch_io_task_array_.destroy(); } int LogIOWorker::BatchLogIOFlushLogTaskMgr::insert(LogIOFlushLogTask *io_task) { - int ret = OB_SUCCESS; - BatchLogIOFlushLogTask *batch_io_task = NULL; - const int64_t palf_id = io_task->get_palf_id(); - if (OB_FAIL(find_usable_batch_io_task_(palf_id, batch_io_task))) { - PALF_LOG(WARN, "find_usable_batch_io_task_ failed", K(ret), K(palf_id)); - } else if (OB_FAIL(batch_io_task->push_back(io_task))) { - PALF_LOG(TRACE, "push_back failed", K(ret), K(palf_id), KPC(io_task)); - } else { + int ret = batch_io_task_.push_back(io_task); + if (OB_FAIL(ret)) { + PALF_LOG(TRACE, "push_back failed", K(ret), KPC(io_task)); } return ret; } @@ -423,18 +377,10 @@ int LogIOWorker::BatchLogIOFlushLogTaskMgr::insert(LogIOFlushLogTask *io_task) int LogIOWorker::BatchLogIOFlushLogTaskMgr::handle(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf_env_impl) { int ret = OB_SUCCESS; - const int64_t count = batch_io_task_array_.count() - usable_count_; - // Each BatchLogIOFlushLogTask is a set LogIOFlushLogTask of one palf instance, - // even if execute 'do_task_' for one of LogIOFlushLogTask failed, we need - // execute 'do_task_' for next LogIOFlushLogTask. - const int64_t first_handle_ts = ObTimeUtility::fast_current_time(); - for (int64_t i = 0; i < count; i++) { - BatchLogIOFlushLogTask *io_task = batch_io_task_array_[i]; - if (OB_ISNULL(io_task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "BatchLogIOFlushLogTask in batch_io_task_array_ is nullptr, unexpected error!!!", - K(ret), KP(io_task), K(i)); - } else if (OB_FAIL(statistics_wait_cost_(first_handle_ts, io_task))) { + BatchLogIOFlushLogTask *io_task = &batch_io_task_; + if (!empty()) { + const int64_t first_handle_ts = ObTimeUtility::fast_current_time(); + if (OB_FAIL(statistics_wait_cost_(first_handle_ts, io_task))) { PALF_LOG(WARN, "do statistics failed", K(ret)); } else if (OB_FAIL(io_task->do_task(cb_thread_pool, palf_env_impl))) { PALF_LOG(WARN, "do_task failed", K(ret), KP(io_task)); @@ -443,61 +389,17 @@ int LogIOWorker::BatchLogIOFlushLogTaskMgr::handle(LogIOTaskCbThreadPool *cb_thr wait_cost_stat_->stat(io_task->get_count(), io_task->get_accum_in_queue_time()); } io_task->reset_accum_in_queue_time(); - PALF_LOG(TRACE, "BatchLogIOFlushLogTaskMgr::handle success", K(ret), K(handle_count_), - KP(io_task)); - } - if (OB_NOT_NULL(io_task)) { - // 'handle_count_' used for statistics - handle_count_ += io_task->get_count(); - io_task->reuse(); - usable_count_++; + PALF_LOG(TRACE, "BatchLogIOFlushLogTaskMgr::handle success", K(ret), K(handle_count_), KP(io_task)); } + handle_count_ += io_task->get_count(); + io_task->reuse(); } return ret; } bool LogIOWorker::BatchLogIOFlushLogTaskMgr::empty() { - return usable_count_ == batch_width_; -} - -int LogIOWorker::BatchLogIOFlushLogTaskMgr::find_usable_batch_io_task_( - const int64_t palf_id, BatchLogIOFlushLogTask *&batch_io_task) -{ - int ret = OB_SUCCESS; - const int64_t count = batch_io_task_array_.count(); - bool found = false; - // 1. check whether the same palf id already exist in 'batch_io_task_array_', - // and get the BatchLogIOFlushLogTask in 'batch_io_task_array_' which has the - // same palf id. - for (int64_t i = 0; i < count && false == found; i++) { - BatchLogIOFlushLogTask *tmp_task = batch_io_task_array_[i]; - if (OB_ISNULL(tmp_task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected error, tmp_task is nullptr", K(ret), KP(tmp_task), K(i)); - } else if (palf_id == tmp_task->get_palf_id()) { - found = true; - batch_io_task = tmp_task; - } - } - // 2. if there is no same palf id in 'batch_io_task_array_', get the first - // available BatchLogIOFlushLogTask in 'batch_io_task_array_'. - for (int64_t i = 0; usable_count_ > 0 && i < count && false == found; i++) { - BatchLogIOFlushLogTask *tmp_task = batch_io_task_array_[i]; - if (OB_ISNULL(tmp_task)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected error, tmp_task is nullptr", K(ret), KP(tmp_task), K(i)); - } else if (INVALID_PALF_ID == tmp_task->get_palf_id()) { - found = true; - batch_io_task = tmp_task; - usable_count_--; - } - } - PALF_LOG(TRACE, "find_usable_batch_io_task_ finished", K(ret), K(usable_count_), K(count), KP(this)); - if (OB_SUCC(ret)) { - ret = true == found ? OB_SUCCESS : OB_SIZE_OVERFLOW; - } - return ret; + return 0 == batch_io_task_.get_count(); } int64_t LogIOWorker::inc_and_fetch_purge_throttling_submitted_seq_() diff --git a/src/logservice/palf/log_io_worker.h b/src/logservice/palf/log_io_worker.h index 9d637735c..8bc2ce074 100644 --- a/src/logservice/palf/log_io_worker.h +++ b/src/logservice/palf/log_io_worker.h @@ -55,20 +55,16 @@ struct LogIOWorkerConfig } bool is_valid() const { - return 0 < io_worker_num_ && 0 < io_queue_capcity_ && 0 <= batch_width_ && 0 <= batch_depth_; + return 0 < io_queue_capcity_ && 0 < batch_depth_; } void reset() { - io_worker_num_ = 0; io_queue_capcity_ = 0; - batch_width_ = 0; batch_depth_ = 0; } - int64_t io_worker_num_; int64_t io_queue_capcity_; - int64_t batch_width_; int64_t batch_depth_; - TO_STRING_KV(K_(io_worker_num), K_(io_queue_capcity), K_(batch_width), K_(batch_depth)); + TO_STRING_KV(K_(io_queue_capcity), K_(batch_depth)); }; class LogIOWorker : public share::ObThreadPool @@ -90,7 +86,7 @@ class LogIOWorker : public share::ObThreadPool int notify_need_writing_throttling(const bool &need_throtting); static constexpr int64_t MAX_THREAD_NUM = 1; - TO_STRING_KV(K_(log_io_worker_num), KP_(cb_thread_pool), K_(purge_throttling_task_handled_seq), K_(purge_throttling_task_submitted_seq)); + TO_STRING_KV(KP_(cb_thread_pool), K_(purge_throttling_task_handled_seq), K_(purge_throttling_task_submitted_seq)); private: bool need_reduce_(LogIOTask *task); int reduce_io_task_(ObLink *task); @@ -109,21 +105,17 @@ class LogIOWorker : public share::ObThreadPool public: BatchLogIOFlushLogTaskMgr(); ~BatchLogIOFlushLogTaskMgr(); - int init(int64_t batch_width, int64_t batch_depth, ObIAllocator *allocator, ObMiniStat::ObStatItem *wait_cost_stat); + int init(int64_t batch_depth, ObIAllocator *allocator, ObMiniStat::ObStatItem *wait_cost_stat); void destroy(); int insert(LogIOFlushLogTask *io_task); int handle(LogIOTaskCbThreadPool *cb_thread_pool, IPalfEnvImpl *palf_env_impl); bool empty(); - TO_STRING_KV(K_(batch_io_task_array), K_(usable_count), K_(batch_width)); + TO_STRING_KV(K_(batch_io_task)); private: - int find_usable_batch_io_task_(const int64_t palf_id, BatchLogIOFlushLogTask *&batch_io_task); int statistics_wait_cost_(int64_t first_handle_time, BatchLogIOFlushLogTask *batch_io_task); private: - typedef ObFixedArray BatchLogIOFlushLogTaskArray; - BatchLogIOFlushLogTaskArray batch_io_task_array_; + BatchLogIOFlushLogTask batch_io_task_; int64_t handle_count_; - int64_t usable_count_; - int64_t batch_width_; ObMiniStat::ObStatItem *wait_cost_stat_; }; typedef common::ObSpinLock SpinLock; @@ -133,8 +125,6 @@ class LogIOWorker : public share::ObThreadPool // will consume it, at nowdays, the io_task_queue is single consumer and mutil // producers model. - // NB: at nowdays, the default 'log_io_worker_num_' is 1. - int64_t log_io_worker_num_; LogIOTaskCbThreadPool *cb_thread_pool_; IPalfEnvImpl *palf_env_impl_; ObTLinkQueue16 queue_; diff --git a/src/logservice/palf/log_io_worker_wrapper.cpp b/src/logservice/palf/log_io_worker_wrapper.cpp index 06067c4b0..8e4119609 100644 --- a/src/logservice/palf/log_io_worker_wrapper.cpp +++ b/src/logservice/palf/log_io_worker_wrapper.cpp @@ -23,12 +23,8 @@ namespace palf { LogIOWorkerWrapper::LogIOWorkerWrapper() - : is_user_tenant_(false), - log_writer_parallelism_(-1), - log_io_workers_(NULL), - throttle_(), - round_robin_idx_(-1), - is_inited_(false) {} + : log_io_worker_(), throttle_(), is_inited_(false) +{} LogIOWorkerWrapper::~LogIOWorkerWrapper() @@ -39,12 +35,8 @@ LogIOWorkerWrapper::~LogIOWorkerWrapper() void LogIOWorkerWrapper::destroy() { is_inited_ = false; - round_robin_idx_ = -1; throttle_.reset(); - destory_and_free_log_io_workers_(); - // reset after destory_and_free_log_io_workers_ - log_writer_parallelism_ = -1; - is_user_tenant_ = false; + log_io_worker_.destroy(); } int LogIOWorkerWrapper::init(const LogIOWorkerConfig &config, LogIOTaskCbThreadPool *cb_thread_pool, @@ -55,19 +47,16 @@ int LogIOWorkerWrapper::init(const LogIOWorkerConfig &config, if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("LogIOWorkerWrapper has inited twice", K(config)); - } else if (!config.is_valid() || OB_UNLIKELY(!true) - || OB_ISNULL(cb_thread_pool) || OB_ISNULL(allocator) || OB_ISNULL(palf_env_impl)) { + } else if (!config.is_valid() || OB_ISNULL(cb_thread_pool) + || OB_ISNULL(allocator) || OB_ISNULL(palf_env_impl)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(config), KP(cb_thread_pool), KP(allocator), KP(palf_env_impl)); - } else if (OB_FAIL(create_and_init_log_io_workers_(config, cb_thread_pool, - allocator, palf_env_impl))) { - LOG_WARN("init_log_io_workers_ failed", K(config)); + } else if (OB_FAIL(log_io_worker_.init(config, cb_thread_pool, allocator, + &throttle_, false, palf_env_impl))) { + LOG_WARN("init log io worker failed", K(config)); } else { - is_user_tenant_ = false; - log_writer_parallelism_ = config.io_worker_num_; throttle_.reset(); - round_robin_idx_ = 0; is_inited_ = true; LOG_INFO("success to init LogIOWorkerWrapper", K(config), KPC(this)); } @@ -77,18 +66,10 @@ int LogIOWorkerWrapper::init(const LogIOWorkerConfig &config, return ret; } -LogIOWorker *LogIOWorkerWrapper::get_log_io_worker(const int64_t palf_id) -{ - int64_t index = palf_id_to_index_(palf_id); - LogIOWorker *iow = log_io_workers_ + index; - PALF_LOG(INFO, "get_log_io_worker success", KPC(this), K(palf_id), K(index), KP(iow)); - return iow; -} - int LogIOWorkerWrapper::start() { - int ret = OB_SUCCESS; - if (OB_FAIL(start_())) { + int ret = log_io_worker_.start(); + if (OB_FAIL(ret)) { LOG_WARN("failed to start log_io_workers_"); } else { LOG_INFO("success to start LogIOWorkerWrapper", KPC(this)); @@ -99,14 +80,14 @@ int LogIOWorkerWrapper::start() void LogIOWorkerWrapper::stop() { PALF_LOG(INFO, "LogIOWorkerWrapper starts stopping", KPC(this)); - stop_(); + log_io_worker_.stop(); PALF_LOG(INFO, "LogIOWorkerWrapper has finished stopping", KPC(this)); } void LogIOWorkerWrapper::wait() { PALF_LOG(INFO, " LogIOWorkerWrapper starts waiting", KPC(this)); - wait_(); + log_io_worker_.wait(); PALF_LOG(INFO, "LogIOWorkerWrapper has finished waiting", KPC(this)); } @@ -130,113 +111,9 @@ int64_t LogIOWorkerWrapper::get_last_working_time() const if (IS_NOT_INIT) { PALF_LOG_RET(ERROR, OB_NOT_INIT, "LogIOWorkerWrapper not inited", KPC(this)); } else { - for (int64_t i = 0; i < log_writer_parallelism_; i++) { - last_working_time = MAX(last_working_time, log_io_workers_[i].get_last_working_time()); - } + last_working_time = log_io_worker_.get_last_working_time(); } return last_working_time; } - -int LogIOWorkerWrapper::create_and_init_log_io_workers_(const LogIOWorkerConfig &config, - LogIOTaskCbThreadPool *cb_thread_pool, - ObIAllocator *allocator, - IPalfEnvImpl *palf_env_impl) -{ - int ret = OB_SUCCESS; - log_writer_parallelism_ = 0; - const int64_t log_writer_parallelism = config.io_worker_num_; - log_io_workers_ = reinterpret_cast(share::mtl_malloc( - (log_writer_parallelism) * sizeof(LogIOWorker), "LogIOWS")); - if (NULL == log_io_workers_) { - ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(WARN, "allocate memory failed", K(log_writer_parallelism)); - } - for (int64_t i = 0; i < log_writer_parallelism && OB_SUCC(ret); i++) { - LogIOWorker *iow = log_io_workers_ + i; - iow = new(iow)LogIOWorker(); - // NB:only sys log streams of user tenants need to ignore throtting - bool need_ignoring_throttling = false; - if (OB_FAIL(iow->init(config, cb_thread_pool, allocator, - &throttle_, need_ignoring_throttling, palf_env_impl))) { - PALF_LOG(WARN, "init LogIOWorker failed", K(i), K(config), - KP(cb_thread_pool), KP(allocator), KP(palf_env_impl)); - } else { - log_writer_parallelism_++; - PALF_LOG(INFO, "init LogIOWorker success", K(i), K(config), - KP(cb_thread_pool), KP(allocator), KP(palf_env_impl), KP(iow), - KP(log_io_workers_)); - } - } - if (OB_FAIL(ret)) { - destory_and_free_log_io_workers_(); - log_writer_parallelism_ = -1; - } - return ret; -} - -int LogIOWorkerWrapper::start_() -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; i < log_writer_parallelism_ && OB_SUCC(ret); i++) { - LogIOWorker *iow = log_io_workers_ + i; - if (OB_FAIL(iow->start())) { - PALF_LOG(WARN, "start LogIOWorker failed", K(i)); - } - } - return ret; -} - -void LogIOWorkerWrapper::stop_() -{ - for (int64_t i = 0; i < log_writer_parallelism_; i++) { - LogIOWorker *iow = log_io_workers_ + i; - iow->stop(); - } -} - -void LogIOWorkerWrapper::wait_() -{ - for (int64_t i = 0; i < log_writer_parallelism_; i++) { - LogIOWorker *iow = log_io_workers_ + i; - iow->wait(); - } -} - -void LogIOWorkerWrapper::destory_and_free_log_io_workers_() -{ - PALF_LOG(INFO, "destory_log_io_workers_ success", KPC(this)); - if (NULL != log_io_workers_) { - for (int64_t i = 0; i < log_writer_parallelism_; i++) { - LogIOWorker *iow = log_io_workers_ + i; - iow->stop(); - iow->wait(); - iow->destroy(); - iow->~LogIOWorker(); - } - share::mtl_free(log_io_workers_); - log_io_workers_ = NULL; - } -} - -int64_t LogIOWorkerWrapper::palf_id_to_index_(const int64_t palf_id) -{ - int64_t index = -1; - // For sys log stream, index set to 0. - if (is_sys_palf_id(palf_id)) { - index = SYS_LOG_IO_WORKER_INDEX; - } else { - const int64_t hash_factor = log_writer_parallelism_ - 1; - if (hash_factor <= 0 && is_user_tenant_) { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected error, log_writer_parallelism_ must be greater than 1 when it's user tenant", - KPC(this), K(palf_id)); - OB_ASSERT(false); - } - // NB: SYS_LOG_IO_WORKER_INDEX is 0, others should not use this LogIOWorker. - index = (round_robin_idx_++ % hash_factor) + 1; - PALF_LOG(INFO, "palf_id_to_index_ success", KPC(this), K(palf_id), K(index)); - OB_ASSERT(index < log_writer_parallelism_); - } - return index; -} }//end of namespace palf }//end of namespace oceanbase diff --git a/src/logservice/palf/log_io_worker_wrapper.h b/src/logservice/palf/log_io_worker_wrapper.h index a794e75d6..33ed83791 100644 --- a/src/logservice/palf/log_io_worker_wrapper.h +++ b/src/logservice/palf/log_io_worker_wrapper.h @@ -37,33 +37,14 @@ class LogIOWorkerWrapper int start(); void stop(); void wait(); - // NB: nowdays, this interface can only be called when create_palf_handle_impl!!! otherwise, round_robin_idx_ - // will not be correct. - LogIOWorker *get_log_io_worker(const int64_t palf_id); + LogIOWorker *get_log_io_worker() { return &log_io_worker_; } int notify_need_writing_throttling(const bool &need_throtting); int64_t get_last_working_time() const; - TO_STRING_KV(K_(is_inited), K_(is_user_tenant), K_(log_writer_parallelism), KP(log_io_workers_), K_(round_robin_idx)); + TO_STRING_KV(K_(is_inited), K_(log_io_worker)); private: - int create_and_init_log_io_workers_(const LogIOWorkerConfig &config, - LogIOTaskCbThreadPool *cb_thread_pool, - ObIAllocator *allocaotr, - IPalfEnvImpl *palf_env_impl); - int start_(); - void stop_(); - void wait_(); - void destory_and_free_log_io_workers_(); - int64_t palf_id_to_index_(const int64_t palf_id); - constexpr static int64_t SYS_LOG_IO_WORKER_INDEX = 0; - -private: - bool is_user_tenant_; - // 'log_writer_parallelism_' has include LogIOWorker which is used for sys log stream. - int64_t log_writer_parallelism_; - // The layout of LogIOWorker: | sys log ioworker | others | - LogIOWorker *log_io_workers_; + LogIOWorker log_io_worker_; LogWritingThrottle throttle_; - int64_t round_robin_idx_; bool is_inited_; }; diff --git a/src/logservice/palf/log_iterator_impl.h b/src/logservice/palf/log_iterator_impl.h index f81311605..61b0579ed 100644 --- a/src/logservice/palf/log_iterator_impl.h +++ b/src/logservice/palf/log_iterator_impl.h @@ -38,7 +38,6 @@ namespace oceanbase using namespace share; namespace palf { -typedef ObFunction GetModeVersion; // =========== LogEntryType start ============= enum class LogEntryType { @@ -93,8 +92,7 @@ class LogIteratorImpl public: LogIteratorImpl(); ~LogIteratorImpl(); - int init(const GetModeVersion &mode_version, - IteratorStorage *log_storage); + int init(IteratorStorage *log_storage); void destroy(); void reuse(); @@ -121,8 +119,7 @@ class LogIteratorImpl // - iterator start lsn is not the header of LogGroupEntry. int next(const share::SCN &replayable_point_scn, LogIOContext &io_ctx); - // param[in] replayable point scn, iterator will ensure that no log will return when the log scn is greater than - // 'replayable_point_scn' and the log is raw write + // param[in] replayable point scn; no log newer than it will be returned. // param[out] the min log scn of next log, is's valid only when return value is OB_ITER_END // param[out] iterate_end_by_replayable_point, return OB_ITER_END whether caused by replayable_point_scn. // OB_SUCCESS. @@ -156,7 +153,7 @@ class LogIteratorImpl // NB: if the last write option success, but the data has been // corrupted, we also regard it as the last write option is // not atomic. - int get_entry(ENTRY &entry, LSN &lsn, bool &is_raw_write); + int get_entry(ENTRY &entry, LSN &lsn); bool is_valid() const; bool check_is_the_last_entry(LogIOContext &io_ctx); @@ -164,8 +161,8 @@ class LogIteratorImpl LSN get_curr_read_lsn() const; TO_STRING_KV(KP(buf_), K_(next_round_pread_size), K_(curr_read_pos), K_(curr_read_buf_start_pos), - K_(curr_read_buf_end_pos), KPC(log_storage_), K_(curr_entry_is_raw_write), K_(curr_entry_size), - K_(prev_entry_scn), K_(curr_entry), K_(init_mode_version), K_(accumulate_checksum), + K_(curr_read_buf_end_pos), KPC(log_storage_), K_(curr_entry_size), + K_(prev_entry_scn), K_(curr_entry), K_(accumulate_checksum), K_(curr_entry_is_padding), K_(padding_entry_size), K_(padding_entry_scn)); private: @@ -196,7 +193,7 @@ class LogIteratorImpl // OB_ITER_END // -- means log entry is iterated end by replayable_point_scn // OB_PARTIAL_LOG - // -- iterator start lsn is not the header of LogGroupEntry. + // -- iterator start lsn is not the header of LogGroupEntry. int parse_one_entry_(const SCN &replayable_point_scn, IterateEndInfo &info); @@ -225,24 +222,16 @@ class LogIteratorImpl if (curr_entry_is_padding_ && OB_FAIL(construct_padding_log_entry_(pos, padding_entry_size_))) { PALF_LOG(WARN, "construct_padding_log_entry_ failed", KPC(this)); } else if (OB_FAIL(curr_entry_.deserialize(buf_, curr_read_buf_end_pos_, pos))) { - } else if (curr_entry_is_raw_write_ && curr_entry_.get_scn() > replayable_point_scn) { - ret = OB_ITER_END; - info.log_scn_ = curr_entry_.get_scn(); - info.reason_ = IterateEndReason::DUE_TO_REPLAYABLE_POINT_SCN_LOG_ENTRY; - PALF_LOG(TRACE, "iterate end by replayable_point", KPC(this), K(replayable_point_scn), K(info)); } } else { ret = OB_PARTIAL_LOG; - PALF_LOG(WARN, "parse LogEntry failed, iterator start lsn may not be aligned", + PALF_LOG(WARN, "entry type does not match LogEntry", KPC(this), K(replayable_point_scn), K(info)); } return ret; } // When entry in file is LogGroupEntry, handle it specifically. - // 1. for raw write LogGroupEntry, if it's controlled by replayable_point_scn, no need update - // 'curr_read_pos_', 'accum_checksum_', because this log may later become unreadable. - // 2. for append LogGroupEntry, handle it normally. int parse_log_group_entry_(const SCN &replayable_point_scn, IterateEndInfo &info) { @@ -252,7 +241,7 @@ class LogIteratorImpl int64_t pos = curr_read_pos_; if (true == matched_type) { if (OB_FAIL(curr_entry_.deserialize(buf_, curr_read_buf_end_pos_, pos))) { - } else if (OB_FAIL(handle_each_log_group_entry_(curr_entry_, replayable_point_scn, info))) { + } else if (OB_FAIL(handle_each_log_group_entry_(curr_entry_))) { if (OB_ITER_END != ret) { PALF_LOG(WARN, "handle_each_log_group_entry_ failed", KPC(this), K(info), K(replayable_point_scn)); } else { @@ -261,7 +250,7 @@ class LogIteratorImpl } } else if (OB_FAIL(actual_entry.deserialize(buf_, curr_read_buf_end_pos_, pos))) { PALF_LOG(TRACE, "deserialize entry failed", K(ret), KPC(this)); - } else if (OB_FAIL(handle_each_log_group_entry_(actual_entry, replayable_point_scn, info))) { + } else if (OB_FAIL(handle_each_log_group_entry_(actual_entry))) { if (OB_ITER_END != ret) { PALF_LOG(WARN, "handle_each_log_group_entry_ failed", KPC(this), K(actual_entry), K(info), K(replayable_point_scn)); } else { @@ -285,13 +274,10 @@ class LogIteratorImpl int read_data_from_storage_(LogIOContext &io_ctx); void advance_read_lsn_(const offset_t step); - void try_clean_up_cache_(); template // when T is not LogGroupEntry, no need do anything - int handle_each_log_group_entry_(const T&entry, - const SCN &replayable_point_scn, - IterateEndInfo &info) + int handle_each_log_group_entry_(const T&entry) { PALF_LOG(TRACE, "T is not LogGroupEntry, do no thing", K(entry)); return OB_SUCCESS; @@ -302,60 +288,15 @@ class LogIteratorImpl // 1. check accumulate checksum: // - if accumulate checksum is not match, return OB_CHECKSUM_ERROR // - if data checksum is not match, return OB_INVALID_DATA - // 2. check this entry whether need control by 'replayable_point_scn': - // - if control by 'replayable_point_scn', return OB_ITER_END, and don't modify - // several fields in LogIteratorImpl('curr_read_pos_', 'curr_entry_is_raw_write_') - // - if not control by 'replayable_point_scn', return OB_SUCCESS. - int handle_each_log_group_entry_(const LogGroupEntry&entry, - const SCN &replayable_point_scn, - IterateEndInfo &info) + int handle_each_log_group_entry_(const LogGroupEntry&entry) { int ret = OB_SUCCESS; - bool curr_entry_is_raw_write = entry.get_header().is_raw_write(); int64_t new_accumulate_checksum = -1; PALF_LOG(TRACE, "T is LogGroupEntry", K(entry)); if (OB_FAIL(verify_accum_checksum_(entry, new_accumulate_checksum))) { PALF_LOG(WARN, "verify_accum_checksum_ failed", K(ret), KPC(this), K(entry)); - // NB: when current entry is raw write, and the log scn of current entry is greater than - // replayable_point_scn, this log may be clean up, therefore we can not update several fields of - // LogIteratorImpl, return OB_ITER_END directly, otherwise, we may not parse new LogGroupEntryHeader - // after the replayable boundary, this will cause one log which is append, but control by replayable_point_scn. - // - // NB: we need check the min scn of LogGroupEntry whether has been greater than - // replayable_point_scn: - // - if LogGroupEntry has two LogEntry, the scn of them are 10, 15 respectively, - // replayable_point_scn is 12. in this case, we can read first LogEntry, and - // we can update several fields like 'curr_entry_is_raw_write_', 'accum_checksum_' - // and the others('curr_read_pos_'...). when the replayable boundary is 12, the LogEntry - // after 10 will be truncated, the new LogGroupEntry will be generated, meanwhile, - // we will advanced 'curr_read_pos_' to the end of first LogEntry and read new LogGroupEntry - // correctly. - // - if LogGroupEntry has one LogEntry, the scn of it is 13, the several fields are - // not been updated because of it's controlled by replayable_point_scn, when the replayable - // scn is 12, we don't need rollback these fields. - // - // NB: for PalfGroupBufferIterator, we should use max scn to control replay and use min scn - // as the log_scn_ of info. consider that, replayable_point_scn is 12, the min scn of group log - // is 7 and max scn of group 15, we should not return this log. meanwhile, we should use - // scn 7 to update next_min_scn. - } else if (true == curr_entry_is_raw_write) { - SCN min_scn; - bool is_group_iterator = std::is_same::value; - if (OB_FAIL(entry.get_log_min_scn(min_scn))) { - PALF_LOG(ERROR, "get_log_min_scn failed", K(ret), KPC(this), K(min_scn), - K(entry), K(replayable_point_scn)); - } else if ((is_group_iterator && entry.get_scn() > replayable_point_scn) - || (!is_group_iterator && min_scn > replayable_point_scn)) { - info.log_scn_ = min_scn; - info.reason_ = IterateEndReason::DUE_TO_REPLAYABLE_POINT_SCN_LOG_GROUP_ENTRY; - ret = OB_ITER_END; - PALF_LOG(TRACE, "iterate end by replayable_point", K(ret), KPC(this), K(min_scn), - K(entry), K(replayable_point_scn), K(info), K(is_group_iterator)); - } else { - } } if (OB_SUCC(ret)) { - curr_entry_is_raw_write_ = entry.get_header().is_raw_write(); accumulate_checksum_ = new_accumulate_checksum; // To support get PADDING entry, need record the meta info of PADDING entry set_padding_info_(entry); @@ -421,11 +362,9 @@ static constexpr int MAX_READ_TIMES_IN_EACH_NEXT = 2; int64_t next_round_pread_size_; IteratorStorage *log_storage_; ENTRY curr_entry_; - bool curr_entry_is_raw_write_; // this fields record the entry size of curr readable entry. // NB: when 'curr_entry_size_' is 0, means it's not readable. int64_t curr_entry_size_; - int64_t init_mode_version_; // The log scn of prev entry, only effect when 'curr_entry_' is invalid. // // Add this field is only used for interface next(replayable_point_scn, &next_min_scn) @@ -433,7 +372,6 @@ static constexpr int MAX_READ_TIMES_IN_EACH_NEXT = 2; // when 'next' return OB_SUCCESS, use 'prev_entry_scn_' to record the log ts of 'curr_entry_'. // share::SCN prev_entry_scn_; - GetModeVersion get_mode_version_; int64_t accumulate_checksum_; // To support get PADDING ENTRY, add following fields: int64_t curr_entry_is_padding_; @@ -451,9 +389,7 @@ LogIteratorImpl::LogIteratorImpl() next_round_pread_size_(0), log_storage_(NULL), curr_entry_(), - curr_entry_is_raw_write_(false), curr_entry_size_(0), - init_mode_version_(0), prev_entry_scn_(), accumulate_checksum_(-1), curr_entry_is_padding_(false), @@ -470,8 +406,7 @@ LogIteratorImpl::~LogIteratorImpl() } template -int LogIteratorImpl::init(const GetModeVersion &get_mode_version, - IteratorStorage *log_storage) +int LogIteratorImpl::init(IteratorStorage *log_storage) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -483,10 +418,7 @@ int LogIteratorImpl::init(const GetModeVersion &get_mode_version, next_round_pread_size_ = MAX_LOG_BUFFER_SIZE; log_storage_ = log_storage; curr_entry_.reset(); - curr_entry_is_raw_write_ = false; curr_entry_size_ = 0; - init_mode_version_ = PALF_INITIAL_PROPOSAL_ID; - get_mode_version_ = get_mode_version; prev_entry_scn_.reset(); accumulate_checksum_ = -1; curr_entry_is_padding_ = false; @@ -506,9 +438,7 @@ void LogIteratorImpl::reuse() curr_read_buf_end_pos_ = 0; next_round_pread_size_ = MAX_LOG_BUFFER_SIZE; curr_entry_.reset(); - curr_entry_is_raw_write_ = false; curr_entry_size_ = 0; - init_mode_version_ = PALF_INITIAL_PROPOSAL_ID; prev_entry_scn_.reset(); accumulate_checksum_ = -1; curr_entry_is_padding_ = false; @@ -526,9 +456,7 @@ void LogIteratorImpl::destroy() curr_entry_is_padding_ = false; accumulate_checksum_ = -1; prev_entry_scn_.reset(); - init_mode_version_ = PALF_INITIAL_PROPOSAL_ID; curr_entry_size_ = 0; - curr_entry_is_raw_write_ = false; curr_entry_.reset(); log_storage_ = NULL; next_round_pread_size_ = 0; @@ -536,7 +464,6 @@ void LogIteratorImpl::destroy() curr_read_buf_end_pos_ = 0; curr_read_buf_start_pos_ = 0; curr_read_pos_ = 0; - init_mode_version_ = 0; } } @@ -647,17 +574,6 @@ int LogIteratorImpl::next(const share::SCN &replayable_point_scn, iterate_end_by_replayable_point = false; IterateEndInfo info; - // NB: when return OB_ITER_END, we need try to clean up cache, and we should clean up cache only when - // the log ts of curr entry is greater than 'replayable_point_scn', otherwise, we would return some logs - // that should not be returned, consider following case: - // 1. T1, 'replayable_point_scn' is 10, the log ts of curr entry is 15.(no any bad effect) - // 2. T2, 'replayable_point_scn' is 10, the logs on disk after 10 are no longer readable, and - // return OB_ITER_END because of 'file end lsn'.(no any bad effect) - // 3. T3, 'replayable_point_scn' has been advanced to 16, and write several logs on disk, however, the cache - // of iterator has not been clean up, the old logs will be returned.(bad effect) - // - // Therefore, we should try_clean_up_cache_ in the beginning of each round of next. - (void) try_clean_up_cache_(); if (!replayable_point_scn.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid argument", K(replayable_point_scn), KPC(this)); @@ -719,13 +635,10 @@ int LogIteratorImpl::next(const share::SCN &replayable_point_scn, next_min_scn); PALF_LOG(TRACE, "update next_min_scn to max of prev_entry_scn_ and next_min_scn", KPC(this), K(info), K(replayable_point_scn)); - // NB: To make 'next_min_scn' newly, advance 'prev_entry_scn_' when 'curr_entry_' need control by readable point scn. - // consider follower case: + // Advance prev_entry_scn_ when the current entry is beyond the readable point: // T1, iterate an entry successfully, and make prev_entry_scn to 5; // T2, iterate control by readable point scn, scn of 'curr_entry_' is 10, and readable point scn is 8. - // T3, the logs after 8 are no longer readable, and no log has been written. - // if we don't advance 'prev_entry_scn_' to 8, the continuous point of replay service can not update - // to 8. + // If no later log is written, advance prev_entry_scn_ so replay can advance to 8. if (info.log_scn_ > replayable_point_scn) { prev_entry_scn_ = MAX(replayable_point_scn, prev_entry_scn_); PALF_LOG(TRACE, "update prev_entry_scn_ to replayable_point_scn", KPC(this), K(info), K(replayable_point_scn)); @@ -746,7 +659,7 @@ int LogIteratorImpl::next(const share::SCN &replayable_point_scn, } template -int LogIteratorImpl::get_entry(ENTRY &entry, LSN &lsn, bool &is_raw_write) +int LogIteratorImpl::get_entry(ENTRY &entry, LSN &lsn) { int ret = OB_SUCCESS; int64_t pos = curr_read_pos_; @@ -763,7 +676,6 @@ int LogIteratorImpl::get_entry(ENTRY &entry, LSN &lsn, bool &is_raw_write PALF_LOG(WARN, "invalid data", K(ret), KPC(this), K(entry)); } else { lsn = log_storage_->get_lsn(curr_read_pos_); - is_raw_write = curr_entry_is_raw_write_; } return ret; } @@ -906,7 +818,6 @@ bool LogIteratorImpl::check_is_the_last_entry(LogIOContext &io_ctx) PALF_LOG(ERROR, "read_data_from_storage_ failed", K(ret), KPC(this)); } else if (OB_ITER_END == ret) { PALF_LOG(INFO, "has iterate end", K(ret), KPC(this)); - // NB: compatibility // If the size of current readable data is smaller than header size, return OB_ITER_END. } else if (curr_read_buf_end_pos_ - curr_read_buf_start_pos_ < header.get_serialize_size()) { ret = OB_ITER_END; @@ -1012,53 +923,6 @@ int LogIteratorImpl::read_data_from_storage_(LogIOContext &io_ctx) return ret; } -template -void LogIteratorImpl::try_clean_up_cache_() -{ - const bool matched_type = std::is_same::value; - const int64_t current_mode_version = get_mode_version_(); - if (true == matched_type) { - // do nothing - } else if (INVALID_PROPOSAL_ID == current_mode_version || init_mode_version_ > current_mode_version) { - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "current_mode_version is unexpected", K(current_mode_version), KPC(this)); - } else if (init_mode_version_ < current_mode_version) { - PALF_LOG_RET(WARN, OB_SUCCESS, "mode version has been changed, need reset cache buf", KPC(this), K(current_mode_version)); - LSN curr_read_lsn = log_storage_->get_lsn(curr_read_pos_); - // reuse LogIteratorStorage firstly, only the log before 'start_lsn_' + 'curr_read_pos_' - // has been consumed. - // NB: need ensure that we can not update 'curr_read_pos_' to 'curr_read_pos_' + sizeof(LogGroupEntryHeader) - // when the LogGroupEntryHeader after 'curr_read_pos_' need be controlled by replayable_point_scn. - log_storage_->reuse(curr_read_lsn); - curr_read_buf_start_pos_ = 0; - curr_read_pos_ = 0; - curr_read_buf_end_pos_ = 0; - curr_entry_.reset(); - - // NB: we can not reset curr_entry_is_raw_write_, otherwise, the log entry after replayable_point_scn may no be - // controlled by replayable_point_scn. - // consider that: - // 1. At T1 timestamp, the LogGroupEntry has three LogEntry, the first LogEntry has been seen by ReplayService, - // however, the remained LogEntry can not been seen due to replayable_point_scn (current replayable_point_scn - // may be lag behind others). - // 2. At T2 timestamp, mode version has changed, and there are several logs whose SCN is greater than - // replayable_point_scn. - // 3. At T3 timestamp, ReplayService use next() function, iterator will try_clean_up_cache_ because mode version - // has been changed. if reset curr_entry_is_raw_write_, the second LogEntry may be seen by ReplayService even - // if the SCN of this LogEntry is greater than replayable_point_scn. - // curr_entry_is_raw_write_ = false; - curr_entry_size_ = 0; - init_mode_version_ = current_mode_version; - - // we can not reset prev_entry_scn_, otherwise, if there is no logs which can be readable on disk, - // we can not return a valid next_min_scn. - // - prev_entry_.reset(); - - // we need reset accum_checksum_, otherwise, the accum_checksum_ is from the previous readable range, and iterate new - // group log will fail. - accumulate_checksum_ = -1; - reset_padding_info_(); - } -} } // end namespace palf } // end namespace oceanbase #endif diff --git a/src/logservice/palf/log_learner.cpp b/src/logservice/palf/log_learner.cpp deleted file mode 100644 index a791e1ffc..000000000 --- a/src/logservice/palf/log_learner.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_learner.h" - -namespace oceanbase -{ -using namespace common; - -namespace palf -{ - -LogLearner::LogLearner() - : server_(), - register_time_us_(OB_INVALID_TIMESTAMP), - keepalive_ts_(OB_INVALID_TIMESTAMP) -{ -} - -LogLearner::LogLearner(const common::ObAddr &server, const int64_t register_time_us) - : server_(server), - register_time_us_(register_time_us), - keepalive_ts_(OB_INVALID_TIMESTAMP) -{ -} - -LogLearner::LogLearner(const LogLearner &child) - : server_(child.server_), - register_time_us_(child.register_time_us_), - keepalive_ts_(child.keepalive_ts_) -{ -} - -LogLearner::~LogLearner() -{ - reset(); -} - -bool LogLearner::is_valid() const -{ - return server_.is_valid() && register_time_us_ >= 0; -} - -void LogLearner::reset() -{ - server_.reset(); - register_time_us_ = OB_INVALID_TIMESTAMP; - keepalive_ts_ = OB_INVALID_TIMESTAMP; -} - -bool LogLearner::is_timeout(const int64_t timeout_us) const -{ - bool bool_ret = false; - const int64_t now_us = ObTimeUtility::current_time(); - if (now_us - keepalive_ts_ > timeout_us) { - bool_ret = true; - } else { - } - return bool_ret; -} - -const common::ObAddr &LogLearner::get_server() const -{ - return server_; -} - - -void LogLearner::update_keepalive_ts() -{ - const int64_t curr_time_us = ObTimeUtility::current_time(); - keepalive_ts_ = (curr_time_us > keepalive_ts_)? curr_time_us: keepalive_ts_; -} - -bool LogLearner::operator<(const LogLearner &val) const -{ - return server_ < val.server_; -} - -bool LogLearner::operator==(const LogLearner &val) const -{ - return server_ == val.server_; -} - -bool LogLearner::operator!=(const LogLearner &val) const -{ - return server_ != val.server_; -} - -LogLearner &LogLearner::operator=(const LogLearner &val) -{ - server_ = val.server_; - register_time_us_ = val.register_time_us_; - keepalive_ts_ = val.keepalive_ts_; - return *this; -} - -OB_SERIALIZE_MEMBER(LogLearner, server_, register_time_us_); -} // namespace palf end -} // namespace oceanbase end diff --git a/src/logservice/palf/log_learner.h b/src/logservice/palf/log_learner.h deleted file mode 100644 index c167c8ec4..000000000 --- a/src/logservice/palf/log_learner.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_LEARNER_H_ -#define OCEANBASE_LOGSERVICE_LOG_LEARNER_H_ - -#include "share/ob_errno.h" // KR -#include "lib/net/ob_addr.h" // common::ObAddr -#include "lib/ob_define.h" // vars... -#include "lib/utility/ob_print_utils.h" // print... -#include "lib/utility/ob_unify_serialize.h" // serialize -#include "lib/utility/ob_macro_utils.h" // some macro -#include "common/ob_learner_list.h" // common::BaseLearnerList - -namespace oceanbase -{ -namespace palf -{ - -class LogLearner -{ - OB_UNIS_VERSION(1); -public: - LogLearner(); - LogLearner(const common::ObAddr &server, const int64_t register_time_us); - LogLearner(const LogLearner &child); - virtual ~LogLearner(); - bool is_valid() const; - void reset(); - bool is_timeout(const int64_t timeout_us) const; - const common::ObAddr &get_server() const; - void update_keepalive_ts(); - bool operator<(const LogLearner &val) const; - bool operator==(const LogLearner &val) const; - bool operator!=(const LogLearner &val) const; - LogLearner &operator=(const LogLearner &val); - TO_STRING_KV(K_(server), K_(register_time_us), K_(keepalive_ts)); - -public: - // net addr of LogLearner - common::ObAddr server_; - // uniquely identifies a available - // registration when this child is in children list of parent - int64_t register_time_us_; - // last recv keepalive timestamp - int64_t keepalive_ts_; -}; - -typedef common::BaseLearnerList LogLearnerList; -typedef common::BaseLearnerList LogCandidateList; -} // namespace palf end -} // namespace oceanbase end - -#endif diff --git a/src/logservice/palf/log_loop_thread.cpp b/src/logservice/palf/log_loop_thread.cpp index acf848ac4..cb3a6e664 100644 --- a/src/logservice/palf/log_loop_thread.cpp +++ b/src/logservice/palf/log_loop_thread.cpp @@ -84,7 +84,7 @@ void LogLoopThread::log_loop_() const int64_t start_ts = ObTimeUtility::current_time(); IPalfHandleImpl *handle = nullptr; - if (OB_SUCCESS != palf_env_impl_->get_palf_handle_impl(SYS_PALF_ID, handle)) { + if (OB_SUCCESS != palf_env_impl_->get_palf_handle_impl(handle)) { ob_usleep(run_interval_, true/*is_idle_sleep*/); continue; } diff --git a/src/logservice/palf/log_meta.cpp b/src/logservice/palf/log_meta.cpp index 6b1ca5aea..f7b586914 100755 --- a/src/logservice/palf/log_meta.cpp +++ b/src/logservice/palf/log_meta.cpp @@ -23,11 +23,8 @@ namespace palf using namespace common; using namespace share; LogMeta::LogMeta() : version_(-1), - log_prepare_meta_(), - log_config_meta_(), log_mode_meta_(), - log_snapshot_meta_(), - log_replica_property_meta_() + log_snapshot_meta_() { } @@ -36,8 +33,7 @@ LogMeta::~LogMeta() { reset(); } LogMeta::LogMeta(const LogMeta &rmeta) { *this = rmeta; } int LogMeta::generate_by_palf_base_info(const PalfBaseInfo &palf_base_info, - const AccessMode &access_mode, - const LogReplicaType &replica_type) + const AccessMode &access_mode) { int ret = OB_SUCCESS; if (false == is_valid_access_mode(access_mode) || false == palf_base_info.is_valid()) { @@ -46,21 +42,10 @@ int LogMeta::generate_by_palf_base_info(const PalfBaseInfo &palf_base_info, } else if (OB_FAIL(log_snapshot_meta_.generate(palf_base_info.curr_lsn_, palf_base_info.prev_log_info_, palf_base_info.curr_lsn_))) { PALF_LOG(WARN, "generate snapshot_meta failed", K(ret), K(palf_base_info)); } else { - const int64_t prev_log_proposal_id = palf_base_info.prev_log_info_.log_proposal_id_; const SCN &prev_scn = palf_base_info.prev_log_info_.scn_; - const int64_t init_log_proposal_id = (prev_log_proposal_id != INVALID_PROPOSAL_ID)? \ - prev_log_proposal_id: PALF_INITIAL_PROPOSAL_ID; const SCN init_ref_scn = (prev_scn.is_valid() ? prev_scn: SCN::min_scn()); - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - init_config_version.generate(init_log_proposal_id, 0); - init_config_info.generate(init_config_version); version_ = LOG_META_VERSION; - log_prepare_meta_.generate(LogVotedFor(), init_log_proposal_id); - log_config_meta_.generate_for_default(init_log_proposal_id, init_config_info, init_config_info); - log_mode_meta_.generate(init_log_proposal_id, init_log_proposal_id, access_mode, init_ref_scn); - const bool allow_vote = (replica_type != ARBITRATION_REPLICA); - log_replica_property_meta_.generate(allow_vote, replica_type); + log_mode_meta_.generate(access_mode, init_ref_scn); PALF_LOG(INFO, "generate_by_palf_base_info success", KPC(this)); } return ret; @@ -81,60 +66,19 @@ int LogMeta::load(const char *buf, int64_t buf_len) bool LogMeta::is_valid() const { - return true == log_prepare_meta_.is_valid() - && true == log_config_meta_.is_valid() + return LOG_META_VERSION == version_ && true == log_mode_meta_.is_valid() - && true == log_snapshot_meta_.is_valid() - && true == log_replica_property_meta_.is_valid(); + && true == log_snapshot_meta_.is_valid(); } void LogMeta::reset() { version_ = -1; - log_prepare_meta_.reset(); - log_config_meta_.reset(); log_mode_meta_.reset(); log_snapshot_meta_.reset(); - log_replica_property_meta_.reset(); version_ = -1; } -int LogMeta::update_log_prepare_meta(const LogPrepareMeta &log_prepare_meta) -{ - int ret = OB_SUCCESS; - if (false == log_prepare_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(log_prepare_meta)); - } else { - log_prepare_meta_ = log_prepare_meta; - } - return ret; -} - -int LogMeta::update_log_config_meta(const LogConfigMeta &log_config_meta) -{ - int ret = OB_SUCCESS; - if (false == log_config_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(log_config_meta)); - } else { - log_config_meta_ = log_config_meta; - } - return ret; -} - -int LogMeta::update_log_mode_meta(const LogModeMeta &log_mode_meta) -{ - int ret = OB_SUCCESS; - if (false == log_mode_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(log_mode_meta)); - } else { - log_mode_meta_ = log_mode_meta; - } - return ret; -} - int LogMeta::update_log_snapshot_meta(const LogSnapshotMeta &log_snapshot_meta) { int ret = OB_SUCCESS; @@ -147,26 +91,11 @@ int LogMeta::update_log_snapshot_meta(const LogSnapshotMeta &log_snapshot_meta) return ret; } -int LogMeta::update_log_replica_property_meta(const LogReplicaPropertyMeta &log_replica_property_meta) -{ - int ret = OB_SUCCESS; - if (false == log_replica_property_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(log_replica_property_meta)); - } else { - log_replica_property_meta_ = log_replica_property_meta; - } - return ret; -} - void LogMeta::operator=(const LogMeta &log_meta) { version_ = log_meta.version_; - log_prepare_meta_ = log_meta.log_prepare_meta_; - log_config_meta_ = log_meta.log_config_meta_; log_mode_meta_ = log_meta.log_mode_meta_; log_snapshot_meta_ = log_meta.log_snapshot_meta_; - log_replica_property_meta_ = log_meta.log_replica_property_meta_; } DEFINE_SERIALIZE(LogMeta) @@ -176,11 +105,8 @@ DEFINE_SERIALIZE(LogMeta) if (OB_UNLIKELY(NULL == buf || buf_len < 0)) { ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) - || OB_FAIL(log_prepare_meta_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(log_config_meta_.serialize(buf, buf_len, new_pos)) || OB_FAIL(log_mode_meta_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(log_snapshot_meta_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(log_replica_property_meta_.serialize(buf, buf_len, new_pos))) { + || OB_FAIL(log_snapshot_meta_.serialize(buf, buf_len, new_pos))) { PALF_LOG(ERROR, "LogMeta serialize failed", K(ret), K(buf), K(buf_len), K(pos), K(new_pos)); } else { pos = new_pos; @@ -195,12 +121,13 @@ DEFINE_DESERIALIZE(LogMeta) int64_t new_pos = pos; if (OB_UNLIKELY(NULL == buf || data_len < 0)) { ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) - || OB_FAIL(log_prepare_meta_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(log_config_meta_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(log_mode_meta_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(log_snapshot_meta_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(log_replica_property_meta_.deserialize(buf, data_len, new_pos))) { + } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_))) { + PALF_LOG(ERROR, "decode LogMeta version failed", K(ret), K(data_len), K(new_pos)); + } else if (LOG_META_VERSION != version_) { + ret = OB_NOT_SUPPORTED; + PALF_LOG(ERROR, "unsupported LogMeta version", K(ret), K_(version)); + } else if (OB_FAIL(log_mode_meta_.deserialize(buf, data_len, new_pos)) + || OB_FAIL(log_snapshot_meta_.deserialize(buf, data_len, new_pos))) { PALF_LOG(ERROR, "LogMeta deserialize failed", K(ret), K(buf), K(data_len), K(pos), K(new_pos)); } else { PALF_LOG(INFO, "LogMeta deserialize", K(buf), K(buf + pos), K(pos), K(new_pos), K(*this)); @@ -213,11 +140,8 @@ DEFINE_GET_SERIALIZE_SIZE(LogMeta) { int64_t size = 0; size += serialization::encoded_length_i64(version_); - size += log_prepare_meta_.get_serialize_size(); - size += log_config_meta_.get_serialize_size(); size += log_mode_meta_.get_serialize_size(); size += log_snapshot_meta_.get_serialize_size(); - size += log_replica_property_meta_.get_serialize_size(); return size; } } // end namespace palf diff --git a/src/logservice/palf/log_meta.h b/src/logservice/palf/log_meta.h index bcb348498..f1a4a12c3 100644 --- a/src/logservice/palf/log_meta.h +++ b/src/logservice/palf/log_meta.h @@ -36,39 +36,28 @@ class LogMeta public: int generate_by_palf_base_info(const PalfBaseInfo &palf_base_info, - const AccessMode &access_mode, - const LogReplicaType &replica_type); + const AccessMode &access_mode); int load(const char *buf, int64_t buf_len); bool is_valid() const; void reset(); - LogPrepareMeta get_log_prepare_meta() const { return log_prepare_meta_; } - LogConfigMeta get_log_config_meta() const { return log_config_meta_; } LogModeMeta get_log_mode_meta() const { return log_mode_meta_; } LogSnapshotMeta get_log_snapshot_meta() const { return log_snapshot_meta_; } - LogReplicaPropertyMeta get_log_replica_property_meta() const { return log_replica_property_meta_; } void operator=(const LogMeta &log_meta); // The follow functions used to set few fields of this object - int update_log_prepare_meta(const LogPrepareMeta &log_prepare_meta); - int update_log_config_meta(const LogConfigMeta &log_config_meta); int update_log_snapshot_meta(const LogSnapshotMeta &log_snapshot_meta); - int update_log_replica_property_meta(const LogReplicaPropertyMeta &log_replica_property_meta); - int update_log_mode_meta(const LogModeMeta &log_mode_meta); - TO_STRING_KV(K_(version), K_(log_prepare_meta), K_(log_config_meta), K_(log_snapshot_meta), - K_(log_replica_property_meta), K_(log_mode_meta)); + TO_STRING_KV(K_(version), K_(log_snapshot_meta), + K_(log_mode_meta)); NEED_SERIALIZE_AND_DESERIALIZE; private: int64_t version_; - LogPrepareMeta log_prepare_meta_; - LogConfigMeta log_config_meta_; LogModeMeta log_mode_meta_; LogSnapshotMeta log_snapshot_meta_; - LogReplicaPropertyMeta log_replica_property_meta_; - static constexpr int64_t LOG_META_VERSION = 1; + static constexpr int64_t LOG_META_VERSION = 4; }; } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_meta_info.cpp b/src/logservice/palf/log_meta_info.cpp old mode 100755 new mode 100644 index 55805eec9..bcc7085af --- a/src/logservice/palf/log_meta_info.cpp +++ b/src/logservice/palf/log_meta_info.cpp @@ -23,1082 +23,8 @@ using namespace share; namespace palf { -LogVotedFor::LogVotedFor() -{ - reset(); -} - -LogVotedFor::LogVotedFor(const ObAddr &addr) -{ - if (addr.using_ipv4()) { - voted_for_[0] = addr.get_ipv4(); - voted_for_[1] = addr.get_port(); - } else { - voted_for_[0] = addr.get_ipv6_high(); - voted_for_[1] = addr.get_ipv6_low(); - voted_for_[2] = addr.get_port(); - } -} - -LogVotedFor::~LogVotedFor() -{ - reset(); -} - -LogVotedFor::LogVotedFor(const LogVotedFor &voted_for) -{ - MEMCPY(voted_for_, voted_for.voted_for_, COUNT*serialization::encoded_length_i64(voted_for_[0])); -} - -LogVotedFor::LogVotedFor(LogVotedFor &&voted_for) -{ - MEMCPY(voted_for_, voted_for.voted_for_, COUNT*serialization::encoded_length_i64(voted_for_[0])); -} - -LogVotedFor & LogVotedFor::operator=(const LogVotedFor &voted_for) -{ - MEMCPY(voted_for_, voted_for.voted_for_, COUNT*serialization::encoded_length_i64(voted_for_[0])); - return *this; -} - -bool LogVotedFor::operator==(const LogVotedFor &voted_for) -{ - return 0 == MEMCMP(voted_for_, voted_for.voted_for_, COUNT*serialization::encoded_length_i64(voted_for_[0])); -} - -void LogVotedFor::reset() -{ - MEMSET(voted_for_, 0, COUNT*serialization::encoded_length_i64(voted_for_[0])); -} - -int64_t LogVotedFor::to_string(char *buf, const int64_t buf_len)const -{ - ObAddr addr(voted_for_[0], voted_for_[1]); - // nowdays, just print IPV4 - return addr.to_string(buf, buf_len); -} - -DEFINE_SERIALIZE(LogVotedFor) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (OB_UNLIKELY(NULL == buf || buf_len < 0)) { - ret = OB_INVALID_ARGUMENT; - } else if (buf_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else { - for (int64_t i = 0; i < COUNT && OB_SUCC(ret); i++) { - ret = serialization::encode_i64(buf, buf_len, new_pos, voted_for_[i]); - } - } - if (OB_SUCC(ret)) { - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogVotedFor) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (OB_UNLIKELY(NULL == buf || data_len < 0)) { - ret = OB_INVALID_ARGUMENT; - } else if (data_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else { - for (int64_t i = 0; i < COUNT && OB_SUCC(ret); i++) { - ret = serialization::decode_i64(buf, data_len, new_pos, &voted_for_[i]); - } - } - if (OB_SUCC(ret)) { - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogVotedFor) -{ - int64_t size = COUNT * serialization::encoded_length_i64(voted_for_[0]); - return size; -} - -LogPrepareMeta::LogPrepareMeta() : version_(-1), voted_for_(), log_proposal_id_(INVALID_PROPOSAL_ID) -{} - -LogPrepareMeta::~LogPrepareMeta() -{ - reset(); -} - -int LogPrepareMeta::generate(const LogVotedFor &voted_for, const int64_t &log_proposal_id) -{ - int ret = OB_SUCCESS; - if (INVALID_PROPOSAL_ID == log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - } else { - version_ = LOG_PREPARE_VERSION; - voted_for_ = voted_for; - log_proposal_id_ = log_proposal_id; - } - return ret; -} - -bool LogPrepareMeta::is_valid() const -{ - // voted_for is unused - return version_ == LOG_PREPARE_VERSION && INVALID_PROPOSAL_ID != log_proposal_id_; -} - -void LogPrepareMeta::reset() -{ - log_proposal_id_ = INVALID_PROPOSAL_ID; - voted_for_.reset(); - version_ = -1; -} - -void LogPrepareMeta::operator=(const LogPrepareMeta &log_prepare_meta) -{ - this->version_ = log_prepare_meta.version_; - this->voted_for_ = log_prepare_meta.voted_for_; - this->log_proposal_id_ = log_prepare_meta.log_proposal_id_; -} - -DEFINE_SERIALIZE(LogPrepareMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (OB_UNLIKELY(NULL == buf || buf_len < 0)) { - ret = OB_INVALID_ARGUMENT; - } else if (buf_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) || - OB_FAIL(voted_for_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, log_proposal_id_))) { - PALF_LOG(ERROR, "LogPrepareMeta serialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogPreareMeta serialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogPrepareMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (OB_UNLIKELY(NULL == buf || data_len < 0)) { - ret = OB_INVALID_ARGUMENT; - } else if (data_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) || - OB_FAIL(voted_for_.deserialize(buf, data_len, new_pos)) || - OB_FAIL( - serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&log_proposal_id_)))) { - PALF_LOG(ERROR, "LogPrepareMeta deserialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogPreareMeta deserialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogPrepareMeta) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += voted_for_.get_serialize_size(); - size += serialization::encoded_length_i64(log_proposal_id_); - return size; -} - -LogConfigVersion::LogConfigVersion() : proposal_id_(INVALID_PROPOSAL_ID), config_seq_(-1) -{} - -LogConfigVersion::~LogConfigVersion() -{ - reset(); -} - -void LogConfigVersion::reset() -{ - proposal_id_ = INVALID_PROPOSAL_ID; - config_seq_ = -1; -} - -bool LogConfigVersion::is_valid() const -{ - return proposal_id_ != INVALID_PROPOSAL_ID && config_seq_ >= 0; -} - -bool LogConfigVersion::is_initial_version() const -{ - return config_seq_ <= 0; -} - -int LogConfigVersion::generate(const int64_t proposal_id, const int64_t config_seq) -{ - int ret = OB_SUCCESS; - if (proposal_id == INVALID_PROPOSAL_ID || config_seq < 0) { - ret = OB_INVALID_ARGUMENT; - } else { - proposal_id_ = proposal_id; - config_seq_ = config_seq; - } - return ret; -} - -int LogConfigVersion::inc_update_version(const int64_t proposal_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(INVALID_PROPOSAL_ID == proposal_id_ || config_seq_ < 0)) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogConfigVersion not init", K(ret), K(proposal_id_), K(config_seq_)); - } else if (OB_UNLIKELY(INVALID_PROPOSAL_ID == proposal_id || proposal_id < proposal_id_)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(proposal_id), K(proposal_id_)); - } else { - proposal_id_ = proposal_id; - config_seq_ += 1; - } - return ret; -} - -void LogConfigVersion::operator=(const LogConfigVersion &config_version) -{ - proposal_id_ = config_version.proposal_id_; - config_seq_ = config_version.config_seq_; -} - -bool LogConfigVersion::operator==(const LogConfigVersion &config_version) const -{ - return proposal_id_ == config_version.proposal_id_ && config_seq_ == config_version.config_seq_; -} - -bool LogConfigVersion::operator!=(const LogConfigVersion &config_version) const -{ - return !(*this == config_version); -} - -bool LogConfigVersion::operator>(const LogConfigVersion &config_version) const -{ - const bool self_valid = is_valid(); - const bool arg_valid = config_version.is_valid(); - bool bool_ret = false; - const bool cmp_ret = (proposal_id_ > config_version.proposal_id_ ) || - (proposal_id_ == config_version.proposal_id_ && config_seq_ > config_version.config_seq_); - if (OB_LIKELY(self_valid && arg_valid)) { - bool_ret = cmp_ret; - } else if (!self_valid) { - bool_ret = false; - } else { - bool_ret = true; - } - return bool_ret; -} - -bool LogConfigVersion::operator<(const LogConfigVersion &config_version) const -{ - const bool self_valid = is_valid(); - const bool arg_valid = config_version.is_valid(); - bool bool_ret = false; - const bool cmp_ret = (proposal_id_ < config_version.proposal_id_) || - (proposal_id_ == config_version.proposal_id_ && config_seq_ < config_version.config_seq_); - if (OB_LIKELY(self_valid && arg_valid)) { - bool_ret = cmp_ret; - } else if (!arg_valid) { - bool_ret = false; - } else { - bool_ret = true; - } - return bool_ret; -} - -bool LogConfigVersion::operator>=(const LogConfigVersion &config_version) const -{ - return (*this > config_version) || (*this == config_version); -} - -bool LogConfigVersion::operator<=(const LogConfigVersion &config_version) const -{ - return (*this < config_version) || (*this == config_version); -} - -int64_t LogConfigVersion::to_string(char *buf, const int64_t buf_len) -{ - int64_t pos = 0; - if (OB_SUCCESS != databuff_print_obj(buf, buf_len, pos, proposal_id_)) { - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "databuff_print_obj failed", K(pos)); - } else if (OB_SUCCESS != databuff_print_obj(buf, buf_len, pos, config_seq_)) { - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "databuff_print_obj failed", K(pos)); - } else { - } - return pos; -} - -DEFINE_SERIALIZE(LogConfigVersion) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, proposal_id_)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, config_seq_))) { - PALF_LOG(ERROR, "LogConfigVersion serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogConfigVersion) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= data_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &proposal_id_)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &config_seq_))) { - PALF_LOG(ERROR, "LogConfigVersion deserialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogConfigVersion) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(proposal_id_); - size += serialization::encoded_length_i64(config_seq_); - return size; -} - -LogConfigInfo::LogConfigInfo() - : log_sync_memberlist_(), - log_sync_replica_num_(0), - arbitration_member_(), - learnerlist_(), - degraded_learnerlist_(), - config_version_() -{} - -LogConfigInfo::~LogConfigInfo() -{ - reset(); -} - -bool LogConfigInfo::is_valid() const -{ - const bool is_all_list_unique = this->is_all_list_unique(); - return true == log_sync_memberlist_.is_valid() && - true == is_all_list_unique && - 0 < log_sync_replica_num_ && - common::OB_MAX_MEMBER_NUMBER >= log_sync_replica_num_ && - config_version_.is_valid(); -} - -// IP addresses of all members and learners should be different -bool LogConfigInfo::is_all_list_unique() const -{ - int ret = OB_SUCCESS; - bool is_all_list_unique = false; - GlobalLearnerList server_list; - if (arbitration_member_.is_valid() && - OB_FAIL(server_list.add_learner(arbitration_member_))) { - PALF_LOG(WARN, "add_learner failed", K(server_list), K_(arbitration_member)); - } else if (OB_FAIL(check_list_unique(server_list, log_sync_memberlist_))) { - PALF_LOG(WARN, "serverlist should not overlap with log_sync_memberlist", - K_(arbitration_member), K_(log_sync_memberlist)); - } else if (OB_FAIL(check_list_unique(server_list, degraded_learnerlist_))) { - PALF_LOG(WARN, "serverlist should not overlap with log_sync_memberlist", - K_(arbitration_member), K_(log_sync_memberlist), K_(degraded_learnerlist)); - } else if (OB_FAIL(check_list_unique(server_list, learnerlist_))) { - PALF_LOG(WARN, "serverlist should not overlap with log_sync_memberlist", - K_(arbitration_member), K_(log_sync_memberlist), K_(degraded_learnerlist), K_(learnerlist)); - } else { - is_all_list_unique = true; - } - return is_all_list_unique; -} - -void LogConfigInfo::reset() -{ - log_sync_memberlist_.reset(); - log_sync_replica_num_ = 0; - arbitration_member_.reset(); - learnerlist_.reset(); - degraded_learnerlist_.reset(); - config_version_.reset(); -} - -int LogConfigInfo::generate(const ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (false == memberlist.is_valid() || false == config_version.is_valid() || - 0 >= replica_num || OB_MAX_MEMBER_NUMBER < replica_num) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(memberlist), K(replica_num), - K(learnerlist), K(config_version)); - } else { - log_sync_memberlist_ = memberlist; - log_sync_replica_num_ = replica_num; - learnerlist_ = learnerlist; - config_version_ = config_version; - } - return ret; -} - -int LogConfigInfo::get_expected_paxos_memberlist(common::ObMemberList &paxos_memberlist, - int64_t &paxos_replica_num) const -{ - int ret = OB_SUCCESS; - if (false == log_sync_memberlist_.is_valid() || - 0 >= log_sync_replica_num_ || - common::OB_MAX_MEMBER_NUMBER < log_sync_replica_num_) { - // memberlist may be empty when bootstraping cluster, just return empty memberlist - paxos_memberlist.reset(); - paxos_replica_num = 0; - } else if (OB_UNLIKELY(degraded_learnerlist_.is_valid())) { - paxos_memberlist = log_sync_memberlist_; - paxos_replica_num = log_sync_replica_num_; - common::ObMember tmp_member; - const int64_t degraded_count = degraded_learnerlist_.get_member_number(); - for (int64_t i = 0; i < degraded_count && OB_SUCC(ret); ++i) { - if (OB_FAIL(degraded_learnerlist_.get_member_by_index(i, tmp_member))) { - PALF_LOG(WARN, "get_member_by_index failed", KR(ret), K(i), K(degraded_learnerlist_)); - } else if (OB_FAIL(paxos_memberlist.add_member(tmp_member))) { - PALF_LOG(WARN, "add_member failed", KR(ret), K(paxos_memberlist), K(tmp_member)); - } else { - paxos_replica_num++; - } - } - } else { - paxos_memberlist = log_sync_memberlist_; - paxos_replica_num = log_sync_replica_num_; - } - return ret; -} - -// generate paxos memberlist including arbitration replica -int LogConfigInfo::convert_to_complete_config(common::ObMemberList &alive_paxos_memberlist, - int64_t &alive_paxos_replica_num, - GlobalLearnerList &all_learners) const -{ - int ret = OB_SUCCESS; - if (false == log_sync_memberlist_.is_valid() || - 0 >= log_sync_replica_num_ || - common::OB_MAX_MEMBER_NUMBER < log_sync_replica_num_) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigInfo not init", KR(ret), K_(log_sync_memberlist), K_(log_sync_replica_num)); - } else if (OB_FAIL(all_learners.deep_copy(learnerlist_))) { - } else if (OB_FAIL(all_learners.append(degraded_learnerlist_))) { - } else if (OB_UNLIKELY(true == arbitration_member_.is_valid())) { - alive_paxos_memberlist = log_sync_memberlist_; - if (OB_FAIL(alive_paxos_memberlist.add_member(arbitration_member_))) { - PALF_LOG(WARN, "add_member failed", KR(ret), K(alive_paxos_memberlist), K(arbitration_member_)); - } else { - alive_paxos_replica_num = log_sync_replica_num_ + 1; - } - } else { - alive_paxos_memberlist = log_sync_memberlist_; - alive_paxos_replica_num = log_sync_replica_num_; - } - return ret; -} - -void LogConfigInfo::operator=(const LogConfigInfo &config_info) -{ - log_sync_memberlist_ = config_info.log_sync_memberlist_; - log_sync_replica_num_ = config_info.log_sync_replica_num_; - arbitration_member_ = config_info.arbitration_member_; - learnerlist_.deep_copy(config_info.learnerlist_); - degraded_learnerlist_.deep_copy(config_info.degraded_learnerlist_); - config_version_ = config_info.config_version_; - PALF_LOG(TRACE, "LogConfigInfo operator =", KPC(this), K(config_info)); -} - -bool LogConfigInfo::operator==(const LogConfigInfo &config_info) const -{ - return true == log_sync_memberlist_.member_addr_equal(config_info.log_sync_memberlist_) && - log_sync_replica_num_ == config_info.log_sync_replica_num_ && - true == learnerlist_.learner_addr_equal(config_info.learnerlist_) && - true == degraded_learnerlist_.learner_addr_equal(config_info.degraded_learnerlist_) && - config_version_ == config_info.config_version_; -} - -DEFINE_SERIALIZE(LogConfigInfo) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(log_sync_memberlist_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, log_sync_replica_num_)) || - OB_FAIL(arbitration_member_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(learnerlist_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(degraded_learnerlist_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(config_version_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigInfo serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogConfigInfo) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= data_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(log_sync_memberlist_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &log_sync_replica_num_)) || - OB_FAIL(arbitration_member_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(learnerlist_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(degraded_learnerlist_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(config_version_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigInfo deserialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogConfigInfo) -{ - int64_t size = 0; - size += log_sync_memberlist_.get_serialize_size(); - size += serialization::encoded_length_i64(log_sync_replica_num_); - size += arbitration_member_.get_serialize_size(); - size += learnerlist_.get_serialize_size(); - size += degraded_learnerlist_.get_serialize_size(); - size += config_version_.get_serialize_size(); - return size; -} - -bool is_valid_config_lock_type(int64_t lock_type) -{ - //only support member change so far - return (LOCK_NOTHING == lock_type || LOCK_PAXOS_MEMBER_CHANGE == lock_type); -} - -void LogLockMeta::reset() -{ - version_ = -1; - lock_owner_ = OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - lock_type_ = ConfigChangeLockType::LOCK_NOTHING; - lock_time_ = OB_INVALID_TIMESTAMP; -} - -bool LogLockMeta::is_valid() const -{ - const bool is_valid_locked_stat = - lock_owner_ > 0 - && LOCK_PAXOS_MEMBER_CHANGE == lock_type_ - && OB_INVALID_TIMESTAMP != lock_time_; - - const bool is_valid_unlocked_stat = (LOCK_NOTHING == lock_type_) - && ((OB_INVALID_CONFIG_CHANGE_LOCK_OWNER == lock_owner_ && OB_INVALID_TIMESTAMP == lock_time_) - || (OB_INVALID_CONFIG_CHANGE_LOCK_OWNER != lock_owner_ && OB_INVALID_TIMESTAMP != lock_time_)); - - return ((LOG_LOCK_META_VERSION == version_) - && (is_valid_locked_stat || is_valid_unlocked_stat)); -} - -int LogLockMeta::generate(const int64_t lock_owner, const int64_t lock_type) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY((LOCK_PAXOS_MEMBER_CHANGE != lock_type) || lock_owner <= 0)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(lock_type), K(lock_owner)); - } else { - version_ = LOG_LOCK_META_VERSION; - lock_owner_ = lock_owner; - lock_type_ = lock_type; - lock_time_ = ObTimeUtility::fast_current_time(); - } - return ret; -} - -void LogLockMeta::unlock() -{ - lock_type_ = LOCK_NOTHING; - lock_time_ = ObTimeUtility::fast_current_time(); -} - -void LogLockMeta::reset_as_unlocked() -{ - version_ = LOG_LOCK_META_VERSION; - lock_owner_ = OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - lock_type_ = LOCK_NOTHING; - lock_time_ = OB_INVALID_TIMESTAMP; -} - -bool LogLockMeta::is_locked() const -{ - return LOCK_PAXOS_MEMBER_CHANGE == lock_type_; -} - -bool LogLockMeta::is_lock_owner_valid() const -{ - return (OB_INVALID_CONFIG_CHANGE_LOCK_OWNER != lock_owner_); -} - -void LogLockMeta::operator=(const LogLockMeta &lock_meta) -{ - version_ = lock_meta.version_; - lock_owner_ = lock_meta.lock_owner_; - lock_type_ = lock_meta.lock_type_; - lock_time_ = lock_meta.lock_time_; -} - -bool LogLockMeta::operator==(const LogLockMeta &lock_meta) const -{ - return version_ == lock_meta.version_ - && lock_owner_ == lock_meta.lock_owner_ - && lock_type_ == lock_meta.lock_type_ - && lock_time_ == lock_meta.lock_time_; -} - -DEFINE_SERIALIZE(LogLockMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_len)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_))) { - PALF_LOG(WARN, "serialize version_ failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, lock_owner_))) { - PALF_LOG(WARN, "serialize lock_owner_ failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, lock_type_))) { - PALF_LOG(WARN, "serialize log_type failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, lock_time_))) { - PALF_LOG(WARN, "serialize lock_ts_ failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogLockMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= data_len) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KP(buf), K(data_len)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_))) { - PALF_LOG(WARN, "deserialize failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &lock_owner_))) { - PALF_LOG(WARN, "deserialize failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &lock_type_))) { - PALF_LOG(WARN, "deserialize failed", K(ret), K(new_pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &lock_time_))) { - PALF_LOG(WARN, "deserialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogLockMeta) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_i64(lock_owner_); - size += serialization::encoded_length_i64(lock_type_); - size += serialization::encoded_length_i64(lock_time_); - return size; -} - -LogConfigInfoV2::LogConfigInfoV2() - : version_(-1), - config_(), - lock_meta_() -{} - -LogConfigInfoV2::~LogConfigInfoV2() -{ - reset(); -} - -bool LogConfigInfoV2::is_valid() const -{ - return (LOG_CONFIG_INFO_VERSION == version_) - && config_.is_valid() - && lock_meta_.is_valid(); -} - -void LogConfigInfoV2::reset() -{ - version_ = -1; - config_.reset(); - lock_meta_.reset(); -} - - -int LogConfigInfoV2::generate(const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!config_version.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(config_version)); - } else { - version_ = LOG_CONFIG_INFO_VERSION; - config_.config_version_ = config_version; - lock_meta_.reset_as_unlocked(); - } - return ret; -} - -int LogConfigInfoV2::generate(const LogConfigInfo &config_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!config_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(config_info)); - } else { - version_ = LOG_CONFIG_INFO_VERSION; - config_ = config_info; - lock_meta_.reset_as_unlocked(); - } - return ret; -} - -int LogConfigInfoV2::transform_for_deserialize(const LogConfigInfo &config_info) -{ - int ret = OB_SUCCESS; - version_ = LOG_CONFIG_INFO_VERSION; - config_ = config_info; - lock_meta_.reset_as_unlocked(); - return ret; -} - -// generate paxos memberlist including arbitration replica -int LogConfigInfoV2::convert_to_complete_config(common::ObMemberList &all_paxos_memberlist, - int64_t &all_paxos_replica_num, - GlobalLearnerList &all_learners) const -{ - return config_.convert_to_complete_config(all_paxos_memberlist, all_paxos_replica_num, all_learners); -} - -bool LogConfigInfoV2::is_config_change_locked() const -{ - return lock_meta_.is_locked(); -} - -int LogConfigInfoV2::generate(const ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version) -{ - LogLockMeta lock_meta; - lock_meta.reset_as_unlocked(); - return generate(memberlist, replica_num, learnerlist, config_version, lock_meta); -} - -int LogConfigInfoV2::generate(const ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version, - const LogLockMeta &lock_meta) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!lock_meta.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K(lock_meta)); - } else if (OB_FAIL(config_.generate(memberlist, replica_num, learnerlist, config_version))) { - PALF_LOG(WARN, "failed to generate config", KR(ret)); - } else { - version_ = LOG_CONFIG_INFO_VERSION; - lock_meta_ = lock_meta; - } - return ret; -} - -void LogConfigInfoV2::operator=(const LogConfigInfoV2 &config_info) -{ - version_ = config_info.version_; - config_ = config_info.config_; - lock_meta_ = config_info.lock_meta_; - PALF_LOG(TRACE, "LogConfigInfoV2 operator =", KPC(this), K(config_info)); -} - -bool LogConfigInfoV2::operator==(const LogConfigInfoV2 &config_info) const -{ - return (version_ == config_info.version_) - && (config_ == config_info.config_) - && (lock_meta_ == config_info.lock_meta_); -} - -DEFINE_SERIALIZE(LogConfigInfoV2) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) - || OB_FAIL(config_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(lock_meta_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigInfoV2 serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogConfigInfoV2) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= data_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) - || OB_FAIL(config_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(lock_meta_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigInfoV2 deserialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogConfigInfoV2) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += config_.get_serialize_size(); - size += lock_meta_.get_serialize_size(); - return size; -} - -LogConfigMeta::LogConfigMeta() - : version_(-1), - proposal_id_(INVALID_PROPOSAL_ID), - prev_(), - curr_(), - prev_log_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(), - prev_mode_pid_(INVALID_PROPOSAL_ID) -{} - -LogConfigMeta::~LogConfigMeta() -{ - reset(); -} - -int LogConfigMeta::generate_for_default( - const int64_t proposal_id, - const LogConfigInfoV2 &prev_config_info, - const LogConfigInfoV2 &curr_config_info) -{ - int ret = OB_SUCCESS; - if (INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - } else { - // Note: we generate a default META_VERSION rather than according to DATA_VERSION, - // because DATA_VERSION of the tenant may be empty it the server is just added - // to the cluster. It's fine because real LogConfigMeta will be set by the - // set_initial_member_list interface. - version_ = LOG_CONFIG_META_VERSION_INC; - proposal_id_ = proposal_id; - prev_ = prev_config_info; - curr_ = curr_config_info; - } - return ret; -} - -int LogConfigMeta::generate( - const int64_t proposal_id, - const LogConfigInfoV2 &prev_config_info, - const LogConfigInfoV2 &curr_config_info, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid) -{ - int ret = OB_SUCCESS; - if (INVALID_PROPOSAL_ID == proposal_id || false == curr_config_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else { - version_ = LOG_CONFIG_META_VERSION_42; - - proposal_id_ = proposal_id; - prev_ = prev_config_info; - curr_ = curr_config_info; - prev_log_proposal_id_ = prev_log_proposal_id; - prev_lsn_ = prev_lsn; - prev_mode_pid_ = prev_mode_pid; - } - return ret; -} - -bool LogConfigMeta::is_valid() const -{ - // NB: prev_config_info is invalid before change config - return (LOG_CONFIG_META_VERSION == version_ - || LOG_CONFIG_META_VERSION_INC == version_ - ||LOG_CONFIG_META_VERSION_42 == version_) - && proposal_id_ != INVALID_PROPOSAL_ID; -} - -void LogConfigMeta::reset() -{ - version_ = -1; - proposal_id_ = INVALID_PROPOSAL_ID; - curr_.reset(); - prev_.reset(); - version_ = -1; - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - prev_mode_pid_ = INVALID_PROPOSAL_ID; -} - -void LogConfigMeta::operator=(const LogConfigMeta &log_config_meta) -{ - this->version_ = log_config_meta.version_; - this->proposal_id_ = log_config_meta.proposal_id_; - this->prev_ = log_config_meta.prev_; - this->curr_ = log_config_meta.curr_; - this->prev_log_proposal_id_ = log_config_meta.prev_log_proposal_id_; - this->prev_lsn_ = log_config_meta.prev_lsn_; - this->prev_mode_pid_ = log_config_meta.prev_mode_pid_; -} - -DEFINE_SERIALIZE(LogConfigMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (buf_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_))) { - PALF_LOG(ERROR, "LogConfigMeta faild to serialize version_", K(ret), K(new_pos), K(buf_len), K(pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, proposal_id_))) { - PALF_LOG(ERROR, "LogConfigMeta failed to serialize proposal_id_", K(ret), K(new_pos), K(buf_len), K(pos)); - } else if (LOG_CONFIG_META_VERSION_INC >= version_) { - if (OB_FAIL(prev_.config_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(curr_.config_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigMeta serialize failed", K(ret), K(new_pos), K(buf_len), K(pos)); - } else if (LOG_CONFIG_META_VERSION_INC == version_) { - if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_log_proposal_id_)) || - OB_FAIL(prev_lsn_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_mode_pid_))) { - PALF_LOG(ERROR, "LogConfigMeta Version 2 serialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogConfigMeta Version 2 serialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - } - } else if (LOG_CONFIG_META_VERSION_42 == version_) { - if (OB_FAIL(prev_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(curr_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_log_proposal_id_)) || - OB_FAIL(prev_lsn_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_mode_pid_))) { - PALF_LOG(ERROR, "LogConfigMeta Version 3 serialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogConfigMeta Version 3 serialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "invalid version", K(ret), K(version_)); - } - - if (OB_SUCC(ret)) { - PALF_LOG(TRACE, "LogConfigMeta serialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogConfigMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || 0 >= data_len) { - ret = OB_INVALID_ARGUMENT; - /* TODO: ObAddr's serialized size is variable, replace it later. - } else if (data_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; */ - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_))) { - PALF_LOG(ERROR, "failed to deserialize version", K(ret)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &proposal_id_))) { - PALF_LOG(ERROR, "failed to deserialize version", K(ret)); - } else if (LOG_CONFIG_META_VERSION == version_ || LOG_CONFIG_META_VERSION_INC == version_) { - LogConfigInfo old_prev; - LogConfigInfo old_curr; - if (OB_FAIL(old_prev.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "LogConfigMeta deserialize failed", K(ret), K(new_pos)); - } else if (OB_FAIL(old_curr.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "deserialize failed", K(ret), K(new_pos)); - } else if (OB_FAIL(prev_.transform_for_deserialize(old_prev))) { - PALF_LOG(ERROR, "failed to generate pre_", K(ret)); - } else if (OB_FAIL(curr_.transform_for_deserialize(old_curr))) { - PALF_LOG(ERROR, " failed to generate curr_", K(ret)); - } else if (LOG_CONFIG_META_VERSION_INC == version_) { - if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &prev_log_proposal_id_)) || - OB_FAIL(prev_lsn_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &prev_mode_pid_))) { - PALF_LOG(ERROR, " failed to decode pre_log_info", K(ret)); - } - } else {/*do nothing*/} - } else if (LOG_CONFIG_META_VERSION_42 == version_) { - if (OB_FAIL(prev_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "failed to deserialize prev_", K(ret)); - } else if (OB_FAIL(curr_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "failed to deserialize curr_", K(ret)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &prev_log_proposal_id_)) || - OB_FAIL(prev_lsn_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &prev_mode_pid_))) { - PALF_LOG(ERROR, "LogConfigMeta Version 3 deserialize failed", K(ret), K(new_pos)); - } else {/*do nothing*/} - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "invalid version", K(ret), K(version_)); - } - - if (OB_SUCC(ret)) { - PALF_LOG(TRACE, "LogConfigMeta deserialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - -DEFINE_GET_SERIALIZE_SIZE(LogConfigMeta) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_i64(proposal_id_); - if (LOG_CONFIG_META_VERSION == version_ ) { - size += prev_.config_.get_serialize_size(); - size += curr_.config_.get_serialize_size(); - } else if (LOG_CONFIG_META_VERSION_INC == version_) { - size += prev_.config_.get_serialize_size(); - size += curr_.config_.get_serialize_size(); - size += serialization::encoded_length_i64(prev_log_proposal_id_); - size += prev_lsn_.get_serialize_size(); - size += serialization::encoded_length_i64(prev_mode_pid_); - } else if (LOG_CONFIG_META_VERSION_42 == version_) { - size += prev_.get_serialize_size(); - size += curr_.get_serialize_size(); - size += serialization::encoded_length_i64(prev_log_proposal_id_); - size += prev_lsn_.get_serialize_size(); - size += serialization::encoded_length_i64(prev_mode_pid_); - } - return size; -} - LogModeMeta::LogModeMeta() - : version_(-1), - proposal_id_(INVALID_PROPOSAL_ID), - mode_version_(INVALID_PROPOSAL_ID), - access_mode_(AccessMode::INVALID_ACCESS_MODE), + : access_mode_(AccessMode::INVALID_ACCESS_MODE), ref_scn_() {} @@ -1107,21 +33,14 @@ LogModeMeta::~LogModeMeta() reset(); } -int LogModeMeta::generate(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, +int LogModeMeta::generate(const AccessMode &access_mode, const SCN &ref_scn) { int ret = OB_SUCCESS; - if (INVALID_PROPOSAL_ID == mode_version || - INVALID_PROPOSAL_ID == proposal_id || - false == is_valid_access_mode(access_mode) || + if (false == is_valid_access_mode(access_mode) || !ref_scn.is_valid()) { ret = OB_INVALID_ARGUMENT; } else { - version_ = LOG_MODE_META_VERSION; - proposal_id_ = proposal_id; - mode_version_ = mode_version; access_mode_ = access_mode; ref_scn_ = ref_scn; } @@ -1130,27 +49,18 @@ int LogModeMeta::generate(const int64_t proposal_id, bool LogModeMeta::is_valid() const { - return LOG_MODE_META_VERSION == version_ && - INVALID_PROPOSAL_ID != proposal_id_ && - INVALID_PROPOSAL_ID != mode_version_ && - is_valid_access_mode(access_mode_) && + return is_valid_access_mode(access_mode_) && ref_scn_.is_valid(); } void LogModeMeta::reset() { - version_ = -1; - proposal_id_ = INVALID_PROPOSAL_ID; - mode_version_ = INVALID_PROPOSAL_ID; access_mode_ = AccessMode::INVALID_ACCESS_MODE; ref_scn_.reset(); } void LogModeMeta::operator=(const LogModeMeta &mode_meta) { - this->version_ = mode_meta.version_; - this->proposal_id_ = mode_meta.proposal_id_; - this->mode_version_ = mode_meta.mode_version_; this->access_mode_ = mode_meta.access_mode_; this->ref_scn_ = mode_meta.ref_scn_; } @@ -1163,10 +73,7 @@ DEFINE_SERIALIZE(LogModeMeta) ret = OB_INVALID_ARGUMENT; } else if (buf_len - new_pos < get_serialize_size()) { ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, proposal_id_)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, mode_version_)) || - OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, static_cast(access_mode_))) || + } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, static_cast(access_mode_))) || OB_FAIL(ref_scn_.fixed_serialize(buf, buf_len, new_pos))) { PALF_LOG(ERROR, "LogModeMeta serialize failed", K(ret), K(new_pos)); } else { @@ -1182,10 +89,7 @@ DEFINE_DESERIALIZE(LogModeMeta) int64_t new_pos = pos; if (NULL == buf || 0 >= data_len) { ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &proposal_id_)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &mode_version_)) || - OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&access_mode_))) || + } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&access_mode_))) || OB_FAIL(ref_scn_.fixed_deserialize(buf, data_len, new_pos))) { PALF_LOG(ERROR, "LogModeMeta deserialize failed", K(ret), K(new_pos)); } else { @@ -1198,16 +102,12 @@ DEFINE_DESERIALIZE(LogModeMeta) DEFINE_GET_SERIALIZE_SIZE(LogModeMeta) { int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_i64(proposal_id_); - size += serialization::encoded_length_i64(mode_version_); size += serialization::encoded_length_i64(static_cast(access_mode_)); size += ref_scn_.get_fixed_serialize_size(); return size; } -constexpr int64_t LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION = 1; -constexpr int64_t LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION_V2 = 2; +constexpr int64_t LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION = 2; LogSnapshotMeta::LogSnapshotMeta() : version_(-1), base_lsn_(), prev_log_info_(), prev_log_tail_lsn_() {} @@ -1222,28 +122,23 @@ int LogSnapshotMeta::generate(const LSN &lsn, const LSN &prev_log_tail_lsn) { int ret = OB_SUCCESS; - if (false == lsn.is_valid()) { + if (!lsn.is_valid() || !prev_log_info.is_valid() || !prev_log_tail_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; } else { - version_ = get_version_(); + version_ = LOG_SNAPSHOT_META_VERSION; base_lsn_ = lsn; - if (LOG_SNAPSHOT_META_VERSION_V2 == version_) { - prev_log_info_ = prev_log_info; - prev_log_tail_lsn_ = prev_log_tail_lsn; - } else if (prev_log_tail_lsn != lsn) { - prev_log_info_.reset(); - prev_log_tail_lsn_.reset(); - } else { - prev_log_info_ = prev_log_info; - prev_log_tail_lsn_.reset(); - } + prev_log_info_ = prev_log_info; + prev_log_tail_lsn_ = prev_log_tail_lsn; } return ret; } bool LogSnapshotMeta::is_valid() const { - return true == base_lsn_.is_valid(); + return LOG_SNAPSHOT_META_VERSION == version_ + && base_lsn_.is_valid() + && prev_log_info_.is_valid() + && prev_log_tail_lsn_.is_valid(); } void LogSnapshotMeta::reset() @@ -1261,22 +156,13 @@ int LogSnapshotMeta::get_prev_log_info(const LSN &curr_lsn, int ret = OB_SUCCESS; log_info.reset(); tail_lsn.reset(); - if (LOG_SNAPSHOT_META_VERSION == version_) { - if (!prev_log_info_.is_valid() || curr_lsn != base_lsn_) { - ret = OB_ENTRY_NOT_EXIST; - } else { - log_info = prev_log_info_; - } - } else if (LOG_SNAPSHOT_META_VERSION_V2 == version_) { - if (!prev_log_info_.is_valid() || curr_lsn != prev_log_tail_lsn_) { - ret = OB_ENTRY_NOT_EXIST; - } else { - log_info = prev_log_info_; - tail_lsn = prev_log_tail_lsn_; - } + if (LOG_SNAPSHOT_META_VERSION != version_) { + ret = OB_NOT_SUPPORTED; + } else if (curr_lsn != prev_log_tail_lsn_) { + ret = OB_ENTRY_NOT_EXIST; } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected error, version is invalid", KPC(this), K(curr_lsn)); + log_info = prev_log_info_; + tail_lsn = prev_log_tail_lsn_; } return ret; } @@ -1297,11 +183,9 @@ DEFINE_SERIALIZE(LogSnapshotMeta) ret = OB_BUF_NOT_ENOUGH; } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) || OB_FAIL(base_lsn_.serialize(buf, buf_len, new_pos)) || - OB_FAIL(prev_log_info_.serialize(buf, buf_len, new_pos))) { + OB_FAIL(prev_log_info_.serialize(buf, buf_len, new_pos)) || + OB_FAIL(prev_log_tail_lsn_.serialize(buf, buf_len, new_pos))) { PALF_LOG(ERROR, "LogSnapshotMeta serialize failed", K(ret), K(new_pos)); - } else if (LOG_SNAPSHOT_META_VERSION_V2 == version_ - && OB_FAIL(prev_log_tail_lsn_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "serialize prev_log_tail_lsn_ failed", K(ret), K(new_pos)); } else { PALF_LOG(TRACE, "LogSnapshotMeta serialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); pos = new_pos; @@ -1317,11 +201,12 @@ DEFINE_DESERIALIZE(LogSnapshotMeta) ret = OB_BUF_NOT_ENOUGH; } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) || OB_FAIL(base_lsn_.deserialize(buf, data_len, new_pos)) || - OB_FAIL(prev_log_info_.deserialize(buf, data_len, new_pos))) { + OB_FAIL(prev_log_info_.deserialize(buf, data_len, new_pos)) || + OB_FAIL(prev_log_tail_lsn_.deserialize(buf, data_len, new_pos))) { PALF_LOG(ERROR, "LogSnapshotMeta deserialize failed", K(ret), K(new_pos)); - } else if (LOG_SNAPSHOT_META_VERSION_V2 == version_ - && OB_FAIL(prev_log_tail_lsn_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "deserialize prev_log_tail_lsn_ failed", K(ret), K(new_pos)); + } else if (LOG_SNAPSHOT_META_VERSION != version_) { + ret = OB_NOT_SUPPORTED; + PALF_LOG(ERROR, "unsupported LogSnapshotMeta version", K(ret), K_(version)); } else { PALF_LOG(TRACE, "LogSnapshotMeta deserialize", K(*this), K(buf + pos), KP(buf), K(pos), K(new_pos)); pos = new_pos; @@ -1335,87 +220,9 @@ DEFINE_GET_SERIALIZE_SIZE(LogSnapshotMeta) size += serialization::encoded_length_i64(version_); size += base_lsn_.get_serialize_size(); size += prev_log_info_.get_serialize_size(); - if (LOG_SNAPSHOT_META_VERSION_V2 == version_) { - size += prev_log_tail_lsn_.get_serialize_size(); - } + size += prev_log_tail_lsn_.get_serialize_size(); return size; } -int64_t LogSnapshotMeta::get_version_() const -{ - return LOG_SNAPSHOT_META_VERSION_V2; -} - -int LogReplicaPropertyMeta::generate(const bool allow_vote, const LogReplicaType replica_type) -{ - int ret = OB_SUCCESS; - version_ = LOG_REPLICA_PROPERTY_META_VERSION; - allow_vote_ = allow_vote; - replica_type_ = replica_type; - return ret; -} - -bool LogReplicaPropertyMeta::is_valid() const -{ - return LOG_REPLICA_PROPERTY_META_VERSION == version_; -} - -void LogReplicaPropertyMeta::reset() -{ - allow_vote_ = false; - replica_type_ = LogReplicaType::INVALID_REPLICA; - version_ = -1; -} - -void LogReplicaPropertyMeta::operator=(const LogReplicaPropertyMeta &replica_meta) -{ - this->version_ = replica_meta.version_; - this->allow_vote_ = replica_meta.allow_vote_; - this->replica_type_ = replica_meta.replica_type_; -} - -DEFINE_SERIALIZE(LogReplicaPropertyMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (buf_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, version_)) - || OB_FAIL(serialization::encode_bool(buf, buf_len, new_pos, allow_vote_)) - || OB_FAIL(serialization::encode_i32(buf, buf_len, new_pos, replica_type_))) { - PALF_LOG(ERROR, "LogReplicaPropertyMeta serialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogReplicaPropertyMeta serialize", K(*this), K(buf+pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - -DEFINE_DESERIALIZE(LogReplicaPropertyMeta) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (data_len - new_pos < get_serialize_size()) { - ret = OB_BUF_NOT_ENOUGH; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, &version_)) - || OB_FAIL(serialization::decode_bool(buf, data_len, new_pos, &allow_vote_)) - || OB_FAIL(serialization::decode_i32(buf, data_len, new_pos, reinterpret_cast(&replica_type_)))) { - PALF_LOG(ERROR, "LogReplicaPropertyMeta deserialize failed", K(ret), K(new_pos)); - } else { - PALF_LOG(TRACE, "LogReplicaPropertyMeta deserialize", K(*this), K(buf+pos), KP(buf), K(pos), K(new_pos)); - pos = new_pos; - } - return ret; -} - - -DEFINE_GET_SERIALIZE_SIZE(LogReplicaPropertyMeta) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_bool(allow_vote_); - size += serialization::encoded_length_i32(replica_type_); - return size; -} } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_meta_info.h b/src/logservice/palf/log_meta_info.h index f5b153083..9558f231a 100644 --- a/src/logservice/palf/log_meta_info.h +++ b/src/logservice/palf/log_meta_info.h @@ -17,313 +17,35 @@ #ifndef OCEANBASE_LOGSERVICE_LOG_META_INFO_ #define OCEANBASE_LOGSERVICE_LOG_META_INFO_ -#include "lib/ob_define.h" // ObReplicaType +#include "lib/ob_define.h" #include "lib/utility/ob_print_utils.h" // Print* -#include "common/ob_learner_list.h" // common::GlobalLearnerList -#include "common/ob_member_list.h" // ObMemberList #include "share/scn.h" // SCN #include "lsn.h" // LSN #include "palf_base_info.h" // LogInfo #include "palf_options.h" // AccessMode -#include "log_define.h" // ReplicaType +#include "log_define.h" namespace oceanbase { namespace palf { -struct LogVotedFor { - LogVotedFor(); - ~LogVotedFor(); - LogVotedFor(const ObAddr &addr); - LogVotedFor(const LogVotedFor &voted_for); - LogVotedFor(LogVotedFor &&voted_for); - LogVotedFor &operator=(const LogVotedFor &voted_for); - bool operator==(const LogVotedFor &voted_for); - void reset(); - int64_t to_string(char *buf, int64_t buf_len) const; - static constexpr int64_t COUNT = 3; - // ipv6 need occupy 16BYTE - // port need occupy 8BYTE - // - // --- ipv6 --- - // --- memory layout ---- - // ---- ipv6 higher ----- - // ---- ipv6 lower ----- - // ---- ipv6 port ----- - // - // --- ipv4 --- - // ---- ip ---- - // --- port --- - int64_t voted_for_[COUNT]; - NEED_SERIALIZE_AND_DESERIALIZE; -}; - -// Prepare meta for consenus -struct LogPrepareMeta -{ -public: - LogPrepareMeta(); - ~LogPrepareMeta(); - -public: - int generate(const LogVotedFor &voted_for, const int64_t &log_proposal_id); - bool is_valid() const; - void reset(); - void operator=(const LogPrepareMeta &log_prepare_meta); - TO_STRING_KV(K_(version), K_(voted_for), K_(log_proposal_id)); - NEED_SERIALIZE_AND_DESERIALIZE; - - int64_t version_; - LogVotedFor voted_for_; - int64_t log_proposal_id_; - static constexpr int64_t LOG_PREPARE_VERSION = 1; -}; - -class LogConfigVersion { -public: - LogConfigVersion(); - ~LogConfigVersion(); - void operator=(const LogConfigVersion &config_version); - static const int64_t CONFIG_VERSION_LEN = 128; - -public: - int generate(const int64_t proposal_id, const int64_t config_seq); - int inc_update_version(const int64_t proposal_id); - bool is_valid() const; - bool is_initial_version() const; - void reset(); - int64_t to_string(char *buf, const int64_t buf_len); - bool operator==(const LogConfigVersion &config_version) const; - bool operator!=(const LogConfigVersion &config_version) const; - bool operator>(const LogConfigVersion &config_version) const; - bool operator<(const LogConfigVersion &config_version) const; - bool operator>=(const LogConfigVersion &config_version) const; - bool operator<=(const LogConfigVersion &config_version) const; - TO_STRING_KV(K_(proposal_id), K_(config_seq)); - NEED_SERIALIZE_AND_DESERIALIZE; - // Ensure that: - // 1. For leader don't has any writes, this field used to ensure leader completeness. - // (NB: can not use the proposal_id of prepare request, because a lagged replica - // may receive the prepare reqeust of new leader). - // 2. For changing config, this field used to ensure inc config_version monotoniclly. -private: - int64_t proposal_id_; - int64_t config_seq_; -}; - -struct LogConfigInfo -{ -public: - LogConfigInfo(); - ~LogConfigInfo(); - void operator=(const LogConfigInfo &config_info); - -public: - bool is_valid() const; - void reset(); - //for unitest - int generate(const common::ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version); - // @brief get the expected paxos member list without arbitraion member, - // including degraded paxos members. - // @param[in/out] ObMemberList, the output member list - // @param[in/out] int64_t, the output replica_num - // @retval - // return OB_SUCCESS if success - // else return other errno - int get_expected_paxos_memberlist(common::ObMemberList &paxos_memberlist, - int64_t &paxos_replica_num) const; - int convert_to_complete_config(common::ObMemberList &alive_paxos_memberlist, - int64_t &alive_paxos_replica_num, - GlobalLearnerList &all_learners) const; - bool is_all_list_unique() const; - // For unittest - bool operator==(const LogConfigInfo &config_info) const; - TO_STRING_KV(K_(config_version), - K_(log_sync_memberlist), - K_(log_sync_replica_num), - K_(arbitration_member), - K_(learnerlist), - K_(degraded_learnerlist)); - NEED_SERIALIZE_AND_DESERIALIZE; - - // paxos members for log sync, don't include arbitration replica - common::ObMemberList log_sync_memberlist_; - // replica number for log sync, don't include arbitration replica - int64_t log_sync_replica_num_; - // arbitration replica - common::ObMember arbitration_member_; - // all learners, don't include learners which have been degraded from members - common::GlobalLearnerList learnerlist_; - // learners which have been degraded from members - common::GlobalLearnerList degraded_learnerlist_; - LogConfigVersion config_version_; -private: - template - int check_list_unique(GlobalLearnerList &server_list, - const LIST &list) const - { - int ret = OB_SUCCESS; - for (int i = 0; OB_SUCC(ret) && i < list.get_member_number(); i++) { - common::ObMember member; - if (OB_FAIL(list.get_member_by_index(i, member))) { - PALF_LOG(WARN, "get_server_by_index failed", K(list)); - } else if (server_list.contains(member.get_server())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "serverlist should not overlap with list", K(server_list), K(list)); - } else if (OB_FAIL(server_list.add_learner(member))) { - PALF_LOG(WARN, "add_learner failed", K(server_list), K(list)); - } - } - return ret; - } -}; - -enum ConfigChangeLockType -{ - LOCK_NOTHING = 0x0, - LOCK_PAXOS_MEMBER_CHANGE = 0x1, // binary 00001 - LOCK_PAXOS_REPLICA_NUMBER_CHANGE = 0x2, // binary 00010 (TODO) - LOCK_LEARNER_CHANGE = 0x4, // binary 00100(TODO) - LOCK_ACCESS_MODE_CHANGE = 0x8, // binary 01000 (TODO) - LOCK_ARBITRATION_MEMBER_CHANGE = 0x10, // binary 10000 (TODO) -}; - -bool is_valid_config_lock_type(int64_t lock_type); - -struct LogLockMeta -{ -public: - LogLockMeta() {reset();} - ~LogLockMeta() {reset();} - void reset(); - bool is_valid() const; - int generate(const int64_t lock_owner, const int64_t lock_type); - void operator=(const LogLockMeta &lock_meta); - bool operator==(const LogLockMeta &lock_meta) const; - void reset_as_unlocked(); - void unlock(); - bool is_locked() const; - bool is_lock_owner_valid() const; - TO_STRING_KV(K_(version), K_(lock_type), K_(lock_owner), K_(lock_time)); - NEED_SERIALIZE_AND_DESERIALIZE; -public: - static constexpr int64_t LOG_LOCK_META_VERSION = 1; - int64_t version_;//for compatibilty - int64_t lock_owner_;// owner of lock - int64_t lock_type_;// ConfigChangeLockType - int64_t lock_time_;// the timestamp of executing locking or unlocking. default as OB_INVALID_TIMESTAMP, using fordebugging -}; - -struct LogConfigInfoV2 -{ -public: - LogConfigInfoV2(); - ~LogConfigInfoV2(); -public: - static constexpr int64_t LOG_CONFIG_INFO_VERSION = 1; - bool is_valid() const; - void reset(); - //for init with default - int generate(const LogConfigVersion &config_version); - //for serialize compaction - int generate(const LogConfigInfo &config_info); - //for deserialization from lower version - int transform_for_deserialize(const LogConfigInfo &config_info); - int convert_to_complete_config(common::ObMemberList &all_paxos_memberlist, - int64_t &all_paxos_replica_num, - GlobalLearnerList &all_learners) const; - bool is_config_change_locked() const; - - //for unitest - int generate(const common::ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version); - //for unitest - int generate(const common::ObMemberList &memberlist, - const int64_t replica_num, - const common::GlobalLearnerList &learnerlist, - const LogConfigVersion &config_version, - const LogLockMeta &lock_meta); - // For unittest - void operator=(const LogConfigInfoV2 &config_info); - bool operator==(const LogConfigInfoV2 &config_info) const; - TO_STRING_KV(K_(version), K_(config), K_(lock_meta)); - NEED_SERIALIZE_AND_DESERIALIZE; -public: - int64_t version_; - LogConfigInfo config_; - LogLockMeta lock_meta_; -}; - -// Change member log for consenus -struct LogConfigMeta { -public: - LogConfigMeta(); - ~LogConfigMeta(); - -public: - // Note: the function will generate a default version_. - int generate_for_default(const int64_t proposal_id, - const LogConfigInfoV2 &prev_config_info, - const LogConfigInfoV2 &curr_config_info); - int generate(const int64_t proposal_id, - const LogConfigInfoV2 &prev_config_info, - const LogConfigInfoV2 &curr_config_info, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t prev_mode_pid); - bool is_valid() const; - void reset(); - void operator=(const LogConfigMeta &log_config_meta); - TO_STRING_KV(K_(version), K_(proposal_id), K_(prev), K_(curr), K_(prev_log_proposal_id), - K_(prev_lsn), K_(prev_mode_pid)); - NEED_SERIALIZE_AND_DESERIALIZE; - int64_t version_; - // ====== members in VERSION 1 ======== - int64_t proposal_id_; - LogConfigInfoV2 prev_;//modified in version_42 - LogConfigInfoV2 curr_;//modified in version_42 - // ====== added members in VERSION 2 ======== - int64_t prev_log_proposal_id_; - LSN prev_lsn_; - int64_t prev_mode_pid_; - - static constexpr int64_t LOG_CONFIG_META_VERSION = 1; - static constexpr int64_t LOG_CONFIG_META_VERSION_INC = 2; - static constexpr int64_t LOG_CONFIG_META_VERSION_42 = 3;//LogConfigInfo-->LogConfigInfoV2 -}; - struct LogModeMeta { public: LogModeMeta(); ~LogModeMeta(); - int generate( - const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const share::SCN &ref_scn); + int generate(const AccessMode &access_mode, const share::SCN &ref_scn); bool is_valid() const; void reset(); void operator=(const LogModeMeta &mode_meta); - TO_STRING_KV(K_(version), K_(proposal_id), K_(mode_version), K_(access_mode), K_(ref_scn)); + TO_STRING_KV(K_(access_mode), K_(ref_scn)); NEED_SERIALIZE_AND_DESERIALIZE; public: - int64_t version_; - // proposal_id of LogModeMeta, changed by leader switching and change_access_mode - int64_t proposal_id_; - // proposal_id of last access_mode_, only changed by change_access_mode - int64_t mode_version_; AccessMode access_mode_; // scn lower bound // after switching over, scn of all submitted log should be bigger than ref_scn_ share::SCN ref_scn_; - static constexpr int64_t LOG_MODE_META_VERSION = 1; }; // Garbage collect controller @@ -343,7 +65,6 @@ struct LogSnapshotMeta LogInfo &log_info, LSN &tail_lsn) const; void operator=(const LogSnapshotMeta &log_snapshot_meta); - int64_t get_version_() const; TO_STRING_KV(K_(version), K_(base_lsn), K_(prev_log_info), K_(prev_log_tail_lsn)); NEED_SERIALIZE_AND_DESERIALIZE; @@ -357,29 +78,6 @@ struct LogSnapshotMeta LSN prev_log_tail_lsn_; static const int64_t LOG_SNAPSHOT_META_VERSION; - static const int64_t LOG_SNAPSHOT_META_VERSION_V2; -}; - -struct LogReplicaPropertyMeta { -public: - LogReplicaPropertyMeta(): version_(-1), allow_vote_(false), replica_type_(LogReplicaType::INVALID_REPLICA) { } - ~LogReplicaPropertyMeta() { } - -public: - int generate(const bool allow_vote, const LogReplicaType replica_type); - bool is_valid() const; - void reset(); - void operator=(const LogReplicaPropertyMeta &replica_meta); - TO_STRING_KV(K_(allow_vote), "replica_type", replica_type_2_str(replica_type_)); - NEED_SERIALIZE_AND_DESERIALIZE; - - int64_t version_; - // a persistent flag which means if this paxos replica - // is allowed to reply ack when receiving logs. It's true by default. - bool allow_vote_; - // persistent replica type flag - LogReplicaType replica_type_; - static constexpr int64_t LOG_REPLICA_PROPERTY_META_VERSION = 1; }; } // end namespace palf diff --git a/src/logservice/palf/log_mode_mgr.cpp b/src/logservice/palf/log_mode_mgr.cpp index 08c9b3e36..2e5e0029a 100644 --- a/src/logservice/palf/log_mode_mgr.cpp +++ b/src/logservice/palf/log_mode_mgr.cpp @@ -16,10 +16,6 @@ #define USING_LOG_PREFIX PALF #include "log_mode_mgr.h" -#include "log_engine.h" -#include "log_config_mgr.h" -#include "log_io_task_cb_utils.h" -#include "log_sliding_window.h" namespace oceanbase { @@ -30,64 +26,20 @@ namespace palf LogModeMgr::LogModeMgr() : is_inited_(false), - palf_id_(-1), self_(), - applied_mode_meta_(), - accepted_mode_meta_(), - last_submit_mode_meta_(), - state_(ModeChangeState::MODE_INIT), - leader_epoch_(-1), - new_proposal_id_(INVALID_PROPOSAL_ID), - ack_list_(), - follower_list_(), - majority_cnt_(-1), - last_submit_req_ts_(OB_INVALID_TIMESTAMP), - max_majority_accepted_mode_meta_(), - local_max_lsn_(), - local_max_log_pid_(INVALID_PROPOSAL_ID), - max_majority_accepted_pid_(INVALID_PROPOSAL_ID), - max_majority_lsn_(), - resend_mode_meta_list_(), - state_mgr_(NULL), - log_engine_(NULL), - config_mgr_(NULL), - sw_(NULL) - { } + applied_mode_meta_() +{} -int LogModeMgr::init(const int64_t palf_id, - const common::ObAddr &self, - const LogModeMeta &log_mode_meta, - LogStateMgr *state_mgr, - LogEngine *log_engine, - LogConfigMgr *config_mgr, - LogSlidingWindow *sw) +int LogModeMgr::init(const common::ObAddr &self, const LogModeMeta &log_mode_meta) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - PALF_LOG(WARN, "LogModeMgr has inited", K(ret), K(palf_id)); - } else if (false == is_valid_palf_id(palf_id) || - false == self.is_valid() || - false == log_mode_meta.is_valid() || - OB_ISNULL(state_mgr) || - OB_ISNULL(log_engine) || - OB_ISNULL(config_mgr) || - OB_ISNULL(sw)) { + } else if (!self.is_valid() || !log_mode_meta.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(palf_id), K(self), K(log_mode_meta), - KP(state_mgr), KP(log_engine), KP(config_mgr), KP(sw)); } else { - palf_id_ = palf_id; self_ = self; - // NB: log_mode_meta_ takes effect immediately on initing; - // changing log_mode_meta_ takes effect on committed. applied_mode_meta_ = log_mode_meta; - accepted_mode_meta_ = log_mode_meta; - last_submit_mode_meta_ = log_mode_meta; - state_mgr_ = state_mgr; - log_engine_ = log_engine; - config_mgr_ = config_mgr; - sw_ = sw; is_inited_ = true; } return ret; @@ -95,20 +47,9 @@ int LogModeMgr::init(const int64_t palf_id, void LogModeMgr::destroy() { - if (IS_INIT) { - is_inited_ = false; - palf_id_ = -1; - self_.reset(); - applied_mode_meta_.reset(); - accepted_mode_meta_.reset(); - last_submit_mode_meta_.reset(); - reset_status_(); - resend_mode_meta_list_.reset(); - state_mgr_ = NULL; - log_engine_ = NULL; - config_mgr_ = NULL; - sw_ = NULL; - } + is_inited_ = false; + self_.reset(); + applied_mode_meta_.reset(); } int LogModeMgr::get_access_mode(AccessMode &access_mode) const @@ -116,703 +57,28 @@ int LogModeMgr::get_access_mode(AccessMode &access_mode) const int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogModeMgr has inited", K(ret)); } else { - access_mode = (AccessMode)ATOMIC_LOAD(reinterpret_cast(&applied_mode_meta_.access_mode_)); - } - return ret; -} - -int LogModeMgr::get_mode_version(int64_t &mode_version) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogModeMgr has inited", K(ret)); - } else { - mode_version = ATOMIC_LOAD(reinterpret_cast(&applied_mode_meta_.mode_version_)); - } - return ret; -} - -int LogModeMgr::get_access_mode(int64_t &mode_version, AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogModeMgr has inited", K(ret)); - } else { - mode_version = applied_mode_meta_.mode_version_; access_mode = applied_mode_meta_.access_mode_; } return ret; } -int LogModeMgr::get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, +int LogModeMgr::get_access_mode_ref_scn(AccessMode &access_mode, SCN &ref_scn) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogModeMgr has inited", K(ret)); } else { - mode_version = applied_mode_meta_.mode_version_; access_mode = applied_mode_meta_.access_mode_; ref_scn = applied_mode_meta_.ref_scn_; } return ret; } -// require rlock of PalfHandleImpl -LogModeMeta LogModeMgr::get_accepted_mode_meta() const -{ - return accepted_mode_meta_; -} - -// require rlock of PalfHandleImpl -LogModeMeta LogModeMgr::get_last_submit_mode_meta() const -{ - return last_submit_mode_meta_; -} - -// proected by lock_ in PalfHandleImpl bool LogModeMgr::can_append() const { - return applied_mode_meta_.access_mode_ == AccessMode::APPEND && can_do_paxos_accept(); -} - -// proected by lock_ in PalfHandleImpl -bool LogModeMgr::can_raw_write() const -{ - return applied_mode_meta_.access_mode_ == AccessMode::RAW_WRITE && can_do_paxos_accept(); -} - -bool LogModeMgr::can_receive_log() const -{ - return is_valid_access_mode(applied_mode_meta_.access_mode_); -} - -// need lock-free -bool LogModeMgr::is_in_pending_state() const -{ - return MODE_INIT != ATOMIC_LOAD(&state_); -} - -// proected by lock_ in PalfHandleImpl -bool LogModeMgr::can_do_paxos_accept() const -{ - return MODE_PREPARE != state_; -} - -// proected by lock_ in PalfHandleImpl -bool LogModeMgr::need_skip_log_barrier() const -{ - return false; -} - -int LogModeMgr::can_change_access_mode_(const int64_t mode_version) const -{ - int ret = OB_SUCCESS; - const bool is_leader_active = state_mgr_->is_leader_active(); - const bool is_epoch_changed = !state_mgr_->check_epoch_is_same_with_election(leader_epoch_); - const bool see_newer_mode_meta = (max_majority_accepted_mode_meta_.is_valid() && - max_majority_accepted_mode_meta_.proposal_id_ > accepted_mode_meta_.proposal_id_); - const bool see_newer_log = \ - (max_majority_accepted_pid_ != INVALID_PROPOSAL_ID && max_majority_accepted_pid_ > local_max_log_pid_) || - (max_majority_accepted_pid_ == local_max_log_pid_ && max_majority_lsn_ > local_max_lsn_); - // leader_epoch_ hasn't been updated when state_ is equal to MODE_INIT, so don't check epoch when init - if ((false == is_leader_active) || (state_ != ModeChangeState::MODE_INIT && is_epoch_changed)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "self is not master, can not change_mode", K(ret), K_(palf_id), K_(self), - K(is_leader_active), K_(state), K(is_epoch_changed)); - } else if (state_ != ModeChangeState::MODE_INIT && (see_newer_mode_meta || see_newer_log)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "see newer mode_meta/log, cann't change_acces_mode", K(ret), K_(palf_id), K_(self), - K(see_newer_mode_meta), K_(max_majority_accepted_mode_meta), K_(accepted_mode_meta), - K(see_newer_log), K_(max_majority_accepted_pid), K_(local_max_log_pid), - K_(max_majority_lsn), K_(local_max_lsn)); - } else if (applied_mode_meta_.mode_version_ != mode_version) { - // mode_version is the proposal_id of PALF when access_mode was applied - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "state don't matches, can not change_mode", K(ret), K_(palf_id), K_(self), - K(mode_version), "curr_proposal_id", state_mgr_->get_proposal_id(), K_(applied_mode_meta)); - } - return ret; -} - -int LogModeMgr::init_change_mode_() -{ - int ret = OB_SUCCESS; - int64_t replica_num = 0; - if (OB_FAIL(config_mgr_->get_alive_member_list_with_arb(follower_list_, replica_num))) { - PALF_LOG(WARN, "get_alive_member_list_with_arb failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(follower_list_.remove_server(self_))) { - PALF_LOG(WARN, "remove_server failed", K(ret), K_(palf_id), K_(self), K_(follower_list)); - } else { - majority_cnt_ = replica_num / 2 + 1; - leader_epoch_ = state_mgr_->get_leader_epoch(); - local_max_lsn_ = sw_->get_max_lsn(); - local_max_log_pid_ = state_mgr_->get_proposal_id(); - max_majority_accepted_pid_ = local_max_log_pid_; - max_majority_lsn_ = local_max_lsn_; - } - return ret; -} - -// State machine for mode changing -// INIT ◄────┐ -// │ │ -// ▼ │ -// PREPARE │ -// │ │ -// ▼ │ -// ACCEPT─────┘ -// require caller hold rlock of PalfHandleImpl -bool LogModeMgr::is_state_changed() const -{ - bool bool_ret = false; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - bool_ret = true; - } else { - switch (state_) { - case (ModeChangeState::MODE_INIT): - { - bool_ret = true; - break; - } - case (ModeChangeState::MODE_PREPARE): - case (ModeChangeState::MODE_ACCEPT): - { - const bool is_reach_majority = is_reach_majority_(); - const bool is_need_retry = is_need_retry_(); - const bool is_epoch_changed = !state_mgr_->check_epoch_is_same_with_election(leader_epoch_); - bool_ret = (is_reach_majority || is_need_retry || is_epoch_changed); - if (true == bool_ret) { - PALF_LOG(INFO, "is_state_changed", K_(palf_id), K_(self), "state", state2str_(state_), - K(is_reach_majority), K(is_need_retry), K(is_epoch_changed), K_(follower_list), - K_(majority_cnt), K_(ack_list), K_(last_submit_req_ts)); - } - break; - } - default: - { - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "Invalid ModeChangeState", K_(palf_id), K_(self), K_(state)); - break; - } - } - } - return bool_ret; -} - -bool LogModeMgr::is_reach_majority_() const -{ - return ack_list_.get_count() >= majority_cnt_; -} - -bool LogModeMgr::can_finish_change_mode_() const -{ - bool bool_ret = false; - LSN last_slide_lsn, committed_end_lsn; - int64_t unused_id = 0; - const LSN max_lsn = sw_->get_max_lsn(); - bool_ret = sw_->is_all_committed_log_slided_out(last_slide_lsn, unused_id, committed_end_lsn) && - (committed_end_lsn >= max_lsn); - if (false == bool_ret && palf_reach_time_interval(500 * 1000, wait_committed_log_slide_warn_ts_)) { - PALF_LOG(INFO, "wait is_all_committed_log_slided_out", K(bool_ret), K(last_slide_lsn), - K(committed_end_lsn), K(max_lsn)); - } - return bool_ret; -} - -bool LogModeMgr::is_need_retry_() const -{ - return (last_submit_req_ts_ == OB_INVALID_TIMESTAMP || - common::ObTimeUtility::current_time() - last_submit_req_ts_ > PREPARE_RETRY_INTERVAL_US); -} - -void LogModeMgr::reset_status() -{ - common::ObSpinLockGuard guard(lock_); - reset_status_(); - resend_mode_meta_list_.reset(); -} - -void LogModeMgr::reset_status_() -{ - ATOMIC_STORE(&state_, ModeChangeState::MODE_INIT); - leader_epoch_ = -1; - new_proposal_id_ = INVALID_PROPOSAL_ID; - ack_list_.reset(); - follower_list_.reset(); - majority_cnt_ = INT64_MAX; - last_submit_req_ts_ = OB_INVALID_TIMESTAMP; - max_majority_accepted_mode_meta_.reset(); - local_max_lsn_.reset(); - local_max_log_pid_ = INVALID_PROPOSAL_ID; - max_majority_accepted_pid_ = INVALID_PROPOSAL_ID; - max_majority_lsn_.reset(); -} - -int LogModeMgr::change_access_mode( - const int64_t mode_version, - const AccessMode &access_mode, - const SCN &ref_scn) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == is_valid_access_mode(access_mode) || - !ref_scn.is_valid() || - INVALID_PROPOSAL_ID == mode_version || - mode_version < 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K_(self), K(access_mode), - K(ref_scn), K(mode_version)); - } else if (OB_FAIL(can_change_access_mode_(mode_version))) { - PALF_LOG(WARN, "can_change_access_mode failed", K(ret), K_(palf_id), K_(self)); - } else if (applied_mode_meta_.access_mode_ == access_mode) { - ret = OB_SUCCESS; - PALF_LOG(INFO, "don't need change access_mode to self", K_(palf_id), K_(self), - K(access_mode), K_(applied_mode_meta)); - } else if (false == can_switch_access_mode_(applied_mode_meta_.access_mode_, access_mode)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "can not switch access_mode", K_(palf_id), K_(self), - K(access_mode), K_(applied_mode_meta)); - } else { - const bool is_reconfirm = false; - ret = switch_state_(access_mode, ref_scn, is_reconfirm); - } - return ret; -} - -int LogModeMgr::reconfirm_mode_meta() -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == state_mgr_->is_leader_reconfirm()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "don't call reconfirm_mode_meta when self_ isn't reconfirming leader", - K(ret), K_(palf_id), K_(self)); - } else { - const bool is_reconfirm = true; - SCN invalid_scn; - ret = switch_state_(AccessMode::INVALID_ACCESS_MODE, invalid_scn, is_reconfirm); - } - return ret; -} - -int LogModeMgr::switch_state_(const AccessMode &access_mode, - const SCN &ref_scn, - const bool is_reconfirm) -{ - int ret = OB_SUCCESS; - bool change_done = false; - switch (state_) { - case (ModeChangeState::MODE_INIT): - { - const bool need_inc_pid = true; - const bool need_send_and_handle_prepare = (!is_reconfirm); - // if access_mode is already changed, just return OB_SUCCESS; - if (OB_FAIL(init_change_mode_())) { - PALF_LOG(WARN, "init_change_mode_ failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(submit_prepare_req_(need_inc_pid, need_send_and_handle_prepare))) { - PALF_LOG(WARN, "submit_prepare_req_ failed", K(ret), K_(palf_id), K_(self), K(is_reconfirm)); - } else { - ATOMIC_STORE(&state_, ModeChangeState::MODE_PREPARE); - } - if (state_ != ModeChangeState::MODE_PREPARE) { - break; - } - } - case (ModeChangeState::MODE_PREPARE): - { - if (is_reach_majority_()) { - ack_list_.reset(); - last_submit_req_ts_ = OB_INVALID_TIMESTAMP; - ATOMIC_STORE(&state_, ModeChangeState::MODE_ACCEPT); - } else if (is_reconfirm || is_need_retry_()) { - // change_access_mode inc update proposal_id only once - const bool need_inc_pid = is_reconfirm; - const bool need_send_and_handle_prepare = (!is_reconfirm); - if (OB_FAIL(submit_prepare_req_(need_inc_pid, need_send_and_handle_prepare))) { - PALF_LOG(WARN, "submit_prepare_req_ failed", K(ret), K_(palf_id), K_(self)); - } - } - if (state_ != ModeChangeState::MODE_ACCEPT) { - break; - } - } - case (ModeChangeState::MODE_ACCEPT): - { - if (is_reach_majority_()) { - //TODO(yunlong):check conflict - - // not reaches majority until LogModeMeta is been flushed by leader, - // otherwise next change_access_mode/reconfirm may learn wrong ModeMeta - if (accepted_mode_meta_.proposal_id_ == new_proposal_id_) { - // wait all committed log slide out - change_done = (true == is_reconfirm)? true: can_finish_change_mode_(); - if (change_done) { - applied_mode_meta_ = accepted_mode_meta_; - const bool need_inc_ref_scn = applied_mode_meta_.ref_scn_.is_valid() && - AccessMode::APPEND == applied_mode_meta_.access_mode_; - if (need_inc_ref_scn && OB_FAIL(sw_->inc_update_scn_base(applied_mode_meta_.ref_scn_))) { - PALF_LOG(ERROR, "inc_update_base_log_ts failed", KR(ret), K_(palf_id), K_(self), - K_(applied_mode_meta)); - } else if (need_inc_ref_scn && sw_->get_max_scn() < ref_scn) { - change_done = false; - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "inc_update_base_log_ts failed", KR(ret), K_(palf_id), K_(self), - K_(applied_mode_meta), KPC(sw_)); - } else if (OB_FAIL(set_resend_mode_meta_list_())) { - PALF_LOG(WARN, "set_resend_mode_meta_list_ failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(resend_applied_mode_meta_())) { - PALF_LOG(WARN, "resend_applied_mode_meta_ failed", K(ret), K_(palf_id), K_(self)); - } else { } - } - } else { - PALF_LOG(INFO, "mode_meta hasn't been flushed in leader", K(ret), K_(palf_id), K_(self), - K_(last_submit_mode_meta), K_(accepted_mode_meta), K_(new_proposal_id)); - } - } else if (is_need_retry_()) { - const int64_t mode_version = new_proposal_id_; - LogModeMeta mode_meta = max_majority_accepted_mode_meta_; - mode_meta.proposal_id_ = new_proposal_id_; - const bool is_applied_mode_meta = false; - if (false == is_reconfirm && OB_FAIL(mode_meta.generate(new_proposal_id_, mode_version, access_mode, ref_scn))) { - PALF_LOG(WARN, "generate mode_meta failed", K(ret), K_(palf_id), K_(self), - K(access_mode), K(ref_scn), K_(new_proposal_id)); - } else if (OB_FAIL(submit_accept_req_(new_proposal_id_, is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "submit_accept_req_ failed", K(ret), K_(palf_id), K_(self), - K(mode_meta), K_(new_proposal_id)); - } - } - break; - } - default: - { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "Invalid ModeChangeState", K_(palf_id), K_(self), K_(state)); - break; - } - } - if (true == change_done) { - ret = OB_SUCCESS; - reset_status_(); - } else if (OB_SUCCESS != ret && OB_EAGAIN != ret) { - PALF_LOG(ERROR, "switch_state failed", K(ret), K_(palf_id), "state", state2str_(state_), K_(follower_list), - K_(majority_cnt), K_(ack_list)); - } else { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(10 * 1000)) { - PALF_LOG(INFO, "change_access_mode waiting retry", K(ret), K_(palf_id), "state", state2str_(state_), - K_(follower_list), K_(majority_cnt), K_(ack_list)); - } - } - return ret; -} - -int LogModeMgr::set_resend_mode_meta_list_() -{ - int ret = OB_SUCCESS; - common::ObMemberList member_list; - common::GlobalLearnerList learner_list; - int64_t replica_num; - resend_mode_meta_list_.reset(); - if (OB_FAIL(config_mgr_->get_alive_member_list_with_arb(member_list, replica_num))) { - PALF_LOG(WARN, "get_alive_member_list_with_arb failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(config_mgr_->get_global_learner_list(learner_list))) { - PALF_LOG(WARN, "get_global_learner_list failed", K(ret), K_(palf_id), K_(self)); - } else { - member_list.remove_server(self_); - (void) learner_list.deep_copy_to(resend_mode_meta_list_); - const int64_t member_number = member_list.get_member_number(); - for (int64_t idx = 0; idx < member_number && OB_SUCC(ret); ++idx) { - common::ObAddr server; - if (OB_FAIL(member_list.get_server_by_index(idx, server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(ret), K(idx)); - } else if (OB_FAIL(resend_mode_meta_list_.add_learner(ObMember(server, 1)))) { - PALF_LOG(WARN, "add_learner failed", K(ret), K(server)); - } - } - } - return ret; -} - -int LogModeMgr::leader_do_loop_work() -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - const bool is_leader = (self_ == state_mgr_->get_leader()); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (true == is_leader && - ModeChangeState::MODE_INIT == state_ && - 0 != resend_mode_meta_list_.get_member_number() && - is_need_retry_()) { - ret = resend_applied_mode_meta_(); - } - return ret; -} - -int LogModeMgr::resend_applied_mode_meta_() -{ - int ret = OB_SUCCESS; - const int64_t proposal_id = state_mgr_->get_proposal_id(); - const bool is_applied_mode_meta = true; - if (resend_mode_meta_list_.is_valid() && - OB_FAIL(log_engine_->submit_change_mode_meta_req(resend_mode_meta_list_, proposal_id, - is_applied_mode_meta, applied_mode_meta_))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", K(ret), K_(palf_id), K_(self), - K_(resend_mode_meta_list), K(proposal_id), K(is_applied_mode_meta), K_(applied_mode_meta)); - } else { - last_submit_req_ts_ = common::ObTimeUtility::current_time(); - } - return ret; -} - -// require wlock in PalfHandleImpl -int LogModeMgr::submit_prepare_req_(const bool need_inc_pid, const bool need_send_and_handle_prepare) -{ - int ret = OB_SUCCESS; - // need reset ack_list when resend prepare msg with bigger proposal_id - ack_list_.reset(); - if (need_inc_pid) { - new_proposal_id_ = state_mgr_->get_proposal_id() + 1; - } - if (need_send_and_handle_prepare && OB_FAIL(state_mgr_->handle_prepare_request(self_, new_proposal_id_))) { - PALF_LOG(WARN, "handle_prepare_request failed", K(ret), K_(palf_id), K_(self), K_(new_proposal_id)); - } else if (need_send_and_handle_prepare && follower_list_.is_valid() && - OB_FAIL(log_engine_->submit_prepare_meta_req(follower_list_, new_proposal_id_))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", K(ret), K_(palf_id), K_(self), - K_(follower_list), K_(new_proposal_id)); - } else if (OB_FAIL(ack_list_.add_server(self_))) { - PALF_LOG(WARN, "add_server failed", K(ret), K_(palf_id), K_(self)); - } else { - last_submit_req_ts_ = common::ObTimeUtility::current_time(); - max_majority_accepted_mode_meta_ = accepted_mode_meta_; - PALF_LOG(INFO, "submit_prepare_meta_req success", K(ret), K_(palf_id), K_(self), - K_(follower_list), K_(ack_list), K_(new_proposal_id)); - } - return ret; -} - -int LogModeMgr::submit_accept_req_(const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - const bool has_accepted = (accepted_mode_meta_.proposal_id_ == mode_meta.proposal_id_); - if (false == has_accepted && OB_FAIL(receive_mode_meta_(self_, proposal_id, is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "receive_mode_meta failed", K(ret), K_(palf_id), K_(self), K(mode_meta)); - } else if (follower_list_.is_valid() && - OB_FAIL(log_engine_->submit_change_mode_meta_req(follower_list_, proposal_id, - is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", K(ret), K_(palf_id), K_(self), - K_(follower_list), K(proposal_id), K(is_applied_mode_meta), K(mode_meta)); - } else { - PALF_LOG(INFO, "submit_change_mode_meta_req success", K(ret), K_(palf_id), K_(self), - K_(follower_list), K_(ack_list), K(proposal_id), K(is_applied_mode_meta), K(mode_meta)); - last_submit_req_ts_ = common::ObTimeUtility::current_time(); - } - return ret; -} - -int LogModeMgr::handle_prepare_response(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t accept_log_proposal_id, - const LSN &last_lsn, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == server.is_valid() || - INVALID_PROPOSAL_ID == msg_proposal_id || - INVALID_PROPOSAL_ID == accept_log_proposal_id || - false == last_lsn.is_valid() || - false == mode_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(server), K(msg_proposal_id), - K(accept_log_proposal_id), K(last_lsn), K(mode_meta)); - } else if (ModeChangeState::MODE_PREPARE != state_ || new_proposal_id_ != msg_proposal_id) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "handle_prepare_response not in PREPARE state", K(ret), K_(palf_id), K_(self), - K_(new_proposal_id), K(server), K(msg_proposal_id), K(mode_meta), "state", state2str_(state_)); - } else if (false == follower_list_.contains(server)) { - PALF_LOG(WARN, "receive prepare req not from follower_list", K(ret), K_(palf_id), K(server), - K_(follower_list)); - } else if (OB_FAIL(ack_list_.add_server(server))) { - PALF_LOG(WARN, "add_server failed", K(ret), K_(palf_id), K_(self), K(server)); - } else { - if (false == max_majority_accepted_mode_meta_.is_valid() || - max_majority_accepted_mode_meta_.proposal_id_ < mode_meta.proposal_id_) { - max_majority_accepted_mode_meta_ = mode_meta; - } - max_majority_accepted_pid_ = max_proposal_id(max_majority_accepted_pid_, accept_log_proposal_id); - max_majority_lsn_ = MAX(max_majority_lsn_, last_lsn); - PALF_LOG(INFO, "handle_prepare_response finish", K(ret), K_(palf_id), K_(self), K(server), K(msg_proposal_id), - K(accept_log_proposal_id), K(last_lsn), K(mode_meta), K_(ack_list), K_(follower_list), K_(majority_cnt), - K_(max_majority_accepted_mode_meta), K_(max_majority_accepted_pid), K_(max_majority_lsn)); - } - return ret; -} - -// require rlock of PalfHandleImpl -bool LogModeMgr::can_receive_mode_meta(const int64_t proposal_id, - const LogModeMeta &mode_meta, - bool &has_accepted) -{ - bool bool_ret = false; - has_accepted = false; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT || - proposal_id == INVALID_PROPOSAL_ID || - false == mode_meta.is_valid()) { - } else if (false == state_mgr_->can_receive_config_log(proposal_id)) { - // for arbitration replica, is_sync_enabled is false, so check can_receive_mode_meta - // with LogStateMgr::can_receive_config_log - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "can_receive_mode_meta failed", K_(palf_id), K_(self), - K(proposal_id), K(mode_meta)); - } else if (accepted_mode_meta_.proposal_id_ > mode_meta.proposal_id_) { - // skip, do not receive mode_meta with smaller proposal_id - } else if (accepted_mode_meta_.proposal_id_ == mode_meta.proposal_id_) { - bool_ret = true; - has_accepted = true; - } else { - bool_ret = true; - } - return bool_ret; -} - -// require wlock of PalfHandleImpl -int LogModeMgr::receive_mode_meta(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = receive_mode_meta_(server, proposal_id, is_applied_mode_meta, mode_meta); - } - return ret; -} - -int LogModeMgr::receive_mode_meta_(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - if (proposal_id == INVALID_PROPOSAL_ID || - false == mode_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(server), K(proposal_id), K(mode_meta)); - } else { - FlushMetaCbCtx flush_ctx; - flush_ctx.type_ = MetaType::MODE_META; - flush_ctx.proposal_id_ = proposal_id; - flush_ctx.log_mode_meta_ = mode_meta; - flush_ctx.is_applied_mode_meta_ = is_applied_mode_meta; - // why need to record LogModeMeta in FlushCtx? - // we need to record 'accepted_mode_meta_' as max flushed LogModeMeta, if a follower - // receives two LogModeMeta A and B successively. - // 1. record last_submit_mode_meta_ to A - // 2. A has been flushed and 'after_flush_mode_meta' is going to be called. - // 3. record last_submit_mode_meta_ to B - // 4. if don't record LogModeMeta in FlushCtx, A will call 'after_flush_mode_meta' - // and records last 'submit_mode_meta_' B to 'accepted_mode_meta_'. But B hasn't been flushed - if (OB_FAIL(log_engine_->submit_flush_mode_meta_task(flush_ctx, mode_meta))) { - PALF_LOG(WARN, "submit_flush_mode_meta_task failed", K(ret), K_(palf_id), K_(self), K(mode_meta)); - } else { - last_submit_mode_meta_ = mode_meta; - } - } - return ret; -} - -// require wlock of PalfHandleImpl -int LogModeMgr::after_flush_mode_meta(const bool is_applied_mode_meta, const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - accepted_mode_meta_ = (mode_meta.proposal_id_ > accepted_mode_meta_.proposal_id_)? - mode_meta: accepted_mode_meta_; - if (is_applied_mode_meta) { - applied_mode_meta_ = (mode_meta.proposal_id_ > applied_mode_meta_.proposal_id_)? - mode_meta: applied_mode_meta_; - } - PALF_LOG(INFO, "after_flush_mode_meta success", K(ret), K_(palf_id), K_(self), - K(is_applied_mode_meta), K(mode_meta), K_(accepted_mode_meta), K_(applied_mode_meta)); - } - return ret; -} - -int LogModeMgr::ack_mode_meta(const common::ObAddr &server, const int64_t proposal_id) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(server), K(proposal_id)); - } else if (ModeChangeState::MODE_ACCEPT != state_) { - PALF_LOG(TRACE, "ack_mode_meta not int ACCEPT state", K(ret), K_(palf_id), - K_(self), K(server), "state", state2str_(state_)); - } else if (new_proposal_id_ != proposal_id) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "ack_mode_meta failed", K(ret), K_(palf_id), K_(self), K(server), - K(proposal_id), K_(new_proposal_id), "state", state2str_(state_)); - } else if (OB_FAIL(ack_list_.add_server(server))) { - PALF_LOG(WARN, "add_server failed", K(ret), K_(palf_id), K_(self), K(server)); - } else { - PALF_LOG(INFO, "ack_mode_meta finish", K(ret), K_(palf_id), K_(self), K(server), - K(proposal_id), K_(follower_list), K_(majority_cnt), K_(ack_list), K_(resend_mode_meta_list)); - } - (void) resend_mode_meta_list_.remove_learner(server); - return ret; -} - -// require rlock in PalfHandleImpl -int LogModeMgr::submit_fetch_mode_meta_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t accepted_mode_pid) -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); - common::ObMemberList member_list; - (void) member_list.add_server(server); - LogModeMeta mode_meta; - bool is_applied_mode_meta = false; - if (applied_mode_meta_.proposal_id_ >= accepted_mode_meta_.proposal_id_) { - is_applied_mode_meta = true; - mode_meta = applied_mode_meta_; - } else { - is_applied_mode_meta = false; - mode_meta = accepted_mode_meta_; - } - if ((accepted_mode_pid == INVALID_PROPOSAL_ID || mode_meta.proposal_id_ >= accepted_mode_pid) && - OB_FAIL(log_engine_->submit_change_mode_meta_req(member_list, msg_proposal_id, is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "submit_change_mode_meta_req failed", K(ret), K_(palf_id), K(server), - K(server), K(msg_proposal_id), K(mode_meta)); - } else { - PALF_LOG(INFO, "submit_change_mode_meta_req success", K(ret), K_(palf_id), K(server), K(mode_meta), - K(accepted_mode_pid)); - } - return ret; + return applied_mode_meta_.access_mode_ == AccessMode::APPEND; } } // namespace palf diff --git a/src/logservice/palf/log_mode_mgr.h b/src/logservice/palf/log_mode_mgr.h index 9a0ce2169..0568392a2 100644 --- a/src/logservice/palf/log_mode_mgr.h +++ b/src/logservice/palf/log_mode_mgr.h @@ -20,8 +20,6 @@ #include "lib/lock/ob_spin_lock.h" // SpinRWLock #include "log_define.h" // utils #include "log_meta_info.h" // LogMembershipMeta -#include "log_req.h" // LogLearnerReqType -#include "log_simple_member_list.h" // LogAckList namespace oceanbase { @@ -34,158 +32,33 @@ namespace palf { class LogStateMgr; class LogEngine; -class LogConfigMgr; class LogSlidingWindow; -enum ModeChangeState -{ - MODE_INIT = 0, - MODE_PREPARE = 1, - MODE_ACCEPT = 2, - MODE_DONE = 3, -}; - class LogModeMgr { public: LogModeMgr(); virtual ~LogModeMgr() { destroy(); } - int init(const int64_t palf_id, - const common::ObAddr &self, - const LogModeMeta &log_mode_meta, - LogStateMgr *state_mgr, - LogEngine *log_engine, - LogConfigMgr *config_mgr, - LogSlidingWindow *sw); + int init(const common::ObAddr &self, + const LogModeMeta &log_mode_meta); virtual void destroy(); - virtual void reset_status(); virtual int get_access_mode(AccessMode &access_mode) const; - virtual int get_mode_version(int64_t &mode_version) const; - virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const; - virtual int get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, + virtual int get_access_mode_ref_scn(AccessMode &access_mode, share::SCN &ref_scn) const; bool can_append() const; - bool can_raw_write() const; - bool can_receive_log() const; - bool is_in_pending_state() const; - bool can_do_paxos_accept() const; - virtual bool is_state_changed() const; - virtual bool need_skip_log_barrier() const; - virtual LogModeMeta get_accepted_mode_meta() const; - virtual LogModeMeta get_last_submit_mode_meta() const; - virtual int reconfirm_mode_meta(); - virtual int change_access_mode(const int64_t mode_version, - const AccessMode &access_mode, - const share::SCN &ref_scn); - virtual int handle_prepare_response(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t accept_log_proposal_id, - const LSN &last_lsn, - const LogModeMeta &mode_meta); - virtual bool can_receive_mode_meta(const int64_t proposal_id, - const LogModeMeta &mode_meta, - bool &has_accepted); - virtual int receive_mode_meta(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta); - virtual int after_flush_mode_meta(const bool is_applied_mode_meta, const LogModeMeta &mode_meta); - virtual int ack_mode_meta(const common::ObAddr &server, const int64_t proposal_id); - virtual int submit_fetch_mode_meta_resp(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t accepted_mode_pid); - int leader_do_loop_work(); int64_t to_string(char* buf, const int64_t buf_len) const { - common::ObSpinLockGuard guard(lock_); int64_t pos = 0; J_OBJ_START(); - J_KV(K_(palf_id), K_(self), K_(applied_mode_meta), K_(accepted_mode_meta), - K_(last_submit_mode_meta), "state", state2str_(state_), K_(new_proposal_id), K_(local_max_lsn), - K_(local_max_log_pid), K_(max_majority_accepted_pid), K_(max_majority_lsn), - K_(max_majority_accepted_mode_meta), K_(follower_list), K_(ack_list), K_(majority_cnt), - K_(last_submit_req_ts), K_(resend_mode_meta_list)); + J_KV(K_(self), K_(applied_mode_meta)); J_OBJ_END(); return pos; } private: - void reset_status_(); - int init_change_mode_(); - int can_change_access_mode_(const int64_t mode_version) const; - bool is_reach_majority_() const; - bool can_finish_change_mode_() const; - bool is_need_retry_() const; - int switch_state_(const AccessMode &access_mode, - const share::SCN &ref_scn, - const bool is_reconfirm); - int submit_prepare_req_(const bool need_inc_pid, const bool need_send_and_handle_prepare); - int submit_accept_req_(const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta); - int receive_mode_meta_(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta); - int set_resend_mode_meta_list_(); - int resend_applied_mode_meta_(); - -private: - static const int64_t PREPARE_RETRY_INTERVAL_US = 2 * 1000 * 1000; // 2s - const char *state2str_(const ModeChangeState state) const - { - #define MODE_CHANGE_TYPE_STR(x) case(ModeChangeState::x): return #x - switch(state) - { - MODE_CHANGE_TYPE_STR(MODE_INIT); - MODE_CHANGE_TYPE_STR(MODE_PREPARE); - MODE_CHANGE_TYPE_STR(MODE_ACCEPT); - MODE_CHANGE_TYPE_STR(MODE_DONE); - default: - return "Invalid"; - } - #undef MODE_CHANGE_TYPE_STR - } - -private: - mutable common::ObSpinLock lock_; bool is_inited_; - int64_t palf_id_; common::ObAddr self_; - // applied_mode_meta, it has been flushed in majority, - // external modules could see this meta LogModeMeta applied_mode_meta_; - // NB: protected by SpinLock - // log_mode_meta has been accepted/flushed - LogModeMeta accepted_mode_meta_; - // NB: protected by SpinLock - // log_mode_meta has been submitted to I/O Worker - LogModeMeta last_submit_mode_meta_; - // above LogModeMetas are protected by lock_ in PalfHandleImpl - // =========access_mode changing state============ - // mode change state - ModeChangeState state_; - int64_t leader_epoch_; - int64_t new_proposal_id_; - // ack server list - LogSimpleMemberList ack_list_; - // all paxos folowers, not include self - common::ObMemberList follower_list_; - int64_t majority_cnt_; - int64_t last_submit_req_ts_; - LogModeMeta max_majority_accepted_mode_meta_; - LSN local_max_lsn_; - int64_t local_max_log_pid_; - int64_t max_majority_accepted_pid_; - LSN max_majority_lsn_; - ResendConfigLogList resend_mode_meta_list_; - mutable int64_t wait_committed_log_slide_warn_ts_; - // =========access_mode changing state============ - LogStateMgr *state_mgr_; - LogEngine *log_engine_; - LogConfigMgr *config_mgr_; - LogSlidingWindow *sw_; }; } // namespace palf } // namespace oceanbase diff --git a/src/logservice/palf/log_net_service.cpp b/src/logservice/palf/log_net_service.cpp deleted file mode 100644 index ebe222e60..000000000 --- a/src/logservice/palf/log_net_service.cpp +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_net_service.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -LogNetService::LogNetService() : palf_id_(), - log_rpc_(NULL), - is_inited_(false) -{ -} - -LogNetService::~LogNetService() -{ - destroy(); -} - -int LogNetService::init(const int64_t palf_id, - LogRpc *log_rpc) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - } else { - palf_id_ = palf_id; - log_rpc_ = log_rpc; - is_inited_ = true; - } - return ret; -} - -void LogNetService::destroy() -{ - if (IS_INIT) { - PALF_LOG(INFO, "LogNetService destroy success", K(palf_id_)); - is_inited_ = false; - log_rpc_ = NULL; - palf_id_ = 0;; - } -} - -int LogNetService::start() -{ - int ret = OB_SUCCESS; - - return ret; -} - -int LogNetService::submit_push_log_req( - const common::ObAddr &server, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); - } else { - LogPushReq push_log_req(push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf); - ret = post_request_to_server_(server, push_log_req); - } - return ret; -} - -int LogNetService::submit_committed_info_req( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); - } else { - CommittedInfo committed_info_req(msg_proposal_id, prev_log_id, - prev_log_proposal_id, committed_end_lsn); - ret = post_request_to_server_(server, committed_info_req); - } - return ret; -} - -int LogNetService::submit_push_log_resp( - const ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool is_batch) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService not inited", K(ret), K(palf_id_), - K(server), K(msg_proposal_id), K(lsn)); - } else if (false == server.is_valid() - || INVALID_PROPOSAL_ID == msg_proposal_id - || false == lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id_), - K(server), K(msg_proposal_id), K(lsn)); - } else if (is_batch) { - LogBatchPushResp push_log_resp(msg_proposal_id, lsn); - ret = post_request_to_server_(server, push_log_resp); - } else { - LogPushResp push_log_resp(msg_proposal_id, lsn); - ret = post_request_to_server_(server, push_log_resp); - } - return ret; -} - -int LogNetService::submit_fetch_log_req( - const ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService not inited!!!", K(ret), K(palf_id_)); - } else { - LogFetchReq fetch_log_req(fetch_type, msg_proposal_id, prev_lsn, lsn, fetch_log_size, - fetch_log_count, accepted_mode_pid); - ret = post_request_to_server_(server, fetch_log_req); - } - return ret; -} - -int LogNetService::submit_batch_fetch_log_resp( - const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService not inited!!!", K(ret), K(palf_id_)); - } else { - LogBatchFetchResp batch_fetch_log_resp(msg_proposal_id, prev_log_proposal_id, - prev_lsn, curr_lsn, write_buf); - ret = post_request_to_server_(server, batch_fetch_log_resp); - } - return ret; -} - -int LogNetService::submit_notify_rebuild_req( - const ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - NotifyRebuildReq notify_rebuild_req(base_lsn, base_prev_log_info); - ret = post_request_to_server_(server, notify_rebuild_req); - } - return ret; -} - -int LogNetService::submit_notify_fetch_log_req(const ObMemberList &dst_list) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - NotifyFetchLogReq notify_req; - ret = post_request_to_member_list_(dst_list, notify_req); - } - return ret; -} - -int LogNetService::submit_prepare_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService not inited!!!", K(ret), K(palf_id_)); - } else { - LogPrepareResp prepare_meta_resp(msg_proposal_id, vote_granted, log_proposal_id, max_flushed_lsn, \ - mode_meta, committed_end_lsn); - ret = post_request_to_server_(server, prepare_meta_resp); - PALF_LOG(INFO, "submit_prepare_meta_resp success", K(ret), K(server), K(msg_proposal_id)); - } - return ret; -} - -int LogNetService::submit_change_config_meta_resp( - const ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogChangeConfigMetaResp req(msg_proposal_id, config_version); - ret = post_request_to_server_(server, req); - } - return ret; -} - -int LogNetService::submit_change_mode_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogChangeModeMetaResp req(msg_proposal_id); - ret = post_request_to_server_(server, req); - } - return ret; -} - -int LogNetService::submit_config_change_pre_check_req( - const ObAddr &server, - const LogConfigVersion &config_version, - const bool need_purge_throttling, - const int64_t timeout_us, - LogGetMCStResp &resp) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogGetMCStReq req(config_version, need_purge_throttling); - ret = post_sync_request_to_server_(server, timeout_us, req, resp); - } - return ret; -} - -int LogNetService::submit_register_parent_req( - const common::ObAddr &server, - const LogLearner &child_itself, - const bool is_to_leader) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogRegisterParentReq req(child_itself, is_to_leader); - ret = post_request_to_server_(server, req); - } - return ret; -} - -int LogNetService::submit_register_parent_resp( - const common::ObAddr &server, - const LogLearner &parent_itself, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogRegisterParentResp resp(parent_itself, candidate_list, reg_ret); - ret = post_request_to_server_(server, resp); - } - return ret; -} - -int LogNetService::submit_retire_parent_req( - const common::ObAddr &server, - const LogLearner &child_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogLearnerReq retire_parent_req(child_itself, LogLearnerReqType::RETIRE_PARENT); - ret = post_request_to_server_(server, retire_parent_req); - } - return ret; -} - -int LogNetService::submit_retire_child_req(const common::ObAddr &server, const LogLearner &parent_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogLearnerReq retire_child_req(parent_itself, LogLearnerReqType::RETIRE_CHILD); - ret = post_request_to_server_(server, retire_child_req); - } - return ret; -} - -int LogNetService::submit_learner_keepalive_req(const common::ObAddr &server, const LogLearner &sender_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogLearnerReq req(sender_itself, LogLearnerReqType::KEEPALIVE_REQ); - ret = post_request_to_server_(server, req); - } - return ret; -} - -int LogNetService::submit_learner_keepalive_resp(const common::ObAddr &server, const LogLearner &sender_itself) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogLearnerReq req(sender_itself, LogLearnerReqType::KEEPALIVE_RESP); - ret = post_request_to_server_(server, req); - } - return ret; -} - -int LogNetService::submit_get_stat_req(const common::ObAddr &server, - const int64_t timeout_us, - const LogGetStatReq &req, - LogGetStatResp &resp) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = post_sync_request_to_server_(server, timeout_us, req, resp); - } - return ret; -} -} // end namespace palf -} // end namespace oceanbase diff --git a/src/logservice/palf/log_net_service.h b/src/logservice/palf/log_net_service.h deleted file mode 100644 index 28e11d142..000000000 --- a/src/logservice/palf/log_net_service.h +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_NET_SERVICE_ -#define OCEANBASE_LOGSERVICE_LOG_NET_SERVICE_ - -#include -#include "lib/ob_errno.h" // ERRNO... -#include "log_meta_info.h" -#include "log_writer_utils.h" -#include "common/ob_member_list.h" // ObMemberList -#include "log_rpc.h" // LogRpc -#include "log_req.h" // PushLogType - -namespace oceanbase -{ -namespace common -{ -class ObAddr; -} -namespace palf -{ -class LSN; -class LogPrepareMeta; -class LogConfigMeta; -class LogRpc; -class LogWriteBuf; - -class LogNetService -{ -public: - LogNetService(); - ~LogNetService(); - -public: - int init(const int64_t palf_id, - LogRpc *log_rpc); - void destroy(); - int start(); - - template - int submit_push_log_req( - const List &member_list, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); - } else { - LogPushReq push_log_req(push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf); - ret = post_request_to_member_list_(member_list, push_log_req); - } - return ret; - } - - template - int submit_batch_push_log_req( - const List &member_list, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); - } else { - LogBatchPushReq push_log_req(push_log_type, - msg_proposal_id, - prev_log_proposal_id, - prev_lsn, - curr_lsn, - write_buf); - ret = post_request_to_member_list_(member_list, push_log_req); - PALF_LOG(TRACE, "post_request_to_member_list_ success", K(member_list), K(push_log_req)); - } - return ret; - } - - int submit_push_log_req( - const ObAddr &server, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - - - int submit_push_log_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool is_batch); - - template - int submit_prepare_meta_req( - const List &member_list, - const int64_t &log_proposal_id) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService not inited!!!", K(ret), K(palf_id_)); - } else if (INVALID_PROPOSAL_ID == log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid arguments!!!", K(ret), K(palf_id_), - K(member_list), K(log_proposal_id)); - } else { - LogPrepareReq prepare_meta_req(log_proposal_id); - ret = post_request_to_member_list_(member_list, prepare_meta_req); - PALF_LOG(INFO, "submit_prepare_meta_req success", K(ret), K(member_list)); - } - return ret; - } - - int submit_prepare_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &mode_meta); - - int submit_fetch_log_req( - const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid); - - int submit_batch_fetch_log_resp( - const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - - int submit_notify_rebuild_req( - const ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info); - - int submit_notify_fetch_log_req( - const ObMemberList &dst_list); - - template - int submit_change_config_meta_req( - const List &member_list, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &config_meta) - { - int ret = OB_SUCCESS; - int64_t pos = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogChangeConfigMetaReq req(msg_proposal_id, prev_log_proposal_id, prev_lsn, - prev_mode_pid, config_meta); - ret = post_request_to_member_list_(member_list, req); - } - return ret; - } - - int submit_change_config_meta_resp( - const common::ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version); - - template - int submit_change_mode_meta_req( - const List &member_list, - const int64_t &msg_proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - int64_t pos = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LogChangeModeMetaReq req(msg_proposal_id, mode_meta, is_applied_mode_meta); - ret = post_request_to_member_list_(member_list, req); - } - return ret; - } - - int submit_change_mode_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id); - - int submit_config_change_pre_check_req( - const common::ObAddr &server, - const LogConfigVersion &config_version, - const bool need_purge_throttling, - const int64_t timeout_us, - LogGetMCStResp &resp); - - int submit_register_parent_req( - const common::ObAddr &server, - const LogLearner &child_itself, - const bool is_to_leader); - - int submit_register_parent_resp( - const common::ObAddr &server, - const LogLearner &parent_itself, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret); - - int submit_retire_parent_req(const common::ObAddr &server, const LogLearner &child_itself); - int submit_retire_child_req(const common::ObAddr &server, const LogLearner &parent_itself); - int submit_learner_keepalive_req(const common::ObAddr &server, const LogLearner &sender_itself); - int submit_learner_keepalive_resp(const common::ObAddr &server, const LogLearner &sender_itself); - int submit_committed_info_req(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn); - template - int submit_committed_info_req( - const List &member_list, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) - { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); - } else { - CommittedInfo committed_info_req(msg_proposal_id, prev_log_id, - prev_log_proposal_id, committed_end_lsn); - ret = post_request_to_member_list_(member_list, committed_info_req); - } - return ret; - } - int submit_get_stat_req(const common::ObAddr &server, - const int64_t timeout_us, - const LogGetStatReq &req, - LogGetStatResp &resp); - -public: - template - int post_request_to_server_(const common::ObAddr &server, - const ReqType &req); - template - int post_request_to_member_list_(const List &member_list, - const ReqType &req); - template - int post_sync_request_to_server_(const common::ObAddr &server, - const int64_t timeout_us, - const ReqType &req, - RespType &resp); -private: - int64_t palf_id_; - LogRpc *log_rpc_; - bool is_inited_; -}; - -template -int LogNetService::post_request_to_server_( - const common::ObAddr &server, - const ReqType &req) -{ - // RPC plumbing neutered: single-replica seekdb never sends inter-replica - // palf/election RPC. Self is excluded before send; self-ack/commit is local. - UNUSED(server); - UNUSED(req); - return common::OB_SUCCESS; -} - -template -int LogNetService::post_request_to_member_list_( - const List &member_list, - const ReqType &req) -{ - // RPC plumbing neutered: single-replica seekdb has self excluded from the - // destination member list, so the send loop is a no-op. Keep arg validation. - int ret = common::OB_SUCCESS; - if (!req.is_valid() || !member_list.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } - return ret; -} - -template -int LogNetService::post_sync_request_to_server_(const common::ObAddr &server, - const int64_t timeout_us, - const ReqType &req, - RespType &resp) -{ - // RPC plumbing neutered: single-replica seekdb never issues inter-replica - // sync palf RPC (config-change pre-check / get-stat target only peers). - UNUSED(server); - UNUSED(timeout_us); - UNUSED(req); - UNUSED(resp); - return common::OB_SUCCESS; -} -} // end namespace palf -} // end namespace oceanbase - -#endif diff --git a/src/logservice/palf/log_reconfirm.cpp b/src/logservice/palf/log_reconfirm.cpp deleted file mode 100644 index 09008cf39..000000000 --- a/src/logservice/palf/log_reconfirm.cpp +++ /dev/null @@ -1,651 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX PALF -#include "log_reconfirm.h" -#include "log_config_mgr.h" -#include "log_mode_mgr.h" -#include "log_sliding_window.h" -#include "log_engine.h" - -namespace oceanbase -{ -using namespace common; - -namespace palf -{ -LogReconfirm::LogReconfirm() - : state_(INITED), - palf_id_(), - self_(), - new_proposal_id_(INVALID_PROPOSAL_ID), - prepare_log_ack_list_(), - curr_paxos_follower_list_(), - majority_cnt_(0), - majority_max_log_server_(), - majority_max_accept_pid_(INVALID_PROPOSAL_ID), - majority_max_lsn_(), - saved_end_lsn_(), - sw_config_version_(), - sw_(NULL), - state_mgr_(NULL), - mm_(NULL), - mode_mgr_(NULL), - log_engine_(NULL), - lock_(ObLatchIds::CLOG_RECONFIRM_LOCK), - last_submit_prepare_req_time_us_(OB_INVALID_TIMESTAMP), - last_fetch_log_time_us_(OB_INVALID_TIMESTAMP), - last_record_sw_start_id_(OB_INVALID_LOG_ID), - wait_slide_print_time_us_(OB_INVALID_TIMESTAMP), - wait_majority_time_us_(OB_INVALID_TIMESTAMP), - last_notify_fetch_time_us_(OB_INVALID_TIMESTAMP), - last_purge_throttling_time_us_(OB_INVALID_TIMESTAMP), - is_inited_(false) -{} - -int LogReconfirm::init(const int64_t palf_id, - const ObAddr &self, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - LogConfigMgr *mm, - LogModeMgr *mode_mgr, - LogEngine *log_engine) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - } else if (OB_ISNULL(sw) || - OB_ISNULL(state_mgr) || - OB_ISNULL(mm) || - OB_ISNULL(mode_mgr) || - OB_ISNULL(log_engine) || - !self.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else { - palf_id_ = palf_id; - self_ = self; - sw_ = sw; - state_mgr_ = state_mgr; - mm_ = mm; - mode_mgr_ = mode_mgr; - log_engine_ = log_engine; - is_inited_ = true; - PALF_LOG(INFO, "LogReconf init success", K(ret), K(palf_id)); - } - return ret; -} - -void LogReconfirm::reset_state() -{ - ObLockGuard guard(lock_); - if (state_ != INITED) { - state_ = INITED; - new_proposal_id_ = INVALID_PROPOSAL_ID; - prepare_log_ack_list_.reset(); - curr_paxos_follower_list_.reset(); - majority_cnt_ = 0; - majority_max_log_server_.reset(); - majority_max_accept_pid_ = INVALID_PROPOSAL_ID; - majority_max_lsn_.reset(); - saved_end_lsn_.reset(); - follower_end_lsn_list_.reset(); - sw_config_version_.reset(); - last_submit_prepare_req_time_us_ = OB_INVALID_TIMESTAMP; - last_fetch_log_time_us_ = OB_INVALID_TIMESTAMP; - last_record_sw_start_id_ = OB_INVALID_LOG_ID; - last_notify_fetch_time_us_ = OB_INVALID_TIMESTAMP; - last_purge_throttling_time_us_ = OB_INVALID_TIMESTAMP; - } -} - -void LogReconfirm::destroy() -{ - ObLockGuard guard(lock_); - if (is_inited_) { - is_inited_ = false; - last_notify_fetch_time_us_ = OB_INVALID_TIMESTAMP; - last_purge_throttling_time_us_ = OB_INVALID_TIMESTAMP; - state_ = INITED; - new_proposal_id_ = INVALID_PROPOSAL_ID; - prepare_log_ack_list_.reset(); - curr_paxos_follower_list_.reset(); - majority_cnt_ = 0; - majority_max_log_server_.reset(); - majority_max_accept_pid_ = INVALID_PROPOSAL_ID; - majority_max_lsn_.reset(); - saved_end_lsn_.reset(); - follower_end_lsn_list_.destroy(); - sw_config_version_.reset(); - sw_ = NULL; - state_mgr_ = NULL; - mm_ = NULL; - mode_mgr_ = NULL; - log_engine_ = NULL; - last_submit_prepare_req_time_us_ = OB_INVALID_TIMESTAMP; - last_fetch_log_time_us_ = OB_INVALID_TIMESTAMP; - last_record_sw_start_id_ = OB_INVALID_LOG_ID; - } -} - -bool LogReconfirm::need_wlock() -{ - ObLockGuard guard(lock_); - // Note: even the wlock is not required in INITED state, - // we still add it to speedup the reconfirmation - return INITED == state_ || - WAITING_LOG_FLUSHED == state_ || - FETCH_MAX_LOG_LSN == state_ || - RECONFIRM_MODE_META == state_ || - RECONFIRMING == state_ || - START_WORKING == state_; -} - -int LogReconfirm::init_reconfirm_() -{ - int ret = OB_SUCCESS; - LSN max_lsn; - int64_t max_log_proposal_id = INVALID_PROPOSAL_ID; - ObMemberList member_list; - int64_t replica_num = 0; - if (OB_FAIL(mm_->get_alive_member_list_with_arb(member_list, replica_num))) { - PALF_LOG(WARN, "get_alive_member_list_with_arb failed", K_(palf_id)); - } else if (!member_list.contains(self_)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "self is not in curr_member_list", K_(palf_id), K(member_list), K(self_)); - } else if (OB_FAIL(curr_paxos_follower_list_.deep_copy(member_list))) { - PALF_LOG(ERROR, "deep_copy failed", K_(palf_id)); - } else if (OB_FAIL(curr_paxos_follower_list_.remove_server(self_))) { - PALF_LOG(ERROR, "remove_server failed", K_(palf_id), K_(curr_paxos_follower_list), K_(self), KP(mm_)); - } else { - majority_cnt_ = replica_num / 2 + 1; - PALF_LOG(INFO, "init_reconfirm_ success", K(ret), K_(palf_id), K(member_list), K_(self), K_(majority_cnt), K(replica_num)); - } - return ret; -} - -int LogReconfirm::submit_prepare_log_() -{ - int ret = OB_SUCCESS; - LSN unused_prev_lsn; - LSN max_flushed_end_lsn; - int64_t max_flushed_log_pid = INVALID_PROPOSAL_ID; - const int64_t now_us = ObTimeUtility::current_time(); - if (OB_FAIL(sw_->get_max_flushed_log_info(unused_prev_lsn, max_flushed_end_lsn, max_flushed_log_pid))) { - PALF_LOG(WARN, "get_max_flushed_log_info failed", K_(palf_id)); - } else if (OB_EAGAIN != (ret = mode_mgr_->reconfirm_mode_meta())) { - PALF_LOG(WARN, "reconfirm_mode_meta failed", K_(palf_id)); - } else { - // clear prepare_log_ack_list_ - prepare_log_ack_list_.reset(); - majority_max_log_server_ = self_; - majority_max_lsn_ = max_flushed_end_lsn; - // the init 'majority_max_accept_pid_' should be the maximum of max_flushed_log_proposl_id and - // proposal_id of config log, otherwise there will are some ghost logs. - // - // assume that, - // first round, replica A is an origin leader, and submit ten logs, but just only the first - // five log are confirmed, and then; - // second round replica B change to leader, and just submit START_WORKING to others; - // third round, replica C change to leader, and it will think the newest server is A, - // so it will reconfirm the logs generated by A, these logs are ghost. - // - // to avoid this problem, A need truncate these logs if it receives the START_WORKING from B. - // if A does not receive the START_WORKING from B, others should return the proposal_id which - // is the maxest of max_flushed_log_proposl_id and proposal_id of config log. - if (INVALID_PROPOSAL_ID != max_flushed_log_pid) { - majority_max_accept_pid_ = max_flushed_log_pid; - } - const int64_t local_accept_proposal_id = mm_->get_accept_proposal_id(); - if (INVALID_PROPOSAL_ID != local_accept_proposal_id - && INVALID_PROPOSAL_ID != majority_max_accept_pid_ - && local_accept_proposal_id > majority_max_accept_pid_) { - majority_max_accept_pid_ = local_accept_proposal_id; - } - - // NB: 'state_mgr_' need guarantee that get_proposal_id will return - // the maxest value of redo log and prepare meta - const int64_t old_proposal_id = state_mgr_->get_proposal_id(); - new_proposal_id_ = old_proposal_id + 1; - - if (OB_FAIL(state_mgr_->handle_prepare_request(self_, new_proposal_id_))) { - PALF_LOG(WARN, "handle_prepare_request failed", K_(palf_id), K_(self), K(new_proposal_id_)); - } else if (curr_paxos_follower_list_.is_valid() - && OB_FAIL(log_engine_->submit_prepare_meta_req(curr_paxos_follower_list_, new_proposal_id_))) { - PALF_LOG(WARN, "submit_prepare_meta_req failed", K_(palf_id), K_(self), K_(new_proposal_id)); - } else { - const int64_t old_prepare_time_us = last_submit_prepare_req_time_us_; - last_submit_prepare_req_time_us_ = now_us; - PALF_LOG(INFO, "submit_prepare_meta_req success", K(ret), K_(palf_id), K(curr_paxos_follower_list_), K(prepare_log_ack_list_), - K(old_proposal_id), K(new_proposal_id_), K(self_), K(majority_max_lsn_), K(majority_max_log_server_), - K(majority_max_accept_pid_), K(now_us), K(old_prepare_time_us), K(last_submit_prepare_req_time_us_), K(max_flushed_log_pid), - K(local_accept_proposal_id)); - } - } - return ret; -} - -int LogReconfirm::wait_all_log_flushed_() -{ - int ret = OB_SUCCESS; - if (false == sw_->check_all_log_has_flushed()) { - ret = OB_EAGAIN; - PALF_LOG(INFO, "There are some logs flushing, need retry", K(ret), K_(palf_id)); - } else { - PALF_LOG(INFO, "wait_all_log_flushed_ success", K(ret), K_(palf_id)); - } - return ret; -} - -int LogReconfirm::handle_prepare_response(const common::ObAddr &server, - const int64_t &src_proposal_id, - const int64_t &accept_proposal_id, - const LSN &last_lsn, - const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - ObLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == src_proposal_id - || INVALID_PROPOSAL_ID == accept_proposal_id || !last_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K_(palf_id), K(server), K(src_proposal_id), - K(accept_proposal_id), K(last_lsn), K(committed_end_lsn)); - // NB: no need change 'majority_max_log_server_' after FETCH_MAX_LOG_LSN - } else if (src_proposal_id != new_proposal_id_ || FETCH_MAX_LOG_LSN != state_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "state not match", K_(palf_id), K(server), K(src_proposal_id), K_(new_proposal_id), - K(state_)); - } else if (false == curr_paxos_follower_list_.contains(server)) { - PALF_LOG(WARN, "receive prepare req not in follower_list", K_(palf_id), K(server), - K_(curr_paxos_follower_list)); - } else if (OB_FAIL(prepare_log_ack_list_.add_server(server))) { - PALF_LOG(WARN, "prepare_log_ack_list_ add_sever failed", K_(palf_id), K(server)); - } else if (INVALID_PROPOSAL_ID != majority_max_accept_pid_ - && accept_proposal_id < majority_max_accept_pid_) { - // accept_proposal_id is smaller than cur majority value, ignore - PALF_LOG(INFO, "server's proposal_id is smaller than me, ignore it", K(ret), K_(palf_id), K(server), K(src_proposal_id), - K(accept_proposal_id), K(last_lsn), K(majority_max_accept_pid_), K(majority_max_lsn_)); - } else { - // if server's proposal_id is equal with me, compare with lsn. - if (accept_proposal_id == majority_max_accept_pid_) { - if (last_lsn > majority_max_lsn_) { - majority_max_log_server_ = server; - majority_max_lsn_ = last_lsn; - } - } else if (INVALID_PROPOSAL_ID == majority_max_accept_pid_ - || accept_proposal_id > majority_max_accept_pid_) { - majority_max_lsn_ = last_lsn; - majority_max_accept_pid_ = accept_proposal_id; - majority_max_log_server_ = server; - PALF_LOG(INFO, "the replica's accept_proposal_id is greater than majority_max_accept_pid_", K(ret), - K_(palf_id), K(server), K(src_proposal_id), K(accept_proposal_id), K(last_lsn)); - } else { - // do nothing - } - // record committed_end_lsn of followers - // Try to update its match_lsn to committed_end_lsn, because the server's committed_end_lsn - // is maybe larger than self(advanced by old leader). - // Suppose A is leader with reconfirm state, B, C are followers: - // max_flushed_end_lsn: A(200), B(200), C(200) - // committed_end_lsn: A(100), B(200), C(200) - // For this case, followers' match_lsn at A will be 100, and it has no chance to be updated during reconfirm, - // which will lead to waiting majority sync timeout. - if (committed_end_lsn.is_valid()) { - // old version observer's resp may not have this parameter. - // So it may be invalid. - (void) update_follower_end_lsn_(server, committed_end_lsn); - } else { - PALF_LOG(INFO, "committed_end_lsn in prepare response is invalid, it may be from old version member", - K(ret), K_(palf_id), K(server), K(accept_proposal_id), K(last_lsn), K(committed_end_lsn)); - } - } - PALF_LOG(INFO, "handle_prepare_response finished", K(ret), K_(palf_id), K(server), K(accept_proposal_id), K(last_lsn), - K(majority_max_accept_pid_), K(majority_max_lsn_), K(prepare_log_ack_list_)); - return ret; -} - -int LogReconfirm::update_follower_end_lsn_(const common::ObAddr &server, const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - int64_t index = -1; - if (false == server.is_valid() || false == committed_end_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (-1 != (index = ack_info_list_get_index(follower_end_lsn_list_, server))) { - if (follower_end_lsn_list_[index].last_flushed_end_lsn_ < committed_end_lsn) { - follower_end_lsn_list_[index].last_flushed_end_lsn_ = committed_end_lsn; - } - } else { - LogMemberAckInfo log_info(common::ObMember(server, 1), 0, committed_end_lsn); - follower_end_lsn_list_.push_back(log_info); - } - return ret; -} - -int LogReconfirm::ack_log_with_end_lsn_() -{ - int ret = OB_SUCCESS; - for (int64_t idx = 0; idx < follower_end_lsn_list_.count(); idx++) { - int tmp_ret = OB_SUCCESS; - const common::ObAddr &server = follower_end_lsn_list_.at(idx).member_.get_server(); - const LSN &end_lsn = follower_end_lsn_list_.at(idx).last_flushed_end_lsn_; - if (OB_SUCCESS != (tmp_ret = sw_->ack_log(server, end_lsn))) { - PALF_LOG(WARN, "ack_log failed", K(tmp_ret), K(server), K(end_lsn)); - } - } - return ret; -} - -bool LogReconfirm::is_fetch_log_finished_() -{ - bool bool_ret = false; - LSN max_flushed_end_lsn; - (void) sw_->get_max_flushed_end_lsn(max_flushed_end_lsn); - const LSN max_lsn = sw_->get_max_lsn(); - if (majority_max_lsn_ == max_flushed_end_lsn) { - bool_ret = true; - } else { - //In a scenario with writhing throttling, ensure that the fetched logs are written to disk as soon - //as possible. - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = purge_throttling_())) { - PALF_LOG_RET(WARN, tmp_ret, "purge throttling failed", K_(palf_id), K_(majority_max_lsn), - K(max_flushed_end_lsn), K(max_lsn)); - } - } - - PALF_LOG(TRACE, "is_fetch_log_finished_", K_(palf_id), K(bool_ret), K_(majority_max_lsn), - K(max_flushed_end_lsn), K(max_lsn)); - return bool_ret; -} - -int LogReconfirm::try_fetch_log_() -{ - int ret = OB_SUCCESS; - const int64_t now_us = ObTimeUtility::current_time(); - bool is_fetched = false; - const int64_t sw_start_id = sw_->get_start_id(); - if (OB_INVALID_TIMESTAMP == last_fetch_log_time_us_ - || OB_INVALID_LOG_ID == last_record_sw_start_id_ - || (sw_start_id == last_record_sw_start_id_ - && now_us - last_fetch_log_time_us_ >= PALF_FETCH_LOG_INTERVAL_US)) { - if (OB_FAIL(sw_->try_fetch_log_for_reconfirm(majority_max_log_server_, majority_max_lsn_, is_fetched))) { - PALF_LOG(WARN, "try_fetch_log_for_reconfirm failed", K_(palf_id)); - } else if (is_fetched) { - // send fetch req success - last_record_sw_start_id_ = sw_start_id; - last_fetch_log_time_us_ = now_us; - PALF_LOG(INFO, "try_fetch_log_ success", K(ret), K_(palf_id), K_(majority_max_lsn), - K_(majority_max_log_server), K_(self), K_(last_record_sw_start_id)); - } - } else if (sw_start_id != last_record_sw_start_id_) { - last_record_sw_start_id_ = sw_start_id; - last_fetch_log_time_us_ = now_us; - } else { - PALF_LOG(INFO, "no need fetch log in current round", K(ret), K_(palf_id), K_(majority_max_log_server), - K_(majority_max_lsn), K(sw_start_id), K_(last_record_sw_start_id)); - } - return ret; -} - -bool LogReconfirm::can_receive_log() const -{ - // Self can receive log only in RECONFIRM_FETCH_LOG stage. - // Or it may receive logs with old proposal_id whose lsn is larger than majority_max_lsn. - // These logs are unexpected, which should be dropped and rewritten. - return (RECONFIRM_FETCH_LOG == state_); -} - -bool LogReconfirm::can_do_degrade() const -{ - // degrade operation can be executed after prepare message reaches majority - // do not protect with lock_ - // Arb Thread 1: hold lock_ of ConfigMgr -> can_do_degrade() -> lock_ - // LogLoop Thread: hold lock_ -> confirm_start_working_log() -> lock_ of ConfigMgr - return (state_ > RECONFIRM_FETCH_LOG); -} - -bool LogReconfirm::is_majority_catch_up_() -{ - // This step is necessary, because if we directly go to start_working stage: - // 1) For 2F1A case, the A member can skip prev log check and reply ack faster than F. - // This will lead to an unexpected state: the other F replica lacks logs when - // reconfirm finishes. If leader crashes later, the data of these logs will lost. - // 2) For all F replica case, start_working log need prev log check, so all followers - // also need wait log sync before process. It's similar with this step. - bool bool_ret = false; - int tmp_ret = OB_SUCCESS; - LSN majority_match_lsn; - if (OB_SUCCESS != (tmp_ret = sw_->get_majority_match_lsn(majority_match_lsn))) { - PALF_LOG_RET(WARN, tmp_ret, "get_majority_match_lsn failed", K(tmp_ret), K_(palf_id)); - } else if (!majority_match_lsn.is_valid()) { - PALF_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "majority_match_lsn is invalid, unexpected", K_(palf_id), K(majority_match_lsn)); - } else if (majority_match_lsn < majority_max_lsn_) { - PALF_LOG_RET(WARN, OB_SUCCESS, "majority_match_lsn is smaller than majority_max_lsn_, need wait", K_(palf_id), - K_(majority_max_lsn), K(majority_match_lsn)); - if (palf_reach_time_interval(100 * 1000, last_notify_fetch_time_us_)) { - ObMemberList lagged_list; - if (OB_SUCCESS != (tmp_ret = sw_->get_lagged_member_list(majority_max_lsn_, lagged_list))) { - PALF_LOG_RET(WARN, tmp_ret, "get_lagged_member_list_ failed", K(tmp_ret), K_(palf_id)); - } else if (OB_SUCCESS != (tmp_ret = log_engine_->submit_notify_fetch_log_req(lagged_list))) { - PALF_LOG_RET(WARN, tmp_ret, "submit_notify_fetch_log_req failed", K(tmp_ret), K_(palf_id), K(lagged_list)); - } else { - PALF_LOG(INFO, "notify_fetch_log success", K(tmp_ret), K_(palf_id), K_(majority_max_lsn), K(lagged_list)); - } - } - } else { - bool_ret = true; - } - return bool_ret; -} - -int LogReconfirm::reconfirm() -{ - int ret = OB_SUCCESS; - ObLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogReconfirm is not init", K_(self), K_(palf_id)); - } else { - const int64_t now_us = ObTimeUtility::current_time(); - switch (state_) { - case INITED: { - if (OB_FAIL(init_reconfirm_())) { - PALF_LOG(WARN, "init reconfirm failed", K_(palf_id)); - } else { - state_ = WAITING_LOG_FLUSHED; - PALF_EVENT("Reconfirm come into WAITING_LOG_FLUSHED state", palf_id_, K_(self), K_(majority_cnt), - K_(curr_paxos_follower_list)); - } - if (state_ != WAITING_LOG_FLUSHED) { - break; - } - } - case WAITING_LOG_FLUSHED: { - if (OB_FAIL(purge_throttling_())) { - PALF_LOG(WARN, "purge throttling failed", K_(palf_id)); - } else if (OB_FAIL(wait_all_log_flushed_())) { - PALF_LOG(WARN, "wait_all_log_flushed_ failed", K_(palf_id)); - } else if (OB_FAIL(submit_prepare_log_())) { - PALF_LOG(WARN, "submit_prepare_log_ failed", K_(palf_id)); - } else { - state_ = FETCH_MAX_LOG_LSN; - //reset last_purge_throttling_time_us_to avoid impacting purging throttling during RECONFIRM_FETCH_LOG - last_purge_throttling_time_us_ = OB_INVALID_TIMESTAMP; - PALF_EVENT("Reconfirm come into FETCH_MAX_LOG_LSN state", palf_id_, K_(self), K_(majority_max_accept_pid)); - } - break; - } - case FETCH_MAX_LOG_LSN: { - const int64_t cost_ts = ObTimeUtility::current_time() - last_submit_prepare_req_time_us_; - // NB: prepare_log_ack_list_ include self defaultly. - if (prepare_log_ack_list_.get_count() + 1 < majority_cnt_) { - // not reach majority in RECONFIRM_PREPARE_RETRY_INTERVAL_US, retry PREPARE phase - // because we do not record vote_for, so leader need advance proposal_id for each retry. - if (RECONFIRM_PREPARE_RETRY_INTERVAL_US <= cost_ts) { - ret = submit_prepare_log_(); - PALF_LOG(WARN, "prepare_log_ack_list_ is not majority in expected interval, submit_prepare_log_ again", - K(cost_ts), K(prepare_log_ack_list_), K(majority_cnt_), K_(palf_id)); - } - } else { - state_ = RECONFIRM_MODE_META; - PALF_EVENT("Reconfirm come into RECONFIRM_MODE_META state", palf_id_, K(prepare_log_ack_list_), - K_(majority_max_log_server), K_(majority_max_lsn)); - } - if (state_ != RECONFIRM_MODE_META) { - break; - } - } - case RECONFIRM_MODE_META: { - if (OB_SUCC(mode_mgr_->reconfirm_mode_meta())) { - if (true == mode_mgr_->need_skip_log_barrier()) { - state_ = RECONFIRMING; - PALF_EVENT("Reconfirm come into RECONFIRMING state", palf_id_, K(prepare_log_ack_list_)); - } else { - state_ = RECONFIRM_FETCH_LOG; - PALF_EVENT("Reconfirm come into RECONFIRM_FETCH_LOG state", palf_id_, K(prepare_log_ack_list_)); - } - } else if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "reconfirm_mode_meta failed", K_(palf_id)); - } - if (state_ != RECONFIRM_FETCH_LOG) { - break; - } - } - case RECONFIRM_FETCH_LOG: { - // Try to update its match_lsn to committed_end_lsn, because the server's committed_end_lsn - // is maybe larger than self(advanced by old leader). - // Suppose A is leader with reconfirm state, B, C are followers: - // max_flushed_end_lsn: A(200), B(200), C(200) - // committed_end_lsn: A(100), B(200), C(200) - // For this case, followers' match_lsn at A will be 100, and it has no chance to be updated during reconfirm, - // which will lead to waiting majority sync timeout. - if (OB_FAIL(ack_log_with_end_lsn_())) { - PALF_LOG(WARN, "ack_log_with_end_lsn_", K_(follower_end_lsn_list)); - } else if (majority_max_log_server_ != self_ && !is_fetch_log_finished_()) { - if (OB_FAIL(try_fetch_log_())) { - PALF_LOG(WARN, "try_fetch_log_ failed", K_(palf_id)); - } - } else { - state_ = RECONFIRMING; - PALF_EVENT("Reconfirm come into RECONFIRMING state", palf_id_, K_(self)); - } - // need break because next state requires wlock - break; - } - case RECONFIRMING: { - LSN last_lsn; - LSN last_end_lsn; - LSN majority_match_lsn; - int64_t last_log_proposal_id = INVALID_PROPOSAL_ID; - const bool need_skip_log_barrier = mode_mgr_->need_skip_log_barrier(); - const int64_t leader_epoch = state_mgr_->get_leader_epoch(); - if (false == sw_->check_all_log_has_flushed()) { - PALF_LOG(WARN, "check_all_log_has_flushed failed, need wait", K_(palf_id)); - // Wait majority match lsn catching up with majority_max_lsn_ - } else if (need_skip_log_barrier == false - && false == is_majority_catch_up_()) { - if (palf_reach_time_interval(1 * 1000 * 1000, wait_majority_time_us_)) { - PALF_LOG(INFO, "is_majority_catch_up_ return false, need wait and retry", K(ret), - K_(self), K_(palf_id), K_(majority_max_lsn)); - } - } else if (OB_FAIL(sw_->get_max_flushed_log_info(last_lsn, last_end_lsn, last_log_proposal_id))) { - PALF_LOG(WARN, "get_max_flushed_log_info failed", K_(self), K_(palf_id)); - } else if (OB_FAIL(mm_->confirm_start_working_log(new_proposal_id_, leader_epoch, sw_config_version_)) - && OB_EAGAIN != ret) { - PALF_LOG(WARN, "confirm_start_working_log failed", K_(self), K_(palf_id)); - } else { - // Record the end_lsn at the start of the start_working phase - saved_end_lsn_ = last_end_lsn; - state_ = START_WORKING; - PALF_EVENT("Reconfirm come into START_WORKING state", palf_id_, K(new_proposal_id_), - K(majority_max_log_server_), K(majority_max_lsn_), K(majority_match_lsn), K(last_lsn), - K(last_log_proposal_id), K(last_end_lsn)); - } - if (state_ != START_WORKING) { - break; - } - } - case START_WORKING: { - LSN last_slide_lsn, committed_end_lsn; - int64_t last_slide_log_id = OB_INVALID_LOG_ID; - const bool need_skip_log_barrier = mode_mgr_->need_skip_log_barrier(); - const int64_t leader_epoch = state_mgr_->get_leader_epoch(); - if (OB_SUCC(mm_->confirm_start_working_log(new_proposal_id_, leader_epoch, sw_config_version_))) { - // check_ms_log_committed() checks if a majority is not reached, it will attempt to resend the start_working log - // Resend using the previously retrieved lsn and proposal_id, therefore saved_end_lsn_ also does not need to be updated - const LSN curr_max_lsn = sw_->get_max_lsn(); - if (curr_max_lsn != saved_end_lsn_) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "max_lsn is not equal to saved_end_lsn_, unexpected", K_(palf_id), K_(self), - K(curr_max_lsn), K_(saved_end_lsn)); - } else if (OB_FAIL(sw_->try_advance_committed_end_lsn(saved_end_lsn_))) { - // After start_working reaches a majority, leader advances committed_end_lsn to the previously recorded point (expected to equal the current max_lsn) - } else if (!sw_->is_all_committed_log_slided_out(last_slide_lsn, last_slide_log_id, committed_end_lsn)) { - // There is some log has not slided out, need wait and retry. - if (palf_reach_time_interval(5 * 1000 * 1000, wait_slide_print_time_us_)) { - PALF_LOG(INFO, "There is some log has not slided out, need wait and retry", K(ret), K_(palf_id), - K_(self), K(last_slide_lsn), K(last_slide_log_id), K(committed_end_lsn)); - } - } else { - PALF_EVENT("Reconfirm come into FINISHED state", palf_id_, K_(self), K_(saved_end_lsn)); - state_ = FINISHED; - } - } - if (state_ != FINISHED) { - PALF_LOG(WARN, "wait start work log majority or log slide", K_(palf_id), K(new_proposal_id_), K(majority_max_log_server_), - K(majority_max_lsn_), K(majority_max_accept_pid_)); - break; - } - } - case FINISHED: { - break; - } - default: { - ret = OB_ERR_UNEXPECTED; - break; - } - } - if (state_ == FINISHED) { - ret = OB_SUCCESS; - } else if (OB_SUCCESS != ret && OB_EAGAIN != ret) { - // election leader may has changed, so ret is maybe OB_NOT_MASTER. - PALF_LOG(WARN, "reconfirm failed", K_(palf_id), K_(state)); - } else { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(10 * 1000)) { - PALF_LOG(INFO, "reconfirm waiting retry", K(ret), K_(palf_id), K_(state)); - } - } - } - return ret; -} - -int LogReconfirm::purge_throttling_() -{ - int ret = OB_SUCCESS; - if (palf_reach_time_interval(100 * 1000L, last_purge_throttling_time_us_)) { - if (OB_FAIL(log_engine_->submit_purge_throttling_task(PurgeThrottlingType::PURGE_BY_RECONFIRM))) { - PALF_LOG(WARN, "submit_purge_throttling_task", K_(palf_id)); - } else { - PALF_LOG(INFO, "submit_purge_throttling_task during reconfirming", K_(palf_id)); - } - } - return ret; -} - -} // namespace palf -} // namespace oceanbase diff --git a/src/logservice/palf/log_reconfirm.h b/src/logservice/palf/log_reconfirm.h deleted file mode 100644 index 5956fb191..000000000 --- a/src/logservice/palf/log_reconfirm.h +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_RECONFIRM_ -#define OCEANBASE_LOGSERVICE_LOG_RECONFIRM_ - -#include "lib/lock/ob_spin_lock.h" -#include "common/ob_member_list.h" -#include "log_ack_info.h" -#include "log_simple_member_list.h" -#include "log_define.h" -#include "log_task.h" -#include "log_meta_info.h" - -namespace oceanbase -{ -namespace common -{ -class ObILogAllocator; -} -namespace palf -{ -class LogSlidingWindow; -class LogStateMgr; -class LogConfigMgr; -class LogEngine; -class LogGroupEntry; -class LogModeMgr; - -class LogReconfirm -{ -public: - LogReconfirm(); - virtual ~LogReconfirm() { destroy(); } -public: - virtual int init(const int64_t palf_id, - const ObAddr &self, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - LogConfigMgr *mm, - LogModeMgr *mode_mgr, - LogEngine *log_engine); - virtual void destroy(); - virtual void reset_state(); - virtual bool need_wlock(); - virtual bool can_receive_log() const; - virtual bool can_do_degrade() const; - virtual int reconfirm(); - virtual int handle_prepare_response(const common::ObAddr &server, - const int64_t &src_proposal_id, - const int64_t &accept_proposal_id, - const LSN &last_lsn, - const LSN &committed_end_lsn); - TO_STRING_KV(K_(palf_id), K_(self), "state", state_to_string(state_), K_(new_proposal_id), K_(prepare_log_ack_list), \ - K_(curr_paxos_follower_list), K_(majority_cnt), K_(majority_max_log_server), \ - K_(majority_max_accept_pid), K_(majority_max_lsn), K_(saved_end_lsn), K_(last_submit_prepare_req_time_us), \ - K_(last_fetch_log_time_us), K_(last_record_sw_start_id), K_(last_notify_fetch_time_us), K_(last_purge_throttling_time_us), KP(this)); -private: - int init_reconfirm_(); - int submit_prepare_log_(); - int wait_all_log_flushed_(); - bool is_fetch_log_finished_(); - bool is_confirm_log_finished_() const; - int try_fetch_log_(); - int update_follower_end_lsn_(const common::ObAddr &server, const LSN &committed_end_lsn); - int ack_log_with_end_lsn_(); - bool is_majority_catch_up_(); - int purge_throttling_(); -private: - enum State - { - INITED = 0, - // this state is not correctness guarantee, if you do not wait all log flused, - // consensus protocol still works, because when START_WORKING has committed, - // all previous logs have been confirmed, the only effect is to avoid unnecessary - // pulling logs. - WAITING_LOG_FLUSHED = 1, - // this state is used to query who is the newest server, records it as 'newest_server_', - FETCH_MAX_LOG_LSN = 2, - // this state is used to reconfirm mode_meta to majority. - RECONFIRM_MODE_META = 3, - // this state is used to fetch all logs from the 'newest_server_'. - RECONFIRM_FETCH_LOG = 4, - // this state is used to wait fetch log finished, and then write START_WORKING log. - RECONFIRMING = 5, - // this state is used to wait write START_WORKING finished. - START_WORKING = 6, - FINISHED = 7, - }; - - const char *state_to_string(const State &state) const - { - switch (state) - { - case(State::INITED): return "INITED"; - case(State::WAITING_LOG_FLUSHED): return "WAITING_LOG_FLUSHED"; - case(State::FETCH_MAX_LOG_LSN): return "FETCH_MAX_LOG_LSN"; - case(State::RECONFIRM_FETCH_LOG): return "RECONFIRM_FETCH_LOG"; - case(State::RECONFIRM_MODE_META): return "RECONFIRM_MODE_META"; - case(State::RECONFIRMING): return "RECONFIRMING"; - case(State::START_WORKING): return "START_WORKING"; - case(State::FINISHED): return "FINISHED"; - default: - return "INVALID_RECONFIRM_STATE"; - } - } - - static const int64_t RECONFIRM_PREPARE_RETRY_INTERVAL_US = 2 * 1000 * 1000; // 2s -private: - State state_; - int64_t palf_id_; - common::ObAddr self_; - int64_t new_proposal_id_; - - // - // - // cur fetch id - // │ - // │un committed id│ ▼ - // ┌─────────────┼───────────────┼───────────────────────┐ - // │ │ │ │ - // └─────────────┴───────────────┴───────────────────────┘ - // ▲ ▲ ▲ ▲ - // │ │ │ │ - // start id committed id max flushed id end id - // - // After reconfirm, election leader will be the newest server of majority. - // 'max_lsn_' is max flushed id; - // 'max_log_proposal_id_' is the proposal of 'max_lsn_'; - // 'fetching_lsn_' is current fetch id, will be updated by receive_log; - // end id is the max flushed id of 'majority_max_log_server_'. - // NB: before reconfirm, election leader will behind others too much, - // however, the maximum numbers of uncommitted logs for any replica can - // not exceed the max size of ObGroupBuffer, we just only advance the - // start id of sliding window, the sliding window will not be full. - // - // current lsn for fetching which is inited by local max log flushed offset - // NB: The correctness of using local max flushed log offset as 'fetch_lsn_' - // 1. If replica receives config log, need forward check; - // 2. Only after start working, can submit new log. - // int64_t fetching_lsn_; - - // prepare response server list - LogAckList prepare_log_ack_list_; - // all paxos folowers, not include self - common::ObMemberList curr_paxos_follower_list_; - int64_t majority_cnt_; - // the server whose log is newest among majority - common::ObAddr majority_max_log_server_; - // the accept_proposal_id of that server - int64_t majority_max_accept_pid_; - // the max_lsn of that server - LSN majority_max_lsn_; - // the max lsn when generate start_working log - LSN saved_end_lsn_; - LogMemberAckInfoList follower_end_lsn_list_; - LogConfigVersion sw_config_version_; - - LogSlidingWindow *sw_; - LogStateMgr *state_mgr_; - LogConfigMgr *mm_; - LogModeMgr *mode_mgr_; - LogEngine *log_engine_; - - mutable common::ObSpinLock lock_; - // last time of sending prepare log - int64_t last_submit_prepare_req_time_us_; - // last time of fetching log - int64_t last_fetch_log_time_us_; - int64_t last_record_sw_start_id_; - int64_t wait_slide_print_time_us_; - int64_t wait_majority_time_us_; - int64_t last_notify_fetch_time_us_; - int64_t last_purge_throttling_time_us_; - bool is_inited_; -private: - DISALLOW_COPY_AND_ASSIGN(LogReconfirm); -}; -} // namespace palf -} // namespace oceanbase -#endif //_OCEANBASE_LOGSERVICE_LOG_RECONFIRM_ diff --git a/src/logservice/palf/log_req.cpp b/src/logservice/palf/log_req.cpp deleted file mode 100644 index d547ab8f3..000000000 --- a/src/logservice/palf/log_req.cpp +++ /dev/null @@ -1,865 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_req.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -// ==================== LogPushReq start ======================== -LogPushReq::LogPushReq() - : push_log_type_(PUSH_LOG), - msg_proposal_id_(INVALID_PROPOSAL_ID), - prev_log_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(), - curr_lsn_(), - write_buf_() -{ -} - -LogPushReq::LogPushReq(const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) - : push_log_type_(push_log_type), - msg_proposal_id_(msg_proposal_id), - prev_log_proposal_id_(prev_log_proposal_id), - prev_lsn_(prev_lsn), - curr_lsn_(curr_lsn), - write_buf_(write_buf) -{ -} - -LogPushReq::~LogPushReq() -{ - reset(); -} - -bool LogPushReq::is_valid() const -{ - // NB: prev_lsn may be invalid - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && true == curr_lsn_.is_valid() - && true == write_buf_.is_valid(); -} - -void LogPushReq::reset() -{ - push_log_type_ = PUSH_LOG; - msg_proposal_id_ = INVALID_PROPOSAL_ID; - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - curr_lsn_.reset(); - write_buf_.reset(); -} - -OB_DEF_SERIALIZE(LogPushReq) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || pos < 0 || pos > buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::encode_i16(buf, buf_len, new_pos, push_log_type_)) - || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, msg_proposal_id_)) - || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_log_proposal_id_)) - || OB_FAIL(prev_lsn_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(curr_lsn_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(write_buf_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "LogPushReq serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -OB_DEF_DESERIALIZE(LogPushReq) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || pos < 0 || pos > data_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i16(buf, data_len, new_pos, &push_log_type_)) - || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&msg_proposal_id_))) - || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, reinterpret_cast(&prev_log_proposal_id_))) - || OB_FAIL(prev_lsn_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(curr_lsn_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(write_buf_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "LogPushReq serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(LogPushReq) -{ - int64_t size = 0; - size += serialization::encoded_length_i16(push_log_type_); - size += serialization::encoded_length_i64(msg_proposal_id_); - size += serialization::encoded_length_i64(prev_log_proposal_id_); - size += prev_lsn_.get_serialize_size(); - size += curr_lsn_.get_serialize_size(); - size += write_buf_.get_serialize_size(); - return size; -} -// ================== LogPushReq end ========================= - -// ================== LogPushResp start ====================== -LogPushResp::LogPushResp() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - lsn_() -{ -} - -LogPushResp::LogPushResp(const int64_t &msg_proposal_id, - const LSN &lsn) - : msg_proposal_id_(msg_proposal_id), - lsn_(lsn) -{ -} - -LogPushResp::~LogPushResp() -{ - reset(); -} - -bool LogPushResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && true == lsn_.is_valid(); -} - -void LogPushResp::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogPushResp, msg_proposal_id_, lsn_); -// ================= LogPushResp end ======================= - -// ================= LogFetchReq start ===================== -LogFetchReq::LogFetchReq() - : fetch_type_(FETCH_LOG_FOLLOWER), - msg_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(), - lsn_(), - fetch_log_size_(0), - fetch_log_count_(0) -{ -} - -LogFetchReq::LogFetchReq(const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) - : fetch_type_(fetch_type), - msg_proposal_id_(msg_proposal_id), - prev_lsn_(prev_lsn), - lsn_(lsn), - fetch_log_size_(fetch_log_size), - fetch_log_count_(fetch_log_count), - accepted_mode_pid_(accepted_mode_pid) -{ -} - -LogFetchReq::~LogFetchReq() -{ - reset(); -} - -bool LogFetchReq::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && true == lsn_.is_valid(); -} - -void LogFetchReq::reset() -{ - fetch_type_ = FETCH_LOG_FOLLOWER; - msg_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - lsn_.reset(); - fetch_log_size_ = 0; - fetch_log_count_ = 0; - accepted_mode_pid_ = INVALID_PROPOSAL_ID; -} - -OB_SERIALIZE_MEMBER(LogFetchReq, fetch_type_, msg_proposal_id_, prev_lsn_, lsn_, fetch_log_size_, - fetch_log_count_, accepted_mode_pid_); -// ================= LogFetchReq end ====================== - -NotifyRebuildReq::NotifyRebuildReq() - : base_lsn_(), base_prev_log_info_() -{ -} - -NotifyRebuildReq::NotifyRebuildReq(const LSN &base_lsn, - const LogInfo &base_prev_log_info) - : base_lsn_(base_lsn), base_prev_log_info_(base_prev_log_info) -{ -} - -NotifyRebuildReq::~NotifyRebuildReq() -{ - reset(); -} - -bool NotifyRebuildReq::is_valid() const -{ - return true == base_lsn_.is_valid() && true == base_prev_log_info_.is_valid(); -} - -void NotifyRebuildReq::reset() -{ - base_lsn_.reset(); - base_prev_log_info_.reset(); -} - -OB_SERIALIZE_MEMBER(NotifyRebuildReq, base_lsn_, base_prev_log_info_); - -NotifyFetchLogReq::NotifyFetchLogReq() -{} - -NotifyFetchLogReq::~NotifyFetchLogReq() -{} - -bool NotifyFetchLogReq::is_valid() const -{ - return true; -} - -OB_SERIALIZE_MEMBER(NotifyFetchLogReq); - -// ================= LogPrepareReq start ================= -LogPrepareReq::LogPrepareReq() - : log_proposal_id_(INVALID_PROPOSAL_ID) -{ -} - -LogPrepareReq::LogPrepareReq(const int64_t &log_proposal_id) - : log_proposal_id_(log_proposal_id) -{ -} - -LogPrepareReq::LogPrepareReq(const LogPrepareReq &req) -{ - log_proposal_id_ = req.log_proposal_id_; -} - - -LogPrepareReq::~LogPrepareReq() -{ - reset(); -} - -bool LogPrepareReq::is_valid() const -{ - return INVALID_PROPOSAL_ID != log_proposal_id_; -} - -void LogPrepareReq::reset() -{ - log_proposal_id_ = INVALID_PROPOSAL_ID; -} - -OB_SERIALIZE_MEMBER(LogPrepareReq, log_proposal_id_); -// ================= LogPrepareReq end =================== - -// ================= LogPrepareResp start ================ -LogPrepareResp::LogPrepareResp() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - vote_granted_(false), - log_proposal_id_(INVALID_PROPOSAL_ID), - max_flushed_lsn_(), - log_mode_meta_(), - committed_end_lsn_() -{ -} - -LogPrepareResp::LogPrepareResp( - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LogModeMeta &mode_meta, - const LSN &committed_end_lsn) - : msg_proposal_id_(msg_proposal_id), - vote_granted_(vote_granted), - log_proposal_id_(log_proposal_id), - max_flushed_lsn_(max_flushed_lsn), - log_mode_meta_(mode_meta), - committed_end_lsn_(committed_end_lsn) -{ -} - -LogPrepareResp::~LogPrepareResp() -{ - reset(); -} - -bool LogPrepareResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && INVALID_PROPOSAL_ID != log_proposal_id_ - && true == max_flushed_lsn_.is_valid() - && true == log_mode_meta_.is_valid(); -} - -void LogPrepareResp::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - vote_granted_ = false; - log_proposal_id_ = INVALID_PROPOSAL_ID; - max_flushed_lsn_.reset(); - log_mode_meta_.reset(); - committed_end_lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogPrepareResp, msg_proposal_id_, vote_granted_, log_proposal_id_, \ - max_flushed_lsn_, log_mode_meta_, committed_end_lsn_); - -LogChangeConfigMetaReq::LogChangeConfigMetaReq() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - prev_log_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(), - prev_mode_pid_(INVALID_PROPOSAL_ID), - meta_() -{ -} - -LogChangeConfigMetaReq::LogChangeConfigMetaReq( - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &meta) - : msg_proposal_id_(msg_proposal_id), - prev_log_proposal_id_(prev_log_proposal_id), - prev_lsn_(prev_lsn), - prev_mode_pid_(prev_mode_pid), - meta_(meta) -{ -} - -LogChangeConfigMetaReq::~LogChangeConfigMetaReq() -{ - reset(); -} - -bool LogChangeConfigMetaReq::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ -// && INVALID_PROPOSAL_ID != prev_log_proposal_id_ // For scenarios where there is no write to the log stream, prev_log is invalid -// && prev_lsn_.is_valid() - && meta_.is_valid(); -} - -void LogChangeConfigMetaReq::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - prev_mode_pid_ = INVALID_PROPOSAL_ID; - meta_.reset(); -} - -OB_SERIALIZE_MEMBER(LogChangeConfigMetaReq, msg_proposal_id_, prev_log_proposal_id_, prev_lsn_, - prev_mode_pid_, meta_); - -LogChangeConfigMetaResp::LogChangeConfigMetaResp() - : proposal_id_(INVALID_PROPOSAL_ID), - config_version_() -{ -} - -LogChangeConfigMetaResp::LogChangeConfigMetaResp(const int64_t proposal_id, - const LogConfigVersion &config_version) - : proposal_id_(proposal_id), - config_version_(config_version) -{ -} - -LogChangeConfigMetaResp::~LogChangeConfigMetaResp() -{ - reset(); -} - -bool LogChangeConfigMetaResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != proposal_id_ && config_version_.is_valid(); -} - -void LogChangeConfigMetaResp::reset() -{ - proposal_id_ = INVALID_PROPOSAL_ID; - config_version_.reset(); -} - -OB_SERIALIZE_MEMBER(LogChangeConfigMetaResp, proposal_id_, config_version_); - -// ========== LogChangeModeMetaReq start========= -LogChangeModeMetaReq::LogChangeModeMetaReq() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - meta_(), - is_applied_mode_meta_(false) -{ -} - -LogChangeModeMetaReq::LogChangeModeMetaReq( - const int64_t &msg_proposal_id, - const LogModeMeta &meta, - const bool is_applied_mode_meta) - : msg_proposal_id_(msg_proposal_id), - meta_(meta), - is_applied_mode_meta_(is_applied_mode_meta) -{ -} - -LogChangeModeMetaReq::~LogChangeModeMetaReq() -{ - reset(); -} - -bool LogChangeModeMetaReq::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && meta_.is_valid(); -} - -void LogChangeModeMetaReq::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - meta_.reset(); - is_applied_mode_meta_ = false; -} - -OB_SERIALIZE_MEMBER(LogChangeModeMetaReq, msg_proposal_id_, meta_, is_applied_mode_meta_); -// ========== LogChangeModeMetaReq end========= - -// ========== LogChangeModeMetaResp start========= -LogChangeModeMetaResp::LogChangeModeMetaResp() : msg_proposal_id_(INVALID_PROPOSAL_ID) -{ -} - -LogChangeModeMetaResp::LogChangeModeMetaResp(const int64_t msg_proposal_id) - : msg_proposal_id_(msg_proposal_id) -{ -} - -LogChangeModeMetaResp::~LogChangeModeMetaResp() -{ - reset(); -} - -bool LogChangeModeMetaResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_; -} - -void LogChangeModeMetaResp::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; -} - -OB_SERIALIZE_MEMBER(LogChangeModeMetaResp, msg_proposal_id_); -// ========== LogChangeModeMetaResp end========= - -// ================= LogGetMCStReq start ================ -LogGetMCStReq::LogGetMCStReq() - : config_version_(), need_purge_throttling_(false) -{ -} - -LogGetMCStReq::LogGetMCStReq(const LogConfigVersion &config_version, - const bool need_purge_throttling) - : config_version_(config_version), need_purge_throttling_(need_purge_throttling) -{ -} - -LogGetMCStReq::~LogGetMCStReq() -{ - reset(); -} - -bool LogGetMCStReq::is_valid() const -{ - return config_version_.is_valid(); -} - -void LogGetMCStReq::reset() -{ - config_version_.reset(); - need_purge_throttling_ = false; -} - -OB_SERIALIZE_MEMBER(LogGetMCStReq, config_version_, need_purge_throttling_); -// ================= LogGetMCStReq end ================ - -// ================= LogGetMCStResp start ================ -LogGetMCStResp::LogGetMCStResp() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - max_flushed_end_lsn_(), - is_normal_replica_(false), - need_update_config_meta_(false), - last_slide_log_id_(INT64_MAX) -{ -} - -LogGetMCStResp::LogGetMCStResp(const int64_t &msg_proposal_id, - const LSN &max_flushed_end_lsn, - const bool is_normal_replica, - const bool need_update_config_meta, - const int64_t last_slide_log_id) - : msg_proposal_id_(msg_proposal_id), - max_flushed_end_lsn_(max_flushed_end_lsn), - is_normal_replica_(is_normal_replica), - need_update_config_meta_(need_update_config_meta), - last_slide_log_id_(last_slide_log_id) -{ -} - -LogGetMCStResp::~LogGetMCStResp() -{ - reset(); -} - -bool LogGetMCStResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ && max_flushed_end_lsn_.is_valid(); -} - -void LogGetMCStResp::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - max_flushed_end_lsn_.reset(); - is_normal_replica_ = false; - need_update_config_meta_ = false; - last_slide_log_id_ = INT64_MAX; -} - -OB_SERIALIZE_MEMBER(LogGetMCStResp, msg_proposal_id_, max_flushed_end_lsn_, \ - is_normal_replica_, need_update_config_meta_, last_slide_log_id_); -// ================= LogGetMCStResp end ================ - -// ============= LogLearnerReq begin ============== -LogLearnerReq::LogLearnerReq(): sender_(), req_type_(INVALID_LEARNER_REQ_TYPE) -{ -} - -LogLearnerReq::LogLearnerReq(const LogLearner &sender, - const LogLearnerReqType req_type) - : sender_(sender), - req_type_(req_type) -{ -} - -LogLearnerReq::~LogLearnerReq() -{ - reset(); -} - -bool LogLearnerReq::is_valid() const -{ - bool bool_ret = false; - if (OB_UNLIKELY(req_type_ == INVALID_LEARNER_REQ_TYPE)) { - // pass - } else { - bool_ret = sender_.is_valid(); - } - return bool_ret; -} - -void LogLearnerReq::reset() -{ - sender_.reset(); - req_type_ = INVALID_LEARNER_REQ_TYPE; -} - -OB_SERIALIZE_MEMBER(LogLearnerReq, sender_, req_type_); -// ============= LogLearnerReq end ============== - -// ============= LogRegisterParentReq begin ============== -LogRegisterParentReq::LogRegisterParentReq() : child_(), is_to_leader_(false) -{ -} - -LogRegisterParentReq::LogRegisterParentReq(const LogLearner &child, - const bool is_to_leader) : child_(child), is_to_leader_(is_to_leader) -{ -} - -LogRegisterParentReq::~LogRegisterParentReq() -{ - reset(); -} - -bool LogRegisterParentReq::is_valid() const -{ - return child_.is_valid(); -} - -void LogRegisterParentReq::reset() -{ - child_.reset(); - is_to_leader_ = false; -} -OB_SERIALIZE_MEMBER(LogRegisterParentReq, child_, is_to_leader_); -// ============= LogRegisterParentReq end ============== - -// ============= LogRegisterParentResp begin ============== -LogRegisterParentResp::LogRegisterParentResp() - : parent_(), - candidate_list_(), - reg_ret_(RegisterReturn::INVALID_REG_RET) -{ -} - -LogRegisterParentResp::LogRegisterParentResp(const LogLearner &parent, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) - : parent_(parent), - candidate_list_(candidate_list), - reg_ret_(reg_ret) -{ -} - -LogRegisterParentResp::~LogRegisterParentResp() -{ - reset(); -} - -bool LogRegisterParentResp::is_valid() const -{ - return parent_.is_valid(); -} - -void LogRegisterParentResp::reset() -{ - parent_.reset(); - candidate_list_.reset(); - reg_ret_ = RegisterReturn::INVALID_REG_RET; -} - -OB_SERIALIZE_MEMBER(LogRegisterParentResp, parent_, candidate_list_, reg_ret_); -// ============= LogRegisterParentResp end ============== - -// ============= CommittedInfo begin ============== -CommittedInfo::CommittedInfo() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - prev_log_id_(OB_INVALID_LOG_ID), - prev_log_proposal_id_(INVALID_PROPOSAL_ID), - committed_end_lsn_() -{ -} - -CommittedInfo::CommittedInfo(const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) - : msg_proposal_id_(msg_proposal_id), - prev_log_id_(prev_log_id), - prev_log_proposal_id_(prev_log_proposal_id), - committed_end_lsn_(committed_end_lsn) -{ -} - -CommittedInfo::~CommittedInfo() -{ - reset(); -} - -bool CommittedInfo::is_valid() const -{ - // NB: prev_lsn may be invalid - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && OB_INVALID_LOG_ID != prev_log_id_ - && INVALID_PROPOSAL_ID != prev_log_proposal_id_ - && true == committed_end_lsn_.is_valid(); -} - -void CommittedInfo::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - prev_log_id_ = OB_INVALID_LOG_ID; - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - committed_end_lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(CommittedInfo, msg_proposal_id_, prev_log_id_, \ - prev_log_proposal_id_, committed_end_lsn_); -// ============= CommittedInfo end ============== - -// ================= LogGetStatReq start ================ -LogGetStatReq::LogGetStatReq() - : get_type_(LogGetStatType::INVALID_SYNC_GET_TYPE) -{ -} - -LogGetStatReq::LogGetStatReq(const LogGetStatType get_type) - : get_type_(get_type) -{ -} - -LogGetStatReq::~LogGetStatReq() -{ - reset(); -} - -bool LogGetStatReq::is_valid() const -{ - return get_type_ != LogGetStatType::INVALID_SYNC_GET_TYPE; -} - -void LogGetStatReq::reset() -{ - get_type_ = LogGetStatType::INVALID_SYNC_GET_TYPE; -} - -OB_SERIALIZE_MEMBER(LogGetStatReq, get_type_); -// ================= LogGetStatReq end ================ - -// ================= LogGetStatResp start ================ -LogGetStatResp::LogGetStatResp() - : max_scn_(), end_lsn_() -{ -} - -LogGetStatResp::LogGetStatResp(const share::SCN &max_scn, const LSN &end_lsn) - : max_scn_(max_scn), end_lsn_(end_lsn) -{ -} - -LogGetStatResp::~LogGetStatResp() -{ - reset(); -} - -bool LogGetStatResp::is_valid() const -{ - return true; -} - -void LogGetStatResp::reset() -{ - max_scn_.reset(); - end_lsn_.reset(); -} - -OB_SERIALIZE_MEMBER(LogGetStatResp, max_scn_, end_lsn_); -// ================= LogGetStatResp end ================ - -LogBatchFetchResp::LogBatchFetchResp() - : msg_proposal_id_(INVALID_PROPOSAL_ID), - prev_log_proposal_id_(INVALID_PROPOSAL_ID), - prev_lsn_(), - curr_lsn_(), - write_buf_() -{ -} - -LogBatchFetchResp::LogBatchFetchResp(const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) - : msg_proposal_id_(msg_proposal_id), - prev_log_proposal_id_(prev_log_proposal_id), - prev_lsn_(prev_lsn), - curr_lsn_(curr_lsn), - write_buf_(write_buf) -{ -} - -LogBatchFetchResp::~LogBatchFetchResp() -{ - reset(); -} - -bool LogBatchFetchResp::is_valid() const -{ - return INVALID_PROPOSAL_ID != msg_proposal_id_ - && true == curr_lsn_.is_valid() - && true == write_buf_.is_valid(); -} - -void LogBatchFetchResp::reset() -{ - msg_proposal_id_ = INVALID_PROPOSAL_ID; - prev_log_proposal_id_ = INVALID_PROPOSAL_ID; - prev_lsn_.reset(); - curr_lsn_.reset(); - write_buf_.reset(); -} - -OB_DEF_SERIALIZE(LogBatchFetchResp) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || pos < 0 || pos > buf_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, msg_proposal_id_)) - || OB_FAIL(serialization::encode_i64(buf, buf_len, new_pos, prev_log_proposal_id_)) - || OB_FAIL(prev_lsn_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(curr_lsn_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(write_buf_.serialize(buf, buf_len, new_pos))) { - PALF_LOG(ERROR, "LogBatchFetchResp serialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -OB_DEF_DESERIALIZE(LogBatchFetchResp) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - if (NULL == buf || pos < 0 || pos > data_len) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, - reinterpret_cast(&msg_proposal_id_))) - || OB_FAIL(serialization::decode_i64(buf, data_len, new_pos, - reinterpret_cast(&prev_log_proposal_id_))) - || OB_FAIL(prev_lsn_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(curr_lsn_.deserialize(buf, data_len, new_pos)) - || OB_FAIL(write_buf_.deserialize(buf, data_len, new_pos))) { - PALF_LOG(ERROR, "LogBatchFetchResp deserialize failed", K(ret), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(LogBatchFetchResp) -{ - int64_t size = 0; - size += serialization::encoded_length_i64(msg_proposal_id_); - size += serialization::encoded_length_i64(prev_log_proposal_id_); - size += prev_lsn_.get_serialize_size(); - size += curr_lsn_.get_serialize_size(); - size += write_buf_.get_serialize_size(); - return size; -} - -} // end namespace palf -} // end namespace oceanbase diff --git a/src/logservice/palf/log_req.h b/src/logservice/palf/log_req.h deleted file mode 100644 index 758628b89..000000000 --- a/src/logservice/palf/log_req.h +++ /dev/null @@ -1,493 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_REQ_ -#define OCEANBASE_LOGSERVICE_LOG_REQ_ - -#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION -#include "lib/utility/ob_print_utils.h" // TO_STRING_KV -#include "log_define.h" -#include "log_meta_info.h" -#include "log_learner.h" // LogLearner, LogLearnerList -#include "logservice/palf/lsn.h" // LSN -#include "log_writer_utils.h" // LogWriteBuf -#include "share/rpc/ob_batch_proxy.h" // BatchRPC - -namespace oceanbase -{ -namespace palf -{ -class LogPrepareMeta; - -enum PushLogType -{ - PUSH_LOG = 0, - FETCH_LOG_RESP = 1, - PUSH_LOG_WO_ACK = 2, -}; - -inline const char *push_log_type_2_str(const PushLogType type) -{ -#define EXTRACT_PUSH_LOG_TYPE(type_var) case(type_var): return #type_var - switch(type) - { - EXTRACT_PUSH_LOG_TYPE(PUSH_LOG); - EXTRACT_PUSH_LOG_TYPE(FETCH_LOG_RESP); - EXTRACT_PUSH_LOG_TYPE(PUSH_LOG_WO_ACK); - - default: - return "Invalid Type"; - } -#undef EXTRACT_PUSH_LOG_TYPE -} - -struct LogPushReq { - OB_UNIS_VERSION(1); -public: - LogPushReq(); - LogPushReq(const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - ~LogPushReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV("push_log_type", push_log_type_2_str((PushLogType)push_log_type_), K_(msg_proposal_id), K_(prev_log_proposal_id), - K_(prev_lsn), K_(curr_lsn), K_(write_buf)); - int16_t push_log_type_; - int64_t msg_proposal_id_; - int64_t prev_log_proposal_id_; - LSN prev_lsn_; - // NB: no need record the proposal_id of curr_lsn, we will deserlize - // to LogGroupEntry. - LSN curr_lsn_; - LogWriteBuf write_buf_; -}; - -struct LogBatchPushReq : public LogPushReq, public obcall::ObIFill { - LogBatchPushReq(const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) : LogPushReq(push_log_type, msg_proposal_id, - prev_log_proposal_id, prev_lsn, curr_lsn, write_buf), - ObIFill() {} - int fill_buffer(char* buf, int64_t size, int64_t &filled_size) const override final - { - filled_size = 0; - return serialize(buf, size, filled_size); - } - int64_t get_req_size() const override final - { - return get_serialize_size(); - } -}; - -struct LogPushResp { - OB_UNIS_VERSION(1); -public: - LogPushResp(); - LogPushResp(const int64_t &msg_proposal_id, - const LSN &lsn); - ~LogPushResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(lsn)); - int64_t msg_proposal_id_; - LSN lsn_; -}; - -struct LogBatchPushResp : public LogPushResp, public obcall::ObIFill { - LogBatchPushResp(const int64_t &msg_proposal_id, - const LSN &lsn) - : LogPushResp(msg_proposal_id, lsn), ObIFill() {} - int fill_buffer(char* buf, int64_t size, int64_t &filled_size) const override final - { - filled_size = 0; - return serialize(buf, size, filled_size); - } - int64_t get_req_size() const override final - { - return get_serialize_size(); - } -}; - -enum FetchLogType -{ - FETCH_LOG_FOLLOWER = 0, - FETCH_LOG_LEADER_RECONFIRM = 1, - FETCH_MODE_META = 2, -}; - -inline const char *fetch_type_2_str(const FetchLogType type) -{ -#define EXTRACT_FETCH_TYPE(type_var) case(type_var): return #type_var - switch(type) - { - EXTRACT_FETCH_TYPE(FETCH_LOG_FOLLOWER); - EXTRACT_FETCH_TYPE(FETCH_LOG_LEADER_RECONFIRM); - EXTRACT_FETCH_TYPE(FETCH_MODE_META); - - default: - return "Invalid Type"; - } -#undef EXTRACT_FETCH_TYPE -} - -struct LogFetchReq { - OB_UNIS_VERSION(1); -public: - LogFetchReq(); - // NB: 'prev_lsn' is used to forward check on leader - // because we don't record these fields in LogGroupEntryHeader - LogFetchReq(const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid); - ~LogFetchReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), "fetch_type", fetch_type_2_str((FetchLogType)fetch_type_), K_(prev_lsn), K_(lsn), K_(fetch_log_size), - K_(fetch_log_count), K_(accepted_mode_pid)); - int16_t fetch_type_; - int64_t msg_proposal_id_; - LSN prev_lsn_; - LSN lsn_; - int64_t fetch_log_size_; - int64_t fetch_log_count_; - int64_t accepted_mode_pid_; -}; - -struct LogBatchFetchResp { - OB_UNIS_VERSION(1); -public: - LogBatchFetchResp(); - LogBatchFetchResp(const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf); - ~LogBatchFetchResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(prev_log_proposal_id), K_(prev_lsn), K_(curr_lsn), K_(write_buf)); - int64_t msg_proposal_id_; - int64_t prev_log_proposal_id_; - LSN prev_lsn_; - LSN curr_lsn_; - LogWriteBuf write_buf_; -}; - -struct NotifyRebuildReq { - OB_UNIS_VERSION(1); -public: - NotifyRebuildReq(); - NotifyRebuildReq(const LSN &base_lsn, const LogInfo &base_prev_log_info); - ~NotifyRebuildReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(base_lsn), K_(base_prev_log_info)); - LSN base_lsn_; - LogInfo base_prev_log_info_; -}; - -struct NotifyFetchLogReq { - OB_UNIS_VERSION(1); -public: - NotifyFetchLogReq(); - ~NotifyFetchLogReq(); - bool is_valid() const; - int64_t to_string(char* buf, const int64_t buf_len) const - { - int64_t pos = 0; - return pos; - } -}; - -struct LogPrepareReq { - OB_UNIS_VERSION(1); -public: - LogPrepareReq(); - LogPrepareReq(const int64_t &log_proposal_id); - LogPrepareReq(const LogPrepareReq &req); - ~LogPrepareReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(log_proposal_id)); - int64_t log_proposal_id_; -}; - -struct LogPrepareResp { - OB_UNIS_VERSION(1); -public: - LogPrepareResp(); - LogPrepareResp(const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &max_flushed_lsn, - const LogModeMeta &mode_meta, - const LSN &committed_end_lsn); - ~LogPrepareResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(vote_granted), K_(log_proposal_id), K_(max_flushed_lsn), - K_(log_mode_meta), K_(committed_end_lsn)); - int64_t msg_proposal_id_; - bool vote_granted_; - int64_t log_proposal_id_; - LSN max_flushed_lsn_; - LogModeMeta log_mode_meta_; - LSN committed_end_lsn_; -}; - -struct LogChangeConfigMetaReq { - OB_UNIS_VERSION(1); -public: - LogChangeConfigMetaReq(); - LogChangeConfigMetaReq(const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &meta); - ~LogChangeConfigMetaReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(prev_log_proposal_id), K_(prev_lsn), - K_(prev_mode_pid), K_(meta)); - int64_t msg_proposal_id_; - // Forward check, used to truncate the ghost logs of prev leader. - int64_t prev_log_proposal_id_; - LSN prev_lsn_; - int64_t prev_mode_pid_; - LogConfigMeta meta_; -}; - -struct LogChangeConfigMetaResp { - OB_UNIS_VERSION(1); -public: - LogChangeConfigMetaResp(); - LogChangeConfigMetaResp(const int64_t proposal_id, const LogConfigVersion &config_version); - ~LogChangeConfigMetaResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(proposal_id), K_(config_version)); - int64_t proposal_id_; - LogConfigVersion config_version_; -}; - -struct LogChangeModeMetaReq { - OB_UNIS_VERSION(1); -public: - LogChangeModeMetaReq(); - LogChangeModeMetaReq(const int64_t &msg_proposal_id, - const LogModeMeta &meta, - const bool is_applied_mode_meta); - ~LogChangeModeMetaReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(meta), K_(is_applied_mode_meta)); - int64_t msg_proposal_id_; - LogModeMeta meta_; - bool is_applied_mode_meta_; -}; - -struct LogChangeModeMetaResp { - OB_UNIS_VERSION(1); -public: - LogChangeModeMetaResp(); - LogChangeModeMetaResp(const int64_t proposal_id); - ~LogChangeModeMetaResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id)); - int64_t msg_proposal_id_; -}; - -struct LogGetMCStReq { - OB_UNIS_VERSION(1); -public: - LogGetMCStReq(); - LogGetMCStReq(const LogConfigVersion &config_version, - const bool need_purge_throttling); - ~LogGetMCStReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(config_version), K_(need_purge_throttling)); - LogConfigVersion config_version_; - bool need_purge_throttling_; -}; - -struct LogGetMCStResp { - OB_UNIS_VERSION(1); -public: - LogGetMCStResp(); - LogGetMCStResp(const int64_t &msg_proposal_id, - const LSN &max_flushed_end_lsn, - const bool is_normal_replica, - const bool need_update_config_meta, - const int64_t last_slide_log_id); - ~LogGetMCStResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(max_flushed_end_lsn), K_(is_normal_replica), - K_(need_update_config_meta), K_(last_slide_log_id)); - int64_t msg_proposal_id_; - LSN max_flushed_end_lsn_; - bool is_normal_replica_; - bool need_update_config_meta_; - int64_t last_slide_log_id_; -}; - -enum LogLearnerReqType -{ - INVALID_LEARNER_REQ_TYPE = 0, - RETIRE_PARENT = 1, - RETIRE_CHILD = 2, - KEEPALIVE_REQ = 3, - KEEPALIVE_RESP = 4, -}; - -struct LogLearnerReq -{ - OB_UNIS_VERSION(1); -public: - LogLearnerReq(); - LogLearnerReq(const LogLearner &sender, - const LogLearnerReqType req_type); - ~LogLearnerReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(sender), K_(req_type)); - - // RETIRE_CHILD: old parent of child that send this msg (self) - // KEEPALIVE_REQ: parent that sends this msg (self) - // RETIRE_PARENT: child that send this msg (self) - // KEEPALIVE_RESP: child that sends this msg (self) - LogLearner sender_; - // ReqType - LogLearnerReqType req_type_; -}; - -struct LogRegisterParentReq -{ - OB_UNIS_VERSION(1); -public: - LogRegisterParentReq(); - LogRegisterParentReq(const LogLearner &child, const bool is_to_leader); - ~LogRegisterParentReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(child), K_(is_to_leader)); - - // child that is registering parent - LogLearner child_; - // if this register req is sended to leader, and msg receiver has not been leader, - // it should return REGISTER_NOT_MASTER - bool is_to_leader_; -}; - -enum RegisterReturn -{ - INVALID_REG_RET = 0, - REGISTER_DONE = 1, - REGISTER_CONTINUE = 2, - REGISTER_NOT_MASTER = 3, -}; - -struct LogRegisterParentResp -{ - OB_UNIS_VERSION(1); -public: - LogRegisterParentResp(); - LogRegisterParentResp(const LogLearner &parent, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret); - ~LogRegisterParentResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(parent), K_(candidate_list), K_(reg_ret)); - - // parent itself, for msg pair varification - LogLearner parent_; - // candidate members that can be parent of this replica - LogCandidateList candidate_list_; - // REGISTER_DONE: parent_ has been registered as parent - // REGISTER_CONTINUE: continue sending register_req to candidate_list_ - // REGISTER_NOT_MASTER: this replica has not been leader anymore - RegisterReturn reg_ret_; -}; - -struct CommittedInfo -{ - OB_UNIS_VERSION(1); -public: - CommittedInfo(); - CommittedInfo(const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn); - ~CommittedInfo(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(msg_proposal_id), K_(prev_log_id), - K_(prev_log_proposal_id), K_(committed_end_lsn)); - int64_t msg_proposal_id_; - int64_t prev_log_id_; - int64_t prev_log_proposal_id_; - LSN committed_end_lsn_; -}; - -enum LogGetStatType -{ - INVALID_SYNC_GET_TYPE = 0, - GET_LEADER_MAX_SCN = 1, -}; - -struct LogGetStatReq { - OB_UNIS_VERSION(1); -public: - LogGetStatReq(); - LogGetStatReq(const LogGetStatType get_type); - ~LogGetStatReq(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(get_type)); - int16_t get_type_; -}; - -struct LogGetStatResp { - OB_UNIS_VERSION(1); -public: - LogGetStatResp(); - LogGetStatResp(const share::SCN &max_scn, const LSN &end_lsn); - ~LogGetStatResp(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(max_scn), K_(end_lsn)); - share::SCN max_scn_; - LSN end_lsn_; -}; - -} // end namespace palf -} // end namespace oceanbase - -#endif diff --git a/src/logservice/palf/log_request_handler.cpp b/src/logservice/palf/log_request_handler.cpp deleted file mode 100644 index 7571d52d9..000000000 --- a/src/logservice/palf/log_request_handler.cpp +++ /dev/null @@ -1,497 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_request_handler.h" -#include "palf_handle_impl_guard.h" -#include "palf_env_impl.h" - -namespace oceanbase -{ -namespace palf -{ - -using namespace election; - -LogRequestHandler::LogRequestHandler(IPalfEnvImpl *palf_env_impl) : palf_env_impl_(palf_env_impl) -{ -} - -LogRequestHandler::~LogRequestHandler() -{ - palf_env_impl_ = NULL; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogPushReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - const char *buf = req.write_buf_.write_buf_[0].buf_; - const int64_t buf_len = req.write_buf_.write_buf_[0].buf_len_; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->receive_log(server, - (PushLogType) req.push_log_type_, - req.msg_proposal_id_, - req.prev_lsn_, - req.prev_log_proposal_id_, - req.curr_lsn_, - buf, buf_len))) { - PALF_LOG(TRACE, "PalfHandleImpl receive_log failed", K(ret), K(palf_id), - K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl receive_log success", K(ret), K(palf_id), - K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogPushResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->ack_log(server, req.msg_proposal_id_, - req.lsn_))){ - PALF_LOG(WARN, "PalfHandleImpl ack_log failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl ack_log success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const NotifyRebuildReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_notify_rebuild_req(server, - req.base_lsn_, req.base_prev_log_info_))){ - PALF_LOG(WARN, "PalfHandleImpl handle_notify_rebuild_req failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl handle_notify_rebuild_req success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const NotifyFetchLogReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_notify_fetch_log_req(server))){ - PALF_LOG(WARN, "PalfHandleImpl handle_notify_fetch_log_req failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl handle_notify_fetch_log_req success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const CommittedInfo &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_committed_info(server, - req.msg_proposal_id_, - req.prev_log_id_, - req.prev_log_proposal_id_, - req.committed_end_lsn_))) { - PALF_LOG(WARN, "PalfHandleImpl handle_committed_info failed", K(ret), K(palf_id), - K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl handle_committed_info success", K(ret), K(palf_id), - K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogFetchReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->get_log(server, (FetchLogType) req.fetch_type_, req.msg_proposal_id_, - req.prev_lsn_, req.lsn_, req.fetch_log_size_, req.fetch_log_count_, req.accepted_mode_pid_))) { - PALF_LOG(WARN, "PalfHandleImpl get_log failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl get_log success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogBatchFetchResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - const char *buf = req.write_buf_.write_buf_[0].buf_; - const int64_t buf_len = req.write_buf_.write_buf_[0].buf_len_; - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->receive_batch_log(server, req.msg_proposal_id_, - req.prev_log_proposal_id_, req.prev_lsn_, req.curr_lsn_, buf, buf_len))) { - PALF_LOG(WARN, "PalfHandleImpl receive_batch_log failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl receive_batch_log success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogPrepareReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id,guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_prepare_request(server, req.log_proposal_id_))) { - PALF_LOG(WARN, "PalfHandleImpl handle_prepare_request failed", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl handle_prepare_request success", K(ret), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogPrepareResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_prepare_response(server, req.msg_proposal_id_, - req.vote_granted_, req.log_proposal_id_, req.max_flushed_lsn_, req.committed_end_lsn_, req.log_mode_meta_))) { - PALF_LOG(WARN, "PalfHandleImpl handle_prepare_response failed", K(ret), K(server), - K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "PalfHandleImpl handle_prepare_response success", K(ret), K(server), - K(palf_id), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogChangeConfigMetaReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) - || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->receive_config_log(server, req.msg_proposal_id_, - req.prev_log_proposal_id_, req.prev_lsn_, req.prev_mode_pid_, req.meta_))) { - PALF_LOG(WARN, "receive_config_log failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "receive_config_log success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogChangeConfigMetaResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->ack_config_log(server, req.proposal_id_, req.config_version_))) { - PALF_LOG(WARN, "ack_config_log failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "ack_config_log success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogChangeModeMetaReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) - || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->receive_mode_meta(server, req.msg_proposal_id_, - req.is_applied_mode_meta_, req.meta_))) { - PALF_LOG(WARN, "receive_mode_meta failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(INFO, "receive_mode_meta success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogChangeModeMetaResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->ack_mode_meta(server, req.msg_proposal_id_))) { - PALF_LOG(WARN, "ack_mode_meta failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "ack_mode_meta success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template<> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogLearnerReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_learner_req(req.sender_, req.req_type_))) { - PALF_LOG(WARN, "handle_learner_req failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "handle_learner_req success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template<> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogRegisterParentReq &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_register_parent_req(req.child_, req.is_to_leader_))) { - PALF_LOG(WARN, "handle_register_parent_req failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "handle_register_parent_req success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -template<> -int LogRequestHandler::handle_request( - const int64_t palf_id, - const ObAddr &server, - const LogRegisterParentResp &req) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_register_parent_resp(req.parent_, req.candidate_list_, req.reg_ret_))) { - PALF_LOG(WARN, "handle_register_parent_resp failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(TRACE, "handle_register_parent_resp success", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } - } - return ret; -} - -// [Election Message] handlers removed: single-replica seekdb has no election -// transport, so these request specializations are dead. - -template <> -int LogRequestHandler::handle_sync_request( - const int64_t palf_id, - const ObAddr &server, - const LogGetMCStReq &req, - LogGetMCStResp &resp) -{ - int ret = common::OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else { - IPalfHandleImplGuard guard; - if (false == palf_env_impl_->check_disk_space_enough()) { - resp.is_normal_replica_ = false; - PALF_LOG(WARN, "check_disk_space_enough returns false", K(req), K(resp)); - } else if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (OB_FAIL(guard.get_palf_handle_impl()->handle_config_change_pre_check(server, req, resp))) { - PALF_LOG(WARN, "PalfHandleImpl config_change_pre_check failed", K(ret), K(palf_id), K(server), K(req), KPC(palf_env_impl_)); - } else { - PALF_LOG(INFO, "PalfHandleImpl config_change_pre_check success", K(ret), K(palf_id), K(server), K(req), K(resp), KPC(palf_env_impl_)); - } - } - return ret; -} - -template <> -int LogRequestHandler::handle_sync_request( - const int64_t palf_id, - const ObAddr &server, - const LogGetStatReq &req, - LogGetStatResp &resp) -{ - int ret = common::OB_SUCCESS; - IPalfHandleImplGuard guard; - if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); - } else if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { - PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id)); - } else if (req.get_type_ == LogGetStatType::GET_LEADER_MAX_SCN) { - common::ObRole role = FOLLOWER; - int64_t unused_pid; - bool is_pending_state = true; - if (OB_FAIL(guard.get_palf_handle_impl()->get_role(role, unused_pid, is_pending_state))) { - CLOG_LOG(WARN, "palf_handle get_role failed", K(ret), K(palf_id), K(server)); - } else if ((role != LEADER || true == is_pending_state)) { - ret = OB_NOT_MASTER; - CLOG_LOG(INFO, "i am not leader", K(ret), K(palf_id), K(req), K(role), K(is_pending_state)); - } else { - resp.max_scn_ = guard.get_palf_handle_impl()->get_max_scn(); - resp.end_lsn_ = guard.get_palf_handle_impl()->get_end_lsn(); - CLOG_LOG(TRACE, "get_leader_max_scn success", K(ret), K(palf_id), K(server), K(req), K(resp)); - } - } - return ret; -} - -} // end namespace palf -} // end namespace oceanbase diff --git a/src/logservice/palf/log_request_handler.h b/src/logservice/palf/log_request_handler.h deleted file mode 100644 index ca9f08c13..000000000 --- a/src/logservice/palf/log_request_handler.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_REQUEST_HANDLER_ -#define OCEANBASE_LOGSERVICE_LOG_REQUEST_HANDLER_ - -#include "lib/ob_errno.h" // OB_SUCCESS... -#include - -namespace oceanbase -{ -namespace common -{ -class ObAddr; -} -namespace palf -{ -class IPalfEnvImpl; -class LogRequestHandler -{ -public: - LogRequestHandler(IPalfEnvImpl *palf_env_impl); - ~LogRequestHandler(); - template - int handle_request(const int64_t palf_id, - const common::ObAddr &server, - const ReqType &req); - template - int handle_sync_request(const int64_t palf_id, - const common::ObAddr &server, - const ReqType &req, - RespType &resp); -private: - IPalfEnvImpl *palf_env_impl_; -}; -} // end namespace palf -} // end namespace oceanbase - -#endif diff --git a/src/logservice/palf/log_rpc.cpp b/src/logservice/palf/log_rpc.cpp deleted file mode 100644 index c2de56135..000000000 --- a/src/logservice/palf/log_rpc.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_rpc.h" -namespace oceanbase -{ -using namespace common; -namespace palf -{ -LogRpc::LogRpc() : opt_lock_(), - options_(), - cluster_id_(0), - is_inited_(false) -{ -} - -LogRpc::~LogRpc() -{ - destroy(); -} - -int LogRpc::init(const ObAddr &self, - const int64_t cluster_id) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - } else { - self_ = self; - - cluster_id_ = cluster_id; - is_inited_ = true; - PALF_LOG(INFO, "LogRpc init success", K(self)); - } - return ret; -} - -void LogRpc::destroy() -{ - if (IS_INIT) { - is_inited_ = false; - PALF_LOG(INFO, "LogRpc destroy success"); - } -} - -int LogRpc::update_transport_compress_options(const PalfTransportCompressOptions &compress_opt) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(opt_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "LogRpc not inited"); - } else { - options_ = compress_opt; - PALF_LOG(INFO, "update_transport_compress_options success", K(compress_opt)); - } - return ret; -} - -const PalfTransportCompressOptions& LogRpc::get_compress_opts() const -{ - ObSpinLockGuard guard(opt_lock_); - return options_; -} - -} // end namespace palf -} // end namespace oceanbase diff --git a/src/logservice/palf/log_rpc.h b/src/logservice/palf/log_rpc.h deleted file mode 100644 index 3b98803f2..000000000 --- a/src/logservice/palf/log_rpc.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_RPC_ -#define OCEANBASE_LOGSERVICE_LOG_RPC_ - -#include "lib/ob_errno.h" -#include "lib/utility/ob_macro_utils.h" // IS_NOT_INIT -#include "lib/net/ob_addr.h" // ObAddr -#include "palf_options.h" // PalfTransportCompressOptions - -namespace oceanbase -{ -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} -namespace common -{ -class ObAddr; -} -namespace palf -{ -// NB: in single-replica seekdb inter-replica palf/election RPC never fires. -// The obcall send/recv plumbing (LogRpcProxyV2, processors, packet/macros) has -// been removed. LogRpc is retained only as the wiring shell that holds the -// transport compress options consumed by the rest of palf; all send paths in -// LogNetService are neutered no-ops. -class LogRpc { -public: - LogRpc(); - ~LogRpc(); - int init(const common::ObAddr &self, - const int64_t cluster_id); - void destroy(); - int update_transport_compress_options(const PalfTransportCompressOptions &compress_opt); - const PalfTransportCompressOptions& get_compress_opts() const; - - TO_STRING_KV(K_(self), K_(is_inited)); -private: - ObAddr self_; - mutable ObSpinLock opt_lock_; - PalfTransportCompressOptions options_; - - int64_t cluster_id_; - bool is_inited_; -}; -} // end namespace palf -} // end namespace oceanbase - -#endif diff --git a/src/logservice/palf/log_shared_queue_thread.h b/src/logservice/palf/log_shared_queue_thread.h index c198da36c..43b9592c8 100644 --- a/src/logservice/palf/log_shared_queue_thread.h +++ b/src/logservice/palf/log_shared_queue_thread.h @@ -46,7 +46,7 @@ class LogSharedQueueTh public: static constexpr int64_t THREAD_NUM = 1; static constexpr int64_t MINI_MODE_THREAD_NUM = 1; - static constexpr int64_t MAX_LOG_HANDLE_TASK_NUM = 10 * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER; + static constexpr int64_t MAX_LOG_HANDLE_TASK_NUM = 10; private: class TaskQueue : public common::ObSimpleThreadPool { diff --git a/src/logservice/palf/log_shared_task.cpp b/src/logservice/palf/log_shared_task.cpp index 9c73e92c6..b22d45252 100644 --- a/src/logservice/palf/log_shared_task.cpp +++ b/src/logservice/palf/log_shared_task.cpp @@ -22,8 +22,8 @@ namespace oceanbase { namespace palf { -LogSharedTask::LogSharedTask(const int64_t palf_id,const int64_t palf_epoch) - : palf_id_(palf_id), palf_epoch_(palf_epoch) +LogSharedTask::LogSharedTask(const int64_t palf_epoch) + : palf_epoch_(palf_epoch) {} LogSharedTask::~LogSharedTask() @@ -38,12 +38,11 @@ void LogSharedTask::destroy() void LogSharedTask::reset() { - palf_id_ = INVALID_PALF_ID; palf_epoch_ = -1; } -LogHandleSubmitTask::LogHandleSubmitTask(const int64_t palf_id,const int64_t palf_epoch) - : LogSharedTask(palf_id, palf_epoch) +LogHandleSubmitTask::LogHandleSubmitTask(const int64_t palf_epoch) + : LogSharedTask(palf_epoch) {} LogHandleSubmitTask::~LogHandleSubmitTask() @@ -60,7 +59,7 @@ int LogHandleSubmitTask::do_task(IPalfEnvImpl *palf_env_impl) int64_t palf_epoch = -1; IPalfHandleImplGuard guard; common::ObTimeGuard time_guard("handle submit task", 100 * 1000); - if (OB_FAIL(palf_env_impl->get_palf_handle_impl(palf_id_, guard))) { + if (OB_FAIL(palf_env_impl->get_palf_handle_impl(guard))) { PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret), KPC(this)); } else if (OB_FAIL(guard.get_palf_handle_impl()->get_palf_epoch(palf_epoch))) { PALF_LOG(WARN, "IPalfEnvImpl get_palf_epoch failed", K(ret), KPC(this)); diff --git a/src/logservice/palf/log_shared_task.h b/src/logservice/palf/log_shared_task.h index bedeb384e..4e2798deb 100644 --- a/src/logservice/palf/log_shared_task.h +++ b/src/logservice/palf/log_shared_task.h @@ -47,7 +47,7 @@ inline const char *shared_type_2_str(const LogSharedTaskType type) class LogSharedTask { public: - LogSharedTask(const int64_t palf_id, const int64_t palf_epoch); + explicit LogSharedTask(const int64_t palf_epoch); virtual ~LogSharedTask(); void destroy(); void reset(); @@ -55,10 +55,8 @@ class LogSharedTask virtual void free_this(IPalfEnvImpl *palf_env_impl) = 0; virtual LogSharedTaskType get_shared_task_type() const = 0; VIRTUAL_TO_STRING_KV("BaseClass", "LogSharedTask", - "palf_id", palf_id_, "palf_epoch", palf_epoch_); protected: - int64_t palf_id_; int64_t palf_epoch_; private: DISALLOW_COPY_AND_ASSIGN(LogSharedTask); @@ -67,7 +65,7 @@ class LogSharedTask class LogHandleSubmitTask : public LogSharedTask { public: - LogHandleSubmitTask(const int64_t palf_id, const int64_t palf_epoch); + explicit LogHandleSubmitTask(const int64_t palf_epoch); ~LogHandleSubmitTask() override; int do_task(IPalfEnvImpl *palf_env_impl) override; void free_this(IPalfEnvImpl *palf_env_impl) override; diff --git a/src/logservice/palf/log_simple_member_list.cpp b/src/logservice/palf/log_simple_member_list.cpp deleted file mode 100644 index b7386fd56..000000000 --- a/src/logservice/palf/log_simple_member_list.cpp +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_simple_member_list.h" - -namespace oceanbase -{ -using namespace common; -namespace palf -{ -int LogSimpleMemberList::add_server(const common::ObAddr &server) -{ - int ret = OB_SUCCESS; - bool found = false; - for (int64_t i = 0; i < count_; i++) { - if (server_[i] == server) { - found = true; - } - } - if (!found) { - if (OB_UNLIKELY(count_ >= common::OB_MAX_MEMBER_NUMBER)) { - ret = OB_ERROR_OUT_OF_RANGE; - } else { - server_[count_] = server; - count_++; - } - } - return ret; -} - -bool LogSimpleMemberList::contains(const common::ObAddr &server) const -{ - bool bool_ret = false; - for (int64_t i = 0; i < count_; i++) { - if (server_[i] == server) { - bool_ret = true; - break; - } - } - return bool_ret; -} - -void LogSimpleMemberList::reset() -{ - count_ = 0; - for (int64_t i = 0; i < OB_MAX_MEMBER_NUMBER; i++) { - server_[i].reset(); - } -} - -int64_t LogSimpleMemberList::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - databuff_printf(buf, buf_len , pos, "%ld", static_cast(count_)); - for (int64_t i = 0; i < count_; i++) { - pos += server_[i].to_string(buf+pos, buf_len - pos); - databuff_printf(buf, buf_len, pos, "%s", ";"); - } - return pos; -} - -int LogSimpleMemberList::deep_copy(const common::ObMemberList &member_list) -{ - int ret = OB_SUCCESS; - reset(); - ObAddr server; - for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); ++i) { - if (OB_FAIL(member_list.get_server_by_index(i, server))) { - CLOG_LOG(ERROR, "get_server_by_index failed", K(ret)); - } else if (OB_FAIL(add_server(server))) { - CLOG_LOG(ERROR, "add_server failed", K(ret)); - } - } - if (OB_SUCC(ret) && get_count() != member_list.get_member_number()) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "count not match", K(ret), "dst_count", get_count(), "src_count", - member_list.get_member_number()); - } - return ret; -} - -int LogSimpleMemberList::deep_copy_to(common::ObMemberList &member_list) -{ - int ret = OB_SUCCESS; - member_list.reset(); - for (int64_t i = 0; OB_SUCC(ret) && i < count_; ++i) { - if (OB_FAIL(member_list.add_server(server_[i]))) { - CLOG_LOG(ERROR, "add_server failed", K(ret)); - } - } - if (OB_SUCC(ret) && get_count() != member_list.get_member_number()) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "count not match", K(ret), "src_count", get_count(), "dst_count", - member_list.get_member_number()); - } - return ret; -} - -int LogSimpleMemberList::remove_server(const common::ObAddr &server) -{ - int ret = OB_SUCCESS; - int64_t index = OB_INVALID_INDEX; - if (OB_UNLIKELY(!server.is_valid())) { - ret = OB_INVALID_ARGUMENT; - } else if (count_ <= 0) { - ret = OB_ENTRY_NOT_EXIST; - } - if (OB_SUCC(ret)) { - for (int64_t i = 0; index == OB_INVALID_INDEX && i < count_; ++i) { - if (server_[i] == server) { - index = i; - } - } - if (index == OB_INVALID_INDEX) { - ret = OB_ENTRY_NOT_EXIST; - } - } - if (OB_SUCC(ret)) { - for (int64_t i = index; i < count_ - 1; ++i) { - server_[i] = server_[i + 1]; - } - server_[count_ - 1].reset(); - --count_; - } - return ret; -} - -int LogAckList::add_server(const common::ObAddr &server) -{ - int ret = OB_SUCCESS; - bool found = false; - const int64_t count = get_count(); - if (!server.is_valid()) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "add_server unexpected server"); - } else { - for (int64_t i = 0; i < count; i++) { - if (server_[i] == server) { - found = true; - } - } - if (!found) { - if (count >= ACK_LIST_SERVER_NUM) { - // do nothing - } else { - server_[count] = server; - } - } - PALF_LOG(INFO, "add_server success", K(count), K(server), K(count)); - } - return ret; -} - -void LogAckList::reset() -{ - for (int64_t i = 0; i < ACK_LIST_SERVER_NUM; i++) { - server_[i].reset(); - } -} - -int64_t LogAckList::get_count() const -{ - int64_t count = 0; - for (int64_t i = 0; i < ACK_LIST_SERVER_NUM; i++) { - if (server_[i].is_valid()) { - count++; - } - } - return count; -} - -int64_t LogAckList::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - const int64_t count = get_count(); - databuff_printf(buf, buf_len , pos, "%ld", count); - for (int64_t i = 0; i < count; i++) { - pos += server_[i].to_string(buf+pos, buf_len - pos); - databuff_printf(buf, buf_len, pos, "%s", ";"); - } - return pos; -} -} // namespace palf -} // namespace oceanbase diff --git a/src/logservice/palf/log_simple_member_list.h b/src/logservice/palf/log_simple_member_list.h deleted file mode 100644 index 47a8504e2..000000000 --- a/src/logservice/palf/log_simple_member_list.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_LOG_SIMPLE_MEMBER_LIST_H_ -#define OCEANBASE_LOGSERVICE_LOG_SIMPLE_MEMBER_LIST_H_ - -#include "common/ob_member_list.h" -#include "lib/container/ob_se_array.h" -#include "lib/net/ob_addr.h" -#include "share/ob_define.h" - -namespace oceanbase -{ -namespace palf -{ -class LogSimpleMemberList final -{ -public: - LogSimpleMemberList() { reset(); } -public: - int add_server(const common::ObAddr &server); - bool contains(const common::ObAddr &server) const; - int64_t get_count() const { return ATOMIC_LOAD(&count_); } - void reset(); - int64_t to_string(char *buf, const int64_t buf_len) const; - int deep_copy(const common::ObMemberList &member_list); - int deep_copy_to(common::ObMemberList &member_list); - int remove_server(const common::ObAddr &server); -private: - int8_t count_; - common::ObAddr server_[common::OB_MAX_MEMBER_NUMBER]; -}; - -typedef common::ObSEArray LogSimpleMemberListArray; - -class LogAckList -{ -public: - LogAckList() { reset(); } - ~LogAckList() {} -public: - int add_server(const common::ObAddr &server); - int64_t get_count() const; - void reset(); - int64_t to_string(char *buf, const int64_t buf_len) const; -private: - const static int64_t ACK_LIST_SERVER_NUM = MAX(1, common::OB_MAX_MEMBER_NUMBER / 2); - common::ObAddr server_[ACK_LIST_SERVER_NUM]; -}; -} // namespace palf -} // namespace oceanbase - -#endif // OCEANBASE_CLOG_OB_SIMPLE_MEMBER_LIST_H_ diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp index 227db823f..4faa59bae 100644 --- a/src/logservice/palf/log_sliding_window.cpp +++ b/src/logservice/palf/log_sliding_window.cpp @@ -18,7 +18,7 @@ #include "log_sliding_window.h" #include "log_engine.h" #include "log_io_task_cb_utils.h" -#include "log_config_mgr.h" +#include "log_state_mgr.h" #include "log_mode_mgr.h" namespace oceanbase { @@ -26,91 +26,35 @@ using namespace share; namespace palf { -bool UpdateMatchLsnFunc::operator()(const common::ObAddr &server, LsnTsInfo &value) -{ - bool bool_ret = true; - if (!value.is_valid()) { - bool_ret = false; - } else if (value.lsn_ <= new_end_lsn_) { - old_end_lsn_ = value.lsn_; - old_advance_time_us_ = value.last_advance_time_us_; - if (value.lsn_ < new_end_lsn_) { - // Update last_advance_time_us_ when lsn really changes. - value.last_advance_time_us_ = new_ack_time_us_; - value.last_ack_time_us_ = new_ack_time_us_; - } - value.lsn_ = new_end_lsn_; - bool_ret = true; - } - return bool_ret; -} - -bool GetLaggedListFunc::operator()(const common::ObAddr &server, LsnTsInfo &value) -{ - bool bool_ret = true; - int tmp_ret = OB_SUCCESS; - if (!value.is_valid()) { - // skip - } else if (value.lsn_ >= dst_lsn_) { - // skip - } else if (OB_SUCCESS != (tmp_ret = lagged_list_.add_server(server))){ - PALF_LOG_RET(ERROR, tmp_ret, "lagged_list_.add_server failed", K(tmp_ret)); - } - return bool_ret; -} - LogSlidingWindow::LogSlidingWindow() : self_(), sw_(), checksum_(), state_mgr_(NULL), - mm_(NULL), mode_mgr_(NULL), log_engine_(NULL), - plugins_(NULL), lsn_allocator_(), group_buffer_(), last_submit_info_lock_(common::ObLatchIds::PALF_SW_SUBMIT_INFO_LOCK), last_submit_lsn_(), last_submit_end_lsn_(), last_submit_log_id_(OB_INVALID_LOG_ID), - last_submit_log_pid_(INVALID_PROPOSAL_ID), max_flushed_info_lock_(), max_flushed_lsn_(), max_flushed_end_lsn_(), - max_flushed_log_pid_(INVALID_PROPOSAL_ID), committed_end_lsn_(), last_slide_info_lock_(common::ObLatchIds::PALF_SW_SLIDE_INFO_LOCK), last_slide_log_id_(OB_INVALID_LOG_ID), last_slide_scn_(), last_slide_lsn_(), - last_slide_log_pid_(INVALID_PROPOSAL_ID), last_slide_log_accum_checksum_(-1), - fetch_info_lock_(common::ObLatchIds::PALF_SW_FETCH_INFO_LOCK), - last_fetch_req_time_(0), - last_fetch_end_lsn_(), - last_fetch_max_log_id_(OB_INVALID_LOG_ID), - last_fetch_committed_end_lsn_(), - last_fetch_trigger_type_(FetchTriggerType::LOG_LOOP_TH), - match_lsn_map_lock_(common::ObLatchIds::PALF_SW_MATCH_LSN_MAP_LOCK), - match_lsn_map_(), - last_truncate_lsn_(), - cannot_fetch_log_warn_time_(OB_INVALID_TIMESTAMP), cannot_freeze_log_warn_time_(OB_INVALID_TIMESTAMP), larger_log_warn_time_(OB_INVALID_TIMESTAMP), log_life_long_warn_time_(OB_INVALID_TIMESTAMP), - lc_cb_get_warn_time_(OB_INVALID_TIMESTAMP), - fetch_failure_print_time_(OB_INVALID_TIMESTAMP), commit_log_handling_lease_(), submit_log_handling_lease_(), - last_fetch_log_renew_leader_ts_us_(OB_INVALID_TIMESTAMP), end_lsn_stat_time_us_(OB_INVALID_TIMESTAMP), - reconfirm_fetch_dest_(), - is_truncating_(false), - is_rebuilding_(false), - last_rebuild_lsn_(), last_record_end_lsn_(PALF_INITIAL_LSN_VAL), - push_log_rpc_post_cost_stat_("[PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME]", PALF_STAT_PRINT_INTERVAL_US), fs_cb_cost_stat_("[PALF STAT FS CB EXCUTE COST TIME]", PALF_STAT_PRINT_INTERVAL_US), log_life_time_stat_("[PALF STAT LOG LIFE TIME]", PALF_STAT_PRINT_INTERVAL_US), accum_slide_log_cnt_(0), @@ -132,89 +76,66 @@ LogSlidingWindow::LogSlidingWindow() void LogSlidingWindow::destroy() { is_inited_ = false; - is_truncating_ = false; - is_rebuilding_ = false; - last_rebuild_lsn_.reset(); int tmp_ret = OB_SUCCESS; sw_.destroy(); group_buffer_.destroy(); - match_lsn_map_.destroy(); - reconfirm_fetch_dest_.reset(); state_mgr_ = NULL; log_engine_ = NULL; - plugins_ = NULL; - mm_ = NULL; mode_mgr_ = NULL; } -int LogSlidingWindow::init(const int64_t palf_id, - const common::ObAddr &self, +int LogSlidingWindow::init(const common::ObAddr &self, LogStateMgr *state_mgr, - LogConfigMgr *mm, LogModeMgr *mode_mgr, LogEngine *log_engine, palf::PalfFSCbWrapper *palf_fs_cb, common::ObILogAllocator *alloc_mgr, - LogPlugins *plugins, - const PalfBaseInfo &palf_base_info, - const bool is_normal_replica) + const PalfBaseInfo &palf_base_info) { int ret = OB_SUCCESS; const LogInfo &prev_log_info = palf_base_info.prev_log_info_; if (is_inited_) { ret = OB_INIT_TWICE; - } else if (false == is_valid_palf_id(palf_id) - || false == self.is_valid() + } else if (false == self.is_valid() || false == palf_base_info.is_valid() || NULL == state_mgr - || NULL == mm || NULL == mode_mgr || NULL == log_engine - || NULL == palf_fs_cb - || NULL == plugins) { + || NULL == palf_fs_cb) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K(palf_id), K(self), K(palf_base_info), - KP(state_mgr), KP(mm), KP(mode_mgr), KP(log_engine), KP(palf_fs_cb), KP(plugins)); - } else if (is_normal_replica && OB_FAIL(do_init_mem_(palf_id, palf_base_info, alloc_mgr))) { - PALF_LOG(WARN, "do_init_mem_ failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K(self), K(palf_base_info), + KP(state_mgr), KP(mode_mgr), KP(log_engine), KP(palf_fs_cb)); + } else if (OB_FAIL(do_init_mem_(palf_base_info, alloc_mgr))) { + PALF_LOG(WARN, "do_init_mem_ failed", K(ret)); } else { - palf_id_ = palf_id; self_ = self; state_mgr_ = state_mgr; - mm_ = mm; mode_mgr_ = mode_mgr; log_engine_ = log_engine; palf_fs_cb_ = palf_fs_cb; - plugins_ = plugins; last_submit_lsn_ = prev_log_info.lsn_; last_submit_end_lsn_ = palf_base_info.curr_lsn_; last_submit_log_id_ = prev_log_info.log_id_; - last_submit_log_pid_ = prev_log_info.log_proposal_id_; max_flushed_lsn_ = prev_log_info.lsn_; max_flushed_end_lsn_ = palf_base_info.curr_lsn_; - max_flushed_log_pid_ = prev_log_info.log_proposal_id_; last_slide_log_id_ = prev_log_info.log_id_; last_slide_scn_ = prev_log_info.scn_; last_slide_lsn_ = prev_log_info.lsn_; last_slide_end_lsn_ = palf_base_info.curr_lsn_; - last_slide_log_pid_ = prev_log_info.log_proposal_id_; last_slide_log_accum_checksum_ = prev_log_info.accum_checksum_; committed_end_lsn_ = palf_base_info.curr_lsn_; MEMSET(append_cnt_array_, 0, APPEND_CNT_ARRAY_SIZE * sizeof(int64_t)); - PALF_REPORT_INFO_KV(K_(palf_id)); - fs_cb_cost_stat_.set_extra_info(EXTRA_INFOS); - log_life_time_stat_.set_extra_info(EXTRA_INFOS); is_inited_ = true; LogGroupEntryHeader group_header; LogEntryHeader log_header; - PALF_LOG(INFO, "sw init success", K(ret), K_(palf_id), K_(self), K(palf_base_info), + PALF_LOG(INFO, "sw init success", K(ret), K_(self), K(palf_base_info), "group header size", LogGroupEntryHeader::HEADER_SER_SIZE, "log entry size", LogEntryHeader::HEADER_SER_SIZE, "group_header ser size", group_header.get_serialize_size(), "log header ser size", log_header.get_serialize_size()); @@ -226,23 +147,20 @@ int LogSlidingWindow::init(const int64_t palf_id, return ret; } -int LogSlidingWindow::do_init_mem_(const int64_t palf_id, - const PalfBaseInfo &palf_base_info, +int LogSlidingWindow::do_init_mem_(const PalfBaseInfo &palf_base_info, common::ObILogAllocator *alloc_mgr) { int ret = OB_SUCCESS; const LogInfo &prev_log_info = palf_base_info.prev_log_info_; if (OB_FAIL(sw_.init(prev_log_info.log_id_ + 1, PALF_SLIDING_WINDOW_SIZE, alloc_mgr))) { - PALF_LOG(WARN, "sw init failed", K(ret), K(palf_id), K(palf_base_info)); + PALF_LOG(WARN, "sw init failed", K(ret), K(palf_base_info)); } else if (OB_FAIL(lsn_allocator_.init(prev_log_info.log_id_, prev_log_info.scn_, palf_base_info.curr_lsn_))) { - PALF_LOG(WARN, "lsn_allocator_ init failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "lsn_allocator_ init failed", K(ret)); } else if (OB_FAIL(group_buffer_.init(palf_base_info.curr_lsn_))) { - PALF_LOG(WARN, "group_buffer_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(checksum_.init(palf_id, prev_log_info.accum_checksum_))) { - PALF_LOG(WARN, "checksum_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(match_lsn_map_.init("MatchLsnMap"))) { - PALF_LOG(WARN, "match_lsn_map_ init failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "group_buffer_ init failed", K(ret)); + } else if (OB_FAIL(checksum_.init(prev_log_info.accum_checksum_))) { + PALF_LOG(WARN, "checksum_ init failed", K(ret)); } return ret; } @@ -258,42 +176,36 @@ bool LogSlidingWindow::can_receive_larger_log_(const int64_t log_id) const // because it is updated after the last slid log_task's ref_cnt decrease to 0. bool_ret = false; if (palf_reach_time_interval(5 * 1000 * 1000, larger_log_warn_time_)) { - PALF_LOG(INFO, "sw is full, cannot submit larger log", K_(palf_id), K_(self), K(start_log_id), \ + PALF_LOG(INFO, "sw is full, cannot submit larger log", K_(self), K(start_log_id), \ K(sw_end_log_id), K(log_id)); } } return bool_ret; } -bool LogSlidingWindow::leader_can_submit_larger_log_(const int64_t log_id) const +bool LogSlidingWindow::can_submit_larger_log_(const int64_t log_id) const { - // Call this interface when leader submits a new log + // Bound the number of in-flight local logs. bool bool_ret = true; const int64_t start_log_id = get_start_id(); // sw_end_log_id may be less than (start_log_id + PALF_SLIDING_WINDOW_SIZE), // because it is updated after the last slid log_task's ref_cnt decrease to 0. const int64_t sw_end_log_id = sw_.get_end_sn(); - if (log_id - start_log_id >= PALF_MAX_LEADER_SUBMIT_LOG_COUNT + if (log_id - start_log_id >= PALF_MAX_SUBMIT_LOG_COUNT || log_id >= sw_end_log_id) { - // should guarantee: - // 1. sliding window in follower - sliding window size in leader > 2, otherwise - // logs in follower may not be slided, because the log which committed_end_lsn - // can commit the first log may be out of sliding window. - // 2. max number of concurrent submitting group log in leader better be half of - // sliding window size in follower. If not, some logs may be rejected by follower - // because sliding window is full. + // The local sliding window is full. bool_ret = false; if (palf_reach_time_interval(5 * 1000 * 1000, larger_log_warn_time_)) { - PALF_LOG(INFO, "sw is full, cannot submit larger log", K_(palf_id), K_(self), K(start_log_id), \ + PALF_LOG(INFO, "sw is full, cannot submit larger log", K_(self), K(start_log_id), \ K(sw_end_log_id), K(log_id)); } } return bool_ret; } -bool LogSlidingWindow::leader_can_submit_new_log_(const int64_t valid_log_size, LSN &lsn_upper_bound) +bool LogSlidingWindow::can_submit_new_log_(const int64_t valid_log_size, LSN &lsn_upper_bound) { - // Check whether leader can submit new log. + // Check whether the local writer can submit a new log. // The valid_log_size does not consider group_header for generating new group log case. bool bool_ret = false; int tmp_ret = OB_SUCCESS; @@ -308,12 +220,11 @@ bool LogSlidingWindow::leader_can_submit_new_log_(const int64_t valid_log_size, lsn_upper_bound = reuse_base_lsn + group_buffer_size; if (OB_SUCCESS != (tmp_ret = lsn_allocator_.get_curr_end_lsn(curr_end_lsn))) { - PALF_LOG_RET(WARN, tmp_ret, "get_curr_end_lsn failed", K(tmp_ret), K_(palf_id), K_(self), K(valid_log_size)); - // NB: Use committed_lsn as the lower bound of the reusable starting point to avoid reusing group_buffer immediately after writing to disk, which leads to the follower's - // group_buffer is filled with uncommitted logs and cannot slide out + PALF_LOG_RET(WARN, tmp_ret, "get_curr_end_lsn failed", K(tmp_ret), K_(self), K(valid_log_size)); + // Use committed_lsn as the lower bound of the reusable starting point. } else if (!group_buffer_.can_handle_new_log(curr_end_lsn, valid_log_size, curr_committed_end_lsn)) { if (REACH_TIME_INTERVAL(1000 * 1000)) { - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "group_buffer_ cannot handle new log now", K(tmp_ret), K_(palf_id), K_(self), + PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "group_buffer_ cannot handle new log now", K(tmp_ret), K_(self), K(valid_log_size), K(curr_end_lsn), K(curr_committed_end_lsn), "start_id", get_start_id(), "max_log_id", get_max_log_id()); } @@ -323,28 +234,7 @@ bool LogSlidingWindow::leader_can_submit_new_log_(const int64_t valid_log_size, return bool_ret; } -bool LogSlidingWindow::leader_can_submit_group_log_(const LSN &lsn, const int64_t group_log_size) -{ - // Check whether leader can submit new group log. - bool bool_ret = false; - int tmp_ret = OB_SUCCESS; - LSN curr_committed_end_lsn; - get_committed_end_lsn_(curr_committed_end_lsn); - // NB: Use committed_lsn as the lower bound of the reusable starting point to avoid reusing group_buffer immediately after writing to disk, which leads to follower's - // group_buffer is filled with uncommitted logs and cannot slide out - if (!group_buffer_.can_handle_new_log(lsn, group_log_size, curr_committed_end_lsn)) { - if (REACH_TIME_INTERVAL(1000 * 1000)) { - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "group_buffer_ cannot handle new log now", K(tmp_ret), K_(palf_id), K_(self), - K(lsn), K(group_log_size), K(curr_committed_end_lsn), - "start_id", get_start_id(), "max_log_id", get_max_log_id()); - } - } else { - bool_ret = true; - } - return bool_ret; -} - -int LogSlidingWindow::leader_wait_sw_slot_ready_(const int64_t log_id) +int LogSlidingWindow::wait_sw_slot_ready_(const int64_t log_id) { // wait for sw slot ready // Because log_id is allocated when sw might be full @@ -352,15 +242,14 @@ int LogSlidingWindow::leader_wait_sw_slot_ready_(const int64_t log_id) LogTask *log_task = NULL; LogTaskGuard guard(this); do { - if (false == leader_can_submit_larger_log_(log_id)) { - // double check log_id whether exceeds leader sw's range limit, - // This step is necessary because multiple threads may concurrently submit using the same max_log_id through the pre-check + if (false == can_submit_larger_log_(log_id)) { + // Multiple submitters may pass the pre-check with the same max_log_id. ret = OB_EAGAIN; } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { if (OB_ERR_OUT_OF_UPPER_BOUND == ret) { ret = OB_EAGAIN; } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(self), K(log_id)); } } else { // get success, end loop @@ -390,28 +279,28 @@ int LogSlidingWindow::submit_log(const char *buf, // group log valid size (without padding part) const int64_t valid_log_size = LogEntryHeader::HEADER_SER_SIZE + buf_len; const int64_t start_log_id = get_start_id(); - const int64_t log_id_upper_bound = start_log_id + PALF_MAX_LEADER_SUBMIT_LOG_COUNT - 1; + const int64_t log_id_upper_bound = start_log_id + PALF_MAX_SUBMIT_LOG_COUNT - 1; LSN tmp_lsn, lsn_upper_bound; if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BODY_SIZE || (!ref_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(buf_len), KP(buf)); - } else if (!leader_can_submit_new_log_(valid_log_size, lsn_upper_bound) - || !leader_can_submit_larger_log_(get_max_log_id() + 1)) { + PALF_LOG(WARN, "invalid arguments", K(ret), K_(self), K(buf_len), KP(buf)); + } else if (!can_submit_new_log_(valid_log_size, lsn_upper_bound) + || !can_submit_larger_log_(get_max_log_id() + 1)) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(1000 * 1000)) { - PALF_LOG(ERROR, "cannot submit new log now, try again", K(ret), K_(palf_id), K_(self), + PALF_LOG(ERROR, "cannot submit new log now, try again", K(ret), K_(self), K(valid_log_size), K(buf_len), "start_id", get_start_id(), "max_log_id", get_max_log_id()); } // sw_ cannot submit larger log } else if (OB_FAIL(lsn_allocator_.alloc_lsn_scn(ref_scn, valid_log_size, log_id_upper_bound, lsn_upper_bound, tmp_lsn, log_id, scn, is_new_log, need_gen_padding_entry, padding_size))) { - PALF_LOG(WARN, "alloc_lsn_scn failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(leader_wait_sw_slot_ready_(log_id))) { - PALF_LOG(WARN, "leader_wait_sw_slot_ready_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(WARN, "alloc_lsn_scn failed", K(ret), K_(self)); + } else if (OB_FAIL(wait_sw_slot_ready_(log_id))) { + PALF_LOG(WARN, "wait_sw_slot_ready_ failed", K(ret), K_(self), K(log_id)); } else { - PALF_LOG(TRACE, "alloc_lsn_scn success", K(ret), K_(palf_id), K_(self), K(tmp_lsn), K(scn), + PALF_LOG(TRACE, "alloc_lsn_scn success", K(ret), K_(self), K(tmp_lsn), K(scn), K(log_id), K(valid_log_size), K(is_new_log), K(need_gen_padding_entry), K(padding_size)); bool is_need_handle_next = false; bool is_need_handle = false; @@ -420,16 +309,16 @@ int LogSlidingWindow::submit_log(const char *buf, const int64_t padding_entry_body_size = padding_size - LogGroupEntryHeader::HEADER_SER_SIZE; if (OB_FAIL(try_freeze_prev_log_(log_id, tmp_lsn, is_need_handle))) { // try freeze previous log - PALF_LOG(ERROR, "try_freeze_prev_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id), K(tmp_lsn), + PALF_LOG(ERROR, "try_freeze_prev_log_ failed", K(ret), K_(self), K(log_id), K(tmp_lsn), K(padding_size), K(is_new_log), K(valid_log_size)); } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { } else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, padding_entry_body_size, LOG_PADDING, \ NULL, padding_entry_body_size, is_need_handle))) { - PALF_LOG(ERROR, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id), K(tmp_lsn), K(padding_size), + PALF_LOG(ERROR, "generate_new_group_log_ failed", K(ret), K_(self), K(log_id), K(tmp_lsn), K(padding_size), K(is_new_log), K(valid_log_size)); } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { } else { - PALF_LOG(INFO, "generate_new_group_log_ for padding log success", K_(palf_id), K_(self), K(log_id), + PALF_LOG(INFO, "generate_new_group_log_ for padding log success", K_(self), K(log_id), K(padding_size), K(tmp_lsn), K(scn), K(is_need_handle), K(is_need_handle_next)); // after gen padding_entry, update lsn to next block tmp_lsn.val_ += padding_size; @@ -444,14 +333,14 @@ int LogSlidingWindow::submit_log(const char *buf, // output lsn does not contains log_group_entry_header lsn.val_ += LogGroupEntryHeader::HEADER_SER_SIZE; if (OB_FAIL(try_freeze_prev_log_(log_id, tmp_lsn, is_need_handle))) { - PALF_LOG(WARN, "try_freeze_prev_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(WARN, "try_freeze_prev_log_ failed", K(ret), K_(self), K(log_id)); } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { } else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, valid_log_size, LOG_SUBMIT, \ buf, buf_len, is_need_handle))) { - PALF_LOG(WARN, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(WARN, "generate_new_group_log_ failed", K(ret), K_(self), K(log_id)); } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { } else { - PALF_LOG(TRACE, "generate_new_group_log_ success", K_(palf_id), K_(self), K(log_id), K(lsn), K(scn), + PALF_LOG(TRACE, "generate_new_group_log_ success", K_(self), K(log_id), K(lsn), K(scn), K(valid_log_size), K(is_need_handle), K(is_need_handle_next)); int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = try_feedback_freeze_log_task_(log_id))) { @@ -461,10 +350,10 @@ int LogSlidingWindow::submit_log(const char *buf, } else { // this log need to be appended to last log if (OB_FAIL(append_to_group_log_(lsn, log_id, scn, valid_log_size, buf, buf_len, is_need_handle))) { - PALF_LOG(WARN, "append_to_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(WARN, "append_to_group_log_ failed", K(ret), K_(self), K(log_id)); } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { } else { - PALF_LOG(TRACE, "append_to_group_log_ success", K_(palf_id), K_(self), K(log_id), K(lsn), K(scn), + PALF_LOG(TRACE, "append_to_group_log_ success", K_(self), K(log_id), K(lsn), K(scn), K(valid_log_size), K(is_need_handle), K(is_need_handle_next)); } } @@ -491,10 +380,10 @@ int LogSlidingWindow::try_freeze_prev_log_(const int64_t next_log_id, const LSN is_need_handle = false; if (OB_INVALID_LOG_ID == next_log_id || !lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(next_log_id), K(lsn)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(next_log_id), K(lsn)); } else if (FIRST_VALID_LOG_ID == next_log_id) { // prev log_id is 0, skip - PALF_LOG(INFO, "next log_id is 1, no need freeze prev log", K_(palf_id), K_(self), K(next_log_id), K(lsn)); + PALF_LOG(INFO, "next log_id is 1, no need freeze prev log", K_(self), K(next_log_id), K(lsn)); } else { const int64_t log_id = next_log_id - 1; LogTask *log_task = NULL; @@ -504,7 +393,7 @@ int LogSlidingWindow::try_freeze_prev_log_(const int64_t next_log_id, const LSN // this log has slide out, ignore ret = OB_SUCCESS; } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(palf_id), K_(self), K(log_id)); + PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(self), K(log_id)); } } else { log_task->lock(); @@ -512,7 +401,7 @@ int LogSlidingWindow::try_freeze_prev_log_(const int64_t next_log_id, const LSN // Setting end_lsn for prev log_task, in case it can be freezed later by itself but not here. log_task->set_end_lsn(lsn); PALF_LOG(INFO, "log_task is invalid, its first log may has not filled, set end_lsn and skip freeze", - K(ret), K(log_id), K_(palf_id), K_(self), KPC(log_task)); + K(ret), K(log_id), K_(self), KPC(log_task)); } else { log_task->try_freeze(lsn); } @@ -530,7 +419,7 @@ int LogSlidingWindow::try_freeze_prev_log_(const int64_t next_log_id, const LSN int LogSlidingWindow::wait_group_buffer_ready_(const LSN &lsn, const int64_t data_len) { int ret = OB_SUCCESS; - // NB: Although 'committed_end_lsn_' has been used to limit 'leader_can_submit_new_log_', we still need to determine if 'group_buffer_' can be reused: + // NB: Although 'committed_end_lsn_' has been used to limit 'can_submit_new_log_', we still need to determine if 'group_buffer_' can be reused: // 1. Concurrent submission of logs will result in all logs entering the submission process; // 2. Cannot use 'committed_end_lsn_' to determine if 'group_buffer_' can be reused, because 'committed_end_lsn_' may be greater than 'max_flushed_end_lsn'. int64_t wait_times = 0; @@ -546,7 +435,7 @@ int LogSlidingWindow::wait_group_buffer_ready_(const LSN &lsn, const int64_t dat sleep_us = MAX_SLEEP_US; } ob_usleep(sleep_us); - PALF_LOG(WARN, "usleep wait", K_(palf_id), K_(self), K(lsn), K(data_len), K(curr_committed_end_lsn)); + PALF_LOG(WARN, "usleep wait", K_(self), K(lsn), K(data_len), K(curr_committed_end_lsn)); get_committed_end_lsn_(curr_committed_end_lsn); } return ret; @@ -567,10 +456,10 @@ int LogSlidingWindow::append_to_group_log_(const LSN &lsn, if (!lsn.is_valid() || !scn.is_valid() || OB_INVALID_LOG_ID == log_id || log_entry_size <= 0 || NULL == log_data || data_len <= 0) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), K(scn), K(log_id), K(log_entry_size), + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(lsn), K(scn), K(log_id), K(log_entry_size), KP(log_data), K(data_len)); } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - PALF_LOG(WARN, "get_log_task_ failed", K(ret), K(log_id), K_(palf_id), K_(self)); + PALF_LOG(WARN, "get_log_task_ failed", K(ret), K(log_id), K_(self)); } else { // Note: There is no need to check if log_task is valid here, because in the concurrent submit scenario, the first log_entry may not have updated log_task yet LogEntryHeader log_entry_header; @@ -584,15 +473,15 @@ int LogSlidingWindow::append_to_group_log_(const LSN &lsn, char tmp_buf[TMP_HEADER_SER_BUF_LEN]; // wait group buffer ready if (OB_FAIL(wait_group_buffer_ready_(lsn, log_entry_size))) { - PALF_LOG(ERROR, "group_buffer wait failed", K(ret), K_(palf_id), K_(self), K(lsn), K(log_entry_size)); + PALF_LOG(ERROR, "group_buffer wait failed", K(ret), K_(self), K(lsn), K(log_entry_size)); } else if (OB_FAIL(group_buffer_.fill(log_entry_data_lsn, log_data, data_len))) { - PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(self)); } else if (OB_FAIL(log_entry_header.generate_header(log_data, data_len, scn))) { - PALF_LOG(WARN, "genearate header failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "genearate header failed", K(ret), K_(self)); } else if (OB_FAIL(log_entry_header.serialize(tmp_buf, TMP_HEADER_SER_BUF_LEN, pos))) { - PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(self)); } else if (OB_FAIL(group_buffer_.fill(lsn, tmp_buf, pos))) { - PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(self)); } else { assert(LogEntryHeader::HEADER_SER_SIZE == pos); // inc ref by log_entry_size(LOG_HEADER_SIZE + date_len) @@ -624,14 +513,13 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, || LOG_UNKNOWN == log_type || (LOG_PADDING != log_type && (NULL == log_data || data_len <= 0))) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), K(scn), K(log_id), K(log_body_size), + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(lsn), K(scn), K(log_id), K(log_body_size), K(log_type), KP(log_data), K(data_len)); } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - PALF_LOG(ERROR, "get_log_task_ failed", K(ret), K(log_id), K_(palf_id), K_(self), "start_log_id", get_start_id(), "max_log_id", get_max_log_id()); + PALF_LOG(ERROR, "get_log_task_ failed", K(ret), K(log_id), K_(self), "start_log_id", get_start_id(), "max_log_id", get_max_log_id()); } else { LogEntryHeader log_entry_header; LogGroupEntryHeader header; - const int64_t proposal_id = state_mgr_->get_proposal_id(); const bool is_padding_log = (LOG_PADDING == log_type); LogTaskHeaderInfo header_info; @@ -641,16 +529,14 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, header_info.min_scn_= scn; header_info.max_scn_ = scn; header_info.data_len_ = log_body_size; - header_info.proposal_id_ = proposal_id; - header_info.is_raw_write_ = false; log_task->lock(); if (log_task->is_valid()) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_task is valid, unexpected", K(ret), K(log_id), K_(palf_id), K_(self), K(lsn), K(scn), + PALF_LOG(ERROR, "log_task is valid, unexpected", K(ret), K(log_id), K_(self), K(lsn), K(scn), K(log_body_size), K(log_type), K(data_len), KPC(log_task)); } else if (OB_FAIL(log_task->set_initial_header_info(header_info))) { - PALF_LOG(WARN, "set_initial_header_info failed", K(ret), K_(palf_id), K_(self), K(log_id), KPC(log_task)); + PALF_LOG(WARN, "set_initial_header_info failed", K(ret), K_(self), K(log_id), KPC(log_task)); } else { // The first log is responsible to try freezing self, if its end_lsn_ has been set by next log. log_task->try_freeze_by_myself(); @@ -660,7 +546,7 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, if (OB_SUCC(ret)) { const LSN log_entry_data_lsn = lsn + LogGroupEntryHeader::HEADER_SER_SIZE + LogEntryHeader::HEADER_SER_SIZE; if (OB_FAIL(wait_group_buffer_ready_(lsn, log_body_size + LogGroupEntryHeader::HEADER_SER_SIZE))) { - PALF_LOG(ERROR, "group_buffer wait failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "group_buffer wait failed", K(ret), K_(self)); } else if (is_padding_log) { const int64_t padding_log_body_size = log_body_size - LogEntryHeader::HEADER_SER_SIZE; const int64_t padding_valid_data_len = LogEntryHeader::PADDING_LOG_ENTRY_SIZE; @@ -673,12 +559,12 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, char padding_valid_data[padding_valid_data_len]; memset(padding_valid_data, 0, padding_valid_data_len); if (OB_FAIL(LogEntryHeader::generate_padding_log_buf(padding_log_body_size, scn, padding_valid_data, padding_valid_data_len))) { - PALF_LOG(ERROR, "generate_padding_log_buf failed", K_(palf_id), K_(self), K(padding_valid_data_len), + PALF_LOG(ERROR, "generate_padding_log_buf failed", K_(self), K(padding_valid_data_len), K(scn), K(padding_log_body_size)); } // padding log, fill log body with PADDING_LOG_CONTENT_CHAR. else if (OB_FAIL(group_buffer_.fill_padding_body(lsn + LogGroupEntryHeader::HEADER_SER_SIZE, padding_valid_data, padding_valid_data_len, log_body_size))) { - PALF_LOG(WARN, "group_buffer fill_padding_body failed", K(ret), K_(palf_id), K_(self), K(log_body_size)); + PALF_LOG(WARN, "group_buffer fill_padding_body failed", K(ret), K_(self), K(log_body_size)); } else { // inc ref log_task->ref(log_body_size); @@ -690,13 +576,13 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, assert(LogEntryHeader::HEADER_SER_SIZE < TMP_HEADER_SER_BUF_LEN); char tmp_buf[TMP_HEADER_SER_BUF_LEN]; if (OB_FAIL(group_buffer_.fill(log_entry_data_lsn, log_data, data_len))) { - PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(self)); } else if (OB_FAIL(log_entry_header.generate_header(log_data, data_len, scn))) { - PALF_LOG(WARN, "genearate header failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "genearate header failed", K(ret), K_(self)); } else if (OB_FAIL(log_entry_header.serialize(tmp_buf, TMP_HEADER_SER_BUF_LEN, pos))) { - PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(self)); } else if (OB_FAIL(group_buffer_.fill(lsn + LogGroupEntryHeader::HEADER_SER_SIZE, tmp_buf, pos))) { - PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(self)); } else { assert(LogEntryHeader::HEADER_SER_SIZE == pos); log_task->ref(log_body_size); @@ -724,7 +610,7 @@ int LogSlidingWindow::handle_committed_log_() LSN committed_end_lsn; if (is_all_committed_log_slided_out_(unused_lsn, unused_id, unused_start_lsn, committed_end_lsn)) { // all logs have slided out, no need continue - PALF_LOG(TRACE, "is_all_committed_log_slided_out_ returns true", K_(palf_id), K_(self), + PALF_LOG(TRACE, "is_all_committed_log_slided_out_ returns true", K_(self), K(committed_end_lsn)); } else { LSN max_flushed_end_lsn; @@ -761,7 +647,7 @@ int LogSlidingWindow::handle_committed_log_() log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + data_len; log_task->unlock(); - PALF_LOG(TRACE, "handle_committed_log", K_(palf_id), K_(self), K(log_end_lsn), K(committed_end_lsn), + PALF_LOG(TRACE, "handle_committed_log", K_(self), K(log_end_lsn), K(committed_end_lsn), K(max_flushed_end_lsn), K(tmp_log_id), KPC(log_task), K(need_check_next), "can_slide_sw", state_mgr_->can_slide_sw()); if (is_freezed @@ -772,7 +658,7 @@ int LogSlidingWindow::handle_committed_log_() if (OB_FAIL(sw_.slide(PALF_MAX_REPLAY_TIMEOUT, this))) { // slide failed, reset tag (void) log_task->reset_pre_slide(); - PALF_LOG(ERROR, "sw slide failed", K_(palf_id), K_(self), K(ret), K(tmp_log_id), KPC(log_task)); + PALF_LOG(ERROR, "sw slide failed", K_(self), K(ret), K(tmp_log_id), KPC(log_task)); } else { // pop successfully, check next log need_check_next = true; @@ -787,63 +673,6 @@ int LogSlidingWindow::handle_committed_log_() return ret; } -int LogSlidingWindow::try_push_log_to_paxos_follower_(const int64_t curr_proposal_id, - const int64_t prev_log_pid, - const LSN &prev_lsn, - const LSN &lsn, - const LogWriteBuf &log_write_buf) -{ - int ret = OB_SUCCESS; - ObMemberList dst_member_list; - int64_t replica_num = 0; - const bool need_send_log = (state_mgr_->is_leader_active()) ? true : false; - const bool is_fetch_log = false; - const bool need_batch_push = need_use_batch_rpc_(log_write_buf.get_total_size(), is_fetch_log); - if (false == need_send_log) { - // no need send log to paxos follower - } else if (OB_FAIL(mm_->get_log_sync_member_list(dst_member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(dst_member_list.remove_server(self_))) { - PALF_LOG(WARN, "dst_member_list remove_server failed", K(ret), K_(palf_id), K_(self)); - } else if (dst_member_list.is_valid() - && OB_FAIL(log_engine_->submit_push_log_req(dst_member_list, PUSH_LOG, curr_proposal_id, - prev_log_pid, prev_lsn, lsn, log_write_buf, need_batch_push))) { - PALF_LOG(WARN, "submit_push_log_req failed", K(ret), K_(palf_id), K_(self)); - } else { - // do nothing - } - return ret; -} - -int LogSlidingWindow::try_push_log_to_children_(const int64_t curr_proposal_id, - const int64_t prev_log_pid, - const LSN &prev_lsn, - const LSN &lsn, - const LogWriteBuf &log_write_buf) -{ - int ret = OB_SUCCESS; - LogLearnerList children_list; - common::GlobalLearnerList degraded_learner_list; - const bool need_presend_log = (state_mgr_->is_leader_active()) ? true : false; - const bool is_fetch_log = false; - const bool need_batch_push = need_use_batch_rpc_(log_write_buf.get_total_size(), is_fetch_log); - const PushLogType to_child_log_type = PUSH_LOG_WO_ACK; - if (OB_FAIL(mm_->get_log_sync_children_list(children_list))) { - PALF_LOG(WARN, "get_children_list failed", K(ret), K_(palf_id)); - } else if (children_list.is_valid() - && OB_FAIL(log_engine_->submit_push_log_req(children_list, to_child_log_type, curr_proposal_id, - prev_log_pid, prev_lsn, lsn, log_write_buf, need_batch_push))) { - PALF_LOG(WARN, "submit_push_log_req failed", K(ret), K_(palf_id), K_(self)); - } else if (false == need_presend_log) { - } else if (OB_FAIL(mm_->get_degraded_learner_list(degraded_learner_list))) { - PALF_LOG(WARN, "get_degraded_learner_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_UNLIKELY(degraded_learner_list.is_valid() && mm_->is_sync_to_degraded_learners())) { - (void) log_engine_->submit_push_log_req(degraded_learner_list, PUSH_LOG, - curr_proposal_id, prev_log_pid, prev_lsn, lsn, log_write_buf, need_batch_push); - } - return ret; -} - int LogSlidingWindow::try_handle_next_submit_log() { int ret = OB_SUCCESS; @@ -884,10 +713,9 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) LSN last_submit_lsn; LSN last_submit_end_lsn; int64_t last_submit_log_id = OB_INVALID_LOG_ID; - int64_t last_submit_log_pid = INVALID_PROPOSAL_ID; - (void) get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id, last_submit_log_pid); + get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id); const int64_t tmp_log_id = last_submit_log_id + 1; - PALF_LOG(TRACE, "handle_next_submit_log_ begin", K(ret), K_(palf_id), K_(self), K(tmp_log_id), K(last_submit_log_id)); + PALF_LOG(TRACE, "handle_next_submit_log_ begin", K(ret), K_(self), K(tmp_log_id), K(last_submit_log_id)); SCN scn; LogTask *log_task = NULL; LogTaskGuard guard(this); @@ -899,7 +727,6 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) } else { LSN begin_lsn; LSN log_end_lsn; - int64_t log_proposal_id = INVALID_PROPOSAL_ID; bool is_need_submit = false; bool is_submitted = false; // log count of this group log @@ -912,7 +739,6 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) is_need_submit = log_task->try_pre_submit(); begin_lsn = log_task->get_begin_lsn(); log_end_lsn = begin_lsn + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); - log_proposal_id = log_task->get_proposal_id(); log_cnt = log_task->get_log_cnt(); log_task->unlock(); @@ -924,104 +750,60 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) if (is_need_submit) { // generate group_entry_header log_task->lock(); - // Check the prev_proposal_id before submit this log. - // Because there is maybe a gap when this replica receives this log. const LSN prev_lsn = log_task->get_prev_lsn(); - const int64_t prev_log_pid = log_task->get_prev_proposal_id(); log_task->unlock(); if (OB_UNLIKELY(last_submit_end_lsn != begin_lsn)) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "Current log's begin_lsn is not continuous with last_submit_end_lsn, unexpected", - K_(palf_id), K_(self), K(prev_lsn), K(last_submit_log_id), K(last_submit_lsn), - K(last_submit_end_lsn), K(last_submit_log_pid), K(tmp_log_id), KPC(log_task)); - } else if (!state_mgr_->is_leader_active() - && tmp_log_id > FIRST_VALID_LOG_ID // the first log doesn't need check prev_proposal_id - && (last_submit_lsn != prev_lsn || last_submit_log_pid != prev_log_pid)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "prev_proposal_id does not match, cannot submit this log", K_(palf_id), K_(self), - K(prev_lsn), K(prev_log_pid), K(last_submit_log_id), K(last_submit_lsn), K(last_submit_log_pid), - K(tmp_log_id), KPC(log_task)); + K_(self), K(prev_lsn), K(last_submit_log_id), K(last_submit_lsn), + K(last_submit_end_lsn), K(tmp_log_id), KPC(log_task)); } else if (OB_FAIL(generate_group_entry_header_(tmp_log_id, log_task, group_entry_header, group_log_data_checksum, is_accum_checksum_acquired))) { - PALF_LOG(WARN, "generate_group_entry_header_ failed", K_(palf_id), K_(self)); + PALF_LOG(WARN, "generate_group_entry_header_ failed", K_(self)); } else { log_task->lock(); - if (!state_mgr_->is_follower_active()) { - // Updating data_checksum, accum_checksum, committed_end_lsn for log_task. - // Active follower can skip because it has done this in receive_log(). - log_task->set_group_log_checksum(group_log_data_checksum); - if (OB_FAIL(log_task->update_header_info(group_entry_header.get_committed_end_lsn(), - group_entry_header.get_accum_checksum()))) { - PALF_LOG(WARN, "update_header_info failed", K(ret), K_(palf_id), K_(self), K(group_entry_header)); - } + log_task->set_group_log_checksum(group_log_data_checksum); + if (OB_FAIL(log_task->update_header_info(group_entry_header.get_committed_end_lsn(), + group_entry_header.get_accum_checksum()))) { + PALF_LOG(WARN, "update_header_info failed", K(ret), K_(self), K(group_entry_header)); } scn = log_task->get_min_scn(); log_task->unlock(); } } else { - PALF_LOG(TRACE, "cannot submit this log, break loop", K(ret), K_(palf_id), K_(self), K(tmp_log_id), KPC(log_task)); + PALF_LOG(TRACE, "cannot submit this log, break loop", K(ret), K_(self), K(tmp_log_id), KPC(log_task)); break; } // serialize group_entry_header without log_task's lock if (OB_SUCC(ret) && is_need_submit) { int64_t pos = 0; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); const int64_t group_entry_size = LogGroupEntryHeader::HEADER_SER_SIZE + group_entry_header.get_data_len(); FlushLogCbCtx flush_log_cb_ctx; flush_log_cb_ctx.log_id_ = tmp_log_id; flush_log_cb_ctx.scn_ = scn; flush_log_cb_ctx.lsn_ = begin_lsn; - flush_log_cb_ctx.curr_proposal_id_ = curr_proposal_id; - flush_log_cb_ctx.log_proposal_id_ = log_proposal_id; flush_log_cb_ctx.total_len_ = group_entry_size; flush_log_cb_ctx.begin_ts_ = ObTimeUtility::current_time(); LogWriteBuf log_write_buf; assert(LogGroupEntryHeader::HEADER_SER_SIZE < TMP_HEADER_SER_BUF_LEN); char tmp_buf[TMP_HEADER_SER_BUF_LEN]; - // Active follower no need serialize group header to group_buffer, it has done this in receive_log(). - const bool need_serialize_header = (state_mgr_->is_follower_active()) ? false : true; - if (OB_FAIL(need_serialize_header - && group_entry_header.serialize(tmp_buf, TMP_HEADER_SER_BUF_LEN, pos))) { - PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(palf_id), K_(self)); - } else if (need_serialize_header - && OB_FAIL(group_buffer_.fill(begin_lsn, tmp_buf, pos))) { - PALF_LOG(WARN, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); + if (OB_FAIL(group_entry_header.serialize(tmp_buf, TMP_HEADER_SER_BUF_LEN, pos))) { + PALF_LOG(WARN, "serialize log_entry_header failed", K(ret), K_(self)); + } else if (OB_FAIL(group_buffer_.fill(begin_lsn, tmp_buf, pos))) { + PALF_LOG(WARN, "fill group buffer failed", K(ret), K_(self)); } else if (OB_FAIL(group_buffer_.get_log_buf(begin_lsn, group_entry_size, log_write_buf))) { - PALF_LOG(WARN, "get log buffer failed", K(ret), K_(palf_id), K_(self)); - } else { - // Try push log to follower/children. - // NB: Sending log before writing to disk, or log_write_buf may be free during sending. - // Using tmp_ret to avoid handling failure because of rpc exception. - int tmp_ret = OB_SUCCESS; - // Push log to paxos follower. - int64_t rpc_begin_ts = ObTimeUtility::current_time(); - if (OB_SUCCESS != (tmp_ret = try_push_log_to_paxos_follower_(curr_proposal_id, - last_submit_log_pid, last_submit_lsn, begin_lsn, log_write_buf))) { - PALF_LOG(WARN, "try_push_log_to_paxos_follower_ failed", K(tmp_ret), K_(palf_id), K_(self)); - } - int64_t rpc_post_cost = ObTimeUtility::current_time() - rpc_begin_ts; - push_log_rpc_post_cost_stat_.stat(rpc_post_cost); - if (rpc_post_cost > 100 * 1000) { - PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "push log to followers rpc post cost too much time", K(tmp_ret), K_(palf_id), - K_(self), K(rpc_post_cost), K(tmp_log_id), K(begin_lsn), K(curr_proposal_id), K(log_proposal_id)); - } - // Push log to children_list. - if (OB_SUCCESS != (tmp_ret = try_push_log_to_children_(curr_proposal_id, - last_submit_log_pid, last_submit_lsn, begin_lsn, log_write_buf))) { - PALF_LOG(WARN, "try_push_log_to_children_ failed", K(tmp_ret), K_(palf_id)); - } + PALF_LOG(WARN, "get log buffer failed", K(ret), K_(self)); } log_task->set_submit_ts(ObTimeUtility::current_time()); if (OB_FAIL(ret)) { } else if (OB_FAIL(log_engine_->submit_flush_log_task(flush_log_cb_ctx, log_write_buf))) { - PALF_LOG(WARN, "submit_flush_log_task failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "submit_flush_log_task failed", K(ret), K_(self)); } else { is_submitted = true; // statistics info for group log - const common::ObRole role = state_mgr_->get_role(); const int64_t total_log_cnt = ATOMIC_AAF(&accum_log_cnt_, log_cnt); const int64_t total_group_log_size = ATOMIC_AAF(&accum_group_log_size_, group_log_size); if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, group_log_stat_time_us_)) { @@ -1029,7 +811,7 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) if (total_group_log_cnt > 0) { const int64_t avg_log_batch_cnt = total_log_cnt / total_group_log_cnt; const int64_t avg_group_log_size = total_group_log_size / total_group_log_cnt; - PALF_LOG(INFO, "[PALF STAT GROUP LOG INFO]", K_(palf_id), K_(self), "role", role_to_string(role), + PALF_LOG(INFO, "[PALF STAT GROUP LOG INFO]", K_(self), K(total_group_log_cnt), K(avg_log_batch_cnt), K(total_group_log_size), K(avg_group_log_size)); } ATOMIC_STORE(&accum_log_cnt_, 0); @@ -1037,22 +819,7 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) ATOMIC_STORE(&last_record_group_log_id_, tmp_log_id); } // submit success, update last_submit_log info - (void) set_last_submit_log_info_(begin_lsn, log_end_lsn, tmp_log_id, \ - group_entry_header.get_log_proposal_id()); - if (FOLLOWER == state_mgr_->get_role() || state_mgr_->is_leader_reconfirm()) { - bool is_need_fetch = false; - const LSN log_committed_end_lsn = group_entry_header.get_committed_end_lsn(); - try_update_committed_lsn_for_fetch_(log_end_lsn, tmp_log_id, \ - log_committed_end_lsn, is_need_fetch); - if (is_need_fetch) { - try_fetch_log_streamingly_(log_committed_end_lsn); - } - // Advance committed_end_lsn_, follower/reconfirm leader needs - // the order with try_update_committed_lsn_for_fetch_() is important, - // this exec order can avoid trigger failure of next round fetch by sliding_cb() - (void) try_advance_committed_lsn_(log_committed_end_lsn); - is_committed_lsn_updated = true; - } + (void) set_last_submit_log_info_(begin_lsn, log_end_lsn, tmp_log_id); } } if (is_need_submit && !is_submitted) { @@ -1062,12 +829,12 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) // rollcack accum_checksum if (is_accum_checksum_acquired && OB_FAIL(checksum_.rollback_accum_checksum(group_entry_header.get_accum_checksum()))) { - PALF_LOG(ERROR, "rollback_accum_checksum failed", K(ret), K_(palf_id), K_(self), KPC(log_task), + PALF_LOG(ERROR, "rollback_accum_checksum failed", K(ret), K_(self), KPC(log_task), K(group_entry_header)); } - PALF_LOG(ERROR, "submit log failed", K(ret), K_(palf_id), K_(self), KPC(log_task), K(is_accum_checksum_acquired)); + PALF_LOG(ERROR, "submit log failed", K(ret), K_(self), KPC(log_task), K(is_accum_checksum_acquired)); } - PALF_LOG(TRACE, "handle one submit log", K(ret), K_(palf_id), K_(self), K(tmp_log_id), K(is_committed_lsn_updated), + PALF_LOG(TRACE, "handle one submit log", K(ret), K_(self), K(tmp_log_id), K(is_committed_lsn_updated), K(is_need_submit), K(is_submitted)); } is_lease_expired = is_handle_thread_lease_expired(thread_lease_begin_ts); @@ -1082,7 +849,7 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) int tmp_ret = OB_SUCCESS; while (OB_TMP_FAIL(log_engine_->submit_handle_submit_task())) { if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "submit_handle_submit_task failed", K(tmp_ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "submit_handle_submit_task failed", K(tmp_ret), K_(self)); } if (OB_IN_STOP_STATE == tmp_ret) { // The thread pool has been stopped, no need retry. @@ -1110,7 +877,7 @@ int LogSlidingWindow::generate_group_entry_header_(const int64_t log_id, if (OB_INVALID_LOG_ID == log_id || NULL == log_task) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(log_id), KPC(log_task)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(log_id), KPC(log_task)); } else { LSN global_committed_end_lsn; get_committed_end_lsn_(global_committed_end_lsn); @@ -1119,38 +886,35 @@ int LogSlidingWindow::generate_group_entry_header_(const int64_t log_id, header_info = log_task->get_header_info(); log_task->unlock(); const bool is_padding_log = header_info.is_padding_log_; - const bool is_raw_write = header_info.is_raw_write_; const LSN begin_lsn = header_info.begin_lsn_; LSN log_committed_end_lsn = header_info.committed_end_lsn_; if (!log_committed_end_lsn.is_valid()) { - // leader generates new log entries through this path - // follower directly use committed_end_lsn_ in log_task to generate group header + // New local entries use the current committed boundary. log_committed_end_lsn = global_committed_end_lsn; } const int64_t data_len = header_info.data_len_; const SCN max_scn = header_info.max_scn_; - const int64_t log_proposal_id = header_info.proposal_id_; const int64_t group_entry_size = LogGroupEntryHeader::HEADER_SER_SIZE + data_len; LogWriteBuf log_write_buf; int64_t accum_checksum = 0; if (log_committed_end_lsn > begin_lsn) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_committed_end_lsn is larger than begin_lsn", K(ret), K_(palf_id), K_(self), K(global_committed_end_lsn), + PALF_LOG(ERROR, "log_committed_end_lsn is larger than begin_lsn", K(ret), K_(self), K(global_committed_end_lsn), K(header_info)); } else if (OB_FAIL(group_buffer_.get_log_buf(begin_lsn, group_entry_size, log_write_buf))) { - PALF_LOG(WARN, "get log buffer failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(group_header.generate(is_raw_write, is_padding_log, log_write_buf, data_len, max_scn, - log_id, log_committed_end_lsn, log_proposal_id, group_log_data_checksum))) { - PALF_LOG(WARN, "group_header generate failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "get log buffer failed", K(ret), K_(self)); + } else if (OB_FAIL(group_header.generate(is_padding_log, log_write_buf, data_len, max_scn, + log_id, log_committed_end_lsn, group_log_data_checksum))) { + PALF_LOG(WARN, "group_header generate failed", K(ret), K_(self)); } else if (OB_FAIL(checksum_.acquire_accum_checksum(group_log_data_checksum, accum_checksum))) { - PALF_LOG(WARN, "update_accumulated_checksum failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "update_accumulated_checksum failed", K(ret), K_(self)); } else { // set flag for rollback accum_checksum is_accum_checksum_acquired = true; (void) group_header.update_accumulated_checksum(accum_checksum); (void) group_header.update_header_checksum(); - PALF_LOG(TRACE, "generate_group_entry_header_ success", K(ret), K_(palf_id), K_(self), K(is_padding_log), - K(is_raw_write), K(group_log_data_checksum), K(group_header), KPC(log_task)); + PALF_LOG(TRACE, "generate_group_entry_header_ success", K(ret), K_(self), K(is_padding_log), + K(group_log_data_checksum), K(group_header), KPC(log_task)); } } return ret; @@ -1164,7 +928,7 @@ int LogSlidingWindow::try_freeze_last_log_task_(const int64_t expected_log_id, is_need_handle = false; if (OB_INVALID_LOG_ID == expected_log_id || !expected_end_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(expected_log_id), K(expected_end_lsn)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(expected_log_id), K(expected_end_lsn)); } else { LogTask *log_task = NULL; LogTaskGuard guard(this); @@ -1173,7 +937,7 @@ int LogSlidingWindow::try_freeze_last_log_task_(const int64_t expected_log_id, // this log has slide out, ignore ret = OB_SUCCESS; } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(palf_id), K_(self), K(expected_log_id)); + PALF_LOG(ERROR, "get_log_task failed", K(ret), K_(self), K(expected_log_id)); } } else { log_task->lock(); @@ -1183,19 +947,19 @@ int LogSlidingWindow::try_freeze_last_log_task_(const int64_t expected_log_id, + log_task->get_data_len(); if (!log_task->is_valid()) { if (palf_reach_time_interval(1 * 1000 * 1000, cannot_freeze_log_warn_time_)) { - PALF_LOG(INFO, "this log_task is invalid, cannot freeze", K_(palf_id), K_(self), + PALF_LOG(INFO, "this log_task is invalid, cannot freeze", K_(self), K(expected_log_id), K(expected_end_lsn), KPC(log_task)); } } else if (log_end_lsn > expected_end_lsn) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "last log's end_lsn is larger than expected", K(ret), K_(palf_id), K_(self), + PALF_LOG(ERROR, "last log's end_lsn is larger than expected", K(ret), K_(self), K(log_end_lsn), K(expected_log_id), K(expected_end_lsn), KPC(log_task)); } else { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = log_task->try_freeze(expected_end_lsn))) { - PALF_LOG(WARN, "try_freeze failed", K(tmp_ret), K(expected_log_id), K_(palf_id), K_(self)); + PALF_LOG(WARN, "try_freeze failed", K(tmp_ret), K(expected_log_id), K_(self)); } else { - PALF_LOG(TRACE, "try_freeze success", K(ret), K_(palf_id), K_(self), K(expected_log_id), KPC(log_task), + PALF_LOG(TRACE, "try_freeze success", K(ret), K_(self), K(expected_log_id), KPC(log_task), K(log_end_lsn), K(expected_end_lsn)); } } @@ -1218,13 +982,13 @@ int LogSlidingWindow::feedback_freeze_last_log_() bool is_need_handle = false; if (FEEDBACK_FREEZE_MODE != freeze_mode_) { // Only FEEDBACK_FREEZE_MODE need exec this fucntion - PALF_LOG(TRACE, "current freeze mode is not feedback", K_(palf_id), K_(self), "freeze_mode", freeze_mode_2_str(freeze_mode_)); + PALF_LOG(TRACE, "current freeze mode is not feedback", K_(self), "freeze_mode", freeze_mode_2_str(freeze_mode_)); } else if (OB_FAIL(lsn_allocator_.try_freeze(last_log_end_lsn, last_log_id))) { - PALF_LOG(WARN, "lsn_allocator try_freeze failed", K(ret), K_(palf_id), K_(self), K(last_log_end_lsn), K(last_log_id)); + PALF_LOG(WARN, "lsn_allocator try_freeze failed", K(ret), K_(self), K(last_log_end_lsn), K(last_log_id)); } else if (last_log_id <= 0) { // no log, no need freeze } else if (OB_FAIL(try_freeze_last_log_task_(last_log_id, last_log_end_lsn, is_need_handle))) { - PALF_LOG(WARN, "try_freeze_last_log_task_ failed", K(ret), K_(palf_id), K_(self), K(last_log_id), K(last_log_end_lsn)); + PALF_LOG(WARN, "try_freeze_last_log_task_ failed", K(ret), K_(self), K(last_log_id), K(last_log_end_lsn)); } else { bool is_committed_lsn_updated = false; (void) handle_next_submit_log_(is_committed_lsn_updated); @@ -1247,13 +1011,13 @@ int LogSlidingWindow::try_feedback_freeze_log_task_(const int64_t expected_log_i // sliding window is full ret = OB_SUCCESS; } else { - PALF_LOG(ERROR, "get_log_task failed", KR(ret), K_(palf_id), K(expected_log_id)); + PALF_LOG(ERROR, "get_log_task failed", KR(ret), K(expected_log_id)); } } else if (OB_ISNULL(log_task)) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_task is NULL", KR(ret), K_(palf_id), K(expected_log_id)); + PALF_LOG(ERROR, "log_task is NULL", KR(ret), K(expected_log_id)); } else if (log_task->is_freezed()) { - PALF_LOG(TRACE, "log_task has been freezed", KR(ret), K_(palf_id), K(expected_log_id)); + PALF_LOG(TRACE, "log_task has been freezed", KR(ret), K(expected_log_id)); } else { log_task_begin_lsn = log_task->get_begin_lsn(); get_max_flushed_end_lsn(max_flushed_end_lsn); @@ -1281,16 +1045,16 @@ int LogSlidingWindow::check_and_switch_freeze_mode() if (FEEDBACK_FREEZE_MODE == freeze_mode_) { if (total_append_cnt >= APPEND_CNT_LB_FOR_PERIOD_FREEZE) { freeze_mode_ = PERIOD_FREEZE_MODE; - PALF_LOG(INFO, "switch freeze_mode to period", K_(palf_id), K_(self), K(total_append_cnt)); + PALF_LOG(INFO, "switch freeze_mode to period", K_(self), K(total_append_cnt)); } } else if (PERIOD_FREEZE_MODE == freeze_mode_) { if (total_append_cnt < APPEND_CNT_LB_FOR_PERIOD_FREEZE) { freeze_mode_ = FEEDBACK_FREEZE_MODE; - PALF_LOG(INFO, "switch freeze_mode to feedback", K_(palf_id), K_(self), K(total_append_cnt)); + PALF_LOG(INFO, "switch freeze_mode to feedback", K_(self), K(total_append_cnt)); (void) feedback_freeze_last_log_(); } } else {} - PALF_LOG(TRACE, "finish check_and_switch_freeze_mode", K_(palf_id), K_(self), K(total_append_cnt), "freeze_mode", freeze_mode_2_str(freeze_mode_)); + PALF_LOG(TRACE, "finish check_and_switch_freeze_mode", K_(self), K(total_append_cnt), "freeze_mode", freeze_mode_2_str(freeze_mode_)); return ret; } @@ -1302,13 +1066,13 @@ int LogSlidingWindow::period_freeze_last_log() bool is_need_handle = false; if (PERIOD_FREEZE_MODE != freeze_mode_) { // Only PERIOD_FREEZE_MODE need exec this fucntion - PALF_LOG(TRACE, "current freeze mode is not period", K_(palf_id), K_(self), "freeze_mode", freeze_mode_2_str(freeze_mode_)); + PALF_LOG(TRACE, "current freeze mode is not period", K_(self), "freeze_mode", freeze_mode_2_str(freeze_mode_)); } else if (OB_FAIL(lsn_allocator_.try_freeze(last_log_end_lsn, last_log_id))) { - PALF_LOG(WARN, "lsn_allocator try_freeze failed", K(ret), K_(palf_id), K_(self), K(last_log_end_lsn), K(last_log_id)); + PALF_LOG(WARN, "lsn_allocator try_freeze failed", K(ret), K_(self), K(last_log_end_lsn), K(last_log_id)); } else if (last_log_id <= 0) { // no log, no need freeze } else if (OB_FAIL(try_freeze_last_log_task_(last_log_id, last_log_end_lsn, is_need_handle))) { - PALF_LOG(WARN, "try_freeze_last_log_task_ failed", K(ret), K_(palf_id), K_(self), K(last_log_id), K(last_log_end_lsn)); + PALF_LOG(WARN, "try_freeze_last_log_task_ failed", K(ret), K_(self), K(last_log_id), K(last_log_end_lsn)); } else { } if (get_max_log_id() > get_last_submit_log_id_()) { @@ -1316,9 +1080,7 @@ int LogSlidingWindow::period_freeze_last_log() bool is_committed_lsn_updated = false; (void) handle_next_submit_log_(is_committed_lsn_updated); } - // handle committed log periodically - // because committed_end_lsn may be advanced during reconfirm, - // so there is maybe some log that can be slid in sw. + // Handle logs committed by local recovery or flushing. (void) handle_committed_log_(); return ret; } @@ -1331,8 +1093,6 @@ int LogSlidingWindow::after_rebuild(const LSN &lsn) } else if (!lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; } else { - is_rebuilding_ = false; - last_rebuild_lsn_.reset(); LSN committed_end_lsn; get_committed_end_lsn_(committed_end_lsn); if (lsn >= committed_end_lsn) { @@ -1342,33 +1102,6 @@ int LogSlidingWindow::after_rebuild(const LSN &lsn) return ret; } -int LogSlidingWindow::after_truncate(const TruncateLogCbCtx &truncate_cb_ctx) -{ - // Caller holds palf_handle_impl's wrlock. - int ret = OB_SUCCESS; - PALF_LOG(INFO, "after_truncate begin", K_(palf_id), K_(self), K(truncate_cb_ctx)); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!truncate_cb_ctx.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(truncate_cb_ctx)); - } else if (!is_truncating_ - || last_truncate_lsn_ != truncate_cb_ctx.lsn_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "state not match", K(ret), K_(palf_id), K_(self), K(truncate_cb_ctx), - K_(is_truncating), K_(last_truncate_lsn)); - } else { - // Set reuse_lsn here, because it may be advanced larger than last_truncate_lsn_ during flush cb. - // New logs beyond last_truncate_lsn_ won't be processed before after_truncate called, - // which ensures correctness. - (void) group_buffer_.truncate(last_truncate_lsn_); - last_truncate_lsn_.reset(); - is_truncating_ = false; - PALF_LOG(INFO, "after_truncate success", K_(palf_id), K_(self), K(truncate_cb_ctx)); - } - return ret; -} - int LogSlidingWindow::after_flush_log(const FlushLogCbCtx &flush_cb_ctx) { int ret = OB_SUCCESS; @@ -1376,101 +1109,31 @@ int LogSlidingWindow::after_flush_log(const FlushLogCbCtx &flush_cb_ctx) const int64_t log_id = flush_cb_ctx.log_id_; const LSN log_end_lsn = flush_cb_ctx.lsn_ + flush_cb_ctx.total_len_; const int64_t cb_begin_ts = ObTimeUtility::current_time(); - bool is_fetch_log = false; - bool need_reply_ack = true; - PALF_LOG(TRACE, "after_flush_log begin", K_(palf_id), K_(self), K(flush_cb_ctx)); + PALF_LOG(TRACE, "after_flush_log begin", K_(self), K(flush_cb_ctx)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (!flush_cb_ctx.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(flush_cb_ctx)); - } else if (is_truncating_ && log_end_lsn > last_truncate_lsn_) { - // Logs after the truncate point do not need to execute callbacks - PALF_LOG(INFO, "this log has been truncated, no need execute flush cb", K(ret), K_(palf_id), K_(self), - K(flush_cb_ctx), K(log_end_lsn), K_(last_truncate_lsn)); - } else if (is_rebuilding_ && log_end_lsn > last_rebuild_lsn_) { - // logs after the rebuild point do not need to execute callbacks, otherwise it will incorrectly update max_flushed_end_lsn - PALF_LOG(INFO, "this replica is rebuilding, no need execute flush cb", K(ret), K_(palf_id), K_(self), - K(flush_cb_ctx), K(log_end_lsn), K_(last_truncate_lsn)); - } else if (state_mgr_->get_proposal_id() != flush_cb_ctx.curr_proposal_id_) { - // If curr_proposal_id_ has changed during flushing log, we need check log_proposal_id_ further. - LogTask *log_task = NULL; - LogTaskGuard guard(this); - if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - PALF_LOG(WARN, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self)); - } else { - log_task->lock(); - if (!log_task->is_valid()) { - can_exec_cb = false; - PALF_LOG(WARN, "log_task is invalid, it may be truncated, skip flush_cb", K_(palf_id), K_(self), - K(log_id), KPC(log_task)); - } else if (log_task->get_proposal_id() != flush_cb_ctx.log_proposal_id_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "proposal_id of log_task has changed", K(ret), K_(palf_id), K_(self), K(flush_cb_ctx), - KPC(log_task)); - } else { - // log_proposal_id_ matches, can exec flush_cb - can_exec_cb = true; - // update log_task's flushed_ts - log_task->set_flushed_ts(cb_begin_ts); - PushLogType push_log_type = log_task->get_push_log_type(); - is_fetch_log = (PushLogType::FETCH_LOG_RESP == push_log_type); - need_reply_ack = (PushLogType::PUSH_LOG_WO_ACK != push_log_type); - } - log_task->unlock(); - } + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(flush_cb_ctx)); } else { can_exec_cb = true; // update log_task's flushed_ts LogTask *log_task = NULL; LogTaskGuard guard(this); if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - PALF_LOG(WARN, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self)); + PALF_LOG(WARN, "get_log_task failed", K(ret), K(log_id), K_(self)); } else { log_task->set_flushed_ts(cb_begin_ts); - PushLogType push_log_type = log_task->get_push_log_type(); - is_fetch_log = (PushLogType::FETCH_LOG_RESP == push_log_type); - need_reply_ack = (PushLogType::PUSH_LOG_WO_ACK != push_log_type); } } common::ObTimeGuard time_guard("after flush log", 100 * 1000); if (OB_SUCC(ret) && can_exec_cb) { - (void) inc_update_max_flushed_log_info_(flush_cb_ctx.lsn_, log_end_lsn, flush_cb_ctx.log_proposal_id_); - if (LEADER == state_mgr_->get_role()) { - // leader update match_lsn_map_ - (void) try_update_match_lsn_map_(self_, log_end_lsn); - } + (void) inc_update_max_flushed_log_info_(flush_cb_ctx.lsn_, log_end_lsn); time_guard.click(); - if (state_mgr_->is_leader_active()) { - // leader try update committed_end_lsn - // leader cannot update commit_offset before writing start_working log successfully during reconfirm + if (state_mgr_->is_active()) { LSN new_committed_end_lsn; (void) gen_committed_end_lsn_(new_committed_end_lsn); - } else if (FOLLOWER == state_mgr_->get_role()) { - // follower need send ack to leader - const ObAddr &leader = (state_mgr_->get_leader().is_valid())? \ - state_mgr_->get_leader(): state_mgr_->get_broadcast_leader(); - LogConfigVersion config_version; - (void) mm_->get_config_version(config_version); - // flush op for different role - // migrating replicas do not send responses for reducing its impact on the leader - if (!need_reply_ack) { - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - PALF_LOG(INFO, "log_type do not need to reply ack", K(ret), K_(palf_id), K_(self), - K(log_end_lsn), K(leader)); - } - } else if (!leader.is_valid()) { - PALF_LOG(INFO, "current leader is invalid, cannot send ack", K(ret), K_(palf_id), K_(self), - K(flush_cb_ctx), K(log_end_lsn), K(leader)); - } else if (OB_UNLIKELY(config_version.is_initial_version())) { - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - PALF_LOG(INFO, "migrating replicas do not send responses", K(ret), K_(palf_id), K_(self), - K(log_end_lsn), K(leader)); - } - } else if (OB_FAIL(submit_push_log_resp_(leader, flush_cb_ctx.curr_proposal_id_, log_end_lsn, is_fetch_log))) { - PALF_LOG(WARN, "submit_push_log_resp failed", K(ret), K_(palf_id), K_(self), K(leader), K(flush_cb_ctx)); - } else {} } else {} time_guard.click("before handle log"); @@ -1489,41 +1152,15 @@ int LogSlidingWindow::after_flush_log(const FlushLogCbCtx &flush_cb_ctx) (void) handle_next_submit_log_(is_committed_lsn_updated); } time_guard.click("after handle next log"); - // Both leader and follower need handle committed logs here. (void) handle_committed_log_(); time_guard.click("after handle committed log"); } } - PALF_LOG(TRACE, "sw after_flush_log success", K(ret), K_(palf_id), K_(self), K(can_exec_cb), K(flush_cb_ctx), + PALF_LOG(TRACE, "sw after_flush_log success", K(ret), K_(self), K(can_exec_cb), K(flush_cb_ctx), K_(max_flushed_lsn), K_(max_flushed_end_lsn), K(time_guard)); return ret; } -int LogSlidingWindow::get_last_submit_log_info(LSN &last_submit_lsn, - int64_t &log_id, int64_t &log_proposal_id) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LSN unused_lsn; - get_last_submit_log_info_(last_submit_lsn, unused_lsn, log_id, log_proposal_id); - } - return ret; -} - -int LogSlidingWindow::get_last_submit_log_info(LSN &last_submit_lsn, - LSN &last_submit_end_lsn, int64_t &log_id, int64_t &log_proposal_id) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, log_id, log_proposal_id); - } - return ret; -} - int64_t LogSlidingWindow::get_last_submit_log_id_() const { ObSpinLockGuard guard(last_submit_info_lock_); @@ -1536,26 +1173,12 @@ void LogSlidingWindow::get_last_submit_end_lsn_(LSN &end_lsn) const } void LogSlidingWindow::get_last_submit_log_info_(LSN &lsn, LSN &end_lsn, - int64_t &log_id, int64_t &log_proposal_id) const + int64_t &log_id) const { ObSpinLockGuard guard(last_submit_info_lock_); lsn = last_submit_lsn_; end_lsn = last_submit_end_lsn_; log_id = last_submit_log_id_; - log_proposal_id = last_submit_log_pid_; -} - -int LogSlidingWindow::get_max_flushed_log_info(LSN &lsn, - LSN &end_lsn, - int64_t &log_proposal_id) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = get_max_flushed_log_info_(lsn, end_lsn, log_proposal_id); - } - return ret; } void LogSlidingWindow::get_max_flushed_end_lsn(LSN &end_lsn) const @@ -1563,18 +1186,6 @@ void LogSlidingWindow::get_max_flushed_end_lsn(LSN &end_lsn) const end_lsn.val_ = ATOMIC_LOAD(&max_flushed_end_lsn_.val_); } -int LogSlidingWindow::get_max_flushed_log_info_(LSN &lsn, - LSN &end_lsn, - int64_t &log_proposal_id) const -{ - int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(max_flushed_info_lock_); - lsn = max_flushed_lsn_; - end_lsn = max_flushed_end_lsn_; - log_proposal_id = max_flushed_log_pid_; - return ret; -} - int LogSlidingWindow::get_last_slide_end_lsn(LSN &out_end_lsn) const { int ret = OB_SUCCESS; @@ -1607,10 +1218,9 @@ void LogSlidingWindow::get_last_slide_end_lsn_(LSN &out_end_lsn) const SCN last_slide_scn; LSN last_slide_lsn; LSN last_slide_end_lsn; - int64_t unused_pid = INVALID_PROPOSAL_ID; int64_t last_slide_accum_checksum = -1; get_last_slide_log_info_(last_slide_log_id, last_slide_scn, \ - last_slide_lsn, last_slide_end_lsn, unused_pid, last_slide_accum_checksum); + last_slide_lsn, last_slide_end_lsn, last_slide_accum_checksum); out_end_lsn = last_slide_end_lsn; } @@ -1618,7 +1228,6 @@ void LogSlidingWindow::get_last_slide_log_info_(int64_t &log_id, SCN &scn, LSN &lsn, LSN &end_lsn, - int64_t &log_proposal_id, int64_t &accum_checksum) const { ObSpinLockGuard guard(last_slide_info_lock_); @@ -1626,29 +1235,24 @@ void LogSlidingWindow::get_last_slide_log_info_(int64_t &log_id, scn = last_slide_scn_; lsn = last_slide_lsn_; end_lsn = last_slide_end_lsn_; - log_proposal_id = last_slide_log_pid_; accum_checksum = last_slide_log_accum_checksum_; } int LogSlidingWindow::set_last_submit_log_info_(const LSN &lsn, const LSN &end_lsn, - const int64_t log_id, - const int64_t &log_proposal_id) + const int64_t log_id) { int ret = OB_SUCCESS; - if (!lsn.is_valid() || !end_lsn.is_valid() || OB_INVALID_LOG_ID == log_id - || (INVALID_PROPOSAL_ID == log_proposal_id && PALF_INITIAL_LSN_VAL < end_lsn.val_)) { + if (!lsn.is_valid() || !end_lsn.is_valid() || OB_INVALID_LOG_ID == log_id) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), K(end_lsn), K(log_id), K(log_proposal_id)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(lsn), K(end_lsn), K(log_id)); } else { ObSpinLockGuard guard(last_submit_info_lock_); const int64_t old_submit_log_id = last_submit_log_id_; last_submit_lsn_ = lsn; ATOMIC_STORE(&last_submit_end_lsn_.val_, end_lsn.val_); last_submit_log_id_ = log_id; - last_submit_log_pid_ = log_proposal_id; - PALF_LOG(TRACE, "set_last_submit_log_info_ success", K_(palf_id), K_(self), K(old_submit_log_id), K(lsn), K(log_id), \ - K(log_proposal_id)); + PALF_LOG(TRACE, "set_last_submit_log_info_ success", K_(self), K(old_submit_log_id), K(lsn), K(log_id)); } return ret; } @@ -1658,27 +1262,24 @@ int LogSlidingWindow::try_update_last_slide_log_info_( const SCN &scn, const LSN &lsn, const LSN &end_lsn, - const int64_t &proposal_id, const int64_t accum_checksum) { int ret = OB_SUCCESS; if (!lsn.is_valid() || !end_lsn.is_valid() || - INVALID_PROPOSAL_ID == proposal_id || OB_INVALID_LOG_ID == log_id || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), K(end_lsn), K(proposal_id), K(log_id), K(scn)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K_(self), K(lsn), K(end_lsn), K(log_id), K(scn)); } else { ObSpinLockGuard guard(last_slide_info_lock_); ATOMIC_STORE(&last_slide_log_id_, log_id); last_slide_scn_ = scn; last_slide_lsn_ = lsn; last_slide_end_lsn_ = end_lsn; - last_slide_log_pid_ = proposal_id; last_slide_log_accum_checksum_ = accum_checksum; - PALF_LOG(TRACE, "try_update_last_slide_log_info_ success", K_(palf_id), K_(self), K(log_id), K(scn), - K(lsn), K(end_lsn), K(proposal_id), K(accum_checksum)); + PALF_LOG(TRACE, "try_update_last_slide_log_info_ success", K_(self), K(log_id), K(scn), + K(lsn), K(end_lsn), K(accum_checksum)); } return ret; } @@ -1693,7 +1294,7 @@ int LogSlidingWindow::try_advance_committed_lsn_(const LSN &end_lsn) int ret = OB_SUCCESS; if (!end_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K_(palf_id), K_(self), K(end_lsn)); + PALF_LOG(WARN, "invalid argumetns", K_(self), K(end_lsn)); } else { LSN old_committed_end_lsn; get_committed_end_lsn_(old_committed_end_lsn); @@ -1704,11 +1305,11 @@ int LogSlidingWindow::try_advance_committed_lsn_(const LSN &end_lsn) get_committed_end_lsn_(old_committed_end_lsn); } } - PALF_LOG(TRACE, "try_advance_committed_lsn_ success", K_(palf_id), K_(self), K_(committed_end_lsn)); + PALF_LOG(TRACE, "try_advance_committed_lsn_ success", K_(self), K_(committed_end_lsn)); if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, end_lsn_stat_time_us_)) { LSN curr_end_lsn; get_committed_end_lsn_(curr_end_lsn); - PALF_LOG(INFO, "[PALF STAT COMMITTED LOG SIZE]", K_(palf_id), K_(self), "committed size", curr_end_lsn.val_ - last_record_end_lsn_.val_); + PALF_LOG(INFO, "[PALF STAT COMMITTED LOG SIZE]", K_(self), "committed size", curr_end_lsn.val_ - last_record_end_lsn_.val_); last_record_end_lsn_ = curr_end_lsn; } } @@ -1721,387 +1322,34 @@ int LogSlidingWindow::inc_update_scn_base(const SCN &scn) if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (OB_FAIL(lsn_allocator_.inc_update_scn_base(scn))) { - PALF_LOG(WARN, "inc_update_scn_base failed", K(ret), K_(palf_id), K_(self), K(scn)); + PALF_LOG(WARN, "inc_update_scn_base failed", K(ret), K_(self), K(scn)); } return ret; } int LogSlidingWindow::inc_update_max_flushed_log_info_(const LSN &lsn, - const LSN &end_lsn, - const int64_t &proposal_id) + const LSN &end_lsn) { int ret = OB_SUCCESS; LSN curr_max_flushed_end_lsn; get_max_flushed_end_lsn(curr_max_flushed_end_lsn); - if (!lsn.is_valid() || !end_lsn.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { + if (!lsn.is_valid() || !end_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K_(palf_id), K_(self), K(lsn), K(end_lsn), K(proposal_id)); + PALF_LOG(WARN, "invalid argumetns", K_(self), K(lsn), K(end_lsn)); } else if (curr_max_flushed_end_lsn.is_valid() && curr_max_flushed_end_lsn >= end_lsn) { // no need update max_flushed_end_lsn_ } else { common::ObSpinLockGuard guard(max_flushed_info_lock_); // double check if (max_flushed_end_lsn_.is_valid() && max_flushed_end_lsn_ >= end_lsn) { - PALF_LOG(WARN, "arg end lsn is not larger than current, no need update", K_(palf_id), K_(self), - K_(max_flushed_lsn), K_(max_flushed_end_lsn), K(lsn), K(end_lsn), K(proposal_id)); + PALF_LOG(WARN, "arg end lsn is not larger than current, no need update", K_(self), + K_(max_flushed_lsn), K_(max_flushed_end_lsn), K(lsn), K(end_lsn)); } else { max_flushed_lsn_ = lsn; ATOMIC_STORE(&max_flushed_end_lsn_.val_, end_lsn.val_); - max_flushed_log_pid_ = proposal_id; - PALF_LOG(TRACE, "inc_update_max_flushed_log_info_ success", K_(palf_id), K_(self), K(lsn), K(end_lsn), - K(proposal_id), K(max_flushed_end_lsn_)); - } - } - return ret; -} - -int LogSlidingWindow::truncate_max_flushed_log_info_(const LSN &lsn, - const LSN &end_lsn, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - if (!lsn.is_valid() - || !end_lsn.is_valid() - || (INVALID_PROPOSAL_ID == proposal_id && PALF_INITIAL_LSN_VAL < end_lsn.val_)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), K(end_lsn), K(proposal_id)); - } else { - common::ObSpinLockGuard guard(max_flushed_info_lock_); - max_flushed_lsn_ = lsn; - max_flushed_end_lsn_ = end_lsn; - max_flushed_log_pid_ = proposal_id; - PALF_LOG(INFO, "truncate_max_flushed_log_info_ success", K_(palf_id), K_(self), K(lsn), K(end_lsn), K(proposal_id)); - } - return ret; -} - -void LogSlidingWindow::get_last_fetch_info_(LSN &last_fetch_end_lsn, - LSN &last_committed_end_lsn, int64_t &last_fetch_max_log_id) const -{ - ObSpinLockGuard guard(fetch_info_lock_); - last_fetch_end_lsn = last_fetch_end_lsn_; - last_committed_end_lsn = last_fetch_committed_end_lsn_; - last_fetch_max_log_id = last_fetch_max_log_id_; -} - -void LogSlidingWindow::try_reset_last_fetch_log_info_(const LSN &expected_end_lsn, const int64_t log_id) -{ - ObSpinLockGuard guard(fetch_info_lock_); - if (expected_end_lsn.is_valid() && expected_end_lsn != last_fetch_end_lsn_) { - PALF_LOG(INFO, "last_fetch_end_lsn_ has changed, skip reset", K_(palf_id), K_(self), - K(expected_end_lsn), K_(last_fetch_end_lsn)); - } else if (log_id <= last_fetch_max_log_id_) { - // If it receives push log whose end_lsn AND log_id is less than or equal to last_fetch_info, - // which means it can receive other logs from leader, so reset last fetch info. - last_fetch_req_time_ = 0; - last_fetch_end_lsn_.reset(); - last_fetch_committed_end_lsn_.reset(); - last_fetch_max_log_id_ = OB_INVALID_LOG_ID; - last_fetch_trigger_type_ = FetchTriggerType::LOG_LOOP_TH; - PALF_LOG(INFO, "reset last fetch log info", K_(palf_id), K_(self), K(expected_end_lsn)); - } else { - // do nothing - } -} - -void LogSlidingWindow::try_update_committed_lsn_for_fetch_( - const LSN &log_end_lsn, - const int64_t &log_id, - const LSN &log_committed_end_lsn, - bool &is_need_fetch) -{ - bool need_update = false; - LSN last_fetch_end_lsn; - LSN last_committed_end_lsn; - last_fetch_end_lsn.val_ = ATOMIC_LOAD(&last_fetch_end_lsn_.val_); - last_committed_end_lsn.val_ = ATOMIC_LOAD(&last_fetch_committed_end_lsn_.val_); - int64_t last_fetch_max_log_id = OB_INVALID_LOG_ID; - get_last_fetch_info_(last_fetch_end_lsn, last_committed_end_lsn, last_fetch_max_log_id); - if (!last_fetch_end_lsn.is_valid() || last_committed_end_lsn.is_valid()) { - // no need update - } else { - if (last_fetch_end_lsn.is_valid() - && !last_committed_end_lsn.is_valid() - && (log_end_lsn >= last_fetch_end_lsn || log_id == last_fetch_max_log_id)) { - // This log is the last one from the previous fetch, update last_fetch_committed_end_lsn_ - // Used for subsequent triggering of streaming fetch - need_update = true; - } - } - - if (need_update) { - ObSpinLockGuard guard(fetch_info_lock_); - if (last_fetch_committed_end_lsn_.is_valid()) { - PALF_LOG(INFO, "last_fetch_committed_end_lsn_ has been set, skip update", K_(palf_id), K_(self), - K(log_id), K_(last_fetch_max_log_id), K_(last_fetch_end_lsn), - K_(last_fetch_committed_end_lsn)); - } else if (last_fetch_end_lsn == last_fetch_end_lsn_ - || last_fetch_max_log_id == last_fetch_max_log_id_) { - LSN committed_end_lsn; - get_committed_end_lsn_(committed_end_lsn); - // The order is fatal: - // 1) update last_fetch_committed_end_lsn_ - // 2) check last slide end_lsn to decide fetching log streamingly or not. - ATOMIC_STORE(&last_fetch_committed_end_lsn_.val_, log_committed_end_lsn.val_); - MEM_BARRIER(); - // The committed_end_lsn may already be updated at least to this value by previous group log. - // And the logs before log_committed_end_lsn have been slid out. - // For this scenario, it need triger fetch log streamingly now. - LSN last_slide_end_lsn; - get_last_slide_end_lsn_(last_slide_end_lsn); - if (committed_end_lsn >= log_committed_end_lsn - && last_slide_end_lsn >= log_committed_end_lsn) { - is_need_fetch = true; - } - PALF_LOG(INFO, "update last fetch log info", K_(palf_id), K_(self), K(last_fetch_end_lsn), K(log_id), - K_(last_fetch_end_lsn), K_(last_fetch_committed_end_lsn), K(committed_end_lsn), K(last_slide_end_lsn)); - } else { - PALF_LOG(INFO, "last_fetch_max_log_id_ has changed, skip update", K_(palf_id), K_(self), - K(last_fetch_end_lsn), K(log_id), K_(last_fetch_max_log_id), K_(last_fetch_end_lsn), - K_(last_fetch_committed_end_lsn)); - } - } -} - -bool LogSlidingWindow::need_execute_fetch_(const FetchTriggerType &fetch_trigger_type) -{ - bool bool_ret = true; - const int64_t now = ObTimeUtility::current_time(); - if (FetchTriggerType::SLIDING_CB == last_fetch_trigger_type_ - && (FetchTriggerType::ADD_MEMBER_PRE_CHECK == fetch_trigger_type - || FetchTriggerType::MODE_META_BARRIER == fetch_trigger_type - || FetchTriggerType::LEARNER_REGISTER == fetch_trigger_type - || FetchTriggerType::RECONFIRM_NOTIFY_FETCH == fetch_trigger_type)) { - // If self is currently in streamingly fetch state, it does not need fetch again - // in some trigger cases. - bool_ret = false; - } else if (now - last_fetch_req_time_ < PALF_FETCH_LOG_OUTER_TRIGGER_INTERVAL_US - && (FetchTriggerType::ADD_MEMBER_PRE_CHECK == fetch_trigger_type - || FetchTriggerType::MODE_META_BARRIER == fetch_trigger_type - || FetchTriggerType::LEARNER_REGISTER == fetch_trigger_type)) { - // Prevent config pre check generating fetch req too frequently. - bool_ret = false; - } else {} - return bool_ret; -} - -bool LogSlidingWindow::need_use_batch_rpc_(const int64_t buf_size, - const bool is_fetch_log) const -{ - UNUSED(buf_size); - UNUSED(is_fetch_log); - // Single-replica: the batched log push transport (obcall::ObBatchRpc) has - // been removed. Always take the direct push path (already neutered / - // self-excluded on a single member), so palf never relies on batch RPC. - return false; -} - -int LogSlidingWindow::try_fetch_log(const FetchTriggerType &fetch_log_type, - const LSN prev_lsn, - const LSN fetch_start_lsn, - const int64_t fetch_start_log_id) -{ - int ret = OB_SUCCESS; - ObAddr fetch_log_dst; - const bool all_valid = prev_lsn.is_valid() && fetch_start_lsn.is_valid() && is_valid_log_id(fetch_start_log_id); - const bool all_invalid = !prev_lsn.is_valid() && !fetch_start_lsn.is_valid() && !is_valid_log_id(fetch_start_log_id); - // default fetch_log_size is assigned by current group_buffer size - const int64_t fetch_log_size = group_buffer_.get_available_buffer_size(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!all_valid && !all_invalid) { - // require argument are all valid or all invalid - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K_(self), K(fetch_log_type), K(prev_lsn), K(fetch_start_lsn), K(fetch_start_log_id)); - } else if (OB_FAIL(get_fetch_log_dst_(fetch_log_dst)) - || !fetch_log_dst.is_valid() - || fetch_log_dst == self_) { - if (palf_reach_time_interval(5 * 1000 * 1000, fetch_failure_print_time_)) { - PALF_LOG(WARN, "get_fetch_log_dst failed or invalid", K(ret), K_(palf_id), K_(self), K(fetch_log_dst)); - } - } else if (false == need_execute_fetch_(fetch_log_type)) { - if (palf_reach_time_interval(5 * 1000 * 1000, fetch_failure_print_time_)) { - PALF_LOG(INFO, "no need execute fetch", K(ret), K_(palf_id), K_(self), K(fetch_log_type), - "last_fetch_trigger_type", fetch_trigger_type_2_str(last_fetch_trigger_type_), K_(last_fetch_req_time)); - } - } else if (FetchTriggerType::MODE_META_BARRIER == fetch_log_type) { - int64_t last_slide_log_id = OB_INVALID_LOG_ID; - SCN last_slide_scn; - LSN last_slide_lsn; - LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; - int64_t last_slide_accum_checksum = -1; - get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); - if (OB_FAIL(do_fetch_log_(fetch_log_type, fetch_log_dst, last_slide_lsn, \ - last_slide_end_lsn, fetch_log_size, last_slide_log_id + 1))) { - PALF_LOG(WARN, "do_fetch_log_ failed", K(ret), K_(palf_id), K_(self), K(fetch_log_type), K(fetch_log_dst), - K(last_slide_lsn), K(last_slide_end_lsn), K(last_slide_log_id)); - } - } else if (all_valid) { - // use assigned arguments - if (OB_FAIL(do_fetch_log_(fetch_log_type, fetch_log_dst, prev_lsn, \ - fetch_start_lsn, fetch_log_size, fetch_start_log_id))) { - PALF_LOG(WARN, "do_fetch_log_ failed", K(ret), K_(palf_id), K_(self), K(fetch_log_type), K(fetch_log_dst), - K(prev_lsn), K(fetch_start_lsn), K(fetch_start_log_id)); - } - } else if (all_invalid) { - // generate default arguments - // This interface is called by state_mgr, and fetch log is periodically triggered when the left boundary of sw does not slide - // Trigger requires all committed logs to have slid out, and use committed_end_lsn as the new fetch starting point - LSN last_slide_lsn; - int64_t last_slide_log_id; - LSN sw_start_lsn; - LSN committed_end_lsn; - if (!is_all_committed_log_slided_out_(last_slide_lsn, last_slide_log_id, sw_start_lsn, committed_end_lsn)) { - if (palf_reach_time_interval(1 * 1000 * 1000, cannot_fetch_log_warn_time_)) { - PALF_LOG(WARN, "is_all_committed_log_slided_out_ return false, cannot fetch log now", K(ret), - K_(palf_id), K_(self), K(committed_end_lsn)); - } - } else if (OB_FAIL(do_fetch_log_(fetch_log_type, fetch_log_dst, last_slide_lsn, \ - sw_start_lsn, fetch_log_size, last_slide_log_id + 1))) { - PALF_LOG(WARN, "do_fetch_log_ failed", K(ret), K_(palf_id), K_(self), K(fetch_log_type), K(fetch_log_dst), - K(last_slide_lsn), K(sw_start_lsn), K(committed_end_lsn), K(last_slide_log_id)); - } - } - PALF_LOG(TRACE, "runlin trace try_fetch_log", K(ret), K(all_valid), K(all_invalid)); - return ret; -} - -int LogSlidingWindow::try_fetch_log_for_reconfirm(const common::ObAddr &dest, const LSN &fetch_end_lsn, bool &is_fetched) -{ - int ret = OB_SUCCESS; - LSN prev_lsn, sw_start_lsn; - int64_t prev_log_id = OB_INVALID_LOG_ID; - LSN committed_end_lsn; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!dest.is_valid() || !fetch_end_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(dest), K(fetch_end_lsn)); - } else if (!is_all_committed_log_slided_out_(prev_lsn, prev_log_id, sw_start_lsn, committed_end_lsn)) { - if (palf_reach_time_interval(1 * 1000 * 1000, cannot_fetch_log_warn_time_)) { - PALF_LOG(WARN, "is_all_committed_log_slided_out_ return false, cannot fetch log now", K(ret), - K_(palf_id), K_(self), K(committed_end_lsn)); - } - } else { - reconfirm_fetch_dest_ = dest; - const int64_t fetch_log_size = MIN(fetch_end_lsn - committed_end_lsn, group_buffer_.get_available_buffer_size()); - if (OB_FAIL(do_fetch_log_(FetchTriggerType::LEADER_RECONFIRM, dest, prev_lsn, \ - committed_end_lsn, fetch_log_size, prev_log_id + 1))) { - PALF_LOG(WARN, "do_fetch_log_ failed", K(ret), K_(palf_id), K_(self)); - } else { - is_fetched = true; - } - } - return ret; -} - -int LogSlidingWindow::do_fetch_log_(const FetchTriggerType &trigger_type, - const common::ObAddr &dest, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_start_log_id) -{ - int ret = OB_SUCCESS; - if (false == state_mgr_->is_sync_enabled()) { - if (palf_reach_time_interval(1 * 1000 * 1000, cannot_fetch_log_warn_time_)) { - PALF_LOG(INFO, "sync is disabled, cannot fetch log", K_(palf_id), K_(self)); - } - } else if (!dest.is_valid() || !fetch_start_lsn.is_valid() || fetch_start_log_id <= 0 - || fetch_log_size <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K_(self), K(prev_lsn), K(fetch_start_lsn), - K(fetch_start_log_id), K(dest), K(fetch_log_size)); - } else { - const int64_t last_slide_log_id = get_last_slide_log_id_(); - // To avoid duplicate pulls, here we use last_slide_log_id instead of sw_start_id, - // Because sliding_cb() has not advanced sw_start_id before it completes, last_slide_log_id has already advanced - int64_t skip_log_count = fetch_start_log_id - (last_slide_log_id + 1); - if (skip_log_count < 0) { - // The log of fetch_start_log_id maybe has slid out - skip_log_count = 0; - } - // NB: follower's fetch_log_count must be bigger than PALF_MAX_LEADER_SUBMIT_LOG_COUNT, - // otherwise committed_end_lsn in follower may never be inc updated, - // so just set it to PALF_SLIDING_WINDOW_SIZE - const int64_t fetch_log_count = PALF_SLIDING_WINDOW_SIZE - skip_log_count; - // Update last_fetch_end_lsn_ AND last_fetch_max_log_id_ - bool need_exec_fetch = true; - do { - ObSpinLockGuard guard(fetch_info_lock_); - const LSN tmp_end_lsn = fetch_start_lsn + fetch_log_size; - if (FetchTriggerType::SLIDING_CB == trigger_type - && !last_fetch_committed_end_lsn_.is_valid()) { - // Streamingly fetching may trigger more than one time, - // we need filter duplicated ops here. - need_exec_fetch = false; - } else { - ATOMIC_STORE(&last_fetch_end_lsn_.val_, tmp_end_lsn.val_); - last_fetch_max_log_id_ = fetch_start_log_id + fetch_log_count - 1; - last_fetch_committed_end_lsn_.reset(); - } - } while(0); - - if (OB_SUCC(ret) && need_exec_fetch) { - FetchLogType fetch_type = FETCH_LOG_FOLLOWER; - if (LEADER_RECONFIRM == trigger_type) { - fetch_type = FETCH_LOG_LEADER_RECONFIRM; - } else if (MODE_META_BARRIER == trigger_type) { - fetch_type = FETCH_MODE_META; - } - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - const int64_t accepted_mode_pid = mode_mgr_->get_accepted_mode_meta().proposal_id_; - if (fetch_log_count <= 0) { - ret = OB_EAGAIN; - } else if (OB_FAIL(log_engine_->submit_fetch_log_req(dest, fetch_type, curr_proposal_id, prev_lsn, - fetch_start_lsn, fetch_log_size, fetch_log_count, accepted_mode_pid))) { - PALF_LOG(WARN, "submit_fetch_log_req failed", K(ret), K_(palf_id), K_(self)); - } else { - // Record fetch trigger type - last_fetch_req_time_ = ObTimeUtility::current_time(); - last_fetch_trigger_type_ = trigger_type; - } - PALF_LOG(INFO, "do_fetch_log_ finished", K(ret), K_(palf_id), K_(self), K(dest), - K(fetch_log_count), K(fetch_start_lsn), K(prev_lsn), K(fetch_start_log_id), - K(last_slide_log_id), K(fetch_log_size), K(accepted_mode_pid), K_(last_fetch_req_time), - K_(last_fetch_end_lsn), K_(last_fetch_max_log_id), K_(last_fetch_committed_end_lsn), - "trigger_type", fetch_trigger_type_2_str(trigger_type), KPC(this)); - } - } - return ret; -} - -int LogSlidingWindow::get_leader_from_cache(common::ObAddr &leader) const -{ - return get_leader_from_cache_(leader); -} - -int LogSlidingWindow::get_leader_from_cache_(common::ObAddr &leader) const -{ - int ret = OB_SUCCESS; - const common::ObAddr state_mgr_leader = state_mgr_->get_leader(); - const common::ObAddr broadcast_leader = state_mgr_->get_broadcast_leader(); - if (state_mgr_leader.is_valid()) { - leader = state_mgr_leader; - } else if (broadcast_leader.is_valid()) { - leader = broadcast_leader; - } else if (OB_FAIL(plugins_->nonblock_get_leader(palf_id_, leader))) { - if (palf_reach_time_interval(5 * 1000 * 1000, lc_cb_get_warn_time_)) { - PALF_LOG(WARN, "nonblock_get_leader failed", KR(ret), K_(palf_id), K_(self)); + PALF_LOG(TRACE, "inc_update_max_flushed_log_info_ success", K_(self), K(lsn), K(end_lsn), + K(max_flushed_end_lsn_)); } - } else {} - return ret; -} - -int LogSlidingWindow::get_fetch_log_dst_(common::ObAddr &fetch_dst) const -{ - int ret = OB_SUCCESS; - const common::ObAddr parent = mm_->get_parent(); - if (parent.is_valid()) { - fetch_dst = parent; - } else if (OB_FAIL(get_leader_from_cache_(fetch_dst))) { } return ret; } @@ -2123,16 +1371,15 @@ bool LogSlidingWindow::is_all_committed_log_slided_out_( SCN last_slide_scn; LSN last_slide_lsn; LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; int64_t last_slide_accum_checksum = -1; get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); + last_slide_end_lsn, last_slide_accum_checksum); get_committed_end_lsn_(committed_end_lsn); if (committed_end_lsn <= last_slide_end_lsn) { bool_ret = true; } else { bool_ret = false; - PALF_LOG(TRACE, "is_all_committed_log_slided_out_ false", K_(palf_id), K_(self), K(bool_ret), K(committed_end_lsn), + PALF_LOG(TRACE, "is_all_committed_log_slided_out_ false", K_(self), K(bool_ret), K(committed_end_lsn), K(last_slide_end_lsn), K(last_slide_log_id)); } prev_lsn = last_slide_lsn; @@ -2148,7 +1395,7 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot ret = OB_NOT_INIT; } else if (OB_ISNULL(data)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "invalid argument", K_(palf_id), K_(self), K(sn), K(ret)); + PALF_LOG(ERROR, "invalid argument", K_(self), K(sn), K(ret)); } else if (!state_mgr_->can_slide_sw()) { // can_slide_sw() returns false ret = OB_EAGAIN; @@ -2159,7 +1406,7 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot const LogTask *log_task = dynamic_cast(data); if (NULL == log_task) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "dynamic_cast return NULL", K_(palf_id), K_(self), K(ret)); + PALF_LOG(ERROR, "dynamic_cast return NULL", K_(self), K(ret)); } else { LogGroupEntryHeader tmp_header; LogTaskHeaderInfo log_task_header; @@ -2170,7 +1417,6 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot const int64_t log_size = LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); log_end_lsn = log_begin_lsn + log_size; log_task_header = log_task->get_header_info(); - const int64_t log_proposal_id = log_task->get_proposal_id(); const int64_t log_accum_checksum = log_task->get_accum_checksum(); const int64_t log_gen_ts = log_task->get_gen_ts(); const int64_t log_freeze_ts = log_task->get_freeze_ts(); @@ -2187,20 +1433,20 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot // Call fs_cb. int tmp_ret = OB_SUCCESS; const int64_t fs_cb_begin_ts = ObTimeUtility::current_time(); - if (OB_SUCCESS != (tmp_ret = palf_fs_cb_->update_end_lsn(palf_id_, log_end_lsn, log_max_scn, log_proposal_id))) { + if (OB_SUCCESS != (tmp_ret = palf_fs_cb_->update_end_lsn(log_end_lsn, log_max_scn))) { if (OB_EAGAIN == tmp_ret) { if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - PALF_LOG(WARN, "update_end_lsn eagain", K(tmp_ret), K_(palf_id), K_(self), K(log_id), KPC(log_task)); + PALF_LOG(WARN, "update_end_lsn eagain", K(tmp_ret), K_(self), K(log_id), KPC(log_task)); } } else { - PALF_LOG(WARN, "update_end_lsn failed", K(tmp_ret), K_(palf_id), K_(self), K(log_id), KPC(log_task)); + PALF_LOG(WARN, "update_end_lsn failed", K(tmp_ret), K_(self), K(log_id), KPC(log_task)); } } const int64_t fs_cb_cost = ObTimeUtility::current_time() - fs_cb_begin_ts; fs_cb_cost_stat_.stat(fs_cb_cost); if (fs_cb_cost > 1 * 1000) { - PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "fs_cb->update_end_lsn() cost too much time", K(tmp_ret), K_(palf_id), K_(self), - K(fs_cb_cost), K(log_id), K(log_begin_lsn), K(log_end_lsn), K(log_proposal_id)); + PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "fs_cb->update_end_lsn() cost too much time", K(tmp_ret), K_(self), + K(fs_cb_cost), K(log_id), K(log_begin_lsn), K(log_end_lsn)); } const int64_t log_life_time = fs_cb_begin_ts - log_gen_ts; @@ -2216,7 +1462,7 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot const int64_t avg_log_gen_to_submit_time = total_log_gen_to_submit_cost / total_slide_log_cnt; const int64_t avg_log_submit_to_flush_time = total_log_submit_to_flush_cost / total_slide_log_cnt; const int64_t avg_log_submit_to_slide_time = total_log_submit_to_slide_cost / total_slide_log_cnt; - PALF_LOG(INFO, "[PALF STAT LOG TASK TIME]", K_(palf_id), K_(self), K(total_slide_log_cnt), + PALF_LOG(INFO, "[PALF STAT LOG TASK TIME]", K_(self), K(total_slide_log_cnt), K(avg_log_gen_to_freeze_time), K(avg_log_gen_to_submit_time), K(avg_log_submit_to_flush_time), K(avg_log_submit_to_slide_time)); ATOMIC_STORE(&accum_slide_log_cnt_, 0); @@ -2228,62 +1474,23 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot if (log_life_time > 100 * 1000) { if (palf_reach_time_interval(100 * 1000, log_life_long_warn_time_)) { - PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "log_task life cost too much time", K_(palf_id), K_(self), K(log_id), KPC(log_task), + PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "log_task life cost too much time", K_(self), K(log_id), KPC(log_task), K(fs_cb_begin_ts), K(log_life_time)); } } // update last_slide_lsn_ if (OB_SUCC(ret)) { - (void) try_update_last_slide_log_info_(log_id, log_max_scn, log_begin_lsn, log_end_lsn, \ - log_proposal_id, log_accum_checksum); + (void) try_update_last_slide_log_info_(log_id, log_max_scn, log_begin_lsn, log_end_lsn, + log_accum_checksum); } - MEM_BARRIER(); // ensure last_slide_log_info_ has been updated before fetch log streamingly - - if (OB_SUCC(ret) - && (FOLLOWER == state_mgr_->get_role() || state_mgr_->is_leader_reconfirm())) { - // Check if need fetch log streamingly, - try_fetch_log_streamingly_(log_end_lsn); - } } } } return ret; } -void LogSlidingWindow::try_fetch_log_streamingly_(const LSN &log_end_lsn) -{ - // This interface is called by sliding_cb() (single-threaded), triggering streaming log pull - LSN last_committed_end_lsn; - last_committed_end_lsn.val_ = ATOMIC_LOAD(&last_fetch_committed_end_lsn_.val_); - if (last_committed_end_lsn.is_valid() && log_end_lsn == last_committed_end_lsn) { - // The end_lsn of this log matches the committed_end_lsn of the previous fetch, indicating that the logs pulled in the previous round - // All committed logs in the middle have slid out, trigger the next round of fetch with (last_submit_log_id + 1) as the new starting point - LSN last_submit_lsn; - LSN last_submit_end_lsn; - int64_t last_submit_log_id = OB_INVALID_LOG_ID; - int64_t last_submit_log_pid = INVALID_PROPOSAL_ID; - (void) get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id, last_submit_log_pid); - const int64_t fetch_start_log_id = last_submit_log_id + 1; - // fetch_log_size need sub MAX_LOG_BUFFER_SIZE to ensure the incoming last fetched log's end_lsn - // is not smaller than last_fetch_end_lsn, then it can successfully trigger next streaming fetch. - // And all the incoming fetched logs can be filled into group_buffer. - const int64_t fetch_log_size = group_buffer_.get_available_buffer_size() - MAX_LOG_BUFFER_SIZE; - const LSN fetch_begin_lsn = last_submit_end_lsn; - const LSN prev_lsn = last_submit_lsn; - ObAddr dest; - if (FOLLOWER == state_mgr_->get_role()) { - get_fetch_log_dst_(dest); - } else if (state_mgr_->is_leader_reconfirm()) { - dest = reconfirm_fetch_dest_; - } else { - // do nothing - } - (void) do_fetch_log_(FetchTriggerType::SLIDING_CB, dest, prev_lsn, \ - fetch_begin_lsn, fetch_log_size, fetch_start_log_id); - } -} bool LogSlidingWindow::is_all_log_flushed_() { @@ -2294,1525 +1501,156 @@ bool LogSlidingWindow::is_all_log_flushed_() get_max_flushed_end_lsn(max_flushed_end_lsn); LSN curr_end_lsn; if (OB_SUCCESS != (tmp_ret = lsn_allocator_.get_curr_end_lsn(curr_end_lsn))) { - PALF_LOG_RET(WARN, tmp_ret, "get_curr_end_lsn failed", K(tmp_ret), K_(palf_id), K_(self)); + PALF_LOG_RET(WARN, tmp_ret, "get_curr_end_lsn failed", K(tmp_ret), K_(self)); } else if (max_flushed_end_lsn < curr_end_lsn) { - PALF_LOG_RET(WARN, OB_EAGAIN, "there is some log has not been flushed", K_(palf_id), K_(self), K(curr_end_lsn), + PALF_LOG_RET(WARN, OB_EAGAIN, "there is some log has not been flushed", K_(self), K(curr_end_lsn), K(max_flushed_end_lsn), K_(max_flushed_lsn)); } else { bool_ret = true; } - PALF_LOG(INFO, "is_all_log_flushed_", K(bool_ret), K_(palf_id), K_(self), K(curr_end_lsn), K(max_flushed_end_lsn)); + PALF_LOG(INFO, "is_all_log_flushed_", K(bool_ret), K_(self), K(curr_end_lsn), K(max_flushed_end_lsn)); return bool_ret; } -int LogSlidingWindow::check_all_log_task_freezed_(bool &is_all_freezed) -{ - int ret = OB_SUCCESS; - is_all_freezed = true; - const int64_t start_log_id = get_start_id(); - const int64_t max_log_id = get_max_log_id(); - LogTask *log_task = NULL; - for (int64_t tmp_log_id = start_log_id; OB_SUCC(ret) && is_all_freezed && tmp_log_id <= max_log_id; ++tmp_log_id) { - LogTaskGuard guard(this); - if (OB_FAIL(guard.get_log_task(tmp_log_id, log_task))) { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K(tmp_log_id), K_(palf_id), K_(self)); - } else { - log_task->lock(); - if (!log_task->is_valid()) { - // Its previous state maybe reconfirm, and it may not fetch all logs form dest follower. - // So maybe there are holes in sw. - PALF_LOG(INFO, "log_task is invalid, this log (hole) may be not received during reconfirm state", K(ret), - K(tmp_log_id), K_(palf_id), K_(self), K(start_log_id), K(max_log_id), KPC(log_task)); - } else if (!log_task->is_freezed()) { - is_all_freezed = false; - PALF_LOG(WARN, "this log_task is not freezed", K(ret), K(tmp_log_id), K(start_log_id), K(max_log_id), - K_(palf_id), K_(self), KPC(log_task)); - } else { - // do nothing - } - log_task->unlock(); - } - } - return ret; -} - -int LogSlidingWindow::freeze_pending_log_(LSN &last_lsn) +int LogSlidingWindow::clean_log() { - // try freeze last log when switch to pending state int ret = OB_SUCCESS; - int64_t last_log_id = OB_INVALID_LOG_ID; - bool is_need_handle = false; - if (OB_FAIL(lsn_allocator_.try_freeze(last_lsn, last_log_id))) { - PALF_LOG(WARN, "lsn_allocator try_freeze failed", K(ret), K_(palf_id), K_(self), K(last_lsn)); - } else if (last_log_id <= 0) { - // no log, no need freeze - } else if (OB_FAIL(try_freeze_last_log_task_(last_log_id, last_lsn, is_need_handle))) { - PALF_LOG(WARN, "try_freeze_last_log_task_ failed", K(ret), K_(palf_id), K_(self), K(last_lsn)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; } else { - const int64_t last_submit_log_id = get_last_submit_log_id_(); - if (last_log_id == last_submit_log_id + 1) { - bool is_committed_lsn_updated = false; - (void) handle_next_submit_log_(is_committed_lsn_updated); - } - PALF_LOG(INFO, "freeze_pending_log success", K(ret), K_(palf_id), K_(self), K(last_lsn)); + ret = clean_log_(); } return ret; } -int LogSlidingWindow::to_follower_pending(LSN &last_lsn) +int LogSlidingWindow::clean_log_() { + // Caller holds palf_handle's wrlock. + // This func is used to clear log tasks beyond the last_submit_log_id in sw. int ret = OB_SUCCESS; - bool is_all_freezed = false; - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); + const int64_t start_log_id = get_start_id(); + const int64_t max_log_id = get_max_log_id(); LSN curr_end_lsn; - if (OB_FAIL(freeze_pending_log_(last_lsn))) { - PALF_LOG(WARN, "freeze_pending_log_ failed", K(ret), K_(palf_id), K_(self), K(last_lsn)); - } else if (OB_FAIL(check_all_log_task_freezed_(is_all_freezed))) { - PALF_LOG(WARN, "freeze_pending_log_ failed", K(ret), K_(palf_id), K_(self), K(last_lsn)); - } else if (!is_all_freezed) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "there is some log has not been freezed", K(ret), K_(palf_id), K_(self), K(last_lsn)); - } else if (OB_FAIL(lsn_allocator_.get_curr_end_lsn(curr_end_lsn))) { - PALF_LOG(WARN, "get_curr_end_lsn failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(group_buffer_.to_follower())) { - PALF_LOG(WARN, "group_buffer_.to_follower failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(reset_match_lsn_map_())) { - PALF_LOG(WARN, "reset_match_lsn_map_ failed", K(ret), K_(palf_id), K_(self)); - } else { - reconfirm_fetch_dest_.reset(); - FLOG_INFO("to_follower_pending success", K(ret), K_(palf_id), K_(self), K(last_lsn), - K(max_flushed_end_lsn), K(curr_end_lsn), K(is_all_freezed), KPC(this)); - } - return ret; -} - -int LogSlidingWindow::clean_cached_log(const int64_t begin_log_id, - const LSN &lsn, - const LSN &prev_lsn, - const int64_t prev_log_pid) -{ - // Caller holds palf_handle's wrlock. - // This func is used to clean cached log_task that has not been processed. - // The arg begin_log_id is expected to be equal to (last_submit_log_id + 1). - // Before executing clean op, we need double check prev_log info. - int ret = OB_SUCCESS; - const int64_t last_submit_log_id = get_last_submit_log_id_(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_INVALID_LOG_ID == begin_log_id) { - ret = OB_INVALID_ARGUMENT; - } else if (begin_log_id != last_submit_log_id + 1) { - // begin_log_id is not larger than last_submit_log_id, last_submit_log maybe changed after check, - // cannot clean cached logs. - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "begin_log_id is not larger than last_submit_log_id, cannot clean", K(ret), - K_(palf_id), K_(self), K(begin_log_id)); - } else { - // double check if prev_log matches with local. - // If not match, report -4109 and do nothing. - bool is_prev_log_exist = false; - const bool is_prev_log_match = is_prev_log_pid_match(begin_log_id, lsn, prev_lsn, \ - prev_log_pid, is_prev_log_exist); - if (false == is_prev_log_match) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "prev_log does not match with local, cannot clean", K(ret), - K_(palf_id), K_(self), K(begin_log_id), K(lsn), K(prev_lsn), K(prev_log_pid)); - // call clean_log_() to reset log_tasks beyond last_submit_log_id. - } else if (OB_FAIL(clean_log_())) { - PALF_LOG(ERROR, "clean_log_ failed", K(ret), K_(palf_id), K_(self), K(last_submit_log_id)); - } else { - // do nothing - } - } - PALF_LOG(INFO, "clean_cached_log finished", K(ret), K_(palf_id), K_(self), K(begin_log_id), K(lsn), K(prev_lsn), K(prev_log_pid)); - return ret; -} - -int LogSlidingWindow::clean_log() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ret = clean_log_(); - } - return ret; -} - -int LogSlidingWindow::clean_log_() -{ - // Caller holds palf_handle's wrlock. - // This func is used to clear log tasks beyond the last_submit_log_id in sw. - int ret = OB_SUCCESS; - const int64_t start_log_id = get_start_id(); - const int64_t max_log_id = get_max_log_id(); - LSN curr_end_lsn; - (void) lsn_allocator_.get_curr_end_lsn(curr_end_lsn); - - int64_t last_slide_log_id = OB_INVALID_LOG_ID; - SCN last_slide_scn; - LSN last_slide_lsn; - LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; - int64_t last_slide_accum_checksum = -1; - get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); - LSN last_submit_lsn; - LSN last_submit_end_lsn; - int64_t last_submit_log_id = OB_INVALID_LOG_ID; - int64_t last_submit_log_pid = INVALID_PROPOSAL_ID; - (void) get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id, last_submit_log_pid); - // new_last_log_xxx are used to truncate lsn_allocator. - int64_t new_last_log_id = OB_INVALID_LOG_ID; - SCN new_last_scn; - LSN new_last_log_end_lsn; - if (last_slide_end_lsn == last_submit_end_lsn) { - new_last_log_id = last_slide_log_id; - new_last_scn = last_slide_scn; - new_last_log_end_lsn = last_slide_end_lsn; - PALF_LOG(INFO, "record last slide log info", K(ret), K(last_slide_log_id), - K(last_slide_scn), K(last_slide_end_lsn), K_(palf_id), K_(self)); - } - - int64_t first_empty_log_id = OB_INVALID_LOG_ID; // record the first hole in sw, just for debug - LogTask *log_task = NULL; - for (int64_t tmp_log_id = start_log_id; OB_SUCC(ret) && tmp_log_id <= max_log_id; ++tmp_log_id) { - LogTaskGuard guard(this); - if (OB_FAIL(guard.get_log_task(tmp_log_id, log_task))) { - // caller hold wrlock, so this step is expected to succeed. - PALF_LOG(ERROR, "get_log_task failed", K(ret), K(tmp_log_id), K_(palf_id), K_(self)); - } else { - log_task->lock(); - if (!log_task->is_valid()) { - PALF_LOG(INFO, "log_task is invalid", K(ret), K(tmp_log_id), K_(palf_id), K_(self), K(first_empty_log_id), - K(max_log_id), KPC(log_task)); - if (OB_INVALID_LOG_ID == first_empty_log_id) { - first_empty_log_id = tmp_log_id; - PALF_LOG(INFO, "found first empty log slot", K(ret), K(tmp_log_id), K_(palf_id), K_(self)); - } - } else { - const SCN curr_scn = log_task->get_max_scn(); - const LSN log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); - PALF_LOG(INFO, "log_task is valid, check if need clean", K(ret), K(tmp_log_id), K_(palf_id), K_(self), KPC(log_task)); - if (log_end_lsn == last_submit_end_lsn) { - if (OB_INVALID_LOG_ID == new_last_log_id) { - // record max flushed log_task info - new_last_log_id = tmp_log_id; - new_last_scn = curr_scn; - new_last_log_end_lsn = log_end_lsn; - PALF_LOG(INFO, "find last submit log_task", K(ret), K(tmp_log_id), K_(palf_id), K_(self), - KPC(log_task), K(last_submit_log_id)); - } - } - if (OB_SUCC(ret)) { - if (OB_INVALID_LOG_ID != last_submit_log_id && tmp_log_id > last_submit_log_id) { - // The logs beyond last_submit_log_id need to be reset(reconfirm). - PALF_LOG(INFO, "clean log task beyond last_submit_log_id", K(ret), K_(palf_id), K_(self), K(max_log_id), K(tmp_log_id), - K(first_empty_log_id), K(last_submit_log_id), KPC(log_task)); - log_task->reset(); - } - } - } - log_task->unlock(); - } - } - if (OB_SUCC(ret)) { - if (OB_INVALID_LOG_ID == new_last_log_id - || !new_last_scn.is_valid() - || !new_last_log_end_lsn.is_valid()) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "last_log info is invalid", K(ret), K(max_log_id), K(first_empty_log_id), K(last_submit_log_id), - K_(palf_id), K_(self), K(new_last_log_end_lsn), K(new_last_log_id), K(new_last_scn), K(start_log_id), K(max_log_id)); - } else if (new_last_log_end_lsn <= curr_end_lsn - && OB_FAIL(truncate_lsn_allocator_(new_last_log_end_lsn, new_last_log_id, new_last_scn))) { - // truncate lsn_allocator_ by new_last_log info - PALF_LOG(ERROR, "truncate_lsn_allocator_ failed", K(ret), K_(palf_id), K_(self), K(new_last_log_id), K(new_last_log_end_lsn), - K(new_last_scn)); - } else { - // do nothing - } - } - PALF_LOG(INFO, "clean log finished", K(ret), K_(palf_id), K_(self), K(max_log_id), K(first_empty_log_id), K(last_submit_log_id), - K(start_log_id), K(max_log_id), K(new_last_log_id), K(new_last_scn), K(new_last_log_end_lsn)); - return ret; -} - -int LogSlidingWindow::to_leader_reconfirm() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(reset_match_lsn_map_())) { - PALF_LOG(WARN, "reset_match_lsn_map_ failed", K(ret), K_(palf_id), K_(self)); - } else { - PALF_LOG(INFO, "to_leader_reconfirm success", K(ret), K_(palf_id), K_(self)); - } - return ret; -} - -int LogSlidingWindow::to_leader_active() -{ - // Check if all group entries have been flushed - // Reset log_tasks' IS_SUBMIT_LOG_EXIST flag - // Resize group_buffer - int ret = OB_SUCCESS; - SCN ref_scn; - int64_t mode_version = INVALID_PROPOSAL_ID; - AccessMode access_mode = AccessMode::INVALID_ACCESS_MODE; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(mode_mgr_->get_access_mode_ref_scn(mode_version, access_mode, ref_scn))) { - PALF_LOG(INFO, "get_access_mode_ref_scn failed", K(ret), K_(palf_id), K_(self)); - } else if (curr_proposal_id < mode_version) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "curr_proposal_id is less than proposal_id in ModeMeta", K(ret), - K_(palf_id), K_(self), K(mode_version), K(curr_proposal_id)); - } else if (!is_all_log_flushed_()) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "to_leader_active need retry, because there is some log has not been flushed", K(ret), - K_(palf_id), K_(self)); - } else if (OB_FAIL(clean_log_())) { - PALF_LOG(INFO, "clean_log_ failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(group_buffer_.to_leader())) { - PALF_LOG(WARN, "group_buffer_.to_leader failed", K(ret), K_(palf_id), K_(self)); - } else if (ref_scn.is_valid() && AccessMode::APPEND == access_mode && - OB_FAIL(lsn_allocator_.inc_update_scn_base(ref_scn))) { - PALF_LOG(ERROR, "inc_update_scn_base failed", K(ret), K_(palf_id), K_(self), K(ref_scn)); - } else if (OB_FAIL(reset_match_lsn_map_())) { - // Reset match_lsn_map to handle case that some follower's match_lsn is larger than - // majority_max_lsn of reconfirm(it has phantom logs generated by old leader). - PALF_LOG(WARN, "reset_match_lsn_map_ failed", K(ret), K_(palf_id), K_(self)); - } else { - reconfirm_fetch_dest_.reset(); - PALF_LOG(INFO, "to_leader_active success", K(ret), K_(palf_id), K_(self)); - } - return ret; -} - -int64_t LogSlidingWindow::get_start_id() const -{ - return sw_.get_begin_sn(); -} - -int LogSlidingWindow::gen_committed_end_lsn_(LSN &new_committed_end_lsn) -{ - int ret = OB_SUCCESS; - ObMemberList curr_member_list, prev_member_list; - int64_t curr_replica_num = 0, prev_replica_num = 0; - LSN curr_result_lsn, prev_result_lsn; - bool is_before_barrier = false; - LSN barrier_lsn; - if (OB_FAIL(mm_->get_log_sync_member_list_for_generate_committed_lsn(prev_member_list, - prev_replica_num, curr_member_list, curr_replica_num, is_before_barrier, barrier_lsn))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(get_majority_lsn_(curr_member_list, curr_replica_num, curr_result_lsn))) { - PALF_LOG(WARN, "get_majority_lsn failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_UNLIKELY(true == is_before_barrier) && - OB_FAIL(get_majority_lsn_(prev_member_list, prev_replica_num, prev_result_lsn))) { - PALF_LOG(WARN, "get_majority_lsn failed", K(ret), K_(palf_id), K_(self)); - } else { - // Note: the leader generates committed_end_lsn based on different memberlists before - // and after a reconfiguration, barrier_lsn is the boundary. - // - Logs which is before barrier_lsn could be committed by previous/current memberlist. - LSN result_lsn = (OB_UNLIKELY(is_before_barrier))? MAX(prev_result_lsn, curr_result_lsn): curr_result_lsn; - // - Logs which is after barrier_lsn should be committed by current memberlist. - // - If current committed_end_lsn is smaller than barrier_lsn, then new committed_end_lsn - // generated by previous memberlist must be smaller than barrier_lsn. - // For example, memberlist:{A} + replica:B. After adding B successfully, Logs after the - // barrier may have been persisted by A, but not B. The leader A can not commit logs after - // the barrier with memberlist:{A}. - result_lsn = (OB_UNLIKELY(is_before_barrier))? MIN(result_lsn, barrier_lsn): result_lsn; - // Note: The leader is not allowed to generate new committed_end_lsn while changing configs with arb. - // 1. {A, B, C(arb)}, A is the leader, end_lsns of A and B are both 100. - // 2. B crashes and A decicdes to degrade B to a learner, A changes memberlist to {A, C(arb)} and - // sends config log to C. While the config log is flying, applications submits logs [100, 200). - // A can not commit [100, 200] logs before A receives ack messages to the config log from C. - // Otherwise, if the config log do not reaches majority and B is elected to be new leader later, - // new leader B may lost committed logs [100, 200). - if (OB_LIKELY(false == state_mgr_->is_changing_config_with_arb())) { - (void) try_advance_committed_lsn_(result_lsn); - } - new_committed_end_lsn = result_lsn; - } - return ret; -} - -int LogSlidingWindow::gen_committed_end_lsn_with_memberlist_( - const ObMemberList &member_list, - const int64_t replica_num) -{ - int ret = OB_SUCCESS; - LSN result_lsn; - if (!member_list.is_valid() || replica_num <= 0 || - replica_num < member_list.get_member_number()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(member_list), K(replica_num)); - } else if (OB_FAIL(get_majority_lsn_(member_list, replica_num, result_lsn))) { - PALF_LOG(WARN, "get_majority_lsn failed", K(ret), K_(palf_id), K_(self)); - } else { - (void) try_advance_committed_lsn_(result_lsn); - PALF_LOG(INFO, "gen_committed_end_lsn_with_memberlist_ finished", K(ret), K_(palf_id), - K_(self), K(result_lsn), K(member_list), K(replica_num)); - } - return ret; -} - -int LogSlidingWindow::get_server_ack_info(const common::ObAddr &server, LsnTsInfo &ack_info) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(false == server.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(server)); - } else { - ObSpinLockGuard guard(match_lsn_map_lock_); - if (OB_FAIL(match_lsn_map_.get(server, ack_info))) { - PALF_LOG(WARN, "match_lsn_map_ get failed", K(ret), K_(palf_id), K_(self), K(server)); - } - } - return ret; -} - -int LogSlidingWindow::get_ack_info_array(LogMemberAckInfoList &ack_info_array) const -{ - int ret = OB_SUCCESS; - common::ObMemberList member_list; - int64_t replica_num = 0; - common::GlobalLearnerList degraded_learner_list; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(mm_->get_log_sync_member_list(member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(mm_->get_degraded_learner_list(degraded_learner_list))) { - PALF_LOG(WARN, "get_degraded_learner_list failed", K(ret), K_(palf_id), K_(self)); - } else { - // TODO by yunlong: optimize for loop - ObSpinLockGuard guard(match_lsn_map_lock_); - ObMember tmp_server; - LsnTsInfo tmp_val; - for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); ++i) { - tmp_server.reset(); - if (OB_FAIL(member_list.get_member_by_index(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(match_lsn_map_.get(tmp_server.get_server(), tmp_val))) { - // Expectation is that it should not fail, synchronize the map with member_list on each member change - PALF_LOG(WARN, "match_lsn_map_ get failed", K(ret), K_(palf_id), K_(self), K(tmp_server)); - } else { - LogMemberAckInfo ack_info; - ack_info.member_ = tmp_server; - ack_info.last_ack_time_us_ = tmp_val.last_ack_time_us_; - ack_info.last_flushed_end_lsn_ = tmp_val.lsn_; - ack_info_array.push_back(ack_info); - PALF_LOG(TRACE, "push ack info for match_lsn_map_ success", K(ack_info)); - } - } - PALF_LOG(TRACE, "begin push ack info for degraded_learner_list", K(degraded_learner_list)); - for (int64_t i = 0; OB_SUCC(ret) && i < degraded_learner_list.get_member_number(); ++i) { - tmp_server.reset(); - if (OB_FAIL(degraded_learner_list.get_learner(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(match_lsn_map_.get(tmp_server.get_server(), tmp_val))) { - ret = OB_SUCCESS; - PALF_LOG(TRACE, "get server from match_lsn_map_ success", K(tmp_server), K(ret)); - } else { - LogMemberAckInfo ack_info; - ack_info.member_ = tmp_server; - ack_info.last_ack_time_us_ = tmp_val.last_ack_time_us_; - ack_info.last_flushed_end_lsn_ = tmp_val.lsn_; - ack_info_array.push_back(ack_info); - PALF_LOG(TRACE, "push ack info for degraded_learner_list success", K(ack_info), K(degraded_learner_list)); - } - } - } - return ret; -} - -int LogSlidingWindow::get_lagged_member_list(const LSN &dst_lsn, ObMemberList &lagged_list) -{ - int ret = OB_SUCCESS; - if (!dst_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else { - GetLaggedListFunc get_lagged_list_func(dst_lsn); - ObSpinLockGuard guard(match_lsn_map_lock_); - if (OB_FAIL(match_lsn_map_.for_each(get_lagged_list_func))) { - PALF_LOG(WARN, "match_lsn_map_.operate() failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(get_lagged_list_func.get_lagged_list(lagged_list))) { - PALF_LOG(WARN, "get_lagged_list failed", K(ret), K_(palf_id), K_(self)); - } else { - PALF_LOG(INFO, "get_lagged_list success", K(ret), K_(palf_id), K_(self), K(dst_lsn), K(lagged_list)); - } - } - return ret; -} - - -int LogSlidingWindow::get_majority_lsn_(const ObMemberList &member_list, - const int64_t replica_num, - LSN &result_lsn) const -{ - int ret = OB_SUCCESS; - assert(replica_num > 0); - LSN lsn_array[OB_MAX_MEMBER_NUMBER]; - int64_t valid_member_cnt = 0; - do { - ObSpinLockGuard guard(match_lsn_map_lock_); - ObAddr tmp_server; - LsnTsInfo tmp_val; - for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); ++i) { - int tmp_ret = OB_SUCCESS; - tmp_server.reset(); - if (OB_FAIL(member_list.get_server_by_index(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_TMP_FAIL(match_lsn_map_.get(tmp_server, tmp_val))) { - // Note: the leader may generate committed_end_lsn based on previous member list, - // members in member_list may do not exist in match_lsn_map. For example, removing D from - // (ABCD), previous member_list is (ABCD) but D has been removed from match_lsn_map. - // Therefore, we just skip members that do not exist in match_lsn_map. - PALF_LOG(WARN, "match_lsn_map_ get failed", K(tmp_ret), K_(palf_id), K_(self), K(tmp_server)); - } else { - lsn_array[valid_member_cnt++] = tmp_val.lsn_; - PALF_LOG(TRACE, "current matched lsn", K_(palf_id), K_(self), "server:", tmp_server, "lsn:", tmp_val.lsn_); - } - } - } while(0); - - if (valid_member_cnt < replica_num / 2 + 1) { - PALF_LOG(WARN, "match_lsn_map do not reach majority", K(ret), K_(palf_id), K_(self), - K(member_list), K(replica_num), K(valid_member_cnt)); - } else if (OB_SUCC(ret)) { - lib::ob_sort(lsn_array, lsn_array + valid_member_cnt, LSNCompare()); - assert(replica_num / 2 < OB_MAX_MEMBER_NUMBER); - result_lsn = lsn_array[replica_num / 2]; - if (!result_lsn.is_valid()) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "result_lsn is invalid, unexpected", K(ret), K_(palf_id), K_(self), K(replica_num)); - } - } - return ret; -} - -bool LogSlidingWindow::is_allow_rebuild() const -{ - // Caller holds palf_handle_impl's rlock. - bool bool_ret = false; - if (IS_INIT) { - bool_ret = !is_truncating_ && !is_rebuilding_; - } - return bool_ret; -} - -int LogSlidingWindow::truncate_for_rebuild(const PalfBaseInfo &palf_base_info) -{ - // Caller holds palf_handle_impl's wrlock. - int ret = OB_SUCCESS; - const LogInfo &prev_log_info = palf_base_info.prev_log_info_; - const int64_t new_start_log_id = prev_log_info.log_id_ + 1; - const int64_t start_log_id = get_start_id(); - const int64_t max_log_id = get_max_log_id(); - // The 'log_id' of migrage src is smaller than self's 'start_log_id' - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (new_start_log_id <= start_log_id) { - PALF_LOG(INFO, "new_start_log_id is smaller than start_log_id, do nothing", K_(palf_id), K_(self), - K(new_start_log_id), K(start_log_id), K(max_log_id), K(palf_base_info)); - // 1) check previous log_tasks - } else { - if (max_log_id >= prev_log_info.log_id_) { - int tmp_ret = OB_SUCCESS; - LSN local_prev_lsn, local_prev_end_lsn; - SCN local_prev_scn; - int64_t local_prev_log_pid = INVALID_PROPOSAL_ID; - int64_t local_prev_log_accum_checksum = -1; - if (OB_SUCCESS != (tmp_ret = get_prev_log_info_(new_start_log_id, local_prev_lsn, \ - local_prev_end_lsn, local_prev_scn, local_prev_log_pid, local_prev_log_accum_checksum))) { - if (OB_ENTRY_NOT_EXIST != tmp_ret) { - ret = tmp_ret; - PALF_LOG(WARN, "get_prev_log_info_ failed", K(ret), K(new_start_log_id), K(palf_base_info), K_(palf_id), K_(self)); - } else { - LSN last_slide_end_lsn; - get_last_slide_end_lsn_(last_slide_end_lsn); - PALF_LOG(WARN, "prev_log does not exist", K(ret), K(tmp_ret), K(new_start_log_id), K(palf_base_info), K_(palf_id), K_(self), - K(last_slide_end_lsn)); - } - } else if (local_prev_lsn != prev_log_info.lsn_ - || local_prev_log_pid != prev_log_info.log_proposal_id_) { - PALF_LOG(INFO, "prev_log does not match with local during rebuild", K(ret), K_(palf_id), K_(self), K(start_log_id), - K(max_log_id), K(new_start_log_id), K(palf_base_info), K(local_prev_lsn), K(local_prev_log_pid)); - } else {} - } - - // 2) truncate log_task before new_start_log_id - if (OB_SUCC(ret)) { - // inc sw's start_id - if (OB_FAIL(sw_.truncate_and_reset_begin_sn(new_start_log_id))) { - PALF_LOG(WARN, "sw_.truncate_and_reset_begin_sn failed", K(ret), K_(palf_id), K_(self), K(start_log_id), - K(new_start_log_id)); - } - } - - // 3) truncate lsn_allocator_ - if (OB_SUCC(ret)) { - // Different from the ordinary truncate log scenario, here we only increase last_log_meta, with no need for rollback - // Only when curr_end_lsn < palf_base_info.curr_lsn_ does an update need to be made - LSN curr_end_lsn; - (void) lsn_allocator_.get_curr_end_lsn(curr_end_lsn); - if (curr_end_lsn.is_valid() - && curr_end_lsn < palf_base_info.curr_lsn_) { - if (OB_FAIL(lsn_allocator_.truncate(palf_base_info.curr_lsn_, prev_log_info.log_id_, prev_log_info.scn_))) { - PALF_LOG(WARN, "truncate lsn_allocator_ failed", K(ret), K_(palf_id), K_(self), K(curr_end_lsn), K(palf_base_info)); - } else { - PALF_LOG(INFO, "truncate lsn_allocator_ success", K(ret), K_(palf_id), K_(self), K(curr_end_lsn), K(palf_base_info)); - } - } - } - // 4) inc update last_submit_log info, max_flushed_log info, last_slide_log info and max_committed_log info - // NB: These values cannot be rolled back, otherwise log overwrite writing will occur subsequently, and the log_storage layer will report an error - if (OB_SUCC(ret)) { - LSN last_submit_lsn; - LSN last_submit_end_lsn; - int64_t last_submit_log_id = OB_INVALID_LOG_ID; - int64_t last_submit_log_pid = INVALID_PROPOSAL_ID; - (void) get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id, last_submit_log_pid); - - if (last_submit_end_lsn <= palf_base_info.curr_lsn_) { - (void) set_last_submit_log_info_(prev_log_info.lsn_, palf_base_info.curr_lsn_, \ - prev_log_info.log_id_, prev_log_info.log_proposal_id_); - // update local accum_checksum when last_submit_log_info updated - checksum_.set_accum_checksum(prev_log_info.accum_checksum_); - } - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - if (max_flushed_end_lsn <= palf_base_info.curr_lsn_) { - (void) truncate_max_flushed_log_info_(prev_log_info.lsn_, palf_base_info.curr_lsn_, - prev_log_info.log_proposal_id_); - (void) group_buffer_.truncate(palf_base_info.curr_lsn_); - } - const int64_t last_slide_log_id = get_last_slide_log_id_(); - if (last_slide_log_id <= prev_log_info.log_id_) { - (void) try_update_last_slide_log_info_(prev_log_info.log_id_, prev_log_info.scn_, \ - prev_log_info.lsn_, palf_base_info.curr_lsn_, prev_log_info.log_proposal_id_, \ - prev_log_info.accum_checksum_); - // update local verify_checksum when last_slide_log_info updated - checksum_.set_verify_checksum(prev_log_info.accum_checksum_); - } - is_rebuilding_ = true; - last_rebuild_lsn_ = palf_base_info.curr_lsn_; - // Note: can not inc update committed_end_lsn in here, otherwise iterator will read uncommitted log unexpectedly. - } - } - // TODO by yunlong: dump all log_info before and after truncating - PALF_LOG(INFO, "truncate_for_rebuild finished", K(ret), K_(palf_id), K_(self), K(palf_base_info), K(start_log_id), - K(max_log_id)); - return ret; -} - -int LogSlidingWindow::truncate(const TruncateLogInfo &truncate_log_info, const LSN &expected_prev_lsn, - const int64_t expected_prev_log_pid) -{ - // Caller holds palf_handle_impl's wrlock. - int ret = OB_SUCCESS; - LogTask *log_task = NULL; - LogTaskGuard guard(this); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!truncate_log_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(truncate_log_info)); - } else if (OB_FAIL(guard.get_log_task(truncate_log_info.truncate_log_id_, log_task))) { - PALF_LOG(WARN, "get_log_task failed when truncate log", K(ret), K(truncate_log_info), K_(palf_id), K_(self)); - } else { - // check if dst log matches with arg - const int64_t truncate_log_id = truncate_log_info.truncate_log_id_; - const LSN &truncate_begin_lsn = truncate_log_info.truncate_begin_lsn_; - const int64_t truncate_log_proposal_id = truncate_log_info.truncate_log_proposal_id_; - LSN log_begin_lsn; - LSN log_end_lsn; - int64_t prev_log_id = truncate_log_id - 1; - LSN prev_lsn, prev_end_lsn; - SCN prev_scn; - int64_t prev_proposal_id = INVALID_PROPOSAL_ID; - int64_t prev_accum_checksum = -1; - SCN max_scn; - // double check the prev log is not changed - bool is_prev_log_exist = false; - const bool is_prev_log_match = is_prev_log_pid_match(truncate_log_id, truncate_begin_lsn, expected_prev_lsn, - expected_prev_log_pid, is_prev_log_exist); - log_task->lock(); - if (false == is_prev_log_match - || !log_task->is_valid() - || log_task->get_log_id() != truncate_log_id - || log_task->get_proposal_id() != truncate_log_proposal_id) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "log/prev_log does not match, it may has changed", K(ret), K_(palf_id), K_(self), K(truncate_log_info), - K(expected_prev_lsn), K(expected_prev_log_pid), KPC(log_task), K(is_prev_log_exist), - K(is_prev_log_match)); - } else { - log_begin_lsn = log_task->get_begin_lsn(); - log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); - prev_lsn = log_task->get_prev_lsn(); - prev_proposal_id = log_task->get_prev_proposal_id(); - max_scn = log_task->get_max_scn(); - log_task->reset(); - } - log_task->unlock(); - // get prev log info - if (OB_SUCC(ret)) { - if (FIRST_VALID_LOG_ID == truncate_log_id) { - // The first log needs to be truncated, prev_log_info is generated as the initial value - LogInfo prev_log_info; - prev_log_info.generate_by_default(); - prev_log_id = prev_log_info.log_id_; - prev_lsn = prev_log_info.lsn_; - prev_scn = prev_log_info.scn_; - prev_proposal_id = prev_log_info.log_proposal_id_; - prev_accum_checksum = prev_log_info.accum_checksum_; - } else if (OB_FAIL(get_prev_log_info_(truncate_log_id, prev_lsn, prev_end_lsn, prev_scn, prev_proposal_id, prev_accum_checksum))) { - PALF_LOG(WARN, "get_prev_log_info_ failed when truncate log", K(ret), K_(palf_id), K_(self), K(truncate_log_info), KPC(log_task)); - } else { - // do nothing - } - } - // NB: must revert log_task manually in here to avoid truncate getting stuck - guard.revert_log_task(); - // truncate sw and log storage - if (OB_SUCC(ret)) { - TruncateLogCbCtx truncate_log_cb_ctx(truncate_begin_lsn); - LSN committed_end_lsn; - get_committed_end_lsn_(committed_end_lsn); - if (truncate_begin_lsn < committed_end_lsn) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "truncate begin lsn is less than committed_end_lsn, unexpected", K(ret), K_(palf_id), K_(self), - K(truncate_log_info), K(committed_end_lsn)); - // Note: order is vital. If the sliding window has already been truncated, - // but it fails to submit truncate task. LogEngine will encounter an error - // when appending logs, because the logs in disk have not been truncated. - } else if (OB_FAIL(log_engine_->submit_truncate_log_task(truncate_log_cb_ctx))) { - PALF_LOG(WARN, "log_engine_ truncate failed", K(ret), K_(palf_id), K_(self), K(truncate_log_info)); - } else if (OB_FAIL(sw_.truncate(truncate_log_id))) { - PALF_LOG(WARN, "sw_ truncate failed", K(ret), K_(palf_id), K_(self), K(truncate_log_info)); - } else { - // update truncating state - is_truncating_ = true; - last_truncate_lsn_ = truncate_begin_lsn; - - // truncate max log meta of lsn_allocator - (void) truncate_lsn_allocator_(truncate_begin_lsn, prev_log_id, prev_scn); - // truncate last_submit_log info - // Truncate last_submit_log_info only when the truncated log has been submitted - const int64_t last_submit_log_id = get_last_submit_log_id_(); - if (last_submit_log_id >= truncate_log_id) { - (void) set_last_submit_log_info_(prev_lsn, truncate_begin_lsn, prev_log_id, prev_proposal_id); - // reset accum_checksum only when last_submit_log_id >= log_id (larger than prev_log_id) - checksum_.set_accum_checksum(prev_accum_checksum); - PALF_LOG(INFO, "truncate last_submit_log_info_ and accum_checksum", K_(palf_id), K_(self), K(truncate_log_info), K(last_submit_log_id), - K(prev_accum_checksum)); - } - // truncate max_flushed_log info - // Considering there might be holes before the truncate point in sw, we need to compare it with the current max_flushed_end_lsn here - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - if (max_flushed_end_lsn > truncate_begin_lsn) { - // flush point has been pushed, need to truncate - (void) truncate_max_flushed_log_info_(prev_lsn, truncate_begin_lsn, prev_proposal_id); - (void) group_buffer_.truncate(truncate_begin_lsn); - PALF_LOG(INFO, "truncate max_flushed_log_info_", K_(palf_id), K_(self), K(truncate_log_info), K(log_end_lsn), - "old flushed_end_lsn", max_flushed_end_lsn); - } - PALF_LOG(INFO, "truncate success", K(ret), K_(palf_id), K_(self), K(truncate_log_info), "max_log_id", get_max_log_id(), - K(log_begin_lsn), K(expected_prev_lsn), K(expected_prev_log_pid), K(prev_accum_checksum)); - } - } - } - return ret; -} - -int LogSlidingWindow::receive_log(const common::ObAddr &src_server, - const PushLogType push_log_type, - const LSN &prev_lsn, - const int64_t &prev_log_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len, - const bool need_check_clean_log, - TruncateLogInfo &truncate_log_info) -{ - int ret = OB_SUCCESS; - int64_t log_id = OB_INVALID_LOG_ID; - LSN last_submit_lsn; - LSN last_submit_end_lsn; - int64_t last_submit_log_id = OB_INVALID_LOG_ID; - int64_t last_submit_log_pid = INVALID_PROPOSAL_ID; - get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id, last_submit_log_pid); - int64_t log_proposal_id = INVALID_PROPOSAL_ID; - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - int64_t pos = 0; - LogGroupEntryHeader group_entry_header; - int64_t group_log_data_checksum = 0; - LSN log_end_lsn; - LogTaskGuard guard(this); - LogTask *log_task = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!src_server.is_valid() || !lsn.is_valid() || NULL == buf || buf_len <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(src_server), K(lsn), - KP(buf), K(buf_len)); - } else if (is_truncating_ || is_rebuilding_) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "sw is truncatiing/rebuilding now, cannot receive log", K(ret), K_(palf_id), K_(self), K_(is_truncating), - K_(last_truncate_lsn), K_(is_rebuilding), K(src_server), K(lsn), KP(buf), K(buf_len)); - } else if (!group_buffer_.can_handle_new_log(lsn, buf_len)) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(1000 * 1000)) { - PALF_LOG(WARN, "group_buffer_ cannot handle new log", K(ret), K_(palf_id), K_(self), K(lsn)); - } - } else if (OB_FAIL(group_entry_header.deserialize(buf, buf_len, pos))) { - PALF_LOG(WARN, "group_entry_header deserialize failed", K(ret), K_(palf_id), K_(self)); - } else if (!group_entry_header.check_integrity(buf + LogGroupEntryHeader::HEADER_SER_SIZE, - buf_len - LogGroupEntryHeader::HEADER_SER_SIZE, group_log_data_checksum)) { - ret = OB_INVALID_DATA; - PALF_LOG(WARN, "group_entry_header check_integrity failed", K(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(log_id = group_entry_header.get_log_id())) { - } else if (!can_receive_larger_log_(log_id)) { - ret = OB_EAGAIN; - if (palf_reach_time_interval(5 * 1000 * 1000, larger_log_warn_time_)) { - PALF_LOG(WARN, "sw is full, cannot receive larger log", K(ret), K_(palf_id), K_(self), K(group_entry_header), - "start_id", get_start_id()); - } - } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "this log has slide out, no need receive", K(ret), K(log_id), K_(palf_id), K_(self)); - } - } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self), K(group_entry_header)); - } - } else { - log_proposal_id = group_entry_header.get_log_proposal_id(); - log_end_lsn = lsn + group_entry_header.get_serialize_size() + group_entry_header.get_data_len(); - bool is_prev_log_exist = false; - const bool is_prev_log_match = is_prev_log_pid_match(log_id, lsn, prev_lsn, \ - prev_log_proposal_id, is_prev_log_exist); - bool need_send_ack = false; - bool is_log_pid_match = false; - SCN min_scn; - if (is_prev_log_exist && !is_prev_log_match) { - // prev log exists and its proposal_id does not match with arg, cannot receive this log - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "previous log does not match", K(ret), K_(palf_id), K_(self), K(log_id), K(log_proposal_id), - K(prev_log_proposal_id), K(is_prev_log_exist), K(group_entry_header)); - } - // If log_id > last_submit_log_id + 1, check if we can cache it. - if (OB_SUCC(ret) - && log_id > (last_submit_log_id + 1) - && log_proposal_id != last_submit_log_pid) { - // if its proposal_id does not equal to last_submit_log_pid, - // and it's not continuous with last_submit_log_id, we cannot receive it. - // Only logs whose proposal_id is equal to last_submit_log_pid can be cached into sw. - ret = OB_EAGAIN; - PALF_LOG(ERROR, "new log's proposal_id does not equal to last submit log's, and log_id is not continuous with last "\ - "submit log, cannot receive(cache) it", K(ret), K_(palf_id), K_(self), K(log_id), K(log_proposal_id), - K(prev_log_proposal_id), K(is_prev_log_exist), K(group_entry_header), K(last_submit_log_id), K(last_submit_log_pid)); - } - // check if need clean cached log_tasks. - if (OB_SUCC(ret) - && is_prev_log_match - && need_check_clean_log - && log_id == last_submit_log_id + 1 - && log_proposal_id != last_submit_log_pid) { - // prev log matches, new log's proposal_id does not equal to last submit log's, - // and it is continuous with last submit log, - // check if need clean cached log_tasks. - if (INVALID_PROPOSAL_ID != last_submit_log_pid && log_proposal_id < last_submit_log_pid) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "new log's proposal_id is smaller than last submit log, unexpected", K(ret), - K_(palf_id), K_(self), K(log_id), K(log_proposal_id), K(is_prev_log_exist), - K(group_entry_header), K(last_submit_log_id), K(last_submit_log_pid)); - } else { - ret = OB_EAGAIN; - // new log's proposal_id is larger than last submit log, and prev log check successfully, - // we need clean cached log_tasks whose log_id is larger than this log. - // And we reuse the truncate_log_id_ of truncate_log_info to output dest log_id. - truncate_log_info.truncate_type_ = TRUNCATE_CACHED_LOG_TASK; - truncate_log_info.truncate_log_id_ = log_id; - } - PALF_LOG(INFO, "new log's proposal_id does not equal to last submit log's, and it is continuous, "\ - "need clean cached log_tasks beyond this log", K(ret), K_(palf_id), K_(self), K(log_id), K(log_proposal_id), - K(is_prev_log_exist), K(group_entry_header), K(last_submit_log_id), K(last_submit_log_pid)); - } - - // check if need update log_task. - if (OB_SUCC(ret)) { - bool is_local_log_valid = false; - if (!need_update_log_task_(group_entry_header, log_task, need_send_ack, is_local_log_valid, is_log_pid_match)) { - PALF_LOG(INFO, "no need update log", K(log_id), K_(palf_id), K_(self), K(need_send_ack), K(is_log_pid_match), - K(is_local_log_valid), K(is_prev_log_exist), K(is_prev_log_match), K(group_entry_header), KPC(log_task)); - if (false == is_local_log_valid) { - // local log_task is invalid, and it does not need update, this means that it's maybe in PRE_FILL state. - } else if (false == is_log_pid_match) { - // local log_task's proposal_id does not match with new log. - // - // (log_id <= last_submit_log_id) must be true. - // - // Because if log_id > last_submit_log_id, there are only too cases: - // 1) new log_proposal_id != last_submit_log_pid, it cannot reach here, because it cannot be received. - // - // 2) new log_proposal_id == last_submit_log_pid, the local log_task's propsal_id should be equal - // with last_submit_log_pid too. If not, local log shouldn't exist (it should already be - // truncated by previous receive operation). - // - // In summary, (log_id <= last_submit_log_id) must be true. - if (lsn <= last_submit_end_lsn) { - // It means that this log is the first mismatch one with request server, - // because it has passed prev log check. - // We need truncate log at this log's begin lsn. - truncate_log_info.truncate_type_ = TRUNCATE_LOG; - truncate_log_info.truncate_log_id_ = log_id; - // lsn is expected to be equal to log_task's end_lsn. - // So we can use lsn as the truncate_begin_lsn_. - truncate_log_info.truncate_begin_lsn_ = lsn; - log_task->lock(); - truncate_log_info.truncate_log_proposal_id_ = log_task->get_proposal_id(); - log_task->unlock(); - // return -4023 when need truncate - ret = OB_EAGAIN; - PALF_LOG(WARN, "need truncate log", K(ret), K(log_id), K_(palf_id), K_(self), K(lsn), K(last_submit_end_lsn), - K(last_submit_log_id), KPC(log_task), K(is_prev_log_exist), K(is_prev_log_match), - K(truncate_log_info)); - } else { - // Unexpected case: - // (log_id <= last_submit_log_id) and (lsn > last_submit_end_lsn). - // If the expected truncating log is some previous one, - // this log cannot pass the prev log check. - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "lsn > last_submit_end_lsn and log_id <= last_submit_log_id, \ - and local log_task's proposal_id != arg proposal_id, unexpected", - K(ret), K(log_id), K_(palf_id), K_(self), K(lsn), K(last_submit_end_lsn), K(last_submit_log_id), KPC(log_task), - K(is_prev_log_exist), K(is_prev_log_match)); - } - if (state_mgr_->is_leader_reconfirm()) { - // NB: In leader reconfirm, it's not possibale truncate log, because we guarantee that: - // 1. election leader has the highest config version; - // 2. changing config must need forward check; - // 3. receiving config log need clean rubbish logs afterwards; - // 3. only after start working, election leader can become palf leader. - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected truncate for reconfirm state", K(ret), K(log_id), K_(palf_id), K_(self), K(need_send_ack), - K(truncate_log_info), K(group_entry_header), KPC(log_task), K(src_server), K(push_log_type)); - } - } else if (need_send_ack) { - // This log matches with msg and it has been flushed, just sending ack directly. - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (OB_FAIL(submit_push_log_resp_(src_server, curr_proposal_id, log_end_lsn, PushLogType::FETCH_LOG_RESP == push_log_type))) { - PALF_LOG(WARN, "submit_push_log_resp failed", K(ret), K_(palf_id), K_(self), K(src_server)); - } else { - PALF_LOG(INFO, "submit_push_log_resp succ", K(ret), K_(palf_id), K_(self), K(src_server), K(curr_proposal_id), - K(log_proposal_id), K(log_end_lsn), K(group_entry_header)); - } - } else { - // This log_task matches with msg and does not need to update. - } - } else { - // Need update log_task, it means that PRE_FILL tag must be set by self. - if (lsn < max_flushed_end_lsn) { - // lsn is smaller than max_flushed_end_lsn, prev log may not exist - // This log may need to be truncated later (e.g., is inconsistent with the source), to avoid overwriting existing local logs, we need to reject it first, - // Wait for the previous log update and then pull again - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "lsn is smaller than max_flushed_end_lsn, this log may need truncate later, cannot receive", - K(ret), K_(palf_id), K_(self), K(lsn), K(log_end_lsn), K(max_flushed_end_lsn), K(group_entry_header), - K(is_prev_log_exist), K(is_prev_log_match), K(last_submit_log_id), K(last_submit_log_pid)); - } else if (lsn < last_submit_end_lsn) { - // lsn < last_submit_end_lsn, it is unexpected. - // This log should have been processed(no need update) or has slid out(get log_task will encounter OUT OF LOWER BOUND). - // For any case, it shouldn't run here. - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "lsn is smaller than last_submit_end_lsn, unexpected", - K(ret), K_(palf_id), K_(self), KPC(log_task), K(lsn), K(log_end_lsn), K(max_flushed_end_lsn), K(group_entry_header), - K(is_prev_log_exist), K(is_prev_log_match), K(last_submit_log_id), K(last_submit_log_pid), - K(last_submit_log_id), K(last_submit_lsn), K(last_submit_end_lsn)); - } else if (OB_FAIL(get_min_scn_from_buf_(group_entry_header, buf + LogGroupEntryHeader::HEADER_SER_SIZE, - buf_len - LogGroupEntryHeader::HEADER_SER_SIZE, min_scn))) { - PALF_LOG(WARN, "get_min_scn_from_buf_ failed", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header)); - // local log_task is invalid, receive it. - // Here wait_group_buffer_ready_ is not needed, because lsn is determined and unique, a prior check is sufficient - // fill needs to do memory copy, hence it cannot hold the log_task's lock, use PRE_FILL to control that at most one thread executes fill. - } else if (OB_FAIL(group_buffer_.fill(lsn, buf, buf_len))) { - PALF_LOG(ERROR, "fill group buffer failed", K(ret), K_(palf_id), K_(self), K(group_entry_header)); - } else if (OB_FAIL(try_update_max_lsn_(lsn, group_entry_header))) { - PALF_LOG(WARN, "try_update_max_lsn_ failed", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header)); - } else { - log_task->lock(); - if (log_task->is_valid()) { - // log_task may be concurrently collected by other threads, expected content is consistent with this thread. - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_task has been updated during filling group buffer, unexpected", K(ret), K_(palf_id), - K_(self), K(lsn), K(group_entry_header), KPC(log_task)); - } else if (OB_FAIL(log_task->set_group_header(lsn, min_scn, group_entry_header))) { - PALF_LOG(ERROR, "log_task->set_group_header failed", K(ret), K_(palf_id), K_(self), K(group_entry_header)); - } else { - // update prev_lsn - log_task->set_prev_lsn(prev_lsn); - // update prev_log_proposal_id - log_task->set_prev_log_proposal_id(prev_log_proposal_id); - // update group log data_checksum - log_task->set_group_log_checksum(group_log_data_checksum); - (void) log_task->set_freezed(); - log_task->set_freeze_ts(ObTimeUtility::current_time()); - log_task->set_push_log_type(push_log_type); - } - log_task->unlock(); - } - // Reset PRE_FILL tag unconditionally. - log_task->reset_pre_fill(); - } - PALF_LOG(TRACE, "receive_log", K(ret), K_(palf_id), K_(self), K(src_server), K(group_entry_header), - K(log_id), KPC(log_task)); - // check if need reset last fetch log info - LSN last_fetch_end_lsn; - last_fetch_end_lsn.val_ = ATOMIC_LOAD(&last_fetch_end_lsn_.val_); - if (OB_SUCC(ret) - && (PUSH_LOG == push_log_type || PUSH_LOG_WO_ACK == push_log_type) - && last_fetch_end_lsn.is_valid() - && log_end_lsn <= last_fetch_end_lsn) { - // Only when the end_lsn and log_id received from push log are within the last fetch range - // can only reset fetch info - (void) try_reset_last_fetch_log_info_(last_fetch_end_lsn, log_id); - } - } - } - // Note: can not use log_task below this line - guard.revert_log_task(); - - if (OB_SUCC(ret)) { - bool is_committed_lsn_updated = false; - // try handle submit log - if (log_id == get_last_submit_log_id_() + 1) { - (void) handle_next_submit_log_(is_committed_lsn_updated); - } - if (is_committed_lsn_updated) { - // handle committed logs - (void) handle_committed_log_(); - } - // check and try fetch log for proposal_id increasing log - if (false == need_check_clean_log - && log_id == last_submit_log_id + 1 - && log_proposal_id != last_submit_log_pid) { - // this log will leads to last_submit_log_pid increase, - // some logs with new proposal_id may have been dropped before it arrives, - // so we need trigger fetching to get these dropped logs quickly. - (void) try_fetch_log(CLEAN_CACHED_LOG, lsn, log_end_lsn, log_id + 1); - } - } - return ret; -} - -int LogSlidingWindow::submit_push_log_resp(const common::ObAddr &server) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - // Follower replies local committed_end_lsn to reconfirming leader. - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - LSN committed_end_lsn; - get_committed_end_lsn_(committed_end_lsn); - const bool is_fetch_log = false; - ret = submit_push_log_resp_(server, curr_proposal_id, committed_end_lsn, is_fetch_log); - } - return ret; -} - -int LogSlidingWindow::submit_push_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &log_end_lsn, - const bool &is_fetch_log) -{ - int ret = OB_SUCCESS; - const bool is_need_batch = need_use_batch_rpc_(0, is_fetch_log); - if (state_mgr_->is_allow_vote() && - OB_FAIL(log_engine_->submit_push_log_resp(server, msg_proposal_id, log_end_lsn, is_need_batch))) { - PALF_LOG(WARN, "submit_push_log_resp failed", K(ret), K_(palf_id), K_(self), K(server), K(log_end_lsn)); - } - return ret; -} - -int LogSlidingWindow::submit_group_log(const LSN &lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - int64_t log_id = OB_INVALID_LOG_ID; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!lsn.is_valid() || NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BUFFER_SIZE) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(lsn), - KP(buf), K(buf_len)); - } else if (is_truncating_) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "sw is executing truncation now, unexpected state", K(ret), K_(palf_id), K_(self), - K(lsn), KP(buf), K(buf_len)); - } else { - LogGroupEntryHeader group_entry_header; - LogTask *log_task = NULL; - LogTaskGuard guard(this); - int64_t group_log_data_checksum = 0; - int64_t pos = 0; - LSN last_slide_end_lsn; - get_last_slide_end_lsn_(last_slide_end_lsn); - - if (OB_FAIL(group_entry_header.deserialize(buf, buf_len, pos))) { - PALF_LOG(WARN, "group_entry_header deserialize failed", K(ret), K_(palf_id), K_(self)); - } else if (lsn < last_slide_end_lsn && group_entry_header.get_log_id() < get_start_id()) { - // raw_write may submit an old group_log which is smaller than start log of sw, - // just return success for this case. - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - PALF_LOG(INFO, "this group_log has slid out, no need submit", K(ret), K_(palf_id), K_(self), - K(lsn), K(last_slide_end_lsn), K(group_entry_header)); - } - } else if (!leader_can_submit_group_log_(lsn, buf_len)) { - LSN curr_max_lsn; - (void) lsn_allocator_.get_curr_end_lsn(curr_max_lsn); - if (lsn >= curr_max_lsn && group_entry_header.get_log_id() < get_start_id()) { - // This group log's lsn is larger than max_lsn, but its log_id is less than sw_start_id. - // This secenio is unexpected. - ret = OB_ERR_UNEXPECTED; - } else { - ret = OB_EAGAIN; - } - if (REACH_TIME_INTERVAL(1000 * 1000)) { - PALF_LOG(ERROR, "leader cannot submit group log", K(ret), K_(palf_id), K_(self), K(lsn), K(buf_len)); - } - } else if (!group_entry_header.check_integrity(buf + LogGroupEntryHeader::HEADER_SER_SIZE, - buf_len - LogGroupEntryHeader::HEADER_SER_SIZE, group_log_data_checksum)) { - ret = OB_INVALID_DATA; - PALF_LOG(WARN, "group_entry_header check_integrity failed", K(ret), K_(palf_id), K_(self)); - } else if (!leader_can_submit_larger_log_(group_entry_header.get_log_id())) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "sw is full, cannot receive larger log", K(ret), K_(palf_id), K_(self), K(group_entry_header), - "start_id", get_start_id(), K(lsn)); - } - } else if ((log_id = group_entry_header.get_log_id()) < get_start_id()) { - PALF_LOG(WARN, "this log has slided out, no need receive", K(ret), K_(palf_id), K_(self), - K(group_entry_header), "start_id", get_start_id()); - // update log proposal_id - } else if (OB_FAIL(group_entry_header.update_log_proposal_id(curr_proposal_id))) { - PALF_LOG(WARN, "group_entry_header update_log_proposal_id failed", K(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(group_entry_header.update_write_mode(true))) { - } else { - // update header's committed_end_lsn - LSN commited_end_lsn; - get_committed_end_lsn_(commited_end_lsn); - if (OB_FAIL(group_entry_header.update_committed_end_lsn(commited_end_lsn))) { - PALF_LOG(WARN, "update commited_end_lsn failed", K(ret), K(commited_end_lsn)); - } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { - if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { - PALF_LOG(WARN, "this log has slide out, no need receive", K(ret), K(log_id), K_(palf_id), K_(self)); - } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self)); - } - } else { - // get log_task success - } - if (OB_SUCC(ret)) { - log_task->lock(); - SCN min_scn; - if (log_task->is_valid()) { - if (lsn != log_task->get_begin_lsn() - || group_entry_header.get_max_scn() != log_task->get_max_scn() - || group_entry_header.get_accum_checksum() != log_task->get_accum_checksum()) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "local log_task is valid, but its content does not match "\ - "with argument, unexpected", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header), KPC(log_task)); - } else { - PALF_LOG(INFO, "log_task is already valid, no need receive log", K(ret), K(log_id), K_(palf_id), K_(self), - K(group_entry_header), KPC(log_task)); - } - } else if (OB_FAIL(get_min_scn_from_buf_(group_entry_header, buf + LogGroupEntryHeader::HEADER_SER_SIZE, - buf_len - LogGroupEntryHeader::HEADER_SER_SIZE, min_scn))) { - PALF_LOG(WARN, "get_min_scn_from_buf_ failed", K(ret), K_(palf_id), K_(self)); - // Here wait_group_buffer_ready_ is not needed, because lsn is determined and unique, thus a prior check is sufficient. - // During the fill period, the lock of log_task cannot be held, because it may take a long time. - } else if (OB_FAIL(group_buffer_.fill(lsn, buf, buf_len))) { - PALF_LOG(WARN, "fill group buffer failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(try_update_max_lsn_(lsn, group_entry_header))) { - PALF_LOG(WARN, "try_update_max_lsn_ failed", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header)); - } else { - // prev_log_proposal_id match or not exist, receive this log - if (log_task->is_valid()) { - // log_task may be concurrently collected by other threads, expected content is consistent with this thread. - if (group_entry_header.get_log_proposal_id() != log_task->get_proposal_id()) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_task has been updated during filling group buffer, and log proposal_id does not match "\ - "with this req, unexpected", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header), KPC(log_task)); - } - } else if (OB_FAIL(log_task->set_group_header(lsn, min_scn, group_entry_header))) { - PALF_LOG(WARN, "log_task->set_group_header failed", K(ret), K_(palf_id), K_(self)); - } else { - // update group log data_checksum - log_task->set_group_log_checksum(group_log_data_checksum); - (void) log_task->set_submit_log_exist(); - (void) log_task->set_freezed(); - log_task->set_freeze_ts(ObTimeUtility::current_time()); - } - - PALF_LOG(TRACE, "submit_group_log", K(ret), K_(palf_id), K_(self), K(group_entry_header), - K(log_id), KPC(log_task)); - } - log_task->unlock(); - } - } - } - if (OB_SUCC(ret)) { - bool is_committed_lsn_updated = false; - // try submit log - const int64_t last_submit_log_id = get_last_submit_log_id_(); - if (log_id == last_submit_log_id + 1) { - (void) handle_next_submit_log_(is_committed_lsn_updated); - } - // handle committed logs - if (is_committed_lsn_updated) { - (void) handle_committed_log_(); - } - } - return ret; -} - -bool LogSlidingWindow::need_update_log_task_(LogGroupEntryHeader &header, - LogTask *log_task, - bool &need_send_ack, - bool &is_local_log_valid, - bool &is_log_pid_match) const -{ - bool bool_ret = false; - const int64_t log_id = header.get_log_id(); - if (IS_NOT_INIT) { - } else if (!header.is_valid() || NULL == log_task) { - PALF_LOG_RET(WARN, OB_INVALID_ARGUMENT, "invalid argumetns", K_(palf_id), K_(self), K(header), KP(log_task)); - } else { - int64_t old_pid = INVALID_PROPOSAL_ID; - LSN log_end_lsn; - - log_task->lock(); - if (!log_task->is_valid()) { - if (log_task->try_pre_fill()) { - bool_ret = true; - } else { - // Other thread has set PRE_FILL tag, skip update. - bool_ret = false; - } - } else { - is_local_log_valid = true; - old_pid = log_task->get_proposal_id(); - log_end_lsn = (log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len()); - } - log_task->unlock(); - - if (false == bool_ret) { - int64_t new_pid = header.get_log_proposal_id(); - if (INVALID_PROPOSAL_ID == old_pid) { - // current pid of log_task is invalid, it's maybe in PRE_FILL state, skip - } else if (old_pid == new_pid) { - is_log_pid_match = true; - // check if need send ack - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - if (max_flushed_end_lsn >= log_end_lsn) { - need_send_ack = true; - } - PALF_LOG(INFO, "receive log with same proposal_id", K(bool_ret), K_(palf_id), K_(self), K(log_id), K(log_end_lsn), - K(need_send_ack), K(old_pid), KPC(log_task), K(header), K(max_flushed_end_lsn), KPC(log_task)); - } else if (old_pid > new_pid) { - // Received a log with a smaller proposal_id, which should not happen. - // Here it indirectly depends on: follower can only return committed logs when responding to fetch log requests, therefore under the same proposal_id, - // There will not be two replicas sending logs with the same log_id but different proposal_id to the same destination. - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "receive log with smaller proposal_id, unexpected error", K_(palf_id), K_(self), K(log_id), - K(new_pid), KPC(log_task), K(header), KPC(log_task)); - } else { - PALF_LOG(INFO, "receive log with larger proposal_id, maybe need truncate", K_(palf_id), K_(self), K(log_id), - K(new_pid), K(header), KPC(log_task)); - } - } - } - return bool_ret; -} - -int LogSlidingWindow::get_prev_log_info_(const int64_t log_id, - LSN &prev_lsn, - LSN &prev_end_lsn, - SCN &prev_scn, - int64_t &prev_log_pid, - int64_t &prev_log_accum_checksum) -{ - int ret = OB_SUCCESS; - if (log_id < 1) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(log_id)); - } else if (FIRST_VALID_LOG_ID == log_id) { - ret = OB_ENTRY_NOT_EXIST; - PALF_LOG(INFO, "this is the first log, previous log not exist", K(ret), - K(log_id), K_(palf_id), K_(self)); - } else { - const int64_t prev_log_id = log_id - 1; - const int64_t start_id = get_start_id(); - LogTask *prev_log_task = NULL; - LogTaskGuard guard(this); - if (OB_FAIL(guard.get_log_task(prev_log_id, prev_log_task))) { - if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { - // log has slide out, need check last_slide_log info - ret = OB_SUCCESS; - if (start_id == log_id) { - int64_t last_slide_log_id = OB_INVALID_LOG_ID; - SCN last_slide_scn; - LSN last_slide_lsn; - LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; - int64_t last_slide_accum_checksum = -1; - get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); - if (get_start_id() == start_id) { - // double check log_id == start_id - prev_lsn = last_slide_lsn; - prev_end_lsn = last_slide_end_lsn; - prev_scn = last_slide_scn; - prev_log_pid = last_slide_log_pid; - prev_log_accum_checksum = last_slide_accum_checksum; - } else { - ret = OB_ENTRY_NOT_EXIST; - } - } else { - ret = OB_ENTRY_NOT_EXIST; - PALF_LOG(WARN, "prev log has slide out, but log_id is not equal to start_id", - K(ret), K_(palf_id), K_(self), K(log_id), K(start_id)); - } - } else { - PALF_LOG(ERROR, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self)); - } - } else { - prev_log_task->lock(); - if (!prev_log_task->is_valid()) { - ret = OB_ENTRY_NOT_EXIST; - } else { - prev_lsn = prev_log_task->get_begin_lsn(); - prev_end_lsn = prev_log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE - + prev_log_task->get_data_len(); - prev_scn = prev_log_task->get_max_scn(); - prev_log_pid = prev_log_task->get_proposal_id(); - prev_log_accum_checksum = prev_log_task->get_accum_checksum(); - } - prev_log_task->unlock(); - } - } - return ret; -} + (void) lsn_allocator_.get_curr_end_lsn(curr_end_lsn); -bool LogSlidingWindow::pre_check_for_config_log(const int64_t &msg_proposal_id, - const LSN &lsn, - const int64_t &log_proposal_id, - TruncateLogInfo &truncate_log_info) -{ - // lsn may be invalid (before writing the first log) - bool bool_ret = false; - int tmp_ret = OB_SUCCESS; int64_t last_slide_log_id = OB_INVALID_LOG_ID; SCN last_slide_scn; LSN last_slide_lsn; LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; int64_t last_slide_accum_checksum = -1; get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); - int64_t match_log_id = OB_INVALID_LOG_ID; - LSN match_log_end_lsn; - if (IS_NOT_INIT) { - bool_ret = false; - } else if (PALF_INITIAL_LSN_VAL == lsn.val_ && INVALID_PROPOSAL_ID == log_proposal_id) { - // prev log of this config log is the first invalid log, no need do pre check - bool_ret = true; - match_log_id = FIRST_VALID_LOG_ID - 1; - } else if (FIRST_VALID_LOG_ID <= last_slide_log_id - && last_slide_lsn >= lsn) { - // the last_slide_log is valid and its lsn >= lsn - if (last_slide_lsn == lsn && last_slide_log_pid == log_proposal_id) { - // 1. When last_slide_lsn == lsn, that means log matches. - bool_ret = true; - match_log_id = last_slide_log_id; - match_log_end_lsn = last_slide_end_lsn; - } else if (last_slide_lsn > lsn && last_slide_log_pid >= log_proposal_id) { - // 2. When last_slide_lsn > lsn, and log_proposal_id of last_slide_log >= log_proposal_id, - // that means: - // case 1. prev log may has been committed by this leader - // case 2. prev log may has been truncated by another new leader - // if case 2 happens, that means old leader and new leader send config log/log with same proposal_id, - // it's impossible in our consensus protocol. - // Therefore only case 1 is possible, we can receive this config log safely. - bool_ret = true; - PALF_LOG(INFO, "last_slide_log_pid matches with arg or lsn has slid", K(bool_ret), K_(palf_id), K_(self), K(msg_proposal_id), - K(last_slide_lsn), K(last_slide_log_pid), K(lsn), K(log_proposal_id)); - } else if (last_slide_lsn > lsn && last_slide_log_pid < log_proposal_id) { - // 3. When last_slide_lsn > lsn, and log_proposal_id of last_slide_log < log_proposal_id - // that means more old/new logs with smaller/begger proposal_id have been committed, - // but current leader did not reconfirm them, it is unexpected error case!!! - bool_ret = false; - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "last_slide_log_pid is unexpected", K(bool_ret), K_(palf_id), K_(self), K(msg_proposal_id), - K(last_slide_lsn), K(last_slide_log_pid), K(lsn), K(log_proposal_id)); - } else { - bool_ret = false; - tmp_ret = OB_STATE_NOT_MATCH; - PALF_LOG_RET(WARN, tmp_ret, "log proposal_id does not match", K(bool_ret), K(msg_proposal_id), K_(palf_id), K_(self), - K(lsn), K(last_slide_lsn), K(last_slide_log_pid), K(log_proposal_id)); - } - } else { - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - for (int64_t tmp_log_id = get_start_id(); !bool_ret && (OB_SUCCESS == tmp_ret); ++tmp_log_id) { - LogTask *log_task = NULL; - LogTaskGuard guard(this); - if (OB_SUCCESS != (tmp_ret = guard.get_log_task(tmp_log_id, log_task))) { - PALF_LOG_RET(WARN, tmp_ret, "get_log_task failed", K(tmp_ret), K_(palf_id), K_(self), K(tmp_log_id)); - } else { - log_task->lock(); - if (!log_task->is_valid()) { - tmp_ret = OB_ENTRY_NOT_EXIST; - PALF_LOG_RET(WARN, tmp_ret, "this log_task is invalid", K(tmp_ret), K(tmp_log_id), K_(palf_id), K_(self), KPC(log_task)); - } else { - const LSN curr_lsn = log_task->get_begin_lsn(); - const LSN curr_log_end_lsn = (log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE - + log_task->get_data_len()); - const int64_t curr_log_proposal_id = log_task->get_proposal_id(); - if (curr_lsn == lsn) { - if (curr_log_proposal_id == log_proposal_id) { - // proposal_id matches - if (max_flushed_end_lsn >= curr_log_end_lsn) { - // this log has been flushed or no need check if it's flushed - bool_ret = true; - match_log_id = tmp_log_id; - match_log_end_lsn = curr_log_end_lsn; - } else { - // this log has not been flushed - PALF_LOG_RET(WARN, OB_STATE_NOT_MATCH, "local log is match with arg, but it has not been flushed", K(bool_ret), - K(msg_proposal_id), K_(palf_id), K_(self), K(max_flushed_end_lsn), K(curr_log_end_lsn), - K(lsn), K(log_proposal_id)); - int local_ret = OB_SUCCESS; - if (OB_SUCCESS != (local_ret = log_engine_->submit_purge_throttling_task(PURGE_BY_PRE_CHECK_FOR_CONFIG))) { - PALF_LOG_RET(WARN, local_ret, "submit_purge_throttling_task failed", K_(palf_id), K_(self)); - } - } - } else { - // proposal_id does not match with arg - tmp_ret = OB_STATE_NOT_MATCH; - PALF_LOG_RET(WARN, tmp_ret, "log proposal_id dest not match", K(bool_ret), K(msg_proposal_id), K_(palf_id), K_(self), - K(lsn), KPC(log_task)); - } - } else if (curr_lsn > lsn) { - // curr_lsn > lsn, dest log was not found - tmp_ret = OB_ENTRY_NOT_EXIST; - PALF_LOG_RET(WARN, tmp_ret, "curr_lsn is larger than than arg lsn, the request is maybe stale", - K(bool_ret), K(msg_proposal_id), K_(palf_id), K_(self), K(curr_lsn), K(lsn), K(log_proposal_id)); - } else { - // curr_lsn < lsn, need check next log_task - } - } - log_task->unlock(); - } - } + last_slide_end_lsn, last_slide_accum_checksum); + LSN last_submit_lsn; + LSN last_submit_end_lsn; + int64_t last_submit_log_id = OB_INVALID_LOG_ID; + get_last_submit_log_info_(last_submit_lsn, last_submit_end_lsn, last_submit_log_id); + // new_last_log_xxx are used to truncate lsn_allocator. + int64_t new_last_log_id = OB_INVALID_LOG_ID; + SCN new_last_scn; + LSN new_last_log_end_lsn; + if (last_slide_end_lsn == last_submit_end_lsn) { + new_last_log_id = last_slide_log_id; + new_last_scn = last_slide_scn; + new_last_log_end_lsn = last_slide_end_lsn; + PALF_LOG(INFO, "record last slide log info", K(ret), K(last_slide_log_id), + K(last_slide_scn), K(last_slide_end_lsn), K_(self)); } - if (bool_ret && OB_INVALID_LOG_ID != match_log_id) { - // if log matches, check if need truncate following logs - // For ghost logs that have been submitted to disk / are about to be submitted to disk (no holes) (proposal_id < msg_proposal_id), - // Need to execute truncate immediately, - // For ghost logs that cannot be committed to disk after the hole, the current leader was unaware of their existence and can be left unhandled for now, - // Because this part of the log will not be submitted to disk later (msg_proposal_id ensures that old leader's logs cannot be received at the gap position), - // Subsequently, if elected as leader, rely on the logic before reconfirm to clean up all uncommitted write log entries truncated. - // For simplicity, we only decide whether to truncate based on the next log: - // - If the next log is a hole, then the hole position can no longer receive old logs from the old leader, hence the ghost entries after the hole - // Logs will not be written to disk; - // - If the next log is not a hole, judge its proposal_id if it is smaller than msg_proposal_id, indicating a ghost - // log, directly trigger truncate, otherwise it is not a ghost log, no need to handle. - LogTask *log_task = NULL; + + int64_t first_empty_log_id = OB_INVALID_LOG_ID; // record the first hole in sw, just for debug + LogTask *log_task = NULL; + for (int64_t tmp_log_id = start_log_id; OB_SUCC(ret) && tmp_log_id <= max_log_id; ++tmp_log_id) { LogTaskGuard guard(this); - const int64_t next_log_id = match_log_id + 1; - if (OB_SUCCESS != (tmp_ret = guard.get_log_task(next_log_id, log_task))) { - PALF_LOG_RET(WARN, tmp_ret, "get_log_task failed", K(tmp_ret), K_(palf_id), K_(self), K(next_log_id)); + if (OB_FAIL(guard.get_log_task(tmp_log_id, log_task))) { + // caller hold wrlock, so this step is expected to succeed. + PALF_LOG(ERROR, "get_log_task failed", K(ret), K(tmp_log_id), K_(self)); } else { log_task->lock(); if (!log_task->is_valid()) { - PALF_LOG(INFO, "next log_task is invalid, no need truncate", K(next_log_id), - K_(palf_id), K_(self), KPC(log_task)); - } else if (log_task->get_proposal_id() < msg_proposal_id) { - truncate_log_info.truncate_type_ = TRUNCATE_LOG; - truncate_log_info.truncate_log_id_ = next_log_id; - if (!match_log_end_lsn.is_valid()) { - // match_log_id is 0, the match_log_end_lsn will be invalid, - // it means there is no prev log for this config_log. - // Here we need set truncate_begin_lsn_ to the begin_lsn of the first log. - assert(FIRST_VALID_LOG_ID == next_log_id); - truncate_log_info.truncate_begin_lsn_ = log_task->get_begin_lsn(); - } else { - truncate_log_info.truncate_begin_lsn_ = match_log_end_lsn; + PALF_LOG(INFO, "log_task is invalid", K(ret), K(tmp_log_id), K_(self), K(first_empty_log_id), + K(max_log_id), KPC(log_task)); + if (OB_INVALID_LOG_ID == first_empty_log_id) { + first_empty_log_id = tmp_log_id; + PALF_LOG(INFO, "found first empty log slot", K(ret), K(tmp_log_id), K_(self)); } - truncate_log_info.truncate_log_proposal_id_ = log_task->get_proposal_id(); - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "next log proposal_id is less than msg_proposal_id, need truncate", K_(palf_id), K_(self), - K(next_log_id), K(msg_proposal_id), K(truncate_log_info), KPC(log_task)); } else { - PALF_LOG(INFO, "next log proposal_id is not less than msg_proposal_id, no need truncate", - K_(palf_id), K_(self), K(next_log_id), K(msg_proposal_id), KPC(log_task), K(truncate_log_info)); + const SCN curr_scn = log_task->get_max_scn(); + const LSN log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); + PALF_LOG(INFO, "log_task is valid, check if need clean", K(ret), K(tmp_log_id), K_(self), KPC(log_task)); + if (log_end_lsn == last_submit_end_lsn) { + if (OB_INVALID_LOG_ID == new_last_log_id) { + // record max flushed log_task info + new_last_log_id = tmp_log_id; + new_last_scn = curr_scn; + new_last_log_end_lsn = log_end_lsn; + PALF_LOG(INFO, "find last submit log_task", K(ret), K(tmp_log_id), K_(self), + KPC(log_task), K(last_submit_log_id)); + } + } + if (OB_SUCC(ret)) { + if (OB_INVALID_LOG_ID != last_submit_log_id && tmp_log_id > last_submit_log_id) { + // Drop stale tasks beyond the last locally submitted log. + PALF_LOG(INFO, "clean log task beyond last_submit_log_id", K(ret), K_(self), K(max_log_id), K(tmp_log_id), + K(first_empty_log_id), K(last_submit_log_id), KPC(log_task)); + log_task->reset(); + } + } } log_task->unlock(); } } - - return bool_ret; + if (OB_SUCC(ret)) { + if (OB_INVALID_LOG_ID == new_last_log_id + || !new_last_scn.is_valid() + || !new_last_log_end_lsn.is_valid()) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "last_log info is invalid", K(ret), K(max_log_id), K(first_empty_log_id), K(last_submit_log_id), + K_(self), K(new_last_log_end_lsn), K(new_last_log_id), K(new_last_scn), K(start_log_id), K(max_log_id)); + } else if (new_last_log_end_lsn <= curr_end_lsn + && OB_FAIL(truncate_lsn_allocator_(new_last_log_end_lsn, new_last_log_id, new_last_scn))) { + // truncate lsn_allocator_ by new_last_log info + PALF_LOG(ERROR, "truncate_lsn_allocator_ failed", K(ret), K_(self), K(new_last_log_id), K(new_last_log_end_lsn), + K(new_last_scn)); + } else { + // do nothing + } + } + PALF_LOG(INFO, "clean log finished", K(ret), K_(self), K(max_log_id), K(first_empty_log_id), K(last_submit_log_id), + K(start_log_id), K(max_log_id), K(new_last_log_id), K(new_last_scn), K(new_last_log_end_lsn)); + return ret; } -bool LogSlidingWindow::is_prev_log_pid_match(const int64_t log_id, - const LSN &lsn, - const LSN &prev_lsn, - const int64_t &prev_log_pid, - bool &is_prev_log_exist) +int LogSlidingWindow::activate() { - bool bool_ret = false; - is_prev_log_exist = false; - LSN local_prev_lsn, local_prev_end_lsn; - SCN local_prev_scn; - int64_t local_prev_log_pid = INVALID_PROPOSAL_ID; - int64_t local_prev_log_accum_checksum = -1; - int tmp_ret = OB_SUCCESS; - if (OB_INVALID_LOG_ID == log_id || !lsn.is_valid()) { - PALF_LOG_RET(WARN, OB_INVALID_ARGUMENT, "invalid argumetns", K(log_id), K_(palf_id), K_(self), K(lsn)); - } else if (FIRST_VALID_LOG_ID == log_id) { - bool_ret = true; - PALF_LOG(INFO, "this is the first log, no need check prev log proposal_id", K(log_id), - K_(palf_id), K_(self), K(lsn), K(prev_log_pid)); - } else if (OB_SUCCESS != (tmp_ret = get_prev_log_info_(log_id, local_prev_lsn, local_prev_end_lsn, - local_prev_scn, local_prev_log_pid, local_prev_log_accum_checksum))) { - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG_RET(WARN, tmp_ret, "get_prev_log_info_ failed", K(tmp_ret), K(log_id), K_(palf_id), K_(self), K(log_id), - K(lsn), K(prev_lsn), K(prev_log_pid)); - } + // Check if all group entries have been flushed + // Reset log_tasks' IS_SUBMIT_LOG_EXIST flag + // Resize group_buffer + int ret = OB_SUCCESS; + SCN ref_scn; + AccessMode access_mode = AccessMode::INVALID_ACCESS_MODE; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(mode_mgr_->get_access_mode_ref_scn(access_mode, ref_scn))) { + PALF_LOG(INFO, "get_access_mode_ref_scn failed", K(ret), K_(self)); + } else if (!is_all_log_flushed_()) { + ret = OB_EAGAIN; + PALF_LOG(WARN, "activate need retry, because there is some log has not been flushed", K(ret), + K_(self)); + } else if (OB_FAIL(clean_log_())) { + PALF_LOG(INFO, "clean_log_ failed", K(ret), K_(self)); + } else if (OB_FAIL(group_buffer_.activate())) { + PALF_LOG(WARN, "group_buffer activate failed", K(ret), K_(self)); + } else if (ref_scn.is_valid() && AccessMode::APPEND == access_mode && + OB_FAIL(lsn_allocator_.inc_update_scn_base(ref_scn))) { + PALF_LOG(ERROR, "inc_update_scn_base failed", K(ret), K_(self), K(ref_scn)); } else { - is_prev_log_exist = true; - if (local_prev_log_pid == prev_log_pid - && local_prev_lsn == prev_lsn - && local_prev_end_lsn == lsn) { - // We also need check if end_lsn of prev log matches with next lsn, because the group entry - // may be truncated at some inner log entry. - bool_ret = true; - } else { - bool_ret = false; - PALF_LOG_RET(WARN, OB_ERR_UNEXPECTED, "prev_log_task does not match with arg", K(log_id), K_(palf_id), K_(self), K(lsn), - K(prev_lsn), K(prev_log_pid), K(local_prev_lsn), K(local_prev_end_lsn), K(local_prev_log_pid)); - } + PALF_LOG(INFO, "activate sliding window success", K(ret), K_(self)); } - return bool_ret; + return ret; +} + +int64_t LogSlidingWindow::get_start_id() const +{ + return sw_.get_begin_sn(); } int LogSlidingWindow::get_committed_end_lsn(LSN &committed_end_lsn) const @@ -3844,7 +1682,7 @@ int64_t LogSlidingWindow::get_max_log_id() const LSN LogSlidingWindow::get_max_lsn() const { LSN max_lsn; - (void) lsn_allocator_.get_curr_end_lsn(max_lsn); + (void)lsn_allocator_.get_curr_end_lsn(max_lsn); return max_lsn; } @@ -3853,311 +1691,15 @@ const SCN LogSlidingWindow::get_max_scn() const return lsn_allocator_.get_max_scn(); } -int LogSlidingWindow::get_majority_match_lsn(LSN &majority_match_lsn) -{ - int ret = OB_SUCCESS; - ObMemberList member_list; - int64_t replica_num = 0; - LSN result_lsn; - if (OB_FAIL(mm_->get_log_sync_member_list(member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), KPC(this)); - } else if (OB_FAIL(get_majority_lsn_(member_list, replica_num, result_lsn))) { - PALF_LOG(WARN, "get_majority_lsn failed", K(ret), KPC(this)); - } else { - majority_match_lsn = result_lsn; - } - return ret; -} - bool LogSlidingWindow::check_all_log_has_flushed() { - bool bool_ret = is_all_log_flushed_(); - return bool_ret; -} - -int LogSlidingWindow::reset_match_lsn_map_() -{ - int ret = OB_SUCCESS; - ObMemberList member_list; - int64_t replica_num = 0; - LSN max_flushed_end_lsn; - LSN committed_end_lsn; - const int64_t now_us = ObTimeUtility::current_time(); - ObSpinLockGuard guard(match_lsn_map_lock_); - if (OB_FAIL(mm_->get_log_sync_member_list(member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(match_lsn_map_.clear())) { - PALF_LOG(WARN, "match_lsn_map_ clear failed", K(ret), K_(palf_id), K_(self)); - } else { - get_max_flushed_end_lsn(max_flushed_end_lsn); - get_committed_end_lsn_(committed_end_lsn); - ObAddr tmp_server; - LSN tmp_match_lsn; - for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); ++i) { - tmp_server.reset(); - if (OB_FAIL(member_list.get_server_by_index(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(ret), KPC(this)); - } else { - // Leader update match_lsn for each paxos member. - // Setting match_lsn to max_flushed_end_lsn for itself and committed_end_lsn for others. - tmp_match_lsn = (self_ == tmp_server) ? max_flushed_end_lsn : committed_end_lsn; - if (OB_FAIL(match_lsn_map_.insert(tmp_server, LsnTsInfo(tmp_match_lsn, now_us)))) { - PALF_LOG(WARN, "match_lsn_map_.insert failed", K(ret), KPC(this)); - } - } - } - } - PALF_LOG(INFO, "reset_match_lsn_map_ finished", K(ret), K_(palf_id), K_(self), K(member_list), - K(max_flushed_end_lsn), K(committed_end_lsn)); - return ret; -} - -// need caller hold wlock in PalfHandleImpl -int LogSlidingWindow::config_change_update_match_lsn_map( - const ObMemberList &added_memberlist, - const ObMemberList &removed_memberlist, - const ObMemberList &new_log_sync_memberlist, - const int64_t new_replica_num) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - LSN init_lsn(PALF_INITIAL_LSN_VAL); - const int64_t now_us = ObTimeUtility::current_time(); - ObSpinLockGuard guard(match_lsn_map_lock_); - int tmp_ret = OB_SUCCESS; - ObAddr tmp_server; - for (int64_t i = 0; OB_SUCC(tmp_ret) && i < added_memberlist.get_member_number(); ++i) { - tmp_server.reset(); - if (OB_SUCCESS != (tmp_ret = added_memberlist.get_server_by_index(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(tmp_ret), K_(palf_id), K_(self)); - } else if (OB_SUCCESS != (tmp_ret = match_lsn_map_.insert(tmp_server, LsnTsInfo(init_lsn, now_us))) && - OB_ENTRY_EXIST != tmp_ret) { - PALF_LOG(WARN, "match_lsn_map_.insert failed", K(tmp_ret), K(tmp_server), K_(palf_id), K_(self)); - } else { - tmp_ret = OB_SUCCESS; - PALF_LOG(INFO, "match_lsn_map_.insert success", K(tmp_ret), K(tmp_server), K_(palf_id), K_(self)); - } - } - for (int64_t i = 0; OB_SUCC(tmp_ret) && i < removed_memberlist.get_member_number(); ++i) { - tmp_server.reset(); - if (OB_SUCCESS != (tmp_ret = removed_memberlist.get_server_by_index(i, tmp_server))) { - PALF_LOG(WARN, "get_server_by_index failed", K(tmp_ret), K_(palf_id), K_(self)); - } else if (OB_SUCCESS != (tmp_ret = match_lsn_map_.erase(tmp_server)) && - OB_ENTRY_NOT_EXIST != tmp_ret) { - PALF_LOG(WARN, "match_lsn_map_.erase failed", K(tmp_ret), K(tmp_server), K_(self), K_(self)); - } else { - tmp_ret = OB_SUCCESS; - } - PALF_LOG(INFO, "match_lsn_map_.erase finish", K(tmp_ret), K(tmp_server), K_(self)); - } - } - if (OB_SUCC(ret)) { - LSN new_committed_end_lsn; - (void) gen_committed_end_lsn_(new_committed_end_lsn); - (void) handle_committed_log_(); - } - // Try to advance committed_end_lsn after updating match_lsn_map_. - if (OB_SUCC(ret)) { - if (state_mgr_->is_leader_active()) { - // Only leader with ACTIVE state can generate new committed_end_lsn. - // This step is necessary because after removing member, current log_sync_memeber_list - // may become self (single member). And if all local logs have been flushed, but - // committed_end_lsn is smaller than flushed lsn, self won't have chance to advance it. - // So we need try to advance committed_end_lsn here. - (void) gen_committed_end_lsn_with_memberlist_(new_log_sync_memberlist, new_replica_num); - } - } - PALF_LOG(INFO, "config_change_update_match_lsn_map_ finished", K(ret), K_(palf_id), K_(self), K(added_memberlist), K(removed_memberlist)); - return ret; -} - -int LogSlidingWindow::try_update_match_lsn_map_(const common::ObAddr &server, const LSN &end_lsn) -{ - int ret = OB_SUCCESS; - if (!server.is_valid() || !end_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(server), K(end_lsn)); - } else { - const int64_t now_us = ObTimeUtility::current_time(); - int tmp_ret = OB_SUCCESS; - LsnTsInfo tmp_val; - UpdateMatchLsnFunc update_func(end_lsn, now_us); - ObSpinLockGuard guard(match_lsn_map_lock_); - if (OB_SUCCESS != (tmp_ret = match_lsn_map_.get(server, tmp_val))) { - if (OB_ENTRY_NOT_EXIST == tmp_ret) { - if (OB_FAIL(match_lsn_map_.insert(server, LsnTsInfo(end_lsn, now_us)))) { - PALF_LOG(WARN, "match_lsn_map_.insert failed", K(ret), KPC(this), K(server)); - } else { - PALF_LOG(INFO, "match_lsn_map_.insert success", K(ret), K_(palf_id), K_(self), K(server), K(end_lsn)); - } - } else { - ret = tmp_ret; - PALF_LOG(WARN, "match_lsn_map_.get failed", K(ret), K_(palf_id), K_(self), K(server)); - } - } else if (OB_FAIL(match_lsn_map_.operate(server, update_func))) { - // entry exists, try inc update it - (void) match_lsn_map_.get(server, tmp_val); - PALF_LOG(WARN, "match_lsn_map_.operate() failed", K(ret), K_(palf_id), K_(self), - K(server), K(end_lsn), "curr val", tmp_val); - } else { - // Update successfully, check if it advances delay too long. - if (update_func.is_advance_delay_too_long()) { - PALF_LOG(WARN, "[MATCH LSN ADVANCE DELAY]match_lsn advance delay too much time", - K(ret), K_(palf_id), K_(self), K(server), K(update_func)); - } - } - } - PALF_LOG(TRACE, "try_update_match_lsn_map_ finished", K(ret), K_(palf_id), K_(self), K(server), K(end_lsn)); - return ret; -} - -int LogSlidingWindow::try_send_committed_info(const common::ObAddr &server, - const LSN &log_lsn, - const LSN &log_end_lsn, - const int64_t &log_proposal_id) -{ - int ret = OB_SUCCESS; - LSN committed_end_lsn; - get_committed_end_lsn_(committed_end_lsn); - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!log_lsn.is_valid() || !log_end_lsn.is_valid() || INVALID_PROPOSAL_ID == log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - } else { - // leader/follower can send committed_info to request server, - // if the arg log has slid out and its end_lsn equals to committed_end_lsn. - int64_t last_slide_log_id = OB_INVALID_LOG_ID; - SCN last_slide_scn; - LSN last_slide_lsn; - LSN last_slide_end_lsn; - int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; - int64_t last_slide_accum_checksum = -1; - get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ - last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); - if (log_lsn == last_slide_lsn - && log_proposal_id == last_slide_log_pid - && committed_end_lsn == log_end_lsn) { - // If arg log does match with last slide log, follower can send committed_info to server. - OB_ASSERT(log_end_lsn == last_slide_end_lsn); - if (OB_FAIL(log_engine_->submit_committed_info_req(server, curr_proposal_id, - last_slide_log_id, log_proposal_id, committed_end_lsn))) { - PALF_LOG(WARN, "submit_committed_info_req failed", K(ret), K_(palf_id), K_(self), K(server)); - } else { - PALF_LOG(TRACE, "try_send_committed_info success", K(ret), K_(palf_id), K_(self), - K(last_slide_log_id), K(log_proposal_id), K(committed_end_lsn)); - } - } - } - return ret; -} - -int LogSlidingWindow::leader_get_committed_log_info_(const LSN &committed_end_lsn, - int64_t &log_id, - int64_t &log_proposal_id) -{ - int ret = OB_SUCCESS; - const int64_t max_log_id = get_max_log_id(); - LSN curr_max_lsn; - (void) lsn_allocator_.get_curr_end_lsn(curr_max_lsn); - LogTask *log_task = NULL; - LogTaskGuard guard(this); - if (curr_max_lsn > committed_end_lsn) { - // There is new log generated, no need broadcast committed_info. - } else if (OB_FAIL(guard.get_log_task(max_log_id, log_task))) { - PALF_LOG(WARN, "get_log_task failed", K(ret), K_(palf_id), K_(self), K(max_log_id), KPC(log_task)); - } else if (!log_task->is_valid() || !log_task->is_freezed()) { - // log_task is invalid or not freezed, that means there is maybe new log after committed_end_lsn. - // No need broadcast commonitted_info. - } else { - LSN log_end_lsn; - log_task->lock(); - log_proposal_id = log_task->get_proposal_id(); - log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); - log_task->unlock(); - if (log_end_lsn == committed_end_lsn) { - log_id = max_log_id; - } - } - return ret; -} - -int LogSlidingWindow::leader_broadcast_committed_info_(const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - int64_t log_id = OB_INVALID_LOG_ID; - int64_t log_proposal_id = INVALID_PROPOSAL_ID; - ObMemberList dst_member_list; - int64_t replica_num = 0; - if (OB_FAIL(leader_get_committed_log_info_(committed_end_lsn, log_id, log_proposal_id)) - || OB_INVALID_LOG_ID == log_id) { - // no need send committed_info - } else if (OB_FAIL(mm_->get_log_sync_member_list(dst_member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(dst_member_list.remove_server(self_))) { - PALF_LOG(WARN, "dst_member_list remove_server failed", K(ret), K_(palf_id), K_(self)); - } else if (dst_member_list.is_valid() - && OB_FAIL(log_engine_->submit_committed_info_req(dst_member_list, curr_proposal_id, - log_id, log_proposal_id, committed_end_lsn))) { - PALF_LOG(WARN, "submit_committed_info_req failed", K(ret), K_(palf_id), K_(self), K(log_id)); - } else { - PALF_LOG(TRACE, "leader_broadcast_committed_info_", K(ret), K_(palf_id), K_(self), K(log_id)); - } - return ret; + return is_all_log_flushed_(); } -int LogSlidingWindow::ack_log(const common::ObAddr &src_server, const LSN &end_lsn) +int LogSlidingWindow::gen_committed_end_lsn_(LSN &new_committed_end_lsn) { - int ret = OB_SUCCESS; - ObMemberList member_list; - int64_t replica_num = 0; - GlobalLearnerList degraded_learner_list; - bool in_member_list = false; - bool in_degraded_list = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!src_server.is_valid() || !end_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K_(self), K(src_server), K(end_lsn)); - } else if (OB_FAIL(mm_->get_log_sync_member_list(member_list, replica_num))) { - PALF_LOG(WARN, "get_log_sync_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(mm_->get_degraded_learner_list(degraded_learner_list))) { - PALF_LOG(WARN, "get_degraded_learner_list failed", K(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(in_member_list = member_list.contains(src_server))) { - } else if (FALSE_IT(in_degraded_list = degraded_learner_list.contains(src_server))) { - } else if (!in_member_list && !in_degraded_list) { - // src_server is not paxos member or degraded learners, skip - // why record end_lsn of degraded learners in match_lsn_map? - // if a degraded server recovers, - PALF_LOG(WARN, "src_server is not in curr_member_list/degraded_learner_list", K(ret), K_(palf_id), K_(self), - K(src_server), K(member_list), K(degraded_learner_list)); - } else if (OB_FAIL(try_update_match_lsn_map_(src_server, end_lsn))) { - PALF_LOG(WARN, "try_update_match_lsn_map_ failed", K(ret), K_(palf_id), K_(self), K(src_server), K(end_lsn)); - } else { - LSN old_committed_end_lsn; - get_committed_end_lsn_(old_committed_end_lsn); - LSN new_committed_end_lsn; - if (state_mgr_->is_leader_active()) { - // Only leader with ACTIVE state can generate new committed_end_lsn. - (void) gen_committed_end_lsn_(new_committed_end_lsn); - if (new_committed_end_lsn > old_committed_end_lsn) { - (void) leader_broadcast_committed_info_(new_committed_end_lsn); - } - // if the leader receives ack from degraded learners and the log - // is in the range of sliding window, start to send logs to degraded learners - if (OB_UNLIKELY(in_degraded_list && - end_lsn >= get_max_lsn() - 2 * LEADER_DEFAULT_GROUP_BUFFER_SIZE)) { - mm_->set_sync_to_degraded_learners(); - } - } - (void) handle_committed_log_(); - PALF_LOG(TRACE, "ack log finished", K(ret), K_(palf_id), K_(self), K(src_server), K(end_lsn), - K(old_committed_end_lsn), K(new_committed_end_lsn)); - } - return ret; + get_max_flushed_end_lsn(new_committed_end_lsn); + return try_advance_committed_lsn_(new_committed_end_lsn); } int LogSlidingWindow::append_disk_log(const LSN &lsn, @@ -4171,11 +1713,11 @@ int LogSlidingWindow::append_disk_log(const LSN &lsn, ret = OB_NOT_INIT; } else if (false == lsn.is_valid() || false == group_entry.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry)); + PALF_LOG(WARN, "invalid arguments", K(ret), K_(self), K(lsn), K(group_entry)); } else if (OB_FAIL(append_disk_log_to_sw_(lsn, group_entry))) { - PALF_LOG(WARN, "append_disk_log_to_sw_ failed", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry)); + PALF_LOG(WARN, "append_disk_log_to_sw_ failed", K(ret), K_(self), K(lsn), K(group_entry)); } else if (OB_FAIL(try_update_max_lsn_(lsn, group_entry_header))){ - PALF_LOG(WARN, "try_update_max_lsn_ failed", K(ret), K_(palf_id), K_(self), K(lsn)); + PALF_LOG(WARN, "try_update_max_lsn_ failed", K(ret), K_(self), K(lsn)); // Update group_buffer's readable_begin_lsn. // Because these logs' data do not fill into group_buffer, so it cannot // be read by hot cache. @@ -4185,14 +1727,13 @@ int LogSlidingWindow::append_disk_log(const LSN &lsn, PALF_LOG(WARN, "inc_update_reuse_lsn failed", K(ret), K(log_end_lsn)); } else { // update max_flushed log info - const int64_t &log_proposal_id = group_entry_header.get_log_proposal_id(); - (void) inc_update_max_flushed_log_info_(lsn, log_end_lsn, log_proposal_id); - (void) set_last_submit_log_info_(lsn, log_end_lsn, group_entry_header.get_log_id(), group_entry_header.get_log_proposal_id()); + (void) inc_update_max_flushed_log_info_(lsn, log_end_lsn); + (void) set_last_submit_log_info_(lsn, log_end_lsn, group_entry_header.get_log_id()); // update saved accum_checksum_ (void) checksum_.set_accum_checksum(group_entry_header.get_accum_checksum()); (void) try_advance_committed_lsn_(group_entry_header.get_committed_end_lsn()); (void) handle_committed_log_(); - PALF_LOG(INFO, "append_disk_log success", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry)); + PALF_LOG(INFO, "append_disk_log success", K(ret), K_(self), K(lsn), K(group_entry)); } return ret; } @@ -4205,10 +1746,7 @@ int LogSlidingWindow::report_log_task_trace(const int64_t log_id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (OB_SUCC(guard.get_log_task(log_id, log_task))) { - LogMemberAckInfoList ack_info_list; - get_ack_info_array(ack_info_list); - PALF_LOG(INFO, "current log_task status", K_(palf_id), K_(self), K(log_id), KPC(log_task), - K(ack_info_list)); + PALF_LOG(INFO, "current log_task status", K_(self), K(log_id), KPC(log_task)); } else { // do nothing } @@ -4229,31 +1767,31 @@ int LogSlidingWindow::append_disk_log_to_sw_(const LSN &lsn, int64_t group_log_data_checksum = 0; if (false == header.check_integrity(buf, buf_len, group_log_data_checksum)) { ret = OB_INVALID_DATA; - PALF_LOG(ERROR, "group_entry_header check_integrity failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(ERROR, "group_entry_header check_integrity failed", K(ret), K_(self)); } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "get log task failed", K(ret), K_(palf_id), K_(self), K(log_id), K(lsn), K(header), "start id", sw_.get_begin_sn()); + PALF_LOG(ERROR, "get log task failed", K(ret), K_(self), K(log_id), K(lsn), K(header), "start id", sw_.get_begin_sn()); } else if (log_task->is_valid()) { - PALF_LOG(ERROR, "it's not possible to get valid log_task from sw successfully in scan disk phase", K(ret), K_(palf_id), K_(self), + PALF_LOG(ERROR, "it's not possible to get valid log_task from sw successfully in scan disk phase", K(ret), K_(self), K(lsn), K(header), "start_id", sw_.get_begin_sn()); } else if (OB_FAIL(get_min_scn_from_buf_(header, buf, buf_len, min_scn))) { - PALF_LOG(WARN, "get_min_scn_from_buf_ failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "get_min_scn_from_buf_ failed", K(ret), K_(self)); } else { LSN max_flushed_lsn; - LSN max_flushed_end_lsn; - int64_t max_flushed_log_pid = INVALID_PROPOSAL_ID; - (void) get_max_flushed_log_info_(max_flushed_lsn, max_flushed_end_lsn, max_flushed_log_pid); + { + ObSpinLockGuard guard(max_flushed_info_lock_); + max_flushed_lsn = max_flushed_lsn_; + } log_task->lock(); if (OB_FAIL(log_task->set_group_header(lsn, min_scn, header))) { - PALF_LOG(WARN, "set_group_header failed", K(ret), K_(palf_id), K_(self), K(lsn), K(header), KPC(log_task)); + PALF_LOG(WARN, "set_group_header failed", K(ret), K_(self), K(lsn), K(header), KPC(log_task)); } else { log_task->set_group_log_checksum(group_log_data_checksum); log_task->set_prev_lsn(max_flushed_lsn); - log_task->set_prev_log_proposal_id(max_flushed_log_pid); log_task->set_freezed(); log_task->set_freeze_ts(ObTimeUtility::current_time()); log_task->try_pre_submit(); - PALF_LOG(TRACE, "append_disk_log success", K(ret), K_(palf_id), K_(self), K(lsn), K(header), KPC(log_task)); + PALF_LOG(TRACE, "append_disk_log success", K(ret), K_(self), K(lsn), K(header), KPC(log_task)); } log_task->unlock(); } @@ -4265,13 +1803,12 @@ int LogSlidingWindow::try_update_max_lsn_(const LSN &lsn, const LogGroupEntryHea int ret = OB_SUCCESS; const SCN &scn = header.get_max_scn(); const int64_t log_id = header.get_log_id(); - const int64_t &log_proposal_id = header.get_log_proposal_id(); const int64_t group_entry_len = header.get_serialize_size() + header.get_data_len(); const LSN end_lsn = lsn + group_entry_len; if (OB_FAIL(lsn_allocator_.inc_update_last_log_info(end_lsn, log_id, scn))) { - PALF_LOG(WARN, "inc_update_last_log_info failed", K(ret), K_(palf_id), K_(self), K(lsn), K(scn)); + PALF_LOG(WARN, "inc_update_last_log_info failed", K(ret), K_(self), K(lsn), K(scn)); } else { - PALF_LOG(TRACE, "try_update_max_lsn_ success", K(ret), K_(palf_id), K_(self), K(lsn), K(end_lsn), + PALF_LOG(TRACE, "try_update_max_lsn_ success", K(ret), K_(self), K(lsn), K(end_lsn), K(log_id), K(scn)); } return ret; @@ -4284,9 +1821,9 @@ int LogSlidingWindow::truncate_lsn_allocator_(const LSN &last_lsn, const int64_t if (!last_lsn.is_valid() || OB_INVALID_LOG_ID == last_log_id || (!last_scn.is_valid() && 0 != last_log_id)) { ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(lsn_allocator_.truncate(last_lsn, last_log_id, last_scn))) { - PALF_LOG(WARN, "lsn_allocator_.truncate failed", K(ret), K_(palf_id), K_(self)); + PALF_LOG(WARN, "lsn_allocator_.truncate failed", K(ret), K_(self)); } else { - PALF_LOG(INFO, "lsn_allocator_.truncate success", K(ret), K_(palf_id), K_(self), K(last_lsn), + PALF_LOG(INFO, "lsn_allocator_.truncate success", K(ret), K_(self), K(last_lsn), K(last_log_id), K(last_scn)); } return ret; @@ -4312,9 +1849,8 @@ void LogSlidingWindow::LogTaskGuard::revert_log_task() { int ret = OB_SUCCESS; if (NULL != sw_ && is_valid_log_id(log_id_)) { if (OB_FAIL(sw_->sw_.revert(log_id_))) { - const int64_t palf_id = sw_->palf_id_; const int64_t begin_sn = sw_->sw_.get_begin_sn(); - PALF_LOG(ERROR, "revert failed", K(ret), K(palf_id), K_(log_id), K(begin_sn)); + PALF_LOG(ERROR, "revert failed", K(ret), K_(log_id), K(begin_sn)); } } sw_ = NULL; @@ -4339,56 +1875,6 @@ int LogSlidingWindow::get_min_scn_from_buf_(const LogGroupEntryHeader &header, return ret; } -int LogSlidingWindow::handle_committed_info(const common::ObAddr &server, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - LogTask *log_task = NULL; - LogTaskGuard guard(this); - const int64_t last_submit_log_id = get_last_submit_log_id_(); - LSN max_flushed_end_lsn; - get_max_flushed_end_lsn(max_flushed_end_lsn); - LSN curr_committed_end_lsn; - get_committed_end_lsn_(curr_committed_end_lsn); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || !committed_end_lsn.is_valid() - || OB_INVALID_LOG_ID == prev_log_id || INVALID_PROPOSAL_ID == prev_log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K_(self), K(server), K(prev_log_id), - K(prev_log_proposal_id), K(committed_end_lsn)); - } else if (last_submit_log_id < prev_log_id) { - // last_submit_log_id is smaller than prev_log_id, ignore msg - } else if (max_flushed_end_lsn < committed_end_lsn) { - // max_flushed_end_lsn is smaller than committed_end_lsn, ignore msg - } else if (curr_committed_end_lsn >= committed_end_lsn) { - // curr_committed_end_lsn is not smaller than committed_end_lsn, ignore msg - } else if (OB_FAIL(guard.get_log_task(prev_log_id, log_task))) { - PALF_LOG(WARN, "get_log_task failed", K(ret), K_(palf_id), K_(self), K(prev_log_id), KPC(log_task)); - } else if (!log_task->is_valid()) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "log_task is invalid, unexpected", K(ret), K_(palf_id), K_(self), K(prev_log_id), KPC(log_task), - K(last_submit_log_id), K(max_flushed_end_lsn)); - } else { - int64_t log_proposal_id = INVALID_PROPOSAL_ID; - LSN log_end_lsn; - log_task->lock(); - log_proposal_id = log_task->get_proposal_id(); - log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); - log_task->unlock(); - if (log_proposal_id == prev_log_proposal_id) { - OB_ASSERT(log_end_lsn == committed_end_lsn); - (void) try_advance_committed_lsn_(committed_end_lsn); - (void) handle_committed_log_(); - } - PALF_LOG(TRACE, "handle_committed_info", K(ret), K_(palf_id), K_(self), K(prev_log_id), K(prev_log_proposal_id), - K(committed_end_lsn), K(last_submit_log_id), K(max_flushed_end_lsn)); - } - return ret; -} - int LogSlidingWindow::advance_reuse_lsn(const LSN &flush_log_end_lsn) { // Do not hold lock here. @@ -4398,9 +1884,9 @@ int LogSlidingWindow::advance_reuse_lsn(const LSN &flush_log_end_lsn) } else if (!flush_log_end_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(group_buffer_.inc_update_reuse_lsn(flush_log_end_lsn))) { - PALF_LOG(WARN, "inc_update_reuse_lsn failed", K(ret), K_(palf_id), K(flush_log_end_lsn)); + PALF_LOG(WARN, "inc_update_reuse_lsn failed", K(ret), K(flush_log_end_lsn)); } else { - PALF_LOG(TRACE, "advance_reuse_lsn success", K(ret), K_(palf_id), K(flush_log_end_lsn)); + PALF_LOG(TRACE, "advance_reuse_lsn success", K(ret), K(flush_log_end_lsn)); } return ret; } @@ -4415,15 +1901,14 @@ int LogSlidingWindow::read_data_from_buffer(const LSN &read_begin_lsn, ret = OB_NOT_INIT; } else if (!read_begin_lsn.is_valid() || in_read_size <= 0 || OB_ISNULL(buf)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argumetns", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size), KP(buf)); + PALF_LOG(WARN, "invalid argumetns", K(ret), K(read_begin_lsn), K(in_read_size), KP(buf)); } else { - RLockGuard guard(group_buffer_lock_); if (OB_FAIL(group_buffer_.read_data(read_begin_lsn, in_read_size, buf, out_read_size))) { if (OB_ERR_OUT_OF_LOWER_BOUND != ret) { - PALF_LOG(WARN, "read_data failed", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size)); + PALF_LOG(WARN, "read_data failed", K(ret), K(read_begin_lsn), K(in_read_size)); } } else { - PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K_(palf_id), K(read_begin_lsn), + PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K(read_begin_lsn), K(in_read_size), K(out_read_size)); } } diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h index 62df7cfa8..4b8bd96ec 100755 --- a/src/logservice/palf/log_sliding_window.h +++ b/src/logservice/palf/log_sliding_window.h @@ -21,12 +21,10 @@ #include "lib/hash/ob_linear_hash_map.h" #include "lib/lock/ob_spin_lock.h" #include "lib/thread/ob_thread_lease.h" -#include "log_ack_info.h" #include "share/scn.h" #include "log_group_entry.h" #include "log_group_buffer.h" #include "log_checksum.h" -#include "log_req.h" #include "lsn.h" #include "lsn_allocator.h" #include "log_task.h" @@ -52,67 +50,9 @@ class FlushLogCbCtx; class LogGroupEntryHeader; class LogTaskHeaderInfo; class LogStateMgr; -class LogConfigMgr; class LogModeMgr; class LogTask; class LogGroupEntry; -class TruncateLogCbCtx; - -enum FetchTriggerType -{ - LOG_LOOP_TH = 0, - SLIDING_CB = 1, - LEADER_RECONFIRM = 2, - NOTIFY_REBUILD = 3, - LEARNER_REGISTER = 4, - CLEAN_CACHED_LOG = 5, - MODE_META_BARRIER = 6, - RECONFIRM_NOTIFY_FETCH = 7, - ADD_MEMBER_PRE_CHECK = 8, -}; - -inline const char *fetch_trigger_type_2_str(const FetchTriggerType type) -{ -#define EXTRACT_TRIGGER_TYPE(type_var) case(type_var): return #type_var - switch(type) - { - EXTRACT_TRIGGER_TYPE(LOG_LOOP_TH); - EXTRACT_TRIGGER_TYPE(SLIDING_CB); - EXTRACT_TRIGGER_TYPE(LEADER_RECONFIRM); - EXTRACT_TRIGGER_TYPE(NOTIFY_REBUILD); - EXTRACT_TRIGGER_TYPE(LEARNER_REGISTER); - EXTRACT_TRIGGER_TYPE(CLEAN_CACHED_LOG); - EXTRACT_TRIGGER_TYPE(MODE_META_BARRIER); - EXTRACT_TRIGGER_TYPE(RECONFIRM_NOTIFY_FETCH); - EXTRACT_TRIGGER_TYPE(ADD_MEMBER_PRE_CHECK); - - default: - return "Invalid Type"; - } -#undef EXTRACT_TRIGGER_TYPE -} - -enum TruncateType -{ - INVALID_TRUNCATE_TYPE = 0, - TRUNCATE_CACHED_LOG_TASK = 1, - TRUNCATE_LOG = 2, -}; - -inline const char *truncate_type_2_str(const TruncateType type) -{ -#define EXTRACT_TRUNCATE_TYPE(type_var) case(type_var): return #type_var - switch(type) - { - EXTRACT_TRUNCATE_TYPE(INVALID_TRUNCATE_TYPE); - EXTRACT_TRUNCATE_TYPE(TRUNCATE_CACHED_LOG_TASK); - EXTRACT_TRUNCATE_TYPE(TRUNCATE_LOG); - - default: - return "Invalid Type"; - } -#undef EXTRACT_TRUNCATE_TYPE -} enum FreezeMode { @@ -134,76 +74,6 @@ inline const char *freeze_mode_2_str(const FreezeMode mode) #undef EXTRACT_FREEZE_MODE } -struct TruncateLogInfo -{ - TruncateType truncate_type_; - int64_t truncate_log_id_; - LSN truncate_begin_lsn_; - int64_t truncate_log_proposal_id_; - TruncateLogInfo() { reset(); } - bool is_valid() const { - return (OB_INVALID_LOG_ID != truncate_log_id_ - && INVALID_PROPOSAL_ID != truncate_log_proposal_id_ - && truncate_begin_lsn_.is_valid()); - } - void reset() { - truncate_type_ = INVALID_TRUNCATE_TYPE; - truncate_log_id_ = OB_INVALID_LOG_ID; - truncate_begin_lsn_.reset(); - truncate_log_proposal_id_ = INVALID_PROPOSAL_ID; - } - TO_STRING_KV("truncate_type", truncate_type_2_str(truncate_type_), K_(truncate_log_id), K_(truncate_begin_lsn), K_(truncate_log_proposal_id)); -}; - -class UpdateMatchLsnFunc -{ -public: - UpdateMatchLsnFunc(const LSN &end_lsn, const int64_t new_ack_time_us) - : new_end_lsn_(end_lsn), old_end_lsn_(), new_ack_time_us_(new_ack_time_us), old_advance_time_us_(OB_INVALID_TIMESTAMP) - {} - ~UpdateMatchLsnFunc() {} - bool operator()(const common::ObAddr &server, LsnTsInfo &value); - bool is_advance_delay_too_long() const { - bool bool_ret = false; - if (old_end_lsn_ < new_end_lsn_ - && (new_ack_time_us_ - old_advance_time_us_) > MATCH_LSN_ADVANCE_DELAY_THRESHOLD_US) { - // Return true when advance delay exceeds 1s. - bool_ret = true; - } - return bool_ret; - } - TO_STRING_KV(K_(old_end_lsn), K_(new_end_lsn), K_(old_advance_time_us), K_(new_ack_time_us), - "advance delay(us)", new_ack_time_us_ - old_advance_time_us_); -private: - LSN new_end_lsn_; - LSN old_end_lsn_; - int64_t new_ack_time_us_; - int64_t old_advance_time_us_; -}; - -class GetLaggedListFunc -{ -public: - GetLaggedListFunc(const LSN &dst_lsn) - : dst_lsn_(dst_lsn), lagged_list_() - {} - ~GetLaggedListFunc() {} - bool operator()(const common::ObAddr &server, LsnTsInfo &value); - int get_lagged_list(common::ObMemberList &out_list) const - { return out_list.deep_copy(lagged_list_); } - TO_STRING_KV(K_(dst_lsn), K_(lagged_list)); -private: - LSN dst_lsn_; - common::ObMemberList lagged_list_; -}; - -inline bool is_need_rebuild(const LSN &end_lsn, const LSN &last_rebuild_lsn) -{ - return (end_lsn.is_valid() && - last_rebuild_lsn.is_valid() && - end_lsn < last_rebuild_lsn); -} - class LogSlidingWindow : public ISlidingCallBack { public: @@ -211,33 +81,21 @@ class LogSlidingWindow : public ISlidingCallBack virtual ~LogSlidingWindow() { destroy(); } public: virtual void destroy(); - virtual int init(const int64_t palf_id, - const common::ObAddr &self, + virtual int init(const common::ObAddr &self, LogStateMgr *state_mgr, - LogConfigMgr *mm, LogModeMgr *mode_mgr, LogEngine *log_engine, palf::PalfFSCbWrapper *palf_fs_cb, common::ObILogAllocator *alloc_mgr, - LogPlugins *plugins, - const PalfBaseInfo &palf_base_info, - const bool is_normal_replica); + const PalfBaseInfo &palf_base_info); virtual int sliding_cb(const int64_t sn, const FixedSlidingWindowSlot *data); virtual int64_t get_max_log_id() const; virtual const share::SCN get_max_scn() const; virtual LSN get_max_lsn() const; virtual int64_t get_start_id() const; virtual int get_committed_end_lsn(LSN &committed_end_lsn) const; - virtual int try_fetch_log(const FetchTriggerType &fetch_log_type, - const LSN prev_lsn = LSN(), - const LSN fetch_start_lsn = LSN(), - const int64_t fetch_start_log_id = OB_INVALID_LOG_ID); - virtual int try_fetch_log_for_reconfirm(const common::ObAddr &dest, const LSN &fetch_end_lsn, bool &is_fetched); - virtual int submit_push_log_resp(const common::ObAddr &server); virtual bool is_empty() const; virtual bool check_all_log_has_flushed(); - virtual int get_majority_match_lsn(LSN &majority_match_lsn); - virtual int get_lagged_member_list(const LSN &dst_lsn, ObMemberList &lagged_list); virtual bool is_all_committed_log_slided_out(LSN &prev_lsn, int64_t &prev_log_id, LSN &committed_end_lsn) const; // ================= log sync part begin virtual int submit_log(const char *buf, @@ -245,149 +103,70 @@ class LogSlidingWindow : public ISlidingCallBack const share::SCN &ref_scn, LSN &lsn, share::SCN &scn); - virtual int submit_group_log(const LSN &lsn, - const char *buf, - const int64_t buf_len); - virtual int receive_log(const common::ObAddr &src_server, - const PushLogType push_log_type, - const LSN &prev_lsn, - const int64_t &prev_log_pid, - const LSN &lsn, - const char *buf, - const int64_t buf_len, - const bool need_check_clean_log, - TruncateLogInfo &truncate_log_info); virtual int after_flush_log(const FlushLogCbCtx &flush_cb_ctx); - virtual int after_truncate(const TruncateLogCbCtx &truncate_cb_ctx); virtual int after_rebuild(const LSN &lsn); - virtual int ack_log(const common::ObAddr &src_server, const LSN &end_lsn); - virtual int truncate(const TruncateLogInfo &truncate_log_info, const LSN &expected_prev_lsn, - const int64_t expected_prev_log_pid); - virtual bool is_allow_rebuild() const; - virtual int truncate_for_rebuild(const PalfBaseInfo &palf_base_info); - virtual bool is_prev_log_pid_match(const int64_t log_id, - const LSN &lsn, - const LSN &prev_lsn, - const int64_t &prev_log_pid, - bool &is_prev_log_exist); - virtual bool pre_check_for_config_log(const int64_t &msg_proposal_id, - const LSN &lsn, - const int64_t &log_proposal_id, - TruncateLogInfo &truncate_log_info); - virtual int handle_committed_info(const common::ObAddr &server, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn); - virtual int config_change_update_match_lsn_map(const ObMemberList &added_memberlist, - const ObMemberList &removed_memberlist, - const ObMemberList &new_log_sync_memberlist, - const int64_t new_replica_num); // ================= log sync part end virtual int append_disk_log(const LSN &lsn, const LogGroupEntry &group_entry); virtual int report_log_task_trace(const int64_t log_id); virtual void get_max_flushed_end_lsn(LSN &end_lsn) const; - virtual int get_max_flushed_log_info(LSN &lsn, - LSN &end_lsn, - int64_t &log_proposal_id) const; virtual int clean_log(); - virtual int clean_cached_log(const int64_t begin_log_id, - const LSN &lsn, - const LSN &prev_lsn, - const int64_t prev_log_pid); - virtual int to_follower_pending(LSN &last_lsn); - virtual int to_leader_reconfirm(); - virtual int to_leader_active(); + virtual int activate(); virtual int try_advance_committed_end_lsn(const LSN &end_lsn); virtual int64_t get_last_submit_log_id_() const; virtual void get_last_submit_end_lsn_(LSN &end_lsn) const; - virtual int get_last_submit_log_info(LSN &last_submit_lsn, int64_t &log_id, int64_t &log_proposal_id) const; - virtual int get_last_submit_log_info(LSN &last_submit_lsn, - LSN &last_submit_end_lsn, int64_t &log_id, int64_t &log_proposal_id) const; virtual int get_last_slide_end_lsn(LSN &out_end_lsn) const; virtual const share::SCN get_last_slide_scn() const; virtual int check_and_switch_freeze_mode(); virtual bool is_in_period_freeze_mode() const; virtual int period_freeze_last_log(); virtual int inc_update_scn_base(const share::SCN &scn); - virtual int get_server_ack_info(const common::ObAddr &server, LsnTsInfo &ack_info) const; - virtual int get_ack_info_array(LogMemberAckInfoList &ack_info_array) const; virtual int advance_reuse_lsn(const LSN &flush_log_end_lsn); - virtual int try_send_committed_info(const common::ObAddr &server, - const LSN &log_lsn, - const LSN &log_end_lsn, - const int64_t &log_proposal_id); - virtual int get_leader_from_cache(common::ObAddr &leader) const; virtual int read_data_from_buffer(const LSN &read_begin_lsn, const int64_t in_read_size, char *buf, int64_t &out_read_size) const; int64_t get_last_slide_log_id() const; virtual int try_handle_next_submit_log(); - TO_STRING_KV(K_(palf_id), K_(self), K_(lsn_allocator), K_(group_buffer), \ - K_(last_submit_lsn), K_(last_submit_end_lsn), K_(last_submit_log_id), K_(last_submit_log_pid), \ - K_(max_flushed_lsn), K_(max_flushed_end_lsn), K_(max_flushed_log_pid), K_(committed_end_lsn), \ + TO_STRING_KV(K_(self), K_(lsn_allocator), K_(group_buffer), \ + K_(last_submit_lsn), K_(last_submit_end_lsn), K_(last_submit_log_id), \ + K_(max_flushed_lsn), K_(max_flushed_end_lsn), K_(committed_end_lsn), \ K_(last_slide_log_id), K_(last_slide_scn), K_(last_slide_lsn), K_(last_slide_end_lsn), \ - K_(last_slide_log_pid), K_(last_slide_log_accum_checksum), K_(last_fetch_end_lsn), \ - K_(last_fetch_max_log_id), K_(last_fetch_committed_end_lsn), K_(last_truncate_lsn), \ - K_(last_fetch_req_time), K_(is_truncating), K_(is_rebuilding), K_(last_rebuild_lsn), \ - "freeze_mode", freeze_mode_2_str(freeze_mode_), K_(has_pending_handle_submit_task), \ - "last_fetch_trigger_type", fetch_trigger_type_2_str(last_fetch_trigger_type_), KP(this)); + K_(last_slide_log_accum_checksum), \ + "freeze_mode", freeze_mode_2_str(freeze_mode_), K_(has_pending_handle_submit_task), KP(this)); protected: virtual bool is_handle_thread_lease_expired(const int64_t thread_lease_begin_ts) const; private: - int do_init_mem_(const int64_t palf_id, - const PalfBaseInfo &palf_base_info, + int do_init_mem_(const PalfBaseInfo &palf_base_info, common::ObILogAllocator *alloc_mgr); - int get_fetch_log_dst_(common::ObAddr &leader) const; - int get_leader_from_cache_(common::ObAddr &leader) const; int clean_log_(); - int reset_match_lsn_map_(); bool is_all_log_flushed_(); - int leader_wait_sw_slot_ready_(const int64_t log_id); + int wait_sw_slot_ready_(const int64_t log_id); bool can_receive_larger_log_(const int64_t log_id) const; - bool leader_can_submit_larger_log_(const int64_t log_id) const; - bool leader_can_submit_new_log_(const int64_t valid_log_size, LSN &lsn_upper_bound); - bool leader_can_submit_group_log_(const LSN &lsn, const int64_t group_log_size); + bool can_submit_larger_log_(const int64_t log_id) const; + bool can_submit_new_log_(const int64_t valid_log_size, LSN &lsn_upper_bound); void get_committed_end_lsn_(LSN &out_lsn) const; - int get_max_flushed_log_info_(LSN &lsn, - LSN &end_lsn, - int64_t &log_proposal_id) const; - int get_prev_log_info_(const int64_t log_id, - LSN &prev_lsn, - LSN &prev_end_lsn, - share::SCN &prev_log_iscn, - int64_t &prev_log_pid, - int64_t &prev_log_accum_checksum); int inc_update_max_flushed_log_info_(const LSN &lsn, - const LSN &end_lsn, - const int64_t &proposal_id); - int truncate_max_flushed_log_info_(const LSN &lsn, - const LSN &end_lsn, - const int64_t &log_proposal_id); + const LSN &end_lsn); void get_last_slide_end_lsn_(LSN &out_end_lsn) const; int64_t get_last_slide_log_id_() const; void get_last_slide_log_info_(int64_t &log_id, share::SCN &scn, LSN &lsn, LSN &end_lsn, - int64_t &log_proposal_id, int64_t &accum_checksum) const; int try_update_last_slide_log_info_(const int64_t log_id, const share::SCN &scn, const LSN &lsn, const LSN &end_lsn, - const int64_t &proposal_id, const int64_t accum_checksum); int try_advance_committed_lsn_(const LSN &end_lsn); void get_last_submit_log_info_(LSN &lsn, LSN &end_lsn, - int64_t &log_id, - int64_t &log_proposal_id) const; + int64_t &log_id) const; int set_last_submit_log_info_(const LSN &lsn, const LSN &end_lsn, - const int64_t log_id, - const int64_t &log_proposal_id); + const int64_t log_id); int try_freeze_prev_log_(const int64_t next_log_id, const LSN &lsn, bool &is_need_handle); int feedback_freeze_last_log_(); int try_feedback_freeze_log_task_(const int64_t expected_log_id); @@ -416,19 +195,7 @@ class LogSlidingWindow : public ISlidingCallBack int64_t &group_log_checksum, bool &is_accum_checksum_acquired); int gen_committed_end_lsn_(LSN &new_committed_end_lsn); - int gen_committed_end_lsn_with_memberlist_( - const ObMemberList &member_list, - const int64_t replica_num); - int get_majority_lsn_(const ObMemberList &member_list, - const int64_t replica_num, - LSN &result_lsn) const; int inc_ref_(LogTask *log_task, const int64_t inc_val, int64_t &result); - bool need_update_log_task_(LogGroupEntryHeader &header, - LogTask *log_task, - bool &need_send_ack, - bool &is_local_log_valid, - bool &is_log_pid_match) const; - int try_update_match_lsn_map_(const common::ObAddr &server, const LSN &end_lsn); int wait_group_buffer_ready_(const LSN &lsn, const int64_t data_len); int append_disk_log_to_sw_(const LSN &lsn, const LogGroupEntry &group_entry); int try_update_max_lsn_(const LSN &lsn, const LogGroupEntryHeader &header); @@ -437,50 +204,11 @@ class LogSlidingWindow : public ISlidingCallBack int64_t &prev_log_id, LSN &start_lsn, LSN &committed_end_lsn) const; - void get_last_fetch_info_(LSN &last_fetch_end_lsn, - LSN &last_committed_end_lsn, - int64_t &last_fetch_max_log_id) const; - void try_reset_last_fetch_log_info_(const LSN &expected_end_lsn, const int64_t log_id); - void try_update_committed_lsn_for_fetch_(const LSN &expected_end_lsn, - const int64_t &expected_log_id, - const LSN &log_committed_end_lsn, - bool &is_need_fetch); - void try_fetch_log_streamingly_(const LSN &log_end_lsn); - int do_fetch_log_(const FetchTriggerType &trigger_type, - const common::ObAddr &dest, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_start_log_id); - int freeze_pending_log_(LSN &last_lsn); - int check_all_log_task_freezed_(bool &is_all_freezed); int get_min_scn_from_buf_(const LogGroupEntryHeader &group_entry_header, const char *buf, const int64_t buf_len, share::SCN &min_scn); - int leader_get_committed_log_info_(const LSN &committed_end_lsn, - int64_t &log_id, - int64_t &log_proposal_id); - int leader_broadcast_committed_info_(const LSN &committed_end_lsn); - int submit_push_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool &is_fetch_log); - inline int try_push_log_to_paxos_follower_(const int64_t curr_proposal_id, - const int64_t prev_log_pid, - const LSN &prev_lsn, - const LSN &lsn, - const LogWriteBuf &log_write_buf); - int try_push_log_to_children_(const int64_t curr_proposal_id, - const int64_t prev_log_pid, - const LSN &prev_lsn, - const LSN &lsn, - const LogWriteBuf &log_write_buf); - bool need_execute_fetch_(const FetchTriggerType &fetch_trigger_type); - bool need_use_batch_rpc_(const int64_t buf_size, - const bool is_fetch_log) const; public: - typedef common::ObLinearHashMap SvrMatchOffsetMap; static const int64_t TMP_HEADER_SER_BUF_LEN = 256; // temporary buffer size for log header serialization static const int64_t APPEND_CNT_ARRAY_SIZE = 32; // size of the append count statistics array static const uint64_t APPEND_CNT_ARRAY_MASK = APPEND_CNT_ARRAY_SIZE - 1; @@ -505,22 +233,14 @@ class LogSlidingWindow : public ISlidingCallBack int64_t log_id_; }; private: - typedef common::RWLock RWLock; - typedef RWLock::RLockGuard RLockGuard; - typedef RWLock::WLockGuard WLockGuard; -private: - int64_t palf_id_; common::ObAddr self_; FixedSlidingWindow sw_; LogChecksum checksum_; LogStateMgr *state_mgr_; - LogConfigMgr *mm_; LogModeMgr *mode_mgr_; LogEngine *log_engine_; - LogPlugins *plugins_; palf::PalfFSCbWrapper *palf_fs_cb_; LSNAllocator lsn_allocator_; - mutable RWLock group_buffer_lock_; LogGroupBuffer group_buffer_; // Record the last submit log info. // It is used to submit logs sequentially, for restarting, set it as last_replay_log_id. @@ -528,79 +248,31 @@ class LogSlidingWindow : public ISlidingCallBack LSN last_submit_lsn_; LSN last_submit_end_lsn_; int64_t last_submit_log_id_; - int64_t last_submit_log_pid_; // Record the max flushed log info. // max_flushed_lsn_: start lsn of max flushed log, it can be used as prev_lsn for fetching log. // max_flushed_end_lsn_: end lsn of max flushed log, it can be used as start_lsn for fetching log. - // max_flushed_log_pid_: the proposal_id of max flushed log. mutable common::ObSpinLock max_flushed_info_lock_; LSN max_flushed_lsn_; LSN max_flushed_end_lsn_; - int64_t max_flushed_log_pid_; // Record committed end lsn. mutable common::ObSpinLock committed_info_lock_; LSN committed_end_lsn_; // Record the last log which has slided out. // last_slide_lsn_: it is used as prev_lsn for fetching log // last_slide_end_lsn_: it is used for checking all committed log slided out when fetching log. - // last_slide_log_pid_: it is used for forward checking when receive log. mutable common::ObSpinLock last_slide_info_lock_; int64_t last_slide_log_id_; // used by clean log share::SCN last_slide_scn_; LSN last_slide_lsn_; LSN last_slide_end_lsn_; - int64_t last_slide_log_pid_; int64_t last_slide_log_accum_checksum_; - // ---------------- fetch log info begin -------------------------- - // last_fetch_req_time_: - // record the request time of the last fetch operation. - // last_fetch_end_lsn_: - // Record the LSN endpoint of this round's fetch, calculated based on the group_buffer capacity - // last_fetch_max_log_id_: - // Record the log_id endpoint of this round fetch, calculated based on sw capacity - // The number of logs fetched in one round is uncertain, and it ends when any of the above conditions are met. - // - // last_fetch_committed_end_lsn_: - // Record the committed_end_lsn after fetching all logs in this round, when handle_next_submit_log_() is called - // Check if the last log of this round fetch is processed (log_id is equal to last_fetch_max_log_id_ or - // update the value if log_end_lsn exceeds last_fetch_end_lsn_. - // - // Streaming fetch mechanism: - // Check if the end_lsn is equal to last_fetch_committed_end_lsn_ when the log slides out, trigger the next fetch if true, - // The starting point for the next fetch is (last_submit_log_id + 1). - // - mutable common::ObSpinLock fetch_info_lock_; - int64_t last_fetch_req_time_; - LSN last_fetch_end_lsn_; - int64_t last_fetch_max_log_id_; - LSN last_fetch_committed_end_lsn_; - FetchTriggerType last_fetch_trigger_type_; - // ---------------- fetch log info end -------------------------- - // used to record synchronization points for each replica - mutable common::ObSpinLock match_lsn_map_lock_; - SvrMatchOffsetMap match_lsn_map_; - // last truncate lsn, protected by palf_handle_impl's wlock - LSN last_truncate_lsn_; - mutable int64_t cannot_fetch_log_warn_time_; mutable int64_t cannot_freeze_log_warn_time_; mutable int64_t larger_log_warn_time_; mutable int64_t log_life_long_warn_time_; - mutable int64_t lc_cb_get_warn_time_; - mutable int64_t fetch_failure_print_time_; common::ObThreadLease commit_log_handling_lease_; // thread lease for handling committed logs common::ObThreadLease submit_log_handling_lease_; // thread lease for handling committed logs - // last_renew_leader_ts in fetch_log - mutable int64_t last_fetch_log_renew_leader_ts_us_; int64_t end_lsn_stat_time_us_; - // fetch log dest server for reconfirm - common::ObAddr reconfirm_fetch_dest_; - // whether sw is executing trucnation - bool is_truncating_; - // whether this replica is rebuilding - bool is_rebuilding_; - LSN last_rebuild_lsn_; LSN last_record_end_lsn_; - ObMiniStat::ObStatItem push_log_rpc_post_cost_stat_; ObMiniStat::ObStatItem fs_cb_cost_stat_; ObMiniStat::ObStatItem log_life_time_stat_; int64_t accum_slide_log_cnt_; diff --git a/src/logservice/palf/log_state_mgr.cpp b/src/logservice/palf/log_state_mgr.cpp index 89222f63d..445521720 100644 --- a/src/logservice/palf/log_state_mgr.cpp +++ b/src/logservice/palf/log_state_mgr.cpp @@ -16,492 +16,43 @@ #define USING_LOG_PREFIX PALF #include "log_state_mgr.h" -#include "log_io_task_cb_utils.h" -#include "log_engine.h" -#include "log_config_mgr.h" -#include "log_reconfirm.h" -#include "log_sliding_window.h" #include "log_mode_mgr.h" +#include "log_sliding_window.h" namespace oceanbase { using namespace common; -using namespace share; - namespace palf { + LogStateMgr::LogStateMgr() : self_(), - palf_id_(), - prepare_meta_(), sw_(NULL), - reconfirm_(NULL), - log_engine_(NULL), - mm_(NULL), mode_mgr_(NULL), - palf_role_change_cb_(NULL), - election_(NULL), - plugins_(NULL), - role_state_val_(0), - leader_(), - leader_epoch_(OB_INVALID_TIMESTAMP), - last_check_start_id_(OB_INVALID_LOG_ID), - last_check_start_id_time_us_(OB_INVALID_TIMESTAMP), - reconfirm_start_time_us_(OB_INVALID_TIMESTAMP), - check_sync_enabled_time_(OB_INVALID_TIMESTAMP), - check_reconfirm_timeout_time_(OB_INVALID_TIMESTAMP), - check_follower_pending_warn_time_(OB_INVALID_TIMESTAMP), - log_sync_timeout_warn_time_(OB_INVALID_TIMESTAMP), - update_leader_warn_time_(OB_INVALID_TIMESTAMP), - pending_end_lsn_(), + state_(INIT), + recovery_start_time_us_(OB_INVALID_TIMESTAMP), scan_disk_log_finished_(false), - is_sync_enabled_(true), - allow_vote_(true), - allow_vote_persisted_(true), - replica_type_(NORMAL_REPLICA), - is_changing_config_with_arb_(false), - last_set_changing_config_with_arb_time_us_(OB_INVALID_TIMESTAMP), - broadcast_leader_(), - last_recv_leader_broadcast_time_us_(OB_INVALID_TIMESTAMP), is_inited_(false) {} -int LogStateMgr::init(const int64_t palf_id, - const common::ObAddr &self, - const LogPrepareMeta &log_prepare_meta, - const LogReplicaPropertyMeta &replica_property_meta, - election::Election* election, +int LogStateMgr::init(const common::ObAddr &self, LogSlidingWindow *sw, - LogReconfirm *reconfirm, - LogEngine *log_engine, - LogConfigMgr *mm, - LogModeMgr *mode_mgr, - palf::PalfRoleChangeCbWrapper *palf_role_change_cb, - LogPlugins *plugins) + LogModeMgr *mode_mgr) { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; - PALF_LOG(ERROR, "init twice", K(ret), K_(palf_id)); - } else if (false == is_valid_palf_id(palf_id) || NULL == sw || NULL == reconfirm || NULL == log_engine - || NULL == election || NULL == mm || NULL == mode_mgr || NULL == palf_role_change_cb || - NULL == plugins || !self.is_valid() || !replica_property_meta.is_valid()) { + } else if (!self.is_valid() || OB_ISNULL(sw) || OB_ISNULL(mode_mgr)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K(palf_id), KP(sw), KP(reconfirm), K(log_prepare_meta), - KP(election), KP(log_engine), KP(mm), KP(mode_mgr), KP(palf_role_change_cb), KP(plugins), - K(replica_property_meta), K(self)); + PALF_LOG(WARN, "invalid arguments", K(ret), K(self), KP(sw), KP(mode_mgr)); } else { - prepare_meta_= log_prepare_meta; - palf_id_ = palf_id; - election_ = election; - plugins_ = plugins; + self_ = self; sw_ = sw; - reconfirm_ = reconfirm; - log_engine_ = log_engine; - mm_ = mm; mode_mgr_ = mode_mgr; - palf_role_change_cb_ = palf_role_change_cb; - self_ = self; - leader_.reset(); - role_ = FOLLOWER; - state_ = INIT; + set_state_(INIT); scan_disk_log_finished_ = false; - allow_vote_ = replica_property_meta.allow_vote_; - allow_vote_persisted_ = replica_property_meta.allow_vote_; - replica_type_ = replica_property_meta.replica_type_; - is_sync_enabled_ = !is_arb_replica(); - broadcast_leader_.reset(); is_inited_ = true; - PALF_LOG(INFO, "LogStateMgr init success", K(ret), K_(palf_id), K_(self)); - } - if (OB_SUCCESS != ret && OB_INIT_TWICE != ret) { - destroy(); - } - return ret; -} - -bool LogStateMgr::is_follower_init_() const -{ - return check_role_and_state_(FOLLOWER, INIT); -} - -bool LogStateMgr::is_follower_active_() const -{ - return check_role_and_state_(FOLLOWER, ACTIVE); -} - -bool LogStateMgr::is_leader_reconfirm_() const -{ - return check_role_and_state_(LEADER, RECONFIRM); -} - -bool LogStateMgr::is_leader_active_() const -{ - return check_role_and_state_(LEADER, ACTIVE); -} - -bool LogStateMgr::check_role_and_state_(const common::ObRole &role, const ObReplicaState &state) const -{ - bool bool_ret = false; - const int64_t tmp_val = ATOMIC_LOAD(&role_state_val_); - if ((tmp_val & UNION_ROLE_VAL_MASK) != role) { // check role_ - } else if (((tmp_val >> 32) & UNION_ROLE_VAL_MASK) != state) { // check state_ - } else { - bool_ret = true; - } - return bool_ret; -} - -void LogStateMgr::get_role_and_state(common::ObRole &role, ObReplicaState &state) const -{ - const int64_t tmp_val = ATOMIC_LOAD(&role_state_val_); - role = (ObRole)(tmp_val & UNION_ROLE_VAL_MASK); - state = (ObReplicaState)((tmp_val >> 32) & UNION_ROLE_VAL_MASK); -} - -void LogStateMgr::update_role_and_state_(const common::ObRole &new_role, const ObReplicaState &new_state) -{ - const int64_t old_val = ATOMIC_LOAD(&role_state_val_); - const int64_t new_val = (((int32_t)new_state & UNION_ROLE_VAL_MASK) << 32) + \ - ((int32_t)new_role & UNION_ROLE_VAL_MASK); - if (ATOMIC_BCAS(&role_state_val_, old_val, new_val)) { - // update success - } else { - // When updating role/state, a write lock on palf_handle is acquired, so the expectation is that it should not fail - PALF_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "update_role_and_state_ failed", K_(palf_id), K(old_val), K(new_val)); - } -} - -bool LogStateMgr::is_follower_pending_() const -{ - return check_role_and_state_(FOLLOWER, PENDING); -} - -bool LogStateMgr::can_append(const int64_t proposal_id, const bool need_check_proposal_id) const -{ - bool bool_ret = false; - if (is_leader_active_() - && ((need_check_proposal_id && proposal_id == get_proposal_id()) - || false == need_check_proposal_id) - && OB_LIKELY(mode_mgr_->can_append())) { - bool_ret = true; - } - return bool_ret; -} - -bool LogStateMgr::can_raw_write(const int64_t proposal_id, const bool need_check_proposal_id) const -{ - bool bool_ret = false; - if (is_leader_active_() - && ((need_check_proposal_id && proposal_id == get_proposal_id()) - || false == need_check_proposal_id) - && OB_LIKELY(mode_mgr_->can_raw_write())) { - bool_ret = true; - } - return bool_ret; -} - - -bool LogStateMgr::can_slide_sw() const -{ - return (is_leader_reconfirm_() || is_leader_active_() || is_follower_active_() - || is_follower_init_() || is_follower_pending_()); -} - -bool LogStateMgr::can_revoke(const int64_t proposal_id) const -{ - return true == is_leader_active_() && get_proposal_id() == proposal_id; -} - -bool LogStateMgr::is_state_changed() -{ - bool state_changed = false; - bool is_error = false; - if (is_follower_init_()) { - state_changed = follower_init_need_switch_(); - } else if (is_follower_active_()) { - (void) check_and_try_fetch_log_(); - state_changed = follower_active_need_switch_() || need_reset_broadcast_leader_(); - } else if (is_follower_pending_()) { - (void) check_and_try_fetch_log_(); - state_changed = follower_pending_need_switch_(); - if (false == state_changed) { - common::ObAddr new_leader; - int64_t new_leader_epoch = OB_INVALID_TIMESTAMP; - state_changed = follower_need_update_role_(new_leader, new_leader_epoch); - } - } else if (is_leader_reconfirm_()) { - state_changed = (leader_reconfirm_need_switch_()); - } else if (is_leader_active_()) { - state_changed = (leader_active_need_switch_(is_error)); - } else {} - return state_changed; -} - -// State machine for normal replica -// ┌──────────────────────────────────────────────────────────┐ -// │ │ -// │ follower init │ -// │ │ │ -// │ │ │ -// │ ▼ │ -// │ follower active◄───────────────┐ │ -// │ │ │ │ -// │ │ │ │ -// │ ▼ │ │ -// │ ┌─────────leader reconfirm <─────┐ │ │ -// │ │ │ │ │ │ -// │ │ │ │ │ │ -// │ │ ▼ │ │ │ -// │ │ leader active │ │ │ -// │ │ │ │ │ │ -// │ │ │ │ │ │ -// │ │ ▼ │ │ │ -// │ └─────────►follower pending──────┘───────┘ │ -// │ │ -// └──────────────────────────────────────────────────────────┘ -// -int LogStateMgr::switch_state() -{ - int ret = OB_SUCCESS; - - common::ObAddr new_leader; - bool need_next_loop = false; - do { - const ObAddr old_leader = leader_; - const common::ObRole old_role = role_; - const int32_t old_state = state_; - int64_t old_leader_epoch = leader_epoch_; - const int64_t start_ts = ObTimeUtility::current_time(); - - need_next_loop = false; - if (is_follower_init_()) { - if (follower_init_need_switch_()) { - ret = init_to_follower_active_(); - need_next_loop = true; // next loop will check new leader - } - } else if (is_follower_pending_()) { - new_leader.reset(); - int64_t new_leader_epoch = OB_INVALID_TIMESTAMP; - if (follower_pending_need_switch_()) { - ret = pending_to_follower_active_(); - if (follower_active_need_switch_()) { - need_next_loop = true; - } - } else if (follower_need_update_role_(new_leader, new_leader_epoch)) { - ret = follower_pending_to_reconfirm_(new_leader_epoch); - need_next_loop = true; // 1) drive reconfirm - } else { - // do nothing - } - } else if (is_follower_active_()) { - if (follower_active_need_switch_()) { - new_leader.reset(); - int64_t new_leader_epoch = OB_INVALID_TIMESTAMP; - if (follower_need_update_role_(new_leader, new_leader_epoch)) { - ret = follower_active_to_reconfirm_(new_leader_epoch); - if (OB_SUCC(ret)) { - ret = reconfirm_to_leader_active_(); - } - } else { - set_leader_and_epoch_(new_leader, new_leader_epoch); - } - need_next_loop = true; // 1) drive reconfirm or 2) fetch log from leader - } else if (need_reset_broadcast_leader_()) { - reset_broadcast_leader_(); - } else { } - } else if (is_leader_reconfirm_()) { - if (is_reconfirm_timeout_()) { - ret = reconfirm_to_follower_pending_(); - } else if (need_update_leader_(new_leader)) { - ret = reconfirm_to_follower_pending_(); - } else if (is_reconfirm_state_changed_(ret)) { - if (OB_SUCC(ret)) { - ret = reconfirm_to_leader_active_(); - } else { - ret = reconfirm_to_follower_pending_(); - } - } else { - // do nothing - } - } else if (is_leader_active_()) { - bool is_error = false; - if (leader_active_need_switch_(is_error)) { - ret = leader_active_to_follower_pending_(); - need_next_loop = true; - } - } else { - // do nothing - } - - PALF_LOG(INFO, "switch_state", K(ret), K_(palf_id), K(old_role), K_(role), - K(old_state), K_(state), K(old_leader), K_(leader), - K(old_leader_epoch), K_(leader_epoch), K(need_next_loop), - "diff", ObTimeUtility::current_time() - start_ts); - } while (need_next_loop && OB_SUCC(ret)); - - if (OB_EAGAIN == ret) { - // rewrite ret code to OB_SUCCESS when it's OB_EAGAIN - ret = OB_SUCCESS; - } - return ret; -} - -int LogStateMgr::set_scan_disk_log_finished() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - scan_disk_log_finished_ = true; - PALF_LOG(INFO, "scan disk log finished", K_(palf_id)); - } - return ret; -} - -int64_t LogStateMgr::get_proposal_id() const -{ - return ATOMIC_LOAD(&(prepare_meta_.log_proposal_id_)); -} - -bool LogStateMgr::can_handle_prepare_response(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id()) { - bool_ret = is_leader_reconfirm_() || is_leader_active_(); - } - return bool_ret; -} - -bool LogStateMgr::can_handle_prepare_request(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id > get_proposal_id()) { - bool_ret = (is_leader_active_() || is_leader_reconfirm_() || is_follower_active_() \ - || is_follower_pending_()); - } - return bool_ret; -} - -bool LogStateMgr::can_handle_leader_broadcast(const common::ObAddr &server, - const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id()) { - bool_ret = (is_follower_active_()); - } - return bool_ret; -} - -bool LogStateMgr::can_handle_committed_info(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id() && true == is_sync_enabled()) { - bool_ret = (is_follower_active_() || is_follower_pending_()); - } - return bool_ret; -} - -bool LogStateMgr::can_receive_log(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id() && true == is_sync_enabled() && mode_mgr_->can_receive_log()) { - bool_ret = (is_follower_active_() || is_follower_pending_() || is_reconfirm_can_receive_log_()); - } - return bool_ret; -} - -bool LogStateMgr::is_reconfirm_can_receive_log_() const -{ - return (is_leader_reconfirm_() && reconfirm_->can_receive_log()); -} - -bool LogStateMgr::can_receive_config_log(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id()) { - // can receive config log even if is_sync_enabled is false - bool_ret = (is_follower_active_() || is_follower_pending_() || is_leader_reconfirm_()); - } - return bool_ret; -} - -int LogStateMgr::handle_prepare_request(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (INVALID_PROPOSAL_ID == proposal_id || !server.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (proposal_id < get_proposal_id()) { - (void) reject_prepare_request_(server, get_proposal_id()); - // NB: 'vote_granted' is not used nowday, so we just only accept prepare request - // with greater 'proposal_id'. - } else if (proposal_id == get_proposal_id()) { - // proposal_id is match with local, no need reply - PALF_LOG(INFO, "accpet same proposal prepare request, ignore it", K(ret), K_(palf_id), - K(server), K(proposal_id), K_(prepare_meta)); - } else { - // proposal_id is greater than local, means another replica has been in the prepare phase, - // this replica need roll back to follower active. - if (server != self_ && LEADER == role_) { - if (is_leader_reconfirm_()) { - ret = reconfirm_to_follower_pending_(); - PALF_LOG(INFO, "there is another replica has been leader which used greater prorposl id", K(server), - K(proposal_id), K(self_), "local proposal_id", get_proposal_id()); - } else if (is_leader_active_()) { - ret = leader_active_to_follower_pending_(); - } else {} - } - - if (OB_SUCC(ret)) { - if (server != leader_) { - // update leader_ - leader_ = server; - broadcast_leader_ = server; - last_recv_leader_broadcast_time_us_ = common::ObTimeUtility::current_time(); - } - - if (OB_FAIL(write_prepare_meta_(proposal_id, server))) { - PALF_LOG(WARN, "write_prepare_meta_ failed", K(ret), K_(palf_id), K(proposal_id), K(server)); - } - } - PALF_LOG(INFO, "LogStateMgr handle_prepare_request success", K(ret), K_(palf_id), - K(self_), K(server), K(proposal_id), K(leader_), K_(broadcast_leader)); - } - return ret; -} - -int LogStateMgr::handle_leader_broadcast(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - broadcast_leader_ = server; - last_recv_leader_broadcast_time_us_ = common::ObTimeUtility::current_time(); - return ret; -} - -int LogStateMgr::reject_prepare_request_(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(server), K(proposal_id)); - } else { - const bool vote_granted = false; - int64_t fake_log_proposal_id = INVALID_PROPOSAL_ID; - LSN fake_lsn; - LogModeMeta fake_mode_meta; - if (OB_FAIL(log_engine_->submit_prepare_meta_resp(server, proposal_id, - vote_granted, fake_log_proposal_id, fake_lsn, fake_lsn, fake_mode_meta))) { - PALF_LOG(WARN, "submit_prepare_response failed", K(ret), K_(palf_id)); - } else { - PALF_LOG(INFO, "reject_prepare_request_ success", K(ret), K_(palf_id), K(server), K(proposal_id), K(vote_granted)); - } + PALF_LOG(INFO, "LogStateMgr init success", K_(self)); } return ret; } @@ -509,761 +60,113 @@ int LogStateMgr::reject_prepare_request_(const common::ObAddr &server, void LogStateMgr::destroy() { is_inited_ = false; - is_sync_enabled_ = true; + scan_disk_log_finished_ = false; sw_ = NULL; - reconfirm_ = NULL; - log_engine_ = NULL; - mm_ = NULL; mode_mgr_ = NULL; - election_ = NULL; - plugins_ = NULL; - scan_disk_log_finished_ = false; - update_role_and_state_(FOLLOWER, INIT); - leader_.reset(); - leader_epoch_ = OB_INVALID_TIMESTAMP; - pending_end_lsn_.reset(); self_.reset(); - is_changing_config_with_arb_ = false; -} - -int LogStateMgr::write_prepare_meta_(const int64_t &proposal_id, - const common::ObAddr &new_leader) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogStateMgr is not inited", K(ret), K_(palf_id)); - } else if (INVALID_PROPOSAL_ID == proposal_id || !new_leader.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K(proposal_id), K(new_leader)); - } else { - LogPrepareMeta prepare_meta; - FlushMetaCbCtx flush_cb_ctx; - flush_cb_ctx.type_ = PREPARE_META; - flush_cb_ctx.proposal_id_ = proposal_id; - if (OB_FAIL(prepare_meta.generate(LogVotedFor(), proposal_id))) { - PALF_LOG(WARN, "prepare_meta generate failed", K(ret), K_(palf_id)); - } else if (OB_FAIL(log_engine_->submit_flush_prepare_meta_task(flush_cb_ctx, prepare_meta))) { - PALF_LOG(WARN, "submit_flush_prepare_meta_task failed", K(ret), K_(palf_id)); - } else { - // update prepare with 'proposal-id' - prepare_meta_ = prepare_meta; - PALF_LOG(INFO, "write_prepare_meta_ success", K(prepare_meta), K(new_leader), K(prepare_meta_)); - } - } - return ret; -} - -int LogStateMgr::init_to_follower_active_() -{ - int ret = OB_SUCCESS; - reset_status_(); - update_role_and_state_(FOLLOWER, ACTIVE); - plugins_->record_role_change_event(palf_id_, FOLLOWER, ObReplicaState::INIT, - FOLLOWER, ObReplicaState::ACTIVE); - PALF_EVENT("init_to_follower_active", palf_id_, K(ret), K_(self)); - return ret; -} - -int LogStateMgr::to_follower_active_() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(palf_role_change_cb_->on_role_change(palf_id_))) { - PALF_LOG(WARN, "on_leader_revoke failed!", K(ret), K_(palf_id)); - } else {} - PALF_LOG(INFO, "to_follower_active_ finished!", K(ret), K_(palf_id), K_(self)); - return ret; -} - -int LogStateMgr::pending_to_follower_active_() -{ - int ret = OB_SUCCESS; - update_role_and_state_(FOLLOWER, ACTIVE); - // Need to update role/state first, then submit role_change_event - // Otherwise handle role_change event execution may occur first leading to role switch failure - if (OB_FAIL(to_follower_active_())) { - PALF_LOG(ERROR, "to_follower_active_ failed", K(ret), K_(palf_id)); - } - PALF_REPORT_INFO_KV(K_(leader), K_(pending_end_lsn)); - plugins_->record_role_change_event(palf_id_, FOLLOWER, ObReplicaState::PENDING, - FOLLOWER, ObReplicaState::ACTIVE, EXTRA_INFOS); - PALF_EVENT("follower_pending_to_follower_active", palf_id_, K(ret), K_(self), K_(leader), - K_(pending_end_lsn)); - return ret; -} - -int LogStateMgr::to_reconfirm_(const int64_t new_leader_epoch) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(sw_->to_leader_reconfirm())) { - PALF_LOG(ERROR, "sw to_leader_reconfirm failed", K(ret), K_(palf_id)); - } else { - reconfirm_start_time_us_ = ObTimeUtility::current_time(); - reset_status_(); - update_role_and_state_(LEADER, RECONFIRM); - reconfirm_->reset_state(); - leader_ = self_; - leader_epoch_ = new_leader_epoch; - // New leader need clear logs after sw' first empty slot(log hole). - // If not, later local flush waiting will fail. - (void) sw_->clean_log(); - } - - return ret; -} - -int LogStateMgr::follower_active_to_reconfirm_(const int64_t new_leader_epoch) -{ - int ret = to_reconfirm_(new_leader_epoch); - PALF_REPORT_INFO_KV(K_(leader), K(new_leader_epoch)); - plugins_->record_role_change_event(palf_id_, FOLLOWER, ObReplicaState::ACTIVE, - LEADER, ObReplicaState::RECONFIRM, EXTRA_INFOS); - PALF_EVENT("follower_active_to_reconfirm", palf_id_, K(ret), K_(self), K(new_leader_epoch), - K(leader_), K_(pending_end_lsn)); - return ret; -} - -int LogStateMgr::follower_pending_to_reconfirm_(const int64_t new_leader_epoch) -{ - int ret = to_reconfirm_(new_leader_epoch); - PALF_REPORT_INFO_KV(K_(leader), K(new_leader_epoch), K_(pending_end_lsn)); - plugins_->record_role_change_event(palf_id_, FOLLOWER, ObReplicaState::PENDING, - LEADER, ObReplicaState::RECONFIRM, EXTRA_INFOS); - PALF_EVENT("follower_pending_to_reconfirm", palf_id_, K(ret), K_(self), K(new_leader_epoch), - K(leader_), K_(pending_end_lsn)); - return ret; -} - -int LogStateMgr::reconfirm_to_follower_pending_() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(to_follower_pending_())) { - PALF_LOG(WARN, "to_follower_pending_ failed, try again", K(ret), K_(palf_id)); - } else { - reset_status_(); - update_role_and_state_(FOLLOWER, PENDING); - PALF_REPORT_INFO_KV(K_(leader)); - plugins_->record_role_change_event(palf_id_, LEADER, ObReplicaState::RECONFIRM, - FOLLOWER, ObReplicaState::PENDING, EXTRA_INFOS); - PALF_EVENT("reconfirm_to_follower_pending", palf_id_, K_(self), K_(leader), K(lbt())); - } - return ret; + set_state_(INIT); } -int LogStateMgr::reconfirm_to_leader_active_() -{ - int ret = OB_SUCCESS; - const int64_t reconfirm_stage_cost = ObTimeUtility::current_time() - reconfirm_start_time_us_; - PALF_EVENT("reconfirm_to_leader_active begin", palf_id_, K_(self), K(reconfirm_stage_cost)); - ObMemberList member_list; - int64_t replica_num = -1; - LogConfigVersion config_version; - if (OB_FAIL(mm_->get_config_version(config_version))) { - PALF_LOG(WARN, "get_config_version failed", K(ret), K_(palf_id)); - } - // set_initial_member_list hasn't been called, so self_ may not be in member_list - // else if (OB_FAIL(mm_->get_alive_member_list_with_arb(member_list, replica_num))) { - // PALF_LOG(WARN, "get_alive_member_list_with_arb failed", K(ret), K_(palf_id)); - // } else if (!member_list.contains(self_)) { - // PALF_LOG(ERROR, "curr_member_list doesn't contain self, revoke", K_(palf_id), - // K(member_list), K_(self)); - // } - else if (OB_FAIL(sw_->to_leader_active())) { - PALF_LOG(WARN, "sw leader_active failed", K(ret), K_(palf_id)); - } else { - update_role_and_state_(LEADER, ACTIVE); - // Need to update role/state and committed_end_lsn first, then submit role_change_event - // Otherwise handle role_change event execution may occur first leading to role switch failure - if (OB_FAIL(to_leader_active_())) { - PALF_LOG(ERROR, "to_leader_active_ failed", K(ret), K_(palf_id)); - } else if (OB_FAIL(mm_->submit_broadcast_leader_info(get_proposal_id()))) { - PALF_LOG(ERROR, "submit_broadcast_leader_info failed", KR(ret)); - } - const int64_t reconfirm_to_active_cost = ObTimeUtility::current_time() - reconfirm_start_time_us_; - PALF_EVENT("reconfirm_to_leader_active end", palf_id_, K(ret), K_(self), K(reconfirm_to_active_cost), K_(role), K_(state)); - PALF_REPORT_INFO_KV(K(reconfirm_stage_cost), K(reconfirm_to_active_cost), K(config_version)); - plugins_->record_role_change_event(palf_id_, LEADER, ObReplicaState::RECONFIRM, - LEADER, ObReplicaState::ACTIVE, EXTRA_INFOS); - } - - return ret; -} - -int LogStateMgr::leader_active_to_follower_pending_() -{ - int ret = OB_SUCCESS; - - if (LEADER == role_) { - if (OB_FAIL(to_follower_pending_())) { - PALF_LOG(WARN, "to_follower_pending_ failed, try again", K(ret), K_(palf_id)); - } else { - reset_status_(); - } - } else { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "unexpected role", K(ret), K_(palf_id), K_(role)); - } - - if (OB_SUCC(ret)) { - update_role_and_state_(FOLLOWER, PENDING); - } - - PALF_REPORT_INFO_KV(K_(pending_end_lsn)); - plugins_->record_role_change_event(palf_id_, LEADER, ObReplicaState::ACTIVE, - FOLLOWER, ObReplicaState::PENDING, EXTRA_INFOS); - PALF_EVENT("leader_active_to_follower_pending_", palf_id_, K(ret), K_(self), K_(role), K_(state), K_(pending_end_lsn)); - return ret; -} - -void LogStateMgr::reset_status_() -{ - leader_.reset(); - leader_epoch_ = OB_INVALID_TIMESTAMP; - last_check_start_id_time_us_ = OB_INVALID_TIMESTAMP; - ATOMIC_STORE(&is_changing_config_with_arb_, false); - mm_->reset_status(); - mode_mgr_->reset_status(); - PALF_LOG(INFO, "reset_status_", K_(palf_id), K_(self), K_(leader_epoch), K(leader_)); -} - -int LogStateMgr::to_leader_active_() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(palf_role_change_cb_->on_role_change(palf_id_))) { - PALF_LOG(WARN, "on_leader_takeover failed!", K(ret), K_(palf_id)); - } - PALF_LOG(INFO, "to_leader_active_ finished!", K(ret), K_(palf_id)); - return ret; -} - -int LogStateMgr::truncate(const LSN &truncate_begin_lsn) -{ - int ret = OB_SUCCESS; - if (!truncate_begin_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (pending_end_lsn_.is_valid() - && truncate_begin_lsn < pending_end_lsn_) { - // If truncate_begin_lsn is less than pending_end_lsn_, - // need update it to avoid waiting slide fail. - pending_end_lsn_ = truncate_begin_lsn; - } else { - // no need update pending_end_lsn_ - } - PALF_LOG(INFO, "state_mgr truncate finished", K(ret), K_(palf_id), K_(pending_end_lsn), - K(truncate_begin_lsn)); - return ret; -} - -int LogStateMgr::to_follower_pending_() -{ - // save curr_end_lsn as pending_end_lsn_ - int ret = OB_SUCCESS; - LSN curr_end_lsn; - if (OB_FAIL(sw_->to_follower_pending(curr_end_lsn))) { - PALF_LOG(WARN, "get_curr_end_lsn failed", K(ret), K_(palf_id)); - } else { - pending_end_lsn_ = curr_end_lsn; - } - PALF_LOG(INFO, "to_follower_pending_", K(ret), K_(palf_id), K_(pending_end_lsn)); - return ret; -} - -bool LogStateMgr::follower_init_need_switch_() -{ - bool state_changed = false; - if (scan_disk_log_finished_) { - state_changed = true; - } - return state_changed; -} - -bool LogStateMgr::is_pending_log_clear_() const -{ - bool bool_ret = false; - int tmp_ret = OB_SUCCESS; - LSN last_slide_end_lsn; - if (!pending_end_lsn_.is_valid()) { - PALF_LOG_RET(ERROR, OB_INVALID_ERROR, "pending_end_lsn_ is invalid", K_(pending_end_lsn), K_(palf_id)); - } else if (OB_SUCCESS != (tmp_ret = sw_->get_last_slide_end_lsn(last_slide_end_lsn))) { - PALF_LOG_RET(WARN, tmp_ret, "get_last_slide_end_lsn failed", K(tmp_ret), K_(palf_id)); - } else if (last_slide_end_lsn >= pending_end_lsn_) { - bool_ret = true; - } else { - bool_ret = false; - if (palf_reach_time_interval(1000 * 1000, check_follower_pending_warn_time_)) { - PALF_LOG(INFO, "follower pending log not clear, need wait", K(bool_ret), K_(palf_id), K(last_slide_end_lsn), K_(pending_end_lsn)); - } - } - return bool_ret; -} - -bool LogStateMgr::follower_pending_need_switch_() -{ - bool state_changed = false; - int tmp_ret = OB_SUCCESS; - LSN last_slide_end_lsn; - if (is_pending_log_clear_()) { - state_changed = true; - PALF_LOG(INFO, "follower_pending_need_switch_", K(state_changed), K_(palf_id)); - } - return state_changed; -} - -bool LogStateMgr::follower_active_need_switch_() -{ - bool state_changed = false; - common::ObAddr new_leader; - if (need_update_leader_(new_leader)) { - state_changed = true; - } else if (new_leader.is_valid() - && self_ == new_leader) { - if (is_arb_replica()) { - PALF_LOG(INFO, "arb replica is leader, skip", K_(palf_id), K_(self), K(new_leader)); - } else { - state_changed = true; - } - } else {} - return state_changed; -} - -bool LogStateMgr::leader_reconfirm_need_switch_() -{ - int state_changed = false; - common::ObAddr new_leader; - int ret = OB_SUCCESS; - if (is_reconfirm_need_wlock_()) { - state_changed = true; - } else if (is_reconfirm_timeout_()) { - state_changed = true; - } else if (is_reconfirm_state_changed_(ret)) { - state_changed = true; - } else if (need_update_leader_(new_leader)) { - state_changed = true; - } else {} - return state_changed; -} - -bool LogStateMgr::is_reconfirm_timeout_() -{ - bool bool_ret = false; - const int64_t now_us = ObTimeUtility::current_time(); - - const int64_t start_id = sw_->get_start_id(); - if (OB_INVALID_TIMESTAMP == last_check_start_id_time_us_ - || last_check_start_id_ != start_id) { - last_check_start_id_ = start_id; - last_check_start_id_time_us_ = now_us; - } else { - bool is_sw_timeout = false; - - if (now_us - last_check_start_id_time_us_ > PALF_LEADER_RECONFIRM_SYNC_TIMEOUT_US) { - // start log of sw is timeout - is_sw_timeout = true; - } - bool_ret = is_sw_timeout; - if (bool_ret) { - if (palf_reach_time_interval(1 * 1000 * 1000, check_reconfirm_timeout_time_)) { - PALF_LOG_RET(WARN, OB_TIMEOUT, "leader reconfirm timeout", K_(palf_id), K(start_id), K(is_sw_timeout), K_(reconfirm)); - LOG_DBA_WARN_V2(OB_LOG_LEADER_RECONFIRM_TIMEOUT, OB_TIMEOUT, "leader reconfirm timeout"); - (void) sw_->report_log_task_trace(start_id); - } - } else if (palf_reach_time_interval(100 * 1000, check_reconfirm_timeout_time_)) { - PALF_LOG(INFO, "leader reconfirm need wait", K_(palf_id), K(start_id), K(is_sw_timeout), - K_(self), K_(reconfirm)); - } else { - // do nothing - } - } - return bool_ret; -} - -bool LogStateMgr::is_reconfirm_need_wlock_() -{ - return reconfirm_->need_wlock(); -} - -bool LogStateMgr::is_reconfirm_state_changed_(int &ret) -{ - bool bool_ret = false; - ret = reconfirm_->reconfirm(); - if (OB_EAGAIN == ret) { - } else if (OB_SUCC(ret)) { - PALF_LOG(TRACE, "is_reconfirm_state_changed_", K(bool_ret)); - bool_ret = true; - } else { - // election leader may has changed, so ret is maybe OB_NOT_MASTER. - PALF_LOG(WARN, "reconfirm failed", K_(palf_id)); - bool_ret = true; - } - return bool_ret; -} - -bool LogStateMgr::leader_active_need_switch_(bool &is_error) -{ - bool state_changed = false; - common::ObAddr new_leader; - is_error = false; - if (need_update_leader_(new_leader)) { - state_changed = true; - } else if (LEADER == role_) { - state_changed = check_leader_log_sync_state_(); - if (state_changed) { - is_error = true; - } - } else { - // do nothing - } - return state_changed; -} - -bool LogStateMgr::check_leader_log_sync_state_() -{ - // This function is called only in state. - bool state_changed = false; - const int64_t start_id = sw_->get_start_id(); - const int64_t now_us = ObTimeUtility::current_time(); - if (OB_INVALID_TIMESTAMP == last_check_start_id_time_us_ - || last_check_start_id_ != start_id) { - last_check_start_id_ = start_id; - last_check_start_id_time_us_ = now_us; - } else if (sw_->is_empty()) { - // sw is empty - last_check_start_id_time_us_ = now_us; - } else { - // sw is not empty, check log sync state - if (now_us - last_check_start_id_time_us_ > PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US) { - if (palf_reach_time_interval(10 * 1000 * 1000, log_sync_timeout_warn_time_)) { - PALF_LOG_RET(WARN, OB_TIMEOUT, "log sync timeout on leader", K_(palf_id), K_(self), K(now_us), K(last_check_start_id_time_us_), K(start_id)); - LOG_DBA_WARN_V2(OB_LOG_LEADER_LOG_SYNC_TIMEOUT, OB_TIMEOUT, "log sync timeout on leader"); - (void) sw_->report_log_task_trace(start_id); - } - } - } - return state_changed; -} - -bool LogStateMgr::need_update_leader_(common::ObAddr &new_leader) -{ - bool bool_ret = false; - int ret = OB_SUCCESS; - int64_t new_leader_epoch = OB_INVALID_TIMESTAMP; - - if (OB_FAIL(get_elect_leader_(new_leader, new_leader_epoch))) { - if (OB_ELECTION_WARN_INVALID_LEADER != ret) { - PALF_LOG(WARN, "get_elect_leader_ failed", K(ret), K_(palf_id)); - } - if (leader_.is_valid()) { - bool_ret = true; - PALF_LOG(WARN, "get_elect_leader_ failed, leader_ is valid, need update", K(ret), - K_(palf_id), K_(self), K(leader_), K(bool_ret)); - } - } else { - if (new_leader != leader_ || new_leader_epoch != leader_epoch_) { - bool_ret = true; - if (palf_reach_time_interval(1 * 1000 * 1000, update_leader_warn_time_)) { - PALF_LOG(WARN, "leader or epoch has changed, need update", K(bool_ret), K(new_leader), - K_(palf_id), K(new_leader_epoch), K(leader_), K(leader_epoch_), K_(self)); - } - } - } - - return bool_ret; -} - -bool LogStateMgr::need_reset_broadcast_leader_() const -{ - const int64_t curr_time_us = common::ObTimeUtility::current_time(); - return broadcast_leader_.is_valid() && - (curr_time_us - last_recv_leader_broadcast_time_us_ > \ - 2 * PALF_BROADCAST_LEADER_INFO_INTERVAL_US); -} - -bool LogStateMgr::follower_need_update_role_(common::ObAddr &new_leader, - int64_t &new_leader_epoch) -{ - bool bool_ret = false; - int ret = OB_SUCCESS; - if (OB_FAIL(get_elect_leader_(new_leader, new_leader_epoch))) { - if (OB_ELECTION_WARN_INVALID_LEADER != ret) { - PALF_LOG(WARN, "get_elect_leader_ failed", K(ret), K_(palf_id)); - } - new_leader.reset(); - new_leader_epoch = OB_INVALID_TIMESTAMP; - } else { - bool_ret = (self_ == new_leader); - if (bool_ret && is_arb_replica()) { - PALF_LOG(INFO, "arb replica is leader, skip", K_(palf_id), K_(self), K(new_leader), K(new_leader_epoch)); - bool_ret = false; - } - PALF_LOG(TRACE, "follower_need_update_role_", K(new_leader), K(new_leader_epoch)); - } - return bool_ret; -} - -void LogStateMgr::set_leader_and_epoch_(const common::ObAddr &new_leader, const int64_t new_leader_epoch) +bool LogStateMgr::is_state_changed() { - leader_ = new_leader; - leader_epoch_ = new_leader_epoch; - PALF_LOG(INFO, "set_leader_and_epoch_", K_(palf_id), K_(self), K(leader_), K(leader_epoch_), K(new_leader)); + return is_inited_ && ((INIT == state_ && scan_disk_log_finished_) || RECOVERING == state_); } -int LogStateMgr::get_elect_leader_(common::ObAddr &leader, - int64_t &leader_epoch) const +int LogStateMgr::switch_state() { int ret = OB_SUCCESS; - ObAddr tmp_addr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_SUCC(election_->get_current_leader_likely(tmp_addr, leader_epoch))) { - leader = tmp_addr; - } else { -// if (palf_reach_time_interval(1000 * 1000, last_get_leader_fail_time_)) { - PALF_LOG(WARN, "get_current_leader_likely failed", K(ret), K_(palf_id)); -// } - } - - return ret; -} - -int LogStateMgr::check_role_leader_and_state() const -{ - int ret = OB_SUCCESS; - if (!check_role_and_state_(LEADER, ACTIVE)) { - ret = OB_NOT_MASTER; - } - return ret; -} - -bool LogStateMgr::check_epoch_is_same_with_election(const int64_t expected_epoch) const -{ - int ret = OB_SUCCESS; - bool bool_ret = false; - common::ObAddr ununsed_leader; - ObRole curr_election_role = ObRole::INVALID_ROLE; - int64_t curr_election_epoch = OB_INVALID_TIMESTAMP; - if (OB_FAIL(election_->get_role(curr_election_role, curr_election_epoch))) { - PALF_LOG(WARN, "get_elect_leader_ failed", K(ret)); - } else if (OB_UNLIKELY(curr_election_role != ObRole::LEADER)) { - PALF_LOG(WARN, "election is not leader now", K(ret), K(curr_election_role), K(curr_election_epoch)); - } else if (OB_UNLIKELY(curr_election_epoch != expected_epoch)) { - PALF_LOG(WARN, "election epoch has changed", K(ret), K(curr_election_epoch), K(expected_epoch)); - } else { - bool_ret = true; - } - return bool_ret; -} - -bool LogStateMgr::can_send_log_ack(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id != get_proposal_id()) { - // proposal_id has changed during disk flushing - } else { - bool_ret = (is_follower_active_() || is_follower_pending_()); - } - return bool_ret; -} - -bool LogStateMgr::can_receive_log_ack(const int64_t &proposal_id) const -{ - bool bool_ret = false; - if (proposal_id == get_proposal_id()) { - bool_ret = (LEADER == role_ && mode_mgr_->can_receive_log()); - } - return bool_ret; -} - -bool LogStateMgr::can_truncate_log() const -{ - bool bool_ret = (is_follower_active_() || is_follower_pending_()); - return bool_ret; -} - -bool LogStateMgr::need_freeze_group_buffer() const -{ - // Replica need freeze group buffer during state. - // PENDING state is not considered here because all logs have been freezed before replica - // switches into PENDING. - return is_leader_active_(); -} - -int LogStateMgr::check_and_try_fetch_log_() -{ - int ret = OB_SUCCESS; - const int64_t now_us = ObTimeUtility::current_time(); - const uint64_t start_id = sw_->get_start_id(); - if (false == ATOMIC_LOAD(&is_sync_enabled_)) { - if (palf_reach_time_interval(5 * 1000 * 1000, check_sync_enabled_time_)) { - PALF_LOG(INFO, "sync is disabled, cannot fetch log", K_(palf_id)); + } else if (INIT == state_) { + if (scan_disk_log_finished_) { + ret = start_recovery_(); } - } else if (OB_INVALID_ID == last_check_start_id_ - || OB_INVALID_TIMESTAMP == last_check_start_id_time_us_ - || (last_check_start_id_ == start_id - && now_us - last_check_start_id_time_us_ > PALF_FETCH_LOG_INTERVAL_US)) { - if (OB_FAIL(sw_->try_fetch_log(FetchTriggerType::LOG_LOOP_TH))) { - PALF_LOG(WARN, "sw try_fetch_log failed", K(ret), K_(palf_id)); - } else { - last_check_start_id_ = start_id; - last_check_start_id_time_us_ = now_us; - PALF_LOG(TRACE, "sw try_fetch_log success", K(ret), K_(palf_id), K(start_id)); + } else if (RECOVERING == state_) { + ret = recover_(); + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; } - } else if (last_check_start_id_ != start_id) { - last_check_start_id_ = start_id; - last_check_start_id_time_us_ = now_us; - } else {} - return ret; -} - -bool LogStateMgr::is_sync_enabled() const -{ - return ATOMIC_LOAD(&is_sync_enabled_); -} - -int LogStateMgr::enable_sync() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_arb_replica()) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(INFO, "can not enable_sync in arb_replica", K(ret), K_(palf_id), K(self_), - "replica_type", replica_type_2_str(replica_type_)); - } else { - ATOMIC_STORE(&is_sync_enabled_, true); - PALF_LOG(INFO, "enable_sync success", K(ret), K_(palf_id), K(self_)); } return ret; } -int LogStateMgr::disable_sync() +int LogStateMgr::set_scan_disk_log_finished() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; -// } else if (true == is_follower_pending_()) { -// ret = OB_STATE_NOT_MATCH; -// PALF_LOG(WARN, "can not disable sycn in follower pending", K(ret), KPC(this)); } else { - ATOMIC_STORE(&is_sync_enabled_, false); - PALF_LOG(INFO, "disable_sync success", K(ret), K_(palf_id), K(self_)); + scan_disk_log_finished_ = true; } return ret; } -bool LogStateMgr::is_allow_vote() const +bool LogStateMgr::can_append() const { - return ATOMIC_LOAD(&allow_vote_); + return is_active() && mode_mgr_->can_append(); } -bool LogStateMgr::is_allow_vote_persisted() const +bool LogStateMgr::can_slide_sw() const { - - return ATOMIC_LOAD(&allow_vote_persisted_); + return is_inited_; } -int LogStateMgr::disable_vote_in_mem() +bool LogStateMgr::need_freeze_group_buffer() const { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ATOMIC_STORE(&allow_vote_, false); - PALF_LOG(INFO, "disable_vote_in_mem success", K(ret), K_(palf_id), K_(self)); - } - return ret; + return is_active(); } -int LogStateMgr::disable_vote() +bool LogStateMgr::is_active() const { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ATOMIC_STORE(&allow_vote_, false); - ATOMIC_STORE(&allow_vote_persisted_, false); - PALF_LOG(INFO, "disable_vote success", K(ret), K_(palf_id), K_(self)); - } - return ret; + return is_state_(ACTIVE); } -int LogStateMgr::enable_vote() +bool LogStateMgr::is_recovering() const { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - ATOMIC_STORE(&allow_vote_, true); - ATOMIC_STORE(&allow_vote_persisted_, true); - PALF_LOG(INFO, "enable_vote success", K(ret), K_(palf_id), K_(self)); - } - return ret; + return is_state_(RECOVERING); } -LogReplicaType LogStateMgr::get_replica_type() const +bool LogStateMgr::is_state_(const LogState state) const { - return ATOMIC_LOAD(&replica_type_); + return state == get_state(); } -bool LogStateMgr::is_arb_replica() const +void LogStateMgr::set_state_(const LogState state) { - return get_replica_type() == LogReplicaType::ARBITRATION_REPLICA; + ATOMIC_STORE(&state_, static_cast(state)); } -int LogStateMgr::get_election_role(common::ObRole &role, int64_t &epoch) const +int LogStateMgr::start_recovery_() { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(election_->get_role(role, epoch))) { - PALF_LOG(WARN, "get elect role failed", K(ret), K_(self), K_(palf_id)); - } else { - // do nothing - } - return ret; -} - -// protected by wlock in PalfHandleImpl -int LogStateMgr::set_changing_config_with_arb() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (true == ATOMIC_BCAS(&is_changing_config_with_arb_, false, true)) { - last_set_changing_config_with_arb_time_us_ = common::ObTimeUtility::current_time(); - PALF_EVENT("set_changing_config_with_arb to true", palf_id_, K(ret), K_(self)); - } - return ret; + recovery_start_time_us_ = ObTimeUtility::current_time(); + set_state_(RECOVERING); + return OB_SUCCESS; } -// protected by wlock in PalfHandleImpl -int LogStateMgr::reset_changing_config_with_arb() +int LogStateMgr::recover_() { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (true == ATOMIC_BCAS(&is_changing_config_with_arb_, true, false)) { - const int64_t cost_time_us = (common::ObTimeUtility::current_time() - - last_set_changing_config_with_arb_time_us_); - PALF_EVENT("set_changing_config_with_arb to false", palf_id_, K(ret), K_(self), K(cost_time_us)); + LSN max_flushed_end_lsn; + LSN last_slide_lsn; + LSN committed_end_lsn; + int64_t last_slide_log_id = OB_INVALID_LOG_ID; + if (!sw_->check_all_log_has_flushed()) { + ret = OB_EAGAIN; + } else { + sw_->get_max_flushed_end_lsn(max_flushed_end_lsn); + if (OB_FAIL(sw_->try_advance_committed_end_lsn(max_flushed_end_lsn))) { + PALF_LOG(WARN, "advance committed end lsn failed", K(ret), K(max_flushed_end_lsn)); + } else if (!sw_->is_all_committed_log_slided_out( + last_slide_lsn, last_slide_log_id, committed_end_lsn)) { + ret = OB_EAGAIN; + } else if (OB_FAIL(sw_->activate())) { + PALF_LOG(WARN, "activate sliding window failed", K(ret)); + } else { + set_state_(ACTIVE); + } } return ret; } -// protected by rlock in PalfHandleImpl -bool LogStateMgr::is_changing_config_with_arb() const -{ - return ATOMIC_LOAD(&is_changing_config_with_arb_); -} - -void LogStateMgr::reset_broadcast_leader_() -{ - if (need_reset_broadcast_leader_()) { - PALF_LOG_RET(INFO, OB_SUCCESS, "reset_broadcast_leader_", K_(self), K_(palf_id), - K_(last_recv_leader_broadcast_time_us), K_(broadcast_leader)); - broadcast_leader_.reset(); - } -} - } // namespace palf } // namespace oceanbase diff --git a/src/logservice/palf/log_state_mgr.h b/src/logservice/palf/log_state_mgr.h index cf3de32a7..8864e76f1 100644 --- a/src/logservice/palf/log_state_mgr.h +++ b/src/logservice/palf/log_state_mgr.h @@ -19,204 +19,53 @@ #include "log_define.h" #include "log_meta_info.h" -#include "common/ob_member_list.h" -#include "common/ob_role.h" -#include "lib/lock/ob_spin_lock.h" -#include "lib/utility/utility.h" -#include "lsn.h" -#include "share/ob_ls_id.h" -#include "election_self.h" +#include "palf_callback_wrapper.h" namespace oceanbase { namespace palf { -class PalfRoleChangeCbWrapper; -} -namespace palf -{ -class ObIElection; -class LogSlidingWindow; -class LogReconfirm; -class LogEngine; -class LogConfigMgr; class LogModeMgr; -class LogPrepareMeta; +class LogSlidingWindow; class LogStateMgr { public: LogStateMgr(); virtual ~LogStateMgr() { destroy(); } -public: + + virtual int init(const common::ObAddr &self, + LogSlidingWindow *sw, + LogModeMgr *mode_mgr); virtual void destroy(); - virtual int init(const int64_t key, - const common::ObAddr &self, - const LogPrepareMeta &log_prepare_meta, - const LogReplicaPropertyMeta &replica_property_meta, - election::Election* election, - LogSlidingWindow *sw, - LogReconfirm *reconfirm, - LogEngine *log_engine, - LogConfigMgr *mm, - LogModeMgr *mode_mgr_, - palf::PalfRoleChangeCbWrapper *palf_role_change_cb, - LogPlugins *plugins); virtual bool is_state_changed(); virtual int switch_state(); - virtual int handle_prepare_request(const common::ObAddr &new_leader, - const int64_t &proposal_id); virtual int set_scan_disk_log_finished(); - virtual bool can_append(const int64_t proposal_id, const bool need_check_proposal_id) const; - virtual bool can_raw_write(const int64_t proposal_id, const bool need_check_proposal_id) const; + + virtual bool can_append() const; virtual bool can_slide_sw() const; - virtual bool can_handle_committed_info(const int64_t &proposal_id) const; - virtual bool can_revoke(const int64_t proposal_id) const; - virtual int check_role_leader_and_state() const; - virtual int64_t get_proposal_id() const; - virtual common::ObRole get_role() const { return role_; } - virtual int16_t get_state() const { return state_; } - virtual void get_role_and_state(common::ObRole &role, ObReplicaState &state) const; - virtual const common::ObAddr &get_leader() const { return leader_; } - virtual int64_t get_leader_epoch() const { return leader_epoch_; } - virtual bool check_epoch_is_same_with_election(const int64_t expected_epoch) const; - virtual bool can_handle_prepare_response(const int64_t &proposal_id) const; - virtual bool can_handle_prepare_request(const int64_t &proposal_id) const; - virtual bool can_receive_log(const int64_t &proposal_id) const; - virtual bool can_receive_config_log(const int64_t &proposal_id) const; - virtual bool can_send_log_ack(const int64_t &proposal_id) const; - virtual bool can_receive_log_ack(const int64_t &proposal_id) const; - virtual bool can_truncate_log() const; + virtual LogState get_state() const { return static_cast(ATOMIC_LOAD(&state_)); } virtual bool need_freeze_group_buffer() const; - virtual bool is_leader_active() const { return is_leader_active_(); } - virtual bool is_follower_pending() const { return is_follower_pending_(); } - virtual bool is_follower_active() const { return is_follower_active_(); } - virtual bool is_leader_reconfirm() const { return is_leader_reconfirm_(); } - virtual int truncate(const LSN &truncate_begin_lsn); - virtual int enable_sync(); - virtual int disable_sync(); - virtual bool is_sync_enabled() const; - virtual bool is_allow_vote() const; - virtual bool is_allow_vote_persisted() const; - //only modify allow_vote_ - virtual int disable_vote_in_mem(); - //modify allow_vote_ and allow_vote_persisted_ - virtual int disable_vote(); - virtual int enable_vote(); - virtual LogReplicaType get_replica_type() const; - virtual int get_election_role(common::ObRole &role, int64_t &epoch) const; - virtual bool is_arb_replica() const; - virtual int set_changing_config_with_arb(); - virtual int reset_changing_config_with_arb(); - virtual bool is_changing_config_with_arb() const; - virtual const common::ObAddr &get_broadcast_leader() const { return broadcast_leader_;} - virtual bool can_handle_leader_broadcast(const common::ObAddr &server, - const int64_t &proposal_id) const; - virtual int handle_leader_broadcast(const common::ObAddr &server, - const int64_t &proposal_id); - TO_STRING_KV(KP(this), K_(self), K_(palf_id), "role", role_to_string(role_), "replica_type", \ - replica_type_2_str(replica_type_), "state", replica_state_to_string(state_), K_(prepare_meta), \ - K_(leader), K_(leader_epoch), K_(is_sync_enabled), K_(allow_vote), K_(allow_vote_persisted), K_(pending_end_lsn), \ - K_(scan_disk_log_finished), K_(last_check_start_id), K_(is_changing_config_with_arb), \ - K_(reconfirm_start_time_us), KP_(palf_role_change_cb), K_(allow_vote)); -private: - bool check_role_and_state_(const common::ObRole &role, const ObReplicaState &state) const; - void update_role_and_state_(const common::ObRole &new_role, const ObReplicaState &new_state); - int reject_prepare_request_(const common::ObAddr &server, - const int64_t &proposal_id); - bool is_reconfirm_can_receive_log_() const; - bool is_follower_active_() const; - bool is_follower_init_() const; - bool is_leader_reconfirm_() const; - bool is_leader_active_() const; - bool is_follower_pending_() const; - - bool follower_init_need_switch_(); - bool follower_active_need_switch_(); - bool leader_reconfirm_need_switch_(); - bool leader_active_need_switch_(bool &is_error); - bool follower_pending_need_switch_(); - bool is_pending_log_clear_() const; - int write_prepare_meta_(const int64_t &proposal_id, - const common::ObAddr &new_leader); - int init_to_follower_active_(); - int to_reconfirm_(const int64_t new_leader_epoch); - int follower_active_to_reconfirm_(const int64_t new_leader_epoch); - int follower_pending_to_reconfirm_(const int64_t new_leader_epoch); - int reconfirm_to_leader_active_(); - int reconfirm_to_follower_pending_(); - int leader_active_to_follower_pending_(); - int pending_to_follower_active_(); + virtual bool is_active() const; + virtual bool is_recovering() const; - int to_leader_active_(); - int to_follower_active_(); - int to_follower_pending_(); + TO_STRING_KV(KP(this), K_(self), "state", log_state_to_string(get_state()), + K_(scan_disk_log_finished), + K_(recovery_start_time_us)); - void reset_status_(); - - bool is_reconfirm_timeout_(); - bool is_reconfirm_need_wlock_(); - bool is_reconfirm_state_changed_(int &ret); - - bool need_update_leader_(common::ObAddr &new_leader); - bool follower_need_update_role_(common::ObAddr &new_leader, int64_t &new_leader_epoch); - void set_leader_and_epoch_(const common::ObAddr &new_leader, const int64_t new_leader_epoch); - int get_elect_leader_(common::ObAddr &leader, int64_t &leader_epoch) const; - bool check_leader_log_sync_state_(); - bool need_fetch_log_() const; - int check_and_try_fetch_log_(); - bool need_reset_broadcast_leader_() const; - void reset_broadcast_leader_(); private: - static const uint64_t UNION_ROLE_VAL_MASK = (1ll << 32) - 1; + bool is_state_(const LogState state) const; + void set_state_(const LogState state); + int start_recovery_(); + int recover_(); + private: common::ObAddr self_; - int64_t palf_id_; - LogPrepareMeta prepare_meta_; LogSlidingWindow *sw_; - LogReconfirm *reconfirm_; - LogEngine *log_engine_; - LogConfigMgr *mm_; LogModeMgr *mode_mgr_; - palf::PalfRoleChangeCbWrapper *palf_role_change_cb_; - election::Election* election_; - LogPlugins *plugins_; - union - { - int64_t role_state_val_; - struct - { - common::ObRole role_ : 32; - ObReplicaState state_ : 32; - }; - }; - common::ObAddr leader_; - int64_t leader_epoch_; - int64_t last_check_start_id_; - int64_t last_check_start_id_time_us_; - int64_t last_check_pending_replay_cnt_; - int64_t reconfirm_start_time_us_; - int64_t check_sync_enabled_time_; - int64_t check_reconfirm_timeout_time_; - mutable int64_t check_follower_pending_warn_time_; - mutable int64_t log_sync_timeout_warn_time_; - mutable int64_t update_leader_warn_time_; - LSN pending_end_lsn_; + int32_t state_; + int64_t recovery_start_time_us_; bool scan_disk_log_finished_; - // whether log sync is enabled, it's true by default - bool is_sync_enabled_; - // whether this replica is allowed to reply ack when receiving logs - // it's true by default - bool allow_vote_; - // value of allow_vote persisted_, will be modified after meta is flushed - bool allow_vote_persisted_; - // whether this replica is an arbitration replica - LogReplicaType replica_type_; - // is changing config with arbitration member, stop appending logs - bool is_changing_config_with_arb_; - int64_t last_set_changing_config_with_arb_time_us_; - common::ObAddr broadcast_leader_; - int64_t last_recv_leader_broadcast_time_us_; bool is_inited_; DISALLOW_COPY_AND_ASSIGN(LogStateMgr); diff --git a/src/logservice/palf/log_storage.cpp b/src/logservice/palf/log_storage.cpp index eb89adf1e..e88b4a95d 100644 --- a/src/logservice/palf/log_storage.cpp +++ b/src/logservice/palf/log_storage.cpp @@ -33,7 +33,6 @@ LogStorage::LogStorage() : ILogStorage(ILogStorageType::DISK_STORAGE), log_block_header_(), curr_block_writable_size_(0), need_append_block_header_(false), - palf_id_(INVALID_PALF_ID), logical_block_size_(0), tail_info_lock_(common::ObLatchIds::PALF_LOG_ENGINE_LOCK), delete_block_lock_(common::ObLatchIds::PALF_LOG_ENGINE_LOCK), @@ -49,7 +48,7 @@ LogStorage::~LogStorage() } int LogStorage::init(const char *base_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, + const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, @@ -61,7 +60,6 @@ int LogStorage::init(const char *base_dir, const char *sub_dir, const LSN &base_ } else if (OB_FAIL(do_init_(base_dir, sub_dir, base_lsn, - palf_id, logical_block_size, align_size, align_buf_size, @@ -70,10 +68,9 @@ int LogStorage::init(const char *base_dir, const char *sub_dir, const LSN &base_ plugins, log_cache, io_adapter))) { - PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir), K(palf_id)); + PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir)); } else { - PALF_LOG(INFO, "LogStorage init success", K(ret), K(base_dir), K(sub_dir), - K(palf_id), K(base_lsn)); + PALF_LOG(INFO, "LogStorage init success", K(ret), K(base_dir), K(sub_dir), K(base_lsn)); } return ret; } @@ -111,11 +108,9 @@ void LogStorage::destroy() { is_inited_ = false; logical_block_size_ = 0; - palf_id_ = INVALID_PALF_ID; need_append_block_header_ = false; curr_block_writable_size_ = 0; log_block_header_.reset(); - readable_log_tail_.reset(); log_tail_.reset(); log_reader_.destroy(); block_mgr_.destroy(); @@ -403,10 +398,10 @@ int LogStorage::truncate_prefix_blocks(const LSN &lsn) reset_log_tail_for_last_block_(lsn, false); block_mgr_.reset(lsn_2_block(lsn, logical_block_size_)); } - PALF_EVENT("truncate_prefix_blocks success", palf_id_, K(ret), KPC(this), + PALF_EVENT("truncate_prefix_blocks success", K(ret), KPC(this), K(lsn), K(block_id), K(min_block_id), K(max_block_id), K(truncate_end_block_id)); - plugins_->record_truncate_event(palf_id_, lsn, min_block_id, max_block_id, truncate_end_block_id); + plugins_->record_truncate_event(lsn, min_block_id, max_block_id, truncate_end_block_id); return ret; } @@ -548,7 +543,6 @@ int LogStorage::load_last_block_(const block_id_t min_block_id, int LogStorage::do_init_(const char *base_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, @@ -577,11 +571,10 @@ int LogStorage::do_init_(const char *base_dir, } else if (OB_FAIL(log_reader_.init(log_dir, logical_block_size + MAX_INFO_BLOCK_SIZE, io_adapter))) { PALF_LOG(ERROR, "LogReader init failed", K(ret), K(log_dir)); } else { - log_tail_ = readable_log_tail_ = base_lsn; + log_tail_ = base_lsn; log_block_header_.reset(); curr_block_writable_size_ = 0; need_append_block_header_ = true; - palf_id_ = palf_id; logical_block_size_ = logical_block_size; update_manifest_cb_ = update_manifest_cb; plugins_ = plugins; @@ -594,15 +587,6 @@ int LogStorage::do_init_(const char *base_dir, return ret; } -// To ensure the integrity of each read data, we need check the 'block_id' whether is integrity -// after read data successfully. -// -// To ensure the integrity of log blocks, we need check the 'block_id' which opened failed whether -// is deleted by others. -// -// 1. For delete block, LogBlockMgr will inc 'min_block_id_' firstly, and then reuse the block, -// after reading, if 'block_id' is smaller than 'min_block_id_', means that the data is not integrity. -// int LogStorage::check_read_out_of_bound_(const block_id_t &block_id, const bool no_such_block) const { @@ -610,7 +594,7 @@ int LogStorage::check_read_out_of_bound_(const block_id_t &block_id, block_id_t min_block_id = LOG_INVALID_BLOCK_ID; block_id_t max_block_id = LOG_INVALID_BLOCK_ID; if (OB_FAIL(get_block_id_range(min_block_id, max_block_id)) && OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(ERROR, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id)); + PALF_LOG(ERROR, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id)); } else if (min_block_id > block_id) { ret = OB_ERR_OUT_OF_LOWER_BOUND; PALF_LOG(INFO, "read something out of lower bound, the block may be deleted by GC or rebuild", @@ -621,8 +605,6 @@ int LogStorage::check_read_out_of_bound_(const block_id_t &block_id, K(min_block_id), K(max_block_id), K(block_id)); } if (OB_SUCC(ret) && no_such_block) { - // if there is no block whose names with 'block_id' and 'block_id' is in range of [min_block_id, max_block_id) - // return OB_ERR_UNEXPECTED. if (min_block_id <= block_id && block_id < max_block_id) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "unexpected error, the block may be deleted by human", KPC(this), @@ -681,8 +663,7 @@ int LogStorage::update_block_header_(const block_id_t block_id, int64_t pos = 0; log_block_header_.update_lsn_and_scn(block_min_lsn, block_min_scn); - log_block_header_.update_palf_id_and_curr_block_id( - palf_id_, lsn_2_block(log_tail_, logical_block_size_)); + log_block_header_.update_curr_block_id(lsn_2_block(log_tail_, logical_block_size_)); log_block_header_.calc_checksum(); if (FALSE_IT(memset(block_header_serialize_buf_, '\0', MAX_INFO_BLOCK_SIZE))) { @@ -711,28 +692,18 @@ void LogStorage::update_log_tail_guarded_by_lock_(const int64_t log_size) { ObSpinLockGuard guard(tail_info_lock_); log_tail_ = log_tail_ + log_size; - if (readable_log_tail_ < log_tail_) { - readable_log_tail_ = log_tail_; - } } void LogStorage::update_log_tail_guarded_by_lock_(const LSN &lsn) { ObSpinLockGuard guard(tail_info_lock_); log_tail_ = lsn; - readable_log_tail_ = log_tail_; } -const LSN &LogStorage::get_log_tail_guarded_by_lock_() const +LSN LogStorage::get_log_tail_guarded_by_lock_() const { ObSpinLockGuard guard(tail_info_lock_); - return readable_log_tail_; -} - -void LogStorage::get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail) const -{ - ObSpinLockGuard guard(tail_info_lock_); - readable_log_tail = readable_log_tail_; + return log_tail_; } offset_t LogStorage::get_phy_offset_(const LSN &lsn) const @@ -750,20 +721,16 @@ int LogStorage::read_block_header_(const block_id_t block_id, ReadBufGuard read_buf_guard("LogStorage", in_read_size); ReadBuf &read_buf = read_buf_guard.read_buf_; - // 'readable_log_tail' and 'block_header' are snapshot, we can read valid data even if the block - // is deleted. NB: we need ensure that the lsn_2_block('readable_log_tail') is smaller than or - // equal to 'max_block_id'. - LSN readable_log_tail; - get_readable_log_tail_guarded_by_lock_(readable_log_tail); - block_id_t max_block_id = lsn_2_block(readable_log_tail, logical_block_size_); - bool last_block_has_data = (0 == lsn_2_offset(readable_log_tail, logical_block_size_) ? false : true); + const LSN log_tail = get_log_tail_guarded_by_lock_(); + block_id_t max_block_id = lsn_2_block(log_tail, logical_block_size_); + bool last_block_has_data = (0 != lsn_2_offset(log_tail, logical_block_size_)); if (!read_buf.is_valid()) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "allocate memory failed"); } else if (block_id > max_block_id || (block_id == max_block_id && false == last_block_has_data)) { ret = OB_ERR_OUT_OF_UPPER_BOUND; PALF_LOG(WARN, "block_id is large than max_block_id", K(ret), K(block_id), - K(readable_log_tail), K(max_block_id), K(log_block_header)); + K(log_tail), K(max_block_id), K(log_block_header)); } else { LogIOContext io_ctx(LogIOUser::META_INFO); if (OB_FAIL(log_reader_.pread(block_id, 0, in_read_size, read_buf, out_read_size, io_ctx))) { @@ -781,11 +748,6 @@ int LogStorage::read_block_header_(const block_id_t block_id, } // to ensure the data integrity, we should check 'block_id' whether has integrity data. int tmp_ret = check_read_out_of_bound_(block_id, OB_NO_SUCH_FILE_OR_DIRECTORY == ret); - // overwrite ret code: - // 1. if ret is OB_NO_SUCH_FILE_OR_DIRECTORY, the block may be recycled or overwritten. - // 2. if ret is OB_INVALID_DATA, the block may be being recycled or overwritten. - // 3. if ret is OB_SUCCESS, we should check the data has been read whether is integrity because the block - // may be being recycled or overwritten. if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret || OB_INVALID_DATA == ret || OB_SUCC(ret)) { @@ -824,11 +786,10 @@ int LogStorage::inner_pread_(const LSN &read_lsn, { int ret = OB_SUCCESS; // NB: don't support read data from diffent file. - LSN readable_log_tail; - get_readable_log_tail_guarded_by_lock_(readable_log_tail); + const LSN log_tail = get_log_tail_guarded_by_lock_(); const block_id_t read_block_id = lsn_2_block(read_lsn, logical_block_size_); const LSN curr_block_end_lsn = LSN((read_block_id + 1) * logical_block_size_); - const LSN &max_readable_lsn = MIN(readable_log_tail, curr_block_end_lsn); + const LSN &max_readable_lsn = MIN(log_tail, curr_block_end_lsn); const int64_t real_in_read_size = MIN(max_readable_lsn - read_lsn, in_read_size); const offset_t read_offset = lsn_2_offset(read_lsn, logical_block_size_); const offset_t real_read_offset = @@ -836,7 +797,7 @@ int LogStorage::inner_pread_(const LSN &read_lsn, const LSN begin_lsn = get_begin_lsn(); - if (read_lsn >= readable_log_tail) { + if (read_lsn >= log_tail) { ret = OB_ERR_OUT_OF_UPPER_BOUND; PALF_LOG(WARN, "read something out of upper bound", K(ret), K(read_lsn), K(log_tail_)); } else if (read_lsn < begin_lsn) { @@ -878,15 +839,11 @@ int LogStorage::inner_pread_(const LSN &read_lsn, K(real_in_read_size), K(read_lsn), K(out_read_size), - K(readable_log_tail)); + K(log_tail)); } // to ensure the data integrity, we should check 'read_block_id' whether has integrity data. int tmp_ret = check_read_out_of_bound_(read_block_id, OB_NO_SUCH_FILE_OR_DIRECTORY == ret); - // overwrite ret code: - // 1. if ret is OB_NO_SUCH_FILE_OR_DIRECTORY, the block may be recycled or overwritten. - // 2. if ret is OB_SUCCESS, we should check the data has been read whether is integrity because the block - // may be being recycled or overwritten. if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret || OB_SUCC(ret)) { ret = tmp_ret; @@ -903,7 +860,7 @@ void LogStorage::reset_log_tail_for_last_block_(const LSN &lsn, bool last_block_ (void)truncate_block_header_(lsn); curr_block_writable_size_ = (true == last_block_exist) ? logical_block_size_ - logical_offset : 0; need_append_block_header_ = (curr_block_writable_size_ == logical_block_size_) ? true : false; - log_tail_ = readable_log_tail_ = lsn; + log_tail_ = lsn; } int LogStorage::update_manifest_(const block_id_t expected_next_block_id, const bool in_restart) diff --git a/src/logservice/palf/log_storage.h b/src/logservice/palf/log_storage.h index 2753139bc..f5bfa0b61 100644 --- a/src/logservice/palf/log_storage.h +++ b/src/logservice/palf/log_storage.h @@ -54,7 +54,6 @@ class LogStorage : public ILogStorage int init(const char *log_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, @@ -68,7 +67,6 @@ class LogStorage : public ILogStorage int load(const char *log_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, @@ -126,7 +124,6 @@ class LogStorage : public ILogStorage int get_logical_block_size(int64_t &logical_block_size) const; TO_STRING_KV(K_(log_tail), - K_(readable_log_tail), K_(log_block_header), K_(block_mgr), K(logical_block_size_), @@ -137,7 +134,6 @@ class LogStorage : public ILogStorage int do_init_(const char *log_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, @@ -172,8 +168,7 @@ class LogStorage : public ILogStorage void update_log_tail_guarded_by_lock_(const int64_t log_size); void update_log_tail_guarded_by_lock_(const LSN &lsn); - const LSN &get_log_tail_guarded_by_lock_() const; - void get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail) const; + LSN get_log_tail_guarded_by_lock_() const; offset_t get_phy_offset_(const LSN &lsn) const; int read_block_header_(const block_id_t block_id, LogBlockHeader &block_header) const; bool check_last_block_is_full_(const block_id_t max_block_id) const; @@ -193,14 +188,11 @@ class LogStorage : public ILogStorage LogBlockMgr block_mgr_; LogReader log_reader_; LSN log_tail_; - // Always advances with 'log_tail_'. - LSN readable_log_tail_; LogBlockHeader log_block_header_; // Used to detemine whether need switch block. int64_t curr_block_writable_size_; // Whether need to append block header; bool need_append_block_header_; - int64_t palf_id_; int64_t logical_block_size_; // used to protect log_tail_ and log_block_header_ mutable ObSpinLock tail_info_lock_; @@ -222,7 +214,6 @@ template int LogStorage::load(const char *base_dir, const char *sub_dir, const LSN &base_lsn, - const int64_t palf_id, const int64_t logical_block_size, const int64_t align_size, const int64_t align_buf_size, @@ -244,7 +235,6 @@ int LogStorage::load(const char *base_dir, } else if (OB_FAIL(do_init_(base_dir, sub_dir, base_lsn, - palf_id, logical_block_size, align_size, align_buf_size, @@ -253,7 +243,7 @@ int LogStorage::load(const char *base_dir, plugins, log_cache, io_adapter))) { - PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir), K(palf_id)); + PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir)); // NB: if there is no valid data on disk, no need to load last block } else if (OB_FAIL(block_mgr_.get_block_id_range(min_block_id, max_block_id)) && OB_ENTRY_NOT_EXIST != ret) { @@ -294,13 +284,13 @@ int LogStorage::locate_log_tail_and_last_valid_entry_header_(const block_id_t mi // NB: 'log_tail_' need point to the tail of 'iterate_block_id', because 'pread' interface // check whether iterating to the end of redo log block depends on this field. log_block_header_.reset(); - log_block_header_.update_palf_id_and_curr_block_id(palf_id_, iterate_block_id); + log_block_header_.update_curr_block_id(iterate_block_id); PalfIterator iterator; auto get_file_end_lsn = []() { return LSN(LOG_MAX_LSN_VAL); }; LSN start_lsn(iterate_block_id * logical_block_size_); if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, this))) { PALF_LOG(WARN, "PalfGroupBufferIterator init failed", K(ret), K(start_lsn)); - } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(palf_id_, palf::LogIOUser::RESTART)))) { + } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(palf::LogIOUser::RESTART)))) { PALF_LOG(WARN, "set_io_context failed", K(ret), K(start_lsn)); } else { iterator.set_need_print_error(need_print_error); diff --git a/src/logservice/palf/log_task.cpp b/src/logservice/palf/log_task.cpp index 173e60c66..afb2c3a31 100644 --- a/src/logservice/palf/log_task.cpp +++ b/src/logservice/palf/log_task.cpp @@ -31,14 +31,11 @@ LogTaskHeaderInfo& LogTaskHeaderInfo::operator=(const LogTaskHeaderInfo &rval) this->min_scn_ = rval.min_scn_; this->max_scn_ = rval.max_scn_; this->data_len_ = rval.data_len_; - this->proposal_id_ = rval.proposal_id_; this->prev_lsn_ = rval.prev_lsn_; - this->prev_proposal_id_ = rval.prev_proposal_id_; this->committed_end_lsn_ = rval.committed_end_lsn_; this->data_checksum_ = rval.data_checksum_; this->accum_checksum_ = rval.accum_checksum_; this->is_padding_log_ = rval.is_padding_log_; - this->is_raw_write_ = rval.is_raw_write_; } return *this; } @@ -51,14 +48,11 @@ void LogTaskHeaderInfo::reset() min_scn_.reset(); max_scn_.reset(); data_len_ = 0; - proposal_id_ = INVALID_PROPOSAL_ID; prev_lsn_.reset(); - prev_proposal_id_ = INVALID_PROPOSAL_ID; committed_end_lsn_.reset(); data_checksum_ = -1; accum_checksum_ = -1; is_padding_log_ = false; - is_raw_write_ = false; } bool LogTaskHeaderInfo::is_valid() const @@ -140,7 +134,6 @@ LogTask::LogTask() freeze_ts_(OB_INVALID_TIMESTAMP), submit_ts_(OB_INVALID_TIMESTAMP), flushed_ts_(OB_INVALID_TIMESTAMP), - push_log_type_(PushLogType::PUSH_LOG), lock_() { reset(); @@ -194,7 +187,7 @@ void LogTask::update_data_len(const int64_t data_len) int LogTask::set_initial_header_info(const LogTaskHeaderInfo &header_info) { // Caller need hold lock. - // This is called by leader when submitting the first log entry for this log_task. + // Called when submitting the first entry for this task. int ret = OB_SUCCESS; if (is_valid()) { ret = OB_ERR_UNEXPECTED; @@ -203,7 +196,6 @@ int LogTask::set_initial_header_info(const LogTaskHeaderInfo &header_info) header_.begin_lsn_ = header_info.begin_lsn_; header_.log_id_ = header_info.log_id_; header_.is_padding_log_ = header_info.is_padding_log_; - header_.proposal_id_ = header_info.proposal_id_; header_.min_scn_ = header_info.min_scn_; update_data_len(header_info.data_len_); // Note: Here direct assignment cannot be used, inc_update must be used instead, because other log_entry may have already updated max_scn_ @@ -252,13 +244,11 @@ int LogTask::set_group_header(const LSN &lsn, const SCN &scn, const LogGroupEntr header_.end_lsn_ = lsn + group_entry_header.get_serialize_size() + group_entry_header.get_data_len(); header_.log_id_ = group_entry_header.get_log_id(); header_.is_padding_log_ = group_entry_header.is_padding_log(); - header_.proposal_id_ = group_entry_header.get_log_proposal_id(); // leader's proposal_id when generate this log header_.min_scn_ = scn; header_.max_scn_ = group_entry_header.get_max_scn(); header_.data_len_ = group_entry_header.get_data_len(); // total len without log_group_entry_header header_.committed_end_lsn_ = group_entry_header.get_committed_end_lsn(); // Out-of-order log reception requires this value after filling in the preceding gaps header_.accum_checksum_ = group_entry_header.get_accum_checksum(); - header_.is_raw_write_ = group_entry_header.is_raw_write(); set_valid(); // set valid } return ret; @@ -269,11 +259,6 @@ void LogTask::set_prev_lsn(const LSN &prev_lsn) header_.prev_lsn_ = prev_lsn; } -void LogTask::set_prev_log_proposal_id(const int64_t &prev_log_proposal_id) -{ - header_.prev_proposal_id_ = prev_log_proposal_id; -} - void LogTask::set_end_lsn(const LSN &end_lsn) { header_.end_lsn_ = end_lsn; @@ -435,15 +420,5 @@ void LogTask::set_flushed_ts(const int64_t ts) ATOMIC_STORE(&flushed_ts_, ts); } -void LogTask::set_push_log_type(const PushLogType &push_log_type) -{ - ATOMIC_STORE(&push_log_type_, push_log_type); -} - -bool LogTask::is_fetch_log_type() const -{ - return PushLogType::FETCH_LOG_RESP == ATOMIC_LOAD(&push_log_type_); -} - } // namespace palf } // namespace oceanbase diff --git a/src/logservice/palf/log_task.h b/src/logservice/palf/log_task.h index 16136199a..b773834e3 100644 --- a/src/logservice/palf/log_task.h +++ b/src/logservice/palf/log_task.h @@ -22,7 +22,6 @@ #include "fixed_sliding_window.h" #include "log_define.h" // block_id_t #include "lsn.h" -#include "log_req.h" // PushLogType namespace oceanbase { @@ -39,23 +38,20 @@ struct LogTaskHeaderInfo share::SCN min_scn_; share::SCN max_scn_; int64_t data_len_; // total len without log_group_entry_header - int64_t proposal_id_; // leader's proposal_id when generate this log LSN prev_lsn_; - int64_t prev_proposal_id_; // Out-of-order log reception requires validation of this value - LSN committed_end_lsn_; // When logs are received out of order, this value is needed to fill in the preceding gaps - int64_t data_checksum_; // follower receives log for verification, local accum_checksum calculation also uses this value - int64_t accum_checksum_; // follower receives logs out of order and performs checksum validation after filling in preceding gaps with local values + LSN committed_end_lsn_; + int64_t data_checksum_; + int64_t accum_checksum_; bool is_padding_log_; - bool is_raw_write_; LogTaskHeaderInfo() { reset(); } ~LogTaskHeaderInfo() { reset(); } LogTaskHeaderInfo& operator=(const LogTaskHeaderInfo &rval); void reset(); bool is_valid() const; - TO_STRING_KV(K_(begin_lsn), K_(end_lsn), K_(log_id), K_(min_scn), K_(max_scn), K_(data_len), K_(proposal_id), - K_(prev_lsn), K_(prev_proposal_id), K_(committed_end_lsn), - K_(data_checksum), K_(accum_checksum), K_(is_padding_log), K_(is_raw_write)); + TO_STRING_KV(K_(begin_lsn), K_(end_lsn), K_(log_id), K_(min_scn), K_(max_scn), K_(data_len), + K_(prev_lsn), K_(committed_end_lsn), + K_(data_checksum), K_(accum_checksum), K_(is_padding_log)); }; class LogSimpleBitMap @@ -130,29 +126,22 @@ class LogTask : public FixedSlidingWindowSlot void set_group_log_checksum(const int64_t data_checksum); void set_prev_lsn(const LSN &prev_lsn); LSN get_prev_lsn() const { return header_.prev_lsn_; } - void set_prev_log_proposal_id(const int64_t &prev_log_proposal_id); - int64_t get_prev_proposal_id() const { return header_.prev_proposal_id_; } LogTaskHeaderInfo get_header_info() const { return header_; } bool is_padding_log() const { return header_.is_padding_log_; } - bool is_raw_write() const { return header_.is_raw_write_; } int64_t get_data_len() const { return ATOMIC_LOAD(&(header_.data_len_)); } int64_t get_log_id() const { return header_.log_id_; } const share::SCN get_min_scn() const {return header_.min_scn_; } const share::SCN get_max_scn() const { return header_.max_scn_; } - int64_t get_proposal_id() const { return header_.proposal_id_; } LSN get_begin_lsn() const { return header_.begin_lsn_; } LSN get_end_lsn() const { return header_.end_lsn_; } int64_t get_accum_checksum() const { return header_.accum_checksum_; } void set_freeze_ts(const int64_t ts); void set_submit_ts(const int64_t ts); void set_flushed_ts(const int64_t ts); - void set_push_log_type(const PushLogType &push_log_type); int64_t get_gen_ts() const { return ATOMIC_LOAD(&(gen_ts_)); } int64_t get_freeze_ts() const { return ATOMIC_LOAD(&(freeze_ts_)); } int64_t get_submit_ts() const { return ATOMIC_LOAD(&(submit_ts_)); } int64_t get_flushed_ts() const { return ATOMIC_LOAD(&(flushed_ts_)); } - PushLogType get_push_log_type() const { return ATOMIC_LOAD(&push_log_type_);} - bool is_fetch_log_type() const; TO_STRING_KV(K_(header), K_(state_map), K_(ref_cnt), K_(gen_ts), K_(freeze_ts), K_(submit_ts), K_(flushed_ts), "gen_to_freeze cost time", freeze_ts_ - gen_ts_, @@ -170,7 +159,6 @@ class LogTask : public FixedSlidingWindowSlot mutable int64_t freeze_ts_; mutable int64_t submit_ts_; mutable int64_t flushed_ts_; - mutable PushLogType push_log_type_; mutable common::ObLatch lock_; }; } // end namespace palf diff --git a/src/logservice/palf/log_updater.cpp b/src/logservice/palf/log_updater.cpp deleted file mode 100644 index 1a299157c..000000000 --- a/src/logservice/palf/log_updater.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_updater.h" -#include "palf_env_impl.h" // IPalfEnvImpl -namespace oceanbase -{ -namespace palf -{ -LogUpdater::LogUpdater() : palf_env_impl_(NULL), timer_(), is_inited_(false) {} - -LogUpdater::~LogUpdater() -{ - destroy(); -} - -int LogUpdater::init(IPalfEnvImpl *palf_env_impl) -{ - int ret = OB_SUCCESS; - if (NULL == palf_env_impl) { - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(timer_.init("LogUpdater", common::ObMemAttr("LogUpdater")))) { - PALF_LOG(ERROR, "LogUpdater timer init failed", K(ret)); - } else { - palf_env_impl_ = palf_env_impl; - is_inited_ = true; - PALF_LOG(INFO, "LogUpdater init success", KPC(palf_env_impl)); - } - return ret; -} - -int LogUpdater::start() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(timer_.schedule(*this, PALF_UPDATE_CACHED_STAT_INTERVAL_US, true))) { - PALF_LOG(WARN, "LogUpdater schedule failed", K(ret)); - } else { - PALF_LOG(INFO, "LogUpdater start success", KPC(palf_env_impl_)); - } - return ret; -} - -void LogUpdater::stop() -{ - if (IS_INIT) { - PALF_LOG(INFO, "LogUpdater stop start", KPC(palf_env_impl_)); - timer_.stop(); - PALF_LOG(INFO, "LogUpdater stop finished", KPC(palf_env_impl_)); - } -} - -void LogUpdater::wait() -{ - if (IS_INIT) { - PALF_LOG(INFO, "LogUpdater wait start", KPC(palf_env_impl_)); - timer_.wait(); - PALF_LOG(INFO, "LogUpdater wait finished", KPC(palf_env_impl_)); - } -} - -void LogUpdater::destroy() -{ - PALF_LOG(INFO, "LogUpdater destroy start", KPC(palf_env_impl_)); - is_inited_ = false; - timer_.destroy(); - palf_env_impl_ = NULL; - PALF_LOG(INFO, "LogUpdater destroy finish", KPC(palf_env_impl_)); -} - -void LogUpdater::runTimerTask() -{ - int64_t start_time_us = ObTimeUtility::current_time(); - int ret = OB_SUCCESS; - if (NULL == palf_env_impl_) { - PALF_LOG(ERROR, "palf_env_impl_ is NULL, unexpected error"); - } else { - IPalfHandleImpl *handle = nullptr; - if (OB_SUCCESS == palf_env_impl_->get_palf_handle_impl(SYS_PALF_ID, handle)) { - handle->update_palf_stat(); - palf_env_impl_->revert_palf_handle_impl(handle); - } - int64_t cost_time_us = ObTimeUtility::current_time() - start_time_us; - if (cost_time_us >= PALF_UPDATE_CACHED_STAT_INTERVAL_US) { - PALF_LOG(WARN, "update_palf cost too much time", K(ret), K(cost_time_us), KPC(palf_env_impl_)); - } - } -} -} // end namespace palf -} // end namespace oceanbase diff --git a/src/logservice/palf/log_updater.h b/src/logservice/palf/log_updater.h deleted file mode 100644 index b8ffe9e70..000000000 --- a/src/logservice/palf/log_updater.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_PALF_LOG_UPDATER_ -#define OCEANBASE_PALF_LOG_UPDATER_ -#include "lib/task/ob_timer.h" -namespace oceanbase -{ -namespace palf -{ -class IPalfEnvImpl; -class LogUpdater : public common::ObTimerTask -{ -public: - LogUpdater(); - virtual ~LogUpdater(); - int init(IPalfEnvImpl *palf_env_impl); - int start(); - void stop(); - void wait(); - void destroy(); - virtual void runTimerTask(); -private: - IPalfEnvImpl *palf_env_impl_; - common::ObTimer timer_; - bool is_inited_; -}; -} // end namespace palf -} // end namespace oceanbase -#endif diff --git a/src/logservice/palf/lsn_allocator.h b/src/logservice/palf/lsn_allocator.h index fe65e5e6d..3ee801c09 100644 --- a/src/logservice/palf/lsn_allocator.h +++ b/src/logservice/palf/lsn_allocator.h @@ -72,7 +72,7 @@ class LSNAllocator // Update last_lsn and log_timestamp // called when receive_log/append_disk_log is invoked int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const share::SCN &scn); - // inc update scn base, called by change access mode and to leader active + // Increase the SCN allocation base. int inc_update_scn_base(const share::SCN &scn); int truncate(const LSN &lsn, const int64_t log_id, const share::SCN &scn); // Get last_lsn and log_timestamp diff --git a/src/logservice/palf/palf_base_info.cpp b/src/logservice/palf/palf_base_info.cpp index 20d5f0b2e..079f34655 100644 --- a/src/logservice/palf/palf_base_info.cpp +++ b/src/logservice/palf/palf_base_info.cpp @@ -26,7 +26,6 @@ LogInfo::LogInfo() log_id_(OB_INVALID_LOG_ID), lsn_(), scn_(), - log_proposal_id_(INVALID_PROPOSAL_ID), accum_checksum_(-1) {} @@ -41,7 +40,6 @@ void LogInfo::reset() log_id_ = OB_INVALID_LOG_ID; lsn_.reset(); scn_.reset(); - log_proposal_id_ = INVALID_PROPOSAL_ID; accum_checksum_ = -1; } @@ -51,7 +49,6 @@ void LogInfo::operator=(const LogInfo &log_info) this->log_id_ = log_info.log_id_; this->lsn_ = log_info.lsn_; this->scn_ = log_info.scn_; - this->log_proposal_id_ = log_info.log_proposal_id_; this->accum_checksum_ = log_info.accum_checksum_; } @@ -66,7 +63,6 @@ bool LogInfo::operator==(const LogInfo &log_info) const return log_id_ == log_info.log_id_ && lsn_ == log_info.lsn_ && scn_ == log_info.scn_ && - log_proposal_id_ == log_info.log_proposal_id_ && accum_checksum_ == log_info.accum_checksum_; } @@ -83,11 +79,10 @@ void LogInfo::generate_by_default() LSN default_prev_lsn(PALF_INITIAL_LSN_VAL); lsn_ = default_prev_lsn; scn_.set_min(); - log_proposal_id_ = INVALID_PROPOSAL_ID; accum_checksum_ = -1; } -OB_SERIALIZE_MEMBER(LogInfo, version_, log_id_, scn_, lsn_, log_proposal_id_, accum_checksum_); +OB_SERIALIZE_MEMBER(LogInfo, version_, log_id_, scn_, lsn_, accum_checksum_); PalfBaseInfo::PalfBaseInfo() : version_(-1), diff --git a/src/logservice/palf/palf_base_info.h b/src/logservice/palf/palf_base_info.h index c90a17185..0eace39ec 100644 --- a/src/logservice/palf/palf_base_info.h +++ b/src/logservice/palf/palf_base_info.h @@ -27,7 +27,7 @@ namespace palf { struct LogInfo { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: LogInfo(); ~LogInfo(); @@ -38,14 +38,13 @@ struct LogInfo bool operator!=(const LogInfo &log_info) const; void generate_by_default(); - TO_STRING_KV(K_(log_id), K_(lsn), K_(scn), K_(log_proposal_id), K_(accum_checksum)); + TO_STRING_KV(K_(log_id), K_(lsn), K_(scn), K_(accum_checksum)); - static constexpr int64_t LOG_INFO_VERSION = 1; + static constexpr int64_t LOG_INFO_VERSION = 2; int64_t version_; int64_t log_id_; LSN lsn_; share::SCN scn_; - int64_t log_proposal_id_; int64_t accum_checksum_; }; diff --git a/src/logservice/palf/palf_callback.h b/src/logservice/palf/palf_callback.h index 8c9e107f1..9e5a1576e 100644 --- a/src/logservice/palf/palf_callback.h +++ b/src/logservice/palf/palf_callback.h @@ -35,99 +35,19 @@ class PalfFSCb { public: // end_lsn returns the position of the next log after the last confirmed log - virtual int update_end_lsn(int64_t id, const LSN &end_lsn, const share::SCN &end_scn, const int64_t proposal_id) = 0; -}; - -class PalfRoleChangeCb -{ -public: - virtual int on_role_change(const int64_t id) = 0; - virtual int on_need_change_leader(const int64_t ls_id, const common::ObAddr &new_leader) = 0; -}; - -class PalfRebuildCb -{ -public: - // lsn indicates the baseline lsn point at the source end when rebuild is triggered - virtual int on_rebuild(const int64_t id, const LSN &lsn) = 0; - virtual bool is_rebuilding(const int64_t id) const = 0; -}; - -class PalfLocationCacheCb -{ -public: - virtual int get_leader(const int64_t id, common::ObAddr &leader) = 0; - virtual int nonblock_get_leader(const int64_t id, common::ObAddr &leader) = 0; + virtual int update_end_lsn(const LSN &end_lsn, const share::SCN &end_scn) = 0; }; class PalfMonitorCb { public: // record events - virtual int record_set_initial_member_list_event(const int64_t palf_id, - const int64_t replica_num, - const char *member_list = NULL, - const char *extra_info = NULL) = 0; - virtual int record_election_leader_change_event(const int64_t palf_id, const common::ObAddr &dest_addr) = 0; - virtual int record_reconfiguration_event(const char *sub_event, - const int64_t palf_id, - const LogConfigVersion& config_version, - const int64_t prev_replica_num, - const int64_t curr_replica_num, - const char *extra_info = NULL) = 0; - virtual int record_replica_type_change_event(const int64_t palf_id, - const LogConfigVersion& config_version, - const char *prev_replica_type, - const char *curr_replica_type, - const char *extra_info = NULL) = 0; - virtual int record_access_mode_change_event(const int64_t palf_id, - const int64_t prev_mode_version, - const int64_t curr_mode_verion, - const AccessMode& prev_access_mode, - const AccessMode& curr_access_mode, - const char *extra_info = NULL) = 0; - virtual int record_set_base_lsn_event(const int64_t palf_id, const LSN &new_base_lsn) = 0; - virtual int record_enable_sync_event(const int64_t palf_id) = 0; - virtual int record_disable_sync_event(const int64_t palf_id) = 0; - virtual int record_enable_vote_event(const int64_t palf_id) = 0; - virtual int record_disable_vote_event(const int64_t palf_id) = 0; - virtual int record_advance_base_info_event(const int64_t palf_id, const PalfBaseInfo &palf_base_info) = 0; - virtual int record_rebuild_event(const int64_t palf_id, - const common::ObAddr &server, - const LSN &base_lsn) = 0; - virtual int record_truncate_event(const int64_t palf_id, - const LSN &lsn, + virtual int record_set_base_lsn_event(const LSN &new_base_lsn) = 0; + virtual int record_advance_base_info_event(const PalfBaseInfo &palf_base_info) = 0; + virtual int record_truncate_event(const LSN &lsn, const int64_t min_block_id, const int64_t max_block_id, const int64_t truncate_end_block_id) = 0; - virtual int record_role_change_event(const int64_t palf_id, - const common::ObRole &prev_role, - const palf::ObReplicaState &prev_state, - const common::ObRole &curr_role, - const palf::ObReplicaState &curr_state, - const char *extra_info = NULL) = 0; - virtual int record_parent_child_change_event(const int64_t palf_id, - const bool is_register, /* true: register; false; retire; */ - const bool is_parent, /* true: parent; false: child; */ - const common::ObAddr &server, - const int64_t register_time_us, - const char *extra_info = NULL) = 0; - - // performance statistic - virtual int add_log_write_stat(const int64_t palf_id, const int64_t log_write_size) = 0; -}; - -class PalfLiteMonitorCb -{ -public: - // @desc: record creating or deleting events - // add/remove cluster: valid cluster_id, invalid tenant, invalid ls_id, - // add/remove tenant: valid cluster_id, valid tenant, invalid ls_id, - // add/remove ls: valid cluster_id, valid tenant, valid ls_id, - virtual int record_create_or_delete_event(const int64_t cluster_id, - const int64_t ls_id, - const bool is_create, - const char *extra_info) = 0; }; } // end namespace palf diff --git a/src/logservice/palf/palf_callback_wrapper.cpp b/src/logservice/palf/palf_callback_wrapper.cpp index e4076d49e..388ce9860 100644 --- a/src/logservice/palf/palf_callback_wrapper.cpp +++ b/src/logservice/palf/palf_callback_wrapper.cpp @@ -40,11 +40,11 @@ void PalfFSCbWrapper::del_cb_impl(PalfFSCbNode *cb_impl) (void)list_.remove(cb_impl); } -int PalfFSCbWrapper::update_end_lsn(int64_t id, const LSN &end_lsn, const share::SCN &end_scn, const int64_t proposal_id) +int PalfFSCbWrapper::update_end_lsn(const LSN &end_lsn, const share::SCN &end_scn) { int ret = common::OB_SUCCESS; if (OB_UNLIKELY(true == list_.is_empty())) { - PALF_LOG(TRACE, "the block size callback list is empty", K(id), K(end_lsn)); + PALF_LOG(TRACE, "the block size callback list is empty", K(end_lsn)); } else { int tmp_ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); @@ -53,130 +53,17 @@ int PalfFSCbWrapper::update_end_lsn(int64_t id, const LSN &end_lsn, const share: if (NULL == cb) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "PalfFSCb is NULL, unexpect error", KPC(node)); - } else if (OB_SUCCESS != (tmp_ret = cb->update_end_lsn(id, end_lsn, end_scn, proposal_id))) { - PALF_LOG(ERROR, "update_end_lsn failed", K(tmp_ret), K(id), K(end_lsn), K(end_scn), K(proposal_id), KPC(node)); + } else if (OB_SUCCESS != (tmp_ret = cb->update_end_lsn(end_lsn, end_scn))) { + PALF_LOG(ERROR, "update_end_lsn failed", K(tmp_ret), K(end_lsn), K(end_scn), KPC(node)); } } } return ret; } -PalfRoleChangeCbWrapper::PalfRoleChangeCbWrapper() : list_() {} -PalfRoleChangeCbWrapper::~PalfRoleChangeCbWrapper() {} - -int PalfRoleChangeCbWrapper::add_cb_impl(PalfRoleChangeCbNode *cb_impl) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(lock_); - if (false == list_.add_last(cb_impl)) { - ret = OB_ERR_UNEXPECTED; - } else { - PALF_LOG(INFO, "PalfRoleChangeCbWrapper add_cb_impl success"); - } - return ret; -} - -void PalfRoleChangeCbWrapper::del_cb_impl(PalfRoleChangeCbNode *cb_impl) -{ - ObSpinLockGuard guard(lock_); - if (NULL == list_.remove(cb_impl)) { - } else { - PALF_LOG(INFO, "PalfRoleChangeCbWrapper del_cb_impl success"); - } -} - -int PalfRoleChangeCbWrapper::on_role_change(int64_t id) -{ - int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(true == list_.is_empty())) { - PALF_LOG(INFO, "the role change callback list is empty", K(id)); - } else { - ObSpinLockGuard guard(lock_); - DLIST_FOREACH(node, list_) { - PalfRoleChangeCb *rc_cb = node->rc_cb_; - if (NULL == rc_cb) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "PalfRoleChangeCb is NULL, unexpect error", K(ret), KPC(node)); - } else if (OB_FAIL(rc_cb->on_role_change(id))) { - PALF_LOG(INFO, "on_role_change failed", K(ret), K(id), KPC(node)); - } - } - } - return ret; -} - -int PalfRoleChangeCbWrapper::on_need_change_leader(const int64_t id, const ObAddr &dest_addr) -{ - int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(true == list_.is_empty())) { - PALF_LOG(INFO, "the role change callback list is empty", K(id), K(dest_addr)); - } else { - ObSpinLockGuard guard(lock_); - DLIST_FOREACH(node, list_) { - PalfRoleChangeCb *rc_cb = node->rc_cb_; - if (NULL == rc_cb) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "PalfRoleChangeCb is NULL, unexpect error", K(ret), KPC(node), K(id), K(dest_addr)); - } else if (OB_FAIL(rc_cb->on_need_change_leader(id, dest_addr))) { - PALF_LOG(WARN, "on_need_change_leader failed", K(ret), K(id), KPC(node), K(dest_addr)); - } - } - } - return ret; -} - -PalfRebuildCbWrapper::PalfRebuildCbWrapper() : list_() {} -PalfRebuildCbWrapper::~PalfRebuildCbWrapper() {} - -int PalfRebuildCbWrapper::add_cb_impl(PalfRebuildCbNode *cb_impl) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(lock_); - if (false == list_.add_last(cb_impl)) { - ret = OB_ERR_UNEXPECTED; - } else { - PALF_LOG(INFO, "PalfRebuildCbWrapper add_cb_impl success"); - } - return ret; -} - -void PalfRebuildCbWrapper::del_cb_impl(PalfRebuildCbNode *cb_impl) -{ - ObSpinLockGuard guard(lock_); - if (NULL == list_.remove(cb_impl)) { - } else { - PALF_LOG(INFO, "PalfRebuildCbWrapper del_cb_impl success"); - } -} - -int PalfRebuildCbWrapper::on_rebuild(const int64_t id, const LSN &lsn) -{ - int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(true == list_.is_empty())) { - PALF_LOG(INFO, "the role change callback list is empty", K(id)); - } else { - ObSpinLockGuard guard(lock_); - DLIST_FOREACH(node, list_) { - PalfRebuildCb *rebuild_cb = node->rebuild_cb_; - if (NULL == rebuild_cb) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "PalfRebuildCb is NULL, unexpect error", K(ret), KPC(node)); - } else if (OB_FAIL(rebuild_cb->on_rebuild(id, lsn))) { - PALF_LOG(INFO, "on_rebuild failed", K(ret), K(id), K(lsn), KPC(node)); - } - } - } - return ret; -} - - LogPlugins::LogPlugins() - : loc_lock_(), - loc_cb_(NULL), - palf_monitor_lock_(), - palf_monitor_(NULL), - palflite_monitor_lock_(), - palflite_monitor_(NULL) { } + : palf_monitor_lock_(), + palf_monitor_(NULL) { } LogPlugins::~LogPlugins() { @@ -185,48 +72,10 @@ LogPlugins::~LogPlugins() void LogPlugins::destroy() { - { - common::RWLock::WLockGuard guard(loc_lock_); - loc_cb_ = NULL; - } { common::RWLock::WLockGuard guard(palf_monitor_lock_); palf_monitor_ = NULL; } - { - common::RWLock::WLockGuard guard(palflite_monitor_lock_); - palflite_monitor_ = NULL; - } -} - -template<> -int LogPlugins::add_plugin(PalfLocationCacheCb *plugin) -{ - int ret = OB_SUCCESS; - common::RWLock::WLockGuard guard(loc_lock_); - if (OB_ISNULL(plugin)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "Palf plugin is NULL", KP(plugin)); - } else if (OB_NOT_NULL(loc_cb_)) { - ret = OB_OP_NOT_ALLOW; - PALF_LOG(INFO, "Palf plugin is not NULL", KP(plugin), KP_(loc_cb)); - } else { - loc_cb_ = plugin; - PALF_LOG(INFO, "add_plugin success", KP(plugin)); - } - return ret; -} - -template<> -int LogPlugins::del_plugin(PalfLocationCacheCb *plugin) -{ - int ret = OB_SUCCESS; - common::RWLock::WLockGuard guard(loc_lock_); - if (OB_NOT_NULL(loc_cb_)) { - PALF_LOG(INFO, "del_plugin success", KP_(loc_cb)); - loc_cb_ = NULL; - } - return ret; } template<> @@ -259,35 +108,5 @@ int LogPlugins::del_plugin(PalfMonitorCb *plugin) return ret; } -template<> -int LogPlugins::add_plugin(PalfLiteMonitorCb *plugin) -{ - int ret = OB_SUCCESS; - common::RWLock::WLockGuard guard(palf_monitor_lock_); - if (OB_ISNULL(plugin)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "Palf plugin is NULL", KP(plugin)); - } else if (OB_NOT_NULL(palflite_monitor_)) { - ret = OB_OP_NOT_ALLOW; - PALF_LOG(INFO, "Palf plugin is not NULL", KP(plugin), KP_(loc_cb)); - } else { - palflite_monitor_ = plugin; - PALF_LOG(INFO, "add_plugin success", KP(plugin)); - } - return ret; -} - -template<> -int LogPlugins::del_plugin(PalfLiteMonitorCb *plugin) -{ - int ret = OB_SUCCESS; - common::RWLock::WLockGuard guard(palf_monitor_lock_); - if (OB_NOT_NULL(palflite_monitor_)) { - PALF_LOG(INFO, "del_plugin success", KP_(palflite_monitor)); - palflite_monitor_ = NULL; - } - return ret; -} - }; // end namespace palf }; // end namespace oceanbase diff --git a/src/logservice/palf/palf_callback_wrapper.h b/src/logservice/palf/palf_callback_wrapper.h index cda2a66e2..de402f905 100644 --- a/src/logservice/palf/palf_callback_wrapper.h +++ b/src/logservice/palf/palf_callback_wrapper.h @@ -42,54 +42,13 @@ class PalfFSCbWrapper ~PalfFSCbWrapper(); virtual int add_cb_impl(PalfFSCbNode *cb_impl); virtual void del_cb_impl(PalfFSCbNode *cb_impl); - virtual int update_end_lsn(int64_t id, const LSN &end_lsn, const share::SCN &end_scn, const int64_t proposal_id); + virtual int update_end_lsn(const LSN &end_lsn, const share::SCN &end_scn); private: // The head of list ObDList list_; ObSpinLock lock_; }; -struct PalfRoleChangeCbNode : public common::ObDLinkBase -{ - PalfRoleChangeCbNode(PalfRoleChangeCb *rc_cb) : rc_cb_(rc_cb) {} - PalfRoleChangeCb *rc_cb_; - TO_STRING_KV(KP(rc_cb_), KP(prev_), KP(next_)); -}; - -class PalfRoleChangeCbWrapper -{ -public: - PalfRoleChangeCbWrapper(); - ~PalfRoleChangeCbWrapper(); - int add_cb_impl(PalfRoleChangeCbNode *cb_impl); - void del_cb_impl(PalfRoleChangeCbNode *cb_impl); - int on_role_change(int64_t id); - int on_need_change_leader(const int64_t id, const ObAddr &new_leader); -private: - ObDList list_; - ObSpinLock lock_; -}; - -struct PalfRebuildCbNode : public common::ObDLinkBase -{ - PalfRebuildCbNode(PalfRebuildCb *rebuild_cb) : rebuild_cb_(rebuild_cb) {} - PalfRebuildCb *rebuild_cb_; - TO_STRING_KV(KP(rebuild_cb_), KP(prev_), KP(next_)); -}; - -class PalfRebuildCbWrapper -{ -public: - PalfRebuildCbWrapper(); - ~PalfRebuildCbWrapper(); - int add_cb_impl(PalfRebuildCbNode *cb_impl); - void del_cb_impl(PalfRebuildCbNode *cb_impl); - int on_rebuild(const int64_t id, const LSN &lsn); -private: - ObDList list_; - ObSpinLock lock_; -}; - #define PALF_PLUGINS_DELEGATE_PTR(delegate_obj, lock_, func_name) \ template \ int func_name(Args &&...args) { \ @@ -114,36 +73,14 @@ class LogPlugins template int del_plugin(T *plugin); - PALF_PLUGINS_DELEGATE_PTR(loc_cb_, loc_lock_, get_leader); - PALF_PLUGINS_DELEGATE_PTR(loc_cb_, loc_lock_, nonblock_get_leader); - - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_set_initial_member_list_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_election_leader_change_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_reconfiguration_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_replica_type_change_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_access_mode_change_event); PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_set_base_lsn_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_enable_sync_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_disable_sync_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_enable_vote_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_disable_vote_event); PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_advance_base_info_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_rebuild_event); PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_truncate_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_role_change_event); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, add_log_write_stat); - PALF_PLUGINS_DELEGATE_PTR(palf_monitor_, palf_monitor_lock_, record_parent_child_change_event); - - PALF_PLUGINS_DELEGATE_PTR(palflite_monitor_, palflite_monitor_lock_, record_create_or_delete_event); - TO_STRING_KV(KP_(loc_cb), KP_(palf_monitor), KP_(palflite_monitor)); + TO_STRING_KV(KP_(palf_monitor)); private: - common::RWLock loc_lock_; - PalfLocationCacheCb *loc_cb_; common::RWLock palf_monitor_lock_; PalfMonitorCb *palf_monitor_; - common::RWLock palflite_monitor_lock_; - PalfLiteMonitorCb *palflite_monitor_; }; } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/palf_env.cpp b/src/logservice/palf/palf_env.cpp index 88d452988..0a6474b36 100644 --- a/src/logservice/palf/palf_env.cpp +++ b/src/logservice/palf/palf_env.cpp @@ -54,7 +54,7 @@ int PalfEnv::create_palf_env( ret = OB_ALLOCATE_MEMORY_FAILED; } else if (OB_FAIL(FileDirectoryUtils::delete_tmp_file_or_directory_at(base_dir))) { CLOG_LOG(WARN, "delete_tmp_file_or_directory_at failed", K(ret), K(base_dir)); - } else if (OB_FAIL(palf_env->palf_env_impl_.init(options, base_dir, self, 1L, + } else if (OB_FAIL(palf_env->palf_env_impl_.init(options, base_dir, self, log_alloc_mgr, log_block_pool, monitor, log_local_device, io_manager))) { PALF_LOG(WARN, "PalfEnvImpl init failed", K(ret), K(base_dir)); @@ -96,19 +96,17 @@ void PalfEnv::destroy_() palf_env_impl_.destroy(); } -int PalfEnv::create(const int64_t id, - const AccessMode &access_mode, +int PalfEnv::create(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, PalfHandle &handle) { int ret = OB_SUCCESS; - int64_t palf_id(id); palf::IPalfHandleImpl *palf_handle_impl = NULL; - if (OB_FAIL(palf_env_impl_.create_palf_handle_impl(palf_id, access_mode, palf_base_info, palf_handle_impl))) { - PALF_LOG(WARN, "create_palf_handle_impl failed", K(ret), K(palf_id)); + if (OB_FAIL(palf_env_impl_.create_palf_handle_impl(access_mode, palf_base_info, palf_handle_impl))) { + PALF_LOG(WARN, "create_palf_handle_impl failed", K(ret)); } else if (FALSE_IT(handle.palf_handle_impl_ = palf_handle_impl)) { } else { - PALF_LOG(INFO, "create palf handle success", K(id)); + PALF_LOG(INFO, "create palf handle success"); } if (OB_FAIL(ret)) { handle.palf_handle_impl_ = NULL; @@ -116,16 +114,15 @@ int PalfEnv::create(const int64_t id, return ret; } -int PalfEnv::open(const int64_t id, PalfHandle &handle) +int PalfEnv::open(PalfHandle &handle) { int ret = OB_SUCCESS; - int64_t palf_id(id); palf::IPalfHandleImpl *palf_handle_impl = NULL; - if (OB_FAIL(palf_env_impl_.get_palf_handle_impl(palf_id, palf_handle_impl))) { - PALF_LOG(TRACE, "get_palf_handle_impl failed", K(ret), K(palf_id)); + if (OB_FAIL(palf_env_impl_.get_palf_handle_impl(palf_handle_impl))) { + PALF_LOG(TRACE, "get_palf_handle_impl failed", K(ret)); } else if (FALSE_IT(handle.palf_handle_impl_ = palf_handle_impl)) { } else { - PALF_LOG(TRACE, "PalfEnv open success", K(ret), K(id), K(handle)); + PALF_LOG(TRACE, "PalfEnv open success", K(ret), K(handle)); } if (OB_FAIL(ret)) { handle.palf_handle_impl_ = NULL; @@ -136,17 +133,14 @@ int PalfEnv::open(const int64_t id, PalfHandle &handle) void PalfEnv::close(PalfHandle &handle) { (void)handle.unregister_file_size_cb(); - (void)handle.unregister_role_change_cb(); - (void)handle.unregister_rebuild_cb(); palf_env_impl_.revert_palf_handle_impl(handle.palf_handle_impl_); handle.palf_handle_impl_ = NULL; PALF_LOG(TRACE, "PalfEnv close success", K(handle)); } -int PalfEnv::remove(int64_t id) +int PalfEnv::remove() { - int64_t palf_id(id); - return palf_env_impl_.remove_palf_handle_impl(palf_id); + return palf_env_impl_.remove_palf_handle_impl(); } int PalfEnv::get_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte) @@ -174,21 +168,10 @@ bool PalfEnv::check_disk_space_enough() return palf_env_impl_.check_disk_space_enough(); } -int PalfEnv::for_each(const ObFunction &func) -{ - return palf_env_impl_.for_each(func); -} - int PalfEnv::get_io_start_time(int64_t &last_working_time) { return palf_env_impl_.get_io_start_time(last_working_time); } -// should be removed in version 4.2.0.0 -int PalfEnv::update_replayable_point(const SCN &replayable_scn) -{ - return palf_env_impl_.update_replayable_point(replayable_scn); -} - } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/palf_env.h b/src/logservice/palf/palf_env.h index 17ec9d45a..3d9e193d9 100644 --- a/src/logservice/palf/palf_env.h +++ b/src/logservice/palf/palf_env.h @@ -43,7 +43,6 @@ class ObLocalDevice; } namespace palf { -class PalfRoleChangeCb; class PalfHandle; class PalfDiskOptions; class ILogBlockPool; @@ -74,21 +73,19 @@ class PalfEnv public: PalfEnv(); ~PalfEnv(); - // Migration scenario destination end replica creation interface - // @param [in] id, the identifier of the log stream to be created - // @param [in] access_mode，palf access mode + // Create the unique log stream. + // @param [in] access_mode, palf access mode // @param [in] palf_base_info, the log start information of palf // @param [out] handle, the generated palf_handle object after successful creation - int create(const int64_t id, - const AccessMode &access_mode, + int create(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, PalfHandle &handle); - // Open a Paxos Replica corresponding to an id, and return the file handle - int open(int64_t id, PalfHandle &handle); + // Open the unique log stream and return its handle. + int open(PalfHandle &handle); // Close a handle void close(PalfHandle &handle); - // Delete the Paxos Replica corresponding to the id, which will also delete the physical file; - int remove(int64_t id); + // Delete the unique log stream and its physical files. + int remove(); // @brief get palf disk usage // @param [out] used_size_byte @@ -116,11 +113,7 @@ class PalfEnv // and will be reset as OB_INVALID_TIMESTAMP when an io task ends, atomically. int get_io_start_time(int64_t &last_working_time); // @brief iterate each PalfHandle of PalfEnv and execute 'func' - int for_each(const ObFunction &func); - // just for LogRpc palf::IPalfEnvImpl *get_palf_env_impl() { return &palf_env_impl_; } - // should be removed in version 4.2.0.0 - int update_replayable_point(const SCN &replayable_scn); int start(); private: void stop_(); diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index f17231052..4d0efe5a1 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -20,9 +20,9 @@ #include #endif #include "palf_handle.h" -#include "share/ob_ls_id.h" // ObLSID #include "share/ob_local_device.h" // ObLocalDevice #include "share/io/ob_io_manager.h" // ObIOManager +#include "lib/ob_running_mode.h" namespace oceanbase { @@ -149,23 +149,18 @@ int PalfDiskOptionsWrapper::update_disk_options_not_guarded_by_lock_(const PalfD PalfEnvImpl::PalfEnvImpl() : palf_meta_lock_(common::ObLatchIds::PALF_ENV_LOCK), log_alloc_mgr_(NULL), log_block_pool_(NULL), - fetch_log_engine_(), - log_rpc_(), cb_thread_pool_(), log_io_worker_wrapper_(), log_shared_queue_th_(), block_gc_timer_task_(), - log_updater_(), monitor_(NULL), disk_options_wrapper_(), disk_not_enough_print_interval_in_gc_thread_(OB_INVALID_TIMESTAMP), disk_not_enough_print_interval_in_loop_thread_(OB_INVALID_TIMESTAMP), self_(), - single_palf_handle_(nullptr), + palf_handle_(nullptr), last_palf_epoch_(0), - rebuild_replica_log_lag_threshold_(0), enable_log_cache_(false), - enable_fetch_log_engine_(true), diskspace_enough_(true), io_adapter_(), is_inited_(false), @@ -183,7 +178,6 @@ PalfEnvImpl::~PalfEnvImpl() int PalfEnvImpl::init( const PalfOptions &options, const char *base_dir, const ObAddr &self, - const int64_t cluster_id, common::ObILogAllocator *log_alloc_mgr, ILogBlockPool *log_block_pool, PalfMonitorCb *monitor, @@ -202,13 +196,8 @@ int PalfEnvImpl::init( ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "invalid arguments", K(ret), K(base_dir), K(self), KP(log_alloc_mgr), KP(log_block_pool), KP(monitor), KP(log_local_device), KP(io_manager)); - } else if (OB_FAIL(init_log_io_worker_config_(options.disk_options_.log_writer_parallelism_, - log_io_worker_config_))) { + } else if (OB_FAIL(init_log_io_worker_config_(log_io_worker_config_))) { PALF_LOG(WARN, "init_log_io_worker_config_ failed", K(options)); - } else if (OB_FAIL(fetch_log_engine_.init(this, log_alloc_mgr, options.enable_fetch_log_engine_))) { - PALF_LOG(ERROR, "FetchLogEngine init failed", K(ret)); - } else if (OB_FAIL(log_rpc_.init(self, cluster_id))) { - PALF_LOG(ERROR, "LogRpc init failed", K(ret)); } else if (OB_FAIL(cb_thread_pool_.init(io_cb_num, this))) { PALF_LOG(ERROR, "LogIOTaskThreadPool init failed", K(ret)); } else if (OB_FAIL(log_io_worker_wrapper_.init(log_io_worker_config_, @@ -231,8 +220,6 @@ int PalfEnvImpl::init( PALF_LOG(ERROR, "log_loop_thread_ init failed", K(ret)); } else if (OB_FAIL(disk_options_wrapper_.init(options.disk_options_))) { PALF_LOG(ERROR, "disk_options_wrapper_ init failed", K(ret)); - } else if (OB_FAIL(log_updater_.init(this))) { - PALF_LOG(ERROR, "LogUpdater init failed", K(ret)); } else if (OB_FAIL(io_adapter_.init(log_local_device, io_manager))) { PALF_LOG(ERROR, "LogIOAdapter init failed", K(ret)); } else { @@ -244,7 +231,6 @@ int PalfEnvImpl::init( is_inited_ = true; is_running_ = true; enable_log_cache_ = options.enable_log_cache_; - enable_fetch_log_engine_ = options.enable_fetch_log_engine_; PALF_LOG(INFO, "PalfEnvImpl init success", K(ret), K(self_), KPC(this)); } if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { @@ -258,8 +244,8 @@ int PalfEnvImpl::start() int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(scan_all_palf_handle_impl_director_())) { - PALF_LOG(WARN, "scan_all_palf_handle_impl_director_ failed", K(ret)); + } else if (OB_FAIL(reload_palf_handle_impl_())) { + PALF_LOG(WARN, "reload_palf_handle_impl_ failed", K(ret)); } else if (OB_FAIL(cb_thread_pool_.start())) { PALF_LOG(ERROR, "LogIOTaskThreadPool start failed", K(ret)); } else if (OB_FAIL(log_io_worker_wrapper_.start())) { @@ -268,12 +254,8 @@ int PalfEnvImpl::start() PALF_LOG(ERROR, "LogIOWorker start failed", K(ret)); } else if (OB_FAIL(block_gc_timer_task_.start())) { PALF_LOG(ERROR, "FileCollectTimerTask start failed", K(ret)); - } else if (OB_FAIL(fetch_log_engine_.start())) { - PALF_LOG(ERROR, "FetchLogEngine start failed", K(ret)); } else if (OB_FAIL(log_loop_thread_.start())) { PALF_LOG(ERROR, "log_loop_thread_ start failed", K(ret)); - } else if (OB_FAIL(log_updater_.start())) { - PALF_LOG(ERROR, "LogUpdater start failed", K(ret)); } else { is_running_ = true; PALF_LOG(INFO, "PalfEnv start success", K(ret)); @@ -290,9 +272,7 @@ void PalfEnvImpl::stop() log_shared_queue_th_.stop(); cb_thread_pool_.stop(); block_gc_timer_task_.stop(); - fetch_log_engine_.stop(); log_loop_thread_.stop(); - log_updater_.stop(); PALF_LOG(INFO, "PalfEnvImpl stop success", KPC(this)); } } @@ -304,9 +284,7 @@ void PalfEnvImpl::wait() log_shared_queue_th_.wait(); cb_thread_pool_.wait(); block_gc_timer_task_.wait(); - fetch_log_engine_.wait(); log_loop_thread_.wait(); - log_updater_.wait(); PALF_LOG(INFO, "PalfEnvImpl wait success", KPC(this)); } @@ -320,9 +298,6 @@ void PalfEnvImpl::destroy() cb_thread_pool_.destroy(); log_loop_thread_.destroy(); block_gc_timer_task_.destroy(); - fetch_log_engine_.destroy(); - log_updater_.destroy(); - log_rpc_.destroy(); log_alloc_mgr_ = NULL; monitor_ = NULL; disk_not_enough_print_interval_in_gc_thread_ = OB_INVALID_TIMESTAMP; @@ -331,15 +306,12 @@ void PalfEnvImpl::destroy() log_dir_[0] = '\0'; tmp_log_dir_[0] = '\0'; disk_options_wrapper_.reset(); - rebuild_replica_log_lag_threshold_ = 0; enable_log_cache_ = false; - enable_fetch_log_engine_ = true; io_adapter_.destroy(); } // NB: not thread safe -int PalfEnvImpl::create_palf_handle_impl(const int64_t palf_id, - const AccessMode &access_mode, +int PalfEnvImpl::create_palf_handle_impl(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, IPalfHandleImpl *&palf_handle_impl) { @@ -348,20 +320,17 @@ int PalfEnvImpl::create_palf_handle_impl(const int64_t palf_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(WARN, "PalfEnvImpl is not inited", K(ret)); - } else if (OB_FAIL(create_palf_handle_impl_(palf_id, access_mode, palf_base_info, - NORMAL_REPLICA, palf_handle_impl))) { + } else if (OB_FAIL(create_palf_handle_impl_(access_mode, palf_base_info, palf_handle_impl))) { palf_handle_impl = NULL; } else { - PALF_LOG(INFO, "PalfEnvImpl create_palf_handle_impl finished", K(ret), K(palf_id), K(access_mode), + PALF_LOG(INFO, "PalfEnvImpl create_palf_handle_impl finished", K(ret), K(access_mode), K(palf_base_info), KPC(this)); } return ret; } -int PalfEnvImpl::create_palf_handle_impl_(const int64_t palf_id, - const AccessMode &access_mode, +int PalfEnvImpl::create_palf_handle_impl_(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, - const LogReplicaType replica_type, IPalfHandleImpl *&ipalf_handle_impl) { int ret = OB_SUCCESS; @@ -375,31 +344,28 @@ int PalfEnvImpl::create_palf_handle_impl_(const int64_t palf_id, } else if (!is_running_) { ret = OB_NOT_RUNNING; PALF_LOG(WARN, "PalfEnvImpl is not running", K(ret)); - } else if (false == is_valid_palf_id(palf_id)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K(palf_id)); - } else if (NULL != single_palf_handle_) { + } else if (NULL != palf_handle_) { ret = OB_ENTRY_EXIST; - PALF_LOG(WARN, "palf_handle has exist, ignore this request", K(ret), K(palf_id)); - } else if (false == check_can_create_palf_handle_impl_()) { + PALF_LOG(WARN, "palf_handle has exist, ignore this request", K(ret)); + } else if (false == has_minimum_log_disk_capacity_()) { ret = OB_LOG_OUTOF_DISK_SPACE; - PALF_LOG(WARN, "PalfEnv can not hold more instance", K(ret), KPC(this), K(palf_id)); - } else if (0 > (pret = snprintf(base_dir, MAX_PATH_SIZE, "%s/%ld", log_dir_, palf_id))) { + PALF_LOG(WARN, "PalfEnv can not hold the log stream", K(ret), KPC(this)); + } else if (0 > (pret = snprintf(base_dir, MAX_PATH_SIZE, "%s/log_stream", log_dir_))) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "snprinf failed", K(pret), K(palf_id)); + PALF_LOG(ERROR, "snprintf failed", K(pret)); // Note:: order is vital, allocate memory may be fail } else if (NULL == (palf_handle_impl = PalfHandleImplFactory::alloc())) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "alloc palf_handle_impl failed", K(ret)); } else if (OB_FAIL(create_directory(base_dir))) { - PALF_LOG(WARN, "prepare_directory_for_creating_ls failed!!!", K(ret), K(palf_id)); - } else if (OB_FAIL(palf_handle_impl->init(palf_id, access_mode, palf_base_info, replica_type, - &fetch_log_engine_, base_dir, log_alloc_mgr_, log_block_pool_, &log_rpc_, - log_io_worker_wrapper_.get_log_io_worker(palf_id), &log_shared_queue_th_, this, + PALF_LOG(WARN, "prepare log stream directory failed", K(ret)); + } else if (OB_FAIL(palf_handle_impl->init(access_mode, palf_base_info, + base_dir, log_alloc_mgr_, log_block_pool_, + log_io_worker_wrapper_.get_log_io_worker(), &log_shared_queue_th_, this, self_, palf_epoch, &io_adapter_))) { - PALF_LOG(ERROR, "IPalfHandleImpl init failed", K(ret), K(palf_id)); + PALF_LOG(ERROR, "IPalfHandleImpl init failed", K(ret)); } else { - single_palf_handle_ = palf_handle_impl; + palf_handle_ = palf_handle_impl; (void) palf_handle_impl->set_monitor_cb(monitor_); palf_handle_impl->set_scan_disk_log_finished(); ipalf_handle_impl = palf_handle_impl; @@ -408,89 +374,65 @@ int PalfEnvImpl::create_palf_handle_impl_(const int64_t palf_id, if (OB_FAIL(ret) && NULL != palf_handle_impl) { PalfHandleImplFactory::free(palf_handle_impl); palf_handle_impl = NULL; - if (NULL == single_palf_handle_) { + if (NULL == palf_handle_) { remove_directory_while_exist_(base_dir); } } - PALF_LOG(INFO, "PalfEnvImpl create_palf_handle_impl_ finished", K(ret), K(palf_id), - K(access_mode), K(palf_base_info), K(replica_type), KPC(this)); + PALF_LOG(INFO, "PalfEnvImpl create_palf_handle_impl_ finished", K(ret), + K(access_mode), K(palf_base_info), KPC(this)); return ret; } -int PalfEnvImpl::remove_palf_handle_impl(const int64_t palf_id) +int PalfEnvImpl::remove_palf_handle_impl() { int ret = OB_SUCCESS; WLockGuard guard(palf_meta_lock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(ERROR, "PalfEnvImpl is not inited", K(ret)); - } else if (OB_FAIL(remove_palf_handle_impl_from_map_not_guarded_by_lock_(palf_id))) { - PALF_LOG(WARN, "palf instance not exist", K(ret), KPC(this), K(palf_id)); + } else if (OB_FAIL(remove_palf_handle_impl_())) { + PALF_LOG(WARN, "log stream not exist", K(ret), KPC(this)); } else { - // sys ls is never removed during normal operation, but handle - // error/shutdown paths where the handle has been marked for removal. - } - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; + // Handle error and shutdown paths where the stream was marked for removal. } return ret; } -int PalfEnvImpl::get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImplGuard &palf_handle_impl_guard) +int PalfEnvImpl::get_palf_handle_impl(IPalfHandleImplGuard &palf_handle_impl_guard) { int ret = OB_SUCCESS; IPalfHandleImpl *palf_handle_impl = NULL; - if (OB_FAIL(get_palf_handle_impl(palf_id, palf_handle_impl))) { - PALF_LOG(TRACE, "get_palf_handle_impl failed", K(ret), K(palf_id)); + if (OB_FAIL(get_palf_handle_impl(palf_handle_impl))) { + PALF_LOG(TRACE, "get_palf_handle_impl failed", K(ret)); } else { palf_handle_impl_guard.palf_env_impl_ = this; palf_handle_impl_guard.palf_handle_impl_ = palf_handle_impl; - palf_handle_impl_guard.palf_id_ = palf_id; - PALF_LOG(TRACE, "get_palf_handle_impl success", K(palf_id), K(palf_handle_impl_guard)); + PALF_LOG(TRACE, "get_palf_handle_impl success", K(palf_handle_impl_guard)); // do nothing } return ret; } -int PalfEnvImpl::get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImpl *&ipalf_handle_impl) +int PalfEnvImpl::get_palf_handle_impl(IPalfHandleImpl *&ipalf_handle_impl) { int ret = OB_SUCCESS; - if (false == is_valid_palf_id(palf_id)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id)); - } else if (OB_ISNULL(single_palf_handle_) - || false == single_palf_handle_->check_can_be_used()) { + if (OB_ISNULL(palf_handle_) + || false == palf_handle_->check_can_be_used()) { ret = OB_ENTRY_NOT_EXIST; } else { - ipalf_handle_impl = single_palf_handle_; + ipalf_handle_impl = palf_handle_; } return ret; } void PalfEnvImpl::revert_palf_handle_impl(IPalfHandleImpl *ipalf_handle_impl) { - // handle is never freed during operation (sys ls lifetime == process lifetime) + // The handle remains owned by the environment during operation. UNUSED(ipalf_handle_impl); } -int PalfEnvImpl::scan_all_palf_handle_impl_director_() -{ - int ret = OB_SUCCESS; - ObTimeGuard guard("PalfEnvImplStart", 0); - ReloadPalfHandleImplFunctor functor(this); - if (OB_FAIL(scan_dir(log_dir_, functor))) { - PALF_LOG(WARN, "scan_dir failed", K(ret)); - } else { - guard.click("scan_dir"); - PALF_LOG(INFO, "scan_all_palf_handle_impl_director_ success", K(ret), K(log_dir_), K(guard)); - } - return ret; -} - int PalfEnvImpl::create_directory(const char *base_dir) { int ret = OB_SUCCESS; @@ -600,13 +542,10 @@ int PalfEnvImpl::try_recycle_blocks() int64_t utl_threshold_to_stop_write = disk_opts_for_stopping_writing.log_disk_utilization_threshold_; utl_threshold_to_recycle_blocks = 0 == utl_threshold_to_recycle_blocks ? DEFAULT_LOG_UTL_THRESHOLD : utl_threshold_to_recycle_blocks; utl_threshold_to_stop_write = 0 == utl_threshold_to_stop_write ? DEFAULT_LOG_UTL_THRESHOLD : utl_threshold_to_stop_write; - int64_t palf_id = 0; - int64_t maximum_used_size = 0; int tmp_ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(get_disk_usage_(total_used_size_byte, total_unrecyclable_size_byte, - palf_id, maximum_used_size))) { + } else if (OB_FAIL(get_disk_usage_(total_used_size_byte, total_unrecyclable_size_byte))) { PALF_LOG(WARN, "get_disk_usage_ failed", K(ret), KPC(this)); } else if (FALSE_IT(disk_options_wrapper_.set_cur_unrecyclable_log_disk_size(total_unrecyclable_size_byte))) { } else if (OB_SUCCESS != (tmp_ret = log_io_worker_wrapper_.notify_need_writing_throttling(disk_options_wrapper_.need_throttling()))) { @@ -670,8 +609,6 @@ int PalfEnvImpl::try_recycle_blocks() "limit_size(MB)", (log_disk_usage_limit_size*log_disk_limit_percent)/100/MB, "limit_percent(%)", log_disk_limit_percent, "total_unrecyclable_size_byte(MB)", total_unrecyclable_size_byte/MB, - "maximum_used_size(MB)", maximum_used_size/MB, - "maximum_log_stream", palf_id, "in_shrinking", in_shrinking); LOG_DBA_ERROR_(OB_LOG_DISK_SPACE_ALMOST_FULL, tmp_ret, "log disk space is almost full", ", total_size(MB)=", log_disk_usage_limit_size/MB, @@ -682,8 +619,6 @@ int PalfEnvImpl::try_recycle_blocks() ", limit_size(MB)=", (log_disk_usage_limit_size*log_disk_limit_percent)/100/MB, ", limit_percent(%)=", log_disk_limit_percent, ", total_unrecyclable_size_byte(MB)=", total_unrecyclable_size_byte/MB, - ", maximum_used_size(MB)=", maximum_used_size/MB, - ", maximum_log_stream=", palf_id, ", in_shrinking=", in_shrinking); } } else { @@ -784,12 +719,6 @@ int PalfEnvImpl::update_options(const PalfOptions &options) } else if (false == options.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid argument", K(options)); - } else if (enable_fetch_log_engine_ != options.enable_fetch_log_engine_) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "fetch log engine cannot be switched after startup", K(ret), K(options), KPC(this)); - } else if (OB_FAIL(log_rpc_.update_transport_compress_options(options.compress_options_))) { - PALF_LOG(WARN, "update_transport_compress_options failed", K(ret), K(options)); - } else if (FALSE_IT(rebuild_replica_log_lag_threshold_ = options.rebuild_replica_log_lag_threshold_)) { } else if (OB_FAIL(check_can_update_log_disk_options_(options.disk_options_))) { PALF_LOG(WARN, "check_can_update_log_disk_options_ failed", K(options)); } else if (OB_FAIL(disk_options_wrapper_.update_disk_options(options.disk_options_))) { @@ -808,42 +737,7 @@ int PalfEnvImpl::get_options(PalfOptions &options) ret = OB_NOT_INIT; } else { options.disk_options_ = disk_options_wrapper_.get_disk_opts_for_recycling_blocks(); - options.compress_options_ = log_rpc_.get_compress_opts(); - options.rebuild_replica_log_lag_threshold_ = rebuild_replica_log_lag_threshold_; options.enable_log_cache_ = enable_log_cache_; - options.enable_fetch_log_engine_ = enable_fetch_log_engine_; - } - return ret; -} - -int PalfEnvImpl::for_each(const common::ObFunction &func) -{ - int ret = OB_SUCCESS; - if (!func.is_valid()) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - IPalfHandleImpl *handle = nullptr; - if (OB_SUCCESS == get_palf_handle_impl(SYS_PALF_ID, handle)) { - func(handle); - revert_palf_handle_impl(handle); - } - } - return ret; -} - -int PalfEnvImpl::for_each(const common::ObFunction &func) -{ - int ret = OB_SUCCESS; - if (!func.is_valid()) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - IPalfHandleImpl *handle = nullptr; - if (OB_SUCCESS == get_palf_handle_impl(SYS_PALF_ID, handle)) { - PalfHandle palf_handle; - palf_handle.palf_handle_impl_ = handle; - func(palf_handle); - revert_palf_handle_impl(handle); - } } return ret; } @@ -853,121 +747,71 @@ common::ObILogAllocator* PalfEnvImpl::get_log_allocator() return log_alloc_mgr_; } -PalfEnvImpl::ReloadPalfHandleImplFunctor::ReloadPalfHandleImplFunctor(PalfEnvImpl *palf_env_impl) : palf_env_impl_(palf_env_impl) -{ -} - -int PalfEnvImpl::ReloadPalfHandleImplFunctor::func(const struct dirent *entry) +int PalfEnvImpl::reload_palf_handle_impl_() { int ret = OB_SUCCESS; int pret = 0; - ObTimeGuard guard("ReloadFunctor"); - struct stat st; - char log_dir[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; - if (OB_ISNULL(entry)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid args", K(ret), KP(entry)); - } else if (0 > (pret = snprintf(log_dir, MAX_PATH_SIZE, "%s/%s", palf_env_impl_->log_dir_, entry->d_name))) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "snprint failed", K(ret), K(pret), K(entry->d_name)); - } else if (0 != stat(log_dir, &st)) { - PALF_LOG(INFO, "this entry is not a block", K(ret), K(log_dir), K(errno)); - } else if (false == S_ISDIR(st.st_mode)) { - PALF_LOG(WARN, "path is not a directory, ignore it", K(ret), K(log_dir), K(st.st_mode)); - } else { - const char *path = entry->d_name; - bool is_number = true; - const size_t path_len = strlen(path); - for (size_t i = 0; is_number && i < path_len; ++i) { - if ('\0' == path[i]) { - break; - } else if (!isdigit(path[i])) { - is_number = false; - } - } - if (!is_number) { - // do nothing, skip invalid block like tmp - } else { - int64_t id = strtol(path, nullptr, 10); - if (OB_FAIL(palf_env_impl_->reload_palf_handle_impl_(id))) { - PALF_LOG(WARN, "reload_palf_handle_impl failed", K(ret)); - } - guard.click("reload_palf_handle_impl"); - PALF_LOG(INFO, "reload_palf_handle_impl_", K(ret), K(id), K(guard)); - } - } - return ret; -} - -int PalfEnvImpl::reload_palf_handle_impl_(const int64_t palf_id) -{ - int ret = OB_SUCCESS; - int pret = 0; - PalfHandleImpl *tmp_palf_handle_impl; + PalfHandleImpl *tmp_palf_handle_impl = nullptr; char base_dir[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; int64_t start_ts = ObTimeUtility::current_time(); bool is_integrity = true; + bool dir_exist = false; const int64_t palf_epoch = ATOMIC_AAF(&last_palf_epoch_, 1); - if (0 > (pret = snprintf(base_dir, MAX_PATH_SIZE, "%s/%ld", log_dir_, palf_id))) { + if (0 > (pret = snprintf(base_dir, MAX_PATH_SIZE, "%s/log_stream", log_dir_))) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "snprint failed", K(ret), K(pret), K(palf_id)); + PALF_LOG(WARN, "snprint failed", K(ret), K(pret)); + } else if (OB_FAIL(FileDirectoryUtils::is_exists(base_dir, dir_exist))) { + PALF_LOG(WARN, "check palf directory failed", K(ret), K(base_dir)); + } else if (!dir_exist) { + PALF_LOG(INFO, "palf directory does not exist", K(base_dir)); } else if (NULL == (tmp_palf_handle_impl = PalfHandleImplFactory::alloc())) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "alloc ipalf_handle_impl failed", K(ret)); - } else if (OB_FAIL(tmp_palf_handle_impl->load(palf_id, &fetch_log_engine_, base_dir, log_alloc_mgr_, - log_block_pool_, &log_rpc_, log_io_worker_wrapper_.get_log_io_worker(palf_id), &log_shared_queue_th_, + } else if (OB_FAIL(tmp_palf_handle_impl->load(base_dir, log_alloc_mgr_, + log_block_pool_, log_io_worker_wrapper_.get_log_io_worker(), &log_shared_queue_th_, this, self_, palf_epoch, &io_adapter_, is_integrity))) { - PALF_LOG(ERROR, "PalfHandleImpl init failed", K(ret), K(palf_id)); + PALF_LOG(ERROR, "PalfHandleImpl init failed", K(ret)); } else { - single_palf_handle_ = tmp_palf_handle_impl; + palf_handle_ = tmp_palf_handle_impl; (void) tmp_palf_handle_impl->set_monitor_cb(monitor_); (void) tmp_palf_handle_impl->set_scan_disk_log_finished(); int64_t cost_ts = ObTimeUtility::current_time() - start_ts; - PALF_LOG(INFO, "reload_palf_handle_impl success", K(ret), K(palf_id), K(cost_ts), KP(this)); + PALF_LOG(INFO, "reload_palf_handle_impl success", K(ret), K(cost_ts), KP(this)); } if (OB_FAIL(ret) && NULL != tmp_palf_handle_impl) { PALF_LOG(ERROR, "reload_palf_handle_impl_ failed, need free tmp_palf_handle_impl", K(ret), K(tmp_palf_handle_impl)); - if (NULL == single_palf_handle_) { + if (NULL == palf_handle_) { PalfHandleImplFactory::free(tmp_palf_handle_impl); tmp_palf_handle_impl = NULL; } } else if (false == is_integrity) { - PALF_LOG(WARN, "palf instance is not integrity, remove it", K(palf_id)); - ret = move_incomplete_palf_into_tmp_dir_(palf_id); + PALF_LOG(WARN, "log stream is incomplete"); + ret = move_incomplete_palf_into_tmp_dir_(); } return ret; } int PalfEnvImpl::get_total_used_disk_space_(int64_t &total_used_disk_space, - int64_t &total_unrecyclable_disk_space, - int64_t &palf_id, - int64_t &maximum_used_size) + int64_t &total_unrecyclable_disk_space) { int ret = OB_SUCCESS; - if (NULL == single_palf_handle_) { + if (NULL == palf_handle_) { // GC timer starts before LS is created, handle not ready yet total_used_disk_space = 0; total_unrecyclable_disk_space = 0; - palf_id = INVALID_PALF_ID; - maximum_used_size = 0; - } else if (OB_FAIL(single_palf_handle_->get_total_used_disk_space( + } else if (OB_FAIL(palf_handle_->get_total_used_disk_space( total_used_disk_space, total_unrecyclable_disk_space))) { PALF_LOG(WARN, "get_total_used_disk_space failed", K(ret)); - } else { - maximum_used_size = total_used_disk_space; - palf_id = SYS_PALF_ID; } return ret; } int PalfEnvImpl::get_disk_usage_(int64_t &used_size_byte, - int64_t &unrecyclable_disk_space, - int64_t &palf_id, - int64_t &maximum_used_size) + int64_t &unrecyclable_disk_space) { int ret = OB_SUCCESS; - if (OB_FAIL(get_total_used_disk_space_(used_size_byte, unrecyclable_disk_space, palf_id, maximum_used_size))) { + if (OB_FAIL(get_total_used_disk_space_(used_size_byte, unrecyclable_disk_space))) { PALF_LOG(WARN, "get_total_used_disk_space failed", K(ret), KPC(this)); } return ret; @@ -977,9 +821,7 @@ int PalfEnvImpl::get_disk_usage_(int64_t &used_size_byte) { int ret = OB_SUCCESS; int64_t unused_unrecyclable_size = 0; - int64_t unused_palf_id = 0; - int64_t unused_size = 0; - if (OB_FAIL(get_disk_usage_(used_size_byte, unused_unrecyclable_size, unused_palf_id, unused_size))) { + if (OB_FAIL(get_disk_usage_(used_size_byte, unused_unrecyclable_size))) { PALF_LOG(WARN, "get_total_used_disk_space failed", K(ret), KPC(this)); } return ret; @@ -989,10 +831,10 @@ int PalfEnvImpl::recycle_blocks_(bool &has_recycled) { int ret = OB_SUCCESS; has_recycled = false; - if (NULL == single_palf_handle_) { + if (NULL == palf_handle_) { // GC timer starts before LS is created, handle not ready yet } else { - const LSN base_lsn = single_palf_handle_->get_base_lsn_used_for_block_gc(); + const LSN base_lsn = palf_handle_->get_base_lsn_used_for_block_gc(); const block_id_t min_using_block_id = lsn_2_block(base_lsn, PALF_BLOCK_SIZE); block_id_t min_block_id = LOG_INVALID_BLOCK_ID; auto need_skip_by_ret = [](const int ret) { @@ -1001,7 +843,7 @@ int PalfEnvImpl::recycle_blocks_(bool &has_recycled) }; if (false == base_lsn.is_valid()) { PALF_LOG(WARN, "base_lsn is invalid", K(base_lsn)); - } else if (OB_FAIL(single_palf_handle_->get_min_block_id_for_gc(min_block_id)) + } else if (OB_FAIL(palf_handle_->get_min_block_id_for_gc(min_block_id)) && !need_skip_by_ret(ret)) { PALF_LOG(WARN, "get_min_block_id_for_gc failed", K(ret)); } else if (need_skip_by_ret(ret) @@ -1009,7 +851,7 @@ int PalfEnvImpl::recycle_blocks_(bool &has_recycled) || min_using_block_id - min_block_id < 2) { PALF_LOG(TRACE, "can not recycle blocks, need keep at least two blocks", K(ret), K(min_block_id), K(min_using_block_id)); - } else if (OB_FAIL(single_palf_handle_->delete_block(min_block_id))) { + } else if (OB_FAIL(palf_handle_->delete_block(min_block_id))) { PALF_LOG(WARN, "delete_block failed", K(ret), K(min_block_id)); } else { has_recycled = true; @@ -1019,39 +861,27 @@ int PalfEnvImpl::recycle_blocks_(bool &has_recycled) return ret; } -int PalfEnvImpl::wait_until_reference_count_to_zero_(const int64_t palf_id) -{ - // sys ls handle is never freed during operation, no ref counting needed. - UNUSED(palf_id); - return OB_SUCCESS; -} - -bool PalfEnvImpl::check_can_create_palf_handle_impl_() const +bool PalfEnvImpl::has_minimum_log_disk_capacity_() const { - bool bool_ret = true; - const int64_t count = (NULL != single_palf_handle_) ? 1 : 0; const PalfDiskOptions disk_opts = disk_options_wrapper_.get_disk_opts_for_recycling_blocks(); - bool_ret = (count + 1) * MIN_DISK_SIZE_PER_PALF_INSTANCE <= disk_opts.log_disk_usage_limit_size_; - return bool_ret; + return MIN_DISK_SIZE_PER_PALF_INSTANCE <= disk_opts.log_disk_usage_limit_size_; } -int PalfEnvImpl::remove_palf_handle_impl_from_map_not_guarded_by_lock_(const int64_t palf_id) +int PalfEnvImpl::remove_palf_handle_impl_() { int ret = OB_SUCCESS; - if (false == is_valid_palf_id(palf_id) || palf_id != SYS_PALF_ID) { - ret = OB_ENTRY_NOT_EXIST; - } else if (NULL == single_palf_handle_) { + if (NULL == palf_handle_) { ret = OB_ENTRY_NOT_EXIST; } else { - single_palf_handle_->set_deleted(); - PalfHandleImplFactory::free(single_palf_handle_); - single_palf_handle_ = nullptr; - PALF_LOG(INFO, "remove_palf_handle_impl success", K(ret), K(palf_id)); + palf_handle_->set_deleted(); + PalfHandleImplFactory::free(palf_handle_); + palf_handle_ = nullptr; + PALF_LOG(INFO, "remove_palf_handle_impl success", K(ret)); } return ret; } -int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_(const int64_t palf_id) +int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_() { int ret = OB_SUCCESS; int pret = 0; @@ -1060,8 +890,8 @@ int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_(const int64_t palf_id) char dest_log_dir[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; bool tmp_dir_exist = false; int64_t timestamp = ObTimeUtility::current_time(); - IPalfHandleImpl *old_handle = single_palf_handle_; - single_palf_handle_ = nullptr; + IPalfHandleImpl *old_handle = palf_handle_; + palf_handle_ = nullptr; if (NULL != old_handle) { PalfHandleImplFactory::free(old_handle); } @@ -1074,10 +904,10 @@ int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_(const int64_t palf_id) #endif ret = convert_sys_errno(); PALF_LOG(ERROR, "mkdir tmp log dir failed", K(ret), KPC(this), K(tmp_log_dir_)); - } else if (0 > (pret = snprintf(src_log_dir, MAX_PATH_SIZE, "%s/%ld", log_dir_, palf_id))) { + } else if (0 > (pret = snprintf(src_log_dir, MAX_PATH_SIZE, "%s/log_stream", log_dir_))) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "snprintf failed, unexpected error", K(ret)); - } else if (0 > (pret = snprintf(dest_log_dir, MAX_PATH_SIZE, "%s/%ld_%ld", tmp_log_dir_, palf_id, timestamp))) { + } else if (0 > (pret = snprintf(dest_log_dir, MAX_PATH_SIZE, "%s/log_stream_%ld", tmp_log_dir_, timestamp))) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "snprintf failed, unexpected error", K(ret)); } else if (OB_FAIL(rename_with_retry(src_log_dir, dest_log_dir))) { @@ -1126,17 +956,6 @@ int PalfEnvImpl::get_io_start_time(int64_t &last_working_time) } -int PalfEnvImpl::update_replayable_point(const SCN &replayable_scn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(fetch_log_engine_.update_replayable_point(replayable_scn))) { - PALF_LOG(WARN, "update_replayable_point failed", KPC(this), K(replayable_scn)); - } - return ret; -} - int PalfEnvImpl::get_throttling_options(PalfThrottleOptions &options) { int ret = OB_SUCCESS; @@ -1199,49 +1018,24 @@ void PalfEnvImpl::period_calc_disk_usage() } -int PalfEnvImpl::init_log_io_worker_config_(const int log_writer_parallelism, - LogIOWorkerConfig &config) +int PalfEnvImpl::init_log_io_worker_config_(LogIOWorkerConfig &config) { int ret = OB_SUCCESS; - // log_writer_parallelism only valid when it's user tenant. - // to support writing throttling, sys log stream must has dependent LogIOWorker. - const int64_t real_log_writer_parallelism = 1; - auto tmp_upper_align_div = [](const int64_t num, const int64_t align) -> int64_t { - return (num + align - 1) / align; - }; - constexpr int64_t default_io_queue_cap = 100 * 1024; - constexpr int64_t default_io_batch_width = 8; - // Due to the uniqueness of the log stream ID, using a hash distribution method can - // naturally balance the load in a single-unit environment, therefore, we set the - // min io queue capacity to PALF_SLIDING_WINDOW_SIZE * 2, and set default_io_batch_width - // to 1. - // TODO by zjf225077: - // to support load balance in a multi-unit environment, LogIOWorker needs to use - // a load factor to ensure that the number of log streams on each LogIOWorker is in - // a balanced state. constexpr int64_t default_min_io_queue_cap = PALF_SLIDING_WINDOW_SIZE * 2; - constexpr int64_t default_min_batch_width = 1; - // Assume that a maximum of 100 * 1024 I/O tasks exist simultaneously in single PalfEnvImpl - config.io_worker_num_ = real_log_writer_parallelism; - config.io_queue_capcity_ = MAX(default_min_io_queue_cap, - tmp_upper_align_div(default_io_queue_cap, real_log_writer_parallelism)); - config.batch_width_ = MAX(default_min_batch_width, - tmp_upper_align_div(default_io_batch_width, real_log_writer_parallelism)); + config.io_queue_capcity_ = MAX(default_min_io_queue_cap, default_io_queue_cap); config.batch_depth_ = PALF_SLIDING_WINDOW_SIZE; - PALF_LOG(INFO, "init_log_io_worker_config_ success", K(config), K(log_writer_parallelism)); + PALF_LOG(INFO, "init_log_io_worker_config_ success", K(config)); return ret; } int PalfEnvImpl::check_can_update_log_disk_options_(const PalfDiskOptions &disk_opts) { int ret = OB_SUCCESS; - const int64_t curr_palf_instance_num = (NULL != single_palf_handle_) ? 1 : 0; - const int64_t curr_min_log_disk_size = curr_palf_instance_num * MIN_DISK_SIZE_PER_PALF_INSTANCE; - if (disk_opts.log_disk_usage_limit_size_ < curr_min_log_disk_size) { + if (NULL != palf_handle_ && + disk_opts.log_disk_usage_limit_size_ < MIN_DISK_SIZE_PER_PALF_INSTANCE) { ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "can not hold current palf instance", K(curr_palf_instance_num), - K(curr_min_log_disk_size), K(disk_opts)); + PALF_LOG(WARN, "log disk is too small", K(disk_opts)); } return ret; } diff --git a/src/logservice/palf/palf_env_impl.h b/src/logservice/palf/palf_env_impl.h index 675e2d244..b605dea61 100644 --- a/src/logservice/palf/palf_env_impl.h +++ b/src/logservice/palf/palf_env_impl.h @@ -17,8 +17,6 @@ #ifndef OCEANBASE_LOGSERVICE_LOG_MGR_ #define OCEANBASE_LOGSERVICE_LOG_MGR_ #include -#include "common/ob_member_list.h" - #include "lib/lock/ob_mutex.h" #include "lib/lock/ob_spin_lock.h" #include "lib/ob_define.h" @@ -27,17 +25,14 @@ #include "lib/utility/utility.h" #include "share/ob_occam_timer.h" #include "share/scn.h" -#include "fetch_log_engine.h" #include "log_define.h" #include "log_shared_queue_thread.h" #include "log_io_task_cb_thread_pool.h" #include "log_loop_thread.h" -#include "log_rpc.h" #include "palf_options.h" #include "palf_handle_impl.h" #include "log_io_worker_wrapper.h" #include "block_gc_timer_task.h" -#include "log_updater.h" #include "log_io_utils.h" #include "log_io_adapter.h" namespace oceanbase @@ -174,26 +169,20 @@ class IPalfEnvImpl IPalfEnvImpl() {} virtual ~IPalfEnvImpl() {} public: - virtual int get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImplGuard &palf_handle_impl) = 0; - virtual int get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImpl *&palf_handle_impl) = 0; - virtual int create_palf_handle_impl(const int64_t palf_id, - const AccessMode &access_mode, + virtual int get_palf_handle_impl(IPalfHandleImplGuard &palf_handle_impl) = 0; + virtual int get_palf_handle_impl(IPalfHandleImpl *&palf_handle_impl) = 0; + virtual int create_palf_handle_impl(const AccessMode &access_mode, const PalfBaseInfo &base_info, IPalfHandleImpl *&palf_handle_impl) = 0; - virtual int remove_palf_handle_impl(const int64_t palf_id) = 0; + virtual int remove_palf_handle_impl() = 0; virtual void revert_palf_handle_impl(IPalfHandleImpl *palf_handle_impl) = 0; virtual common::ObILogAllocator *get_log_allocator() = 0; - virtual int for_each(const common::ObFunction &func) = 0; virtual int create_directory(const char *base_dir) = 0; virtual int remove_directory(const char *base_dir) = 0; virtual bool check_disk_space_enough() = 0; - virtual int64_t get_rebuild_replica_log_lag_threshold() const = 0; virtual int get_io_start_time(int64_t &last_working_time) = 0; // should be removed in version 4.2.0.0 - virtual int update_replayable_point(const SCN &replayable_scn) = 0; virtual int get_throttling_options(PalfThrottleOptions &option) = 0; virtual void period_calc_disk_usage() = 0; virtual LogSharedQueueTh *get_log_shared_queue_thread() = 0; @@ -211,7 +200,6 @@ class PalfEnvImpl : public IPalfEnvImpl int init(const PalfOptions &options, const char *base_dir, const common::ObAddr &self, - const int64_t cluster_id, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, PalfMonitorCb *monitor, @@ -220,7 +208,7 @@ class PalfEnvImpl : public IPalfEnvImpl // start function contains two meanings: // // 1. Start all the worker threads contained in PalfEnvImpl - // 2. According to the log_storage and meta_storage files included in base_dir, load all the required metadata and logs for the log streams, and perform fault recovery + // 2. According to the log_storage and meta_storage files included in base_dir, load the required metadata and logs for the log stream, and perform fault recovery // // @return :TODO int start(); @@ -229,24 +217,17 @@ class PalfEnvImpl : public IPalfEnvImpl void destroy(); public: // Create log stream interface - // @param [in] palf_id, identifier of the log stream to be created // @param [in] palf_base_info, palf's log start information // @param [out] palf_handle_impl, the generated palf_handle_impl object after successful creation // When the palf_handle_impl object is no longer in use, the caller needs to execute revert_palf_handle_impl - int create_palf_handle_impl(const int64_t palf_id, - const AccessMode &access_mode, + int create_palf_handle_impl(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, IPalfHandleImpl *&palf_handle_impl) override final; // Delete log stream, called by Garbage Collector - // - // @param [in] palf_id, the identifier of the log stream to be deleted - // // @return :TODO - int remove_palf_handle_impl(const int64_t palf_id) override final; - int get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImpl *&palf_handle_impl) override final; - int get_palf_handle_impl(const int64_t palf_id, - IPalfHandleImplGuard &guard) override final; + int remove_palf_handle_impl() override final; + int get_palf_handle_impl(IPalfHandleImpl *&palf_handle_impl) override final; + int get_palf_handle_impl(IPalfHandleImplGuard &guard) override final; void revert_palf_handle_impl(IPalfHandleImpl *palf_handle_impl) override final; // =================== disk space management ================== @@ -256,14 +237,9 @@ class PalfEnvImpl : public IPalfEnvImpl int get_stable_disk_usage(int64_t &used_size_byte, int64_t &total_usable_size_byte); int update_options(const PalfOptions &options); int get_options(PalfOptions &options); - int64_t get_rebuild_replica_log_lag_threshold() const - {return rebuild_replica_log_lag_threshold_;} - int for_each(const common::ObFunction &func); - int for_each(const common::ObFunction &func) override final; common::ObILogAllocator* get_log_allocator() override final; int get_io_start_time(int64_t &last_working_time) override final; - int update_replayable_point(const SCN &replayable_scn) override final; int get_throttling_options(PalfThrottleOptions &option); void period_calc_disk_usage() override final; LogSharedQueueTh *get_log_shared_queue_thread() override final; @@ -275,15 +251,7 @@ class PalfEnvImpl : public IPalfEnvImpl int remove_directory(const char *base_dir) override final; private: - class ReloadPalfHandleImplFunctor : public ObBaseDirFunctor - { - public: - ReloadPalfHandleImplFunctor(PalfEnvImpl *palf_env_impl); - int func(const struct dirent *entry) override; - private: - PalfEnvImpl *palf_env_impl_; - }; - int reload_palf_handle_impl_(const int64_t palf_id); + int reload_palf_handle_impl_(); struct RemoveStaleIncompletePalfFunctor : public ObBaseDirFunctor { RemoveStaleIncompletePalfFunctor(PalfEnvImpl *palf_env_impl); @@ -293,33 +261,23 @@ class PalfEnvImpl : public IPalfEnvImpl }; private: - int create_palf_handle_impl_(const int64_t palf_id, - const AccessMode &access_mode, + int create_palf_handle_impl_(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, - const LogReplicaType replica_type, IPalfHandleImpl *&palf_handle_impl); - int scan_all_palf_handle_impl_director_(); const PalfDiskOptions &get_disk_options_guarded_by_lock_() const; int get_total_used_disk_space_(int64_t &total_used_disk_space, - int64_t &total_unrecyclable_disk_space, - int64_t &palf_id, - int64_t &maximum_used_size); + int64_t &total_unrecyclable_disk_space); int get_disk_usage_(int64_t &used_size_byte); int get_disk_usage_(int64_t &used_size_byte, - int64_t &unrecyclable_disk_space, - int64_t &palf_id, - int64_t &maximum_used_size); + int64_t &unrecyclable_disk_space); int recycle_blocks_(bool &has_recycled); - int wait_until_reference_count_to_zero_(const int64_t palf_id); - // check the diskspace whether is enough to hold a new palf instance. - bool check_can_create_palf_handle_impl_() const; - int remove_palf_handle_impl_from_map_not_guarded_by_lock_(const int64_t palf_id); - int move_incomplete_palf_into_tmp_dir_(const int64_t palf_id); + bool has_minimum_log_disk_capacity_() const; + int remove_palf_handle_impl_(); + int move_incomplete_palf_into_tmp_dir_(); int check_tmp_log_dir_exist_(bool &exist) const; int remove_stale_incomplete_palf_(); - int init_log_io_worker_config_(const int log_writer_parallelism, - LogIOWorkerConfig &config); + int init_log_io_worker_config_(LogIOWorkerConfig &config); int check_can_update_log_disk_options_(const PalfDiskOptions &disk_options); int remove_directory_while_exist_(const char *log_dir); @@ -330,13 +288,10 @@ class PalfEnvImpl : public IPalfEnvImpl RWLock palf_meta_lock_; common::ObILogAllocator *log_alloc_mgr_; ILogBlockPool *log_block_pool_; - FetchLogEngine fetch_log_engine_; - LogRpc log_rpc_; LogIOTaskCbThreadPool cb_thread_pool_; LogIOWorkerWrapper log_io_worker_wrapper_; LogSharedQueueTh log_shared_queue_th_; BlockGCTimerTask block_gc_timer_task_; - LogUpdater log_updater_; PalfMonitorCb *monitor_; PalfDiskOptionsWrapper disk_options_wrapper_; @@ -347,14 +302,12 @@ class PalfEnvImpl : public IPalfEnvImpl char tmp_log_dir_[common::MAX_PATH_SIZE]; common::ObAddr self_; - IPalfHandleImpl *single_palf_handle_; + IPalfHandleImpl *palf_handle_; LogLoopThread log_loop_thread_; - // last_palf_epoch_ is used to assign increasing epoch for each palf instance. + // The epoch changes whenever the unique log stream is recreated. int64_t last_palf_epoch_; - int64_t rebuild_replica_log_lag_threshold_;//for rebuild test bool enable_log_cache_; - bool enable_fetch_log_engine_; LogIOWorkerConfig log_io_worker_config_; bool diskspace_enough_; diff --git a/src/logservice/palf/palf_handle.cpp b/src/logservice/palf/palf_handle.cpp index 2e9c3d07a..207f38c40 100755 --- a/src/logservice/palf/palf_handle.cpp +++ b/src/logservice/palf/palf_handle.cpp @@ -24,9 +24,7 @@ namespace palf #define CHECK_VALID if (NULL == palf_handle_impl_) { return OB_NOT_INIT; } PalfHandle::PalfHandle() : palf_handle_impl_(NULL), - rc_cb_(NULL), - fs_cb_(NULL), - rebuild_cb_(NULL) + fs_cb_(NULL) { } @@ -38,9 +36,7 @@ PalfHandle::PalfHandle(const PalfHandle& rhs) PalfHandle::~PalfHandle() { palf_handle_impl_ = NULL; - rc_cb_ = NULL; fs_cb_ = NULL; - rebuild_cb_ = NULL; } bool PalfHandle::is_valid() const @@ -54,9 +50,7 @@ PalfHandle& PalfHandle::operator=(const PalfHandle &rhs) return *this; } palf_handle_impl_ = rhs.palf_handle_impl_; - rc_cb_ = rhs.rc_cb_; fs_cb_ = rhs.fs_cb_; - rebuild_cb_ = rhs.rebuild_cb_; return *this; } @@ -65,12 +59,10 @@ bool PalfHandle::operator==(const PalfHandle &rhs) const return palf_handle_impl_ == rhs.palf_handle_impl_; } -int PalfHandle::set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) +int PalfHandle::bootstrap() { CHECK_VALID; - return palf_handle_impl_->set_initial_member_list(member_list, paxos_replica_num, learner_list); + return palf_handle_impl_->bootstrap(); } int PalfHandle::append(const PalfAppendOptions &opts, @@ -86,17 +78,6 @@ int PalfHandle::append(const PalfAppendOptions &opts, return ret; } -int PalfHandle::raw_write(const PalfAppendOptions &opts, - const LSN &lsn, - const void *buffer, - const int64_t nbytes) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->submit_group_log(opts, lsn, static_cast(buffer), nbytes); - return ret; -} - int PalfHandle::seek(const LSN &lsn, PalfBufferIterator &iter) { CHECK_VALID; @@ -151,36 +132,12 @@ int PalfHandle::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) return palf_handle_impl_->locate_by_lsn_coarsely(lsn, result_scn); } -int PalfHandle::enable_sync() -{ - CHECK_VALID; - return palf_handle_impl_->enable_sync(); -} - -int PalfHandle::disable_sync() -{ - CHECK_VALID; - return palf_handle_impl_->disable_sync(); -} - -bool PalfHandle::is_sync_enabled() const -{ - CHECK_VALID; - return palf_handle_impl_->is_sync_enabled(); -} - int PalfHandle::advance_base_lsn(const LSN &lsn) { CHECK_VALID; return palf_handle_impl_->set_base_lsn(lsn); } -int PalfHandle::advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild) -{ - CHECK_VALID; - return palf_handle_impl_->advance_base_info(palf_base_info, is_rebuild); -} - int PalfHandle::get_begin_lsn(LSN &lsn) const { CHECK_VALID; @@ -215,14 +172,6 @@ int PalfHandle::get_end_lsn(LSN &lsn) const return ret; } -int PalfHandle::get_last_rebuild_lsn(LSN &last_rebuild_lsn) const -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->get_last_rebuild_lsn(last_rebuild_lsn); - return ret; -} - int PalfHandle::get_max_lsn(LSN &lsn) const { int ret = OB_SUCCESS; @@ -239,24 +188,6 @@ int PalfHandle::get_max_scn(SCN &scn) const return ret; } -int PalfHandle::get_config_version(LogConfigVersion &config_version) const -{ - CHECK_VALID; - return palf_handle_impl_->get_config_version(config_version); -} - -int PalfHandle::get_role(common::ObRole &role, int64_t &proposal_id, bool &is_pending_state) const -{ - CHECK_VALID; - return palf_handle_impl_->get_role(role, proposal_id, is_pending_state); -} - -int PalfHandle::get_palf_id(int64_t &palf_id) const -{ - CHECK_VALID; - return palf_handle_impl_->get_palf_id(palf_id); -} - int PalfHandle::get_palf_epoch(int64_t &palf_epoch) const { CHECK_VALID; @@ -271,218 +202,12 @@ int PalfHandle::get_end_scn(SCN &scn) const return ret; } -int PalfHandle::get_global_learner_list(common::GlobalLearnerList &learner_list) const -{ - CHECK_VALID; - return palf_handle_impl_->get_global_learner_list(learner_list); -} - -int PalfHandle::get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const -{ - CHECK_VALID; - return palf_handle_impl_->get_paxos_member_list(member_list, paxos_replica_num); -} - -int PalfHandle::get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - GlobalLearnerList &learner_list) const -{ - CHECK_VALID; - return palf_handle_impl_->get_paxos_member_list_and_learner_list(member_list, paxos_replica_num, learner_list); -} - -int PalfHandle::get_stable_membership(LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const -{ - CHECK_VALID; - return palf_handle_impl_->get_stable_membership(config_version, member_list, paxos_replica_num, learner_list); -} - -int PalfHandle::get_election_leader(common::ObAddr &addr) const -{ - CHECK_VALID; - return palf_handle_impl_->get_election_leader(addr); -} - -int PalfHandle::get_parent(common::ObAddr &parent) const -{ - CHECK_VALID; - return palf_handle_impl_->get_parent(parent); -} - -int PalfHandle::change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->change_replica_num(member_list, curr_replica_num, new_replica_num, timeout_us); -} -int PalfHandle::force_set_as_single_replica() -{ - CHECK_VALID; - return palf_handle_impl_->force_set_as_single_replica(); -} -int PalfHandle::force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num) -{ - CHECK_VALID; - return palf_handle_impl_->force_set_member_list(new_member_list, new_replica_num); -} -int PalfHandle::get_ack_info_array(LogMemberAckInfoList &ack_info_array, - common::GlobalLearnerList °raded_list) const -{ - CHECK_VALID; - return palf_handle_impl_->get_ack_info_array(ack_info_array, degraded_list); -} - -int PalfHandle::add_member(const common::ObMember &member, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->add_member(member, new_replica_num, config_version, timeout_us); -} - -int PalfHandle::remove_member(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->remove_member(member, new_replica_num, timeout_us); -} - -int PalfHandle::replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->replace_member(added_member, removed_member, config_version, timeout_us); -} - -int PalfHandle::add_learner(const common::ObMember &added_learner, const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->add_learner(added_learner, timeout_us); -} - -int PalfHandle::remove_learner(const common::ObMember &removed_learner, const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->remove_learner(removed_learner, timeout_us); -} - -int PalfHandle::switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->switch_learner_to_acceptor(learner, new_replica_num, config_version, timeout_us); -} - -int PalfHandle::switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->switch_acceptor_to_learner(member, new_replica_num, timeout_us); -} - -int PalfHandle::replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->replace_learners(added_learners, removed_learners, timeout_us); -} - -int PalfHandle::replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->replace_member_with_learner(added_member, removed_member, config_version, timeout_us); -} - -int PalfHandle::change_leader_to(const common::ObAddr &dst_addr) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->change_leader_to(dst_addr); - return ret; -} - -int PalfHandle::change_access_mode(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const SCN &ref_scn) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->change_access_mode(proposal_id, mode_version, access_mode, ref_scn); - return ret; -} - -int PalfHandle::get_access_mode(int64_t &mode_version, AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->get_access_mode(mode_version, access_mode); - return ret; -} - -int PalfHandle::get_access_mode(AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->get_access_mode(access_mode); - return ret; -} - -int PalfHandle::get_access_mode_version(int64_t &mode_version) const -{ - int ret = OB_SUCCESS; - CHECK_VALID; - return palf_handle_impl_->get_access_mode_version(mode_version); -} - -int PalfHandle::get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, +int PalfHandle::get_access_mode_ref_scn(AccessMode &access_mode, SCN &ref_scn) const { int ret = OB_SUCCESS; CHECK_VALID; - ret = palf_handle_impl_->get_access_mode_ref_scn(mode_version, access_mode, ref_scn); - return ret; -} - -int PalfHandle::disable_vote(const bool need_check_log_missing) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->disable_vote(need_check_log_missing); - return ret; -} - -bool PalfHandle::is_vote_enabled() const -{ - int ret = OB_SUCCESS; - bool bool_ret = false; - CHECK_VALID; - bool_ret = palf_handle_impl_->is_vote_enabled(); - return bool_ret; -} - -int PalfHandle::enable_vote() -{ - int ret = OB_SUCCESS; - CHECK_VALID; - ret = palf_handle_impl_->enable_vote(); + ret = palf_handle_impl_->get_access_mode_ref_scn(access_mode, ref_scn); return ret; } @@ -519,7 +244,7 @@ int PalfHandle::unregister_file_size_cb() if (NULL == fs_cb_) { PALF_LOG(TRACE, "no need unregister_file_size_cb", K(fs_cb_)); } else if (OB_FAIL(palf_handle_impl_->unregister_file_size_cb(fs_cb_))) { - PALF_LOG(WARN, "unregister_role_change_cb failed", K(ret)); + PALF_LOG(WARN, "unregister_file_size_cb failed", K(ret)); } else { MTL_DELETE(PalfFSCbNode, "PalfFSCbNode", fs_cb_); fs_cb_ = NULL; @@ -527,172 +252,12 @@ int PalfHandle::unregister_file_size_cb() return ret; } -int PalfHandle::register_role_change_cb(PalfRoleChangeCb *rc_cb) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (NULL == rc_cb) { - PALF_LOG(TRACE, "no need register_role_change_cb", K(ret)); - } else if (NULL != rc_cb_) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "PalfHandle has register_role_change_cb, not support regist repeatedly", K(ret), K(rc_cb), K(rc_cb_)); - } else { - PalfRoleChangeCbNode *rc_cb_node = MTL_NEW(PalfRoleChangeCbNode, "PalfRCCbNode", rc_cb); - if (NULL == rc_cb_node) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_FAIL(palf_handle_impl_->register_role_change_cb(rc_cb_node))) { - PALF_LOG(WARN, "register_role_change_cb failed", K(ret)); - } else { - rc_cb_ = rc_cb_node; - } - if (OB_FAIL(ret)) { - MTL_DELETE(PalfRoleChangeCbNode, "PalfRCCbNode", rc_cb_node); - rc_cb_ = NULL; - } - } - return ret; -} - -int PalfHandle::unregister_role_change_cb() -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (NULL == rc_cb_) { - PALF_LOG(TRACE, "no need unregister_role_change_cb", K(rc_cb_)); - } else if (OB_FAIL(palf_handle_impl_->unregister_role_change_cb(rc_cb_))) { - PALF_LOG(WARN, "unregister_role_change_cb failed", K(ret)); - } else { - MTL_DELETE(PalfRoleChangeCbNode, "PalfRCCbNode", rc_cb_); - rc_cb_ = NULL; - } - return ret; -} - -int PalfHandle::register_rebuild_cb(PalfRebuildCb *rebuild_cb) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (NULL == rebuild_cb) { - PALF_LOG(TRACE, "no need rebuild_cb", K(ret)); - } else if (NULL != rebuild_cb_) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "PalfHandle has register_rebuild_cb, not support regist repeatedly", - K(ret), K(rebuild_cb), K(rebuild_cb_)); - } else { - PalfRebuildCbNode *rebuild_cb_node = MTL_NEW(PalfRebuildCbNode, "RebuildCbNode", rebuild_cb); - if (NULL == rebuild_cb_node) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_FAIL(palf_handle_impl_->register_rebuild_cb(rebuild_cb_node))) { - PALF_LOG(WARN, "register_rebuild_cb failed", K(ret)); - } else { - rebuild_cb_ = rebuild_cb_node; - } - if (OB_FAIL(ret)) { - MTL_DELETE(PalfRebuildCbNode, "RebuildCbNode", rebuild_cb_node); - rebuild_cb_ = NULL; - } - } - return ret; -} - -int PalfHandle::unregister_rebuild_cb() -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (NULL == rebuild_cb_) { - PALF_LOG(TRACE, "no need unregister_rebuild_cb", K(ret)); - } else if (OB_FAIL(palf_handle_impl_->unregister_rebuild_cb(rebuild_cb_))) { - PALF_LOG(WARN, "unregister_rebuild_cb failed", K(ret)); - } else { - MTL_DELETE(PalfRebuildCbNode, "RebuildCbNode", rebuild_cb_); - rebuild_cb_ = NULL; - PALF_LOG(INFO, "unregister_rebuild_cb success", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandle::set_location_cache_cb(PalfLocationCacheCb *lc_cb) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (OB_ISNULL(lc_cb)) { - PALF_LOG(INFO, "no need set_location_cache_cb", KR(ret)); - } else if (OB_FAIL(palf_handle_impl_->set_location_cache_cb(lc_cb))) { - PALF_LOG(WARN, "set_location_cache_cb failed", KR(ret)); - } else { - } - return ret; -} - -int PalfHandle::reset_location_cache_cb() -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (OB_FAIL(palf_handle_impl_->reset_location_cache_cb())) { - PALF_LOG(WARN, "reset_location_cache_cb failed", KR(ret)); - } else { - PALF_LOG(INFO, "reset_location_cache_cb success", KR(ret)); - } - return ret; -} - -int PalfHandle::set_election_priority(election::ElectionPriority *priority) -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (OB_ISNULL(priority)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "priority is nullptr", KR(ret), KP(priority)); - } else if (OB_FAIL(palf_handle_impl_->set_election_priority(priority))) { - PALF_LOG(WARN, "set election priority to palf failed", KR(ret)); - } - return ret; -} - -int PalfHandle::reset_election_priority() -{ - int ret = OB_SUCCESS; - CHECK_VALID; - if (OB_FAIL(palf_handle_impl_->reset_election_priority())) { - PALF_LOG(WARN, "set election priority to palf failed", KR(ret)); - } - return ret; -} - -int PalfHandle::advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, - const int64_t downgrade_priority_time_us, - const char *reason) -{ - CHECK_VALID; - return palf_handle_impl_->advance_election_epoch_and_downgrade_priority(proposal_id, - downgrade_priority_time_us, - reason); -} - int PalfHandle::stat(PalfStat &palf_stat) const { CHECK_VALID; return palf_handle_impl_->stat(palf_stat); } -int PalfHandle::try_lock_config_change(int64_t lock_owner, - int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->try_lock_config_change(lock_owner, timeout_us); -} - -int PalfHandle::unlock_config_change(int64_t lock_owner, int64_t timeout_us) -{ - CHECK_VALID; - return palf_handle_impl_->unlock_config_change(lock_owner, timeout_us); -} - -int PalfHandle::get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) -{ - CHECK_VALID; - return palf_handle_impl_->get_config_change_lock_stat(lock_owner, is_locked); -} int PalfHandle::diagnose(PalfDiagnoseInfo &diagnose_info) const { diff --git a/src/logservice/palf/palf_handle.h b/src/logservice/palf/palf_handle.h index 99723ae7d..87ec88c9a 100755 --- a/src/logservice/palf/palf_handle.h +++ b/src/logservice/palf/palf_handle.h @@ -16,9 +16,7 @@ #ifndef OCEANBASE_LOGSERVICE_PALF_HANDLE_ #define OCEANBASE_LOGSERVICE_PALF_HANDLE_ -#include "common/ob_member_list.h" #include "common/ob_role.h" -#include "election_self.h" #include "lsn.h" #include "palf_handle_impl.h" #include "palf_handle_impl_guard.h" @@ -33,8 +31,6 @@ namespace palf { class PalfAppendOptions; class PalfFSCb; -class PalfRoleChangeCb; -class PalfLocationCacheCb; class PalfHandle { public: @@ -53,22 +49,7 @@ class PalfHandle // @brief move-assignment operator PalfHandle& operator=(PalfHandle &&rhs); bool operator==(const PalfHandle &rhs) const; - // After successfully creating the log stream, set the initial member list information, only allowed to execute once - // - // @param [in] member_list, member list of the log stream - // @param [in] paxos_replica_num, the number of replicas in the log stream paxos member group - // - // @return :TODO - // @brief set the initial member list of paxos group after creating - // palf successfully, it can only be called once - // @param[in] ObMemberList, the initial member list, do not include arbitration replica - // @param[in] int64_t, the paxos relica num - // @retval - // return OB_SUCCESS if success - // else return other errno - int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list); + int bootstrap(); //================ File access related interfaces ======================= int append(const PalfAppendOptions &opts, const void *buffer, @@ -77,11 +58,6 @@ class PalfHandle LSN &lsn, share::SCN &scn); - int raw_write(const PalfAppendOptions &opts, - const LSN &lsn, - const void *buffer, - const int64_t nbytes); - // @brief: read up to 'nbytes' from palf at offset of 'lsn' into the 'read_buf', and // there are alignment restrictions on the length and address of user-space buffers // and the file offset. @@ -165,15 +141,8 @@ class PalfHandle // - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled // - others: bug int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn); - // Enable log synchronization - int enable_sync(); - // Close log synchronization - int disable_sync(); - bool is_sync_enabled() const; // Advance the file's recyclable point int advance_base_lsn(const LSN &lsn); - // Migration/rebuild scenario advances base_lsn - int advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild); // Return the position information of the earliest readable log in the file int get_begin_lsn(LSN &lsn) const; int get_begin_scn(share::SCN &scn) const; @@ -192,247 +161,18 @@ class PalfHandle int get_end_scn(share::SCN &scn) const; int get_max_lsn(LSN &lsn) const; int get_max_scn(share::SCN &scn) const; - int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const; - // @brief get readable end lsn for this replica, all logs before it can be readable. + // @brief get readable end lsn; all logs before it are readable. // @param[out] lsn, readable end lsn. // -- OB_NOT_INIT not_init // -- OB_SUCCESS int get_readable_end_lsn(LSN &lsn) const; - //================= Distributed related interfaces ========================= - // Return the current replica's role, only Leader and Follower roles exist - // - // @param [out] role, current replica's role - // @param [out] leader_epoch, indicates a term of the leader, ensuring monotonic increase in scenarios of leader-follower switch and restart - // @param [out] is_pending_state, indicates whether the current replica is in a pending state - // - // @return :TODO - int get_role(common::ObRole &role, int64_t &proposal_id, bool &is_pending_state) const; - int get_palf_id(int64_t &palf_id) const; int get_palf_epoch(int64_t &palf_epoch) const; - int get_global_learner_list(common::GlobalLearnerList &learner_list) const; - int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const; - int get_config_version(LogConfigVersion &config_version) const; - int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - GlobalLearnerList &learner_list) const; - int get_stable_membership(LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const; - int get_election_leader(common::ObAddr &addr) const; - int get_parent(common::ObAddr &parent) const; - - // @brief: a special config change interface, change replica number of paxos group - // @param[in] common::ObMemberList: current memberlist, for pre-check - // @param[in] const int64_t curr_replica_num: current replica num, for pre-check - // @param[in] const int64_t new_replica_num: new replica num - // @param[in] const int64_t timeout_us: timeout, us - // @return - // - OB_SUCCESS: change_replica_num successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: change_replica_num timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - int change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us); - // @brief: force set self as single member - int force_set_as_single_replica(); - // @brief: force set member list. - // @param[in] const common::ObMemberList &new_member_list: members which will be added - // @param[in] const int64_t new_replica_num: replica number of paxos group after forcing to set member list - // @return - // - OB_SUCCESS: force_set_member_list successfully - // - OB_TIMEOUT: force_set_member_list timeout - // - OB_NOT_RUNNING: log stream is stopped - // - OB_INVALID_ARGUMENT: invalid argument - // - other: bug - int force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num); - - int get_ack_info_array(LogMemberAckInfoList &ack_info_array, - common::GlobalLearnerList °raded_list) const; - // @brief, add a member to paxos group, can be called only in leader - // @param[in] common::ObMember &member: member which will be added - // @param[in] const int64_t new_replica_num: replica number of paxos group after adding 'member' - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us: add member timeout, us - // @return - // - OB_SUCCESS: add member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: add member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - OB_STATE_NOT_MATCH: not the same leader - // - other: bug - int add_member(const common::ObMember &member, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us); - - // @brief, remove a member from paxos group, can be called only in leader - // @param[in] common::ObMember &member: member which will be removed - // @param[in] const int64_t new_replica_num: replica number of paxos group after removing 'member' - // @param[in] const int64_t timeout_us: remove member timeout, us - // @return - // - OB_SUCCESS: remove member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: remove member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - int remove_member(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us); - - // @brief, replace old_member with new_member, can be called only in leader - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - int replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us); - - // @brief: add a learner(read only replica) in this clsuter - // @param[in] const common::ObMember &added_learner: learner will be added - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: add_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - int add_learner(const common::ObMember &added_learner, - const int64_t timeout_us); - - // @brief: remove a learner(read only replica) in this clsuter - // @param[in] const common::ObMember &removed_learner: learner will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: remove_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - int remove_learner(const common::ObMember &removed_learner, - const int64_t timeout_us); - - // @brief: switch a learner(read only replica) to acceptor(full replica) in this clsuter - // @param[in] const common::ObMember &learner: learner will be switched to acceptor - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // learner to acceptor (similar to add_member) - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_learner_to_acceptor timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - int switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us); - - // @brief: switch an acceptor(full replica) to learner(read only replica) in this clsuter - // @param[in] const common::ObMember &member: acceptor will be switched to learner - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // acceptor to learner (similar to remove_member) - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_acceptor_to_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - int switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us); - // @brief, replace removed_learners with added_learners - // @param[in] const common::ObMemberList &added_learners: learners will be added - // @param[in] const common::ObMemberList &removed_learners: learners will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace learner successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - int replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us); - // @brief, replace removed_member with learner - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - int replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us); - int advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, - const int64_t downgrade_priority_time_us, - const char *reason); - int change_leader_to(const common::ObAddr &dst_addr); - // @brief: change AccessMode of palf. - // @param[in] const int64_t &proposal_id: current proposal_id of leader - // @param[in] const int64_t &mode_version: mode_version corresponding to AccessMode, - // can be gotted by get_access_mode - // @param[in] const palf::AccessMode access_mode: access_mode will be changed to - // @param[in] const int64_t ref_scn: scn of all submitted logs after changing access mode - // are bigger than ref_scn - // NB: ref_scn will take effect only when: - // a. ref_scn is bigger than/equal to max_ts(get_max_scn()) - // b. AccessMode is set to APPEND - // @retval - // OB_SUCCESS - // OB_NOT_MASTER: self is not active leader - // OB_EAGAIN: another change_acess_mode is running, try again later - // NB: 1. if return OB_EAGAIN, caller need execute 'change_access_mode' again. - // 2. before execute 'change_access_mode', caller need execute 'get_access_mode' to - // get 'mode_version' and pass it to 'change_access_mode' - int change_access_mode(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const share::SCN &ref_scn); - // @brief: query the access_mode of palf and it's corresponding mode_version - // @param[out] palf::AccessMode &access_mode: current access_mode - // @param[out] int64_t &mode_version: mode_version corresponding to AccessMode - // @retval - // OB_SUCCESS - int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const; - int get_access_mode(AccessMode &access_mode) const; - int get_access_mode_version(int64_t &mode_version) const; - int get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, + int get_access_mode_ref_scn(AccessMode &access_mode, SCN &ref_scn) const; - // @brief: check whether the palf instance is allowed to vote for logs - // By default, return true; - // After calling disable_vote(), return false. - bool is_vote_enabled() const; - // @brief: store a persistent flag which means this paxos replica - // can not reply ack when receiving logs. - // @param[in] need_check_log_missing: for rebuildinng caused by log missing, need check whether log - // By default, paxos replica can reply ack. - // @return: - int disable_vote(const bool need_check_log_missing); - // @brief: store a persistent flag which means this paxos replica - // can reply ack when receiving logs. - // By default, paxos replica can reply ack. - // @return: - int enable_vote(); //================= Callback function registration =========================== // @brief: register a callback to PalfHandleImpl, and do something in // this callback when file size has changed. @@ -443,66 +183,16 @@ class PalfHandle // NB: not thread safe int unregister_file_size_cb(); - // @brief: register a callback to PalfHandleImpl, and do something in - // this callback when role has changed. - // NB: not thread safe - int register_role_change_cb(PalfRoleChangeCb *rc_cb); - - // @brief: unregister a callback from PalfHandleImpl - // NB: not thread safe - int unregister_role_change_cb(); - - // @brief: register a callback to PalfHandleImpl, and do something in - // this callback when there is a rebuild operation. - // NB: not thread safe - int register_rebuild_cb(PalfRebuildCb *rebuild_cb); - - // @brief: unregister a callback from PalfHandleImpl - // NB: not thread safe - int unregister_rebuild_cb(); //================= Dependency function registration =========================== - int set_location_cache_cb(PalfLocationCacheCb *lc_cb); - int reset_location_cache_cb(); - int set_election_priority(election::ElectionPriority *priority); - int reset_election_priority(); int stat(PalfStat &palf_stat) const; - //---------config change lock related--------// - //@return - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS successfull lock - // -- OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT failed to lock because of locked by others - // -- OB_TIMEOUT timeout, may lock successfully or not - // -- OB_EAGAIN other config change operation is going on,need retry later - // -- OB_NOT_MASTER this replica is not leader, not refresh location and retry with actual leader - // -- OB_STATE_NOT_MATCH lock_owner is smaller than previous lock_owner - int try_lock_config_change(int64_t lock_owner, int64_t timeout_us); - //@return - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS successfull unlock - // -- OB_TIMEOUT timeout, may unlock successfully or not - // -- OB_EAGAIN other config change operation is going on,need retry later - // -- OB_NOT_MASTER this replica is not leader, need refresh location and retry with actual leader - // -- OB_STATE_NOT_MATCH lock_owner is smaller than previous lock_owner,or lock_owner is bigger than previous lock_owner - int unlock_config_change(int64_t lock_owner, int64_t timeout_us); - //@return - // -- OB_NOT_INIT not_init - // -- OB_SUCCESS success - // -- OB_NOT_MASTER this replica is not leader, not refresh location and retry with actual leader - // -- OB_EAGAIN is_locking or unlocking - int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked); - - - // @param [out] diagnose info, current diagnose info of palf int diagnose(PalfDiagnoseInfo &diagnose_info) const; - TO_STRING_KV(KP(palf_handle_impl_), KP(rc_cb_), KP(fs_cb_)); + TO_STRING_KV(KP(palf_handle_impl_), KP(fs_cb_)); private: palf::IPalfHandleImpl *palf_handle_impl_; - palf::PalfRoleChangeCbNode *rc_cb_; palf::PalfFSCbNode *fs_cb_; - palf::PalfRebuildCbNode *rebuild_cb_; }; } // end namespace oceanbase } // end namespace palf diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 0aa698a4f..a719f5e50 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -23,40 +23,26 @@ namespace oceanbase { using namespace common; using namespace share; -using namespace palf::election; namespace palf { PalfHandleImpl::PalfHandleImpl() : lock_(common::ObLatchIds::PALF_HANDLE_IMPL_LOCK), sw_(), - config_mgr_(), mode_mgr_(), state_mgr_(), - reconfirm_(), log_engine_(), - election_(), log_cache_(), - fetch_log_engine_(NULL), allocator_(NULL), - palf_id_(INVALID_PALF_ID), self_(), fs_cb_wrapper_(), - role_change_cb_wrpper_(), - rebuild_cb_wrapper_(), plugins_(), last_locate_scn_(), last_locate_block_(LOG_INVALID_BLOCK_ID), cannot_recv_log_warn_time_(OB_INVALID_TIMESTAMP), - cannot_handle_committed_info_time_(OB_INVALID_TIMESTAMP), log_disk_full_warn_time_(OB_INVALID_TIMESTAMP), - last_check_parent_child_time_us_(OB_INVALID_TIMESTAMP), wait_slide_print_time_us_(OB_INVALID_TIMESTAMP), append_size_stat_time_us_(OB_INVALID_TIMESTAMP), - replace_member_print_time_us_(OB_INVALID_TIMESTAMP), - config_change_print_time_us_(OB_INVALID_TIMESTAMP), - last_rebuild_lsn_(), - last_rebuild_meta_info_(), last_record_append_lsn_(PALF_INITIAL_LSN_VAL), has_set_deleted_(false), palf_env_impl_(NULL), @@ -65,20 +51,7 @@ PalfHandleImpl::PalfHandleImpl() handle_submit_log_cost_stat_("[PALF STAT HANDLE SUBMIT LOG COST TIME]", PALF_STAT_PRINT_INTERVAL_US), last_accum_write_statistic_time_(OB_INVALID_TIMESTAMP), accum_write_log_size_(0), - last_accum_fetch_statistic_time_(OB_INVALID_TIMESTAMP), - accum_fetch_log_size_(0), - replica_meta_lock_(), - rebuilding_lock_(), - config_change_lock_(), - mode_change_lock_(), last_dump_info_time_us_(OB_INVALID_TIMESTAMP), - last_check_sync_time_us_(OB_INVALID_TIMESTAMP), - last_renew_loc_time_us_(OB_INVALID_TIMESTAMP), - last_print_in_sync_time_us_(OB_INVALID_TIMESTAMP), - last_hook_fetch_log_time_us_(OB_INVALID_TIMESTAMP), - chaning_config_warn_time_(OB_INVALID_TIMESTAMP), - cached_is_in_sync_(false), - has_higher_prio_config_change_(false), is_inited_(false) { log_dir_[0] = '\0'; @@ -89,15 +62,11 @@ PalfHandleImpl::~PalfHandleImpl() destroy(); } -int PalfHandleImpl::init(const int64_t palf_id, - const AccessMode &access_mode, +int PalfHandleImpl::init(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, - const LogReplicaType replica_type, - FetchLogEngine *fetch_log_engine, const char *log_dir, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, IPalfEnvImpl *palf_env_impl, @@ -111,45 +80,37 @@ int PalfHandleImpl::init(const int64_t palf_id, LogSnapshotMeta snapshot_meta; if (IS_INIT) { ret = OB_INIT_TWICE; - PALF_LOG(ERROR, "LogServer has inited", K(ret), K(palf_id)); - } else if (false == is_valid_palf_id(palf_id) - || false == is_valid_access_mode(access_mode) + PALF_LOG(ERROR, "LogServer has inited", K(ret)); + } else if (false == is_valid_access_mode(access_mode) || false == palf_base_info.is_valid() - || INVALID_REPLICA == replica_type - || NULL == fetch_log_engine || NULL == log_dir || NULL == alloc_mgr || NULL == log_block_pool - || NULL == log_rpc || NULL == log_io_worker || NULL == log_shared_queue_th || NULL == palf_env_impl || false == self.is_valid() || palf_epoch < 0) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(palf_base_info), K(replica_type), - K(access_mode), K(log_dir), K(alloc_mgr), K(log_block_pool), K(log_rpc), + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_base_info), + K(access_mode), K(log_dir), K(alloc_mgr), K(log_block_pool), K(log_io_worker), K(log_shared_queue_th), K(palf_env_impl), K(self), K(palf_epoch)); - } else if (OB_FAIL(log_meta.generate_by_palf_base_info(palf_base_info, access_mode, replica_type))) { - PALF_LOG(WARN, "generate_by_palf_base_info failed", K(ret), K(palf_id), K(palf_base_info), K(access_mode), K(replica_type)); + } else if (OB_FAIL(log_meta.generate_by_palf_base_info(palf_base_info, access_mode))) { + PALF_LOG(WARN, "generate_by_palf_base_info failed", K(ret), K(palf_base_info), K(access_mode)); } else if ((pret = snprintf(log_dir_, MAX_PATH_SIZE, "%s", log_dir)) && false) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "error unexpected", K(ret), K(palf_id)); - } else if (OB_FAIL(log_engine_.init(palf_id, log_dir, log_meta, alloc_mgr, log_block_pool, &log_cache_, \ - log_rpc, log_io_worker, log_shared_queue_th, &plugins_, palf_epoch, PALF_BLOCK_SIZE, PALF_META_BLOCK_SIZE, io_adapter))) { - PALF_LOG(WARN, "LogEngine init failed", K(ret), K(palf_id), K(log_dir), K(alloc_mgr), - K(log_rpc), K(log_io_worker), K(log_shared_queue_th)); - } else if (OB_FAIL(do_init_mem_(palf_id, palf_base_info, log_meta, log_dir, self, fetch_log_engine, - alloc_mgr, log_rpc, palf_env_impl))) { - PALF_LOG(WARN, "PalfHandleImpl do_init_mem_ failed", K(ret), K(palf_id)); + PALF_LOG(ERROR, "error unexpected", K(ret)); + } else if (OB_FAIL(log_engine_.init(log_dir, log_meta, alloc_mgr, log_block_pool, &log_cache_, \ + log_io_worker, log_shared_queue_th, &plugins_, palf_epoch, PALF_BLOCK_SIZE, PALF_META_BLOCK_SIZE, io_adapter))) { + PALF_LOG(WARN, "LogEngine init failed", K(ret), K(log_dir), K(alloc_mgr), + K(log_io_worker), K(log_shared_queue_th)); + } else if (OB_FAIL(do_init_mem_(palf_base_info, log_meta, log_dir, self, + alloc_mgr, palf_env_impl))) { + PALF_LOG(WARN, "PalfHandleImpl do_init_mem_ failed", K(ret)); } else { - PALF_REPORT_INFO_KV(K_(palf_id)); - append_cost_stat_.set_extra_info(EXTRA_INFOS); - flush_cb_cost_stat_.set_extra_info(EXTRA_INFOS); last_accum_write_statistic_time_ = ObTimeUtility::current_time(); - last_accum_fetch_statistic_time_ = ObTimeUtility::current_time(); - PALF_EVENT("PalfHandleImpl init success", palf_id_, K(ret), K(self), K(access_mode), K(palf_base_info), - K(replica_type), K(log_dir), K(log_meta), K(palf_epoch)); + PALF_EVENT("PalfHandleImpl init success", K(ret), K(self), K(access_mode), K(palf_base_info), + K(log_dir), K(log_meta), K(palf_epoch)); } return ret; } @@ -159,12 +120,9 @@ bool PalfHandleImpl::check_can_be_used() const return false == ATOMIC_LOAD(&has_set_deleted_); } -int PalfHandleImpl::load(const int64_t palf_id, - FetchLogEngine *fetch_log_engine, - const char *log_dir, +int PalfHandleImpl::load(const char *log_dir, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, IPalfEnvImpl *palf_env_impl, @@ -181,39 +139,36 @@ int PalfHandleImpl::load(const int64_t palf_id, LogSnapshotMeta snapshot_meta; if (IS_INIT) { ret = OB_INIT_TWICE; - } else if (false == is_valid_palf_id(palf_id) - || NULL == log_dir - || NULL == fetch_log_engine + } else if (NULL == log_dir || NULL == alloc_mgr - || NULL == log_rpc || NULL == log_io_worker || NULL == log_shared_queue_th || false == self.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(log_dir), K(alloc_mgr), - K(log_rpc), K(log_io_worker), K(log_shared_queue_th)); - } else if (OB_FAIL(log_engine_.load(palf_id, log_dir, alloc_mgr, log_block_pool, &log_cache_, log_rpc, + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(log_dir), K(alloc_mgr), + K(log_io_worker), K(log_shared_queue_th)); + } else if (OB_FAIL(log_engine_.load(log_dir, alloc_mgr, log_block_pool, &log_cache_, log_io_worker, log_shared_queue_th, &plugins_, last_group_entry_header_lsn, entry_header, palf_epoch, PALF_BLOCK_SIZE, PALF_META_BLOCK_SIZE, io_adapter, is_integrity))) { - PALF_LOG(WARN, "LogEngine load failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "LogEngine load failed", K(ret)); // NB: when 'entry_header' is invalid, means that there is no data on disk, and set max_committed_end_lsn - // to 'base_lsn_', we will generate default PalfBaseInfo or get it from LogSnapshotMeta(rebuild). + // to 'base_lsn_', we will generate default PalfBaseInfo or get it from LogSnapshotMeta. } else if (false == is_integrity) { - PALF_LOG(INFO, "palf instance is not integrity", KPC(this)); + PALF_LOG(INFO, "log stream is incomplete", KPC(this)); } else if (FALSE_IT(snapshot_meta = log_engine_.get_log_meta().get_log_snapshot_meta())) { } else if (FALSE_IT(max_committed_end_lsn = (true == entry_header.is_valid() ? last_group_entry_header_lsn + entry_header.get_serialize_size() + entry_header.get_data_len() : snapshot_meta.base_lsn_))) { } else if (OB_FAIL(construct_palf_base_info_(max_committed_end_lsn, palf_base_info))) { - PALF_LOG(WARN, "construct_palf_base_info_ failed", K(ret), K(palf_id), K(entry_header), K(palf_base_info)); - } else if (OB_FAIL(do_init_mem_(palf_id, palf_base_info, log_engine_.get_log_meta(), log_dir, self, - fetch_log_engine, alloc_mgr, log_rpc, palf_env_impl))) { - PALF_LOG(WARN, "PalfHandleImpl do_init_mem_ failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "construct_palf_base_info_ failed", K(ret), K(entry_header), K(palf_base_info)); + } else if (OB_FAIL(do_init_mem_(palf_base_info, log_engine_.get_log_meta(), log_dir, self, + alloc_mgr, palf_env_impl))) { + PALF_LOG(WARN, "PalfHandleImpl do_init_mem_ failed", K(ret)); } else if (OB_FAIL(append_disk_log_to_sw_(max_committed_end_lsn))) { - PALF_LOG(WARN, "append_disk_log_to_sw_ failed", K(ret), K(palf_id)); + PALF_LOG(WARN, "append_disk_log_to_sw_ failed", K(ret)); } else { - PALF_EVENT("PalfHandleImpl load success", palf_id_, K(ret), K(palf_base_info), K(log_dir), K(palf_epoch)); + PALF_EVENT("PalfHandleImpl load success", K(ret), K(palf_base_info), K(log_dir), K(palf_epoch)); } return ret; } @@ -222,21 +177,15 @@ void PalfHandleImpl::destroy() { WLockGuard guard(lock_); if (IS_INIT) { - PALF_EVENT("PalfHandleImpl destroy", palf_id_, KPC(this)); + PALF_EVENT("PalfHandleImpl destroy", KPC(this)); is_inited_ = false; diskspace_enough_ = true; - cached_is_in_sync_ = false; plugins_.destroy(); self_.reset(); - palf_id_ = INVALID_PALF_ID; - fetch_log_engine_ = NULL; allocator_ = NULL; - election_.stop(); log_cache_.destroy(); log_engine_.destroy(); - reconfirm_.destroy(); state_mgr_.destroy(); - config_mgr_.destroy(); mode_mgr_.destroy(); sw_.destroy(); if (false == check_can_be_used()) { @@ -245,8 +194,6 @@ void PalfHandleImpl::destroy() palf_env_impl_ = NULL; last_accum_write_statistic_time_ = OB_INVALID_TIMESTAMP; accum_write_log_size_ = 0; - last_accum_fetch_statistic_time_ = OB_INVALID_TIMESTAMP; - accum_fetch_log_size_ = 0; } } @@ -262,35 +209,9 @@ int PalfHandleImpl::start() return ret; } -int PalfHandleImpl::set_initial_member_list( - const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) +int PalfHandleImpl::bootstrap() { - int ret = OB_SUCCESS; - LogConfigVersion config_version; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl has not inited!!!", K(ret)); - } else { - { - WLockGuard guard(lock_); - const int64_t proposal_id = state_mgr_.get_proposal_id(); - if (OB_FAIL(config_mgr_.set_initial_member_list(member_list, paxos_replica_num, learner_list, - proposal_id, config_version))) { - PALF_LOG(WARN, "LogConfigMgr set_initial_member_list failed", K(ret), KPC(this)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(config_mgr_.wait_config_log_persistence(config_version))) { - PALF_LOG(WARN, "want_config_log_persistence failed", K(ret), KPC(this)); - } else { - PALF_EVENT("set_initial_member_list success", palf_id_, K(ret), K(member_list), - K(learner_list), K(paxos_replica_num)); - report_set_initial_member_list_(paxos_replica_num, member_list); - } - } - return ret; + return IS_NOT_INIT ? OB_NOT_INIT : OB_SUCCESS; } int PalfHandleImpl::get_begin_lsn(LSN &lsn) const @@ -364,7 +285,7 @@ int PalfHandleImpl::submit_log( } else if (NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BODY_SIZE || !ref_scn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K_(palf_id), KP(buf), K(buf_len), K(ref_scn)); + PALF_LOG(WARN, "invalid argument", KP(buf), K(buf_len), K(ref_scn)); } else { RLockGuard guard(lock_); if (false == palf_env_impl_->check_disk_space_enough()) { @@ -372,11 +293,10 @@ int PalfHandleImpl::submit_log( if (palf_reach_time_interval(1 * 1000 * 1000, log_disk_full_warn_time_)) { PALF_LOG(WARN, "log outof disk space", KPC(this), K(opts), K(ref_scn)); } - } else if (!state_mgr_.can_append(opts.proposal_id, opts.need_check_proposal_id)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "cannot submit_log", KPC(this), KP(buf), K(buf_len), "role", - state_mgr_.get_role(), "state", state_mgr_.get_state(), "proposal_id", - state_mgr_.get_proposal_id(), K(opts), "mode_mgr can_append", mode_mgr_.can_append()); + } else if (!state_mgr_.can_append()) { + ret = OB_STATE_NOT_MATCH; + PALF_LOG(WARN, "cannot submit_log", KPC(this), KP(buf), K(buf_len), + "state", state_mgr_.get_state(), K(opts), "mode_mgr can_append", mode_mgr_.can_append()); } else if (OB_FAIL(sw_.submit_log(buf, buf_len, ref_scn, lsn, scn))) { if (OB_EAGAIN != ret) { PALF_LOG(WARN, "submit_log failed", KPC(this), KP(buf), K(buf_len)); @@ -392,3483 +312,628 @@ int PalfHandleImpl::submit_log( return ret; } -int PalfHandleImpl::get_palf_id(int64_t &palf_id) const +int PalfHandleImpl::set_base_lsn( + const LSN &lsn) { + // NB: Guarded by lock is important, otherwise there are some problems concurrent with rebuild or migrate. + // + // Thread1(assume it's migrate thread) + // 1. if the 'base_lsn' of data source is greater than or equal to local, and then + // 2. avoid the hole between blocks, delete all blocks before 'base_lsn', submit truncate prefix blocks task + // 3. to update base lsn, submit update snpshot meta task. + // + // + // Thread2(checkpoint thread) + // 1. the clog disk is not enough, and update it via 'set_base_lsn' + // + // Consider that: + // + // Time1: thread1 has executed step1, assume the base lsn of data source is 100, local base lsn is 50, + // and then thread2 execute 'set_base_lsn', the local base lsn set to 150 + // + // Time2: thread1 submit truncate prefix blocks, and this task will be failed because the base lsn in this task + // is smaller than local base lsn. + // + // Therefore, we need guarded by lock. int ret = OB_SUCCESS; + RLockGuard guard(lock_); + LSN end_lsn = get_end_lsn(); + LogSnapshotMeta log_snapshot_meta = log_engine_.get_log_meta().get_log_snapshot_meta(); + const LSN new_base_lsn(lsn_2_block(lsn, PALF_BLOCK_SIZE) * PALF_BLOCK_SIZE); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); + } else if (!lsn.is_valid() || lsn > end_lsn) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(end_lsn), K(lsn)); + } else if (log_snapshot_meta.base_lsn_ >= new_base_lsn) { + PALF_LOG(WARN, "no need to set new base lsn, curr base lsn is greater than or equal to new base lsn", + KPC(this), K(log_snapshot_meta), K(new_base_lsn), K(lsn)); } else { - palf_id = palf_id_; + FlushMetaCbCtx flush_meta_cb_ctx; + flush_meta_cb_ctx.type_ = SNAPSHOT_META; + flush_meta_cb_ctx.base_lsn_ = new_base_lsn; + if (OB_FAIL(log_snapshot_meta.generate(new_base_lsn, + log_snapshot_meta.prev_log_info_, + log_snapshot_meta.prev_log_tail_lsn_))) { + PALF_LOG(WARN, "LogSnapshotMeta generate failed", K(ret), KPC(this), K(log_snapshot_meta)); + } else if (OB_FAIL(log_engine_.submit_flush_snapshot_meta_task(flush_meta_cb_ctx, log_snapshot_meta))) { + PALF_LOG(WARN, "submit_flush_snapshot_meta_task failed", K(ret), KPC(this)); + } else { + PALF_EVENT("set_base_lsn success", K(ret), K(self_), K(lsn), + K(log_snapshot_meta), K(new_base_lsn), K(flush_meta_cb_ctx)); + plugins_.record_set_base_lsn_event(new_base_lsn); + } } return ret; } -int PalfHandleImpl::get_role(common::ObRole &role, - int64_t &proposal_id, - bool &is_pending_state) const +int PalfHandleImpl::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) { int ret = OB_SUCCESS; + block_id_t result_block_id = LOG_INVALID_BLOCK_ID; + result_lsn.reset(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); + } else if (OB_UNLIKELY(!scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(scn)); + } else if (OB_FAIL(get_block_id_by_scn_(scn, result_block_id))) { + PALF_LOG(WARN, "get_block_id_by_scn_ failed", KR(ret), KPC(this), K(scn)); } else { - ObRole curr_role = INVALID_ROLE; - ObReplicaState curr_state = INVALID_STATE; - int64_t curr_leader_epoch = -1; - do { - is_pending_state = mode_mgr_.is_in_pending_state(); - // get the current proposal_id - proposal_id = state_mgr_.get_proposal_id(); - // Get the current leader_epoch - curr_leader_epoch = state_mgr_.get_leader_epoch(); - // Get the current role, state - state_mgr_.get_role_and_state(curr_role, curr_state); - if (LEADER != curr_role || ACTIVE != curr_state) { - // not - role = common::FOLLOWER; - if (RECONFIRM == curr_state || PENDING == curr_state) { - is_pending_state = true; - } - } else if (false == state_mgr_.check_epoch_is_same_with_election(curr_leader_epoch)) { - // PALF is currently , but the epoch changed at the election layer, - // Explanation that election has switched to leader, PALF will eventually switch to , this situation is also considered as pending state, - // Otherwise the caller sees PALF as , but proposal_id is still the value from LEADER, - // May lead to unexpected behavior, such as the role change service may prematurely execute the relinquishment operation. - role = common::FOLLOWER; - is_pending_state = true; - } else { - // Return LEADER - role = common::LEADER; - } - // double check proposal_id - if (proposal_id == state_mgr_.get_proposal_id()) { - // proposal_id does not change, exit loop - break; - } else { - // proposal_id has changed, need retry - } - } while(true); + } + // 2. convert block_id to lsn + if (OB_SUCC(ret)) { + result_lsn = LSN(result_block_id * PALF_BLOCK_SIZE); + inc_update_last_locate_block_scn_(result_block_id, scn); } return ret; } -int PalfHandleImpl::change_leader_to(const common::ObAddr &dest_addr) +int PalfHandleImpl::get_block_id_by_scn_(const SCN &scn, block_id_t &result_block_id) { int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret), K(dest_addr)); - } else if (OB_FAIL(election_.change_leader_to(dest_addr))) { - PALF_LOG(WARN, "election can not change leader", K(ret), KPC(this), K(dest_addr)); + block_id_t mid_block_id = LOG_INVALID_BLOCK_ID, min_block_id = LOG_INVALID_BLOCK_ID; + block_id_t max_block_id = LOG_INVALID_BLOCK_ID; + int64_t mid_ts = OB_INVALID_TIMESTAMP; + if (OB_FAIL(get_binary_search_range_(scn, min_block_id, max_block_id, result_block_id))) { + PALF_LOG(WARN, "get_binary_search_range_ failed", KR(ret), KPC(this), K(scn)); } else { - PALF_EVENT("election leader will be changed", palf_id_, K(ret), KPC(this), K(dest_addr)); - plugins_.record_election_leader_change_event(palf_id_, dest_addr); + // 1. get lower bound lsn (result_lsn) by binary search + SCN mid_scn; + while(OB_SUCC(ret) && min_block_id <= max_block_id) { + mid_block_id = min_block_id + ((max_block_id - min_block_id) >> 1); + if (OB_FAIL(log_engine_.get_block_min_scn(mid_block_id, mid_scn))) { + PALF_LOG(WARN, "get_block_min_scn failed", KR(ret), KPC(this), K(mid_block_id)); + // OB_ERR_OUT_OF_UPPER_BOUND: this block is a empty active block, just return + // OB_ERR_OUT_OF_LOWER_BOUND: block_id_ is smaller than min_block_id, this block may be recycled + // OB_ERR_UNEXPECTED: log files lost unexpectedly, just return + // OB_IO_ERROR: just return + if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { + // block mid_lsn.block_id_ may be recycled, get_binary_search_range_ again + if (OB_FAIL(get_binary_search_range_(scn, min_block_id, max_block_id, result_block_id))) { + PALF_LOG(WARN, "get_binary_search_range_ failed", KR(ret), KPC(this), K(scn)); + } + } else if (OB_ERR_OUT_OF_UPPER_BOUND == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + } else if (mid_scn < scn) { + min_block_id = mid_block_id; + if (max_block_id == min_block_id) { + result_block_id = mid_block_id; + break; + } else if (max_block_id == min_block_id + 1) { + SCN next_min_scn; + if (OB_FAIL(log_engine_.get_block_min_scn(max_block_id, next_min_scn))) { + // if fail to read next block, just return prev block lsn + ret = OB_SUCCESS; + result_block_id = mid_block_id; + } else if (scn < next_min_scn) { + result_block_id = mid_block_id; + } else { + result_block_id = max_block_id; + } + break; + } + } else if (mid_scn > scn) { + // block_id is uint64_t, so check == 0 firstly. + if (mid_block_id == 0 || mid_block_id - 1 < min_block_id) { + ret = OB_ERR_OUT_OF_LOWER_BOUND; + PALF_LOG(WARN, "scn is smaller than min scn of first block", KR(ret), KPC(this), K(min_block_id), + K(max_block_id), K(mid_block_id), K(scn), K(mid_scn)); + } else { + max_block_id = mid_block_id - 1; + } + } else { + result_block_id = mid_block_id; + break; + } + } } return ret; } -int PalfHandleImpl::get_global_learner_list(common::GlobalLearnerList &learner_list) const -{ - RLockGuard guard(lock_); - return config_mgr_.get_global_learner_list(learner_list); -} - -int PalfHandleImpl::get_paxos_member_list( - common::ObMemberList &member_list, - int64_t &paxos_replica_num) const +void PalfHandleImpl::set_deleted() { - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); - } else if (OB_FAIL(config_mgr_.get_curr_member_list(member_list, paxos_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", K(ret), KPC(this)); - } else {} - return ret; + ATOMIC_STORE(&has_set_deleted_, true); + PALF_LOG(INFO, "set_deleted success", KPC(this)); } -int PalfHandleImpl::get_config_version(LogConfigVersion &config_version) const +int PalfHandleImpl::get_binary_search_range_(const SCN &scn, + block_id_t &min_block_id, + block_id_t &max_block_id, + block_id_t &result_block_id) { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret), K_(palf_id)); + result_block_id = LOG_INVALID_BLOCK_ID; + const LSN committed_lsn = get_end_lsn(); + if (OB_FAIL(log_engine_.get_block_id_range(min_block_id, max_block_id))) { + // there is no log whose scn is smaller than 'scn' now and in the future, + // return OB_ERR_OUT_OF_LOWER_BOUND. + if (OB_ENTRY_NOT_EXIST == ret && scn <= get_end_scn()) { + ret = OB_ERR_OUT_OF_LOWER_BOUND; + } } else { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.get_config_version(config_version))) { - PALF_LOG(WARN, "failed to get_config_version", K(ret), K_(palf_id)); + block_id_t committed_block_id = lsn_2_block(committed_lsn, PALF_BLOCK_SIZE); + max_block_id = (committed_block_id < max_block_id)? committed_block_id : max_block_id; + // optimization: cache last_locate_scn_ to shrink binary search range + SpinLockGuard guard(last_locate_lock_); + if (is_valid_block_id(last_locate_block_) && + min_block_id <= last_locate_block_ && + max_block_id >= last_locate_block_) { + if (scn < last_locate_scn_) { + max_block_id = last_locate_block_; + } else if (scn > last_locate_scn_) { + min_block_id = last_locate_block_; + } else { + result_block_id = last_locate_block_; + // result_lsn hits last_locate_block_ cache, don't need binary search + // let min_block_id > max_block_id + min_block_id = 1; + max_block_id = 0; + } } + PALF_LOG(INFO, "get_binary_search_range_", K(ret), KPC(this), K(min_block_id), K(max_block_id), + K(result_block_id), K(committed_lsn), K(scn), K_(last_locate_scn), K_(last_locate_block)); } return ret; } -int PalfHandleImpl::get_paxos_member_list_and_learner_list( - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - GlobalLearnerList &learner_list) const +void PalfHandleImpl::inc_update_last_locate_block_scn_(const block_id_t &block_id, const SCN &scn) { - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); - } else if (OB_FAIL(config_mgr_.get_curr_member_list(member_list, paxos_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", K(ret), KPC(this)); - } else if (OB_FAIL(config_mgr_.get_global_learner_list(learner_list))) { - PALF_LOG(WARN, "get_global_learner_list failed", K(ret), KPC(this)); - } else {} - return ret; + SpinLockGuard guard(last_locate_lock_); + if (block_id > last_locate_block_) { + last_locate_block_ = block_id; + last_locate_scn_ = scn; + } } -int PalfHandleImpl::get_stable_membership(LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const +int PalfHandleImpl::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) { int ret = OB_SUCCESS; - const int get_lock = config_change_lock_.trylock(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); - } else if (OB_SUCCESS != get_lock) { - ret = OB_EAGAIN; - if (palf_reach_time_interval(1000 * 1000, config_change_print_time_us_)) { - PALF_LOG(WARN, "another config_change is running, try again", KR(ret), KPC(this)); - } + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); + } else if (!lsn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(lsn)); + } else if (lsn < log_engine_.get_begin_lsn()) { + ret = OB_ERR_OUT_OF_LOWER_BOUND; + PALF_LOG(WARN, "lsn is too small, this block has been recycled", KR(ret), KPC(this), + K(lsn), "begin_lsn", log_engine_.get_begin_lsn()); } else { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.get_config_version(config_version))) { - PALF_LOG(WARN, "failed to get_config_version", K(ret), K_(palf_id)); - } else if (OB_FAIL(config_mgr_.get_curr_member_list(member_list, paxos_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", K(ret), KPC(this)); - } else if (OB_FAIL(config_mgr_.get_global_learner_list(learner_list))) { - PALF_LOG(WARN, "get_global_learner_list failed", K(ret), KPC(this)); + const LSN committed_lsn = get_end_lsn(); + LSN curr_lsn = (committed_lsn <= lsn) ? committed_lsn: lsn; + block_id_t curr_block_id = lsn_2_block(curr_lsn, PALF_BLOCK_SIZE); + if (OB_FAIL(log_engine_.get_block_min_scn(curr_block_id, result_scn))) { + // if this block is a empty active block, read prev block if exists + if (OB_ERR_OUT_OF_UPPER_BOUND == ret && + curr_block_id > 0 && + OB_FAIL(log_engine_.get_block_min_scn(curr_block_id - 1, result_scn))) { + PALF_LOG(WARN, "get_block_min_scn failed", KR(ret), KPC(this), K(curr_lsn), K(lsn)); + } } - } - if (OB_SUCCESS == get_lock) { - config_change_lock_.unlock(); + PALF_LOG(INFO, "locate_by_lsn_coarsely", KR(ret), KPC(this), K(lsn), K(committed_lsn), K(result_scn)); } return ret; } -int PalfHandleImpl::get_election_leader(ObAddr &addr) const +int PalfHandleImpl::get_min_block_info_for_gc(block_id_t &min_block_id, SCN &max_scn) { int ret = OB_SUCCESS; - RLockGuard guard(lock_); - ret = get_election_leader_without_lock_(addr); +// if (false == end_lsn.is_valid()) { +// ret = OB_ENTRY_NOT_EXIST; +// } + if (OB_FAIL(log_engine_.get_min_block_info_for_gc(min_block_id, max_scn))) { + PALF_LOG(WARN, "get_min_block_info_for_gc failed", K(ret), KPC(this)); + } else { + PALF_LOG(TRACE, "get_min_block_info_for_gc success", K(ret), KPC(this), K(min_block_id), K(max_scn)); + } return ret; } -int PalfHandleImpl::get_parent(common::ObAddr &parent) const +int PalfHandleImpl::get_min_block_id_for_gc(block_id_t &min_block_id) { int ret = OB_SUCCESS; + block_id_t max_block_id = LOG_INVALID_BLOCK_ID; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); - } else { - parent = config_mgr_.get_parent(); - ret = (parent.is_valid())? OB_SUCCESS: OB_ENTRY_NOT_EXIST; + } else if (OB_FAIL(log_engine_.get_block_id_range(min_block_id, max_block_id))) { + PALF_LOG(WARN, "get_block_id_range failed", K(ret), KPC(this)); } return ret; } -int PalfHandleImpl::handle_config_change_pre_check(const ObAddr &server, - const LogGetMCStReq &req, - LogGetMCStResp &resp) +int PalfHandleImpl::delete_block(const block_id_t &block_id) { int ret = OB_SUCCESS; - uint64_t tenant_data_version = 0; - int tmp_ret = common::OB_SUCCESS; - const bool is_vote_enabled = state_mgr_.is_allow_vote(); - const bool is_sync_enabled = state_mgr_.is_sync_enabled(); - LSN last_rebuild_lsn; - do { - SpinLockGuard guard(last_rebuild_meta_info_lock_); - last_rebuild_lsn = last_rebuild_lsn_; - } while (0); - const bool need_rebuild = is_need_rebuild(get_end_lsn(), last_rebuild_lsn); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret), K_(palf_id)); - } else if (OB_TMP_FAIL(oceanbase::common::ObClusterVersion::get_instance().get_tenant_data_version(tenant_data_version))) { - // Note: if the DATA_VERSION in the replica is empty, it is not allowed - // to be added in the Paxos group. Check PalfHandleImpl only - resp.is_normal_replica_ = false; - PALF_LOG(WARN, "get tenant data version failed", K(tmp_ret), K(req), K(resp)); - } else if (false == is_vote_enabled || false == is_sync_enabled || true == need_rebuild) { - resp.is_normal_replica_ = false; - PALF_LOG(WARN, "replica has been disabled vote/sync", K(ret), K(req), K(resp), - K(is_vote_enabled), K(is_sync_enabled), K(need_rebuild)); + } else if (OB_FAIL(log_engine_.delete_block(block_id))) { + PALF_LOG(WARN, "delete block failed", K(ret), KPC(this), K(block_id)); } else { - RLockGuard guard(lock_); - if (req.need_purge_throttling_) { - int tmp_ret = OB_SUCCESS; - const PurgeThrottlingType purge_type = PurgeThrottlingType::PURGE_BY_GET_MC_REQ; - if (OB_SUCCESS != (tmp_ret = log_engine_.submit_purge_throttling_task(purge_type))) { - PALF_LOG_RET(WARN, tmp_ret, "failed to submit_purge_throttling_task with handle_config_change_pre_check", K_(palf_id)); - } - } - resp.msg_proposal_id_ = state_mgr_.get_proposal_id(); - LogConfigVersion curr_config_version; - sw_.get_max_flushed_end_lsn(resp.max_flushed_end_lsn_); - resp.last_slide_log_id_ = sw_.get_last_slide_log_id(); - if (OB_FAIL(config_mgr_.get_config_version(curr_config_version))) { - } else { - resp.need_update_config_meta_ = (req.config_version_ != curr_config_version); - } - resp.is_normal_replica_ = true; - - // it's a optimization. To add one F members into 1F1A group, - // leader will not accept appended logs until max_flushed_end_lsn of - // added F member reaches log barrier. So we fetch log from leader right now - // to reduce waiting time. - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = sw_.try_fetch_log(FetchTriggerType::ADD_MEMBER_PRE_CHECK))) { - PALF_LOG(WARN, "try_fetch_log with ADD_MEMBER_PRE_CHECK failed", - KR(tmp_ret), KPC(this), K(server)); - } else { - PALF_LOG(INFO, "try_fetch_log with ADD_MEMBER_PRE_CHECK success", KR(tmp_ret), KPC(this)); - } - PALF_LOG(INFO, "handle_config_change_pre_check success", K(ret), KPC(this), K(server), - K(req), K(resp), K(curr_config_version)); + PALF_LOG(WARN, "delete block success", K(ret), KPC(this), K(block_id)); } return ret; } -int PalfHandleImpl::force_set_as_single_replica() +int PalfHandleImpl::inner_append_log(const LSN &lsn, + const LogWriteBuf &write_buf, + const SCN &scn) { int ret = OB_SUCCESS; + const int64_t begin_ts = ObTimeUtility::current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else { - const int64_t now_time_us = common::ObTimeUtility::current_time(); - const ObMember member(self_, now_time_us); - const int64_t new_replica_num = 1; - LogConfigChangeArgs args(member, new_replica_num, FORCE_SINGLE_MEMBER); - const int64_t timeout_us = 10 * 1000 * 1000L; - int64_t prev_replica_num; - (void) config_mgr_.get_replica_num(prev_replica_num); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "one_stage_config_change_ failed", KR(ret), KPC(this), K(member), K(new_replica_num)); - } else { - PALF_EVENT("force_set_as_single_replica success", palf_id_, KR(ret), KPC(this), K(member), K(new_replica_num)); - report_force_set_as_single_replica_(prev_replica_num, new_replica_num, member); - } + PALF_LOG(ERROR, "PalfHandleImpl not inited", K(ret), KPC(this)); + } else if (false == lsn.is_valid() + || false == write_buf.is_valid()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(lsn), K(write_buf)); + } else if (OB_FAIL(log_engine_.append_log(lsn, write_buf, scn))) { + PALF_LOG(ERROR, "LogEngine pwrite failed", K(ret), KPC(this), K(lsn), K(scn)); + } else { + const int64_t curr_size = write_buf.get_total_size(); + const int64_t accum_size = ATOMIC_AAF(&accum_write_log_size_, curr_size); + const int64_t now = ObTimeUtility::current_time(); + const int64_t time_cost = now - begin_ts; + append_cost_stat_.stat(time_cost); + if (time_cost >= 5 * 1000) { + PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "write log cost too much time", K(ret), KPC(this), + K(lsn), K(scn), "size", write_buf.get_total_size(), K(accum_size), K(time_cost)); + } + if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_accum_write_statistic_time_)) { + PALF_LOG(INFO, "[PALF STAT INNER APPEND LOG SIZE]", KPC(this), K(accum_size)); + ATOMIC_STORE(&accum_write_log_size_, 0); + } } return ret; } -int PalfHandleImpl::force_set_member_list(const common::ObMemberList &new_member_list, - const int64_t new_replica_num) +int PalfHandleImpl::inner_append_log(const LSNArray &lsn_array, + const LogWriteBufArray &write_buf_array, + const SCNArray &scn_array) { int ret = OB_SUCCESS; + const int64_t begin_ts = ObTimeUtility::current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl is not inited!", KR(ret), KPC(this)); - } else if (!new_member_list.is_valid() || !is_valid_replica_num(new_replica_num) || - new_replica_num != new_member_list.get_member_number()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(new_member_list), K(new_replica_num)); - } else if (OB_FAIL(force_set_member_list_(new_member_list, new_replica_num))){ - PALF_LOG(WARN, "failed to force to set member list", KR(ret), KPC(this), K(new_member_list), K(new_replica_num)); + PALF_LOG(ERROR, "PalfHandleImpl not inited", K(ret), KPC(this)); + } else if (OB_FAIL(log_engine_.append_log(lsn_array, write_buf_array, scn_array))) { + PALF_LOG(ERROR, "LogEngine pwrite failed", K(ret), KPC(this), K(lsn_array), K(scn_array)); } else { - PALF_LOG(INFO, "force to set member list successfully", K_(palf_id), K(new_member_list), K(new_replica_num)); + int64_t accum_size = 0; + int64_t curr_size = 0; + int64_t lsn_array_count = lsn_array.count(); + int64_t write_buf_array_count = write_buf_array.count(); + if (0 < lsn_array_count && 0 < write_buf_array_count) { + int64_t last_log_buf_len = write_buf_array[write_buf_array_count - 1]->get_total_size(); + curr_size = lsn_array[lsn_array_count - 1].val_ - lsn_array[0].val_ + last_log_buf_len; + } + accum_size = ATOMIC_AAF(&accum_write_log_size_, curr_size); + const int64_t now = ObTimeUtility::current_time(); + const int64_t time_cost = now - begin_ts; + append_cost_stat_.stat(time_cost); + if (time_cost > 10 * 1000) { + PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "write log cost too much time", K(ret), KPC(this), K(lsn_array), + K(scn_array), K(curr_size), K(accum_size), K(time_cost)); + } + if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_accum_write_statistic_time_)) { + PALF_LOG(INFO, "[PALF STAT INNER APPEND LOG SIZE]", KPC(this), K(accum_size)); + ATOMIC_STORE(&accum_write_log_size_, 0); + } } - return ret; } -int PalfHandleImpl::change_replica_num( - const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) +int PalfHandleImpl::inner_append_meta(const char *buf, + const int64_t buf_len) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (!member_list.is_valid() || - false == is_valid_replica_num(curr_replica_num) || - false == is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { + PALF_LOG(ERROR, "PalfHandleImpl not inited"); + } else if (NULL == buf + || 0 >= buf_len) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(member_list), K(curr_replica_num), - K(new_replica_num), K(timeout_us)); + PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(buf), K(buf_len)); + } else if (OB_FAIL(log_engine_.append_meta(buf, buf_len))) { + PALF_LOG(ERROR, "LogEngine append_meta failed", K(ret), KPC(this)); } else { - LogConfigChangeArgs args(member_list, curr_replica_num, new_replica_num, CHANGE_REPLICA_NUM); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "change_replica_num failed", KR(ret), KPC(this), K(member_list), - K(curr_replica_num), K(new_replica_num)); - } else { - PALF_EVENT("change_replica_num success", palf_id_, KR(ret), KPC(this), K(member_list), - K(curr_replica_num), K(new_replica_num)); - report_change_replica_num_(curr_replica_num, new_replica_num, member_list); - } } return ret; } -int PalfHandleImpl::add_member( - const common::ObMember &member, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) +int PalfHandleImpl::inner_truncate_prefix_blocks(const LSN &lsn) { int ret = OB_SUCCESS; - int64_t prev_replica_num; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (!member.is_valid() || - false == is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { + PALF_LOG(ERROR, "PalfHandleImpl not inited"); + } else if (false == lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(member), K(new_replica_num), K(timeout_us)); - } else if (OB_FAIL(config_mgr_.get_replica_num(prev_replica_num))) { - PALF_LOG(WARN, "get prev_replica_num failed", KR(ret), KPC(this)); + PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(lsn)); + } else if (OB_FAIL(log_engine_.truncate_prefix_blocks(lsn))) { + PALF_LOG(WARN, "LogEngine truncate_prefix_blocks failed", K(ret), KPC(this), K(lsn)); } else { - LogConfigChangeArgs args(member, new_replica_num, config_version, ADD_MEMBER); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "add_member failed", KR(ret), KPC(this), K(member), K(new_replica_num)); - } else { - PALF_EVENT("add_member success", palf_id_, KR(ret), KPC(this), K(member), K(new_replica_num)); - report_add_member_(prev_replica_num, new_replica_num, member); - } + PALF_LOG(INFO, "LogEngine truncate_prefix_blocks success", K(ret), KPC(this), K(lsn)); } return ret; } -int PalfHandleImpl::remove_member( - const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) +int PalfHandleImpl::set_scan_disk_log_finished() { int ret = OB_SUCCESS; - int64_t prev_replica_num; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (!member.is_valid() || - false == is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(member), K(new_replica_num), K(timeout_us)); - } else if (OB_FAIL(config_mgr_.get_replica_num(prev_replica_num))) { - PALF_LOG(WARN, "get prev_replica_num failed", KR(ret), KPC(this)); - } else { - LogConfigChangeArgs args(member, new_replica_num, REMOVE_MEMBER); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "remove_member failed", KR(ret), KPC(this), K(member), K(new_replica_num)); - } else { - PALF_EVENT("remove_member success", palf_id_, KR(ret), KPC(this), K(member), K(new_replica_num)); - report_remove_member_(prev_replica_num, new_replica_num, member); - } + if (OB_FAIL(state_mgr_.set_scan_disk_log_finished())) { + PALF_LOG(WARN, "set_scan_disk_log_finished failed", K(ret), KPC(this)); } return ret; } -int PalfHandleImpl::replace_member( - const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) +int PalfHandleImpl::get_access_mode_ref_scn(AccessMode &access_mode, + SCN &ref_scn) const { int ret = OB_SUCCESS; + RLockGuard guard(lock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (!added_member.is_valid() || - !removed_member.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(added_member), K(removed_member), - K(timeout_us)); - } else { - ObMemberList old_member_list, curr_member_list; - int64_t old_replica_num = -1, curr_replica_num = -1; - LogConfigChangeArgs args(added_member, 0, config_version, ADD_MEMBER_AND_NUM); - const int64_t begin_time_us = common::ObTimeUtility::current_time(); - if (OB_FAIL(config_mgr_.get_curr_member_list(old_member_list, old_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); - } else if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "add_member in replace_member failed", KR(ret), KPC(this), K(args)); - } else if (FALSE_IT(args.server_ = removed_member)) { - } else if (FALSE_IT(args.type_ = REMOVE_MEMBER_AND_NUM)) { - } else if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - common::ObTimeUtility::current_time()))) { - if (palf_reach_time_interval(100 * 1000, replace_member_print_time_us_)) { - PALF_LOG(WARN, "remove_member in replace_member failed", KR(ret), K(args), KPC(this)); - } - } else if (OB_FAIL(config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); - } else { - PALF_EVENT("replace_member success", palf_id_, KR(ret), KPC(this), K(added_member), - K(removed_member), K(timeout_us), K(old_member_list), K(old_replica_num), - K(curr_member_list), K(curr_replica_num), - "leader replace_member cost time(us)", common::ObTimeUtility::current_time() - begin_time_us); - report_replace_member_(added_member, removed_member, curr_member_list, "REPLACE_MEMBER"); - } + PALF_LOG(WARN, "PalfHandleImpl is not inited", K(ret), KPC(this)); + } else if (OB_FAIL(mode_mgr_.get_access_mode_ref_scn(access_mode, ref_scn))) { + PALF_LOG(WARN, "get_access_mode_ref_scn failed", K(ret), KPC(this)); } return ret; } -int PalfHandleImpl::add_learner(const common::ObMember &added_learner, const int64_t timeout_us) +int PalfHandleImpl::alloc_palf_buffer_iterator(const LSN &offset, + PalfBufferIterator &iterator) { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!added_learner.is_valid() || timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; + auto get_file_end_lsn = [this]() { + LSN max_flushed_end_lsn; + (void)sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); + LSN committed_end_lsn; + sw_.get_committed_end_lsn(committed_end_lsn); + return MIN(committed_end_lsn, max_flushed_end_lsn); + }; + if (OB_FAIL(iterator.init(offset, get_file_end_lsn, log_engine_.get_log_storage()))) { + PALF_LOG(ERROR, "PalfBufferIterator init failed", K(ret), KPC(this)); } else { - LogConfigChangeArgs args(added_learner, 0, ADD_LEARNER); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "add_learner failed", KR(ret), KPC(this), K(args), K(timeout_us)); - } else { - PALF_EVENT("add_learner success", palf_id_, K(ret), KPC(this), K(args), K(timeout_us)); - report_add_learner_(added_learner); - } } return ret; } -int PalfHandleImpl::remove_learner(const common::ObMember &removed_learner, const int64_t timeout_us) +int PalfHandleImpl::alloc_palf_buffer_iterator(const SCN &scn, + PalfBufferIterator &iterator) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!removed_learner.is_valid() || timeout_us <= 0) { + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - } else { - LogConfigChangeArgs args(removed_learner, 0, REMOVE_LEARNER); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "remove_learner failed", KR(ret), KPC(this), K(args), K(timeout_us)); - } else { - PALF_EVENT("remove_learner success", palf_id_, K(ret), KPC(this), K(args), K(timeout_us)); - report_remove_learner_(removed_learner); - } - } + PALF_LOG(WARN, "invalid argument", KR(ret), K(scn)); + } else if (OB_FAIL(alloc_iterator_from_scn_(scn, iterator))) { + PALF_LOG(WARN, "alloc_iterator_from_scn_ failed", KR(ret), K(scn)); + } else {} return ret; } -int PalfHandleImpl::switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) +int PalfHandleImpl::alloc_palf_group_buffer_iterator(const LSN &offset, + PalfGroupBufferIterator &iterator) { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!learner.is_valid() || - !is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KPC(this), K(learner), K(timeout_us)); + auto get_file_end_lsn = [&]() { + LSN max_flushed_end_lsn; + (void)sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); + LSN committed_end_lsn; + sw_.get_committed_end_lsn(committed_end_lsn); + return MIN(committed_end_lsn, max_flushed_end_lsn); + }; + if (OB_FAIL(iterator.init(offset, get_file_end_lsn, log_engine_.get_log_storage()))) { + PALF_LOG(ERROR, "PalfGroupBufferIterator init failed", K(ret), KPC(this)); } else { - LogConfigChangeArgs args(learner, new_replica_num, config_version, SWITCH_LEARNER_TO_ACCEPTOR); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "switch_learner_to_acceptor failed", KR(ret), KPC(this), K(args), K(timeout_us)); - } else { - PALF_EVENT("switch_learner_to_acceptor success", palf_id_, K(ret), KPC(this), K(args), K(timeout_us)); - report_switch_learner_to_acceptor_(learner); - } } return ret; } -int PalfHandleImpl::switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) +int PalfHandleImpl::alloc_palf_group_buffer_iterator(const SCN &scn, + PalfGroupBufferIterator &iterator) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!member.is_valid() || - !is_valid_replica_num(new_replica_num) || - timeout_us <= 0) { + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - } else { - LogConfigChangeArgs args(member, new_replica_num, SWITCH_ACCEPTOR_TO_LEARNER); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "switch_acceptor_to_learner failed", KR(ret), KPC(this), K(args), K(timeout_us)); - } else { - PALF_EVENT("switch_acceptor_to_learner success", palf_id_, K(ret), KPC(this), K(args), K(timeout_us)); - report_switch_acceptor_to_learner_(member); - } - } + PALF_LOG(WARN, "invalid argument", KR(ret), K(scn)); + } else if (OB_FAIL(alloc_iterator_from_scn_(scn, iterator))) { + PALF_LOG(WARN, "alloc_iterator_from_scn_ failed", KR(ret), K(scn)); + } else {} return ret; } -int PalfHandleImpl::replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) +int PalfHandleImpl::register_file_size_cb(palf::PalfFSCbNode *fs_cb) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!added_learners.is_valid() || !removed_learners.is_valid() || timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(added_learners), K(removed_learners), K(timeout_us)); } else { - LogConfigChangeArgs args(added_learners, removed_learners, REPLACE_LEARNERS); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "replace_learners failed", KR(ret), KPC(this), K(args), K(timeout_us)); + if (OB_FAIL(fs_cb_wrapper_.add_cb_impl(fs_cb))) { + PALF_LOG(WARN, "add_file_size_cb_impl failed", K(ret), KPC(this), KPC(fs_cb)); } else { - PALF_EVENT("replace_learners success", palf_id_, K(ret), KPC(this), K(args), K(timeout_us)); - report_replace_learners_(added_learners, removed_learners); + PALF_LOG(INFO, "register_file_size_cb success", KPC(this)); } } return ret; } -ERRSIM_POINT_DEF(ERRSIM_REPLACE_MEMBER_NOT_REMOVE_ERROR); -int PalfHandleImpl::replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const palf::LogConfigVersion &config_version, - const int64_t timeout_us) +int PalfHandleImpl::unregister_file_size_cb(palf::PalfFSCbNode *fs_cb) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (!added_member.is_valid() || - !removed_member.is_valid() || - false == config_version.is_valid() || - timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(added_member), - K(removed_member), K(config_version), K(timeout_us)); } else { - ObMemberList old_member_list, curr_member_list; - int64_t old_replica_num = -1, curr_replica_num = -1; - LogConfigChangeArgs args(added_member, 0, config_version, SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM); - const int64_t begin_time_us = common::ObTimeUtility::current_time(); - if (OB_FAIL(config_mgr_.get_curr_member_list(old_member_list, old_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); - } else if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "add_member in replace_member_with_learner failed", KR(ret), KPC(this), K(args)); - } else if (OB_UNLIKELY(ERRSIM_REPLACE_MEMBER_NOT_REMOVE_ERROR)) { - // do nothing - } else if (FALSE_IT(args.server_ = removed_member)) { - } else if (FALSE_IT(args.type_ = REMOVE_MEMBER_AND_NUM)) { - } else if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - common::ObTimeUtility::current_time()))) { - if (palf_reach_time_interval(100 * 1000, replace_member_print_time_us_)) { - PALF_LOG(WARN, "remove_member in replace_member_with_learner failed", KR(ret), K(args), KPC(this)); - } - } else if (OB_FAIL(config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); - } else { - PALF_EVENT("replace_member_with_learner success", palf_id_, KR(ret), KPC(this), K(added_member), - K(removed_member), K(config_version), K(timeout_us), K(old_member_list), K(old_replica_num), - K(curr_member_list), K(curr_replica_num), - "leader replace_member_with_learner cost time(us)", common::ObTimeUtility::current_time() - begin_time_us); - report_replace_member_(added_member, removed_member, curr_member_list, "REPLACE_MEMBER_WITH_LEARNER"); - } + fs_cb_wrapper_.del_cb_impl(fs_cb); + PALF_LOG(INFO, "unregister_file_size_cb success", KPC(this)); } return ret; } -int PalfHandleImpl::change_access_mode(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const SCN &ref_scn) +int PalfHandleImpl::set_monitor_cb(PalfMonitorCb *monitor_cb) { int ret = OB_SUCCESS; - AccessMode prev_access_mode; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (INVALID_PROPOSAL_ID == proposal_id || - INVALID_PROPOSAL_ID == mode_version || - !ref_scn.is_valid() || - false == is_valid_access_mode(access_mode)) { - // ref_scn is reasonable only when access_mode is APPEND - // mode_version is the proposal_id of PALF when access_mode was applied + PALF_LOG(WARN, "not initted", KR(ret), KPC(this)); + } else if (OB_ISNULL(monitor_cb)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K_(self), - K(proposal_id), K(mode_version), K(access_mode), K(ref_scn)); - } else if (OB_FAIL(mode_change_lock_.trylock())) { - // require lock to protect status from concurrent change_access_mode - ret = OB_EAGAIN; - PALF_LOG(WARN, "another change_access_mode is running, try again", K(ret), K_(palf_id), - K_(self), K(proposal_id),K(access_mode), K(ref_scn)); - } else if (OB_FAIL(config_change_lock_.trylock())) { - // forbid to change access mode when reconfiguration is doing - mode_change_lock_.unlock(); - ret = OB_EAGAIN; - PALF_LOG(WARN, "reconfiguration is running, try again", K(ret), K_(palf_id), - K_(self), K(proposal_id), K(access_mode), K(ref_scn)); - } else if (OB_FAIL(mode_mgr_.get_access_mode(prev_access_mode))) { - PALF_LOG(WARN, "get old change_access mode failed", K(ret), K_(palf_id), - K_(self), K(proposal_id),K(access_mode), K(ref_scn)); + PALF_LOG(WARN, "lc_cb is NULL, can't register", KR(ret), KPC(this)); + } else if (OB_FAIL(plugins_.add_plugin(monitor_cb))) { + PALF_LOG(WARN, "add_plugin failed", KR(ret), KPC(this), KP(monitor_cb), K_(plugins)); } else { - PALF_EVENT("start change_access_mode", palf_id_, K(ret), KPC(this), - K(proposal_id), K(access_mode), K(ref_scn), K_(sw)); - ret = OB_EAGAIN; - bool first_run = true; - common::ObTimeGuard time_guard("change_access_mode", 10 * 1000); - while (OB_EAGAIN == ret) { - bool is_state_changed = false; - { - RLockGuard guard(lock_); - is_state_changed = mode_mgr_.is_state_changed(); - } - if (is_state_changed) { - WLockGuard guard(lock_); - if (first_run && proposal_id != state_mgr_.get_proposal_id()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "can not change_access_mode", K(ret), K_(palf_id), - K_(self), K(proposal_id), "palf proposal_id", state_mgr_.get_proposal_id()); - } else if (OB_SUCC(mode_mgr_.change_access_mode(mode_version, access_mode, ref_scn))) { - PALF_LOG(INFO, "change_access_mode success", K(ret), K_(palf_id), K_(self), K(proposal_id), - K(mode_version), K(access_mode), K(ref_scn)); - } else if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "change_access_mode failed", K(ret), K_(palf_id), K_(self), K(proposal_id), - K(mode_version), K(access_mode), K(ref_scn)); - } - first_run = false; - time_guard.click("do_once_change"); - } - // do role_change after change_access_mode success - if (OB_SUCC(ret)) { - RLockGuard guard(lock_); - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = role_change_cb_wrpper_.on_role_change(palf_id_))) { - PALF_LOG(WARN, "on_role_change failed", K(tmp_ret), K_(palf_id), K_(self)); - } - PALF_EVENT("change_access_mode success", palf_id_, K(ret), KPC(this), - K(proposal_id), K(access_mode), K(ref_scn), K(time_guard), K_(sw)); - int64_t curr_mode_version; - mode_mgr_.get_mode_version(curr_mode_version); - PALF_REPORT_INFO_KV(K(proposal_id), K(ref_scn)); - plugins_.record_access_mode_change_event(palf_id_, mode_version, curr_mode_version, prev_access_mode, access_mode, EXTRA_INFOS); - } - if (OB_EAGAIN == ret) { - ob_usleep(1000); - } - } - PALF_EVENT("change_access_mode finish", palf_id_, K(ret), KPC(this), - K(proposal_id), K(access_mode), K(ref_scn), K(time_guard), K_(sw)); - config_change_lock_.unlock(); - mode_change_lock_.unlock(); + PALF_LOG(INFO, "set_monitor_cb success", KPC(this), K_(plugins), KP(monitor_cb)); } return ret; } -int PalfHandleImpl::check_args_and_generate_config_(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - bool &is_already_finished, - LogConfigInfoV2 &new_config_info) const +int PalfHandleImpl::reset_monitor_cb() { int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.check_args_and_generate_config(args, proposal_id, - election_epoch, is_already_finished, new_config_info))) { - if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { - PALF_LOG(WARN, "check_args_and_generate_config failed", K(ret), KPC(this), K(args)); - } - } else { } + PalfMonitorCb *monitor_cb = NULL; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(plugins_.del_plugin(monitor_cb))) { + PALF_LOG(WARN, "del_plugin failed", KR(ret), KPC(this), K_(plugins)); + } return ret; } -int PalfHandleImpl::wait_log_barrier_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - TimeoutChecker ¬_timeout) +int PalfHandleImpl::check_and_switch_freeze_mode() { int ret = OB_SUCCESS; - bool has_renew_barrier = false; - while (OB_SUCC(ret) && OB_SUCC(not_timeout())) { - bool need_wlock = !has_renew_barrier; - bool need_rlock = !need_wlock; - if (DEGRADE_ACCEPTOR_TO_LEARNER != args.type_ && - true == ATOMIC_LOAD(&has_higher_prio_config_change_)) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "reconfiguration is interrupted, try again", K(ret), - K_(palf_id), K_(self), K(args)); - break; - } - if (true == need_wlock) { - WLockGuard guard(lock_); - // if the reconfiguration request do not change memberlist or replica_num, - // it's safe to commit logs after the reconfiguration barrier before the - // reconfiguration log is committed. - const bool do_not_change_quorum = is_must_not_change_replica_num(args.type_); - if (OB_FAIL(config_mgr_.renew_config_change_barrier())) { - PALF_LOG(WARN, "renew_config_change_barrier failed", KR(ret), KPC(this), K(args)); - } else if (false == do_not_change_quorum && - OB_FAIL(state_mgr_.set_changing_config_with_arb())) { - PALF_LOG(WARN, "set_changing_config_with_arb failed", KR(ret), KPC(this), K(args)); - } else { - has_renew_barrier = true; - } - } else if (true == need_rlock) { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.wait_log_barrier(args, new_config_info)) && OB_EAGAIN != ret) { - PALF_LOG(WARN, "wait_log_barrier_ failed", KR(ret), KPC(this), K(args)); - } else if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } else { - break; - } - } - if (OB_SUCC(ret) && true == need_rlock) { - ob_usleep(10 * 1000); - } + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + RLockGuard guard(lock_); + sw_.check_and_switch_freeze_mode(); } return ret; } -int PalfHandleImpl::one_stage_config_change_(const LogConfigChangeArgs &args, - const int64_t timeout_us) +bool PalfHandleImpl::is_in_period_freeze_mode() const { - int ret = OB_SUCCESS; - bool doing_degrade = false; - int64_t proposal_id = INVALID_PROPOSAL_ID; - int64_t election_epoch = INVALID_PROPOSAL_ID; - bool is_already_finished = false; - int get_lock = OB_EAGAIN; - LogConfigInfoV2 new_config_info; - if (DEGRADE_ACCEPTOR_TO_LEARNER == args.type_) { - // for concurrent DEGRADE - if (ATOMIC_BCAS(&has_higher_prio_config_change_, false, true)) { - get_lock = config_change_lock_.lock(); - } - } else if (false == ATOMIC_LOAD(&has_higher_prio_config_change_)) { - get_lock = config_change_lock_.trylock(); - } - LogConfigVersion config_version; - if (OB_SUCCESS != get_lock) { - ret = OB_EAGAIN; - if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { - PALF_LOG(WARN, "another config_change is running, try again", KR(ret), KPC(this), K(args), K(timeout_us)); - } - } else if (!args.is_valid() || timeout_us <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(args)); - // a degrade operation will interrupt startworking log running in background - } else if (OB_FAIL(config_mgr_.start_change_config(proposal_id, election_epoch, args.type_))) { - PALF_LOG(WARN, "start_change_config failed", KR(ret), KPC(this), K(args)); - } else if (FALSE_IT(doing_degrade = (args.type_ == DEGRADE_ACCEPTOR_TO_LEARNER))) { - } else if (OB_FAIL(check_args_and_generate_config_(args, proposal_id, election_epoch, - is_already_finished, new_config_info))) { - if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { - PALF_LOG(WARN, "check_args_and_generate_config failed", KR(ret), KPC(this), K(args)); - } - } else if (is_already_finished) { - if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { - PALF_LOG(INFO, "one_stage_config_change has already finished", K(ret), KPC(this), K(args)); - } - } else { - PALF_LOG(INFO, "one_stage_config_change start", KPC(this), K(proposal_id), K(args), K(timeout_us)); - TimeoutChecker not_timeout(timeout_us); - ObTimeGuard time_guard("config_change"); - // step 1: pre sync config log when adding member - // The reason for this is when a new member is created and is going to add Paxos Group. - // Its member_list is empty, so it cann't vote for any leader candidate. - // If we just add D to (A, B, C) without pre syncing config log, if leader A crashes after - // adding D, and member_list of D if still empty, D cann't vote for any one, therefore no one - // can be elected to be leader. - if (is_add_member_list(args.type_)) { - RLockGuard guard(lock_); - (void) config_mgr_.pre_sync_config_log_and_mode_meta(args.server_, proposal_id); - } - // step 2: config change remote precheck - // Eg. 1. remove C from (A, B, C), A is leader. If log of B is far behind majority, we should not remove C, - // otherwise paxos group will become unavailable - // Eg. 2. add D to (A, B, C), A is leader. If log of C and D is far behind majority (A, B), we should not add D, - // otherwise paxos group will become unavailable - while (is_may_change_replica_num(args.type_) && OB_SUCC(ret) && OB_SUCC(not_timeout())) { - bool added_member_has_new_version = true; - const int64_t curr_proposal_id = state_mgr_.get_proposal_id(); - if (DEGRADE_ACCEPTOR_TO_LEARNER != args.type_ && true == ATOMIC_LOAD(&has_higher_prio_config_change_)) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "reconfiguration is interrupted, try again", K(ret), K_(palf_id), K_(self), K(args)); - } else if (proposal_id != curr_proposal_id) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "leader has been switched, try to change config again", KR(ret), KPC(this), - K(proposal_id), K(curr_proposal_id)); - } else if (OB_SUCC(config_mgr_.check_follower_sync_status(args, new_config_info, - added_member_has_new_version))) { - // check log synchronization progress of new memberlist majority synchronically - break; - } else if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "check_follower_sync_status_ fails", K(ret), K_(palf_id), K(new_config_info)); - } else if (is_upgrade_or_degrade(args.type_)) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "degrade/upgrade eagain, arb_reason: check_follower_sync_status_ return false", - K(ret), K_(palf_id), K_(self), K(args)); - } else { - if (false == added_member_has_new_version) { - RLockGuard guard(lock_); - (void) config_mgr_.pre_sync_config_log_and_mode_meta(args.server_, proposal_id); - } - ret = OB_SUCCESS; - ob_usleep(100 * 1000); - } - } - time_guard.click("precheck"); - // step 4: check whether the new config info can be set to the election module - while (OB_SUCCESS == ret && OB_SUCC(not_timeout())) { - { - RLockGuard guard(lock_); - LogConfigVersion config_version; - if (OB_FAIL(config_mgr_.get_config_version(config_version))) { - PALF_LOG(WARN, "get_config_version failed", KR(ret), KPC(this), K(config_version)); - } else if (OB_FAIL(config_version.inc_update_version(proposal_id))) { - PALF_LOG(WARN, "inc_update_version failed", KR(ret), KPC(this), K(config_version)); - } else if (OB_FAIL(election_.can_set_memberlist(config_version))) { - ret = (OB_OP_NOT_ALLOW == ret)? OB_SUCCESS: ret; - } else { - break; - } - } - ob_usleep(50 * 1000); - } - time_guard.click("wait_ele"); - // step 5: waiting for log barrier if a arbitration member exists - if (OB_SUCC(ret) && true == new_config_info.config_.arbitration_member_.is_valid()) { - ret = wait_log_barrier_(args, new_config_info, not_timeout); - } - time_guard.click("wait_barrier"); - // step 6: motivate reconfiguration - while (OB_SUCCESS == ret && OB_SUCC(not_timeout())) { - bool need_wlock = false; - bool need_rlock = false; - bool can_be_interrupted = !config_version.is_valid(); - { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.is_state_changed(need_rlock, need_wlock))) { - PALF_LOG(WARN, "is_state_changed failed", KR(ret), KPC(this), K(need_wlock), K(need_wlock)); - } - } - // higher priority request arrives and can be interrupted. - // 1. the reason: 2F1A, reconfiguration requests will not change member list until new committed_end_lsn - // reaches log barrier. But if one F crashes, leader will not exit until timeout reaches, - // degrade operation may get stuck. So normal reconfiguration can be interrupted by degrade operation. - // 2. the condition: if config_version is invalid, that means config meta hasn't been changed, - // can be interrupted - if (DEGRADE_ACCEPTOR_TO_LEARNER != args.type_ && - true == ATOMIC_LOAD(&has_higher_prio_config_change_) && - true == can_be_interrupted) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "reconfiguration is interrupted, try again", K(ret), K_(palf_id), K_(self), K(args)); - break; - } - if (false == need_rlock && false == need_wlock) { - const int64_t SLEEP_US = (state_mgr_.is_changing_config_with_arb())? 1 * 1000: 50 * 1000; - ob_usleep(SLEEP_US); - } - if (true == need_wlock) { - WLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.change_config(args, proposal_id, election_epoch, config_version)) && OB_EAGAIN != ret) { - PALF_LOG(WARN, "change_config failed", KR(ret), KPC(this), K(args), K(proposal_id), K(election_epoch), K(config_version)); - } else if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } else { - break; - } - } else if (true == need_rlock) { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.change_config(args, proposal_id, election_epoch, config_version)) && OB_EAGAIN != ret) { - PALF_LOG(WARN, "change_config failed", KR(ret), KPC(this), K(args), K(proposal_id), K(election_epoch), K(config_version)); - } else if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } else { - break; - } - } - } - time_guard.click("reconfigure"); - PALF_LOG(INFO, "one_stage_config_change finish", KR(ret), KPC(this), K(args), K(config_version), - K(timeout_us), K(time_guard)); - ret = (OB_LOG_NOT_SYNC == ret)? OB_EAGAIN: ret; - if (OB_FAIL(ret)) { - if (config_version.is_valid()) { - config_mgr_.after_config_change_timeout(config_version); - } else { - // encounter unexpected error, reset flag - WLockGuard guard(lock_); - state_mgr_.reset_changing_config_with_arb(); - } - } - } - if (OB_SUCCESS == get_lock) { - config_change_lock_.unlock(); - if (DEGRADE_ACCEPTOR_TO_LEARNER == args.type_) { - ATOMIC_STORE(&has_higher_prio_config_change_, false); - } - } - if (doing_degrade) { - (void) config_mgr_.end_degrade(); - } - return ret; + return sw_.is_in_period_freeze_mode(); } -int PalfHandleImpl::handle_register_parent_req(const LogLearner &child, const bool is_to_leader) +int PalfHandleImpl::period_freeze_last_log() { int ret = OB_SUCCESS; - RLockGuard guard(lock_); - const bool is_leader_active = state_mgr_.is_leader_active(); - bool to_leader = is_to_leader; - common::ObMemberList member_list; - int64_t replica_num = -1; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!child.is_valid()) { - ret = OB_INVALID_ARGUMENT; - } else if (is_leader_active && OB_FAIL(config_mgr_.get_curr_member_list(member_list, replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), K_(self), K_(palf_id), K(child)); - } else if (is_leader_active && member_list.contains(child.get_server())) { - PALF_LOG(INFO, "receive register_req from acceptor, ignore", K_(self), K_(palf_id), K(child)); - } else if (!is_leader_active && !state_mgr_.is_leader_reconfirm() && to_leader) { - // avoid learner sends registering req frequently when leader is reconfirming - LogLearner parent; - LogCandidateList candidate_list; - parent.server_ = self_; - parent.register_time_us_ = child.register_time_us_; - if (OB_FAIL(log_engine_.submit_register_parent_resp(child.server_, parent, candidate_list, RegisterReturn::REGISTER_NOT_MASTER))) { - PALF_LOG(WARN, "submit_register_parent_resp failed", KR(ret), K_(palf_id), K_(self), K(child)); - } } else { - if (is_leader_active && !to_leader) { - // child send register req to candidate, but the candidate has switched to leader - to_leader = true; - } - if (OB_SUCC(ret) && OB_FAIL(config_mgr_.handle_register_parent_req(child, to_leader))) { - PALF_LOG(WARN, "handle_register_parent_req failed", KR(ret), K_(self), K(child), K(is_to_leader), K(to_leader)); - } + RLockGuard guard(lock_); + sw_.period_freeze_last_log(); } - PALF_LOG(INFO, "handle_register_parent_req finished", KR(ret), K_(palf_id), K_(self), K(child), K(is_to_leader)); return ret; } -int PalfHandleImpl::handle_register_parent_resp(const LogLearner &server, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) +int PalfHandleImpl::check_and_switch_state() { int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || RegisterReturn::INVALID_REG_RET == reg_ret) { - ret = OB_INVALID_ARGUMENT; - } else if (!state_mgr_.is_follower_active()) { - ret = OB_STATE_NOT_MATCH; - } else if (OB_FAIL(config_mgr_.handle_register_parent_resp(server, candidate_list, reg_ret))) { - PALF_LOG(WARN, "handle_register_parent_resp failed", KR(ret), K_(palf_id), K(server), K(candidate_list), K(reg_ret)); - } - return ret; -} - -int PalfHandleImpl::handle_learner_req(const LogLearner &server, const LogLearnerReqType req_type) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || LogLearnerReqType::INVALID_LEARNER_REQ_TYPE == req_type) { - ret = OB_INVALID_ARGUMENT; - } else if (LogLearnerReqType::RETIRE_PARENT == req_type && OB_FAIL(config_mgr_.handle_retire_parent(server))) { - PALF_LOG(WARN, "handle_reture_parent failed", KR(ret), K_(palf_id), K(server)); - } else if (LogLearnerReqType::RETIRE_CHILD == req_type && OB_FAIL(config_mgr_.handle_retire_child(server))) { - PALF_LOG(WARN, "handle_retire_child failed", KR(ret), K_(palf_id), K(server)); - } else if (LogLearnerReqType::KEEPALIVE_REQ == req_type && OB_FAIL(config_mgr_.handle_learner_keepalive_req(server))) { - PALF_LOG(WARN, "handle_learner_keepalive_req failed", KR(ret), K_(palf_id), K(server)); - } else if (LogLearnerReqType::KEEPALIVE_RESP == req_type && OB_FAIL(config_mgr_.handle_learner_keepalive_resp(server))) { - PALF_LOG(WARN, "handle_learner_keepalive_resp failed", KR(ret), K_(palf_id), K(server)); - } - return ret; -} - -int PalfHandleImpl::set_base_lsn( - const LSN &lsn) -{ - // NB: Guarded by lock is important, otherwise there are some problems concurrent with rebuild or migrate. - // - // Thread1(assume it's migrate thread) - // 1. if the 'base_lsn' of data source is greater than or equal to local, and then - // 2. avoid the hole between blocks, delete all blocks before 'base_lsn', submit truncate prefix blocks task - // 3. to update base lsn, submit update snpshot meta task. - // - // Execute above steps in 'advance_base_info' - // - // Thread2(checkpoint thread) - // 1. the clog disk is not enough, and update it via 'set_base_lsn' - // - // Consider that: - // - // Time1: thread1 has executed step1, assume the base lsn of data source is 100, local base lsn is 50, - // and then thread2 execute 'set_base_lsn', the local base lsn set to 150 - // - // Time2: thread1 submit truncate prefix blocks, and this task will be failed because the base lsn in this task - // is smaller than local base lsn. - // - // Therefore, we need guarded by lock. - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - LSN end_lsn = get_end_lsn(); - LogSnapshotMeta log_snapshot_meta = log_engine_.get_log_meta().get_log_snapshot_meta(); - const LSN new_base_lsn(lsn_2_block(lsn, PALF_BLOCK_SIZE) * PALF_BLOCK_SIZE); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!lsn.is_valid() || lsn > end_lsn) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(end_lsn), K(lsn)); - } else if (log_snapshot_meta.base_lsn_ >= new_base_lsn) { - PALF_LOG(WARN, "no need to set new base lsn, curr base lsn is greater than or equal to new base lsn", - KPC(this), K(log_snapshot_meta), K(new_base_lsn), K(lsn)); - } else { - FlushMetaCbCtx flush_meta_cb_ctx; - flush_meta_cb_ctx.type_ = SNAPSHOT_META; - flush_meta_cb_ctx.base_lsn_ = new_base_lsn; - if (OB_FAIL(log_snapshot_meta.generate(new_base_lsn, - log_snapshot_meta.prev_log_info_, - log_snapshot_meta.prev_log_tail_lsn_))) { - PALF_LOG(WARN, "LogSnapshotMeta generate failed", K(ret), KPC(this), K(log_snapshot_meta)); - } else if (OB_FAIL(log_engine_.submit_flush_snapshot_meta_task(flush_meta_cb_ctx, log_snapshot_meta))) { - PALF_LOG(WARN, "submit_flush_snapshot_meta_task failed", K(ret), KPC(this)); - } else { - PALF_EVENT("set_base_lsn success", palf_id_, K(ret), K_(palf_id), K(self_), K(lsn), - K(log_snapshot_meta), K(new_base_lsn), K(flush_meta_cb_ctx)); - plugins_.record_set_base_lsn_event(palf_id_, new_base_lsn); - } - } - return ret; -} - -bool PalfHandleImpl::is_sync_enabled() const -{ - bool bool_ret = false; - if (IS_NOT_INIT) { - } else { - bool_ret = state_mgr_.is_sync_enabled(); - } - return bool_ret; -} - -int PalfHandleImpl::enable_sync() -{ - int ret = OB_SUCCESS; - WLockGuard guard(lock_); - bool is_sync_enabled = state_mgr_.is_sync_enabled(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (is_sync_enabled) { - PALF_LOG(INFO, "sync has been enabled", KPC(this)); - } else if (OB_FAIL(state_mgr_.enable_sync())) { - PALF_LOG(WARN, "enable_sync failed", K(ret), KPC(this)); - } else { - PALF_EVENT("enable_sync success", palf_id_, K(ret), KPC(this)); - plugins_.record_enable_sync_event(palf_id_); - } - return ret; -} - -int PalfHandleImpl::disable_sync() -{ - int ret = OB_SUCCESS; - WLockGuard guard(lock_); - bool is_sync_enabled = state_mgr_.is_sync_enabled(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!is_sync_enabled) { - PALF_LOG(INFO, "sync has been disabled", KPC(this)); - } else if (OB_FAIL(state_mgr_.disable_sync())) { - PALF_LOG(WARN, "disable_sync failed", K(ret), KPC(this)); - } else { - PALF_EVENT("disable_sync success", palf_id_, K(ret), KPC(this)); - plugins_.record_disable_sync_event(palf_id_); - } - return ret; -} - -bool PalfHandleImpl::is_vote_enabled() const -{ - bool bool_ret = false; - if (IS_NOT_INIT) { - } else { - bool_ret = state_mgr_.is_allow_vote(); - } - return bool_ret; -} - -/*brief:disable_vote(need_check_log_missing), this function is reenterable. - * step 1: check voting status, if already disabled, just return - * step 2: for need_check_log_missing situation, double check whether it is really necessary to rebuild - * step 3: set voting flag as false when necessary - */ -int PalfHandleImpl::disable_vote(const bool need_check_log_missing) -{ - int ret = OB_SUCCESS; - const PRIORITY_SEED_BIT new_election_inner_priority_seed = PRIORITY_SEED_BIT::SEED_IN_REBUILD_PHASE_BIT; if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - //step 1: check vote status. - bool vote_disabled = false; + bool state_changed = false; do { - common::ObMemberList memberlist; - int64_t replica_num = 0; - AccessMode access_mode; RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.get_curr_member_list(memberlist, replica_num))) { - PALF_LOG(WARN, "get_curr_member_list failed", KPC(this)); - } else if (OB_FAIL(mode_mgr_.get_access_mode(access_mode))) { - PALF_LOG(WARN, "get_access_mode failed", K(ret), KPC(this)); - } else if (state_mgr_.is_leader_active() && - (1 == replica_num || AccessMode::APPEND == access_mode)) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "can not disable_vote in leader", KPC(this), - K(memberlist), K(replica_num), K(access_mode)); - } else if (!state_mgr_.is_allow_vote()) { - PALF_LOG(INFO, "vote has already been disabled", KPC(this)); - vote_disabled = true; - } - } while(0); - - if (OB_SUCC(ret) && !vote_disabled) { - if (OB_FAIL(election_.add_inner_priority_seed_bit(new_election_inner_priority_seed)) && OB_ENTRY_EXIST != ret) { - // Because this interface is idempotent, so we need ignore err code OB_ENTRY_EXIST. - PALF_LOG(WARN, "election add_inner_priority_seed_bit for rebuild failed", KPC(this)); - // Update allow_vote flag - } else if (OB_FAIL(set_allow_vote_flag_(false, need_check_log_missing))) { - PALF_LOG(WARN, "set_allow_vote_flag failed", KPC(this)); - // rollback election priority when it encounters failure - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = election_.clear_inner_priority_seed_bit(new_election_inner_priority_seed))) { - PALF_LOG(WARN, "election clear_inner_priority_seed_bit for rebuild failed", K(tmp_ret), KPC(this)); - } - } else { - PALF_EVENT("disable_vote success", palf_id_, KPC(this), K(need_check_log_missing)); - PALF_REPORT_INFO_KV(K(need_check_log_missing)); - plugins_.record_disable_vote_event(palf_id_); + state_changed = state_mgr_.is_state_changed(); + } while (0); + if (state_changed) { + WLockGuard guard(lock_); + if (OB_FAIL(state_mgr_.switch_state())) { + PALF_LOG(WARN, "switch_state failed", K(ret)); } + PALF_LOG(TRACE, "check_and_switch_state finished", K(ret), KPC(this), K(state_changed)); } - } - return ret; -} - -int PalfHandleImpl::enable_vote() -{ - int ret = OB_SUCCESS; - const PRIORITY_SEED_BIT election_inner_priority_seed = PRIORITY_SEED_BIT::SEED_IN_REBUILD_PHASE_BIT; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - // Update allow_vote flag firstly - } else if (OB_FAIL(set_allow_vote_flag_(true, false/*no need check log misingg*/))) { - PALF_LOG(WARN, "set_allow_vote_flag failed", KPC(this)); - } else if (OB_FAIL(election_.clear_inner_priority_seed_bit(election_inner_priority_seed)) - && OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(WARN, "election clear_inner_priority_seed_bit for rebuild failed", KPC(this)); - // rollback allow_vote flag when it encounters failure - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = set_allow_vote_flag_(false, false/*no need check log misingg*/))) { - PALF_LOG(WARN, "rollback allow_vote flag failed", K(tmp_ret), KPC(this)); + if (palf_reach_time_interval(PALF_DUMP_DEBUG_INFO_INTERVAL_US, last_dump_info_time_us_)) { + RLockGuard guard(lock_); + FLOG_INFO("[PALF_DUMP]", K_(self), "[SlidingWindow]", sw_, "[StateMgr]", state_mgr_, + "[ModeMgr]", mode_mgr_, "[LogEngine]", log_engine_); + (void) sw_.report_log_task_trace(sw_.get_start_id()); } - } else { - PALF_EVENT("enable_vote success", palf_id_, KPC(this)); - plugins_.record_enable_vote_event(palf_id_); } - return ret; + return OB_SUCCESS; } -int PalfHandleImpl::set_allow_vote_flag_(const bool allow_vote, - const bool need_check_log_missing) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(replica_meta_lock_); - if (state_mgr_.is_arb_replica()) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "can not enable_vote/disable_vote in arb_member", K(ret), KPC(this)); - } else { - WLockGuard guard(lock_); - if (!allow_vote && need_check_log_missing) { - //disable_vote and need check whether log is actually missing - RebuildMetaInfo last_rebuild_meta_info; - RebuildMetaInfo rebuild_meta_info; - get_last_rebuild_meta_info_(last_rebuild_meta_info); - if (last_rebuild_meta_info.is_valid()) { - //check with local rebuild meta info - (void)gen_rebuild_meta_info_(rebuild_meta_info); - ret = (last_rebuild_meta_info == rebuild_meta_info) ? OB_SUCCESS : OB_OP_NOT_ALLOW; - PALF_LOG(INFO, "double check whether need disable_vote", K(last_rebuild_meta_info), - K(rebuild_meta_info), KPC(this)); - } else { - ret = OB_OP_NOT_ALLOW; - PALF_LOG(INFO, "maybe restart during rebuild, just return OB_OP_NOT_ALLOW", KPC(this)); - } - } - if (OB_SUCC(ret)) { - FlushMetaCbCtx flush_meta_cb_ctx; - flush_meta_cb_ctx.type_ = REPLICA_PROPERTY_META; - flush_meta_cb_ctx.allow_vote_ = allow_vote; - LogReplicaPropertyMeta replica_property_meta = log_engine_.get_log_meta().get_log_replica_property_meta(); - replica_property_meta.allow_vote_ = allow_vote; - if (OB_FAIL(log_engine_.submit_flush_replica_property_meta_task(flush_meta_cb_ctx, replica_property_meta))) { - PALF_LOG(WARN, "submit_flush_replica_property_meta_task failed", K(ret), K(flush_meta_cb_ctx), K(replica_property_meta)); - } else { - if (!allow_vote) { - //for disble_vote, modify allow_vote in memory under protection of wlock - state_mgr_.disable_vote_in_mem(); - } - } - } - } - // wait until replica_property_meta has been flushed - if (OB_SUCC(ret)) { - while(allow_vote != state_mgr_.is_allow_vote_persisted()) { - ob_throttle_usleep(500, 0, palf_id_); - } - } - return ret; -} -int PalfHandleImpl::advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) -{ - int ret = OB_SUCCESS; - const LSN new_base_lsn = palf_base_info.curr_lsn_; - const LogInfo prev_log_info = palf_base_info.prev_log_info_; - LSN last_slide_lsn, committed_end_lsn; - int64_t last_slide_log_id; - common::ObTimeGuard time_guard("advance_base_info"); - ObSpinLockGuard rebuilding_guard(rebuilding_lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!palf_base_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(palf_base_info)); - } else if (true == state_mgr_.is_sync_enabled()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "is_sync_enabled, can not advance_base_info", K(ret), K_(palf_id), K(palf_base_info), K(is_rebuild)); - } else { - while (!sw_.is_all_committed_log_slided_out(last_slide_lsn, last_slide_log_id, committed_end_lsn)) { - if (palf_reach_time_interval(1 * 1000 * 1000, wait_slide_print_time_us_)) { - PALF_LOG(INFO, "There is some log has not slided out, need wait and retry", K(ret), K_(palf_id), K_(self), - K(palf_base_info), K(is_rebuild), K(last_slide_lsn), K(last_slide_log_id), K(committed_end_lsn)); - } - ob_usleep(1000); - } - time_guard.click("wait_slide"); - // require wlock in here to ensure: - // 1. snapshot_meta.base_lsn_ cann't be updated concurrently - // 2. can not receive logs before submitting truncate prefix block task - // require wlock "after" waitting all committed logs slided out. If not, log sliding will be locked - // by wlock, so committed logs will not slide forever. - WLockGuard guard(lock_); - time_guard.click("wait_wlock"); - const LSN curr_base_lsn = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_; - assert(get_end_lsn() >= curr_base_lsn); - LogSnapshotMeta log_snapshot_meta; - FlushMetaCbCtx flush_meta_cb_ctx; - TruncatePrefixBlocksCbCtx truncate_prefix_cb_ctx(new_base_lsn); - flush_meta_cb_ctx.type_ = SNAPSHOT_META; - flush_meta_cb_ctx.base_lsn_ = new_base_lsn; - // Note: can not rebuild while a truncate operation is doing, because group_buffer may be - // truncated by LogCallback again after it has been advanced by rebuild operation. - if (false == sw_.is_allow_rebuild()) { - ret = OB_EAGAIN; - PALF_LOG(WARN, "can not advance_base_info for now, try again failed", K(ret), KPC(this), K(palf_base_info), K(is_rebuild)); - } else if (OB_FAIL(check_need_advance_base_info_(new_base_lsn, prev_log_info, is_rebuild))) { - PALF_LOG(WARN, "check_need_advance_base_info failed", K(ret), KPC(this), K(palf_base_info), K(is_rebuild)); - } else if (OB_FAIL(sw_.truncate_for_rebuild(palf_base_info))) { - // The difference from the truncate in receive_log is that the expected truncate point in this scenario is greater than the left boundary of sw - // and logs that are less than the truncate point need to be callbacked and discarded - PALF_LOG(WARN, "sw_ truncate_for_rebuild failed", K(ret), KPC(this), K(palf_base_info)); - } else if (OB_FAIL(log_snapshot_meta.generate(new_base_lsn, prev_log_info, new_base_lsn))) { - PALF_LOG(WARN, "LogSnapshotMeta generate failed", K(ret), KPC(this), K(palf_base_info)); - } else if (OB_FAIL(log_engine_.submit_flush_snapshot_meta_task(flush_meta_cb_ctx, log_snapshot_meta))) { - PALF_LOG(WARN, "submit_flush_snapshot_meta_task failed", K(ret), KPC(this), K(flush_meta_cb_ctx), K(log_snapshot_meta)); - } else if (OB_FAIL(log_engine_.submit_truncate_prefix_blocks_task(truncate_prefix_cb_ctx))) { - PALF_LOG(WARN, "submit_truncate_prefix_blocks_task failed", K(ret), KPC(this), K(truncate_prefix_cb_ctx)); - } else { - time_guard.click("sw_truncate"); - PALF_LOG(INFO, "sw_ truncate_for_rebuild success", K(ret), KPC(this), K(palf_base_info)); - } - } - PALF_EVENT("advance_base_info finished", palf_id_, K(ret), KPC(this), K(palf_base_info), K(time_guard)); - plugins_.record_advance_base_info_event(palf_id_, palf_base_info); - return ret; -} -int PalfHandleImpl::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) -{ - int ret = OB_SUCCESS; - block_id_t result_block_id = LOG_INVALID_BLOCK_ID; - result_lsn.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); - } else if (OB_UNLIKELY(!scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(scn)); - } else if (OB_FAIL(get_block_id_by_scn_(scn, result_block_id))) { - PALF_LOG(WARN, "get_block_id_by_scn_ failed", KR(ret), KPC(this), K(scn)); - } else { - } - // 2. convert block_id to lsn - if (OB_SUCC(ret)) { - result_lsn = LSN(result_block_id * PALF_BLOCK_SIZE); - inc_update_last_locate_block_scn_(result_block_id, scn); - } - return ret; -} -int PalfHandleImpl::get_block_id_by_scn_(const SCN &scn, block_id_t &result_block_id) -{ - int ret = OB_SUCCESS; - block_id_t mid_block_id = LOG_INVALID_BLOCK_ID, min_block_id = LOG_INVALID_BLOCK_ID; - block_id_t max_block_id = LOG_INVALID_BLOCK_ID; - int64_t mid_ts = OB_INVALID_TIMESTAMP; - if (OB_FAIL(get_binary_search_range_(scn, min_block_id, max_block_id, result_block_id))) { - PALF_LOG(WARN, "get_binary_search_range_ failed", KR(ret), KPC(this), K(scn)); - } else { - // 1. get lower bound lsn (result_lsn) by binary search - SCN mid_scn; - while(OB_SUCC(ret) && min_block_id <= max_block_id) { - mid_block_id = min_block_id + ((max_block_id - min_block_id) >> 1); - if (OB_FAIL(log_engine_.get_block_min_scn(mid_block_id, mid_scn))) { - PALF_LOG(WARN, "get_block_min_scn failed", KR(ret), KPC(this), K(mid_block_id)); - // OB_ERR_OUT_OF_UPPER_BOUND: this block is a empty active block, just return - // OB_ERR_OUT_OF_LOWER_BOUND: block_id_ is smaller than min_block_id, this block may be recycled - // OB_ERR_UNEXPECTED: log files lost unexpectedly, just return - // OB_IO_ERROR: just return - if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { - // block mid_lsn.block_id_ may be recycled, get_binary_search_range_ again - if (OB_FAIL(get_binary_search_range_(scn, min_block_id, max_block_id, result_block_id))) { - PALF_LOG(WARN, "get_binary_search_range_ failed", KR(ret), KPC(this), K(scn)); - } - } else if (OB_ERR_OUT_OF_UPPER_BOUND == ret) { - ret = OB_ENTRY_NOT_EXIST; - } - } else if (mid_scn < scn) { - min_block_id = mid_block_id; - if (max_block_id == min_block_id) { - result_block_id = mid_block_id; - break; - } else if (max_block_id == min_block_id + 1) { - SCN next_min_scn; - if (OB_FAIL(log_engine_.get_block_min_scn(max_block_id, next_min_scn))) { - // if fail to read next block, just return prev block lsn - ret = OB_SUCCESS; - result_block_id = mid_block_id; - } else if (scn < next_min_scn) { - result_block_id = mid_block_id; - } else { - result_block_id = max_block_id; - } - break; - } - } else if (mid_scn > scn) { - // block_id is uint64_t, so check == 0 firstly. - if (mid_block_id == 0 || mid_block_id - 1 < min_block_id) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(WARN, "scn is smaller than min scn of first block", KR(ret), KPC(this), K(min_block_id), - K(max_block_id), K(mid_block_id), K(scn), K(mid_scn)); - } else { - max_block_id = mid_block_id - 1; - } - } else { - result_block_id = mid_block_id; - break; - } - } - } - return ret; -} -void PalfHandleImpl::set_deleted() -{ - ATOMIC_STORE(&has_set_deleted_, true); - PALF_LOG(INFO, "set_deleted success", KPC(this)); -} - -// NB: 1) if min_block_id_ == max_block_id_, then block max_lsn.block_id_ may be active block -// 2) if min_block_id_ < max_block_id_, then block max_lsn.block_id_ must not be active block -// 3) if min_block_id_ > max_block_id_, cache hits -int PalfHandleImpl::get_binary_search_range_(const SCN &scn, - block_id_t &min_block_id, - block_id_t &max_block_id, - block_id_t &result_block_id) -{ - int ret = OB_SUCCESS; - result_block_id = LOG_INVALID_BLOCK_ID; - const LSN committed_lsn = get_end_lsn(); - if (OB_FAIL(log_engine_.get_block_id_range(min_block_id, max_block_id))) { - // there is no log whose scn is smaller than 'scn' now and in the future, - // return OB_ERR_OUT_OF_LOWER_BOUND. - if (OB_ENTRY_NOT_EXIST == ret && scn <= get_end_scn()) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - } - } else { - block_id_t committed_block_id = lsn_2_block(committed_lsn, PALF_BLOCK_SIZE); - max_block_id = (committed_block_id < max_block_id)? committed_block_id : max_block_id; - // optimization: cache last_locate_scn_ to shrink binary search range - SpinLockGuard guard(last_locate_lock_); - if (is_valid_block_id(last_locate_block_) && - min_block_id <= last_locate_block_ && - max_block_id >= last_locate_block_) { - if (scn < last_locate_scn_) { - max_block_id = last_locate_block_; - } else if (scn > last_locate_scn_) { - min_block_id = last_locate_block_; - } else { - result_block_id = last_locate_block_; - // result_lsn hits last_locate_block_ cache, don't need binary search - // let min_block_id > max_block_id - min_block_id = 1; - max_block_id = 0; - } - } - PALF_LOG(INFO, "get_binary_search_range_", K(ret), KPC(this), K(min_block_id), K(max_block_id), - K(result_block_id), K(committed_lsn), K(scn), K_(last_locate_scn), K_(last_locate_block)); - } - return ret; -} - -void PalfHandleImpl::inc_update_last_locate_block_scn_(const block_id_t &block_id, const SCN &scn) -{ - SpinLockGuard guard(last_locate_lock_); - if (block_id > last_locate_block_) { - last_locate_block_ = block_id; - last_locate_scn_ = scn; - } -} - -int PalfHandleImpl::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); - } else if (!lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(lsn)); - } else if (lsn < log_engine_.get_begin_lsn()) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(WARN, "lsn is too small, this block has been recycled", KR(ret), KPC(this), - K(lsn), "begin_lsn", log_engine_.get_begin_lsn()); - } else { - const LSN committed_lsn = get_end_lsn(); - LSN curr_lsn = (committed_lsn <= lsn) ? committed_lsn: lsn; - block_id_t curr_block_id = lsn_2_block(curr_lsn, PALF_BLOCK_SIZE); - if (OB_FAIL(log_engine_.get_block_min_scn(curr_block_id, result_scn))) { - // if this block is a empty active block, read prev block if exists - if (OB_ERR_OUT_OF_UPPER_BOUND == ret && - curr_block_id > 0 && - OB_FAIL(log_engine_.get_block_min_scn(curr_block_id - 1, result_scn))) { - PALF_LOG(WARN, "get_block_min_scn failed", KR(ret), KPC(this), K(curr_lsn), K(lsn)); - } - } - PALF_LOG(INFO, "locate_by_lsn_coarsely", KR(ret), KPC(this), K(lsn), K(committed_lsn), K(result_scn)); - } - return ret; -} - -int PalfHandleImpl::get_min_block_info_for_gc(block_id_t &min_block_id, SCN &max_scn) -{ - int ret = OB_SUCCESS; -// if (false == end_lsn.is_valid()) { -// ret = OB_ENTRY_NOT_EXIST; -// } - if (OB_FAIL(log_engine_.get_min_block_info_for_gc(min_block_id, max_scn))) { - PALF_LOG(WARN, "get_min_block_info_for_gc failed", K(ret), KPC(this)); - } else { - PALF_LOG(TRACE, "get_min_block_info_for_gc success", K(ret), KPC(this), K(min_block_id), K(max_scn)); - } - return ret; -} - -int PalfHandleImpl::get_min_block_id_for_gc(block_id_t &min_block_id) -{ - int ret = OB_SUCCESS; - block_id_t max_block_id = LOG_INVALID_BLOCK_ID; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(log_engine_.get_block_id_range(min_block_id, max_block_id))) { - PALF_LOG(WARN, "get_block_id_range failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::delete_block(const block_id_t &block_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(log_engine_.delete_block(block_id))) { - PALF_LOG(WARN, "delete block failed", K(ret), KPC(this), K(block_id)); - } else { - PALF_LOG(WARN, "delete block success", K(ret), KPC(this), K(block_id)); - } - return ret; -} - -int PalfHandleImpl::inner_append_log(const LSN &lsn, - const LogWriteBuf &write_buf, - const SCN &scn) -{ - int ret = OB_SUCCESS; - const int64_t begin_ts = ObTimeUtility::current_time(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl not inited", K(ret), KPC(this)); - } else if (false == lsn.is_valid() - || false == write_buf.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(lsn), K(write_buf)); - } else if (OB_FAIL(log_engine_.append_log(lsn, write_buf, scn))) { - PALF_LOG(ERROR, "LogEngine pwrite failed", K(ret), KPC(this), K(lsn), K(scn)); - } else { - const int64_t curr_size = write_buf.get_total_size(); - const int64_t accum_size = ATOMIC_AAF(&accum_write_log_size_, curr_size); - const int64_t now = ObTimeUtility::current_time(); - const int64_t time_cost = now - begin_ts; - append_cost_stat_.stat(time_cost); - if (time_cost >= 5 * 1000) { - PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "write log cost too much time", K(ret), KPC(this), - K(lsn), K(scn), "size", write_buf.get_total_size(), K(accum_size), K(time_cost)); - } - if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_accum_write_statistic_time_)) { - PALF_LOG(INFO, "[PALF STAT INNER APPEND LOG SIZE]", KPC(this), K(accum_size)); - ATOMIC_STORE(&accum_write_log_size_, 0); - } - } - return ret; -} - -int PalfHandleImpl::inner_append_log(const LSNArray &lsn_array, - const LogWriteBufArray &write_buf_array, - const SCNArray &scn_array) -{ - int ret = OB_SUCCESS; - const int64_t begin_ts = ObTimeUtility::current_time(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl not inited", K(ret), KPC(this)); - } else if (OB_FAIL(log_engine_.append_log(lsn_array, write_buf_array, scn_array))) { - PALF_LOG(ERROR, "LogEngine pwrite failed", K(ret), KPC(this), K(lsn_array), K(scn_array)); - } else { - int64_t accum_size = 0; - int64_t curr_size = 0; - int64_t lsn_array_count = lsn_array.count(); - int64_t write_buf_array_count = write_buf_array.count(); - if (0 < lsn_array_count && 0 < write_buf_array_count) { - int64_t last_log_buf_len = write_buf_array[write_buf_array_count - 1]->get_total_size(); - curr_size = lsn_array[lsn_array_count - 1].val_ - lsn_array[0].val_ + last_log_buf_len; - } - accum_size = ATOMIC_AAF(&accum_write_log_size_, curr_size); - const int64_t now = ObTimeUtility::current_time(); - const int64_t time_cost = now - begin_ts; - append_cost_stat_.stat(time_cost); - if (time_cost > 10 * 1000) { - PALF_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "write log cost too much time", K(ret), KPC(this), K(lsn_array), - K(scn_array), K(curr_size), K(accum_size), K(time_cost)); - } - if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_accum_write_statistic_time_)) { - PALF_LOG(INFO, "[PALF STAT INNER APPEND LOG SIZE]", KPC(this), K(accum_size)); - ATOMIC_STORE(&accum_write_log_size_, 0); - } - } - return ret; -} - -int PalfHandleImpl::inner_append_meta(const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl not inited"); - } else if (NULL == buf - || 0 >= buf_len) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(buf), K(buf_len)); - } else if (OB_FAIL(log_engine_.append_meta(buf, buf_len))) { - PALF_LOG(ERROR, "LogEngine append_meta failed", K(ret), KPC(this)); - } else { - } - return ret; -} - -int PalfHandleImpl::inner_truncate_log(const LSN &lsn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl not inited"); - } else if (false == lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument", K(ret), K(lsn), KPC(this)); - } else if (OB_FAIL(log_engine_.truncate(lsn))) { - PALF_LOG(ERROR, "LogEngine truncate failed", K(ret), K(lsn), KPC(this)); - } else { - PALF_LOG(INFO, "PalfHandleImpl inner_truncate_log success", K(lsn), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::inner_truncate_prefix_blocks(const LSN &lsn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl not inited"); - } else if (false == lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(lsn)); - } else if (OB_FAIL(log_engine_.truncate_prefix_blocks(lsn))) { - PALF_LOG(WARN, "LogEngine truncate_prefix_blocks failed", K(ret), KPC(this), K(lsn)); - } else { - PALF_LOG(INFO, "LogEngine truncate_prefix_blocks success", K(ret), KPC(this), K(lsn)); - } - return ret; -} - -int PalfHandleImpl::set_scan_disk_log_finished() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(state_mgr_.set_scan_disk_log_finished())) { - PALF_LOG(WARN, "set_scan_disk_log_finished failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::get_access_mode(AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl is not inited", K(ret), KPC(this)); - } else if (OB_FAIL(mode_mgr_.get_access_mode(access_mode))) { - PALF_LOG(WARN, "get_access_mode failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::get_access_mode_version(int64_t &mode_version) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl is not inited", K(ret), KPC(this)); - } else if (OB_FAIL(mode_mgr_.get_mode_version(mode_version))) { - PALF_LOG(WARN, "get_mode_version failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::get_access_mode(int64_t &mode_version, AccessMode &access_mode) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl is not inited", K(ret), KPC(this)); - } else if (OB_FAIL(mode_mgr_.get_access_mode(mode_version, access_mode))) { - PALF_LOG(WARN, "get_access_mode failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, - SCN &ref_scn) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl is not inited", K(ret), KPC(this)); - } else if (OB_FAIL(mode_mgr_.get_access_mode_ref_scn(mode_version, access_mode, ref_scn))) { - PALF_LOG(WARN, "get_access_mode_ref_scn failed", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::alloc_palf_buffer_iterator(const LSN &offset, - PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - auto get_file_end_lsn = [this]() { - LSN max_flushed_end_lsn; - (void)sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); - LSN committed_end_lsn; - sw_.get_committed_end_lsn(committed_end_lsn); - return MIN(committed_end_lsn, max_flushed_end_lsn); - }; - auto get_mode_version = [this]() -> int64_t { - int64_t mode_version = INVALID_PROPOSAL_ID; - int ret = OB_SUCCESS; - if (OB_FAIL(this->get_access_mode_version(mode_version))) { - PALF_LOG(WARN, "get_access_mode_version failed", K(ret), KPC(this)); - mode_version = INVALID_PROPOSAL_ID; - } - return mode_version; - }; - if (OB_FAIL(iterator.init(offset, get_file_end_lsn, get_mode_version, log_engine_.get_log_storage()))) { - PALF_LOG(ERROR, "PalfBufferIterator init failed", K(ret), KPC(this)); - } else { - } - return ret; -} - -int PalfHandleImpl::alloc_palf_buffer_iterator(const SCN &scn, - PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K(scn)); - } else if (OB_FAIL(alloc_iterator_from_scn_(scn, iterator))) { - PALF_LOG(WARN, "alloc_iterator_from_scn_ failed", KR(ret), K_(palf_id), K(scn)); - } else {} - return ret; -} - -int PalfHandleImpl::alloc_palf_group_buffer_iterator(const LSN &offset, - PalfGroupBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - auto get_file_end_lsn = [&]() { - LSN max_flushed_end_lsn; - (void)sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); - LSN committed_end_lsn; - sw_.get_committed_end_lsn(committed_end_lsn); - return MIN(committed_end_lsn, max_flushed_end_lsn); - }; - auto get_mode_version = [this]() -> int64_t { - int64_t mode_version = INVALID_PROPOSAL_ID; - int ret = OB_SUCCESS; - if (OB_FAIL(this->get_access_mode_version(mode_version))) { - PALF_LOG(WARN, "get_access_mode_version failed", K(ret), KPC(this)); - mode_version = INVALID_PROPOSAL_ID; - } - return mode_version; - }; - if (OB_FAIL(iterator.init(offset, get_file_end_lsn, get_mode_version, log_engine_.get_log_storage()))) { - PALF_LOG(ERROR, "PalfGroupBufferIterator init failed", K(ret), KPC(this)); - } else { - } - return ret; -} - -int PalfHandleImpl::alloc_palf_group_buffer_iterator(const SCN &scn, - PalfGroupBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K(scn)); - } else if (OB_FAIL(alloc_iterator_from_scn_(scn, iterator))) { - PALF_LOG(WARN, "alloc_iterator_from_scn_ failed", KR(ret), K_(palf_id), K(scn)); - } else {} - return ret; -} - -int PalfHandleImpl::register_file_size_cb(palf::PalfFSCbNode *fs_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - if (OB_FAIL(fs_cb_wrapper_.add_cb_impl(fs_cb))) { - PALF_LOG(WARN, "add_file_size_cb_impl failed", K(ret), KPC(this), KPC(fs_cb)); - } else { - PALF_LOG(INFO, "register_file_size_cb success", KPC(this)); - } - } - return ret; -} - -int PalfHandleImpl::unregister_file_size_cb(palf::PalfFSCbNode *fs_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - fs_cb_wrapper_.del_cb_impl(fs_cb); - PALF_LOG(INFO, "unregister_file_size_cb success", KPC(this)); - } - return ret; -} - -int PalfHandleImpl::register_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - if (OB_FAIL(role_change_cb_wrpper_.add_cb_impl(role_change_cb))) { - PALF_LOG(WARN, "add_role_change_cb_impl failed", K(ret), KPC(this), KPC(role_change_cb)); - } else if (OB_FAIL(role_change_cb->rc_cb_->on_role_change(palf_id_))) { - PALF_LOG(WARN, "on_role_change failed", K(ret), KPC(this)); - } else { - PALF_LOG(INFO, "register_role_change_cb success", KPC(this)); - } - } - return ret; -} - -int PalfHandleImpl::unregister_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - role_change_cb_wrpper_.del_cb_impl(role_change_cb); - PALF_LOG(INFO, "unregister_role_change_cb success", KPC(this)); - } - return ret; -} - -int PalfHandleImpl::register_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (NULL == rebuild_cb) { - ret = OB_INVALID_ARGUMENT; - } else { - if (OB_FAIL(rebuild_cb_wrapper_.add_cb_impl(rebuild_cb))) { - PALF_LOG(WARN, "add_rebuild_cb_impl failed", K(ret), KPC(this), KPC(rebuild_cb)); - } else { - PALF_LOG(INFO, "register_rebuild_cb success", KPC(this), KP(rebuild_cb)); - } - } - return ret; -} - -int PalfHandleImpl::unregister_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - rebuild_cb_wrapper_.del_cb_impl(rebuild_cb); - PALF_LOG(INFO, "unregister_rebuild_cb success", KPC(this), KP(rebuild_cb)); - } - return ret; -} - -int PalfHandleImpl::set_location_cache_cb(PalfLocationCacheCb *lc_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "not initted", KR(ret), KPC(this)); - } else if (OB_ISNULL(lc_cb)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "lc_cb is NULL, can't register", KR(ret), KPC(this)); - } else if (OB_FAIL(plugins_.add_plugin(lc_cb))) { - PALF_LOG(WARN, "add_plugin failed", KR(ret), KPC(this), KP(lc_cb), K_(plugins)); - } else { - PALF_LOG(INFO, "set_location_cache_cb success", KPC(this), K_(plugins), KP(lc_cb)); - } - return ret; -} - -int PalfHandleImpl::set_election_priority(election::ElectionPriority *priority) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_ISNULL(priority)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "priority is NULL, can't setted", KR(ret), KPC(this)); - } else { - WLockGuard guard(lock_); - if (OB_FAIL(election_.set_priority(priority))) { - PALF_LOG(WARN, "set election priority failed", KR(ret), KPC(this)); - } - } - return ret; -} - -int PalfHandleImpl::reset_election_priority() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "palf handle not init", KR(ret), KPC(this)); - } else { - WLockGuard guard(lock_); - if (OB_FAIL(election_.reset_priority())) { - PALF_LOG(WARN, "fail to reset election priority", KR(ret), KPC(this)); - } - } - return ret; -} - -int PalfHandleImpl::reset_location_cache_cb() -{ - int ret = OB_SUCCESS; - PalfLocationCacheCb *loc_cb = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(plugins_.del_plugin(loc_cb))) { - PALF_LOG(WARN, "del_plugin failed", KR(ret), KPC(this), K_(plugins)); - } - return ret; -} - -int PalfHandleImpl::set_monitor_cb(PalfMonitorCb *monitor_cb) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "not initted", KR(ret), KPC(this)); - } else if (OB_ISNULL(monitor_cb)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "lc_cb is NULL, can't register", KR(ret), KPC(this)); - } else if (OB_FAIL(plugins_.add_plugin(monitor_cb))) { - PALF_LOG(WARN, "add_plugin failed", KR(ret), KPC(this), KP(monitor_cb), K_(plugins)); - } else { - PALF_LOG(INFO, "set_monitor_cb success", KPC(this), K_(plugins), KP(monitor_cb)); - } - return ret; -} - -int PalfHandleImpl::reset_monitor_cb() -{ - int ret = OB_SUCCESS; - PalfMonitorCb *monitor_cb = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(plugins_.del_plugin(monitor_cb))) { - PALF_LOG(WARN, "del_plugin failed", KR(ret), KPC(this), K_(plugins)); - } - return ret; -} - -int PalfHandleImpl::check_and_switch_freeze_mode() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - RLockGuard guard(lock_); - sw_.check_and_switch_freeze_mode(); - } - return ret; -} - -bool PalfHandleImpl::is_in_period_freeze_mode() const -{ - return sw_.is_in_period_freeze_mode(); -} - -int PalfHandleImpl::period_freeze_last_log() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(true == state_mgr_.is_arb_replica())) { - } else { - RLockGuard guard(lock_); - sw_.period_freeze_last_log(); - } - return ret; -} - -int PalfHandleImpl::check_and_switch_state() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - bool state_changed = false; - bool config_state_changed = false; - do { - RLockGuard guard(lock_); - state_changed = state_mgr_.is_state_changed(); - } while (0); - if (state_changed) { - WLockGuard guard(lock_); - if (OB_FAIL(state_mgr_.switch_state())) { - PALF_LOG(WARN, "switch_state failed", K(ret)); - } - PALF_LOG(TRACE, "check_and_switch_state finished", K(ret), KPC(this), K(state_changed)); - } - do { - RLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.leader_do_loop_work(config_state_changed))) { - // overwrite ret - PALF_LOG(WARN, "LogConfigMgr::leader_do_loop_work failed", KR(ret), K_(self), K_(palf_id)); - } else if (OB_FAIL(mode_mgr_.leader_do_loop_work())) { - PALF_LOG(WARN, "LogModeMgr::leader_do_loop_work failed", KR(ret), K_(self), K_(palf_id)); - } - (void) config_mgr_.forward_initial_config_meta_to_arb(); - } while (0); - if (OB_UNLIKELY(config_state_changed)) { - WLockGuard guard(lock_); - if (OB_FAIL(config_mgr_.switch_state())) { - // overwrite ret - PALF_LOG(WARN, "switch_state failed", K(ret)); - } - } - if (palf_reach_time_interval(PALF_CHECK_PARENT_CHILD_INTERVAL_US, last_check_parent_child_time_us_)) { - RLockGuard guard(lock_); - if (state_mgr_.is_follower_active()) { - (void) config_mgr_.check_parent_health(); - } - (void) config_mgr_.check_children_health(); - } - if (palf_reach_time_interval(PALF_DUMP_DEBUG_INFO_INTERVAL_US, last_dump_info_time_us_)) { - RLockGuard guard(lock_); - FLOG_INFO("[PALF_DUMP]", K_(palf_id), K_(self), "[SlidingWindow]", sw_, "[StateMgr]", state_mgr_, - "[ConfigMgr]", config_mgr_, "[ModeMgr]", mode_mgr_, "[LogEngine]", log_engine_, "[Reconfirm]", - reconfirm_); - if (false == state_mgr_.is_arb_replica()) { - LogMemberAckInfoList ack_info_list; - sw_.get_ack_info_array(ack_info_list); - FLOG_INFO("[PALF_DUMP]", K_(palf_id), K_(self), K(ack_info_list)); - } - (void) sw_.report_log_task_trace(sw_.get_start_id()); - } - } - return OB_SUCCESS; -} - -int PalfHandleImpl::handle_prepare_request(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - bool can_handle_prepare_request = false; - bool can_handle_leader_broadcast = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(server), K(proposal_id)); - } else { - RLockGuard guard(lock_); - if (state_mgr_.can_handle_prepare_request(proposal_id)) { - can_handle_prepare_request = true; - } else if (state_mgr_.can_handle_leader_broadcast(server, proposal_id)) { - can_handle_leader_broadcast = true; - } - } - if (OB_FAIL(ret)) { - } else if (can_handle_prepare_request) { - WLockGuard guard(lock_); - if (!state_mgr_.can_handle_prepare_request(proposal_id)) { - // can not handle prepare request - } else if (OB_FAIL(state_mgr_.handle_prepare_request(server, proposal_id))) { - PALF_LOG(WARN, "handle_prepare_request failed", K(ret), KPC(this), K(server), K(proposal_id)); - } else { - // Call clean_log() when updating proposal_id to delete phantom logs(if it exists). - (void) sw_.clean_log(); - PALF_LOG(INFO, "handle_prepare_request success", K(ret), KPC(this), K(server), K_(self), K(proposal_id)); - } - } else if (can_handle_leader_broadcast) { - WLockGuard guard(lock_); - if (!state_mgr_.can_handle_leader_broadcast(server, proposal_id)) { - // can not handle leader broadcast - } else if (OB_FAIL(state_mgr_.handle_leader_broadcast(server, proposal_id))) { - PALF_LOG(WARN, "handle_leader_broadcast failed", K(ret), KPC(this), K(server), K(proposal_id)); - } else { - PALF_LOG(TRACE, "handle_leader_broadcast success", K(ret), KPC(this), K(server), K(proposal_id)); - } - } - PALF_LOG(TRACE, "handle_prepare_request", K(ret), KPC(this), K(server), - K(can_handle_prepare_request), K(can_handle_leader_broadcast)); - return ret; -} - -int PalfHandleImpl::handle_prepare_response(const common::ObAddr &server, - const int64_t &proposal_id, - const bool vote_granted, - const int64_t &accept_proposal_id, - const LSN &last_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &log_mode_meta) -{ - int ret = OB_SUCCESS; - bool can_handle_prepare_resp = true; - int64_t curr_proposal_id = INVALID_PROPOSAL_ID; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(server), K(proposal_id)); - } else { - RLockGuard guard(lock_); - curr_proposal_id = state_mgr_.get_proposal_id(); - if (!state_mgr_.can_handle_prepare_response(proposal_id)) { - can_handle_prepare_resp = false; - PALF_LOG(WARN, "cannot handle parepare response", K(ret), KPC(this), K(server), - K(proposal_id), K(curr_proposal_id)); - } - } - if (OB_SUCC(ret)) { - if (!can_handle_prepare_resp) { - // check proposal_id - if (proposal_id > curr_proposal_id) { - WLockGuard guard(lock_); - if (!state_mgr_.can_handle_prepare_request(proposal_id)) { - // can not handle prepare request - } else if (OB_FAIL(state_mgr_.handle_prepare_request(server, proposal_id))) { - PALF_LOG(WARN, "handle_prepare_request failed", K(ret), KPC(this)); - } else { - // Call clean_log() when updating proposal_id to delete phantom logs(if it exists). - (void) sw_.clean_log(); - } - } - } else if (vote_granted) { - // server grant vote for me, process preapre response - RLockGuard guard(lock_); - if (OB_FAIL(mode_mgr_.handle_prepare_response(server, proposal_id, accept_proposal_id, - last_lsn, log_mode_meta))) { - PALF_LOG(WARN, "log_mode_mgr.handle_prepare_response failed", K(ret), KPC(this), K(proposal_id), - K(accept_proposal_id), K(last_lsn), K(log_mode_meta)); - } else if (OB_FAIL(reconfirm_.handle_prepare_response(server, proposal_id, accept_proposal_id, - last_lsn, committed_end_lsn))) { - PALF_LOG(WARN, "reconfirm.handle_prepare_response failed", K(ret), KPC(this), - K(proposal_id), K(accept_proposal_id), K(last_lsn)); - } - } else { - // do nothing - } - } - return ret; -} - -int PalfHandleImpl::receive_mode_meta(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - int lock_ret = OB_EAGAIN; - bool has_accepted = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == server.is_valid() || - INVALID_PROPOSAL_ID == proposal_id || - false == mode_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), KPC(this), K(server), K(proposal_id), K(mode_meta)); - } else if (OB_FAIL(try_update_proposal_id_(server, proposal_id))) { - PALF_LOG(WARN, "try_update_proposal_id_ failed", KR(ret), KPC(this), K(server), K(proposal_id)); - } else if (OB_SUCCESS != (lock_ret = lock_.wrlock())) { - } else if (false == mode_mgr_.can_receive_mode_meta(proposal_id, mode_meta, has_accepted)) { - PALF_LOG(WARN, "can_receive_mode_meta failed", KR(ret), KPC(this), K(proposal_id), K(mode_meta)); - } else if (true == has_accepted) { - if (OB_FAIL(log_engine_.submit_change_mode_meta_resp(server, proposal_id))) { - PALF_LOG(WARN, "submit_change_mode_meta_resp failed", KR(ret), KPC(this), K(proposal_id), K(mode_meta)); - } - if (true == is_applied_mode_meta) { - // update LogModeMgr::applied_mode_meta requires wlock - (void) mode_mgr_.after_flush_mode_meta(is_applied_mode_meta, mode_meta); - } - } else if (OB_FAIL(mode_mgr_.receive_mode_meta(server, proposal_id, is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "receive_mode_meta failed", KR(ret), KPC(this), K(server), K(proposal_id), - K(mode_meta)); - } else { } - PALF_LOG(INFO, "receive_mode_meta finish", KR(ret), KPC(this), K(server), K(proposal_id), - K(is_applied_mode_meta), K(mode_meta)); - if (OB_SUCCESS == lock_ret) { - lock_.wrunlock(); - } - return ret; -} - -int PalfHandleImpl::ack_mode_meta(const common::ObAddr &server, - const int64_t msg_proposal_id) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - const int64_t &curr_proposal_id = state_mgr_.get_proposal_id(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); - } else if (msg_proposal_id != curr_proposal_id) { - PALF_LOG(WARN, "proposal_id does not match", KR(ret), KPC(this), K(msg_proposal_id), - K(server), K(curr_proposal_id)); - } else if (self_ != state_mgr_.get_leader()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "self is not leader, state not match", KR(ret), KPC(this), - K(server), K(msg_proposal_id), K(curr_proposal_id)); - } else if (OB_FAIL(mode_mgr_.ack_mode_meta(server, msg_proposal_id))) { - PALF_LOG(WARN, "ack_mode_meta failed", KR(ret), KPC(this), K(server), K(msg_proposal_id)); - } else { - PALF_LOG(TRACE, "ack_mode_meta success", KR(ret), KPC(this), K(server), K(msg_proposal_id)); - } - return ret; -} - -int PalfHandleImpl::do_init_mem_( - const int64_t palf_id, - const PalfBaseInfo &palf_base_info, - const LogMeta &log_meta, - const char *log_dir, - const common::ObAddr &self, - FetchLogEngine *fetch_log_engine, - ObILogAllocator *alloc_mgr, - LogRpc *log_rpc, - IPalfEnvImpl *palf_env_impl) -{ - int ret = OB_SUCCESS; - int pret = -1; - const bool is_normal_replica = (log_meta.get_log_replica_property_meta().replica_type_ == NORMAL_REPLICA); - // inner priority seed: smaller means higher priority - // reserve some bits for future requirements - uint64_t election_inner_priority_seed = is_normal_replica ? - static_cast(PRIORITY_SEED_BIT::DEFAULT_SEED) : - 0ULL | static_cast(PRIORITY_SEED_BIT::SEED_NOT_NORMOL_REPLICA_BIT); - const bool allow_vote = log_meta.get_log_replica_property_meta().allow_vote_; - if (false == allow_vote) { - election_inner_priority_seed |= static_cast(PRIORITY_SEED_BIT::SEED_IN_REBUILD_PHASE_BIT); - } - palf::PalfRoleChangeCbWrapper &role_change_cb_wrpper = role_change_cb_wrpper_; - if ((pret = snprintf(log_dir_, MAX_PATH_SIZE, "%s", log_dir)) && false) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "error unexpected", K(ret), K(palf_id)); - } else if (OB_FAIL(sw_.init(palf_id, self, &state_mgr_, &config_mgr_, &mode_mgr_, - &log_engine_, &fs_cb_wrapper_, alloc_mgr, &plugins_, palf_base_info, is_normal_replica))) { - PALF_LOG(WARN, "sw_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(election_.init_and_start(palf_id, - self, - election_inner_priority_seed, - 1, - [&role_change_cb_wrpper](int64_t id, - const ObAddr &dest_addr){ - return role_change_cb_wrpper.on_need_change_leader(id, dest_addr); - }))) { - PALF_LOG(WARN, "election_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(log_cache_.init(palf_id, this))) { - PALF_LOG(WARN, "log_cache_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(state_mgr_.init(palf_id, self, log_meta.get_log_prepare_meta(), log_meta.get_log_replica_property_meta(), - &election_, &sw_, &reconfirm_, &log_engine_, &config_mgr_, &mode_mgr_, &role_change_cb_wrpper_, &plugins_))) { - PALF_LOG(WARN, "state_mgr_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(config_mgr_.init(palf_id, self, log_meta.get_log_config_meta(), &log_engine_, - &sw_, &state_mgr_, &election_, &mode_mgr_, &reconfirm_, &plugins_))) { - PALF_LOG(WARN, "config_mgr_ init failed", K(ret), K(palf_id)); - } else if (is_normal_replica && OB_FAIL(reconfirm_.init(palf_id, self, &sw_, &state_mgr_, &config_mgr_, &mode_mgr_, &log_engine_))) { - PALF_LOG(WARN, "reconfirm_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(mode_mgr_.init(palf_id, self, log_meta.get_log_mode_meta(), &state_mgr_, &log_engine_, &config_mgr_, &sw_))) { - PALF_LOG(WARN, "mode_mgr_ init failed", K(ret), K(palf_id)); - } else { - palf_id_ = palf_id; - fetch_log_engine_ = fetch_log_engine; - allocator_ = alloc_mgr; - self_ = self; - has_set_deleted_ = false; - palf_env_impl_ = palf_env_impl; - is_inited_ = true; - PALF_LOG(INFO, "PalfHandleImpl do_init_ success", K(ret), K(palf_id), K(self), K(log_dir), K(palf_base_info), - K(log_meta), K(fetch_log_engine), K(alloc_mgr), K(log_rpc)); - } - if (OB_FAIL(ret)) { - is_inited_ = true; - destroy(); - } - return ret; -} - -int PalfHandleImpl::get_palf_epoch(int64_t &palf_epoch) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - palf_epoch = log_engine_.get_palf_epoch(); - } - return ret; -} - -int PalfHandleImpl::try_update_proposal_id_(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(server), K(proposal_id)); - } else { - bool need_handle_prepare_req = false; - do { - // check if need update proposal_id - RLockGuard guard(lock_); - if (proposal_id > state_mgr_.get_proposal_id()) { - need_handle_prepare_req = true; - } - } while(0); - // try update proposal_id - if (need_handle_prepare_req) { - WLockGuard guard(lock_); - if (!state_mgr_.can_handle_prepare_request(proposal_id)) { - // can not handle prepare request - } else if (OB_FAIL(state_mgr_.handle_prepare_request(server, proposal_id))) { - PALF_LOG(WARN, "handle_prepare_request failed", K(ret), K(server), K(proposal_id)); - } else { - // Call clean_log() when updating proposal_id to delete phantom logs(if it exists). - (void) sw_.clean_log(); - PALF_LOG(INFO, "try_update_proposal_id_ finished", K(ret), K(server), K(proposal_id)); - } - } - } - return ret; -} - -int PalfHandleImpl::handle_committed_info(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == msg_proposal_id || !committed_end_lsn.is_valid() - || OB_INVALID_LOG_ID == prev_log_id || INVALID_PROPOSAL_ID == prev_log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(server), K(msg_proposal_id), K(prev_log_id), - K(prev_log_proposal_id), K(committed_end_lsn)); - } else if (OB_FAIL(try_update_proposal_id_(server, msg_proposal_id))) { - PALF_LOG(WARN, "try_update_proposal_id_ failed", K(ret), KPC(this), K(server), K(msg_proposal_id)); - } else { - RLockGuard guard(lock_); - if (!state_mgr_.can_handle_committed_info(msg_proposal_id)) { - ret = OB_STATE_NOT_MATCH; - if (palf_reach_time_interval(2 * 1000 * 1000, cannot_handle_committed_info_time_)) { - PALF_LOG(WARN, "can not handle_committed_info", K(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn), - "local proposal_id", state_mgr_.get_proposal_id(), "role", state_mgr_.get_role(), - "state", state_mgr_.get_state(), "is_sync_enabled", state_mgr_.is_sync_enabled()); - } - } else if (OB_FAIL(sw_.handle_committed_info(server, prev_log_id, prev_log_proposal_id, committed_end_lsn))) { - PALF_LOG(WARN, "handle_committed_info failed", K(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn)); - } else { - PALF_LOG(TRACE, "handle_committed_info success", K(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn)); - } - } - return ret; -} - -int PalfHandleImpl::receive_log_(const common::ObAddr &server, - const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_log_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - TruncateLogInfo truncate_log_info; -#ifdef ERRSIM - if (!GCONF.palf_inject_receive_log_error_zone.get_value_string().empty()) { - if (0 == strcmp(GCONF.zone.str(), GCONF.palf_inject_receive_log_error_zone.str())) { - ret = OB_ERROR; - LOG_WARN("palf receive log errsim", K(ret)); - return ret; - } - } -#endif - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == msg_proposal_id || !lsn.is_valid() - || NULL == buf || buf_len <= 0) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), - KP(buf), K(buf_len)); - } else if (OB_FAIL(try_update_proposal_id_(server, msg_proposal_id))) { - PALF_LOG(WARN, "try_update_proposal_id_ failed", K(ret), KPC(this), K(server), K(msg_proposal_id)); - } else { - // rdlock - RLockGuard guard(lock_); - if (false == palf_env_impl_->check_disk_space_enough()) { - ret = OB_LOG_OUTOF_DISK_SPACE; - if (palf_reach_time_interval(1 * 1000 * 1000, log_disk_full_warn_time_)) { - PALF_LOG(WARN, "log outof disk space", K(ret), KPC(this), K(server), K(push_log_type), K(lsn)); - } - } else if (!state_mgr_.can_receive_log(msg_proposal_id)) { - ret = OB_STATE_NOT_MATCH; - if (palf_reach_time_interval(2 * 1000 * 1000, cannot_recv_log_warn_time_)) { - PALF_LOG(WARN, "can not receive log", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), - "local proposal_id", state_mgr_.get_proposal_id(), "role", state_mgr_.get_role(), - "state", state_mgr_.get_state(), "is_sync_enabled", state_mgr_.is_sync_enabled()); - } - } else if (OB_FAIL(sw_.receive_log(server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, - buf, buf_len, true, truncate_log_info))) { - if(OB_EAGAIN == ret) { - // rewrite -4023 to 0 - ret = OB_SUCCESS; - } else { - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); - } - } - } else { - PALF_LOG(TRACE, "receive_log success", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), K(buf_len)); - } - } - // check if need truncate - if (OB_SUCC(ret) && (INVALID_TRUNCATE_TYPE != truncate_log_info.truncate_type_)) { - PALF_LOG(INFO, "begin clean/truncate log", K(ret), KPC(this), K(server), K_(self), K(msg_proposal_id), - K(lsn), K(buf_len), K(truncate_log_info)); - // write lock - WLockGuard guard(lock_); - if (!state_mgr_.can_receive_log(msg_proposal_id)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "can not receive log", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), - "role", state_mgr_.get_role()); - } else if (TRUNCATE_CACHED_LOG_TASK == truncate_log_info.truncate_type_) { - if (OB_FAIL(sw_.clean_cached_log(truncate_log_info.truncate_log_id_, lsn, prev_lsn, prev_log_proposal_id))) { - PALF_LOG(WARN, "sw_ clean_cached_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), - K(truncate_log_info)); - } - } else if (TRUNCATE_LOG == truncate_log_info.truncate_type_) { - if (!state_mgr_.can_truncate_log()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(ERROR, "can not truncate log", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), - "role", state_mgr_.get_role(), "state", state_mgr_.get_state()); - } else if (OB_FAIL(sw_.truncate(truncate_log_info, prev_lsn, prev_log_proposal_id))) { - PALF_LOG(WARN, "sw_ truncate failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); - } else if (OB_FAIL(state_mgr_.truncate(truncate_log_info.truncate_begin_lsn_))) { - PALF_LOG(WARN, "state_mgr_ truncate failed", K(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_lsn), K(prev_log_proposal_id), K(lsn), K(lsn), K(lsn)); - } else { - // do nothing - } - } else { - // do nothing - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(sw_.receive_log(server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, buf, - buf_len, false, truncate_log_info))) { - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); - } - } else { - PALF_LOG(INFO, "receive_log success", K(ret), KPC(this), K(server), K_(self), K(msg_proposal_id), K(prev_lsn), - K(prev_log_proposal_id), K(lsn), K(truncate_log_info)); - } - } - return ret; -} - -int PalfHandleImpl::receive_log(const common::ObAddr &server, - const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_log_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len) -{ - return receive_log_(server, push_log_type, msg_proposal_id, prev_lsn, prev_log_proposal_id, lsn, buf, buf_len); -} - -int PalfHandleImpl::receive_batch_log(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - int64_t start_ts = ObTimeUtility::current_time(); - MemoryStorage storage; - MemPalfGroupBufferIterator iterator; - auto get_file_end_lsn = [curr_lsn, buf_len] () { return curr_lsn + buf_len; }; - if (OB_FAIL(iterator.init(curr_lsn, get_file_end_lsn, &storage))) { - PALF_LOG(ERROR, "init iterator failed", K(ret), KPC(this)); - } else if (OB_FAIL(storage.init(curr_lsn))) { - PALF_LOG(ERROR, "init storage failed", K(ret), KPC(this)); - } else if (OB_FAIL(storage.append(buf, buf_len))) { - PALF_LOG(ERROR, "storage append failed", K(ret), KPC(this)); - } else { - LSN prev_lsn_each_round = prev_lsn; - int64_t prev_log_proposal_id_each_round = prev_log_proposal_id; - LSN curr_lsn_each_round = curr_lsn; - int64_t curr_log_proposal_id = 0; - const char *buf_each_round = NULL; - int64_t buf_len_each_round = 0; - int64_t count = 0, success_count = 0; - while (OB_SUCC(iterator.next())) { - if (OB_FAIL(iterator.get_entry(buf_each_round, buf_len_each_round, curr_lsn_each_round, curr_log_proposal_id))) { - PALF_LOG(ERROR, "get_entry failed", K(ret), KPC(this), K(iterator), KP(buf_each_round)); - } else if (OB_FAIL(receive_log_(server, FETCH_LOG_RESP, msg_proposal_id, prev_lsn_each_round, - prev_log_proposal_id_each_round, curr_lsn_each_round, buf_each_round, buf_len_each_round))) { - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "receive_log failed", K(ret), KPC(this), K(iterator), K(server), K(FETCH_LOG_RESP), - K(msg_proposal_id), K(prev_lsn_each_round), K(prev_log_proposal_id_each_round), - K(curr_lsn_each_round), KP(buf_each_round), K(buf_len_each_round)); - } - } else { - success_count++; - } - prev_lsn_each_round = curr_lsn_each_round; - prev_log_proposal_id_each_round = curr_log_proposal_id; - count++; - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - int64_t cost_us = ObTimeUtility::current_time() - start_ts; - if (cost_us > 200 * 1000) { - PALF_LOG(INFO, "receive_batch_log cost too much time", K(ret), KPC(this), K(server), K(count), - K(success_count), K(cost_us), K(prev_lsn), K(curr_lsn), K(buf_len), K_(sw), K(iterator)); - } - } - return ret; -} - -int PalfHandleImpl::submit_group_log(const PalfAppendOptions &opts, - const LSN &lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!lsn.is_valid() || NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BUFFER_SIZE) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(lsn), KP(buf), K(buf_len)); - } else { - int64_t wait_times = 0; - while (true) { - do { - RLockGuard guard(lock_); - if (false == palf_env_impl_->check_disk_space_enough()) { - ret = OB_LOG_OUTOF_DISK_SPACE; - if (palf_reach_time_interval(1 * 1000 * 1000, log_disk_full_warn_time_)) { - PALF_LOG(WARN, "log outof disk space", K(ret), KPC(this), K(opts), K(lsn)); - } - } else if (!state_mgr_.can_raw_write(opts.proposal_id, opts.need_check_proposal_id)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "cannot submit_group_log", K(ret), K_(self), K_(palf_id), KP(buf), K(buf_len), - "role", state_mgr_.get_role(), "state", state_mgr_.get_state(), - "current proposal_id", state_mgr_.get_proposal_id(), - "mode_mgr can_raw_write", mode_mgr_.can_raw_write(), K(opts)); - } else if (OB_FAIL(sw_.submit_group_log(lsn, buf, buf_len))) { - if (OB_EAGAIN != ret) { - PALF_LOG(WARN, "submit_group_log failed", K(ret), K_(palf_id), K_(self), KP(buf), K(buf_len)); - } - } else { - PALF_LOG(TRACE, "submit_group_log success", K(ret), K_(palf_id), K_(self), K(buf_len), K(lsn)); - } - } while(0); - - if (opts.need_nonblock) { - // nonblock mode, end loop - break; - } else if (OB_EAGAIN == ret) { - // block mode, need sleep and retry for -4023 ret code - static const int64_t MAX_SLEEP_US = 100; - ++wait_times; - int64_t sleep_us = wait_times * 10; - if (sleep_us > MAX_SLEEP_US) { - sleep_us = MAX_SLEEP_US; - } - ob_usleep(sleep_us); - } else { - // other ret code, end loop - break; - } - } - } - return ret; -} - -int PalfHandleImpl::ack_log(const common::ObAddr &server, - const int64_t &proposal_id, - const LSN &log_end_lsn) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id || !log_end_lsn.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(server), K(proposal_id), K(log_end_lsn)); - } else if (!state_mgr_.can_receive_log_ack(proposal_id)) { - // cannot handle log ack, skip - } else if (OB_FAIL(sw_.ack_log(server, log_end_lsn))) { - PALF_LOG(WARN, "ack_log failed", K(ret), KPC(this), K(server), K(proposal_id), K(log_end_lsn)); - } else { - PALF_LOG(TRACE, "ack_log success", K(ret), KPC(this), K(server), K(proposal_id), K(log_end_lsn)); - } - return ret; -} - -int PalfHandleImpl::get_last_rebuild_lsn(LSN &last_rebuild_lsn) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - SpinLockGuard guard(last_rebuild_meta_info_lock_); - last_rebuild_lsn = last_rebuild_lsn_; - } - return ret; -} - -int PalfHandleImpl::check_need_advance_base_info_(const LSN &base_lsn, - const LogInfo &base_prev_log_info, - const bool is_rebuild) -{ - int ret = OB_SUCCESS; - LSN committed_end_lsn; - AccessMode curr_access_mode; - bool unused_bool = false, need_rebuild = false; - int64_t unused_mode_version; - (void)mode_mgr_.get_access_mode(unused_mode_version, curr_access_mode); - if (!base_lsn.is_valid() || !base_prev_log_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(base_lsn), K(base_prev_log_info)); - } else if (state_mgr_.is_leader_active() && curr_access_mode != AccessMode::RAW_WRITE) { - // when access mode is RAW_WRITE, that means this palf is restoring, advance_base_info can be allowed - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "self is active leader, can not advance_base_info", K(ret), K_(palf_id), K(curr_access_mode), K(base_lsn)); - } else if (OB_FAIL(sw_.get_committed_end_lsn(committed_end_lsn))) { - PALF_LOG(WARN, "get_committed_end_lsn failed", KR(ret), K_(palf_id)); - } else if (base_lsn < committed_end_lsn) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "base_lsn is less than local committed_end_lsn, it's maybe a stale msg", - K(ret), K_(palf_id), K(base_lsn), K(committed_end_lsn)); - } else if (true == is_rebuild && OB_FAIL(check_need_rebuild_(base_lsn, base_prev_log_info, need_rebuild, unused_bool))) { - PALF_LOG(WARN, "check_need_rebuild failed", K(ret), KPC(this), K(base_lsn), K(base_prev_log_info), K(is_rebuild)); - } else if (true == is_rebuild && false == need_rebuild) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "do not need rebuild", K(ret), KPC(this), K(base_lsn), K(base_prev_log_info), K(is_rebuild)); - } - return ret; -} - -void PalfHandleImpl::gen_rebuild_meta_info_(RebuildMetaInfo &rebuild_meta) const -{ - int64_t unused_log_id = -1; - sw_.get_committed_end_lsn(rebuild_meta.committed_end_lsn_); - sw_.get_last_submit_log_info(rebuild_meta.last_submit_lsn_, unused_log_id, rebuild_meta.last_submit_log_pid_); -} - -void PalfHandleImpl::get_last_rebuild_meta_info_(RebuildMetaInfo &rebuild_meta_info) const -{ - SpinLockGuard guard(last_rebuild_meta_info_lock_); - rebuild_meta_info = last_rebuild_meta_info_; -} - -// caller should hold wlock when calling this function -int PalfHandleImpl::check_need_rebuild_(const LSN &base_lsn, - const LogInfo &base_prev_log_info, - bool &need_rebuild, - bool &need_fetch_log) -{ - int ret = OB_SUCCESS; - LSN committed_end_lsn; - LSN last_submit_lsn; - int64_t last_submit_log_id; - int64_t last_submit_log_pid; - bool unused_bool; - need_rebuild = false; - need_fetch_log = false; - if (!base_lsn.is_valid() || !base_prev_log_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), KPC(this), K(base_lsn), K(base_prev_log_info)); - } else if (state_mgr_.is_leader_active()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "self is active leader, no need execute rebuild", K(ret), K_(palf_id), K(base_lsn)); - } else if (OB_FAIL(sw_.get_committed_end_lsn(committed_end_lsn))) { - PALF_LOG(WARN, "get_committed_end_lsn failed", KR(ret), K_(palf_id)); - } else if (base_lsn <= committed_end_lsn) { - PALF_LOG(INFO, "base_lsn is less than or equal to local committed_end_lsn", - K(ret), K_(palf_id), K(base_lsn), K(committed_end_lsn)); - } else if (OB_FAIL(sw_.get_last_submit_log_info(last_submit_lsn, last_submit_log_id, last_submit_log_pid))) { - PALF_LOG(WARN, "get_last_submit_log_info failed", KR(ret), K_(palf_id)); - } else if (last_submit_lsn < base_prev_log_info.lsn_) { - // previous log hasn't been submitted, need rebuild - need_rebuild = true; - PALF_LOG(INFO, "palf need rebuild, reason: previous log hasn't been submitted", K(ret), KPC(this), - K(need_rebuild), K(base_lsn), K(base_prev_log_info), K(last_submit_lsn)); - } else if (last_submit_lsn == base_prev_log_info.lsn_) { - // previous log has just been submitted, need check if matches - if (last_submit_log_id == base_prev_log_info.log_id_ && - last_submit_log_pid == base_prev_log_info.log_proposal_id_) { - // previous log is in sliding window and matches, don't rebuild, need fetch log - need_fetch_log = true; - } else { - // previous log don't match, need rebuild - need_rebuild = true; - PALF_LOG(INFO, "palf need rebuild, previous log don't match", K(ret), KPC(this), - K(need_rebuild), K(base_lsn), K(base_prev_log_info), K(last_submit_log_id), K(last_submit_log_pid)); - } - } else if (sw_.is_prev_log_pid_match(base_prev_log_info.log_id_ + 1, base_lsn, - base_prev_log_info.lsn_, base_prev_log_info.log_proposal_id_, unused_bool)) { - // when last_submit_lsn > base_prev_log_info.lsn_, to check if previous log matches. - // if matches, don't rebuild, need fetch log - need_fetch_log = true; - } else { - // if prev log is in sliding window and don't match, do rebuild - // if prev log isn't in sliding window, do rebuild - need_rebuild = true; - PALF_LOG(INFO, "palf need rebuild, previous log don't match", K(ret), KPC(this), - K(need_rebuild), K(base_lsn), K(base_prev_log_info), K(last_submit_log_id), K(last_submit_log_pid)); - } - return ret; -} - -int PalfHandleImpl::handle_notify_fetch_log_req(const common::ObAddr &server) -{ - // This req is sent by reconfirming leader. - // Self is lag behind majority_max_lsn, so it need fetch log immediately. - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(log_engine_.submit_purge_throttling_task(PurgeThrottlingType::PURGE_BY_NOTIFY_FETCH_LOG))) { - PALF_LOG(WARN, "failed to submit_purge_throttling_task with notify_fetch_log", KPC(this), K(server)); - } else if (OB_FAIL(sw_.try_fetch_log(FetchTriggerType::RECONFIRM_NOTIFY_FETCH))) { - PALF_LOG(WARN, "try_fetch_log failed", KR(ret), KPC(this), K(server)); - } else if (OB_FAIL(sw_.submit_push_log_resp(server))) { - PALF_LOG(WARN, "submit_push_log_resp failed", KR(ret), KPC(this), K(server)); - } else {} - PALF_LOG(INFO, "handle_notify_fetch_log_req finished", KR(ret), KPC(this), K(server)); - return ret; -} - -int PalfHandleImpl::handle_notify_rebuild_req(const common::ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) -{ - int ret = OB_SUCCESS; - // This function will be called when: - // This node A has sent a fetch log req to B, B find that A's fetch_start_lsn is smaller than B's base_lsn - // (B's log file may be recycled), therefore B will notify A may need do rebuild, send it's base_lsn and prev_log_info - // of base_lsn to A. This function will be called when A receive this message. - // After receiving rebuild_req, A will check whether previous log entries of base_lsn are matching in this function. - // The main check logic is: - // 1. require write lock - // 2. ensure base_lsn > committed_end_lsn. If base_lsn <= committed_end_lsn, that means it's possible - // for A to get log from B in next fetch log, so ignore. - // 3. check if previous log is cached in sliding window and matches, - // If it does, do not need rebuild and fetch log, if not, do rebuild!!! - - // NB: Why require write lock in here? - // In step 2, base_lsn > committed_end_lsn is guaranteed. if require write lock and don't find - // previous log in sliding window finally, therefore we can ensure this node must do rebuild. - // (When this function hold write lock, committed log can not be slid.) - // If we require read lock, previous log may has slid when we read it from sliding window, - // this will cause wrong rebuild. - bool need_rebuild = false; - bool need_fetch_log = false; - RebuildMetaInfo rebuild_meta_info; - do { - int tmp_ret = OB_SUCCESS; - // leader may send multiple notify_rebuild_req, when next req arrives, previous on_rebuild may - // hold rlock, so try hold wlock and release it after timeout (1ms). - const int64_t until_timeout_us = common::ObTimeUtility::current_time() + 1000; - WLockGuardWithTimeout guard(lock_, until_timeout_us, tmp_ret); - if (OB_SUCCESS != tmp_ret) { - PALF_LOG(INFO, "notify_rebuild wait lock timeout", K(ret), KPC(this), K(server), K(base_lsn), - K(base_prev_log_info)); - } else if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || !base_lsn.is_valid() || - !base_prev_log_info.is_valid() || base_lsn.val_ == PALF_INITIAL_LSN_VAL) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K(server), K(base_lsn)); - } else if (OB_FAIL(check_need_rebuild_(base_lsn, base_prev_log_info, need_rebuild, need_fetch_log))) { - PALF_LOG(WARN, "check_need_rebuild failed", K(ret), KPC(this), K(server), K(base_lsn), K(base_prev_log_info)); - } else if (need_rebuild) { - //set rebuild_meta_info - gen_rebuild_meta_info_(rebuild_meta_info); - } else {} - } while (0); - - if (OB_SUCC(ret)) { - // can not hold wlock when exec on_rebuild - if (need_rebuild) { - if (OB_FAIL(rebuild_cb_wrapper_.on_rebuild(palf_id_, base_lsn))) { - PALF_LOG(WARN, "on_rebuild failed", K(ret), K(server), K(base_lsn)); - } else { - PALF_EVENT("on_rebuild success", palf_id_, K(ret), K_(self), K(server), K(base_lsn)); - plugins_.record_rebuild_event(palf_id_, server, base_lsn); - } - // Whether on_rebuild returns OB_SUCCESS or not, set value for rebuild_base_lsn_ - SpinLockGuard rebuild_guard(last_rebuild_meta_info_lock_); - last_rebuild_lsn_ = base_lsn; - last_rebuild_meta_info_ = rebuild_meta_info; - } else if (need_fetch_log && OB_FAIL(sw_.try_fetch_log(FetchTriggerType::NOTIFY_REBUILD, - base_prev_log_info.lsn_, base_lsn, base_prev_log_info.log_id_+1))) { - PALF_LOG(WARN, "try_fetch_log failed", KR(ret), KPC(this), K(server), K(base_lsn), K(base_prev_log_info)); - } - } - return ret; -} - -int PalfHandleImpl::fetch_log_from_storage(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid, - const SCN &replayable_point, - FetchLogStat &fetch_stat) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(fetch_log_from_storage_(server, fetch_type, msg_proposal_id, prev_lsn, - fetch_start_lsn, fetch_log_size, fetch_log_count, replayable_point, fetch_stat))) { - PALF_LOG(WARN, "fetch_log_from_storage_ failed", K(ret), K_(palf_id), K_(self), - K(server), K(fetch_type), K(msg_proposal_id), K(prev_lsn), K(fetch_start_lsn), - K(fetch_log_size), K(fetch_log_count), K(accepted_mode_pid)); - } - { - // try fetch mode_meta when handle every fetch_log_req - RLockGuard guard(lock_); - if (OB_FAIL(mode_mgr_.submit_fetch_mode_meta_resp(server, msg_proposal_id, accepted_mode_pid))) { - PALF_LOG(WARN, "submit_fetch_mode_meta_resp failed", K(ret), K_(palf_id), K_(self), - K(msg_proposal_id), K(accepted_mode_pid)); - } - const int64_t accum_size = ATOMIC_AAF(&accum_fetch_log_size_, fetch_log_size); - if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_accum_fetch_statistic_time_)) { - PALF_LOG(INFO, "[PALF STAT FETCH LOG SIZE]", KPC(this), K(accum_size)); - ATOMIC_STORE(&accum_fetch_log_size_, 0); - } - } - return ret; -} - -int PalfHandleImpl::try_send_committed_info_(const ObAddr &server, - const LSN &log_lsn, - const LSN &log_end_lsn, - const int64_t &log_proposal_id) -{ - int ret = OB_SUCCESS; - AccessMode access_mode; - if (!log_lsn.is_valid() || !log_end_lsn.is_valid() || INVALID_PROPOSAL_ID == log_proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), KPC(this), K(server), K(log_lsn), K(log_end_lsn), - K(log_proposal_id)); - } else if (OB_FAIL(mode_mgr_.get_access_mode(access_mode))) { - PALF_LOG(WARN, "get_access_mode failed", K(ret), KPC(this)); - } else if (AccessMode::APPEND == access_mode) { - // No need send committed_info in APPEND mode, because leader will generate keeapAlive log periodically. - } else if (OB_FAIL(sw_.try_send_committed_info(server, log_lsn, log_end_lsn, log_proposal_id))) { - PALF_LOG(TRACE, "try_send_committed_info failed", K(ret), K_(palf_id), K_(self), - K(server), K(log_lsn), K(log_end_lsn), K(log_proposal_id)); - } else { - PALF_LOG(TRACE, "try_send_committed_info_ success", K(ret), K_(palf_id), K_(self), K(server), - K(log_lsn), K(log_end_lsn), K(log_proposal_id)); - } - return ret; -} - -int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const SCN &replayable_point, - FetchLogStat &fetch_stat) -{ - int ret = OB_SUCCESS; - LSN prev_lsn_each_round = prev_lsn; - LSN prev_end_lsn_each_round = prev_lsn; - int64_t prev_log_proposal_id_each_round = INVALID_PROPOSAL_ID; - PalfGroupBufferIterator iterator; - const LSN fetch_end_lsn = fetch_start_lsn + fetch_log_size; - int64_t fetched_count = 0; - const bool need_check_prev_log = (prev_lsn.is_valid() && PALF_INITIAL_LSN_VAL < fetch_start_lsn.val_); - LSN max_flushed_end_lsn; - LSN committed_end_lsn; - bool is_limitted_by_end_lsn = true; - AccessMode access_mode = AccessMode::INVALID_ACCESS_MODE; - common::ObMemberList member_list; - int64_t replica_num = 0; - GlobalLearnerList learner_list; - // Assign values for max_flushed_end_lsn/committed_end_lsn/is_limitted_by_end_lsn with rdlock - // to avoid concurrent update with switch_state/truncate (with wrlock). - do { - RLockGuard guard(lock_); - sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); - committed_end_lsn = get_end_lsn(); - // Note: Self may revoke after unlock, so the uncommitted logs may be sent later. - // But we use req's msg_proposal_id to send response, which gurantees no harm. - // Because if another new leader has taken over, the majority must have advanced - // its proposal_id to reject fetch response with old proposal_id. - if (FETCH_LOG_LEADER_RECONFIRM == fetch_type - || state_mgr_.is_leader_active() - || state_mgr_.is_leader_reconfirm()) { - // leader reconfirm state also needs to allow sending unconfirmed log, otherwise start_working log may not achieve a majority - // Because the majority replica log is behind, the forward validation before receive config_log will fail - // In the reconfirm state, unconfirmed logs can be safely sent because this part of the logs is expected not to be truncated - is_limitted_by_end_lsn = false; - } - int64_t unused_mode_version; - (void) mode_mgr_.get_access_mode(unused_mode_version, access_mode); - (void) config_mgr_.get_curr_member_list(member_list, replica_num); - (void) config_mgr_.get_global_learner_list(learner_list); - } while(0); - - // max_flushed_end_lsn may be truncated by concurrent truncate, so itreator need handle this - // case when it try to read some log which is being truncated. - auto get_file_end_lsn = [&]() { return max_flushed_end_lsn; }; - LogInfo prev_log_info; - const bool no_need_fetch_log = prev_lsn >= max_flushed_end_lsn; - const bool is_dest_in_memberlist = (member_list.contains(server) || learner_list.contains(server)); - // Rpc delay increases enormously when it's size exceeds 2M. - const int64_t MAX_BATCH_LOG_SIZE_EACH_ROUND = 2 * 1024 * 1024 - 1024; - char *batch_log_buf = NULL; - if (no_need_fetch_log) { - PALF_LOG(INFO, "no need fetch_log_from_storage", K(ret), KPC(this), K(server), K(fetch_start_lsn), K(prev_lsn), - K(max_flushed_end_lsn), K(access_mode)); - } else if (true == need_check_prev_log - && OB_FAIL(get_prev_log_info_for_fetch_(prev_lsn, fetch_start_lsn, prev_log_info))) { - PALF_LOG(WARN, "get_prev_log_info_for_fetch_ failed", K(ret), K_(palf_id), K(prev_lsn), K(fetch_start_lsn)); - } else if (true == need_check_prev_log && prev_log_info.lsn_ != prev_lsn) { - if (is_dest_in_memberlist) { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "the LSN between each replica is not same, unexpected error!!!", K(ret), - K_(palf_id), K(fetch_start_lsn), K(prev_log_info)); - } else { - PALF_LOG(INFO, "the LSN between leader and non paxos member is not same, do not fetch log", - K_(palf_id), K(fetch_start_lsn), K(prev_log_info)); - } - } else if (check_need_hook_fetch_log_(fetch_type, fetch_start_lsn)) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - } else if (FALSE_IT(prev_log_proposal_id_each_round = prev_log_info.log_proposal_id_)) { - } else if (OB_FAIL(iterator.init(fetch_start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { - PALF_LOG(ERROR, "init iterator failed", K(ret), K_(palf_id)); - } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(palf_id_, LogIOUser::FETCHLOG)))) { - PALF_LOG(ERROR, "iterator set_io_context failed", K(ret), K_(palf_id)); - } else if (FALSE_IT(iterator.set_need_print_error(false))) { - // NB: Fetch log will be concurrent with truncate, the content on disk will not integrity, need igore - // read log error. - } else if (OB_NOT_NULL(batch_log_buf = (char *)mtl_malloc(MAX_BATCH_LOG_SIZE_EACH_ROUND, "BatchLogBuf"))) { - // When group log size > 10K, we just send it without aggregating. - const int64_t MAX_NEED_BATCH_LOG_SIZE = 10 * 1024; - // When batched log count reaches max_batch_log_count_each_round, we end the aggregation. - const int64_t max_batch_log_count_each_round = PALF_SLIDING_WINDOW_SIZE; - int64_t remained_count = fetch_log_count; - bool is_reach_end = false; - bool skip_next = false; - BatchFetchParams batch_fetch_params; - batch_fetch_params.batch_log_buf_ = batch_log_buf; - batch_fetch_params.can_batch_size_ = MAX_BATCH_LOG_SIZE_EACH_ROUND; - batch_fetch_params.last_log_lsn_prev_round_ = prev_lsn_each_round; - batch_fetch_params.last_log_end_lsn_prev_round_ = prev_end_lsn_each_round; - batch_fetch_params.last_log_proposal_id_prev_round_ = prev_log_proposal_id_each_round; - while (OB_SUCC(ret) && remained_count > 0 && !is_reach_end && - batch_fetch_params.last_log_end_lsn_prev_round_ < fetch_end_lsn) { - batch_fetch_params.can_batch_count_ = MIN(remained_count, max_batch_log_count_each_round); - batch_fetch_params.has_consumed_count_ = 0; - if (OB_FAIL(batch_fetch_log_each_round_(server, msg_proposal_id, iterator, is_limitted_by_end_lsn, - is_dest_in_memberlist, replayable_point, fetch_end_lsn, committed_end_lsn, MAX_NEED_BATCH_LOG_SIZE, - batch_fetch_params, skip_next, is_reach_end, fetch_stat)) && OB_ITER_END != ret) { - PALF_LOG(WARN, "batch_fetch_log_each_round_ failed", K(ret), KPC(this), K(iterator)); - } else { - remained_count -= batch_fetch_params.has_consumed_count_; - } - } - prev_lsn_each_round = batch_fetch_params.last_log_lsn_prev_round_; - prev_end_lsn_each_round = batch_fetch_params.last_log_end_lsn_prev_round_; - prev_log_proposal_id_each_round = batch_fetch_params.last_log_proposal_id_prev_round_; - fetched_count = fetch_log_count - remained_count; - } else { - - PALF_LOG(WARN, "allocate batch_log_buf memory failed", KPC(this), K(server), K(prev_lsn)); - - LogGroupEntry curr_group_entry; - LSN curr_lsn; - LSN curr_log_end_lsn; - bool is_reach_size_limit = false; // whether the total fetched size exceeds fetch_log_size - bool is_reach_count_limit = false; - bool is_reach_end = false; - int64_t total_size = 0; - int64_t send_cost = 0; - int64_t send_begin_time = ObTimeUtility::current_time(); - while (OB_SUCC(ret) && !is_reach_size_limit && !is_reach_count_limit && !is_reach_end - && OB_SUCC(iterator.next())) { - if (OB_FAIL(iterator.get_entry(curr_group_entry, curr_lsn))) { - PALF_LOG(ERROR, "PalfGroupBufferIterator get_entry failed", K(ret), K_(palf_id), - K(curr_group_entry), K(curr_lsn), K(iterator)); - } else if (FALSE_IT(curr_log_end_lsn = curr_lsn + curr_group_entry.get_group_entry_size())) { - } else if (is_limitted_by_end_lsn && curr_log_end_lsn > committed_end_lsn) { - // Only leader replica can send uncommitted logs to others, - // the other replicas just send committed logs to avoid unexpected rewriting. - is_reach_end = true; - PALF_LOG(INFO, "reach committed_end_lsn(not leader active replica), end fetch", K(ret), K_(palf_id), K(server), - K(msg_proposal_id), K(curr_lsn), K(curr_log_end_lsn), K(committed_end_lsn)); - } else if (false == is_dest_in_memberlist && - curr_group_entry.get_header().is_raw_write() && - replayable_point.is_valid() && - curr_group_entry.get_scn() > replayable_point) { - is_reach_end = true; - PALF_LOG(INFO, "non paxos member could not fetch logs which scn is bigger than replayable_point, end fetch", - K_(palf_id), K(server), K(msg_proposal_id), K(curr_lsn), K(replayable_point)); - } else if (FALSE_IT(send_begin_time = ObTimeUtility::current_time())) { - } else if (OB_FAIL(submit_fetch_log_resp_(server, msg_proposal_id, prev_log_proposal_id_each_round, \ - prev_lsn_each_round, curr_lsn, curr_group_entry))) { - PALF_LOG(WARN, "submit_fetch_log_resp_ failed", K(ret), K_(palf_id), K(server), - K(msg_proposal_id), K(prev_lsn_each_round), K(fetch_start_lsn)); - } else { - send_cost += ObTimeUtility::current_time() - send_begin_time; - fetched_count++; - total_size += curr_group_entry.get_group_entry_size(); - if (fetched_count >= fetch_log_count) { - is_reach_count_limit = true; - } - // check if reach size limit - if (curr_log_end_lsn >= fetch_end_lsn) { - is_reach_size_limit = true; - } - PALF_LOG(TRACE, "fetch one log success", K(ret), K_(palf_id), K_(self), K(server), K(prev_lsn), - K(fetch_start_lsn), K(prev_lsn_each_round), K(curr_lsn), K(curr_group_entry), - K(prev_log_proposal_id_each_round), K(fetch_end_lsn), K(curr_log_end_lsn), K(is_reach_size_limit), - K(fetch_log_size), K(fetched_count), K(is_reach_count_limit)); - prev_lsn_each_round = curr_lsn; - prev_end_lsn_each_round = curr_log_end_lsn; - prev_log_proposal_id_each_round = curr_group_entry.get_header().get_log_proposal_id(); - } - } - // update fetch statistic info - fetch_stat.total_size_ = total_size; - fetch_stat.group_log_cnt_ = fetched_count; - fetch_stat.send_cost_ = send_cost; - } - if (batch_log_buf != NULL) { - mtl_free(batch_log_buf); - batch_log_buf = NULL; - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - PALF_LOG(INFO, "fetch_log_from_storage_ finished", K(ret), KPC(this), K(prev_lsn), K(iterator), - K(fetch_log_count), "has_fetched_log_count", fetched_count); - // try send committed_info to server - if (OB_SUCC(ret)) { - RLockGuard guard(lock_); - (void) try_send_committed_info_(server, prev_lsn_each_round, prev_end_lsn_each_round, - prev_log_proposal_id_each_round); - } - if (OB_FAIL(ret) && OB_ERR_OUT_OF_LOWER_BOUND == ret) { - // ret is OB_ERR_OUT_OF_LOWER_BOUND, need notify dst server to trigger rebuild - LSN base_lsn = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_; - LogInfo base_prev_log_info; - if (!base_lsn.is_valid()) { - PALF_LOG(WARN, "local base_lsn is invalid, cannot notify server to rebuild", K(ret), - K_(palf_id), K(server), K(base_lsn), K(base_prev_log_info)); - } else if (OB_FAIL(get_prev_log_info_(base_lsn, base_prev_log_info))) { - // NB: snapshot_meta.prev_log_info_ is invalid in most common case, so we need to read base_prev_log_info from disk - PALF_LOG(WARN, "local base_prev_log_info is invalid, cannot notify server to rebuild", K(ret), - K_(palf_id), K(server), K(base_lsn), K(base_prev_log_info)); - } else if (OB_FAIL(log_engine_.submit_notify_rebuild_req(server, base_lsn, base_prev_log_info))) { - PALF_LOG(WARN, "submit_notify_rebuild_req failed", K(ret), K_(palf_id), K(server), K(base_lsn), K(base_prev_log_info)); - } else { - PALF_LOG(INFO, "submit_notify_rebuild_req success", K(ret), K_(palf_id), K(server), K(prev_lsn), - K(fetch_start_lsn), K(base_lsn), K(base_prev_log_info)); - } - } - return ret; -} - -int PalfHandleImpl::batch_fetch_log_each_round_(const common::ObAddr &server, - const int64_t msg_proposal_id, - PalfGroupBufferIterator &iterator, - const bool is_limitted_by_end_lsn, - const bool is_dest_in_memberlist, - const share::SCN& replayable_point, - const LSN &fetch_end_lsn, - const LSN &committed_end_lsn, - const int64_t max_need_batch_log_size, - BatchFetchParams &batch_fetch_params, - bool &skip_next, - bool &is_reach_end, - FetchLogStat &fetch_stat) -{ - int ret = OB_SUCCESS; - int64_t remained_count = batch_fetch_params.can_batch_count_; - int64_t remained_size = batch_fetch_params.can_batch_size_; - LSN prev_lsn = batch_fetch_params.last_log_lsn_prev_round_; - LSN prev_end_lsn = batch_fetch_params.last_log_end_lsn_prev_round_; - int64_t prev_log_proposal_id = batch_fetch_params.last_log_proposal_id_prev_round_; - int64_t has_consumed_count = batch_fetch_params.has_consumed_count_; - LSN curr_lsn; - LSN curr_end_lsn; - int64_t curr_log_proposal_id; - LSN first_log_lsn; - const char *curr_log_buf = NULL; - int64_t curr_log_buf_len = 0; - int64_t log_end_pos = 0; - bool has_reach_threshold = false; - share::SCN curr_scn; - bool is_raw_write; - - int64_t start_ts = ObTimeUtility::current_time(); - while (OB_SUCC(ret) && !is_reach_end && remained_count > 0 && remained_size > 0) { - if (!skip_next && OB_FAIL(iterator.next())) { - PALF_LOG(WARN, "iterator next failed", K(ret), KPC(this), K(iterator), K(replayable_point), K(prev_lsn)); - } else if (FALSE_IT(skip_next = false)) { - } else if (OB_FAIL(iterator.get_entry(curr_log_buf, curr_log_buf_len, curr_scn, curr_lsn, curr_log_proposal_id, - is_raw_write))){ - PALF_LOG(WARN, "iterator get_entry failed", K(ret), KPC(this), K(iterator), K(log_end_pos)); - } else if (false == is_dest_in_memberlist && is_raw_write && replayable_point.is_valid() && - curr_scn > replayable_point) { - is_reach_end = true; - PALF_LOG(INFO, "non paxos member could not fetch logs which scn is bigger than replayable_point, end fetch", - K_(palf_id), K(server), K(msg_proposal_id), K(curr_lsn), K(replayable_point)); - } else if (FALSE_IT(curr_end_lsn = curr_lsn + curr_log_buf_len)) { - } else if (is_limitted_by_end_lsn && curr_end_lsn > committed_end_lsn){ - // Only leader replica can send uncommitted logs to others, - // the other replicas just send committed logs to avoid unexpected rewriting. - is_reach_end = true; - PALF_LOG(INFO, "reach committed_end_lsn(not leader active replica), end fetch", K(ret), K_(palf_id), K(server), - K(msg_proposal_id), K(curr_lsn), K(curr_end_lsn), K(committed_end_lsn)); - } else if (curr_log_buf_len >= max_need_batch_log_size ) { - has_reach_threshold = true; - PALF_LOG(TRACE, "group log is bigger than batch_log_size_threshold", K(ret), K_(palf_id), K(server), - K(msg_proposal_id), K(curr_lsn), K(curr_end_lsn), K(max_need_batch_log_size)); - break; - } else if (remained_size >= curr_log_buf_len ) { - if (!first_log_lsn.is_valid()) { - first_log_lsn = curr_lsn; - } - MEMCPY(batch_fetch_params.batch_log_buf_ + log_end_pos, curr_log_buf, curr_log_buf_len); - log_end_pos += curr_log_buf_len; - prev_lsn = curr_lsn; - prev_end_lsn = curr_end_lsn; - prev_log_proposal_id = curr_log_proposal_id; - has_consumed_count++; - remained_count--; - remained_size -= curr_log_buf_len; - if (curr_end_lsn >= fetch_end_lsn) { - PALF_LOG(INFO, "fetched log has reached fetch_end_lsn", K(ret), KPC(this), K(curr_lsn), K(curr_end_lsn), - K(fetch_end_lsn)); - is_reach_end = true; - } - } else { - // batch the group log next round - skip_next = true; - PALF_LOG(TRACE, "batched log has exceeded can_batch_size", K(ret), KPC(this), K(batch_fetch_params.can_batch_size_), - K(remained_size), K(prev_lsn), K(curr_lsn)); - break; - } - } - int64_t batch_end_ts = common::ObTimeUtility::current_time(); - if (OB_ITER_END == ret) { - is_reach_end = true; - ret = OB_SUCCESS; - } - if (OB_FAIL(ret)) { - PALF_LOG(WARN, "batch log failed", K(ret), KPC(this), K(iterator), K(log_end_pos), - K(curr_lsn), K(has_consumed_count), K(first_log_lsn)); - } else { - if (1 < has_consumed_count) { - ret = submit_batch_fetch_log_resp_(server, msg_proposal_id, batch_fetch_params.last_log_proposal_id_prev_round_, - batch_fetch_params.last_log_lsn_prev_round_, first_log_lsn, batch_fetch_params.batch_log_buf_, log_end_pos); - } else if (1 == has_consumed_count) { - ret = submit_fetch_log_resp_(server, msg_proposal_id, batch_fetch_params.last_log_proposal_id_prev_round_, - batch_fetch_params.last_log_lsn_prev_round_, first_log_lsn, batch_fetch_params.batch_log_buf_, log_end_pos); - } else { - PALF_LOG(TRACE, "no log is aggregated", K(ret), KPC(this), K(log_end_pos), K(curr_lsn), - K(has_consumed_count), K(first_log_lsn)); - } - } - - if (OB_SUCC(ret) && has_reach_threshold) { - if (OB_FAIL(submit_fetch_log_resp_(server, msg_proposal_id, prev_log_proposal_id, prev_lsn, - curr_lsn, curr_log_buf, curr_log_buf_len))) { - PALF_LOG(WARN, "submit_fetch_log_resp_ failed", K(ret), KPC(this), K(prev_lsn), - K(curr_lsn), K(curr_log_buf_len)); - } else { - log_end_pos += curr_log_buf_len; - prev_lsn = curr_lsn; - prev_end_lsn = curr_end_lsn; - prev_log_proposal_id = curr_log_proposal_id; - has_consumed_count++; - remained_count--; - remained_size -= curr_log_buf_len; - PALF_LOG(TRACE, "submit_fetch_log_resp_ success", K(ret), KPC(this), K(prev_lsn), - K(curr_lsn), K(curr_log_buf_len)); - } - } - int64_t send_end_ts = common::ObTimeUtility::current_time(); - if (OB_SUCC(ret) && 0 != has_consumed_count) { - batch_fetch_params.last_log_lsn_prev_round_ = prev_lsn; - batch_fetch_params.last_log_end_lsn_prev_round_ = prev_end_lsn; - batch_fetch_params.last_log_proposal_id_prev_round_ = prev_log_proposal_id; - batch_fetch_params.has_consumed_count_ = has_consumed_count; - int64_t total_size = log_end_pos; - int64_t batch_cost = batch_end_ts - start_ts; - int64_t send_cost = send_end_ts - batch_end_ts; - int64_t total_cost = send_end_ts - start_ts; - fetch_stat.total_size_ += total_size; - fetch_stat.group_log_cnt_ += has_consumed_count; - fetch_stat.send_cost_ += send_cost; - int64_t avg_log_size = total_size / has_consumed_count; - int64_t avg_log_cost = total_cost / has_consumed_count; - int64_t avg_batch_cost = batch_cost / has_consumed_count; - int64_t avg_send_cost = send_cost/ has_consumed_count; - PALF_LOG(TRACE, "batch_fetch_log_one_round_ success", K(ret), KPC(this), K(is_reach_end), - K(batch_fetch_params.can_batch_size_), K(remained_size), K(batch_fetch_params.can_batch_count_), - K(has_consumed_count), K(total_size), K(total_cost), K(batch_cost), K(send_cost), K(avg_log_size), - K(avg_log_cost), K(avg_send_cost), K(avg_batch_cost), K(first_log_lsn), - K(batch_fetch_params.last_log_lsn_prev_round_), K(iterator)); - } - return ret; -} - -int PalfHandleImpl::submit_fetch_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogGroupEntry &curr_group_entry) -{ - int ret = OB_SUCCESS; - LogWriteBuf write_buf; - // NB: 'curr_group_entry' generates by PalfGroupBufferIterator, the memory is safe before next(); - const char *buf = curr_group_entry.get_data_buf() - curr_group_entry.get_header().get_serialize_size(); - const int64_t buf_len = curr_group_entry.get_group_entry_size(); - int64_t pos = 0; - const int64_t curr_log_proposal_id = curr_group_entry.get_header().get_log_proposal_id(); - if (OB_FAIL(write_buf.push_back(buf, buf_len))) { - PALF_LOG(WARN, "push_back buf into LogWriteBuf failed", K(ret)); - } else if (OB_FAIL(log_engine_.submit_push_log_req(server, FETCH_LOG_RESP, msg_proposal_id, prev_log_proposal_id, - prev_lsn, curr_lsn, write_buf))) { - PALF_LOG(WARN, "submit_push_log_req failed", K(ret), K(server), K(msg_proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(curr_lsn), K(curr_log_proposal_id), K(write_buf)); - } else { - PALF_LOG(TRACE, "submit_fetch_log_resp_ success", K(ret), K(server), K(msg_proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(curr_lsn), K(curr_log_proposal_id), K(write_buf)); - } - return ret; -} - -int PalfHandleImpl::submit_fetch_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - LogWriteBuf write_buf; - // NB: 'curr_group_entry' generates by PalfGroupBufferIterator, the memory is safe before next(); - if (OB_FAIL(write_buf.push_back(buf, buf_len))) { - PALF_LOG(WARN, "push_back buf into LogWriteBuf failed", K(ret)); - } else if (OB_FAIL(log_engine_.submit_push_log_req(server, FETCH_LOG_RESP, msg_proposal_id, prev_log_proposal_id, - prev_lsn, curr_lsn, write_buf))) { - PALF_LOG(WARN, "submit_push_log_req failed", K(ret), K(server), K(msg_proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(curr_lsn), K(write_buf)); - } else { - PALF_LOG(TRACE, "submit_fetch_log_resp_ success", K(ret), K(server), K(msg_proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(curr_lsn), K(write_buf)); - } - return ret; -} - -int PalfHandleImpl::submit_batch_fetch_log_resp_(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len) -{ - int ret = OB_SUCCESS; - LogWriteBuf write_buf; - int64_t pos = 0; - if (OB_FAIL(write_buf.push_back(buf, buf_len))) { - PALF_LOG(WARN, "push_back buf into LogWriteBuf failed", K(ret)); - } else if (OB_FAIL(log_engine_.submit_batch_fetch_log_resp(server, msg_proposal_id, prev_log_proposal_id, - prev_lsn, curr_lsn, write_buf))) { - PALF_LOG(WARN, "submit_push_log_req failed", K(ret), K(server), K(msg_proposal_id), K(prev_log_proposal_id), - K(prev_lsn), K(curr_lsn), K(write_buf)); - } else { - PALF_LOG(TRACE, "submit_batch_fetch_log_resp_ success", K(ret), K(server), KPC(this), K(msg_proposal_id), - K(prev_log_proposal_id), K(buf_len), K(prev_lsn), K(curr_lsn), K(write_buf)); - } - return ret; -} - -// NB: 1. there is no need to distinguish between reconfirm or follower active; -// 2. whether msg_proposal_id is equal to currentry proposal_id or not, response logs to the requester. -int PalfHandleImpl::get_log(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) +int PalfHandleImpl::do_init_mem_( + const PalfBaseInfo &palf_base_info, + const LogMeta &log_meta, + const char *log_dir, + const common::ObAddr &self, + ObILogAllocator *alloc_mgr, + IPalfEnvImpl *palf_env_impl) { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (false == server.is_valid() - || INVALID_PROPOSAL_ID == msg_proposal_id - || false == start_lsn.is_valid() - || 0 >= fetch_log_size - || 0 >= fetch_log_count - || INVALID_PROPOSAL_ID == accepted_mode_pid) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K(server), K(msg_proposal_id), K(prev_lsn), - K(start_lsn), K(fetch_log_size), K(fetch_log_count), K(accepted_mode_pid)); - } else if (OB_ISNULL(fetch_log_engine_)) { + int pret = -1; + if ((pret = snprintf(log_dir_, MAX_PATH_SIZE, "%s", log_dir)) && false) { ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "fetch log engine is null", K(ret), KPC(this)); - } else if (!fetch_log_engine_->is_enabled()) { - ret = OB_NOT_SUPPORTED; - PALF_LOG(WARN, "fetch log engine is disabled", K(ret), K_(palf_id)); - } else if (OB_FAIL(try_update_proposal_id_(server, msg_proposal_id))) { - PALF_LOG(WARN, "try_update_proposal_id_ failed", KR(ret), KPC(this), K(server), K(msg_proposal_id)); + PALF_LOG(ERROR, "error unexpected", K(ret)); + } else if (OB_FAIL(sw_.init(self, &state_mgr_, &mode_mgr_, + &log_engine_, &fs_cb_wrapper_, alloc_mgr, palf_base_info))) { + PALF_LOG(WARN, "sw_ init failed", K(ret)); + } else if (OB_FAIL(log_cache_.init(this))) { + PALF_LOG(WARN, "log_cache_ init failed", K(ret)); + } else if (OB_FAIL(state_mgr_.init(self, &sw_, &mode_mgr_))) { + PALF_LOG(WARN, "state_mgr_ init failed", K(ret)); + } else if (OB_FAIL(mode_mgr_.init(self, log_meta.get_log_mode_meta()))) { + PALF_LOG(WARN, "mode_mgr_ init failed", K(ret)); } else { - FetchLogTask *task = NULL; - RLockGuard guard(lock_); - if (NULL == (task = fetch_log_engine_->alloc_fetch_log_task())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - PALF_LOG(WARN, "alloc fetch log task failed", K(ret), K_(palf_id)); - } else if (OB_FAIL(task->set(palf_id_, server, fetch_type, msg_proposal_id, - prev_lsn, start_lsn, fetch_log_size, fetch_log_count, accepted_mode_pid))) { - PALF_LOG(WARN, "set fetch log task error", K(ret), K_(palf_id), - K(msg_proposal_id), K(prev_lsn), K(start_lsn), K(fetch_log_size), K(fetch_log_count), K(accepted_mode_pid)); - } else if (OB_FAIL(fetch_log_engine_->submit_fetch_log_task(task))) { - PALF_LOG(WARN, "submit fetch log task error", K(ret), K_(palf_id), K(server), - K(prev_lsn), K(start_lsn), K(fetch_log_size), K(fetch_log_count), K(accepted_mode_pid)); - } else { - PALF_LOG(INFO, "submit fetch log task success", K(ret), K_(palf_id), K(server), K(fetch_type), K(prev_lsn), - K(start_lsn), K(fetch_log_size), K(fetch_log_count), K(accepted_mode_pid)); - } - if (OB_SUCCESS != ret) { - fetch_log_engine_->free_fetch_log_task(task); - } + allocator_ = alloc_mgr; + self_ = self; + has_set_deleted_ = false; + palf_env_impl_ = palf_env_impl; + is_inited_ = true; + PALF_LOG(INFO, "PalfHandleImpl do_init_ success", K(ret), K(self), K(log_dir), K(palf_base_info), + K(log_meta), K(alloc_mgr)); + } + if (OB_FAIL(ret)) { + is_inited_ = true; + destroy(); } return ret; } -int PalfHandleImpl::receive_config_log(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &meta) +int PalfHandleImpl::get_palf_epoch(int64_t &palf_epoch) const { int ret = OB_SUCCESS; - bool receive_newer_config_log = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); - } else if (false == server.is_valid() || - self_ == server || - INVALID_PROPOSAL_ID == msg_proposal_id || - false == prev_lsn.is_valid() || - INVALID_PROPOSAL_ID == prev_mode_pid || - false == meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(server), - K(msg_proposal_id), K(prev_lsn), K(prev_mode_pid), K(meta)); - } else if (OB_FAIL(try_update_proposal_id_(server, msg_proposal_id))) { - PALF_LOG(WARN, "try_update_proposal_id_ failed", KR(ret), KPC(this), K(server), K(msg_proposal_id)); } else { - TruncateLogInfo truncate_log_info; - // need wlock in case of truncating log and writing log_ms_meta in LogConfigMgr - WLockGuard guard(lock_); - const bool skip_log_barrier = mode_mgr_.need_skip_log_barrier(); - if (false == state_mgr_.can_receive_config_log(msg_proposal_id) || - false == config_mgr_.can_receive_config_log(server, meta)) { - ret = OB_STATE_NOT_MATCH; - if (REACH_TIME_INTERVAL(100 * 1000)) { - PALF_LOG(WARN, "can not receive log", KR(ret), KPC(this), K(msg_proposal_id), "role", state_mgr_.get_role()); - } - } else if (FALSE_IT(receive_newer_config_log = true)) { - } else if (mode_mgr_.get_accepted_mode_meta().proposal_id_ < prev_mode_pid) { - // need fetch mode_meta - if (OB_FAIL(sw_.try_fetch_log(FetchTriggerType::MODE_META_BARRIER))) { - PALF_LOG(WARN, "try_fetch_log with MODE_META_BARRIER failed", - KR(ret), KPC(this), K(server), K(msg_proposal_id), K(prev_mode_pid), K(meta)); - } else { - PALF_LOG(INFO, "pre_check_for_mode_meta don't match, try fetch mode meta", - KR(ret), KPC(this), K(server), K(msg_proposal_id), K(prev_mode_pid), K(meta)); - } - } else if (!skip_log_barrier && !sw_.pre_check_for_config_log(msg_proposal_id, prev_lsn, prev_log_proposal_id, truncate_log_info)) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "pre_check_for_config_log failed, cannot receive config log", - KR(ret), KPC(this), K(server), K(msg_proposal_id), K(prev_lsn), K(prev_log_proposal_id), K(meta)); - } else if (!skip_log_barrier - && TRUNCATE_LOG == truncate_log_info.truncate_type_ - && OB_FAIL(sw_.truncate(truncate_log_info, prev_lsn, prev_log_proposal_id))) { - PALF_LOG(WARN, "sw truncate failed", KR(ret), KPC(this), K(truncate_log_info)); - } else if (OB_FAIL(config_mgr_.receive_config_log(server, meta))) { - PALF_LOG(WARN, "receive_config_log failed", KR(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_log_proposal_id), K(prev_lsn)); - } else { - PALF_LOG(INFO, "receive_config_log success", KR(ret), KPC(this), K(server), K(msg_proposal_id), - K(prev_lsn), K(prev_log_proposal_id), K(meta)); - } - } - if (receive_newer_config_log) { - RLockGuard guard(lock_); - const bool in_paxos_member_list = meta.curr_.config_.log_sync_memberlist_.contains(self_) || - meta.curr_.config_.degraded_learnerlist_.contains(self_); - // 1. self is not in paxos memberlist, try register parent - if (!in_paxos_member_list && OB_FAIL(config_mgr_.register_parent())) { - // overwrite ret - PALF_LOG(WARN, "register_parent failed", KPC(this), K(server), K(meta)); - } - // 2. self is in paxos memberlist, try retire parent - if (in_paxos_member_list && OB_FAIL(config_mgr_.retire_parent())) { - // overwrite ret - PALF_LOG(WARN, "register_parent failed", KPC(this), K(server), K(meta)); - } - } - return ret; -} - -int PalfHandleImpl::ack_config_log(const common::ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - bool is_majority = false; - do { - RLockGuard guard(lock_); - const int64_t &curr_proposal_id = state_mgr_.get_proposal_id(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret)); - } else if (msg_proposal_id != curr_proposal_id) { - PALF_LOG(WARN, "proposal_id does not match", KR(ret), KPC(this), K(msg_proposal_id), - K(server), K(curr_proposal_id)); - } else if (self_ != state_mgr_.get_leader()) { - ret = OB_STATE_NOT_MATCH; - PALF_LOG(WARN, "self is not leader, state not match", KR(ret), KPC(this), - K(server), K(msg_proposal_id), K(curr_proposal_id)); - } else if (OB_FAIL(config_mgr_.ack_config_log(server, msg_proposal_id, config_version, is_majority))) { - PALF_LOG(WARN, "ObLogConfigMgr ack_config_log failed", KR(ret), KPC(this), K(server), K(msg_proposal_id), K(config_version)); - } else { - PALF_LOG(INFO, "ack_config_log success", KR(ret), KPC(this), K(server), K(msg_proposal_id), K(config_version)); - } - } while (0); - if (is_majority) { - WLockGuard guard(lock_); - (void) config_mgr_.after_config_log_majority(msg_proposal_id, config_version); + palf_epoch = log_engine_.get_palf_epoch(); } return ret; } @@ -3942,30 +1007,11 @@ int PalfHandleImpl::inner_after_flush_meta(const FlushMetaCbCtx &flush_meta_cb_c PALF_LOG(INFO, "inner_after_flush_meta", K(flush_meta_cb_ctx)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (MODE_META == flush_meta_cb_ctx.type_) { - WLockGuard guard(lock_); - ret = after_flush_mode_meta_(flush_meta_cb_ctx.proposal_id_, - flush_meta_cb_ctx.is_applied_mode_meta_, - flush_meta_cb_ctx.log_mode_meta_); + } else if (SNAPSHOT_META != flush_meta_cb_ctx.type_) { + ret = OB_ERR_UNEXPECTED; } else { RLockGuard guard(lock_); - switch(flush_meta_cb_ctx.type_) { - case PREPARE_META: - ret = after_flush_prepare_meta_(flush_meta_cb_ctx.proposal_id_); - break; - case CHANGE_CONFIG_META: - ret = after_flush_config_change_meta_(flush_meta_cb_ctx.proposal_id_, flush_meta_cb_ctx.config_version_); - break; - case SNAPSHOT_META: - ret = after_flush_snapshot_meta_(flush_meta_cb_ctx.base_lsn_); - break; - case REPLICA_PROPERTY_META: - ret = after_flush_replica_property_meta_(flush_meta_cb_ctx.allow_vote_); - break; - default: - ret = OB_ERR_UNEXPECTED; - break; - } + ret = after_flush_snapshot_meta_(flush_meta_cb_ctx.base_lsn_); } return ret; } @@ -3980,142 +1026,11 @@ int PalfHandleImpl::inner_after_truncate_prefix_blocks(const TruncatePrefixBlock return ret; } -int PalfHandleImpl::after_flush_prepare_meta_(const int64_t &proposal_id) -{ - // caller holds lock - int ret = OB_SUCCESS; - const ObAddr &leader = state_mgr_.get_leader(); - if (proposal_id != state_mgr_.get_proposal_id()) { - PALF_LOG(WARN, "proposal_id has changed during flushing", K(ret), K(proposal_id), - "curr_proposal_id", state_mgr_.get_proposal_id()); - } else if (false == leader.is_valid() || self_ == leader) { - // leader is self, no need submit response - PALF_LOG(INFO, "do not need submit prepare response to self", K_(self), "leader:", leader); - } else if (OB_FAIL(submit_prepare_response_(leader, proposal_id))) { - PALF_LOG(WARN, "submit_prepare_response_ failed", K(ret), K(proposal_id), "leader", leader); - } else { - PALF_LOG(INFO, "after_flush_prepare_meta_ success", K(proposal_id)); - // do nothing - } - return ret; -} - -int PalfHandleImpl::after_flush_config_change_meta_(const int64_t proposal_id, const LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - const ObAddr &leader = state_mgr_.get_leader(); - if (proposal_id != state_mgr_.get_proposal_id()) { - PALF_LOG(WARN, "proposal_id has changed during flushing", K(ret), K_(palf_id), K_(self), K(proposal_id), - "curr_proposal_id", state_mgr_.get_proposal_id()); - } else if (OB_FAIL(config_mgr_.after_flush_config_log(config_version))) { - PALF_LOG(WARN, "LogConfigMgr after_flush_config_log failed", K(ret), KPC(this), K(proposal_id), - K(config_version)); - } else if (self_ == leader) { - bool unused_bool = false; - if (OB_FAIL(config_mgr_.ack_config_log(self_, proposal_id, config_version, unused_bool))) { - PALF_LOG(WARN, "ack_config_log failed", K(ret), KPC(this), K(config_version)); - } - } else if (false == leader.is_valid()) { - PALF_LOG(WARN, "leader is invalid", KPC(this), K(proposal_id), K(config_version)); - } else if (OB_FAIL(log_engine_.submit_change_config_meta_resp(leader, proposal_id, config_version))) { - PALF_LOG(WARN, "submit_change_config_meta_resp failed", K(ret), KPC(this), K(proposal_id), K(config_version)); - } else { - PALF_LOG(INFO, "submit_change_config_meta_resp success", K(ret), KPC(this), K(proposal_id), K(config_version)); - } - return ret; -} -// 1. Update snapshot_meta serialization, implement inc update semantics. -// 2. The application layer first submits the truncate task, then submits the update meta task, and finally starts pulling logs. (No longer dependent, the upper layer guarantees that base_lsn will not roll back) -// NB: TODO by runlin, after supporting multiple writers, truncate and updating meta need to be made with 'similar' two-way barrier semantics int PalfHandleImpl::after_flush_snapshot_meta_(const LSN &lsn) { return log_engine_.update_base_lsn_used_for_gc(lsn); } -int PalfHandleImpl::after_flush_replica_property_meta_(const bool allow_vote) -{ - return (true == allow_vote)? state_mgr_.enable_vote(): state_mgr_.disable_vote(); -} - -int PalfHandleImpl::after_flush_mode_meta_(const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) -{ - int ret = OB_SUCCESS; - const ObAddr &leader = state_mgr_.get_leader(); - if (proposal_id != state_mgr_.get_proposal_id()) { - PALF_LOG(WARN, "proposal_id has changed during flushing", K(ret), K(proposal_id), - "curr_proposal_id", state_mgr_.get_proposal_id()); - } else if (OB_FAIL(mode_mgr_.after_flush_mode_meta(is_applied_mode_meta, mode_meta))) { - PALF_LOG(WARN, "after_flush_mode_meta failed", K(ret), K_(palf_id), K(proposal_id), K(mode_meta)); - } else if (self_ == leader) { - if (OB_FAIL(mode_mgr_.ack_mode_meta(self_, mode_meta.proposal_id_))) { - PALF_LOG(WARN, "ack_mode_meta failed", K(ret), K_(palf_id), K_(self), K(proposal_id)); - } - } else if (false == leader.is_valid()) { - PALF_LOG(WARN, "leader is invalid", KPC(this), K(proposal_id)); - } else if (OB_FAIL(log_engine_.submit_change_mode_meta_resp(leader, proposal_id))) { - PALF_LOG(WARN, "submit_change_mode_meta_resp failed", K(ret), K_(self), K(leader), K(proposal_id)); - } else { - PALF_LOG(INFO, "submit_change_mode_meta_resp success", K(ret), K_(self), K(leader), K(proposal_id)); - } - return ret; -} - -int PalfHandleImpl::inner_after_truncate_log(const TruncateLogCbCtx &truncate_log_cb_ctx) -{ - int ret = OB_SUCCESS; - WLockGuard guard(lock_); - if (OB_FAIL(sw_.after_truncate(truncate_log_cb_ctx))) { - PALF_LOG(WARN, "inner_after_truncate_log failed", K(ret), K(truncate_log_cb_ctx)); - } else { - PALF_LOG(INFO, "after_truncate_log success", K(ret), K_(self), K_(palf_id)); - } - return ret; -} - -int PalfHandleImpl::get_prev_log_info_for_fetch_(const LSN &prev_lsn, - const LSN &curr_lsn, - LogInfo &prev_log_info) -{ - int ret = OB_SUCCESS; - PalfGroupBufferIterator iterator; - auto get_file_end_lsn = [&]() { return curr_lsn; }; - auto get_mode_version = [this]() -> int64_t { - int64_t mode_version = INVALID_PROPOSAL_ID; - int ret = OB_SUCCESS; - if (OB_FAIL(this->get_access_mode_version(mode_version))) { - PALF_LOG(WARN, "get_access_mode_version failed", K(ret), KPC(this)); - mode_version = INVALID_PROPOSAL_ID; - } - return mode_version; - }; - if (OB_FAIL(iterator.init(prev_lsn, get_file_end_lsn, get_mode_version, log_engine_.get_log_storage()))) { - PALF_LOG(WARN, "LogGroupEntryIterator init failed", K(ret), K(iterator), K(prev_lsn), K(curr_lsn)); - } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(palf_id_, LogIOUser::FETCHLOG)))) { - PALF_LOG(WARN, "LogGroupEntryIterator set_io_context failed", K(ret), K(iterator), K(prev_lsn), K(curr_lsn)); - } else if (FALSE_IT(iterator.set_need_print_error(false))) { - } else { - LogGroupEntry entry; - LSN lsn; - if (OB_SUCC(iterator.next())) { - if (OB_FAIL(iterator.get_entry(entry, lsn))) { - PALF_LOG(ERROR, "get_entry failed", K(ret), K(iterator)); - } else { - const LogGroupEntryHeader &header = entry.get_header(); - prev_log_info.log_id_ = header.get_log_id(); - prev_log_info.scn_ = header.get_max_scn(); - prev_log_info.accum_checksum_ = header.get_accum_checksum(); - prev_log_info.log_proposal_id_ = header.get_log_proposal_id(); - prev_log_info.lsn_ = prev_lsn; - } - } else if (OB_FAIL(get_prev_log_info_(curr_lsn, prev_log_info))) { - PALF_LOG(WARN, "get_prev_log_info_ failed", K(ret), KPC(this)); - } - } - return ret; -} - int PalfHandleImpl::get_prev_log_info_(const LSN &lsn, LogInfo &prev_log_info) { @@ -4140,7 +1055,7 @@ int PalfHandleImpl::get_prev_log_info_(const LSN &lsn, PALF_LOG(INFO, "lsn is same as base_lsn, and log_snapshot_meta is valid", K(lsn), K(log_snapshot_meta)); } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { PALF_LOG(WARN, "LogGroupEntryIterator init failed", K(ret), K(start_lsn), K(lsn)); - } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(palf_id_, LogIOUser::FETCHLOG)))) { + } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(LogIOUser::FETCHLOG)))) { PALF_LOG(WARN, "LogGroupEntryIterator set_io_context failed", K(ret), K(start_lsn), K(lsn)); } else { LSN curr_lsn; @@ -4175,7 +1090,6 @@ int PalfHandleImpl::get_prev_log_info_(const LSN &lsn, prev_log_info.log_id_ = prev_entry_header.get_log_id(); prev_log_info.scn_ = prev_entry_header.get_max_scn(); prev_log_info.accum_checksum_ = prev_entry_header.get_accum_checksum(); - prev_log_info.log_proposal_id_ = prev_entry_header.get_log_proposal_id(); prev_log_info.lsn_ = prev_lsn; ret = OB_SUCCESS; } @@ -4188,45 +1102,6 @@ int PalfHandleImpl::get_prev_log_info_(const LSN &lsn, return ret; } -int PalfHandleImpl::submit_prepare_response_(const common::ObAddr &server, - const int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - const bool vote_granted = true; - LSN unused_prev_lsn; - LSN max_flushed_end_lsn; - int64_t max_flushed_log_pid = INVALID_PROPOSAL_ID; - LSN committed_end_lsn; - if (!is_inited_) { - ret = OB_NOT_INIT; - } else if (!server.is_valid() || INVALID_PROPOSAL_ID == proposal_id) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(server), K(proposal_id)); - } else if (OB_FAIL(sw_.get_max_flushed_log_info(unused_prev_lsn, max_flushed_end_lsn, max_flushed_log_pid))) { - PALF_LOG(WARN, "get_max_flushed_log_info failed", K(ret), K_(palf_id)); - } else if (OB_FAIL(sw_.get_committed_end_lsn(committed_end_lsn))) { - PALF_LOG(WARN, "get_committed_end_lsn failed", K(ret), K_(palf_id)); - } else { - const LogModeMeta accepted_mode_meta = mode_mgr_.get_accepted_mode_meta(); - // the last log info include proposal_id and lsn, the proposal_id should be the maxest - // of redo lod and config log. - // because config log and redo log are stored separately - int64_t accept_proposal_id = config_mgr_.get_accept_proposal_id(); - if (INVALID_PROPOSAL_ID != max_flushed_log_pid) { - accept_proposal_id = MAX(config_mgr_.get_accept_proposal_id(), max_flushed_log_pid); - } - if (OB_FAIL(log_engine_.submit_prepare_meta_resp(server, proposal_id, vote_granted, accept_proposal_id, - max_flushed_end_lsn, committed_end_lsn, accepted_mode_meta))) { - PALF_LOG(WARN, "submit_prepare_response failed", K(ret), K_(palf_id)); - } else { - PALF_LOG(INFO, "submit_prepare_response success", K(ret), K_(palf_id), K_(self), K(server), - K(vote_granted), K(accept_proposal_id), K(max_flushed_end_lsn), K(committed_end_lsn), - K(accepted_mode_meta)); - } - } - return ret; -} - int PalfHandleImpl::construct_palf_base_info_(const LSN &max_committed_lsn, PalfBaseInfo &palf_base_info) { @@ -4235,7 +1110,7 @@ int PalfHandleImpl::construct_palf_base_info_(const LSN &max_committed_lsn, const LSN base_lsn = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_; if (false == max_committed_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K_(palf_id), K(max_committed_lsn), K(base_lsn)); + PALF_LOG(WARN, "invalid argument", K(ret), K(max_committed_lsn), K(base_lsn)); // NB: // 1. for rebuild, there may be no valid block on disk, however, the 'prev_log_info' has been saved // in LogMeta, if 'max_committed_end_lsn' is same as 'base_lsn', we can construct PalfBaseInfo @@ -4257,15 +1132,12 @@ int PalfHandleImpl::append_disk_log_to_sw_(const LSN &start_lsn) { int ret = OB_SUCCESS; PalfGroupBufferIterator iterator; - // NB: we need append each log locate in [start_lsn, log_tail_); - auto get_file_end_lsn = [&](){ return LSN(LOG_MAX_LSN_VAL); }; - // if there is no valid data on disk, the 'start_lsn' which meash max committed log offset - // of this log stream may be invalid. + auto get_file_end_lsn = []() { return LSN(LOG_MAX_LSN_VAL); }; if (false == start_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { PALF_LOG(WARN, "PalfGroupBufferIterator init failed", K(ret), K(log_engine_), K(start_lsn)); - } else if (OB_FAIL(iterator.set_io_context(palf::LogIOContext(LogIOUser::RESTART)))) { + } else if (OB_FAIL(iterator.set_io_context(LogIOContext(LogIOUser::RESTART)))) { PALF_LOG(WARN, "set_io_context failed", K(ret), KPC(this), K(start_lsn)); } else { LogGroupEntry group_entry; @@ -4275,7 +1147,6 @@ int PalfHandleImpl::append_disk_log_to_sw_(const LSN &start_lsn) PALF_LOG(ERROR, "get_entry failed", K(ret), K(group_entry), K(lsn)); } else if (OB_FAIL(sw_.append_disk_log(lsn, group_entry))) { PALF_LOG(WARN, "append_disk_log failed", K(ret), K(lsn), K(group_entry)); - } else { } } if (OB_ITER_END == ret) { @@ -4286,78 +1157,10 @@ int PalfHandleImpl::append_disk_log_to_sw_(const LSN &start_lsn) return ret; } -int PalfHandleImpl::get_election_leader_without_lock_(ObAddr &addr) const -{ - int ret = OB_SUCCESS; - int64_t unused_leader_epoch = -1; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); - } else if (OB_FAIL(election_.get_current_leader_likely(addr, unused_leader_epoch))) { - PALF_LOG(WARN, "get_election_leader failed", K(ret), KPC(this)); - } else if (OB_UNLIKELY(!addr.is_valid())) { - ret = OB_LEADER_NOT_EXIST; - PALF_LOG(WARN, "election has no leader", K(ret), KPC(this)); - } - return ret; -} - -int PalfHandleImpl::advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, - const int64_t downgrade_priority_time_us, - const char *reason) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not inited", K(ret), K_(palf_id), K(reason)); - } else if (false == state_mgr_.can_revoke(proposal_id)) { - ret = OB_NOT_MASTER; - PALF_LOG(WARN, "advance election epoch failed, not master", K(ret), K_(palf_id), K(proposal_id), K(reason)); - } else if (OB_FAIL(election_.change_leader_to(GCTX.self_addr()))) { - PALF_LOG(WARN, "fail to change leader to self", K(ret), K_(palf_id), K(proposal_id), K(reason)); - } else if (OB_FAIL(election_.temporarily_downgrade_protocol_priority(downgrade_priority_time_us, reason))) { - PALF_LOG(WARN, "PalfHandleImpl revoke leader failed", K(ret), K_(palf_id), K(reason)); - } else { - PALF_LOG(INFO, "PalfHandleImpl revoke leader success", K(ret), K_(palf_id), K(reason)); - } - return ret; -} - int PalfHandleImpl::diagnose(PalfDiagnoseInfo &diagnose_info) const { int ret = OB_SUCCESS; - state_mgr_.get_role_and_state(diagnose_info.palf_role_, diagnose_info.palf_state_); - diagnose_info.palf_proposal_id_ = state_mgr_.get_proposal_id(); - state_mgr_.get_election_role(diagnose_info.election_role_, diagnose_info.election_epoch_); - diagnose_info.enable_sync_ = state_mgr_.is_sync_enabled(); - diagnose_info.enable_vote_ = state_mgr_.is_allow_vote(); - diagnose_info.parent_ = config_mgr_.get_parent(); - return ret; -} - -// TODO by yunlong: this function needs refactoring in 4.2.0.0 -int PalfHandleImpl::get_ack_info_array(LogMemberAckInfoList &ack_info_array, - common::GlobalLearnerList °raded_list) const -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - const bool is_leader = (state_mgr_.is_leader_active() || - (state_mgr_.is_leader_reconfirm() && reconfirm_.can_do_degrade())); - common::ObMember arb_member; - const bool need_degrade_or_upgrade = arb_member.is_valid(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not inited!!!", K(ret)); - } else if (false == is_leader) { - ret = OB_NOT_MASTER; - } else if (false == need_degrade_or_upgrade) { - // do not need degrade or upgrade, skip - } else if (OB_FAIL(sw_.get_ack_info_array(ack_info_array))) { - PALF_LOG(WARN, "get_ack_info_array failed", K(ret), KPC(this)); - } else if (OB_FAIL(config_mgr_.get_degraded_learner_list(degraded_list))) { - PALF_LOG(WARN, "get_degraded_learner_list failed", K(ret), KPC(this)); - } + diagnose_info.log_state_ = state_mgr_.get_state(); return ret; } @@ -4367,30 +1170,12 @@ int PalfHandleImpl::stat(PalfStat &palf_stat) if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - LSN last_rebuild_lsn; - do { - SpinLockGuard guard(last_rebuild_meta_info_lock_); - last_rebuild_lsn = last_rebuild_lsn_; - } while (0); - // following members should be protected by rlock_ RLockGuard guard(lock_); block_id_t min_block_id = LOG_INVALID_BLOCK_ID; SCN min_block_min_scn; - ObRole curr_role = INVALID_ROLE; - ObReplicaState curr_state = INVALID_STATE; palf_stat.self_ = self_; - palf_stat.palf_id_ = palf_id_; - state_mgr_.get_role_and_state(curr_role, curr_state); - palf_stat.role_ = (LEADER == curr_role && curr_state == ACTIVE)? LEADER: FOLLOWER; - palf_stat.log_proposal_id_ = state_mgr_.get_proposal_id(); - (void)config_mgr_.get_config_version(palf_stat.config_version_); - (void)mode_mgr_.get_access_mode(palf_stat.mode_version_, palf_stat.access_mode_); - (void)config_mgr_.get_curr_member_list(palf_stat.paxos_member_list_, palf_stat.paxos_replica_num_); - (void)config_mgr_.get_degraded_learner_list(palf_stat.degraded_list_); - (void)config_mgr_.get_global_learner_list(palf_stat.learner_list_); - palf_stat.allow_vote_ = state_mgr_.is_allow_vote(); - palf_stat.replica_type_ = state_mgr_.get_replica_type(); + (void)mode_mgr_.get_access_mode(palf_stat.access_mode_); palf_stat.base_lsn_ = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_; (void)log_engine_.get_min_block_info(min_block_id, min_block_min_scn); palf_stat.begin_lsn_ = LSN(min_block_id * PALF_BLOCK_SIZE); @@ -4399,400 +1184,33 @@ int PalfHandleImpl::stat(PalfStat &palf_stat) palf_stat.end_scn_ = get_end_scn(); palf_stat.max_lsn_ = get_max_lsn(); palf_stat.max_scn_ = get_max_scn(); - palf_stat.is_need_rebuild_ = is_need_rebuild(palf_stat.end_lsn_, last_rebuild_lsn); - palf_stat.is_in_sync_ = (LEADER == palf_stat.role_)? true: cached_is_in_sync_; PALF_LOG(TRACE, "PalfHandleImpl stat", K(palf_stat)); } return OB_SUCCESS; } -int PalfHandleImpl::update_palf_stat() -{ - int ret = OB_SUCCESS; - bool is_in_sync = false; - bool is_use_sync_cache = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else { - is_in_sync_(is_in_sync, is_use_sync_cache); - if (false == is_use_sync_cache) { - cached_is_in_sync_ = is_in_sync; - } - } - return OB_SUCCESS; -} - -int PalfHandleImpl::try_lock_config_change(int64_t lock_owner, int64_t timeout_us) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(lock_owner <= 0 || timeout_us <= 0)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(lock_owner), K(timeout_us)); - } else { - LogConfigChangeArgs args(lock_owner, ConfigChangeLockType::LOCK_PAXOS_MEMBER_CHANGE, TRY_LOCK_CONFIG_CHANGE); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "try_lock_config_change failed", KR(ret), KPC(this), K(lock_owner)); - } else { - PALF_EVENT("try_lock_config_change success", palf_id_, KR(ret), KPC(this), K(lock_owner)); - } - } - return ret; -} - -int PalfHandleImpl::unlock_config_change(int64_t lock_owner, int64_t timeout_us) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(lock_owner <= 0 || timeout_us <= 0)) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(lock_owner), K(timeout_us)); - } else { - LogConfigChangeArgs args(lock_owner, ConfigChangeLockType::LOCK_NOTHING, UNLOCK_CONFIG_CHANGE); - if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { - PALF_LOG(WARN, "unlock_config_change failed", KR(ret), KPC(this), K(lock_owner)); - } else { - PALF_EVENT("unlock_config_change success", palf_id_, KR(ret), KPC(this), K(lock_owner)); - } - } - return ret; -} - -int PalfHandleImpl::get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImpl has not inited", K(ret)); - } else if (OB_FAIL(config_mgr_.get_config_change_lock_stat(lock_owner, is_locked))) { - PALF_LOG(WARN, "get_curr_member_list failed", K(ret), KPC(this)); - } else {} - return ret; -} - -void PalfHandleImpl::is_in_sync_(bool &is_log_sync, bool &is_use_cache) -{ - int ret = OB_SUCCESS; - SCN leader_max_scn; - LSN leader_end_lsn; - is_log_sync = false; - is_use_cache = false; - share::SCN local_max_scn = sw_.get_max_scn(); - LSN local_end_lsn; - - if (state_mgr_.get_leader() == self_) { - is_log_sync = true; - } else if (false == local_max_scn.is_valid()) { - } else if (palf_reach_time_interval(PALF_LOG_SYNC_DELAY_THRESHOLD_US, last_check_sync_time_us_)) { - // if reachs time interval, get max_scn of leader with sync RPC - if (OB_FAIL(get_leader_max_scn_(leader_max_scn, leader_end_lsn))) { - CLOG_LOG(WARN, "get_palf_max_scn failed", K(ret), K_(self), K_(palf_id)); - last_check_sync_time_us_ = OB_INVALID_TIMESTAMP; - } else if (leader_max_scn.is_valid() && leader_end_lsn.is_valid()) { - sw_.get_committed_end_lsn(local_end_lsn); - const bool is_scn_sync = (leader_max_scn.convert_to_ts() - local_max_scn.convert_to_ts() <= PALF_LOG_SYNC_DELAY_THRESHOLD_US); - const bool is_log_size_sync = (leader_end_lsn - local_end_lsn) < 2 * PALF_BLOCK_SIZE; - // Note: do not consider the gap of LSN (120 MB is too much to catch up for migration dest) - UNUSED(is_log_size_sync); - is_log_sync = is_scn_sync; - } - } else { - is_use_cache = true; - } - - const bool is_in_sync = (is_use_cache) ? cached_is_in_sync_ : is_log_sync; - const int64_t log_print_interval = (is_in_sync)? 600 * 1000 * 1000: 10 * 1000 * 1000; - if (palf_reach_time_interval(log_print_interval, last_print_in_sync_time_us_)) { - CLOG_LOG(INFO, "is_in_sync", K(ret), K_(palf_id), K(is_in_sync), K(is_use_cache), - "remote_check_is_log_sync", is_log_sync, K(leader_max_scn), K(local_max_scn)); - } -} - -int PalfHandleImpl::get_leader_max_scn_(SCN &max_scn, LSN &end_lsn) -{ - int ret = OB_SUCCESS; - common::ObAddr leader; - LogGetStatReq req(LogGetStatType::GET_LEADER_MAX_SCN); - LogGetStatResp resp; - bool need_renew_leader = false; - - max_scn.reset(); - end_lsn.reset(); - // use lc_cb_ in here without rlock is safe, because we don't reset lc_cb_ - // until this PalfHandleImpl is destoryed. - if (OB_FAIL(plugins_.nonblock_get_leader(palf_id_, leader))) { - CLOG_LOG(WARN, "get_leader failed", K(ret), K_(self), K_(palf_id)); - need_renew_leader = true; - } else if (false == leader.is_valid()) { - need_renew_leader = true; - } else if (OB_FAIL(log_engine_.submit_get_stat_req(leader, PALF_SYNC_RPC_TIMEOUT_US, req, resp))) { - CLOG_LOG(WARN, "get_palf_max_scn failed", K(ret), K_(palf_id)); - need_renew_leader = true; - } else { - max_scn = resp.max_scn_; - end_lsn = resp.end_lsn_; - } - return ret; -} - -void PalfHandleImpl::report_set_initial_member_list_(const int64_t paxos_replica_num, const common::ObMemberList &member_list) -{ - ObSqlString member_list_buf; - (void) member_list_to_string(member_list, member_list_buf); - plugins_.record_set_initial_member_list_event(palf_id_, paxos_replica_num, member_list_buf.ptr()); -} -void PalfHandleImpl::report_force_set_as_single_replica_(const int64_t prev_replica_num, const int64_t curr_replica_num, const ObMember &member) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObSqlString member_buf; - member_to_string(member, member_buf); - PALF_REPORT_INFO_KV("member", member_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::FORCE_SINGLE_MEMBER), - palf_id_, config_version, prev_replica_num, curr_replica_num, EXTRA_INFOS); -} - -void PalfHandleImpl::report_force_set_member_list(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMemberList &member_list) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObSqlString member_list_buf; - (void) member_list_to_string(member_list, member_list_buf); - PALF_REPORT_INFO_KV("member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::FORCE_SET_MEMBER_LIST), - palf_id_, config_version, prev_replica_num, curr_replica_num, EXTRA_INFOS); -} - -void PalfHandleImpl::report_change_replica_num_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMemberList &member_list) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObSqlString member_list_buf; - (void) member_list_to_string(member_list, member_list_buf); - PALF_REPORT_INFO_KV("member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::CHANGE_REPLICA_NUM), - palf_id_, config_version, prev_replica_num, curr_replica_num, EXTRA_INFOS); -} -void PalfHandleImpl::report_add_member_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMember &added_member) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObMemberList curr_member_list; - int64_t replica_num; - config_mgr_.get_curr_member_list(curr_member_list, replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(added_member, member_buf); - PALF_REPORT_INFO_KV( - "added_member", member_buf, - "member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::ADD_MEMBER), - palf_id_, config_version, prev_replica_num, curr_replica_num, EXTRA_INFOS); -} -void PalfHandleImpl::report_remove_member_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMember &removed_member) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObMemberList curr_member_list; - int64_t replica_num; - config_mgr_.get_curr_member_list(curr_member_list, replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(removed_member, member_buf); - PALF_REPORT_INFO_KV( - "removed_member", member_buf, - "member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::REMOVE_MEMBER), - palf_id_, config_version, prev_replica_num, curr_replica_num, EXTRA_INFOS); -} -void PalfHandleImpl::report_replace_member_(const common::ObMember &added_member, - const common::ObMember &removed_member, - const common::ObMemberList &member_list, - const char *event_name) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObSqlString member_list_buf; - (void) member_list_to_string(member_list, member_list_buf); - ObSqlString added_member_buf; - ObSqlString removed_member_buf; - member_to_string(added_member, added_member_buf); - member_to_string(removed_member, removed_member_buf); - PALF_REPORT_INFO_KV( - "added_member", added_member_buf, - "removed_member", removed_member_buf, - "member_list", member_list_buf); - int64_t curr_replica_num; - (void) config_mgr_.get_replica_num(curr_replica_num); - plugins_.record_reconfiguration_event(event_name, palf_id_, config_version, curr_replica_num, curr_replica_num, EXTRA_INFOS); -} -void PalfHandleImpl::report_add_learner_(const common::ObMember &added_learner) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - common::ObMemberList curr_member_list; - int64_t curr_replica_num; - (void) config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(added_learner, member_buf); - PALF_REPORT_INFO_KV( - "added_learner", member_buf, - "member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::ADD_LEARNER), - palf_id_, config_version, curr_replica_num, curr_replica_num, EXTRA_INFOS); -} -void PalfHandleImpl::report_remove_learner_(const common::ObMember &removed_learner) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - common::ObMemberList curr_member_list; - int64_t curr_replica_num; - (void) config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(removed_learner, member_buf); - PALF_REPORT_INFO_KV( - "removed_learner", member_buf, - "member_list", member_list_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::REMOVE_LEARNER), - palf_id_, config_version, curr_replica_num, curr_replica_num, EXTRA_INFOS); -} - -void PalfHandleImpl::report_switch_learner_to_acceptor_(const common::ObMember &learner) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObMemberList curr_member_list; - int64_t curr_replica_num; - (void) config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(learner, member_buf); - PALF_REPORT_INFO_KV( - "member", member_buf, - "curr_member_list", member_list_buf, - "curr_replica_num", curr_replica_num); - const char *replica_readonly_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY); - const char *replica_full_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL); - plugins_.record_replica_type_change_event(palf_id_, config_version, replica_readonly_name_, replica_full_name_, EXTRA_INFOS); -} - -void PalfHandleImpl::report_switch_acceptor_to_learner_(const common::ObMember &acceptor) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - ObMemberList curr_member_list; - int64_t curr_replica_num; - (void) config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num); - ObSqlString member_list_buf; - (void) member_list_to_string(curr_member_list, member_list_buf); - ObSqlString member_buf; - member_to_string(acceptor, member_buf); - PALF_REPORT_INFO_KV( - "member", member_buf, - "curr_member_list", member_list_buf, - "curr_replica_num", curr_replica_num); - const char *replica_readonly_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY); - const char *replica_full_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL); - plugins_.record_replica_type_change_event(palf_id_, config_version, replica_full_name_, replica_readonly_name_, EXTRA_INFOS); -} - -void PalfHandleImpl::report_replace_learners_(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners) -{ - LogConfigVersion config_version; - (void) config_mgr_.get_config_version(config_version); - common::ObMemberList curr_member_list; - int64_t curr_replica_num; - (void) config_mgr_.get_curr_member_list(curr_member_list, curr_replica_num); - ObSqlString added_learners_buf, removed_learners_buf; - member_list_to_string(added_learners, added_learners_buf); - member_list_to_string(removed_learners, removed_learners_buf); - PALF_REPORT_INFO_KV( - "added_learners", added_learners_buf, - "removed_learners", removed_learners_buf); - plugins_.record_reconfiguration_event(LogConfigChangeType2Str(LogConfigChangeType::REPLACE_LEARNERS), - palf_id_, config_version, curr_replica_num, curr_replica_num, EXTRA_INFOS); -} - -bool PalfHandleImpl::check_need_hook_fetch_log_(const FetchLogType fetch_type, const LSN &start_lsn) -{ - bool bool_ret = false; - const int64_t rebuild_replica_log_lag_threshold = palf_env_impl_->get_rebuild_replica_log_lag_threshold(); - if (rebuild_replica_log_lag_threshold > 0 && (FETCH_LOG_FOLLOWER == fetch_type)) { - LSN max_lsn = get_max_lsn(); - LSN base_lsn = get_base_lsn_used_for_block_gc(); - bool_ret = (start_lsn < base_lsn) && ((max_lsn - start_lsn) > rebuild_replica_log_lag_threshold); - - if (bool_ret && palf_reach_time_interval(1 * 1000 * 1000L, last_hook_fetch_log_time_us_)) { - PALF_LOG(INFO, "hook fetch_log because of rebuild_replica_log_lag_threshold", K(palf_id_), - K(rebuild_replica_log_lag_threshold), K(start_lsn), K(max_lsn), K(base_lsn)); - } - } - return bool_ret; -} - PalfStat::PalfStat() : self_(), - palf_id_(INVALID_PALF_ID), - role_(common::ObRole::INVALID_ROLE), - log_proposal_id_(INVALID_PROPOSAL_ID), - config_version_(), - mode_version_(INVALID_PROPOSAL_ID), access_mode_(AccessMode::INVALID_ACCESS_MODE), - paxos_member_list_(), - paxos_replica_num_(-1), - arbitration_member_(), - degraded_list_(), - allow_vote_(true), - replica_type_(LogReplicaType::INVALID_REPLICA), begin_lsn_(), begin_scn_(), base_lsn_(), end_lsn_(), end_scn_(), max_lsn_(), - max_scn_(), - is_in_sync_(false), - is_need_rebuild_(false) { } + max_scn_() +{} bool PalfStat::is_valid() const { return self_.is_valid() && - palf_id_ != INVALID_PALF_ID && - role_ != common::ObRole::INVALID_ROLE; + access_mode_ != AccessMode::INVALID_ACCESS_MODE; } void PalfStat::reset() { self_.reset(); - palf_id_ = INVALID_PALF_ID; - role_ = common::ObRole::INVALID_ROLE; - log_proposal_id_ = INVALID_PROPOSAL_ID; - config_version_.reset(); - mode_version_ = INVALID_PROPOSAL_ID; access_mode_ = AccessMode::INVALID_ACCESS_MODE; - paxos_member_list_.reset(); - paxos_replica_num_ = -1; - learner_list_.reset(); - arbitration_member_.reset(); - degraded_list_.reset(); - allow_vote_ = true; - replica_type_ = LogReplicaType::INVALID_REPLICA; begin_lsn_.reset(); begin_scn_.reset(); base_lsn_.reset(); @@ -4800,8 +1218,6 @@ void PalfStat::reset() end_scn_.reset(); max_lsn_.reset(); max_scn_.reset(); - is_in_sync_ = false; - is_need_rebuild_ = false; } int PalfHandleImpl::read_data_from_buffer(const LSN &read_begin_lsn, @@ -4814,15 +1230,15 @@ int PalfHandleImpl::read_data_from_buffer(const LSN &read_begin_lsn, ret = OB_NOT_INIT; } else if (!read_begin_lsn.is_valid() || in_read_size <= 0 || OB_ISNULL(buf)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size), + PALF_LOG(WARN, "invalid arguments", K(ret), K(read_begin_lsn), K(in_read_size), KP(buf)); } else if (OB_FAIL(sw_.read_data_from_buffer(read_begin_lsn, in_read_size, buf, out_read_size))) { if (OB_ERR_OUT_OF_LOWER_BOUND != ret) { - PALF_LOG(WARN, "read_data_from_buffer failed", K(ret), K_(palf_id), K(read_begin_lsn), + PALF_LOG(WARN, "read_data_from_buffer failed", K(ret), K(read_begin_lsn), K(in_read_size)); } } else { - PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K_(palf_id), K(read_begin_lsn), + PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K(read_begin_lsn), K(in_read_size), K(out_read_size)); } return ret; @@ -4844,92 +1260,33 @@ int PalfHandleImpl::raw_read(const LSN &lsn, ReadBuf read_buf(buffer, nbytes); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - PALF_LOG(WARN, "PalfHandleImp not inited", K(ret), K_(palf_id), K(lsn), K(nbytes), K(read_buf)); + PALF_LOG(WARN, "PalfHandleImp not inited", K(ret), K(lsn), K(nbytes), K(read_buf)); } else if (!lsn.is_valid() || !is_valid_raw_read_buf(read_buf, lsn_2_offset(lsn, PALF_BLOCK_SIZE), nbytes)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K(lsn), K(nbytes), K(read_buf)); + PALF_LOG(WARN, "invalid arguments", K(ret), K(lsn), K(nbytes), K(read_buf)); } else if (OB_FAIL(get_begin_lsn(readable_begin_lsn))) { - PALF_LOG(WARN, "get_begin_lsn failed", K(ret), K_(palf_id), K(lsn), K(nbytes), K(read_buf)); + PALF_LOG(WARN, "get_begin_lsn failed", K(ret), K(lsn), K(nbytes), K(read_buf)); } else if (lsn < readable_begin_lsn) { ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(WARN, "read something out of lower bound", K(ret), K_(palf_id), K(lsn), K(nbytes), + PALF_LOG(WARN, "read something out of lower bound", K(ret), K(lsn), K(nbytes), K(read_size), K(readable_begin_lsn)); } else if (lsn >= readable_end_lsn) { ret = OB_ERR_OUT_OF_UPPER_BOUND; - PALF_LOG(WARN, "read something out of upper bound", K(ret), K_(palf_id), K(lsn), + PALF_LOG(WARN, "read something out of upper bound", K(ret), K(lsn), K(nbytes), K(read_size), K(readable_end_lsn)); // only read the data before readable_end_lsn } else if (FALSE_IT(real_read_size = MIN(nbytes, readable_end_lsn - lsn))) { } else if (OB_FAIL(log_engine_.raw_read(lsn, real_read_size, need_read_block_header, read_buf, read_size, io_ctx))) { - PALF_LOG(WARN, "raw_read from storage failed", K(ret), K_(palf_id), K(lsn), + PALF_LOG(WARN, "raw_read from storage failed", K(ret), K(lsn), K(nbytes), K(real_read_size), K(readable_end_lsn), K(read_size)); } else { - PALF_LOG(TRACE, "raw_read success", K(ret), K_(palf_id), K(lsn), K(nbytes), + PALF_LOG(TRACE, "raw_read success", K(ret), K(lsn), K(nbytes), K(real_read_size), K(readable_end_lsn), K(read_size), K(time_guard)); } return ret; } -int PalfHandleImpl::force_set_member_list_(const common::ObMemberList &new_member_list, - const int64_t new_replica_num) -{ - int ret = OB_SUCCESS; - int64_t prev_replica_num = 0; - ObMemberList prev_member_list; - LogConfigInfoV2 new_config_info; - - { - WLockGuard guard(lock_); - LogConfigChangeArgs args(new_member_list, new_replica_num, FORCE_SET_MEMBER_LIST); - if (OB_FAIL(config_mgr_.get_curr_member_list(prev_member_list, prev_replica_num))) { - PALF_LOG(WARN, "failed to get prev member list", KR(ret), KPC(this)); - } else { - ObMember tmp_member; - // get removed members which are in prev_member_list but not in new_member_list - // args.removed_list_ will be used for update_match_lsn_map_() - for (int64_t i = 0; OB_SUCC(ret) && i < prev_member_list.get_member_number(); ++i) { - tmp_member.reset(); - if (OB_FAIL(prev_member_list.get_member_by_index(i, tmp_member))) { - PALF_LOG(WARN, "get_server_by_index from memberlist failed", K(ret), K_(palf_id), K(prev_member_list)); - } else if (new_member_list.contains(tmp_member.get_server())) { - } else if (OB_FAIL(args.removed_list_.add_member(tmp_member))) { - PALF_LOG(WARN, "removed_list_.add_member failed", K(ret), K(tmp_member), K(args)); - } - } - } - - int64_t proposal_id = INVALID_PROPOSAL_ID; - bool is_already_finished = false; - int64_t election_epoch = INVALID_PROPOSAL_ID; - if (OB_FAIL(ret)) { - PALF_LOG(WARN, "some errors happen, is not allowed to start change config", K(ret)); - } else if (OB_FAIL(config_mgr_.start_change_config(proposal_id, election_epoch, args.type_))) { - PALF_LOG(WARN, "start_change_config failed", KR(ret), KPC(this), K(args)); - } else if (OB_FAIL(config_mgr_.check_args_and_generate_config( - args, proposal_id, election_epoch, is_already_finished, - new_config_info))) { - PALF_LOG(WARN, "check_args_and_generate_config failed", K(ret), KPC(this), K(args)); - } else if (OB_FAIL(config_mgr_.force_set_member_list(args, proposal_id))) { - PALF_LOG(WARN, "failed to force to set member list", KR(ret), KPC(this), K(args), K(proposal_id)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(config_mgr_.wait_config_log_persistence(new_config_info.config_.config_version_))) { - PALF_LOG(WARN, "wait_config_log_persistence failed", K(ret), KPC(this), K(new_config_info)); - } else { - PALF_EVENT("force_set_member_list success", palf_id_, KR(ret), KPC(this), K(new_member_list), K(new_replica_num)); - report_force_set_member_list(prev_replica_num, new_replica_num, new_member_list); - } - - return ret; -} - -OB_SERIALIZE_MEMBER(PalfStat, self_, palf_id_, role_, log_proposal_id_, config_version_, - mode_version_, access_mode_, paxos_member_list_, paxos_replica_num_, allow_vote_, - replica_type_, begin_lsn_, begin_scn_, base_lsn_, end_lsn_, end_scn_, max_lsn_, max_scn_, - arbitration_member_, degraded_list_, is_in_sync_, is_need_rebuild_, learner_list_); template int PalfHandleImpl::alloc_iterator_from_scn_(const SCN &scn, @@ -4950,7 +1307,7 @@ int PalfHandleImpl::alloc_iterator_from_scn_(const SCN &scn, PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); } else if (OB_UNLIKELY(!scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", KR(ret), K_(palf_id), K(scn)); + PALF_LOG(WARN, "invalid argument", KR(ret), K(scn)); } else if (OB_FAIL(locate_by_scn_coarsely(scn, start_lsn)) && OB_ERR_OUT_OF_LOWER_BOUND != ret) { PALF_LOG(WARN, "locate_by_scn_coarsely failed", KR(ret), KPC(this), K(scn)); diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index 30c0529d1..6d21f80e1 100755 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -17,52 +17,40 @@ #ifndef OCEANBASE_PALF_LOG_SERVICE_ #define OCEANBASE_PALF_LOG_SERVICE_ -#include "common/ob_member_list.h" #include "common/ob_role.h" -#include "lib/hash/ob_link_hashmap_deps.h" #include "lib/lock/ob_tc_rwlock.h" #include "lib/lock/ob_spin_lock.h" #include "lib/ob_define.h" -#include "election_self.h" #include "share/scn.h" #include "palf_callback_wrapper.h" #include "log_engine.h" // LogEngine #include "log_meta.h" #include "log_cache.h" #include "lsn.h" -#include "log_config_mgr.h" #include "log_mode_mgr.h" -#include "log_reconfirm.h" #include "log_sliding_window.h" #include "log_state_mgr.h" #include "log_io_task_cb_utils.h" #include "palf_options.h" #include "palf_iterator.h" -#include "log_net_service.h" namespace oceanbase { namespace common { -class ObMember; class ObILogAllocator; } namespace palf { class FlushLogCbCtx; -class ObMemberChangeCtx; class LSN; -class FetchLogEngine; -class TruncateLogCbCtx; class FlushMetaCbCtx; class LogIOFlushLogTask; -class LogIOTruncateLogTask; class LogIOFlushMetaTask; class ReadBuf; class LogWriteBuf; class LogIOWorker; class LogSharedQueueTh; -class LogRpc; class IPalfEnvImpl; class LogEngine; class LogCache; @@ -76,19 +64,7 @@ struct PalfStat { void reset(); common::ObAddr self_; - int64_t palf_id_; - common::ObRole role_; - int64_t log_proposal_id_; - LogConfigVersion config_version_; - int64_t mode_version_; AccessMode access_mode_; - ObMemberList paxos_member_list_; - int64_t paxos_replica_num_; - common::ObMember arbitration_member_; - common::GlobalLearnerList degraded_list_; - common::GlobalLearnerList learner_list_; - bool allow_vote_; - LogReplicaType replica_type_; LSN begin_lsn_; share::SCN begin_scn_; LSN base_lsn_; @@ -96,127 +72,22 @@ struct PalfStat { share::SCN end_scn_; LSN max_lsn_; share::SCN max_scn_; - bool is_in_sync_; - bool is_need_rebuild_; - TO_STRING_KV(K_(self), K_(palf_id), K_(role), K_(log_proposal_id), K_(config_version), K_(mode_version), - K_(access_mode), K_(paxos_member_list), K_(paxos_replica_num), K_(learner_list), K_(allow_vote), K_(replica_type), - K_(begin_lsn), K_(begin_scn), K_(base_lsn), K_(end_lsn), K_(end_scn), K_(max_lsn), K_(max_scn), - K_(is_in_sync), K_(is_need_rebuild)); + TO_STRING_KV(K_(self), K_(access_mode), + K_(begin_lsn), K_(begin_scn), K_(base_lsn), K_(end_lsn), K_(end_scn), K_(max_lsn), K_(max_scn)); }; struct PalfDiagnoseInfo { PalfDiagnoseInfo() { reset(); } ~PalfDiagnoseInfo() { reset(); } - common::ObRole election_role_; - int64_t election_epoch_; - common::ObRole palf_role_; - palf::ObReplicaState palf_state_; - int64_t palf_proposal_id_; - bool enable_sync_; - bool enable_vote_; - common::ObAddr parent_; + palf::LogState log_state_; void reset() { - election_role_ = FOLLOWER; - election_epoch_ = 0; - palf_role_ = FOLLOWER; - palf_state_ = ObReplicaState::INVALID_STATE; - palf_proposal_id_ = INVALID_PROPOSAL_ID; - enable_sync_ = false; - enable_vote_ = false; - parent_.reset(); + log_state_ = LogState::INVALID_STATE; } - TO_STRING_KV(K(election_role_), - K(election_epoch_), - K(palf_role_), - K(palf_state_), - K(palf_proposal_id_), - K(enable_sync_), - K(enable_vote_), - K(parent_)); + TO_STRING_KV(K(log_state_)); }; -struct FetchLogStat { - FetchLogStat() { reset(); } - ~FetchLogStat() { reset(); } - int64_t total_size_; - int64_t group_log_cnt_; - int64_t read_cost_; // time cost of reading and deserializing log - int64_t get_cost_; // time cost of checking integrity - int64_t send_cost_; // time cost of sending logs by rpc - void reset() { - total_size_ = 0; - group_log_cnt_ = 0; - read_cost_ = 0; - get_cost_ = 0; - send_cost_ = 0; - } - TO_STRING_KV(K_(total_size), - K_(group_log_cnt), - K_(read_cost), - K_(get_cost), - K_(send_cost)); -}; - -struct BatchFetchParams { - BatchFetchParams() { reset(); } - ~BatchFetchParams() { reset(); } - void reset() { - can_batch_count_ = 0; - can_batch_size_ = 0; - batch_log_buf_ = NULL; - last_log_lsn_prev_round_.reset(); - last_log_end_lsn_prev_round_.reset(); - last_log_proposal_id_prev_round_ = 0; - } - TO_STRING_KV(K_(can_batch_count), - K_(can_batch_size), - KP_(batch_log_buf), - K_(last_log_lsn_prev_round), - K_(last_log_end_lsn_prev_round), - K_(last_log_proposal_id_prev_round), - K_(has_consumed_count)); - int64_t can_batch_count_; - int64_t can_batch_size_; - char *batch_log_buf_; - LSN last_log_lsn_prev_round_; - LSN last_log_end_lsn_prev_round_; - int64_t last_log_proposal_id_prev_round_; - int64_t has_consumed_count_; -}; - -struct RebuildMetaInfo -{ -public: - RebuildMetaInfo() {reset();} - ~ RebuildMetaInfo() {reset();} - void reset() - { - committed_end_lsn_.reset(); - last_submit_lsn_.reset(); - last_submit_log_pid_ = INVALID_PROPOSAL_ID; - } - bool is_valid() const - { - //last_submit_log_pid_ may be INVALID_PROPOSAL_ID when newly created replica has never synced any log - return (committed_end_lsn_.is_valid() && last_submit_lsn_.is_valid()); - } - - bool operator==(const RebuildMetaInfo &other) const - { - return (committed_end_lsn_ == other.committed_end_lsn_ - && last_submit_lsn_ == other.last_submit_lsn_ - && last_submit_log_pid_ == other.last_submit_log_pid_); - } - TO_STRING_KV(K_(committed_end_lsn), - K_(last_submit_lsn), - K_(last_submit_log_pid)); -public: - LSN committed_end_lsn_; - LSN last_submit_lsn_; - int64_t last_submit_log_pid_; -}; // The interface class of the log service, modules other than logservice are only allowed to call the interfaces of IPalfHandleImpl when using the log service -class IPalfHandleImpl : public common::LinkHashValue +class IPalfHandleImpl { public: IPalfHandleImpl() {}; @@ -224,25 +95,9 @@ class IPalfHandleImpl : public common::LinkHashValue public: virtual bool check_can_be_used() const = 0; - // after creating palf successfully, set initial memberlist(can only be called once) - // - // @param [in] member_list, paxos memberlist - // @param [in] paxos_replica_num, number of paxos replicas - // @param [in] learner_list, learner_list - // - // @return :TODO - virtual int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) = 0; - // Submit content that needs to be persisted, the submitted content is logged locally and synchronized to other replicas in the Paxos member list - // At the same time, if there are read-only replicas or physical standby databases, logs will be synchronized to the corresponding nodes when conditions are met - // After the majority of replicas in the Paxos member list successfully persist, it will call log_ctx->on_success() to notify the caller - // - // If a leader-follower switch (Leader->Follower) occurs during execution, and this log eventually forms a majority, - // then it will still call log_ctx->on_success() rather than replay log - // - // In the scenario where the function returns successfully, on any replica, log_ctx->on_success()/log_ctx->on_failure()/replay log - // One of them ensures the final call and only one call + virtual int bootstrap() = 0; + // Persist one log and invoke exactly one completion path: callback success, + // callback failure, or replay. // // @param [in] opts, some optional parameters for submitting logs, see the definition of PalfAppendOptions for details // @param [in] buf, the starting pointer of the content to be persisted, buf can be released after ::submit_log function returns @@ -262,223 +117,14 @@ class IPalfHandleImpl : public common::LinkHashValue const share::SCN &ref_scn, LSN &lsn, share::SCN &scn) = 0; - // submit group_log to palf - // Usage scenario: Follower leader processes logs received from the primary - // @param [in] opts, some optional parameters for submitting logs, see the definition of PalfAppendOptions for details - // @param [in] lsn, log sequence number corresponding to the log - // @param [in] buf, the starting pointer of the content to be persisted, buf can be released after ::submit_group_log function returns - // @param [in] buf_len, the length of the content to be persisted - virtual int submit_group_log(const PalfAppendOptions &opts, - const LSN &lsn, - const char *buf, - const int64_t buf_len) = 0; - // Return the current replica's role, which only exists in Leader/StandbyLeader/Follower three roles - // - // @param [out] role, current replica's role - // @param [out] proposal_id, unique identifier for the leader, monotonically increasing across machines. - // - // @return :TODO - virtual int get_role(common::ObRole &role, - int64_t &proposal_id, - bool &is_pending_state) const = 0; - // get palf_id - virtual int get_palf_id(int64_t &palf_id) const = 0; - // Primary switch interface, used for internal debugging only, no formal functionality should depend on this interface - // The cut-in action in the formal function should be described by a priority strategy, unified into a set of rules - // This interface is an asynchronous interface, after the interface call returns successfully, the Leader may take several seconds to complete the switch, theoretically there is also a very small probability of the switch failing - // In a normal system, it is expected that the leader will complete the switch within several ms - // - // @param [in] dest_addr, the destination replica for leader switch, must be in the current member list - // - // @return : - // OB_ENTRY_NOT_EXIST : The target replica for the leader switch is not in palf's current member list - // OB_NOT_MASTER : This replica is not currently the leader and cannot accept a leader switch request - virtual int change_leader_to(const common::ObAddr &dest_addr) = 0; - - virtual int get_global_learner_list(common::GlobalLearnerList &learner_list) const = 0; - virtual int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const = 0; - virtual int get_config_version(LogConfigVersion &config_version) const = 0; - virtual int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const = 0; - virtual int get_stable_membership(LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const = 0; - virtual int get_election_leader(common::ObAddr &addr) const = 0; - virtual int get_parent(common::ObAddr &parent) const = 0; - - // @brief: a special config change interface, change replica number of paxos group - // @param[in] common::ObMemberList: current memberlist, for pre-check - // @param[in] const int64_t curr_replica_num: current replica num, for pre-check - // @param[in] const int64_t new_replica_num: new replica num - // @param[in] const int64_t timeout_us: timeout, us - // @return - // - OB_SUCCESS: change_replica_num successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: change_replica_num timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) = 0; - // @brief: force set self as single replica. - virtual int force_set_as_single_replica() = 0; - - // @brief: force set member list. - // @param[in] const common::ObMemberList &new_member_list: members which will be set forcely - // @param[in] const int64_t new_replica_num: replica number of paxos group after forcing to set member list - // @return - // - OB_SUCCESS: force_set_member_list successfully - // - OB_TIMEOUT: force_set_member_list timeout because of config log persistence - // - OB_NOT_RUNNING: log stream is stopped - // - OB_INVALID_ARGUMENT: invalid argument - // - other: bug - virtual int force_set_member_list(const common::ObMemberList &new_member_list, const int64_t new_replica_num) = 0; - - // @brief, add a member into paxos group - // @param[in] common::ObMember &member: member which will be added - // @param[in] const int64_t new_replica_num: replica number of paxos group after adding 'member' - // @param[in] const int64_t timeout_us: add member timeout, us - // @return - // - OB_SUCCESS: add member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: add member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int add_member(const common::ObMember &member, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - - // @brief, remove a member from paxos group - // @param[in] common::ObMember &member: member which will be removed - // @param[in] const int64_t new_replica_num: replica number of paxos group after removing 'member' - // @param[in] const int64_t timeout_us: remove member timeout, us - // @return - // - OB_SUCCESS: remove member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: remove member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int remove_member(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) = 0; - - // @brief, replace old_member with new_member - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - - // @brief: add a learner(read only replica) in this clsuter - // @param[in] const common::ObMember &added_learner: learner will be added - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: add_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - virtual int add_learner(const common::ObMember &added_learner, const int64_t timeout_us) = 0; - - // @brief: remove a learner(read only replica) in this clsuter - // @param[in] const common::ObMember &removed_learner: learner will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: remove_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - virtual int remove_learner(const common::ObMember &removed_learner, const int64_t timeout_us) = 0; - - // @brief: switch a learner(read only replica) to acceptor(full replica) in this clsuter - // @param[in] const common::ObMember &learner: learner will be switched to acceptor - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // learner to acceptor (similar to add_member) - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_learner_to_acceptor timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - virtual int switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) = 0; - - // @brief: switch an acceptor(full replica) to learner(read only replica) in this clsuter - // @param[in] const common::ObMember &member: acceptor will be switched to learner - // @param[in] const int64_t new_replica_num: replica number of paxos group after switching - // acceptor to learner (similar to remove_member) - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS - // - OB_INVALID_ARGUMENT: invalid argument - // - OB_TIMEOUT: switch_acceptor_to_learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - virtual int switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) = 0; - - // @brief, replace removed_learners with added_learners - // @param[in] const common::ObMemberList &added_learners: learners will be added - // @param[in] const common::ObMemberList &removed_learners: learners will be removed - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace learner successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace learner timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) = 0; - - // @brief, replace removed_member with learner - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const LogConfigVersion &config_version: config_version for leader checking - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - virtual int replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) = 0; // Set the recyclable point of the log file, log files with LSN less than or equal to lsn can be safely recycled // // @param [in] lsn, the log file position that can be recycled // // @return :TODO virtual int set_base_lsn(const LSN &lsn) = 0; - // Allow palf to pull logs - virtual int enable_sync() = 0; - // Prohibit palf from pulling logs, to prevent log disk overflow in rebuild/migrate scenarios - virtual int disable_sync() = 0; - // Mark the palf instance as deleted + // Mark the log stream as deleted. virtual void set_deleted() = 0; - virtual bool is_sync_enabled() const = 0; - // Migration/rebuild scenario purpose end advancing base_lsn - // - // @param [in] palf_base_info, recyclable log file position - virtual int advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) = 0; - // @desc: query coarse lsn by scn, that means there is a LogGroupEntry in disk, // its lsn and scn are result_lsn and result_scn, and result_scn <= scn. // result_lsn result_scn @@ -539,20 +185,11 @@ class IPalfHandleImpl : public common::LinkHashValue virtual LSN get_max_lsn() const = 0; virtual const share::SCN get_max_scn() const = 0; virtual const share::SCN get_end_scn() const = 0; - virtual int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const = 0; virtual const LSN get_readable_end_lsn() const = 0; virtual int get_total_used_disk_space(int64_t &total_used_disk_space, int64_t &unrecyclable_disk_space) const = 0; virtual const LSN &get_base_lsn_used_for_block_gc() const = 0; - // @desc: get ack_info_array and degraded_list for judging to degrade/upgrade - // @params [in] member_ts_array: ack info array of all paxos members - // @params [in] degraded_list: members which have been degraded - // @return: - virtual int get_ack_info_array(LogMemberAckInfoList &ack_info_array, - common::GlobalLearnerList °raded_list) const = 0; - virtual int delete_block(const block_id_t &block_id) = 0; virtual int inner_after_flush_log(const FlushLogCbCtx &flush_log_cb_ctx) = 0; - virtual int inner_after_truncate_log(const TruncateLogCbCtx &truncate_log_cb_ctx) = 0; virtual int inner_after_flush_meta(const FlushMetaCbCtx &flush_meta_cb_ctx) = 0; virtual int inner_after_truncate_prefix_blocks(const TruncatePrefixBlocksCbCtx &truncate_prefix_cb_ctx) = 0; virtual int advance_reuse_lsn(const LSN &flush_log_end_lsn) = 0; @@ -564,124 +201,14 @@ class IPalfHandleImpl : public common::LinkHashValue const SCNArray &scn_array) = 0; virtual int inner_append_meta(const char *buf, const int64_t buf_len) = 0; - virtual int inner_truncate_log(const LSN &lsn) = 0; virtual int inner_truncate_prefix_blocks(const LSN &lsn) = 0; virtual int check_and_switch_state() = 0; virtual int check_and_switch_freeze_mode() = 0; virtual bool is_in_period_freeze_mode() const = 0; virtual int period_freeze_last_log() = 0; - virtual int handle_prepare_request(const common::ObAddr &server, - const int64_t &proposal_id) = 0; - virtual int handle_prepare_response(const common::ObAddr &server, - const int64_t &proposal_id, - const bool vote_granted, - const int64_t &accept_proposal_id, - const LSN &last_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &log_mode_meta) = 0; - virtual int receive_log(const common::ObAddr &server, - const PushLogType push_log_type, - const int64_t &proposal_id, - const LSN &prev_lsn, - const int64_t &prev_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len) = 0; - virtual int receive_batch_log(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len) = 0; - virtual int ack_log(const common::ObAddr &server, - const int64_t &proposal_id, - const LSN &log_end_lsn) = 0; - virtual int get_log(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) = 0; - virtual int fetch_log_from_storage(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &req_proposal_id, - const LSN &prev_log_offset, - const LSN &log_offset, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid, - const SCN &replayable_point, - FetchLogStat &fetch_stat) = 0; - virtual int receive_config_log(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &meta) = 0; - virtual int ack_config_log(const common::ObAddr &server, - const int64_t proposal_id, - const LogConfigVersion &config_version) = 0; - virtual int receive_mode_meta(const common::ObAddr &server, - const int64_t msg_proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &meta) = 0; - virtual int ack_mode_meta(const common::ObAddr &server, - const int64_t proposal_id) = 0; - virtual int handle_notify_fetch_log_req(const common::ObAddr &server) = 0; - virtual int handle_notify_rebuild_req(const common::ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) = 0; - virtual int handle_config_change_pre_check(const ObAddr &server, - const LogGetMCStReq &req, - LogGetMCStResp &resp) = 0; - virtual int handle_register_parent_req(const LogLearner &child, - const bool is_to_leader) = 0; - virtual int handle_register_parent_resp(const LogLearner &server, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) = 0; - virtual int handle_learner_req(const LogLearner &server, const LogLearnerReqType req_type) = 0; virtual int set_scan_disk_log_finished() = 0; - virtual int change_access_mode(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const share::SCN &ref_scn) = 0; - virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const = 0; - virtual int get_access_mode(AccessMode &access_mode) const = 0; - virtual int get_access_mode_version(int64_t &mode_version) const = 0; - virtual int get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, + virtual int get_access_mode_ref_scn(AccessMode &access_mode, SCN &ref_scn) const = 0; - virtual int handle_committed_info(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) = 0; - - // @brief: check whether the palf instance is allowed to vote for logs - // By default, return true; - // After calling disable_vote(), return false. - virtual bool is_vote_enabled() const = 0; - // @brief: store a persistent flag which means this paxos replica - // can not reply ack when receiving logs. - // By default, paxos replica can reply ack. - // This interface is idempotent. - // @param[in] need_check_log_missing: reason for rebuilding. True means log missing, False means data - // missing - // @return: - // OB_NOT_INIT: not inited - // OB_NOT_RUNNING: in stop state - // OB_OP_NOT_ALLOWED: no need to rebuilds. rebuilding should be abandoned. - // OB_LEADER_NOT_EXIST: no leader when double checking. rebuilding should retry. - virtual int disable_vote(const bool need_check_log_missing) = 0; - // @brief: store a persistent flag which means this paxos replica - // can reply ack when receiving logs. - // By default, paxos replica can reply ack. - // This interface is idempotent. - virtual int enable_vote() = 0; - // ===================== Iterator start ======================= virtual int alloc_palf_buffer_iterator(const LSN &offset, PalfBufferIterator &iterator) = 0; @@ -696,25 +223,10 @@ class IPalfHandleImpl : public common::LinkHashValue // ==================== Callback start ====================== virtual int register_file_size_cb(palf::PalfFSCbNode *fs_cb) = 0; virtual int unregister_file_size_cb(palf::PalfFSCbNode *fs_cb) = 0; - virtual int register_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) = 0; - virtual int unregister_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) = 0; - virtual int register_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) = 0; - virtual int unregister_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) = 0; - virtual int set_location_cache_cb(PalfLocationCacheCb *lc_cb) = 0; - virtual int reset_location_cache_cb() = 0; - virtual int set_election_priority(election::ElectionPriority *priority) = 0; - virtual int reset_election_priority() = 0; // ==================== Callback end ======================== - virtual int advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, - const int64_t downgrade_priority_time_us, - const char *reason) = 0; virtual int stat(PalfStat &palf_stat) = 0; virtual int get_palf_epoch(int64_t &palf_epoch) const = 0; - virtual int try_lock_config_change(int64_t lock_owner, int64_t timeout_us) = 0; - virtual int unlock_config_change(int64_t lock_owner, int64_t timeout_us) = 0; - virtual int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) = 0; virtual int diagnose(PalfDiagnoseInfo &diagnose_info) const = 0; - virtual int update_palf_stat() = 0; virtual int read_data_from_buffer(const LSN &read_begin_lsn, const int64_t in_read_size, char *buf, @@ -735,15 +247,11 @@ class PalfHandleImpl : public IPalfHandleImpl public: PalfHandleImpl(); ~PalfHandleImpl() override; - int init(const int64_t palf_id, - const AccessMode &access_mode, + int init(const AccessMode &access_mode, const PalfBaseInfo &palf_base_info, - const LogReplicaType replica_type, - FetchLogEngine *fetch_log_engine, const char *log_dir, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, IPalfEnvImpl *palf_env_impl, @@ -756,12 +264,9 @@ class PalfHandleImpl : public IPalfHandleImpl // 2. Read the latest data from meta storage, initialize dio_aligned_buf; // 3. Initialize dio_aligned_buf in log_storage; // 4. Initialize other fields of palf_handle_impl. - int load(const int64_t palf_id, - FetchLogEngine *fetch_log_engine, - const char *log_dir, + int load(const char *log_dir, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogRpc *log_rpc, LogIOWorker*log_io_worker, LogSharedQueueTh *log_shared_queue_th, IPalfEnvImpl *palf_env_impl, @@ -771,9 +276,7 @@ class PalfHandleImpl : public IPalfHandleImpl bool &is_integrity); void destroy(); int start(); - int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) override final; + int bootstrap() override final; int submit_log(const PalfAppendOptions &opts, const char *buf, const int64_t buf_len, @@ -781,74 +284,10 @@ class PalfHandleImpl : public IPalfHandleImpl LSN &lsn, share::SCN &scn) override final; - int submit_group_log(const PalfAppendOptions &opts, - const LSN &lsn, - const char *buf, - const int64_t buf_len) override final; - int get_role(common::ObRole &role, - int64_t &proposal_id, - bool &is_pending_state) const override final; - int get_palf_id(int64_t &palf_id) const override final; - int change_leader_to(const common::ObAddr &dest_addr) override final; - int get_global_learner_list(common::GlobalLearnerList &learner_list) const override final; - int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const override final; - int get_config_version(LogConfigVersion &config_version) const; - int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const override final; - int get_stable_membership(LogConfigVersion &config_version, - common::ObMemberList &member_list, - int64_t &paxos_replica_num, - common::GlobalLearnerList &learner_list) const override final; - int get_election_leader(common::ObAddr &addr) const; - int get_parent(common::ObAddr &parent) const; - int force_set_as_single_replica() override final; - int force_set_member_list(const common::ObMemberList &member_list, - const int64_t new_replica_num) override final; - int change_replica_num(const common::ObMemberList &member_list, - const int64_t curr_replica_num, - const int64_t new_replica_num, - const int64_t timeout_us) override final; - int add_member(const common::ObMember &member, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) override final; - int remove_member(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) override final; - int replace_member(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) override final; - int add_learner(const common::ObMember &added_learner, - const int64_t timeout_us) override final; - int remove_learner(const common::ObMember &removed_learner, - const int64_t timeout_us) override final; - int switch_learner_to_acceptor(const common::ObMember &learner, - const int64_t new_replica_num, - const LogConfigVersion &config_version, - const int64_t timeout_us) override final; - int switch_acceptor_to_learner(const common::ObMember &member, - const int64_t new_replica_num, - const int64_t timeout_us) override final; - int replace_learners(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners, - const int64_t timeout_us) override final; - int replace_member_with_learner(const common::ObMember &added_member, - const common::ObMember &removed_member, - const LogConfigVersion &config_version, - const int64_t timeout_us) override final; int set_base_lsn(const LSN &lsn) override final; - int enable_sync() override final; - int disable_sync() override final; void set_deleted() override final; - bool is_sync_enabled() const override final; - int advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) override final; int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) override final; int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn) override final; - bool is_vote_enabled() const override final; - int disable_vote(const bool need_check_log_missing) override final; - int enable_vote() override final; int read_data_from_buffer(const LSN &read_begin_lsn, const int64_t in_read_size, char *buf, @@ -862,15 +301,7 @@ class PalfHandleImpl : public IPalfHandleImpl public: int delete_block(const block_id_t &block_id) override final; int set_scan_disk_log_finished() override; - int change_access_mode(const int64_t proposal_id, - const int64_t mode_version, - const AccessMode &access_mode, - const share::SCN &ref_scn) override final; - int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const override final; - int get_access_mode(AccessMode &access_mode) const override final; - int get_access_mode_version(int64_t &mode_version) const override final; - int get_access_mode_ref_scn(int64_t &mode_version, - AccessMode &access_mode, + int get_access_mode_ref_scn(AccessMode &access_mode, SCN &ref_scn) const override final; // =========================== Iterator start ============================ int alloc_palf_buffer_iterator(const LSN &offset, PalfBufferIterator &iterator) override final; @@ -882,16 +313,8 @@ class PalfHandleImpl : public IPalfHandleImpl // ==================== Callback start ====================== int register_file_size_cb(palf::PalfFSCbNode *fs_cb) override final; int unregister_file_size_cb(palf::PalfFSCbNode *fs_cb) override final; - int register_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) override final; - int unregister_role_change_cb(palf::PalfRoleChangeCbNode *role_change_cb) override final; - int register_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) override final; - int unregister_rebuild_cb(palf::PalfRebuildCbNode *rebuild_cb) override final; - int set_location_cache_cb(PalfLocationCacheCb *lc_cb) override final; - int reset_location_cache_cb() override final; int set_monitor_cb(PalfMonitorCb *monitor_cb); int reset_monitor_cb(); - int set_election_priority(election::ElectionPriority *priority) override final; - int reset_election_priority() override final; // ==================== Callback end ======================== public: int get_begin_lsn(LSN &lsn) const override final; @@ -922,11 +345,8 @@ class PalfHandleImpl : public IPalfHandleImpl const share::SCN get_end_scn() const override final { - // Based on implementation complexity, directly use last_slide_scn as end_scn - // Otherwise, it is necessary to maintain scn additionally in match_lsn_map return sw_.get_last_slide_scn(); } - int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const override final; const LSN get_readable_end_lsn() const override final { LSN committed_end_lsn; @@ -941,11 +361,8 @@ class PalfHandleImpl : public IPalfHandleImpl { return log_engine_.get_base_lsn_used_for_block_gc(); } - int get_ack_info_array(LogMemberAckInfoList &ack_info_array, - common::GlobalLearnerList °raded_list) const override final; // ===================== LogIOTask start ========================== int inner_after_flush_log(const FlushLogCbCtx &flush_log_cb_ctx) override final; - int inner_after_truncate_log(const TruncateLogCbCtx &truncate_log_cb_ctx) override final; int inner_after_flush_meta(const FlushMetaCbCtx &flush_meta_cb_ctx) override final; int inner_after_truncate_prefix_blocks(const TruncatePrefixBlocksCbCtx &truncate_prefix_cb_ctx) override final; int advance_reuse_lsn(const LSN &flush_log_end_lsn); @@ -957,242 +374,42 @@ class PalfHandleImpl : public IPalfHandleImpl const SCNArray &scn_array); int inner_append_meta(const char *buf, const int64_t buf_len) override final; - int inner_truncate_log(const LSN &lsn) override final; int inner_truncate_prefix_blocks(const LSN &lsn) override final; // ================================================================== int check_and_switch_state() override final; int check_and_switch_freeze_mode() override final; bool is_in_period_freeze_mode() const override final; int period_freeze_last_log() override final; - int handle_prepare_request(const common::ObAddr &server, - const int64_t &proposal_id) override final; - int handle_prepare_response(const common::ObAddr &server, - const int64_t &proposal_id, - const bool vote_granted, - const int64_t &accept_proposal_id, - const LSN &last_lsn, - const LSN &committed_end_lsn, - const LogModeMeta &log_mode_meta) override final; - int receive_log(const common::ObAddr &server, - const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_log_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len) override final; - int receive_batch_log(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len) override final; - int ack_log(const common::ObAddr &server, - const int64_t &proposal_id, - const LSN &log_end_lsn) override final; - int get_log(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) override final; - int fetch_log_from_storage(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid, - const SCN &replayable_point, - FetchLogStat &fetch_stat) override final; - int receive_config_log(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &meta) override final; - int ack_config_log(const common::ObAddr &server, - const int64_t proposal_id, - const LogConfigVersion &config_version) override final; - int receive_mode_meta(const common::ObAddr &server, - const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &meta) override final; - int ack_mode_meta(const common::ObAddr &server, - const int64_t proposal_id) override final; - int handle_notify_fetch_log_req(const common::ObAddr &server) override final; - int handle_notify_rebuild_req(const common::ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) override final; - int handle_committed_info(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t prev_log_id, - const int64_t &prev_log_proposal_id, - const LSN &committed_end_lsn) override final; - int handle_config_change_pre_check(const ObAddr &server, - const LogGetMCStReq &req, - LogGetMCStResp &resp) override final; - int advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, - const int64_t downgrade_priority_time_us, - const char *reason) override final; int stat(PalfStat &palf_stat) override final; - int handle_register_parent_req(const LogLearner &child, - const bool is_to_leader) override final; - int handle_register_parent_resp(const LogLearner &server, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) override final; - int handle_learner_req(const LogLearner &server, const LogLearnerReqType req_type) override final; int get_palf_epoch(int64_t &palf_epoch) const override final; + //config change lock related function - int try_lock_config_change(int64_t lock_owner, int64_t timeout_us); - int unlock_config_change(int64_t lock_owner, int64_t timeout_us); - int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked); int diagnose(PalfDiagnoseInfo &diagnose_info) const; - int update_palf_stat() override final; - TO_STRING_KV(K_(palf_id), K_(self), K_(has_set_deleted)); + TO_STRING_KV(K_(self), K_(has_set_deleted)); private: - int do_init_mem_(const int64_t palf_id, - const PalfBaseInfo &palf_base_info, + int do_init_mem_(const PalfBaseInfo &palf_base_info, const LogMeta &log_meta, const char *log_dir, const common::ObAddr &self, - FetchLogEngine *fetch_log_engine, ObILogAllocator *alloc_mgr, - LogRpc *log_rpc, IPalfEnvImpl *palf_env_impl); - int after_flush_prepare_meta_(const int64_t &proposal_id); - int after_flush_config_change_meta_(const int64_t proposal_id, const LogConfigVersion &config_version); - int after_flush_mode_meta_(const int64_t proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta); int after_flush_snapshot_meta_(const LSN &lsn); - int after_flush_replica_property_meta_(const bool allow_vote); - /* - *param[in] need_check_log_missing: for disable_vote invoke by rebuilding, - true means need double check whether log is actually missing - * */ - int set_allow_vote_flag_(const bool allow_vote, const bool need_check_log_missing); int get_prev_log_info_(const LSN &lsn, LogInfo &log_info); - int get_prev_log_info_for_fetch_(const LSN &prev_lsn, - const LSN &curr_lsn, - LogInfo &prev_log_info); - int submit_prepare_response_(const common::ObAddr &server, - const int64_t &proposal_id); int construct_palf_base_info_(const LSN &max_committed_lsn, PalfBaseInfo &palf_base_info); int append_disk_log_to_sw_(const LSN &start_lsn); - int try_send_committed_info_(const common::ObAddr &server, - const LSN &log_lsn, - const LSN &log_end_lsn, - const int64_t &log_proposal_id); - int receive_log_(const common::ObAddr &server, - const PushLogType push_log_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_log_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len); - int fetch_log_from_storage_(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const SCN &replayable_point, - FetchLogStat &fetch_stat); - int batch_fetch_log_each_round_(const common::ObAddr &server, - const int64_t msg_proposal_id, - PalfGroupBufferIterator &iterator, - const bool is_limitted_by_end_lsn, - const bool is_dest_in_memberlist, - const share::SCN& replayable_point, - const LSN &fetch_end_lsn, - const LSN &committed_end_lsn, - const int64_t batch_log_size_threshold, - BatchFetchParams &batch_fetch_params, - bool &skip_next, - bool &is_reach_end, - FetchLogStat &fetch_stat); - int submit_fetch_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogGroupEntry &curr_group_entry); - int submit_fetch_log_resp_(const common::ObAddr &server, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len); - int submit_batch_fetch_log_resp_(const common::ObAddr &server, - const int64_t msg_proposal_id, - const int64_t prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const char *buf, - const int64_t buf_len); - int try_update_proposal_id_(const common::ObAddr &server, - const int64_t &proposal_id); int get_binary_search_range_(const share::SCN &scn, block_id_t &min_block_id, block_id_t &max_block_id, block_id_t &result_block_id); int get_block_id_by_scn_(const share::SCN &scn, block_id_t &result_block_id); void inc_update_last_locate_block_scn_(const block_id_t &block_id, const share::SCN &scn); - int can_change_config_(const LogConfigChangeArgs &args, int64_t &proposal_id); - int check_args_and_generate_config_(const LogConfigChangeArgs &args, - const int64_t proposal_id, - const int64_t election_epoch, - bool &is_already_finished, - LogConfigInfoV2 &new_config_info) const; - int wait_log_barrier_(const LogConfigChangeArgs &args, - const LogConfigInfoV2 &new_config_info, - TimeoutChecker ¬_timeout); - int one_stage_config_change_(const LogConfigChangeArgs &args, const int64_t timeout_us); - int check_need_rebuild_(const LSN &base_lsn, - const LogInfo &base_prev_log_info, - bool &need_rebuild, - bool &need_fetch_log); - int check_need_advance_base_info_(const LSN &base_lsn, - const LogInfo &base_prev_log_info, - const bool is_rebuild); - int get_election_leader_without_lock_(ObAddr &addr) const; - int force_set_member_list_(const common::ObMemberList &new_member_list, const int64_t new_replica_num); - int leader_sync_mode_meta_to_arb_member_(); - void is_in_sync_(bool &is_log_sync, bool &is_use_cache); - int get_leader_max_scn_(SCN &max_scn, LSN &end_lsn); - void gen_rebuild_meta_info_(RebuildMetaInfo &rebuild_meta) const; - void get_last_rebuild_meta_info_(RebuildMetaInfo &rebuild_meta_info) const; + // ======================= report event begin ======================================= - void report_set_initial_member_list_(const int64_t paxos_replica_num, const common::ObMemberList &member_list); - void report_force_set_as_single_replica_(const int64_t prev_replica_num, const int64_t curr_replica_num, const ObMember &member); - void report_force_set_member_list(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMemberList &member_list); - void report_change_replica_num_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMemberList &member_list); - void report_add_member_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMember &added_member); - void report_remove_member_(const int64_t prev_replica_num, const int64_t curr_replica_num, const common::ObMember &removed_member); - void report_replace_member_(const common::ObMember &added_member, - const common::ObMember &removed_member, - const common::ObMemberList &member_list, - const char *event_name); - void report_add_learner_(const common::ObMember &added_learner); - void report_remove_learner_(const common::ObMember &removed_learner); - void report_switch_learner_to_acceptor_(const common::ObMember &learner); - void report_switch_acceptor_to_learner_(const common::ObMember &acceptor); - void report_replace_learners_(const common::ObMemberList &added_learners, - const common::ObMemberList &removed_learners); // ======================= report event end ======================================= - bool check_need_hook_fetch_log_(const FetchLogType fetch_type, const LSN &start_lsn); template int alloc_iterator_from_scn_(const SCN &scn, PalfIterator &iterator); @@ -1207,20 +424,13 @@ class PalfHandleImpl : public IPalfHandleImpl mutable RWLock lock_; char log_dir_[common::MAX_PATH_SIZE]; LogSlidingWindow sw_; - LogConfigMgr config_mgr_; LogModeMgr mode_mgr_; LogStateMgr state_mgr_; - LogReconfirm reconfirm_; LogEngine log_engine_; - election::ElectionImpl election_; LogCache log_cache_; - FetchLogEngine *fetch_log_engine_; common::ObILogAllocator *allocator_; - int64_t palf_id_; common::ObAddr self_; palf::PalfFSCbWrapper fs_cb_wrapper_; - palf::PalfRoleChangeCbWrapper role_change_cb_wrpper_; - palf::PalfRebuildCbWrapper rebuild_cb_wrapper_; LogPlugins plugins_; // ======optimization for locate_by_scn_coarsely========= mutable SpinLock last_locate_lock_; @@ -1228,16 +438,9 @@ class PalfHandleImpl : public IPalfHandleImpl block_id_t last_locate_block_; // ======optimization for locate_by_scn_coarsely========= int64_t cannot_recv_log_warn_time_; - int64_t cannot_handle_committed_info_time_; int64_t log_disk_full_warn_time_; - int64_t last_check_parent_child_time_us_; int64_t wait_slide_print_time_us_; int64_t append_size_stat_time_us_; - int64_t replace_member_print_time_us_; - mutable int64_t config_change_print_time_us_; - mutable SpinLock last_rebuild_meta_info_lock_;//protect last_rebuild_lsn_ and last_rebuild_meta_info_ - LSN last_rebuild_lsn_; - RebuildMetaInfo last_rebuild_meta_info_;//used for double checking whether it is necessary to rebuild LSN last_record_append_lsn_; // NB: only set has_set_deleted_ to true when this palf_handle has been deleted. bool has_set_deleted_; @@ -1248,22 +451,7 @@ class PalfHandleImpl : public IPalfHandleImpl ObMiniStat::ObStatItem handle_submit_log_cost_stat_; int64_t last_accum_write_statistic_time_; int64_t accum_write_log_size_; // the accum size of written logs - int64_t last_accum_fetch_statistic_time_; - int64_t accum_fetch_log_size_; - // a spin lock for read/write replica_meta mutex - SpinLock replica_meta_lock_; - SpinLock rebuilding_lock_; - mutable SpinLock config_change_lock_; - SpinLock mode_change_lock_; - // a spin lock for single replica mutex int64_t last_dump_info_time_us_; - int64_t last_check_sync_time_us_; - int64_t last_renew_loc_time_us_; - int64_t last_print_in_sync_time_us_; - int64_t last_hook_fetch_log_time_us_; - int64_t chaning_config_warn_time_; - bool cached_is_in_sync_; - bool has_higher_prio_config_change_; bool is_inited_; }; } // end namespace palf diff --git a/src/logservice/palf/palf_handle_impl_guard.cpp b/src/logservice/palf/palf_handle_impl_guard.cpp index 64260d264..3333a6d34 100644 --- a/src/logservice/palf/palf_handle_impl_guard.cpp +++ b/src/logservice/palf/palf_handle_impl_guard.cpp @@ -23,8 +23,7 @@ using namespace common; namespace palf { -IPalfHandleImplGuard::IPalfHandleImplGuard() : palf_id_(), - palf_handle_impl_(NULL), +IPalfHandleImplGuard::IPalfHandleImplGuard() : palf_handle_impl_(NULL), palf_env_impl_(NULL) { } @@ -36,7 +35,7 @@ IPalfHandleImplGuard::~IPalfHandleImplGuard() bool IPalfHandleImplGuard::is_valid() const { - return true == is_valid_palf_id(palf_id_) && NULL != palf_handle_impl_ && NULL != palf_env_impl_; + return NULL != palf_handle_impl_ && NULL != palf_env_impl_; } void IPalfHandleImplGuard::reset() @@ -46,7 +45,6 @@ void IPalfHandleImplGuard::reset() } palf_handle_impl_ = NULL; palf_env_impl_ = NULL; - palf_id_ = -1; }; } // end namespace palf diff --git a/src/logservice/palf/palf_handle_impl_guard.h b/src/logservice/palf/palf_handle_impl_guard.h index 114c13e57..9613c4feb 100644 --- a/src/logservice/palf/palf_handle_impl_guard.h +++ b/src/logservice/palf/palf_handle_impl_guard.h @@ -35,9 +35,8 @@ struct IPalfHandleImplGuard bool is_valid() const; void reset(); IPalfHandleImpl *get_palf_handle_impl() const { return palf_handle_impl_; } - TO_STRING_KV(K_(palf_id), KP_(palf_handle_impl), KP_(palf_env_impl)); + TO_STRING_KV(KP_(palf_handle_impl), KP_(palf_env_impl)); - int64_t palf_id_; IPalfHandleImpl *palf_handle_impl_; IPalfEnvImpl *palf_env_impl_; private: diff --git a/src/logservice/palf/palf_iterator.h b/src/logservice/palf/palf_iterator.h index ce92dc4dc..d8996e9bd 100644 --- a/src/logservice/palf/palf_iterator.h +++ b/src/logservice/palf/palf_iterator.h @@ -18,7 +18,7 @@ #define OCEANBASE_LOGSERVICE_PALF_ITERATOR_ #include "log_iterator_impl.h" // LogIteratorImpl #include "log_iterator_storage.h" // LogIteratorStorage -#include "log_define.h" // PALF_INITIAL_PROPOSAL_ID +#include "log_define.h" #include "lib/profile/ob_trace_id.h" // ObCurTraceId(self-contained include fix) namespace oceanbase { @@ -43,10 +43,9 @@ class PalfIterator ILogStorage *log_storage) { int ret = OB_SUCCESS; - auto get_mode_version = []() { return PALF_INITIAL_PROPOSAL_ID; }; if (IS_INIT) { ret = OB_INIT_TWICE; - } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage))) { + } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, log_storage))) { PALF_LOG(WARN, "PalfIterator init failed", K(ret)); } else { PALF_LOG(TRACE, "PalfIterator init success", K(ret), K(start_offset), KPC(this)); @@ -54,23 +53,6 @@ class PalfIterator } return ret; } - int init(const LSN &start_offset, - const GetFileEndLSN &get_file_end_lsn, - const GetModeVersion &get_mode_version, - ILogStorage *log_storage) - { - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage))) { - PALF_LOG(WARN, "PalfIterator init failed", K(ret)); - } else { - PALF_LOG(TRACE, "PalfIterator init success", K(ret), K(start_offset), KPC(this)); - is_inited_ = true; - } - return ret; - } - int set_io_context(const LogIOContext &io_ctx) { int ret = OB_SUCCESS; @@ -133,8 +115,7 @@ class PalfIterator } // @brief access next log entry of palf - // parma[in] replayable point scn, iterate will ensure that no log will return when the log scn is greater - // than 'replayable_point_scn' and the log is raw write + // param[in] replayable point scn; no log newer than it will be returned. // @retval // OB_SUCCESS. // OB_INVALID_DATA. @@ -158,8 +139,7 @@ class PalfIterator } // @brief access next log entry of palf - // parma[in] replayable point scn, iterate will ensure that no log will return when the log scn is greater - // than 'replayable_point_scn' and the log is raw write + // param[in] replayable point scn; no log newer than it will be returned. // param[out] the min log scn of next log, it's valid only when return value is OB_ITER_END // param[out] return OB_ITER_END whether casused by replayable_point_scn, it's valid only when return value is OB_ITER_END // @retval @@ -167,9 +147,8 @@ class PalfIterator // OB_INVALID_DATA. // OB_ITER_END, has iterated to the end of block. // OB_NEED_RETRY: - // 1. the data in cache is not integrity, and the integrity data has been truncate from disk, - // need read data from storage eagain. - // 2. data is truncated concurrently while reading from disk. + // 1. cached data was truncated from disk; read from storage again. + // 2. data was truncated concurrently while reading from disk. // OB_ERR_OUT_LOWER_BOUND, block has been recycled // OB_PARTIAL_LOG, iterator start lsn is not the header of LogGroupEntry. int next(const share::SCN &replayable_point_scn, @@ -200,10 +179,9 @@ class PalfIterator int get_entry(LogEntryType &entry, LSN &lsn) { int ret = OB_SUCCESS; - bool unused_is_raw_write = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn, unused_is_raw_write)) && OB_ITER_END != ret) { + } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn)) && OB_ITER_END != ret) { PALF_LOG(WARN, "PalfIterator get_entry failed", K(ret), K(entry), K(lsn), KPC(this)); } else { PALF_LOG(TRACE, "PalfIterator get_entry success", K(ret), KPC(this), @@ -214,11 +192,10 @@ class PalfIterator int get_entry(const char *&buffer, LogEntryType &entry, LSN& lsn) { int ret = OB_SUCCESS; - bool unused_is_raw_write = false; // OB_ASSERT((std::is_same::value) == true); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn, unused_is_raw_write)) && OB_ITER_END != ret) { + } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn)) && OB_ITER_END != ret) { PALF_LOG(WARN, "PalfIterator get_entry failed", K(ret), K(entry), K(lsn), KPC(this)); } else { buffer = entry.get_data_buf() - entry.get_header_size(); @@ -226,20 +203,9 @@ class PalfIterator } return ret; } - int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, bool &is_raw_write) + int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn) { - return get_entry_(buffer, nbytes, scn, lsn, is_raw_write); - } - int get_entry(const char *&buffer, int64_t &nbytes, LSN &lsn, int64_t &log_proposal_id) - { - share::SCN unused_scn; - bool unused_is_raw_write = false; - return get_entry_(buffer, nbytes, unused_scn, lsn, log_proposal_id, unused_is_raw_write); - } - int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, int64_t &log_proposal_id, - bool &is_raw_write) - { - return get_entry_(buffer, nbytes, scn, lsn, log_proposal_id, is_raw_write); + return get_entry_(buffer, nbytes, scn, lsn); } bool is_inited() const { @@ -286,21 +252,17 @@ class PalfIterator private: int do_init_(const LSN &start_offset, const GetFileEndLSN &get_file_end_lsn, - const GetModeVersion &get_mode_version, ILogStorage *log_storage) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - } else if (!get_file_end_lsn.is_valid() - || !get_mode_version.is_valid() - || NULL == log_storage) { + } else if (!get_file_end_lsn.is_valid() || NULL == log_storage) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "invalid argument", K(ret), K(start_offset), K(get_file_end_lsn), K(get_mode_version), - K(log_storage)); + PALF_LOG(WARN, "invalid argument", K(ret), K(start_offset), K(get_file_end_lsn), K(log_storage)); } else if (OB_FAIL(iterator_storage_.init(start_offset, LogEntryType::BLOCK_SIZE, get_file_end_lsn, log_storage))) { PALF_LOG(WARN, "IteratorStorage init failed", K(ret)); - } else if (OB_FAIL(iterator_impl_.init(get_mode_version, &iterator_storage_))) { + } else if (OB_FAIL(iterator_impl_.init(&iterator_storage_))) { PALF_LOG(WARN, "PalfIterator init failed", K(ret)); } else { io_ctx_.set_start_lsn(start_offset); @@ -310,14 +272,14 @@ class PalfIterator return ret; } - int get_entry_(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, bool &is_raw_write) + int get_entry_(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn) { int ret = OB_SUCCESS; LogEntryType entry; OB_ASSERT((std::is_same::value) == true); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn, is_raw_write)) && OB_ITER_END != ret) { + } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn)) && OB_ITER_END != ret) { PALF_LOG(WARN, "PalfIterator get_entry failed", K(ret), K(entry), K(lsn), KPC(this)); } else { buffer = entry.get_data_buf(); @@ -328,26 +290,6 @@ class PalfIterator return ret; } - int get_entry_(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, int64_t &log_proposal_id, - bool &is_raw_write) - { - int ret = OB_SUCCESS; - LogEntryType entry; - OB_ASSERT((std::is_same::value) == true); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (OB_FAIL(iterator_impl_.get_entry(entry, lsn, is_raw_write)) && OB_ITER_END != ret) { - PALF_LOG(WARN, "PalfIterator get_group_entry failed", K(ret), K(entry), K(lsn), KPC(this)); - } else { - buffer = entry.get_data_buf() - entry.get_header_size(); - nbytes = entry.get_serialize_size(); - scn = entry.get_scn(); - log_proposal_id = entry.get_header().get_log_proposal_id(); - PALF_LOG(TRACE, "PalfIterator get_group_entry success", K(iterator_impl_), K(ret), KPC(this), K(entry)); - } - return ret; - } - private: IteratorStorage iterator_storage_; LogIteratorImpl iterator_impl_; diff --git a/src/logservice/palf/palf_options.cpp b/src/logservice/palf/palf_options.cpp index c0aad4990..ddec6bf86 100644 --- a/src/logservice/palf/palf_options.cpp +++ b/src/logservice/palf/palf_options.cpp @@ -24,15 +24,12 @@ namespace palf void PalfOptions::reset() { disk_options_.reset(); - compress_options_.reset(); - rebuild_replica_log_lag_threshold_ = 0; enable_log_cache_ = false; - enable_fetch_log_engine_ = true; } bool PalfOptions::is_valid() const { - return disk_options_.is_valid() && compress_options_.is_valid() && (rebuild_replica_log_lag_threshold_ >= 0); + return disk_options_.is_valid(); } void PalfDiskOptions::reset() @@ -42,7 +39,6 @@ void PalfDiskOptions::reset() log_disk_utilization_threshold_ = -1; log_disk_throttling_percentage_ = -1; log_disk_throttling_maximum_duration_ = -1; - log_writer_parallelism_ = -1; } bool PalfDiskOptions::is_valid() const @@ -56,8 +52,7 @@ bool PalfDiskOptions::is_valid() const && log_disk_throttling_percentage_ >= MIN_WRITING_THTOTTLING_TRIGGER_PERCENTAGE && log_disk_throttling_percentage_ <= 100 && log_disk_throttling_maximum_duration_ >= MIN_DURATION - && log_disk_throttling_maximum_duration_ <= MAX_DURATION - && log_writer_parallelism_ >= 1 && log_writer_parallelism_ <= 8; + && log_disk_throttling_maximum_duration_ <= MAX_DURATION; } bool PalfDiskOptions::operator==(const PalfDiskOptions &palf_disk_options) const @@ -66,8 +61,7 @@ bool PalfDiskOptions::operator==(const PalfDiskOptions &palf_disk_options) const && log_disk_utilization_threshold_ == palf_disk_options.log_disk_utilization_threshold_ && log_disk_utilization_limit_threshold_ == palf_disk_options.log_disk_utilization_limit_threshold_ && log_disk_throttling_percentage_ == palf_disk_options.log_disk_throttling_percentage_ - && log_disk_throttling_maximum_duration_ == palf_disk_options.log_disk_throttling_maximum_duration_ - && log_writer_parallelism_ == palf_disk_options.log_writer_parallelism_; + && log_disk_throttling_maximum_duration_ == palf_disk_options.log_disk_throttling_maximum_duration_; } bool PalfDiskOptions::operator!=(const PalfDiskOptions &palf_disk_options) const @@ -82,39 +76,12 @@ PalfDiskOptions &PalfDiskOptions::operator=(const PalfDiskOptions &other) log_disk_utilization_limit_threshold_ = other.log_disk_utilization_limit_threshold_; log_disk_throttling_percentage_ = other.log_disk_throttling_percentage_; log_disk_throttling_maximum_duration_ = other.log_disk_throttling_maximum_duration_; - log_writer_parallelism_ = other.log_writer_parallelism_; - return *this; -} - -void PalfTransportCompressOptions::reset() -{ - enable_transport_compress_ = false; - transport_compress_func_ = ObCompressorType::INVALID_COMPRESSOR; -} - -bool PalfTransportCompressOptions::is_valid() const -{ - return !enable_transport_compress_ || (ObCompressorType::INVALID_COMPRESSOR != transport_compress_func_); -} -//To use without locking, the order of modification needs to be considered -PalfTransportCompressOptions &PalfTransportCompressOptions::operator=(const PalfTransportCompressOptions &other) -{ - if (!other.enable_transport_compress_) { - enable_transport_compress_ = other.enable_transport_compress_; - MEM_BARRIER(); - transport_compress_func_ = other.transport_compress_func_; - } else { - transport_compress_func_ = other.transport_compress_func_; - MEM_BARRIER(); - enable_transport_compress_ = other.enable_transport_compress_; - } return *this; } static const char *access_mode_strs[] = { "INVALID_ACCESS_MODE", - "APPEND", - "RAW_WRITE" + "APPEND" }; int get_access_mode(const common::ObString &str, AccessMode &mode) diff --git a/src/logservice/palf/palf_options.h b/src/logservice/palf/palf_options.h index d86de78bc..dcdbec1eb 100644 --- a/src/logservice/palf/palf_options.h +++ b/src/logservice/palf/palf_options.h @@ -28,15 +28,13 @@ namespace palf // 2. log_disk_utilization_threshold_, log_disklog disk utilization threshold before reuse log files. // 3. log_disk_utilization_limit_threshold_, maximum of log disk usage percentage before stop submitting or receiving logs. // 4. log_disk_throttling_percentage_, the threshold of the size of the log disk when writing_limit will be triggered. -// 5. log_writer_parallelism, the number of parallel log writer processes that can be used to write redo log entries to disk. struct PalfDiskOptions { PalfDiskOptions() : log_disk_usage_limit_size_(-1), log_disk_utilization_threshold_(-1), log_disk_utilization_limit_threshold_(-1), log_disk_throttling_percentage_(-1), - log_disk_throttling_maximum_duration_(-1), - log_writer_parallelism_(-1) + log_disk_throttling_maximum_duration_(-1) {} ~PalfDiskOptions() { reset(); } static constexpr int64_t MB = 1024*1024ll; @@ -50,13 +48,11 @@ struct PalfDiskOptions int log_disk_utilization_limit_threshold_; int64_t log_disk_throttling_percentage_; int64_t log_disk_throttling_maximum_duration_; - int log_writer_parallelism_; TO_STRING_KV("log_disk_size(MB)", log_disk_usage_limit_size_ / MB, "log_disk_utilization_threshold(%)", log_disk_utilization_threshold_, "log_disk_utilization_limit_threshold(%)", log_disk_utilization_limit_threshold_, "log_disk_throttling_percentage(%)", log_disk_throttling_percentage_, - "log_disk_throttling_maximum_duration(s)", log_disk_throttling_maximum_duration_ / (1000 * 1000), - "log_writer_parallelism", log_writer_parallelism_); + "log_disk_throttling_maximum_duration(s)", log_disk_throttling_maximum_duration_ / (1000 * 1000)); }; @@ -84,19 +80,12 @@ struct PalfAppendOptions // // Default value is NONBLOCK bool need_nonblock = true; - bool need_check_proposal_id = true; - int64_t proposal_id = 0; - TO_STRING_KV(K(need_nonblock), K(need_check_proposal_id), K(proposal_id)); + TO_STRING_KV(K(need_nonblock)); }; -// Palf supports switching between access modes -// // APPEND: In this mode, PALF assigns LSN and TS to the logs to be committed -// -// RAW_WRITE: In this mode, PALF does not have the capability to allocate LSN and TS for pending logs enum class AccessMode { INVALID_ACCESS_MODE = 0, APPEND = 1, - RAW_WRITE = 2, }; inline int access_mode_to_string(const AccessMode access_mode, char *str_buf_, const int64_t str_len) @@ -104,8 +93,6 @@ inline int access_mode_to_string(const AccessMode access_mode, char *str_buf_, c int ret = OB_SUCCESS; if (AccessMode::APPEND == access_mode) { strncpy(str_buf_, "APPEND", str_len); - } else if (AccessMode::RAW_WRITE == access_mode) { - strncpy(str_buf_, "RAW_WRITE", str_len); } else { ret = OB_INVALID_ARGUMENT; } @@ -116,63 +103,21 @@ int get_access_mode(const common::ObString &str, AccessMode &mode); inline bool is_valid_access_mode(const AccessMode &access_mode) { - return AccessMode::APPEND == access_mode - || AccessMode::RAW_WRITE == access_mode; + return AccessMode::APPEND == access_mode; } -inline bool can_switch_access_mode_(const AccessMode &src_access_mode, const AccessMode &dst_access_mode) -{ - bool bool_ret = true; - if (false == is_valid_access_mode(dst_access_mode)) { - // can not switch to invalid AccessMode - bool_ret = false; - } else if (src_access_mode == dst_access_mode) { - // can not switch to itself - bool_ret = false; - } - return bool_ret; -} - -struct PalfTransportCompressOptions -{ -public: - PalfTransportCompressOptions() : - enable_transport_compress_(false), - transport_compress_func_(ObCompressorType::INVALID_COMPRESSOR) - {} - ~PalfTransportCompressOptions() { reset(); } - void reset(); - bool is_valid() const; - PalfTransportCompressOptions &operator=(const PalfTransportCompressOptions &other); -public: - bool enable_transport_compress_; - ObCompressorType transport_compress_func_; - TO_STRING_KV(K(enable_transport_compress_), - K(transport_compress_func_)); -}; - struct PalfOptions { PalfOptions() : disk_options_(), - compress_options_(), - rebuild_replica_log_lag_threshold_(0), - enable_log_cache_(false), - enable_fetch_log_engine_(true) + enable_log_cache_(false) {} ~PalfOptions() { reset(); } void reset(); bool is_valid() const; - TO_STRING_KV(K(disk_options_), - K(compress_options_), - K(rebuild_replica_log_lag_threshold_), - K(enable_log_cache_), - K(enable_fetch_log_engine_)); + TO_STRING_KV(K(disk_options_), K(enable_log_cache_)); public: PalfDiskOptions disk_options_; - PalfTransportCompressOptions compress_options_; - int64_t rebuild_replica_log_lag_threshold_; bool enable_log_cache_; - bool enable_fetch_log_engine_; }; struct PalfThrottleOptions diff --git a/src/logservice/palf_handle_guard.h b/src/logservice/palf_handle_guard.h index 593ef8929..f5a3dd6a0 100644 --- a/src/logservice/palf_handle_guard.h +++ b/src/logservice/palf_handle_guard.h @@ -55,15 +55,6 @@ class PalfHandleGuard palf_env_ = palf_env; palf_handle.palf_handle_impl_ = NULL; } - // @brief set the initial member list of paxos group - // @param[in] ObMemberList, the initial member list - // @param[in] int64_t, the paxos relica num - // @retval - // return OB_SUCCESS if success - // else return other errno - // int set_initial_member_list(const common::ObMemberList &member_list, - // const int64_t paxos_replica_num); - DELEGATE_WITH_RET(palf_handle_, set_initial_member_list, int); // @brief append count bytes from the buffer starting at buf to the palf handle, return the LSN and timestamp // @param[in] cost PalfAppendOptions&, decide this append option whether need block thread. // @param[in] const void *, the data buffer. @@ -79,7 +70,6 @@ class PalfHandleGuard // LSN &lsn, // int64_t &scn); DELEGATE_WITH_RET(palf_handle_, append, int); - DELEGATE_WITH_RET(palf_handle_, raw_write, int); DELEGATE_WITH_RET(palf_handle_, raw_read, int); // @breif, query lsn by timestamp, note that this function may be time-consuming @@ -98,7 +88,6 @@ class PalfHandleGuard DELEGATE_WITH_RET(palf_handle_, advance_base_lsn, int); CONST_DELEGATE_WITH_RET(palf_handle_, get_base_lsn, int); - DELEGATE_WITH_RET(palf_handle_, change_leader_to, int); // @breif, get begin lsn, begin lsn maybe smaller than recycable lsn, because palf will not delete data before // recycable lsn immediately. // @param[out] int64_t&, begin lsn. @@ -126,115 +115,13 @@ class PalfHandleGuard // @param[out] int64_t, LSN. // int get_max_lsn(LSN &lsn) const; CONST_DELEGATE_WITH_RET(palf_handle_, get_max_lsn, int); - // @brief get readable end lsn for this replica, all logs before it can be readable. + // @brief get readable end lsn; all logs before it are readable. // @param[out] lsn, readable end lsn. // -- OB_NOT_INIT not_init // -- OB_SUCCESS CONST_DELEGATE_WITH_RET(palf_handle_, get_readable_end_lsn, int); - // @brief, get role of this replica - // @param[out] common::ObRole& - // @param[out] int64_t&, the proposal_id of current leader, palf will ensure increasing this field monotonically. - // @param[out] bool&, whether it's in pending state - // int get_role(common::ObRole &role, int64_t &proposal_id) const; - int get_role(common::ObRole &role, int64_t &proposal_id) - { - bool unused_state; - return palf_handle_.get_role(role, proposal_id, unused_state); - } - - int get_role(common::ObRole &role, int64_t &proposal_id, bool &is_pending_state) - { - return palf_handle_.get_role(role, proposal_id, is_pending_state); - } - // @brief, get paxos member list of this paxos group - // @param[out] common::ObMemberList& - // int get_paxos_member_list(common::ObMemberList &member_list) const override final; - CONST_DELEGATE_WITH_RET(palf_handle_, get_paxos_member_list, int); - // @brief, get paxos member list and learner_list of this paxos group - // @param[out] common::ObMemberList& - // int get_paxos_member_list(common::ObMemberList &member_list) const override final; - CONST_DELEGATE_WITH_RET(palf_handle_, get_paxos_member_list_and_learner_list, int); - - // @brief: a special config change interface, change replica number of paxos group - // @param[in] common::ObMemberList: current memberlist, for pre-check - // @param[in] const int64_t curr_replica_num: current replica num, for pre-check - // @param[in] const int64_t new_replica_num: new replica num - // @param[in] const int64_t timeout_us: timeout, ns - // @return - // - OB_SUCCESS: change_replica_num successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: change_replica_num timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - DELEGATE_WITH_RET(palf_handle_, change_replica_num, int); - -// @brief, add a member to paxos group, can be called only in leader - // @param[in] common::ObMember &member: member which will be added - // @param[in] const int64_t paxos_replica_num: replica number of paxos group after adding 'member' - // @param[in] const int64_t timeout_us: add member timeout, ns - // @param[in] const palf::LogConfigVersion &config_version: config_version for checking leader's - // config_version - // @return - // - OB_SUCCESS: add member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: add member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - OB_STATE_NOT_MATCH: leader has switched - // - other: bug - DELEGATE_WITH_RET(palf_handle_, add_member, int); - - // @brief, get config_version - // @return - // - OB_SUCCESS: get_config_version successfully - // - OB_NOT_INIT - DELEGATE_WITH_RET(palf_handle_, get_config_version, int); - - // @brief, remove a member from paxos group, can be called only in leader - // @param[in] common::ObMember &member: member which will be removed - // @param[in] const int64_t paxos_replica_num: replica number of paxos group after removing 'member' - // @param[in] const int64_t timeout_us: remove member timeout, ns - // @return - // - OB_SUCCESS: remove member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: remove member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - // int remove_member(const common::ObMember &member, - // const int64_t paxos_replica_num, - // const int64_t timeout_us) - DELEGATE_WITH_RET(palf_handle_, remove_member, int); - - // @brief, replace old_member with new_member, can be called only in leader - // @param[in] const common::ObMember &removed_member: member will be removed - // @param[in] const common::ObMember &added_member: member will be added - // @param[in] const int64_t timeout_us - // @return - // - OB_SUCCESS: replace member successfully - // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change - // - OB_TIMEOUT: replace member timeout - // - OB_NOT_MASTER: not leader or rolechange during membership changing - // - other: bug - DELEGATE_WITH_RET(palf_handle_, replace_member, int); - DELEGATE_WITH_RET(palf_handle_, add_learner, int); - DELEGATE_WITH_RET(palf_handle_, remove_learner, int); - DELEGATE_WITH_RET(palf_handle_, switch_learner_to_acceptor, int); - DELEGATE_WITH_RET(palf_handle_, switch_acceptor_to_learner, int); - DELEGATE_WITH_RET(palf_handle_, set_location_cache_cb, int); - DELEGATE_WITH_RET(palf_handle_, change_access_mode, int); - DELEGATE_WITH_RET(palf_handle_, get_access_mode, int); CONST_DELEGATE_WITH_RET(palf_handle_, stat, int); - DELEGATE_WITH_RET(palf_handle_, try_lock_config_change, int); - DELEGATE_WITH_RET(palf_handle_, unlock_config_change, int); - DELEGATE_WITH_RET(palf_handle_, get_config_change_lock_stat, int); - DELEGATE_WITH_RET(palf_handle_, disable_sync, int); - DELEGATE_WITH_RET(palf_handle_, enable_sync, int); - DELEGATE_WITH_RET(palf_handle_, advance_base_info, int); - DELEGATE_WITH_RET(palf_handle_, is_vote_enabled, bool); - DELEGATE_WITH_RET(palf_handle_, get_election_leader, int); - DELEGATE_WITH_RET(palf_handle_, is_sync_enabled, bool); - DELEGATE_WITH_RET(palf_handle_, get_access_mode_version, int); - CONST_DELEGATE_WITH_RET(palf_handle_, get_election_leader, int); CONST_DELEGATE_WITH_RET(palf_handle_, get_palf_epoch, int); TO_STRING_KV(K(palf_handle_)); private: diff --git a/src/logservice/rcservice/ob_role_change_handler.cpp b/src/logservice/rcservice/ob_role_change_handler.cpp deleted file mode 100644 index b91ff6609..000000000 --- a/src/logservice/rcservice/ob_role_change_handler.cpp +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_role_change_handler.h" -#include "share/rc/ob_tenant_base.h" -namespace oceanbase -{ -using namespace common; -namespace logservice -{ -void RCDiagnoseInfo::reset() -{ - id_ = -1; - state_ = TakeOverState::INVALID_TAKE_OVER_STATE; - log_type_ = ObLogBaseType::INVALID_LOG_BASE_TYPE; -} - -ObRoleChangeHandler::ObRoleChangeHandler(): lock_(common::ObLatchIds::RCS_LOCK), - sub_role_change_handler_arr_() -{ - reset(); -} - -ObRoleChangeHandler::~ObRoleChangeHandler() -{ - reset(); -} - -void ObRoleChangeHandler::reset() -{ - for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { - sub_role_change_handler_arr_[i] = NULL; - } -} - -int ObRoleChangeHandler::register_handler(const ObLogBaseType &type, - ObIRoleChangeSubHandler *role_change_handler) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(lock_); - if (false == is_valid_log_base_type(type)) { - ret = OB_INVALID_ARGUMENT; - } else { - sub_role_change_handler_arr_[type] = role_change_handler; - CLOG_LOG(INFO, "register_handler success", K(ret), K(type), KP(role_change_handler)); - } - return ret; -} - -void ObRoleChangeHandler::unregister_handler(const ObLogBaseType &type) -{ - ObSpinLockGuard guard(lock_); - if (true == is_valid_log_base_type(type)) { - sub_role_change_handler_arr_[type] = NULL; - CLOG_LOG(INFO, "unregister_handler success", K(type)); - } -} - -void ObRoleChangeHandler::switch_to_follower_forcedly() -{ - ObSpinLockGuard guard(lock_); - for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { - ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; - char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; - ObLogBaseType base_type = static_cast(i); - bool has_defined_to_string = false; - if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, - OB_LOG_BASE_TYPE_STR_MAX_LEN)) { - has_defined_to_string = true; - } - if (NULL != handler) { - handler->switch_to_follower_forcedly(); - CLOG_LOG(INFO, "leader to follower forcedly, current role change handler is", - "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - } - } -} - -int ObRoleChangeHandler::switch_to_leader(RCDiagnoseInfo &diagnose_info) -{ - int ret = OB_SUCCESS; - CLOG_LOG(INFO, "ObRoleChangeHandler::switch_to_leader called"); - ObSpinLockGuard guard(lock_); - for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE && OB_SUCC(ret); i++) { - ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; - char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; - ObLogBaseType base_type = static_cast(i); - diagnose_info.log_type_ = base_type; - bool has_defined_to_string = false; - if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, - OB_LOG_BASE_TYPE_STR_MAX_LEN)) { - has_defined_to_string = true; - } - if (NULL == handler) { - if (i == static_cast(ObLogBaseType::TIMESTAMP_LOG_BASE_TYPE)) { - CLOG_LOG(WARN, "TIMESTAMP_LOG_BASE_TYPE handler is NULL", K(i)); - } - } else if (OB_FAIL(handler->switch_to_leader())) { - CLOG_LOG(WARN, "switch_to_leader failed", K(ret), KP(handler), K(i), - "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - } else { - CLOG_LOG(INFO, "follower to leader, current role change handler is", - "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - } - } - CLOG_LOG(INFO, "ObRoleChangeHandler::switch_to_leader finished", K(ret)); - return ret; -} - -int ObRoleChangeHandler::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - ObSpinLockGuard guard(lock_); - int64_t cursor = 0; - const int64_t count = ObLogBaseType::MAX_LOG_BASE_TYPE; - while (cursor < count && OB_SUCC(ret)) { - char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; - ObLogBaseType base_type = static_cast(cursor); - bool has_defined_to_string = false; - if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, - OB_LOG_BASE_TYPE_STR_MAX_LEN)) { - has_defined_to_string = true; - } - ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[cursor]; - if (NULL == handler) { - cursor++; - } else if (OB_FAIL(handler->switch_to_follower_gracefully()) && OB_LS_NEED_REVOKE != ret) { - CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need resume other sub modules", K(ret), - KP(handler), K(cursor), - "cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - // NB: resume_leader failed, need revoke leader. - } else if (OB_LS_NEED_REVOKE == ret) { - CLOG_LOG(WARN, "ObIRoleChangeSubHandler resume leader failed", K(ret), K(cursor)); - } else { - CLOG_LOG(INFO, "leader to follower gracefully, current role change handler is", - "cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - cursor++; - } - } - // if any sub role handler switch_to_follower_gracefully failed, and no need to revoke leader, - // we should resume other sub role handler, meanwhile, we should overrite 'ret' only if - // resume_leader_when_switch_failure_ failed. - if (OB_FAIL(ret) && OB_LS_NEED_REVOKE != ret) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = resume_leader_when_switch_failure_(cursor))) { - CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed", K(tmp_ret), K(cursor)); - ret = tmp_ret; - } else { - CLOG_LOG(WARN, "resume_leader_when_switch_failure_ success, no need to excut follower to leader gracefully", - K(ret), K(tmp_ret)); - } - } - return ret; -} - -int ObRoleChangeHandler::resume_to_leader() -{ - int ret = OB_SUCCESS; - int cursor = ObLogBaseType::MAX_LOG_BASE_TYPE; - if (OB_FAIL(resume_leader_when_switch_failure_(cursor))) { - CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed"); - } else { - CLOG_LOG(INFO, "resume_to_leader success"); - } - return ret; -} - -int ObRoleChangeHandler::resume_leader_when_switch_failure_(const int64_t cursor) -{ - int ret = OB_SUCCESS; - for (int64_t i = cursor - 1; i >= 0 && OB_SUCC(ret); i--) { - ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; - ObLogBaseType base_type = static_cast(i); - char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; - bool has_defined_to_string = false; - if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, - OB_LOG_BASE_TYPE_STR_MAX_LEN)) { - has_defined_to_string = true; - } - if (NULL == handler){ - CLOG_LOG(INFO, "not register into role change service", K(ret), K(i)); - } else if (OB_FAIL(handler->resume_leader())) { - CLOG_LOG(WARN, "resume_leader failed", K(ret), K(i), KP(handler), - "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - } else { - CLOG_LOG(INFO, "resume_leader success", K(ret), K(i), KP(handler), - "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); - } - } - if (OB_FAIL(ret)) { - ret = OB_LS_NEED_REVOKE; - } - return ret; -} -} // end namespace logservice -} // end namespace oceanbase diff --git a/src/logservice/rcservice/ob_role_change_handler.h b/src/logservice/rcservice/ob_role_change_handler.h deleted file mode 100644 index d58d423a6..000000000 --- a/src/logservice/rcservice/ob_role_change_handler.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_ROLE_CHANGE_HANDLER_ -#define OCEANBASE_LOGSERVICE_OB_ROLE_CHANGE_HANDLER_ -#include "lib/container/ob_fixed_array.h" -#include "lib/container/ob_se_array.h" -#include "lib/lock/ob_spin_lock.h" -#include "lib/utility/ob_print_utils.h" -#include "logservice/ob_log_base_header.h" -#include "logservice/ob_log_base_type.h" -#include "share/ob_errno.h" -namespace oceanbase -{ -namespace logservice -{ -enum TakeOverState { - INVALID_TAKE_OVER_STATE = 0, - WAIT_REPLAY_DONE = 1, - WAIT_RC_HANDLER_DONE = 2, - TAKE_OVER_FINISH = 3, - UNKNOWN_TAKE_OVER_STATE = 4, - MAX_TAKE_OVER_STATE = 5 -}; - -static inline -int takeover_state_to_string(const TakeOverState log_type, - char *str, - const int64_t str_len) -{ - int ret = OB_SUCCESS; - if (log_type == INVALID_TAKE_OVER_STATE) { - strncpy(str ,"INVALID_STATE", str_len); - } else if (log_type == WAIT_REPLAY_DONE) { - strncpy(str ,"WAIT_REPLAY_DONE", str_len); - } else if (log_type == WAIT_RC_HANDLER_DONE) { - strncpy(str ,"WAIT_RC_HANDLER_DONE", str_len); - } else if (log_type == TAKE_OVER_FINISH) { - strncpy(str ,"FINISH", str_len); - } else if (log_type == UNKNOWN_TAKE_OVER_STATE) { - strncpy(str ,"UNKNOWN", str_len); - } else { - ret = OB_INVALID_ARGUMENT; - } - return ret; -} - -struct RCDiagnoseInfo { - RCDiagnoseInfo() { reset(); } - ~RCDiagnoseInfo() { reset(); } - void reset(); - int64_t id_; - TakeOverState state_; - ObLogBaseType log_type_; //Only meaningful when in WAIT_RC_HANDLER_DONE state - TO_STRING_KV(K(state_), - K(log_type_)); -}; - -class ObRoleChangeHandler { -public: - ObRoleChangeHandler(); - ~ObRoleChangeHandler(); - void reset(); - int register_handler(const ObLogBaseType &type, ObIRoleChangeSubHandler *role_change_hander); - void unregister_handler(const ObLogBaseType &type); - - void switch_to_follower_forcedly(); - int switch_to_leader(RCDiagnoseInfo &diagnose_info); - // @retval: - // 1. OB_SUCCESS - // 2. OB_LS_NEED_REVOKE, ObRoleChangeService need revoke this LS. - // 3. OTHERS, switch_to_follower_gracefully failed, but no need to revoke leader. - int switch_to_follower_gracefully(); - int resume_to_leader(); - -private: - int resume_leader_when_switch_failure_(int64_t cursor); -private: - ObSpinLock lock_; - ObIRoleChangeSubHandler* sub_role_change_handler_arr_[ObLogBaseType::MAX_LOG_BASE_TYPE]; -}; -} -} -#endif diff --git a/src/logservice/rcservice/ob_role_change_service.cpp b/src/logservice/rcservice/ob_role_change_service.cpp deleted file mode 100644 index ad5f81269..000000000 --- a/src/logservice/rcservice/ob_role_change_service.cpp +++ /dev/null @@ -1,841 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_role_change_service.h" -#include "observer/ob_server_event_history_table_operator.h" - -namespace oceanbase -{ -using namespace common; -using namespace palf; -using namespace logservice; -using namespace share; -namespace logservice -{ - -RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type, - const share::ObLSID &ls_id) : event_type_(event_type), - ls_id_(ls_id) -{ -} -RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type, - const share::ObLSID &ls_id, - const common::ObAddr &dst_addr) : event_type_(event_type), - ls_id_(ls_id), - dst_addr_(dst_addr) -{ -} - -bool RoleChangeEvent::is_valid() const -{ - return RoleChangeEventType::INVALID_RC_EVENT_TYPE != event_type_ - && false != ls_id_.is_valid(); -} - -void RoleChangeEvent::reset() -{ - event_type_ = RoleChangeEventType::INVALID_RC_EVENT_TYPE; - ls_id_.reset(); - dst_addr_.reset(); -} - -bool RoleChangeEvent::operator==(const RoleChangeEvent &rhs) const -{ - // for change leader event, we just check 'ls_id'. - return event_type_ == rhs.event_type_ && ls_id_ == rhs.ls_id_; -} - -RoleChangeEventSet::RoleChangeEventSet() -{} - -RoleChangeEventSet::~RoleChangeEventSet() -{} - -int RoleChangeEventSet::insert(const RoleChangeEvent &event) -{ - int ret = OB_SUCCESS; - int64_t free_idx = -1; - ObSpinLockGuard guard(lock_); - for (int64_t i = 0; i < MAX_ARRAY_SIZE; i++) { - if (event == events_[i]) { - ret = OB_ENTRY_EXIST; - } - } - for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == free_idx && OB_SUCC(ret); i++) { - if (false == events_[i].is_valid()) { - free_idx = i; - } - } - if (OB_ENTRY_EXIST == ret) { - } else if (-1 != free_idx) { - events_[free_idx] = event; - } else { - ret = OB_SIZE_OVERFLOW; - } - CLOG_LOG(INFO, "insert event into set success", K(ret), K(event), K(free_idx)); - return ret; -} - -int RoleChangeEventSet::remove(const RoleChangeEvent &event) -{ - int ret = OB_SUCCESS; - int64_t delete_idx = -1; - ObSpinLockGuard guard(lock_); - for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == delete_idx; i++) { - if (event == events_[i]) { - delete_idx = i; - } - }; - if (-1 != delete_idx) { - events_[delete_idx].reset(); - } else { - ret = OB_ENTRY_NOT_EXIST; - } - CLOG_LOG(INFO, "remove slog from set success", K(ret), K(delete_idx), K(event)); - return ret; -} - -ObRoleChangeService::ObRoleChangeService() : ls_service_(NULL), - apply_service_(NULL), - replay_service_(NULL), - cur_task_info_(), - is_inited_(false) -{ -} - -ObRoleChangeService::~ObRoleChangeService() -{ - if (IS_INIT) { - destroy(); - } -} - -int ObRoleChangeService::init(storage::ObLSService *ls_service, - logservice::ObLogApplyService *apply_service, - logservice::ObILogReplayService *replay_service) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - } else if (OB_ISNULL(ls_service) || OB_ISNULL(apply_service) || OB_ISNULL(replay_service)) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(ret), KP(ls_service), KP(apply_service), KP(replay_service)); - } else if (OB_FAIL(common::ObSimpleThreadPool::init(MAX_THREAD_NUM, MAX_RC_EVENT_TASK, "RCSrv"))) { - CLOG_LOG(WARN, "ObRoleChangeService thread pool init failed", K(ret)); - } else if (OB_FAIL(common::ObSimpleThreadPool::set_adaptive_thread(MAX_THREAD_NUM, MAX_THREAD_NUM))) { - CLOG_LOG(WARN, "ObRoleChangeService set thread count failed", K(ret)); - } else { - cur_task_info_.reset(); - ls_service_ = ls_service; - apply_service_ = apply_service; - replay_service_ = replay_service; - is_inited_ = true; - CLOG_LOG(INFO, "ObRoleChangeService init success", K(ret), KP(ls_service), KP(apply_service), KP(replay_service)); - } - if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { - common::ObSimpleThreadPool::destroy(); - } - return ret; -} - -int ObRoleChangeService::start() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - } else if (common::ObSimpleThreadPool::get_thread_count() <= 0 - && !common::ObSimpleThreadPool::try_expand_one(MAX_THREAD_NUM)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "ObRoleChangeService start failed", K(ret)); - } else { - CLOG_LOG(INFO, "ObRoleChangeService start success", K(ret)); - } - return ret; -} - -void ObRoleChangeService::wait() -{ - if (IS_INIT) { - common::ObSimpleThreadPool::stop(); - common::ObSimpleThreadPool::wait(); - } - CLOG_LOG(INFO, "ObRoleChangeService wait finish"); -} - -void ObRoleChangeService::stop() -{ - if (IS_INIT) { - common::ObSimpleThreadPool::stop(); - } - CLOG_LOG(INFO, "ObRoleChangeService stop finish"); -} - -void ObRoleChangeService::destroy() -{ - if (IS_INIT) { - (void)stop(); - (void)wait(); - common::ObSimpleThreadPool::destroy(); - is_inited_ = false; - cur_task_info_.reset(); - ls_service_ = NULL; - apply_service_ = NULL; - replay_service_ = NULL; - CLOG_LOG(INFO, "ObRoleChangeService destroy success"); - } -} - -void ObRoleChangeService::handle(void *task) -{ - ObDIActionGuard ag("LogService", "RoleChangeService", "HandleEvent"); - int ret = OB_SUCCESS; - // When role chage service hang exceeds 30 seconds, we think there is dead lock in 'handle_role_change_event_', - // TIMEGUARD will pring lbt(). - TIMEGUARD_INIT(CLOG, 30_s); - RoleChangeEvent *event = reinterpret_cast(task); - const int64_t ls_id = event->ls_id_.id(); - const int64_t start_ts = ObTimeUtility::current_time(); - RetrySubmitRoleChangeEventCtx retry_ctx; - CLOG_LOG(INFO, "begin handle_role_change_event_", "sequence:", start_ts, KPC(event)); - if (NULL == event) { - CLOG_LOG(WARN, "unexpected error, task is nullptr", KP(event)); - } else if (OB_FAIL(handle_role_change_event_(*event, retry_ctx))) { - CLOG_LOG(WARN, "handle_role_change_event_ failed", K(ret), KPC(event), K(retry_ctx)); - } else { - CLOG_LOG(INFO, "end handle_role_change_event_", "sequence:", start_ts, KPC(event)); - } - if (NULL != event) { - OB_DELETE(RoleChangeEvent, "RCService", event); - } - if (retry_ctx.need_retry() && OB_FAIL(on_role_change(ls_id))) { - CLOG_LOG(WARN, "retry submit role change event failed", K(ls_id), K(retry_ctx)); - } -} - -int ObRoleChangeService::on_role_change(const int64_t id) -{ - int ret = OB_SUCCESS; - share::ObLSID ls_id(id); - RoleChangeEvent event(RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE, ls_id); - // TODO by runlin: if task queue has been full, push task will be failed, the role change event - // will be lost. - if (OB_FAIL(submit_role_change_event_(event))) { - CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event)); - } else { - CLOG_LOG(INFO, "on_role_change success", K(ret), K(event)); - } - return ret; -} - -int ObRoleChangeService::on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr) -{ - int ret = OB_SUCCESS; - RoleChangeEvent event(RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE, share::ObLSID(ls_id), dst_addr); - if (OB_FAIL(submit_role_change_event_(event))) { - CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event)); - } else { - CLOG_LOG(INFO, "change_leader success", K(ret), K(event)); - } - return ret; -} - -int ObRoleChangeService::submit_role_change_event_(const RoleChangeEvent &event) -{ - int ret = OB_SUCCESS; - if(OB_FAIL(rc_set_.insert(event)) && OB_ENTRY_EXIST != ret) { - CLOG_LOG(ERROR, "insert into rc_set failed", K(ret), K(event)); - } else if (OB_ENTRY_EXIST == ret) { - CLOG_LOG(INFO, "repeat role change event, filter it", K(ret), K(event)); - ret = OB_SUCCESS; - } else if (OB_FAIL(push_event_into_queue_(event))) { - CLOG_LOG(WARN, "push_event_into_queue_ failed", K(ret), K(event)); - } else { - CLOG_LOG(INFO, "submit_role_change_event_ success", K(ret), K(event)); - } - return ret; -} - -// TODO: use poll to avoid alloc memory failed. -int ObRoleChangeService::push_event_into_queue_(const RoleChangeEvent &event) -{ - int ret = OB_SUCCESS; - RoleChangeEvent *rc_event = NULL; - - int64_t warn_time = OB_INVALID_TIMESTAMP; - do { - if (NULL == (rc_event = - MTL_NEW(RoleChangeEvent, "RCService", event.event_type_, event.ls_id_, event.dst_addr_))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - if (palf_reach_time_interval(1 * 1000 * 1000, warn_time)) { - CLOG_LOG(WARN, "allocate memory failed", K(ret), K(event)); - } - usleep(1 * 1000); - } else { - ret = OB_SUCCESS; - } - } while(OB_FAIL(ret)); - - if (OB_FAIL(common::ObSimpleThreadPool::push(rc_event))) { - CLOG_LOG(WARN, "ObRoleChangeTask push task failed", K(ret), K(event)); - } - if (OB_FAIL(ret) && NULL != rc_event) { - MTL_DELETE(RoleChangeEvent, "RCService", rc_event); - } - return ret; -} - -int ObRoleChangeService::handle_role_change_event_(const RoleChangeEvent &event, - RetrySubmitRoleChangeEventCtx &retry_ctx) -{ - int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = NULL; - AccessMode curr_access_mode; - int64_t unused_mode_version; - OB_ASSERT(OB_SUCCESS == rc_set_.remove(event)); - if (false == event.is_valid()) { - ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(event)); - } else if (OB_FAIL(ls_service_->get_ls(event.ls_id_, ls_handle, ObLSGetMod::LOG_MOD)) - || NULL == (ls = ls_handle.get_ls())) { - ret = OB_ENTRY_NOT_EXIST; - CLOG_LOG(WARN, "get log stream from ObLSService failed", K(ret), K(event)); - } else if (OB_FAIL(ls->get_log_handler()->get_access_mode(unused_mode_version, curr_access_mode))) { - CLOG_LOG(WARN, "ObLogHandler get_access_mode failed", K(ret)); - } else { - switch (event.event_type_) { - case RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE: - ObDIActionGuard("change leader event"); - CLOG_LOG(INFO, "begin change leader", K(curr_access_mode), K(event), KPC(ls)); -#ifdef ERRSIM - ret = OB_E(EventTable::EN_RC_ONLY_LEADER_TO_LEADER) OB_SUCCESS; - if (OB_FAIL(ret)) { - CLOG_LOG(INFO, "fake EN_RC_ONLY_LEADER_TO_LEADER with change_leader_event", KPC(ls), K(event)); - } -#endif - if (OB_FAIL(ret)) { - } else if (is_append_mode(curr_access_mode) - && OB_FAIL(handle_change_leader_event_for_log_handler_(event.dst_addr_, ls))) { - CLOG_LOG(WARN, "ObLogHandler change leader failed", K(ret), K(event), KPC(ls)); - } - CLOG_LOG(INFO, "end change leader", K(ret), K(curr_access_mode), K(event), KPC(ls)); - break; - case RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE: - ObDIActionGuard("role change cb event"); - CLOG_LOG(INFO, "begin log handler role change", K(curr_access_mode), K(event), KPC(ls)); - if (OB_FAIL(handle_role_change_cb_event_for_log_handler_(curr_access_mode, ls, retry_ctx))) { - CLOG_LOG(WARN, "handle_role_change_cb_event_for_log_handler_ failed", K(ret), - K(curr_access_mode), KPC(ls)); - } - CLOG_LOG(INFO, "end log handler role change", K(ret), K(curr_access_mode), K(event), KPC(ls), K(retry_ctx)); - CLOG_LOG(INFO, "begin restore handler role change", K(curr_access_mode), K(event), KPC(ls)); - CLOG_LOG(INFO, "end restore handler role change", K(ret), K(curr_access_mode), K(event), KPC(ls)); - break; - default: - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "unexpected role change event type", K(ret)); - } - } - if (OB_SUCC(ret) && OB_NOT_NULL(ls) && !retry_ctx.need_retry()) { - (void)ls->report_replica_info(); - } - return ret; -} - -int ObRoleChangeService::handle_role_change_cb_event_for_log_handler_( - const AccessMode &curr_access_mode, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx) -{ - int ret = OB_SUCCESS; - const bool log_handler_is_offline = ls->get_log_handler()->is_offline(); - - // If log handler is offline, need execute LEADER_2_FOLLOWER or FOLLOWER_2_FOLLOWER - // - // when access mode is APPEND, log_handler need execute leader to follower or - // follower to leader. otherwise, only need execute leader to follower or follower - // to follower, therefore, we set 'need_transform_by_access_mode' to false when - // 'curr_access_mode' is APPEND. - const bool only_need_change_to_follower = !is_append_mode(curr_access_mode) || log_handler_is_offline; - RoleChangeOptType opt_type; - ObRole curr_role = ObRole::INVALID_ROLE; - ObRole new_role = ObRole::INVALID_ROLE; - bool is_pending_state = false; - int64_t curr_proposal_id = -1; - int64_t new_proposal_id = -1; - const share::ObLSID &ls_id = ls->get_ls_id(); - CLOG_LOG(INFO, "handle_role_change_cb_event_for_log_handler_ called", K(ls_id), K(curr_access_mode), K(log_handler_is_offline), K(only_need_change_to_follower)); - if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role, - curr_proposal_id, new_role, new_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id), - K(new_role), K(new_proposal_id)); - } else if (false == need_execute_role_change(curr_proposal_id, curr_role, new_proposal_id, - new_role, is_pending_state, log_handler_is_offline)) { - CLOG_LOG(INFO, "no need change role", K(ret), K(ls_id), K(is_pending_state), K(curr_role), K(curr_proposal_id), - K(new_role), K(new_proposal_id), K(is_pending_state), K(log_handler_is_offline)); - } else { - opt_type = get_role_change_opt_type_(curr_role, new_role, only_need_change_to_follower); - CLOG_LOG(INFO, "get_role_change_opt_type_ result", K(ls_id), K(opt_type), K(curr_role), K(new_role), K(only_need_change_to_follower)); - -#ifdef ERRSIM - ret = OB_E(EventTable::EN_RC_ONLY_LEADER_TO_LEADER) OB_SUCCESS; - if (RoleChangeOptType::LEADER_2_LEADER == opt_type) { - ret = OB_SUCCESS; - } - if (OB_FAIL(ret)) { - CLOG_LOG(INFO, "fake EN_RC_ONLY_LEADER_TO_LEADER with role_change_event", KPC(ls), K(opt_type)); - } -#endif - - switch (opt_type) { - // leader -> follower - case RoleChangeOptType::LEADER_2_FOLLOWER: - if (OB_SUCC(ret) && OB_FAIL(switch_leader_to_follower_forcedly_(new_proposal_id, ls))) { - CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(curr_role), - K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); - } - break; - // follower -> follower - case RoleChangeOptType::FOLLOWER_2_LEADER: - CLOG_LOG(INFO, "FOLLOWER_2_LEADER: about to call switch_follower_to_leader_", K(ls_id), K(curr_role), K(new_role), K(curr_proposal_id), K(new_proposal_id)); - if (OB_SUCC(ret) && OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) { - CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(curr_role), K(curr_proposal_id), K(new_role), - K(curr_access_mode), K(new_proposal_id)); - } else { - CLOG_LOG(INFO, "FOLLOWER_2_LEADER: switch_follower_to_leader_ success", K(ls_id), K(curr_role), K(new_role)); - } - break; - // leader -> leader - case RoleChangeOptType::LEADER_2_LEADER: - if (OB_SUCC(ret) && OB_FAIL(switch_leader_to_leader_(new_proposal_id, curr_proposal_id, - ls, retry_ctx))) { - CLOG_LOG(WARN, "switch_leader_to_leader_ failed", K(ret), K(curr_role), - K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); - } - break; - // follower -> follower - case RoleChangeOptType::FOLLOWER_2_FOLLOWER: - if (OB_SUCC(ret) && OB_FAIL(switch_follower_to_follower_(new_proposal_id, ls))) { - CLOG_LOG(WARN, "switch_follower_to_follower_ failed", K(curr_role), K(curr_proposal_id), - K(new_role), K(curr_access_mode), K(new_proposal_id)); - } - break; - default: - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "unexpected error, can not handle role change", K(ret), - K(curr_role), K(curr_proposal_id), K(new_role), K(new_proposal_id), KPC(ls)); - } - } - return ret; -} - -int ObRoleChangeService::handle_change_leader_event_for_log_handler_( - const common::ObAddr &dst_addr, - ObLS *ls) -{ - int ret = OB_SUCCESS; - ObRole curr_role = ObRole::INVALID_ROLE; - ObRole new_role = ObRole::INVALID_ROLE; - bool is_pending_state = false; - int64_t curr_proposal_id = -1; - int64_t new_proposal_id = -1; - if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role, - curr_proposal_id, new_role, new_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id), - K(new_role), K(new_proposal_id)); - } else if (true == is_pending_state - || curr_proposal_id != new_proposal_id || LEADER != curr_role || LEADER != new_role) { - // when log handler is not LEDAER, we also need execute change_leader_to, otherwise, the leader can not be changed by election. - ls->get_log_handler()->change_leader_to(dst_addr); - CLOG_LOG(INFO, "no need execute switch_leader_to_follower_gracefully, change leader directlly", - K(ret), K(is_pending_state), K(curr_proposal_id), K(new_proposal_id), K(curr_role), K(new_role)); - } else if (OB_FAIL(switch_leader_to_follower_gracefully_(new_proposal_id, curr_proposal_id, - dst_addr, ls))) { - CLOG_LOG(WARN, "switch_leader_to_follower_gracefully_ failed", K(ret), KPC(ls), - K(curr_role), K(curr_proposal_id), K(new_role), K(new_proposal_id)); - } else { - CLOG_LOG(INFO, "handle_change_leader_event_for_log_handler_ success", K(ret), K(curr_role), - K(curr_proposal_id), K(new_role), K(new_proposal_id), K(dst_addr)); - } - return ret; -} - -int ObRoleChangeService::switch_follower_to_leader_( - const int64_t new_proposal_id, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx) -{ - int ret = OB_SUCCESS; - const ObRole new_role = LEADER; - const share::ObLSID &ls_id = ls->get_ls_id(); - palf::LSN end_lsn; - ObTimeGuard time_guard("switch_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME); - ObLogHandler *log_handler = ls->get_log_handler(); - ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); - ATOMIC_SET(&cur_task_info_.state_, TakeOverState::WAIT_REPLAY_DONE); - ATOMIC_SET(&cur_task_info_.id_, ls->get_ls_id().id()); - if (OB_FAIL(log_handler->get_end_lsn(end_lsn))) { - CLOG_LOG(WARN, "get_end_lsn failed", K(ret), KPC(ls)); - // NB: order is vital!!! - // We must guarantee that 'replay_service_' has replayed complete data, and before - // stop 'replay_service_', other components can not submit log. - } else if (FALSE_IT(time_guard.click("wait_replay_service_apply_done_")) - || OB_FAIL(wait_replay_service_replay_done_(ls_id, end_lsn, WAIT_REPLAY_DONE_TIMEOUT_US))) { - if (need_retry_submit_role_change_event_(ret)) { - retry_ctx.set_retry_reason(RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT); - } else { - CLOG_LOG(WARN, "wait_replay_service_replay_done_ failed", K(ret), K(end_lsn)); - } - } else if (FALSE_IT(time_guard.click("apply_service->switch_to_leader")) - || OB_FAIL(apply_service_->switch_to_leader(ls_id, new_proposal_id))) { - CLOG_LOG(WARN, "apply_service_ switch_to_leader failed", K(ret), K(new_role), K(new_proposal_id)); - } else if (FALSE_IT(time_guard.click("replay_service->switch_to_leader")) - || OB_FAIL(replay_service_->switch_to_leader(ls_id))) { - CLOG_LOG(WARN, "replay_service_ switch_to_leader failed", K(new_role), K(new_proposal_id)); - } else if (FALSE_IT(time_guard.click("wait_replay_service_submit_task_clear_")) - || OB_FAIL(wait_replay_service_submit_task_clear_(ls_id))) { - CLOG_LOG(ERROR, "wait_replay_service_submit_task_clear_ failed", K(new_role), K(new_proposal_id)); - } else if (FALSE_IT(log_handler->switch_role(new_role, new_proposal_id))) { - CLOG_LOG(WARN, "ObLogHandler switch role failed", K(ret), K(new_role), K(new_proposal_id)); - } else if (FALSE_IT(ATOMIC_SET(&cur_task_info_.state_, TakeOverState::WAIT_RC_HANDLER_DONE))) { - } else { - CLOG_LOG(INFO, "about to call role_change_handler->switch_to_leader", K(ls_id), K(new_proposal_id), KPC(ls)); - if (FALSE_IT(time_guard.click("role_change_handler->switch_to_leader")) - || OB_FAIL(role_change_handler->switch_to_leader(cur_task_info_))) { - CLOG_LOG(WARN, "ObRoleChangeHandler switch_to_leader failed", K(ret), K(ls_id), K(new_proposal_id), KPC(ls)); - } else { - ATOMIC_SET(&cur_task_info_.state_, TakeOverState::TAKE_OVER_FINISH); - ATOMIC_SET(&cur_task_info_.log_type_, ObLogBaseType::INVALID_LOG_BASE_TYPE); - CLOG_LOG(INFO, "switch_follower_to_leader_ success", K(ret), KPC(ls)); - } - } - if (OB_FAIL(ret) && !retry_ctx.need_retry()) { - log_handler->advance_election_epoch_and_downgrade_priority(0_s, "palf switch follower to leader failed"); - CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), KPC(ls)); - } - return ret; -} - -int ObRoleChangeService::switch_leader_to_follower_forcedly_( - const int64_t new_proposal_id, - ObLS *ls) -{ - int ret = OB_SUCCESS; - const ObRole new_role = FOLLOWER; - const share::ObLSID &ls_id = ls->get_ls_id(); - ObLogHandler *log_handler = ls->get_log_handler(); - ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); - palf::LSN end_lsn; - ObTimeGuard time_guard("switch_leader_to_follower_forcedly_", EACH_ROLE_CHANGE_COST_MAX_TIME); - - // Why need wait_apply_sync? - // - // when we can execute 'switch_to_follower_forcedly', means that there is no possibility to submit log via log handler successfully. - // however, the flying callback may have not been pushed into apply service, and then, 'switch_to_follower' will be executed, for trans, - // if the callback be executed after 'switch_to_follower', will cause abort. - if (OB_FAIL(apply_service_->wait_append_sync(ls_id))) { - CLOG_LOG(WARN, "wait_apply_sync failed", K(ret), K(ls_id)); - } else if (FALSE_IT(time_guard.click("apply_service->wait_apply_sync")) - || OB_FAIL(apply_service_->switch_to_follower(ls_id))) { - CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id)); - } else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower")) - || OB_FAIL(wait_apply_service_apply_done_(ls_id, end_lsn))) { - CLOG_LOG(WARN, "wait_apply_service_apply_done_ failed", K(ret), K(end_lsn)); - } else { - time_guard.click("wait_apply_service_apply_done_"); - role_change_handler->switch_to_follower_forcedly(); - // NB: order is vital - // We must guarantee that this replica will not submit any logs after switch_role. - log_handler->switch_role(new_role, new_proposal_id); - // NB: in case of leader reovke, do we no need retry. - (void)replay_service_->switch_to_follower(ls_id, end_lsn); - CLOG_LOG(INFO, "switch_leader_to_follower_forcedly_ success", K(ret), KPC(ls)); - } - if (OB_FAIL(ret)) { - log_handler->advance_election_epoch_and_downgrade_priority(0_s, "palf switch leader to follower forcedly failed"); - CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(new_proposal_id), K(new_role)); - } - return ret; -} - -int ObRoleChangeService::switch_leader_to_follower_gracefully_( - const int64_t new_proposal_id, - const int64_t curr_proposal_id, - const common::ObAddr &dst_addr, - ObLS *ls) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - const ObRole new_role = FOLLOWER; - const share::ObLSID &ls_id = ls->get_ls_id(); - ObLogHandler *log_handler = ls->get_log_handler(); - ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); - LSN end_lsn; - ObTimeGuard time_guard("switch_leader_to_follower_gracefully_", EACH_ROLE_CHANGE_COST_MAX_TIME); - // 1. OB_SUCCESS means execute transaction successfully, we need execute follow steps. - // 2. OB_LS_NEED_REVOKE means the transaction execute failed, and can't been rollback, need revoke LS. - // 3. OTHERS, switch_to_follower_gracefully failed, and 'role_change_handler' has rollback success, - // no need to execute follow steps. - if (FALSE_IT(time_guard.click("role_change_handler->switch_to_follower_gracefully")) - || OB_SUCCESS != (tmp_ret = role_change_handler->switch_to_follower_gracefully())) { - CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need revoke leader", K(tmp_ret), - K(new_role), K(new_proposal_id), K(dst_addr)); - // NB: order is vital!!! - // we must ensure that the 'end_lsn' provid by 'apply_service_' is correctly. - // just switch_role to follower firstly, avoid sync log failed because palf has changed leader. - } else if (FALSE_IT(log_handler->switch_role(new_role, curr_proposal_id))) { - // apply service will not update end_lsn after switch_to_follower, so wait apply done first here - } else if (FALSE_IT(time_guard.click("wait_apply_service_apply_done_when_change_leader_")) - || OB_FAIL(wait_apply_service_apply_done_when_change_leader_(log_handler, curr_proposal_id, ls_id, end_lsn))) { - CLOG_LOG(WARN, "wait_apply_service_apply_done_when_change_leader_ failed", K(ret), - K(new_role), K(new_proposal_id), K(dst_addr)); - // wait apply service done my fail, we need : - // 1. switch log handler to origin status. - // 2. resume role change handler - log_handler->switch_role(LEADER, curr_proposal_id); - if (OB_FAIL(role_change_handler->resume_to_leader())) { - CLOG_LOG(WARN, "resume to leader failed", K(ret), KPC(ls)); - } - // NB: the following steps mustn't be failed. - } else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower")) - || OB_FAIL(apply_service_->switch_to_follower(ls_id))) { - CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); - } else if (FALSE_IT(time_guard.click("replay_service->switch_to_follower")) - || OB_FAIL(replay_service_->switch_to_follower(ls_id, end_lsn))) { - CLOG_LOG(WARN, "replay_service_ switch_to_follower failed", K(ret), KPC(ls), K(new_role), K(new_proposal_id)); - // NB: execute 'change_leader_to' lastly, can make 'wait_apply_service_apply_done_when_change_leader_' finish quickly. - } else if (OB_FAIL(log_handler->change_leader_to(dst_addr))) { - CLOG_LOG(WARN, "ObLogHandler change_leader failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); - } else { - CLOG_LOG(INFO, "switch_to_follower_gracefully success", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); - } - if (OB_FAIL(ret) || OB_LS_NEED_REVOKE == tmp_ret) { - log_handler->advance_election_epoch_and_downgrade_priority(0_s, "palf switch leader to follower gracefully failed"); - CLOG_LOG(WARN, "switch_leader_to_follower_gracefully failed, revoke leader", K(ret), K(tmp_ret), K(dst_addr), - K(new_role), K(new_proposal_id)); - ret = (OB_SUCCESS == ret ? tmp_ret : ret); - } - return ret; -} - -int ObRoleChangeService::switch_leader_to_leader_( - const int64_t new_proposal_id, - const int64_t curr_proposal_id, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx) -{ - int ret = OB_SUCCESS; - #ifdef ERRSIM - SERVER_EVENT_SYNC_ADD("LOGSERVICE", "BEFORE_LEADER_TO_LEADER_RC"); - #endif - ObTimeGuard time_guard("switch_leader_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME); - if (FALSE_IT(time_guard.click("switch_leader_to_follower_forcedly_")) - || OB_FAIL(switch_leader_to_follower_forcedly_(curr_proposal_id, ls))) { - CLOG_LOG(WARN, "switch_leader_to_leader_, switch leader to follower failed", K(ret), KPC(ls)); - } else if (FALSE_IT(time_guard.click("switch_follower_to_leader_")) - || OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) { - CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), K(new_proposal_id)); - } else { - CLOG_LOG(INFO, "switch_leader_to_leader_ success", K(ret), KPC(ls)); - } - return ret; -} - -int ObRoleChangeService::switch_follower_to_follower_(const int64_t new_proposal_id, ObLS *ls) -{ - int ret = OB_SUCCESS; - // need update proposal_id - const share::ObLSID &ls_id = ls->get_ls_id(); - ObLogHandler *log_handler = ls->get_log_handler(); - (void) log_handler->switch_role(common::ObRole::FOLLOWER, new_proposal_id); - CLOG_LOG(INFO, "switch_follower_to_follower_"); - return ret; -} - -int ObRoleChangeService::wait_replay_service_replay_done_( - const share::ObLSID &ls_id, - const palf::LSN &end_lsn, - const int64_t timeout_us) -{ - int ret = OB_SUCCESS; - bool is_done = false; - const int64_t start_ts = ObTimeUtility::current_time(); - palf::TimeoutChecker not_timeout(timeout_us); - while (OB_SUCC(ret) && false == is_done && OB_SUCC(not_timeout())) { - if (OB_FAIL(replay_service_->is_replay_done(ls_id, end_lsn, is_done))) { - CLOG_LOG(WARN, "replay_service_ is_replay_done failed", K(ret), K(is_done), K(end_lsn)); - } else if (false == is_done) { - ob_throttle_usleep(50*1000, 0, ls_id.id()); - CLOG_LOG(INFO, "wait replay done return false, need retry", K(ls_id), K(end_lsn), K(start_ts)); - } else { - } - } - CLOG_LOG(INFO, "wait_replay_service_replay_done_ finish", K(ret), K(ls_id), K(end_lsn), K(is_done)); - return ret; -} - -int ObRoleChangeService::wait_replay_service_submit_task_clear_(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - bool is_clear = false; - const int64_t start_ts = ObTimeUtility::current_time(); - while (OB_SUCC(ret) && (!is_clear)) { - if (OB_FAIL(replay_service_->is_submit_task_clear(ls_id, is_clear))) { - CLOG_LOG(WARN, "replay_service_ is_submit_task_clean failed", K(is_clear)); - } else if (!is_clear) { - ob_throttle_usleep(1 * 1000, 0, ls_id.id()); - if (REACH_TIME_INTERVAL(100 * 1000L)) { - CLOG_LOG(WARN, "submit_task is not clear, need retry", K(ls_id), K(start_ts)); - } - } else {/*do nothing*/} - } - CLOG_LOG(INFO, "wait_replay_service_submit_task_clear_ finish", K(ls_id), K(is_clear)); - return ret; -} - -int ObRoleChangeService::wait_apply_service_apply_done_( - const share::ObLSID &ls_id, - palf::LSN &end_lsn) -{ - int ret = OB_SUCCESS; - bool is_done = false; - const int64_t start_ts = ObTimeUtility::current_time(); - while (OB_SUCC(ret) && false == is_done) { - if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) { - CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn)); - } else if (false == is_done) { - ob_throttle_usleep(5*1000, ls_id.id()); - CLOG_LOG(WARN, "wait apply done return false, need retry", K(ls_id), K(is_done), K(end_lsn), K(start_ts)); - } else { - } - } - return ret; -} - -int ObRoleChangeService::wait_apply_service_apply_done_when_change_leader_( - const ObLogHandler *log_handler, - const int64_t proposal_id, - const share::ObLSID &ls_id, - palf::LSN &end_lsn) -{ - int ret = OB_SUCCESS; - bool is_done = false; - palf::LSN max_lsn; - common::ObRole unused_curr_role; - int64_t unused_curr_proposal_id; - common::ObRole new_role; - int64_t new_proposal_id; - bool is_pending_state = false; - while (OB_SUCC(ret) && (false == is_done || end_lsn != max_lsn)) { - if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) { - CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn)); - // NB: ApplyService execute on_failure only when it's FOLLOWER, therefore ApplyService my not return apply done - // when it's LEADER, we need check the role of palf when has changed. - } else if (OB_FAIL(log_handler->get_max_lsn(max_lsn))) { - CLOG_LOG(WARN, "get_end_lsn from palf failed", K(ret), K(ls_id), K(end_lsn)); - } else if (OB_FAIL(log_handler->prepare_switch_role(unused_curr_role, unused_curr_proposal_id, - new_role, new_proposal_id, is_pending_state))) { - CLOG_LOG(WARN, "failed prepare_switch_role", K(ret), K(new_role), K(proposal_id), K(ls_id)); - // if palf has changed role, return OB_STATE_NOT_MATCH, change leader failed. - } else if (LEADER != new_role || proposal_id != new_proposal_id) { - ret = OB_STATE_NOT_MATCH; - CLOG_LOG(WARN, "palf has changed leader, wait_apply_service_apply_done_when_change_leader_ failed", K(ret), K(proposal_id), - K(new_proposal_id)); - } else if (false == is_done || end_lsn != max_lsn) { - CLOG_LOG(INFO, "wait apply done return false, need retry", K(ls_id), K(is_done), - K(end_lsn), K(max_lsn)); - ob_throttle_usleep(5*1000, 0, ls_id.id()); - } else { - } - } - return ret; -} - -ObRoleChangeService::RoleChangeOptType ObRoleChangeService::get_role_change_opt_type_( - const ObRole &old_role, - const ObRole &new_role, - const bool need_transform_by_access_mode) const -{ - RoleChangeOptType rc_opt_type = RoleChangeOptType::INVALID_RC_OPT_TYPE; - if (FOLLOWER == old_role && LEADER == new_role) { - rc_opt_type = RoleChangeOptType::FOLLOWER_2_LEADER; - } else if (LEADER == old_role && FOLLOWER == new_role) { - rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER; - } else if (FOLLOWER == old_role && FOLLOWER == new_role) { - rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER; - } else if (LEADER == old_role && LEADER == new_role) { - rc_opt_type = RoleChangeOptType::LEADER_2_LEADER; - } else { - } - if (true == need_transform_by_access_mode) { - if (LEADER == old_role) { - rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER; - } else { - rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER; - } - } - return rc_opt_type; -} - -// NB: -// 1. when log handler is offline, need execute role change; -// 2. when palf is not in pending: -// 1. proposal_id is not same or -// 2. role is not same.(If there are no pending logs in sliding window, -// leader to follower will not advance proposal_id) -bool ObRoleChangeService::need_execute_role_change(const int64_t curr_proposal_id, - const common::ObRole curr_role, - const int64_t new_proposal_id, - const common::ObRole new_role, - const bool is_pending_state, - const bool is_offline) const -{ - return is_offline - || (!is_pending_state - && (curr_proposal_id != new_proposal_id || curr_role != new_role)); -} - -int ObRoleChangeService::diagnose(RCDiagnoseInfo &diagnose_info) -{ - int ret = OB_SUCCESS; - if (diagnose_info.id_ == ATOMIC_LOAD(&cur_task_info_.id_)) { - // The leader switch task for the current log stream is being processed - diagnose_info.state_ = ATOMIC_LOAD(&cur_task_info_.state_); - diagnose_info.log_type_ = ATOMIC_LOAD(&cur_task_info_.log_type_); - } else { - // The current log stream leader switch task has not been processed, possibly because the leader switch task of another log stream is stuck - diagnose_info.state_ = logservice::TakeOverState::UNKNOWN_TAKE_OVER_STATE; - diagnose_info.log_type_ = logservice::ObLogBaseType::INVALID_LOG_BASE_TYPE; - } - return ret; -} - -bool ObRoleChangeService::is_append_mode(const AccessMode &mode) const -{ - return (AccessMode::APPEND == mode); -} - -bool ObRoleChangeService::need_retry_submit_role_change_event_(int ret) const -{ - bool bool_ret = false; - if (OB_TIMEOUT == ret) { - bool_ret = true; - } - return bool_ret; -} - -} // end namespace logservice -} // end namespace oceanbase diff --git a/src/logservice/rcservice/ob_role_change_service.h b/src/logservice/rcservice/ob_role_change_service.h deleted file mode 100644 index db64a5819..000000000 --- a/src/logservice/rcservice/ob_role_change_service.h +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_LOGSERVICE_OB_ROLE_CHANGE_SERVICE_ -#define OCEANBASE_LOGSERVICE_OB_ROLE_CHANGE_SERVICE_ -#include "lib/function/ob_function.h" -#include "lib/thread/ob_simple_thread_pool.h" -#include "lib/utility/ob_macro_utils.h" -#include "share/ob_ls_id.h" -#include "storage/tx_storage/ob_ls_service.h" -#include "logservice/palf/palf_options.h" -#include "logservice/ob_log_handler.h" -#include "logservice/palf/palf_callback.h" -#include "logservice/applyservice/ob_log_apply_service.h" -#include "logservice/replayservice/ob_log_replay_service.h" -#include "logservice/rcservice/ob_role_change_handler.h" -namespace oceanbase -{ -namespace logservice -{ -enum class RoleChangeEventType { - INVALID_RC_EVENT_TYPE = 0, - CHANGE_LEADER_EVENT_TYPE = 1, - ROLE_CHANGE_CB_EVENT_TYPE = 2, - MAX_RC_EVENT_TYPE = 3 -}; - -struct RoleChangeEvent { - RoleChangeEvent() { reset(); } - RoleChangeEvent(const RoleChangeEventType &event_type, - const share::ObLSID &ls_id); - RoleChangeEvent(const RoleChangeEventType &event_type, - const share::ObLSID &ls_id, - const common::ObAddr &dst_addr); - bool is_valid() const; - void reset(); - bool operator == (const RoleChangeEvent &event) const; - RoleChangeEventType event_type_; - share::ObLSID ls_id_; - ObAddr dst_addr_; - TO_STRING_KV(K_(event_type), K_(ls_id), K_(dst_addr)); -}; - -class RoleChangeEventSet { -public: - RoleChangeEventSet(); - ~RoleChangeEventSet(); - int insert(const RoleChangeEvent &event); - int remove(const RoleChangeEvent &event); - static constexpr int64_t MAX_ARRAY_SIZE = 128; -private: - // Assumed there are sixteen log streams at most. - // 64 for normal role change events. - // 64 for leader change events. - RoleChangeEvent events_[MAX_ARRAY_SIZE]; - mutable ObSpinLock lock_; - DISALLOW_COPY_AND_ASSIGN(RoleChangeEventSet); -}; - -class ObRoleChangeService : public common::ObSimpleThreadPool, public palf::PalfRoleChangeCb { -public: - ObRoleChangeService(); - ~ObRoleChangeService(); - int init(storage::ObLSService *ls_service, - logservice::ObLogApplyService *apply_service, - logservice::ObILogReplayService *replay_service); - int start(); - void wait(); - void stop(); - void destroy(); - void handle(void *task) override; - int on_role_change(const int64_t id) final override; - int on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr) final override; - int diagnose(RCDiagnoseInfo &diagnose_info); - -private: - enum class RoleChangeOptType { - INVALID_RC_OPT_TYPE = 0, - FOLLOWER_2_LEADER = 1, - LEADER_2_FOLLOWER = 2, - FOLLOWER_2_FOLLOWER = 3, - LEADER_2_LEADER = 4, - MAX_RC_OPT_TYPE = 5 - }; - enum class RetrySubmitRoleChangeEventReason { - INVALID_TYPE = 0, - WAIT_REPLAY_DONE_TIMEOUT = 1, - MAX_TYPE = 2 - }; - class RetrySubmitRoleChangeEventCtx { - public: - RetrySubmitRoleChangeEventCtx() : reason_(RetrySubmitRoleChangeEventReason::INVALID_TYPE) {} - ~RetrySubmitRoleChangeEventCtx() - { - reason_ = RetrySubmitRoleChangeEventReason::INVALID_TYPE; - } - bool need_retry() const - { - return RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT == reason_; - } - void set_retry_reason(const RetrySubmitRoleChangeEventReason &reason) - { - reason_ = reason; - } - TO_STRING_KV(K_(reason)); - private: - RetrySubmitRoleChangeEventReason reason_; - }; -private: - int submit_role_change_event_(const RoleChangeEvent &event); - int push_event_into_queue_(const RoleChangeEvent &event); - int handle_role_change_event_(const RoleChangeEvent &event, - RetrySubmitRoleChangeEventCtx &retry_ctx); - - - int handle_role_change_cb_event_for_log_handler_(const palf::AccessMode &curr_access_mode, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx); - int handle_change_leader_event_for_log_handler_(const common::ObAddr &dst_addr, - ObLS *ls); - - // retval - // - OB_SUCCESS - // - OB_TIMEOUT, means wait replay finish timeout. - int switch_follower_to_leader_(const int64_t new_proposal_id, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx); - int switch_leader_to_follower_forcedly_(const int64_t new_proposal_id, - ObLS *ls); - int switch_leader_to_follower_gracefully_(const int64_t new_proposal_id, - const int64_t curr_proposal_id, - const common::ObAddr &dst_addr, - ObLS *ls); - int switch_follower_to_follower_(const int64_t new_proposal_id, ObLS *ls); - int switch_leader_to_leader_(const int64_t new_proposal_id, - const int64_t curr_proposal_id, - ObLS *ls, - RetrySubmitRoleChangeEventCtx &retry_ctx); - - // wait replay finish with timeout. - int wait_replay_service_replay_done_(const share::ObLSID &ls_id, - const palf::LSN &end_lsn, - const int64_t timeout_us); - - int wait_replay_service_submit_task_clear_(const share::ObLSID &ls_id); - int wait_apply_service_apply_done_(const share::ObLSID &ls_id, - palf::LSN &end_lsn); - int wait_apply_service_apply_done_when_change_leader_(const ObLogHandler *log_handler, - const int64_t proposal_id, - const share::ObLSID &ls_id, - palf::LSN &end_lsn); - bool need_execute_role_change(const int64_t curr_proposal_id, - const common::ObRole curr_role, - const int64_t new_proposal_id, - const common::ObRole new_role, - const bool is_pending_state, - const bool is_offline) const; - - bool is_append_mode(const palf::AccessMode &access_mode) const; -private: - RoleChangeOptType get_role_change_opt_type_(const common::ObRole &old_role, - const common::ObRole &new_role, - const bool need_transform_by_access_mode) const; - // retry submit role change event - // NB: nowdays, we only support retry submit role change event when wait replay finished - // timeout. - bool need_retry_submit_role_change_event_(int ret) const; -public: - static const int64_t MAX_THREAD_NUM = 1; - static const int64_t MAX_RC_EVENT_TASK = 1024; -private: - DISALLOW_COPY_AND_ASSIGN(ObRoleChangeService); -private: - static constexpr int64_t EACH_ROLE_CHANGE_COST_MAX_TIME = 1 * 1000 * 1000; - static constexpr int64_t WAIT_REPLAY_DONE_TIMEOUT_US = 2 * 1000 * 1000; - storage::ObLSService *ls_service_; - logservice::ObLogApplyService *apply_service_; - logservice::ObILogReplayService *replay_service_; - RoleChangeEventSet rc_set_; - RCDiagnoseInfo cur_task_info_; // for diagnose - bool is_inited_; -}; -} // end namespace logservice -} // end namespce oceanbase -#endif diff --git a/src/logservice/replayservice/ob_log_replay_service.cpp b/src/logservice/replayservice/ob_log_replay_service.cpp index 7cfe34430..650ab7394 100644 --- a/src/logservice/replayservice/ob_log_replay_service.cpp +++ b/src/logservice/replayservice/ob_log_replay_service.cpp @@ -118,12 +118,12 @@ void ReplayProcessStat::runTimerTask() int64_t estimate_time = 0; if (NULL == rp_sv_) { CLOG_LOG(ERROR, "rp_sv_ is NULL, unexpected error"); - } else if (OB_FAIL(rp_sv_->stat_all_ls_replay_process(submitted_log_size, unsubmitted_log_size, - replayed_log_size, unreplayed_log_size))) { - CLOG_LOG(WARN, "stat_all_ls_replay_process failed", K(ret)); + } else if (OB_FAIL(rp_sv_->stat_replay_process(submitted_log_size, unsubmitted_log_size, + replayed_log_size, unreplayed_log_size))) { + CLOG_LOG(WARN, "stat_replay_process failed", K(ret)); } else if (0 > submitted_log_size || 0 > unsubmitted_log_size || 0 > replayed_log_size || 0 > unreplayed_log_size) { - CLOG_LOG(WARN, "stat_all_ls_replay_process failed", K(ret)); + CLOG_LOG(WARN, "stat_replay_process failed", K(ret)); } else if (-1 == last_replayed_log_size_) { last_replayed_log_size_ = replayed_log_size; last_submitted_log_size_ = submitted_log_size; @@ -184,7 +184,8 @@ ObLogReplayService::ObLogReplayService() palf_env_(NULL), allocator_(NULL), replayable_point_(), - replay_status_map_(), + replay_status_(NULL), + lock_(), pending_replay_log_size_(0), wait_cost_stat_("[REPLAY STAT REPLAY TASK IN QUEUE TIME]", PALF_STAT_PRINT_INTERVAL_US), replay_cost_stat_("[REPLAY STAT REPLAY TASK EXECUTE COST TIME]", PALF_STAT_PRINT_INTERVAL_US) @@ -212,13 +213,13 @@ int ObLogReplayService::init(PalfEnv *palf_env, CLOG_LOG(WARN, "invalid argument", K(ret), KP(palf_env), KP(ls_adapter), KP(allocator)); } else if (OB_FAIL(common::ObLinkQueueThreadPool::init( 1, - (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET, + common::REPLAY_TASK_QUEUE_SIZE + 1, "ReplaySrv"))) { CLOG_LOG(WARN, "fail to init replay service thread pool", K(ret)); } else if (OB_FAIL(common::ObLinkQueueThreadPool::set_adaptive_thread(0, thread_quota * MTL_CPU_COUNT()))) { CLOG_LOG(WARN, "set adaptive thread failed", K(ret)); - } else if (OB_FAIL(replay_status_map_.init("REPLAY_STATUS"))) { - CLOG_LOG(WARN, "replay_status_map_ init error", K(ret)); + } else if (OB_FAIL(lock_.init(ObMemAttr("ReplayStatus")))) { + CLOG_LOG(WARN, "replay status lock init error", K(ret)); } else if (OB_FAIL(replay_stat_.init(this))) { CLOG_LOG(WARN, "replay_stat_ init error", K(ret)); } else { @@ -290,7 +291,9 @@ void ObLogReplayService::wait() void ObLogReplayService::destroy() { - (void)remove_all_ls_(); + if (is_inited_) { + (void)remove_status(); + } is_inited_ = false; is_running_ = false; CLOG_LOG(INFO, "replay service destroy"); @@ -303,7 +306,7 @@ void ObLogReplayService::destroy() allocator_ = NULL; ls_adapter_ = NULL; palf_env_ = NULL; - replay_status_map_.destroy(); + lock_.destroy(); } void ObLogReplayService::handle(common::LinkTask *task) @@ -380,7 +383,7 @@ void ObLogReplayService::handle(common::LinkTask *task) } } -int ObLogReplayService::add_ls(const share::ObLSID &id) +int ObLogReplayService::create_status() { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -390,48 +393,57 @@ int ObLogReplayService::add_ls(const share::ObLSID &id) CLOG_LOG(WARN, "replay service not init", K(ret)); } else if (NULL == (replay_status = static_cast(mtl_malloc(sizeof(ObReplayStatus), attr)))){ ret = OB_ALLOCATE_MEMORY_FAILED; - CLOG_LOG(WARN, "failed to alloc replay status", K(ret), K(id)); + CLOG_LOG(WARN, "failed to alloc replay status", K(ret)); } else { new (replay_status) ObReplayStatus(); - if (OB_FAIL(replay_status->init(id, palf_env_, this))) { + if (OB_FAIL(replay_status->init(palf_env_, this))) { mtl_free(replay_status); replay_status = NULL; - CLOG_LOG(ERROR, "failed to init replay status", K(ret), K(id), K(palf_env_), K(this)); + CLOG_LOG(ERROR, "failed to init replay status", K(ret), K(palf_env_), K(this)); } else { replay_status->inc_ref(); - if (OB_FAIL(replay_status_map_.insert(id, replay_status))) { - // enable after playback has started, cannot be freed directly - CLOG_LOG(ERROR, "failed to insert log stream", K(ret), K(id), KPC(replay_status)); + { + ObQSyncLockWriteGuard guard(lock_); + if (OB_NOT_NULL(replay_status_)) { + ret = OB_ENTRY_EXIST; + } else { + replay_status_ = replay_status; + replay_status = NULL; + } + } + if (OB_NOT_NULL(replay_status)) { revert_replay_status_(replay_status); - } else { - CLOG_LOG(INFO, "add_ls success", K(ret), K(id), KPC(replay_status)); } } } return ret; } // First remove from the map and then attempt to free memory -int ObLogReplayService::remove_ls(const share::ObLSID &id) +int ObLogReplayService::remove_status() { int ret = OB_SUCCESS; - RemoveReplayStatusFunctor functor; + ObReplayStatus *replay_status = NULL; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(WARN, "replay service not inited", K(ret), K(id)); - } else if (OB_FAIL(replay_status_map_.erase_if(id, functor))) { - CLOG_LOG(ERROR, "failed to remove log stream", K(ret), K(id)); - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } + CLOG_LOG(WARN, "replay service not inited", K(ret)); } else { - CLOG_LOG(INFO, "replay service remove ls", K(ret), K(id)); + { + ObQSyncLockWriteGuard guard(lock_); + replay_status = replay_status_; + replay_status_ = NULL; + } + if (OB_NOT_NULL(replay_status)) { + if (OB_FAIL(replay_status->disable())) { + CLOG_LOG(WARN, "failed to disable replay status", K(ret)); + } + revert_replay_status_(replay_status); + } } return ret; } // base_lsn can be not completely corresponding to base_scn, filter and replay based on base_scn, // The caller should note that logs with log scn as base_scn need to be replayed -int ObLogReplayService::enable(const share::ObLSID &id, - const LSN &base_lsn, +int ObLogReplayService::enable(const LSN &base_lsn, const SCN &base_scn) { int ret = OB_SUCCESS; @@ -440,20 +452,20 @@ int ObLogReplayService::enable(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->enable(base_lsn, base_scn))) { - CLOG_LOG(WARN, "replay status enable failed", K(ret), K(id), K(base_lsn), K(base_scn)); + CLOG_LOG(WARN, "replay status enable failed", K(ret), K(base_lsn), K(base_scn)); } else { (void)update_replayable_point(SCN::max_scn()); } return ret; } -int ObLogReplayService::disable(const share::ObLSID &id) +int ObLogReplayService::disable() { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -461,18 +473,18 @@ int ObLogReplayService::disable(const share::ObLSID &id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->disable())) { - CLOG_LOG(WARN, "replay status disable failed", K(ret), K(id)); + CLOG_LOG(WARN, "replay status disable failed", K(ret)); } return ret; } -int ObLogReplayService::is_enabled(const share::ObLSID &id, bool &is_enabled) +int ObLogReplayService::is_enabled(bool &is_enabled) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -480,18 +492,18 @@ int ObLogReplayService::is_enabled(const share::ObLSID &id, bool &is_enabled) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else { is_enabled = replay_status->is_enabled(); } return ret; } -int ObLogReplayService::block_submit_log(const share::ObLSID &id) +int ObLogReplayService::block_submit_log() { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -499,18 +511,18 @@ int ObLogReplayService::block_submit_log(const share::ObLSID &id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else { replay_status->block_submit(); } return ret; } -int ObLogReplayService::unblock_submit_log(const share::ObLSID &id) +int ObLogReplayService::unblock_submit_log() { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -518,18 +530,18 @@ int ObLogReplayService::unblock_submit_log(const share::ObLSID &id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else { replay_status->unblock_submit(); } return ret; } -int ObLogReplayService::switch_to_leader(const share::ObLSID &id) +int ObLogReplayService::disable_local_replay() { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -537,19 +549,18 @@ int ObLogReplayService::switch_to_leader(const share::ObLSID &id) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else { - replay_status->switch_to_leader(); + replay_status->disable_local_replay(); } return ret; } -int ObLogReplayService::switch_to_follower(const share::ObLSID &id, - const palf::LSN &begin_lsn) +int ObLogReplayService::enable_local_replay(const palf::LSN &begin_lsn) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -557,19 +568,18 @@ int ObLogReplayService::switch_to_follower(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else { - replay_status->switch_to_follower(begin_lsn); + replay_status->enable_local_replay(begin_lsn); } return ret; } -int ObLogReplayService::is_replay_done(const share::ObLSID &id, - const LSN &end_lsn, +int ObLogReplayService::is_replay_done(const LSN &end_lsn, bool &is_done) { int ret = OB_SUCCESS; @@ -578,20 +588,20 @@ int ObLogReplayService::is_replay_done(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->is_replay_done(end_lsn, is_done))){ - CLOG_LOG(WARN, "check replay done failed", K(ret), K(id)); + CLOG_LOG(WARN, "check replay done failed", K(ret)); } else { // do nothing } return ret; } -int ObLogReplayService::is_submit_task_clear(const share::ObLSID &id, bool &is_clear) +int ObLogReplayService::is_submit_task_clear(bool &is_clear) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -599,20 +609,20 @@ int ObLogReplayService::is_submit_task_clear(const share::ObLSID &id, bool &is_c if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->is_submit_task_clear(is_clear))){ - CLOG_LOG(WARN, "check replay done failed", K(id)); + CLOG_LOG(WARN, "check replay done failed", K(ret)); } else { // do nothing } return ret; } //Generic interface, the final return value during controlled replay is the log_ts of the last log before the controlled replay point -int ObLogReplayService::get_max_replayed_scn(const share::ObLSID &id, SCN &scn) +int ObLogReplayService::get_max_replayed_scn(SCN &scn) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -620,22 +630,22 @@ int ObLogReplayService::get_max_replayed_scn(const share::ObLSID &id, SCN &scn) if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->get_max_replayed_scn(scn))) { if (OB_STATE_NOT_MATCH != ret) { - CLOG_LOG(WARN, "get_max_replayed_scn failed", K(ret), K(id)); + CLOG_LOG(WARN, "get_max_replayed_scn failed", K(ret)); } else if (REACH_TIME_INTERVAL(1000 * 1000)) { - CLOG_LOG(ERROR, "get_max_replayed_scn failed, replay status is not enabled", K(ret), K(id)); + CLOG_LOG(ERROR, "get_max_replayed_scn failed, replay status is not enabled", K(ret)); } } return ret; } -int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, SCN &scn) +int ObLogReplayService::get_min_unreplayed_scn(SCN &scn) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -649,17 +659,17 @@ int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, SCN &scn if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->get_min_unreplayed_log_info(unused_lsn, scn, unused_replay_hint, unused_log_type, unused_first_handle_ts, unused_replay_cost, unused_retry_cost))) { if (OB_STATE_NOT_MATCH != ret) { - CLOG_LOG(WARN, "get_min_unreplayed_log_info failed", K(ret), K(id)); + CLOG_LOG(WARN, "get_min_unreplayed_log_info failed", K(ret)); } else if (REACH_TIME_INTERVAL(1000 * 1000)) { - CLOG_LOG(ERROR, "get_min_unreplayed_log_info failed, replay status is not enabled", K(ret), K(id)); + CLOG_LOG(ERROR, "get_min_unreplayed_log_info failed, replay status is not enabled", K(ret)); } } else {} return ret; @@ -711,8 +721,7 @@ void ObLogReplayService::free_replay_task_log_buf(ObLogReplayTask *task) } } -int ObLogReplayService::has_fatal_error(const ObLSID &ls_id, - bool &bool_ret) +int ObLogReplayService::has_fatal_error(bool &bool_ret) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -720,17 +729,17 @@ int ObLogReplayService::has_fatal_error(const ObLSID &ls_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(ls_id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(ls_id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(ls_id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (replay_status->try_rdlock()){ bool_ret = replay_status->has_fatal_error(); replay_status->unlock(); } else { ret = OB_EAGAIN; - CLOG_LOG(WARN, "try_rdlock failed", K(ret), K(ls_id)); + CLOG_LOG(WARN, "try_rdlock failed", K(ret)); } return ret; } @@ -744,7 +753,8 @@ void ObLogReplayService::free_replay_log_buf_(ObLogReplayBuffer *&replay_log_buf int ObLogReplayService::update_replayable_point(const SCN &replayable_scn) { int ret = OB_SUCCESS; - FetchLogFunctor functor; + ObReplayStatusGuard guard; + ObReplayStatus *replay_status = NULL; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); @@ -752,10 +762,13 @@ int ObLogReplayService::update_replayable_point(const SCN &replayable_scn) if (replayable_scn > replayable_point_) { replayable_point_.atomic_set(replayable_scn) ; } - if (OB_FAIL(replay_status_map_.for_each(functor))) { + if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "failed to get replay status", K(ret)); + } else if (OB_ISNULL(replay_status = guard.get_replay_status())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "replay status is null", K(ret)); + } else if (OB_FAIL(replay_status->trigger_fetch_log())) { CLOG_LOG(WARN, "failed to trigger submit log task", K(ret)); - } else { - // do nothing } } return ret; @@ -783,44 +796,47 @@ share::SCN ObLogReplayService::inner_get_replayable_point_() const return replayable_scn; } -int ObLogReplayService::stat_for_each(const common::ObFunction &func) +int ObLogReplayService::stat(LSReplayStat &replay_stat) { - auto stat_func = [&func](const ObLSID &id, ObReplayStatus *replay_status) -> bool { - int ret = OB_SUCCESS; - bool bret = true; - if (OB_FAIL(func(*replay_status))) { - bret = false; - CLOG_LOG(WARN, "iter replay stat failed", K(ret)); - } - return bret; - }; int ret = OB_SUCCESS; - if (!func.is_valid()) { - // ObFunction will be invalid when allocating memory failed. - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - ret = replay_status_map_.for_each(stat_func); + ObReplayStatusGuard guard; + ObReplayStatus *replay_status = NULL; + if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "failed to get replay status", K(ret)); + } else if (OB_ISNULL(replay_status = guard.get_replay_status())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "replay status is null", K(ret)); + } else if (OB_FAIL(replay_status->stat(replay_stat))) { + CLOG_LOG(WARN, "stat replay failed", K(ret), KPC(replay_status)); } return ret; } -int ObLogReplayService::stat_all_ls_replay_process(int64_t &submitted_log_size, - int64_t &unsubmitted_log_size, - int64_t &replayed_log_size, - int64_t &unreplayed_log_size) +int ObLogReplayService::stat_replay_process(int64_t &submitted_log_size, + int64_t &unsubmitted_log_size, + int64_t &replayed_log_size, + int64_t &unreplayed_log_size) { int ret = OB_SUCCESS; - StatReplayProcessFunctor functor; + ObReplayStatusGuard guard; + ObReplayStatus *replay_status = NULL; + submitted_log_size = 0; + unsubmitted_log_size = 0; + replayed_log_size = 0; + unreplayed_log_size = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not inited", K(ret)); - } else if (OB_FAIL(replay_status_map_.for_each(functor))) { - CLOG_LOG(WARN, "failed to remove log stream", K(ret)); - } else { - submitted_log_size = functor.get_submitted_log_size(); - unsubmitted_log_size = functor.get_unsubmitted_log_size(); - replayed_log_size = functor.get_replayed_log_size(); - unreplayed_log_size = functor.get_unreplayed_log_size(); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "failed to get replay status", K(ret)); + } else if (OB_ISNULL(replay_status = guard.get_replay_status())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "replay status is null", K(ret)); + } else if (OB_FAIL(replay_status->get_replay_process(submitted_log_size, + unsubmitted_log_size, + replayed_log_size, + unreplayed_log_size))) { + CLOG_LOG(WARN, "get replay process failed", K(ret), KPC(replay_status)); } return ret; } @@ -848,18 +864,19 @@ void *ObLogReplayService::alloc_replay_task(const int64_t size) return allocator_->alloc_replay_task(size); } -int ObLogReplayService::get_replay_status_(const share::ObLSID &id, - ObReplayStatusGuard &guard) +int ObLogReplayService::get_replay_status_(ObReplayStatusGuard &guard) { int ret = OB_SUCCESS; - GetReplayStatusFunctor functor(guard); if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(replay_status_map_.operate(id, functor))) { - CLOG_LOG(WARN, "replay service get replay status failed", K(ret), K(id)); } else { - CLOG_LOG(TRACE, "replay service get success", K(ret), K(id)); + ObQSyncLockReadGuard lock_guard(lock_); + if (OB_ISNULL(replay_status_)) { + ret = OB_ENTRY_NOT_EXIST; + } else { + guard.set_replay_status(replay_status_); + } } return ret; } @@ -939,7 +956,6 @@ int ObLogReplayService::do_replay_task_(ObLogReplayTask *replay_task, const int64_t replay_queue_idx) { int ret = OB_SUCCESS; - ObLS *ls; get_replay_queue_index() = replay_queue_idx; ObLogReplayBuffer *replay_log_buff = NULL; bool need_replay = false; @@ -1050,45 +1066,38 @@ int ObLogReplayService::fetch_pre_barrier_log_(ObReplayStatus &replay_status, const int64_t header_pos) { int ret = OB_SUCCESS; - ObLSID id; - if (OB_FAIL(replay_status.get_ls_id(id))) { - CLOG_LOG(ERROR, "replay status get log stream id failed", KPC(submit_task), - K(replay_status), KR(ret)); + const int64_t share_log_size = sizeof(ObLogReplayBuffer) + log_size; + const int64_t task_size = sizeof(ObLogReplayTask); + char *share_log_buf = NULL; + void *task_buf = NULL; + if (OB_UNLIKELY(NULL == (share_log_buf = static_cast(allocator_->alloc_replay_log_buf(share_log_size))))) { + ret = OB_EAGAIN; + if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(cur_lsn)); + } + } else if (OB_UNLIKELY(NULL == (task_buf = alloc_replay_task(task_size)))) { + ret = OB_EAGAIN; + allocator_->free_replay_log_buf(share_log_buf); + share_log_buf = NULL; + if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(cur_lsn)); + } } else { - const int64_t share_log_size = sizeof(ObLogReplayBuffer) + log_size; - const int64_t task_size = sizeof(ObLogReplayTask); - char *share_log_buf = NULL; - void *task_buf = NULL; - if (OB_UNLIKELY(NULL == (share_log_buf =static_cast(allocator_->alloc_replay_log_buf(share_log_size))))) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(id), K(cur_lsn)); - } - } else if (OB_UNLIKELY(NULL == (task_buf = alloc_replay_task(task_size)))) { - ret = OB_EAGAIN; - allocator_->free_replay_log_buf(share_log_buf); - share_log_buf = NULL; - if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(id), K(cur_lsn)); - } + replay_task = new (task_buf) ObLogReplayTask(header, cur_lsn, cur_log_submit_scn, + log_size, header_pos, NULL, 0); + ObLogReplayBuffer *replay_log_buffer = new (share_log_buf) ObLogReplayBuffer(); + char *real_log_buf = share_log_buf + sizeof(ObLogReplayBuffer); + replay_log_buffer->log_buf_ = real_log_buf; + MEMCPY(real_log_buf, log_buf, log_size); + if (OB_FAIL(replay_task->init(replay_log_buffer))) { + free_replay_log_buf_(replay_log_buffer); + CLOG_LOG(ERROR, "init replay task failed", K(ret), K(cur_lsn), K(cur_log_submit_scn), + K(log_size), KP(replay_log_buffer), K(header), K(header_pos)); } else { - replay_task = new (task_buf) ObLogReplayTask(id, header, cur_lsn, cur_log_submit_scn, - log_size, header_pos, NULL, 0); - ObLogReplayBuffer *replay_log_buffer = new (share_log_buf) ObLogReplayBuffer(); - char *real_log_buf = share_log_buf + sizeof(ObLogReplayBuffer); - replay_log_buffer->log_buf_ = real_log_buf; - MEMCPY(real_log_buf, log_buf, log_size); - if (OB_FAIL(replay_task->init(replay_log_buffer))) { - free_replay_log_buf_(replay_log_buffer); - CLOG_LOG(ERROR, "init replay task failed", K(ret), K(id), K(id), K(cur_lsn), K(cur_log_submit_scn), - K(log_size), KP(replay_log_buffer), K(header), K(header_pos)); - } else { - CLOG_LOG(TRACE, "fetch_and_submit_pre_barrier_log_", K(ret), K(replay_status), K(replay_task), - KPC(replay_task), K(replay_log_buffer->log_buf_)); - } + CLOG_LOG(TRACE, "fetch_and_submit_pre_barrier_log_", K(ret), K(replay_status), K(replay_task), + KPC(replay_task), K(replay_log_buffer->log_buf_)); } } - return ret; } @@ -1102,7 +1111,6 @@ int ObLogReplayService::fetch_and_submit_single_log_(ObReplayStatus &replay_stat int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; const char *log_buf = NULL; - ObLSID id; bool need_skip = false; bool need_iterate_next_log = false; ObLogBaseHeader header; @@ -1116,21 +1124,19 @@ int ObLogReplayService::fetch_and_submit_single_log_(ObReplayStatus &replay_stat CLOG_LOG(WARN, "submit task is NULL when fetch log", K(replay_status), KPC(submit_task)); } else if (OB_FAIL(submit_task->get_log(log_buf, log_size, cur_log_submit_scn, cur_lsn))) { CLOG_LOG(WARN, "submit task get log value failed", K(replay_status), K(submit_task)); - } else if (OB_FAIL(replay_status.get_ls_id(id))) { - CLOG_LOG(ERROR, "replay status get log stream id failed", KPC(submit_task), K(replay_status)); } else if (OB_FAIL(submit_task->need_skip(cur_log_submit_scn, need_skip))) { CLOG_LOG(INFO, "check need_skip failed", K(replay_status), K(cur_lsn), K(log_size), K(cur_log_submit_scn)); } else if (need_skip) { need_iterate_next_log = true; - CLOG_LOG(TRACE, "skip current log", K(id), K(replay_status), K(cur_lsn), K(log_size), K(cur_log_submit_scn)); + CLOG_LOG(TRACE, "skip current log", K(replay_status), K(cur_lsn), K(log_size), K(cur_log_submit_scn)); } else if (OB_FAIL(header.deserialize(log_buf, log_size, header_pos))) { - CLOG_LOG(WARN, "basic header deserialize failed", K(ret), K(header_pos), K(id)); + CLOG_LOG(WARN, "basic header deserialize failed", K(ret), K(header_pos)); } else if (ObLogBaseType::PADDING_LOG_BASE_TYPE == header.get_log_type()) { // For padding log entry, iterate next log directly. need_iterate_next_log = true; - CLOG_LOG(INFO, "no need to replay padding log entry", K(id), KPC(submit_task), K(header)); + CLOG_LOG(INFO, "no need to replay padding log entry", KPC(submit_task), K(header)); #ifdef OB_BUILD_LOG_STORAGE_COMPRESS - } else if (OB_FAIL(prepare_decompression_buf_(id, log_buf, header, log_size, header_pos, + } else if (OB_FAIL(prepare_decompression_buf_(log_buf, header, log_size, header_pos, decompression_buf, decompressed_log_size))) { CLOG_LOG(WARN, "failed to prepare_decompression_buf", KPC(submit_task), K(header)); #endif @@ -1154,17 +1160,17 @@ int ObLogReplayService::fetch_and_submit_single_log_(ObReplayStatus &replay_stat if (OB_UNLIKELY(NULL == (task_buf = static_cast(alloc_replay_task(task_size))))) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { - CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(id), K(cur_lsn)); + CLOG_LOG(WARN, "failed to alloc log replay task buf", K(ret), K(cur_lsn)); } } else { - replay_task = new (task_buf) ObLogReplayTask(id, header, cur_lsn, cur_log_submit_scn, log_size, + replay_task = new (task_buf) ObLogReplayTask(header, cur_lsn, cur_log_submit_scn, log_size, header_pos, decompression_buf, decompressed_log_size); decompression_buf = NULL;//decompression_buf will be free by replay_task char *task_log_buf = task_buf + sizeof(ObLogReplayTask); MEMCPY(task_log_buf, log_buf, log_size); if (OB_FAIL(replay_task->init(task_log_buf))) { // print log details inside - CLOG_LOG(ERROR, "init replay task failed", K(ret), K(id)); + CLOG_LOG(ERROR, "init replay task failed", K(ret)); } } } @@ -1421,7 +1427,7 @@ int ObLogReplayService::submit_log_replay_task_(ObLogReplayTask &replay_task, #ifdef OB_BUILD_LOG_STORAGE_COMPRESS -int ObLogReplayService::prepare_decompression_buf_(const ObLSID id, const char *log_buf, +int ObLogReplayService::prepare_decompression_buf_(const char *log_buf, const ObLogBaseHeader &header, const int64_t log_size, const int64_t base_header_len, @@ -1432,17 +1438,17 @@ int ObLogReplayService::prepare_decompression_buf_(const ObLSID id, const char * if (header.is_compressed()) { if (OB_UNLIKELY(header.need_pre_replay_barrier())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "compression of pre_barrier log is not supported", K(id), K(header)); + CLOG_LOG(ERROR, "compression of pre_barrier log is not supported", K(header)); } else { LogCompressedPayloadHeader comp_header; int64_t pos = 0; if (OB_FAIL(comp_header.deserialize(log_buf + base_header_len, log_size - base_header_len, pos))) { - CLOG_LOG(ERROR, "failed to deserialize LogCompressedPayloadHeader", K(id)); + CLOG_LOG(ERROR, "failed to deserialize LogCompressedPayloadHeader"); } else if (OB_UNLIKELY(NULL == (decompression_buf = (allocator_->alloc_replay_decompression_buf(base_header_len + comp_header.get_original_len()))))) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(2 * 1000 * 1000L)) { - CLOG_LOG(WARN, "failed to allocate decompression_buf", K(id)); + CLOG_LOG(WARN, "failed to allocate decompression_buf"); } } else { decompressed_log_size = comp_header.get_original_len(); @@ -1516,23 +1522,7 @@ void ObLogReplayService::on_replay_error_() } } -int ObLogReplayService::remove_all_ls_() -{ - int ret = OB_SUCCESS; - RemoveReplayStatusFunctor functor; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "replay service not inited", K(ret)); - } else if (OB_FAIL(replay_status_map_.for_each(functor))) { - CLOG_LOG(WARN, "failed to remove log stream", K(ret)); - } else { - CLOG_LOG(INFO, "replay service remove all ls", K(ret)); - } - return ret; -} - -int ObLogReplayService::diagnose(const share::ObLSID &id, - ReplayDiagnoseInfo &diagnose_info) +int ObLogReplayService::diagnose(ReplayDiagnoseInfo &diagnose_info) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -1540,91 +1530,16 @@ int ObLogReplayService::diagnose(const share::ObLSID &id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay service not init", K(ret)); - } else if (OB_FAIL(get_replay_status_(id, guard))) { - CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(id)); + } else if (OB_FAIL(get_replay_status_(guard))) { + CLOG_LOG(WARN, "guard get replay status failed", K(ret)); } else if (NULL == (replay_status = guard.get_replay_status())) { ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is not exist", K(ret), K(id)); + CLOG_LOG(WARN, "replay status is not exist", K(ret)); } else if (OB_FAIL(replay_status->diagnose(diagnose_info))) { - CLOG_LOG(WARN, "replay status enable failed", K(ret), K(id)); + CLOG_LOG(WARN, "replay status enable failed", K(ret)); } return ret; } -bool ObLogReplayService::GetReplayStatusFunctor::operator()(const share::ObLSID &id, - ObReplayStatus *replay_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(replay_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is NULL", K(id), KR(ret)); - } else { - guard_.set_replay_status(replay_status); - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - -bool ObLogReplayService::RemoveReplayStatusFunctor::operator()(const share::ObLSID &id, - ObReplayStatus *replay_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(replay_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is NULL", K(id), KR(ret)); - } else if (OB_FAIL(replay_status->disable())) { - CLOG_LOG(WARN, "failed to disable replay status", K(ret), K(id), KPC(replay_status)); - } - if (OB_SUCCESS == ret) { - if (0 == replay_status->dec_ref()) { - CLOG_LOG(INFO, "free replay status", KPC(replay_status)); - replay_status->~ObReplayStatus(); - mtl_free(replay_status); - } - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - -bool ObLogReplayService::StatReplayProcessFunctor::operator()(const share::ObLSID &id, - ObReplayStatus *replay_status) -{ - int ret = OB_SUCCESS; - int64_t submitted_log_size = 0; - int64_t unsubmitted_log_size = 0; - int64_t replayed_log_size = 0; - int64_t unreplayed_log_size = 0; - if (OB_ISNULL(replay_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is NULL", K(id), KR(ret)); - } else if (OB_FAIL(replay_status->get_replay_process(submitted_log_size, unsubmitted_log_size, - replayed_log_size, unreplayed_log_size))){ - CLOG_LOG(WARN, "get_replay_process failed", K(id), KR(ret), KPC(replay_status)); - } else { - submitted_log_size_ += submitted_log_size; - unsubmitted_log_size_ += unsubmitted_log_size; - replayed_log_size_ += replayed_log_size; - unreplayed_log_size_ += unreplayed_log_size; - CLOG_LOG(INFO, "get_replay_process success", K(id), K(submitted_log_size), K(unsubmitted_log_size), - K(replayed_log_size), K(unreplayed_log_size)); - } - ret_code_ = ret; - return true; -} - -bool ObLogReplayService::FetchLogFunctor::operator()(const share::ObLSID &id, - ObReplayStatus *replay_status) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(replay_status)) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "replay status is NULL", K(id), KR(ret)); - } else if (OB_FAIL(replay_status->trigger_fetch_log())) { - CLOG_LOG(WARN, "failed to trigger fetch log", K(ret), K(id), KPC(replay_status)); - } - ret_code_ = ret; - return OB_SUCCESS == ret; -} - } // namespace replayService } // namespace oceanbase diff --git a/src/logservice/replayservice/ob_log_replay_service.h b/src/logservice/replayservice/ob_log_replay_service.h index 68786eac1..90ae96706 100644 --- a/src/logservice/replayservice/ob_log_replay_service.h +++ b/src/logservice/replayservice/ob_log_replay_service.h @@ -21,7 +21,7 @@ #include "logservice/ob_log_base_header.h" #include "lib/task/ob_timer.h" #include "lib/thread/ob_simple_thread_pool.h" -#include "lib/hash/ob_linear_hash_map.h" +#include "lib/lock/ob_qsync_lock.h" #include "share/scn.h" namespace oceanbase @@ -62,10 +62,10 @@ class ReplayProcessStat : public common::ObTimerTask class ObILogReplayService { public: - virtual int is_replay_done(const share::ObLSID &id, const palf::LSN &end_lsn, bool &is_done) = 0; - virtual int is_submit_task_clear(const share::ObLSID &id, bool &is_clear) = 0; - virtual int switch_to_follower(const share::ObLSID &id, const palf::LSN &begin_lsn) = 0; - virtual int switch_to_leader(const share::ObLSID &id) = 0; + virtual int is_replay_done(const palf::LSN &end_lsn, bool &is_done) = 0; + virtual int is_submit_task_clear(bool &is_clear) = 0; + virtual int enable_local_replay(const palf::LSN &begin_lsn) = 0; + virtual int disable_local_replay() = 0; }; /* TODO(yaoying.yyy): memory management of replayservice needs to be documented @@ -84,108 +84,41 @@ class ObLogReplayService: public common::ObLinkQueueThreadPool, public ObILogRep void stop(); void wait(); void destroy(); -public: - class GetReplayStatusFunctor - { - public: - GetReplayStatusFunctor(ObReplayStatusGuard &guard) - : ret_code_(common::OB_SUCCESS), guard_(guard){} - ~GetReplayStatusFunctor(){} - bool operator()(const share::ObLSID &id, ObReplayStatus *replay_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - ObReplayStatusGuard &guard_; - }; - class RemoveReplayStatusFunctor - { - public: - explicit RemoveReplayStatusFunctor() - : ret_code_(common::OB_SUCCESS) {} - ~RemoveReplayStatusFunctor(){} - bool operator()(const share::ObLSID &id, ObReplayStatus *replay_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - }; - class StatReplayProcessFunctor - { - public: - explicit StatReplayProcessFunctor() - : ret_code_(common::OB_SUCCESS), - submitted_log_size_(0), - unsubmitted_log_size_(0), - replayed_log_size_(0), - unreplayed_log_size_(0) {} - ~StatReplayProcessFunctor(){} - bool operator()(const share::ObLSID &id, ObReplayStatus *replay_status); - int get_ret_code() const { return ret_code_; } - int64_t get_submitted_log_size() const { return submitted_log_size_; } - int64_t get_unsubmitted_log_size() const { return unsubmitted_log_size_; } - int64_t get_replayed_log_size() const { return replayed_log_size_; } - int64_t get_unreplayed_log_size() const { return unreplayed_log_size_; } - TO_STRING_KV(K(ret_code_), K(replayed_log_size_), K(unreplayed_log_size_)); - private: - int ret_code_; - int64_t submitted_log_size_; - int64_t unsubmitted_log_size_; - int64_t replayed_log_size_; - int64_t unreplayed_log_size_; - }; - class FetchLogFunctor - { - public: - explicit FetchLogFunctor() - : ret_code_(common::OB_SUCCESS) {} - ~FetchLogFunctor(){} - bool operator()(const share::ObLSID &id, ObReplayStatus *replay_status); - int get_ret_code() const { return ret_code_; } - TO_STRING_KV(K(ret_code_)); - private: - int ret_code_; - }; public: void handle(common::LinkTask *task); - int add_ls(const share::ObLSID &id); - int remove_ls(const share::ObLSID &id); - int enable(const share::ObLSID &id, - const palf::LSN &base_lsn, + int create_status(); + int remove_status(); + int enable(const palf::LSN &base_lsn, const share::SCN &base_scn); - int disable(const share::ObLSID &id); - int is_enabled(const share::ObLSID &id, bool &is_enabled); - int block_submit_log(const share::ObLSID &id); - int unblock_submit_log(const share::ObLSID &id); - int switch_to_leader(const share::ObLSID &id); - int switch_to_follower(const share::ObLSID &id, - const palf::LSN &begin_lsn); - int is_replay_done(const share::ObLSID &id, - const palf::LSN &end_lsn, + int disable(); + int is_enabled(bool &is_enabled); + int block_submit_log(); + int unblock_submit_log(); + int disable_local_replay(); + int enable_local_replay(const palf::LSN &begin_lsn); + int is_replay_done(const palf::LSN &end_lsn, bool &is_done); - int is_submit_task_clear(const share::ObLSID &id, bool &is_clear); - int get_max_replayed_scn(const share::ObLSID &id, share::SCN &scn); - int get_min_unreplayed_scn(const share::ObLSID &id, SCN &scn); + int is_submit_task_clear(bool &is_clear); + int get_max_replayed_scn(share::SCN &scn); + int get_min_unreplayed_scn(SCN &scn); int submit_task(ObReplayServiceTask *task); int update_replayable_point(const share::SCN &replayable_scn); int get_replayable_point(share::SCN &replayable_scn); - int stat_for_each(const common::ObFunction &func); - int stat_all_ls_replay_process(int64_t &submitted_log_size, - int64_t &unsubmitted_log_size, - int64_t &replayed_log_size, - int64_t &unreplayed_log_size); - int diagnose(const share::ObLSID &id, ReplayDiagnoseInfo &diagnose_info); + int stat(LSReplayStat &replay_stat); + int stat_replay_process(int64_t &submitted_log_size, + int64_t &unsubmitted_log_size, + int64_t &replayed_log_size, + int64_t &unreplayed_log_size); + int diagnose(ReplayDiagnoseInfo &diagnose_info); void inc_pending_task_size(const int64_t log_size); void dec_pending_task_size(const int64_t log_size); int64_t get_pending_task_size() const; void *alloc_replay_task(const int64_t size); void free_replay_task(ObLogReplayTask *task); void free_replay_task_log_buf(ObLogReplayTask *task); - int has_fatal_error(const share::ObLSID &ls_id, - bool &bool_ret); + int has_fatal_error(bool &bool_ret); private: - int get_replay_status_(const share::ObLSID &id, - ObReplayStatusGuard &guard); + int get_replay_status_(ObReplayStatusGuard &guard); int pre_check_(ObReplayStatus &replay_status, ObReplayServiceTask &task); void process_replay_ret_code_(const int ret_code, @@ -224,11 +157,8 @@ class ObLogReplayService: public common::ObLinkQueueThreadPool, public ObILogRep const int64_t first_handle_time); void on_replay_error_(ObLogReplayTask &replay_task, int ret); void on_replay_error_(); - // Called before destruction, return the replay status count of all log streams - int remove_all_ls_(); #ifdef OB_BUILD_LOG_STORAGE_COMPRESS - int prepare_decompression_buf_(const ObLSID id, - const char *log_buf, + int prepare_decompression_buf_(const char *log_buf, const ObLogBaseHeader &header, const int64_t log_size, const int64_t base_header_len, @@ -247,7 +177,7 @@ class ObLogReplayService: public common::ObLinkQueueThreadPool, public ObILogRep const int64_t MAX_SUBMIT_TIME_PER_ROUND = 100 * 1000; //100ms const int64_t TASK_QUEUE_WAIT_IN_GLOBAL_QUEUE_TIME_THRESHOLD = 5 * 1000 * 1000; //5s const int64_t PENDING_TASK_MEMORY_LIMIT = 128 * (1LL << 20); //128MB - // Each log stream accumulates log pulls to the threshold and batch submits all task queues + // Accumulate replay tasks to the threshold before submitting a batch. static const int64_t BATCH_PUSH_REPLAY_TASK_COUNT_THRESOLD = 1024; static const int64_t BATCH_PUSH_REPLAY_TASK_SIZE_THRESOLD = 16 * (1LL << 20); //16MB // params of adaptive thread pool @@ -263,8 +193,8 @@ class ObLogReplayService: public common::ObLinkQueueThreadPool, public ObILogRep palf::PalfEnv *palf_env_; ObILogAllocator *allocator_; share::SCN replayable_point_; - // Considering the migration scenario, the lifecycle of replay status cannot be managed by map alone - common::ObLinearHashMap replay_status_map_; + ObReplayStatus *replay_status_; + mutable common::ObQSyncLock lock_; int64_t pending_replay_log_size_; ObMiniStat::ObStatItem wait_cost_stat_; ObMiniStat::ObStatItem replay_cost_stat_; diff --git a/src/logservice/replayservice/ob_replay_handler.h b/src/logservice/replayservice/ob_replay_handler.h index 1480931d5..b9fd9c10a 100644 --- a/src/logservice/replayservice/ob_replay_handler.h +++ b/src/logservice/replayservice/ob_replay_handler.h @@ -20,7 +20,6 @@ #include "lib/lock/ob_tc_rwlock.h" #include "logservice/ob_log_base_type.h" #include "logservice/palf/lsn.h" -#include "share/ob_ls_id.h" namespace oceanbase { diff --git a/src/logservice/replayservice/ob_replay_status.cpp b/src/logservice/replayservice/ob_replay_status.cpp index 905497b70..2009ef5ca 100644 --- a/src/logservice/replayservice/ob_replay_status.cpp +++ b/src/logservice/replayservice/ob_replay_status.cpp @@ -89,7 +89,6 @@ bool ObReplayServiceTask::need_replay_immediately() const //---------------ObReplayServiceSubmitTask---------------// int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, const SCN &base_scn, - const share::ObLSID &id, ObReplayStatus *replay_status) { int ret = OB_SUCCESS; @@ -97,10 +96,11 @@ int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, if (OB_ISNULL(replay_status)) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "invalid argument", K(type_), K(ret), K(replay_status)); - } else if (OB_FAIL(seek_log_iterator(id, base_lsn, iterator_))) { - CLOG_LOG(WARN, "seek iterator failed", KR(ret), K(type_), K(id), K(base_lsn)); - } else if (OB_FAIL(iterator_.set_io_context(palf::LogIOContext(id.id(), palf::LogIOUser::REPLAY)))) { - CLOG_LOG(WARN, "iterator set_io_context failed", KR(ret), K(id)); + } else if (OB_FAIL(seek_log_iterator_no_shared_storage( + replay_status->palf_env_, base_lsn, iterator_))) { + CLOG_LOG(WARN, "seek iterator failed", KR(ret), K(type_), K(base_lsn)); + } else if (OB_FAIL(iterator_.set_io_context(palf::LogIOContext(palf::LogIOUser::REPLAY)))) { + CLOG_LOG(WARN, "iterator set_io_context failed", KR(ret)); } else if (OB_UNLIKELY(!base_scn.is_valid())) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "base_scn is invalid", K(type_), K(base_lsn), K(base_scn), KR(ret)); @@ -222,8 +222,7 @@ int ObReplayServiceSubmitTask::need_skip(const SCN &scn, bool &need_skip) int ObReplayServiceSubmitTask::get_log(const char *&buffer, int64_t &nbytes, SCN &scn, palf::LSN &offset) { - bool unused_is_raw_write = true; - return iterator_.get_entry(buffer, nbytes, scn, offset, unused_is_raw_write); + return iterator_.get_entry(buffer, nbytes, scn, offset); } int ObReplayServiceSubmitTask::next_log(const SCN &replayable_point, @@ -264,26 +263,12 @@ int ObReplayServiceSubmitTask::next_log(const SCN &replayable_point, return ret; } -int ObReplayServiceSubmitTask::reset_iterator(const share::ObLSID &id, - const LSN &begin_lsn) +int ObReplayServiceSubmitTask::reset_iterator(const LSN &begin_lsn) { int ret = OB_SUCCESS; - /* - * next_to_submit_lsn_ may be bigger than begin_lsn; - current replica is A - first revoke ObRoleChangeService::switch_follower_to_leader_(): - T1 A invoke log_handler->get_end_lsn(end_lsn) and end_lsn is 100. - T2 another replica B switches to leader and submits log with lsn[101, 200]. - T3 A invoke replay_service_->switch_to_leader(ls_id) - T4 A failed with invoking role_change_handler->switch_to_leader(cur_task_info_). - replica A replayed log [101, 150) between T1 and T3. - then - T5 replica A invoke ObRoleChangeService::leader_to_follower_forcedly_() - within leader_to_follower_forcedly_(), parameter end_lsn passed to - replay_service_->switch_to_follower(ls_id, end_lsn) is 100 while next_to_submit_lsn_ is 150. - */ next_to_submit_lsn_ = std::max(next_to_submit_lsn_, begin_lsn); - if (OB_FAIL(seek_log_iterator(id, next_to_submit_lsn_, iterator_))) { + if (OB_FAIL(seek_log_iterator_no_shared_storage( + replay_status_->palf_env_, next_to_submit_lsn_, iterator_))) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(WARN, "seek interator failed", K(type_), K(begin_lsn), K(ret)); } else if (OB_FAIL(iterator_.next())) { @@ -460,7 +445,6 @@ int ObLogReplayTask::init(void *log_buf) void ObLogReplayTask::reset() { - ls_id_.reset(); scn_.reset(); lsn_.reset(); read_log_size_ = 0; @@ -483,8 +467,7 @@ void ObLogReplayTask::reset() bool ObLogReplayTask::is_valid() { bool b_ret = false; - b_ret = ls_id_.is_valid() - && scn_.is_valid() + b_ret = scn_.is_valid() && lsn_.is_valid() && read_log_size_ > 0 && NULL != read_log_buf_; @@ -502,7 +485,6 @@ int64_t ObLogReplayTask::get_replay_payload_size() const void ObLogReplayTask::shallow_copy(const ObLogReplayTask &other) { - ls_id_ = other.ls_id_; log_type_ = other.log_type_; lsn_ = other.lsn_; scn_ = other.scn_; @@ -524,8 +506,7 @@ int64_t ObLogReplayTask::to_string(char* buf, const int64_t buf_len) const char log_base_type_str[logservice::OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; (void) log_base_type_to_string(log_type_, log_base_type_str, logservice::OB_LOG_BASE_TYPE_STR_MAX_LEN); J_OBJ_START(); - J_KV(K(ls_id_), - K_(log_type), + J_KV(K_(log_type), "log_type", log_base_type_str, K(lsn_), K(scn_), @@ -546,14 +527,10 @@ int64_t ObLogReplayTask::to_string(char* buf, const int64_t buf_len) const } //---------------ObReplayFsCb---------------// -int ObReplayFsCb::update_end_lsn(int64_t id, - const LSN &end_offset, - const SCN &end_scn, - const int64_t proposal_id) +int ObReplayFsCb::update_end_lsn(const LSN &end_offset, + const SCN &end_scn) { - UNUSED(id); UNUSED(end_scn); - UNUSED(proposal_id); return replay_status_->update_end_offset(end_offset); } @@ -562,15 +539,14 @@ ObReplayStatus::ObReplayStatus(): is_inited_(false), is_enabled_(false), is_submit_blocked_(true), - role_(FOLLOWER), - ls_id_(), + local_replay_enabled_(true), ref_cnt_(0), post_barrier_lsn_(), err_info_(), pending_task_count_(0), last_check_memstore_lsn_(), rwlock_(common::ObLatchIds::REPLAY_STATUS_LOCK), - rolelock_(common::ObLatchIds::REPLAY_STATUS_LOCK), + local_replay_lock_(common::ObLatchIds::REPLAY_STATUS_LOCK), rp_sv_(NULL), submit_log_task_(), palf_env_(NULL), @@ -587,8 +563,7 @@ ObReplayStatus::~ObReplayStatus() destroy(); } -int ObReplayStatus::init(const share::ObLSID &id, - PalfEnv *palf_env, +int ObReplayStatus::init(PalfEnv *palf_env, ObLogReplayService *rp_sv) { //TODO: use replica type init need_replay @@ -596,13 +571,12 @@ int ObReplayStatus::init(const share::ObLSID &id, if (is_inited_) { ret = OB_INIT_TWICE; CLOG_LOG(WARN, "replay status has already been inited", K(ret)); - } else if (!id.is_valid() || OB_ISNULL(palf_env) || OB_ISNULL(rp_sv)) { + } else if (OB_ISNULL(palf_env) || OB_ISNULL(rp_sv)) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(id), K(rp_sv), KP(palf_env), K(ret)); - } else if (OB_FAIL(palf_env->open(id.id(), palf_handle_))) { - CLOG_LOG(ERROR, "failed to open palf handle", K(palf_env), K(id)); + CLOG_LOG(WARN, "invalid argument", K(rp_sv), KP(palf_env), K(ret)); + } else if (OB_FAIL(palf_env->open(palf_handle_))) { + CLOG_LOG(ERROR, "failed to open palf handle", K(palf_env)); } else { - ls_id_ = id; get_log_info_debug_time_ = OB_INVALID_TIMESTAMP; try_wrlock_debug_time_ = OB_INVALID_TIMESTAMP; check_enable_debug_time_ = OB_INVALID_TIMESTAMP; @@ -645,8 +619,7 @@ void ObReplayStatus::destroy() task_queues_[i].destroy(); } is_submit_blocked_ = true; - role_ = FOLLOWER; - ls_id_.reset(); + local_replay_enabled_ = true; post_barrier_lsn_.reset(); err_info_.reset(); last_check_memstore_lsn_.reset(); @@ -669,9 +642,9 @@ int ObReplayStatus::enable(const LSN &base_lsn, const SCN &base_scn) } else { WLockGuard wlock_guard(rwlock_); if (OB_FAIL(enable_(base_lsn, base_scn))) { - CLOG_LOG(WARN, "enable replay status failed", K(ret), K(base_lsn), K(base_scn), K(ls_id_)); + CLOG_LOG(WARN, "enable replay status failed", K(ret), K(base_lsn), K(base_scn)); } else { - CLOG_LOG(INFO, "enable replay status success", K(ret), K(base_lsn), K(base_scn), K(ls_id_)); + CLOG_LOG(INFO, "enable replay status success", K(ret), K(base_lsn), K(base_scn)); } } return ret; @@ -687,7 +660,7 @@ int ObReplayStatus::enable_(const LSN &base_lsn, const SCN &base_scn) //Defense check for reuse scenario ret = OB_ERR_UNEXPECTED; CLOG_LOG(WARN, "remain pending task when enable replay status", K(ret), KPC(this)); - } else if (OB_FAIL(submit_log_task_.init(base_lsn, base_scn, ls_id_, this))) { + } else if (OB_FAIL(submit_log_task_.init(base_lsn, base_scn, this))) { CLOG_LOG(WARN, "failed to init submit_log_task", K(ret), K(&palf_handle_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < REPLAY_TASK_QUEUE_SIZE; ++i) { @@ -714,18 +687,18 @@ int ObReplayStatus::disable() { int ret = OB_SUCCESS; if (!is_enabled()) { - CLOG_LOG(INFO, "replay status already disable", K(ls_id_)); + CLOG_LOG(INFO, "replay status already disable"); } else { do { - WLockGuard guard(rolelock_); + WLockGuard guard(local_replay_lock_); is_submit_blocked_ = true; } while (0); int64_t abs_timeout_us = WRLOCK_TRY_THRESHOLD + ObTimeUtility::current_time(); if (OB_SUCC(rwlock_.wrlock(abs_timeout_us))) { if (OB_FAIL(disable_())) { - CLOG_LOG(WARN, "disable replay status failed", K(ls_id_)); + CLOG_LOG(WARN, "disable replay status failed"); } else { - CLOG_LOG(INFO, "disable replay status success", K(ls_id_)); + CLOG_LOG(INFO, "disable replay status success"); } rwlock_.unlock(); } else { @@ -766,7 +739,7 @@ bool ObReplayStatus::is_enabled_without_lock() const void ObReplayStatus::block_submit() { - WLockGuard guard(rolelock_); + WLockGuard guard(local_replay_lock_); is_submit_blocked_ = true; CLOG_LOG(INFO, "replay status block submit", KPC(this)); } @@ -775,7 +748,7 @@ void ObReplayStatus::unblock_submit() { int ret = OB_SUCCESS; do { - WLockGuard guard(rolelock_); + WLockGuard guard(local_replay_lock_); is_submit_blocked_ = false; CLOG_LOG(INFO, "replay status unblock submit", KPC(this)); } while (0); @@ -796,35 +769,32 @@ bool ObReplayStatus::is_replay_enabled_() const bool ObReplayStatus::need_submit_log() const { - RLockGuard guard(rolelock_); - return (FOLLOWER == role_ && !is_submit_blocked_); + RLockGuard guard(local_replay_lock_); + return local_replay_enabled_ && !is_submit_blocked_; } -void ObReplayStatus::switch_to_leader() +void ObReplayStatus::disable_local_replay() { - WLockGuard guard(rolelock_); - role_ = LEADER; - CLOG_LOG(INFO, "replay status switch_to_leader", KPC(this)); + WLockGuard guard(local_replay_lock_); + local_replay_enabled_ = false; + CLOG_LOG(INFO, "disable local replay", KPC(this)); } -void ObReplayStatus::switch_to_follower(const palf::LSN &begin_lsn) +void ObReplayStatus::enable_local_replay(const palf::LSN &begin_lsn) { int ret = OB_SUCCESS; - // 1.switch role after reset iterator, or fscb may push submit task with - // old iterator, which will fetch logs smaller than max decided scn. - // 2.submit task after switch role, or this task may be discarded if role - // still be leader, and no more submit task being submitted. + // Reset the iterator before allowing new replay submissions. do { WLockGuardWithRetryInterval wguard(rwlock_, WRLOCK_TRY_THRESHOLD, WRLOCK_RETRY_INTERVAL); if (!is_enabled_) { // do nothing } else { - (void)submit_log_task_.reset_iterator(ls_id_, begin_lsn); + (void)submit_log_task_.reset_iterator(begin_lsn); } } while (0); do { - WLockGuard role_guard(rolelock_); - role_ = FOLLOWER; + WLockGuard replay_guard(local_replay_lock_); + local_replay_enabled_ = true; } while (0); #ifdef ERRSIM @@ -833,7 +803,7 @@ if (OB_SUCCESS != tmp_ret) { CLOG_LOG(INFO, "fake EN_REPLAY_SERVICE_SUBMIT_TASK_SLEEP ", KPC(this), K(begin_lsn)); SERVER_EVENT_SYNC_ADD("REPLAYSERVICE", "BEFORE_PUSH_SUBMIT_TASK"); } - DEBUG_SYNC(REPLAY_SWITCH_TO_FOLLOWER_BEFORE_PUSH_SUBMIT_TASK); + DEBUG_SYNC(REPLAY_ENABLE_LOCAL_BEFORE_PUSH_SUBMIT_TASK); #endif RLockGuard rguard(rwlock_); @@ -845,7 +815,7 @@ if (OB_SUCCESS != tmp_ret) { } else { // success } - CLOG_LOG(INFO, "replay status switch_to_follower", KPC(this), K(begin_lsn)); + CLOG_LOG(INFO, "enable local replay", KPC(this), K(begin_lsn)); } @@ -872,9 +842,9 @@ int ObReplayStatus::is_replay_done(const LSN &end_lsn, //TODO: @keqing.llt Change rate limiting to within the class if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { if (is_done) { - CLOG_LOG(INFO, "log stream finished replay", K(ls_id_), K(min_unreplayed_lsn), K(end_lsn)); + CLOG_LOG(INFO, "log stream finished replay", K(min_unreplayed_lsn), K(end_lsn)); } else { - CLOG_LOG(INFO, "log stream has not finished replay", K(ls_id_), K(min_unreplayed_lsn), K(end_lsn)); + CLOG_LOG(INFO, "log stream has not finished replay", K(min_unreplayed_lsn), K(end_lsn)); } } } @@ -903,16 +873,16 @@ int ObReplayStatus::update_end_offset(const LSN &lsn) RLockGuard rlock_guard(rwlock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - CLOG_LOG(ERROR, "replay status is not init", K(ls_id_), K(lsn), K(ret)); + CLOG_LOG(ERROR, "replay status is not init", K(lsn), K(ret)); } else if (!is_enabled_) { if (palf_reach_time_interval(100 * 1000, check_enable_debug_time_)) { CLOG_LOG(INFO, "replay status is not enabled", K(this), K(ret), K(lsn)); } } else if (OB_UNLIKELY(!lsn.is_valid())) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(ERROR, "invalid arguments", K(ls_id_), K(lsn), K(ret)); + CLOG_LOG(ERROR, "invalid arguments", K(lsn), K(ret)); } else if (!need_submit_log()) { - // leader do nothing, keep submit_log_task recording last round status as follower + // Local append handles new entries while replay submission is disabled. } else if (OB_FAIL(submit_task_to_replay_service_(submit_log_task_))) { CLOG_LOG(ERROR, "failed to submit submit_log_task to replay Service", K(submit_log_task_), KPC(this), K(ret)); @@ -920,18 +890,6 @@ int ObReplayStatus::update_end_offset(const LSN &lsn) return ret; } -int ObReplayStatus::get_ls_id(share::ObLSID &id) -{ - int ret = OB_SUCCESS; - if (!ls_id_.is_valid()) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(ERROR, "ls_id_ is invalid", K(ret), K(ls_id_)); - } else { - id = ls_id_; - } - return ret; -} - int ObReplayStatus::get_min_unreplayed_lsn(LSN &lsn) { SCN unused_scn; @@ -1055,7 +1013,7 @@ int ObReplayStatus::get_replay_process(int64_t &submitted_log_size, unsubmitted_log_size = 0; replayed_log_size = min_unreplayed_lsn.val_ - base_lsn.val_; unreplayed_log_size = 0; - CLOG_LOG(INFO, "replay status is not follower", K(min_unreplayed_lsn), K(base_lsn), KPC(this)); + CLOG_LOG(INFO, "local replay is disabled", K(min_unreplayed_lsn), K(base_lsn), KPC(this)); } else if (OB_FAIL(palf_handle_.get_end_lsn(committed_end_lsn))) { CLOG_LOG(WARN, "get_end_lsn failed", K(ret), KPC(this)); } else { @@ -1101,7 +1059,6 @@ int ObReplayStatus::push_log_replay_task(ObLogReplayTask &task) if (OB_SUCC(ret)) { int index = 0; ObLogBaseType log_type = task.log_type_; - share::ObLSID ls_id = task.ls_id_; palf::LSN lsn = task.lsn_; share::SCN scn = task.scn_; bool is_pre_barrier = task.is_pre_barrier_; @@ -1123,7 +1080,7 @@ int ObReplayStatus::push_log_replay_task(ObLogReplayTask &task) } task_queues_[index].set_batch_push_finish(); } - CLOG_LOG(INFO, "submit pre barrier log success", K(log_type), K(ls_id), K(lsn), K(scn), + CLOG_LOG(INFO, "submit pre barrier log success", K(log_type), K(lsn), K(scn), K(is_pre_barrier), K(is_post_barrier), K(log_size)); } else { for (int64_t i = 1; i < broadcast_task_array.count(); ++i) { @@ -1330,8 +1287,6 @@ int ObReplayStatus::stat(LSReplayStat &stat) const if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else { - stat.ls_id_ = ls_id_.id(); - stat.role_ = role_; stat.enabled_ = is_enabled_; stat.pending_cnt_ = pending_task_count_; if (OB_FAIL(submit_log_task_.get_next_to_submit_log_info(stat.unsubmitted_lsn_, @@ -1425,8 +1380,8 @@ int ObReplayStatus::check_can_replay() const { int ret = OB_SUCCESS; { - RLockGuard guard(rolelock_); - if (FOLLOWER != role_) { + RLockGuard guard(local_replay_lock_); + if (!local_replay_enabled_) { ret = OB_EAGAIN; } } diff --git a/src/logservice/replayservice/ob_replay_status.h b/src/logservice/replayservice/ob_replay_status.h index 4a0bcfad8..5add860ed 100644 --- a/src/logservice/replayservice/ob_replay_status.h +++ b/src/logservice/replayservice/ob_replay_status.h @@ -32,7 +32,6 @@ #include "lib/utility/ob_print_utils.h" #include "share/ob_define.h" #include "share/ob_errno.h" -#include "share/ob_ls_id.h" #include "logservice/ob_log_handler.h" namespace oceanbase @@ -63,17 +62,13 @@ enum class ObReplayServiceTaskType //Virtual table statistics struct LSReplayStat { - int64_t ls_id_; - common::ObRole role_; palf::LSN end_lsn_; bool enabled_; palf::LSN unsubmitted_lsn_; share::SCN unsubmitted_scn_; int64_t pending_cnt_; - TO_STRING_KV(K(ls_id_), - K(role_), - K(end_lsn_), + TO_STRING_KV(K(end_lsn_), K(enabled_), K(unsubmitted_lsn_), K(unsubmitted_scn_), @@ -123,16 +118,14 @@ struct ObLogReplayTask : common::ObLink { reset(); } - ObLogReplayTask(const share::ObLSID &ls_id, - const ObLogBaseHeader &header, + ObLogReplayTask(const ObLogBaseHeader &header, const palf::LSN &lsn, const share::SCN &scn, const int64_t log_size, const int64_t base_header_len, void *decompression_buf, int64_t decompressed_log_size) - : ls_id_(ls_id), - log_type_(header.get_log_type()), + : log_type_(header.get_log_type()), lsn_(lsn), scn_(scn), is_pre_barrier_(header.need_pre_replay_barrier()), @@ -159,7 +152,6 @@ struct ObLogReplayTask : common::ObLink int64_t get_replay_payload_size() const; void shallow_copy(const ObLogReplayTask &other); public: - share::ObLSID ls_id_; ObLogBaseType log_type_; palf::LSN lsn_; share::SCN scn_; @@ -260,10 +252,7 @@ class ObReplayServiceTask : public common::LinkTask ObReplayServiceTaskType type_; //for debug: task wait in queue too much time int64_t enqueue_ts_; - // If only linkhashmap manages the lifecycle of replay status, ObReplayServiceTask only stores ls_id, - // Then in the ABA scenario, the remaining tasks will get a new replay status and replay, - // Therefore the task itself needs to record the replay status, and this will make the replay status not only used at the linkhashmap. - // So it is necessary for replay status to manage its own reference count, the reference count of linkhashmap is redundant. + // Tasks retain the replay status so queued work cannot observe a recycled status object. ObReplayStatus *replay_status_; TaskErrInfo err_info_; //control state transition of queue @@ -289,7 +278,6 @@ class ObReplayServiceSubmitTask : public ObReplayServiceTask } int init(const palf::LSN &base_lsn, const share::SCN &base_scn, - const share::ObLSID &id, ObReplayStatus *replay_status); void reset() override; void destroy() override; @@ -309,8 +297,7 @@ class ObReplayServiceSubmitTask : public ObReplayServiceTask int next_log(const share::SCN &replayable_point, bool &iterate_end_by_replayable_point); // Reset the iterator with the current endpoint as the new starting point - int reset_iterator(const share::ObLSID &id, - const palf::LSN &begin_lsn); + int reset_iterator(const palf::LSN &begin_lsn); INHERIT_TO_STRING_KV("ObReplayServiceSubmitTask", ObReplayServiceTask, K(next_to_submit_lsn_), @@ -406,13 +393,14 @@ class ObReplayFsCb : public palf::PalfFSCb replay_status_ = NULL; } // Callback interface, call the update_end_offset interface of replay status - int update_end_lsn(int64_t id, const palf::LSN &end_offset, const share::SCN &end_scn, const int64_t proposal_id); + int update_end_lsn(const palf::LSN &end_offset, const share::SCN &end_scn); private: ObReplayStatus *replay_status_; }; class ObReplayStatus { + friend class ObReplayServiceSubmitTask; public: typedef common::RWLock RWLock; typedef RWLock::RLockGuard RLockGuard; @@ -453,8 +441,7 @@ class ObReplayStatus public: ObReplayStatus(); ~ObReplayStatus(); - int init(const share::ObLSID &id, - palf::PalfEnv *palf_env, + int init(palf::PalfEnv *palf_env, ObLogReplayService *rp_sv); void destroy(); public: @@ -466,7 +453,7 @@ class ObReplayStatus bool is_enabled() const; // for replay service when holding rdlock bool is_enabled_without_lock() const; - // for follower speed_limit + // Replay submission flow control. // 1. avoid more replay cause OOM because speed_limit cannot work when freeze // 2. quick improving max_undecided_log to reduce freeze cost void block_submit(); @@ -482,11 +469,11 @@ class ObReplayStatus rwlock_.unlock(); } - void switch_to_leader(); - void switch_to_follower(const palf::LSN &begin_lsn); + void disable_local_replay(); + void enable_local_replay(const palf::LSN &begin_lsn); // check whether all logs has finished replaying // - // during Leader Reconfirm->Leader Takeover，demanding that there is no log that need to be replayed, + // Before enabling local append, there must be no remaining log to replay. // this function will be invoked and check whether the returned is_done is true // @param [out] is_done，true if all logs have been replayed // @@ -514,7 +501,6 @@ class ObReplayStatus //Release the special log_buf in ObLogReplayTask, only forward barrier logs are effective void free_replay_task_log_buf(ObLogReplayTask *task); - int get_ls_id(share::ObLSID &id); int get_min_unreplayed_lsn(palf::LSN &lsn); int get_max_replayed_scn(share::SCN &scn); int get_min_unreplayed_log_info(palf::LSN &lsn, @@ -576,10 +562,9 @@ class ObReplayStatus last_check_memstore_lsn_ = lsn; } - TO_STRING_KV(K(ls_id_), - K(is_enabled_), + TO_STRING_KV(K(is_enabled_), K(is_submit_blocked_), - K(role_), + K(local_replay_enabled_), K(err_info_), K(ref_cnt_), K(post_barrier_lsn_), @@ -609,8 +594,7 @@ class ObReplayStatus bool is_inited_; bool is_enabled_; // forbidden replay and fetch log if false bool is_submit_blocked_; // allow replay log if true - common::ObRole role_; // leader do not need replay - share::ObLSID ls_id_; + bool local_replay_enabled_; // guarantee the effectiveness of self memory: // inc_ref() before push task into replay_service, dec_ref() after replay_service finished handling task int64_t ref_cnt_; @@ -624,8 +608,7 @@ class ObReplayStatus // Hold the read lock until the log replay is complete when replaying a log entry // Ensure that no log replay will occur after write lock is disabled mutable RWLock rwlock_; - // protect is_submit_blocked_ and role_ - mutable RWLock rolelock_; + mutable RWLock local_replay_lock_; ObLogReplayService *rp_sv_; // be sure to clear these queues when the partition is offline to prevent old replay task is replayed in situation of migrating out and then migrating in @@ -641,7 +624,7 @@ class ObReplayStatus DISALLOW_COPY_AND_ASSIGN(ObReplayStatus); }; -// get replay status with ref protection, for map in replay service +// Keeps the single replay status alive while it is in use. class ObReplayStatusGuard { public: diff --git a/src/logservice/replayservice/ob_tablet_replay_executor.cpp b/src/logservice/replayservice/ob_tablet_replay_executor.cpp index 6ffe9d1d4..b626f54fd 100644 --- a/src/logservice/replayservice/ob_tablet_replay_executor.cpp +++ b/src/logservice/replayservice/ob_tablet_replay_executor.cpp @@ -16,6 +16,7 @@ #include "logservice/replayservice/ob_tablet_replay_executor.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -61,46 +62,43 @@ int ObTabletReplayExecutor::replay_check_restore_status(storage::ObTabletHandle } -int ObTabletReplayExecutor::execute(const share::SCN &scn, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) +int ObTabletReplayExecutor::execute(const share::SCN &scn, const common::ObTabletID &tablet_id) { MDS_TG(5_ms); int ret = OB_SUCCESS; storage::ObTabletHandle tablet_handle; bool can_skip_replay = false; ObTablet *tablet = nullptr; - ObLSHandle ls_handle; - ObLSService *ls_service = nullptr; + ObLS *ls = nullptr; + ObLSService *const ls_service = share::g_mp->ls_service(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "replay executor not init", KR(ret), K_(is_inited)); } else if (!scn.is_valid()) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "replay executor get invalid argument", KR(ret), K(scn), K(ls_id)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "ls service should not be null", K(ret), KP(ls_service)); - } else if (CLICK_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { - CLOG_LOG(WARN, "fail to get log stream", KR(ret), K(ls_id)); - } else if (CLICK_FAIL(check_can_skip_replay_(ls_handle, scn, can_skip_replay))) { - CLOG_LOG(WARN, "failed to check can skip reply", K(ret), K(scn), K(ls_id)); + CLOG_LOG(WARN, "replay executor get invalid argument", KR(ret), K(scn)); + } else if (CLICK_FAIL(ls_service->get_ls(ls))) { + CLOG_LOG(WARN, "fail to get log stream", KR(ret)); + } else if (CLICK_FAIL(check_can_skip_replay_(ls, scn, can_skip_replay))) { + CLOG_LOG(WARN, "failed to check can skip reply", K(ret), K(scn)); } else if (can_skip_replay) { // do nothing - } else if (CLICK_FAIL(replay_get_tablet_(ls_handle, tablet_id, scn, tablet_handle))) { + } else if (CLICK_FAIL(replay_get_tablet_(ls, tablet_id, scn, tablet_handle))) { if (OB_OBSOLETE_CLOG_NEED_SKIP == ret) { - CLOG_LOG(INFO, "clog is already obsolete, should skip replay", K(ret), K(ls_id), K(scn)); + CLOG_LOG(INFO, "clog is already obsolete, should skip replay", K(ret), K(scn)); ret = OB_SUCCESS; } else if (OB_EAGAIN == ret) { - CLOG_LOG_LIMIT(WARN, "need retry to get tablet", K(ret), K(ls_id), K(scn)); + CLOG_LOG_LIMIT(WARN, "need retry to get tablet", K(ret), K(scn)); } else { - CLOG_LOG(WARN, "failed to get tablet", K(ret), K(ls_id), K(scn)); + CLOG_LOG(WARN, "failed to get tablet", K(ret), K(scn)); } } else if (CLICK_FAIL(replay_check_restore_status_(tablet_handle))) { if (OB_NO_NEED_UPDATE == ret) { - CLOG_LOG(WARN, "no need replay after check restore status, skip this log", K(ret), K(ls_id), K(scn)); + CLOG_LOG(WARN, "no need replay after check restore status, skip this log", K(ret), K(scn)); } else if (OB_EAGAIN == ret) { - CLOG_LOG_LIMIT(WARN, "need retry after check restore status", K(ret), K(ls_id), K(scn)); + CLOG_LOG_LIMIT(WARN, "need retry after check restore status", K(ret), K(scn)); } else { - CLOG_LOG(ERROR, "failed to check restore status", K(ret), K(ls_id), K(scn)); + CLOG_LOG(ERROR, "failed to check restore status", K(ret), K(scn)); } } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -111,25 +109,23 @@ int ObTabletReplayExecutor::execute(const share::SCN &scn, const share::ObLSID & } else { ObTabletMdsSharedLockGuard mds_truncate_lock_guard(tablet->get_tablet_pointer_()->get_mds_truncate_lock()); if (OB_FAIL(mds_truncate_lock_guard.get_ret())) { - CLOG_LOG(WARN, "failed to add truncate lock", K(ret), K(ls_id), K(scn), K(tablet_handle)); + CLOG_LOG(WARN, "failed to add truncate lock", K(ret), K(scn), K(tablet_handle)); } else if (CLICK_FAIL(check_can_skip_replay_to_mds_(scn, tablet_handle, can_skip_replay))) { - CLOG_LOG(WARN, "failed to check can skip reply to mds", K(ret), K(ls_id), K(scn), K(tablet_handle)); + CLOG_LOG(WARN, "failed to check can skip reply to mds", K(ret), K(scn), K(tablet_handle)); } else if (can_skip_replay) { //do nothing } else if (CLICK_FAIL(do_replay_(tablet_handle))) { if (OB_NO_NEED_UPDATE == ret) { - CLOG_LOG(WARN, "no need replay, skip this log", K(ret), K(ls_id), K(scn)); + CLOG_LOG(WARN, "no need replay, skip this log", K(ret), K(scn)); } else if (OB_EAGAIN == ret) { - CLOG_LOG_LIMIT(WARN, "failed to replay, need retry", K(ret), K(ls_id), K(scn)); + CLOG_LOG_LIMIT(WARN, "failed to replay, need retry", K(ret), K(scn)); } else { - CLOG_LOG(ERROR, "failed to replay", K(ret), K(ls_id), K(scn)); + CLOG_LOG(ERROR, "failed to replay", K(ret), K(scn)); } } #ifdef ERRSIM if (OB_SUCC(ret)) { - const int64_t errsim_migration_ls_id = GCONF.errsim_migration_ls_id; - const ObLSID errsim_ls_id(errsim_migration_ls_id); const ObString &errsim_migration_dest_server_addr = GCONF.errsim_migration_dest_server_addr.str(); common::ObAddr addr; const ObAddr &my_addr = GCONF.self_addr_; @@ -137,8 +133,7 @@ int ObTabletReplayExecutor::execute(const share::SCN &scn, const share::ObLSID & if (!errsim_migration_dest_server_addr.empty() && OB_FAIL(addr.parse_from_string(errsim_migration_dest_server_addr))) { CLOG_LOG(WARN, "failed to parse from string to addr", K(ret), K(errsim_migration_dest_server_addr)); } else { - if ((ls_id == errsim_ls_id && my_addr == addr) - || GCONF.errsim_test_tablet_id == tablet->get_tablet_id().id()) { + if (my_addr == addr || GCONF.errsim_test_tablet_id == tablet->get_tablet_id().id()) { ret = EN_REPLAY_FATAL_ERROR ? : OB_SUCCESS; if (OB_FAIL(ret)) { STORAGE_LOG(ERROR, "fake EN_REPLAY_FATAL_ERROR", K(ret)); @@ -153,39 +148,34 @@ int ObTabletReplayExecutor::execute(const share::SCN &scn, const share::ObLSID & } int ObTabletReplayExecutor::replay_get_tablet_( - const storage::ObLSHandle &ls_handle, + storage::ObLS *ls, const common::ObTabletID &tablet_id, const share::SCN &scn, storage::ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; const bool is_update_mds_table = is_replay_update_mds_table_(); if (!scn.is_valid() || !tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "check can skip replay to mds get invalid argument", K(ret), K(scn), K(tablet_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "log stream should not be NULL", KR(ret), K(scn)); } else { - const share::ObLSID &ls_id = ls->get_ls_id(); if (is_replay_update_tablet_status_() || is_replay_ddl_control_log_()) { const bool allow_tablet_not_exist = replay_allow_tablet_not_exist_(); if (!is_replay_update_tablet_status_() && is_replay_ddl_control_log_()) { share::ObTaskController::get().allow_next_syslog(); - CLOG_LOG(INFO, "force replay ddl control log", K(ls_id), K(tablet_id), K(scn), K(allow_tablet_not_exist)); + CLOG_LOG(INFO, "force replay ddl control log", K(tablet_id), K(scn), K(allow_tablet_not_exist)); } if (OB_FAIL(ls->replay_get_tablet_no_check(tablet_id, scn, allow_tablet_not_exist, tablet_handle))) { - CLOG_LOG(WARN, "replay get table failed", KR(ret), K(ls_id), K(tablet_id)); + CLOG_LOG(WARN, "replay get table failed", KR(ret), K(tablet_id)); } } else if (OB_FAIL(ls->replay_get_tablet(tablet_id, scn, is_update_mds_table, tablet_handle))) { - CLOG_LOG(WARN, "replay get table failed", KR(ret), K(ls_id), K(tablet_id)); + CLOG_LOG(WARN, "replay get table failed", KR(ret), K(tablet_id)); } if (OB_FAIL(ret)) { if (OB_TIMEOUT == ret) { ret = OB_EAGAIN; - CLOG_LOG(WARN, "retry get tablet for timeout error", KR(ret), K(ls_id), K(tablet_id)); + CLOG_LOG(WARN, "retry get tablet for timeout error", KR(ret), K(tablet_id)); } } } @@ -223,26 +213,23 @@ int ObTabletReplayExecutor::check_can_skip_replay_to_mds_( } int ObTabletReplayExecutor::check_can_skip_replay_( - const storage::ObLSHandle &ls_handle, + storage::ObLS *ls, const share::SCN &scn, bool &can_skip) { int ret = OB_SUCCESS; can_skip = false; - ObLS *ls = nullptr; - share::SCN tablet_change_scn = share::SCN::min_scn(); if (!is_replay_update_tablet_status_()) { can_skip = false; } else if (!scn.is_valid()) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "check can skip replay to mds get invalid argument", K(ret), K(scn)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "log stream should not be NULL", KR(ret), K(scn)); - } else if (FALSE_IT(tablet_change_scn = ls->get_tablet_change_checkpoint_scn())) { - } else if (scn <= tablet_change_scn) { - can_skip = true; - CLOG_LOG(INFO, "can skip replay", "ls_id", ls->get_ls_id(), K(tablet_change_scn), K(scn)); + } else { + const share::SCN tablet_change_scn = ls->get_tablet_change_checkpoint_scn(); + if (scn <= tablet_change_scn) { + can_skip = true; + CLOG_LOG(INFO, "can skip replay", K(tablet_change_scn), K(scn)); + } } return ret; @@ -252,8 +239,7 @@ int ObTabletReplayExecutor::replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletCreateDeleteMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = OB_SUCCESS; storage::ObTablet *tablet = tablet_handle.get_obj(); @@ -268,22 +254,13 @@ int ObTabletReplayExecutor::replay_to_mds_table_( CLOG_LOG(WARN, "inner tablets have no mds table", KR(ret)); } else { ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - const share::ObLSID &ls_id = tablet->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { - CLOG_LOG(WARN, "failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "ls is null", K(ret), K(ls_id), KP(ls)); - } else if (for_old_mds) { - if (OB_FAIL(ls->get_tablet_svr()->set_tablet_status(tablet_id, mds, ctx))) { - CLOG_LOG(WARN, "failed to set mds data", K(ret), K(ls_id), K(tablet_id), K(scn), K(mds)); - } + if (OB_FAIL(ls_svr->get_ls(ls))) { + CLOG_LOG(WARN, "failed to get ls", K(ret)); } else { if (OB_FAIL(ls->get_tablet_svr()->replay_set_tablet_status(tablet_id, scn, mds, ctx))) { - CLOG_LOG(WARN, "failed to replay set tablet status", K(ret), K(ls_id), K(tablet_id), K(scn), K(mds)); + CLOG_LOG(WARN, "failed to replay set tablet status", K(ret), K(tablet_id), K(scn), K(mds)); } } } @@ -294,8 +271,7 @@ int ObTabletReplayExecutor::replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletBindingMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = OB_SUCCESS; storage::ObTablet *tablet = tablet_handle.get_obj(); @@ -310,22 +286,13 @@ int ObTabletReplayExecutor::replay_to_mds_table_( CLOG_LOG(WARN, "inner tablets have no mds table", KR(ret)); } else { ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - const share::ObLSID &ls_id = tablet->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { - CLOG_LOG(WARN, "failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "ls is null", K(ret), K(ls_id), KP(ls)); - } else if (for_old_mds) { - if (OB_FAIL(ls->get_tablet_svr()->set_ddl_info(tablet_id, mds, ctx, 0))) { - CLOG_LOG(WARN, "failed to save tablet binding info", K(ret), K(ls_id), K(tablet_id), K(scn), K(mds)); - } + if (OB_FAIL(ls_svr->get_ls(ls))) { + CLOG_LOG(WARN, "failed to get ls", K(ret)); } else { if (OB_FAIL(ls->get_tablet_svr()->replay_set_ddl_info(tablet_id, scn, mds, ctx))) { - CLOG_LOG(WARN, "failed to replay set ddl info", K(ret), K(ls_id), K(tablet_id), K(scn), K(mds)); + CLOG_LOG(WARN, "failed to replay set ddl info", K(ret), K(tablet_id), K(scn), K(mds)); } } } @@ -336,15 +303,11 @@ int ObTabletReplayExecutor::replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletDDLCompleteMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = OB_SUCCESS; storage::ObTablet *tablet = tablet_handle.get_obj(); - if (for_old_mds) { - ret = OB_ERR_UNDEFINED; - CLOG_LOG(WARN, "ddl complete only support new interface", K(ret)); - } else if (!is_replay_update_mds_table_()) { + if (!is_replay_update_mds_table_()) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "replay log do not update mds table, cannot replay to mds table", K(ret), K(tablet_handle)); } else if (OB_ISNULL(tablet)) { @@ -355,19 +318,14 @@ int ObTabletReplayExecutor::replay_to_mds_table_( CLOG_LOG(WARN, "inner tablets have no mds table", KR(ret)); } else { ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - const share::ObLSID &ls_id = tablet->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { - CLOG_LOG(WARN, "failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - CLOG_LOG(WARN, "ls is null", K(ret), K(ls_id), KP(ls)); + if (OB_FAIL(ls_svr->get_ls(ls))) { + CLOG_LOG(WARN, "failed to get ls", K(ret)); } else { if (OB_FAIL(ls->get_tablet_svr()->replay_set_ddl_complete( tablet_id, scn, mds::DummyKey(), mds, ctx))) { - CLOG_LOG(WARN, "failed to replay set tablet status", K(ret), K(ls_id), K(tablet_id), K(scn), K(mds)); + CLOG_LOG(WARN, "failed to replay set tablet status", K(ret), K(tablet_id), K(scn), K(mds)); } } } diff --git a/src/logservice/replayservice/ob_tablet_replay_executor.h b/src/logservice/replayservice/ob_tablet_replay_executor.h index 5c00bc6e4..e3177c012 100644 --- a/src/logservice/replayservice/ob_tablet_replay_executor.h +++ b/src/logservice/replayservice/ob_tablet_replay_executor.h @@ -30,11 +30,11 @@ namespace oceanbase namespace share { class SCN; -class ObLSID; } namespace storage { +class ObLS; namespace mds { class MdsCtx; @@ -57,7 +57,7 @@ class ObTabletReplayExecutor // @return OB_TABLET_NOT_EXIST, the tablet is not exist. // @return OB_NO_NEED_UPDATE, this log needs to be ignored. // @return other error codes, failed to replay. - int execute(const share::SCN &scn, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); + int execute(const share::SCN &scn, const common::ObTabletID &tablet_id); // check restore status before replay // @return OB_SUCCESS, need replay. @@ -69,13 +69,9 @@ class ObTabletReplayExecutor protected: // Check if this replay operation will update the tablet status(ObTabletCreateDeleteMdsUserData), for example, the following - // 6 types of ObTabletMdsUserDataType need return TRUE. + // types of ObTxDataSourceType need return TRUE. // 1. CREATE_TABLET // 2. REMOVE_TABLET - // 3. RESERVED_3 - // 4. RESERVED_4 - // 5. RESERVED_5 - // 6. RESERVED_6 virtual bool is_replay_update_tablet_status_() const = 0; // replay to the tablet @@ -118,20 +114,17 @@ class ObTabletReplayExecutor storage::ObTabletHandle &tablet_handle, const ObTabletCreateDeleteMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds = false); + const share::SCN &scn); int replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletDDLCompleteMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds); + const share::SCN &scn); int replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletBindingMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds = false); + const share::SCN &scn); template int replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, @@ -153,12 +146,12 @@ class ObTabletReplayExecutor bool &can_skip); // The replay of multi-source log modified by ObTabletCreateDeleteMdsUserData needs to be filtered by tablet_change_checkpoint_scn int check_can_skip_replay_( - const storage::ObLSHandle &ls_handle, + storage::ObLS *ls, const share::SCN &scn, bool &can_skip); int replay_get_tablet_( - const storage::ObLSHandle &ls_handle, + storage::ObLS *ls, const common::ObTabletID &tablet_id, const share::SCN &scn, storage::ObTabletHandle &tablet_handle); diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 96f48a568..9d5205be9 100644 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -985,7 +985,6 @@ typedef enum ObItemType T_FUN_SYS_GTID_SUBTRACT = 2013, T_FUN_SYS_WAIT_FOR_EXECUTED_GTID_SET = 2014, T_FUN_SYS_WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS = 2015, - T_FUN_SYS_LAST_REFRESH_SCN = 2016, T_FUN_SUM_OPNSIZE = 2017, T_FUN_SYS_GET_LOCK = 2018, T_FUN_SYS_IS_FREE_LOCK = 2019, @@ -1630,7 +1629,7 @@ typedef enum ObItemType T_LS_SERVER_TENANT = 3786, T_ZONE_TENANT = 3787, T_CREATE_TIMESTAMP = 3790, - T_RECYCLE_REPLICA = 3791, + T_RESERVED_3791 = 3791, T_MERGE_CONTROL = 3792, T_TENANT_NAME = 3793, T_CACHE_NAME = 3794, @@ -1862,8 +1861,8 @@ typedef enum ObItemType T_EVENT_JOB_ALTER = 4023, T_EVENT_JOB_DROP = 4024, - T_CLEAR_BALANCE_TASK = 4025, - T_BALANCE_TASK_TYPE = 4026, + T_RESERVED_4025 = 4025, + T_RESERVED_4026 = 4026, T_CREATE_SYNONYM = 4027, T_DROP_SYNONYM = 4028, T_CREATE_FUNC = 4029, @@ -2164,7 +2163,6 @@ typedef enum ObItemType T_AUDIT_CLUSTER = 4303, T_AUDIT_CONTEXT = 4304, T_AUDIT_INDEX = 4306, - T_AUDIT_MATERIALIZED_VIEW = 4307, T_AUDIT_NOT_EXIST = 4308, T_AUDIT_OUTLINE = 4309, T_AUDIT_PROCEDURE = 4310, @@ -2462,34 +2460,6 @@ typedef enum ObItemType T_NO_DECORRELATE = 4580, T_COLUMN_ADD_WITH_LOB_PARAMS = 4582, - //for materialized view - T_MV_REFRESH_INFO = 4583, - T_MV_REFRESH_ON_CLAUSE = 4584, - T_MV_REFRESH_METHOD = 4585, - T_MV_REFRESH_INTERVAL = 4586, - T_MV_REFRESH_START_EXPR = 4587, - - // for materialized view log - T_CREATE_MLOG = 4588, - T_MLOG_WITH = 4589, - T_MLOG_WITH_VALUES = 4590, - T_MLOG_WITH_SPECIAL_COLUMN_LIST = 4591, - T_MLOG_WITH_SPECIAL_COLUMN = 4592, - T_MLOG_WITH_REFERENCE_COLUMN = 4593, - T_MLOG_WITH_PRIMARY_KEY = 4594, - T_MLOG_WITH_ROWID = 4595, - T_MLOG_WITH_SEQUENCE = 4596, - T_MLOG_NEW_VALUES = 4597, - T_MLOG_INCLUDING_NEW_VALUES = 4598, - T_MLOG_EXCLUDING_NEW_VALUES = 4599, - T_MLOG_PURGE = 4600, - T_MLOG_PURGE_IMMEDIATE = 4601, - T_MLOG_PURGE_IMMEDIATE_SYNC = 4602, - T_MLOG_PURGE_IMMEDIATE_ASYNC = 4603, - T_MLOG_PURGE_START_NEXT = 4604, - T_MLOG_PURGE_START_TIME_EXPR = 4605, - T_DROP_MLOG = 4606, - T_COL_SKIP_INDEX = 4607, T_COL_SKIP_INDEX_LIST = 4608, T_COL_SKIP_INDEX_MIN_MAX = 4609, @@ -2525,9 +2495,6 @@ typedef enum ObItemType T_PARALLEL_DAS_DML = 4651, T_DISABLE_PARALLEL_DAS_DML = 4652, T_ENABLE_LOB_PREFETCH = 4653, - T_MV_OPTIONS = 4654, - T_MV_REWRITE = 4655, - T_MV_NO_REWRITE = 4656, // select into outfile T_INTO_FILE_LIST = 4657, T_SINGLE_OPT = 4658, @@ -2566,9 +2533,6 @@ typedef enum ObItemType T_VEC_INDEX_COLUMN = 4691, T_VEC_INDEX_COLUMN_EXPR = 4692, // 4693: T_BACKUP_CLUSTER_PARAMETERS abandoned, id reserved - // create mv build deferred - T_MV_REFRESH_OPT = 4694, - T_MV_BUILD_OPT = 4695, T_PSEUDO_EXTERNAL_FILE_ROW = 4696, T_EXTERNAL_TABLE_AUTO_REFRESH = 4697, @@ -2629,7 +2593,6 @@ typedef enum ObItemType T_UNION_MERGE_HINT = 4740, T_UNION_MERGE_LIST = 4741, - T_PSEUDO_OLD_NEW_COL = 4742, T_TRANSFORM_DISTINCT_AGG = 4743, T_NO_TRANSFORM_DISTINCT_AGG = 4744, @@ -2684,7 +2647,6 @@ typedef enum ObItemType T_PARSER_DICT_TABLE = 4783, T_PARSER_QUANTIFIER_TABLE = 4784, - T_ALTER_MLOG_OPTIONS = 4785, // macro block bloom filter T_ENABLE_MACRO_BLOCK_BLOOM_FILTER = 4786, T_DISTRIBUTE_HASH_LOCAL = 4787, @@ -2795,7 +2757,6 @@ typedef enum ObItemType T_UDF_PROPERTY_LIST = 4866, T_MICRO_BLOCK_FORMAT_VERSION = 4867, - T_MV_NESTED_REFRESH_CLAUSE = 4868, // placeholder for mview T_ALTER_SUBPARTITION_EXCHANGE = 4869, T_CREATE_SENSITIVE_RULE = 4870, T_DROP_SENSITIVE_RULE = 4871, diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 6ef1d2381..f0933e21b 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -180,10 +180,6 @@ ob_set_subtarget(ob_server table_load table_load/dag/ob_table_load_dag.cpp table_load/dag/ob_table_load_empty_insert_dag.cpp table_load/resource/ob_table_load_resource_manager.cpp - table_load/resource/ob_table_load_resource_processor.cpp - table_load/resource/ob_table_load_resource_rpc_executor.cpp - table_load/resource/ob_table_load_resource_rpc.cpp - table_load/resource/ob_table_load_resource_rpc_struct.cpp table_load/resource/ob_table_load_resource_service.cpp table_load/plan/ob_table_load_data_channel.cpp table_load/plan/ob_table_load_data_table_row_handler.cpp @@ -204,7 +200,6 @@ ob_set_subtarget(ob_server table_load table_load/plan/ob_table_load_unique_index_table_row_handler.cpp table_load/plan/ob_table_load_write_op.cpp table_load/ob_table_load_assigned_memory_manager.cpp - table_load/ob_table_load_assigned_task_manager.cpp table_load/ob_table_load_autoinc_nextval.cpp table_load/ob_table_load_bucket.cpp table_load/ob_table_load_client_service.cpp @@ -425,13 +420,11 @@ ob_set_subtarget(ob_server virtual_table virtual_table/ob_all_virtual_nic_info.cpp virtual_table/ob_all_virtual_sys_variable_default_value.cpp virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp - virtual_table/ob_all_virtual_tenant_mview_running_job.cpp virtual_table/ob_all_virtual_compatibility_control.cpp virtual_table/ob_all_virtual_vector_index_info.cpp virtual_table/ob_all_virtual_session_ps_info.cpp virtual_table/ob_all_virtual_sql_stat.cpp virtual_table/ob_all_virtual_tmp_file.cpp - virtual_table/ob_all_virtual_log_transport_dest_stat.cpp virtual_table/ob_all_virtual_plugin_info.cpp virtual_table/ob_all_virtual_change_stream_refresh_stat.cpp virtual_table/ob_all_virtual_dynamic_partition_table.cpp diff --git a/src/observer/change_stream/ob_change_stream_fetcher.cpp b/src/observer/change_stream/ob_change_stream_fetcher.cpp index 628345866..d03cda0ba 100644 --- a/src/observer/change_stream/ob_change_stream_fetcher.cpp +++ b/src/observer/change_stream/ob_change_stream_fetcher.cpp @@ -36,7 +36,6 @@ #include "logservice/ob_log_base_header.h" #include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "logservice/ob_log_handler.h" #include "logservice/palf/log_io_context.h" #include "share/rc/ob_tenant_base.h" @@ -51,7 +50,6 @@ ObCSFetcher::ObCSFetcher() : share::ObThreadPool(1), is_inited_(false), dispatcher_(nullptr), - ls_id_(ObLSID::SYS_LS_ID), iter_(), current_lsn_(), current_scn_(), @@ -113,7 +111,7 @@ int ObCSFetcher::init_consumption_position_() start_lsn = current_lsn_; } if (OB_SUCC(ret)) { - if (OB_FAIL(logservice::seek_log_iterator(ls_id_, start_lsn, iter_))) { + if (OB_FAIL(logservice::seek_log_iterator(start_lsn, iter_))) { LOG_WARN("CSFetcher: fail to seek_log_iterator by min_dep_lsn", KR(ret), K(start_lsn)); } else { current_lsn_ = start_lsn; @@ -122,7 +120,7 @@ int ObCSFetcher::init_consumption_position_() } } if (OB_SUCC(ret)) { - palf::LogIOContext io_ctx(ls_id_.id(), palf::LogIOUser::CDC); + palf::LogIOContext io_ctx(palf::LogIOUser::CDC); if (OB_FAIL(iter_.set_io_context(io_ctx))) { LOG_WARN("CSFetcher: fail to set_io_context", KR(ret)); } @@ -167,7 +165,7 @@ int ObCSFetcher::init_consumption_position_() LOG_INFO("CSFetcher: schema version initialized by SCN", K(current_schema_version_)); } } - if (OB_SUCC(ret) && OB_FAIL(logservice::seek_log_iterator(ls_id_, start_lsn, iter_))) { + if (OB_SUCC(ret) && OB_FAIL(logservice::seek_log_iterator(start_lsn, iter_))) { LOG_WARN("CSFetcher: fail to seek back after schema init", KR(ret)); } } @@ -246,14 +244,10 @@ int ObCSFetcher::get_min_dep_lsn(palf::LSN &min_lsn) min_lsn.reset(); palf::LSN end_lsn; - storage::ObLSHandle tmp_handle; storage::ObLS *ls = nullptr; - logservice::ObLogHandler *log_handler = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, tmp_handle, storage::ObLSGetMod::LOG_MOD)) - || OB_ISNULL(ls = tmp_handle.get_ls()) - || OB_ISNULL(log_handler = ls->get_log_handler()) - || OB_FAIL(log_handler->get_end_lsn(end_lsn))) { - LOG_WARN("CSFetcher: fail to get end_lsn for min_dep_lsn", KR(ret), K(ls_id_)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls)) + || OB_FAIL(ls->get_log_handler()->get_end_lsn(end_lsn))) { + LOG_WARN("CSFetcher: fail to get end_lsn for min_dep_lsn", KR(ret)); return ret; } @@ -302,7 +296,7 @@ int ObCSFetcher::get_refresh_scn(SCN &refresh_scn) refresh_scn.reset(); SCN gts_scn; - if (OB_FAIL(OB_TS_MGR.get_gts(NULL, gts_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts(gts_scn))) { LOG_WARN("CSFetcher: fail to get GTS for refresh_scn", KR(ret)); return ret; } @@ -326,14 +320,10 @@ int ObCSFetcher::get_refresh_scn(SCN &refresh_scn) // Fetch max_lsn to distinguish case 3 and case 4. palf::LSN max_lsn; { - storage::ObLSHandle tmp_handle; storage::ObLS *ls = nullptr; - logservice::ObLogHandler *log_handler = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, tmp_handle, storage::ObLSGetMod::LOG_MOD)) - || OB_ISNULL(ls = tmp_handle.get_ls()) - || OB_ISNULL(log_handler = ls->get_log_handler()) - || OB_FAIL(log_handler->get_max_lsn(max_lsn))) { - LOG_WARN("CSFetcher: fail to get max_lsn in get_refresh_scn", KR(ret), K(ls_id_)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls)) + || OB_FAIL(ls->get_log_handler()->get_max_lsn(max_lsn))) { + LOG_WARN("CSFetcher: fail to get max_lsn in get_refresh_scn", KR(ret)); return ret; } } @@ -341,7 +331,7 @@ int ObCSFetcher::get_refresh_scn(SCN &refresh_scn) if (current_lsn_.is_valid() && current_lsn_ >= max_lsn) { // Case 3: caught up — no pending logs, advance to GTS. SCN gts_scn; - if (OB_FAIL(OB_TS_MGR.get_gts(NULL, gts_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts(gts_scn))) { LOG_WARN("CSFetcher: fail to get GTS for refresh_scn (caught-up)", KR(ret)); } else { refresh_scn = gts_scn; @@ -877,7 +867,7 @@ void ObCSFetcher::run1() if (OB_SUCC(init_consumption_position_())) { iter_ready = true; FLOG_INFO("CSFetcher: iterator initialized, starting consumption", - K(current_lsn_), K(current_scn_), K(ls_id_)); + K(current_lsn_), K(current_scn_)); } else { LOG_WARN("CSFetcher: init consumption position failed, retry", KR(ret)); usleep(CS_FETCHER_INIT_FAIL_SLEEP_US); diff --git a/src/observer/change_stream/ob_change_stream_fetcher.h b/src/observer/change_stream/ob_change_stream_fetcher.h index 78b557917..2245224a8 100644 --- a/src/observer/change_stream/ob_change_stream_fetcher.h +++ b/src/observer/change_stream/ob_change_stream_fetcher.h @@ -26,7 +26,6 @@ #include "lib/hash/ob_hashmap.h" #include "common/ob_tablet_id.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "share/ob_thread_pool.h" #include "lib/lock/ob_thread_cond.h" #include "logservice/palf/lsn.h" @@ -185,7 +184,6 @@ class ObCSFetcher : public share::ObThreadPool bool is_inited_; ObCSDispatcher *dispatcher_; - share::ObLSID ls_id_; palf::PalfBufferIterator iter_; palf::LSN current_lsn_; SCN current_scn_; diff --git a/src/observer/change_stream/ob_change_stream_mgr.cpp b/src/observer/change_stream/ob_change_stream_mgr.cpp index 7169d139d..6c0df371e 100644 --- a/src/observer/change_stream/ob_change_stream_mgr.cpp +++ b/src/observer/change_stream/ob_change_stream_mgr.cpp @@ -138,8 +138,8 @@ int ObChangeStreamMgr::wait_refresh_scn( const int64_t SLEEP_INTERVAL_US = 100 * 1000; // 100ms const int64_t abs_timeout_us = ObTimeUtility::current_time() + timeout_us; - if (OB_FAIL(OB_TS_MGR.get_ts_sync(abs_timeout_us - ObTimeUtility::current_time(), - safe_visible_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts_sync(abs_timeout_us - ObTimeUtility::current_time(), + safe_visible_scn))) { LOG_WARN("get gts for safe visible scn failed", KR(ret)); } else { ObChangeStreamMgr *mgr = share::g_mp->change_stream_mgr(); diff --git a/src/observer/change_stream/ob_cs_plugin_async_index.cpp b/src/observer/change_stream/ob_cs_plugin_async_index.cpp index b12c05803..1480e3d80 100644 --- a/src/observer/change_stream/ob_cs_plugin_async_index.cpp +++ b/src/observer/change_stream/ob_cs_plugin_async_index.cpp @@ -21,7 +21,7 @@ #include "share/schema/ob_schema_getter_guard.h" #include "share/schema/ob_tenant_schema_service.h" #include "share/schema/ob_multi_version_schema_service.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/schema/ob_table_schema.h" #include "share/schema/ob_column_schema.h" #include "observer/vector_index/ob_vector_index_util.h" @@ -35,12 +35,10 @@ #include "storage/blocksstable/ob_datum_row.h" #include "storage/tx_storage/ob_access_service.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/ls/ob_ls.h" #include "storage/access/ob_table_scan_iterator.h" #include "share/table/ob_table_util.h" #include "share/ob_server_struct.h" -#include "share/location_cache/ob_location_service.h" #include "common/mysqlclient/ob_isql_connection.h" #include "storage/tx/ob_trans_define.h" #include "observer/ob_inner_sql_connection.h" @@ -482,20 +480,20 @@ int ObCSAsyncIndexProcessor::resolve_table_id_from_tablet_id_( LOG_WARN("sql_proxy is null", K(ret)); } else { common::ObSEArray tablet_ids; - common::ObSEArray infos; + common::ObSEArray infos; if (OB_FAIL(tablet_ids.push_back(tablet_id))) { LOG_WARN("fail to push back tablet_id", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get( + } else if (OB_FAIL(ObTabletMappingTableOperator::batch_get( *GCTX.sql_proxy_, tablet_ids, infos))) { if (common::OB_ITEM_NOT_MATCH == ret) { - LOG_WARN("tablet not in __all_tablet_to_ls", K(ret), K(tablet_id)); + LOG_WARN("tablet mapping not found", K(ret), K(tablet_id)); ret = common::OB_TABLET_NOT_EXIST; table_id = common::OB_INVALID_ID; if (OB_FAIL(tablet_to_table_.set_refactored(tablet_id.id(), common::OB_INVALID_ID))) { LOG_WARN("fail to cache invalid tablet_id", K(ret), K(tablet_id)); } } else { - LOG_WARN("fail to get table_id from __all_tablet_to_ls", K(ret), K(tablet_id)); + LOG_WARN("fail to get table_id from tablet mapping", K(ret), K(tablet_id)); } } else if (OB_UNLIKELY(infos.count() != 1)) { ret = common::OB_ERR_UNEXPECTED; @@ -504,7 +502,7 @@ int ObCSAsyncIndexProcessor::resolve_table_id_from_tablet_id_( table_id = infos.at(0).get_table_id(); if (common::OB_INVALID_ID == table_id) { ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("table_id invalid in __all_tablet_to_ls", K(ret), K(tablet_id)); + LOG_WARN("table_id invalid in tablet mapping", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_to_table_.set_refactored(tablet_id.id(), table_id))) { LOG_WARN("set tablet_to_table failed", KR(ret)); } @@ -958,7 +956,6 @@ int ObCSAsyncIndexProcessor::set_das_insert_context_(const common::ObIArrayset_trans_desc(tx_desc); - share::ObLSID ls_id = share::SYS_LS; const common::ObTabletID &data_tablet_id = events.at(0).tablet_id_; @@ -985,7 +982,6 @@ int ObCSAsyncIndexProcessor::set_das_insert_context_(const common::ObIArrayset_tablet_id(vbitmap_tablet_id); - insert_op->set_ls_id(ls_id); } if (OB_SUCC(ret)) { @@ -1001,15 +997,14 @@ int ObCSAsyncIndexProcessor::set_das_insert_context_(const common::ObIArrayget_ls_read_snapshot(*tx_desc, - transaction::ObTxIsolationLevel::RC, - ls_id, - timeout_ts, - *snapshot))) { - LOG_WARN("Failed to get ls read snapshot", K(ret), K(ls_id), K(timeout_ts)); + if (OB_FAIL(txs->get_read_snapshot(*tx_desc, + transaction::ObTxIsolationLevel::RC, + timeout_ts, + *snapshot))) { + LOG_WARN("Failed to get read snapshot", K(ret), K(timeout_ts)); } else if (!snapshot->is_valid()) { ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("Invalid snapshot from get_ls_read_snapshot", K(ret), KPC(snapshot)); + LOG_WARN("Invalid snapshot from get_read_snapshot", K(ret), KPC(snapshot)); } else { insert_op->set_snapshot(snapshot); } @@ -1079,7 +1074,7 @@ int ObCSAsyncIndexProcessor::insert_vector_index_log_batch_( K(ret), "event_cnt", events.count(), K(target_vec_info), "first_event", events.at(0), "index_id_table_id", target_vec_info.index_id_table_id_, "data_tablet_id", events.at(0).tablet_id_, - "ls_id", insert_op->get_ls_id(), "tablet_id", insert_op->get_tablet_id(), + "tablet_id", insert_op->get_tablet_id(), "tx_desc_valid", (tx_desc != nullptr && tx_desc->is_valid()), "schema_version", ctx_.schema_version_, "timeout_ts", rtdef != nullptr ? rtdef->timeout_ts_ : -1, "column_ids_cnt", ctdef != nullptr ? ctdef->column_ids_.count() : 0, @@ -1123,7 +1118,6 @@ int ObCSAsyncIndexProcessor::write_to_vsag_( if (!need_vsag) { } else { const common::ObTabletID &data_tablet_id = events.at(0).tablet_id_; - const share::ObLSID ls_id = share::SYS_LS; ObPluginVectorIndexService *vec_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexAdapterGuard adapter_guard; ObPluginVectorIndexAdaptor *adaptor = nullptr; @@ -1171,14 +1165,13 @@ int ObCSAsyncIndexProcessor::write_to_vsag_( } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(ls_id, - inc_tablet_id, + } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(inc_tablet_id, schema::INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL, adapter_guard, &vec_idx_param, vec_info.dim_))) { LOG_WARN("Failed to acquire adapter for inc tablet", - K(ret), K(inc_tablet_id), K(ls_id), K(vec_info.dim_)); + K(ret), K(inc_tablet_id), K(vec_info.dim_)); } else { adaptor = adapter_guard.get_adatper(); if (OB_ISNULL(adaptor)) { @@ -1271,7 +1264,7 @@ int ObCSAsyncIndexProcessor::write_to_vsag_( nullptr, insert_count))) { LOG_WARN("Failed to add vectors to vsag index", - K(ret), K(insert_count), K(dim), K(ls_id), K(vec_info.index_id_table_id_)); + K(ret), K(insert_count), K(dim), K(vec_info.index_id_table_id_)); } else { int tmp_ret = adaptor->update_incr_bitmap(vids, insert_count); if (OB_SUCCESS != tmp_ret) { diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_service.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_service.cpp index 74b6b23f8..eae069a90 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_service.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_service.cpp @@ -117,41 +117,24 @@ void ObDBMSSchedService::destroy() LOG_INFO("[DBMS_SCHED_SERVICE] ObDBMSSchedService destroy success"); } -void ObDBMSSchedService::switch_to_follower_forcedly() +void ObDBMSSchedService::deactivate() { - switch_to_follower_gracefully(); -} -int ObDBMSSchedService::switch_to_leader() -{ - int ret = OB_SUCCESS; - if (job_master_.is_inited()) { - job_master_.switch_to_leader(); - ObTenantThreadHelper::switch_to_leader(); - LOG_INFO("[DBMS_SCHED_SERVICE] ObDBMSSchedService switch leader"); - } - return ret; -} -int ObDBMSSchedService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; if (job_master_.is_inited()) { job_master_.switch_to_follower(); - ObTenantThreadHelper::switch_to_follower_gracefully(); - LOG_INFO("[DBMS_SCHED_SERVICE] ObDBMSSchedService switch follower"); + ObTenantThreadHelper::deactivate(); + LOG_INFO("[DBMS_SCHED_SERVICE] ObDBMSSchedService stopped"); } - return ret; } -int ObDBMSSchedService::resume_leader() +int ObDBMSSchedService::activate() { int ret = OB_SUCCESS; - if (!is_leader()) { - if (OB_FAIL(switch_to_leader())) { - LOG_INFO("[DBMS_SCHED_SERVICE] resume leader failed"); - } + if (job_master_.is_inited()) { + job_master_.switch_to_leader(); + ObTenantThreadHelper::activate(); + LOG_INFO("[DBMS_SCHED_SERVICE] ObDBMSSchedService switch leader"); } return ret; } - void ObDBMSSchedService::wakeup_scheduler() { int ret = OB_SUCCESS; diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_service.h b/src/observer/dbms_scheduler/ob_dbms_sched_service.h index 3a6a687eb..7cf533b0d 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_service.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_service.h @@ -18,7 +18,7 @@ #define OCEANBASE_ROOTSERVER_OB_DBMS_SCHEDULER_SERVICE_H #include "share/ob_define.h" -#include "logservice/ob_log_base_type.h" //ObIRoleChangeSubHandler ObICheckpointSubHandler ObIReplaySubHandler +#include "logservice/ob_log_base_type.h" //ObILocalLogHandler ObICheckpointSubHandler ObIReplaySubHandler #include "observer/dbms_scheduler/ob_dbms_sched_job_master.h" #include "rootserver/ob_tenant_thread_helper.h" // for ObTenantThreadHelper @@ -71,10 +71,8 @@ class ObDBMSSchedService : public ObTenantThreadHelper, } // for role change - virtual void switch_to_follower_forcedly() override; - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override; - virtual int resume_leader() override; + void deactivate() override; + int activate() override; private: dbms_scheduler::ObDBMSSchedJobMaster job_master_; diff --git a/src/observer/mysql/ob_async_plan_driver.cpp b/src/observer/mysql/ob_async_plan_driver.cpp index d4b5a63ab..55705809d 100644 --- a/src/observer/mysql/ob_async_plan_driver.cpp +++ b/src/observer/mysql/ob_async_plan_driver.cpp @@ -113,7 +113,7 @@ int ObAsyncPlanDriver::response_result(ObMySQLResultSet &result) if (OB_SUCCESS != (sret = sender_.send_error_packet(ret, NULL, is_partition_hit))) { LOG_WARN("send error packet fail", K(sret), K(ret)); } - //According to the agreement with the transaction layer, regardless of whether end_participant and end_stmt succeed or not, + //According to the agreement with the transaction layer, regardless of whether end_stmt succeeds or not, //Determine whether the transaction commit or rollback is successful by only checking if the final end_trans is successful, // and SQL must ensure that end_trans is called, when calling end_trans it checks if the connection needs to be terminated, //So here there is no need to check if the connection needs to be terminated diff --git a/src/observer/mysql/obmp_query.cpp b/src/observer/mysql/obmp_query.cpp index 0a8fd04ab..c14a36342 100644 --- a/src/observer/mysql/obmp_query.cpp +++ b/src/observer/mysql/obmp_query.cpp @@ -1213,73 +1213,6 @@ int ObMPQuery::store_params_value_to_str(ObIAllocator &allocator, return ret; } -//int ObMPQuery::fill_feedback_session_info(ObMySQLResultSet &result, -// ObSQLSessionInfo &session) -//{ -// int ret = OB_SUCCESS; -// ObPhysicalPlan *temp_plan = NULL; -// ObTaskExecutorCtx *temp_task_ctx = NULL; -// ObSchemaGetterGuard *schema_guard = NULL; -// if (session.is_abundant_feedback_support() && -// NULL != (temp_plan = result.get_physical_plan()) && -// NULL != (temp_task_ctx = result.get_exec_context().get_task_executor_ctx()) && -// NULL != (schema_guard = ctx_.schema_guard_) && -// temp_plan->get_plan_type() == ObPhyPlanType::OB_PHY_PLAN_REMOTE && -// temp_plan->get_location_type() != ObPhyPlanType::OB_PHY_PLAN_UNCERTAIN && -// temp_task_ctx->get_table_locations().count() == 1 && -// temp_task_ctx->get_table_locations().at(0).get_partition_location_list().count() == 1) { -// bool is_cache_hit = false; -// ObFBPartitionParam param; -// //FIXME: should remove ObPartitionKey -// ObPartitionKey partition_key; -// ObPartitionLocation partition_loc; -// const ObTableSchema *table_schema = NULL; -// ObPartitionReplicaLocationIArray &pl_array = -// temp_task_ctx->get_table_locations().at(0).get_partition_location_list(); -// if (OB_FAIL(pl_array.at(0).get_partition_key(partition_key))) { -// LOG_WARN("failed to get partition key", K(ret)); -// } else if (OB_FAIL(temp_cache->get(partition_key, -// partition_loc, -// 0, -// is_cache_hit))) { -// LOG_WARN("failed to get partition location", K(ret)); -// partition_key.get_table_id(), -// table_schema))) { -// LOG_WARN("failed to get table schema", K(ret), K(partition_key)); -// } else if (OB_ISNULL(table_schema)) { -// ret = OB_ERR_UNEXPECTED; -// LOG_WARN("null table schema", K(ret)); -// } else if (OB_FAIL(build_fb_partition_param(*table_schema, partition_loc, param))) { -// LOG_WARN("failed to build fb partition pararm", K(ret)); -// } else if (OB_FAIL(session.set_partition_location_feedback(param))) { -// LOG_WARN("failed to set partition location feedback", K(param), K(ret)); -// } else { /*do nothing*/ } -// } else { /*do nothing*/} -// return ret; -//} - -//int ObMPQuery::build_fb_partition_param( -// const ObTableSchema &table_schema, -// const ObPartitionLocation &partition_loc, -// ObFBPartitionParam ¶m) { -// INIT_SUCC(ret); -// param.schema_version_ = table_schema.get_schema_version(); -// int64_t origin_partition_idx = OB_INVALID_ID; -// if (OB_FAIL(param.pl_.assign(partition_loc))) { -// LOG_WARN("fail to assign pl", K(partition_loc), K(ret)); -// } -// // when table partition_id to client, we need convert it to -// // real partition idx(e.g. hash partition split) -// else if (OB_FAIL(table_schema.convert_partition_id_to_idx( -// partition_loc.get_partition_id(), origin_partition_idx))) { -// LOG_WARN("fail to convert partition id", K(partition_loc), K(ret)); -// } else { -// param.original_partition_id_ = origin_partition_idx; -// } -// -// return ret; -//} - int ObMPQuery::check_readonly_stmt(ObMySQLResultSet &result) { int ret = OB_SUCCESS; diff --git a/src/observer/mysql/obmp_query.h b/src/observer/mysql/obmp_query.h index d901a7501..4b04e58ec 100644 --- a/src/observer/mysql/obmp_query.h +++ b/src/observer/mysql/obmp_query.h @@ -38,7 +38,6 @@ namespace schema { class ObTableSchema; } -class ObPartitionLocation; } namespace observer { diff --git a/src/observer/net/ob_ingress_bw_alloc_service.cpp b/src/observer/net/ob_ingress_bw_alloc_service.cpp index f3fa69ce2..7fe42fee4 100644 --- a/src/observer/net/ob_ingress_bw_alloc_service.cpp +++ b/src/observer/net/ob_ingress_bw_alloc_service.cpp @@ -338,30 +338,15 @@ void ObIngressBWAllocService::runTimerTask() } } -void ObIngressBWAllocService::switch_to_follower_forcedly() +void ObIngressBWAllocService::deactivate() { ATOMIC_STORE(&is_leader_, false); } -int ObIngressBWAllocService::switch_to_leader() +int ObIngressBWAllocService::activate() { int ret = OB_SUCCESS; ATOMIC_STORE(&is_leader_, true); return ret; } -int ObIngressBWAllocService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - ATOMIC_STORE(&is_leader_, false); - return ret; -} -int ObIngressBWAllocService::resume_leader() -{ - int ret = OB_SUCCESS; - if (!is_leader()) { - ATOMIC_STORE(&is_leader_, true); - } - return ret; -} - } // namespace rootserver } // namespace oceanbase diff --git a/src/observer/net/ob_ingress_bw_alloc_service.h b/src/observer/net/ob_ingress_bw_alloc_service.h index 24ea2c97a..4a8909f21 100644 --- a/src/observer/net/ob_ingress_bw_alloc_service.h +++ b/src/observer/net/ob_ingress_bw_alloc_service.h @@ -19,7 +19,7 @@ #include "share/ob_define.h" #include "lib/task/ob_timer.h" -#include "logservice/ob_log_base_type.h" //ObIRoleChangeSubHandler ObICheckpointSubHandler ObIReplaySubHandler +#include "logservice/ob_log_base_type.h" //ObILocalLogHandler ObICheckpointSubHandler ObIReplaySubHandler #include "observer/net/ob_net_endpoint_ingress_rpc_struct.h" #include "observer/ob_server_struct.h" @@ -51,7 +51,7 @@ class ObNetEndpointIngressManager DISALLOW_COPY_AND_ASSIGN(ObNetEndpointIngressManager); }; class ObIngressBWAllocService : public common::ObTimerTask, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler, public logservice::ObIReplaySubHandler { @@ -104,10 +104,8 @@ class ObIngressBWAllocService : public common::ObTimerTask, } // for role change - void switch_to_follower_forcedly() override final; - int switch_to_leader() override final; - int switch_to_follower_gracefully() override final; - int resume_leader() override final; + void deactivate() override final; + int activate() override final; private: bool is_inited_; diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index 2d0eaa15c..e2bdc6fde 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -937,7 +937,6 @@ template int ObInnerSQLConnection::retry_while_no_tenant_resource(const int64_t cluster_id, T function) { int ret = OB_SUCCESS; - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); const int64_t max_retry_us = 128 * 1000; int64_t retry_us = 2 * 1000; bool need_retry = is_in_trans() ? false : true; @@ -962,7 +961,7 @@ int ObInnerSQLConnection::retry_while_no_tenant_resource(const int64_t cluster_i LOG_WARN("timeout, do not need retry", K(ret), K(abs_timeout_us), K(now)); } else if (OB_FAIL(function())) { if (is_unit_migrate(ret)) { - LOG_INFO("failed to get newest location and will force renew", K(ret), K(ls_id)); + LOG_INFO("failed to get newest location and will force renew", K(ret)); ob_usleep(retry_us); if (retry_us < max_retry_us) { retry_us = retry_us * 2; @@ -1032,8 +1031,6 @@ int ObInnerSQLConnection::start_transaction_inner( // remote } else { TimeoutGuard timeout_guard(*this); // backup && restore worker/session timeout - common::ObAddr resource_server_addr; // MYADDR - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); int64_t query_timeout = OB_INVALID_TIMESTAMP; int64_t trx_timeout = OB_INVALID_TIMESTAMP; ObSQLMode sql_mode = 0; @@ -1047,10 +1044,7 @@ int ObInnerSQLConnection::start_transaction_inner( ret = OB_ERR_UNEXPECTED; LOG_WARN("resource_conn_id is not invalid or resource_svr is valid in start_transaction", K(ret), K(get_resource_conn_id()), K(get_resource_svr())); - } else if (OB_FAIL(nonblock_get_leader(GCONF.cluster_id, ls_id, - resource_server_addr))) { - LOG_WARN("nonblock get leader failed", KR(ret), K(ls_id), K(resource_server_addr)); - } else if (FALSE_IT(set_resource_svr(resource_server_addr))) { + } else if (FALSE_IT(set_resource_svr(GCTX.self_addr()))) { } else if (OB_FAIL(get_session_timeout_for_rpc(query_timeout, trx_timeout))) { LOG_WARN("fail to get_session_timeout_for_rpc", K(ret), K(query_timeout), K(trx_timeout)); } else { @@ -1071,7 +1065,6 @@ int ObInnerSQLConnection::start_transaction_inner( } int ObInnerSQLConnection::register_multi_data_source( - const share::ObLSID ls_id, const transaction::ObTxDataSourceType type, const char *buf, const int64_t buf_len, @@ -1104,16 +1097,14 @@ int ObInnerSQLConnection::register_multi_data_source( if (OB_ISNULL(tx_desc = get_session().get_tx_desc())) { // TODO ADD LOG and check get_session ret = OB_ERR_UNEXPECTED; - LOG_WARN("Invalid tx_desc", K(ret), K(ls_id), K(type)); + LOG_WARN("Invalid tx_desc", K(ret), K(type)); } else { MOD_SCOPE { if (OB_FAIL(share::g_mp->trans_service()->register_mds_into_tx(*tx_desc, - ls_id, type, buf, buf_len, - 0, register_flag))) { LOG_WARN("regiser multi data source failed", K(ret), K(type)); } else if (OB_FAIL(res.close())) { @@ -1144,8 +1135,8 @@ int ObInnerSQLConnection::register_multi_data_source( char *tmp_str = nullptr; int64_t pos = 0; ObString sql; - if (OB_FAIL(mds_str.set(buf, buf_len, type, ls_id, register_flag))) { - LOG_WARN("set multi source data in msd_str error", K(ret), K(type), K(ls_id)); + if (OB_FAIL(mds_str.set(buf, buf_len, type, register_flag))) { + LOG_WARN("set multi source data in msd_str error", K(ret), K(type)); } else if (OB_ISNULL(tmp_str = static_cast( ob_malloc(mds_str.get_serialize_size(), "MulTxDataStr")))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1193,7 +1184,6 @@ int ObInnerSQLConnection::forward_request_(const int64_t op_type, ObInnerSqlWaitGuard guard(is_inner_session(), nullptr, inner_session_); TimeoutGuard timeout_guard(*this); // backup && restore worker/session timeout common::ObAddr resource_server_addr; // MYADDR - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); int64_t query_timeout = OB_INVALID_TIMESTAMP; int64_t trx_timeout = OB_INVALID_TIMESTAMP; ObSQLMode sql_mode = 0; @@ -1398,15 +1388,8 @@ int ObInnerSQLConnection::execute_write_inner(const ObString &sql, set_resource_svr(*sql_exec_addr); set_resource_conn_id(OB_INVALID_ID); } else { // not in trans - common::ObAddr resource_server_addr; - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); - if (OB_FAIL(nonblock_get_leader( - GCONF.cluster_id, ls_id, resource_server_addr))) { - LOG_WARN("nonblock get leader failed", K(ret), K(ls_id)); - } else { - set_resource_svr(resource_server_addr); - set_resource_conn_id(OB_INVALID_ID); - } + set_resource_svr(GCTX.self_addr()); + set_resource_conn_id(OB_INVALID_ID); } if (FAILEDx(get_session_timeout_for_rpc(query_timeout, trx_timeout))) { LOG_WARN("fail to get_session_timeout_for_rpc", K(ret), K(query_timeout), K(trx_timeout)); @@ -1547,15 +1530,8 @@ int ObInnerSQLConnection::execute_read_inner(const int64_t cluster_id, set_resource_conn_id(OB_INVALID_ID); LOG_INFO("set sql exec addr", KR(ret), K(*sql_exec_addr)); } else { - common::ObAddr resource_server_addr; - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); - if (OB_FAIL(nonblock_get_leader( - cluster_id, ls_id, resource_server_addr))) { - LOG_WARN("nonblock get leader failed", K(ret), K(ls_id), K(cluster_id)); - } else { - set_resource_svr(resource_server_addr); - set_resource_conn_id(OB_INVALID_ID); - } + set_resource_svr(GCTX.self_addr()); + set_resource_conn_id(OB_INVALID_ID); } if (FAILEDx(get_session_timeout_for_rpc(query_timeout, trx_timeout))) { LOG_WARN("fail to get_session_timeout_for_rpc", K(ret), K(query_timeout), K(trx_timeout)); @@ -1582,50 +1558,6 @@ int ObInnerSQLConnection::execute_read_inner(const int64_t cluster_id, return ret; } -int ObInnerSQLConnection::nonblock_get_leader( - const int64_t cluster_id, - const share::ObLSID ls_id, - common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - TimeoutGuard timeout_guard(*this); // backup && restore worker/session timeout - int64_t abs_timeout_us = 0; - int64_t start_time = ObTimeUtility::current_time(); - int64_t old_query_start_time = get_session().get_query_start_time(); - get_session().set_query_start_time(start_time); - - if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_NOT_INIT; - LOG_WARN("location cache is NULL", K(ret)); - } else if (OB_FAIL(set_timeout(abs_timeout_us))) { - LOG_WARN("set timeout failed", K(ret)); - } else { - int64_t tmp_abs_timeout_us = 0; - const int64_t retry_interval_us = 200 * 1000; // 200ms - do { - ret = OB_SUCCESS; - tmp_abs_timeout_us = ObTimeUtility::current_time() + GCONF.location_cache_refresh_sql_timeout; - if (THIS_WORKER.is_timeout()) { - ret = OB_TIMEOUT; - LOG_WARN("already timeout", K(ret), K(THIS_WORKER.get_timeout_ts())); - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(GCTX.location_service_->get_leader_with_retry_until_timeout( - cluster_id, ls_id, leader, tmp_abs_timeout_us, retry_interval_us))) { - LOG_WARN("get leader with retry until timeout failed", KR(ret), K(ls_id), - K(leader), K(cluster_id), K(tmp_abs_timeout_us), K(retry_interval_us)); - } else if (OB_UNLIKELY(!leader.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("leader addr invalid", K(ret), K(cluster_id), K(ls_id), K(leader)); - } else { - LOG_DEBUG("get participants", K(ls_id), K(leader), K(cluster_id)); - } - } while (is_location_service_renew_error(ret)); - } - get_session().set_query_start_time(old_query_start_time); - return ret; -} - int ObInnerSQLConnection::execute( sqlclient::ObIExecutor &executor) { diff --git a/src/observer/ob_inner_sql_connection.h b/src/observer/ob_inner_sql_connection.h index 698d16a75..bababdd24 100644 --- a/src/observer/ob_inner_sql_connection.h +++ b/src/observer/ob_inner_sql_connection.h @@ -28,7 +28,6 @@ #include "observer/mysql/ob_query_retry_ctrl.h" #include "observer/ob_inner_sql_transmit_struct.h" #include "common/mysqlclient/ob_isql_client.h" -#include "share/location_cache/ob_location_service.h" #include "storage/tablelock/ob_table_lock_common.h" //ObTableLockMode #include "sql/session/ob_sql_session_mgr.h" @@ -271,7 +270,6 @@ class ObInnerSQLConnection public:// for mds int register_multi_data_source( - const share::ObLSID ls_id, const transaction::ObTxDataSourceType type, const char *buf, const int64_t buf_len, @@ -352,11 +350,6 @@ class ObInnerSQLConnection lib::Worker::CompatMode get_compat_mode() const; - int nonblock_get_leader( - const int64_t cluster_id, - const share::ObLSID ls_id, - common::ObAddr &leader); - int execute_read_inner(const int64_t cluster_id, const ObString &sql, common::ObISQLClient::ReadResult &res, bool is_user_sql = false, const common::ObAddr *sql_exec_addr = nullptr); diff --git a/src/observer/ob_root_service_monitor.cpp b/src/observer/ob_root_service_monitor.cpp index c86ac5f9e..a74c027f6 100644 --- a/src/observer/ob_root_service_monitor.cpp +++ b/src/observer/ob_root_service_monitor.cpp @@ -21,6 +21,7 @@ #include "rootserver/ob_root_service.h" #include "logservice/ob_log_service.h" +#include "logservice/palf_handle_guard.h" namespace oceanbase { using namespace common; @@ -124,65 +125,21 @@ int ObRootServiceMonitor::monitor_root_service() } else { MOD_SCOPE { - ObRole role = FOLLOWER; - bool palf_exist = false; - int64_t proposal_id = 0; // unused - palf::PalfHandleGuard palf_handle_guard; - logservice::ObLogService *log_service = nullptr; - if (OB_ISNULL(log_service = share::g_mp->log_service())) { - ret = OB_ERR_UNEXPECTED; - FLOG_WARN("MTL ObLogService is null", KR(ret)); - } else if (OB_FAIL(log_service->check_palf_exist(SYS_LS, palf_exist))) { - FLOG_WARN("fail to check palf exist", KR(ret), K(SYS_LS)); - } else if (!palf_exist) { - // bypass - } else if (OB_FAIL(log_service->open_palf(SYS_LS, palf_handle_guard))) { - FLOG_WARN("open palf failed", KR(ret), K(SYS_LS)); - } else if (OB_FAIL(palf_handle_guard.get_role(role, proposal_id))) { - FLOG_WARN("get role failed", KR(ret)); - } - if (OB_FAIL(ret)) { - } else if (root_service_.is_stopping()) { + if (root_service_.is_stopping()) { //need exit if (OB_FAIL(root_service_.stop_service())) { FLOG_WARN("root_service stop_service failed", KR(ret)); } } else if (root_service_.is_need_stop()) { FLOG_INFO("root service is starting, stop_service need wait"); + } else if (root_service_.in_service()) { + // already started or is starting + } else if (!root_service_.can_start_service()) { + LOG_ERROR("bug here. root service can not start service"); } else { - if (is_strong_leader(role)) { - if (root_service_.in_service()) { - //already started or is starting - //nothing todo - } else if (!root_service_.can_start_service()) { - LOG_ERROR("bug here. root service can not start service"); - } else { - DEBUG_SYNC(BEFORE_START_RS); - if (OB_FAIL(try_start_root_service())) { - FLOG_WARN("fail to start root_service", KR(ret)); - } - } - } else { - // possible follower or doesn't have role yet - //DEBUG_SYNC(BEFORE_STOP_RS); - //leader does not exist. - if (!root_service_.is_start()) { - //nothing todo - } else { - if (OB_FAIL(root_service_.revoke_rs())) { - FLOG_WARN("fail to revoke rootservice", KR(ret)); - if (root_service_.is_need_stop()) { - //nothing todo - } else if (root_service_.is_stopping()) { - if (OB_FAIL(root_service_.stop_service())) { - FLOG_WARN("root_service stop_service failed", KR(ret)); - } - } else { - ret = OB_ERR_UNEXPECTED; - FLOG_WARN("inalid root service status", KR(ret)); - } - } - } + DEBUG_SYNC(BEFORE_START_RS); + if (OB_FAIL(try_start_root_service())) { + FLOG_WARN("fail to start root_service", KR(ret)); } } } else { diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index f3dc8b6fe..16ca1a0ad 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -156,7 +156,6 @@ ObServer::ObServer() tenant_timezone_mgr_(omt::ObTenantTimezoneMgr::get_instance()), schema_service_(share::schema::ObMultiVersionSchemaService::get_instance()), tablet_operator_(), - location_service_(), bandwidth_throttle_(), sys_bkgd_net_percentage_(0), ethernet_speed_(0), @@ -371,10 +370,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_ERROR("init pl failed", K(ret)); } else if (OB_FAIL(tablet_operator_.init(&meta_db_pool_))) { LOG_ERROR("tablet table operator init failed", KR(ret)); - } else if (OB_FAIL(location_service_.init( - schema_service_, - sql_proxy_))) { - LOG_ERROR("init location service failed", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(init_autoincrement_service())) { LOG_ERROR("init auto-increment service failed", KR(ret)); @@ -403,8 +398,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_ERROR("init tmp block cache failed", KR(ret)); } else if (OB_FAIL(tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache"))) { LOG_ERROR("init tmp page cache failed", KR(ret)); - } else if (OB_FAIL(init_ts_mgr())) { - LOG_ERROR("init ts mgr failed", KR(ret)); } else if (OB_FAIL(ObTenantMutilAllocatorMgr::get_instance().init())) { LOG_ERROR("init ObTenantMutilAllocatorMgr failed", KR(ret)); } else if (OB_FAIL(ObCachedCatalogSchemaMgr::get_instance().init())) { @@ -591,14 +584,6 @@ void ObServer::destroy() tmp_file::ObTmpPageCache::get_instance().destroy(); FLOG_INFO("tmp page cache destroyed"); - FLOG_INFO("begin to destroy location service"); - location_service_.destroy(); - FLOG_INFO("location service destroyed"); - - FLOG_INFO("begin to destroy ts mgr"); - OB_TS_MGR.destroy(); - FLOG_INFO("ts mgr destroyed"); - FLOG_INFO("begin to destroy net frame"); net_frame_.destroy(); FLOG_INFO("net frame destroyed"); @@ -716,12 +701,6 @@ int ObServer::start() } else { FLOG_INFO("success to start server startup task handler"); } - if (FAILEDx(OB_TS_MGR.start())) { - LOG_ERROR("fail to start ts mgr", KR(ret)); - } else { - FLOG_INFO("success to start ts mgr"); - } - // Services are registered once; start() is triggered by reload_config(). if (FAILEDx(ObMdsSchemaHelper::get_instance().init())) { @@ -803,12 +782,6 @@ int ObServer::start() } #endif - if (FAILEDx(location_service_.start())) { - LOG_ERROR("fail to start location service", KR(ret)); - } else { - FLOG_INFO("success to start location service"); - } - if (OB_SUCC(ret)) { FLOG_INFO("[OBSERVER_NOTICE] server instance start succeed"); LOG_DBA_INFO_V2(OB_SERVER_INSTANCE_START_SUCCESS, @@ -1123,10 +1096,6 @@ int ObServer::stop() OB_STORAGE_OBJECT_MGR.stop(); FLOG_INFO("storage object mgr stopped"); - FLOG_INFO("begin to stop location service"); - location_service_.stop(); - FLOG_INFO("location service stopped"); - FLOG_INFO("begin to stop timer monitor"); ObTimerMonitor::get_instance().stop(); FLOG_INFO("timer monitor stopped"); @@ -1167,10 +1136,6 @@ int ObServer::stop() FLOG_INFO("root service stopped"); } - FLOG_INFO("begin to stop ts mgr"); - OB_TS_MGR.stop(); - FLOG_INFO("ts mgr stopped"); - FLOG_INFO("begin to stop memory dump"); ObMemoryDump::get_instance().stop(); FLOG_INFO("memory dump stopped"); @@ -1886,8 +1851,7 @@ int ObServer::init_schema() int ObServer::init_autoincrement_service() { int ret = OB_SUCCESS; - if (OB_FAIL(ObAutoincrementService::get_instance().init(self_addr_, - &sql_proxy_, + if (OB_FAIL(ObAutoincrementService::get_instance().init(&sql_proxy_, &schema_service_))) { LOG_ERROR("init autoincrement_service_ fail", KR(ret)); } @@ -2082,7 +2046,6 @@ int ObServer::init_global_context() gctx_.conn_res_mgr_ = &conn_res_mgr_; gctx_.omt_ = &multi_tenant_; gctx_.vt_iter_creator_ = &vt_data_service_.get_vt_iter_factory().get_vt_iter_creator(); - gctx_.location_service_ = &location_service_; gctx_.start_time_ = start_time_; gctx_.warm_up_start_time_ = &warm_up_start_time_; gctx_.status_ = SS_INIT; @@ -2146,20 +2109,6 @@ int ObServer::init_version() return ObClusterVersion::get_instance().init(&config_); } -int ObServer::init_ts_mgr() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(OB_TS_MGR.init(self_addr_, - schema_service_, - location_service_))) { - LOG_ERROR("gts cache mgr init failed", K_(self_addr), KR(ret)); - } else { - LOG_INFO("gts cache mgr init success"); - } - - return ret; -} - int ObServer::init_px_target_mgr() { int ret = OB_SUCCESS; diff --git a/src/observer/ob_server.h b/src/observer/ob_server.h index 380f8b505..19d524bd3 100644 --- a/src/observer/ob_server.h +++ b/src/observer/ob_server.h @@ -29,7 +29,6 @@ #include "share/config/ob_config_manager.h" #include "share/tablet/ob_tablet_table_operator.h" -#include "share/location_cache/ob_location_service.h" #include "share/storage/ob_sqlite_connection_pool.h" #include "share/ob_kv_storage.h" #include "sql/ob_sql.h" @@ -64,6 +63,7 @@ #include "storage/ddl/ob_ddl_heart_beat_task.h" #include "storage/ob_disk_usage_reporter.h" +#include "storage/ob_storage_rpc.h" #include "logservice/ob_server_log_block_mgr.h" @@ -207,7 +207,6 @@ class ObServer : public share::ObIModuleProvider common::ObMySQLProxy &get_mysql_proxy() { return sql_proxy_; } int64_t get_start_time() const { return start_time_; } sql::ObConnectResourceMgr& get_conn_res_mgr() { return conn_res_mgr_; } - share::ObLocationService &get_location_service() { return location_service_; } private: int stop(); int wait_no_client(); @@ -244,7 +243,6 @@ class ObServer : public share::ObIModuleProvider int init_global_context(); int parse_role_and_restore_source(const ObServerOptions &opts); int init_version(); - int init_ts_mgr(); int init_px_target_mgr(); int init_storage(); int init_tx_data_cache(); @@ -335,7 +333,6 @@ class ObServer : public share::ObIModuleProvider // The Oceanbase partition table relating to share::ObTabletTableOperator tablet_operator_; - share::ObLocationService location_service_; // storage related common::ObInOutBandwidthThrottle bandwidth_throttle_; @@ -397,7 +394,7 @@ class ObServer : public share::ObIModuleProvider oceanbase::sql::ObTenantSQLSessionMgr * tenant_sql_session_mgr() override { return mods_tenant_sql_session_mgr_; } storage::ObTenantMetaMemMgr * tenant_meta_mem_mgr() override { return mods_tenant_meta_mem_mgr_; } storage::ObTenantFTPluginMgr * tenant_ft_plugin_mgr() override { return mods_tenant_ft_plugin_mgr_; } - ObPartTransCtxObjPool * part_trans_ctx_obj_pool() override { return mods_part_trans_ctx_obj_pool_; } + ObTxCtxObjPool * part_trans_ctx_obj_pool() override { return mods_part_trans_ctx_obj_pool_; } ObTableScanIteratorObjPool * table_scan_iterator_obj_pool() override { return mods_table_scan_iterator_obj_pool_; } common::ObTenantIOManager * tenant_io_manager() override { return mods_tenant_io_manager_; } storage::mds::ObTenantMdsService * tenant_mds_service() override { return mods_tenant_mds_service_; } @@ -442,7 +439,6 @@ class ObServer : public share::ObIModuleProvider sql::dtl::ObDTLIntermResultManager * dtl_interm_result_manager() override { return mods_dtl_interm_result_manager_; } sql::ObPlanMonitorNodeList * plan_monitor_node_list() override { return mods_plan_monitor_node_list_; } sql::ObDataAccessService * data_access_service() override { return mods_data_access_service_; } - sql::ObDASIDService * dasid_service() override { return mods_dasid_service_; } share::schema::ObTenantSchemaService * tenant_schema_service() override { return mods_tenant_schema_service_; } storage::ObTenantFreezer * tenant_freezer() override { return mods_tenant_freezer_; } storage::checkpoint::ObCheckPointService * check_point_service() override { return mods_check_point_service_; } @@ -471,7 +467,6 @@ class ObServer : public share::ObIModuleProvider table::ObTableObjectPoolMgr * table_object_pool_mgr() override { return mods_table_object_pool_mgr_; } share::ObIndexUsageInfoMgr * index_usage_info_mgr() override { return mods_index_usage_info_mgr_; } storage::ObTabletMemtableMgrPool * tablet_memtable_mgr_pool() override { return mods_tablet_memtable_mgr_pool_; } - rootserver::ObMViewMaintenanceService * m_view_maintenance_service() override { return mods_m_view_maintenance_service_; } share::ObResourceLimitCalculator * resource_limit_calculator() override { return mods_resource_limit_calculator_; } storage::ObGlobalIteratorPool * global_iterator_pool() override { return mods_global_iterator_pool_; } common::ObRbMemMgr * rb_mem_mgr() override { return mods_rb_mem_mgr_; } @@ -503,7 +498,7 @@ class ObServer : public share::ObIModuleProvider oceanbase::sql::ObTenantSQLSessionMgr * mods_tenant_sql_session_mgr_ = nullptr; storage::ObTenantMetaMemMgr * mods_tenant_meta_mem_mgr_ = nullptr; storage::ObTenantFTPluginMgr * mods_tenant_ft_plugin_mgr_ = nullptr; - ObPartTransCtxObjPool * mods_part_trans_ctx_obj_pool_ = nullptr; + ObTxCtxObjPool * mods_part_trans_ctx_obj_pool_ = nullptr; ObTableScanIteratorObjPool * mods_table_scan_iterator_obj_pool_ = nullptr; common::ObTenantIOManager * mods_tenant_io_manager_ = nullptr; storage::mds::ObTenantMdsService * mods_tenant_mds_service_ = nullptr; @@ -546,7 +541,6 @@ class ObServer : public share::ObIModuleProvider sql::dtl::ObDTLIntermResultManager * mods_dtl_interm_result_manager_ = nullptr; sql::ObPlanMonitorNodeList * mods_plan_monitor_node_list_ = nullptr; sql::ObDataAccessService * mods_data_access_service_ = nullptr; - sql::ObDASIDService * mods_dasid_service_ = nullptr; share::schema::ObTenantSchemaService * mods_tenant_schema_service_ = nullptr; storage::ObTenantFreezer * mods_tenant_freezer_ = nullptr; storage::checkpoint::ObCheckPointService * mods_check_point_service_ = nullptr; @@ -575,7 +569,6 @@ class ObServer : public share::ObIModuleProvider table::ObTableObjectPoolMgr * mods_table_object_pool_mgr_ = nullptr; share::ObIndexUsageInfoMgr * mods_index_usage_info_mgr_ = nullptr; storage::ObTabletMemtableMgrPool * mods_tablet_memtable_mgr_pool_ = nullptr; - rootserver::ObMViewMaintenanceService * mods_m_view_maintenance_service_ = nullptr; share::ObResourceLimitCalculator * mods_resource_limit_calculator_ = nullptr; storage::ObGlobalIteratorPool * mods_global_iterator_pool_ = nullptr; common::ObRbMemMgr * mods_rb_mem_mgr_ = nullptr; diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 9dc797752..116a20de4 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -50,6 +50,7 @@ // ObLogRestoreSourceMgr removed - using config parameter instead #include "share/ob_all_tenant_info.h" // ObAllTenantInfoProxy #include "share/ob_server_struct.h" // GCTX +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" // ObLSService #include "share/ob_rpc_struct.h" // ObCreateLSArg #include "share/schema/ob_multi_version_schema_service.h" // hook registration @@ -563,8 +564,7 @@ int ObService::calc_column_checksum_request(const obcall::ObCalcColumnChecksumRe LOG_WARN("ddl sim failure: calcualte column checksum rpc slow", K(tmp_ret), K(arg.task_id_)); } else if (OB_TMP_FAIL(dag_scheduler->alloc_dag(dag))) { STORAGE_LOG(WARN, "fail to alloc dag", KR(tmp_ret)); - } else if (OB_TMP_FAIL(dag->init(calc_item.ls_id_, - calc_item.tablet_id_, + } else if (OB_TMP_FAIL(dag->init(calc_item.tablet_id_, calc_item.calc_table_id_ == arg.target_table_id_, arg.target_table_id_, arg.schema_version_, @@ -626,10 +626,10 @@ int ObService::minor_freeze(const obcall::ObMinorFreezeArg &arg, } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); - } else if (arg.ls_id_.is_valid() || arg.tablet_id_.is_valid()) { - ret = handle_ls_freeze_req_(arg); + } else if (arg.tablet_id_.is_valid()) { + ret = handle_tablet_freeze_req_(arg.tablet_id_); } else { - ret = handle_tenant_freeze_req_(arg); + ret = handle_tenant_freeze_req_(); } result = ret; @@ -638,10 +638,9 @@ int ObService::minor_freeze(const obcall::ObMinorFreezeArg &arg, return ret; } -int ObService::handle_tenant_freeze_req_(const obcall::ObMinorFreezeArg &arg) +int ObService::handle_tenant_freeze_req_() { int ret = OB_SUCCESS; - UNUSED(arg); int tmp_ret = OB_SUCCESS; if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = tenant_freeze_()))) { LOG_WARN("fail to freeze tenant", K(tmp_ret)); @@ -652,17 +651,7 @@ int ObService::handle_tenant_freeze_req_(const obcall::ObMinorFreezeArg &arg) return ret; } -int ObService::handle_ls_freeze_req_(const obcall::ObMinorFreezeArg &arg) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(handle_ls_freeze_req_(arg.ls_id_, arg.tablet_id_))) { - LOG_WARN("fail to freeze tablet", K(ret), K(arg)); - } - return ret; -} - -int ObService::handle_ls_freeze_req_(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +int ObService::handle_tablet_freeze_req_(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -672,11 +661,9 @@ int ObService::handle_ls_freeze_req_(const share::ObLSID &ls_id, if (OB_ISNULL(freezer = share::g_mp->tenant_freezer())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObTenantFreezer shouldn't be null", K(ret)); - } else if (tablet_id.is_valid()) { - // tablet freeze + } else { const bool is_sync = true; - if (OB_FAIL(freezer->tablet_freeze(ls_id, - tablet_id, + if (OB_FAIL(freezer->tablet_freeze(tablet_id, is_sync, 0 /*max_retry_time_us*/, false, /*rewrite_tablet_meta*/ @@ -684,21 +671,10 @@ int ObService::handle_ls_freeze_req_(const share::ObLSID &ls_id, if (OB_EAGAIN == ret) { ret = OB_SUCCESS; } else { - LOG_ERROR("fail to freeze tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("fail to freeze tablet", K(ret), K(tablet_id)); } } else { - LOG_INFO("succeed to freeze tablet", K(ret), K(ls_id), K(tablet_id)); - } - } else { - // logstream freeze - if (OB_FAIL(freezer->ls_freeze_all_unit(ls_id, ObFreezeSourceFlag::USER_MINOR_FREEZE))) { - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } else { - LOG_ERROR("fail to freeze ls", K(ret), K(ls_id), K(tablet_id)); - } - } else { - LOG_INFO("succeed to freeze ls", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("succeed to freeze tablet", K(ret), K(tablet_id)); } } } @@ -752,7 +728,7 @@ int ObService::tablet_major_freeze(const obcall::ObTabletMajorFreezeArg &arg, } else { MOD_SCOPE { if (OB_FAIL(share::g_mp->tenant_tablet_scheduler()->user_request_schedule_medium_merge( - arg.ls_id_, arg.tablet_id_))) { + arg.tablet_id_))) { LOG_WARN("failed to try schedule tablet major freeze", K(ret), K(arg)); } } @@ -778,7 +754,6 @@ int ObService::check_modify_time_elapsed( LOG_WARN("invalid argument", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSHandle ls_handle; SCN tmp_scn; transaction::ObTransService *txs = share::g_mp->trans_service(); ObLSService *ls_service = share::g_mp->ls_service(); @@ -788,17 +763,16 @@ int ObService::check_modify_time_elapsed( for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablets_.count(); ++i) { ObTabletHandle tablet_handle; - ObLSHandle ls_handle; - const ObLSID &ls_id = arg.tablets_.at(i).ls_id_; + ObLS *ls = nullptr; const ObTabletID &tablet_id = arg.tablets_.at(i).tablet_id_; SCN snapshot_version; ObCheckTransElapsedResult single_result; int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(DDL_SIM(arg.ddl_task_id_, CHECK_MODIFY_TIME_ELAPSED_SLOW))) { LOG_WARN("ddl sim failure: check modify time elapsed slow", K(tmp_ret), K(arg.ddl_task_id_)); - } else if (OB_TMP_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(tmp_ret), K(ls_id)); - } else if (OB_TMP_FAIL(ls_handle.get_ls()->check_modify_time_elapsed(tablet_id, + } else if (OB_TMP_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(tmp_ret)); + } else if (OB_TMP_FAIL(ls->check_modify_time_elapsed(tablet_id, arg.sstable_exist_ts_, single_result.pending_tx_id_))) { if (OB_EAGAIN != tmp_ret) { @@ -844,31 +818,24 @@ int ObService::check_schema_version_elapsed( } for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablets_.count(); ++i) { ObTabletHandle tablet_handle; - ObLSHandle ls_handle; - const ObLSID &ls_id = arg.tablets_.at(i).ls_id_; + ObLS *ls = nullptr; const ObTabletID &tablet_id = arg.tablets_.at(i).tablet_id_; ObCheckTransElapsedResult single_result; int tmp_ret = OB_SUCCESS; - bool is_leader_serving = false; if (OB_TMP_FAIL(DDL_SIM(arg.ddl_task_id_, CHECK_SCHEMA_TRANS_END_SLOW))) { LOG_WARN("ddl sim failure: check schema version elapsed slow", K(tmp_ret), K(arg)); - } else if (OB_TMP_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(tmp_ret), K(i), K(ls_id)); - } else if (OB_TMP_FAIL(ls_handle.get_ls()->get_tx_svr()->check_in_leader_serving_state(is_leader_serving))) { - LOG_WARN("fail to check ls in leader serving state", K(tmp_ret), K(ls_id)); - } else if (!is_leader_serving) { - tmp_ret = OB_NOT_MASTER; // check is leader ready - LOG_WARN("ls leader is not ready, should not provide service", K(ret)); - } else if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, + } else if (OB_TMP_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(tmp_ret), K(i)); + } else if (OB_TMP_FAIL(ls->get_tablet(tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("fail to get tablet", K(tmp_ret), K(i), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get tablet", K(tmp_ret), K(i), K(tablet_id)); } else if (OB_TMP_FAIL(tablet_handle.get_obj()->check_schema_version_elapsed(arg.schema_version_, arg.need_wait_trans_end_, single_result.snapshot_, single_result.pending_tx_id_))) { - LOG_WARN("check schema version elapsed failed", K(tmp_ret), K(arg), K(ls_id), K(tablet_id)); + LOG_WARN("check schema version elapsed failed", K(tmp_ret), K(arg), K(tablet_id)); } if (OB_SUCC(ret)) { single_result.ret_code_ = tmp_ret; @@ -898,9 +865,8 @@ int ObService::check_memtable_cnt( LOG_WARN("invalid argument", K(ret), K(arg)); } else { ObMinorFreezeArg minor_freeze_arg; - minor_freeze_arg.ls_id_ = arg.ls_id_; minor_freeze_arg.tablet_id_ = arg.tablet_id_; - if (OB_FAIL(handle_ls_freeze_req_(minor_freeze_arg))) { + if (OB_FAIL(handle_tablet_freeze_req_(minor_freeze_arg.tablet_id_))) { LOG_WARN("failed to handle tablet freeze", K(ret)); } else { MOD_SCOPE { @@ -908,23 +874,13 @@ int ObService::check_memtable_cnt( ObTabletID tablet_id = arg.tablet_id_; const int64_t expire_renew_time = INT64_MAX; bool is_cache_hit = false; - ObLSID ls_id = arg.ls_id_; - ObLSService *ls_srv = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - ObLSTabletService *ls_tablet_service = nullptr; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObArray memtable_handles; - if (OB_FAIL(ls_srv->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("fail to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet service should not be null", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_tablet_service->get_tablet(tablet_id, + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get ls", K(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet(tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, ObMDSGetTabletMode::READ_ALL_COMMITED))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); @@ -939,8 +895,7 @@ int ObService::check_memtable_cnt( ObTabletFreezeLog freeze_log; freeze_log.tablet_id_ = tablet_id; if (OB_FAIL(storage::ObDDLRedoLogWriter:: - write_auto_split_log(ls_id, - ObDDLClogType::DDL_TABLET_FREEZE_LOG, + write_auto_split_log(ObDDLClogType::DDL_TABLET_FREEZE_LOG, logservice::ObReplayBarrierType::STRICT_BARRIER, freeze_log, unused_scn))) { LOG_WARN("write tablet freeze log failed", K(ret), K(freeze_log)); @@ -977,7 +932,7 @@ int ObService::prepare_tablet_split_task_ranges( } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(arg)); - } else if (OB_FAIL(ObTabletSplitUtil::split_task_ranges(result.rowkey_allocator_, arg.ddl_type_, arg.ls_id_, + } else if (OB_FAIL(ObTabletSplitUtil::split_task_ranges(result.rowkey_allocator_, arg.ddl_type_, arg.tablet_id_, arg.user_parallelism_, arg.schema_tablet_size_, result.parallel_datum_rowkey_list_))) { LOG_WARN("split task ranges failed", K(ret)); } @@ -1002,7 +957,7 @@ int ObService::check_ddl_tablet_merge_status( for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); ++i) { const common::ObTabletID &tablet_id = arg.tablet_ids_.at(i); ObTabletHandle tablet_handle; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObDDLKvMgrHandle ddl_kv_mgr_handle; ObLSService *ls_service = nullptr; bool status = false; @@ -1013,9 +968,9 @@ int ObService::check_ddl_tablet_merge_status( } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(arg)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle))) { + } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { LOG_WARN("get tablet failed", K(ret)); } // check and update major status @@ -1220,57 +1175,6 @@ int ObService::bootstrap() return ret; } -int ObService::init_tenant_merge_info_() -{ - int ret = OB_SUCCESS; - // Same logic as ObTenantDDLService::insert_tenant_merge_info_ - HEAP_VARS_2((ObGlobalMergeInfo, global_info), - (ObZoneMergeInfo, zone_merge_info)) { - - - // Insert global merge info - if (OB_FAIL(ObGlobalMergeTableOperator::insert_global_merge_info(*GCTX.sql_proxy_, global_info))) { - LOG_WARN("fail to insert global merge info", KR(ret), K(global_info)); - } else { - // Insert zone merge info (one zone for standby) - ObSEArray merge_info_array; - if (OB_FAIL(merge_info_array.push_back(zone_merge_info))) { - LOG_WARN("fail to push_back zone merge info", KR(ret)); - } else if (OB_FAIL(ObZoneMergeTableOperator::insert_zone_merge_infos(*GCTX.sql_proxy_, merge_info_array))) { - LOG_WARN("fail to insert zone merge infos", KR(ret)); - } else { - LOG_INFO("succ to init tenant merge info for standby bootstrap", K(global_info), K(zone_merge_info)); - } - } - } - return ret; -} - -int ObService::create_sys_ls() -{ - int ret = OB_SUCCESS; - LOG_INFO("create empty LS for standby"); - storage::ObLSService *ls_service = nullptr; - logservice::ObLogService *log_service = nullptr; - if (!GCTX.is_standby_cluster()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("create_sys_tenant_ls_if_not_exists called but not standby cluster", KR(ret)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls_service is null"); - } else if (OB_ISNULL(log_service = share::g_mp->log_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log_service is null"); - } else { - if (OB_FAIL(ls_service->create_ls_for_ha())) { - LOG_WARN("create ls failed", KR(ret)); - } - } - - return ret; -} - - int ObService::set_server_id_(const int64_t server_id) { int ret = OB_SUCCESS; @@ -1676,23 +1580,9 @@ int ObService::fetch_split_tablet_info(const ObFetchSplitTabletInfoArg &arg, LOG_WARN("invalid arg", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - ObRole role = INVALID_ROLE; - if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls_service or log_service", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { - ret = OB_NOT_MASTER; - LOG_WARN("ls not leader", K(ret), K(arg.ls_id_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.tablet_ids_.at(i); @@ -1702,7 +1592,7 @@ int ObService::fetch_split_tablet_info(const ObFetchSplitTabletInfoArg &arg, LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_tablet_status( share::SCN::max_scn(), user_data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US))) { - LOG_WARN("failed to get tablet status", K(ret), K(arg.ls_id_), K(tablet_id)); + LOG_WARN("failed to get tablet status", K(ret), K(tablet_id)); } else if (OB_FAIL(res.create_commit_versions_.push_back(user_data.create_commit_version_))) { LOG_WARN("failed to push back", K(ret)); } else if (OB_FAIL(res.tablet_sizes_.push_back(tablet_handle.get_obj()->get_tablet_meta().space_usage_.all_sstable_data_required_size_))) { @@ -1901,7 +1791,6 @@ int ObService::inner_fill_tablet_info_( share::ObTabletReplicaChecksumItem &tablet_checksum, const bool need_checksum) { - ObLSHandle ls_handle; ObTabletHandle tablet_handle; int ret = OB_SUCCESS; ObTablet *tablet = nullptr; @@ -1933,13 +1822,12 @@ int ObService::inner_fill_tablet_info_( } else if (OB_FAIL(tablet->get_tablet_report_info( gctx_.self_addr(), tablet_replica, tablet_checksum, need_checksum))) { LOG_WARN("fail to get tablet report info from tablet", KR(ret), - "ls_id", ls->get_ls_id(), K(tablet_id)); + K(tablet_id)); } return ret; } -int ObService::fill_tablet_report_info(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObService::fill_tablet_report_info(const ObTabletID &tablet_id, ObTabletReplica &tablet_replica, share::ObTabletReplicaChecksumItem &tablet_checksum, const bool need_checksum) @@ -1948,35 +1836,24 @@ int ObService::fill_tablet_report_info(const ObLSID &ls_id, if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("service not inited", KR(ret)); - } else if (!tablet_id.is_valid() || !ls_id.is_valid() || false) { + } else if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_id), K(ls_id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_id)); } else { MOD_SCOPE { - ObSharedGuard ls_iter; ObTabletHandle tablet_handle; - ObLSHandle ls_handle; - storage::ObLS *ls = nullptr; - ObLSService* ls_svr = nullptr; - if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLSService is null", KR(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("fail to get log_stream's ls_handle", KR(ret), K(ls_id)); - } else { - LOG_TRACE("log stream not exist in this tenant", KR(ret), K(ls_id)); - } - } else if (FALSE_IT(ls = ls_handle.get_ls())) { + ObLS *ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get log stream", KR(ret)); } else if (OB_FAIL(inner_fill_tablet_info_(tablet_id, ls, tablet_replica, tablet_checksum, need_checksum))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("fail to inner fill tenant's tablet replica", KR(ret), K(tablet_id), K(ls), K(tablet_replica), K(tablet_checksum), K(need_checksum)); + LOG_WARN("fail to inner fill tenant's tablet replica", KR(ret), K(tablet_id), K(tablet_replica), K(tablet_checksum), K(need_checksum)); } else { - LOG_TRACE("tablet not exist in this log stream", KR(ret), K(tablet_id), K(ls), K(tablet_replica), K(tablet_checksum), K(need_checksum)); + LOG_TRACE("tablet not exist in this log stream", KR(ret), K(tablet_id), K(tablet_replica), K(tablet_checksum), K(need_checksum)); } } } @@ -1998,163 +1875,6 @@ int ObService::estimate_tablet_block_count(const obcall::ObEstBlockArg &arg, return ret; } -ERRSIM_POINT_DEF(ERRSIM_GET_LS_SYNC_SCN_ERROR); -ERRSIM_POINT_DEF(ERRSIM_GET_SYS_LS_SYNC_SCN_ERROR); -int ObService::force_set_ls_as_single_replica( - const ObForceSetLSAsSingleReplicaArg &arg) -{ - int ret = OB_SUCCESS; - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - ObLSService *ls_svr = nullptr; - LOG_INFO("force_set_ls_as_single_replica", K(arg)); - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("arg is invaild", KR(ret), K(arg)); - } else if (false && OB_FAIL(guard.switch_to())) { - LOG_WARN("switch tenant failed", KR(ret), K(arg)); - } - - if (OB_SUCC(ret)) { - ls_svr = share::g_mp->ls_service(); - logservice::ObLogService *log_ls_svr = share::g_mp->log_service(); - ObLS *ls = nullptr; - ObLSHandle handle; - logservice::ObLogHandler *log_handler = NULL; - ObLSID ls_id = arg.get_ls_id(); - if (OB_ISNULL(ls_svr) || OB_ISNULL(log_ls_svr)) { - ret = OB_ERR_UNEXPECTED; - COMMON_LOG(ERROR, "should not be null", KR(ret), KP(ls_svr), KP(log_ls_svr)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::OBSERVER_MOD))) { - COMMON_LOG(WARN, "get ls failed", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - COMMON_LOG(ERROR, "ls should not be null", KR(ret), K(ls_id)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log_handler is null", KR(ret), K(ls_id), KP(ls)); - } else if (OB_FAIL(log_handler->force_set_as_single_replica())) { - LOG_WARN("failed to force_set_as_single_replica", KR(ret), K(ls_id), KPC(ls)); - } - } - LOG_INFO("finish force_set_ls_as_single_replica", KR(ret), K(arg)); - return ret; -} - -int ObService::force_set_server_list(const obcall::ObForceSetServerListArg &arg, obcall::ObForceSetServerListResult &result) -{ - int ret = OB_SUCCESS; - LOG_INFO("force_set_server_list", K(arg)); - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ob_service is not inited", KR(ret)); - } else if (OB_ISNULL(GCTX.omt_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("GCTX.omt_ is null", KR(ret), K(arg)); - } else { - const int64_t new_membership_timestamp = ObTimeUtility::current_time(); - bool all_succeed = true; - { - COMMON_LOG(INFO, "start to excute force_set_server_list"); - MOD_SCOPE { - ObLSService *ls_svr = share::g_mp->ls_service(); - logservice::ObLogService *log_service = share::g_mp->log_service(); - ObSharedGuard ls_iter_guard; - ObForceSetServerListResult::ResultInfo result_info; - - if (OB_ISNULL(ls_svr) || OB_ISNULL(log_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ptr is null", KR(ret), KP(ls_svr), KP(log_service)); - } else if (OB_FAIL(ls_svr->get_ls_iter(ls_iter_guard, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("fail to get ls iter guard", KR(ret)); - } - - if (OB_FAIL(ret)) { - all_succeed = false; - COMMON_LOG(WARN, "force_set_server_list with current tenant failed", KR(ret)); - } - - while (OB_SUCC(ret)) { - COMMON_LOG(INFO, "start to iterate every log stream of tenant"); - ObLS *ls = nullptr; - logservice::ObLogHandler *log_handler = NULL; - if (OB_FAIL(ls_iter_guard->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next ls", KR(ret)); - } - } else if (OB_ISNULL(ls)){ - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is nullptr", KR(ret)); - } else { - common::ObMemberList old_member_list; - int64_t old_replica_num = 0; - ObLSID ls_id = ls->get_ls_id(); - - if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - COMMON_LOG(ERROR, "log_handler is null", KR(ret), K(ls_id), KP(ls)); - } else if (OB_FAIL(log_handler->get_paxos_member_list(old_member_list, old_replica_num))) { - LOG_WARN("get old_member_list failed", KR(ret), K(ls_id), KP(ls)); - } else { - common::ObMemberList new_member_list; - // new_member_list is the intersection of args.server_list_ and old_member_list - for (int64_t j = 0; OB_SUCC(ret) && j < arg.server_list_.size(); ++j) { - const common::ObAddr &server = arg.server_list_[j]; - if (!old_member_list.contains(server)) { - } else if (OB_FAIL(new_member_list.add_member(ObMember(server, new_membership_timestamp)))){ - LOG_WARN("new_member_list add_member failed", K(ret), K(server)); - } - } - - if (OB_FAIL(ret)) { - } else if (arg.replica_num_ != new_member_list.get_member_number()) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("new_member_list number does not equal to arg.replica_num", K(ret), K(arg), K(new_member_list.get_member_number())); - } else if (OB_FAIL(log_handler->force_set_member_list(new_member_list, arg.replica_num_))) { - LOG_WARN("force_set_server_list failed", KR(ret), K(arg), K(ls_id)); - } else { - COMMON_LOG(INFO, "execute force_set_server_list successfully with " - "current tenant and ls", K(arg), K(ls_id)); - } - } - - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(result_info.add_ls_info(ls_id, ret))) { - LOG_WARN("add_result_info failed", K(tmp_ret), K(ls_id), "actual ret", ret); - } - - if (OB_FAIL(ret)) { - COMMON_LOG(WARN, "failed to execute force_set_server_list with " - "current tenant and ls", KR(ret), K(arg), K(ls_id)); - ret = OB_SUCCESS; // ignore failed return code, keep executing next ls - } - } // end if - } // end while - - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(result.result_list_.push_back(result_info))) { - LOG_WARN("result_list_ push_back failed", K(tmp_ret), K(result_info)); - } - if (0 != result_info.failed_ls_info_.size()) { - all_succeed = false; - } - ret = OB_SUCCESS; // ignore failed return code, keep executing next tenant - } // MTL_SWITCH end - } // for end - - if (all_succeed) { - result.ret_ = OB_SUCCESS; - } else { - result.ret_ = OB_PARTIAL_FAILED; - } - } - return ret; -} - int ObService::init_tenant_config( const obcall::ObInitTenantConfigArg &arg, obcall::ObInitTenantConfigRes &result) diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index 4603a4bb3..a55071ae4 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -87,15 +87,12 @@ class ObService //fill_tablet_replica: to build a tablet replica locally // @params[in] tenant: tablet belongs to which tenant - // @params[in] ls_id: tablet belongs to which log stream // @params[in] tablet_id: the tablet to build // @params[out] tablet_replica: infos about this tablet replica // @params[out] tablet_checksum: infos about this tablet data/column checksum // @params[in] need_checksum: whether to fill tablet_checksum - // ATTENTION: If ls not exist, then OB_LS_NOT_EXIST - // If tablet not exist on that ls, then OB_TABLET_NOT_EXIST - int fill_tablet_report_info(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, + // ATTENTION: If tablet does not exist, returns OB_TABLET_NOT_EXIST. + int fill_tablet_report_info(const ObTabletID &tablet_id, share::ObTabletReplica &tablet_replica, share::ObTabletReplicaChecksumItem &tablet_checksum, const bool need_checksum = true); @@ -113,7 +110,6 @@ class ObService int calc_column_checksum_request(const obcall::ObCalcColumnChecksumRequestArg &arg, obcall::ObCalcColumnChecksumRequestRes &res); int build_split_tablet_data_start_request(const obcall::ObTabletSplitStartArg &arg, obcall::ObTabletSplitStartResult &res); int build_split_tablet_data_finish_request(const obcall::ObTabletSplitFinishArg &arg, obcall::ObTabletSplitFinishResult &res); - int freeze_split_src_tablet(const obcall::ObFreezeSplitSrcTabletArg &arg, obcall::ObFreezeSplitSrcTabletRes &res, const int64_t abs_timeout_us); int fetch_split_tablet_info(const obcall::ObFetchSplitTabletInfoArg &arg, obcall::ObFetchSplitTabletInfoRes &res, const int64_t abs_timeout_us); int build_ddl_single_replica_request(const obcall::ObDDLBuildSingleReplicaRequestArg &arg); int build_ddl_single_replica_request(const obcall::ObDDLBuildSingleReplicaRequestArg &arg, obcall::ObDDLBuildSingleReplicaRequestResult &res); @@ -126,8 +122,6 @@ class ObService const obcall::ObBroadcastConsensusVersionArg &arg, obcall::ObBroadcastConsensusVersionRes &result); //////////////////////////////////////////////////////////////// - int force_set_ls_as_single_replica(const obcall::ObForceSetLSAsSingleReplicaArg &arg); - int force_set_server_list(const obcall::ObForceSetServerListArg &arg, obcall::ObForceSetServerListResult &result); int estimate_partition_rows(const obcall::ObEstPartArg &arg, obcall::ObEstPartRes &res) const; int estimate_tablet_block_count(const obcall::ObEstBlockArg &arg, @@ -182,9 +176,8 @@ class ObService //////////////////////////////////////////////////////////////// int load_leader_cluster_login_info(); - // ObDropReplicaP @RS::admin to drop replica + // ObCallSetDebugSyncActionP @RS::admin to set debug sync action int set_ds_action(const obcall::ObDebugSyncActionArg &arg); - int report_replica(const obcall::ObReportSingleReplicaArg &arg); // ObSyncPartitionTableP @RS empty_server_checker int sync_partition_table(const obcall::Int64 &arg); // ObCallSetTPP @RS::admin to set tracepoint @@ -206,8 +199,6 @@ class ObService private: int bootstrap(); - int create_sys_ls(); - int init_tenant_merge_info_(); int inner_fill_tablet_info_( const ObTabletID &tablet_id, storage::ObLS *ls, @@ -216,10 +207,9 @@ class ObService const bool need_checksum); int set_server_id_(const int64_t server_id); - int handle_tenant_freeze_req_(const obcall::ObMinorFreezeArg &arg); - int handle_ls_freeze_req_(const obcall::ObMinorFreezeArg &arg); + int handle_tenant_freeze_req_(); + int handle_tablet_freeze_req_(const common::ObTabletID &tablet_id); int tenant_freeze_(); - int handle_ls_freeze_req_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); private: bool inited_; volatile bool stopped_; diff --git a/src/observer/ob_srv_xlator.cpp b/src/observer/ob_srv_xlator.cpp index 1948b735d..5177048d8 100644 --- a/src/observer/ob_srv_xlator.cpp +++ b/src/observer/ob_srv_xlator.cpp @@ -269,7 +269,6 @@ ObReqProcessor *ObSrvXlator::get_processor(ObRequest &req) } else if (ObRequest::OB_MYSQL == req.get_type()) { ret = mysql_xlator_.translate(req, processor); } else if (ObRequest::OB_TASK == req.get_type() || - ObRequest::OB_TS_TASK == req.get_type() || ObRequest::OB_SQL_TASK == req.get_type() || ObRequest::OB_DAS_PARALLEL_TASK == req.get_type()) { processor = &static_cast(req).get_processor(); @@ -329,11 +328,6 @@ int ObSrvXlator::release(ObReqProcessor *processor) //Deal with sqltask memory release ob_delete(req); req = NULL; - } else if (ObRequest::OB_TS_TASK == req_type) { - //Deal with the memory release of the transaction task - ObTsResponseTaskFactory::free(static_cast(req)); - //op_reclaim_free(req); - req = NULL; } else if (ObRequest::OB_SQL_TASK == req_type) { ObSqlTaskFactory::get_instance().free(static_cast(req)); req = NULL; diff --git a/src/observer/ob_sys_tenant_load_sys_package_service.cpp b/src/observer/ob_sys_tenant_load_sys_package_service.cpp index 4b7315ab1..516718f65 100644 --- a/src/observer/ob_sys_tenant_load_sys_package_service.cpp +++ b/src/observer/ob_sys_tenant_load_sys_package_service.cpp @@ -113,7 +113,7 @@ void ObSysTenantLoadSysPackageService::destroy() FLOG_INFO("finish to destroy sys package load service"); } -int ObSysTenantLoadSysPackageService::switch_to_leader() +int ObSysTenantLoadSysPackageService::activate() { int ret = OB_SUCCESS; if (OB_UNLIKELY(!inited_)) { @@ -125,24 +125,11 @@ int ObSysTenantLoadSysPackageService::switch_to_leader() return ret; } -void ObSysTenantLoadSysPackageService::switch_to_follower_forcedly() +void ObSysTenantLoadSysPackageService::deactivate() { - int ret = switch_to_follower_gracefully(); - if (OB_FAIL(switch_to_follower_gracefully())) { - LOG_WARN("failed to switch to follower gracefully", KR(ret)); - } -} - -int ObSysTenantLoadSysPackageService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("service not inited", KR(ret), K_(inited)); - } else { + if (inited_) { stop(); } - return ret; } } // namespace rootserver diff --git a/src/observer/ob_sys_tenant_load_sys_package_service.h b/src/observer/ob_sys_tenant_load_sys_package_service.h index f160d05bf..20a3ebec3 100644 --- a/src/observer/ob_sys_tenant_load_sys_package_service.h +++ b/src/observer/ob_sys_tenant_load_sys_package_service.h @@ -26,7 +26,7 @@ namespace oceanbase { namespace rootserver { -class ObSysTenantLoadSysPackageService : public logservice::ObIRoleChangeSubHandler, +class ObSysTenantLoadSysPackageService : public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler, public logservice::ObIReplaySubHandler { @@ -42,11 +42,9 @@ class ObSysTenantLoadSysPackageService : public logservice::ObIRoleChangeSubHand int wait(); void destroy(); - // for ObIRoleChangeSubHandler - virtual int switch_to_leader() override; - virtual void switch_to_follower_forcedly() override; - virtual int switch_to_follower_gracefully() override; - virtual int resume_leader() override { return switch_to_leader(); } + // for ObILocalLogHandler + int activate() override; + void deactivate() override; // for checkpoint/replay virtual share::SCN get_rec_scn() override { return share::SCN::max_scn(); } diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 20dcbc496..233935aa8 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -28,7 +28,6 @@ #include "share/schema/ob_tenant_schema_service.h" #include "observer/mysql/obsm_conn_callback.h" #include "sql/dtl/ob_dtl_fc_server.h" -#include "sql/das/ob_das_id_service.h" #include "storage/allocator/ob_shared_memory_allocator_mgr.h" // ObSharedMemAllocMgr #include "share/ob_global_autoinc_service.h" #include "ob_tenant_mtl_helper.h" @@ -38,7 +37,7 @@ #include "storage/tx/ob_timestamp_access.h" #include "storage/tx/ob_trans_id_service.h" #include "storage/tx/ob_unique_id_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/tx_storage/ob_checkpoint_service.h" #include "storage/tx_storage/ob_tenant_memory_printer.h" @@ -70,7 +69,6 @@ #include "observer/ob_server_event_history_table_operator.h" #include "share/index_usage/ob_index_usage_info_mgr.h" #include "sql/optimizer/stat/ob_opt_stat_monitor_manager.h" -#include "rootserver/mview/ob_mview_maintenance_service.h" #include "observer/vector_index/ob_plugin_vector_index_service.h" #include "observer/change_stream/ob_change_stream_mgr.h" #include "share/roaringbitmap/ob_rb_memory_mgr.h" @@ -1951,7 +1949,7 @@ int ObServer::obs_construct_modules() if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_shared_timer_))) { SERVER_LOG(WARN, "mods_shared_timer_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObTenantSQLSessionMgr::mtl_new(mods_tenant_sql_session_mgr_))) { SERVER_LOG(WARN, "mods_tenant_sql_session_mgr_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObTenantMetaMemMgr::mtl_new(mods_tenant_meta_mem_mgr_))) { SERVER_LOG(WARN, "mods_tenant_meta_mem_mgr_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(server_obj_pool_mtl_new(mods_part_trans_ctx_obj_pool_))) { SERVER_LOG(WARN, "mods_part_trans_ctx_obj_pool_ fail", KR(ret)); } + if (OB_SUCC(ret) && OB_FAIL(server_obj_pool_mtl_new(mods_part_trans_ctx_obj_pool_))) { SERVER_LOG(WARN, "mods_part_trans_ctx_obj_pool_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(server_obj_pool_mtl_new(mods_table_scan_iterator_obj_pool_))) { SERVER_LOG(WARN, "mods_table_scan_iterator_obj_pool_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObTenantIOManager::mtl_new(mods_tenant_io_manager_))) { SERVER_LOG(WARN, "mods_tenant_io_manager_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_tenant_mds_service_))) { SERVER_LOG(WARN, "mods_tenant_mds_service_ fail", KR(ret)); } @@ -1991,7 +1989,6 @@ int ObServer::obs_construct_modules() if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_dtl_interm_result_manager_))) { SERVER_LOG(WARN, "mods_dtl_interm_result_manager_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_plan_monitor_node_list_))) { SERVER_LOG(WARN, "mods_plan_monitor_node_list_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_data_access_service_))) { SERVER_LOG(WARN, "mods_data_access_service_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_dasid_service_))) { SERVER_LOG(WARN, "mods_dasid_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_tenant_schema_service_))) { SERVER_LOG(WARN, "mods_tenant_schema_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_tenant_freezer_))) { SERVER_LOG(WARN, "mods_tenant_freezer_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_check_point_service_))) { SERVER_LOG(WARN, "mods_check_point_service_ fail", KR(ret)); } @@ -2014,7 +2011,6 @@ int ObServer::obs_construct_modules() if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_tenant_srs_))) { SERVER_LOG(WARN, "mods_tenant_srs_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_index_usage_info_mgr_))) { SERVER_LOG(WARN, "mods_index_usage_info_mgr_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_tablet_memtable_mgr_pool_))) { SERVER_LOG(WARN, "mods_tablet_memtable_mgr_pool_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_m_view_maintenance_service_))) { SERVER_LOG(WARN, "mods_m_view_maintenance_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_resource_limit_calculator_))) { SERVER_LOG(WARN, "mods_resource_limit_calculator_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_global_iterator_pool_))) { SERVER_LOG(WARN, "mods_global_iterator_pool_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_new_default(mods_rb_mem_mgr_))) { SERVER_LOG(WARN, "mods_rb_mem_mgr_ fail", KR(ret)); } @@ -2075,7 +2071,6 @@ int ObServer::obs_init_modules() if (OB_SUCC(ret) && OB_FAIL(ObDTLIntermResultManager::mtl_init(mods_dtl_interm_result_manager_))) { SERVER_LOG(WARN, "mods_dtl_interm_result_manager_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObPlanMonitorNodeList::mtl_init(mods_plan_monitor_node_list_))) { SERVER_LOG(WARN, "mods_plan_monitor_node_list_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObDataAccessService::mtl_init(mods_data_access_service_))) { SERVER_LOG(WARN, "mods_data_access_service_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(ObDASIDService::mtl_init(mods_dasid_service_))) { SERVER_LOG(WARN, "mods_dasid_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObTenantSchemaService::mtl_init(mods_tenant_schema_service_))) { SERVER_LOG(WARN, "mods_tenant_schema_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObTenantFreezer::mtl_init(mods_tenant_freezer_))) { SERVER_LOG(WARN, "mods_tenant_freezer_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObCheckPointService::mtl_init(mods_check_point_service_))) { SERVER_LOG(WARN, "mods_check_point_service_ fail", KR(ret)); } @@ -2098,7 +2093,6 @@ int ObServer::obs_init_modules() if (OB_SUCC(ret) && OB_FAIL(omt::ObTenantSrs::mtl_init(mods_tenant_srs_))) { SERVER_LOG(WARN, "mods_tenant_srs_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObIndexUsageInfoMgr::mtl_init(mods_index_usage_info_mgr_))) { SERVER_LOG(WARN, "mods_index_usage_info_mgr_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(storage::ObTabletMemtableMgrPool::mtl_init(mods_tablet_memtable_mgr_pool_))) { SERVER_LOG(WARN, "mods_tablet_memtable_mgr_pool_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(rootserver::ObMViewMaintenanceService::mtl_init(mods_m_view_maintenance_service_))) { SERVER_LOG(WARN, "mods_m_view_maintenance_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObResourceLimitCalculator::mtl_init(mods_resource_limit_calculator_))) { SERVER_LOG(WARN, "mods_resource_limit_calculator_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(ObGlobalIteratorPool::mtl_init(mods_global_iterator_pool_))) { SERVER_LOG(WARN, "mods_global_iterator_pool_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(common::ObRbMemMgr::mtl_init(mods_rb_mem_mgr_))) { SERVER_LOG(WARN, "mods_rb_mem_mgr_ fail", KR(ret)); } @@ -2150,7 +2144,6 @@ int ObServer::obs_start_modules() if (OB_SUCC(ret) && OB_FAIL(ObOptStatMonitorManager::mtl_start(mods_opt_stat_monitor_manager_))) { SERVER_LOG(WARN, "mods_opt_stat_monitor_manager_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_tenant_srs_))) { SERVER_LOG(WARN, "mods_tenant_srs_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_index_usage_info_mgr_))) { SERVER_LOG(WARN, "mods_index_usage_info_mgr_ fail", KR(ret)); } - if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_m_view_maintenance_service_))) { SERVER_LOG(WARN, "mods_m_view_maintenance_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_rb_mem_mgr_))) { SERVER_LOG(WARN, "mods_rb_mem_mgr_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_plugin_vector_index_service_))) { SERVER_LOG(WARN, "mods_plugin_vector_index_service_ fail", KR(ret)); } if (OB_SUCC(ret) && OB_FAIL(mtl_start_default(mods_tenant_ai_service_))) { SERVER_LOG(WARN, "mods_tenant_ai_service_ fail", KR(ret)); } @@ -2165,7 +2158,6 @@ void ObServer::obs_stop_modules() rootserver::ObDDLScheduler::mtl_stop(mods_ddl_scheduler_); mtl_stop_default(mods_plugin_vector_index_service_); mtl_stop_default(mods_rb_mem_mgr_); - mtl_stop_default(mods_m_view_maintenance_service_); mtl_stop_default(mods_index_usage_info_mgr_); mtl_stop_default(mods_tenant_srs_); ObOptStatMonitorManager::mtl_stop(mods_opt_stat_monitor_manager_); @@ -2215,7 +2207,6 @@ void ObServer::obs_wait_modules() rootserver::ObDDLScheduler::mtl_wait(mods_ddl_scheduler_); mtl_wait_default(mods_plugin_vector_index_service_); mtl_wait_default(mods_rb_mem_mgr_); - mtl_wait_default(mods_m_view_maintenance_service_); mtl_wait_default(mods_index_usage_info_mgr_); mtl_wait_default(mods_tenant_srs_); ObOptStatMonitorManager::mtl_wait(mods_opt_stat_monitor_manager_); @@ -2244,7 +2235,6 @@ void ObServer::obs_wait_modules() mtl_wait_default(mods_tenant_tablet_stat_mgr_); mtl_wait_default(mods_tenant_tmp_file_manager_); mtl_wait_default(mods_tenant_storage_meta_service_); - mtl_wait_default(mods_ls_service_); mtl_wait_default(mods_log_service_); mtl_wait_default(mods_trans_service_); mtl_wait_default(mods_shared_mem_alloc_mgr_); @@ -2253,6 +2243,8 @@ void ObServer::obs_wait_modules() mtl_wait_default(mods_tenant_meta_mem_mgr_); ObTenantSQLSessionMgr::mtl_wait(mods_tenant_sql_session_mgr_); ObSharedTimer::mtl_wait(mods_shared_timer_); + // ObLS pointers are non-owning, so every tenant worker must drain before LS is released. + mtl_wait_default(mods_ls_service_); } void ObServer::obs_destroy_modules() @@ -2269,7 +2261,6 @@ void ObServer::obs_destroy_modules() mtl_destroy_default(mods_rb_mem_mgr_); ObGlobalIteratorPool::mtl_destroy(mods_global_iterator_pool_); mtl_destroy_default(mods_resource_limit_calculator_); - mtl_destroy_default(mods_m_view_maintenance_service_); mtl_destroy_default(mods_tablet_memtable_mgr_pool_); mtl_destroy_default(mods_index_usage_info_mgr_); mtl_destroy_default(mods_tenant_srs_); @@ -2292,7 +2283,6 @@ void ObServer::obs_destroy_modules() mtl_destroy_default(mods_check_point_service_); mtl_destroy_default(mods_tenant_freezer_); mtl_destroy_default(mods_tenant_schema_service_); - mtl_destroy_default(mods_dasid_service_); ObDataAccessService::mtl_destroy(mods_data_access_service_); ObPlanMonitorNodeList::mtl_destroy(mods_plan_monitor_node_list_); ObDTLIntermResultManager::mtl_destroy(mods_dtl_interm_result_manager_); @@ -2332,7 +2322,7 @@ void ObServer::obs_destroy_modules() mtl_destroy_default(mods_tenant_mds_service_); ObTenantIOManager::mtl_destroy(mods_tenant_io_manager_); server_obj_pool_mtl_destroy(mods_table_scan_iterator_obj_pool_); - server_obj_pool_mtl_destroy(mods_part_trans_ctx_obj_pool_); + server_obj_pool_mtl_destroy(mods_part_trans_ctx_obj_pool_); mtl_destroy_default(mods_tenant_meta_mem_mgr_); ObTenantSQLSessionMgr::mtl_destroy(mods_tenant_sql_session_mgr_); mtl_destroy_default(mods_shared_timer_); diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 04975b158..63a4fea25 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -18,7 +18,6 @@ #include "ob_tenant.h" #include "observer/ob_server.h" // T3d #include "share/rc/ob_module_provider.h" -#include "share/ob_server_struct.h" #ifdef _WIN32 #include @@ -429,14 +428,7 @@ int ObTenant::construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantMo mtl_init_ctx_->palf_options_.disk_options_.log_disk_utilization_limit_threshold_ = 95; mtl_init_ctx_->palf_options_.disk_options_.log_disk_throttling_percentage_ = 100; mtl_init_ctx_->palf_options_.disk_options_.log_disk_throttling_maximum_duration_ = 2LL * 60 * 60 * 1000 * 1000;//2h - mtl_init_ctx_->palf_options_.disk_options_.log_writer_parallelism_ = 3; - mtl_init_ctx_->palf_options_.enable_fetch_log_engine_ = !GCTX.is_embedded_mode(); - if (OB_UNLIKELY(!true)) { - ret = false ? OB_SUCCESS : OB_ENTRY_NOT_EXIST; - } else { - mtl_init_ctx_->palf_options_.disk_options_.log_writer_parallelism_ = GCONF._log_writer_parallelism; - mtl_init_ctx_->palf_options_.enable_log_cache_ = GCONF._enable_log_cache; - } + mtl_init_ctx_->palf_options_.enable_log_cache_ = GCONF._enable_log_cache; LOG_INFO("construct_mtl_init_ctx success", "palf_options", mtl_init_ctx_->palf_options_.disk_options_ ); } @@ -767,7 +759,7 @@ int ObTenant::recv_request(ObRequest &req) break; } case ObRequest::OB_TASK: - case ObRequest::OB_TS_TASK: { + { ATOMIC_INC(&recv_task_cnt_); if (OB_FAIL(req_queue_.push(&req, RQ_HIGH, true))) { LOG_WARN("push request to queue fail", K(ret), K(this)); diff --git a/src/observer/report/ob_tablet_table_updater.cpp b/src/observer/report/ob_tablet_table_updater.cpp index e652e30a1..c69c0daa8 100644 --- a/src/observer/report/ob_tablet_table_updater.cpp +++ b/src/observer/report/ob_tablet_table_updater.cpp @@ -78,20 +78,16 @@ ObTabletTableUpdateTask::~ObTabletTableUpdateTask() } int ObTabletTableUpdateTask::init( - const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t add_timestamp, const bool need_diagnose/*false*/) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() - || !tablet_id.is_valid() + if (OB_UNLIKELY(!tablet_id.is_valid() || 0 >= add_timestamp)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("task init failed", KR(ret), K(ls_id), K(tablet_id), - K(add_timestamp)); + LOG_WARN("task init failed", KR(ret), K(tablet_id), K(add_timestamp)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; add_timestamp_ = add_timestamp; need_diagnose_ = need_diagnose; @@ -103,7 +99,6 @@ int ObTabletTableUpdateTask::assign(const ObTabletTableUpdateTask &other) { int ret = OB_SUCCESS; if (this != &other) { - ls_id_ = other.get_ls_id(); tablet_id_ = other.get_tablet_id(); add_timestamp_ = other.get_add_timestamp(); need_diagnose_ = other.need_diagnose_; @@ -118,15 +113,13 @@ bool ObTabletTableUpdateTask::operator ==(const ObTabletTableUpdateTask &other) if (this == &other) { // same pointer equal = true; } else { - equal = (ls_id_ == other.ls_id_ - && tablet_id_ == other.tablet_id_); + equal = tablet_id_ == other.tablet_id_; } return equal; } void ObTabletTableUpdateTask::reset() { - ls_id_.reset(); tablet_id_.reset(); add_timestamp_ = OB_INVALID_TIMESTAMP; need_diagnose_ = false; @@ -139,8 +132,7 @@ bool ObTabletTableUpdateTask::compare_without_version( if (&other == this) { equal = true; } else { - equal = (ls_id_ == other.ls_id_ - && tablet_id_ == other.tablet_id_); + equal = tablet_id_ == other.tablet_id_; } return equal; } @@ -168,8 +160,7 @@ int64_t ObTabletTableUpdateTask::get_start_timestamp() const bool ObTabletTableUpdateTask::is_valid() const { - return ls_id_.is_valid() - && tablet_id_.is_valid() + return tablet_id_.is_valid() && 0 < add_timestamp_; } @@ -181,7 +172,6 @@ bool ObTabletTableUpdateTask::is_barrier() const int64_t ObTabletTableUpdateTask::hash() const { uint64_t hash_val = 0; - hash_val = murmurhash(&ls_id_, sizeof(ls_id_), hash_val); hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); return hash_val; } @@ -270,7 +260,6 @@ int64_t ObTabletTableUpdater::cal_thread_count_() } int ObTabletTableUpdater::submit_tablet_update_task( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_diagnose/*false*/) { @@ -278,17 +267,16 @@ int ObTabletTableUpdater::submit_tablet_update_task( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTabletTableUpdater is not inited", KR(ret)); - } else if (!ls_id.is_valid() || !tablet_id.is_valid()) { + } else if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(async_update(ls_id, tablet_id, need_diagnose))) { - LOG_WARN("fail to async update tablet", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_id)); + } else if (OB_FAIL(async_update(tablet_id, need_diagnose))) { + LOG_WARN("fail to async update tablet", KR(ret), K(tablet_id)); } return ret; } int ObTabletTableUpdater::async_update( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const bool need_diagnose/*false*/) { @@ -300,17 +288,16 @@ int ObTabletTableUpdater::async_update( LOG_WARN("ObTabletTableUpdater is not inited", KR(ret)); } else if (tablet_id.is_reserved_tablet()) { LOG_TRACE("no need to report reserved tablet", KR(ret), K(tablet_id)); - } else if (!ls_id.is_valid() || !tablet_id.is_valid()) { + } else if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(task.init(ls_id, - tablet_id, + LOG_WARN("invalid argument", KR(ret), K(tablet_id)); + } else if (OB_FAIL(task.init(tablet_id, add_timestamp, need_diagnose))) { - LOG_WARN("set update task failed", KR(ret), K(ls_id), K(tablet_id), + LOG_WARN("set update task failed", KR(ret), K(tablet_id), K(add_timestamp)); } else if (OB_FAIL(add_task_(task))){ - LOG_WARN("fail to add task", KR(ret), K(ls_id), K(tablet_id), + LOG_WARN("fail to add task", KR(ret), K(tablet_id), K(add_timestamp)); } return ret; @@ -347,7 +334,7 @@ int ObTabletTableUpdater::add_task_( LOG_WARN("add tablet table update task failed", KR(ret), K(task)); if (task.need_diagnose() && OB_TMP_FAIL(compaction::ADD_SUSPECT_INFO( compaction::MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_REPORT, - task.get_ls_id(), task.get_tablet_id(), + task.get_tablet_id(), ObSuspectInfoType::SUSPECT_COMPACTION_REPORT_ADD_FAILED, static_cast(ret)))) { LOG_WARN_RET(tmp_ret, "fail to add suspect info", K(tmp_ret)); @@ -356,7 +343,7 @@ int ObTabletTableUpdater::add_task_( if (task.need_diagnose()) { DEL_SUSPECT_INFO( compaction::MEDIUM_MERGE, - task.get_ls_id(), task.get_tablet_id(), + task.get_tablet_id(), share::ObDiagnoseTabletType::TYPE_REPORT); } LOG_TRACE("add tablet table update task success", KR(ret), K(task)); @@ -427,17 +414,16 @@ int ObTabletTableUpdater::set_thread_count() } int ObTabletTableUpdater::check_exist( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, bool &exist) { int ret = OB_SUCCESS; exist = false; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_id)); } else { - ObTabletTableUpdateTask task(ls_id, tablet_id, ObClockGenerator::getClock()); + ObTabletTableUpdateTask task(tablet_id, ObClockGenerator::getClock()); if (OB_FAIL(update_queue_.check_exist(task, exist))) { LOG_WARN("fail to check task exist", K(ret), K(task), K(exist)); } @@ -446,17 +432,16 @@ int ObTabletTableUpdater::check_exist( } int ObTabletTableUpdater::check_processing_exist( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, bool &exist) { int ret = OB_SUCCESS; exist = false; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_id)); } else { - ObTabletTableUpdateTask task(ls_id, tablet_id, ObClockGenerator::getClock()); + ObTabletTableUpdateTask task(tablet_id, ObClockGenerator::getClock()); if (OB_FAIL(update_queue_.check_processing_exist(task, exist))) { LOG_WARN("fail to check processing task exist", K(ret), K(task), K(exist)); } @@ -488,7 +473,7 @@ void ObTabletTableUpdater::diagnose_batch_tasks_( if (task.need_diagnose()) { if (OB_TMP_FAIL(compaction::ADD_SUSPECT_INFO( compaction::MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - task.get_ls_id(), task.get_tablet_id(), + task.get_tablet_id(), ObSuspectInfoType::SUSPECT_COMPACTION_REPORT_PROGRESS_FAILED, static_cast(error_code)))) { LOG_WARN_RET(tmp_ret, "fail to add suspect info", K(tmp_ret)); @@ -502,7 +487,7 @@ void ObTabletTableUpdater::diagnose_batch_tasks_( int ObTabletTableUpdater::push_task_info_( const ObTabletTableUpdateTask &task, const share::ObTabletReplica &replica, - ObArray &replicas, + ObArray &tablet_meta_rows, ObArray &task_list) { int ret = OB_SUCCESS; @@ -512,25 +497,25 @@ int ObTabletTableUpdater::push_task_info_( LOG_WARN("fail to reserver task_list", KR(ret), K(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM)); } else if (OB_FAIL(task_list.push_back(task))) { LOG_WARN("fail to push back remove task", KR(ret), K(task)); - } else if (OB_FAIL(replicas.reserve(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM))) { - LOG_WARN("fail to reserver replicas", KR(ret), K(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("fail to push back replica", KR(ret), K(replica)); + } else if (OB_FAIL(tablet_meta_rows.reserve(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM))) { + LOG_WARN("fail to reserve tablet meta rows", KR(ret), K(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM)); + } else if (OB_FAIL(tablet_meta_rows.push_back(replica))) { + LOG_WARN("fail to push back tablet meta row", KR(ret), K(replica)); } return ret; } int ObTabletTableUpdater::generate_tasks_( const ObIArray &batch_tasks, - ObArray &update_tablet_replicas, - ObArray &remove_tablet_replicas, + ObArray &update_tablet_meta_rows, + ObArray &remove_tablet_meta_rows, ObArray &update_tablet_checksums, UpdateTaskList &update_tablet_tasks, RemoveTaskList &remove_tablet_tasks) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - int64_t retry_tablet_replica_count = 0; + int64_t retry_tablet_meta_row_count = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTabletTableUpdater is not inited", KR(ret)); @@ -552,8 +537,7 @@ int ObTabletTableUpdater::generate_tasks_( } else if (FALSE_IT(task->check_task_status())) { } else if (FALSE_IT(replica.reset())) { } else if (FALSE_IT(checksum_item.reset())) { - } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_report_info(task->get_ls_id(), - task->get_tablet_id(), + } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_report_info(task->get_tablet_id(), replica, checksum_item))) { bool is_remove_task = false; @@ -561,16 +545,15 @@ int ObTabletTableUpdater::generate_tasks_( if (OB_TMP_FAIL(add_task_(*task))) { LOG_WARN("fail to add task", KR(tmp_ret), KPC(task)); } else { - retry_tablet_replica_count++; + retry_tablet_meta_row_count++; ret = OB_SUCCESS; // do not affect report of other tablets } - } else if (OB_TENANT_NOT_IN_SERVER != ret && OB_LS_NOT_EXIST != ret && OB_TABLET_NOT_EXIST != ret) { + } else if (OB_TENANT_NOT_IN_SERVER != ret && OB_TABLET_NOT_EXIST != ret) { LOG_WARN("failed to fill tablet replica info", KR(ret), KPC(task)); } else if (OB_TENANT_NOT_IN_SERVER == ret) { is_remove_task = true; ret = OB_SUCCESS; } else { - bool is_ls_not_exist = OB_LS_NOT_EXIST == ret; is_remove_task = true; ret = OB_SUCCESS; } @@ -578,7 +561,6 @@ int ObTabletTableUpdater::generate_tasks_( if (OB_FAIL(ret) || !is_remove_task) { // do nothing } else if (OB_FAIL(replica.init(task->get_tablet_id(), - task->get_ls_id(), GCONF.self_addr_, 1/*snapshot_version*/, 1/*data_size*/, @@ -586,12 +568,12 @@ int ObTabletTableUpdater::generate_tasks_( 0/*report_scn*/, ObTabletReplica::SCN_STATUS_IDLE))) { LOG_WARN("fail to init ObTabletReplica", KR(ret), KPC(task), "server", GCONF.self_addr_); - } else if (OB_FAIL(push_task_info_(*task, replica, remove_tablet_replicas, remove_tablet_tasks))) { + } else if (OB_FAIL(push_task_info_(*task, replica, remove_tablet_meta_rows, remove_tablet_tasks))) { LOG_WARN("failed to push remove task", K(ret), KPC(task)); } } else { LOG_TRACE("fill tablet success", K(task), K(replica)); - if (OB_FAIL(push_task_info_(*task, replica, update_tablet_replicas, update_tablet_tasks))) { + if (OB_FAIL(push_task_info_(*task, replica, update_tablet_meta_rows, update_tablet_tasks))) { LOG_WARN("failed to push update task info", KR(ret), KPC(task), K(replica)); } else if (OB_FAIL(update_tablet_checksums.reserve(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM))) { // reserve() is reentrant, do not have to check whether first time @@ -603,16 +585,16 @@ int ObTabletTableUpdater::generate_tasks_( } //FOREACH if (OB_FAIL(ret)) { - } else if (update_tablet_tasks.count() != update_tablet_replicas.count() + } else if (update_tablet_tasks.count() != update_tablet_meta_rows.count() || update_tablet_tasks.count() != update_tablet_checksums.count() - || ((update_tablet_tasks.count() + remove_tablet_tasks.count() + retry_tablet_replica_count != batch_tasks.count()))) { + || ((update_tablet_tasks.count() + remove_tablet_tasks.count() + retry_tablet_meta_row_count != batch_tasks.count()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet task count and replica count not match", KR(ret), + LOG_WARN("tablet task count and meta row count not match", KR(ret), "tablet_update_tasks count", update_tablet_tasks.count(), - "tablet_update_replicas count", update_tablet_replicas.count(), + "tablet_update_meta_rows count", update_tablet_meta_rows.count(), "tablet_update_checksums count", update_tablet_checksums.count(), "tablet_remove_tasks count", remove_tablet_tasks.count(), - K(retry_tablet_replica_count), + K(retry_tablet_meta_row_count), "batch_tasks count", batch_tasks.count()); } return ret; @@ -628,8 +610,8 @@ int ObTabletTableUpdater::batch_process_tasks( bool schema_not_ready = false; const int64_t start_time = ObTimeUtility::current_time(); - ObArray update_tablet_replicas; - ObArray remove_tablet_replicas; + ObArray update_tablet_meta_rows; + ObArray remove_tablet_meta_rows; ObArray update_tablet_checksums; UpdateTaskList update_tablet_tasks; RemoveTaskList remove_tablet_tasks; @@ -665,40 +647,40 @@ int ObTabletTableUpdater::batch_process_tasks( } } else if (OB_FAIL(generate_tasks_( batch_tasks, - update_tablet_replicas, - remove_tablet_replicas, + update_tablet_meta_rows, + remove_tablet_meta_rows, update_tablet_checksums, update_tablet_tasks, remove_tablet_tasks))) { //There is a situation where there are too many tablet holds and cannot be obtained LOG_WARN("generate_tasks failed", KR(ret), "batch_tasks count", batch_tasks.count(), - "update_tablet_replicas", update_tablet_replicas.count(), - "remove_tablet_replicas", remove_tablet_replicas.count(), + "update_tablet_meta_rows", update_tablet_meta_rows.count(), + "remove_tablet_meta_rows", remove_tablet_meta_rows.count(), "update_tablet_checksums", update_tablet_checksums.count(), "update_tablet_tasks", update_tablet_tasks.count(), "remove_tablet_tasks", remove_tablet_tasks.count()); } else { - update_task_cnt = update_tablet_replicas.count(); - remove_task_cnt = remove_tablet_replicas.count(); + update_task_cnt = update_tablet_meta_rows.count(); + remove_task_cnt = remove_tablet_meta_rows.count(); if (update_tablet_tasks.count() > 0) { - tmp_ret = do_batch_update_(start_time, update_tablet_tasks, update_tablet_replicas, update_tablet_checksums); + tmp_ret = do_batch_update_(start_time, update_tablet_tasks, update_tablet_meta_rows, update_tablet_checksums); if (OB_SUCCESS != tmp_ret) { ret = OB_SUCC(ret) ? tmp_ret : ret; LOG_WARN("do_batch_update_ failed", KR(tmp_ret), K(start_time), "tasks count", update_tablet_tasks.count(), - "tablet replicas count", update_tablet_replicas.count()); + "tablet meta rows count", update_tablet_meta_rows.count()); diagnose_batch_tasks_(update_tablet_tasks, tmp_ret); } else { succ_cnt += update_task_cnt; } } if (remove_tablet_tasks.count() > 0) { - tmp_ret = do_batch_remove_(start_time, remove_tablet_tasks, remove_tablet_replicas); + tmp_ret = do_batch_remove_(start_time, remove_tablet_tasks, remove_tablet_meta_rows); if (OB_SUCCESS != tmp_ret) { ret = OB_SUCC(ret) ? tmp_ret : ret; LOG_WARN("do_batch_remove_ failed", KR(tmp_ret), K(start_time), "tasks count", remove_tablet_tasks.count(), - "remove replicas count", remove_tablet_replicas.count()); + "remove tablet meta rows count", remove_tablet_meta_rows.count()); diagnose_batch_tasks_(remove_tablet_tasks, tmp_ret); } else { succ_cnt += remove_task_cnt; @@ -725,7 +707,7 @@ int ObTabletTableUpdater::batch_process_tasks( int ObTabletTableUpdater::do_batch_remove_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas) + const ObIArray &tablet_meta_rows) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -734,7 +716,7 @@ int ObTabletTableUpdater::do_batch_remove_( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(tasks_count != replicas.count() || OB_ISNULL(GCTX.tablet_operator_))) { + } else if (OB_UNLIKELY(tasks_count != tablet_meta_rows.count() || OB_ISNULL(GCTX.tablet_operator_))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tasks count", KR(ret), K(tasks_count), KP(GCTX.tablet_operator_)); } else if (OB_ISNULL(GCTX.meta_db_pool_)) { @@ -748,10 +730,10 @@ int ObTabletTableUpdater::do_batch_remove_( LOG_WARN("failed to acquire connection", K(ret)); } else if (OB_FAIL(guard->begin_transaction())) { LOG_WARN("fail to start transaction", KR(ret)); - } else if (OB_FAIL(GCTX.tablet_operator_->batch_remove(guard.get_connection(), replicas))) { + } else if (OB_FAIL(GCTX.tablet_operator_->batch_remove(guard.get_connection(), tablet_meta_rows))) { LOG_WARN("do tablet table remove failed, try to reput to queue", KR(ret), "escape time", ObTimeUtility::current_time() - start_time); - } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_remove_with_trans(guard.get_connection(), replicas))) { + } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_remove_with_trans(guard.get_connection(), tablet_meta_rows))) { LOG_WARN("do tablet table checksum remove failed, try to reput to queue", KR(ret), "escape time", ObTimeUtility::current_time() - start_time); } @@ -787,7 +769,7 @@ int ObTabletTableUpdater::do_batch_remove_( int ObTabletTableUpdater::do_batch_update_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas, + const ObIArray &tablet_meta_rows, const ObIArray &checksums) { int ret = OB_SUCCESS; @@ -796,12 +778,12 @@ int ObTabletTableUpdater::do_batch_update_( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (tasks.count() != replicas.count() + } else if (tasks.count() != tablet_meta_rows.count() || tasks.count() != checksums.count() || OB_ISNULL(GCTX.tablet_operator_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tasks num not match or invalid tablet_operator", KR(ret), "task_cnt", tasks.count(), - "replica_cnt", replicas.count(), "checksum_cnt", checksums.count(), K(GCTX.tablet_operator_)); + "tablet_meta_row_cnt", tablet_meta_rows.count(), "checksum_cnt", checksums.count(), K(GCTX.tablet_operator_)); } else { #ifdef ERRSIM if (OB_SUCC(ret)) { @@ -830,7 +812,7 @@ int ObTabletTableUpdater::do_batch_update_( LOG_WARN("failed to acquire connection", K(ret)); } else if (OB_FAIL(guard->begin_transaction())) { LOG_WARN("fail to start transaction", KR(ret)); - } else if (OB_FAIL(GCTX.tablet_operator_->batch_update(guard.get_connection(), replicas))) { + } else if (OB_FAIL(GCTX.tablet_operator_->batch_update(guard.get_connection(), tablet_meta_rows))) { LOG_WARN("do tablet table update failed, try to reput to queue", KR(ret), "escape time", ObTimeUtility::current_time() - start_time); } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_update_with_trans(guard.get_connection(), checksums))) { @@ -862,7 +844,7 @@ int ObTabletTableUpdater::do_batch_update_( } } } - LOG_TRACE("REPORT: batch update tablets finished", KR(ret), K(replicas.count()), K(tasks), + LOG_TRACE("REPORT: batch update tablets finished", KR(ret), K(tablet_meta_rows.count()), K(tasks), "cost_time", ObTimeUtility::current_time() - batch_update_start_time); return ret; } diff --git a/src/observer/report/ob_tablet_table_updater.h b/src/observer/report/ob_tablet_table_updater.h index 4c117e3fd..b88fdf7e7 100644 --- a/src/observer/report/ob_tablet_table_updater.h +++ b/src/observer/report/ob_tablet_table_updater.h @@ -19,7 +19,6 @@ #include "observer/ob_uniq_task_queue.h" // for ObIUniqTaskQueueTask #include "common/ob_tablet_id.h" // for ObTablet -#include "share/ob_ls_id.h" // for ObLSID namespace oceanbase { @@ -30,7 +29,6 @@ class ObTabletID; } namespace share { -class ObLSID; class ObTabletReplica; class ObTabletTableOperator; struct ObTabletReplicaChecksumItem; @@ -66,30 +64,23 @@ class ObTabletTableUpdateTask : public ObIUniqTaskQueueTask &batch_tasks, - ObArray &update_tablet_replicas, - ObArray &remove_tablet_replicas, + ObArray &update_tablet_meta_rows, + ObArray &remove_tablet_meta_rows, ObArray &update_tablet_checksums, UpdateTaskList &update_tablet_tasks, RemoveTaskList &remove_tablet_tasks); @@ -215,22 +199,22 @@ class ObTabletTableUpdater // do_batch_update - the real action to update a batch of tasks // @parma [in] start_time, the time to start this execution // @parma [in] tasks, batch of tasks to execute - // @parma [in] replicas, related replica to each task + // @parma [in] tablet_meta_rows, related tablet meta row to each task // @parma [in] checksums, related checksum to each task int do_batch_update_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas, + const ObIArray &tablet_meta_rows, const ObIArray &checksums); // do_batch_remove - the real action to remove a batch of tasks // @parma [in] start_time, the time to start this execution // @parma [in] tasks, batch of tasks to execute - // @parma [in] replicas, related replica to each task + // @parma [in] tablet_meta_rows, related tablet meta row to each task int do_batch_remove_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas); + const ObIArray &tablet_meta_rows); // add_update_task - add a task to task_queue // @parma [in] task, task to add @@ -254,7 +238,7 @@ class ObTabletTableUpdater int push_task_info_( const ObTabletTableUpdateTask &task, const share::ObTabletReplica &replica, - ObArray &replicas, + ObArray &tablet_meta_rows, ObArray &task_list); private: const int64_t MINI_MODE_UPDATE_TASK_THREAD_CNT = 1; diff --git a/src/observer/report/ob_tenant_meta_checker.cpp b/src/observer/report/ob_tenant_meta_checker.cpp index 4fdd5ad59..1f064e5ad 100644 --- a/src/observer/report/ob_tenant_meta_checker.cpp +++ b/src/observer/report/ob_tenant_meta_checker.cpp @@ -20,7 +20,8 @@ #include "share/rc/ob_module_provider.h" #include "storage/tablet/ob_tablet_iterator.h" // ObLSTabletIterator #include "share/tablet/ob_tablet_table_iterator.h" // ObTenantTabletTableIterator -#include "storage/tx_storage/ob_ls_service.h" // ObLSService, ObLSIterator +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_service.h" // ObLSService #include "share/ob_tablet_replica_checksum_operator.h" // ObTabletReplicaChecksumItem #include "observer/report/ob_tablet_table_updater.h" // ObTabletTableUpdater #include "observer/ob_service.h" // ObService @@ -136,26 +137,27 @@ void ObTenantMetaChecker::destroy() int ObTenantMetaChecker::check_tablet_table() { int ret = OB_SUCCESS; - int64_t dangling_count = 0; // replica only in tablet meta table - int64_t report_count = 0; // replica not in/match tablet meta table + int64_t stale_row_count = 0; + int64_t missing_or_changed_row_count = 0; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else { const int64_t start_time = ObTimeUtility::current_time(); - ObTabletReplicaMap replica_map; - if (OB_FAIL(build_replica_map_(replica_map))) { - LOG_WARN("build replica map from tablet table failed", KR(ret)); - } else if (OB_FAIL(check_dangling_replicas_(replica_map, dangling_count))) { - LOG_WARN("check replicas exist in tablet table but not in local failed", KR(ret)); - } else if (OB_FAIL(check_report_replicas_(replica_map, report_count))) { - LOG_WARN("check replicas not in/match tablet table failed", KR(ret)); - } else if (dangling_count != 0 || report_count != 0) { - LOG_INFO("checker found and corrected dangling or to report replicas for tablet meta table", - KR(ret), K(dangling_count), K(report_count)); + ObTabletMetaRowMap tablet_meta_row_map; + if (OB_FAIL(build_tablet_meta_row_map_(tablet_meta_row_map))) { + LOG_WARN("build tablet meta row map failed", KR(ret)); + } else if (OB_FAIL(check_stale_tablet_meta_rows_(tablet_meta_row_map, stale_row_count))) { + LOG_WARN("check stale tablet meta rows failed", KR(ret)); + } else if (OB_FAIL(check_missing_or_changed_tablet_meta_rows_( + tablet_meta_row_map, missing_or_changed_row_count))) { + LOG_WARN("check missing or changed tablet meta rows failed", KR(ret)); + } else if (stale_row_count != 0 || missing_or_changed_row_count != 0) { + LOG_INFO("checker found and corrected stale or missing tablet meta rows", + KR(ret), K(stale_row_count), K(missing_or_changed_row_count)); } LOG_TRACE("finish checking tablet table", KR(ret), - K(dangling_count), K(report_count), + K(stale_row_count), K(missing_or_changed_row_count), K(start_time), "cost_time", ObTimeUtility::current_time() - start_time); } return ret; @@ -183,7 +185,7 @@ int ObTenantMetaChecker::schedule_tablet_meta_check_task() return ret; } -int ObTenantMetaChecker::build_replica_map_(ObTabletReplicaMap &replica_map) +int ObTenantMetaChecker::build_tablet_meta_row_map_(ObTabletMetaRowMap &tablet_meta_row_map) { int ret = OB_SUCCESS; ObTenantTabletTableIterator tt_iter; @@ -193,11 +195,11 @@ int ObTenantMetaChecker::build_replica_map_(ObTabletReplicaMap &replica_map) } else if (OB_UNLIKELY(stopped_)) { ret = OB_CANCELED; LOG_WARN("ObTenantMetaChecker is stopped", KR(ret)); - } else if (OB_FAIL(replica_map.create( - hash::cal_next_prime(TABLET_REPLICA_MAP_BUCKET_NUM), + } else if (OB_FAIL(tablet_meta_row_map.create( + hash::cal_next_prime(TABLET_META_ROW_MAP_BUCKET_NUM), "TabletCheckMap", ObModIds::OB_HASH_NODE))) { - LOG_WARN("fail to create replica_map", KR(ret)); + LOG_WARN("fail to create tablet meta row map", KR(ret)); } else if (OB_FAIL(tt_iter.init(*tt_operator_))) { LOG_WARN("fail to init tablet meta table iter", KR(ret)); } else if (OB_FAIL(tt_iter.get_filters().set_reserved_server(GCONF.self_addr_))) { @@ -213,18 +215,14 @@ int ObTenantMetaChecker::build_replica_map_(ObTabletReplicaMap &replica_map) if (OB_ITER_END != ret) { LOG_WARN("tablet table iterator next failed", KR(ret)); } - } else if (0 == tablet_info.replica_count()) { + } else if (!tablet_info.has_replica()) { continue; - } else if (1 == tablet_info.replica_count()) { - const ObTabletReplica &replica = tablet_info.get_replicas().at(0); - if (OB_FAIL(replica_map.set_refactored( - ObTabletLSPair(replica.get_tablet_id(), replica.get_ls_id()), - replica))) { - LOG_WARN("fail to set_refactored", KR(ret), K(replica)); - } } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet_info should have one local replica at most", KR(ret), K(tablet_info)); + const ObTabletReplica &tablet_meta_row = tablet_info.get_replica(); + if (OB_FAIL(tablet_meta_row_map.set_refactored( + tablet_meta_row.get_tablet_id(), tablet_meta_row))) { + LOG_WARN("fail to set tablet meta row", KR(ret), K(tablet_meta_row)); + } } } // end while if (OB_ITER_END == ret) { @@ -234,12 +232,12 @@ int ObTenantMetaChecker::build_replica_map_(ObTabletReplicaMap &replica_map) return ret; } -int ObTenantMetaChecker::check_dangling_replicas_( - ObTabletReplicaMap &replica_map, - int64_t &dangling_count) +int ObTenantMetaChecker::check_stale_tablet_meta_rows_( + ObTabletMetaRowMap &tablet_meta_row_map, + int64_t &stale_row_count) { int ret = OB_SUCCESS; - dangling_count = 0; + stale_row_count = 0; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -251,22 +249,21 @@ int ObTenantMetaChecker::check_dangling_replicas_( LOG_WARN("ob_service is null", KR(ret)); } else { bool not_exist = false; - FOREACH_X(it, replica_map, OB_SUCC(ret)) { - const ObLSID &ls_id = it->first.get_ls_id(); - const ObTabletID &tablet_id = it->first.get_tablet_id(); + FOREACH_X(it, tablet_meta_row_map, OB_SUCC(ret)) { + const ObTabletID &tablet_id = it->first; if (OB_UNLIKELY(stopped_)) { ret = OB_CANCELED; LOG_WARN("ObTenantMetaChecker is stopped", KR(ret)); - } else if (OB_FAIL(check_tablet_not_exist_in_local_(ls_id, tablet_id, not_exist))) { - LOG_WARN("fail to check tablet whether exist in local", KR(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(check_tablet_not_exist_in_local_(tablet_id, not_exist))) { + LOG_WARN("fail to check tablet whether exist in local", KR(ret), K(tablet_id)); } else if (not_exist) { - ++dangling_count; - if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id))) { + ++stale_row_count; + if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id))) { LOG_WARN("fail to submit tablet update task", - KR(ret), K(ls_id), K(tablet_id)); + KR(ret), K(tablet_id)); } else { - LOG_INFO("add async task to remove replica from tablet table", - "replica", it->second); + LOG_INFO("add async task to remove stale tablet meta row", + "tablet_meta_row", it->second); } } } // end for @@ -275,39 +272,28 @@ int ObTenantMetaChecker::check_dangling_replicas_( } int ObTenantMetaChecker::check_tablet_not_exist_in_local_( - const ObLSID &ls_id, const ObTabletID &tablet_id, bool ¬_exist) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; not_exist = false; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid tablet id", KR(ret), K(tablet_id)); } else if (tablet_id.is_reserved_tablet()) { // skip reserved tablet } else if (OB_FAIL(share::g_mp->ls_service()->get_ls( - ls_id, - ls_handle, - ObLSGetMod::OBSERVER_MOD))) { - if (OB_LS_NOT_EXIST == ret) { - ret = OB_SUCCESS; - not_exist = true; - } else { - LOG_WARN("fail to get sys_ls handle", KR(ret)); - } - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls handle of sys ls is null", KR(ret)); - } else if (OB_ISNULL(ls_handle.get_ls()->get_tablet_svr())) { + ls))) { + LOG_WARN("fail to get ls", KR(ret)); + } else if (OB_ISNULL(ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet_svr of sys ls is null", KR(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( + LOG_WARN("tablet service is null", KR(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet( tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, @@ -316,19 +302,19 @@ int ObTenantMetaChecker::check_tablet_not_exist_in_local_( ret = OB_SUCCESS; not_exist = true; } else { - LOG_WARN("fail to get tablet", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get tablet", KR(ret), K(tablet_id)); } } return ret; } -int ObTenantMetaChecker::check_report_replicas_( - ObTabletReplicaMap &replica_map, - int64_t &report_count) +int ObTenantMetaChecker::check_missing_or_changed_tablet_meta_rows_( + ObTabletMetaRowMap &tablet_meta_row_map, + int64_t &missing_or_changed_row_count) { int ret = OB_SUCCESS; - report_count = 0; - ObSharedGuard ls_iter; + missing_or_changed_row_count = 0; + ObLS *ls = nullptr; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -341,94 +327,78 @@ int ObTenantMetaChecker::check_report_replicas_( } else if (OB_UNLIKELY(GCTX.is_shared_storage_mode())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("compaction should be local mode", KR(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls_iter( - ls_iter, - ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("failed to get ls iter", KR(ret)); + } else if (OB_ISNULL(share::g_mp->ls_service())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls service is null", KR(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get single log stream", KR(ret)); } else { - ObLS *ls = NULL; ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_ALL_COMMITED); - while(OB_SUCC(ret)) { - if (OB_UNLIKELY(stopped_)) { - ret = OB_CANCELED; - LOG_WARN("ObTenantMetaChecker is stopped", KR(ret)); - } else if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("scan next ls failed.", KR(ret)); - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get ls", KR(ret)); - } else if (OB_ISNULL(ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet svr", KR(ret)); - } else if (OB_FAIL(ls->get_tablet_svr()->build_tablet_iter(tablet_iter))) { - LOG_WARN("failed to build ls tablet iter", KR(ret)); - } else { - ObTabletHandle tablet_handle; - ObTabletID tablet_id; - ObTabletReplica local_replica; // replica from local - ObTabletReplica table_replica; // replica from meta table - share::ObTabletReplicaChecksumItem tablet_checksum; // TODO(@donglou.zl) check tablet_replica_checksum - const bool need_checksum = false; - const ObLSID &ls_id = ls->get_ls_id(); - while (OB_SUCC(ret)) { - if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get next tablet", KR(ret)); - } - } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet handle", KR(ret), K(tablet_handle)); - } else if (FALSE_IT(tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_)) { - } else if (tablet_id.is_reserved_tablet()) { - continue; - } else if (OB_FAIL(replica_map.get_refactored( - ObTabletLSPair(tablet_id, ls_id), - table_replica))) { - if (OB_HASH_NOT_EXIST == ret) { // not exist in table while exist in local - ret = OB_SUCCESS; - if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id))) { - LOG_WARN("fail to submit tablet update task", - KR(ret), K(ls_id), K(tablet_id)); - } else { - ++report_count; - LOG_INFO("add missing replica to tablet meta table success", - KR(ret), K(ls_id), K(tablet_id)); - } - } else { - LOG_WARN("get replica from hashmap failed", - KR(ret), K(ls_id), K(tablet_id)); - } - } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_report_info(ls_id, - tablet_id, - local_replica, - tablet_checksum, - need_checksum))) { - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; // do not affect report of other tablets - } else { - LOG_WARN("fail to fill tablet replica", KR(ret), K(ls_id), K(tablet_id)); - } - } else if (table_replica.is_equal_for_report(local_replica)) { - continue; - } else { // not equal - if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id))) { + if (OB_UNLIKELY(stopped_)) { + ret = OB_CANCELED; + LOG_WARN("ObTenantMetaChecker is stopped", KR(ret)); + } else if (OB_ISNULL(ls->get_tablet_svr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get tablet svr", KR(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->build_tablet_iter(tablet_iter))) { + LOG_WARN("failed to build ls tablet iter", KR(ret)); + } else { + ObTabletHandle tablet_handle; + ObTabletID tablet_id; + ObTabletReplica local_tablet_meta_row; + ObTabletReplica tablet_meta_row; + share::ObTabletReplicaChecksumItem tablet_checksum; // TODO(@donglou.zl) check tablet_replica_checksum + const bool need_checksum = false; + while (OB_SUCC(ret)) { + if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next tablet", KR(ret)); + } + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tablet handle", KR(ret), K(tablet_handle)); + } else if (FALSE_IT(tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_)) { + } else if (tablet_id.is_reserved_tablet()) { + continue; + } else if (OB_FAIL(tablet_meta_row_map.get_refactored(tablet_id, tablet_meta_row))) { + if (OB_HASH_NOT_EXIST == ret) { // not exist in table while exist in local + ret = OB_SUCCESS; + if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id))) { LOG_WARN("fail to submit tablet update task", - KR(ret), K(ls_id), K(tablet_id)); + KR(ret), K(tablet_id)); } else { - ++report_count; - LOG_INFO("modify replica success", KR(ret), K(local_replica), K(table_replica)); + ++missing_or_changed_row_count; + LOG_INFO("add missing tablet meta row success", + KR(ret), K(tablet_id)); } + } else { + LOG_WARN("get tablet meta row from hashmap failed", + KR(ret), K(tablet_id)); + } + } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_report_info(tablet_id, + local_tablet_meta_row, + tablet_checksum, + need_checksum))) { + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; // do not affect report of other tablets + } else { + LOG_WARN("fail to fill tablet meta row", KR(ret), K(tablet_id)); + } + } else if (tablet_meta_row.is_equal_for_report(local_tablet_meta_row)) { + continue; + } else { // not equal + if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id))) { + LOG_WARN("fail to submit tablet update task", + KR(ret), K(tablet_id)); + } else { + ++missing_or_changed_row_count; + LOG_INFO("modify tablet meta row success", KR(ret), K(local_tablet_meta_row), K(tablet_meta_row)); } - } // end while for tablet_iter - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; } + } // end while for tablet_iter + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; } - } // end while for ls_iter - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; } } return ret; diff --git a/src/observer/report/ob_tenant_meta_checker.h b/src/observer/report/ob_tenant_meta_checker.h index 402c6e28d..64e5c276f 100644 --- a/src/observer/report/ob_tenant_meta_checker.h +++ b/src/observer/report/ob_tenant_meta_checker.h @@ -59,14 +59,15 @@ class ObTenantMetaChecker int check_tablet_table(); int schedule_tablet_meta_check_task(); private: - static const int64_t TABLET_REPLICA_MAP_BUCKET_NUM = 64 * 1024; - typedef common::hash::ObHashMap ObTabletReplicaMap; + static const int64_t TABLET_META_ROW_MAP_BUCKET_NUM = 64 * 1024; + typedef common::hash::ObHashMap ObTabletMetaRowMap; - int build_replica_map_(ObTabletReplicaMap &replica_map); - int check_dangling_replicas_(ObTabletReplicaMap &replica_map, int64_t &dangling_count); - int check_report_replicas_(ObTabletReplicaMap &replica_map, int64_t &report_count); + int build_tablet_meta_row_map_(ObTabletMetaRowMap &tablet_meta_row_map); + int check_stale_tablet_meta_rows_(ObTabletMetaRowMap &tablet_meta_row_map, int64_t &stale_row_count); + int check_missing_or_changed_tablet_meta_rows_( + ObTabletMetaRowMap &tablet_meta_row_map, + int64_t &missing_or_changed_row_count); int check_tablet_not_exist_in_local_( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, bool ¬_exist); diff --git a/src/observer/scheduler/ob_dag_scheduler_config.h b/src/observer/scheduler/ob_dag_scheduler_config.h index 788ab509e..3c28b2eb2 100644 --- a/src/observer/scheduler/ob_dag_scheduler_config.h +++ b/src/observer/scheduler/ob_dag_scheduler_config.h @@ -16,17 +16,7 @@ #ifdef DAG_SCHEDULER_DAG_NET_TYPE_DEF // DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_ENUM, DAG_NET_TYPE_STR) -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_MIGRATION, "DAG_NET_MIGRATION") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_PREPARE_MIGARTION, "DAG_NET_PREPARE_MIGRATION") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_COMPLETE_MIGARTION, "DAG_NET_COMPLETE_MIGRATION") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_RESERVED_3, "DAG_NET_RESERVED_3") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_BACKUP, "DAG_NET_BACKUP") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_RESTORE, "DAG_NET_RESTORE") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_BACKUP_CLEAN, "DAG_NET_TYPE_BACKUP_CLEAN") DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_RESERVED_7, "DAG_NET_RESERVED_7") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_RESERVED_8, "DAG_NET_RESERVED_8") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_REBUILD_TABLET, "DAG_NET_REBUILD_TABLET") -DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_RESTORE_COMPLETE, "DAG_NET_RESTORE_COMPLETE") DAG_SCHEDULER_DAG_NET_TYPE_DEF(DAG_NET_TYPE_MAX, "DAG_NET_TYPE_MAX") #endif @@ -48,182 +38,50 @@ DAG_SCHEDULER_DAG_PRIO_DEF(DAG_PRIO_MAX, 0, "INVALID", MAX_FUNCTION_N #ifdef DAG_SCHEDULER_DAG_TYPE_DEF // DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_ENUM, DAG_DEFAULT_PRIO, SYS_TASK_TYPE, DAG_TYPE_STR, DAG_MODULE_STR, DIAGNOSE_WITH_COMMENT, DIAGNOSE_PRIORITY, DIAGNOSE_INT_INFO_CNT, DIAGNOSE_INT_FMT_STR) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MINI_MERGE, ObDagPrio::DAG_PRIO_COMPACTION_HIGH, ObSysTaskType::SSTABLE_MINI_MERGE_TASK, "MINI_MERGE", "COMPACTION", - true, 3, {"ls_id", "tablet_id", "compaction_scn"}) + true, 2, {"tablet_id", "compaction_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MERGE_EXECUTE, ObDagPrio::DAG_PRIO_COMPACTION_MID, ObSysTaskType::SSTABLE_MINOR_MERGE_TASK, "MINOR_EXECUTE", "COMPACTION", - true, 3, {"ls_id", "tablet_id", "compaction_scn"}) + true, 2, {"tablet_id", "compaction_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MAJOR_MERGE, ObDagPrio::DAG_PRIO_COMPACTION_LOW, ObSysTaskType::SSTABLE_MAJOR_MERGE_TASK, "MAJOR_MERGE/MEDIUM_MERGE", "COMPACTION", - true, 3, {"ls_id", "tablet_id", "compaction_scn"}) + true, 2, {"tablet_id", "compaction_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TX_TABLE_MERGE, ObDagPrio::DAG_PRIO_COMPACTION_HIGH, ObSysTaskType::SPECIAL_TABLE_MERGE_TASK, "TX_TABLE_MERGE", "COMPACTION", - false, 3, {"ls_id", "tablet_id", "compaction_scn"}) + false, 2, {"tablet_id", "compaction_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MDS_MINI_MERGE, ObDagPrio::DAG_PRIO_COMPACTION_HIGH, ObSysTaskType::MDS_MINI_MERGE_TASK, "MDS_MINI_MERGE", "COMPACTION", - false, 3, {"ls_id", "tablet_id", "flush_scn"}) + false, 2, {"tablet_id", "flush_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_VERIFY_CKM, ObDagPrio::DAG_PRIO_COMPACTION_LOW, ObSysTaskType::SSTABLE_MAJOR_MERGE_TASK, "VERIFY_CKM", "COMPACTION", - false, 2, {"ls_id", "tablet_count"}) + false, 1, {"tablet_count"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BATCH_FREEZE_TABLETS, ObDagPrio::DAG_PRIO_COMPACTION_LOW, ObSysTaskType::BATCH_FREEZE_TABLET_TASK, "BATCH_FREEZE", "COMPACTION", - false, 2, {"ls_id", "tablet_count"}) + false, 1, {"tablet_count"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_UPDATE_SKIP_MAJOR, ObDagPrio::DAG_PRIO_COMPACTION_LOW, ObSysTaskType::SSTABLE_MAJOR_MERGE_TASK, "UPDATE_SKIP_MAJOR", "COMPACTION", - false, 2, {"ls_id", "tablet_count"}) + false, 1, {"tablet_count"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_REFRESH_SSTABLES, ObDagPrio::DAG_PRIO_COMPACTION_LOW, ObSysTaskType::SSTABLE_MAJOR_MERGE_TASK, "REFRESH_MAJOR", "COMPACTION", - false, 3, {"ls_id", "tablet_id", "compaction_scn"}) + false, 2, {"tablet_id", "compaction_scn"}) /* * NOTICE: if you add/delete a compaction dag type here, remember to alter function is_compaction_dag and get_diagnose_tablet_type in ob_tenant_dag_scheduler.h * AND update check_ls_compaction_dag_exist_with_cancel */ DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_DDL, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TASK, "DDL_COMPLEMENT", "DDL", - true, 7, {"ls_id", "source_tablet_id", "dest_tablet_id", "data_table_id", "target_table_id", "schema_version", "snapshot_version"}) + true, 6, {"source_tablet_id", "dest_tablet_id", "data_table_id", "target_table_id", "schema_version", "snapshot_version"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_UNIQUE_CHECKING, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TASK, "UNIQUE_CHECK", "DDL", true, 2, {"tablet_id", "index_id"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_SQL_BUILD_INDEX, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TASK, "SQL_BUILD_INDEX", "DDL", true, 0, {}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_DDL_KV_MERGE, ObDagPrio::DAG_PRIO_DDL_HIGH, ObSysTaskType::DDL_KV_MERGE_TASK, "DDL_KV_MERGE", "DDL", - true, 3, {"ls_id", "tablet_id", "rec_scn"}) + true, 2, {"tablet_id", "rec_scn"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_DDL_DEL_LOB_META, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TASK, "DDL_DEL_LOB_META", "DROP_VEC_INDEX", - true, 7, {"ls_id", "source_tablet_id", "dest_tablet_id", "data_table_id", "target_table_id", "schema_version", "snapshot_version"}) + true, 5, {"table_id", "tablet_id", "dest_tablet_id", "schema_version", "snapshot_version"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_SPLIT, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TABLET_SPLIT, "DDL_TABLET_SPLIT", "DDL", - true, 2, {"ls_id", "source_tablet_id"}) + true, 1, {"source_tablet_id"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_LOB_SPLIT, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TABLET_SPLIT, "DDL_TABLET_SPLIT", "DDL", - true, 2, {"ls_id", "source_tablet_id"}) + true, 1, {"source_tablet_id"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FORK_TABLE, ObDagPrio::DAG_PRIO_DDL, ObSysTaskType::DDL_TASK, "DDL_FORK_TABLE", "DDL", - true, 2, {"ls_id", "source_tablet_id"}) + true, 1, {"source_tablet_id"}) -// DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MIGRATE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "MIGRATE", "MIGRATE") -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_COMPLETE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "INITIAL_COMPLETE_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MIGRATE_WARMUP, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "MIGRATE_WARMUP", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_WAIT_DATA_READY, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "WAIT_DATA_READY", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_COMPLETE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "FINISH_COMPLETE_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "INITIAL_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "START_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_SYS_TABLETS_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "SYS_TABLETS_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "TABLET_MIGRATION", "MIGRATE", - true, 3, {"ls_id", "tablet_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_DATA_TABLETS_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "DATA_TABLETS_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_GROUP_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "TABLET_GROUP_MIGRATION", "MIGRATE", - true, 3, {"ls_id", "first_tablet_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MIGRATION_FINISH, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "MIGRATION_FINISH", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_PREPARE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "INITIAL_PREPARE_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_PREPARE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "START_PREPARE_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_PREPARE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "FINISH_PREPARE_MIGRATION", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_CHECK_CONVERT, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "TABLET_CHECKE_CONVERT", "MIGRATE", - true, 2, {"ls_id", "op_type"}) -// DAG_TYPE_MIGRATE END -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FAST_MIGRATE, ObDagPrio::DAG_PRIO_HA_MID, ObSysTaskType::MIGRATION_TASK, "FAST_MIGRATE", "MIGRATE", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_VALIDATE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::MIGRATION_TASK, "VALIDATE", "MIGRATE", - false, 0, {}) - -// DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKFILL_TX, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::BACKFILL_TX_TASK, "BACKFILL_TX", "BACKFILL_TX") DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_BACKFILL_TX, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::BACKFILL_TX_TASK, "TABLET_BACKFILL_TX", "BACKFILL_TX", - true, 2, {"ls_id", "tablet_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_BACKFILL_TX, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::BACKFILL_TX_TASK, "FINISH_BACKFILL_TX", "BACKFILL_TX", - true, 1, {"ls_id"}) - -// DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP", "BACKUP") -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_META, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_META", "BACKUP", - false, 1, {"ls_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_PREPARE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_PREPARE", "BACKUP", - false, 4, {"backup_set_id", "ls_id", "turn_id", "retry_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_FINISH, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_FINISH", "BACKUP", - false, 2, {"backup_set_id", "ls_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_DATA, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_DATA", "BACKUP", - false, 6, {"backup_set_id", "backup_data_type", "ls_id", "turn_id", "retry_id", "task_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_PREFETCH_BACKUP_INFO, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "PREFETCH_BACKUP_INFO", "BACKUP", - false, 6, {"backup_set_id", "backup_data_type", "ls_id", "turn_id", "retry_id", "task_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_INDEX_REBUILD, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_INDEX_REBUILD", "BACKUP", - false, 5, {"backup_set_id", "backup_data_type", "ls_id", "turn_id", "retry_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_LS_LOG_GROUP, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_COMPLEMENT_LOG_LS_GROUP", "BACKUP", - false, 2, {"backup_set_id", "ls_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_LS_LOG_GROUP_FINISH, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_COMPLEMENT_LOG_LS_GROUP", "BACKUP", - false, 2, {"backup_set_id", "ls_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_LS_LOG, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_COMPLEMENT_LOG_LS", "BACKUP", - false, 2, {"backup_set_id", "ls_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_BACKUP_FUSE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "INITIAL_BACKUP_FUSE", "BACKUP", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_BACKUP_FUSE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "START_BACKUP_FUSE", "BACKUP", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_TABLET_FUSE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "BACKUP_TABLET_FUSE", "BACKUP", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_BACKUP_FUSE, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_TASK, "FINISH_BACKUP_FUSE", "BACKUP", - false, 0, {}) -// DAG_TYPE_BACKUP END -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_BACKUPSET, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_BACKUPSET_TASK, "BACKUP_BACKUPSET", "BACKUP", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_ARCHIVELOG, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_ARCHIVELOG_TASK, "BACKUP_ARCHIVELOG", "BACKUP", - false, 0, {}) - -// DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "RESTORE", "RESTORE",) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_LS_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "INITIAL_LS_RESTORE", "RESTORE", - true, 2, {"ls_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_LS_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "START_LS_RESTORE", "RESTORE", - true, 2, {"ls_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_SYS_TABLETS_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "SYS_TABLETS_RESTORE", "RESTORE", - true, 2, {"ls_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_DATA_TABLETS_META_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "DATA_TABLETS_META_RESTORE", "RESTORE", - true, 2, {"ls_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_GROUP_META_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "TABLET_GROUP_META_RESTORE", "RESTORE", - true, 3, {"ls_id", "first_tablet_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_LS_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "FINISH_LS_RESTORE", "RESTORE", - true, 2, {"ls_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_TABLET_GROUP_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "INITIAL_TABLET_GROUP_RESTORE", "RESTORE", - true, 3, {"ls_id", "first_tablet_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_TABLET_GROUP_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "START_TABLET_GROUP_RESTORE", "RESTORE", - true, 3, {"ls_id", "first_tablet_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_TABLET_GROUP_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "FINISH_TABLET_GROUP_RESTORE", "RESTORE", - true, 3, {"ls_id", "first_tablet_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "TABLET_RESTORE", "RESTORE", - true, 3, {"ls_id", "tablet_id", "is_leader"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_COMPLETE_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "INITIAL_COMPLETE_RESTORE", "RESTORE", - true, 2, {"task_type", "src_info"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_WAIT_DATA_READY_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "WAIT_DATA_READY_RESTORE", "RESTORE", - true, 2, {"task_type", "src_info"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_COMPLETE_RESTORE, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::RESTORE_TASK, "FINISH_COMPLETE_RESTORE", "RESTORE", - true, 2, {"task_type", "src_info"}) -// DAG_TYPE_RESTORE END - -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_BACKUP_CLEAN, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::BACKUP_CLEAN_TASK, "BACKUP_CLEAN", "BACKUP_CLEAN", - true, 3, {"task_id", "ls_id", "id"}) - -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_REMOVE_MEMBER, ObDagPrio::DAG_PRIO_HA_MID, ObSysTaskType::REMOVE_MEMBER_TASK, "REMOVE_MEMBER", "REMOVE_MEMBER", - true, 1, {"ls_id"}) - -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_RESERVED_0, ObDagPrio::DAG_PRIO_MAX, ObSysTaskType::MAX_SYS_TASK_TYPE, "RESERVED_0", "INVALID", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_RESERVED_1, ObDagPrio::DAG_PRIO_MAX, ObSysTaskType::MAX_SYS_TASK_TYPE, "RESERVED_1", "INVALID", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_RESERVED_2, ObDagPrio::DAG_PRIO_MAX, ObSysTaskType::MAX_SYS_TASK_TYPE, "RESERVED_2", "INVALID", - false, 0, {}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TTL, ObDagPrio::DAG_PRIO_TTL, ObSysTaskType::TTL_TASK, "TTL_DELETE_DAG", "TTL", - false, 3, {"ls_id", "table_id", "tablet_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TENANT_SNAPSHOT_CREATE, ObDagPrio::DAG_PRIO_HA_MID, ObSysTaskType::TENANT_SNAPSHOT_CREATE_TASK, "TENANT_SNAPSHOT_CREATE", "TSNAP_CR8", - false, 1, {"tsnap_id"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TENANT_SNAPSHOT_GC, ObDagPrio::DAG_PRIO_HA_LOW, ObSysTaskType::TENANT_SNAPSHOT_GC_TASK, "TENANT_SNAPSHOT_GC","TSNAP_GC", - false, 1, {"tsnap_id"}) + true, 1, {"tablet_id"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_VECTOR_INDEX, ObDagPrio::DAG_PRIO_TTL, ObSysTaskType::VECTOR_INDEX_TASK, "VECTOR_INDEX_DAG", "VECTOR_INDEX", - false, 3, {"ls_id", "table_id", "tablet_id"}) - -// DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_REBUILD_TABLET, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::REBUILD_TABLET_TASK, "REBUILD_TABLET", "REBUILD_TABLET") -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_INITIAL_REBUILD_TABLET, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::REBUILD_TABLET_TASK, "INITIAL_REBUILD_TBALET", "REBUILD_TABLET", - false, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_REBUILD_TABLET, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::REBUILD_TABLET_TASK, "START_REBUILD_TABLET", "REBUILD_TABLET", - false, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_REBUILD_TABLET, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::REBUILD_TABLET_TASK, "FINISH_REBUILD_TABLET", "REBUILD_TABLET", - false, 2, {"ls_id", "op_type"}) -DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_REBUILD, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::REBUILD_TABLET_TASK, "TABLET_REBUILD", "REBUILD_TABLET", - false, 2, {"ls_id", "op_type"}) -// DAG_TYPE_REBUILD_TABLET END + false, 2, {"table_id", "tablet_id"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_MAX, ObDagPrio::DAG_PRIO_MAX, ObSysTaskType::MAX_SYS_TASK_TYPE, "DAG_TYPE_MAX", "INVALID", false, 0, {}) #endif @@ -267,15 +125,6 @@ struct ObDagPrio #define DAG_SCHEDULER_DAG_PRIO_DEF(dag_prio, score, dag_prio_str, dag_function_type) dag_prio, #include "ob_dag_scheduler_config.h" #undef DAG_SCHEDULER_DAG_PRIO_DEF - // HA_MID / HA_LOW are folded into HA_HIGH in observer-lite: the dag types - // mapped to them (FAST_MIGRATE / BACKUP_* / VALIDATE / TENANT_SNAPSHOT_*) are - // all runtime-stubbed (OB_NOT_SUPPORTED), so they need no dedicated scheduler. - // Keeping them as aliases lets those dag-type definitions and backup code - // compile unchanged while routing any such dag to the HA_HIGH scheduler. - // These aliases must NOT appear in the X-macro list above, otherwise they - // would add duplicate slots to OB_DAG_PRIOS[] / prio_sche_[DAG_PRIO_MAX]. - DAG_PRIO_HA_MID = DAG_PRIO_HA_HIGH, - DAG_PRIO_HA_LOW = DAG_PRIO_HA_HIGH, }; }; diff --git a/src/observer/scheduler/ob_partition_auto_split_helper.cpp b/src/observer/scheduler/ob_partition_auto_split_helper.cpp index 45196598c..b25d1f2bc 100644 --- a/src/observer/scheduler/ob_partition_auto_split_helper.cpp +++ b/src/observer/scheduler/ob_partition_auto_split_helper.cpp @@ -72,7 +72,6 @@ int ObAutoSplitTask::assign(const ObAutoSplitTask &other) LOG_WARN("invalid argument", K(ret), K(other)); } else { auto_split_tablet_size_ = other.auto_split_tablet_size_; - ls_id_ = other.ls_id_; retry_times_ = other.retry_times_; tablet_id_ = other.tablet_id_; used_disk_space_ = other.used_disk_space_; @@ -688,7 +687,6 @@ int ObServerAutoSplitScheduler::check_sstable_limit(const storage::ObTablet &tab } int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage::ObTabletHandle &tablet_handle, - storage::ObLS &ls, bool &can_split, ObAutoSplitTask &task) { @@ -698,8 +696,6 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: int64_t real_auto_split_size = OB_INVALID_SIZE; ObTablet *tablet = nullptr; ObTabletPointer *tablet_ptr = nullptr; - ObRole role = INVALID_ROLE; - const share::ObLSID ls_id = ls.get_ls_id(); bool num_sstables_exceed_limit = false; can_split = false; task.reset(); @@ -708,9 +704,9 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: mds::TwoPhaseCommitState trans_stat;// will be removed later share::SCN trans_version;// will be removed later - if (OB_UNLIKELY(!tablet_handle.is_valid() || !ls_id.is_valid())) { + if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_handle), K(ls_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_handle)); } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("pointer to tablet is nullptr", K(ret), KP(tablet)); @@ -742,8 +738,6 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: can_split = false; } else if (OB_FAIL(check_sstable_limit(*tablet, num_sstables_exceed_limit))) { LOG_WARN("fail to check sstable limit", K(ret), KPC(tablet)); - } else if (OB_FAIL(ls.get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(ls_id)); } else if (OB_FAIL(check_tablet_creation_limit(ObAutoSplitArgBuilder::get_max_split_partition_num(), 0.8/*safe_ratio*/, auto_split_tablet_size, real_auto_split_size))) { LOG_WARN("check_create_new_tablets fail", K(ret)); if (OB_TOO_MANY_PARTITIONS_ERROR == ret) { @@ -752,7 +746,7 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: } } else { can_split = tablet->get_major_table_count() > 0 && tablet->get_data_tablet_id() == tablet->get_tablet_id() - && common::ObRole::LEADER == role && !num_sstables_exceed_limit; + && !num_sstables_exceed_limit; // TODO gaishun.gs resident_info const int64_t used_disk_space = tablet->get_tablet_meta().space_usage_.all_sstable_data_required_size_; can_split &= (used_disk_space > real_auto_split_size); @@ -769,7 +763,6 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: can_split = false; } else if ((can_split = user_data.get_tablet_status() == ObTabletStatus::Status::NORMAL && (medium_info_list->size() == 0))) { - task.ls_id_ = ls_id; task.tablet_id_ = tablet->get_tablet_id(); task.auto_split_tablet_size_ = auto_split_tablet_size; task.used_disk_space_ = used_disk_space; @@ -780,7 +773,7 @@ int ObServerAutoSplitScheduler::check_and_fetch_tablet_split_info(const storage: return ret; } -int ObServerAutoSplitScheduler::push_task(const storage::ObTabletHandle &tablet_handle, oceanbase::storage::ObLS &ls) +int ObServerAutoSplitScheduler::push_task(const storage::ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; ObArray task_array; @@ -789,7 +782,7 @@ int ObServerAutoSplitScheduler::push_task(const storage::ObTabletHandle &tablet_ if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_handle)); - } else if (OB_FAIL(check_and_fetch_tablet_split_info(tablet_handle, ls, can_split, task))) { + } else if (OB_FAIL(check_and_fetch_tablet_split_info(tablet_handle, can_split, task))) { if (OB_UNLIKELY(OB_NOT_SUPPORTED != ret)) { LOG_ERROR("failed to check and fetch tablet split info", K(ret), K(task)); } @@ -831,7 +824,6 @@ int ObServerAutoSplitScheduler::batch_send_split_request(const ObArray ranges_num ? @@ -1253,7 +1242,7 @@ int ObAutoSplitArgBuilder::acquire_table_id_of_tablets(const ObIArray= 0 + return tablet_id_.is_valid() && retry_times_ >= 0 && auto_split_tablet_size_ > 0 && used_disk_space_ > 0 && used_disk_space_ > auto_split_tablet_size_; } void reset() @@ -82,13 +81,11 @@ struct ObAutoSplitTask final auto_split_tablet_size_ = OB_INVALID_SIZE; used_disk_space_ = OB_INVALID_SIZE; retry_times_ = OB_INVALID_COUNT; - ls_id_.reset(); tablet_id_.reset(); } int assign(const ObAutoSplitTask &other); void increment_retry_times() { ++retry_times_; } public: - ObLSID ls_id_; ObTabletID tablet_id_; int64_t auto_split_tablet_size_; int64_t used_disk_space_; @@ -198,8 +195,7 @@ class ObAutoSplitArgBuilder final ObAutoSplitArgBuilder() {} ~ObAutoSplitArgBuilder() {} inline static int32_t get_max_split_partition_num() { return MAX_SPLIT_PARTITION_NUM; } - int build_arg(const share::ObLSID ls_id, - const ObTabletID tablet_id, + int build_arg(const ObTabletID tablet_id, const int64_t auto_split_tablet_size, const int64_t used_disk_space, obcall::ObAlterTableArg &arg); @@ -317,7 +313,7 @@ class ObServerAutoSplitScheduler final const static int64_t OB_SERVER_DELAYED_TIME = (10 * 1000L * 1000L); //10s public: static ObServerAutoSplitScheduler &get_instance(); - int push_task(const storage::ObTabletHandle &teblet_handle, storage::ObLS &ls); + int push_task(const storage::ObTabletHandle &teblet_handle); int init() { return polling_manager_.init(); } void reset() { polling_manager_.reset(); } static int cal_real_auto_split_size(const double base_ratio, const double cur_ratio, const int64_t auto_split_size, int64_t &real_auto_split_size); @@ -328,7 +324,7 @@ class ObServerAutoSplitScheduler final {} ~ObServerAutoSplitScheduler () {} int batch_send_split_request(const ObArray> &task_array); - int check_and_fetch_tablet_split_info(const storage::ObTabletHandle &teblet_handle, storage::ObLS &ls, bool &can_split, ObAutoSplitTask &task); + int check_and_fetch_tablet_split_info(const storage::ObTabletHandle &teblet_handle, bool &can_split, ObAutoSplitTask &task); int check_sstable_limit(const storage::ObTablet &tablet, bool &exceed_limit); private: const static int64_t MAX_SPLIT_RPC_IN_BATCH = 20; diff --git a/src/observer/scheduler/ob_sys_task_stat.cpp b/src/observer/scheduler/ob_sys_task_stat.cpp index cd20941f3..3c55cc304 100644 --- a/src/observer/scheduler/ob_sys_task_stat.cpp +++ b/src/observer/scheduler/ob_sys_task_stat.cpp @@ -40,7 +40,7 @@ const static char *ObSysTaskTypeStr[] = { "BACKUP_CLEAN", "BACKFILL_TX", "REMOVE_MEMBER", - "RESERVED_SYS_TASK_TYPE_18", + "RESERVED", "MDS_MINI_MERGE", "TTL_TASK", "TENANT_SNAPSHOT_CREATE", diff --git a/src/observer/scheduler/ob_sys_task_stat.h b/src/observer/scheduler/ob_sys_task_stat.h index 43bae01bc..7541a0db1 100644 --- a/src/observer/scheduler/ob_sys_task_stat.h +++ b/src/observer/scheduler/ob_sys_task_stat.h @@ -53,7 +53,7 @@ enum ObSysTaskType BACKUP_CLEAN_TASK, BACKFILL_TX_TASK, REMOVE_MEMBER_TASK, - RESERVED_SYS_TASK_TYPE_18 = 18, + RESERVED_TASK, MDS_MINI_MERGE_TASK, TTL_TASK, TENANT_SNAPSHOT_CREATE_TASK, diff --git a/src/observer/scheduler/ob_tenant_dag_scheduler.cpp b/src/observer/scheduler/ob_tenant_dag_scheduler.cpp index da703ec2f..fc3841fda 100644 --- a/src/observer/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/observer/scheduler/ob_tenant_dag_scheduler.cpp @@ -309,7 +309,7 @@ const char *ObITask::ObITaskTypeStr[] = { "FAST_MIGRATE_ASYNC_TASK", "VALIDATE_BACKUP", "VALIDATE_FINISH", - "BUILD_CHANGE_REPLICA", + "RESERVED_TASK_TYPE_39", "RESTORE_TAILORED_PREPARE", "RESTORE_TAILORED_PROCESS", "RESTORE_TAILORED_FINISH", @@ -321,8 +321,8 @@ const char *ObITask::ObITaskTypeStr[] = { "BACKUP_CLEAN", "DDL_KV_DUMP", "DDL_KV_MERGE", - "RESERVED_51", - "RESERVED_52", + "RESERVED_TASK_TYPE_51", + "RESERVED_TASK_TYPE_52", "MDS_MINI_MERGE", "TTL_DELETE", "TENANT_SNAPSHOT_CREATE", @@ -352,7 +352,7 @@ const char *ObITask::ObITaskTypeStr[] = { "CHECK_CONVERT_TABLET", "VECTOR_INDEX_MEMDATA_SYNC", "DELETE_LOB_META_ROW", - "RESERVED_82", + "RESERVED_TASK_TYPE_82", "BACKUP_LS_LOG_GROUP", "BACKUP_LS_LOG", "BACKUP_LS_LOG_FILE", @@ -371,13 +371,13 @@ const char *ObITask::ObITaskTypeStr[] = { "SS_MIGRATE_START", "MIGRATE_START_FINISH", "SS_MIGRATE_FINISH", - "RESERVED_101", - "RESERVED_102", - "RESERVED_103", - "RESERVED_104", - "RESERVED_105", - "RESERVED_106", - "RESERVED_107", + "RESERVED_TASK_TYPE_101", + "RESERVED_TASK_TYPE_102", + "RESERVED_TASK_TYPE_103", + "RESERVED_TASK_TYPE_104", + "RESERVED_TASK_TYPE_105", + "RESERVED_TASK_TYPE_106", + "RESERVED_TASK_TYPE_107", "MIGRATE_START_PHYSICAL", "SS_PHYSICAL_CREATE_TABLETS_CONSUMER", "UNUSED", @@ -440,7 +440,7 @@ const char *ObITask::ObITaskTypeStr[] = { "DIRECT_LOAD_INSERT_SSTABLE_CLEAR", "DIRECT_LOAD_COMPACT_SSTABLE_CLEAR", "DIRECT_LOAD_INC_MAJOR_UPDATE_SS_INC_MAJOR", - "RESERVED_170", + "RESERVED_TASK_TYPE_170", "DDL_FORK_PREPARE", "DDL_FORK_REUSE", "DDL_FORK_REWRITE", @@ -1817,17 +1817,6 @@ const char *ObIDag::get_dag_module_str(const enum ObDagType::ObDagTypeEnum type) } -const char *ObIDagNet::get_dag_net_type_str(ObDagNetType::ObDagNetTypeEnum type) -{ - const char *str = ""; - if (type >= ObDagNetType::DAG_NET_TYPE_MAX || type < ObDagNetType::DAG_NET_TYPE_MIGRATION) { - str = "invalid_type"; - } else { - str = OB_DAG_NET_TYPES[type].dag_net_type_str_; - } - return str; -} - ObDagInfo::ObDagInfo() : dag_type_(ObDagType::DAG_TYPE_MAX), dag_net_type_(ObDagNetType::DAG_NET_TYPE_MAX), @@ -1869,10 +1858,8 @@ bool ObDagInfo::is_valid() const && add_time_ >= 0 && start_time_ >= 0 && indegree_ >= 0 - && ((dag_net_type_ >= ObDagNetType::DAG_NET_TYPE_MIGRATION - && dag_net_type_ < ObDagNetType::DAG_NET_TYPE_MAX) - || (dag_type_ >= share::ObDagType::DAG_TYPE_MINI_MERGE - && dag_type_ < share::ObDagType::DAG_TYPE_MAX))); + && dag_type_ >= share::ObDagType::DAG_TYPE_MINI_MERGE + && dag_type_ < share::ObDagType::DAG_TYPE_MAX); } const char *ObDagSchedulerInfo::ObValueTypeStr[VALUE_TYPE_MAX] = @@ -2843,15 +2830,15 @@ int ObDagPrioScheduler::finish_dag_( compaction::ObTabletMergeDag &merge_dag = static_cast(*dag); if (OB_SUCCESS != dag->get_dag_ret()) { if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet( - merge_dag.ls_id_, merge_dag.tablet_id_, ObIDag::get_diagnose_tablet_type(dag->get_type())))) { + merge_dag.tablet_id_, ObIDag::get_diagnose_tablet_type(dag->get_type())))) { COMMON_LOG(WARN, "failed to add diagnose tablet", K(tmp_ret), - "ls_id", merge_dag.ls_id_, "tablet_id", merge_dag.tablet_id_); + "tablet_id", merge_dag.tablet_id_); } } else if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->delete_diagnose_tablet( - merge_dag.ls_id_, merge_dag.tablet_id_, ObIDag::get_diagnose_tablet_type(dag->get_type())))) { + merge_dag.tablet_id_, ObIDag::get_diagnose_tablet_type(dag->get_type())))) { if (OB_HASH_NOT_EXIST != tmp_ret) { COMMON_LOG(ERROR, "failed to delete diagnose tablet", K(tmp_ret), - "ls_id", merge_dag.ls_id_, "tablet_id", merge_dag.tablet_id_); + "tablet_id", merge_dag.tablet_id_); } } } @@ -3221,9 +3208,7 @@ void ObDagPrioScheduler::add_compaction_info( } } -int ObDagPrioScheduler::check_ls_compaction_dag_exist_with_cancel( - const ObLSID &ls_id, - bool &exist) +int ObDagPrioScheduler::check_compaction_dag_exist_with_cancel(bool &exist) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -3243,9 +3228,9 @@ int ObDagPrioScheduler::check_ls_compaction_dag_exist_with_cancel( if (OB_ISNULL(cur)) { // do nothing } else if (ObDagType::DAG_TYPE_BATCH_FREEZE_TABLETS == cur->get_type()) { - cancel_flag = (ls_id == static_cast(cur)->get_param().ls_id_); + cancel_flag = true; } else { - cancel_flag = (ls_id == static_cast(cur)->get_ls_id()); + cancel_flag = true; } if (cancel_flag) { @@ -3275,12 +3260,12 @@ int ObDagPrioScheduler::check_ls_compaction_dag_exist_with_cancel( } } if (OB_SUCC(ret)) { - COMMON_LOG(INFO, "cancel dag when check ls compaction dag exist", KR(ret), K(ls_id), K(cancel_dag_cnt), K(exist)); + COMMON_LOG(INFO, "cancel dag when checking compaction dag", KR(ret), K(cancel_dag_cnt), K(exist)); } return OB_SUCCESS; } -int ObDagPrioScheduler::get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn) +int ObDagPrioScheduler::get_min_end_scn_from_major_dag(SCN &min_end_scn) { int ret = OB_SUCCESS; ObDagListIndex loop_list[2] = { READY_DAG_LIST, RANK_DAG_LIST }; @@ -3295,8 +3280,7 @@ int ObDagPrioScheduler::get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN SCN tmp_min_end_scn = SCN::max_scn(); if (ObDagType::DAG_TYPE_MAJOR_MERGE == cur->get_type()) { compaction::ObTabletMergeDag * dag = static_cast(cur); - if (ls_id == dag->get_ls_id() - && cur->get_dag_status() != ObIDag::DAG_STATUS_READY) { + if (cur->get_dag_status() != ObIDag::DAG_STATUS_READY) { if (OB_FAIL(dag->get_min_sstable_end_scn(tmp_min_end_scn))) { LOG_WARN("failed to get min end scn from dag", KR(ret), KPC(cur)); } else { @@ -3309,7 +3293,7 @@ int ObDagPrioScheduler::get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN } // for if (OB_SUCC(ret)) { min_end_scn = dag_min_end_scn; - LOG_INFO("[MIN_END_SCN]success to get min end scn from major dag", KR(ret), K(ls_id), K(dag_min_end_scn)); + LOG_INFO("[MIN_END_SCN]success to get min end scn from major dag", KR(ret), K(dag_min_end_scn)); } return ret; } @@ -3437,10 +3421,9 @@ int ObDagPrioScheduler::diagnose_compaction_dags() } else if (OB_ISNULL(merge_dag = static_cast(dag))) { tmp_ret = OB_ERR_UNEXPECTED; COMMON_LOG(WARN, "get unexpected null stored dag", K(tmp_ret), KPC(dag)); - } else if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(merge_dag->ls_id_, - merge_dag->tablet_id_, + } else if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(merge_dag->tablet_id_, ObIDag::get_diagnose_tablet_type(dag->get_type())))) { - COMMON_LOG(WARN, "failed to add diagnose tablet", K(tmp_ret), "ls_id", merge_dag->ls_id_, "tablet_id", merge_dag->tablet_id_); + COMMON_LOG(WARN, "failed to add diagnose tablet", K(tmp_ret), "tablet_id", merge_dag->tablet_id_); } else { COMMON_LOG(TRACE, "dag maybe abormal", KPC(merge_dag)); } @@ -4138,20 +4121,19 @@ int ObDagNetScheduler::get_first_dag_net(ObIDagNet *&dag_net) return ret; } -int ObDagNetScheduler::check_ls_compaction_dag_exist_with_cancel(const ObLSID &ls_id, bool &exist) +int ObDagNetScheduler::check_compaction_dag_exist_with_cancel(bool &exist) { int ret = OB_SUCCESS; - UNUSED(ls_id); exist = false; - LOG_INFO("skip compaction dag net cancel", KR(ret), K(ls_id), K(exist)); + LOG_INFO("skip compaction dag net cancel", KR(ret), K(exist)); return ret; } -int ObDagNetScheduler::get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn) +int ObDagNetScheduler::get_min_end_scn_from_major_dag(SCN &min_end_scn) { int ret = OB_SUCCESS; min_end_scn = SCN::max_scn(); - LOG_INFO("[MIN_END_SCN]skip compaction dag net", KR(ret), K(ls_id), K(min_end_scn)); + LOG_INFO("[MIN_END_SCN]skip compaction dag net", KR(ret), K(min_end_scn)); return ret; } @@ -4706,7 +4688,7 @@ int ObTenantDagScheduler::get_minor_exe_dag_info( return ret; } -int ObTenantDagScheduler::check_ls_compaction_dag_exist_with_cancel(const ObLSID &ls_id, bool &exist) +int ObTenantDagScheduler::check_compaction_dag_exist_with_cancel(bool &exist) { int ret = OB_SUCCESS; exist = false; @@ -4719,8 +4701,8 @@ int ObTenantDagScheduler::check_ls_compaction_dag_exist_with_cancel(const ObLSID * - Need cancel dag for the final desctruction of dag net. * - Need cancel the dag net to avoid scheduling more dags. */ - if (OB_FAIL(dag_net_sche_.check_ls_compaction_dag_exist_with_cancel(ls_id, tmp_exist))) { - LOG_WARN("failed to check ls compaction dag exist", K(ret), K(ls_id)); + if (OB_FAIL(dag_net_sche_.check_compaction_dag_exist_with_cancel(tmp_exist))) { + LOG_WARN("failed to check compaction dag exist", K(ret)); } else if (tmp_exist) { exist = true; } @@ -4728,8 +4710,8 @@ int ObTenantDagScheduler::check_ls_compaction_dag_exist_with_cancel(const ObLSID } else { for (int64_t i = 0; OB_SUCC(ret) && i < ObIDag::MergeDagPrioCnt; ++i) { tmp_exist = false; - if (OB_FAIL(prio_sche_[ObIDag::MergeDagPrio[i]].check_ls_compaction_dag_exist_with_cancel(ls_id, tmp_exist))) { - LOG_WARN("failed to check ls compaction dag exist", K(ret), K(ls_id)); + if (OB_FAIL(prio_sche_[ObIDag::MergeDagPrio[i]].check_compaction_dag_exist_with_cancel(tmp_exist))) { + LOG_WARN("failed to check compaction dag exist", K(ret)); } else if (tmp_exist) { exist = true; } @@ -4738,22 +4720,21 @@ int ObTenantDagScheduler::check_ls_compaction_dag_exist_with_cancel(const ObLSID return ret; } -int ObTenantDagScheduler::get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn) +int ObTenantDagScheduler::get_min_end_scn_from_major_dag(SCN &min_end_scn) { int ret = OB_SUCCESS; SCN min_end_scn_from_dag_net = SCN::max_scn(); SCN min_end_scn_from_dag = SCN::max_scn(); - if (OB_FAIL(dag_net_sche_.get_min_end_scn_from_major_dag(ls_id, min_end_scn_from_dag_net))) { - LOG_WARN("failed to check ls compaction dag exist", K(ret), K(ls_id)); - } else if (OB_FAIL(prio_sche_[ObDagPrio::DAG_PRIO_COMPACTION_LOW].get_min_end_scn_from_major_dag(ls_id, min_end_scn_from_dag))) { - LOG_WARN("failed to check ls compaction dag exist", K(ret), K(ls_id)); + if (OB_FAIL(dag_net_sche_.get_min_end_scn_from_major_dag(min_end_scn_from_dag_net))) { + LOG_WARN("failed to check compaction dag exist", K(ret)); + } else if (OB_FAIL(prio_sche_[ObDagPrio::DAG_PRIO_COMPACTION_LOW].get_min_end_scn_from_major_dag(min_end_scn_from_dag))) { + LOG_WARN("failed to check compaction dag exist", K(ret)); } else { min_end_scn = std::min(min_end_scn_from_dag, min_end_scn_from_dag_net); #ifdef ERRSIM if (!min_end_scn.is_max()) { SERVER_EVENT_SYNC_ADD("merge_errsim", "get_min_end_scn", - "ls_id", ls_id.id(), - "min_end_scn", min_end_scn.get_val_for_tx()); + "min_end_scn", min_end_scn.get_val_for_tx()); } #endif } diff --git a/src/observer/scheduler/ob_tenant_dag_scheduler.h b/src/observer/scheduler/ob_tenant_dag_scheduler.h index bd9fc2020..de3deee92 100644 --- a/src/observer/scheduler/ob_tenant_dag_scheduler.h +++ b/src/observer/scheduler/ob_tenant_dag_scheduler.h @@ -197,7 +197,7 @@ class ObITask : public common::ObDLinkBase, public ObINodeWithChild TASK_TYPE_FAST_MIGRATE_ASYNC_TASK = 36, TASK_TYPE_VALIDATE_BACKUP = 37, TASK_TYPE_VALIDATE_FINISH = 38, - TASK_TYPE_BUILD_CHANGE_REPLICA = 39, + TASK_TYPE_RESERVED_39 = 39, TASK_TYPE_RESTORE_TAILORED_PREPARE = 40, TASK_TYPE_RESTORE_TAILORED_PROCESS = 41, TASK_TYPE_RESTORE_TAILORED_FINISH = 42, @@ -1008,8 +1008,8 @@ class ObDagNetScheduler { const ObDagId &dag_id, bool &exist); int cancel_dag_net(const ObDagId &dag_id); int get_first_dag_net(ObIDagNet *&dag_net); - int check_ls_compaction_dag_exist_with_cancel(const ObLSID &ls_id, bool &exist); - int get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn); + int check_compaction_dag_exist_with_cancel(bool &exist); + int get_min_end_scn_from_major_dag(SCN &min_end_scn); private: bool is_dag_map_full_(); typedef common::ObDList DagNetList; @@ -1127,8 +1127,8 @@ class ObDagPrioScheduler // 1. check ls compaction exist // 2. cancel ls compaction waiting dag - int check_ls_compaction_dag_exist_with_cancel(const ObLSID &ls_id, bool &exist); - int get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn); + int check_compaction_dag_exist_with_cancel(bool &exist); + int get_min_end_scn_from_major_dag(SCN &min_end_scn); int get_compaction_dag_count(int64_t dag_count); int get_max_major_finish_time(const int64_t version, int64_t &estimated_finish_time); int diagnose_dag( @@ -1384,8 +1384,8 @@ class ObTenantDagScheduler : public lib::ThreadPool // 1. check ls compaction exist // 2. cancel ls compaction waiting dag - int check_ls_compaction_dag_exist_with_cancel(const ObLSID &ls_id, bool &exist); - int get_min_end_scn_from_major_dag(const ObLSID &ls_id, SCN &min_end_scn); + int check_compaction_dag_exist_with_cancel(bool &exist); + int get_min_end_scn_from_major_dag(SCN &min_end_scn); int check_dag_net_exist( const ObDagId &dag_id, bool &exist); int cancel_dag_net(const ObDagId &dag_id); diff --git a/src/observer/table_load/control/ob_table_load_control_rpc_struct.h b/src/observer/table_load/control/ob_table_load_control_rpc_struct.h index dd7817e5d..e90fab3ce 100644 --- a/src/observer/table_load/control/ob_table_load_control_rpc_struct.h +++ b/src/observer/table_load/control/ob_table_load_control_rpc_struct.h @@ -197,8 +197,8 @@ class ObDirectLoadControlPreBeginArg final bool online_opt_stat_gather_; ObTableLoadDDLParam ddl_param_; // partition info - table::ObTableLoadArray partition_id_array_; // origin table - table::ObTableLoadArray target_partition_id_array_; // target table + table::ObTableLoadArray partition_id_array_; // origin table + table::ObTableLoadArray target_partition_id_array_; // target table sql::ObSQLSessionInfo *session_info_; sql::ObFreeSessionCtx free_session_ctx_; int64_t avail_memory_; @@ -491,8 +491,8 @@ class ObDirectLoadControlInitEmptyTabletsArg final public: uint64_t table_id_; ObTableLoadDDLParam ddl_param_; - ObSArray partition_id_array_; // origin table - ObSArray target_partition_id_array_; // target table + ObSArray partition_id_array_; // origin table + ObSArray target_partition_id_array_; // target table }; } // namespace observer diff --git a/src/observer/table_load/dag/ob_table_load_dag_direct_write_channel.cpp b/src/observer/table_load/dag/ob_table_load_dag_direct_write_channel.cpp index f59907ffb..cc7691721 100644 --- a/src/observer/table_load/dag/ob_table_load_dag_direct_write_channel.cpp +++ b/src/observer/table_load/dag/ob_table_load_dag_direct_write_channel.cpp @@ -135,10 +135,10 @@ int ObTableLoadDagDirectChunkWriter::init(ObTableLoadDagWriteChannel *write_chan write_channel_ = static_cast(write_channel); max_batch_size_ = store_ctx_->ctx_->param_.batch_size_; const int64_t tablet_cnt = - write_channel_->op_->op_ctx_->store_table_ctx_->ls_partition_ids_.count(); + write_channel_->op_->op_ctx_->store_table_ctx_->partition_ids_.count(); if (1 == tablet_cnt) { is_single_part_ = true; - single_tablet_id_ = write_channel_->op_->op_ctx_->store_table_ctx_->ls_partition_ids_[0] + single_tablet_id_ = write_channel_->op_->op_ctx_->store_table_ctx_->partition_ids_[0] .part_tablet_id_.tablet_id_; } if (OB_FAIL(batch_writer_map_.create(64, "TLD_BW_Map", "TLD_BW_Map"))) { diff --git a/src/observer/table_load/dag/ob_table_load_dag_parallel_merger.cpp b/src/observer/table_load/dag/ob_table_load_dag_parallel_merger.cpp index b5041f829..2256657b4 100644 --- a/src/observer/table_load/dag/ob_table_load_dag_parallel_merger.cpp +++ b/src/observer/table_load/dag/ob_table_load_dag_parallel_merger.cpp @@ -52,7 +52,7 @@ int ObTableLoadDagParallelMerger::init_merge_ctx() param.trans_param_ = op_ctx_->insert_table_ctx_->get_param().trans_param_; param.file_mgr_ = store_ctx_->tmp_file_mgr_; param.ctx_ = store_ctx_->ctx_; - if (OB_FAIL(merge_ctx_.init(param, store_table_ctx->ls_partition_ids_))) { + if (OB_FAIL(merge_ctx_.init(param, store_table_ctx->partition_ids_))) { LOG_WARN("fail to init merge ctx", KR(ret), K(param)); } return ret; diff --git a/src/observer/table_load/dag/ob_table_load_dag_store_write_channel.cpp b/src/observer/table_load/dag/ob_table_load_dag_store_write_channel.cpp index dc6b75dff..a600a19c3 100644 --- a/src/observer/table_load/dag/ob_table_load_dag_store_write_channel.cpp +++ b/src/observer/table_load/dag/ob_table_load_dag_store_write_channel.cpp @@ -136,7 +136,7 @@ int ObTableLoadDagStoreChunkWriter::init(ObTableLoadDagWriteChannel *write_chann store_ctx_ = write_channel->store_ctx_; is_single_part_ = (store_ctx_->write_ctx_.is_multiple_mode_ || - 1 == write_channel_->op_->op_ctx_->store_table_ctx_->ls_partition_ids_.count()); + 1 == write_channel_->op_->op_ctx_->store_table_ctx_->partition_ids_.count()); if (OB_FAIL(table_builder_map_.create(64, "TLD_TB_Map", "TLD_TB_Map"))) { LOG_WARN("fail to create hashmap", KR(ret)); } else if (OB_FAIL(datum_row_.init( diff --git a/src/observer/table_load/ob_table_load_assigned_task_manager.cpp b/src/observer/table_load/ob_table_load_assigned_task_manager.cpp deleted file mode 100644 index 181b2b58b..000000000 --- a/src/observer/table_load/ob_table_load_assigned_task_manager.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER - -#include "observer/table_load/ob_table_load_assigned_task_manager.h" - -namespace oceanbase -{ -namespace observer -{ -using namespace common; -using namespace common::hash; -using namespace lib; -using namespace table; - -/** - * ObTableLoadAssignedTaskManager - */ -ObTableLoadAssignedTaskManager::ObTableLoadAssignedTaskManager() - : is_inited_(false) -{ -} - -ObTableLoadAssignedTaskManager::~ObTableLoadAssignedTaskManager() -{ -} - -int ObTableLoadAssignedTaskManager::init() -{ - int ret = OB_SUCCESS; - const int64_t bucket_num = 1024; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObTableLoadAssignedTaskManager init twice", KR(ret), KP(this)); - } else if (OB_FAIL(assigned_tasks_map_.create(bucket_num, "TLD_AssignedMgr", "TLD_AssignedMgr"))) { - LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num)); - } else { - is_inited_ = true; - } - - return ret; -} - -int ObTableLoadAssignedTaskManager::add_assigned_task(ObDirectLoadResourceApplyArg &arg) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadAssignedTaskManager not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); - } else { - ObMutexGuard guard(mutex_); - if (OB_FAIL(assigned_tasks_map_.set_refactored(arg.task_key_, arg))) { - if (OB_UNLIKELY(OB_HASH_EXIST != ret)) { - LOG_WARN("fail to set refactored", KR(ret), K(arg.task_key_)); - } else { - ret = OB_ENTRY_EXIST; - } - } - } - LOG_INFO("ObTableLoadAssignedTaskManager::add_assigned_task", KR(ret), K(arg)); - - return ret; -} - -int ObTableLoadAssignedTaskManager::delete_assigned_task(ObTableLoadUniqueKey &task_key) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadAssignedTaskManager not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!task_key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(task_key)); - } else { - ObMutexGuard guard(mutex_); - if (OB_FAIL(assigned_tasks_map_.erase_refactored(task_key))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } - } - } - LOG_INFO("ObTableLoadAssignedTaskManager::delete_assigned_task", KR(ret), K(task_key)); - - return ret; -} - -int ObTableLoadAssignedTaskManager::get_assigned_tasks(ObSArray &assigned_tasks) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadAssignedTaskManager not init", KR(ret), KP(this)); - } else { - ObMutexGuard guard(mutex_); - if (OB_FAIL(assigned_tasks.reserve(assigned_tasks_map_.size()))) { - LOG_WARN("fail to reserve assigned_tasks", KR(ret), K(assigned_tasks)); - } else { - for(ResourceApplyMap::iterator iter = assigned_tasks_map_.begin(); iter != assigned_tasks_map_.end(); iter++) { - if (OB_FAIL(assigned_tasks.push_back(iter->second))) { - LOG_WARN("fail to push_back", KR(ret), K(assigned_tasks)); - break; - } - } - } - } - - return ret; -} - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_assigned_task_manager.h b/src/observer/table_load/ob_table_load_assigned_task_manager.h deleted file mode 100644 index 46df198ea..000000000 --- a/src/observer/table_load/ob_table_load_assigned_task_manager.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "lib/hash/ob_hashmap.h" -#include "lib/lock/ob_mutex.h" -#include "observer/table_load/ob_table_load_struct.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc_struct.h" - - -namespace oceanbase -{ -namespace observer -{ - -class ObTableLoadAssignedTaskManager -{ -public: - ObTableLoadAssignedTaskManager(); - ~ObTableLoadAssignedTaskManager(); - int init(); - int add_assigned_task(ObDirectLoadResourceApplyArg &arg); - int delete_assigned_task(ObTableLoadUniqueKey &task_key); - int get_assigned_tasks(common::ObSArray &assigned_tasks); -private: - typedef common::hash::ObHashMap - ResourceApplyMap; - ResourceApplyMap assigned_tasks_map_; - mutable lib::ObMutex mutex_; - bool is_inited_; -}; - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_bucket.cpp b/src/observer/table_load/ob_table_load_bucket.cpp index 718afd1be..cddaf6661 100644 --- a/src/observer/table_load/ob_table_load_bucket.cpp +++ b/src/observer/table_load/ob_table_load_bucket.cpp @@ -26,21 +26,6 @@ namespace observer using namespace common; using namespace table; -int ObTableLoadBucket::init(const ObAddr &leader_addr) { - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", KR(ret)); - } else if (!leader_addr.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid addr", KR(ret), K(leader_addr)); - } else { - leader_addr_ = leader_addr; - is_inited_ = true; - } - return ret; -} - int ObTableLoadBucket::add_row(const ObTabletID &tablet_id, const ObTableLoadObjRow &obj_row, int64_t batch_size, diff --git a/src/observer/table_load/ob_table_load_bucket.h b/src/observer/table_load/ob_table_load_bucket.h index efad58a9d..0ae297f48 100644 --- a/src/observer/table_load/ob_table_load_bucket.h +++ b/src/observer/table_load/ob_table_load_bucket.h @@ -16,7 +16,6 @@ #pragma once -#include "lib/net/ob_addr.h" #include "observer/table_load/ob_table_load_row_array.h" #include "common/row/ob_row.h" @@ -32,7 +31,7 @@ namespace observer class ObTableLoadBucket { public: - ObTableLoadBucket() : is_inited_(false), row_size_(0), sequence_no_(0) {} + ObTableLoadBucket() : row_size_(0), sequence_no_(0) {} int add_row(const common::ObTabletID &tablet_id, const table::ObTableLoadObjRow &obj_row, @@ -41,8 +40,6 @@ class ObTableLoadBucket bool &flag); void reset() { - is_inited_ = false; - leader_addr_.reset(); row_array_.reset(); row_size_ = 0; } @@ -52,17 +49,10 @@ class ObTableLoadBucket row_size_ = 0; } - bool is_inited() const { - return is_inited_; - } - int init(const common::ObAddr &leader_addr); - - TO_STRING_KV(K_(leader_addr), K_(sequence_no)); + TO_STRING_KV(K_(sequence_no)); public: // data members - bool is_inited_; - common::ObAddr leader_addr_; table::ObTableLoadTabletObjRowArray row_array_; int64_t row_size_; uint64_t sequence_no_; diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index d3821541e..f89475e1a 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -269,7 +269,7 @@ int ObTableLoadCoordinator::check_need_sort_for_lob_or_index(bool &need_sort) co int ObTableLoadCoordinator::calc_session_count( const int64_t total_session_count, const int64_t limit_session_count, - const ObTableLoadArray all_leader_info_array, + const ObTableLoadPartitionLocation::LocalInfo &local_info, ObArray &partitions, ObDirectLoadResourceApplyArg &apply_arg, int64_t &coord_session_count, @@ -277,84 +277,19 @@ int ObTableLoadCoordinator::calc_session_count( int64_t &write_session_count) { int ret = OB_SUCCESS; - int64_t remain_session_count = total_session_count; - ObAddr coord_addr = ObServer::get_instance().get_self(); - bool include_coord_addr = false; - int64_t total_partitions = 0; - int64_t store_server_count = all_leader_info_array.count(); - // Determine if the coordinator node also serves as a store node - for (int64_t i = 0; i < store_server_count; i++) { - total_partitions += all_leader_info_array[i].partition_id_array_.count(); - if (coord_addr == all_leader_info_array[i].addr_) { - include_coord_addr = true; - } - } - // Resource control first determines the thread, the first traversal allocates threads proportionally by partition - for (int64_t i = 0; OB_SUCC(ret) && i < store_server_count; i++) { - ObDirectLoadResourceUnit unit; - unit.addr_ = all_leader_info_array[i].addr_; - if (OB_FAIL(partitions.push_back(all_leader_info_array[i].partition_id_array_.count()))) { - LOG_WARN("fail to push back", KR(ret)); - } else { - unit.thread_count_ = MAX(MIN(limit_session_count, total_session_count * partitions[i] / total_partitions), MIN_THREAD_COUNT); - if (OB_FAIL(apply_arg.apply_array_.push_back(unit))) { - LOG_WARN("fail to push back", KR(ret)); - } else { - remain_session_count -= unit.thread_count_; - } - } - } - // The first traversal cannot allocate all threads, continue to average allocate the remaining threads to each node until all threads are allocated - if (OB_SUCC(ret)) { - bool need_break = false; - while (remain_session_count > 0 && !need_break) { - need_break = true; - for (int64_t i = 0; remain_session_count > 0 && i < store_server_count; i++) { - ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; - if (unit.thread_count_ < limit_session_count) { - unit.thread_count_++; - remain_session_count--; - need_break = false; - } - } - } - } - - /* - When the coordinator node does not have data partitions, thread resources need to be requested, but memory does not need to be requested, and it is placed at the last position of the resource request array - coord_session_count indicates the number of available threads on the coordinator node, min_session_count indicates the minimum value of available threads across all nodes - */ - if (OB_SUCC(ret)) { - if (!include_coord_addr) { - ObDirectLoadResourceUnit unit; - unit.addr_ = coord_addr; - unit.thread_count_ = MAX(MIN(limit_session_count, total_session_count / store_server_count), MIN_THREAD_COUNT); - unit.memory_size_ = 0; - coord_session_count = unit.thread_count_; - min_session_count = MIN(min_session_count, unit.thread_count_); - if (OB_FAIL(apply_arg.apply_array_.push_back(unit))) { - LOG_WARN("fail to push back", KR(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < store_server_count; i++) { - ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; - if (all_leader_info_array[i].addr_ == coord_addr) { - coord_session_count = unit.thread_count_; - } - min_session_count = MIN(min_session_count, unit.thread_count_); - } - } - - /* - Determine write_session_count, which represents the number of available threads on the store node during the data sending phase - For load data mode, if the coordinator node and data node are on the same node, half of the threads are allocated for data parsing and the other half for data storage - */ - if (OB_SUCC(ret)) { - if (include_coord_addr && ObDirectLoadMode::is_load_data(ctx_->param_.load_mode_)) { - write_session_count = MIN(min_session_count, (coord_session_count + 1) / 2); - } else { - write_session_count = min_session_count; - } + ObDirectLoadResourceUnit unit; + unit.addr_ = local_info.addr_; + unit.thread_count_ = MAX(MIN(limit_session_count, total_session_count), MIN_THREAD_COUNT); + if (OB_FAIL(partitions.push_back(local_info.partition_id_array_.count()))) { + LOG_WARN("add local partition count failed", KR(ret)); + } else if (OB_FAIL(apply_arg.apply_array_.push_back(unit))) { + LOG_WARN("add local resource unit failed", KR(ret)); + } else { + coord_session_count = unit.thread_count_; + min_session_count = MIN(min_session_count, unit.thread_count_); + write_session_count = ObDirectLoadMode::is_load_data(ctx_->param_.load_mode_) + ? (coord_session_count + 1) / 2 + : coord_session_count; } return ret; } @@ -478,7 +413,7 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar while (OB_SUCC(ret)) { apply_arg.apply_array_.reset(); int64_t memory_limit = 0; - table::ObTableLoadArray all_leader_info_array; + ObTableLoadPartitionLocation::LocalInfo local_info; if (THIS_WORKER.is_timeout()) { ret = OB_TIMEOUT; LOG_WARN("gen_apply_arg wait too long", KR(ret)); @@ -488,32 +423,27 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar LOG_WARN("fail to check status", KR(ret)); } else if (OB_FAIL(coordinator_ctx_->init_partition_location_and_store_infos())) { LOG_WARN("fail to init partition location and store infos", KR(ret)); - } else if (OB_FAIL(coordinator_ctx_->partition_location_.get_all_leader_info(all_leader_info_array))) { - LOG_WARN("fail to get all leader info", KR(ret)); + } else if (OB_FAIL(coordinator_ctx_->partition_location_.get_local_info(local_info))) { + LOG_WARN("get local partition info failed", KR(ret)); } else if (OB_FAIL(ObTableLoadService::get_memory_limit(memory_limit))) { LOG_WARN("fail to get memory_limit", K(ret)); } else if (OB_FAIL(ObSchemaUtils::get_tenant_int_variable(SYS_VAR_PARALLEL_SERVERS_TARGET, parallel_servers_target))) { LOG_WARN("fail read tenant variable", KR(ret)); } else { - bool include_cur_addr = false; bool task_need_sort = false; // Indicates whether the entire import task will go through the sorting process bool main_need_sort = false; // Indicates whether the main table will go through sorting - int64_t total_partitions = 0; ObArray partitions; - int64_t store_server_count = all_leader_info_array.count(); + const int64_t store_server_count = 1; int64_t coord_session_count = 0; int64_t write_session_count = 0; int64_t min_session_count = ctx_->param_.parallel_; - int64_t max_session_count = 0; int64_t limit_session_count = MIN(memory_limit / min_part_memory, ObMacroDataSeq::MAX_PARALLEL_IDX + 1); // The parallelism within the node cannot exceed this value int64_t total_limit_session_count = MIN(limit_session_count * store_server_count, parallel_servers_target); // The total number of sessions across all nodes cannot exceed this level of parallelism int64_t total_session_count = MIN(ctx_->param_.parallel_, total_limit_session_count); - ObDirectLoadResourceOpRes apply_res; - if (OB_FAIL(calc_session_count(total_session_count, limit_session_count, - all_leader_info_array, + local_info, partitions, apply_arg, coord_session_count, @@ -530,9 +460,9 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar main_need_sort, task_need_sort))) { LOG_WARN("fail to calc memory size", KR(ret)); - } else if (OB_FAIL(ObTableLoadResourceService::apply_resource(apply_arg, apply_res))) { + } else if (OB_FAIL(ObTableLoadResourceService::apply_resource(apply_arg))) { if (retry_count % 100 == 0) { - LOG_WARN("fail to apply resource", KR(ret), K(apply_res.error_code_), K(retry_count), K(param_.exe_mode_), K(main_need_sort), K(task_need_sort), K(partitions), K(coordinator_addr), K(apply_arg)); + LOG_WARN("fail to apply resource", KR(ret), K(retry_count), K(param_.exe_mode_), K(main_need_sort), K(task_need_sort), K(partitions), K(coordinator_addr), K(apply_arg)); } if (ret == OB_EAGAIN) { retry_count++; @@ -548,14 +478,10 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar (main_need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) : (main_need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT)); ctx_->job_stat_->parallel_ = coord_session_count; - if (OB_FAIL(ObTableLoadService::add_assigned_task(apply_arg))) { - LOG_WARN("fail to add_assigned_task", KR(ret)); - } else { - ctx_->set_assigned_resource(); - FLOG_INFO("gen_apply_arg", K(retry_count), K(ctx_->param_), K(parallel_servers_target), K(limit_session_count), K(total_session_count), - K(memory_limit), K(part_unsort_memory), K(min_part_memory), K(partitions), K(main_need_sort), K(task_need_sort), K(apply_arg)); - break; - } + ctx_->set_assigned_resource(); + FLOG_INFO("gen_apply_arg", K(retry_count), K(ctx_->param_), K(parallel_servers_target), K(limit_session_count), K(total_session_count), + K(memory_limit), K(part_unsort_memory), K(min_part_memory), K(partitions), K(main_need_sort), K(task_need_sort), K(apply_arg)); + break; } } } @@ -567,22 +493,14 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar int ObTableLoadCoordinator::pre_begin_peers(ObDirectLoadResourceApplyArg &apply_arg) { int ret = OB_SUCCESS; - ObTableLoadArray all_leader_info_array; - ObTableLoadArray target_all_leader_info_array; + ObTableLoadPartitionLocation::LocalInfo local_info; + ObTableLoadPartitionLocation::LocalInfo target_local_info; coordinator_ctx_->set_enable_heart_beat(true); - if (OB_FAIL(coordinator_ctx_->partition_location_.get_all_leader_info(all_leader_info_array))) { - LOG_WARN("fail to get all leader info", KR(ret)); - } else if (OB_FAIL(coordinator_ctx_->target_partition_location_.get_all_leader_info(target_all_leader_info_array))) { - LOG_WARN("fail to get all leader info", KR(ret)); - } else if (OB_UNLIKELY(coordinator_ctx_->store_infos_.count() != all_leader_info_array.count() - || coordinator_ctx_->store_infos_.count() != target_all_leader_info_array.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("store infos count must equal to leader info array count", - KR(ret), - K(coordinator_ctx_->store_infos_), - K(all_leader_info_array), K(target_all_leader_info_array)); + if (OB_FAIL(coordinator_ctx_->partition_location_.get_local_info(local_info))) { + LOG_WARN("get source local info failed", KR(ret)); + } else if (OB_FAIL(coordinator_ctx_->target_partition_location_.get_local_info(target_local_info))) { + LOG_WARN("get target local info failed", KR(ret)); } else { - LOG_INFO("route_pre_begin_peer_request begin", K(all_leader_info_array.count())); ObDirectLoadControlPreBeginArg arg; arg.table_id_ = param_.table_id_; arg.config_.max_error_row_count_ = param_.max_error_row_count_; @@ -609,48 +527,30 @@ int ObTableLoadCoordinator::pre_begin_peers(ObDirectLoadResourceApplyArg &apply_ LOG_WARN("fail to set exec ctx", KR(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < coordinator_ctx_->store_infos_.count(); ++i) { - ObTableLoadCoordinatorCtx::StoreInfo &store_info = coordinator_ctx_->store_infos_.at(i); - const ObAddr &addr = store_info.addr_; - const ObTableLoadPartitionLocation::LeaderInfo &leader_info = all_leader_info_array.at(i); - const ObTableLoadPartitionLocation::LeaderInfo &target_leader_info = - target_all_leader_info_array.at(i); - //The partition information of the source table and the target table, along with the address of each partition, is completely the same - if (OB_UNLIKELY(addr != leader_info.addr_ || addr != target_leader_info.addr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("addr must be same", K(addr), - K(leader_info.addr_), - K(target_leader_info.addr_)); - } + if (OB_SUCC(ret)) { if (arg.exe_mode_ == ObTableLoadExeMode::MAX_TYPE) { arg.config_.parallel_ = ctx_->param_.session_count_; } else { - arg.config_.parallel_ = apply_arg.apply_array_[i].thread_count_; - arg.avail_memory_ = apply_arg.apply_array_[i].memory_size_; + arg.config_.parallel_ = apply_arg.apply_array_[0].thread_count_; + arg.avail_memory_ = apply_arg.apply_array_[0].memory_size_; } - arg.partition_id_array_ = leader_info.partition_id_array_; - arg.target_partition_id_array_ = target_leader_info.partition_id_array_; - if (OB_FAIL(ret)) { - - } else if (ObTableLoadUtils::is_local_addr(addr)) { // local address - ctx_->param_.session_count_ = arg.config_.parallel_; - ctx_->param_.avail_memory_ = arg.avail_memory_; - LOG_INFO("table load local pre begin", K(arg)); - if (OB_FAIL(ObTableLoadStore::init_ctx(ctx_, arg.partition_id_array_, arg.target_partition_id_array_))) { - LOG_WARN("fail to store init ctx", KR(ret)); - } else { - ObTableLoadStore store(ctx_); - if (OB_FAIL(store.init())) { - LOG_WARN("fail to init store", KR(ret)); - } else if (OB_FAIL(store.pre_begin())) { - LOG_WARN("fail to store pre begin", KR(ret)); - } + arg.partition_id_array_ = local_info.partition_id_array_; + arg.target_partition_id_array_ = target_local_info.partition_id_array_; + ctx_->param_.session_count_ = arg.config_.parallel_; + ctx_->param_.avail_memory_ = arg.avail_memory_; + LOG_INFO("table load local pre begin", K(arg)); + if (OB_FAIL(ObTableLoadStore::init_ctx(ctx_, arg.partition_id_array_, arg.target_partition_id_array_))) { + LOG_WARN("fail to store init ctx", KR(ret)); + } else { + ObTableLoadStore store(ctx_); + if (OB_FAIL(store.init())) { + LOG_WARN("fail to init store", KR(ret)); + } else if (OB_FAIL(store.pre_begin())) { + LOG_WARN("fail to store pre begin", KR(ret)); } - } else { // peer, send rpc - TABLE_LOAD_CONTROL_RPC_CALL(pre_begin, addr, arg); } if (OB_SUCC(ret)) { - store_info.enable_heart_beat_ = true; + coordinator_ctx_->store_infos_.at(0).enable_heart_beat_ = true; } } } @@ -1301,12 +1201,12 @@ int ObTableLoadCoordinator::commit_peers(ObTableLoadSqlStatistics &sql_statistic if (ctx_->is_assigned_resource()) { int tmp_ret = OB_SUCCESS; ObDirectLoadResourceReleaseArg release_arg; - release_arg.task_key_ = ObTableLoadUniqueKey(ctx_->param_.table_id_, ctx_->ddl_param_.task_id_); - if (OB_TMP_FAIL(ObTableLoadService::delete_assigned_task(release_arg))) { - LOG_WARN("fail to delete assigned task", KR(tmp_ret), K(release_arg)); + if (OB_TMP_FAIL(ObTableLoadResourceService::release_resource(release_arg))) { + LOG_WARN("fail to release assigned resource", KR(tmp_ret), K(release_arg)); + } else { + ctx_->reset_assigned_resource(); } - ctx_->reset_assigned_resource(); } } } @@ -2127,10 +2027,10 @@ int ObTableLoadCoordinator::flush(ObTableLoadCoordinatorTrans *trans) return ret; } -int ObTableLoadCoordinator::write_peer_leader(const ObTableLoadTransId &trans_id, - int32_t session_id, uint64_t sequence_no, - const ObTableLoadTabletObjRowArray &tablet_obj_rows, - const ObAddr &addr) +int ObTableLoadCoordinator::write_local(const ObTableLoadTransId &trans_id, + int32_t session_id, + uint64_t sequence_no, + const ObTableLoadTabletObjRowArray &tablet_obj_rows) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -2140,35 +2040,11 @@ int ObTableLoadCoordinator::write_peer_leader(const ObTableLoadTransId &trans_id ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret)); } else { - LOG_DEBUG("coordinator write peer leader", K(addr)); - if (ObTableLoadUtils::is_local_addr(addr)) { // local address - ObTableLoadStore store(ctx_); - if (OB_FAIL(store.init())) { - LOG_WARN("fail to init store", KR(ret)); - } else if (OB_FAIL(store.write(trans_id, session_id, sequence_no, tablet_obj_rows))) { - LOG_WARN("fail to store write", KR(ret), K(trans_id)); - } - } else { // remote, send rpc - common::ObArenaAllocator allocator("TLD_Coord"); - - int64_t pos = 0; - int64_t buf_len = tablet_obj_rows.get_serialize_size(); - char *buf = static_cast(allocator.alloc(buf_len)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", KR(ret), K(buf_len)); - } else if (OB_FAIL(tablet_obj_rows.serialize(buf, buf_len, pos))) { - LOG_WARN("failed to serialize obj row array", KR(ret), KP(buf), K(buf_len), K(pos)); - } else { - ObDirectLoadControlInsertTransArg arg; - arg.table_id_ = param_.table_id_; - arg.task_id_ = ctx_->ddl_param_.task_id_; - arg.trans_id_ = trans_id; - arg.session_id_ = session_id; - arg.sequence_no_ = sequence_no; - arg.payload_.assign(buf, buf_len); - TABLE_LOAD_CONTROL_RPC_CALL(insert_trans, addr, arg); - } + ObTableLoadStore store(ctx_); + if (OB_FAIL(store.init())) { + LOG_WARN("fail to init store", KR(ret)); + } else if (OB_FAIL(store.write(trans_id, session_id, sequence_no, tablet_obj_rows))) { + LOG_WARN("fail to store write", KR(ret), K(trans_id)); } } return ret; diff --git a/src/observer/table_load/ob_table_load_coordinator.h b/src/observer/table_load/ob_table_load_coordinator.h index e40a143b7..ba15d4b78 100644 --- a/src/observer/table_load/ob_table_load_coordinator.h +++ b/src/observer/table_load/ob_table_load_coordinator.h @@ -25,8 +25,7 @@ #include "storage/direct_load/ob_table_load_dml_stat.h" #include "storage/direct_load/ob_table_load_sql_statistics.h" #include "observer/table_load/ob_table_load_row_array.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc_struct.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc.h" +#include "observer/table_load/resource/ob_table_load_resource_struct.h" #include "observer/table_load/resource/ob_table_load_resource_service.h" #include "observer/table_load/ob_table_load_assigned_memory_manager.h" #include "observer/table_load/ob_table_load_partition_location.h" @@ -74,7 +73,7 @@ class ObTableLoadCoordinator int check_need_sort_for_lob_or_index(bool &need_sort) const; int calc_session_count(const int64_t total_session_count, const int64_t max_session_count, - const table::ObTableLoadArray all_leader_info_array, + const observer::ObTableLoadPartitionLocation::LocalInfo &local_info, ObArray &partitions, ObDirectLoadResourceApplyArg &apply_arg, int64_t &coord_session_count, @@ -147,10 +146,9 @@ class ObTableLoadCoordinator const table::ObTableLoadObjRowArray &obj_rows); // for client int flush(ObTableLoadCoordinatorTrans *trans); - // Only write to the leader node - int write_peer_leader(const table::ObTableLoadTransId &trans_id, int32_t session_id, - uint64_t sequence_no, const table::ObTableLoadTabletObjRowArray &tablet_obj_rows, - const common::ObAddr &addr); + int write_local(const table::ObTableLoadTransId &trans_id, int32_t session_id, + uint64_t sequence_no, + const table::ObTableLoadTabletObjRowArray &tablet_obj_rows); private: class WriteTaskProcessor; class WriteTaskCallback; diff --git a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp index 175124661..c32a0069b 100644 --- a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp +++ b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp @@ -600,22 +600,17 @@ int ObTableLoadCoordinatorCtx::init_partition_location_and_store_infos() { int ret = OB_SUCCESS; store_infos_.reset(); - ObTableLoadArray all_addr_array; if (OB_FAIL(ObTableLoadPartitionLocation::init_partition_location(partition_ids_, target_partition_ids_, partition_location_, target_partition_location_))) { LOG_WARN("fail to init partition location", KR(ret)); - } else if (OB_FAIL(partition_location_.get_all_leader(all_addr_array))) { - LOG_WARN("fail to get all leader", KR(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) { - StoreInfo store_info; - store_info.addr_ = all_addr_array.at(i); - store_info.enable_heart_beat_ = false; - if (OB_FAIL(store_infos_.push_back(store_info))) { - LOG_WARN("fail to push back store info", KR(ret)); - } + StoreInfo store_info; + store_info.addr_ = GCTX.self_addr(); + store_info.enable_heart_beat_ = false; + if (OB_FAIL(store_infos_.push_back(store_info))) { + LOG_WARN("add local store info failed", KR(ret)); } } return ret; diff --git a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp index f98ad158b..431a2de85 100644 --- a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp +++ b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp @@ -34,7 +34,6 @@ using namespace table; ObTableLoadEmptyInsertTabletCtxManager::ObTableLoadEmptyInsertTabletCtxManager() : thread_count_(0), - idx_(0), start_(0), is_inited_(false) { @@ -58,10 +57,10 @@ int ObTableLoadEmptyInsertTabletCtxManager::init( partition_location_, target_partition_location_))) { LOG_WARN("fail to init partition location", KR(ret)); - } else if (OB_FAIL(partition_location_.get_all_leader_info(all_leader_info_array_))) { - LOG_WARN("fail to get all origin leader info", KR(ret)); - } else if (OB_FAIL(target_partition_location_.get_all_leader_info(target_all_leader_info_array_))) { - LOG_WARN("fail to get all target leader info", KR(ret)); + } else if (OB_FAIL(partition_location_.get_local_info(local_info_))) { + LOG_WARN("get source local info failed", KR(ret)); + } else if (OB_FAIL(target_partition_location_.get_local_info(target_local_info_))) { + LOG_WARN("get target local info failed", KR(ret)); } } if (OB_SUCC(ret)) { @@ -72,8 +71,8 @@ int ObTableLoadEmptyInsertTabletCtxManager::init( int ObTableLoadEmptyInsertTabletCtxManager::get_next_task( ObAddr &addr, - ObIArray &partition_ids, - ObIArray &target_partition_ids) + ObIArray &partition_ids, + ObIArray &target_partition_ids) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -81,24 +80,18 @@ int ObTableLoadEmptyInsertTabletCtxManager::get_next_task( LOG_WARN("empty insert tablet ctx manager is not init", KR(ret)); } else { ObMutexGuard guard(op_lock_); - if (all_leader_info_array_.count() == idx_) { + if (target_local_info_.partition_id_array_.count() == start_) { ret = OB_ITER_END; } else { - const LeaderInfo &leader_info = all_leader_info_array_.at(idx_); - const LeaderInfo &target_leader_info = target_all_leader_info_array_.at(idx_); - addr = target_leader_info.addr_; - for (; OB_SUCC(ret) && start_ < target_leader_info.partition_id_array_.count() + addr = target_local_info_.addr_; + for (; OB_SUCC(ret) && start_ < target_local_info_.partition_id_array_.count() && partition_ids.count() < TABLET_COUNT_PER_TASK; ++start_) { - if (OB_FAIL(partition_ids.push_back(leader_info.partition_id_array_.at(start_)))) { + if (OB_FAIL(partition_ids.push_back(local_info_.partition_id_array_.at(start_)))) { LOG_WARN("fail to push back partition ids", KR(ret)); - } else if (OB_FAIL(target_partition_ids.push_back(target_leader_info.partition_id_array_.at(start_)))) { + } else if (OB_FAIL(target_partition_ids.push_back(target_local_info_.partition_id_array_.at(start_)))) { LOG_WARN("fail to push back target partition ids", KR(ret)); } } - if (target_leader_info.partition_id_array_.count() == start_) { - start_ = 0; - ++idx_; - } } } return ret; @@ -137,8 +130,8 @@ int ObTableLoadEmptyInsertTabletCtxManager::handle_thread_finish(bool &is_finish int ObTableLoadEmptyInsertTabletCtxManager::execute( const uint64_t &table_id, const ObTableLoadDDLParam &ddl_param, - const ObIArray &ls_part_ids, - const ObIArray &target_ls_part_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids) { int ret = OB_SUCCESS; ObTableLoadSchema table_load_schema; @@ -174,8 +167,8 @@ int ObTableLoadEmptyInsertTabletCtxManager::execute( if (OB_FAIL(ret)) { // do nothing } else if (OB_FAIL(tmp_insert_table_ctx.init(insert_table_param, - ls_part_ids, - target_ls_part_ids))) { + partition_ids, + target_partition_ids))) { LOG_WARN("fail to init tmp insert table ctx", KR(ret)); } FOREACH_X(it, tmp_insert_table_ctx.get_tablet_ctx_map(), OB_SUCC(ret)) { @@ -207,8 +200,8 @@ int ObTableLoadEmptyInsertTabletCtxManager::execute( int ObTableLoadEmptyInsertTabletCtxManager::execute_for_dag( const uint64_t &table_id, const ObTableLoadDDLParam &ddl_param, - const ObIArray &ls_part_ids, - const ObIArray &target_ls_part_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids) { int ret = OB_SUCCESS; ObDirectLoadType direct_load_type = ObDirectLoadMgrAgent::load_data_get_direct_load_type( @@ -223,11 +216,10 @@ int ObTableLoadEmptyInsertTabletCtxManager::execute_for_dag( dag_init_param.ddl_task_param_.schema_version_ = ddl_param.schema_version_; dag_init_param.ddl_task_param_.is_no_logging_ = ddl_param.is_no_logging_; dag_init_param.ddl_task_param_.snapshot_version_ = ddl_param.snapshot_version_; - for (int64_t i = 0; OB_SUCC(ret) && i < target_ls_part_ids.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_ids.count(); i++) { if (OB_FAIL(add_var_to_array_no_dup( - dag_init_param.ls_tablet_ids_, - std::make_pair(target_ls_part_ids.at(i).ls_id_, - target_ls_part_ids.at(i).part_tablet_id_.tablet_id_)))) { + dag_init_param.tablet_ids_, + target_partition_ids.at(i).part_tablet_id_.tablet_id_))) { LOG_WARN("add var to array no dup failed", KR(ret)); } } diff --git a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.h b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.h index 5f245a7a5..f4683d933 100644 --- a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.h +++ b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.h @@ -32,7 +32,7 @@ namespace observer class ObTableLoadEmptyInsertTabletCtxManager { - using LeaderInfo = ObTableLoadPartitionLocation::LeaderInfo; + using LocalInfo = ObTableLoadPartitionLocation::LocalInfo; static const int64_t TABLET_COUNT_PER_TASK = 20; public: ObTableLoadEmptyInsertTabletCtxManager(); @@ -41,26 +41,25 @@ class ObTableLoadEmptyInsertTabletCtxManager const common::ObIArray &partition_ids, const common::ObIArray &target_partition_ids); int get_next_task(ObAddr &addr, - ObIArray &partition_ids, - ObIArray &target_partition_ids); + ObIArray &partition_ids, + ObIArray &target_partition_ids); int set_thread_count(const int64_t thread_count); int handle_thread_finish(bool &is_finish); static int execute(const uint64_t &table_id, const ObTableLoadDDLParam &ddl_param, - const ObIArray &ls_part_ids, - const ObIArray &target_ls_part_ids); + const ObIArray &partition_ids, + const ObIArray &target_partition_ids); static int execute_for_dag(const uint64_t &table_id, const ObTableLoadDDLParam &ddl_param, - const ObIArray &ls_part_ids, - const ObIArray &target_ls_part_ids); + const ObIArray &partition_ids, + const ObIArray &target_partition_ids); private: ObTableLoadPartitionLocation partition_location_; ObTableLoadPartitionLocation target_partition_location_; - table::ObTableLoadArray all_leader_info_array_; - table::ObTableLoadArray target_all_leader_info_array_; + LocalInfo local_info_; + LocalInfo target_local_info_; int64_t thread_count_ CACHE_ALIGNED; lib::ObMutex op_lock_; - int64_t idx_; int64_t start_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_instance.cpp b/src/observer/table_load/ob_table_load_instance.cpp index 486ffa670..2eb3c7ce7 100644 --- a/src/observer/table_load/ob_table_load_instance.cpp +++ b/src/observer/table_load/ob_table_load_instance.cpp @@ -19,7 +19,7 @@ #include "lib/stat/ob_diagnostic_info_guard.h" #include "observer/table_load/ob_table_load_instance.h" #include "share/rc/ob_module_provider.h" -#include "storage/tx/ob_ts_mgr.h" // transaction::get_tenant_gts +#include "storage/tx/ob_ts_mgr.h" #include "observer/table_load/ob_table_load_coordinator.h" #include "observer/table_load/ob_table_load_index_long_wait.h" #include "observer/table_load/ob_table_load_redef_table.h" @@ -493,7 +493,7 @@ int ObTableLoadInstance::init_ddl_param_for_inc_direct_load() LOG_WARN("fail to get table schema", KR(ret), K(table_id)); } else if (OB_FAIL(ObCommonIDUtils::gen_unique_id_by_rpc( raw_id))) { LOG_WARN("failed to gen unique id by rpc", KR(ret)); - } else if (OB_FAIL(transaction::get_tenant_gts(current_scn))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts_sync(GCONF.rpc_timeout, current_scn))) { LOG_WARN("failed to get gts", KR(ret)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_data_version))) { LOG_WARN("failed to get min data version", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_parallel_merger.cpp b/src/observer/table_load/ob_table_load_parallel_merger.cpp index 82adbdc54..6553862af 100644 --- a/src/observer/table_load/ob_table_load_parallel_merger.cpp +++ b/src/observer/table_load/ob_table_load_parallel_merger.cpp @@ -59,7 +59,7 @@ int ObTableLoadParallelMerger::init_merge_ctx(ObTableLoadMergeTableBaseOp *op) param.trans_param_ = op->merge_phase_ctx_->trans_param_; param.file_mgr_ = op->store_ctx_->tmp_file_mgr_; param.ctx_ = op->ctx_; - if (OB_FAIL(merge_ctx_.init(param, store_table_ctx->ls_partition_ids_))) { + if (OB_FAIL(merge_ctx_.init(param, store_table_ctx->partition_ids_))) { LOG_WARN("fail to init merge ctx", KR(ret), K(param)); } return ret; diff --git a/src/observer/table_load/ob_table_load_partition_location.cpp b/src/observer/table_load/ob_table_load_partition_location.cpp index e73e73f86..1bde32851 100644 --- a/src/observer/table_load/ob_table_load_partition_location.cpp +++ b/src/observer/table_load/ob_table_load_partition_location.cpp @@ -17,373 +17,72 @@ #define USING_LOG_PREFIX SERVER #include "observer/table_load/ob_table_load_partition_location.h" -#include "share/rc/ob_module_provider.h" #include "observer/ob_server.h" #include "observer/table_load/ob_table_load_utils.h" -#include "storage/tx_storage/ob_ls_service.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" namespace oceanbase { namespace observer { using namespace common; -using namespace common::hash; -using namespace share; -using namespace storage; using namespace table; -int ObTableLoadPartitionLocation::check_tablet_has_same_leader(const ObTableLoadPartitionLocation &other, bool &result) -{ - int ret = OB_SUCCESS; - result = true; - if (tablet_ids_.count() != other.tablet_ids_.count()) { - result = false; - } - for (int64_t i = 0; OB_SUCC(ret) && result && i < tablet_ids_.count(); i ++) { - PartitionLocationInfo info1; - PartitionLocationInfo info2; - if (OB_FAIL(partition_map_.get_refactored(tablet_ids_.at(i), info1))) { - LOG_WARN("fail to get location info", KR(ret)); - } else if (OB_FAIL(other.partition_map_.get_refactored(other.tablet_ids_.at(i), info2))) { - LOG_WARN("fail to get location info", KR(ret)); - } else if (info1.leader_addr_ != info2.leader_addr_) { - result = false; - } - } - return ret; -} - int ObTableLoadPartitionLocation::init_partition_location( - const ObIArray &partition_ids, - const ObIArray &target_partition_ids, - ObTableLoadPartitionLocation &partition_location, - ObTableLoadPartitionLocation &target_partition_location) -{ - int ret = OB_SUCCESS; - int retry = 0; - bool flag = false; - while (retry < 3 && OB_SUCC(ret)) { - partition_location.reset(); - target_partition_location.reset(); - // init partition_location_ - if (OB_FAIL(partition_location.init(partition_ids))) { - LOG_WARN("fail to init partition location", KR(ret)); - } else if (OB_FAIL(target_partition_location.init(target_partition_ids))) { - LOG_WARN("fail to init origin partition location", KR(ret)); - } else if (OB_FAIL(partition_location.check_tablet_has_same_leader(target_partition_location, flag))) { - LOG_WARN("fail to check_tablet_has_same_leader", KR(ret)); - } - if (OB_SUCC(ret)) { - if (flag) { - break; - } else { - LOG_WARN("invalid leader info, maybe change master"); - } - } - retry ++; - } - - if (OB_SUCC(ret)) { - if (!flag) { - ret = OB_EAGAIN; - LOG_WARN("invalid leader info", KR(ret)); - } - } - - return ret; -} - -int ObTableLoadPartitionLocation::fetch_ls_id(const ObTabletID &tablet_id, - ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLocationService &location_service = OBSERVER.get_location_service(); - const int64_t cluster_id = GCONF.cluster_id.get_value(); - MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard); - bool is_cache_hit = false; - if (OB_FAIL(tenant_guard.switch_to())) { - LOG_WARN("fail to switch tenant", KR(ret)); - } else if (OB_FAIL(location_service.get(tablet_id, INT64_MAX, is_cache_hit, ls_id))) { - LOG_WARN("fail to get ls id", KR(ret)); - } - return ret; -} - -int ObTableLoadPartitionLocation::fetch_ls_location(const ObTabletID &tablet_id, - ObLSLocation &ls_location, ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLocationService &location_service = OBSERVER.get_location_service(); - const int64_t cluster_id = GCONF.cluster_id.get_value(); - MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard); - bool is_cache_hit = false; - if (OB_FAIL(tenant_guard.switch_to())) { - LOG_WARN("fail to switch tenant", KR(ret)); - } else if (OB_FAIL(location_service.get(tablet_id, INT64_MAX, is_cache_hit, ls_id))) { - LOG_WARN("fail to get ls id", KR(ret)); - } else if (OB_FAIL(location_service.get(cluster_id, ls_id, INT64_MAX, is_cache_hit, - ls_location))) { - LOG_WARN("fail to get location", KR(ret)); - } - return ret; -} - -int ObTableLoadPartitionLocation::fetch_ls_locations(const ObIArray &partition_ids) -{ - int ret = OB_SUCCESS; - ObArray ls_ids; - - - for (int64_t i = 0; OB_SUCC(ret) && (i < partition_ids.count()); ++i) { - const ObTabletID &tablet_id = partition_ids.at(i).tablet_id_; - if (OB_FAIL(tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet_id", K(tablet_id), K(i)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(*(GCTX.sql_proxy_), tablet_ids_, ls_ids))) { - if (OB_LIKELY(OB_ITEM_NOT_MATCH == ret)) { - ret = OB_SCHEMA_NOT_UPTODATE; - } - LOG_WARN("table_load_partition failed to batch get ls", KR(ret)); - } else { - ObLSLocation location; - ObHashMap ls_location_map; - ObLocationService &location_service = OBSERVER.get_location_service(); - const int64_t cluster_id = GCONF.cluster_id.get_value(); - MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard); - bool is_cache_hit = false; - - if (OB_FAIL(tenant_guard.switch_to())) { - LOG_WARN("fail to switch tenant", KR(ret)); - } else if (OB_FAIL(ls_location_map.create(1024, "TLD_PartLoc", "TLD_PartLoc"))) { - LOG_WARN("fail to create location info map", KR(ret)); - } else { - // avoid redundant location info lookups - for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { - const ObLSID &ls_id = ls_ids.at(i); - PartitionLocationInfo info; - info.partition_id_.part_tablet_id_ = partition_ids.at(i); - info.partition_id_.ls_id_ = ls_id; - - if (OB_FAIL(ls_location_map.get_refactored(ls_id, info.leader_addr_))) { - if (ret != OB_HASH_NOT_EXIST) { - LOG_WARN("failed to get refactored", K(ret), K(i), K(ls_id)); - } else if (OB_FAIL(location_service.get(cluster_id, ls_id, INT64_MAX, is_cache_hit, location))) { - LOG_WARN("fail to get location", KR(ret)); - } else if (OB_FAIL(location.get_leader(info.leader_addr_))) { - LOG_WARN("fail to get leader", KR(ret)); - } else if (OB_FAIL(ls_location_map.set_refactored(ls_id, info.leader_addr_))) { - LOG_WARN("failed to set refactored", K(ret), K(ls_id), K(info)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(partition_map_.set_refactored(tablet_ids_.at(i), info))) { - LOG_WARN("fail to set refactored", KR(ret), K(i), K(info)); - } - } - } - } - - return ret; -} - - -int ObTableLoadPartitionLocation::fetch_tablet_handle(const ObLSID &ls_id, - const ObTabletID &tablet_id, - ObTabletHandle &tablet_handle) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids, + ObTableLoadPartitionLocation &partition_location, + ObTableLoadPartitionLocation &target_partition_location) { int ret = OB_SUCCESS; - ObLSService *ls_svr = nullptr; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObLSTabletService *tablet_service = nullptr; - if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLSService failed", KR(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - if (OB_UNLIKELY(OB_LS_NOT_EXIST == ret)) { - LOG_WARN("get ls handle failed", KR(ret), "log_stream_id", ls_id.id()); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls_handle.get_ls() is nullptr", KR(ret)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet service should not be NULL", KR(ret), KP(tablet_service)); - } else if (OB_FAIL(tablet_service->get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("fail to get tablet", KR(ret)); + partition_location.reset(); + target_partition_location.reset(); + if (OB_UNLIKELY(partition_ids.count() != target_partition_ids.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("source and target partition count differ", K(ret), K(partition_ids), K(target_partition_ids)); + } else if (OB_FAIL(partition_location.init(partition_ids))) { + LOG_WARN("init source partitions failed", K(ret)); + } else if (OB_FAIL(target_partition_location.init(target_partition_ids))) { + LOG_WARN("init target partitions failed", K(ret)); } return ret; } - -int ObTableLoadPartitionLocation::init( - const ObIArray &partition_ids) +int ObTableLoadPartitionLocation::init(const ObIArray &partition_ids) { int ret = OB_SUCCESS; + ObArray tablet_ids; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("ObTableLoadPartitionLocation init twice", KR(ret)); + LOG_WARN("table load local partitions init twice", K(ret)); } else if (OB_UNLIKELY(partition_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(partition_ids)); - } else { - if (OB_FAIL(partition_map_.create(1024, "TLD_PartLoc", "TLD_PartLoc"))) { - LOG_WARN("fail to create map", KR(ret)); - } else if (OB_FAIL(init_all_partition_location(partition_ids))) { - LOG_WARN("fail to init all partition location", KR(ret)); - } else if (OB_FAIL(init_all_leader_info())) { - LOG_WARN("fail to init all leader info", KR(ret)); - } else { - is_inited_ = true; - } - } - return ret; -} - -int ObTableLoadPartitionLocation::init_all_partition_location( - const ObIArray &partition_ids) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(fetch_ls_locations(partition_ids))) { - LOG_WARN("fail to fetch locations", KR(ret)); - } - return ret; -} - -int ObTableLoadPartitionLocation::init_all_leader_info() -{ - int ret = OB_SUCCESS; - ObArenaAllocator tmp_allocator("TLD_PL_Tmp"); - ObHashMap *> addr_map; - ObHashMap *>::const_iterator addr_iter; - int64_t pos = 0; - - // Store all addr in the set - if (OB_FAIL(addr_map.create(64, "TLD_PL_Tmp", "TLD_PL_Tmp"))) { - LOG_WARN("fail to create hashmap", KR(ret)); - } else { - ObHashMap::const_iterator iter; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids_.count(); i ++) { - PartitionLocationInfo info; - if (OB_FAIL(partition_map_.get_refactored(tablet_ids_.at(i), info))) { - LOG_WARN("fail to get tablet info", K(tablet_ids_.at(i)), KR(ret)); - } - const ObTableLoadLSIdAndPartitionId &partition_id = info.partition_id_; - const ObAddr &addr = info.leader_addr_; - ObIArray *partition_id_array = nullptr; - if (OB_SUCC(ret)) { - if (OB_FAIL(addr_map.get_refactored(addr, partition_id_array))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { - LOG_WARN("fail to get refactored", KR(ret), K(addr)); - } else { - ObArray *new_array = nullptr; - if (OB_ISNULL(new_array = - OB_NEWx(ObArray, (&tmp_allocator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new array", KR(ret)); - } else if (OB_FAIL(addr_map.set_refactored(addr, new_array))) { - LOG_WARN("fail to set refactored", KR(ret), K(addr)); - } else { - - partition_id_array = new_array; - } - if (OB_FAIL(ret)) { - if (nullptr != new_array) { - new_array->~ObArray(); - tmp_allocator.free(new_array); - new_array = nullptr; - } - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(partition_id_array->push_back(partition_id))) { - LOG_WARN("fail to push back partition id", KR(ret)); - } - } - } - } - } - // Store the addr in set to array - if (OB_SUCC(ret)) { - if (OB_FAIL(all_leader_addr_array_.create(addr_map.size(), allocator_))) { - LOG_WARN("fail to create leader addr array", KR(ret)); - } else if (OB_FAIL(all_leader_info_array_.create(addr_map.size(), allocator_))) { - LOG_WARN("fail to create leader info array", KR(ret)); - } - } - ObArray sort_array; - - for (addr_iter = addr_map.begin(); OB_SUCC(ret) && addr_iter != addr_map.end(); ++pos, ++addr_iter) { - LeaderInfoForSort item; - item.addr_ = addr_iter->first; - item.partition_id_array_ptr_ = addr_iter->second; - if (OB_FAIL(sort_array.push_back(item))) { - LOG_WARN("fail to push_back item", KR(ret)); - } + LOG_WARN("empty table load partitions", K(ret)); } - if (OB_SUCC(ret)) { - lib::ob_sort(sort_array.begin(), sort_array.end(), [](const ObTableLoadPartitionLocation::LeaderInfoForSort &a, - ObTableLoadPartitionLocation::LeaderInfoForSort &b) { - return a.addr_ < b.addr_; - }); - } - for (int64_t i = 0; OB_SUCC(ret) && i < sort_array.count(); i ++) { - const ObAddr &addr = sort_array.at(i).addr_; - ObIArray *partition_id_array = sort_array.at(i).partition_id_array_ptr_; - all_leader_addr_array_[i] = addr; - LeaderInfo &leader_info = all_leader_info_array_[i]; - leader_info.addr_ = addr; - if (OB_FAIL(ObTableLoadUtils::deep_copy(*partition_id_array, leader_info.partition_id_array_, - allocator_))) { - LOG_WARN("fail to deep copy partition id array", KR(ret)); + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + if (OB_FAIL(tablet_ids.push_back(ObTableLoadTabletId(partition_ids.at(i))))) { + LOG_WARN("add local tablet failed", K(ret), K(i)); } - partition_id_array->~ObIArray(); - tmp_allocator.free(partition_id_array); } - - return ret; -} - -int ObTableLoadPartitionLocation::get_leader(ObTabletID tablet_id, PartitionLocationInfo &info) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadPartitionLocation not init", KR(ret)); - } else { - if (OB_FAIL(partition_map_.get_refactored(tablet_id, info))) { - LOG_WARN("fail to get refactored", KR(ret), K(tablet_id)); - } - } - return ret; -} - -int ObTableLoadPartitionLocation::get_all_leader(ObTableLoadArray &addr_array) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadPartitionLocation not init", KR(ret)); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObTableLoadUtils::deep_copy(tablet_ids, + local_info_.partition_id_array_, + allocator_))) { + LOG_WARN("copy local tablets failed", K(ret)); } else { - addr_array = all_leader_addr_array_; + local_info_.addr_ = ObServer::get_instance().get_self(); + is_inited_ = true; } return ret; } -int ObTableLoadPartitionLocation::get_all_leader_info(ObTableLoadArray &info_array) const +int ObTableLoadPartitionLocation::get_local_info(LocalInfo &info) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadPartitionLocation not init", KR(ret)); + LOG_WARN("table load local partitions not initialized", K(ret)); } else { - info_array = all_leader_info_array_; + info = local_info_; } return ret; } diff --git a/src/observer/table_load/ob_table_load_partition_location.h b/src/observer/table_load/ob_table_load_partition_location.h index 4428ed258..2cde5dc12 100644 --- a/src/observer/table_load/ob_table_load_partition_location.h +++ b/src/observer/table_load/ob_table_load_partition_location.h @@ -16,94 +16,48 @@ #pragma once -#include "common/ob_tablet_id.h" -#include "lib/hash/ob_hashmap.h" #include "lib/net/ob_addr.h" -#include "share/ob_ls_id.h" #include "share/table/ob_table_load_array.h" #include "share/table/ob_table_load_define.h" namespace oceanbase { -namespace share -{ -class ObLSLocation; -} // namespace share -namespace storage -{ -class ObTabletHandle; -} // namespace storage namespace observer { class ObTableLoadPartitionLocation { public: - struct PartitionLocationInfo - { - table::ObTableLoadLSIdAndPartitionId partition_id_; - common::ObAddr leader_addr_; - TO_STRING_KV(K_(partition_id), K_(leader_addr)); - }; - struct LeaderInfo + struct LocalInfo { common::ObAddr addr_; - table::ObTableLoadArray partition_id_array_; + table::ObTableLoadArray partition_id_array_; TO_STRING_KV(K_(addr), K_(partition_id_array)); }; - struct LeaderInfoForSort - { - common::ObAddr addr_; - common::ObIArray *partition_id_array_ptr_; - TO_STRING_KV(K_(addr), KP_(partition_id_array_ptr)); - }; -public: - ObTableLoadPartitionLocation() - : allocator_("TLD_PL"), - is_inited_(false) - { - - - } + + ObTableLoadPartitionLocation() + : allocator_("TLD_PL"), is_inited_(false) + {} + int init(const common::ObIArray &partition_ids); - int get_leader(common::ObTabletID tablet_id, PartitionLocationInfo &info) const; - int get_all_leader(table::ObTableLoadArray &addr_array) const; - int get_all_leader_info(table::ObTableLoadArray &info_array) const; - void reset() { - tablet_ids_.reset(); - partition_map_.destroy(); - all_leader_addr_array_.reset(); - all_leader_info_array_.reset(); + int get_local_info(LocalInfo &info) const; + void reset() + { + local_info_.addr_.reset(); + local_info_.partition_id_array_.reset(); allocator_.reset(); is_inited_ = false; } - int check_tablet_has_same_leader(const ObTableLoadPartitionLocation &other, bool &result); -public: - static int init_partition_location(const ObIArray &partition_ids, - const ObIArray &target_partition_ids, - ObTableLoadPartitionLocation &partition_location, - ObTableLoadPartitionLocation &target_partition_location); - // Get through tablet_id - static int fetch_ls_id(const common::ObTabletID &tablet_id, - share::ObLSID &ls_id); - static int fetch_ls_location(const common::ObTabletID &tablet_id, - share::ObLSLocation &ls_location, share::ObLSID &ls_id); - static int fetch_tablet_handle(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - storage::ObTabletHandle &tablet_handle); + static int init_partition_location( + const common::ObIArray &partition_ids, + const common::ObIArray &target_partition_ids, + ObTableLoadPartitionLocation &partition_location, + ObTableLoadPartitionLocation &target_partition_location); + private: - int init_all_partition_location( - const common::ObIArray &partition_ids); - int init_all_leader_info(); - int fetch_ls_locations( - const common::ObIArray &partition_ids); - private: common::ObArenaAllocator allocator_; - common::ObArray tablet_ids_; // Ensure the order remains unchanged when traversing partition_map_ - common::hash::ObHashMap partition_map_; - table::ObTableLoadArray all_leader_addr_array_; - table::ObTableLoadArray all_leader_info_array_; + LocalInfo local_info_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_redef_table.cpp b/src/observer/table_load/ob_table_load_redef_table.cpp index 8da42173c..c794646e5 100644 --- a/src/observer/table_load/ob_table_load_redef_table.cpp +++ b/src/observer/table_load/ob_table_load_redef_table.cpp @@ -86,30 +86,6 @@ int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { - ObExecContext *exec_ctx = session_info.get_cur_exec_ctx(); - const ObPhysicalPlanCtx *plan_ctx = nullptr; - const ObPhysicalPlan *plan = nullptr; - if (OB_ISNULL(exec_ctx) - || OB_ISNULL(plan_ctx = exec_ctx->get_physical_plan_ctx()) - || OB_ISNULL(plan = plan_ctx->get_phy_plan())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null physical plan (ctx)", KR(ret), KP(plan_ctx), KP(plan)); - } else { - res.task_id_ = plan->get_ddl_task_id(); - share::ObDDLTaskStatus status = share::ObDDLTaskStatus::PREPARE; - bool unused_is_offline_index_rebuild = false; - if (OB_FAIL(ObDDLUtil::get_data_information(res.task_id_, - res.data_format_version_, - res.snapshot_version_, - status, - res.dest_table_id_, - res.schema_version_, - res.is_no_logging_, - unused_is_offline_index_rebuild))) { - LOG_WARN("fail to get ddl task info", KR(ret), K(arg)); - } - } } else { const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); ObCreateHiddenTableArg create_table_arg; @@ -172,8 +148,6 @@ int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { - //pass } else { int64_t foreign_key_checks = 0; session_info.get_foreign_key_checks(foreign_key_checks); @@ -203,8 +177,6 @@ int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg, build_single_replica_response_arg.dest_schema_id_ = arg.dest_table_id_; build_single_replica_response_arg.schema_version_ = arg.schema_version_; build_single_replica_response_arg.dest_schema_version_ = arg.schema_version_; - build_single_replica_response_arg.ls_id_ = share::ObLSID(1); - build_single_replica_response_arg.dest_ls_id_ = share::ObLSID(1); build_single_replica_response_arg.tablet_id_ = ObTableID(-1); build_single_replica_response_arg.snapshot_version_ = 1; build_single_replica_response_arg.execution_id_ = 1; @@ -232,8 +204,6 @@ int ObTableLoadRedefTable::abort(const ObTableLoadRedefTableAbortArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { - //pass } else { const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); ObAbortRedefTableArg abort_redef_table_arg; diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 133f0545d..246cba23e 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -405,9 +405,6 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu if (table_schema->is_view_table()) { LOG_WARN("direct-load does not support view table", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support view table", tmp_prefix); - } else if (table_schema->is_mlog_table()) { - LOG_WARN("direct-load does not support materialized view log table", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support materialized view log table", tmp_prefix); } else { LOG_WARN("direct-load does not support non-user table", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support non-user table", tmp_prefix); @@ -437,16 +434,6 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu LOG_WARN("direct-load does not support table with trigger enabled", KR(ret), K(trigger_enabled)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table with trigger enabled", tmp_prefix); } - // check if table has mlog - else if (table_schema->has_mlog_table() && !table_schema->mv_container_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support table with materialized view log", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table with materialized view log", tmp_prefix); - } else if (table_schema->table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support table required by materialized view", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table required by materialized view", tmp_prefix); - } // check for full text search index else if (OB_FAIL(check_support_direct_load_for_fts_index(schema_guard, table_schema, method, load_mode))) { LOG_WARN("fail to check support direct load for fts index", KR(ret)); @@ -709,20 +696,20 @@ int ObTableLoadService::remove_ctx(ObTableLoadTableCtx *table_ctx) } else { int tmp_ret = OB_SUCCESS; ObDirectLoadResourceReleaseArg release_arg; - release_arg.task_key_ = ObTableLoadUniqueKey(table_ctx->param_.table_id_, table_ctx->ddl_param_.task_id_); if (OB_FAIL(service->get_manager().remove_table_ctx(release_arg.task_key_, table_ctx))) { LOG_WARN("fail to remove_table_ctx", KR(ret), K(release_arg.task_key_)); } else { if (table_ctx->is_assigned_memory()) { - if (OB_TMP_FAIL(service->assigned_memory_manager_.recycle_memory(table_ctx->param_.task_need_sort_, table_ctx->param_.avail_memory_))) { - LOG_WARN("fail to recycle_memory", KR(tmp_ret), K(release_arg.task_key_)); + if (OB_TMP_FAIL(service->assigned_memory_manager_.recycle_memory( + table_ctx->param_.task_need_sort_, table_ctx->param_.avail_memory_))) { + LOG_WARN("fail to recycle memory", KR(tmp_ret), K(release_arg.task_key_)); } table_ctx->reset_assigned_memory(); - } + } if (table_ctx->is_assigned_resource()) { - if (OB_TMP_FAIL(ObTableLoadService::delete_assigned_task(release_arg))) { - LOG_WARN("fail to delete assigned task", KR(tmp_ret), K(release_arg)); + if (OB_TMP_FAIL(ObTableLoadResourceService::release_resource(release_arg))) { + LOG_WARN("fail to release assigned resource", KR(tmp_ret), K(release_arg)); } table_ctx->reset_assigned_resource(); } @@ -777,8 +764,6 @@ int ObTableLoadService::init() LOG_WARN("fail to init table ctx manager", KR(ret)); } else if (OB_FAIL(assigned_memory_manager_.init())) { LOG_WARN("fail to init assigned memory manager", KR(ret)); - } else if (OB_FAIL(assigned_task_manager_.init())) { - LOG_WARN("fail to init assigned task manager", KR(ret)); } else { is_inited_ = true; } @@ -935,38 +920,6 @@ int ObTableLoadService::get_memory_limit(int64_t &memory_limit) return ret; } -int ObTableLoadService::add_assigned_task(ObDirectLoadResourceApplyArg &arg) -{ - int ret = OB_SUCCESS; - ObTableLoadService *service = nullptr; - if (OB_ISNULL(service = share::g_mp->table_load_service())) { - ret = OB_ERR_SYS; - LOG_WARN("null table load service", KR(ret)); - } else { - ret = service->assigned_task_manager_.add_assigned_task(arg); - } - return ret; -} - -int ObTableLoadService::delete_assigned_task(ObDirectLoadResourceReleaseArg &arg) -{ - int ret = OB_SUCCESS; - ObTableLoadService *service = nullptr; - if (OB_ISNULL(service = share::g_mp->table_load_service())) { - ret = OB_ERR_SYS; - LOG_WARN("null table load service", KR(ret)); - } else { - if (OB_FAIL(service->assigned_task_manager_.delete_assigned_task(arg.task_key_))) { - LOG_WARN("fail to delete_assigned_task", KR(ret), K(arg.task_key_)); - } else if (OB_FAIL(ObTableLoadResourceService::release_resource(arg))) { - LOG_WARN("fail to release resource", KR(ret)); - ret = OB_SUCCESS; // Allow failure, the resource management module can reclaim - } - } - - return ret; -} - int ObTableLoadService::assign_memory(bool is_sort, int64_t assign_memory) { int ret = OB_SUCCESS; @@ -994,7 +947,7 @@ int ObTableLoadService::recycle_memory(bool is_sort, int64_t assign_memory) return ret; } -int ObTableLoadService::get_sort_memory(int64_t &sort_memory) +int ObTableLoadService::refresh_avail_memory(int64_t avail_memory) { int ret = OB_SUCCESS; ObTableLoadService *service = nullptr; @@ -1002,12 +955,12 @@ int ObTableLoadService::get_sort_memory(int64_t &sort_memory) ret = OB_ERR_SYS; LOG_WARN("null table load service", KR(ret)); } else { - ret = service->assigned_memory_manager_.get_sort_memory(sort_memory); + ret = service->assigned_memory_manager_.refresh_avail_memory(avail_memory); } return ret; } -int ObTableLoadService::refresh_and_check_resource(ObDirectLoadResourceCheckArg &arg, ObDirectLoadResourceOpRes &res) +int ObTableLoadService::get_sort_memory(int64_t &sort_memory) { int ret = OB_SUCCESS; ObTableLoadService *service = nullptr; @@ -1015,12 +968,7 @@ int ObTableLoadService::refresh_and_check_resource(ObDirectLoadResourceCheckArg ret = OB_ERR_SYS; LOG_WARN("null table load service", KR(ret)); } else { - res.avail_memory_ = service->assigned_memory_manager_.get_avail_memory(); - if (!arg.first_check_ && OB_FAIL(service->assigned_memory_manager_.refresh_avail_memory(arg.avail_memory_))) { - LOG_WARN("fail to refresh_avail_memory", KR(ret)); - } else if (OB_FAIL(service->assigned_task_manager_.get_assigned_tasks(res.assigned_array_))) { - LOG_WARN("fail to get_assigned_tasks", KR(ret)); - } + ret = service->assigned_memory_manager_.get_sort_memory(sort_memory); } return ret; } diff --git a/src/observer/table_load/ob_table_load_service.h b/src/observer/table_load/ob_table_load_service.h index 1961efaa7..99f44715a 100644 --- a/src/observer/table_load/ob_table_load_service.h +++ b/src/observer/table_load/ob_table_load_service.h @@ -19,12 +19,10 @@ #include "lib/task/ob_timer.h" #include "observer/table_load/control/ob_table_load_control_rpc.h" #include "observer/table_load/ob_table_load_assigned_memory_manager.h" -#include "observer/table_load/ob_table_load_assigned_task_manager.h" #include "observer/table_load/ob_table_load_client_service.h" #include "observer/table_load/ob_table_load_coordinator.h" #include "observer/table_load/ob_table_load_manager.h" #include "observer/table_load/ob_table_load_struct.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc.h" #include "observer/table_load/resource/ob_table_load_resource_service.h" #include "storage/direct_load/ob_direct_load_struct.h" #include "src/observer/table_load/ob_table_load_assigned_memory_manager.h" @@ -89,19 +87,11 @@ class ObTableLoadService { return ObTableLoadControlRpcProxy::dispatch(request, result, allocator); } - // for direct load resource api - static int direct_load_resource(const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result, common::ObIAllocator &allocator) - { - return ObTableLoadResourceRpcProxy::dispatch(request, result, allocator); - } static int get_memory_limit(int64_t &memory_limit); - static int add_assigned_task(ObDirectLoadResourceApplyArg &arg); - static int delete_assigned_task(ObDirectLoadResourceReleaseArg &arg); static int assign_memory(bool is_sort, int64_t assign_memory); static int recycle_memory(bool is_sort, int64_t assign_memory); - static int get_sort_memory(int64_t &sort_memory); - static int refresh_and_check_resource(ObDirectLoadResourceCheckArg &arg, ObDirectLoadResourceOpRes &res); + static int refresh_avail_memory(int64_t avail_memory); + static int get_sort_memory(int64_t &sort_memory); public: ObTableLoadService(); int init(); @@ -168,7 +158,6 @@ class ObTableLoadService private: ObTableLoadManager manager_; ObTableLoadAssignedMemoryManager assigned_memory_manager_; - ObTableLoadAssignedTaskManager assigned_task_manager_; common::ObTimer timer_; ObGCTask gc_task_; ObReleaseTask release_task_; diff --git a/src/observer/table_load/ob_table_load_store.cpp b/src/observer/table_load/ob_table_load_store.cpp index e58cd9d42..2d20873bb 100644 --- a/src/observer/table_load/ob_table_load_store.cpp +++ b/src/observer/table_load/ob_table_load_store.cpp @@ -52,8 +52,8 @@ ObTableLoadStore::ObTableLoadStore(ObTableLoadTableCtx *ctx) int ObTableLoadStore::init_ctx( ObTableLoadTableCtx *ctx, - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (OB_ISNULL(ctx)) { diff --git a/src/observer/table_load/ob_table_load_store.h b/src/observer/table_load/ob_table_load_store.h index d4a09adbe..cd43e7809 100644 --- a/src/observer/table_load/ob_table_load_store.h +++ b/src/observer/table_load/ob_table_load_store.h @@ -42,8 +42,8 @@ class ObTableLoadStore ObTableLoadStore(ObTableLoadTableCtx *ctx); static int init_ctx( ObTableLoadTableCtx *ctx, - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); static void abort_ctx(ObTableLoadTableCtx *ctx, int error_code, bool &is_stopped); int init(); private: diff --git a/src/observer/table_load/ob_table_load_store_ctx.cpp b/src/observer/table_load/ob_table_load_store_ctx.cpp index adc599787..4da1b9d8c 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_ctx.cpp @@ -208,8 +208,8 @@ void ObTableLoadStoreCtx::destroy() } int ObTableLoadStoreCtx::init( - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -470,8 +470,8 @@ int ObTableLoadStoreCtx::init_trans_param(storage::ObDirectLoadTransParam &trans } int ObTableLoadStoreCtx::init_store_table_ctxs( - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; // init data store_table_ctx_ @@ -608,7 +608,7 @@ int ObTableLoadStoreCtx::init_write_ctx() break; // No resource control case ObTableLoadExeMode::MAX_TYPE: { - const int64_t part_cnt = data_store_table_ctx_->ls_partition_ids_.count(); + const int64_t part_cnt = data_store_table_ctx_->partition_ids_.count(); int64_t wa_mem_limit = 0; if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) { LOG_WARN("fail to get memory limit", KR(ret)); @@ -711,11 +711,11 @@ int ObTableLoadStoreCtx::init_write_ctx() } } if (OB_SUCC(ret) && ctx_->param_.px_mode_) { - const ObArray &ls_partition_ids = - data_store_table_ctx_->ls_partition_ids_; - write_ctx_.is_single_part_ = (1 == ls_partition_ids.count()); + const ObArray &partition_ids = + data_store_table_ctx_->partition_ids_; + write_ctx_.is_single_part_ = (1 == partition_ids.count()); if (write_ctx_.is_single_part_) { - write_ctx_.single_tablet_id_ = ls_partition_ids[0].part_tablet_id_.tablet_id_; + write_ctx_.single_tablet_id_ = partition_ids[0].part_tablet_id_.tablet_id_; if (OB_FAIL(ObDirectLoadVectorUtils::make_const_tablet_id_vector(write_ctx_.single_tablet_id_, allocator_, write_ctx_.single_tablet_id_vector_))) { @@ -725,8 +725,8 @@ int ObTableLoadStoreCtx::init_write_ctx() if (OB_FAIL(write_ctx_.tablet_idx_map_.create(1024, "TLD_TbtIdxMap", "TLD_TbtIdxMap"))) { LOG_WARN("fail to create hashmap", KR(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTabletID &tablet_id = ls_partition_ids[i].part_tablet_id_.tablet_id_; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTabletID &tablet_id = partition_ids[i].part_tablet_id_.tablet_id_; if (OB_FAIL(write_ctx_.tablet_idx_map_.set_refactored(tablet_id.id(), i))) { LOG_WARN("fail to set refactored", KR(ret), K(tablet_id)); if (OB_HASH_EXIST == ret) { @@ -781,7 +781,7 @@ int ObTableLoadStoreCtx::init_write_ctx_for_dag() break; // No resource control case ObTableLoadExeMode::MAX_TYPE: { - const int64_t part_cnt = data_store_table_ctx_->ls_partition_ids_.count(); + const int64_t part_cnt = data_store_table_ctx_->partition_ids_.count(); int64_t wa_mem_limit = 0; if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) { LOG_WARN("fail to get memory limit", KR(ret)); @@ -872,11 +872,11 @@ int ObTableLoadStoreCtx::init_write_ctx_for_dag() } } if (OB_SUCC(ret) && ctx_->param_.px_mode_) { - const ObArray &ls_partition_ids = - data_store_table_ctx_->ls_partition_ids_; - write_ctx_.is_single_part_ = (1 == ls_partition_ids.count()); + const ObArray &partition_ids = + data_store_table_ctx_->partition_ids_; + write_ctx_.is_single_part_ = (1 == partition_ids.count()); if (write_ctx_.is_single_part_) { - write_ctx_.single_tablet_id_ = ls_partition_ids[0].part_tablet_id_.tablet_id_; + write_ctx_.single_tablet_id_ = partition_ids[0].part_tablet_id_.tablet_id_; if (OB_FAIL(ObDirectLoadVectorUtils::make_const_tablet_id_vector( write_ctx_.single_tablet_id_, allocator_, write_ctx_.single_tablet_id_vector_))) { LOG_WARN("fail to make const tablet id vector", KR(ret), K(write_ctx_.single_tablet_id_)); @@ -886,8 +886,8 @@ int ObTableLoadStoreCtx::init_write_ctx_for_dag() write_ctx_.tablet_idx_map_.create(1024, "TLD_TbtIdxMap", "TLD_TbtIdxMap"))) { LOG_WARN("fail to create hashmap", KR(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTabletID &tablet_id = ls_partition_ids[i].part_tablet_id_.tablet_id_; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTabletID &tablet_id = partition_ids[i].part_tablet_id_.tablet_id_; if (OB_FAIL(write_ctx_.tablet_idx_map_.set_refactored(tablet_id.id(), i))) { LOG_WARN("fail to set refactored", KR(ret), K(tablet_id)); if (OB_HASH_EXIST == ret) { diff --git a/src/observer/table_load/ob_table_load_store_ctx.h b/src/observer/table_load/ob_table_load_store_ctx.h index ff6a54211..402a0f30c 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.h +++ b/src/observer/table_load/ob_table_load_store_ctx.h @@ -127,8 +127,8 @@ class ObTableLoadStoreCtx ~ObTableLoadStoreCtx(); void destroy(); int init( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); void stop(); bool is_valid() const { return is_inited_; } bool is_stopped() const; @@ -216,8 +216,8 @@ class ObTableLoadStoreCtx int alloc_trans(const table::ObTableLoadTransId &trans_id, ObTableLoadStoreTrans *&trans); int init_session_ctx_array(); int init_store_table_ctxs( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); int init_sort_param(); int generate_autoinc_params(share::AutoincParam &autoinc_param); int init_sequence(); diff --git a/src/observer/table_load/ob_table_load_store_table_ctx.cpp b/src/observer/table_load/ob_table_load_store_table_ctx.cpp index 14c6edc7f..5dd03be8e 100644 --- a/src/observer/table_load/ob_table_load_store_table_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_table_ctx.cpp @@ -29,6 +29,7 @@ #include "storage/direct_load/ob_direct_load_insert_lob_table_ctx.h" #include "storage/direct_load/ob_direct_load_struct.h" #include "storage/tablet/ob_tablet.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -276,8 +277,8 @@ ObTableLoadStoreDataTableCtx::~ObTableLoadStoreDataTableCtx() int ObTableLoadStoreDataTableCtx::init( const uint64_t table_id, - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -299,7 +300,7 @@ int ObTableLoadStoreDataTableCtx::init( LOG_WARN("fail to init table store", KR(ret)); } else if (OB_FAIL(ack_table_store_.init())) { LOG_WARN("fail to init table store", KR(ret)); - } else if (OB_FAIL(init_ls_partition_ids(partition_id_array, target_partition_id_array))) { + } else if (OB_FAIL(init_partition_ids(partition_id_array, target_partition_id_array))) { LOG_WARN("fail to init ls partition ids", KR(ret)); } else if (!schema_->lob_column_idxs_.empty()) { if (OB_ISNULL(lob_table_ctx_ = @@ -353,47 +354,44 @@ int ObTableLoadStoreDataTableCtx::init_data_project() return ret; } -int ObTableLoadStoreDataTableCtx::init_ls_partition_ids( - const ObTableLoadArray &ls_partition_ids, - const ObTableLoadArray &target_ls_partition_ids) +int ObTableLoadStoreDataTableCtx::init_partition_ids( + const ObTableLoadArray &partition_ids, + const ObTableLoadArray &target_partition_ids) { int ret = OB_SUCCESS; const bool is_incremental = ObDirectLoadMethod::is_incremental(store_ctx_->ctx_->param_.method_); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = ls_partition_ids[i]; - if (is_incremental && OB_FAIL(check_tablet(ls_partition_id.ls_id_, - ls_partition_id.part_tablet_id_.tablet_id_, + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = partition_ids[i]; + if (is_incremental && OB_FAIL(check_tablet(partition_id.part_tablet_id_.tablet_id_, store_ctx_->ctx_->ddl_param_.schema_version_))) { LOG_WARN("fail to check tablet", KR(ret), - K(ls_partition_id.ls_id_), - K(ls_partition_id.part_tablet_id_.tablet_id_)); - } else if (OB_FAIL(ls_partition_ids_.push_back(ls_partition_id))) { + K(partition_id.part_tablet_id_.tablet_id_)); + } else if (OB_FAIL(partition_ids_.push_back(partition_id))) { LOG_WARN("fail to push back ", KR(ret)); } } - for (int64_t i = 0; OB_SUCC(ret) && i < target_ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = target_ls_partition_ids[i]; - if (OB_FAIL(target_ls_partition_ids_.push_back(ls_partition_id))) { + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = target_partition_ids[i]; + if (OB_FAIL(target_partition_ids_.push_back(partition_id))) { LOG_WARN("fail to push back", KR(ret)); } } return ret; } -int ObTableLoadStoreDataTableCtx::check_tablet(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObTableLoadStoreDataTableCtx::check_tablet(const ObTabletID &tablet_id, const int64_t schema_version) { int ret = OB_SUCCESS; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls service is nullptr", KR(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("fail to get ls", KR(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -459,8 +457,8 @@ int ObTableLoadStoreDataTableCtx::init_insert_table_ctx(const ObDirectLoadTransP OB_NEWx(ObDirectLoadInsertDataTableContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertDataTableContext", KR(ret)); - } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, ls_partition_ids_, - target_ls_partition_ids_))) { + } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, partition_ids_, + target_partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } // init lob insert_table_ctx_ @@ -545,8 +543,8 @@ int ObTableLoadStoreDataTableCtx::open_insert_table_ctx( OB_NEWx(ObDirectLoadInsertDataTableContext, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertDataTableContext", KR(ret)); - } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, ls_partition_ids_, - target_ls_partition_ids_))) { + } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, partition_ids_, + target_partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } else { insert_table_ctx = insert_data_table_ctx; @@ -638,8 +636,8 @@ ObTableLoadStoreLobTableCtx::~ObTableLoadStoreLobTableCtx() int ObTableLoadStoreLobTableCtx::init( const uint64_t table_id, - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -657,7 +655,7 @@ int ObTableLoadStoreLobTableCtx::init( LOG_WARN("fail to init delete table store", KR(ret)); } else if (OB_FAIL(tablet_id_map_.create(1024, ObMemAttr("TLD_TbltIDMap")))) { LOG_WARN("fail to create hashmap", KR(ret)); - } else if (OB_FAIL(init_ls_partition_ids(partition_id_array, target_partition_id_array))) { + } else if (OB_FAIL(init_partition_ids(partition_id_array, target_partition_id_array))) { LOG_WARN("fail to init ls partition ids", KR(ret)); } else { is_inited_ = true; @@ -665,38 +663,36 @@ int ObTableLoadStoreLobTableCtx::init( return ret; } -int ObTableLoadStoreLobTableCtx::init_ls_partition_ids( - const ObTableLoadArray &ls_partition_ids, - const ObTableLoadArray &target_ls_partition_ids) +int ObTableLoadStoreLobTableCtx::init_partition_ids( + const ObTableLoadArray &partition_ids, + const ObTableLoadArray &target_partition_ids) { int ret = OB_SUCCESS; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletBindingMdsUserData ddl_data; - ObTableLoadLSIdAndPartitionId lob_ls_partition_id; + ObTableLoadTabletId lob_partition_id; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = ls_partition_ids[i]; - const ObLSID &ls_id = ls_partition_id.ls_id_; - const ObTabletID &tablet_id = ls_partition_id.part_tablet_id_.tablet_id_; - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = partition_ids[i]; + const ObTabletID &tablet_id = partition_id.part_tablet_id_.tablet_id_; + if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet handle failed", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(SCN::max_scn(), ddl_data))) { LOG_WARN("get ddl data failed", K(ret)); } else { - lob_ls_partition_id.ls_id_ = ls_id; - lob_ls_partition_id.part_tablet_id_.partition_id_ = - ls_partition_id.part_tablet_id_.partition_id_; - lob_ls_partition_id.part_tablet_id_.tablet_id_ = ddl_data.lob_meta_tablet_id_; - if (OB_FAIL(ls_partition_ids_.push_back(lob_ls_partition_id))) { + lob_partition_id.part_tablet_id_.partition_id_ = + partition_id.part_tablet_id_.partition_id_; + lob_partition_id.part_tablet_id_.tablet_id_ = ddl_data.lob_meta_tablet_id_; + if (OB_FAIL(partition_ids_.push_back(lob_partition_id))) { LOG_WARN("fail to push back ", KR(ret)); } // set map @@ -705,24 +701,22 @@ int ObTableLoadStoreLobTableCtx::init_ls_partition_ids( } } } - for (int64_t i = 0; OB_SUCC(ret) && i < target_ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = target_ls_partition_ids[i]; - const ObLSID &ls_id = ls_partition_id.ls_id_; - const ObTabletID &tablet_id = ls_partition_id.part_tablet_id_.tablet_id_; - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = target_partition_ids[i]; + const ObTabletID &tablet_id = partition_id.part_tablet_id_.tablet_id_; + if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet handle failed", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(SCN::max_scn(), ddl_data))) { LOG_WARN("get ddl data failed", K(ret)); } else { - lob_ls_partition_id.ls_id_ = ls_id; - lob_ls_partition_id.part_tablet_id_.partition_id_ = - ls_partition_id.part_tablet_id_.partition_id_; - lob_ls_partition_id.part_tablet_id_.tablet_id_ = ddl_data.lob_meta_tablet_id_; - if (OB_FAIL(target_ls_partition_ids_.push_back(lob_ls_partition_id))) { + lob_partition_id.part_tablet_id_.partition_id_ = + partition_id.part_tablet_id_.partition_id_; + lob_partition_id.part_tablet_id_.tablet_id_ = ddl_data.lob_meta_tablet_id_; + if (OB_FAIL(target_partition_ids_.push_back(lob_partition_id))) { LOG_WARN("fail to push back", KR(ret)); } } @@ -812,8 +806,8 @@ int ObTableLoadStoreLobTableCtx::init_insert_table_ctx(const ObDirectLoadTransPa } else if (OB_FAIL(insert_lob_table_ctx->init(insert_table_param, static_cast( data_table_ctx_->insert_table_ctx_), - ls_partition_ids_, target_ls_partition_ids_, - data_table_ctx_->ls_partition_ids_))) { + partition_ids_, target_partition_ids_, + data_table_ctx_->partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } } @@ -886,7 +880,7 @@ int ObTableLoadStoreLobTableCtx::open_insert_table_ctx( LOG_WARN("fail to new ObDirectLoadInsertDataTableContext", KR(ret)); } else if (OB_FAIL(insert_lob_table_ctx->init( insert_table_param, - ls_partition_ids_, target_ls_partition_ids_))) { + partition_ids_, target_partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } else { insert_table_ctx = insert_lob_table_ctx; @@ -971,8 +965,8 @@ ObTableLoadStoreIndexTableCtx::~ObTableLoadStoreIndexTableCtx() int ObTableLoadStoreIndexTableCtx::init( const uint64_t table_id, - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -992,7 +986,7 @@ int ObTableLoadStoreIndexTableCtx::init( LOG_WARN("fail to init delete table store", KR(ret)); } else if (OB_FAIL(init_index_projector())) { LOG_WARN("fail to init index projector", KR(ret)); - } else if (OB_FAIL(init_ls_partition_ids(partition_id_array, target_partition_id_array))) { + } else if (OB_FAIL(init_partition_ids(partition_id_array, target_partition_id_array))) { LOG_WARN("fail to init ls partition ids", KR(ret)); } else { is_inited_ = true; @@ -1035,33 +1029,31 @@ int ObTableLoadStoreIndexTableCtx::init_index_projector() return ret; } -int ObTableLoadStoreIndexTableCtx::init_ls_partition_ids( - const ObTableLoadArray &ls_partition_ids, - const ObTableLoadArray &target_ls_partition_ids) +int ObTableLoadStoreIndexTableCtx::init_partition_ids( + const ObTableLoadArray &partition_ids, + const ObTableLoadArray &target_partition_ids) { int ret = OB_SUCCESS; - ObTableLoadLSIdAndPartitionId index_ls_partition_id; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = ls_partition_ids[i]; - const ObTabletID &data_tablet_id = ls_partition_id.part_tablet_id_.tablet_id_; - index_ls_partition_id.ls_id_ = ls_partition_id.ls_id_; + ObTableLoadTabletId index_partition_id; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = partition_ids[i]; + const ObTabletID &data_tablet_id = partition_id.part_tablet_id_.tablet_id_; if (OB_FAIL(project_->get_dest_tablet_id_and_part_id_by_src_tablet_id( - data_tablet_id, index_ls_partition_id.part_tablet_id_.tablet_id_, - index_ls_partition_id.part_tablet_id_.partition_id_))) { + data_tablet_id, index_partition_id.part_tablet_id_.tablet_id_, + index_partition_id.part_tablet_id_.partition_id_))) { LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id)); - } else if (OB_FAIL(ls_partition_ids_.push_back(index_ls_partition_id))) { + } else if (OB_FAIL(partition_ids_.push_back(index_partition_id))) { LOG_WARN("fail to push back", KR(ret)); } } - for (int64_t i = 0; OB_SUCC(ret) && i < target_ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = target_ls_partition_ids[i]; - const ObTabletID &data_tablet_id = ls_partition_id.part_tablet_id_.tablet_id_; - index_ls_partition_id.ls_id_ = ls_partition_id.ls_id_; + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = target_partition_ids[i]; + const ObTabletID &data_tablet_id = partition_id.part_tablet_id_.tablet_id_; if (OB_FAIL(project_->get_dest_tablet_id_and_part_id_by_src_tablet_id( - data_tablet_id, index_ls_partition_id.part_tablet_id_.tablet_id_, - index_ls_partition_id.part_tablet_id_.partition_id_))) { + data_tablet_id, index_partition_id.part_tablet_id_.tablet_id_, + index_partition_id.part_tablet_id_.partition_id_))) { LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id)); - } else if (OB_FAIL(target_ls_partition_ids_.push_back(index_ls_partition_id))) { + } else if (OB_FAIL(target_partition_ids_.push_back(index_partition_id))) { LOG_WARN("fail to push back", KR(ret)); } } @@ -1120,8 +1112,8 @@ int ObTableLoadStoreIndexTableCtx::init_insert_table_ctx(const ObDirectLoadTrans OB_NEWx(ObDirectLoadInsertDataTableContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertDataTableContext", KR(ret)); - } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, ls_partition_ids_, - target_ls_partition_ids_))) { + } else if (OB_FAIL(insert_data_table_ctx->init(insert_table_param, partition_ids_, + target_partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } } @@ -1192,8 +1184,8 @@ int ObTableLoadStoreIndexTableCtx::open_insert_table_ctx( OB_NEWx(ObDirectLoadInsertDataTableContext, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertDataTableContext", KR(ret)); - } else if (OB_FAIL(insert_index_table_ctx->init(insert_table_param, ls_partition_ids_, - target_ls_partition_ids_))) { + } else if (OB_FAIL(insert_index_table_ctx->init(insert_table_param, partition_ids_, + target_partition_ids_))) { LOG_WARN("fail to init insert table ctx", KR(ret)); } else { insert_table_ctx = insert_index_table_ctx; diff --git a/src/observer/table_load/ob_table_load_store_table_ctx.h b/src/observer/table_load/ob_table_load_store_table_ctx.h index abc332fd8..371bfab63 100644 --- a/src/observer/table_load/ob_table_load_store_table_ctx.h +++ b/src/observer/table_load/ob_table_load_store_table_ctx.h @@ -77,8 +77,8 @@ class ObTableLoadStoreTableCtx ~ObTableLoadStoreTableCtx(); virtual int init( const uint64_t table_id, - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) = 0; int get_table_data_desc(const ObTableLoadInputDataType::Type input_data_type, @@ -125,8 +125,8 @@ public: ObTableLoadStoreCtx *store_ctx_; uint64_t table_id_; ObTableLoadSchema *schema_; - common::ObArray ls_partition_ids_; - common::ObArray target_ls_partition_ids_; + common::ObArray partition_ids_; + common::ObArray target_partition_ids_; storage::ObDirectLoadInsertTableContext *insert_table_ctx_; protected: @@ -140,20 +140,19 @@ class ObTableLoadStoreDataTableCtx : public ObTableLoadStoreTableCtx ObTableLoadStoreDataTableCtx(ObTableLoadStoreCtx *store_ctx); virtual ~ObTableLoadStoreDataTableCtx(); int init(const uint64_t table_id, - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) override; ObTableLoadStoreLobTableCtx *get_lob_table_ctx() const { return lob_table_ctx_; } private: int init_data_project(); - int init_ls_partition_ids( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + int init_partition_ids( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); int acquire_table_builder(ObTableLoadDataTableBuilder *&table_builder, ObIAllocator &allocator, ObDirectLoadTableDataDesc table_data_desc); - int check_tablet(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, + int check_tablet(const ObTabletID &tablet_id, const int64_t schema_version); //////////////////////// insert_table_ctx //////////////////////// public: @@ -182,16 +181,16 @@ class ObTableLoadStoreLobTableCtx : public ObTableLoadStoreTableCtx ObTableLoadStoreDataTableCtx *data_table_ctx); virtual ~ObTableLoadStoreLobTableCtx(); int init(const uint64_t table_id, - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) override; ObTableLoadStoreDataTableCtx *get_data_table_ctx() const { return data_table_ctx_; } int get_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &tablet_id); private: - int init_ls_partition_ids( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + int init_partition_ids( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); int acquire_table_builder(ObTableLoadLobTableBuilder *&table_builder, ObIAllocator &allocator, ObDirectLoadTableDataDesc table_data_desc); //////////////////////// insert_table_ctx //////////////////////// @@ -223,15 +222,15 @@ class ObTableLoadStoreIndexTableCtx : public ObTableLoadStoreTableCtx ObTableLoadStoreIndexTableCtx(ObTableLoadStoreCtx *store_ctx); virtual ~ObTableLoadStoreIndexTableCtx(); int init(const uint64_t table_id, - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) override; private: int init_index_projector(); - int init_ls_partition_ids( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + int init_partition_ids( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); int acquire_table_builder(ObTableLoadIndexTableBuilder *&table_builder, ObIAllocator &allocator, ObDirectLoadTableDataDesc table_data_desc); //////////////////////// insert_table_ctx //////////////////////// diff --git a/src/observer/table_load/ob_table_load_table_ctx.cpp b/src/observer/table_load/ob_table_load_table_ctx.cpp index 00bed2f1a..433489b4c 100644 --- a/src/observer/table_load/ob_table_load_table_ctx.cpp +++ b/src/observer/table_load/ob_table_load_table_ctx.cpp @@ -233,8 +233,8 @@ int ObTableLoadTableCtx::init_coordinator_ctx(const ObIArray &column_i } int ObTableLoadTableCtx::init_store_ctx( - const ObTableLoadArray &partition_id_array, - const ObTableLoadArray &target_partition_id_array) + const ObTableLoadArray &partition_id_array, + const ObTableLoadArray &target_partition_id_array) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { diff --git a/src/observer/table_load/ob_table_load_table_ctx.h b/src/observer/table_load/ob_table_load_table_ctx.h index 42211d9c2..e7ea84a58 100644 --- a/src/observer/table_load/ob_table_load_table_ctx.h +++ b/src/observer/table_load/ob_table_load_table_ctx.h @@ -77,8 +77,8 @@ class ObTableLoadTableCtx : public common::ObDLinkBase const common::ObIArray &tablet_ids, ObTableLoadExecCtx *exec_ctx); int init_store_ctx( - const table::ObTableLoadArray &partition_id_array, - const table::ObTableLoadArray &target_partition_id_array); + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); public: int alloc_task(ObTableLoadTask *&task); void free_task(ObTableLoadTask *task); diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index 83eaab747..13fa84b7c 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -40,7 +40,7 @@ using namespace table; using namespace common::number; ObTableLoadTransBucketWriter::SessionContext::SessionContext() - : session_id_(0), allocator_("TLD_TB_SessCtx"), last_receive_sequence_no_(0) + : session_id_(0), last_receive_sequence_no_(0) { @@ -53,13 +53,7 @@ ObTableLoadTransBucketWriter::SessionContext::~SessionContext() void ObTableLoadTransBucketWriter::SessionContext::reset() { - for (int64_t i = 0; i < load_bucket_array_.count(); ++i) { - ObTableLoadBucket *load_bucket = load_bucket_array_.at(i); - load_bucket->~ObTableLoadBucket(); - allocator_.free(load_bucket); - } - load_bucket_array_.reset(); - load_bucket_map_.reuse(); + load_bucket_.reset(); } ObTableLoadTransBucketWriter::ObTableLoadTransBucketWriter(ObTableLoadTransCtx *trans_ctx) @@ -132,22 +126,12 @@ int ObTableLoadTransBucketWriter::init_session_ctx_array() SessionContext *session_ctx = session_ctx_array_ + i; session_ctx->session_id_ = i + 1; if (!is_partitioned_) { - ObTableLoadPartitionLocation::PartitionLocationInfo info; if (OB_UNLIKELY(1 != coordinator_ctx_->partition_ids_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected partition id num in non partitioned table", KR(ret), "count", coordinator_ctx_->partition_ids_.count()); } else if (FALSE_IT(session_ctx->partition_id_ = coordinator_ctx_->partition_ids_[0])) { - } else if (OB_FAIL(coordinator_ctx_->partition_location_.get_leader( - session_ctx->partition_id_.tablet_id_, info))) { - LOG_WARN("failed to get leader addr", K(ret)); - } else if (OB_FAIL(session_ctx->load_bucket_.init(info.leader_addr_))) { - LOG_WARN("fail to init bucket", KR(ret)); - } - } else { - if (OB_FAIL(session_ctx->load_bucket_map_.create(1024, "TLD_BucketMap", "TLD_BucketMap"))) { - LOG_WARN("fail to init partition bucket map", KR(ret)); } } } @@ -401,7 +385,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ for (int64_t i = 0; OB_SUCC(ret) && i < row_idxs.count(); ++i) { const ObTableLoadPartitionId &partition_id = partition_ids.at(i); const ObTableLoadObjRow &row = obj_rows.at(row_idxs.at(i)); - ObTableLoadBucket *load_bucket = nullptr; + ObTableLoadBucket *load_bucket = &session_ctx.load_bucket_; bool need_write = false; if (OB_UNLIKELY(!partition_id.is_valid())) { ret = OB_NO_PARTITION_FOR_GIVEN_VALUE; @@ -411,9 +395,6 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ } else { ret = OB_SUCCESS; } - } else if (OB_FAIL(get_load_bucket(session_ctx, partition_id, load_bucket))) { - LOG_WARN("fail to get partition bucket", KR(ret), K(session_ctx.session_id_), - K(partition_id)); } else if (OB_FAIL(load_bucket->add_row(partition_id.tablet_id_, row, param_.batch_size_, @@ -439,21 +420,10 @@ int ObTableLoadTransBucketWriter::flush(int32_t session_id, bool &is_finished) LOG_WARN("invalid args", KR(ret), K(session_id)); } else { SessionContext &session_ctx = session_ctx_array_[session_id - 1]; - if (!is_partitioned_) { - ObTableLoadBucket *load_bucket = &session_ctx.load_bucket_; - if (!(load_bucket->row_array_.empty())) { - if (OB_FAIL(write_load_bucket(session_ctx, load_bucket))) { - LOG_WARN("fail to write partition bucket", KR(ret), KPC(load_bucket)); - } - } - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < session_ctx.load_bucket_array_.count(); ++i) { - ObTableLoadBucket *load_bucket = session_ctx.load_bucket_array_.at(i); - if (!(load_bucket->row_array_.empty())) { - if (OB_FAIL(write_load_bucket(session_ctx, load_bucket))) { - LOG_WARN("fail to write partition bucket", KR(ret), KPC(load_bucket)); - } - } + ObTableLoadBucket *load_bucket = &session_ctx.load_bucket_; + if (!load_bucket->row_array_.empty()) { + if (OB_FAIL(write_load_bucket(session_ctx, load_bucket))) { + LOG_WARN("fail to write tablet bucket", KR(ret), KPC(load_bucket)); } } if (OB_SUCC(ret)) { @@ -466,50 +436,6 @@ int ObTableLoadTransBucketWriter::flush(int32_t session_id, bool &is_finished) return ret; } -int ObTableLoadTransBucketWriter::get_load_bucket(SessionContext &session_ctx, - const ObTableLoadPartitionId &partition_id, - ObTableLoadBucket *&load_bucket) -{ - OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, get_part_bucket_time_us); - int ret = OB_SUCCESS; - load_bucket = nullptr; - if (OB_UNLIKELY(!is_partitioned_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected non partitioned table", KR(ret)); - } else { - ObTableLoadPartitionLocation::PartitionLocationInfo info; - if (OB_FAIL(coordinator_ctx_->partition_location_.get_leader(partition_id.tablet_id_, info))) { - LOG_WARN("failed to get leader addr", K(ret)); - } - if (OB_SUCC(ret)) { - ret = session_ctx.load_bucket_map_.get_refactored(info.leader_addr_, load_bucket); - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - if (OB_ISNULL(load_bucket = OB_NEWx(ObTableLoadBucket, (&session_ctx.allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to new partition bucket", KR(ret)); - } else if (OB_FAIL(load_bucket->init(info.leader_addr_))) { - LOG_WARN("fail to init", KR(ret)); - } else if (OB_FAIL(session_ctx.load_bucket_map_.set_refactored(info.leader_addr_, load_bucket))) { - LOG_WARN("fail to put bucket", KR(ret)); - } else if (OB_FAIL(session_ctx.load_bucket_array_.push_back(load_bucket))) { - LOG_WARN("fail to push back bucket", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != load_bucket) { - load_bucket->~ObTableLoadBucket(); - session_ctx.allocator_.free(load_bucket); - load_bucket = nullptr; - } - } - } else if (OB_FAIL(ret)) { - LOG_WARN("fail to get bucket", KR(ret), K(partition_id)); - } - } - } - return ret; -} - int ObTableLoadTransBucketWriter::write_load_bucket(SessionContext &session_ctx, ObTableLoadBucket *load_bucket) { @@ -521,10 +447,10 @@ int ObTableLoadTransBucketWriter::write_load_bucket(SessionContext &session_ctx, ObTableLoadCoordinator coordinator(coordinator_ctx_->ctx_); if (OB_FAIL(coordinator.init())) { LOG_WARN("fail to init coordinator", KR(ret)); - } else if (OB_FAIL(coordinator.write_peer_leader( + } else if (OB_FAIL(coordinator.write_local( trans_ctx_->trans_id_, session_ctx.session_id_, ++load_bucket->sequence_no_, - load_bucket->row_array_, load_bucket->leader_addr_))) { - LOG_WARN("fail to coordinator write peer leader", KR(ret), K(session_ctx.session_id_), + load_bucket->row_array_))) { + LOG_WARN("fail to write local tablet rows", KR(ret), K(session_ctx.session_id_), KPC(load_bucket)); } } diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.h b/src/observer/table_load/ob_table_load_trans_bucket_writer.h index 8c1a900c4..97a23c047 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.h +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.h @@ -62,8 +62,6 @@ class ObTableLoadTransBucketWriter // partition table int write_for_partitioned(SessionContext &session_ctx, const table::ObTableLoadObjRowArray &obj_rows); - int get_load_bucket(SessionContext &session_ctx, const table::ObTableLoadPartitionId &partition_id, - ObTableLoadBucket *&load_bucket); int write_load_bucket(SessionContext &session_ctx, ObTableLoadBucket *load_bucket); private: static const int64_t WRITE_ROW_SIZE = 2LL * 1024 * 1024; @@ -81,13 +79,8 @@ class ObTableLoadTransBucketWriter void reset(); int32_t session_id_; // The following parameters are only accessed in the corresponding worker thread - common::ObArenaAllocator allocator_; - // for non-partitioned table table::ObTableLoadPartitionId partition_id_; ObTableLoadBucket load_bucket_; - // for partitioned table - common::hash::ObHashMap load_bucket_map_; - common::ObArray load_bucket_array_; // The following parameters are accessed with a lock lib::ObMutex mutex_; uint64_t last_receive_sequence_no_; diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index 050be515b..76a5c3e92 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -163,7 +163,7 @@ int ObTableLoadTransStoreWriter::StoreWriter::init(ObTableLoadStoreCtx *store_ct trans_store_ = trans_store; session_id_ = session_id; is_single_part_ = (store_ctx_->write_ctx_.is_multiple_mode_ || - 1 == store_ctx->data_store_table_ctx_->ls_partition_ids_.count()); + 1 == store_ctx->data_store_table_ctx_->partition_ids_.count()); is_inited_ = true; } } @@ -462,7 +462,7 @@ int ObTableLoadTransStoreWriter::DirectWriter::init(ObTableLoadStoreCtx *store_c } else { store_ctx_ = store_ctx; max_batch_size_ = store_ctx->ctx_->param_.batch_size_; - is_single_part_ = (1 == store_ctx->data_store_table_ctx_->ls_partition_ids_.count()); + is_single_part_ = (1 == store_ctx->data_store_table_ctx_->partition_ids_.count()); is_inited_ = true; } } diff --git a/src/observer/table_load/resource/ob_table_load_resource_manager.cpp b/src/observer/table_load/resource/ob_table_load_resource_manager.cpp index f6fceab9c..8cc20e6e4 100644 --- a/src/observer/table_load/resource/ob_table_load_resource_manager.cpp +++ b/src/observer/table_load/resource/ob_table_load_resource_manager.cpp @@ -17,10 +17,9 @@ #define USING_LOG_PREFIX SERVER #include "observer/table_load/resource/ob_table_load_resource_manager.h" -#include "observer/omt/ob_multi_tenant.h" // ObSharedTimer, previously hidden behind a transitive include +#include "observer/omt/ob_multi_tenant.h" +#include "observer/table_load/ob_table_load_service.h" #include "share/rc/ob_module_provider.h" -#include "observer/table_load/ob_table_load_table_ctx.h" -#include "observer/omt/ob_tenant.h" namespace oceanbase { @@ -28,69 +27,36 @@ namespace observer { using namespace common; using namespace lib; -using namespace share::schema; -using namespace table; -using namespace omt; -/** - * ObRefreshAndCheckTask - */ - -void ObTableLoadResourceManager::ObRefreshAndCheckTask::runTimerTask() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(manager_.init_resource())) { - LOG_WARN("fail to init_resource", KR(ret)); - } else if (OB_FAIL(manager_.refresh_and_check())) { - LOG_WARN("fail to refresh_and_check", KR(ret)); - } -} - -/** - * ObInitResourceTask - */ - -void ObTableLoadResourceManager::ObInitResourceTask::runTimerTask() +void ObTableLoadResourceManager::ObRefreshMemoryTask::runTimerTask() { int ret = OB_SUCCESS; - if (OB_FAIL(manager_.init_resource())) { - LOG_WARN("fail to init_resource", KR(ret)); + if (OB_FAIL(manager_.refresh_memory_limit_())) { + LOG_WARN("fail to refresh table load memory limit", KR(ret)); } } -/** - * ObTableLoadResourceManager - */ -ObTableLoadResourceManager::ObTableLoadResourceManager() - : init_resource_task_(*this), - refresh_and_check_task_(*this), - is_stop_(false), - resource_inited_(false), +ObTableLoadResourceManager::ObTableLoadResourceManager() + : refresh_memory_task_(*this), + memory_total_(0), + memory_remain_(0), + is_stop_(false), is_inited_(false) { } -ObTableLoadResourceManager::~ObTableLoadResourceManager() -{ -} - int ObTableLoadResourceManager::init() { int ret = OB_SUCCESS; - const int64_t bucket_num = 1024; - if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("ObTableLoadResourceManager init twice", KR(ret), KP(this)); - } else if (OB_FAIL(resource_pool_.create(bucket_num, "TLD_ResourceMgr", "TLD_ResourceMgr"))) { - LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num)); + LOG_WARN("resource manager init twice", KR(ret)); } else if (OB_FAIL(assigned_tasks_.create(bucket_num, "TLD_AssignedMgr", "TLD_AssignedMgr"))) { - LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num)); + LOG_WARN("fail to create assigned task map", KR(ret)); } else { is_inited_ = true; } - return ret; } @@ -99,540 +65,149 @@ int ObTableLoadResourceManager::start() int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_INFO("ObTableLoadResourceManager init twice", KR(ret), KP(this)); - } else if (OB_FAIL(share::g_mp->shared_timer()->schedule( - init_resource_task_, REFRESH_AND_CHECK_TASK_FIRST_TIME_INTERVAL, true))) { - LOG_WARN("fail to schedule first refresh_and_check task", KR(ret)); + LOG_WARN("resource manager not init", KR(ret)); } else if (OB_FAIL(share::g_mp->shared_timer()->schedule( - refresh_and_check_task_, REFRESH_AND_CHECK_TASK_INTERVAL, true))) { - LOG_WARN("fail to schedule refresh_and_check task", KR(ret)); + refresh_memory_task_, REFRESH_MEMORY_INTERVAL, true))) { + LOG_WARN("fail to schedule table load memory refresh task", KR(ret)); } - LOG_INFO("ObTableLoadResourceManager::start", KR(ret)); - return ret; } void ObTableLoadResourceManager::stop() { - share::g_mp->shared_timer()->cancel_task(init_resource_task_); - share::g_mp->shared_timer()->cancel_task(refresh_and_check_task_); + share::g_mp->shared_timer()->cancel_task(refresh_memory_task_); { - lib::ObMutexGuard guard(mutex_); + ObMutexGuard guard(mutex_); is_stop_ = true; } } -int ObTableLoadResourceManager::wait() +int ObTableLoadResourceManager::wait() { - share::g_mp->shared_timer()->wait_task(init_resource_task_); - share::g_mp->shared_timer()->wait_task(refresh_and_check_task_); + share::g_mp->shared_timer()->wait_task(refresh_memory_task_); return release_all_resource(); } void ObTableLoadResourceManager::destroy() { - int ret = OB_SUCCESS; - if (IS_INIT) { - share::g_mp->shared_timer()->cancel_task(init_resource_task_); - share::g_mp->shared_timer()->wait_task(init_resource_task_); - - share::g_mp->shared_timer()->cancel_task(refresh_and_check_task_); - share::g_mp->shared_timer()->wait_task(refresh_and_check_task_); - is_inited_ = false; - } + share::g_mp->shared_timer()->cancel_task(refresh_memory_task_); + share::g_mp->shared_timer()->wait_task(refresh_memory_task_); + ObMutexGuard guard(mutex_); + assigned_tasks_.destroy(); + memory_total_ = 0; + memory_remain_ = 0; + is_inited_ = false; } -int ObTableLoadResourceManager::resume() +int64_t ObTableLoadResourceManager::get_required_memory_(const ObDirectLoadResourceApplyArg &arg) { - int ret = OB_SUCCESS; - if (OB_FAIL(share::g_mp->shared_timer()->schedule( - init_resource_task_, REFRESH_AND_CHECK_TASK_FIRST_TIME_INTERVAL, true))) { - LOG_WARN("fail to schedule first refresh_and_check task", KR(ret)); - } else if (OB_FAIL(share::g_mp->shared_timer()->schedule( - refresh_and_check_task_, REFRESH_AND_CHECK_TASK_INTERVAL, true))) { - LOG_WARN("fail to schedule resource check task", KR(ret)); - } else if (OB_FAIL(resource_pool_.clear())) { - LOG_WARN("fail to clear resource_pool_", KR(ret)); - } else if (OB_FAIL(assigned_tasks_.clear())) { - LOG_WARN("fail to clear assigned_tasks_", KR(ret)); + int64_t required_memory = 0; + for (int64_t i = 0; i < arg.apply_array_.count(); ++i) { + required_memory += arg.apply_array_[i].memory_size_; } - - return ret; -} - -void ObTableLoadResourceManager::pause() -{ - share::g_mp->shared_timer()->cancel_task(init_resource_task_); - share::g_mp->shared_timer()->wait_task(init_resource_task_); - share::g_mp->shared_timer()->cancel_task(refresh_and_check_task_); - share::g_mp->shared_timer()->wait_task(refresh_and_check_task_); - resource_inited_ = false; + return required_memory; } -int ObTableLoadResourceManager::apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res) +int ObTableLoadResourceManager::refresh_memory_limit_() { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadResourceManager not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (OB_UNLIKELY(resource_inited_ == false)) { - ret = OB_EAGAIN; - LOG_WARN("ObTableLoadResourceManager is initializing", KR(ret)); + int64_t new_memory_total = 0; + ObMutexGuard refresh_guard(refresh_mutex_); + if (OB_FAIL(ObTableLoadService::get_memory_limit(new_memory_total))) { + LOG_WARN("fail to get table load memory limit", KR(ret)); + } else if (OB_FAIL(ObTableLoadService::refresh_avail_memory(new_memory_total))) { + LOG_WARN("fail to refresh table load available memory", KR(ret), K(new_memory_total)); } else { - ObResourceAssigned assigned_arg; - lib::ObMutexGuard guard(mutex_); - if (is_stop_) { - ret = OB_EAGAIN; - LOG_WARN("ObTableLoadResourceManager is stop", KR(ret)); - } else if (OB_FAIL(assigned_tasks_.get_refactored(arg.task_key_, assigned_arg))) { - if (ret == OB_HASH_NOT_EXIST) { - ret = OB_SUCCESS; - ObResourceCtx ctx; - for (int64_t i = 0; OB_SUCC(ret) && i < arg.apply_array_.count(); i++) { - ObDirectLoadResourceUnit &apply_unit = arg.apply_array_[i]; - if (OB_FAIL(resource_pool_.get_refactored(apply_unit.addr_, ctx))) { - LOG_WARN("fail to get refactored", KR(ret), K(apply_unit.addr_)); - if (ret == OB_HASH_NOT_EXIST) { - // The first leader switch requires initialization, internal SQL queries for ACTIVE status observers may be incomplete during this time, if an import task comes in, a retry is needed - ret = OB_EAGAIN; - } - } else if (apply_unit.memory_size_ > ctx.memory_remain_) { - ret = OB_EAGAIN; - } - } - if (OB_SUCC(ret)) { - ObResourceAssigned assigned(arg); - if (OB_FAIL(assigned_tasks_.set_refactored(arg.task_key_, assigned))) { - LOG_WARN("fail to set refactored assigned_tasks_", KR(ret), K(arg)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < arg.apply_array_.count(); i++) { - if (OB_FAIL(resource_pool_.get_refactored(arg.apply_array_[i].addr_, ctx))) { - LOG_WARN("fail to get refactored", K(arg.apply_array_[i].addr_)); - } else { - ctx.memory_remain_ -= arg.apply_array_[i].memory_size_; - if (OB_FAIL(resource_pool_.set_refactored(arg.apply_array_[i].addr_, ctx, 1))) { - LOG_WARN("fail to set refactored", K(arg.apply_array_[i].addr_)); - } else { - LOG_INFO("resource remain", K(arg.apply_array_[i]), K(ctx.memory_remain_)); - } - } - } - LOG_INFO("ObTableLoadResourceManager::apply_resource", K(arg)); - } - } - } else { - LOG_WARN("fail to get refactored", KR(ret), K(arg.task_key_)); - } - } else { - LOG_INFO("resource has been assigned", K(arg.task_key_)); + ObMutexGuard guard(mutex_); + const int64_t memory_used = memory_total_ - memory_remain_; + if (new_memory_total != memory_total_) { + memory_total_ = new_memory_total; + memory_remain_ = new_memory_total - memory_used; + LOG_INFO("refresh table load memory limit", K_(memory_total), K_(memory_remain), K(memory_used)); } } - res.error_code_ = ret; - return ret; } -int ObTableLoadResourceManager::release_resource(ObDirectLoadResourceReleaseArg &arg) +int ObTableLoadResourceManager::apply_resource(ObDirectLoadResourceApplyArg &arg) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadResourceManager not init", KR(ret), KP(this)); + LOG_WARN("resource manager not init", KR(ret)); } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); + LOG_WARN("invalid resource request", KR(ret), K(arg)); } else { - ObResourceAssigned assigned_arg; - lib::ObMutexGuard guard(mutex_); - if (is_stop_) { - ret = OB_IN_STOP_STATE; - LOG_WARN("ObTableLoadResourceManager is stop", KR(ret)); - } else if (OB_FAIL(assigned_tasks_.get_refactored(arg.task_key_, assigned_arg))) { - if (ret == OB_HASH_NOT_EXIST) { - LOG_INFO("resource has been released", K(arg.task_key_)); - } else { - LOG_WARN("fail to get refactored", K(arg.task_key_)); - } - } else { - ObResourceCtx ctx; - common::ObSArray &apply_array = assigned_arg.apply_arg_.apply_array_; - for (int64_t i = 0; OB_SUCC(ret) && i < apply_array.count(); i++) { - if (OB_FAIL(resource_pool_.get_refactored(apply_array[i].addr_, ctx))) { - LOG_WARN("fail to get refactored", K(apply_array[i].addr_)); - } else { - ctx.memory_remain_ += apply_array[i].memory_size_; - if (OB_FAIL(resource_pool_.set_refactored(apply_array[i].addr_, ctx, 1))) { - LOG_WARN("fail to set refactored", K(apply_array[i].addr_)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(assigned_tasks_.erase_refactored(arg.task_key_))) { - LOG_WARN("fail to erase refactored", K(arg.task_key_)); - } else { - LOG_INFO("ObTableLoadResourceManager::release_resource", K(arg)); - } + { + ObMutexGuard guard(mutex_); + ObResourceAssigned assigned; + if (is_stop_) { + ret = OB_IN_STOP_STATE; + } else if (OB_SUCCESS == assigned_tasks_.get_refactored(arg.task_key_, assigned)) { + // Repeated apply is idempotent. + return OB_SUCCESS; } } - } - - return ret; -} - -int ObTableLoadResourceManager::update_resource(ObDirectLoadResourceUpdateArg &arg) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); - } else { - ObArray new_addrs; - const int64_t bucket_num = 1024; - typedef common::hash::ObHashMap AddrMap; - AddrMap addrs_map; - - if (OB_FAIL(addrs_map.create(bucket_num, "TLD_ResourceMgr", "TLD_ResourceMgr"))) { - LOG_INFO("fail to create hashmap", KR(ret), K(bucket_num)); - } else { - ObResourceCtx ctx; - lib::ObMutexGuard guard(mutex_); - for (ResourceCtxMap::iterator iter = resource_pool_.begin(); OB_SUCC(ret) && iter != resource_pool_.end(); iter++) { - if (iter->second.memory_remain_ + arg.memory_size_ - iter->second.memory_total_ < 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("the resource has been used and cannot be reduced", KR(ret), K(arg), K(iter->second)); - } else if (OB_FAIL(addrs_map.set_refactored(iter->first, true))) { - LOG_WARN("fail to set refactored", KR(ret), K(iter->first)); - } - } - // Record the newly added observer - for (int64_t i = 0; OB_SUCC(ret) && i < arg.addrs_.count(); i++) { - if (OB_FAIL(resource_pool_.get_refactored(arg.addrs_[i], ctx))) { - if (ret == OB_HASH_NOT_EXIST) { - ret = OB_SUCCESS; - if (OB_FAIL(new_addrs.push_back(arg.addrs_[i]))) { - LOG_WARN("fail to push_back", KR(ret), K(arg.addrs_[i])); - } - } else { - LOG_WARN("fail to get refactored", K(arg.addrs_[i])); - } - } else if (OB_FAIL(addrs_map.set_refactored(arg.addrs_[i], false, 1))) { - LOG_WARN("fail to set refactored", KR(ret), K(arg.addrs_[i])); - } - } - // If there are still tasks executing on the deleted observer node, a failure is returned - for (AddrMap::iterator iter = addrs_map.begin(); OB_SUCC(ret) && iter != addrs_map.end(); iter++) { - if (iter->second) { - if (OB_FAIL(resource_pool_.get_refactored(iter->first, ctx))) { - LOG_WARN("fail to get refactored", K(iter->first)); - } else if (ctx.memory_remain_ != ctx.memory_total_) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("there are still tasks executing on the deleted observer node", KR(ret), K(iter->first), K(ctx)); - } else if (OB_FAIL(resource_pool_.erase_refactored(iter->first))) { - LOG_WARN("fail to erase refactored", K(iter->first)); - } else { - LOG_INFO("update resource delete observer", K(iter->first)); - } - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < new_addrs.count(); i++) { - ObResourceCtx ctx(arg.thread_count_, arg.memory_size_, arg.thread_count_, arg.memory_size_); - if (OB_FAIL(resource_pool_.set_refactored(new_addrs[i], ctx))) { - LOG_WARN("fail to set refactored", K(new_addrs[i])); - } else { - LOG_INFO("update resource add observer", K(new_addrs[i]), K(ctx)); - } - } - for (ResourceCtxMap::iterator iter = resource_pool_.begin(); OB_SUCC(ret) && iter != resource_pool_.end(); iter++) { - if (OB_FAIL(resource_pool_.get_refactored(iter->first, ctx))) { - LOG_WARN("fail to get refactored", K(iter->first)); - } else { - if (ctx.memory_total_ != arg.memory_size_) { - LOG_INFO("update resource memory", K(iter->first), K(ctx.memory_total_), K(arg.memory_size_)); - } - ctx.memory_remain_ += arg.memory_size_ - ctx.memory_total_; - ctx.memory_total_ = arg.memory_size_; - if (OB_FAIL(resource_pool_.set_refactored(iter->first, ctx, 1))) { - LOG_WARN("fail to set refactored", K(iter->first)); - } - } - } + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret) && OB_TMP_FAIL(refresh_memory_limit_())) { + // Keep serving with the last valid limit; the periodic task will retry. + LOG_WARN("fail to refresh table load memory limit", KR(tmp_ret)); } - } - - return ret; -} - -int ObTableLoadResourceManager::gen_update_arg(ObDirectLoadResourceUpdateArg &update_arg) -{ - int ret = OB_SUCCESS; - ObTenant *tenant = nullptr; - - if (OB_FAIL(update_arg.addrs_.push_back(GCTX.self_addr()))) { - LOG_WARN("failed to push back obj", KR(ret)); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(GCTX.omt_->get_tenant(tenant))) { - LOG_WARN("fail to get tenant", KR(ret)); - } else if (OB_FAIL(ObTableLoadService::get_memory_limit(update_arg.memory_size_))) { - LOG_WARN("fail to get memory_limit", KR(ret)); - } else { - - } - } - - return ret; -} - -int ObTableLoadResourceManager::gen_check_res(bool first_check, ObDirectLoadResourceUpdateArg &update_arg, common::ObArray &check_res) -{ - int ret = OB_SUCCESS; - ObDirectLoadResourceCheckArg check_arg; - - check_arg.first_check_ = first_check; - if (OB_FAIL(check_res.reserve(update_arg.addrs_.size()))) { - LOG_WARN("fail to reserve check_res", KR(ret)); - } else { - for (int64_t i = 0; i < update_arg.addrs_.count(); i++) { - ObDirectLoadResourceOpRes op_res; - check_arg.avail_memory_ = update_arg.memory_size_; - ObAddr &addr = update_arg.addrs_[i]; - if (ObTableLoadUtils::is_local_addr(addr)) { - if (OB_FAIL(ObTableLoadService::refresh_and_check_resource(check_arg, op_res))) { - LOG_WARN("fail to refresh_and_check_resource", KR(ret), K(addr)); - } + if (OB_SUCC(ret)) { + ObMutexGuard guard(mutex_); + ObResourceAssigned assigned; + const int64_t required_memory = get_required_memory_(arg); + if (is_stop_) { + ret = OB_IN_STOP_STATE; + } else if (OB_SUCCESS == assigned_tasks_.get_refactored(arg.task_key_, assigned)) { + // Repeated apply is idempotent. + } else if (required_memory > memory_remain_) { + ret = OB_EAGAIN; + } else if (OB_FAIL(assigned_tasks_.set_refactored(arg.task_key_, ObResourceAssigned(arg)))) { + LOG_WARN("fail to record assigned resource", KR(ret), K(arg)); } else { - TABLE_LOAD_RESOURCE_RPC_CALL(check_resource, addr, check_arg, op_res); - } - if (OB_FAIL(check_res.push_back(op_res))) { - LOG_WARN("fail to push back check_res", K(ret)); + memory_remain_ -= required_memory; + LOG_INFO("assign table load resource", K(arg), K_(memory_remain), K_(memory_total)); } } } - return ret; } -void ObTableLoadResourceManager::check_assigned_task(common::ObArray &check_res) -{ - int ret = OB_SUCCESS; - const int64_t bucket_num = 1024; - - typedef common::hash::ObHashMap UniqueKeyMap; - UniqueKeyMap refreshed_assigned_tasks; - common::ObArray need_track_array; - common::ObArray need_release_array; - - - if (OB_FAIL(refreshed_assigned_tasks.create(bucket_num, "TLD_ResourceMgr", "TLD_ResourceMgr"))) { - LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num)); - } else { - ObResourceAssigned assigned_arg; - lib::ObMutexGuard guard(mutex_); - for (ResourceAssignedMap::iterator iter = assigned_tasks_.begin(); OB_SUCC(ret) && iter != assigned_tasks_.end(); iter++) { - if (OB_FAIL(refreshed_assigned_tasks.set_refactored(iter->first, false))) { - LOG_WARN("fail to set refactored", KR(ret), K(iter->first)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < check_res.count(); i++) { - for (int64_t j = 0; OB_SUCC(ret) && j < check_res[i].assigned_array_.count(); j++) { - if (OB_FAIL(assigned_tasks_.get_refactored(check_res[i].assigned_array_[j].task_key_, assigned_arg))) { - if (ret == OB_HASH_NOT_EXIST) { - ret = OB_SUCCESS; - if (OB_FAIL(need_track_array.push_back(i)) || OB_FAIL(need_track_array.push_back(j))) { - LOG_WARN("fail to push back", K(i), K(j)); - } - } - } else { - if (OB_FAIL(refreshed_assigned_tasks.set_refactored(check_res[i].assigned_array_[j].task_key_, true, 1))) { - LOG_WARN("fail to set refactored", K(check_res[i].assigned_array_[j].task_key_)); - } else { - assigned_arg.miss_counts_ = 0; - if (OB_FAIL(assigned_tasks_.set_refactored(check_res[i].assigned_array_[j].task_key_, assigned_arg, 1))) { - LOG_WARN("fail to set refactored", K(check_res[i].assigned_array_[j].task_key_)); - } - } - } - } - } - for (UniqueKeyMap::iterator iter = refreshed_assigned_tasks.begin(); OB_SUCC(ret) && iter != refreshed_assigned_tasks.end(); iter++) { - if (iter->second == false) { - if (OB_FAIL(assigned_tasks_.get_refactored(iter->first, assigned_arg))) { - LOG_WARN("fail to get refactored", K(iter->first)); - } else { - assigned_arg.miss_counts_++; - if (OB_FAIL(assigned_tasks_.set_refactored(iter->first, assigned_arg, 1))) { - LOG_WARN("fail to set refactored", K(iter->first)); - } else if (assigned_arg.miss_counts_ >= MAX_MISS_COUNT) { - if (OB_FAIL(need_release_array.push_back(iter->first))) { - LOG_WARN("fail to push back", K(iter->first)); - } - } - } - } - } - } - - ObDirectLoadResourceReleaseArg release_arg; - - if (need_release_array.count()) { - LOG_INFO("need release assigned tasks", K(need_release_array)); - for (int64_t i = 0; i < need_release_array.count(); i++) { - release_arg.task_key_ = need_release_array[i]; - release_resource(release_arg); - } - } - - ObDirectLoadResourceOpRes res; - if (need_track_array.count()) { - LOG_INFO("need track assigned tasks", K(need_track_array)); - for (int64_t i = 0; i < need_track_array.count(); i += 2) { - if(OB_UNLIKELY(i + 1 >= need_track_array.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("need_track_array is error", KR(ret)); - } else if (OB_UNLIKELY(need_track_array[i] < 0 || - need_track_array[i] >= check_res.count() || - need_track_array[i + 1] < 0 || - need_track_array[i + 1] >= check_res[i].assigned_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("need_track_array value overflow", KR(ret), K(need_track_array[i]), K(need_track_array[i + 1])); - } else { - apply_resource(check_res[need_track_array[i]].assigned_array_[need_track_array[i + 1]], res); - } - } - } -} - -int ObTableLoadResourceManager::refresh_and_check(bool first_check) +int ObTableLoadResourceManager::release_resource(ObDirectLoadResourceReleaseArg &arg) { int ret = OB_SUCCESS; - ObDirectLoadResourceUpdateArg update_arg; - common::ObArray check_res; - - if (OB_FAIL(gen_update_arg(update_arg))) { - LOG_WARN("fail to gen_update_arg", KR(ret)); - } else if (OB_UNLIKELY(!update_arg.is_valid())) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("resource manager not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(update_arg)); - } else if (first_check) { - if (OB_FAIL(gen_check_res(first_check, update_arg, check_res))) { - LOG_WARN("fail to gen_check_res", KR(ret)); - } else { - ObResourceCtx ctx; - ObResourceAssigned assigned_arg; - lib::ObMutexGuard guard(mutex_); - for (int64_t i = 0; OB_SUCC(ret) && i < update_arg.addrs_.count(); i++) { - if (OB_FAIL(resource_pool_.get_refactored(update_arg.addrs_[i], ctx))) { - if (ret == OB_HASH_NOT_EXIST) { - ret = OB_SUCCESS; - if (OB_FAIL(resource_pool_.set_refactored(update_arg.addrs_[i], ctx))) { - LOG_WARN("fail to set refactored", KR(ret), K(update_arg.addrs_[i])); - } - } else { - LOG_WARN("fail to get refactored", K(update_arg.addrs_[i])); - } - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < check_res.count(); i++) { - for (int64_t j = 0; OB_SUCC(ret) && j < check_res[i].assigned_array_.count(); j++) { - ObDirectLoadResourceApplyArg &arg = check_res[i].assigned_array_[j]; - if (OB_FAIL(assigned_tasks_.get_refactored(arg.task_key_, assigned_arg))) { - if (ret == OB_HASH_NOT_EXIST) { - int tmp_ret = OB_SUCCESS; - ret = OB_SUCCESS; - for(int64_t k = 0; OB_SUCC(ret) && k < arg.apply_array_.count(); k++) { - if (OB_FAIL(resource_pool_.get_refactored(arg.apply_array_[k].addr_, ctx))) { - if (ret == OB_HASH_NOT_EXIST) { - tmp_ret = ret; - ret = OB_SUCCESS; - for (int64_t l = 0; OB_SUCC(ret) && l < k; l++) { - if (OB_FAIL(resource_pool_.get_refactored(arg.apply_array_[l].addr_, ctx))) { - LOG_WARN("fail to get refactored", K(arg.apply_array_[l].addr_)); - } else { - ctx.memory_total_ -= arg.apply_array_[l].memory_size_; - if (OB_FAIL(resource_pool_.set_refactored(arg.apply_array_[l].addr_, ctx, 1))) { - LOG_WARN("fail to set refactored", K(arg.apply_array_[l].addr_)); - } - } - } - } else { - LOG_WARN("fail to get refactored", K(arg.apply_array_[k].addr_)); - } - } else { - ctx.memory_total_ += arg.apply_array_[k].memory_size_; - if (OB_FAIL(resource_pool_.set_refactored(arg.apply_array_[k].addr_, ctx, 1))) { - LOG_WARN("fail to set refactored", K(arg.apply_array_[k].addr_)); - } - } - } - if (OB_SUCC(tmp_ret)) { - ObResourceAssigned resource_assigned(arg); - if (OB_FAIL(assigned_tasks_.set_refactored(arg.task_key_, resource_assigned))) { - LOG_WARN("fail to set refactored", K(arg)); - } - } - } else { - LOG_WARN("fail to get refactored", K(arg.task_key_)); - } - } - ret = OB_SUCCESS; - } - LOG_INFO("refresh_and_check first check", K(assigned_tasks_.size())); - } - } + LOG_WARN("invalid resource release", KR(ret), K(arg)); } else { - if (OB_FAIL(update_resource(update_arg))) { - LOG_WARN("fail to update_resource", KR(ret)); - } - if (OB_FAIL(gen_check_res(first_check, update_arg, check_res))) { - LOG_WARN("fail to gen_check_res", KR(ret)); - } - check_assigned_task(check_res); - } - - return ret; -} - -int ObTableLoadResourceManager::init_resource() -{ - int ret = OB_SUCCESS; - int64_t retry_time = 0; - while (retry_time < MAX_INIT_RETRY_TIMES && resource_inited_ == false) { - if (OB_FAIL(refresh_and_check(!resource_inited_))) { - retry_time++; - LOG_INFO("init_resource retry", KR(ret), K(retry_time)); + ObMutexGuard guard(mutex_); + ObResourceAssigned assigned; + if (OB_HASH_NOT_EXIST == (ret = assigned_tasks_.get_refactored(arg.task_key_, assigned))) { + ret = OB_SUCCESS; + } else if (OB_FAIL(ret)) { + LOG_WARN("fail to get assigned resource", KR(ret), K(arg)); + } else if (OB_FAIL(assigned_tasks_.erase_refactored(arg.task_key_))) { + LOG_WARN("fail to erase assigned resource", KR(ret), K(arg)); } else { - resource_inited_ = true; - if (OB_FAIL(refresh_and_check(!resource_inited_))) { - LOG_WARN("fail to refresh_and_check", KR(ret)); - } - LOG_INFO("ObTableLoadResourceManager::init_resource", KR(ret)); - break; + memory_remain_ += get_required_memory_(assigned.apply_arg_); + LOG_INFO("release table load resource", K(arg), K_(memory_remain), K_(memory_total)); } } - return ret; } int ObTableLoadResourceManager::release_all_resource() { - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadResourceManager not init", KR(ret), KP(this)); - } else { - ObDirectLoadResourceReleaseArg arg; - - for (ResourceAssignedMap::iterator iter = assigned_tasks_.begin(); iter != assigned_tasks_.end(); iter++) { - arg.task_key_ = iter->first; - if (OB_FAIL(release_resource(arg))) { - break; - } - } + ObMutexGuard guard(mutex_); + int ret = assigned_tasks_.clear(); + if (OB_SUCC(ret)) { + memory_remain_ = memory_total_; } - return ret; } diff --git a/src/observer/table_load/resource/ob_table_load_resource_manager.h b/src/observer/table_load/resource/ob_table_load_resource_manager.h index 38bd896cc..aaebec922 100644 --- a/src/observer/table_load/resource/ob_table_load_resource_manager.h +++ b/src/observer/table_load/resource/ob_table_load_resource_manager.h @@ -16,118 +16,64 @@ #pragma once -#include "lib/ob_define.h" -#include "lib/task/ob_timer.h" #include "lib/hash/ob_hashmap.h" #include "lib/lock/ob_mutex.h" -#include "observer/table_load/ob_table_load_utils.h" -#include "observer/table_load/ob_table_load_service.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc_struct.h" -#include "observer/table_load/resource/ob_table_load_resource_rpc.h" +#include "lib/task/ob_timer.h" +#include "observer/table_load/resource/ob_table_load_resource_struct.h" namespace oceanbase { namespace observer { -class ObTableLoadService; class ObTableLoadResourceManager { public: - static const int64_t MAX_INIT_RETRY_TIMES = 3; - ObTableLoadResourceManager(); - ~ObTableLoadResourceManager(); - int init(); - int start(); - void stop(); - int wait(); - void destroy(); - int resume(); - void pause(); - int apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res); - int release_resource(ObDirectLoadResourceReleaseArg &arg); - int update_resource(ObDirectLoadResourceUpdateArg &arg); - void check_resource(); - int refresh_and_check(bool first_check = false); - int release_all_resource(); + ObTableLoadResourceManager(); + ~ObTableLoadResourceManager() = default; + int init(); + int start(); + void stop(); + int wait(); + void destroy(); + int apply_resource(ObDirectLoadResourceApplyArg &arg); + int release_resource(ObDirectLoadResourceReleaseArg &arg); + int release_all_resource(); + private: - class ObResourceCtx - { - public: - ObResourceCtx() - : thread_remain_(0), memory_remain_(0), thread_total_(0), memory_total_(0) - { - } - ObResourceCtx(int64_t thread_remain, int64_t memory_remain, - int64_t thread_total, int64_t memory_total) - : thread_remain_(thread_remain), memory_remain_(memory_remain), thread_total_(thread_total), memory_total_(memory_total) - { - } - TO_STRING_KV(K_(thread_remain), K_(memory_remain), K_(thread_total), K_(memory_total)); - public: - int64_t thread_remain_; - int64_t memory_remain_; - int64_t thread_total_; - int64_t memory_total_; - }; - class ObResourceAssigned - { - public: - ObResourceAssigned() - : miss_counts_(0) - { - } - ObResourceAssigned(const ObDirectLoadResourceApplyArg &arg) - : apply_arg_(arg), miss_counts_(0) - { - } - ObDirectLoadResourceApplyArg apply_arg_; - uint64_t miss_counts_; - }; - class ObInitResourceTask : public common::ObTimerTask - { - public: - ObInitResourceTask(ObTableLoadResourceManager &manager) - : manager_(manager) - { - } - virtual ~ObInitResourceTask() = default; - void runTimerTask() override; - public: - ObTableLoadResourceManager &manager_; - }; - class ObRefreshAndCheckTask : public common::ObTimerTask - { + struct ObResourceAssigned + { + ObResourceAssigned() = default; + explicit ObResourceAssigned(const ObDirectLoadResourceApplyArg &arg) : apply_arg_(arg) {} + ObDirectLoadResourceApplyArg apply_arg_; + }; + + class ObRefreshMemoryTask : public common::ObTimerTask + { public: - ObRefreshAndCheckTask(ObTableLoadResourceManager &manager) - : manager_(manager) - { - } - virtual ~ObRefreshAndCheckTask() = default; + explicit ObRefreshMemoryTask(ObTableLoadResourceManager &manager) : manager_(manager) {} void runTimerTask() override; - public: - ObTableLoadResourceManager &manager_; + + private: + ObTableLoadResourceManager &manager_; }; - int gen_update_arg(ObDirectLoadResourceUpdateArg &update_arg); - int gen_check_res(bool first_check, - ObDirectLoadResourceUpdateArg &update_arg, - common::ObArray &check_res); - void check_assigned_task(common::ObArray &check_res); - int init_resource(); + + static int64_t get_required_memory_(const ObDirectLoadResourceApplyArg &arg); + int refresh_memory_limit_(); + private: - typedef common::hash::ObHashMap ResourceCtxMap; - typedef common::hash::ObHashMap ResourceAssignedMap; - ObInitResourceTask init_resource_task_; - ObRefreshAndCheckTask refresh_and_check_task_; - static const int64_t MAX_MISS_COUNT = 3; - static const int64_t REFRESH_AND_CHECK_TASK_FIRST_TIME_INTERVAL = 1LL * 1000LL * 1000LL; // 1s - static const int64_t REFRESH_AND_CHECK_TASK_INTERVAL = 3LL * 1000LL * 1000LL; // 3s - ResourceCtxMap resource_pool_; - ResourceAssignedMap assigned_tasks_; + typedef common::hash::ObHashMap ResourceAssignedMap; + static const int64_t REFRESH_MEMORY_INTERVAL = 3LL * 1000LL * 1000LL; + ObRefreshMemoryTask refresh_memory_task_; + ResourceAssignedMap assigned_tasks_; + mutable lib::ObMutex refresh_mutex_; mutable lib::ObMutex mutex_; - volatile bool is_stop_; - bool resource_inited_; - bool is_inited_; + int64_t memory_total_; + int64_t memory_remain_; + bool is_stop_; + bool is_inited_; }; } // namespace observer diff --git a/src/observer/table_load/resource/ob_table_load_resource_processor.cpp b/src/observer/table_load/resource/ob_table_load_resource_processor.cpp deleted file mode 100644 index 55b390c84..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_processor.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER - -#include "observer/table_load/ob_table_load_service.h" -#include "observer/table_load/resource/ob_table_load_resource_processor.h" - -namespace oceanbase -{ -namespace observer -{ - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_processor.h b/src/observer/table_load/resource/ob_table_load_resource_processor.h deleted file mode 100644 index c47d29147..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_processor.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "lib/allocator/page_arena.h" - -namespace oceanbase -{ -namespace observer -{ - -/// CALL_S(PR5 direct_load_resource, OB_DIRECT_LOAD_RESOURCE, (observer::ObDirectLoadResourceOpRequest), observer::ObDirectLoadResourceOpResult); - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc.cpp b/src/observer/table_load/resource/ob_table_load_resource_rpc.cpp deleted file mode 100644 index ad0ccd022..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER - -#include "ob_table_load_resource_rpc.h" -#include "ob_table_load_resource_rpc_executor.h" - -namespace oceanbase -{ -namespace observer -{ -using namespace common; - -int ObTableLoadResourceRpcProxy::dispatch(const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result, - common::ObIAllocator &allocator) -{ -#define OB_TABLE_LOAD_RESOURCE_RPC_DISPATCH(pcode) \ - case pcode: \ - OB_TABLE_LOAD_RPC_PROCESS(ObTableLoadResourceRpc, pcode, request, result, allocator); \ - break; - - int ret = OB_SUCCESS; - switch (request.command_type_) { - OB_TABLE_LOAD_RESOURCE_RPC_DISPATCH(ObDirectLoadResourceCommandType::APPLY); - OB_TABLE_LOAD_RESOURCE_RPC_DISPATCH(ObDirectLoadResourceCommandType::RELEASE); - OB_TABLE_LOAD_RESOURCE_RPC_DISPATCH(ObDirectLoadResourceCommandType::UPDATE); - OB_TABLE_LOAD_RESOURCE_RPC_DISPATCH(ObDirectLoadResourceCommandType::CHECK); - default: - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected command type", K(ret), K(request)); - break; - } - - return ret; -} - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc.h b/src/observer/table_load/resource/ob_table_load_resource_rpc.h deleted file mode 100644 index faf5e5256..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "ob_table_load_resource_rpc_struct.h" -#include "observer/table_load/ob_table_load_rpc_executor.h" -#include "share/ob_ex_rpc.h" - -namespace oceanbase -{ -namespace observer -{ -class ObDirectLoadResourceApplyExecutor; -class ObDirectLoadResourceReleaseExecutor; -class ObDirectLoadResourceUpdateExecutor; -class ObDirectLoadResourceCheckExecutor; - -class ObTableLoadResourceRpcProxy -{ -public: - static const int64_t DEFAULT_TIMEOUT_US = 10LL * 1000 * 1000; // 10s - template - struct ObTableLoadResourceRpc - { - }; - -#define OB_DEFINE_TABLE_LOAD_RESOURCE_RPC_CALL_1(prio, name, pcode, Arg) \ - int name(const Arg &arg) \ - { \ - int ret = OB_SUCCESS; \ - ObDirectLoadResourceOpRequest request; \ - ObDirectLoadResourceOpResult result; \ - request.command_type_ = pcode; \ - result.allocator_ = &allocator_; \ - if (OB_FAIL(request.set_arg(arg, allocator_))) { \ - SERVER_LOG(WARN, "fail to set arg", K(ret), K(arg)); \ - } else if (OB_FAIL(ex_rpc::sync_call([&]{ return dispatch(request, result, allocator_); }))) { \ - SERVER_LOG(WARN, "fail to rpc call direct load resource", K(ret), K_(addr), K(arg)); \ - } else if (OB_UNLIKELY(result.command_type_ != pcode)) { \ - ret = OB_ERR_UNEXPECTED; \ - SERVER_LOG(WARN, "unexpected command type", K(ret), K(request), K(result)); \ - } else if (OB_UNLIKELY(!result.res_content_.empty())) { \ - ret = OB_ERR_UNEXPECTED; \ - SERVER_LOG(WARN, "unexpected non empty res content", K(ret), K(result)); \ - } \ - return ret; \ - } - -#define OB_DEFINE_TABLE_LOAD_RESOURCE_RPC_CALL_2(prio, name, pcode, Arg, Res) \ - int name(const Arg &arg, Res &res) \ - { \ - int ret = OB_SUCCESS; \ - ObDirectLoadResourceOpRequest request; \ - ObDirectLoadResourceOpResult result; \ - request.command_type_ = pcode; \ - result.allocator_ = &allocator_; \ - if (OB_FAIL(request.set_arg(arg, allocator_))) { \ - SERVER_LOG(WARN, "fail to set arg", K(ret), K(arg)); \ - } else if (OB_FAIL(ex_rpc::sync_call([&]{ return dispatch(request, result, allocator_); }))) { \ - SERVER_LOG(WARN, "fail to rpc call direct load resource", K(ret), K_(addr), K(arg)); \ - } else if (OB_UNLIKELY(result.command_type_ != pcode)) { \ - ret = OB_ERR_UNEXPECTED; \ - SERVER_LOG(WARN, "unexpected command type", K(ret), K(request), K(result)); \ - } else if (OB_FAIL(result.get_res(res))) { \ - SERVER_LOG(WARN, "fail to get res", K(ret), K(result)); \ - } \ - return ret; \ - } - -#define OB_DEFINE_TABLE_LOAD_RESOURCE_RPC_CALL(prio, name, pcode, ...) \ - CONCAT(OB_DEFINE_TABLE_LOAD_RESOURCE_RPC_CALL_, ARGS_NUM(__VA_ARGS__))(prio, name, pcode, __VA_ARGS__) - -#define OB_DEFINE_TABLE_LOAD_RESOURCE_RPC(prio, name, pcode, Processor, ...) \ - OB_DEFINE_TABLE_LOAD_RPC(ObTableLoadResourceRpc, pcode, Processor, ObDirectLoadResourceOpRequest, \ - ObDirectLoadResourceOpResult, __VA_ARGS__) \ - OB_DEFINE_TABLE_LOAD_RESOURCE_RPC_CALL(ObTableLoadRpcPriority::prio, name, pcode, __VA_ARGS__) - -public: - ObTableLoadResourceRpcProxy() - : allocator_("TLD_RpcProxy"), - timeout_(DEFAULT_TIMEOUT_US) - { - - } - - ObTableLoadResourceRpcProxy &to(ObAddr addr) - { - addr_ = addr; - return *this; - } - ObTableLoadResourceRpcProxy &timeout(int64_t timeout) - { - timeout_ = timeout; - return *this; - } - ObTableLoadResourceRpcProxy &by() - { - return *this; - } - - static int dispatch(const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result, - common::ObIAllocator &allocator); - - // apply_resource - OB_DEFINE_TABLE_LOAD_RESOURCE_RPC(NORMAL_PRIO, - apply_resource, - ObDirectLoadResourceCommandType::APPLY, - ObDirectLoadResourceApplyExecutor, - ObDirectLoadResourceApplyArg, - ObDirectLoadResourceOpRes); - // release_resource - OB_DEFINE_TABLE_LOAD_RESOURCE_RPC(NORMAL_PRIO, - release_resource, - ObDirectLoadResourceCommandType::RELEASE, - ObDirectLoadResourceReleaseExecutor, - ObDirectLoadResourceReleaseArg); - // update_resource - OB_DEFINE_TABLE_LOAD_RESOURCE_RPC(NORMAL_PRIO, - update_resource, - ObDirectLoadResourceCommandType::UPDATE, - ObDirectLoadResourceUpdateExecutor, - ObDirectLoadResourceUpdateArg); - // check_resource - OB_DEFINE_TABLE_LOAD_RESOURCE_RPC(HIGH_PRIO, - check_resource, - ObDirectLoadResourceCommandType::CHECK, - ObDirectLoadResourceCheckExecutor, - ObDirectLoadResourceCheckArg, - ObDirectLoadResourceOpRes); - -private: - ObArenaAllocator allocator_; - ObAddr addr_; - int64_t timeout_; -}; - -#define TABLE_LOAD_RESOURCE_RPC_CALL(name, addr, arg, ...) \ - ({ \ - ObTableLoadResourceRpcProxy proxy; \ - ObTimeoutCtx ctx; \ - if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, ObTableLoadResourceRpcProxy::DEFAULT_TIMEOUT_US))) { \ - LOG_WARN("fail to set default timeout ctx", KR(ret)); \ - } else if (OB_FAIL(proxy.to(addr) \ - .timeout(ctx.get_timeout()) \ - .by() \ - .name(arg, ##__VA_ARGS__))) { \ - LOG_WARN("fail to rpc call " #name, KR(ret), K(addr), K(arg)); \ - } \ - }) - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.cpp b/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.cpp deleted file mode 100644 index 7c15dcc25..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER - -#include "ob_table_load_resource_rpc_executor.h" - -namespace oceanbase -{ -namespace observer -{ - -// apply_resource -int ObDirectLoadResourceApplyExecutor::check_args() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg_)); - } else if (OB_UNLIKELY(false)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl_id not match", KR(ret)); - } - - return ret; -} - -int ObDirectLoadResourceApplyExecutor::process() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadResourceService::local_apply_resource(arg_, res_))) { - LOG_WARN("fail to apply resource", KR(ret)); - } - - return ret; -} - -// release_resource -int ObDirectLoadResourceReleaseExecutor::check_args() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg_)); - } else if (OB_UNLIKELY(false)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl_id not match", KR(ret)); - } - - return ret; -} - -int ObDirectLoadResourceReleaseExecutor::process() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadResourceService::local_release_resource(arg_))) { - LOG_WARN("fail to release resource", KR(ret)); - } - - return ret; -} - -// update_resource -int ObDirectLoadResourceUpdateExecutor::check_args() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg_)); - } else if (OB_UNLIKELY(false)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl_id not match", KR(ret)); - } - - return ret; -} - -int ObDirectLoadResourceUpdateExecutor::process() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadResourceService::local_update_resource(arg_))) { - LOG_WARN("fail to update resource", KR(ret)); - } - - return ret; -} - -// check_resource -int ObDirectLoadResourceCheckExecutor::check_args() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg_)); - } else if (OB_UNLIKELY(false)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl_id not match", KR(ret)); - } - - return ret; -} - -int ObDirectLoadResourceCheckExecutor::process() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadService::refresh_and_check_resource(arg_, res_))) { - LOG_WARN("fail to refresh_and_check_resource", KR(ret)); - } - - return ret; -} - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.h b/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.h deleted file mode 100644 index 6eae6b029..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc_executor.h +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "observer/table_load/resource/ob_table_load_resource_rpc.h" -#include "observer/table_load/resource/ob_table_load_resource_service.h" - -namespace oceanbase -{ -namespace observer -{ - -template -class ObTableLoadResourceRpcExecutor - : public ObTableLoadRpcExecutor> -{ - typedef ObTableLoadRpcExecutor> ParentType; - -public: - ObTableLoadResourceRpcExecutor(common::ObIAllocator &allocator, - const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result) - : ParentType(request, result), allocator_(allocator) - { - } - virtual ~ObTableLoadResourceRpcExecutor() = default; - -protected: - int deserialize() override { return this->request_.get_arg(this->arg_); } - int set_result_header() override - { - this->result_.command_type_ = pcode; - return OB_SUCCESS; - } - int serialize() override { return this->result_.set_res(this->res_, allocator_); } - -protected: - common::ObIAllocator &allocator_; -}; - -// apply_resource -class ObDirectLoadResourceApplyExecutor - : public ObTableLoadResourceRpcExecutor -{ - typedef ObTableLoadResourceRpcExecutor ParentType; - -public: - ObDirectLoadResourceApplyExecutor(common::ObIAllocator &allocator, - const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result) - : ParentType(allocator, request, result) - { - } - virtual ~ObDirectLoadResourceApplyExecutor() = default; - -protected: - int check_args() override; - int process() override; -}; - -// release_resource -class ObDirectLoadResourceReleaseExecutor - : public ObTableLoadResourceRpcExecutor -{ - typedef ObTableLoadResourceRpcExecutor ParentType; - -public: - ObDirectLoadResourceReleaseExecutor(common::ObIAllocator &allocator, - const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result) - : ParentType(allocator, request, result) - { - } - virtual ~ObDirectLoadResourceReleaseExecutor() = default; - -protected: - int check_args() override; - int process() override; -}; - -// update_resource -class ObDirectLoadResourceUpdateExecutor - : public ObTableLoadResourceRpcExecutor -{ - typedef ObTableLoadResourceRpcExecutor ParentType; - -public: - ObDirectLoadResourceUpdateExecutor(common::ObIAllocator &allocator, - const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result) - : ParentType(allocator, request, result) - { - } - virtual ~ObDirectLoadResourceUpdateExecutor() = default; - -protected: - int check_args() override; - int process() override; -}; - -// check_resource -class ObDirectLoadResourceCheckExecutor - : public ObTableLoadResourceRpcExecutor -{ - typedef ObTableLoadResourceRpcExecutor ParentType; - -public: - ObDirectLoadResourceCheckExecutor(common::ObIAllocator &allocator, - const ObDirectLoadResourceOpRequest &request, - ObDirectLoadResourceOpResult &result) - : ParentType(allocator, request, result) - { - } - virtual ~ObDirectLoadResourceCheckExecutor() = default; - -protected: - int check_args() override; - int process() override; -}; - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.cpp b/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.cpp deleted file mode 100644 index 54905a8f0..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER - -#include "ob_table_load_resource_rpc_struct.h" - -namespace oceanbase -{ -namespace observer -{ - -OB_SERIALIZE_MEMBER(ObDirectLoadResourceOpRequest, - command_type_, - arg_content_); - -OB_UNIS_DEF_SERIALIZE(ObDirectLoadResourceOpResult, - command_type_, - res_content_); - -OB_UNIS_DEF_SERIALIZE_SIZE(ObDirectLoadResourceOpResult, - command_type_, - res_content_); - -OB_DEF_DESERIALIZE(ObDirectLoadResourceOpResult) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null allocator in deserialize", K(ret)); - } else { - ObString tmp_res_content; - LST_DO_CODE(OB_UNIS_DECODE, - command_type_, - tmp_res_content); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ob_write_string(*allocator_, tmp_res_content, res_content_))) { - LOG_WARN("fail to copy string", K(ret)); - } - } - - return ret; -} - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceUnit, - addr_, - thread_count_, - memory_size_); - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceApplyArg, - task_key_, - apply_array_); - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceReleaseArg, - task_key_); - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceUpdateArg, - thread_count_, - memory_size_, - addrs_); - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceCheckArg, - avail_memory_, - first_check_); - -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadResourceOpRes, - error_code_, - avail_memory_, - assigned_array_); - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.h b/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.h deleted file mode 100644 index 8551c7644..000000000 --- a/src/observer/table_load/resource/ob_table_load_resource_rpc_struct.h +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "lib/container/ob_array_serialization.h" -#include "lib/utility/ob_print_utils.h" -#include "lib/utility/ob_unify_serialize.h" -#include "share/table/ob_table_load_define.h" -#include "sql/session/ob_sql_session_mgr.h" -#include "observer/table_load/ob_table_load_struct.h" - -namespace oceanbase -{ -namespace observer -{ -enum class ObDirectLoadResourceCommandType -{ - APPLY = 0, - RELEASE = 1, - UPDATE = 2, - CHECK = 3, - MAX_TYPE -}; - -class ObDirectLoadResourceUnit -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceUnit() - : thread_count_(0), memory_size_(0) - { - } - ObDirectLoadResourceUnit(const ObDirectLoadResourceUnit &unit) - : addr_(unit.addr_), thread_count_(unit.thread_count_), memory_size_(unit.memory_size_) - { - } - TO_STRING_KV(K_(addr), K_(thread_count), K_(memory_size)); -public: - common::ObAddr addr_; - int64_t thread_count_; - int64_t memory_size_; -}; - -class ObDirectLoadResourceOpRequest -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceOpRequest() - : command_type_(observer::ObDirectLoadResourceCommandType::MAX_TYPE) - { - } - template - int set_arg(const Arg &arg, common::ObIAllocator &allocator) - { - int ret = common::OB_SUCCESS; - const int64_t size = arg.get_serialize_size(); - char *buf = nullptr; - int64_t pos = 0; - if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - SERVER_LOG(WARN, "fail to alloc memory", KR(ret), K(size)); - } else if (OB_FAIL(arg.serialize(buf, size, pos))) { - SERVER_LOG(WARN, "fail to serialize arg", KR(ret), K(arg)); - } else { - arg_content_.assign_ptr(buf, size); - } - - return ret; - } - template - int get_arg(Arg &arg) const - { - int ret = common::OB_SUCCESS; - int64_t pos = 0; - if (OB_UNLIKELY(arg_content_.empty())) { - ret = OB_INVALID_ARGUMENT; - SERVER_LOG(WARN, "invalid args", KR(ret), KPC(this)); - } else if (OB_FAIL(arg.deserialize(arg_content_.ptr(), arg_content_.length(), pos))) { - SERVER_LOG(WARN, "fail to deserialize arg content", KR(ret), KPC(this)); - } - - return ret; - } - TO_STRING_KV(K_(command_type), "arg_content", common::ObHexStringWrap(arg_content_)); -public: - observer::ObDirectLoadResourceCommandType command_type_; - ObString arg_content_; -}; - -class ObDirectLoadResourceOpResult -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceOpResult() - : allocator_(nullptr), command_type_(observer::ObDirectLoadResourceCommandType::MAX_TYPE) - { - } - template - int set_res(const Res &res, common::ObIAllocator &allocator) - { - int ret = common::OB_SUCCESS; - const int64_t size = res.get_serialize_size(); - if (size > 0) { - char *buf = nullptr; - int64_t pos = 0; - if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - SERVER_LOG(WARN, "fail to alloc memory", KR(ret), K(size)); - } else if (OB_FAIL(res.serialize(buf, size, pos))) { - SERVER_LOG(WARN, "fail to serialize res", KR(ret), K(res)); - } else { - res_content_.assign_ptr(buf, size); - } - } - - return ret; - } - template - int get_res(Res &res) const - { - int ret = common::OB_SUCCESS; - int64_t pos = 0; - if (OB_UNLIKELY(res_content_.empty())) { - ret = OB_INVALID_ARGUMENT; - SERVER_LOG(WARN, "invalid args", KR(ret), KPC(this)); - } else if (OB_FAIL(res.deserialize(res_content_.ptr(), res_content_.length(), pos))) { - SERVER_LOG(WARN, "fail to deserialize res content", KR(ret), KPC(this)); - } - - return ret; - } - TO_STRING_KV(K_(command_type), "res_content", common::ObHexStringWrap(res_content_)); -public: - common::ObIAllocator *allocator_; // for deserialize - observer::ObDirectLoadResourceCommandType command_type_; - ObString res_content_; -}; - -////////////////////////////////////////////////////////////////////// - -class ObDirectLoadResourceApplyArg final -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceApplyArg() - { - } - ObDirectLoadResourceApplyArg(const ObDirectLoadResourceApplyArg& arg) - : task_key_(arg.task_key_), apply_array_(arg.apply_array_) - { - } - bool is_valid() - { - bool valid = true && task_key_.is_valid(); - for (int64_t i = 0; valid && i < apply_array_.count(); i++) { - valid = (apply_array_[i].addr_.is_valid() && apply_array_[i].thread_count_ >= 0 && apply_array_[i].memory_size_ >= 0); - } - return valid; - } - TO_STRING_KV(K_(task_key), K_(apply_array)); -public: - ObTableLoadUniqueKey task_key_; - common::ObSArray apply_array_; -}; - -class ObDirectLoadResourceReleaseArg final -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceReleaseArg() - { - } - ObDirectLoadResourceReleaseArg(const ObDirectLoadResourceReleaseArg &arg) - : task_key_(arg.task_key_) - { - } - bool is_valid() - { - return true && task_key_.is_valid(); - } - TO_STRING_KV(K_(task_key)); -public: - ObTableLoadUniqueKey task_key_; -}; - -class ObDirectLoadResourceUpdateArg final -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceUpdateArg() - : thread_count_(0), memory_size_(0) - { - } - ObDirectLoadResourceUpdateArg(const ObDirectLoadResourceUpdateArg &arg) - : thread_count_(arg.thread_count_), - memory_size_(arg.memory_size_), - addrs_(arg.addrs_) - { - } - bool is_valid() - { - return true && thread_count_ >= 0 && memory_size_ >= 0; - } - TO_STRING_KV(K_(thread_count), K_(memory_size), K_(addrs)); -public: - int64_t thread_count_; - int64_t memory_size_; - common::ObSArray addrs_; -}; - -class ObDirectLoadResourceCheckArg final -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceCheckArg() - : avail_memory_(0), first_check_(false) - { - } - ObDirectLoadResourceCheckArg(const ObDirectLoadResourceCheckArg &arg) - : avail_memory_(arg.avail_memory_), - first_check_(arg.first_check_) - { - } - bool is_valid() - { - return true; - } - TO_STRING_KV(K_(avail_memory)); -public: - int64_t avail_memory_; - bool first_check_; -}; - -class ObDirectLoadResourceOpRes final -{ - OB_UNIS_VERSION(1); - -public: - ObDirectLoadResourceOpRes() - : error_code_(common::OB_SUCCESS), - avail_memory_(0) - { - } - ObDirectLoadResourceOpRes(const ObDirectLoadResourceOpRes &res) - : error_code_(res.error_code_), - avail_memory_(res.avail_memory_), - assigned_array_(res.assigned_array_) - { - } - TO_STRING_KV(K_(error_code), K_(avail_memory), K_(assigned_array)); -public: - int error_code_; - int64_t avail_memory_; - common::ObSArray assigned_array_; -}; - -} // namespace observer -} // namespace oceanbase diff --git a/src/observer/table_load/resource/ob_table_load_resource_service.cpp b/src/observer/table_load/resource/ob_table_load_resource_service.cpp index d330a7b01..86627680c 100644 --- a/src/observer/table_load/resource/ob_table_load_resource_service.cpp +++ b/src/observer/table_load/resource/ob_table_load_resource_service.cpp @@ -50,6 +50,10 @@ int ObTableLoadResourceService::init() LOG_WARN("init twice", KR(ret)); } else { is_inited_ = true; + if (OB_FAIL(alloc_resource_manager())) { + is_inited_ = false; + LOG_WARN("fail to alloc resource manager", KR(ret)); + } } return ret; @@ -103,49 +107,6 @@ void ObTableLoadResourceService::destroy() LOG_INFO("resource_service finish to destroy"); } -int ObTableLoadResourceService::switch_to_leader() -{ - int ret = OB_SUCCESS; - ObMutexGuard switch_guard(switch_lock_); - int64_t start_time_us = ObTimeUtility::current_time(); - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("fail to check_inner_stat", KR(ret)); - } else { - if (OB_ISNULL(resource_manager_)) { - obsys::ObWLockGuard w_guard(rw_lock_); - if (OB_FAIL(alloc_resource_manager())) { - LOG_WARN("fail to alloc resource_manager", KR(ret)); - } - } else { - obsys::ObRLockGuard r_guard(rw_lock_); - ret = resource_manager_->resume(); - LOG_INFO("resource_service finish to resume",KR(ret)); - } - } - const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; - FLOG_INFO("resource_manager: switch_to_leader", KR(ret), K(cost_us), KP_(resource_manager)); - - return ret; -} - -int ObTableLoadResourceService::switch_to_follower_gracefully() { - int ret = OB_SUCCESS; - LOG_INFO("switch_to_follower_gracefully"); - if (OB_FAIL(inner_switch_to_follower())) { - LOG_WARN("fail to switch to follower", KR(ret)); - } - - return ret; -} - -void ObTableLoadResourceService::switch_to_follower_forcedly() { - int ret = OB_SUCCESS; - LOG_INFO("switch_to_follower_forcedly"); - if (OB_FAIL(inner_switch_to_follower())) { - LOG_WARN("fail to switch to follower", KR(ret)); - } -} - int ObTableLoadResourceService::alloc_resource_manager() { int ret = OB_SUCCESS; @@ -207,21 +168,6 @@ int ObTableLoadResourceService::delete_resource_manager() return ret; } -int ObTableLoadResourceService::inner_switch_to_follower() -{ - int ret = OB_SUCCESS; - ObMutexGuard switch_guard(switch_lock_); - obsys::ObRLockGuard r_guard(rw_lock_); - const int64_t start_time_us = ObTimeUtility::current_time(); - if (OB_NOT_NULL(resource_manager_)) { - resource_manager_->pause(); - } - const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; - FLOG_INFO("resource_manager: switch_to_follower", KR(ret), K(cost_us), KP_(resource_manager)); - - return ret; -} - int ObTableLoadResourceService::check_inner_stat() { int ret = OB_SUCCESS; @@ -233,88 +179,14 @@ int ObTableLoadResourceService::check_inner_stat() return ret; } -int ObTableLoadResourceService::get_leader_addr( - const share::ObLSID &ls_id, - ObAddr &leader) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(GCTX.location_service_->get_leader_with_retry_until_timeout( - GCONF.cluster_id, ls_id, leader, GET_LEADER_RETRY_TIMEOUT))) { - LOG_WARN("fail to get ls location leader", KR(ret), K(ls_id)); - if (is_location_service_renew_error(ret)) { - ret = OB_EAGAIN; - } - } - - return ret; -} - -int ObTableLoadResourceService::local_apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res) -{ - int ret = OB_SUCCESS; - ObTableLoadResourceService *service = nullptr; - if (OB_ISNULL(service = share::g_mp->table_load_resource_service())) { - ret = OB_ERR_SYS; - LOG_WARN("null table load resource service", KR(ret)); - } else if(OB_ISNULL(service->resource_manager_)) { - ret = OB_EAGAIN; - LOG_WARN("resource_manager_ is null", KR(ret)); - } else { - ret = service->resource_manager_->apply_resource(arg, res); - } - - return ret; -} - -int ObTableLoadResourceService::local_release_resource(ObDirectLoadResourceReleaseArg &arg) -{ - int ret = OB_SUCCESS; - ObTableLoadResourceService *service = nullptr; - if (OB_ISNULL(service = share::g_mp->table_load_resource_service())) { - ret = OB_ERR_SYS; - LOG_WARN("null table load resource service", KR(ret)); - } else if(OB_ISNULL(service->resource_manager_)) { - ret = OB_EAGAIN; - LOG_WARN("resource_manager_ is null", KR(ret)); - } else { - ret = service->resource_manager_->release_resource(arg); - } - - return ret; -} - -int ObTableLoadResourceService::local_update_resource(ObDirectLoadResourceUpdateArg &arg) -{ - int ret = OB_SUCCESS; - ObTableLoadResourceService *service = nullptr; - if (OB_ISNULL(service = share::g_mp->table_load_resource_service())) { - ret = OB_ERR_SYS; - LOG_WARN("null table load resource service", KR(ret)); - } else if(OB_ISNULL(service->resource_manager_)) { - ret = OB_EAGAIN; - LOG_WARN("resource_manager_ is null", KR(ret)); - } else { - ret = service->resource_manager_->update_resource(arg); - } - - return ret; -} - -int ObTableLoadResourceService::apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res) +int ObTableLoadResourceService::apply_resource(ObDirectLoadResourceApplyArg &arg) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(arg), KR(ret)); } else { - ObAddr leader; - if (OB_FAIL(get_leader_addr(share::SYS_LS, leader))) { - LOG_WARN("fail to get leader addr", KR(ret)); - } else if (ObTableLoadUtils::is_local_addr(leader)) { - ret = local_apply_resource(arg, res); - } else { - TABLE_LOAD_RESOURCE_RPC_CALL(apply_resource, leader, arg, res); - } + ret = share::g_mp->table_load_resource_service()->resource_manager_->apply_resource(arg); } return ret; @@ -327,14 +199,7 @@ int ObTableLoadResourceService::release_resource(ObDirectLoadResourceReleaseArg ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(arg), KR(ret)); } else { - ObAddr leader; - if (OB_FAIL(get_leader_addr(share::SYS_LS, leader))) { - LOG_WARN("fail to get leader addr", KR(ret)); - } else if (ObTableLoadUtils::is_local_addr(leader)) { - ret = local_release_resource(arg); - } else { - TABLE_LOAD_RESOURCE_RPC_CALL(release_resource, leader, arg); - } + ret = share::g_mp->table_load_resource_service()->resource_manager_->release_resource(arg); } return ret; diff --git a/src/observer/table_load/resource/ob_table_load_resource_service.h b/src/observer/table_load/resource/ob_table_load_resource_service.h index b54e3b593..525d520ba 100644 --- a/src/observer/table_load/resource/ob_table_load_resource_service.h +++ b/src/observer/table_load/resource/ob_table_load_resource_service.h @@ -16,13 +16,7 @@ #pragma once -#include "lib/hash/ob_hashmap.h" -#include "share/scn.h" -#include "share/ob_ls_id.h" -#include "logservice/ob_log_base_type.h" -#include "lib/lock/ob_mutex.h" #include "lib/lock/ob_spin_lock.h" -#include "lib/lock/ob_recursive_mutex.h" #include "observer/table_load/ob_table_load_struct.h" #include "observer/table_load/resource/ob_table_load_resource_manager.h" @@ -32,12 +26,8 @@ namespace observer { class ObTableLoadResourceManager; -class ObTableLoadResourceService : public logservice::ObIReplaySubHandler, - public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler +class ObTableLoadResourceService { -public: - static const int64_t GET_LEADER_RETRY_TIMEOUT = 1 * 1000LL * 1000LL; // 1s public: ObTableLoadResourceService() : resource_manager_(nullptr), @@ -52,43 +42,14 @@ class ObTableLoadResourceService : public logservice::ObIReplaySubHandler, void wait(); void destroy(); - // for replay, do nothing - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) - { - UNUSED(buffer); - UNUSED(nbytes); - UNUSED(lsn); - UNUSED(scn); - return OB_SUCCESS; - } - // for checkpoint, do nothing - share::SCN get_rec_scn() { return share::SCN::max_scn(); } - - int flush(share::SCN &scn) - { - UNUSED(scn); - return OB_SUCCESS; - } - - int resume_leader() { return switch_to_leader(); } - int switch_to_leader(); - int switch_to_follower_gracefully(); - void switch_to_follower_forcedly(); - - static int get_leader_addr(const share::ObLSID &ls_id, common::ObAddr &leader); - static int local_apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res); - static int local_release_resource(ObDirectLoadResourceReleaseArg &arg); - static int local_update_resource(ObDirectLoadResourceUpdateArg &arg); - static int apply_resource(ObDirectLoadResourceApplyArg &arg, ObDirectLoadResourceOpRes &res); + static int apply_resource(ObDirectLoadResourceApplyArg &arg); static int release_resource(ObDirectLoadResourceReleaseArg &arg); private: int alloc_resource_manager(); int delete_resource_manager(); - int inner_switch_to_follower(); int check_inner_stat(); private: - mutable lib::ObMutex switch_lock_; mutable obsys::ObRWLock rw_lock_; ObTableLoadResourceManager *resource_manager_; bool is_inited_; diff --git a/src/observer/table_load/resource/ob_table_load_resource_struct.h b/src/observer/table_load/resource/ob_table_load_resource_struct.h new file mode 100644 index 000000000..620fb5cd3 --- /dev/null +++ b/src/observer/table_load/resource/ob_table_load_resource_struct.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "lib/container/ob_array.h" +#include "lib/utility/ob_print_utils.h" +#include "observer/table_load/ob_table_load_struct.h" + +namespace oceanbase +{ +namespace observer +{ + +struct ObDirectLoadResourceUnit +{ + ObDirectLoadResourceUnit() : thread_count_(0), memory_size_(0) {} + TO_STRING_KV(K_(addr), K_(thread_count), K_(memory_size)); + + common::ObAddr addr_; + int64_t thread_count_; + int64_t memory_size_; +}; + +struct ObDirectLoadResourceApplyArg +{ + bool is_valid() const + { + bool valid = task_key_.is_valid(); + for (int64_t i = 0; valid && i < apply_array_.count(); ++i) { + const ObDirectLoadResourceUnit &unit = apply_array_[i]; + valid = unit.addr_.is_valid() && unit.thread_count_ >= 0 && unit.memory_size_ >= 0; + } + return valid; + } + TO_STRING_KV(K_(task_key), K_(apply_array)); + + ObTableLoadUniqueKey task_key_; + common::ObSArray apply_array_; +}; + +struct ObDirectLoadResourceReleaseArg +{ + bool is_valid() const { return task_key_.is_valid(); } + TO_STRING_KV(K_(task_key)); + + ObTableLoadUniqueKey task_key_; +}; + +} // namespace observer +} // namespace oceanbase diff --git a/src/observer/vector_index/ob_hybrid_vector_refresh_task.cpp b/src/observer/vector_index/ob_hybrid_vector_refresh_task.cpp index 93c0ebaa6..7ca11eff7 100644 --- a/src/observer/vector_index/ob_hybrid_vector_refresh_task.cpp +++ b/src/observer/vector_index/ob_hybrid_vector_refresh_task.cpp @@ -32,23 +32,23 @@ namespace share int ObVecEmbeddingAsyncTaskExecutor::load_task(uint64_t &task_trace_base_num) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; ObArray task_ctx_array; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector async task not init", KR(ret)); } else if (!check_operation_allow()) { // skip - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { // skip - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { // skip + LOG_WARN("fail to get index ls mgr", K(ret)); } else { - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); ObIAllocator *allocator = task_opt.get_allocator(); const int64_t current_task_cnt = ObVecIndexAsyncTaskUtil::get_processing_task_cnt(task_opt); ObSEArray adapter_array; ObAdapterMapFunc adapter_map_func(adapter_array); - RWLock::RLockGuard lock_guard(index_ls_mgr->get_adapter_map_lock()); - if (OB_FAIL(index_ls_mgr->get_complete_adapter_map().foreach_refactored(adapter_map_func))) { + RWLock::RLockGuard lock_guard(index_mgr->get_adapter_map_lock()); + if (OB_FAIL(index_mgr->get_complete_adapter_map().foreach_refactored(adapter_map_func))) { LOG_WARN("fail to foreach adapter map", KR(ret)); } FOREACH_X(iter, adapter_array, OB_SUCC(ret) && (task_ctx_array.count() + current_task_cnt <= MAX_ASYNC_TASK_PROCESSING_COUNT)) { @@ -78,7 +78,7 @@ int ObVecEmbeddingAsyncTaskExecutor::load_task(uint64_t &task_trace_base_num) } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::fetch_new_trace_id(++task_trace_base_num, allocator, new_trace_id))) { LOG_WARN("fail to fetch new trace id", K(ret), K(tablet_id)); } else { - LOG_DEBUG("start load task", K(ret), K(tablet_id), K(task_trace_base_num), K(ls_->get_ls_id())); + LOG_DEBUG("start load task", K(ret), K(tablet_id), K(task_trace_base_num)); // 1. update task_ctx to async task map task_ctx->ls_ = ls_; @@ -92,7 +92,7 @@ int ObVecEmbeddingAsyncTaskExecutor::load_task(uint64_t &task_trace_base_num) task_ctx->task_status_.trace_id_ = new_trace_id; task_ctx->task_status_.target_scn_.convert_from_ts(ObTimeUtility::current_time()); - if (OB_FAIL(index_ls_mgr->get_async_task_opt().add_task_ctx(tablet_id, task_ctx, inc_new_task))) { // not overwrite + if (OB_FAIL(index_mgr->get_async_task_opt().add_task_ctx(tablet_id, task_ctx, inc_new_task))) { // not overwrite LOG_WARN("fail to add task ctx", K(ret)); } else if (inc_new_task && OB_FAIL(task_ctx_array.push_back(task_ctx))) { LOG_WARN("fail to push back task status", K(ret), K(task_ctx)); @@ -107,15 +107,15 @@ int ObVecEmbeddingAsyncTaskExecutor::load_task(uint64_t &task_trace_base_num) } } } - LOG_INFO("finish load async task", K(ret), K(ls_->get_ls_id()), K(task_ctx_array.count()), K(current_task_cnt)); + LOG_INFO("finish load async task", K(ret), K(task_ctx_array.count()), K(current_task_cnt)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(insert_new_task(task_ctx_array))) { - LOG_WARN("fail to insert new tasks", K(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to insert new tasks", K(ret)); } // clear on fail if (OB_FAIL(ret) && !task_ctx_array.empty()) { - if (OB_FAIL(clear_task_ctxs(index_ls_mgr->get_async_task_opt(), task_ctx_array))) { + if (OB_FAIL(clear_task_ctxs(index_mgr->get_async_task_opt(), task_ctx_array))) { LOG_WARN("fail to clear task ctx", K(ret)); } } @@ -155,7 +155,7 @@ ObHybridVectorRefreshTask::do_work() │ └── ObEmbeddingTask::is_completed() ├── prepare_index_id_data() // generate data for deleting table 4 ├── ObInsertLobColumnHelper::start_trans() - ├── txs->get_ls_read_snapshot() + ├── txs->get_read_snapshot() ├── do_refresh_only() │ ├── init_dml_param() │ │ ├── table_param.convert() @@ -184,7 +184,7 @@ int ObHybridVectorRefreshTask::do_work() ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret), KPC(task_ctx)); } else { - LOG_INFO("start do_work", K(ret), K(task_ctx->task_status_), K(ls_id_)); + LOG_INFO("start do_work", K(ret), K(task_ctx->task_status_)); } while (OB_SUCC(ret) && !exec_finish) { switch (current_status()) { @@ -478,7 +478,7 @@ int ObHybridVectorRefreshTask::init_dml_param(uint64_t table_id, dml_param.store_ctx_guard_ = &store_ctx_guard; dml_param.schema_version_ = table_schema->get_schema_version(); dml_param.dml_allocator_ = &allocator_; - if (OB_FAIL(oas->get_write_store_ctx_guard(ls_id_, timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))) { + if (OB_FAIL(oas->get_write_store_ctx_guard(timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))) { LOG_WARN("failed to get write store context guard", K(ret)); } } @@ -550,8 +550,7 @@ int ObHybridVectorRefreshTask::prepare_for_embedding(ObPluginVectorIndexAdaptor LOG_WARN("allocate memory failed", K(ret), K(table_param)); } else if (FALSE_IT(table_param = new(table_param)schema::ObTableParam(task_ctx->allocator_))) { } else if (FALSE_IT(ctx_->task_status_.target_scn_.convert_from_ts(ObTimeUtility::current_time()))) { - } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL, task_ctx->allocator_, @@ -737,7 +736,7 @@ int ObHybridVectorRefreshTask::do_refresh_only( LOG_WARN("unexpected error", K(ret), KPC(task_ctx), K(oas)); } else if (OB_FAIL(init_dml_param(adaptor.get_vbitmap_table_id(), dml_param, table_dml_param, task_ctx->index_id_column_ids_, tx_desc, snapshot, store_ctx_guard))) { LOG_WARN("failed to init dml param", K(ret), K(dml_param), K(table_dml_param)); - } else if (OB_FAIL(oas->insert_rows(ls_id_, adaptor.get_vbitmap_tablet_id(), *tx_desc, dml_param, task_ctx->index_id_column_ids_, &index_id_iter, affected_rows))) { + } else if (OB_FAIL(oas->insert_rows(adaptor.get_vbitmap_tablet_id(), *tx_desc, dml_param, task_ctx->index_id_column_ids_, &index_id_iter, affected_rows))) { LOG_WARN("failed to insert rows to index id table", K(ret), K(adaptor.get_vbitmap_table_id())); } store_ctx_guard.reset(); @@ -751,7 +750,7 @@ int ObHybridVectorRefreshTask::do_refresh_only( LOG_WARN("failed to remove ora_rowscn column", K(ret)); } else if (OB_FAIL(init_dml_param(adaptor.get_inc_table_id(), dml_param, index_id_dml_param, delta_delete_column_id, tx_desc, snapshot, store_ctx_guard))) { LOG_WARN("failed to init dml param", K(ret), K(dml_param), K(index_id_dml_param)); - } else if (OB_FAIL(oas->delete_rows(ls_id_, adaptor.get_inc_tablet_id(), *tx_desc, dml_param, delta_delete_column_id, &delta_delete_iter, affected_rows))) { + } else if (OB_FAIL(oas->delete_rows(adaptor.get_inc_tablet_id(), *tx_desc, dml_param, delta_delete_column_id, &delta_delete_iter, affected_rows))) { LOG_WARN("failed to insert rows to index id table", K(ret), K(adaptor.get_inc_table_id())); } store_ctx_guard.reset(); @@ -850,8 +849,7 @@ int ObHybridVectorRefreshTask::delete_embedded_table(ObPluginVectorIndexAdaptor common::ObNewRowIterator *scan_iter = nullptr; ObStorageDatumUtils util; ObArenaAllocator scan_allocator("VecEmbedding", OB_MALLOC_NORMAL_BLOCK_SIZE); - if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, snapshot.version(), INDEX_TYPE_HYBRID_INDEX_EMBEDDED_LOCAL, allocator_, @@ -912,7 +910,7 @@ int ObHybridVectorRefreshTask::delete_embedded_table(ObPluginVectorIndexAdaptor LOG_WARN("unexpected error", K(ret), KPC(task_ctx), K(oas)); } else if (OB_FAIL(init_dml_param(adaptor.get_embedded_table_id(), dml_param, table_dml_param, dml_column_ids, tx_desc, snapshot, store_ctx_guard))) { LOG_WARN("failed to init dml param", K(ret), K(dml_param), K(table_dml_param)); - } else if (OB_FAIL(oas->delete_rows(ls_id_, adaptor.get_embedded_tablet_id(), *tx_desc, dml_param, dml_column_ids, &delete_iter, affected_rows))) { + } else if (OB_FAIL(oas->delete_rows(adaptor.get_embedded_tablet_id(), *tx_desc, dml_param, dml_column_ids, &delete_iter, affected_rows))) { LOG_WARN("failed to delete rows from embedded table", K(ret), K(adaptor.get_embedded_tablet_id())); } store_ctx_guard.reset(); @@ -953,10 +951,10 @@ int ObHybridVectorRefreshTask::after_embedding(ObPluginVectorIndexAdaptor &adapt } else if (OB_ISNULL(txs)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ptr", K(ret), KPC(task_ctx), K(txs)); - } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(ls_id_, false/*is_for_read*/, timeout_us, tx_desc))) { + } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(false/*is_for_read*/, timeout_us, tx_desc))) { LOG_WARN("fail to start trans", K(ret)); } else if (FALSE_IT(trans_start = true)) { - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ls_id_, timeout_us, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout_us, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else if (OB_FAIL(do_refresh_only(adaptor, tx_desc, snapshot, store_ctx_guard, index_id_iter, delta_delete_iter))) { LOG_WARN("fail to do refresh", K(ret)); @@ -982,8 +980,7 @@ int ObHybridVectorRefreshTask::after_embedding(ObPluginVectorIndexAdaptor &adapt int64_t loop_cnt = 0; if (OB_FAIL(new_row.init(task_ctx->embedded_table_column_ids_.count()))) { LOG_WARN("fail to init datum row", K(ret), K(task_ctx->embedded_table_column_ids_), K(new_row)); - } else if (adaptor.get_is_need_vid() && OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (adaptor.get_is_need_vid() && OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, INDEX_TYPE_VEC_VID_ROWKEY_LOCAL, allocator_, @@ -1060,8 +1057,7 @@ int ObHybridVectorRefreshTask::after_embedding(ObPluginVectorIndexAdaptor &adapt ObTableScanIterator *embedded_table_scan_iter = nullptr; ObArenaAllocator embedde_scan_allocator("VecEmbedding", OB_MALLOC_NORMAL_BLOCK_SIZE); ObRowkey rowkey(obj_ptr, embedded_rowkey_count); - if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, snapshot.version(), INDEX_TYPE_HYBRID_INDEX_EMBEDDED_LOCAL, allocator_, @@ -1130,7 +1126,7 @@ int ObHybridVectorRefreshTask::after_embedding(ObPluginVectorIndexAdaptor &adapt LOG_WARN("unexpected error", K(ret), KPC(task_ctx), K(oas)); } else if (OB_FAIL(init_dml_param(adaptor.get_embedded_table_id(), dml_param, table_dml_param, task_ctx->embedded_table_column_ids_, tx_desc, snapshot, store_ctx_guard))) { LOG_WARN("failed to init dml param", K(ret), K(dml_param), K(table_dml_param)); - } else if (OB_FAIL(oas->update_rows(ls_id_, adaptor.get_embedded_tablet_id(), *tx_desc, dml_param, task_ctx->embedded_table_column_ids_, task_ctx->embedded_table_update_ids_, &embedded_iter, affected_rows))) { + } else if (OB_FAIL(oas->update_rows(adaptor.get_embedded_tablet_id(), *tx_desc, dml_param, task_ctx->embedded_table_column_ids_, task_ctx->embedded_table_update_ids_, &embedded_iter, affected_rows))) { LOG_WARN("failed to insert rows to embedded table", K(ret), K(adaptor.get_embedded_tablet_id())); } store_ctx_guard.reset(); diff --git a/src/observer/vector_index/ob_ivf_async_task.cpp b/src/observer/vector_index/ob_ivf_async_task.cpp index 01b395a01..7946f6158 100644 --- a/src/observer/vector_index/ob_ivf_async_task.cpp +++ b/src/observer/vector_index/ob_ivf_async_task.cpp @@ -17,7 +17,6 @@ #include "ob_ivf_async_task.h" #include "share/rc/ob_module_provider.h" #include "observer/vector_index/ob_plugin_vector_index_service.h" -#include "share/ob_ls_id.h" #include "observer/vector_index/ob_vector_index_ivf_cache_util.h" namespace oceanbase @@ -27,18 +26,10 @@ namespace share int ObIvfAsyncTask::delete_deprecated_cache(ObPluginVectorIndexService &vector_index_service) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - if (OB_FAIL(vector_index_service.get_ls_index_mgr_map().get_refactored(ls_id_, ls_index_mgr))) { - if (ret == OB_HASH_NOT_EXIST) { - // do not need delete - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id_)); - } - } else if (OB_FAIL(ls_index_mgr->erase_ivf_cache_mgr(ctx_->task_status_.tablet_id_))) { + ObPluginVectorIndexMgr *index_mgr = &vector_index_service.get_index_mgr(); + if (OB_FAIL(index_mgr->erase_ivf_cache_mgr(ctx_->task_status_.tablet_id_))) { if (ret != OB_HASH_NOT_EXIST) { LOG_WARN("failed to erase vector index ivf cache mgr", - K(ls_id_), K(ctx_->task_status_.tablet_id_), KR(ret)); } else { // already removed @@ -78,15 +69,13 @@ int ObIvfAsyncTask::write_cache(ObPluginVectorIndexService &vector_index_service K(1UL), K(ctx_->task_status_.table_id_), KPC(aux_table_info)); - } else if (OB_FAIL(vector_index_service.acquire_ivf_cache_mgr_guard(ls_id_, - ctx_->task_status_.tablet_id_, + } else if (OB_FAIL(vector_index_service.acquire_ivf_cache_mgr_guard(ctx_->task_status_.tablet_id_, vec_param, vec_param.dim_, ctx_->task_status_.table_id_, cache_guard))) { LOG_WARN("fail to acquire ivf cache mgr with vec param", K(ret), - K(ls_id_), K(ctx_->task_status_), K(vec_param)); } else if (OB_ISNULL(cache_mgr = cache_guard.get_ivf_cache_mgr())) { @@ -133,7 +122,7 @@ int ObIvfAsyncTask::do_work() LOG_WARN("unexpected nullptr", K(ret), KP(ctx_), KP(vector_index_service)); } else if (OB_ISNULL(vec_idx_mgr_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_id_)); + LOG_WARN("get invalid vector index manager", KR(ret)); } else if (ctx_->task_status_.task_type_ == OB_VECTOR_ASYNC_INDEX_IVF_CLEAN) { if (OB_FAIL(delete_deprecated_cache(*vector_index_service))) { LOG_WARN("fail to delete deprecated cache", K(ret)); diff --git a/src/observer/vector_index/ob_ivf_async_task.h b/src/observer/vector_index/ob_ivf_async_task.h index add26d780..3bfb57cd1 100644 --- a/src/observer/vector_index/ob_ivf_async_task.h +++ b/src/observer/vector_index/ob_ivf_async_task.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_SHARE_OB_IVF_ASYNC_TASK_H_ #define OCEANBASE_SHARE_OB_IVF_ASYNC_TASK_H_ -#include "share/ob_ls_id.h" #include "share/scn.h" #include "share/rc/ob_tenant_base.h" #include "observer/vector_index/ob_vector_index_ivf_cache_mgr.h" diff --git a/src/observer/vector_index/ob_ivf_async_task_executor.cpp b/src/observer/vector_index/ob_ivf_async_task_executor.cpp index d3599f150..31ae528bd 100644 --- a/src/observer/vector_index/ob_ivf_async_task_executor.cpp +++ b/src/observer/vector_index/ob_ivf_async_task_executor.cpp @@ -132,16 +132,14 @@ int ObIvfAsyncTaskExector::LoadTaskCallback::is_cache_writable(const ObIvfAuxTab ObVectorIndexType::VIT_IVF_INDEX, vec_param))) { LOG_WARN("fail to get vector index param with dim", K(ret), K(table_info)); - } else if (OB_FAIL(ObIvfCacheUtil::is_cache_writable(ls_->get_ls_id(), - table_info.centroid_table_id_, + } else if (OB_FAIL(ObIvfCacheUtil::is_cache_writable(table_info.centroid_table_id_, table_info.centroid_tablet_ids_[idx], vec_param, vec_param.dim_, is_writable))) { LOG_WARN("fail to check is cache writable", K(ret), K(table_info)); } else if (!is_writable && table_info.type_ == ObVectorIndexAlgorithmType::VIAT_IVF_PQ) { - if (OB_FAIL(ObIvfCacheUtil::is_cache_writable(ls_->get_ls_id(), - table_info.centroid_table_id_, + if (OB_FAIL(ObIvfCacheUtil::is_cache_writable(table_info.centroid_table_id_, table_info.centroid_tablet_ids_[idx], vec_param, vec_param.dim_, @@ -247,17 +245,17 @@ bool ObIvfAsyncTaskExector::check_operation_allow() int ObIvfAsyncTaskExector::check_and_set_thread_pool() { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector index load task not inited", K(ret)); } else if (OB_ISNULL(vector_index_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { + } else if (OB_FAIL(get_index_mgr(index_mgr))) { LOG_WARN("fail to get index ls mgr", K(ret)); } else { - ObIAllocator *allocator = index_ls_mgr->get_async_task_opt().get_allocator(); + ObIAllocator *allocator = index_mgr->get_async_task_opt().get_allocator(); ObVecIndexAsyncTaskHandler &thread_pool_handle = vector_index_service_->get_vec_async_task_handle(); if (thread_pool_handle.is_inited()) { // no need to init twice, skip @@ -489,7 +487,7 @@ int ObIvfAsyncTaskExector::generate_aux_table_info_map(ObIvfAuxTableInfoMap &aux int ObIvfAsyncTaskExector::load_task(uint64_t &task_trace_base_num) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; ObSchemaGetterGuard schema_guard; bool is_active_time = true; if (IS_NOT_INIT) { @@ -503,10 +501,9 @@ int ObIvfAsyncTaskExector::load_task(uint64_t &task_trace_base_num) } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::in_active_time(is_active_time))) { LOG_WARN("fail to get active time", KR(ret)); } else if (!is_active_time) { - LOG_INFO("skip auto-create per-tablet ivf maintenance tasks, not in active time", - K(ls_->get_ls_id())); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { // skip - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + LOG_INFO("skip auto-create per-tablet ivf maintenance tasks, not in active time"); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { // skip + LOG_WARN("fail to get index ls mgr", K(ret)); } else if (OB_ISNULL(ls_)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("invalid null ls", K(ret)); @@ -516,22 +513,22 @@ int ObIvfAsyncTaskExector::load_task(uint64_t &task_trace_base_num) } else { ObVecIndexTaskCtxArray task_status_array; LoadTaskCallback load_task_func( - index_ls_mgr->get_async_task_opt(), *ls_, task_status_array, schema_guard, task_trace_base_num); + index_mgr->get_async_task_opt(), *ls_, task_status_array, schema_guard, task_trace_base_num); ObIvfAuxTableInfoMap aux_table_info_map; - if (OB_FAIL(index_ls_mgr->get_ivf_cache_mgr_map().foreach_refactored( + if (OB_FAIL(index_mgr->get_ivf_cache_mgr_map().foreach_refactored( load_task_func))) { // ivf clean task LOG_WARN("fail to do load task each entry", K(ret), K(load_task_func)); } else if (OB_FAIL(generate_aux_table_info_map(aux_table_info_map))) { - LOG_WARN("fail to generate aux table info map", K(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to generate aux table info map", K(ret)); } else if (OB_FAIL(aux_table_info_map.foreach_refactored(load_task_func))) { // ivf load task LOG_WARN("fail to do load task each entry", K(ret), K(load_task_func)); } else if (OB_FAIL(insert_new_task(task_status_array))) { - LOG_WARN("fail to insert new task", K(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to insert new task", K(ret)); } // clear on fail if (OB_FAIL(ret) && !task_status_array.empty()) { - if (OB_FAIL(clear_task_ctxs(index_ls_mgr->get_async_task_opt(), task_status_array))) { + if (OB_FAIL(clear_task_ctxs(index_mgr->get_async_task_opt(), task_status_array))) { LOG_WARN("fail to clear task ctx", K(ret)); } } diff --git a/src/observer/vector_index/ob_plugin_vector_index_adaptor.cpp b/src/observer/vector_index/ob_plugin_vector_index_adaptor.cpp index 312fa6aa5..9b27056fb 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_adaptor.cpp +++ b/src/observer/vector_index/ob_plugin_vector_index_adaptor.cpp @@ -24,9 +24,10 @@ #include "share/roaringbitmap/ob_rb_memory_mgr.h" #include "observer/vector_index/ob_plugin_vector_index_utils.h" #include "storage/allocator/ob_tenant_vector_allocator.h" +#include "storage/tx/ob_ts_mgr.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_getter_guard.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/ob_server_struct.h" #include "share/ob_share_util.h" #include "common/ob_timeout_ctx.h" @@ -37,8 +38,7 @@ namespace share { ObVectorIndexInfo::ObVectorIndexInfo() - : ls_id_(share::ObLSID::INVALID_LS_ID), - rowkey_vid_table_id_(common::OB_INVALID_ID), + : rowkey_vid_table_id_(common::OB_INVALID_ID), vid_rowkey_table_id_(common::OB_INVALID_ID), inc_index_table_id_(common::OB_INVALID_ID), vbitmap_table_id_(common::OB_INVALID_ID), @@ -59,7 +59,6 @@ ObVectorIndexInfo::ObVectorIndexInfo() void ObVectorIndexInfo::reset() { - ls_id_ = share::ObLSID::INVALID_LS_ID; rowkey_vid_table_id_ = common::OB_INVALID_ID; vid_rowkey_table_id_ = common::OB_INVALID_ID; inc_index_table_id_ = common::OB_INVALID_ID; @@ -1131,7 +1130,7 @@ int ObPluginVectorIndexAdaptor::get_current_scn(share::SCN ¤t_scn) if (OB_ISNULL(txs)) { ret = OB_ERR_SYS; LOG_WARN("trans service is null", KR(ret)); - } else if (OB_FAIL(txs->get_ts_mgr()->get_gts(stc, NULL, current_scn, rts))) { + } else if (OB_FAIL(txs->get_ts_mgr()->get_gts(stc, current_scn, rts))) { LOG_WARN("get scn from cache.", KR(ret)); } return ret; @@ -2533,8 +2532,7 @@ int ObPluginVectorIndexAdaptor::complete_delta_buffer_table_data(ObVectorQueryAd int ObPluginVectorIndexAdaptor::check_index_id_table_readnext_status(ObVectorQueryAdaptorResultContext *ctx, common::ObNewRowIterator *row_iter, SCN query_scn, - bool is_async_mode, - ObLSID ls_id) + bool is_async_mode) { LOG_INFO("check_index_id_table_readnext_status"); INIT_SUCC(ret); @@ -2563,7 +2561,7 @@ int ObPluginVectorIndexAdaptor::check_index_id_table_readnext_status(ObVectorQue } } else if (is_async_mode) { // Async mode: branch early, no read_scn parsing at all - ret = check_index_id_table_readnext_status_async(ctx, row_iter, query_scn, ls_id); + ret = check_index_id_table_readnext_status_async(ctx, row_iter, query_scn); } else if (OB_FAIL(table_scan_iter->get_next_row(datum_row))) { if (ret == OB_ITER_END) { ret = OB_SUCCESS; @@ -2618,8 +2616,7 @@ int ObPluginVectorIndexAdaptor::check_index_id_table_readnext_status(ObVectorQue int ObPluginVectorIndexAdaptor::check_index_id_table_readnext_status_async( ObVectorQueryAdaptorResultContext *ctx, common::ObNewRowIterator *row_iter, - SCN query_scn, - ObLSID ls_id) + SCN query_scn) { // Async mode: no read_scn parsing. Use complete_index_mem_data_incremental for both // full/incremental (no check_if_complete_index, no complete_index_mem_data) @@ -2638,7 +2635,7 @@ int ObPluginVectorIndexAdaptor::check_index_id_table_readnext_status_async( OB_FAIL(ctx->init_bitmaps())) { LOG_WARN("failed to init ctx bitmaps.", K(ret)); } else { - ret = complete_index_mem_data_incremental(ctx, ls_id, query_scn, i_vids); + ret = complete_index_mem_data_incremental(ctx, query_scn, i_vids); if (OB_FAIL(ret)) { LOG_WARN("failed to complete index mem data incrementally", K(ret), K(vbitmap_data_->scn_)); } else { @@ -2968,7 +2965,6 @@ int ObPluginVectorIndexAdaptor::complete_index_mem_data(SCN read_scn, } int ObPluginVectorIndexAdaptor::complete_index_mem_data_incremental(ObVectorQueryAdaptorResultContext *ctx, - ObLSID ls_id, SCN query_scn, ObArray &i_vids) { @@ -3002,20 +2998,20 @@ int ObPluginVectorIndexAdaptor::complete_index_mem_data_incremental(ObVectorQuer ret = common::OB_ERR_UNEXPECTED; LOG_WARN("sql_proxy is null", K(ret)); } else { - // Query __all_tablet_to_ls to get table_id by tablet_id + // Query legacy tablet mapping to get table_id by tablet_id. common::ObSEArray tablet_ids; - common::ObSEArray<::oceanbase::share::ObTabletToLSInfo, 1> tablet_infos; + common::ObSEArray<::oceanbase::share::ObTabletTablePair, 1> tablet_infos; if (OB_FAIL(tablet_ids.push_back(vbitmap_tablet_id_))) { LOG_WARN("fail to push back tablet_id", K(ret), K(vbitmap_tablet_id_)); - } else if (OB_FAIL(::oceanbase::share::ObTabletToLSTableOperator::batch_get( + } else if (OB_FAIL(::oceanbase::share::ObTabletMappingTableOperator::batch_get( *GCTX.sql_proxy_, tablet_ids, tablet_infos))) { if (common::OB_ITEM_NOT_MATCH == ret) { - // Tablet not in __all_tablet_to_ls yet (table just created), retry later + // Tablet mapping is not visible yet (table just created), retry later. ret = common::OB_EAGAIN; - LOG_WARN("vbitmap tablet not found in __all_tablet_to_ls, may need retry", + LOG_WARN("vbitmap tablet mapping not found, may need retry", K(ret), K(vbitmap_tablet_id_)); } else { - LOG_WARN("fail to get tablet info from __all_tablet_to_ls", K(ret), K(vbitmap_tablet_id_)); + LOG_WARN("fail to get tablet mapping", K(ret), K(vbitmap_tablet_id_)); } } else if (tablet_infos.count() != 1) { ret = common::OB_ERR_UNEXPECTED; @@ -3085,8 +3081,7 @@ int ObPluginVectorIndexAdaptor::complete_index_mem_data_incremental(ObVectorQuer storage::ObTableScanParam scan_param; schema::ObTableParam table_param(tmp_allocator); // Read index_id_table using copied base_scn (thread-safe) - if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id, - this, + if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(this, query_scn, INDEX_TYPE_VEC_INDEX_ID_LOCAL, tmp_allocator, @@ -3098,7 +3093,7 @@ int ObPluginVectorIndexAdaptor::complete_index_mem_data_incremental(ObVectorQuer false, false, &base_scn))) { - LOG_WARN("failed to read index_id table incrementally", K(ret), K(ls_id), K(base_scn)); + LOG_WARN("failed to read index_id table incrementally", K(ret), K(base_scn)); } else { ObTableScanIterator *table_scan_iter = static_cast(incr_iter); while (OB_SUCC(ret)) { @@ -3187,8 +3182,7 @@ int ObPluginVectorIndexAdaptor::complete_index_mem_data_incremental(ObVectorQuer // --------------------------------------------------------------------------- // refresh_bitmap_background(): background bitmap refresh that mirrors the // query-time path (complete_index_mem_data_incremental) without a live query -// context. Uses SYS_LS, which is consistent with write_to_vsag_() in the -// change_stream plugin. The result bitmaps written into the synthetic ctx +// context. The result bitmaps written into the synthetic ctx // are discarded; the valuable side-effect is the update to vbitmap_data_. // --------------------------------------------------------------------------- int ObPluginVectorIndexAdaptor::refresh_bitmap_background() @@ -3210,7 +3204,7 @@ int ObPluginVectorIndexAdaptor::refresh_bitmap_background() LOG_WARN("fail to get read snapshot version", KR(ret)); } else if (OB_FAIL(ctx.init_bitmaps())) { LOG_WARN("failed to init bitmaps for background bitmap refresh", K(ret)); - } else if (OB_FAIL(complete_index_mem_data_incremental(&ctx, share::SYS_LS, snapshot_scn, i_vids))) { + } else if (OB_FAIL(complete_index_mem_data_incremental(&ctx, snapshot_scn, i_vids))) { LOG_WARN("background bitmap refresh failed", K(ret), K(vbitmap_tablet_id_)); } else { FLOG_INFO("refresh_bitmap_background", K(snapshot_scn), K(i_vids.count())); @@ -4248,8 +4242,7 @@ int ObPluginVectorIndexAdaptor::query_next_result(ObVectorQueryAdaptorResultCont return ret; } -int ObPluginVectorIndexAdaptor::query_result(ObLSID &ls_id, - ObVectorQueryAdaptorResultContext *ctx, +int ObPluginVectorIndexAdaptor::query_result(ObVectorQueryAdaptorResultContext *ctx, ObVectorQueryConditions *query_cond, ObVectorQueryVidIterator *&vids_iter) { @@ -4314,7 +4307,7 @@ int ObPluginVectorIndexAdaptor::query_result(ObLSID &ls_id, if (get_create_type() == CreateTypeComplete) { ctx->status_ = PVQ_REFRESH; LOG_INFO("query result need refresh adapter, ls leader", - K(ret), K(ls_id), K(ctx->get_ls_leader()), K(snapshot_tablet_id_), K(get_snapshot_key_prefix()), K(row->storage_datums_[0].get_string())); + K(ret), K(ctx->get_ls_leader()), K(snapshot_tablet_id_), K(get_snapshot_key_prefix()), K(row->storage_datums_[0].get_string())); } else if (OB_FAIL(deserialize_snap_data(query_cond, row))) { if (ret == OB_ERR_VSAG_RETURN_ERROR) { // snapshot data may be transiently incomplete under concurrent DDL/DML; @@ -4322,7 +4315,7 @@ int ObPluginVectorIndexAdaptor::query_result(ObLSID &ls_id, ctx->status_ = PVQ_REFRESH; ret = OB_SUCCESS; LOG_INFO("deserialize snap data got vsag transient error, mark refresh", - K(ls_id), K(snapshot_tablet_id_), K(get_snapshot_key_prefix()), + K(snapshot_tablet_id_), K(get_snapshot_key_prefix()), K(row->storage_datums_[0].get_string())); } else { LOG_WARN("failed to deserialize snap data", K(ret)); diff --git a/src/observer/vector_index/ob_plugin_vector_index_adaptor.h b/src/observer/vector_index/ob_plugin_vector_index_adaptor.h index 30a3eda31..490e098b9 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_adaptor.h +++ b/src/observer/vector_index/ob_plugin_vector_index_adaptor.h @@ -25,7 +25,6 @@ #include "common/row/ob_row_iterator.h" #include "observer/vector_index/ob_plugin_vector_index_util.h" #include "storage/ob_i_store.h" -#include "share/ob_ls_id.h" #include "share/rc/ob_tenant_base.h" #include "lib/oblog/ob_log_module.h" #include "observer/vector_index/ob_plugin_vector_index_serialize.h" @@ -50,14 +49,12 @@ struct ObVectorIndexInfo void reset(); static const int64_t OB_VECTOR_INDEX_STATISTICS_SIZE = 2048; static const int64_t OB_VECTOR_INDEX_SYNC_INFO_SIZE = 1024; - TO_STRING_KV(K_(ls_id), - K_(rowkey_vid_table_id), K_(vid_rowkey_table_id), K_(inc_index_table_id), + TO_STRING_KV(K_(rowkey_vid_table_id), K_(vid_rowkey_table_id), K_(inc_index_table_id), K_(vbitmap_table_id), K_(snapshot_index_table_id), K_(data_table_id), K_(embedded_table_id), K_(rowkey_vid_tablet_id), K_(vid_rowkey_tablet_id), K_(inc_index_tablet_id), K_(vbitmap_tablet_id), K_(snapshot_index_tablet_id), K_(data_tablet_id), K_(embedded_tablet_id), K_(statistics), K_(sync_info)); public: - int64_t ls_id_; // table_id int64_t rowkey_vid_table_id_; int64_t vid_rowkey_table_id_; @@ -674,13 +671,11 @@ class ObPluginVectorIndexAdaptor int check_index_id_table_readnext_status(ObVectorQueryAdaptorResultContext *ctx, common::ObNewRowIterator *row_iter, SCN query_scn, - bool is_async_mode = false, - ObLSID ls_id = ObLSID()); + bool is_async_mode = false); // Async mode only: no read_scn parsing, uses complete_index_mem_data_incremental / build_temp_bitmap int check_index_id_table_readnext_status_async(ObVectorQueryAdaptorResultContext *ctx, common::ObNewRowIterator *row_iter, - SCN query_scn, - ObLSID ls_id); + SCN query_scn); int build_temp_bitmap_from_index_id_table(ObVectorQueryAdaptorResultContext *ctx, common::ObNewRowIterator *row_iter, SCN query_scn, @@ -688,8 +683,7 @@ class ObPluginVectorIndexAdaptor // Query Processor third int check_snapshot_table_wait_status(ObVectorQueryAdaptorResultContext *ctx); - int query_result(ObLSID &ls_id, - ObVectorQueryAdaptorResultContext *ctx, + int query_result(ObVectorQueryAdaptorResultContext *ctx, ObVectorQueryConditions *query_cond, ObVectorQueryVidIterator *&vids_iter); int deserialize_snap_data(ObVectorQueryConditions *query_cond, blocksstable::ObDatumRow *row); @@ -712,12 +706,9 @@ class ObPluginVectorIndexAdaptor blocksstable::ObDatumRow *last_row, ObArray &i_vids); int complete_index_mem_data_incremental(ObVectorQueryAdaptorResultContext *ctx, - ObLSID ls_id, SCN query_scn, ObArray &i_vids); - // Background bitmap refresh: constructs a minimal synthetic context and calls - // complete_index_mem_data_incremental with SYS_LS (consistent with write_to_vsag_). - // Same table-scan code path as a query-time refresh, but without a live query context. + // Background bitmap refresh uses the same table-scan code path as query-time refresh. int refresh_bitmap_background(); int update_incr_bitmap(const int64_t *vids, int64_t count); int prepare_delta_mem_data(roaring::api::roaring64_bitmap_t *gene_bitmap, diff --git a/src/observer/vector_index/ob_plugin_vector_index_scheduler.cpp b/src/observer/vector_index/ob_plugin_vector_index_scheduler.cpp index c73a2187c..89308b32f 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_scheduler.cpp +++ b/src/observer/vector_index/ob_plugin_vector_index_scheduler.cpp @@ -80,22 +80,12 @@ void ObPluginVectorIndexLoadScheduler::clean_deprecated_adapters() ObSEArray delete_tablet_id_array; delete_tablet_id_array.reset(); - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; - if (OB_FAIL(vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), index_ls_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } + ObPluginVectorIndexMgr *index_mgr = &vector_index_service_->get_index_mgr(); - if (OB_SUCC(ret) && OB_NOT_NULL(index_ls_mgr)) { + if (OB_SUCC(ret)) { { - RWLock::RLockGuard lock_guard(index_ls_mgr->get_adapter_map_lock()); - FOREACH_X(iter, index_ls_mgr->get_complete_adapter_map(), OB_SUCC(ret)) { + RWLock::RLockGuard lock_guard(index_mgr->get_adapter_map_lock()); + FOREACH_X(iter, index_mgr->get_complete_adapter_map(), OB_SUCC(ret)) { ObPluginVectorIndexAdaptor *adapter = iter->second; ObSchemaGetterGuard schema_guard; const ObTableSchema *table_schema; @@ -134,15 +124,15 @@ void ObPluginVectorIndexLoadScheduler::clean_deprecated_adapters() } } if (delete_tablet_id_array.count() > 0) { - LOG_INFO("try erase complete vector index adapter", - K(index_ls_mgr->get_ls_id()), K(delete_tablet_id_array.count())); + LOG_INFO("try erase complete vector index adapter", + K(delete_tablet_id_array.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < delete_tablet_id_array.count(); i++) { - if (OB_FAIL(index_ls_mgr->erase_complete_adapter(delete_tablet_id_array.at(i)))) { + if (OB_FAIL(index_mgr->erase_complete_adapter(delete_tablet_id_array.at(i)))) { if (ret != OB_HASH_NOT_EXIST) { - LOG_WARN("failed to erase full vector index adapter", - K(index_ls_mgr->get_ls_id()), K(delete_tablet_id_array.at(i)), KR(ret)); + LOG_WARN("failed to erase full vector index adapter", + K(delete_tablet_id_array.at(i)), KR(ret)); } else { // already removed ret = OB_SUCCESS; } @@ -151,11 +141,11 @@ void ObPluginVectorIndexLoadScheduler::clean_deprecated_adapters() delete_tablet_id_array.reset(); common::hash::ObHashSet full_partial_adaptor_hash_set; - if (index_ls_mgr->get_partial_adapter_map().size() > 0) { - if (OB_FAIL(full_partial_adaptor_hash_set.create(index_ls_mgr->get_partial_adapter_map().size()))){ - LOG_WARN("fail to create full partial adaptor set failed", KR(ret), K(index_ls_mgr->get_partial_adapter_map().size())); + if (index_mgr->get_partial_adapter_map().size() > 0) { + if (OB_FAIL(full_partial_adaptor_hash_set.create(index_mgr->get_partial_adapter_map().size()))){ + LOG_WARN("fail to create full partial adaptor set failed", KR(ret), K(index_mgr->get_partial_adapter_map().size())); } else { - FOREACH_X(iter, index_ls_mgr->get_partial_adapter_map(), OB_SUCC(ret)) { + FOREACH_X(iter, index_mgr->get_partial_adapter_map(), OB_SUCC(ret)) { ObPluginVectorIndexAdaptor *adapter = iter->second; ObTabletID tablet_id = iter->first; ObTabletHandle tablet_handle; @@ -208,15 +198,15 @@ void ObPluginVectorIndexLoadScheduler::clean_deprecated_adapters() } if (delete_tablet_id_array.count() > 0) { - LOG_INFO("try erase partial vector index adapter", - K(index_ls_mgr->get_ls_id()), K(delete_tablet_id_array.count())); + LOG_INFO("try erase partial vector index adapter", + K(delete_tablet_id_array.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < delete_tablet_id_array.count(); i++) { - if (OB_FAIL(index_ls_mgr->erase_partial_adapter(delete_tablet_id_array.at(i)))) { + if (OB_FAIL(index_mgr->erase_partial_adapter(delete_tablet_id_array.at(i)))) { if (ret != OB_HASH_NOT_EXIST) { - LOG_WARN("failed to erase full vector index adapter", - K(index_ls_mgr->get_ls_id()), K(delete_tablet_id_array.at(i)), KR(ret)); + LOG_WARN("failed to erase full vector index adapter", + K(delete_tablet_id_array.at(i)), KR(ret)); } else { // already removed ret = OB_SUCCESS; } @@ -308,7 +298,6 @@ int ObPluginVectorIndexLoadScheduler::acquire_adapter_in_maintenance( { int ret = OB_SUCCESS; ObIndexType index_type = table_schema->get_index_type(); - ObLSID ls_id = ls_->get_ls_id(); ObArray tablet_ids; if (OB_FAIL(table_schema->get_tablet_ids(tablet_ids))) { @@ -336,22 +325,21 @@ int ObPluginVectorIndexLoadScheduler::acquire_adapter_in_maintenance( ObTabletID data_tablet_id = tablet_handle.get_obj()->get_data_tablet_id(); if (OB_FAIL(ObVectorIndexUtil::get_vector_index_column_dim(*table_schema, dim))) { LOG_WARN("fail to get vec_index_col_param", K(ret)); - } else if (OB_FAIL(vector_index_service_->acquire_adapter_guard(ls_id, - tablet_ids.at(i), + } else if (OB_FAIL(vector_index_service_->acquire_adapter_guard(tablet_ids.at(i), index_type, adapter_guard, &vec_idx_params, dim))) { - LOG_WARN("fail to acquire adapter gurad", K(ret), K(ls_id)); + LOG_WARN("fail to acquire adapter gurad", K(ret)); } else if (adapter_guard.get_adatper()->is_complete()) { // get create type ? // already exist full adapter, bypass } else if (OB_FAIL(adapter_guard.get_adatper()-> set_table_id(ObPluginVectorIndexUtils::index_type_to_record_type(index_type), table_id))) { - LOG_WARN("fail to set table id", K(ret), K(ls_id), K(tablet_ids.at(i))); + LOG_WARN("fail to set table id", K(ret), K(tablet_ids.at(i))); } else if (OB_FAIL(adapter_guard.get_adatper()->set_tablet_id(VIRT_DATA, data_tablet_id))) { - LOG_WARN("fail to fill partial index adapter info", K(ret), K(ls_id), K(tablet_ids.at(i)), K(data_tablet_id)); + LOG_WARN("fail to fill partial index adapter info", K(ret), K(tablet_ids.at(i)), K(data_tablet_id)); } else if (OB_FAIL(adapter_guard.get_adatper()->set_table_id(VIRT_DATA, table_schema->get_data_table_id()))) { - LOG_WARN("fail to fill data table id into partial adapter", K(ret), K(ls_id), K(table_id), K(data_tablet_id)); + LOG_WARN("fail to fill data table id into partial adapter", K(ret), K(table_id), K(data_tablet_id)); } else if (OB_FAIL(ObPluginVectorIndexUtils::get_vector_index_prefix(*table_schema, index_identity))) { LOG_WARN("fail to get index identity", KR(ret)); @@ -527,7 +515,7 @@ int ObPluginVectorIndexLoadScheduler::execute_adapter_maintenance(ObIArraycheck_and_merge_adapter(ls_->get_ls_id(), shared_table_info_map))) { + } else if (OB_FAIL(vector_index_service_->check_and_merge_adapter(shared_table_info_map))) { LOG_WARN("fail to merge parital adapter task", KR(ret)); } else { mark_tenant_checked(); @@ -619,11 +607,11 @@ int ObPluginVectorIndexLoadScheduler::reload_tenant_task(bool &has_ivf_index) int ret = OB_SUCCESS; ObVectorIndexTenantStatus tenant_task; ObVectorIndexTaskStatus expected_state; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObPluginVectorIndexLoadScheduler not init", KR(ret)); - } else if (OB_FAIL(get_ls_mgr(index_ls_mgr))) { + } else if (OB_FAIL(get_index_mgr(index_mgr))) { LOG_WARN("fail to get ls mgr", K(ret)); } else if (OB_FAIL(read_tenant_task_status(NULL, tenant_task))) { LOG_WARN("fail to read vector index tenant task", KR(ret)); @@ -638,18 +626,17 @@ int ObPluginVectorIndexLoadScheduler::reload_tenant_task(bool &has_ivf_index) LOG_WARN("get invalid vector index tenant task status", KR(ret), K(tenant_task.status_), K(expected_state), K(local_tenant_task_)); } else { - if (OB_ISNULL(index_ls_mgr)) { - } else if (index_ls_mgr->get_ls_task_ctx().state_ != expected_state) { + if (index_mgr->get_task_ctx().state_ != expected_state) { if (expected_state != OB_TTL_TASK_RUNNING) { - FLOG_INFO("vector index schedular is not running now", KR(ret), K(index_ls_mgr->get_ls_task_ctx())); + FLOG_INFO("vector index schedular is not running now", KR(ret), K(index_mgr->get_task_ctx())); } // currently, only finish/running vs running // if change from running to finish/cancel release context - index_ls_mgr->get_ls_task_ctx().reuse(); - index_ls_mgr->get_ls_task_ctx().task_id_++; // not used, ++ if overall task status changed - index_ls_mgr->get_ls_task_ctx().need_check_ = true; + index_mgr->get_task_ctx().reuse(); + index_mgr->get_task_ctx().task_id_++; // not used, ++ if overall task status changed + index_mgr->get_task_ctx().need_check_ = true; // all finish - index_ls_mgr->get_ls_task_ctx().state_ = expected_state; + index_mgr->get_task_ctx().state_ = expected_state; } } // vector index async task @@ -676,25 +663,25 @@ int ObPluginVectorIndexLoadScheduler::execute_one_memdata_sync_task(ObPluginVect LOG_WARN("memdata load vector index adapter or ctx is null", KR(ret), KPC(mgr), KPC(task_ctx)); } else { common::ObSpinLockGuard ctx_guard(task_ctx->lock_); - if (task_ctx->task_status_ != mgr->get_ls_task_ctx().state_) { + if (task_ctx->task_status_ != mgr->get_task_ctx().state_) { // only pending task could be changed to running, reuse ttl task status - if (OB_TTL_TASK_RUNNING == mgr->get_ls_task_ctx().state_) { + if (OB_TTL_TASK_RUNNING == mgr->get_task_ctx().state_) { if (OB_TTL_TASK_PREPARE == task_ctx->task_status_) { try_schedule = true; } else if (OB_TTL_TASK_FINISH == task_ctx->task_status_ || OB_TTL_TASK_CANCEL == task_ctx->task_status_) { // do nothing - LOG_INFO("memdata load task finish or cancelled", K(mgr->get_ls_task_ctx()), KPC(task_ctx)); + LOG_INFO("memdata load task finish or cancelled", K(mgr->get_task_ctx()), KPC(task_ctx)); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("memdata load no expected task status", KR(ret), K(mgr->get_ls_task_ctx()), KPC(task_ctx)); + LOG_WARN("memdata load no expected task status", KR(ret), K(mgr->get_task_ctx()), KPC(task_ctx)); } } else { // ls not running ret = OB_ERR_UNEXPECTED; LOG_WARN("memdata load unexpect ls task status", KR(ret), KPC(mgr), KPC(task_ctx)); } } else { // if is running do nothing, if not need schedular. - LOG_INFO("nmemdata load o need to schedule task", K(mgr->get_ls_task_ctx()), KPC(task_ctx)); + LOG_INFO("nmemdata load o need to schedule task", K(mgr->get_task_ctx()), KPC(task_ctx)); } if (OB_SUCC(ret) @@ -919,7 +906,7 @@ int ObPluginVectorIndexLoadScheduler::check_task_state(ObPluginVectorIndexMgr *m return ret; } -int ObPluginVectorIndexLoadScheduler::check_ls_task_state(ObPluginVectorIndexMgr *mgr) +int ObPluginVectorIndexLoadScheduler::check_task_state(ObPluginVectorIndexMgr *mgr) { int ret = OB_SUCCESS; // mgr cannot be null here. @@ -928,20 +915,18 @@ int ObPluginVectorIndexLoadScheduler::check_ls_task_state(ObPluginVectorIndexMgr uint32_t finished_count = 0; if (OB_ISNULL(mgr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null mgr", K(ls_->get_ls_id())); + LOG_WARN("get null mgr"); } else if (OB_FAIL(mgr->get_mem_sync_info().count_processing_finished(processing_finished, total_count, finished_count))) { LOG_WARN("failed to count finished tasks", KR(ret)); } else if (processing_finished) { - ObPluginVectorIndexLSTaskCtx &ls_task_ctx = mgr->get_ls_task_ctx(); - ls_task_ctx.all_finished_ = true; - ls_task_ctx.need_memdata_sync_ = false; - LOG_INFO("memdata sync all task finished", - K(ls_->get_ls_id()), K(finished_count), K(total_count)); + ObPluginVectorIndexScheduleCtx &task_ctx = mgr->get_task_ctx(); + task_ctx.all_finished_ = true; + task_ctx.need_memdata_sync_ = false; + LOG_INFO("memdata sync all task finished", K(finished_count), K(total_count)); } else { - LOG_INFO("memdata sync task remaining", - K(ls_->get_ls_id()), K(finished_count), K(total_count)); + LOG_INFO("memdata sync task remaining", K(finished_count), K(total_count)); } return ret; } @@ -961,15 +946,8 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_adapter_maintenance_task if (OB_FAIL(execute_adapter_maintenance(vec_table_id_array))) { LOG_WARN("fail to generate tablet tasks"); } - int tmp_ret = OB_SUCCESS; if (OB_ISNULL(mgr)) { - tmp_ret = vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), mgr); - if (tmp_ret == OB_SUCCESS) { - } else if (tmp_ret == OB_HASH_NOT_EXIST) { - tmp_ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(tmp_ret), K(ls_->get_ls_id())); - } + mgr = &vector_index_service_->get_index_mgr(); } if (OB_NOT_NULL(mgr)) { mgr->dump_all_inst(); // for debug, remove later @@ -1001,18 +979,18 @@ int ObPluginVectorIndexLoadScheduler::log_tablets_need_memdata_sync(ObPluginVect bool need_sync = false; if (OB_ISNULL(adapter)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null adapter", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("get null adapter", KR(ret)); } else if (iter->first != adapter->get_inc_tablet_id()) { // do nothing } else if (tablet_id_array_.count() >= ObVectorIndexSyncLogCb::VECTOR_INDEX_MAX_SYNC_COUNT) { // do nothing, wait for next schedule } else if (!need_refresh_ && OB_FAIL(adapter->check_need_sync_to_follower_or_do_opt_task(need_sync))) { - LOG_WARN("fail to check need memdata sync", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to check need memdata sync", KR(ret)); } else if ((need_refresh_ || need_sync) && is_leader_) { if (OB_FAIL(tablet_id_array_.push_back(iter->first))) { - LOG_WARN("fail to push tablet id need memdata sync", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to push tablet id need memdata sync", KR(ret)); } else if (OB_FAIL(table_id_array_.push_back(adapter->get_inc_table_id()))) { - LOG_WARN("fail to push table id need memdata sync", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to push table id need memdata sync", KR(ret)); } else { need_submit_log = true; } @@ -1031,9 +1009,9 @@ int ObPluginVectorIndexLoadScheduler::log_tablets_need_memdata_sync(ObPluginVect TRANS_LOG(WARN, "fail to add complete adaptor to waiting map",KR(ret)); } } else if (OB_FAIL(submit_log_())) { - TRANS_LOG(WARN, "fail to submit vector index memdata sync log",KR(ret), K(ls_->get_ls_id())); + TRANS_LOG(WARN, "fail to submit vector index memdata sync log",KR(ret)); } else { - TRANS_LOG(INFO, "submit vector index memdata sync log success", KR(ret), K(need_refresh_), K(ls_->get_ls_id())); + TRANS_LOG(INFO, "submit vector index memdata sync log success", KR(ret), K(need_refresh_)); } } else if (!is_leader_ && need_refresh_) { if (OB_FAIL(mgr->get_mem_sync_info().add_task_to_waiting_map(mgr->get_complete_adapter_map()))) { @@ -1050,7 +1028,7 @@ int ObPluginVectorIndexLoadScheduler::execute_all_memdata_sync_task(ObPluginVect int ret = OB_SUCCESS; if (OB_ISNULL(mgr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null mgr", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("get null mgr", KR(ret)); } else { // other threads will not process current map, it is save to just use iter VectorIndexMemSyncMap ¤t_map = mgr->get_mem_sync_info().get_processing_map(); @@ -1072,22 +1050,22 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_memdata_sync_task(ObPlug bool force_mem_data_sync = false; if (OB_ISNULL(mgr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null mgr", KR(ret), K(ls_->get_ls_id())); + LOG_WARN("get null mgr", KR(ret)); } else if (OB_FAIL(mgr->check_need_mem_data_sync_task(need_mem_data_sync))) { LOG_WARN("fail to check need mem data sync task", - KR(ret), K(mgr->get_ls_task_ctx()), K(ls_->get_ls_id())); + KR(ret), K(mgr->get_task_ctx())); } else if (need_mem_data_sync) { - mgr->get_ls_task_ctx().non_memdata_task_cycle_ = 0; - mgr->get_ls_task_ctx().need_memdata_sync_ = true; + mgr->get_task_ctx().non_memdata_task_cycle_ = 0; + mgr->get_task_ctx().need_memdata_sync_ = true; } else { - mgr->get_ls_task_ctx().non_memdata_task_cycle_++; - if (mgr->get_ls_task_ctx().non_memdata_task_cycle_ - > ObPluginVectorIndexLSTaskCtx::NON_MEMDATA_TASK_CYCLE_MAX) { - mgr->get_ls_task_ctx().non_memdata_task_cycle_ = 0; + mgr->get_task_ctx().non_memdata_task_cycle_++; + if (mgr->get_task_ctx().non_memdata_task_cycle_ + > ObPluginVectorIndexScheduleCtx::NON_MEMDATA_TASK_CYCLE_MAX) { + mgr->get_task_ctx().non_memdata_task_cycle_ = 0; // disable force sync currently - // mgr->get_ls_task_ctx().need_memdata_sync_ = true; + // mgr->get_task_ctx().need_memdata_sync_ = true; // force_mem_data_sync = true; - FLOG_INFO("not receive any sync task log", K(ls_->get_ls_id())); + FLOG_INFO("not receive any sync task log"); } } @@ -1102,7 +1080,7 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_memdata_sync_task(ObPlug } } - if (OB_SUCC(ret) && mgr->get_ls_task_ctx().need_memdata_sync_) { + if (OB_SUCC(ret) && mgr->get_task_ctx().need_memdata_sync_) { if (OB_FAIL(execute_all_memdata_sync_task(mgr))) { if (OB_SIZE_OVERFLOW != ret) { LOG_WARN("fail to try schedule memedata_sync dag task", KR(ret)); @@ -1110,7 +1088,7 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_memdata_sync_task(ObPlug ret = OB_SUCCESS; } } - check_ls_task_state(mgr); + check_task_state(mgr); } else { // do nothing } @@ -1118,22 +1096,14 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_memdata_sync_task(ObPlug return ret; } -int ObPluginVectorIndexLoadScheduler::get_ls_mgr(ObPluginVectorIndexMgr *&index_ls_mgr) +int ObPluginVectorIndexLoadScheduler::get_index_mgr(ObPluginVectorIndexMgr *&index_mgr) { int ret = OB_SUCCESS; - index_ls_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("tablet ttl manager not init", KR(ret)); - } else if (OB_FAIL(vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), index_ls_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_->get_ls_id())); + } else { + index_mgr = &vector_index_service_->get_index_mgr(); } return ret; } @@ -1157,25 +1127,24 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_tasks(ObIArray int ret = OB_SUCCESS; ObTimeGuard guard("ObPluginVectorIndexLoadScheduler::check_and_handle_event", VEC_INDEX_LOAD_TIME_NORMAL_THRESHOLD); - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("tablet ttl manager not init", KR(ret)); - } else if (OB_FAIL(get_ls_mgr(index_ls_mgr))) { + } else if (OB_FAIL(get_index_mgr(index_mgr))) { LOG_WARN("fail to get ls mgr", K(ret)); } if (OB_FAIL(ret)) { - } else if (OB_NOT_NULL(index_ls_mgr) && index_ls_mgr->get_ls_task_ctx().state_ != OB_TTL_TASK_RUNNING) { + } else if (index_mgr->get_task_ctx().state_ != OB_TTL_TASK_RUNNING) { // do nothing, ToDo: change log level later LOG_INFO("not vector index schedular running", - K(index_ls_mgr->get_ls_task_ctx().state_), K(ls_->get_ls_id())); + K(index_mgr->get_task_ctx().state_)); } else { - // Notice: index_ls_mgr maybe null // create / remove adapter, check need update & write mem sync log - if (can_schedule(ObVectorTaskScheduleType::ADAPTER_MAINTENANCE) && OB_FAIL(check_and_execute_adapter_maintenance_task(index_ls_mgr, vec_table_id_array))) { // Tips: do merge + if (can_schedule(ObVectorTaskScheduleType::ADAPTER_MAINTENANCE) && OB_FAIL(check_and_execute_adapter_maintenance_task(index_mgr, vec_table_id_array))) { // Tips: do merge LOG_WARN("fail to check and execute adapter maintenance task", - KR(ret), K(ls_->get_ls_id())); + KR(ret)); } // Notice: leader write sync log, do memdata_sync only one loop(role changed from follower to leader) // explicit cover error code @@ -1188,19 +1157,19 @@ int ObPluginVectorIndexLoadScheduler::check_and_execute_tasks(ObIArray } // write tablets need memdata sync to clog if (OB_FAIL(ret)) { - } else if (OB_NOT_NULL(index_ls_mgr) - && (current_memory_config_ != 0) + } else if ((current_memory_config_ != 0) && can_schedule(ObVectorTaskScheduleType::FOLLOWER_SYNC) - && OB_FAIL(log_tablets_need_memdata_sync(index_ls_mgr))) { // Tips: check if need check to follower - LOG_ERROR("fail to log tablets need memdata sync", KR(ret), K(ls_->get_ls_id())); + && OB_FAIL(log_tablets_need_memdata_sync(index_mgr))) { // Tips: check if need check to follower + LOG_ERROR("fail to log tablets need memdata sync", KR(ret)); } // explicit cover error code ret = OB_SUCCESS; // mem_sync task - if (can_schedule(ObVectorTaskScheduleType::FOLLOWER_SYNC) && OB_NOT_NULL(index_ls_mgr) && OB_FAIL(check_and_execute_memdata_sync_task(index_ls_mgr))) { + if (can_schedule(ObVectorTaskScheduleType::FOLLOWER_SYNC) + && OB_FAIL(check_and_execute_memdata_sync_task(index_mgr))) { LOG_WARN("fail to check and execute memdata sync task", - KR(ret), K(ls_->get_ls_id())); + KR(ret)); } } return ret; @@ -1232,7 +1201,7 @@ void ObPluginVectorIndexLoadScheduler::run_task() } else if (ATOMIC_BCAS(&need_do_for_switch_, true, false)) { // reserved, do nothing int tmp_ret = OB_SUCCESS; - LOG_INFO("switch leader", K(ls_->get_ls_id()), K(is_leader_), K(is_stopped_)); + LOG_INFO("switch leader", K(is_leader_), K(is_stopped_)); if (!check_can_do_work() || is_stopped() || !is_leader_) { // skip } else if (OB_TMP_FAIL(resume_task_executors())) { LOG_WARN("fail to resume async task", K(tmp_ret)); @@ -1321,8 +1290,7 @@ int ObPluginVectorIndexLoadScheduler::submit_log_() KR(ret), KP(cb_.log_buffer_), K(pos)); } else { is_logging_ = true; - TRANS_LOG(INFO, "submit vector index memdata sync log success", - K(ls_->get_ls_id()), K(base_scn), K(lsn), K(scn)); + TRANS_LOG(INFO, "submit vector index memdata sync log success", K(base_scn), K(lsn), K(scn)); } tablet_id_array_.reuse(); table_id_array_.reuse(); @@ -1335,22 +1303,16 @@ int ObPluginVectorIndexLoadScheduler::handle_submit_callback(const bool success) int ret = OB_SUCCESS; common::ObSpinLockGuard ctx_guard(logging_lock_); is_logging_ = false; - TRANS_LOG(INFO, "submit vector index memdata sync log success", - K(ls_->get_ls_id()), K(success)); + TRANS_LOG(INFO, "submit vector index memdata sync log success", K(success)); return ret; } int ObPluginVectorIndexLoadScheduler::handle_replay_result(ObVectorIndexSyncLog &ls_log) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *mgr = nullptr; - if (OB_FAIL(vector_index_service_->acquire_vector_index_mgr(ls_->get_ls_id(), mgr))) { - LOG_WARN("fail to acquire vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } else if (OB_ISNULL(mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } else if (OB_FAIL(mgr->get_mem_sync_info().add_task_to_waiting_map(ls_log))){ - LOG_WARN("memdata sync failed to add task", KR(ret), K(ls_->get_ls_id())); + ObPluginVectorIndexMgr *mgr = &vector_index_service_->get_index_mgr(); + if (OB_FAIL(mgr->get_mem_sync_info().add_task_to_waiting_map(ls_log))){ + LOG_WARN("memdata sync failed to add task", KR(ret)); } if (ret == OB_ALLOCATE_MEMORY_FAILED) { ret = OB_SUCCESS; @@ -1388,17 +1350,10 @@ int ObPluginVectorIndexLoadScheduler::replay(const void *buffer, void ObPluginVectorIndexLoadScheduler::refresh_adapter_rb_flag() { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; - if (OB_FAIL(vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), index_ls_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } - } - if (OB_SUCC(ret) && OB_NOT_NULL(index_ls_mgr)) { - RWLock::RLockGuard lock_guard(index_ls_mgr->get_adapter_map_lock()); - FOREACH_X(iter, index_ls_mgr->get_complete_adapter_map(), OB_SUCC(ret)) { + ObPluginVectorIndexMgr *index_mgr = &vector_index_service_->get_index_mgr(); + if (OB_SUCC(ret)) { + RWLock::RLockGuard lock_guard(index_mgr->get_adapter_map_lock()); + FOREACH_X(iter, index_mgr->get_complete_adapter_map(), OB_SUCC(ret)) { ObPluginVectorIndexAdaptor *adapter = iter->second; if (OB_ISNULL(adapter->get_snap_data_()) || !adapter->get_snap_data_()->is_inited()) { LOG_INFO("snap_data index is empty or not init, won't set rb_flag"); @@ -1408,7 +1363,7 @@ void ObPluginVectorIndexLoadScheduler::refresh_adapter_rb_flag() snap_memdata->rb_flag_ = true; } } - LOG_INFO("finish refresh adapter rb flag", K(ret), K(ls_->get_ls_id())); + LOG_INFO("finish refresh adapter rb flag", K(ret)); } } @@ -1426,7 +1381,7 @@ share::SCN ObPluginVectorIndexLoadScheduler::get_rec_scn() // role change interfaces -int ObPluginVectorIndexLoadScheduler::switch_to_leader() +int ObPluginVectorIndexLoadScheduler::activate() { int64_t start_time_us = ObTimeUtility::current_time(); FLOG_INFO("vector index scheduler: begin to switch_to_leader", KPC_(ls), K(start_time_us)); @@ -1439,7 +1394,7 @@ int ObPluginVectorIndexLoadScheduler::switch_to_leader() ATOMIC_STORE(&need_do_for_switch_, true); } if (OB_SUCC(ret) && check_can_do_work()) { - (void) ObPluginVectorIndexUtils::set_ls_leader_flag(ls_->get_ls_id(), is_leader_); + ObPluginVectorIndexUtils::set_leader_flag(is_leader_); refresh_adapter_rb_flag(); } const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; @@ -1447,14 +1402,7 @@ int ObPluginVectorIndexLoadScheduler::switch_to_leader() return ret; } -int ObPluginVectorIndexLoadScheduler::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - inner_switch_to_follower_(); - return ret; -} - -void ObPluginVectorIndexLoadScheduler::switch_to_follower_forcedly() +void ObPluginVectorIndexLoadScheduler::deactivate() { inner_switch_to_follower_(); } @@ -1465,65 +1413,24 @@ void ObPluginVectorIndexLoadScheduler::inner_switch_to_follower_() const int64_t start_time_us = ObTimeUtility::current_time(); ATOMIC_STORE(&is_leader_, false); ATOMIC_STORE(&need_do_for_switch_, true); - (void) ObPluginVectorIndexUtils::set_ls_leader_flag(ls_->get_ls_id(), is_leader_); + ObPluginVectorIndexUtils::set_leader_flag(is_leader_); const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; FLOG_INFO("vector index scheduler: finish to switch_to_follower", KPC_(ls), K(cost_us)); } -int ObPluginVectorIndexLoadScheduler::safe_to_destroy(bool &is_safe) -{ - int ret = OB_SUCCESS; - is_safe = true; - - int64_t async_task_ref = 0; - if (OB_NOT_NULL(vector_index_service_)) { - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; - if (OB_FAIL(vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), index_ls_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } - } else if (OB_NOT_NULL(index_ls_mgr)) { - async_task_ref = index_ls_mgr->get_async_task_opt().get_ls_processing_task_cnt(); - if (0 == async_task_ref) { - ObVecIndexTaskCtxArray task_ctx_array; - ObAsyncTaskMapFunc task_map_func(task_ctx_array); - if (OB_FAIL(index_ls_mgr->get_async_task_opt().get_async_task_map().foreach_refactored(task_map_func))) { - LOG_WARN("fail to foreach adapter map", KR(ret)); - } else if (OB_FAIL(async_task_exec_.clear_task_ctxs(index_ls_mgr->get_async_task_opt(), task_ctx_array))) { - LOG_WARN("fail to clean task ctx", K(ret), K(task_ctx_array)); - } - } - } - } - - int64_t dag_ref = get_dag_ref(); - if (0 != dag_ref || 0 != async_task_ref) { - if (REACH_TIME_INTERVAL(60L * 1000000)) { // 60s - LOG_WARN("vector index scheduler can't destroy", K(dag_ref)); - } - is_safe = false; - } - return ret; -} - void ObPluginVectorIndexLoadScheduler::stop() { int ret = OB_SUCCESS; is_stopped_= true; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (OB_NOT_NULL(vector_index_service_) && OB_NOT_NULL(ls_)) { - if (OB_FAIL(vector_index_service_->get_ls_index_mgr_map().get_refactored(ls_->get_ls_id(), index_ls_mgr))) { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_->get_ls_id())); - } else if (OB_NOT_NULL(index_ls_mgr)) { - index_ls_mgr->get_async_task_opt().set_stop(); - } + index_mgr = &vector_index_service_->get_index_mgr(); + index_mgr->get_async_task_opt().set_stop(); } if (OB_NOT_NULL(ls_)) { - FLOG_INFO("vector index task scheduler stop", K(ls_->get_ls_id()), KP(index_ls_mgr)); + FLOG_INFO("vector index task scheduler stop", KP(index_mgr)); } else { - FLOG_INFO("vector index task scheduler stop, ls is null", KP(index_ls_mgr)); + FLOG_INFO("vector index task scheduler stop, ls is null", KP(index_mgr)); } }; @@ -1547,7 +1454,6 @@ int ObVectorIndexDag::init(ObPluginVectorIndexMgr *mgr, ObPluginVectorIndexTaskC } else { compat_mode_ = lib::Worker::CompatMode::MYSQL; // only support mysql now - param_.ls_id_ = mgr->get_ls_id(); param_.table_id_ = task_ctx->index_table_id_; param_.tablet_id_ = task_ctx->index_tablet_id_; param_.task_ctx_ = task_ctx; @@ -1579,7 +1485,6 @@ uint64_t ObVectorIndexDag::hash() const if (OB_UNLIKELY(!is_inited_ || !param_.is_valid())) { LOG_ERROR_RET(OB_ERR_SYS, "invalid argument", K(is_inited_), K_(param)); } else { - hash_value += param_.ls_id_.hash(); hash_value += param_.tablet_id_.hash(); } return hash_value; @@ -1607,7 +1512,6 @@ int ObVectorIndexDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, ret = OB_NOT_INIT; LOG_WARN("ObVectorIndexDag has not been initialized", K(is_inited_), K_(param)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - static_cast(param_.ls_id_.id()), static_cast(param_.table_id_), static_cast(param_.tablet_id_.id())))) { LOG_WARN("fail to fill info param", KR(ret), K_(param)); @@ -1628,7 +1532,6 @@ int ObVectorIndexTask::init(ObPluginVectorIndexLoadScheduler *schedular, LOG_WARN("invalid argument", KR(ret), KP(schedular), KP(mgr), KP(task_ctx)); } else { - ls_id_ = mgr->get_ls_id(); vec_idx_scheduler_ = schedular; vec_idx_mgr_ = mgr; task_ctx_ = task_ctx; @@ -1662,12 +1565,12 @@ int ObVectorIndexTask::process() .set_properties(lib::USE_TL_PAGE_OPTIONAL); CREATE_WITH_TEMP_CONTEXT(param) { if (OB_FAIL(process_one())) { - LOG_WARN("fail to process one", KR(ret), K(ls_id_), KPC(task_ctx_)); + LOG_WARN("fail to process one", KR(ret), KPC(task_ctx_)); } ret = OB_SUCCESS; // continue to try schedular remainig tasks if (OB_FAIL(vec_idx_scheduler_->check_task_state(vec_idx_mgr_, task_ctx_, need_stop))) { - LOG_WARN("fail to check task state", KR(ret), K(ls_id_), KPC(task_ctx_)); + LOG_WARN("fail to check task state", KR(ret), KPC(task_ctx_)); ret = OB_SUCCESS; // cover memdata sync failure } } @@ -1684,17 +1587,16 @@ int ObVectorIndexTask::process_one() ObPluginVectorIndexAdapterGuard adpt_guard; ObPluginVectorIndexAdapterGuard new_adpt_guard; - if (OB_FAIL(ObPluginVectorIndexUtils::get_task_read_snapshot(ls_id_, read_snapshot_))) { - LOG_WARN("memdata sync fail to get task read snapshot", KR(ret), K(ls_id_), KPC(task_ctx_)); + if (OB_FAIL(ObPluginVectorIndexUtils::get_task_read_snapshot(read_snapshot_))) { + LOG_WARN("memdata sync fail to get task read snapshot", KR(ret), KPC(task_ctx_)); } else if (OB_FAIL(vec_idx_mgr_->get_adapter_inst_guard(task_ctx_->index_tablet_id_, adpt_guard))) { - LOG_WARN("memdata sync fail to get adapter instance", KR(ret), K(ls_id_), KPC(task_ctx_)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::refresh_memdata(ls_id_, - adpt_guard.get_adatper(), + LOG_WARN("memdata sync fail to get adapter instance", KR(ret), KPC(task_ctx_)); + } else if (OB_FAIL(ObPluginVectorIndexUtils::refresh_memdata(adpt_guard.get_adatper(), read_snapshot_, allocator_))) { - LOG_WARN("memdata sync fail to refresh memdata", KR(ret), K(ls_id_), KPC(task_ctx_)); + LOG_WARN("memdata sync fail to refresh memdata", KR(ret), KPC(task_ctx_)); } else if (OB_FAIL(vec_idx_mgr_->get_adapter_inst_guard(task_ctx_->index_tablet_id_, new_adpt_guard))) { - LOG_WARN("memdata sync fail to get adapter instance", KR(ret), K(ls_id_), KPC(task_ctx_)); + LOG_WARN("memdata sync fail to get adapter instance", KR(ret), KPC(task_ctx_)); } if (OB_SUCC(ret)) { @@ -1716,21 +1618,21 @@ int ObVectorIndexTask::process_one() int64_t cost = ObTimeUtil::current_time() - start_time; LOG_INFO("memdata sync finish process one", K(cost), K(allocator_.used()), K(allocator_.total()), - K(ls_id_), KPC(task_ctx_)); + KPC(task_ctx_)); allocator_.reset(); LOG_INFO("memdata sync check allocator use", K(allocator_.used()), K(allocator_.total()), - K(ls_id_), KPC(task_ctx_)); + KPC(task_ctx_)); return ret; } -int ObVectorIndexMemSyncInfo::init(int64_t hash_capacity, ObLSID &ls_id) +int ObVectorIndexMemSyncInfo::init(int64_t hash_capacity) { int ret = OB_SUCCESS; if (OB_FAIL(first_mem_sync_map_.create(hash_capacity, "VecIdxTaskMap", "VecIdxTaskMap"))) { - LOG_WARN("fail to create first mem sync set", K(ls_id), KR(ret)); + LOG_WARN("fail to create first mem sync set", KR(ret)); } else if (OB_FAIL(second_mem_sync_map_.create(hash_capacity, "VecIdxTaskMap", "VecIdxTaskMap"))) { - LOG_WARN("fail to create second mem sync set", K(ls_id), K(ls_id), KR(ret)); + LOG_WARN("fail to create second mem sync set", KR(ret)); } return ret; } diff --git a/src/observer/vector_index/ob_plugin_vector_index_scheduler.h b/src/observer/vector_index/ob_plugin_vector_index_scheduler.h index 023d3480a..18da7b1fc 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_scheduler.h +++ b/src/observer/vector_index/ob_plugin_vector_index_scheduler.h @@ -16,7 +16,6 @@ #ifndef OCEANBASE_OBSERVER_OB_PLUGIN_VECTOR_INDEX_SCHEDULER_DEFINE_H_ #define OCEANBASE_OBSERVER_OB_PLUGIN_VECTOR_INDEX_SCHEDULER_DEFINE_H_ -#include "share/ob_ls_id.h" #include "share/scn.h" #include "lib/lock/ob_recursive_mutex.h" #include "share/rc/ob_tenant_base.h" @@ -157,8 +156,8 @@ class ObPluginVectorIndexTenantTaskCtx ObVectorIndexTaskStatus state_; }; -// task context of a ls -struct ObPluginVectorIndexLSTaskCtx +// Scheduling state shared by the tenant's vector-index tablet tasks. +struct ObPluginVectorIndexScheduleCtx { void reuse() { @@ -208,7 +207,7 @@ typedef hash::ObHashMap ObVecIdxShared class ObPluginVectorIndexLoadScheduler : public common::ObTimerTask, public logservice::ObIReplaySubHandler, public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler + public logservice::ObILocalLogHandler { public: ObPluginVectorIndexLoadScheduler() @@ -271,7 +270,7 @@ class ObPluginVectorIndexLoadScheduler : public common::ObTimerTask, int log_tablets_need_memdata_sync(ObPluginVectorIndexMgr *mgr); int execute_all_memdata_sync_task(ObPluginVectorIndexMgr *mgr); int execute_one_memdata_sync_task(ObPluginVectorIndexMgr *mgr, ObPluginVectorIndexTaskCtx *ctx); - int check_ls_task_state(ObPluginVectorIndexMgr *mgr); + int check_task_state(ObPluginVectorIndexMgr *mgr); int check_has_vector_index(bool &has_ivf_index, ObIArray &vec_table_id_array); // task generation interfaces @@ -280,9 +279,8 @@ class ObPluginVectorIndexLoadScheduler : public common::ObTimerTask, int try_schedule_task(ObPluginVectorIndexMgr *mgr, ObPluginVectorIndexTaskCtx *task_ctx); int try_schedule_remaining_tasks(ObPluginVectorIndexMgr *mgr, ObPluginVectorIndexTaskCtx *current_ctx); int generate_vec_idx_memdata_dag(ObPluginVectorIndexMgr *mgr, ObPluginVectorIndexTaskCtx *task_ctx); - int get_ls_mgr(ObPluginVectorIndexMgr *&mgr); + int get_index_mgr(ObPluginVectorIndexMgr *&mgr); void refresh_adapter_rb_flag(); - void set_ls_leader_flag(const bool is_leader); // logger interfaces int handle_submit_callback(const bool success); @@ -294,10 +292,8 @@ class ObPluginVectorIndexLoadScheduler : public common::ObTimerTask, share::SCN get_rec_scn(); // role change interfaces - int switch_to_follower_gracefully(); - void switch_to_follower_forcedly(); - int resume_leader() { return OB_SUCCESS; } - int switch_to_leader(); + void deactivate() override; + int activate() override; // task save destory void stop(); @@ -307,8 +303,6 @@ class ObPluginVectorIndexLoadScheduler : public common::ObTimerTask, void dec_dag_ref() { ATOMIC_DEC(&dag_ref_cnt_); } int64_t get_dag_ref() const { return ATOMIC_LOAD(&dag_ref_cnt_); } - int safe_to_destroy(bool &is_safe); - TO_STRING_KV(K_(is_inited), K_(is_leader), K_(need_do_for_switch), K_(is_stopped), K_(is_logging), K_(need_refresh), K_(interval_factor), K_(basic_period), K_(current_memory_config), K_(dag_ref_cnt), @@ -385,7 +379,6 @@ class ObVectorIndexTask : public share::ObITask ObVectorIndexTask() : ObITask(ObITaskType::TASK_TYPE_VECTOR_INDEX_MEMDATA_SYNC), is_inited_(false), - ls_id_(share::ObLSID::INVALID_LS_ID), vec_idx_scheduler_(nullptr), vec_idx_mgr_(nullptr), task_ctx_(nullptr), @@ -398,13 +391,12 @@ class ObVectorIndexTask : public share::ObITask ObPluginVectorIndexTaskCtx *task_ctx); common::ObIAllocator &get_allocator() { return allocator_; } virtual int process() override; - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(read_snapshot), KPC_(task_ctx)); + TO_STRING_KV(K_(is_inited), K_(read_snapshot), KPC_(task_ctx)); private: int process_one(); private: bool is_inited_; - share::ObLSID ls_id_; ObPluginVectorIndexLoadScheduler *vec_idx_scheduler_; ObPluginVectorIndexMgr *vec_idx_mgr_; ObPluginVectorIndexTaskCtx *task_ctx_; @@ -418,8 +410,7 @@ class ObVectorIndexTaskParam final { public: ObVectorIndexTaskParam() - : ls_id_(share::ObLSID::INVALID_LS_ID), - table_id_(OB_INVALID_ID), + : table_id_(OB_INVALID_ID), tablet_id_(common::OB_INVALID_ID), task_ctx_(nullptr) {} @@ -427,21 +418,17 @@ class ObVectorIndexTaskParam final bool is_valid() const { return true - && ls_id_.is_valid() && table_id_ != OB_INVALID_ID && tablet_id_.is_valid(); } bool operator==(const ObVectorIndexTaskParam& param) const { return true - && ls_id_ == param.ls_id_ && table_id_ == param.table_id_ && tablet_id_ == param.tablet_id_; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), KP_(task_ctx)); + TO_STRING_KV(K_(tablet_id), KP_(task_ctx)); public: - - share::ObLSID ls_id_; uint64_t table_id_; common::ObTabletID tablet_id_; ObPluginVectorIndexTaskCtx *task_ctx_; @@ -485,7 +472,7 @@ class ObVectorIndexMemSyncInfo ~ObVectorIndexMemSyncInfo(){} - int init(int64_t hash_capacity, ObLSID &ls_id); + int init(int64_t hash_capacity); void destroy(); int add_task_to_waiting_map(ObVectorIndexSyncLog &ls_log); diff --git a/src/observer/vector_index/ob_plugin_vector_index_serialize.cpp b/src/observer/vector_index/ob_plugin_vector_index_serialize.cpp index 498b27471..ff07457ba 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_serialize.cpp +++ b/src/observer/vector_index/ob_plugin_vector_index_serialize.cpp @@ -388,7 +388,6 @@ int ObHNSWSerializeCallback::operator()(const char *data, const int64_t data_siz ObLobAccessParam lob_param; lob_param.set_tmp_allocator(param.tmp_allocator_); lob_param.allocator_ = param.allocator_; - lob_param.ls_id_ = vctx->get_ls_id(); lob_param.tablet_id_ = vctx->get_data_tablet_id(); lob_param.lob_meta_tablet_id_ = vctx->get_lob_meta_tablet_id(); lob_param.lob_piece_tablet_id_ = vctx->get_lob_piece_tablet_id(); diff --git a/src/observer/vector_index/ob_plugin_vector_index_service.cpp b/src/observer/vector_index/ob_plugin_vector_index_service.cpp index c8b4b15d8..0fd60490d 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_service.cpp +++ b/src/observer/vector_index/ob_plugin_vector_index_service.cpp @@ -47,10 +47,9 @@ ObPluginVectorIndexMgr::~ObPluginVectorIndexMgr() void ObPluginVectorIndexMgr::destroy() { if (IS_INIT) { - LOG_INFO("LS Vector Index Mgr destroy", K(ls_id_)); + LOG_INFO("vector index manager destroy"); is_inited_ = false; need_check_ = false; - ls_id_.reset(); release_all_adapters(); partial_index_adpt_map_.destroy(); complete_index_adpt_map_.destroy(); @@ -99,32 +98,30 @@ void ObPluginVectorIndexMgr::release_all_adapters() } } -int ObPluginVectorIndexMgr::init(ObLSID ls_id, - lib::MemoryContext &memory_context, +int ObPluginVectorIndexMgr::init(lib::MemoryContext &memory_context, uint64_t *all_vsag_use_mem) { int ret = OB_SUCCESS; int64_t hash_capacity = common::hash::cal_next_prime(DEFAULT_ADAPTER_HASH_SIZE); if (OB_FAIL(complete_index_adpt_map_.create(hash_capacity, "VecIdxAdptMap", "VecIdxAdptMap"))) { - LOG_WARN("fail to create full index adapter map", KR(ret), K(ls_id)); + LOG_WARN("fail to create full index adapter map", KR(ret)); } else if (OB_FAIL(partial_index_adpt_map_.create(hash_capacity, "VecIdxAdptMap", "VecIdxAdptMap"))) { - LOG_WARN("fail to create partial index adapter map", KR(ret), K(ls_id)); + LOG_WARN("fail to create partial index adapter map", KR(ret)); } else if (OB_FAIL(ivf_index_helper_map_.create(hash_capacity, "IvfIdxHpMap", "IvfIdxHpMap"))) { - LOG_WARN("fail to create ivf index build helper map", KR(ret), K(ls_id)); + LOG_WARN("fail to create ivf index build helper map", KR(ret)); } else if (OB_FAIL(ivf_cache_mgr_map_.create(hash_capacity, "IvfMgrMap", "IvfMgrMap"))) { - LOG_WARN("fail to create ivf mgr map", KR(ret), K(ls_id)); - } else if (OB_FAIL(mem_sync_info_.init(hash_capacity, ls_id))) { - LOG_WARN("fail to create first mem sync set", K(ls_id), KR(ret)); - } else if (OB_FAIL(async_task_opt_.init(hash_capacity, ls_id))) { - LOG_WARN("fail to create async task option", KR(ret), K(ls_id)); + LOG_WARN("fail to create ivf mgr map", KR(ret)); + } else if (OB_FAIL(mem_sync_info_.init(hash_capacity))) { + LOG_WARN("fail to create first mem sync set", KR(ret)); + } else if (OB_FAIL(async_task_opt_.init(hash_capacity))) { + LOG_WARN("fail to create async task option", KR(ret)); } else { - ls_tablet_task_ctx_.task_id_ = 0; - ls_tablet_task_ctx_.non_memdata_task_cycle_ = 0; - ls_tablet_task_ctx_.need_memdata_sync_ = false; - ls_tablet_task_ctx_.state_ = OB_TTL_TASK_PREPARE; + task_ctx_.task_id_ = 0; + task_ctx_.non_memdata_task_cycle_ = 0; + task_ctx_.need_memdata_sync_ = false; + task_ctx_.state_ = OB_TTL_TASK_PREPARE; need_check_ = false; - - ls_id_ = ls_id; + memory_context_ = memory_context; all_vsag_use_mem_ = all_vsag_use_mem; is_inited_ = true; @@ -264,7 +261,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_(ObTabletID tablet_id, ObPluginVect if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("failed to get full vector index inst", K(tablet_id), KR(ret)); } else { - // ret is OB_HASH_NOT_EXIST not found, + // ret is OB_HASH_NOT_EXIST not found, } } else if (OB_ISNULL(index_inst)) { ret = OB_ERR_UNEXPECTED; @@ -445,7 +442,7 @@ int ObPluginVectorIndexMgr::get_or_create_partial_adapter_(ObTabletID tablet_id, } else { // not exist create new if (OB_FAIL(create_partial_adapter(tablet_id, ObTabletID(), type, allocator, OB_INVALID_ID, OB_INVALID_ID, vec_index_param, dim))) { LOG_WARN("failed to create tmp vector index instance with ls", K(tablet_id), K(type), KR(ret)); - } + } if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { } else if (OB_FAIL(get_adapter_inst_guard(tablet_id, adapter_guard))) { LOG_WARN("failed to get tmp vector index instance with ls", K(tablet_id), K(type), KR(ret)); @@ -468,7 +465,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx int ret = OB_SUCCESS; need_merge = true; - if (!ctx.inc_tablet_id_.is_valid() + if (!ctx.inc_tablet_id_.is_valid() || !ctx.snapshot_tablet_id_.is_valid() || !ctx.vbitmap_tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -483,7 +480,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx vec_index_param, dim, allocator))) { - LOG_WARN("failed to get vector index adapter", K(ctx.inc_tablet_id_), KR(ret)); + LOG_WARN("failed to get vector index adapter", K(ctx.inc_tablet_id_), KR(ret)); } else if (FALSE_IT(adapter = candidate.inc_adatper_guard_.get_adatper())) { } else if (adapter->get_create_type() == CreateTypeFullPartial || adapter->get_create_type() == CreateTypeComplete) { @@ -502,7 +499,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx vec_index_param, dim, allocator))) { - LOG_WARN("failed to get vector index adapter", K(ctx.vbitmap_tablet_id_), KR(ret)); + LOG_WARN("failed to get vector index adapter", K(ctx.vbitmap_tablet_id_), KR(ret)); } else if (FALSE_IT(adapter = candidate.bitmp_adatper_guard_.get_adatper())) { } else if (adapter->get_create_type() == CreateTypeFullPartial || adapter->get_create_type() == CreateTypeComplete) { @@ -512,7 +509,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx need_merge = false; } } - + if (OB_FAIL(ret) || need_merge == false) { // do nothing } else if (OB_FAIL(get_or_create_partial_adapter_(ctx.snapshot_tablet_id_, @@ -521,7 +518,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx vec_index_param, dim, allocator))) { - LOG_WARN("failed to get vector index adapter", K(ctx.snapshot_tablet_id_), KR(ret)); + LOG_WARN("failed to get vector index adapter", K(ctx.snapshot_tablet_id_), KR(ret)); } else if (FALSE_IT(adapter = candidate.sn_adatper_guard_.get_adatper())) { } else if (adapter->get_create_type() == CreateTypeFullPartial || adapter->get_create_type() == CreateTypeComplete) { @@ -542,7 +539,7 @@ int ObPluginVectorIndexMgr::get_adapter_inst_by_ctx(ObVectorIndexAcquireCtx &ctx vec_index_param, dim, allocator))) { - LOG_WARN("failed to get vector index adapter", K(ctx.embedded_tablet_id_), KR(ret)); + LOG_WARN("failed to get vector index adapter", K(ctx.embedded_tablet_id_), KR(ret)); } else if (FALSE_IT(adapter = candidate.embedded_adatper_guard_.get_adatper())) { } else if (adapter->get_create_type() == CreateTypeFullPartial || adapter->get_create_type() == CreateTypeComplete) { @@ -567,7 +564,7 @@ int ObPluginVectorIndexMgr::get_and_merge_adapter(ObVectorIndexAcquireCtx &ctx, ObVectorIndexAdapterCandiate candidate; if (OB_FAIL(get_adapter_inst_by_ctx(ctx, need_merge, allocator, adapter_guard, candidate, vec_index_param, dim))) { - LOG_WARN("failed to get and merge adapter", K(ls_id_), K(ctx), KR(ret)); + LOG_WARN("failed to get and merge adapter", K(ctx), KR(ret)); } if (OB_SUCC(ret) && need_merge @@ -582,26 +579,23 @@ int ObPluginVectorIndexMgr::get_and_merge_adapter(ObVectorIndexAcquireCtx &ctx, int ObPluginVectorIndexMgr::check_need_mem_data_sync_task(bool &need_sync) { need_sync = false; - mem_sync_info_.check_and_switch_if_needed(need_sync, ls_tablet_task_ctx_.all_finished_); - LOG_INFO("memdata sync check", K(ls_id_), K(need_sync), K(ls_tablet_task_ctx_)); + mem_sync_info_.check_and_switch_if_needed(need_sync, task_ctx_.all_finished_); + LOG_INFO("memdata sync check", K(need_sync), K(task_ctx_)); // both map empty, do nothing return OB_SUCCESS; } -int ObPluginVectorIndexService::acquire_adapter_guard(ObLSID ls_id, - ObVectorIndexAcquireCtx &ctx, +int ObPluginVectorIndexService::acquire_adapter_guard(ObVectorIndexAcquireCtx &ctx, ObPluginVectorIndexAdapterGuard &adapter_guard, ObString *vec_index_param, int64_t dim) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = single_index_mgr_; - if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - LOG_WARN("failed to acquire vector index mgr", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_and_merge_adapter(ctx, allocator_, adapter_guard, + if (OB_FAIL(index_mgr->get_and_merge_adapter(ctx, allocator_, adapter_guard, vec_index_param, dim))) { - LOG_WARN("failed to get and merge adapter", K(ls_id), K(ctx), KR(ret)); + LOG_WARN("failed to get and merge adapter", K(ctx), KR(ret)); } else { share::ObPluginVectorIndexAdaptor* adaptor = adapter_guard.get_adatper(); if (OB_NOT_NULL(adaptor) && !adaptor->validate_tablet_ids(ctx)) { @@ -614,44 +608,25 @@ int ObPluginVectorIndexService::acquire_adapter_guard(ObLSID ls_id, } int ObPluginVectorIndexService::acquire_ivf_build_helper_guard( - ObLSID ls_id, const ObIvfHelperKey &key, ObIndexType type, ObIvfBuildHelperGuard &helper_guard, ObString &vec_index_param) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, ls_index_mgr))) { + ObPluginVectorIndexMgr *index_mgr = single_index_mgr_; + if (OB_FAIL(index_mgr->get_build_helper_inst_guard(key, helper_guard))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index mgr for ls", K(ret), K(ls_id)); - } else { // create new ls index mgr if not exist - ret = OB_SUCCESS; - if (OB_FAIL(create_ivf_build_helper(ls_id, key, type, vec_index_param))) { - LOG_WARN("failed to create tmp vector index instance", K(ret), K(ls_id), K(key), K(type), K(vec_index_param)); - } - if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { - } else if (OB_FAIL(get_build_helper_inst_guard(ls_id, key, helper_guard))) { - LOG_WARN("failed to get tmp ivf build helper instance", K(ret), K(ls_id), K(key), K(type)); - } else { - LOG_INFO("create ivf build helper success", K(ret), K(ls_id), KPC(helper_guard.get_helper())); - } - } - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_build_helper_inst_guard(key, helper_guard))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get ivf build helper", KR(ret), K(ls_id), K(key)); + LOG_WARN("failed to get ivf build helper", KR(ret), K(key)); } else { // not exist create new - if (OB_FAIL(ls_index_mgr->create_ivf_build_helper(key, type, vec_index_param, allocator_))) { - LOG_WARN("failed to create tmp vector index instance with ls", KR(ret), K(ls_id), K(key), K(type)); - } + if (OB_FAIL(index_mgr->create_ivf_build_helper(key, type, vec_index_param, allocator_))) { + LOG_WARN("failed to create tmp vector index instance with ls", KR(ret), K(key), K(type)); + } if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { - } else if (OB_FAIL(ls_index_mgr->get_build_helper_inst_guard(key, helper_guard))) { - LOG_WARN("failed to get tmp vector index instance with ls", KR(ret), K(ls_id), K(key), K(type)); + } else if (OB_FAIL(index_mgr->get_build_helper_inst_guard(key, helper_guard))) { + LOG_WARN("failed to get tmp vector index instance with ls", KR(ret), K(key), K(type)); } else { - LOG_INFO("create partial index adapter success", K(ret), K(ls_id), K(key), KPC(helper_guard.get_helper())); + LOG_INFO("create partial index adapter success", K(ret), K(key), KPC(helper_guard.get_helper())); } } } else { @@ -661,58 +636,39 @@ int ObPluginVectorIndexService::acquire_ivf_build_helper_guard( return ret; } -int ObPluginVectorIndexService::acquire_adapter_guard(ObLSID ls_id, - ObTabletID tablet_id, +int ObPluginVectorIndexService::acquire_adapter_guard(ObTabletID tablet_id, ObIndexType type, ObPluginVectorIndexAdapterGuard &adapter_guard, ObString *vec_index_param, int64_t dim) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, ls_index_mgr))) { + ObPluginVectorIndexMgr *index_mgr = single_index_mgr_; + if (OB_FAIL(index_mgr->get_adapter_inst_guard(tablet_id, adapter_guard))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } else { // create new ls index mgr if not exist - ret = OB_SUCCESS; - if (OB_FAIL(create_partial_adapter(ls_id, tablet_id, ObTabletID(), type, OB_INVALID_ID, OB_INVALID_ID, vec_index_param, dim))) { - LOG_WARN("failed to create tmp vector index instance", K(ls_id), K(tablet_id), K(type), KR(ret)); - } - if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { - } else if (OB_FAIL(get_adapter_inst_guard(ls_id, tablet_id, adapter_guard))) { - LOG_WARN("failed to get tmp vector index instance", K(ls_id), K(tablet_id), K(type), KR(ret)); - } else { - LOG_INFO("create partial index adapter success", K(ret), K(ls_id), KPC(adapter_guard.get_adatper())); - } - } - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_adapter_inst_guard(tablet_id, adapter_guard))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index adapter", K(ls_id), K(tablet_id), KR(ret)); + LOG_WARN("failed to get vector index adapter", K(tablet_id), KR(ret)); } else { // not exist create new - if (OB_FAIL(ls_index_mgr->create_partial_adapter(tablet_id, ObTabletID(), type, allocator_, OB_INVALID_ID, OB_INVALID_ID, vec_index_param, dim))) { - LOG_WARN("failed to create tmp vector index instance with ls", K(ls_id), K(tablet_id), K(type), KR(ret)); - } + if (OB_FAIL(index_mgr->create_partial_adapter(tablet_id, ObTabletID(), type, allocator_, OB_INVALID_ID, OB_INVALID_ID, vec_index_param, dim))) { + LOG_WARN("failed to create tmp vector index instance with ls", K(tablet_id), K(type), KR(ret)); + } if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { - } else if (OB_FAIL(ls_index_mgr->get_adapter_inst_guard(tablet_id, adapter_guard))) { - LOG_WARN("failed to get tmp vector index instance with ls", K(ls_id), K(tablet_id), K(type), KR(ret)); + } else if (OB_FAIL(index_mgr->get_adapter_inst_guard(tablet_id, adapter_guard))) { + LOG_WARN("failed to get tmp vector index instance with ls", K(tablet_id), K(type), KR(ret)); } else { - LOG_INFO("create partial index adapter success", K(ret), K(ls_id), KP(adapter_guard.get_adatper()), KPC(adapter_guard.get_adatper())); + LOG_INFO("create partial index adapter success", K(ret), KP(adapter_guard.get_adatper()), KPC(adapter_guard.get_adatper())); } } } else { // get from existed ls index mgr } - if (OB_SUCC(ret) + if (OB_SUCC(ret) && OB_NOT_NULL(adapter_guard.get_adatper()) && adapter_guard.get_adatper()->get_index_type() >= ObVectorIndexAlgorithmType::VIAT_MAX) { // check index param, if it is emtpy, may get partial adapter during maintenance - if (OB_NOT_NULL(vec_index_param) + if (OB_NOT_NULL(vec_index_param) && !vec_index_param->empty() && OB_FAIL(adapter_guard.get_adatper()->set_param(*vec_index_param, dim))) { - LOG_WARN("failed to set param", K(ret), K(ls_id), K(tablet_id), K(type), KPC(vec_index_param), K(dim)); + LOG_WARN("failed to set param", K(ret), K(tablet_id), K(type), KPC(vec_index_param), K(dim)); } LOG_INFO("may get get partial adapter during maintenance", KPC(adapter_guard.get_adatper())); } @@ -731,7 +687,7 @@ int ObPluginVectorIndexMgr::check_and_merge_partial_inner(ObVecIdxSharedTableInf LOG_WARN("fail to create hash map for data tablet id to vec index adapter", KR(ret)); } else { // build candidate and save to data_tablet_id_map - // query process may merge adapters and delete partial adapters from hashmap, + // query process may merge adapters and delete partial adapters from hashmap, // use lock here to avoid merge race condition for simple RLockGuard lock_guard(adapter_map_rwlock_); @@ -827,100 +783,41 @@ int ObPluginVectorIndexMgr::check_and_merge_partial_inner(ObVecIdxSharedTableInf return ret; } -int ObPluginVectorIndexService::check_and_merge_adapter(ObLSID ls_id, ObVecIdxSharedTableInfoMap &info_map) +int ObPluginVectorIndexService::check_and_merge_adapter(ObVecIdxSharedTableInfoMap &info_map) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, index_ls_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_id)); - } - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_id)); - } else if (!index_ls_mgr->get_partial_adapter_map().empty()) { - if (OB_FAIL(index_ls_mgr->check_and_merge_partial_inner(info_map, allocator_))) { + if (!single_index_mgr_->get_partial_adapter_map().empty()) { + if (OB_FAIL(single_index_mgr_->check_and_merge_partial_inner(info_map, allocator_))) { LOG_WARN("failed to check and merge partial adapter", KR(ret)); } - + } return ret; } -int ObPluginVectorIndexService::get_adapter_inst_guard(ObLSID ls_id, - ObTabletID tablet_id, +int ObPluginVectorIndexService::get_adapter_inst_guard(ObTabletID tablet_id, ObPluginVectorIndexAdapterGuard &adpt_guard) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, ls_index_mgr))) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_adapter_inst_guard(tablet_id, adpt_guard))) { - LOG_WARN("failed to get vector index adapter", K(ls_id), K(tablet_id), KR(ret)); + if (OB_FAIL(single_index_mgr_->get_adapter_inst_guard(tablet_id, adpt_guard))) { + LOG_WARN("failed to get vector index adapter", K(tablet_id), KR(ret)); } return ret; } int ObPluginVectorIndexService::get_build_helper_inst_guard( - ObLSID ls_id, const ObIvfHelperKey &key, ObIvfBuildHelperGuard &helper_guard) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, ls_index_mgr))) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_build_helper_inst_guard(key, helper_guard))) { - LOG_WARN("failed to get ivf build helper guard", KR(ret), K(ls_id), K(key)); + if (OB_FAIL(single_index_mgr_->get_build_helper_inst_guard(key, helper_guard))) { + LOG_WARN("failed to get ivf build helper guard", KR(ret), K(key)); } return ret; } -int ObPluginVectorIndexService::acquire_vector_index_mgr(ObLSID ls_id, ObPluginVectorIndexMgr *&mgr) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - void *mgr_buff = allocator_.alloc(sizeof(ObPluginVectorIndexMgr)); - if (OB_ISNULL(mgr_buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memeory for new vector index mgr", KR(ret)); - } else { - ObPluginVectorIndexMgr *new_ls_index_mgr = new(mgr_buff)ObPluginVectorIndexMgr(memory_context_); - if (OB_FAIL(new_ls_index_mgr->init(ls_id, memory_context_, all_vsag_use_mem_))) { - LOG_WARN("failed to init ls vector index mgr", KR(ret), K(ls_id)); - } else if (OB_FAIL(get_ls_index_mgr_map().set_refactored(ls_id, new_ls_index_mgr))) { - if (ret != OB_HASH_EXIST) { - LOG_WARN("set vector index mgr map faild", KR(ret), K(ls_id)); - } - } - if (OB_FAIL(ret)) { - new_ls_index_mgr->~ObPluginVectorIndexMgr(); - allocator_.free(mgr_buff); - new_ls_index_mgr = nullptr; - mgr_buff = nullptr; - } - if (OB_FAIL(ret) && (OB_HASH_EXIST != ret)) { - } else if (OB_FAIL(get_ls_index_mgr_map().get_refactored(ls_id, mgr))) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } - } - } - } - return ret; -} - -int ObPluginVectorIndexService::create_partial_adapter(ObLSID ls_id, - ObTabletID idx_tablet_id, +int ObPluginVectorIndexService::create_partial_adapter(ObTabletID idx_tablet_id, ObTabletID data_tablet_id, ObIndexType type, int64_t index_table_id, @@ -929,21 +826,7 @@ int ObPluginVectorIndexService::create_partial_adapter(ObLSID ls_id, int64_t dim) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - ObPluginVectorIndexMgr *new_ls_index_mgr = nullptr; - ObPluginVectorIndexAdaptor *tmp_vec_idx_adpt = nullptr; - - if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->create_partial_adapter(idx_tablet_id, + if (OB_FAIL(single_index_mgr_->create_partial_adapter(idx_tablet_id, data_tablet_id, type, allocator_, @@ -951,59 +834,33 @@ int ObPluginVectorIndexService::create_partial_adapter(ObLSID ls_id, data_table_id, vec_index_param, dim))) { - LOG_WARN("set vector index adapter faild", K(ls_id), K(idx_tablet_id), KR(ret)); + LOG_WARN("set vector index adapter faild", K(idx_tablet_id), KR(ret)); } return ret; } int ObPluginVectorIndexService::create_ivf_build_helper( - ObLSID ls_id, const ObIvfHelperKey &key, ObIndexType type, ObString &vec_index_param) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - ObIvfBuildHelper *tmp_ivf_build_helper = nullptr; - - if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->create_ivf_build_helper(key, + if (OB_FAIL(single_index_mgr_->create_ivf_build_helper(key, type, vec_index_param, allocator_))) { - LOG_WARN("failed to create ivf build helper", KR(ret), K(ls_id), K(key), K(type), K(vec_index_param)); + LOG_WARN("failed to create ivf build helper", KR(ret), K(key), K(type), K(vec_index_param)); } return ret; } -int ObPluginVectorIndexService::erase_ivf_build_helper(ObLSID ls_id, const ObIvfHelperKey &key) +int ObPluginVectorIndexService::erase_ivf_build_helper(const ObIvfHelperKey &key) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; - - if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get vector index mgr for ls", KR(ret), K(ls_id)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(ls_index_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null vector index mgr for ls", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->erase_ivf_build_helper(key))) { - LOG_WARN("failed to create ivf build helper", KR(ret), K(ls_id), K(key)); + if (OB_FAIL(single_index_mgr_->erase_ivf_build_helper(key))) { + LOG_WARN("failed to create ivf build helper", KR(ret), K(key)); } return ret; @@ -1020,22 +877,15 @@ void ObPluginVectorIndexService::destroy() FLOG_INFO("destroy vector index service"); is_inited_ = false; has_start_ = false; - + is_ls_or_tablet_changed_ = false; schema_service_ = NULL; ls_service_ = NULL; sql_proxy_ = NULL; - FOREACH(iter, index_ls_mgr_map_) { - const ObLSID &ls_id = iter->first; - ObPluginVectorIndexMgr *ls_index_mgr = iter->second; - if (OB_NOT_NULL(ls_index_mgr)) { - ls_index_mgr->destroy(); - allocator_.free(ls_index_mgr); - ls_index_mgr = nullptr; - } - } - index_ls_mgr_map_.destroy(); + single_index_mgr_->destroy(); + allocator_.free(single_index_mgr_); + single_index_mgr_ = nullptr; allocator_.reset(); if (memory_context_ != nullptr) { DESTROY_CONTEXT(memory_context_); @@ -1065,27 +915,37 @@ int ObPluginVectorIndexService::init(schema::ObMultiVersionSchemaService *schema || OB_ISNULL(ls_service)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument to init ObPluginVectorIndexService", KR(ret)); - } else if (OB_FAIL(index_ls_mgr_map_.create(common::hash::cal_next_prime(DEFAULT_LS_HASH_SIZE), - "VecIdxLSMgr", - "VecIdxLSMgr"))) { - LOG_WARN("create ls mgr ", KR(ret)); } else if (OB_FAIL(allocator_.init(nullptr, OB_MALLOC_MIDDLE_BLOCK_SIZE, mem_attr))) { LOG_WARN("ObTenantSrs allocator init failed.", K(ret)); } else { ObSharedMemAllocMgr *shared_mem_mgr = share::g_mp->shared_mem_alloc_mgr(); memory_context_ = shared_mem_mgr->vector_allocator().get_mem_context(); all_vsag_use_mem_ = shared_mem_mgr->vector_allocator().get_used_mem_ptr(); - - schema_service_ = schema_service; - ls_service_ = ls_service; - sql_proxy_ = GCTX.sql_proxy_; - is_inited_ = true; - LOG_INFO("plugin vector index service: init", KR(ret)); + + void *mgr_buf = allocator_.alloc(sizeof(ObPluginVectorIndexMgr)); + if (OB_ISNULL(mgr_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate vector index manager", KR(ret)); + } else if (FALSE_IT(single_index_mgr_ = new(mgr_buf) ObPluginVectorIndexMgr(memory_context_))) { + } else if (OB_FAIL(single_index_mgr_->init(memory_context_, all_vsag_use_mem_))) { + LOG_WARN("failed to init vector index manager", KR(ret)); + } else { + schema_service_ = schema_service; + ls_service_ = ls_service; + sql_proxy_ = GCTX.sql_proxy_; + is_inited_ = true; + LOG_INFO("plugin vector index service: init", KR(ret)); + } + if (OB_FAIL(ret) && OB_NOT_NULL(single_index_mgr_)) { + single_index_mgr_->~ObPluginVectorIndexMgr(); + allocator_.free(single_index_mgr_); + single_index_mgr_ = nullptr; + } } return ret; } -int ObPluginVectorIndexService::switch_to_leader() +int ObPluginVectorIndexService::activate() { int ret = OB_SUCCESS; int64_t start_time_us = ObTimeUtility::current_time(); @@ -1116,14 +976,7 @@ int ObPluginVectorIndexService::switch_to_leader() return ret; } -int ObPluginVectorIndexService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - inner_switch_to_follower(); - return ret; -} - -void ObPluginVectorIndexService::switch_to_follower_forcedly() +void ObPluginVectorIndexService::deactivate() { inner_switch_to_follower(); } @@ -1214,7 +1067,7 @@ void ObPluginVectorIndexService::wait() } kmeans_build_task_handler_.wait(); embedding_task_handler_.wait(); - } + } } // for debug @@ -1226,23 +1079,22 @@ void ObPluginVectorIndexMgr::dump_all_inst() const ObTabletID &tablet_id = iter->first; ObPluginVectorIndexAdaptor *adapter = iter->second; ObVectorIndexParam *hnsw_param = (adapter == nullptr)? nullptr : (ObVectorIndexParam *)(adapter->get_algo_data()); - LOG_INFO("dump partial index adapter", K(ls_id_), K(tablet_id), KP(adapter), KPC(adapter), KPC(hnsw_param)); + LOG_INFO("dump partial index adapter", K(tablet_id), KP(adapter), KPC(adapter), KPC(hnsw_param)); } FOREACH(iter, complete_index_adpt_map_) { const ObTabletID &tablet_id = iter->first; ObPluginVectorIndexAdaptor *adapter = iter->second; ObVectorIndexParam *hnsw_param = (adapter == nullptr)? nullptr : (ObVectorIndexParam *)(adapter->get_algo_data()); - LOG_INFO("dump complete index adapter", K(ls_id_), K(tablet_id), KP(adapter), KPC(adapter), KPC(hnsw_param)); + LOG_INFO("dump complete index adapter", K(tablet_id), KP(adapter), KPC(adapter), KPC(hnsw_param)); } } -int ObPluginVectorIndexMgr::get_cache_tablet_ids(ObLSID &ls_id, ObIArray &cache_tablet_ids) +int ObPluginVectorIndexMgr::get_cache_tablet_ids(ObIArray &cache_tablet_ids) { int ret = OB_SUCCESS; - ObLSTabletPair pair; + ObTabletPair pair; FOREACH_X(iter, ivf_cache_mgr_map_, OB_SUCC(ret)) { - pair.ls_id_ = ls_id; pair.tablet_id_ = iter->first; if (OB_FAIL(cache_tablet_ids.push_back(pair))) { LOG_WARN("failed to push array", K(ret)); @@ -1252,15 +1104,14 @@ int ObPluginVectorIndexMgr::get_cache_tablet_ids(ObLSID &ls_id, ObIArray &complete_tablet_ids, - ObIArray &partial_tablet_ids) + ObIArray &complete_tablet_ids, + ObIArray &partial_tablet_ids) { int ret = OB_SUCCESS; - ObLSTabletPair pair; + ObTabletPair pair; RLockGuard lock_guard(adapter_map_rwlock_); FOREACH_X(iter, partial_index_adpt_map_, OB_SUCC(ret)) { const ObTabletID &tablet_id = iter->first; - pair.ls_id_ = ls_id_; pair.tablet_id_ = tablet_id; if (OB_FAIL(partial_tablet_ids.push_back(pair))) { LOG_WARN("failed to push array", K(ret)); @@ -1268,7 +1119,6 @@ int ObPluginVectorIndexMgr::get_snapshot_tablet_ids( } FOREACH_X(iter, complete_index_adpt_map_, OB_SUCC(ret)) { const ObTabletID &tablet_id = iter->first; - pair.ls_id_ = ls_id_; pair.tablet_id_ = tablet_id; if (OB_FAIL(complete_tablet_ids.push_back(pair))) { LOG_WARN("failed to push array", K(ret)); @@ -1278,30 +1128,21 @@ int ObPluginVectorIndexMgr::get_snapshot_tablet_ids( } int ObPluginVectorIndexService::get_snapshot_ids( - ObIArray &complete_tablet_ids, - ObIArray &partial_tablet_ids) + ObIArray &complete_tablet_ids, + ObIArray &partial_tablet_ids) { int ret = OB_SUCCESS; - FOREACH_X(iter, index_ls_mgr_map_, OB_SUCC(ret)) { - const ObLSID &ls_id = iter->first; - ObPluginVectorIndexMgr *index_ls_mgr = iter->second; - if (OB_FAIL(index_ls_mgr->get_snapshot_tablet_ids(complete_tablet_ids, partial_tablet_ids))) { - LOG_WARN("failed to get snapshot tablet ids", K(ret)); - } + if (OB_FAIL(single_index_mgr_->get_snapshot_tablet_ids(complete_tablet_ids, partial_tablet_ids))) { + LOG_WARN("failed to get snapshot tablet ids", K(ret)); } return ret; } -int ObPluginVectorIndexService::get_cache_ids(ObIArray &cache_tablet_ids) +int ObPluginVectorIndexService::get_cache_ids(ObIArray &cache_tablet_ids) { int ret = OB_SUCCESS; - FOREACH_X(iter, index_ls_mgr_map_, OB_SUCC(ret)) - { - ObLSID &ls_id = iter->first; - ObPluginVectorIndexMgr *index_ls_mgr = iter->second; - if (OB_FAIL(index_ls_mgr->get_cache_tablet_ids(ls_id, cache_tablet_ids))) { - LOG_WARN("failed to get cache tablet ids", K(ret)); - } + if (OB_FAIL(single_index_mgr_->get_cache_tablet_ids(cache_tablet_ids))) { + LOG_WARN("failed to get cache tablet ids", K(ret)); } return ret; } @@ -1426,12 +1267,11 @@ int ObPluginVectorIndexMgr::replace_old_adapter(ObPluginVectorIndexAdaptor *new_ } if (OB_FAIL(tmp_ret)) { ret = tmp_ret; - LOG_WARN("failed to get old complete vector index adapter before replace", - K(ret), K(ls_id_), KPC(new_adapter)); + LOG_WARN("failed to get old complete vector index adapter before replace", K(ret), KPC(new_adapter)); } else if (OB_NOT_NULL(old_adapter) && old_adapter != new_adapter && OB_FAIL(new_adapter->inherit_index_id_watermarks_from(*old_adapter))) { LOG_WARN("failed to inherit adapter index id watermarks before replace", - K(ret), K(ls_id_), KPC(old_adapter), KPC(new_adapter)); + K(ret), KPC(old_adapter), KPC(new_adapter)); } // should not fail in following process if (OB_FAIL(ret)) { @@ -1883,36 +1723,31 @@ int ObPluginVectorIndexService::generate_get_aux_info_sql( return ret; } int ObPluginVectorIndexService::acquire_ivf_cache_mgr_guard( - ObLSID ls_id, const ObIvfCacheMgrKey &key, ObIvfCacheMgrGuard &cache_mgr_guard) + const ObIvfCacheMgrKey &key, ObIvfCacheMgrGuard &cache_mgr_guard) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = single_index_mgr_; - if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - LOG_WARN("failed to acquire vector index mgr", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_ivf_cache_mgr_guard(key, cache_mgr_guard))) { + if (OB_FAIL(index_mgr->get_ivf_cache_mgr_guard(key, cache_mgr_guard))) { LOG_WARN("fail to get ivf cache mgr guard", K(ret), K(key)); } return ret; } -int ObPluginVectorIndexService::acquire_ivf_cache_mgr_guard(ObLSID ls_id, - const ObIvfCacheMgrKey &key, +int ObPluginVectorIndexService::acquire_ivf_cache_mgr_guard(const ObIvfCacheMgrKey &key, const ObVectorIndexParam &vec_index_param, int64_t dim, int64_t table_id, ObIvfCacheMgrGuard &cache_mgr_guard) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *ls_index_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = single_index_mgr_; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObPluginVectorIndexService is not inited", KR(ret)); - } else if (OB_FAIL(acquire_vector_index_mgr(ls_id, ls_index_mgr))) { - LOG_WARN("failed to acquire vector index mgr", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_index_mgr->get_or_create_ivf_cache_mgr_guard(allocator_, key, vec_index_param, dim, table_id, cache_mgr_guard))) { - LOG_WARN("failed to get or create ivf cache mgr guard", K(ls_id), KR(ret)); + } else if (OB_FAIL(index_mgr->get_or_create_ivf_cache_mgr_guard(allocator_, key, vec_index_param, dim, table_id, cache_mgr_guard))) { + LOG_WARN("failed to get or create ivf cache mgr guard", KR(ret)); } else { ObIvfCacheMgr* cache_mgr = cache_mgr_guard.get_ivf_cache_mgr(); if (OB_ISNULL(cache_mgr)) { diff --git a/src/observer/vector_index/ob_plugin_vector_index_service.h b/src/observer/vector_index/ob_plugin_vector_index_service.h index 7a950d133..3350cd70f 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_service.h +++ b/src/observer/vector_index/ob_plugin_vector_index_service.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_OB_PLUGIN_VECTOR_INDEX_SERVICE_DEFINE_H_ #include // For std::invoke_result #include "sql/engine/expr/ob_expr_lob_utils.h" -#include "share/ob_ls_id.h" #include "logservice/ob_log_base_type.h" #include "share/scn.h" #include "lib/lock/ob_recursive_mutex.h" @@ -126,28 +125,26 @@ class ObPluginVectorIndexMgr { public: ObPluginVectorIndexMgr(lib::MemoryContext &memory_context) - : is_inited_(false), + : is_inited_(false), need_check_(false), - ls_id_(), complete_index_adpt_map_(), partial_index_adpt_map_(), ivf_index_helper_map_(), adapter_map_rwlock_(), - ls_tablet_task_ctx_(), + task_ctx_(), interval_factor_(0), vector_index_service_(nullptr), mem_sync_info_{}, memory_context_(memory_context), all_vsag_use_mem_(nullptr), async_task_opt_{}, - ls_leader_(false) + is_leader_(false) {} virtual ~ObPluginVectorIndexMgr(); - int init(ObLSID ls_id, lib::MemoryContext &memory_context, uint64_t *all_vsag_use_mem); - ObLSID& get_ls_id() { return ls_id_; } + int init(lib::MemoryContext &memory_context, uint64_t *all_vsag_use_mem); - ObPluginVectorIndexLSTaskCtx& get_ls_task_ctx() { return ls_tablet_task_ctx_; } + ObPluginVectorIndexScheduleCtx& get_task_ctx() { return task_ctx_; } VectorIndexAdaptorMap& get_partial_adapter_map() { return partial_index_adpt_map_; } VectorIndexAdaptorMap& get_complete_adapter_map() { return complete_index_adpt_map_; } IvfCacheMgrMap& get_ivf_cache_mgr_map() { return ivf_cache_mgr_map_; } @@ -159,8 +156,8 @@ class ObPluginVectorIndexMgr void destroy(); void release_all_adapters(); - bool get_ls_leader() { return ls_leader_; } - void set_ls_leader(const bool ls_leader) { ls_leader_ = ls_leader; } + bool is_leader() const { return is_leader_; } + void set_leader(bool is_leader) { is_leader_ = is_leader; } int get_adapter_inst_guard(ObTabletID tablet_id, ObPluginVectorIndexAdapterGuard &adpt_guard); int get_adapter_inst_guard_in_lock(ObTabletID tablet_id, ObPluginVectorIndexAdapterGuard &adpt_guard); @@ -230,10 +227,10 @@ class ObPluginVectorIndexMgr // for debug void dump_all_inst(); // for virtual table - int get_snapshot_tablet_ids(ObIArray &complete_tablet_ids, ObIArray &partial_tablet_ids); - int get_cache_tablet_ids(ObLSID &ls_id, ObIArray &cache_tablet_ids); + int get_snapshot_tablet_ids(ObIArray &complete_tablet_ids, ObIArray &partial_tablet_ids); + int get_cache_tablet_ids(ObIArray &cache_tablet_ids); - TO_STRING_KV(K_(is_inited), K_(need_check), K_(ls_id), K_(ls_tablet_task_ctx)); + TO_STRING_KV(K_(is_inited), K_(need_check), K_(task_ctx)); private: // non-thread save inner functions @@ -266,14 +263,13 @@ class ObPluginVectorIndexMgr typedef RWLock::WLockGuard WLockGuard; bool is_inited_; - bool need_check_; // schema version change, or ls/tablet not existed - share::ObLSID ls_id_; + bool need_check_; // schema version changed or storage is unavailable VectorIndexAdaptorMap complete_index_adpt_map_; // map of complete index adapters with full info VectorIndexAdaptorMap partial_index_adpt_map_; // map of passive created index adapters IvfVectorIndexHelperMap ivf_index_helper_map_; // map of ivf inder build helper IvfCacheMgrMap ivf_cache_mgr_map_; // map of ivf cache managers TCRWLock adapter_map_rwlock_; // lock for adapter maps - ObPluginVectorIndexLSTaskCtx ls_tablet_task_ctx_; // task ctx of ls level + ObPluginVectorIndexScheduleCtx task_ctx_; uint32_t interval_factor_; // used to expand real execute interval ObPluginVectorIndexService *vector_index_service_; @@ -282,7 +278,7 @@ class ObPluginVectorIndexMgr lib::MemoryContext &memory_context_; uint64_t *all_vsag_use_mem_; ObVecIndexAsyncTaskOption async_task_opt_; - bool ls_leader_; + bool is_leader_; }; // id to unique identify an vector index adapter @@ -319,11 +315,10 @@ struct ObPluginVectorIndexIdentity ObString index_identity_; // index_name_prefix }; -typedef common::hash::ObHashMap LSIndexMgrMap; // Manage all vector index adapters of a tenant class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler + public logservice::ObILocalLogHandler { public: ObPluginVectorIndexService() @@ -336,7 +331,8 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, memory_context_(NULL), all_vsag_use_mem_(NULL), tenant_vec_async_task_sched_(nullptr), - is_vec_async_task_started_(false) + is_vec_async_task_started_(false), + single_index_mgr_(nullptr) {} virtual ~ObPluginVectorIndexService(); int init(schema::ObMultiVersionSchemaService *schema_service, @@ -369,10 +365,8 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, return OB_SUCCESS; } void inner_switch_to_follower(); - void switch_to_follower_forcedly(); - int switch_to_leader(); - int switch_to_follower_gracefully(); - int resume_leader() { return switch_to_leader(); } + void deactivate() override; + int activate() override; int alloc_tenant_vec_async_task_sched(); ObFIFOAllocator &get_allocator() { return allocator_; } @@ -380,39 +374,34 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, ObVecIndexAsyncTaskHandler &get_vec_async_task_handle() { return vec_async_task_handle_; } ObKmeansBuildTaskHandler& get_kmeans_build_handler() { return kmeans_build_task_handler_; }; int get_embedding_task_handler(ObEmbeddingTaskHandler *&handler); - LSIndexMgrMap &get_ls_index_mgr_map() { return index_ls_mgr_map_; }; - int get_adapter_inst_guard(ObLSID ls_id, ObTabletID tablet_id, ObPluginVectorIndexAdapterGuard &adapter_guard); - int get_build_helper_inst_guard(ObLSID ls_id, const ObIvfHelperKey &key, ObIvfBuildHelperGuard &helper_guard); - int create_partial_adapter(ObLSID ls_id, - ObTabletID idx_tablet_id, + int get_adapter_inst_guard(ObTabletID tablet_id, ObPluginVectorIndexAdapterGuard &adapter_guard); + int get_build_helper_inst_guard(const ObIvfHelperKey &key, ObIvfBuildHelperGuard &helper_guard); + int create_partial_adapter(ObTabletID idx_tablet_id, ObTabletID data_tablet_id, ObIndexType type, int64_t index_table_id, int64_t data_table_id = OB_INVALID_ID, ObString *vec_index_param = nullptr, int64_t dim = 0); - int create_ivf_build_helper(ObLSID ls_id, - const ObIvfHelperKey &key, - ObIndexType type, - ObString &vec_index_param); - int erase_ivf_build_helper(ObLSID ls_id, const ObIvfHelperKey &key); - int check_and_merge_adapter(ObLSID ls_id, ObVecIdxSharedTableInfoMap &info_map); - int acquire_vector_index_mgr(ObLSID ls_id, ObPluginVectorIndexMgr *&mgr); + int create_ivf_build_helper(const ObIvfHelperKey &key, + ObIndexType type, + ObString &vec_index_param); + int erase_ivf_build_helper(const ObIvfHelperKey &key); + int check_and_merge_adapter(ObVecIdxSharedTableInfoMap &info_map); + ObPluginVectorIndexMgr &get_index_mgr() { return *single_index_mgr_; } + const ObPluginVectorIndexMgr &get_index_mgr() const { return *single_index_mgr_; } // user interfaces - int acquire_adapter_guard(ObLSID ls_id, - ObTabletID tablet_id, + int acquire_adapter_guard(ObTabletID tablet_id, ObIndexType type, ObPluginVectorIndexAdapterGuard &adapter_guard, ObString *vec_index_param = nullptr, int64_t dim = 0); - int acquire_adapter_guard(ObLSID ls_id, - ObVectorIndexAcquireCtx &ctx, + int acquire_adapter_guard(ObVectorIndexAcquireCtx &ctx, ObPluginVectorIndexAdapterGuard &adapter_guard, ObString *vec_index_param = nullptr, int64_t dim = 0); - int acquire_ivf_build_helper_guard(ObLSID ls_id, - const ObIvfHelperKey &key, + int acquire_ivf_build_helper_guard(const ObIvfHelperKey &key, ObIndexType type, ObIvfBuildHelperGuard &helper_guard, ObString &vec_index_param); @@ -420,8 +409,8 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, // for debug int dump_all_inst(); // for virtual table - int get_snapshot_ids(ObIArray &complete_tablet_ids, ObIArray &partial_tablet_ids); - int get_cache_ids(ObIArray &cache_tablet_ids); + int get_snapshot_ids(ObIArray &complete_tablet_ids, ObIArray &partial_tablet_ids); + int get_cache_ids(ObIArray &cache_tablet_ids); // for ivf // ivfflat index needs center ids // ivfsq index needs sq metas and center ids @@ -441,13 +430,12 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, const ObTabletID tablet_id, ObIAllocator &allocator, CallbackFunc &callback_func); - int acquire_ivf_cache_mgr_guard(ObLSID ls_id, - const ObIvfCacheMgrKey &key, + int acquire_ivf_cache_mgr_guard(const ObIvfCacheMgrKey &key, const ObVectorIndexParam &vec_index_param, int64_t dim, int64_t table_id, ObIvfCacheMgrGuard &cache_mgr_guard); - int acquire_ivf_cache_mgr_guard(ObLSID ls_id, const ObIvfCacheMgrKey &key, ObIvfCacheMgrGuard &cache_mgr_guard); + int acquire_ivf_cache_mgr_guard(const ObIvfCacheMgrKey &key, ObIvfCacheMgrGuard &cache_mgr_guard); lib::MemoryContext &get_memory_context() { return memory_context_; } uint64_t *get_all_vsag_use_mem() { return all_vsag_use_mem_; } @@ -463,11 +451,8 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, private: static const int64_t BASIC_TIMER_INTERVAL = 30 * 1000 * 1000; // 30s static const int64_t VEC_INDEX_LOAD_TIME_TASKER_THRESHOLD = 30 * 1000 * 1000; // 30s - static const int64_t DEFAULT_LS_HASH_SIZE = 64; bool is_inited_; bool has_start_; - - LSIndexMgrMap index_ls_mgr_map_; bool is_ls_or_tablet_changed_; share::schema::ObMultiVersionSchemaService *schema_service_; @@ -480,6 +465,7 @@ class ObPluginVectorIndexService : public logservice::ObIReplaySubHandler, uint64_t *all_vsag_use_mem_; ObTenantVecAsyncTaskScheduler *tenant_vec_async_task_sched_; bool is_vec_async_task_started_; + ObPluginVectorIndexMgr *single_index_mgr_; ObVecIndexAsyncTaskHandler vec_async_task_handle_; ObKmeansBuildTaskHandler kmeans_build_task_handler_; ObEmbeddingTaskHandler embedding_task_handler_; diff --git a/src/observer/vector_index/ob_plugin_vector_index_utils.cpp b/src/observer/vector_index/ob_plugin_vector_index_utils.cpp index 218f2ee6f..3a6f34c8f 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_utils.cpp +++ b/src/observer/vector_index/ob_plugin_vector_index_utils.cpp @@ -18,6 +18,7 @@ #include "share/rc/ob_module_provider.h" #include "observer/vector_index/ob_plugin_vector_index_service.h" #include "sql/resolver/ddl/ob_vec_index_builder_util.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -25,18 +26,13 @@ namespace oceanbase namespace share { -int ObPluginVectorIndexUtils::get_task_read_snapshot(ObLSID &ls_id, SCN &read_version) +int ObPluginVectorIndexUtils::get_task_read_snapshot(SCN &read_version) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; ObLS *ls = nullptr; // ObLSWRSHandler::get_ls_weak_read_ts - storage::ObLSService *ls_svr = share::g_mp->ls_service(); - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::SHARE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null ls", K(ret), K(ls_id)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); } else { read_version = ls->get_ls_wrs_handler()->get_ls_weak_read_ts(); } @@ -446,7 +442,6 @@ int ObPluginVectorIndexUtils::get_data_table_out_column_id( int ObPluginVectorIndexUtils::read_vector_info(ObPluginVectorIndexAdaptor *adapter, ObIAllocator &allocator, - ObLSID &ls_id, SCN target_scn, ObVectorQueryAdaptorResultContext &ada_ctx) { @@ -478,8 +473,7 @@ int ObPluginVectorIndexUtils::read_vector_info(ObPluginVectorIndexAdaptor *adapt } else if (OB_ISNULL(output_vec_obj = static_cast(allocator.alloc(sizeof(ObObj) * alloc_size)))) { // use lots of memory ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc mem.", K(ret)); - } else if (adapter->get_is_need_vid() && OB_FAIL(read_local_tablet(ls_id, - adapter, + } else if (adapter->get_is_need_vid() && OB_FAIL(read_local_tablet(adapter, target_scn, type, allocator, @@ -488,8 +482,7 @@ int ObPluginVectorIndexUtils::read_vector_info(ObPluginVectorIndexAdaptor *adapt vid_table_param, vid_id_iter))) { LOG_WARN("failed to read vid id table local tablet.", K(ret)); - } else if (OB_FAIL(read_local_tablet(ls_id, - adapter, + } else if (OB_FAIL(read_local_tablet(adapter, target_scn, data_table_type, allocator, @@ -672,8 +665,7 @@ int ObPluginVectorIndexUtils::read_vector_info(ObPluginVectorIndexAdaptor *adapt // debug interface, remove later -int ObPluginVectorIndexUtils::try_sync_vbitmap_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *adapter, +int ObPluginVectorIndexUtils::try_sync_vbitmap_memdata(ObPluginVectorIndexAdaptor *adapter, SCN &target_scn, ObIAllocator &allocator, ObVectorQueryAdaptorResultContext &ada_ctx) @@ -685,8 +677,7 @@ int ObPluginVectorIndexUtils::try_sync_vbitmap_memdata(ObLSID &ls_id, storage::ObTableScanParam vbitmap_scan_param; schema::ObTableParam vbitmap_table_param(allocator); - if (OB_FAIL(read_local_tablet(ls_id, - adapter, + if (OB_FAIL(read_local_tablet(adapter, target_scn, index_type, allocator, @@ -694,8 +685,8 @@ int ObPluginVectorIndexUtils::try_sync_vbitmap_memdata(ObLSID &ls_id, vbitmap_scan_param, vbitmap_table_param, index_id_iter))) { // read_local_tablet 4rd aux index get rowkey, backword - LOG_WARN("fail to read local tablet", KR(ret), K(ls_id), K(index_type), KPC(adapter)); - } else if (OB_FAIL(adapter->check_index_id_table_readnext_status(&ada_ctx, index_id_iter, target_scn, false, ls_id))) { + LOG_WARN("fail to read local tablet", KR(ret), K(index_type), KPC(adapter)); + } else if (OB_FAIL(adapter->check_index_id_table_readnext_status(&ada_ctx, index_id_iter, target_scn))) { LOG_WARN("fail to check and sync vbitmap.", KR(ret)); } // ToDo: may also need to sync vector to incr memdata @@ -710,8 +701,7 @@ int ObPluginVectorIndexUtils::try_sync_vbitmap_memdata(ObLSID &ls_id, return ret; } -int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *&adapter, +int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObPluginVectorIndexAdaptor *&adapter, const bool create_new_adp, SCN &target_scn, ObIAllocator &allocator) @@ -728,8 +718,7 @@ int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, int64_t index_count = 0; void *adpt_buff = nullptr; - if (OB_FAIL(read_local_tablet(ls_id, - adapter, + if (OB_FAIL(read_local_tablet(adapter, target_scn, index_type, allocator, @@ -737,7 +726,7 @@ int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, snapshot_scan_param, snapshot_table_param, snapshot_idx_iter))) { // read_local_tablet 5th aux index get rowkey - LOG_WARN("fail to read local tablet", KR(ret), K(ls_id), K(index_type), KPC(new_adapter)); + LOG_WARN("fail to read local tablet", KR(ret), K(index_type), KPC(new_adapter)); } else { blocksstable::ObDatumRow *row = nullptr; ObTableScanIterator *table_scan_iter = static_cast(snapshot_idx_iter); @@ -757,15 +746,7 @@ int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, vec_idx_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret)); - } - } else if (OB_ISNULL(vec_idx_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret)); + } else if (FALSE_IT(vec_idx_mgr = &vector_index_service->get_index_mgr())) { } else { if (create_new_adp) { adpt_buff = vector_index_service->get_allocator().alloc(sizeof(ObPluginVectorIndexAdaptor)); @@ -835,10 +816,10 @@ int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, // adapter->get_allocator()->free(snap_memdata->mem_ctx_); // snap_memdata->mem_ctx_ = nullptr; //} - LOG_INFO("memdata sync snapshot index complement no data", K(index_count), K(ls_id), K(index_type), KPC(new_adapter)); + LOG_INFO("memdata sync snapshot index complement no data", K(index_count), K(index_type), KPC(new_adapter)); } else { // index_count > 0 new_adapter->close_snap_data_rb_flag(); - LOG_INFO("memdata sync snapshot index complement data", K(index_count), K(ls_id), K(index_type), KPC(new_adapter)); + LOG_INFO("memdata sync snapshot index complement data", K(index_count), K(index_type), KPC(new_adapter)); } } } @@ -864,7 +845,6 @@ int ObPluginVectorIndexUtils::try_sync_snapshot_memdata(ObLSID &ls_id, } int ObPluginVectorIndexUtils::refresh_adp_from_table( - ObLSID &ls_id, ObPluginVectorIndexAdaptor *&adapter, const bool create_new_adapter, SCN target_scn, @@ -883,10 +863,9 @@ int ObPluginVectorIndexUtils::refresh_adp_from_table( schema::ObTableParam inc_table_param(allocator); int64_t extra_info_actual_size = 0; schema::ObIndexType delta_type = adapter->is_hybrid_index()? INDEX_TYPE_HYBRID_INDEX_LOG_LOCAL: INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL; - if (OB_FAIL(try_sync_snapshot_memdata(ls_id, adapter, create_new_adapter, target_scn, allocator))) { + if (OB_FAIL(try_sync_snapshot_memdata(adapter, create_new_adapter, target_scn, allocator))) { LOG_WARN("failed to refresh mem snapshots without refresh incr", KR(ret)); - } else if (OB_FAIL(read_local_tablet(ls_id, - adapter, + } else if (OB_FAIL(read_local_tablet(adapter, target_scn, delta_type, allocator, @@ -894,7 +873,7 @@ int ObPluginVectorIndexUtils::refresh_adp_from_table( inc_scan_param, inc_table_param, delta_buf_iter))) { - LOG_WARN("fail to read local tablet", KR(ret), K(ls_id), K(delta_type), KP(adapter)); + LOG_WARN("fail to read local tablet", KR(ret), K(delta_type), KP(adapter)); } else if (OB_FAIL(adapter->get_extra_info_actual_size(extra_info_actual_size))) { LOG_WARN("fail to get extra info actual size", K(ret), KPC(adapter)); } else { @@ -906,10 +885,10 @@ int ObPluginVectorIndexUtils::refresh_adp_from_table( ObVectorQueryAdaptorResultContext ada_ctx(extra_info_column_count, &allocator, &tmp_allocator); if (OB_FAIL(adapter->check_delta_buffer_table_readnext_status(&ada_ctx, delta_buf_iter,target_scn))) { LOG_WARN("fail to check_delta_buffer_table_readnext_status.", K(ret)); - } else if (OB_FAIL(try_sync_vbitmap_memdata(ls_id, adapter, target_scn, allocator, ada_ctx))) { + } else if (OB_FAIL(try_sync_vbitmap_memdata(adapter, target_scn, allocator, ada_ctx))) { LOG_WARN("failed to sync vbitmap", KR(ret)); } else if (ada_ctx.get_status() == PVQ_COM_DATA) { - if (OB_FAIL(read_vector_info(adapter, allocator, ls_id, target_scn, ada_ctx))) { + if (OB_FAIL(read_vector_info(adapter, allocator, target_scn, ada_ctx))) { LOG_WARN("failed to read vector_info", KR(ret)); } } @@ -925,14 +904,14 @@ int ObPluginVectorIndexUtils::refresh_adp_from_table( return ret; } -int ObPluginVectorIndexUtils::query_need_refresh_memdata(ObPluginVectorIndexAdaptor *adapter, ObLSID &ls_id) +int ObPluginVectorIndexUtils::query_need_refresh_memdata(ObPluginVectorIndexAdaptor *adapter) { int ret = OB_SUCCESS; ObArenaAllocator allocator("VectorAdaptor", OB_MALLOC_NORMAL_BLOCK_SIZE); SCN target_scn; if (OB_ISNULL(adapter)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), KP(adapter), K(ls_id)); + LOG_WARN("unexpected nullptr", K(ret), KP(adapter)); } else { bool need_retry = false; common::ObSpinLockGuard ctx_guard(adapter->get_reload_lock()); @@ -940,7 +919,7 @@ int ObPluginVectorIndexUtils::query_need_refresh_memdata(ObPluginVectorIndexAdap need_retry = true; } else if (OB_FAIL(target_scn.convert_from_ts(ObTimeUtility::fast_current_time()))) { LOG_WARN("failed to convert ts to scn", K(ret)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::refresh_memdata(ls_id, adapter, target_scn, allocator))) { + } else if (OB_FAIL(ObPluginVectorIndexUtils::refresh_memdata(adapter, target_scn, allocator))) { LOG_WARN("fail to refresh adapter", K(ret)); } else if (OB_FALSE_IT(adapter->set_reload_finish(true))) { } else { @@ -954,28 +933,11 @@ int ObPluginVectorIndexUtils::query_need_refresh_memdata(ObPluginVectorIndexAdap return ret; } -int ObPluginVectorIndexUtils::refresh_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *adapter, +int ObPluginVectorIndexUtils::refresh_memdata(ObPluginVectorIndexAdaptor *adapter, SCN target_scn, ObIAllocator &allocator) { int ret = OB_SUCCESS; - // ToDo: remove test interface later -#if 0 - if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_VEC_ROWKEY_VID_LOCAL, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_VEC_ROWKEY_VID_LOCAL)); - } else if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_VEC_VID_ROWKEY_LOCAL, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_VEC_VID_ROWKEY_LOCAL)); - } else if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL)); - } else if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_VEC_INDEX_ID_LOCAL, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_VEC_INDEX_ID_LOCAL)); - } else if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL)); - } else if (OB_FAIL(test_read_local_data(ls_id, adapter, INDEX_TYPE_IS_NOT, target_scn, allocator))) { - LOG_WARN("fail to test read local data.", K(ret), K(ls_id), K(INDEX_TYPE_IS_NOT)); - } -#endif if (OB_ISNULL(adapter)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid adapter", K(ret), KPC(adapter)); @@ -987,16 +949,8 @@ int ObPluginVectorIndexUtils::refresh_memdata(ObLSID &ls_id, if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, vec_idx_mgr))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret)); - } - } else if (OB_ISNULL(vec_idx_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret)); - } else if (OB_FAIL(refresh_adp_from_table(ls_id, new_adapter, true, target_scn, allocator))) { + } else if (FALSE_IT(vec_idx_mgr = &vector_index_service->get_index_mgr())) { + } else if (OB_FAIL(refresh_adp_from_table(new_adapter, true, target_scn, allocator))) { LOG_WARN("failed to refresh adapter from table", K(ret), KPC(adapter)); } if (adapter != new_adapter && OB_NOT_NULL(new_adapter)) { @@ -1028,8 +982,7 @@ static bool is_non_shared_vec_index_aux_table(schema::ObIndexType type) return bret; } -int ObPluginVectorIndexUtils::read_local_tablet(ObLSID &ls_id, - ObPluginVectorIndexAdaptor* adapter, +int ObPluginVectorIndexUtils::read_local_tablet(ObPluginVectorIndexAdaptor* adapter, SCN target_scn, schema::ObIndexType type, ObIAllocator &allocator, @@ -1047,17 +1000,17 @@ int ObPluginVectorIndexUtils::read_local_tablet(ObLSID &ls_id, scan_iter = nullptr; // init scan param refer to ObLocalIndexLookupOp::init_scan_param() - // assign ls_id, tablet_id, tx_snapshot + // assign tablet_id and transaction snapshot // set need_scn_ = true if need ora_rowscn ObTabletID tablet_id; uint64_t table_id = OB_INVALID_ID; ObTabletHandle tablet_handle; - ObLSHandle ls_handle; + ObLS *ls = nullptr; // INDEX_TYPE_IS_NOT means data tablet - if (!ls_id.is_valid() || OB_ISNULL(adapter)) { + if (OB_ISNULL(adapter)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls id or adapter", KR(ret), K(ls_id), KPC(adapter)); + LOG_WARN("invalid adapter", KR(ret), KPC(adapter)); } else if (is_vec_delta_buffer_type(type) || is_hybrid_vec_index_log_type(type)) { tablet_id = adapter->get_inc_tablet_id(); table_id = adapter->get_inc_table_id(); @@ -1088,23 +1041,19 @@ int ObPluginVectorIndexUtils::read_local_tablet(ObLSID &ls_id, } if (OB_SUCC(ret)) { - LOG_INFO("read table tablet", K(ls_id), K(tablet_id), K(table_id), K(type), K(target_scn)); + LOG_INFO("read table tablet", K(tablet_id), K(table_id), K(type), K(target_scn)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::SHARE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_with_timeout(tablet_id, + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet_with_timeout(tablet_id, tablet_handle, 0, // timeout ObMDSGetTabletMode::READ_READABLE_COMMITED, target_scn))) { LOG_WARN("fail to get tablet handle", KR(ret), K(tablet_id), K(type)); } else { - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; scan_param.schema_version_ = tablet_handle.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; if (OB_FAIL(init_common_scan_param(scan_param, adapter, target_scn, &allocator, &scan_allocator, type, table_id))) { @@ -1905,18 +1854,18 @@ int ObPluginVectorIndexUtils::get_vector_index_name_prefix(const ObTableSchema & return ret; } -int ObPluginVectorIndexUtils::erase_ivf_build_helper(ObLSID ls_id, const ObIvfHelperKey &key) +int ObPluginVectorIndexUtils::erase_ivf_build_helper(const ObIvfHelperKey &key) { int ret = OB_SUCCESS; ObPluginVectorIndexService *vec_index_service = share::g_mp->plugin_vector_index_service(); if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->erase_ivf_build_helper(ls_id, key))) { - LOG_WARN("failed to erase ivf build helper", K(ret), K(ls_id), K(key)); + } else if (OB_FAIL(vec_index_service->erase_ivf_build_helper(key))) { + LOG_WARN("failed to erase ivf build helper", K(ret), K(key)); } if (ret == OB_HASH_NOT_EXIST) { - LOG_WARN("erase ivf build helper, key not exist", K(ret), K(ls_id), K(key)); + LOG_WARN("erase ivf build helper, key not exist", K(ret), K(key)); ret = OB_SUCCESS; } return ret; @@ -1969,44 +1918,34 @@ int ObPluginVectorIndexUtils::split_snapshot_prefix(const ObString &src, const O return ret; } -void ObPluginVectorIndexUtils::set_ls_leader_flag(const ObLSID &ls_id, const bool is_leader) +void ObPluginVectorIndexUtils::set_leader_flag(const bool is_leader) { int ret = OB_SUCCESS; ObPluginVectorIndexService *vector_index_service = share::g_mp->plugin_vector_index_service(); - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; - if (!ls_id.is_valid() || OB_ISNULL(vector_index_service)) { + ObPluginVectorIndexMgr *index_mgr = nullptr; + if (OB_ISNULL(vector_index_service)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), KP(vector_index_service)); - } else if (OB_FAIL(vector_index_service->acquire_vector_index_mgr(ls_id, index_ls_mgr))) { - LOG_WARN("fail to acquire vector index mgr", K(ret), K(ls_id)); - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), KP(index_ls_mgr)); + LOG_WARN("invalid argument", K(ret), KP(vector_index_service)); } else { - index_ls_mgr->set_ls_leader(is_leader); - LOG_INFO("success to set ls leader", K(ls_id), K(is_leader)); + index_mgr = &vector_index_service->get_index_mgr(); + index_mgr->set_leader(is_leader); + LOG_INFO("success to set leader", K(is_leader)); } } -int ObPluginVectorIndexUtils::get_ls_leader_flag(const ObLSID &ls_id, bool &is_leader) +int ObPluginVectorIndexUtils::get_leader_flag(bool &is_leader) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; ObPluginVectorIndexService *vector_index_service = share::g_mp->plugin_vector_index_service(); - if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, index_ls_mgr))) { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_id)); - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret)); - } else { - is_leader = index_ls_mgr->get_ls_leader(); - LOG_TRACE("success to set ls leader", K(ls_id), K(is_leader)); - } + index_mgr = &vector_index_service->get_index_mgr(); + is_leader = index_mgr->is_leader(); + LOG_TRACE("success to get leader", K(is_leader)); return ret; } -int ObPluginVectorIndexUtils::fill_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos) +int ObPluginVectorIndexUtils::fill_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; hash::ObHashSet adaptor_ptr_set; @@ -2014,12 +1953,11 @@ int ObPluginVectorIndexUtils::fill_mem_context_detail_info(ObPluginVectorIndexSe LOG_WARN("fail to create tablet_id set", KR(ret)); } for (int i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - ObLSID ls_id = tablet_ids.at(i).ls_id_; ObTabletID tablet_id = tablet_ids.at(i).tablet_id_; ObPluginVectorIndexAdapterGuard adapter_guard; - if (OB_FAIL(service->get_adapter_inst_guard(ls_id, tablet_id, adapter_guard))) { + if (OB_FAIL(service->get_adapter_inst_guard(tablet_id, adapter_guard))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get adapter inst guard", K(ls_id), K(tablet_id), KR(ret)); + LOG_WARN("failed to get adapter inst guard", K(tablet_id), KR(ret)); } } else { ObPluginVectorIndexAdaptor *adaptor = adapter_guard.get_adatper(); @@ -2068,9 +2006,9 @@ int ObPluginVectorIndexUtils::fill_mem_context_detail_info(ObPluginVectorIndexSe } int ObPluginVectorIndexUtils::get_mem_context_detail_info(ObPluginVectorIndexService *service, - ObIArray &complete_tablet_ids, - ObIArray &partial_tablet_ids, - ObIArray &cache_tablet_ids, + ObIArray &complete_tablet_ids, + ObIArray &partial_tablet_ids, + ObIArray &cache_tablet_ids, char *buf, int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; @@ -2085,7 +2023,7 @@ int ObPluginVectorIndexUtils::get_mem_context_detail_info(ObPluginVectorIndexSer return ret; } -int ObPluginVectorIndexUtils::fill_ivf_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos) +int ObPluginVectorIndexUtils::fill_ivf_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; hash::ObHashSet adaptor_ptr_set; @@ -2093,12 +2031,11 @@ int ObPluginVectorIndexUtils::fill_ivf_mem_context_detail_info(ObPluginVectorInd LOG_WARN("fail to create tablet_id set", KR(ret)); } for (int i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - ObLSID ls_id = tablet_ids.at(i).ls_id_; ObTabletID tablet_id = tablet_ids.at(i).tablet_id_; ObIvfCacheMgrGuard cache_mgr_guard; - if (OB_FAIL(service->acquire_ivf_cache_mgr_guard(ls_id, tablet_id, cache_mgr_guard))) { + if (OB_FAIL(service->acquire_ivf_cache_mgr_guard(tablet_id, cache_mgr_guard))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get adapter inst guard", K(ls_id), K(tablet_id), KR(ret)); + LOG_WARN("failed to get adapter inst guard", K(tablet_id), KR(ret)); } } else { ObIvfCacheMgr *adaptor = cache_mgr_guard.get_ivf_cache_mgr(); diff --git a/src/observer/vector_index/ob_plugin_vector_index_utils.h b/src/observer/vector_index/ob_plugin_vector_index_utils.h index 3d2745cf0..25d6aa04c 100644 --- a/src/observer/vector_index/ob_plugin_vector_index_utils.h +++ b/src/observer/vector_index/ob_plugin_vector_index_utils.h @@ -15,7 +15,6 @@ */ #ifndef OCEANBASE_OBSERVER_OB_MOCK_PLUGIN_VECTOR_INDEX_UTILS_DEFINE_H_ #define OCEANBASE_OBSERVER_OB_MOCK_PLUGIN_VECTOR_INDEX_UTILS_DEFINE_H_ -#include "share/ob_ls_id.h" #include "share/scn.h" #include "share/rc/ob_tenant_base.h" #include "observer/vector_index/ob_plugin_vector_index_adaptor.h" @@ -56,13 +55,11 @@ class ObVsagLoggerSingleton class ObPluginVectorIndexUtils { public: - static int get_task_read_snapshot(ObLSID &ls_id, SCN &read_version); - static int refresh_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *adapter, + static int get_task_read_snapshot(SCN &read_version); + static int refresh_memdata(ObPluginVectorIndexAdaptor *adapter, SCN target_scn, ObIAllocator &allocator); - static int refresh_adp_from_table(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *&adapter, + static int refresh_adp_from_table(ObPluginVectorIndexAdaptor *&adapter, const bool create_new_adapter, SCN target_scn, ObIAllocator &allocator); @@ -83,9 +80,9 @@ class ObPluginVectorIndexUtils return obvectorutil::init_vasg_logger(&ObVsagLoggerSingleton::getInstance()); } - static void set_ls_leader_flag(const ObLSID &ls_id, const bool is_leader); - static int get_ls_leader_flag(const ObLSID &ls_id, bool &is_leader); - static int query_need_refresh_memdata(ObPluginVectorIndexAdaptor *adapter, ObLSID &ls_id); + static void set_leader_flag(bool is_leader); + static int get_leader_flag(bool &is_leader); + static int query_need_refresh_memdata(ObPluginVectorIndexAdaptor *adapter); static int add_key_ranges(uint64_t table_id, ObRowkey& rowkey, storage::ObTableScanParam &scan_param); static int add_key_ranges(uint64_t table_id, ObRowkey& start_key, ObRowkey& end_key, storage::ObTableScanParam &scan_param); @@ -134,11 +131,9 @@ class ObPluginVectorIndexUtils int offset = 1); static int read_vector_info(ObPluginVectorIndexAdaptor *adapter, ObIAllocator &allocator, - ObLSID &ls_id, SCN target_scn, ObVectorQueryAdaptorResultContext &ada_ctx); - static int read_local_tablet(ObLSID &ls_id, - ObPluginVectorIndexAdaptor* adapter, + static int read_local_tablet(ObPluginVectorIndexAdaptor* adapter, SCN target_scn, schema::ObIndexType type, ObIAllocator &allocator, @@ -151,11 +146,11 @@ class ObPluginVectorIndexUtils const bool need_ora_scn = false, const SCN *min_scn = nullptr); - static int erase_ivf_build_helper(ObLSID ls_id, const ObIvfHelperKey &key); + static int erase_ivf_build_helper(const ObIvfHelperKey &key); static int get_mem_context_detail_info(ObPluginVectorIndexService *service, - ObIArray &complete_tablet_ids, - ObIArray &partial_tablet_ids, - ObIArray &cache_tablet_ids, + ObIArray &complete_tablet_ids, + ObIArray &partial_tablet_ids, + ObIArray &cache_tablet_ids, char *buf, int64_t buf_len, int64_t &pos); static int get_tenant_vector_index_ids(bool &has_ivf_index, common::ObIArray &table_id_array); @@ -186,18 +181,16 @@ class ObPluginVectorIndexUtils static int get_shared_table_rowkey_colum_count(schema::ObIndexType type, uint64_t table_id, uint32 &col_cnt); - static int try_sync_snapshot_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *&adapter, + static int try_sync_snapshot_memdata(ObPluginVectorIndexAdaptor *&adapter, const bool create_new_adp, SCN &target_scn, ObIAllocator &allocator); - static int try_sync_vbitmap_memdata(ObLSID &ls_id, - ObPluginVectorIndexAdaptor *adapter, + static int try_sync_vbitmap_memdata(ObPluginVectorIndexAdaptor *adapter, SCN &target_scn, ObIAllocator &allocator, ObVectorQueryAdaptorResultContext &ada_ctx); - static int fill_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos); - static int fill_ivf_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos); + static int fill_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos); + static int fill_ivf_mem_context_detail_info(ObPluginVectorIndexService *service, ObIArray &tablet_ids, char *buf, int64_t buf_len, int64_t &pos); }; diff --git a/src/observer/vector_index/ob_vector_index_async_task.cpp b/src/observer/vector_index/ob_vector_index_async_task.cpp index c0c0fedc4..ec1d44370 100644 --- a/src/observer/vector_index/ob_vector_index_async_task.cpp +++ b/src/observer/vector_index/ob_vector_index_async_task.cpp @@ -45,7 +45,7 @@ bool ObVecAsyncTaskExector::check_operation_allow() int ObVecAsyncTaskExector::check_and_set_thread_pool() { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; const bool is_not_support = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -55,12 +55,12 @@ int ObVecAsyncTaskExector::check_and_set_thread_pool() } else if (OB_ISNULL(vector_index_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { + } else if (OB_FAIL(get_index_mgr(index_mgr))) { LOG_WARN("fail to get index ls mgr", K(ret)); } else { - ObIAllocator *allocator = index_ls_mgr->get_async_task_opt().get_allocator(); + ObIAllocator *allocator = index_mgr->get_async_task_opt().get_allocator(); ObVecIndexAsyncTaskHandler &thread_pool_handle = vector_index_service_->get_vec_async_task_handle(); - if (0 == index_ls_mgr->get_complete_adapter_map().size()) { // no vector index exist, skip + if (0 == index_mgr->get_complete_adapter_map().size()) { // no vector index exist, skip } else { common::ObSpinLockGuard init_guard(thread_pool_handle.lock_); // lock thread pool init to avoid init twice if (thread_pool_handle.is_inited()) { // no need to init twice, skip @@ -77,7 +77,7 @@ int ObVecAsyncTaskExector::check_and_set_thread_pool() int ObVecAsyncTaskExector::load_task(uint64_t &task_trace_base_num) { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; ObArray task_ctx_array; bool is_active_time = true; if (IS_NOT_INIT) { @@ -91,17 +91,16 @@ int ObVecAsyncTaskExector::load_task(uint64_t &task_trace_base_num) } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::in_active_time(is_active_time))) { LOG_WARN("fail to get active time", KR(ret)); } else if (!is_active_time) { - LOG_INFO("skip auto-create per-tablet hnsw optimize tasks, not in active time", - K(ls_->get_ls_id())); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { // skip - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + LOG_INFO("skip auto-create per-tablet hnsw optimize tasks, not in active time"); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { // skip + LOG_WARN("fail to get index ls mgr", K(ret)); } else { - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); ObIAllocator *allocator = task_opt.get_allocator(); const int64_t current_task_cnt = ObVecIndexAsyncTaskUtil::get_processing_task_cnt(task_opt); - RWLock::RLockGuard lock_guard(index_ls_mgr->get_adapter_map_lock()); - FOREACH_X(iter, index_ls_mgr->get_complete_adapter_map(), + RWLock::RLockGuard lock_guard(index_mgr->get_adapter_map_lock()); + FOREACH_X(iter, index_mgr->get_complete_adapter_map(), OB_SUCC(ret) && (task_ctx_array.count() + current_task_cnt <= MAX_ASYNC_TASK_PROCESSING_COUNT)) { ObTabletID tablet_id = iter->first; ObPluginVectorIndexAdaptor *adapter = iter->second; @@ -129,12 +128,12 @@ int ObVecAsyncTaskExector::load_task(uint64_t &task_trace_base_num) } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::fetch_new_trace_id(++task_trace_base_num, allocator, new_trace_id))) { LOG_WARN("fail to fetch new trace id", K(ret), K(tablet_id)); } else { - LOG_DEBUG("start load task", K(ret), K(tablet_id), K(task_trace_base_num), K(ls_->get_ls_id())); + LOG_DEBUG("start load task", K(ret), K(tablet_id), K(task_trace_base_num)); // 1. update task_ctx to async task map - + task_ctx->ls_ = ls_; task_ctx->task_status_.tablet_id_ = tablet_id.id(); - + task_ctx->task_status_.table_id_ = index_table_id; task_ctx->task_status_.task_id_ = new_task_id; task_ctx->task_status_.task_type_ = ObVecIndexAsyncTaskType::OB_VECTOR_ASYNC_INDEX_OPTINAL; @@ -142,8 +141,8 @@ int ObVecAsyncTaskExector::load_task(uint64_t &task_trace_base_num) task_ctx->task_status_.status_ = ObVecIndexAsyncTaskStatus::OB_VECTOR_ASYNC_TASK_PREPARE; task_ctx->task_status_.trace_id_ = new_trace_id; task_ctx->task_status_.target_scn_.convert_from_ts(ObTimeUtility::current_time()); - - if (OB_FAIL(index_ls_mgr->get_async_task_opt().add_task_ctx(tablet_id, task_ctx, inc_new_task))) { // not overwrite + + if (OB_FAIL(index_mgr->get_async_task_opt().add_task_ctx(tablet_id, task_ctx, inc_new_task))) { // not overwrite LOG_WARN("fail to add task ctx", K(ret)); } else if (inc_new_task && OB_FAIL(task_ctx_array.push_back(task_ctx))) { LOG_WARN("fail to push back task status", K(ret), K(task_ctx)); @@ -158,15 +157,15 @@ int ObVecAsyncTaskExector::load_task(uint64_t &task_trace_base_num) } } } - LOG_INFO("finish load async task", K(ret), K(ls_->get_ls_id()), K(task_ctx_array.count()), K(current_task_cnt)); + LOG_INFO("finish load async task", K(ret), K(task_ctx_array.count()), K(current_task_cnt)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(insert_new_task(task_ctx_array))) { - LOG_WARN("fail to insert new tasks", K(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to insert new tasks", K(ret)); } // clear on fail if (OB_FAIL(ret) && !task_ctx_array.empty()) { - if (OB_FAIL(clear_task_ctxs(index_ls_mgr->get_async_task_opt(), task_ctx_array))) { + if (OB_FAIL(clear_task_ctxs(index_mgr->get_async_task_opt(), task_ctx_array))) { LOG_WARN("fail to clear task ctx", K(ret)); } } @@ -185,7 +184,7 @@ int ObVecTaskManager::process_task() } else { ob_usleep(1LL * 1000 * 1000); } - + if (REACH_TIME_INTERVAL(10 * 60L * 1000000)) { // 60s LOG_INFO("vector index task not finished", K(ret), K(task_ids_)); } @@ -218,10 +217,10 @@ int ObVecTaskManager::create_task() } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::fetch_new_trace_id(++trace_base_num, &allocator, new_trace_id))) { LOG_WARN("fail to fetch new trace id", K(ret), K(tablet_id)); } else { - - + + task_ctx->task_status_.tablet_id_ = tablet_id.id(); - + task_ctx->task_status_.table_id_ = index_table_id_; task_ctx->task_status_.task_id_ = new_task_id; task_ctx->task_status_.task_type_ = task_type_; @@ -262,7 +261,7 @@ int ObVecTaskManager::check_task_status() field.data_.uint_ = task_ids_.at(i); if (OB_FAIL(filters.push_back(field))) { LOG_WARN("fail to push back field", K(ret)); - } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::construct_read_task_sql(OB_ALL_VECTOR_INDEX_TASK_HISTORY_TNAME, + } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::construct_read_task_sql(OB_ALL_VECTOR_INDEX_TASK_HISTORY_TNAME, false, false, filters, *sql_proxy, sql))) { LOG_WARN("fail to construct read task sql", K(ret)); } else { diff --git a/src/observer/vector_index/ob_vector_index_async_task_util.cpp b/src/observer/vector_index/ob_vector_index_async_task_util.cpp index 1bf7c493d..82d853eea 100644 --- a/src/observer/vector_index/ob_vector_index_async_task_util.cpp +++ b/src/observer/vector_index/ob_vector_index_async_task_util.cpp @@ -29,6 +29,7 @@ #include "storage/ob_table_dml_param.h" #include "share/table/ob_ttl_util.h" #include "share/ob_common_id.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/ob_value_row_iterator.h" #include "observer/vector_index/ob_ivf_async_task.h" @@ -65,11 +66,11 @@ ObVecIndexAsyncTaskOption::~ObVecIndexAsyncTaskOption() destroy(); } -int ObVecIndexAsyncTaskOption::init(const int64_t capacity, ObLSID &ls_id) +int ObVecIndexAsyncTaskOption::init(const int64_t capacity) { int ret = OB_SUCCESS; if (OB_FAIL(task_ctx_map_.create(capacity, mem_attr_.label_, mem_attr_.label_))) { - LOG_WARN("fail to create vector index task ctx map", K(ret), K(ls_id)); + LOG_WARN("fail to create vector index task ctx map", K(ret)); } return ret; } @@ -478,7 +479,7 @@ int ObVecIndexAsyncTaskUtil::resume_task_from_inner_table( ObSqlString sql; if (OB_FAIL(construct_read_task_sql(tname, for_update, is_read_tenant_async_task, filters, proxy, sql))) { - LOG_WARN("fail to construct read task sql", K(ret), K(ls->get_ls_id())); + LOG_WARN("fail to construct read task sql", K(ret)); } else { ObIAllocator *allocator = async_task_opt.get_allocator(); SMART_VAR(ObMySQLProxy::MySQLResult, res) { @@ -517,8 +518,7 @@ int ObVecIndexAsyncTaskUtil::resume_task_from_inner_table( need_resumed = false; } - LOG_INFO("resume task", K(ret), K(need_resumed), K(is_read_tenant_async_task), - K(OB_NOT_NULL(ls) ? ls->get_ls_id() : ObLSID()), K(task_result)); + LOG_INFO("resume task", K(ret), K(need_resumed), K(is_read_tenant_async_task), K(task_result)); if (OB_FAIL(ret) || !need_resumed) { // skip } else if (task_result.status_ != ObVecIndexAsyncTaskStatus::OB_VECTOR_ASYNC_TASK_FINISH) { // resume not finish task ObVecIndexAsyncTaskCtx *task_ctx = nullptr; @@ -861,7 +861,6 @@ void ObVecIndexAsyncTaskHandler::destroy() } int ObVecIndexAsyncTaskHandler::push_task( - const ObLSID &ls_id, ObVecIndexAsyncTaskCtx *ctx, ObIAllocator *allocator) { @@ -872,16 +871,16 @@ int ObVecIndexAsyncTaskHandler::push_task( LOG_WARN("handler is not init", KR(ret)); } else if (OB_UNLIKELY(!true || OB_ISNULL(ctx) || OB_ISNULL(allocator))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), KP(ctx), KP(allocator)); + LOG_WARN("invalid argument", K(ret), KP(ctx), KP(allocator)); } else if (ctx->task_status_.status_ != ObVecIndexAsyncTaskStatus::OB_VECTOR_ASYNC_TASK_PREPARE) { // skip not PREPARE status task } else if (ctx->task_status_.task_type_ == ObVecIndexAsyncTaskType::OB_VECTOR_ASYNC_INDEX_OPTINAL) { ObVecIndexAsyncTask *async_task = nullptr; if (OB_ISNULL(async_task = static_cast(allocator->alloc(sizeof(ObVecIndexAsyncTask))))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory of ObVecIndexAsyncTask", K(ret), K(ls_id)); + LOG_WARN("fail to alloc memory of ObVecIndexAsyncTask", K(ret)); } else if (FALSE_IT(async_task = new (async_task) ObVecIndexAsyncTask())) { - } else if (OB_FAIL(async_task->init(ls_id, ctx->task_status_.task_type_, ctx))) { - LOG_WARN("fail to init opt async task", KR(ret), K(ls_id)); + } else if (OB_FAIL(async_task->init(ctx->task_status_.task_type_, ctx))) { + LOG_WARN("fail to init opt async task", KR(ret)); } else if (OB_FAIL(common::ObSimpleThreadPool::push(async_task))) { LOG_WARN("fail to push task", KR(ret), KPC(async_task)); } else { @@ -896,16 +895,16 @@ int ObVecIndexAsyncTaskHandler::push_task( } if (OB_FAIL(ret)) { } else if (OB_NOT_NULL(async_task)) { - handle_ls_process_task_cnt(async_task->get_ls_id(), true); + update_processing_task_count(true); } } else if (ctx->task_status_.task_type_ == ObVecIndexAsyncTaskType::OB_VECTOR_ASYNC_HYBRID_VECTOR_EMBEDDING) { ObHybridVectorRefreshTask *task = nullptr; if (OB_ISNULL(task = static_cast(allocator->alloc(sizeof(ObHybridVectorRefreshTask))))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory of ObHybridVectorRefreshTask", K(ret), K(ls_id)); + LOG_WARN("fail to alloc memory of ObHybridVectorRefreshTask", K(ret)); } else if (FALSE_IT(task = new (task) ObHybridVectorRefreshTask())) { - } else if (OB_FAIL(task->init(ls_id, ctx->task_status_.task_type_, ctx))) { - LOG_WARN("fail to init opt async task", KR(ret), K(ls_id)); + } else if (OB_FAIL(task->init(ctx->task_status_.task_type_, ctx))) { + LOG_WARN("fail to init opt async task", KR(ret)); } else if (OB_FAIL(common::ObSimpleThreadPool::push(task))) { LOG_WARN("fail to push task", KR(ret), KPC(task)); } @@ -919,7 +918,7 @@ int ObVecIndexAsyncTaskHandler::push_task( } else if (OB_NOT_NULL(task) && (task->current_status() == ObHybridVectorRefreshTaskStatus::TASK_PREPARE || task->current_status() == ObHybridVectorRefreshTaskStatus::TASK_FINISH)) { task->reset_status(); - handle_ls_process_task_cnt(task->get_ls_id(), true); + update_processing_task_count(true); inc_async_task_ref(); } } else if (ctx->task_status_.task_type_ == OB_VECTOR_ASYNC_INDEX_IVF_LOAD @@ -927,9 +926,9 @@ int ObVecIndexAsyncTaskHandler::push_task( ObIvfAsyncTask *ivf_task = nullptr; if (OB_ISNULL(ivf_task = OB_NEWx(ObIvfAsyncTask, allocator))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory of ObIvfAsyncTask", K(ret), K(ls_id)); - } else if (OB_FAIL(ivf_task->init(ls_id, ctx->task_status_.task_type_, ctx))) { - LOG_WARN("fail to init opt async task", KR(ret), K(ls_id)); + LOG_WARN("fail to alloc memory of ObIvfAsyncTask", K(ret)); + } else if (OB_FAIL(ivf_task->init(ctx->task_status_.task_type_, ctx))) { + LOG_WARN("fail to init opt async task", KR(ret)); } else if (OB_FAIL(common::ObSimpleThreadPool::push(ivf_task))) { LOG_WARN("fail to push task", KR(ret), KPC(ivf_task)); } else { @@ -944,7 +943,7 @@ int ObVecIndexAsyncTaskHandler::push_task( } if (OB_FAIL(ret)) { } else if (OB_NOT_NULL(ivf_task)) { - handle_ls_process_task_cnt(ivf_task->get_ls_id(), true); + update_processing_task_count(true); } } else { ret = OB_ERR_UNEXPECTED; @@ -953,23 +952,16 @@ int ObVecIndexAsyncTaskHandler::push_task( return ret; } -int ObVecIndexAsyncTaskHandler::get_allocator_by_ls(const ObLSID &ls_id, ObIAllocator *&allocator) +int ObVecIndexAsyncTaskHandler::get_allocator(ObIAllocator *&allocator) { int ret = OB_SUCCESS; ObPluginVectorIndexService *vector_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexMgr *vec_idx_mgr = nullptr; - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ls id", K(ret), K(ls_id)); - } else if (OB_ISNULL(vector_index_service)) { + if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(vector_index_service)); - } else if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, vec_idx_mgr))) { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_id)); - } else if (OB_ISNULL(vec_idx_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), KP(vec_idx_mgr)); } else { + vec_idx_mgr = &vector_index_service->get_index_mgr(); ObVecIndexAsyncTaskOption &task_opt = vec_idx_mgr->get_async_task_opt(); allocator = task_opt.get_allocator(); } @@ -1019,7 +1011,7 @@ void ObVecIndexAsyncTaskHandler::handle(void *task) } if (OB_NOT_NULL(async_task) && (async_task->get_task_type() != ObVecIndexAsyncTaskType::OB_VECTOR_ASYNC_HYBRID_VECTOR_EMBEDDING || async_task->all_finished())) { - handle_ls_process_task_cnt(async_task->get_ls_id(), false); + update_processing_task_count(false); dec_async_task_ref(); } // free memory @@ -1027,11 +1019,11 @@ void ObVecIndexAsyncTaskHandler::handle(void *task) int tmp_ret = OB_SUCCESS; ObIAllocator *allocator = nullptr; async_task->~ObVecIndexIAsyncTask(); - if (OB_TMP_FAIL(get_allocator_by_ls(async_task->get_ls_id(), allocator))) { - LOG_WARN("fail to get allocator by ls id", K(tmp_ret), K(async_task->get_ls_id())); + if (OB_TMP_FAIL(get_allocator(allocator))) { + LOG_WARN("fail to get async task allocator", K(tmp_ret)); } else if (OB_ISNULL(allocator)) { tmp_ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null allocator", K(tmp_ret), K(async_task->get_ls_id())); + LOG_WARN("get null allocator", K(tmp_ret)); } else { allocator->free(async_task); } @@ -1053,17 +1045,17 @@ void ObVecIndexAsyncTaskHandler::handle_drop(void *task) async_task = static_cast(task); LOG_INFO("finish ObVecIndexAsyncTaskHandler::handle_drop", KPC(async_task)); if (OB_NOT_NULL(async_task)) { - handle_ls_process_task_cnt(async_task->get_ls_id(), false); + update_processing_task_count(false); } if (OB_NOT_NULL(async_task)) { int tmp_ret = OB_SUCCESS; ObIAllocator *allocator = nullptr; async_task->~ObVecIndexAsyncTask(); - if (OB_TMP_FAIL(get_allocator_by_ls(async_task->get_ls_id(), allocator))) { - LOG_WARN("fail to get allocator by ls id", K(tmp_ret), K(async_task->get_ls_id())); + if (OB_TMP_FAIL(get_allocator(allocator))) { + LOG_WARN("fail to get async task allocator", K(tmp_ret)); } else if (OB_ISNULL(allocator)) { tmp_ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null allocator", K(tmp_ret), K(async_task->get_ls_id())); + LOG_WARN("get null allocator", K(tmp_ret)); } else { allocator->free(async_task); } @@ -1073,32 +1065,25 @@ void ObVecIndexAsyncTaskHandler::handle_drop(void *task) } } -void ObVecIndexAsyncTaskHandler::handle_ls_process_task_cnt(const ObLSID &ls_id, const bool is_inc) +void ObVecIndexAsyncTaskHandler::update_processing_task_count(const bool is_inc) { int ret = OB_SUCCESS; ObPluginVectorIndexService *vector_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexMgr *vec_idx_mgr = nullptr; - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("[handle ls process task] invalid ls id", K(ret), K(ls_id)); - } else if (OB_ISNULL(vector_index_service)) { + if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[handle ls process task] unexpected nullptr", K(ret), KP(vector_index_service)); - } else if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, vec_idx_mgr))) { - LOG_WARN("[handle ls process task] fail to get vector index ls mgr", KR(ret), K(ls_id)); - } else if (OB_ISNULL(vec_idx_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[handle ls process task] unexpected nullptr", K(ret), KP(vec_idx_mgr)); } else if (is_inc) { - vec_idx_mgr->get_async_task_opt().inc_ls_task_cnt(); + vec_idx_mgr = &vector_index_service->get_index_mgr(); + vec_idx_mgr->get_async_task_opt().inc_processing_task_cnt(); } else { - vec_idx_mgr->get_async_task_opt().dec_ls_task_cnt(); + vec_idx_mgr = &vector_index_service->get_index_mgr(); + vec_idx_mgr->get_async_task_opt().dec_processing_task_cnt(); } } /**************************** ObVecIndexIAsyncTask ******************************/ int ObVecIndexIAsyncTask::init( - const ObLSID &ls_id, const int task_type, ObVecIndexAsyncTaskCtx *ctx) { @@ -1110,20 +1095,13 @@ int ObVecIndexIAsyncTask::init( } else if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(vector_index_service->get_ls_index_mgr_map().get_refactored(ls_id, - vec_idx_mgr_))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get vector index ls mgr", KR(ret), K(ls_id_)); - } - } else if (OB_UNLIKELY(!true || (!ls_id.is_valid()) || OB_ISNULL(ctx))) { + } else if (FALSE_IT(vec_idx_mgr_ = &vector_index_service->get_index_mgr())) { + } else if (OB_ISNULL(ctx)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), KP(ctx)); + LOG_WARN("invalid arguments", K(ret), KP(ctx)); } else { ctx_ = ctx; - ls_id_ = ls_id; task_type_ = task_type; is_inited_ = true; } @@ -1179,7 +1157,7 @@ int ObVecIndexAsyncTask::do_work() ObPluginVectorIndexAdapterGuard adpt_guard; ObPluginVectorIndexService *vector_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexAdaptor *new_adapter = nullptr; - LOG_INFO("start do_work", K(ret), K(ctx_->task_status_), K(ls_id_)); + LOG_INFO("start do_work", K(ret), K(ctx_->task_status_)); void *adpt_buff = nullptr; DEBUG_SYNC(HANDLE_VECTOR_INDEX_ASYNC_TASK); if (IS_NOT_INIT) { @@ -1190,7 +1168,7 @@ int ObVecIndexAsyncTask::do_work() LOG_WARN("unexpected nullptr", K(ret)); } else if (OB_ISNULL(vec_idx_mgr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid vector index ls mgr", KR(ret), K(ls_id_)); + LOG_WARN("get invalid vector index manager", KR(ret)); } else if (OB_FAIL(vec_idx_mgr_->get_adapter_inst_guard(ctx_->task_status_.tablet_id_, adpt_guard))) { if (OB_HASH_NOT_EXIST == ret) { ret = OB_EAGAIN; @@ -1334,8 +1312,7 @@ int ObVecIndexAsyncTask::process_data_for_index(ObPluginVectorIndexAdaptor &adap } else if (OB_ISNULL(vids = static_cast(allocator_.alloc(sizeof(int64_t) * VEC_INDEX_HNSWSQ_BUILD_COUNT_THRESHOLD)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc new mem.", K(ret)); - } else if (!is_hybrid_index && adaptor.get_is_need_vid() && OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (!is_hybrid_index && adaptor.get_is_need_vid() && OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, vid_table_type, allocator_, @@ -1344,8 +1321,7 @@ int ObVecIndexAsyncTask::process_data_for_index(ObPluginVectorIndexAdaptor &adap vid_table_param, vid_id_iter))) { LOG_WARN("failed to read vid id table local tablet.", K(ret)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, data_table_type, allocator_, @@ -1559,13 +1535,13 @@ int ObVecIndexAsyncTask::optimize_vector_index(ObPluginVectorIndexAdaptor &adapt bool trans_start = false; oceanbase::transaction::ObTransService *txs = share::g_mp->trans_service(); const uint64_t timeout_us = ObTimeUtility::current_time() + ObInsertLobColumnHelper::LOB_TX_TIMEOUT; - if (OB_FAIL(ObInsertLobColumnHelper::start_trans(ls_id_, false/*is_for_read*/, timeout_us, tx_desc))) { + if (OB_FAIL(ObInsertLobColumnHelper::start_trans(false/*is_for_read*/, timeout_us, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); } else if (FALSE_IT(trans_start = true)) { } else if (OB_ISNULL(tx_desc) || OB_ISNULL(txs)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get tx desc or ob access service, get nullptr", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ls_id_, timeout_us, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout_us, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else if (FALSE_IT(ctx_->task_status_.target_scn_ = snapshot.version())) { } else if (OB_FAIL(process_data_for_index(adaptor))) { @@ -1604,22 +1580,18 @@ int ObVecIndexAsyncTask::refresh_snapshot_index_data(ObPluginVectorIndexAdaptor { int ret = OB_SUCCESS; ObVecIdxSnapshotDataWriteCtx ctx; - ctx.ls_id_ = ls_id_; ctx.data_tablet_id_ = adaptor.get_data_tablet_id(); // get lob tablet id - HEAP_VARS_3((ObLSHandle, ls_handle), (ObTabletHandle, data_tablet_handle), (ObTabletBindingMdsUserData, ddl_data)) + HEAP_VARS_3((ObLS *, ls, nullptr), (ObTabletHandle, data_tablet_handle), (ObTabletBindingMdsUserData, ddl_data)) { ObLSService *ls_service = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ctx.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ctx.ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(ctx.data_tablet_id_, data_tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet(ctx.data_tablet_id_, data_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(ctx.data_tablet_id_)); } else if (OB_FAIL(data_tablet_handle.get_obj()->get_ddl_data(ddl_data))) { LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet_handle)); @@ -1671,8 +1643,7 @@ int ObVecIndexAsyncTask::refresh_snapshot_index_data(ObPluginVectorIndexAdaptor LOG_WARN("table schema not exist", K(ret), K(adaptor.get_snapshot_table_id()), K(adaptor.get_data_table_id()), KP(snapshot_table_schema), KP(data_table_schema)); } else if (FALSE_IT(lob_inrow_threshold = snapshot_table_schema->get_lob_inrow_threshold())) { - } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, allocator_, @@ -1864,11 +1835,11 @@ int ObVecIndexAsyncTask::refresh_snapshot_index_data(ObPluginVectorIndexAdaptor dml_param.dml_allocator_ = &allocator_; if (OB_ISNULL(adaptor.get_snap_data_()) || !adaptor.get_snap_data_()->is_inited()) { // adaptor created by vector index async task, there won't be access from other threads. LOG_INFO("data table is empty, won't create snapshot index"); - } else if (OB_FAIL(oas->get_write_store_ctx_guard(ls_id_, timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))) { + } else if (OB_FAIL(oas->get_write_store_ctx_guard(timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))) { LOG_WARN("failed to get write store context guard", K(ret)); } else { int64_t affected_rows = 0; - if (OB_FAIL(oas->insert_rows(ls_id_, adaptor.get_snap_tablet_id(), *tx_desc, dml_param, dml_column_ids, &row_iter, affected_rows))) { + if (OB_FAIL(oas->insert_rows(adaptor.get_snap_tablet_id(), *tx_desc, dml_param, dml_column_ids, &row_iter, affected_rows))) { LOG_WARN("failed to insert rows to snapshot table", K(ret), K(adaptor.get_snap_tablet_id())); } store_ctx_guard.reset(); @@ -1880,9 +1851,9 @@ int ObVecIndexAsyncTask::refresh_snapshot_index_data(ObPluginVectorIndexAdaptor dml_param.table_param_ = &table_delete_dml_param; int64_t affected_rows = 0; if (OB_FAIL(ret)) { - } else if (OB_FAIL(oas->get_write_store_ctx_guard(ls_id_, timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))){ + } else if (OB_FAIL(oas->get_write_store_ctx_guard(timeout_us, *tx_desc, snapshot, 0, dml_param.write_flag_, store_ctx_guard))){ LOG_WARN("failed to get write store context guard", K(ret)); - } else if (OB_FAIL(oas->delete_rows(ls_id_, adaptor.get_snap_tablet_id(), *tx_desc, dml_param, dml_column_ids, &delete_row_iter, affected_rows))) { + } else if (OB_FAIL(oas->delete_rows(adaptor.get_snap_tablet_id(), *tx_desc, dml_param, dml_column_ids, &delete_row_iter, affected_rows))) { LOG_WARN("failed to delete rows from snapshot table", K(ret), K(adaptor.get_snap_tablet_id())); } delete_row_iter.reset(); @@ -2054,7 +2025,7 @@ int ObVecIndexAsyncTask::delete_tablet_data( if (OB_FAIL(ret)) { } else if (cur_row_count == 0) { delete_unfinish = false; - } else if (OB_FAIL(oas->delete_rows(ls_id_, tablet_id, *tx_desc, dml_param, dml_column_ids, &row_iter, delta_table_affected_rows))) { + } else if (OB_FAIL(oas->delete_rows(tablet_id, *tx_desc, dml_param, dml_column_ids, &row_iter, delta_table_affected_rows))) { LOG_WARN("failed to delete rows from delta table", K(ret), K(tablet_id)); } else if (delta_table_affected_rows != cur_row_count) { LOG_WARN("delete rows count unexpected", K(cur_row_count), K(delta_table_affected_rows)); @@ -2108,8 +2079,7 @@ int ObVecIndexAsyncTask::delete_incr_table_data(ObPluginVectorIndexAdaptor &adap } else if (OB_ISNULL(index_table_schema) || index_table_schema->is_in_recyclebin()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("vector index table not exist", K(ret), K(adaptor.get_vbitmap_table_id())); - } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL, allocator_, @@ -2120,8 +2090,7 @@ int ObVecIndexAsyncTask::delete_incr_table_data(ObPluginVectorIndexAdaptor &adap &delta_dml_column_ids, true))) { LOG_WARN("failed to read vid id table local tablet.", K(ret)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(ls_id_, - &adaptor, + } else if (OB_FAIL(ObPluginVectorIndexUtils::read_local_tablet(&adaptor, ctx_->task_status_.target_scn_, INDEX_TYPE_VEC_INDEX_ID_LOCAL, allocator_, diff --git a/src/observer/vector_index/ob_vector_index_async_task_util.h b/src/observer/vector_index/ob_vector_index_async_task_util.h index 71819032a..ae32ad771 100644 --- a/src/observer/vector_index/ob_vector_index_async_task_util.h +++ b/src/observer/vector_index/ob_vector_index_async_task_util.h @@ -22,7 +22,6 @@ #include "lib/container/ob_array.h" #include "common/ob_tablet_id.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "lib/thread/ob_simple_thread_pool.h" #include "storage/access/ob_dml_param.h" #include "storage/tx/ob_trans_define_v4.h" @@ -203,21 +202,21 @@ class ObVecIndexAsyncTaskOption ObVecIndexAsyncTaskOption() : mem_attr_("VecIdxATaskCtx"), allocator_(mem_attr_), - ls_task_cnt_(0), + processing_task_cnt_(0), stop_(false) { } ~ObVecIndexAsyncTaskOption(); - int init(const int64_t capacity, ObLSID &ls_id); + int init(const int64_t capacity); void destroy(); int add_task_ctx(ObTabletID &tablet_id, ObVecIndexAsyncTaskCtx *task, bool &inc_new_task); int del_task_ctx(ObTabletID &tablet_id); int is_task_ctx_exist(ObTabletID &tablet_id, bool &is_exist); - void inc_ls_task_cnt() { ATOMIC_INC(&ls_task_cnt_); } - void dec_ls_task_cnt() { ATOMIC_DEC(&ls_task_cnt_); } - int64_t get_ls_processing_task_cnt() const { return ATOMIC_LOAD(&ls_task_cnt_); } + void inc_processing_task_cnt() { ATOMIC_INC(&processing_task_cnt_); } + void dec_processing_task_cnt() { ATOMIC_DEC(&processing_task_cnt_); } + int64_t get_processing_task_cnt() const { return ATOMIC_LOAD(&processing_task_cnt_); } void set_stop() { stop_ = true; } bool is_stop() { return stop_; } VecIndexAsyncTaskMap &get_async_task_map() { return task_ctx_map_; } @@ -228,7 +227,7 @@ class ObVecIndexAsyncTaskOption ObMemAttr mem_attr_; VecIndexAsyncTaskMap task_ctx_map_; ObArenaAllocator allocator_; - volatile int64_t ls_task_cnt_; + volatile int64_t processing_task_cnt_; bool stop_; }; @@ -242,14 +241,14 @@ class ObVecIndexAsyncTaskHandler : public common::ObSimpleThreadPool int start(); void stop(); void destroy(); - int push_task(const ObLSID &ls_id, ObVecIndexAsyncTaskCtx *ctx, ObIAllocator *allocator); - int get_allocator_by_ls(const ObLSID &ls_id, ObIAllocator *&allocator); + int push_task(ObVecIndexAsyncTaskCtx *ctx, ObIAllocator *allocator); + int get_allocator(ObIAllocator *&allocator); bool is_inited() const { return is_inited_; } void inc_async_task_ref() { ATOMIC_INC(&async_task_ref_cnt_); } void dec_async_task_ref() { ATOMIC_DEC(&async_task_ref_cnt_); } int64_t get_async_task_ref() const { return ATOMIC_LOAD(&async_task_ref_cnt_); } - void handle_ls_process_task_cnt(const ObLSID &ls_id, const bool is_inc); + void update_processing_task_count(const bool is_inc); bool is_stopped() { return stopped_; } void set_stop() { stopped_ = true; } @@ -275,7 +274,6 @@ class ObVecIndexIAsyncTask ObVecIndexIAsyncTask(const ObMemAttr &mem_attr) : is_inited_(false), task_type_(ObVecIndexAsyncTaskType::OB_VECTOR_ASYNC_TASK_TYPE_INVALID), - ls_id_(ObLSID::INVALID_LS_ID), ctx_(nullptr), vec_idx_mgr_(nullptr), old_adapter_(nullptr), @@ -285,23 +283,20 @@ class ObVecIndexIAsyncTask all_finished_(false) {} virtual ~ObVecIndexIAsyncTask() {} - int init(const ObLSID &ls_id, const int task_type, - ObVecIndexAsyncTaskCtx *ctx); + int init(const int task_type, ObVecIndexAsyncTaskCtx *ctx); int get_task_type() { return task_type_; } - ObLSID &get_ls_id() { return ls_id_; } ObVecIndexAsyncTaskCtx *get_task_ctx() { return ctx_; } void set_old_adapter(ObPluginVectorIndexAdaptor* adapter) { old_adapter_ = adapter; } bool all_finished() { return all_finished_; } virtual void check_task_free() {} virtual int do_work() = 0; - VIRTUAL_TO_STRING_KV(K_(is_inited), K_(task_type), K_(ls_id), KPC(ctx_)); + VIRTUAL_TO_STRING_KV(K_(is_inited), K_(task_type), KPC(ctx_)); protected: bool is_inited_; int task_type_; // 0. built; 1. opt; 2. ivf load; 3. ivf clean - ObLSID ls_id_; ObVecIndexAsyncTaskCtx *ctx_; ObPluginVectorIndexMgr *vec_idx_mgr_; ObPluginVectorIndexAdaptor* old_adapter_; diff --git a/src/observer/vector_index/ob_vector_index_i_task_executor.cpp b/src/observer/vector_index/ob_vector_index_i_task_executor.cpp index 82f3156d3..dde3cf650 100644 --- a/src/observer/vector_index/ob_vector_index_i_task_executor.cpp +++ b/src/observer/vector_index/ob_vector_index_i_task_executor.cpp @@ -40,18 +40,14 @@ int ObVecITaskExecutor::init(storage::ObLS *ls) return ret; } -int ObVecITaskExecutor::get_index_ls_mgr(ObPluginVectorIndexMgr *&index_ls_mgr) +int ObVecITaskExecutor::get_index_mgr(ObPluginVectorIndexMgr *&index_mgr) { int ret = OB_SUCCESS; - index_ls_mgr = nullptr; if (OB_ISNULL(ls_) || OB_ISNULL(vector_index_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(ls_), KP(vector_index_service_)); - } else if (OB_FAIL(vector_index_service_->acquire_vector_index_mgr(ls_->get_ls_id(), index_ls_mgr))) { - LOG_WARN("fail to acquire vector index mgr", K(ret), K(ls_->get_ls_id())); - } else if (OB_ISNULL(index_ls_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), KP(index_ls_mgr)); + } else { + index_mgr = &vector_index_service_->get_index_mgr(); } return ret; } @@ -60,16 +56,16 @@ int ObVecITaskExecutor::get_index_ls_mgr(ObPluginVectorIndexMgr *&index_ls_mgr) int ObVecITaskExecutor::resume_task() { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector index load task not inited", KR(ret)); } else if (!check_operation_allow()) { // skip - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { // skip - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { // skip + LOG_WARN("fail to get index ls mgr", K(ret)); } else { const bool for_update = true; // select for update - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); ObVecIndexFieldArray filters; if (OB_FAIL(ObVecIndexAsyncTaskUtil::resume_task_from_inner_table( @@ -84,16 +80,16 @@ int ObVecITaskExecutor::load_task_from_inner_table() { int ret = OB_SUCCESS; ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector index load task not inited", KR(ret)); } else if (!check_operation_allow()) { // skip - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); - } else if (OB_ISNULL(index_ls_mgr) || OB_ISNULL(sql_proxy)) { + } else if (OB_FAIL(get_index_mgr(index_mgr))) { + LOG_WARN("fail to get index ls mgr", K(ret)); + } else if (OB_ISNULL(sql_proxy)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null pointer", K(ret), K(index_ls_mgr), K(sql_proxy)); + LOG_WARN("get unexpected null pointer", K(ret), K(sql_proxy)); } else { ObVecIndexFieldArray filters; ObVecIndexTaskStatusField field1; @@ -102,7 +98,7 @@ int ObVecITaskExecutor::load_task_from_inner_table() ObVecIndexTaskStatusField field2; field2.field_name_ = "status"; field2.data_.uint_ = ObVecIndexAsyncTaskStatus::OB_VECTOR_ASYNC_TASK_PREPARE; - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); if (OB_FAIL(filters.push_back(field1))) { LOG_WARN("fail to push back field", K(ret)); @@ -119,7 +115,7 @@ int ObVecITaskExecutor::load_task_from_inner_table() int ObVecITaskExecutor::start_task() { int ret = OB_SUCCESS; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector async task not init", K(ret)); @@ -127,11 +123,11 @@ int ObVecITaskExecutor::start_task() } else if (OB_ISNULL(vector_index_service_) || OB_ISNULL(ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(vector_index_service_), KP(ls_)); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { + LOG_WARN("fail to get index ls mgr", K(ret)); } else { ObVecIndexTaskCtxArray task_ctx_array; - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); FOREACH_X(iter, task_opt.get_async_task_map(), OB_SUCC(ret)) { ObTabletID tablet_id = iter->first; ObVecIndexAsyncTaskCtx *task_ctx = iter->second; @@ -150,12 +146,12 @@ int ObVecITaskExecutor::start_task() int tmp_ret = OB_SUCCESS; if (task_ctx->in_thread_pool_) { // skip push task LOG_DEBUG("task is in thread pool already", KPC(task_ctx)); - } else if (OB_FAIL(task_handle.push_task(ls_->get_ls_id(), task_ctx, task_opt.get_allocator()))) { - LOG_WARN("fail to push task to thread pool", K(ret), K(ls_->get_ls_id()), K(*task_ctx)); + } else if (OB_FAIL(task_handle.push_task(task_ctx, task_opt.get_allocator()))) { + LOG_WARN("fail to push task to thread pool", K(ret), K(*task_ctx)); } else if (FALSE_IT(task_ctx->in_thread_pool_ = true)) { } else if (OB_FAIL(update_status_and_ret_code(task_ctx))) { LOG_WARN("fail to update task status to inner table", - K(ret), K(ls_->get_ls_id()), K(*task_ctx)); + K(ret), K(*task_ctx)); } else if (task_ctx->sys_task_id_.is_invalid() && OB_TMP_FAIL(ObVecIndexAsyncTaskUtil::add_sys_task(task_ctx))) { LOG_WARN("add sys task failed", K(tmp_ret)); } @@ -165,7 +161,7 @@ int ObVecITaskExecutor::start_task() { if (OB_FAIL(update_status_and_ret_code(task_ctx))) { LOG_WARN("fail to update task status to inner table", - K(ret), K(ls_->get_ls_id()), K(*task_ctx)); + K(ret), K(*task_ctx)); } break; } @@ -174,7 +170,7 @@ int ObVecITaskExecutor::start_task() // update task status in inner table if (OB_FAIL(update_status_and_ret_code(task_ctx))) { LOG_WARN("fail to update task status to inner table", - K(ret), K(ls_->get_ls_id()), K(*task_ctx)); + K(ret), K(*task_ctx)); } else if (OB_FAIL(task_ctx_array.push_back(task_ctx))) { LOG_WARN("fail to push back task_ctx_array", K(ret), K(task_ctx)); } @@ -373,7 +369,7 @@ int ObVecITaskExecutor::insert_new_task(ObVecIndexTaskCtxArray &task_ctx_array) LOG_WARN("fail start transaction", K(ret)); } else if (OB_FAIL(ObVecIndexAsyncTaskUtil::batch_insert_vec_task( OB_ALL_VECTOR_INDEX_TASK_TNAME, trans, task_ctx_array))) { - LOG_WARN("fail to insert vec tasks", K(ret), K(ls_->get_ls_id())); + LOG_WARN("fail to insert vec tasks", K(ret)); } if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -391,14 +387,14 @@ int ObVecITaskExecutor::clear_old_task_ctx_if_need() { int ret = OB_SUCCESS; bool all_task_is_finish = true; - ObPluginVectorIndexMgr *index_ls_mgr = nullptr; + ObPluginVectorIndexMgr *index_mgr = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("vector async task not init", KR(ret)); - } else if (OB_FAIL(get_index_ls_mgr(index_ls_mgr))) { // skip - LOG_WARN("fail to get index ls mgr", K(ret), K(ls_->get_ls_id())); + } else if (OB_FAIL(get_index_mgr(index_mgr))) { // skip + LOG_WARN("fail to get index ls mgr", K(ret)); } else { - ObVecIndexAsyncTaskOption &task_opt = index_ls_mgr->get_async_task_opt(); + ObVecIndexAsyncTaskOption &task_opt = index_mgr->get_async_task_opt(); FOREACH_X(iter, task_opt.get_async_task_map(), OB_SUCC(ret) && all_task_is_finish) { ObTabletID tablet_id = iter->first; ObVecIndexAsyncTaskCtx *task_ctx = iter->second; @@ -411,7 +407,7 @@ int ObVecITaskExecutor::clear_old_task_ctx_if_need() all_task_is_finish = false; // break if has unfinish task } } - if (OB_SUCC(ret) && all_task_is_finish && (0 == task_opt.get_ls_processing_task_cnt())) { + if (OB_SUCC(ret) && all_task_is_finish && (0 == task_opt.get_processing_task_cnt())) { // all tasks is finish and task record in map should be removed expectedly. // when map size > 0, is not expected. if (task_opt.get_async_task_map().size() > 0) { @@ -429,14 +425,13 @@ int ObVecITaskExecutor::clear_old_task_ctx_if_need() } } } else { - index_ls_mgr->get_async_task_opt().get_allocator()->reset(); + index_mgr->get_async_task_opt().get_allocator()->reset(); LOG_DEBUG("reset vector async task ctx memory", K(ret), - K(index_ls_mgr->get_async_task_opt().get_allocator()), - K(ls_->get_ls_id()), K(all_task_is_finish)); + K(index_mgr->get_async_task_opt().get_allocator()), K(all_task_is_finish)); } } else { LOG_DEBUG("not reset vector async task ctx memory", - K(ret), K(all_task_is_finish), K(task_opt.get_ls_processing_task_cnt())); + K(ret), K(all_task_is_finish), K(task_opt.get_processing_task_cnt())); } } return OB_SUCCESS; diff --git a/src/observer/vector_index/ob_vector_index_i_task_executor.h b/src/observer/vector_index/ob_vector_index_i_task_executor.h index 8fdfb2067..bdab9be0d 100644 --- a/src/observer/vector_index/ob_vector_index_i_task_executor.h +++ b/src/observer/vector_index/ob_vector_index_i_task_executor.h @@ -47,7 +47,7 @@ class ObVecITaskExecutor static const int64_t VEC_INDEX_TASK_MAX_RETRY_TIME = 3; // 200 static const int64_t MAX_ASYNC_TASK_PROCESSING_COUNT = 128; // the thread pool max paralell processing cnt is 8 - int get_index_ls_mgr(ObPluginVectorIndexMgr *&index_ls_mgr); + int get_index_mgr(ObPluginVectorIndexMgr *&index_mgr); virtual bool check_operation_allow() = 0; int update_status_and_ret_code(ObVecIndexAsyncTaskCtx *task_ctx); int clear_task_ctx(ObVecIndexAsyncTaskOption &task_opt, ObVecIndexAsyncTaskCtx *task_ctx); diff --git a/src/observer/vector_index/ob_vector_index_ivf_cache_mgr.h b/src/observer/vector_index/ob_vector_index_ivf_cache_mgr.h index 84d31e683..0ab131068 100644 --- a/src/observer/vector_index/ob_vector_index_ivf_cache_mgr.h +++ b/src/observer/vector_index/ob_vector_index_ivf_cache_mgr.h @@ -18,7 +18,6 @@ #define OCEANBASE_SHARE_IVF_CACHE_MGR_H_ #include "common/datum/ob_datum.h" #include "storage/ob_i_store.h" -#include "share/ob_ls_id.h" #include "share/rc/ob_tenant_base.h" #include "lib/oblog/ob_log_module.h" #include "ob_vector_index_util.h" diff --git a/src/observer/vector_index/ob_vector_index_ivf_cache_util.cpp b/src/observer/vector_index/ob_vector_index_ivf_cache_util.cpp index bdb6c7326..0459d17b1 100644 --- a/src/observer/vector_index/ob_vector_index_ivf_cache_util.cpp +++ b/src/observer/vector_index/ob_vector_index_ivf_cache_util.cpp @@ -73,7 +73,7 @@ int ObIvfCacheUtil::scan_and_write_ivf_cent_cache(ObPluginVectorIndexService &se return ret; } -int ObIvfCacheUtil::is_cache_writable(const ObLSID &ls_id, int64_t table_id, +int ObIvfCacheUtil::is_cache_writable(int64_t table_id, const ObTabletID &tablet_id, const ObVectorIndexParam &vec_param, int64_t dim, bool &is_writable) @@ -87,11 +87,9 @@ int ObIvfCacheUtil::is_cache_writable(const ObLSID &ls_id, int64_t table_id, if (OB_ISNULL(vector_index_service)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("unexpected nullptr", K(ret), KP(vector_index_service)); - } else if (OB_FAIL(vector_index_service->acquire_ivf_cache_mgr_guard( - ls_id, tablet_id, vec_param, dim, table_id, cache_guard))) { + } else if (OB_FAIL(vector_index_service->acquire_ivf_cache_mgr_guard(tablet_id, vec_param, dim, table_id, cache_guard))) { LOG_WARN("fail to acquire ivf cache mgr with vec param", K(ret), - K(ls_id), K(tablet_id), K(vec_param), K(dim)); diff --git a/src/observer/vector_index/ob_vector_index_ivf_cache_util.h b/src/observer/vector_index/ob_vector_index_ivf_cache_util.h index 8e41a44c4..bbb9fea40 100644 --- a/src/observer/vector_index/ob_vector_index_ivf_cache_util.h +++ b/src/observer/vector_index/ob_vector_index_ivf_cache_util.h @@ -26,7 +26,7 @@ namespace share class ObIvfCacheUtil { public: - static int is_cache_writable(const ObLSID &ls_id, int64_t table_id, const ObTabletID &tablet_id, + static int is_cache_writable(int64_t table_id, const ObTabletID &tablet_id, const ObVectorIndexParam &vec_param, int64_t dim, bool &is_writable); static int scan_and_write_ivf_cent_cache(ObPluginVectorIndexService &service, const ObTableID &table_id, const ObTabletID &tablet_id, diff --git a/src/observer/vector_index/ob_vector_index_util.cpp b/src/observer/vector_index/ob_vector_index_util.cpp index e27427805..15347c9f5 100644 --- a/src/observer/vector_index/ob_vector_index_util.cpp +++ b/src/observer/vector_index/ob_vector_index_util.cpp @@ -3938,7 +3938,6 @@ int ObVectorIndexUtil::get_vector_index_type( void ObVecIdxSnapshotDataWriteCtx::reset() { - ls_id_.reset(); data_tablet_id_.reset(); lob_meta_tablet_id_.reset(); lob_piece_tablet_id_.reset(); @@ -4826,11 +4825,11 @@ int ObVectorIndexUtil::check_rename_rebuild_confilt( LOG_WARN("failed to get index schema by name", K(ret), K(ori_index_name)); } else if (OB_NOT_NULL(orig_index_schema) && orig_index_schema->is_vec_index()) { bool has_rebuild_index_task = false; - if (OB_FAIL(rootserver::ObDDLTaskRecordOperator::check_has_index_or_mlog_task(trans, + if (OB_FAIL(rootserver::ObDDLTaskRecordOperator::check_has_index_task(trans, *orig_index_schema, origin_table_schema.get_table_id(), has_rebuild_index_task))) { - LOG_WARN("fail to check has index or mlog task", KR(ret), K(ori_index_name)); + LOG_WARN("fail to check has index task", KR(ret), K(ori_index_name)); } else if (has_rebuild_index_task) { ret = OB_NOT_SUPPORTED; LOG_WARN("not support to rename vector index while rebuild is in progress", KR(ret), K(ori_index_name)); diff --git a/src/observer/vector_index/ob_vector_index_util.h b/src/observer/vector_index/ob_vector_index_util.h index 6047828fc..5aeedd889 100644 --- a/src/observer/vector_index/ob_vector_index_util.h +++ b/src/observer/vector_index/ob_vector_index_util.h @@ -918,12 +918,10 @@ class ObVecIdxSnapshotDataWriteCtx final { public: ObVecIdxSnapshotDataWriteCtx() - : ls_id_(), data_tablet_id_(), lob_meta_tablet_id_(), lob_piece_tablet_id_(), + : data_tablet_id_(), lob_meta_tablet_id_(), lob_piece_tablet_id_(), vals_() {} ~ObVecIdxSnapshotDataWriteCtx() {} - ObLSID& get_ls_id() { return ls_id_; } - const ObLSID& get_ls_id() const { return ls_id_; } ObTabletID& get_data_tablet_id() { return data_tablet_id_; } const ObTabletID& get_data_tablet_id() const { return data_tablet_id_; } ObTabletID& get_lob_meta_tablet_id() { return lob_meta_tablet_id_; } @@ -932,9 +930,8 @@ class ObVecIdxSnapshotDataWriteCtx final const ObTabletID& get_lob_piece_tablet_id() const { return lob_piece_tablet_id_; } ObIArray& get_vals() { return vals_; } void reset(); - TO_STRING_KV(K(ls_id_), K(data_tablet_id_), K(lob_meta_tablet_id_), K(lob_piece_tablet_id_), K(vals_)); + TO_STRING_KV(K(data_tablet_id_), K(lob_meta_tablet_id_), K(lob_piece_tablet_id_), K(vals_)); public: - ObLSID ls_id_; ObTabletID data_tablet_id_; ObTabletID lob_meta_tablet_id_; ObTabletID lob_piece_tablet_id_; diff --git a/src/observer/virtual_table/ob_all_virtual_activity_metrics.cpp b/src/observer/virtual_table/ob_all_virtual_activity_metrics.cpp index 339747c84..d179b0843 100644 --- a/src/observer/virtual_table/ob_all_virtual_activity_metrics.cpp +++ b/src/observer/virtual_table/ob_all_virtual_activity_metrics.cpp @@ -26,9 +26,7 @@ namespace observer ObAllVirtualActivityMetric::ObAllVirtualActivityMetric() : ObVirtualTableScannerIterator(), current_pos_(0), - length_(0), - addr_(), - ip_buffer_() + length_(0) { } @@ -41,8 +39,6 @@ void ObAllVirtualActivityMetric::reset() { current_pos_ = 0; length_ = 0; - addr_.reset(); - ip_buffer_[0] = '\0'; ObVirtualTableScannerIterator::reset(); } diff --git a/src/observer/virtual_table/ob_all_virtual_activity_metrics.h b/src/observer/virtual_table/ob_all_virtual_activity_metrics.h index acb7d40f7..387b6e15c 100644 --- a/src/observer/virtual_table/ob_all_virtual_activity_metrics.h +++ b/src/observer/virtual_table/ob_all_virtual_activity_metrics.h @@ -26,10 +26,6 @@ namespace oceanbase { -namespace common -{ -class ObAddr; -} namespace observer { class ObAllVirtualActivityMetric : public common::ObVirtualTableScannerIterator @@ -51,7 +47,6 @@ class ObAllVirtualActivityMetric : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) { addr_ = addr; } private: int get_next_freezer_stat_(storage::ObTenantFreezerStat& stat); int prepare_start_to_read_(); @@ -59,8 +54,6 @@ class ObAllVirtualActivityMetric : public common::ObVirtualTableScannerIterator private: int64_t current_pos_; int64_t length_; - common::ObAddr addr_; - char ip_buffer_[common::OB_IP_STR_BUFF]; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualActivityMetric); }; diff --git a/src/observer/virtual_table/ob_all_virtual_apply_stat.cpp b/src/observer/virtual_table/ob_all_virtual_apply_stat.cpp index bc261b57c..041f99804 100644 --- a/src/observer/virtual_table/ob_all_virtual_apply_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_apply_stat.cpp @@ -26,44 +26,26 @@ int ObAllVirtualApplyStat::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { - auto func_iter_ls = [&](const logservice::ObApplyStatus &apply_status) -> int - { - int ret = OB_SUCCESS; - logservice::LSApplyStat apply_stat; - if (OB_FAIL(apply_status.stat(apply_stat))) { - SERVER_LOG(WARN, "apply_status stat failed", K(ret), K(apply_status)); - } else if (OB_FAIL(insert_stat_(apply_stat))) { - SERVER_LOG(WARN, "insert stat failed", K(ret), K(apply_stat)); - } else if (OB_FAIL(scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "iter apply stat faild", KR(ret), K(apply_stat)); - } else { - SERVER_LOG(INFO, "iter apply stat succ", K(apply_stat)); - } - return ret; - }; + logservice::LSApplyStat apply_stat; logservice::ObLogService *log_service = share::g_mp->log_service(); if (NULL == log_service) { - SERVER_LOG(INFO, "tenant has no ObLogService"); - } else if (OB_FAIL(log_service->iterate_apply(func_iter_ls))) { - SERVER_LOG(WARN, "iter ls failed", K(ret)); + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "tenant has no ObLogService", K(ret)); + } else if (OB_FAIL(log_service->stat_apply(apply_stat))) { + SERVER_LOG(WARN, "stat apply failed", K(ret)); + } else if (OB_FAIL(insert_stat_(apply_stat))) { + SERVER_LOG(WARN, "insert stat failed", K(ret), K(apply_stat)); } else { - SERVER_LOG(INFO, "iter ls succ", K(ret)); + SERVER_LOG(INFO, "stat apply success", K(apply_stat)); } if (OB_FAIL(ret)) { SERVER_LOG(WARN, "iter tenant failed", K(ret)); } else { - scanner_it_ = scanner_.begin(); start_to_read_ = true; - } - } - if (OB_SUCC(ret) && start_to_read_) { - if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get next row failed", K(ret)); - } - } else { row = &cur_row_; } + } else { + ret = OB_ITER_END; } return ret; } @@ -76,21 +58,9 @@ int ObAllVirtualApplyStat::insert_stat_(logservice::LSApplyStat &apply_stat) uint64_t col_id = output_column_ids_.at(i); switch (col_id) { case OB_APP_MIN_COLUMN_ID: - if (OB_FAIL(role_to_string(apply_stat.role_, role_str_, sizeof(role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(apply_stat)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - case OB_APP_MIN_COLUMN_ID + 1: cur_row_.cells_[i].set_uint64(apply_stat.end_lsn_.val_); break; - case OB_APP_MIN_COLUMN_ID + 2: - cur_row_.cells_[i].set_int(apply_stat.proposal_id_); - break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 1: cur_row_.cells_[i].set_int(apply_stat.pending_cnt_); break; default: diff --git a/src/observer/virtual_table/ob_all_virtual_apply_stat.h b/src/observer/virtual_table/ob_all_virtual_apply_stat.h index 87e04379f..39964dbb5 100644 --- a/src/observer/virtual_table/ob_all_virtual_apply_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_apply_stat.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_OB_ALL_VIRTUAL_APPLY_STAT_H_ #include "common/row/ob_row.h" -#include "observer/omt/ob_multi_tenant.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "logservice/applyservice/ob_log_apply_service.h" @@ -29,17 +28,10 @@ namespace observer { class ObAllVirtualApplyStat : public common::ObVirtualTableScannerIterator { -public: - explicit ObAllVirtualApplyStat(omt::ObMultiTenant *omt) : omt_(omt) {} public: virtual int inner_get_next_row(common::ObNewRow *&row); private: int insert_stat_(logservice::LSApplyStat &apply_stat); -private: - static const int64_t VARCHAR_32 = 32; - char role_str_[VARCHAR_32] = {'\0'}; - char ip_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; - omt::ObMultiTenant *omt_; }; } // namespace observer } // namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_checkpoint.cpp b/src/observer/virtual_table/ob_all_virtual_checkpoint.cpp index be30ca1a3..3707cc54e 100644 --- a/src/observer/virtual_table/ob_all_virtual_checkpoint.cpp +++ b/src/observer/virtual_table/ob_all_virtual_checkpoint.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_checkpoint.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -29,8 +30,7 @@ namespace observer ObAllVirtualCheckpointInfo::ObAllVirtualCheckpointInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_() + ls_(nullptr) { } @@ -41,49 +41,30 @@ ObAllVirtualCheckpointInfo::~ObAllVirtualCheckpointInfo() void ObAllVirtualCheckpointInfo::reset() { - ls_iter_guard_.reset(); + ls_ = nullptr; ob_checkpoint_iter_.reset(); - addr_.reset(); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualCheckpointInfo::get_next_ls_(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - if (ls_iter_guard_.get_ptr() == nullptr - && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); - } else if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } - - return ret; -} - int ObAllVirtualCheckpointInfo::prepare_to_read_() { int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObArray infos; ob_checkpoint_iter_.reset(); - if (OB_FAIL(get_next_ls_(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } else if (NULL == ls) { + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ret), K(ls)); + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); } else if (FALSE_IT(infos.reset())) { - } else if (OB_FAIL(ls->get_checkpoint_info(infos))) { - SERVER_LOG(WARN, "get checkpoint info failed", K(ret), KPC(ls)); + } else if (OB_FAIL(ls_->get_checkpoint_info(infos))) { + SERVER_LOG(WARN, "get checkpoint info failed", K(ret), KPC(ls_)); } else { int64_t idx = 0; for (; idx < infos.count() && OB_SUCC(ret); ++idx) { if (OB_FAIL(ob_checkpoint_iter_.push(infos.at(idx)))) { - SERVER_LOG(ERROR, "ob_checkpoint_iter push failed", K(ret), KPC(ls)); + SERVER_LOG(ERROR, "ob_checkpoint_iter push failed", K(ret), KPC(ls_)); } } } @@ -102,25 +83,12 @@ int ObAllVirtualCheckpointInfo::prepare_to_read_() int ObAllVirtualCheckpointInfo::get_next_(ObCheckpointVTInfo &checkpoint) { int ret = OB_SUCCESS; - // ensure inner_get_next_row can get new data - bool need_retry = true; - while (need_retry) { - if (!ob_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { - if (OB_ITER_END == ret) { - SERVER_LOG(DEBUG, "iterate checkpoint info iter end", K(ret)); - } else { - SERVER_LOG(WARN, "prepare data failed", K(ret)); - } - } else if (OB_FAIL(ob_checkpoint_iter_.get_next(checkpoint))) { - if (OB_ITER_END == ret) { - ob_checkpoint_iter_.reset(); - SERVER_LOG(DEBUG, "iterate checkpoint info iter in the ls end", K(ret)); - continue; - } else { - SERVER_LOG(WARN, "get next checkpoint info error.", K(ret)); - } + if (!ob_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { + SERVER_LOG(WARN, "prepare data failed", K(ret)); + } else if (OB_FAIL(ob_checkpoint_iter_.get_next(checkpoint))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "get next checkpoint info error.", K(ret)); } - need_retry = false; } return ret; } diff --git a/src/observer/virtual_table/ob_all_virtual_checkpoint.h b/src/observer/virtual_table/ob_all_virtual_checkpoint.h index 5c6760776..9438695ce 100644 --- a/src/observer/virtual_table/ob_all_virtual_checkpoint.h +++ b/src/observer/virtual_table/ob_all_virtual_checkpoint.h @@ -19,12 +19,14 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "storage/checkpoint/ob_checkpoint_executor.h" -#include "storage/tx_storage/ob_ls_map.h" -#include "observer/omt/ob_multi_tenant.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { static constexpr const char OB_SERVICE_CHECKPOINT[] = "ob_service_checkpoint"; @@ -40,20 +42,12 @@ class ObAllVirtualCheckpointInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls_(ObLS *&ls); int prepare_to_read_(); int get_next_(storage::checkpoint::ObCheckpointVTInfo &checkpoint); private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; char service_type_buf_[common::MAX_SERVICE_TYPE_BUF_LENGTH]; - // These resources must be released in their own tenant - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; ObCheckpointVTIterator ob_checkpoint_iter_; private: diff --git a/src/observer/virtual_table/ob_all_virtual_dag.cpp b/src/observer/virtual_table/ob_all_virtual_dag.cpp index 22c7d225a..0b03e9579 100644 --- a/src/observer/virtual_table/ob_all_virtual_dag.cpp +++ b/src/observer/virtual_table/ob_all_virtual_dag.cpp @@ -157,10 +157,6 @@ int ObAllVirtualDag::fill_cells(share::ObDagInfo &dag_info) if (dag_info.dag_type_ >= ObDagType::DAG_TYPE_MINI_MERGE && dag_info.dag_type_ < ObDagType::DAG_TYPE_MAX) { cells[i].set_varchar(share::ObIDag::get_dag_type_str(dag_info.dag_type_)); cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } else if (dag_info.dag_net_type_ >= ObDagNetType::DAG_NET_TYPE_MIGRATION - && dag_info.dag_net_type_ < ObDagNetType::DAG_NET_TYPE_MAX) { - cells[i].set_varchar(share::ObIDagNet::get_dag_net_type_str(dag_info.dag_net_type_)); - cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); } else { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "unexpected dag info, ", K(ret), K(dag_info)); diff --git a/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.cpp b/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.cpp index a67e76abb..cff7ad946 100644 --- a/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.cpp @@ -14,17 +14,14 @@ * limitations under the License. */ -#include "observer/ob_server_struct.h" #include "ob_all_virtual_dump_tenant_info.h" #include "observer/omt/ob_tenant.h" namespace oceanbase { -using namespace lib; namespace observer { ObAllVirtualDumpTenantInfo::ObAllVirtualDumpTenantInfo() - : is_inited_(false) { } @@ -35,176 +32,112 @@ ObAllVirtualDumpTenantInfo::~ObAllVirtualDumpTenantInfo() int ObAllVirtualDumpTenantInfo::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - auto func = [&] (omt::ObTenant &t) { - int ret = OB_SUCCESS; - const int64_t col_count = output_column_ids_.count(); - ObObj *cells = cur_row_.cells_; - // Column order after removing svr_ip and svr_port: - // OB_APP_MIN_COLUMN_ID (16): compat_mode - // OB_APP_MIN_COLUMN_ID + 1 (17): unit_min_cpu - // ... and so on - for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { - uint64_t col_id = output_column_ids_.at(i); - switch (col_id) { - case OB_APP_MIN_COLUMN_ID: - //compat_mode - cells[i].set_int(static_cast(t.get_compat_mode())); - break; - case OB_APP_MIN_COLUMN_ID + 1: - //unit_min_cpu - cells[i].set_double(t.unit_min_cpu_); - break; - case OB_APP_MIN_COLUMN_ID + 2: - //unit_max_cpu - cells[i].set_double(t.unit_max_cpu_); - break; - case OB_APP_MIN_COLUMN_ID + 3: - //slice - cells[i].set_double(0); - break; - case OB_APP_MIN_COLUMN_ID + 4: - //slice_remain - cells[i].set_double(0); - break; - case OB_APP_MIN_COLUMN_ID + 5: - //token_cnt - cells[i].set_int(t.worker_count()); - break; - case OB_APP_MIN_COLUMN_ID + 6: - //ass_token_cnt - cells[i].set_int(t.worker_count()); - break; - case OB_APP_MIN_COLUMN_ID + 7: - //lq_tokens - cells[i].set_int(0); - break; - case OB_APP_MIN_COLUMN_ID + 8: - //used_lq_tokens - cells[i].set_int(0); - break; - case OB_APP_MIN_COLUMN_ID + 9: - //stopped - cells[i].set_int(t.stopped_); - break; - case OB_APP_MIN_COLUMN_ID + 10: - //idle_us - cells[i].set_int(0); - break; - case OB_APP_MIN_COLUMN_ID + 11: - //recv_hp_rpc_cnt - cells[i].set_int(t.recv_hp_rpc_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 12: - //recv_np_rpc_cnt - cells[i].set_int(t.recv_np_rpc_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 13: - //recv_lp_rpc_cnt - cells[i].set_int(t.recv_lp_rpc_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 14: - //recv_mysql_cnt - cells[i].set_int(t.recv_mysql_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 15: - //recv_task_cnt - cells[i].set_int(t.recv_task_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 16: - //recv_large_req_cnt - cells[i].set_int(t.recv_large_req_cnt_); - break; - case OB_APP_MIN_COLUMN_ID + 17: - //tt_large_quries - cells[i].set_int(t.tt_large_quries_); - break; - case OB_APP_MIN_COLUMN_ID + 18: - //actives - cells[i].set_int(t.workers_.get_size()); - break; - case OB_APP_MIN_COLUMN_ID + 19: - //workers - cells[i].set_int(t.workers_.get_size()); - break; - case OB_APP_MIN_COLUMN_ID + 20: - //lq_waiting_workers - cells[i].set_int(0); - break; - case OB_APP_MIN_COLUMN_ID + 21: - //req_queue_total_size - cells[i].set_int(t.req_queue_.size()); - break; - case OB_APP_MIN_COLUMN_ID + 22: - //queue_0 - cells[i].set_int(t.req_queue_.queue_size(0)); - break; - case OB_APP_MIN_COLUMN_ID + 23: - //queue_1 - cells[i].set_int(t.req_queue_.queue_size(1)); - break; - case OB_APP_MIN_COLUMN_ID + 24: - //queue_2 - cells[i].set_int(t.req_queue_.queue_size(2)); - break; - case OB_APP_MIN_COLUMN_ID + 25: - //queue_3 - cells[i].set_int(t.req_queue_.queue_size(3)); - break; - case OB_APP_MIN_COLUMN_ID + 26: - //queue_4 - cells[i].set_int(t.req_queue_.queue_size(4)); - break; - case OB_APP_MIN_COLUMN_ID + 27: - //queue_5 - cells[i].set_int(t.req_queue_.queue_size(5)); - break; - case OB_APP_MIN_COLUMN_ID + 28: - //large_queued - cells[i].set_int(t.lq_retry_queue_size()); - break; - default: - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "invalid column id, ", K(ret), K(col_id)); - } - } - if (OB_SUCC(ret)) { - // The scanner supports up to 64M, so the overflow situation is not considered for the time being - if (OB_FAIL(scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "fail to add row", K(ret), K(cur_row_)); - } - } - return ret; - }; - - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_ISNULL(omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "nullptr", K(ret)); - } else { - omt::ObTenant *the_tenant = nullptr; - if (OB_SUCCESS == omt->get_tenant(the_tenant) && OB_NOT_NULL(the_tenant)) { - ret = func(*the_tenant); - } - if (OB_FAIL(ret)) { - SERVER_LOG(WARN, "run tenant func failed", K(ret)); - } else { - scanner_it_ = scanner_.begin(); - is_inited_ = true; + if (start_to_read_) { + ret = OB_ITER_END; + } else { + start_to_read_ = true; + omt::ObTenant &tenant = *static_cast(MTL_CTX()); + common::ObObj *cells = cur_row_.cells_; + for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); ++i) { + const uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case OB_APP_MIN_COLUMN_ID: + cells[i].set_int(static_cast(tenant.get_compat_mode())); + break; + case OB_APP_MIN_COLUMN_ID + 1: + cells[i].set_double(tenant.unit_min_cpu_); + break; + case OB_APP_MIN_COLUMN_ID + 2: + cells[i].set_double(tenant.unit_max_cpu_); + break; + case OB_APP_MIN_COLUMN_ID + 3: + cells[i].set_double(0); + break; + case OB_APP_MIN_COLUMN_ID + 4: + cells[i].set_double(0); + break; + case OB_APP_MIN_COLUMN_ID + 5: + cells[i].set_int(tenant.worker_count()); + break; + case OB_APP_MIN_COLUMN_ID + 6: + cells[i].set_int(tenant.worker_count()); + break; + case OB_APP_MIN_COLUMN_ID + 7: + cells[i].set_int(0); + break; + case OB_APP_MIN_COLUMN_ID + 8: + cells[i].set_int(0); + break; + case OB_APP_MIN_COLUMN_ID + 9: + cells[i].set_int(tenant.stopped_); + break; + case OB_APP_MIN_COLUMN_ID + 10: + cells[i].set_int(0); + break; + case OB_APP_MIN_COLUMN_ID + 11: + cells[i].set_int(tenant.recv_hp_rpc_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 12: + cells[i].set_int(tenant.recv_np_rpc_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 13: + cells[i].set_int(tenant.recv_lp_rpc_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 14: + cells[i].set_int(tenant.recv_mysql_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 15: + cells[i].set_int(tenant.recv_task_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 16: + cells[i].set_int(tenant.recv_large_req_cnt_); + break; + case OB_APP_MIN_COLUMN_ID + 17: + cells[i].set_int(tenant.tt_large_quries_); + break; + case OB_APP_MIN_COLUMN_ID + 18: + cells[i].set_int(tenant.workers_.get_size()); + break; + case OB_APP_MIN_COLUMN_ID + 19: + cells[i].set_int(tenant.workers_.get_size()); + break; + case OB_APP_MIN_COLUMN_ID + 20: + cells[i].set_int(0); + break; + case OB_APP_MIN_COLUMN_ID + 21: + cells[i].set_int(tenant.req_queue_.size()); + break; + case OB_APP_MIN_COLUMN_ID + 22: + cells[i].set_int(tenant.req_queue_.queue_size(0)); + break; + case OB_APP_MIN_COLUMN_ID + 23: + cells[i].set_int(tenant.req_queue_.queue_size(1)); + break; + case OB_APP_MIN_COLUMN_ID + 24: + cells[i].set_int(tenant.req_queue_.queue_size(2)); + break; + case OB_APP_MIN_COLUMN_ID + 25: + cells[i].set_int(tenant.req_queue_.queue_size(3)); + break; + case OB_APP_MIN_COLUMN_ID + 26: + cells[i].set_int(tenant.req_queue_.queue_size(4)); + break; + case OB_APP_MIN_COLUMN_ID + 27: + cells[i].set_int(tenant.req_queue_.queue_size(5)); + break; + case OB_APP_MIN_COLUMN_ID + 28: + cells[i].set_int(tenant.lq_retry_queue_size()); + break; + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id)); + break; } } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next row", K(ret)); - } - } else { + if (OB_SUCC(ret)) { row = &cur_row_; } } - return ret; } diff --git a/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.h b/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.h index fc6784f56..783022348 100644 --- a/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.h +++ b/src/observer/virtual_table/ob_all_virtual_dump_tenant_info.h @@ -29,8 +29,6 @@ class ObAllVirtualDumpTenantInfo : public common::ObVirtualTableScannerIterator virtual ~ObAllVirtualDumpTenantInfo(); virtual int inner_get_next_row(common::ObNewRow *&row); private: - char ip_buf_[common::OB_IP_STR_BUFF]; - bool is_inited_; DISALLOW_COPY_AND_ASSIGN(ObAllVirtualDumpTenantInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.cpp b/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.cpp index 59467efd3..9c2065a91 100644 --- a/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.cpp +++ b/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.cpp @@ -26,10 +26,8 @@ namespace observer ObAllVirtualDynamicPartitionTable::ObAllVirtualDynamicPartitionTable() : is_inited_(false), schema_service_(NULL), - batch_ids_(), - cur_tenant_table_ids_(), - t_loop_idx_(-1), - table_idx_(-1) + table_ids_(), + table_idx_(0) {} ObAllVirtualDynamicPartitionTable::~ObAllVirtualDynamicPartitionTable() @@ -49,10 +47,8 @@ int ObAllVirtualDynamicPartitionTable::init(share::schema::ObMultiVersionSchemaS } else { is_inited_ = true; schema_service_ = schema_service; - batch_ids_.reset(); - cur_tenant_table_ids_.reset(); - - table_idx_ = -1; + table_ids_.reset(); + table_idx_ = 0; } return ret; } @@ -61,10 +57,8 @@ void ObAllVirtualDynamicPartitionTable::destroy() { is_inited_ = false; schema_service_ = NULL; - batch_ids_.reset(); - cur_tenant_table_ids_.reset(); - - table_idx_ = -1; + table_ids_.reset(); + table_idx_ = 0; } int ObAllVirtualDynamicPartitionTable::inner_get_next_row(common::ObNewRow *&row) @@ -73,98 +67,49 @@ int ObAllVirtualDynamicPartitionTable::inner_get_next_row(common::ObNewRow *&row if (OB_ISNULL(schema_service_)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "schema service is null", KR(ret)); - } else if (t_loop_idx_ >= batch_ids_.count() || batch_ids_.empty()) { - ret = OB_ITER_END; } else { bool found_next_table = false; ObSchemaGetterGuard schema_guard; const ObTableSchema *table_schema = NULL; int64_t tenant_schema_version = OB_INVALID_VERSION; - do { - bool found_next_tenant = false; - do { - if (table_idx_ + 1 >= cur_tenant_table_ids_.count()) { - if (OB_FAIL(next_tenant_())) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "switch to next tenant failed", KR(ret)); - } - } - } else { - found_next_tenant = true; - } - } while (OB_SUCC(ret) && !found_next_tenant); - - if (OB_SUCC(ret)) { - do { - table_idx_ += 1; - - const uint64_t table_id = cur_tenant_table_ids_.at(table_idx_); - - if (OB_FAIL(schema_service_->get_tenant_schema_guard(schema_guard))) { - SERVER_LOG(WARN, "fail to get tenant schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( table_id, table_schema))) { - SERVER_LOG(WARN, "fail to get table schema", KR(ret), K(table_id)); - } else if (OB_ISNULL(table_schema) || !table_schema->with_dynamic_partition_policy()) { - // table may be dropped or altered, skip - SERVER_LOG(INFO, "table is altered, skip", KR(ret), K(table_id)); - } else { - found_next_table = true; - if (OB_FAIL(schema_guard.get_schema_version(tenant_schema_version))) { - SERVER_LOG(WARN, "fail to get schema version", KR(ret)); - } - } - } while (OB_SUCC(ret) && table_idx_ + 1 < cur_tenant_table_ids_.count() && !found_next_table); + if (OB_FAIL(schema_service_->get_tenant_schema_guard(schema_guard))) { + SERVER_LOG(WARN, "fail to get tenant schema guard", KR(ret)); + } + while (OB_SUCC(ret) && !found_next_table && table_idx_ < table_ids_.count()) { + const uint64_t table_id = table_ids_.at(table_idx_++); + table_schema = NULL; + if (OB_FAIL(schema_guard.get_table_schema(table_id, table_schema))) { + SERVER_LOG(WARN, "fail to get table schema", KR(ret), K(table_id)); + } else if (OB_ISNULL(table_schema) || !table_schema->with_dynamic_partition_policy()) { + // The table may have been dropped or altered after inner_open(). + SERVER_LOG(INFO, "table is altered, skip", K(table_id)); + } else if (OB_FAIL(schema_guard.get_schema_version(tenant_schema_version))) { + SERVER_LOG(WARN, "fail to get schema version", KR(ret)); + } else { + found_next_table = true; } - } while (OB_SUCC(ret) && !found_next_table); + } if (OB_SUCC(ret) && found_next_table) { - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "table schema is null", KR(ret)); - } else if (OB_UNLIKELY(!table_schema->with_dynamic_partition_policy())) { - SERVER_LOG(WARN, "table schema has no dynamic_partition_policy", KR(ret)); - } else if (OB_FAIL(build_new_row_(tenant_schema_version, *table_schema, row))) { + if (OB_FAIL(build_new_row_(tenant_schema_version, *table_schema, row))) { SERVER_LOG(WARN, "fail to build new row", KR(ret), K(table_schema)); } + } else if (OB_SUCC(ret)) { + ret = OB_ITER_END; } } return ret; } int ObAllVirtualDynamicPartitionTable::inner_open() -{ - int ret = OB_SUCCESS; - // single-tenant: no per-tenant batch list to build; batch_ids_ stays empty. - if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "schema service is null", KR(ret)); - } - - if (FAILEDx(next_tenant_())) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "switch to next tenant failed", KR(ret)); - } else { - ret = OB_SUCCESS; - } - } - - return ret; -} - -int ObAllVirtualDynamicPartitionTable::next_tenant_() { int ret = OB_SUCCESS; if (OB_ISNULL(schema_service_)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "schema service is null", KR(ret)); - } else if (t_loop_idx_ + 1 >= batch_ids_.count()) { - ret = OB_ITER_END; } else { - - table_idx_ = -1; - cur_tenant_table_ids_.reset(); - - + table_idx_ = 0; + table_ids_.reset(); ObSchemaGetterGuard schema_guard; ObArray table_schemas; if (OB_FAIL(schema_service_->get_tenant_schema_guard(schema_guard))) { @@ -182,11 +127,11 @@ int ObAllVirtualDynamicPartitionTable::next_tenant_() || table_schema->is_in_recyclebin() || !table_schema->is_normal_schema()) { // skip - } else if (OB_FAIL(schema_guard.get_database_schema( table_schema->get_database_id(), database_schema))) { + } else if (OB_FAIL(schema_guard.get_database_schema(table_schema->get_database_id(), database_schema))) { SERVER_LOG(WARN, "fail to get database schema", KR(ret), K(table_schema->get_database_id())); } else if (OB_ISNULL(database_schema) || database_schema->is_in_recyclebin()) { // skip - } else if (OB_FAIL(cur_tenant_table_ids_.push_back(table_schema->get_table_id()))) { + } else if (OB_FAIL(table_ids_.push_back(table_schema->get_table_id()))) { SERVER_LOG(WARN, "fail to push back table_id", KR(ret), K(table_schema->get_table_id())); } } @@ -206,11 +151,7 @@ int ObAllVirtualDynamicPartitionTable::build_new_row_( if (OB_ISNULL(cells) || OB_ISNULL(allocator_) || OB_ISNULL(schema_service_)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "cells or allocator or schema service is null", KR(ret)); - } else if (t_loop_idx_ >= batch_ids_.count()) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "tenant idx out of bound", KR(ret)); } else { - const int64_t col_count = output_column_ids_.count(); for (int64_t cell_idx = 0; OB_SUCC(ret) && cell_idx < col_count; cell_idx++) { uint64_t col_id = output_column_ids_.at(cell_idx); diff --git a/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.h b/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.h index f323340af..cdb3ff7b8 100644 --- a/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.h +++ b/src/observer/virtual_table/ob_all_virtual_dynamic_partition_table.h @@ -33,7 +33,6 @@ class ObAllVirtualDynamicPartitionTable : public common::ObVirtualTableScannerIt virtual int inner_open(); virtual int inner_get_next_row(common::ObNewRow *&row); private: - int next_tenant_(); int build_new_row_(const int64_t tenant_schema_version, const share::schema::ObTableSchema &table_schema, common::ObNewRow *&row); @@ -56,9 +55,7 @@ class ObAllVirtualDynamicPartitionTable : public common::ObVirtualTableScannerIt private: bool is_inited_; share::schema::ObMultiVersionSchemaService *schema_service_; - common::ObArray batch_ids_; - common::ObArray cur_tenant_table_ids_; - int64_t t_loop_idx_; + common::ObArray table_ids_; int64_t table_idx_; }; } // namespace observer diff --git a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp index 780192e8c..450144bc3 100644 --- a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp @@ -26,56 +26,26 @@ int ObAllVirtualHADiagnose::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { - auto func_iter_ls = [&](const storage::ObLS &ls) -> int - { - int ret = OB_SUCCESS; - storage::DiagnoseInfo diagnose_info; - if (OB_FAIL(ls.diagnose(diagnose_info))) { - if (OB_ENTRY_NOT_EXIST == ret) { - SERVER_LOG(WARN, "ls may have been removed, just skip", K(ls)); - ret = OB_SUCCESS; - } else if (OB_NOT_RUNNING == ret) { - SERVER_LOG(WARN, "ls may be during rebalancing ", K(ls)); - ret = OB_SUCCESS; - } else { - SERVER_LOG(WARN, "ls stat diagnose info failed", K(ret), K(ls)); - } - } else if (OB_FAIL(insert_stat_(diagnose_info))) { - SERVER_LOG(WARN, "insert stat failed", K(ret), K(diagnose_info)); - } else if (OB_FAIL(scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "iter diagnose info faild", KR(ret), K(diagnose_info)); - } else { - SERVER_LOG(INFO, "iter diagnose info succ", K(diagnose_info)); - } - return ret; - }; + storage::DiagnoseInfo diagnose_info; storage::ObLSService *ls_service = share::g_mp->ls_service(); if (NULL == ls_service) { - SERVER_LOG(INFO, "tenant has no ObLSService"); - } else if (OB_FAIL(ls_service->iterate_diagnose(func_iter_ls))) { - if (OB_NOT_RUNNING == ret) { - ret = OB_SUCCESS; - } else { - SERVER_LOG(WARN, "iter ls failed", K(ret)); - } + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->diagnose(diagnose_info))) { + SERVER_LOG(WARN, "diagnose ls failed", K(ret)); + } else if (OB_FAIL(insert_stat_(diagnose_info))) { + SERVER_LOG(WARN, "insert stat failed", K(ret), K(diagnose_info)); } else { - SERVER_LOG(INFO, "iter ls succ", K(ret)); + SERVER_LOG(INFO, "diagnose ls success", K(diagnose_info)); } if (OB_FAIL(ret)) { SERVER_LOG(WARN, "iter tenant failed", K(ret)); } else { - scanner_it_ = scanner_.begin(); start_to_read_ = true; - } - } - if (OB_SUCC(ret) && start_to_read_) { - if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get next row failed", K(ret)); - } - } else { row = &cur_row_; } + } else { + ret = OB_ITER_END; } return ret; } @@ -87,76 +57,15 @@ int ObAllVirtualHADiagnose::insert_stat_(storage::DiagnoseInfo &diagnose_info) for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { uint64_t col_id = output_column_ids_.at(i); switch (col_id) { - case ELECTION_ROLE: - if (OB_FAIL(role_to_string(diagnose_info.palf_diagnose_info_.election_role_, - election_role_str_, sizeof(election_role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(diagnose_info)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(election_role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - case ELECTION_EPOCH: - cur_row_.cells_[i].set_int(diagnose_info.palf_diagnose_info_.election_epoch_); - break; - case PALF_ROLE: - if (OB_FAIL(role_to_string(diagnose_info.palf_diagnose_info_.palf_role_, - palf_role_str_, sizeof(palf_role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(diagnose_info)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(palf_role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; case PALF_STATE: - cur_row_.cells_[i].set_varchar(ObString::make_string(replica_state_to_string(diagnose_info.palf_diagnose_info_.palf_state_))); + cur_row_.cells_[i].set_varchar(ObString::make_string(log_state_to_string(diagnose_info.palf_diagnose_info_.log_state_))); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( ObCharset::get_default_charset())); break; - case PALF_PROPOSAL_ID: - cur_row_.cells_[i].set_int(diagnose_info.palf_diagnose_info_.palf_proposal_id_); - break; - case LOG_HANDLER_ROLE: - if (OB_FAIL(role_to_string(diagnose_info.log_handler_diagnose_info_.log_handler_role_, - log_handler_role_str_, sizeof(log_handler_role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(diagnose_info)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(log_handler_role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - case LOG_HANDLER_PROPOSAL_ID: - cur_row_.cells_[i].set_int(diagnose_info.log_handler_diagnose_info_.log_handler_proposal_id_); - break; - case LOG_HANDLER_TAKEOVER_STATE: - if (OB_FAIL(takeover_state_to_string(diagnose_info.rc_diagnose_info_.state_, - log_handler_takeover_state_str_, - sizeof(log_handler_takeover_state_str_)))) { - SERVER_LOG(WARN, "takeover_state_to_string failed", K(ret), K(diagnose_info)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(log_handler_takeover_state_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - case LOG_HANDLER_TAKEOVER_LOG_TYPE: - if (OB_FAIL(log_base_type_to_string(diagnose_info.rc_diagnose_info_.log_type_, - log_handler_takeover_log_type_str_, - sizeof(log_handler_takeover_log_type_str_)))) { - SERVER_LOG(WARN, "log_base_type_to_string failed", K(ret), K(diagnose_info)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(log_handler_takeover_log_type_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; case MAX_APPLIED_SCN: cur_row_.cells_[i].set_uint64(diagnose_info.apply_diagnose_info_.max_applied_scn_.get_val_for_inner_table_field()); break; - case MAX_REPALYED_LSN: + case MAX_REPLAYED_LSN: cur_row_.cells_[i].set_uint64(diagnose_info.replay_diagnose_info_.max_replayed_lsn_.val_); break; case MAX_REPLAYED_SCN: @@ -167,18 +76,6 @@ int ObAllVirtualHADiagnose::insert_stat_(storage::DiagnoseInfo &diagnose_info) cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( ObCharset::get_default_charset())); break; - case GC_STATE: - cur_row_.cells_[i].set_varchar(ObString::make_string("")); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - case GC_START_TS: - cur_row_.cells_[i].set_int(-1); - break; - //TODO: @keqing.llt archive_scn column is currently just a placeholder - case ARCHIVE_SCN: - cur_row_.cells_[i].set_uint64(0); - break; case CHECKPOINT_SCN: cur_row_.cells_[i].set_uint64(diagnose_info.ls_clog_checkpoint_stat_.clog_checkpoint_scn_.get_val_for_inner_table_field()); break; @@ -196,48 +93,6 @@ int ObAllVirtualHADiagnose::insert_stat_(storage::DiagnoseInfo &diagnose_info) ObCharset::get_default_charset())); } break; - case RESTORE_HANDLER_ROLE: - cur_row_.cells_[i].set_varchar(ObString::make_string("")); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - case RESTORE_HANDLER_PROPOSAL_ID: - cur_row_.cells_[i].set_int(palf::INVALID_PROPOSAL_ID); - break; - case RESTORE_CONTEXT_INFO: - cur_row_.cells_[i].set_varchar(""); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - case RESTORE_ERR_CONTEXT_INFO: - cur_row_.cells_[i].set_varchar(""); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - case ENABLE_SYNC: - cur_row_.cells_[i].set_bool(diagnose_info.palf_diagnose_info_.enable_sync_); - break; - case ENABLE_VOTE: - cur_row_.cells_[i].set_bool(diagnose_info.palf_diagnose_info_.enable_vote_); - break; - case ARB_SRV_INFO: - cur_row_.cells_[i].set_varchar(ObString("")); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - case PARENT: - cur_row_.cells_[i].set_varchar(ObString("")); - if (diagnose_info.palf_diagnose_info_.parent_.is_valid()) { - if (OB_SUCC(diagnose_info.palf_diagnose_info_.parent_.\ - ip_port_to_string(parent_, common::OB_IP_PORT_STR_BUFF))) { - cur_row_.cells_[i].set_varchar(ObString::make_string(parent_)); - } else { - SERVER_LOG(WARN, "ip_port_to_string failed", K(ret), K(diagnose_info), K(parent_)); - } - } - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; case READ_TX: cur_row_.cells_[i].set_varchar(diagnose_info.read_only_tx_info_); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( diff --git a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.h b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.h index 797bf01ce..c9cda75d0 100644 --- a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.h +++ b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_OB_ALL_VIRTUAL_HA_DIAGNOSE_H_ #include "common/row/ob_row.h" -#include "observer/omt/ob_multi_tenant.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "storage/ls/ob_ls.h" @@ -29,57 +28,26 @@ namespace observer { enum IOStatColumn { - ELECTION_ROLE = common::OB_APP_MIN_COLUMN_ID, - ELECTION_EPOCH, - PALF_ROLE, - PALF_STATE, - PALF_PROPOSAL_ID, - LOG_HANDLER_ROLE, - LOG_HANDLER_PROPOSAL_ID, - LOG_HANDLER_TAKEOVER_STATE, - LOG_HANDLER_TAKEOVER_LOG_TYPE, + PALF_STATE = common::OB_APP_MIN_COLUMN_ID, MAX_APPLIED_SCN, - MAX_REPALYED_LSN, + MAX_REPLAYED_LSN, MAX_REPLAYED_SCN, REPLAY_DIAGNOSE_INFO, - GC_STATE, - GC_START_TS, - ARCHIVE_SCN, CHECKPOINT_SCN, MIN_REC_SCN, MIN_REC_SCN_LOG_TYPE, - RESTORE_HANDLER_ROLE, - RESTORE_HANDLER_PROPOSAL_ID, - RESTORE_CONTEXT_INFO, - RESTORE_ERR_CONTEXT_INFO, - ENABLE_SYNC, - ENABLE_VOTE, - ARB_SRV_INFO, - PARENT, READ_TX, }; class ObAllVirtualHADiagnose : public common::ObVirtualTableScannerIterator { -public: - explicit ObAllVirtualHADiagnose(omt::ObMultiTenant *omt) : omt_(omt) {} public: virtual int inner_get_next_row(common::ObNewRow *&row); private: int insert_stat_(storage::DiagnoseInfo &diagnose_info); private: static const int64_t VARCHAR_32 = 32; - char ip_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; - char election_role_str_[VARCHAR_32] = {'\0'}; - char palf_role_str_[VARCHAR_32] = {'\0'}; - char log_handler_role_str_[VARCHAR_32] = {'\0'}; - char log_handler_takeover_state_str_[VARCHAR_32] = {'\0'}; - char log_handler_takeover_log_type_str_[VARCHAR_32] = {'\0'}; - char gc_state_str_[VARCHAR_32] = {'\0'}; char min_rec_log_scn_log_type_str_[VARCHAR_32] = {'\0'}; - char restore_handler_role_str_[VARCHAR_32] = {'\0'}; - char parent_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; - omt::ObMultiTenant *omt_; }; } // namespace observer } // namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_id_service.cpp b/src/observer/virtual_table/ob_all_virtual_id_service.cpp index ca9ff129c..7a18a0632 100644 --- a/src/observer/virtual_table/ob_all_virtual_id_service.cpp +++ b/src/observer/virtual_table/ob_all_virtual_id_service.cpp @@ -16,7 +16,6 @@ #include "observer/virtual_table/ob_all_virtual_id_service.h" #include "share/rc/ob_module_provider.h" -#include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -30,37 +29,24 @@ namespace observer void ObAllVirtualIDService::reset() { - init_ = false; - service_types_index_ = -1; for(int i=0; ils_service()->check_ls_exist(IDS_LS, exist))) { - SERVER_LOG(WARN, "check ls exist fail", K(ret)); - } else if (!exist) { - ret = OB_LS_NOT_EXIST; + transaction::ObIDService *id_service = NULL; + if (OB_FAIL(transaction::ObIDService::get_id_service(service_type_[service_types_index_], id_service))) { + SERVER_LOG(WARN, "get id service fail", K(ret), K(service_type_), K(service_types_index_)); + } else if (OB_ISNULL(id_service)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "id service is null", K(ret), K(service_type_[service_types_index_])); } else { - transaction::ObIDService *id_service = NULL; - if (OB_FAIL(transaction::ObIDService::get_id_service(service_type_[service_types_index_], id_service))) { - SERVER_LOG(WARN, "get id service fail", K(ret), K(service_type_), K(service_types_index_)); - } else if (OB_ISNULL(id_service)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "id service is null", K(ret), K(service_type_[service_types_index_])); - } else { - id_service->get_virtual_info(last_id_, limit_id_, rec_log_ts_, latest_log_ts_, - pre_allocated_range_, submit_log_ts_, is_master_); - } + id_service->get_virtual_info(last_id_, limit_id_, rec_log_ts_, latest_log_ts_, + pre_allocated_range_, submit_log_ts_); } } } @@ -97,36 +76,19 @@ int ObAllVirtualIDService::get_next_info_() return ret; } -int ObAllVirtualIDService::fill_ids_() -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(GCTX.omt_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "failed to get multi tenant from GCTX", K(ret)); - } - - return ret; -} - int ObAllVirtualIDService::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; if (!start_to_read_ && OB_FAIL(prepare_start_to_read_())) { SERVER_LOG(WARN, "prepare start to read error", K(ret), K(start_to_read_)); - } else { - do { - if (OB_FAIL(get_next_info_())) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "ObAllVirtualIDService iter error", K(ret)); - } - } - } while (OB_TENANT_NOT_IN_SERVER == ret || OB_LS_NOT_EXIST == ret); + } else if (OB_FAIL(get_next_info_())) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "ObAllVirtualIDService iter error", K(ret)); + } } if (OB_SUCC(ret)) { SERVER_LOG(INFO, "ObAllVirtualIDService iter success", K(*this)); - const ObAddr self = GCTX.self_addr(); const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); @@ -161,10 +123,6 @@ int ObAllVirtualIDService::inner_get_next_row(ObNewRow *&row) cur_row_.cells_[i].set_int(submit_log_ts_); break; } - case OB_APP_MIN_COLUMN_ID + 7: { // is_master - cur_row_.cells_[i].set_bool(is_master_); - break; - } default: { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid coloum_id", K(ret), K(col_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_id_service.h b/src/observer/virtual_table/ob_all_virtual_id_service.h index 2324c3889..c62af0211 100644 --- a/src/observer/virtual_table/ob_all_virtual_id_service.h +++ b/src/observer/virtual_table/ob_all_virtual_id_service.h @@ -19,19 +19,9 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "storage/tx/ob_id_service.h" -#include "lib/container/ob_array.h" namespace oceanbase { -namespace share -{ -namespace schema -{ -class ObMultiVersionSchemaService; -class ObSchemaGetterGuard; -} -} - namespace observer { class ObAllVirtualIDService: public common::ObVirtualTableScannerIterator @@ -50,26 +40,19 @@ class ObAllVirtualIDService: public common::ObVirtualTableScannerIterator "rec_log_ts", rec_log_ts_, "latest_log_ts", latest_log_ts_, "pre_allocated_range", pre_allocated_range_, - "submit_log_ts", submit_log_ts_, - "is_master", is_master_); + "submit_log_ts", submit_log_ts_); private: int prepare_start_to_read_(); int get_next_info_(); - int fill_ids_(); private: - bool init_; int64_t service_types_index_; int64_t service_type_[transaction::ObIDService::MAX_SERVICE_TYPE]; - int64_t expire_time_; - char ip_buf_[common::OB_IP_STR_BUFF]; - int64_t last_id_; int64_t limit_id_; share::SCN rec_log_ts_; share::SCN latest_log_ts_; int64_t pre_allocated_range_; int64_t submit_log_ts_; - bool is_master_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualIDService); }; diff --git a/src/observer/virtual_table/ob_all_virtual_load_data_stat.cpp b/src/observer/virtual_table/ob_all_virtual_load_data_stat.cpp index 20376089c..a5734acf4 100644 --- a/src/observer/virtual_table/ob_all_virtual_load_data_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_load_data_stat.cpp @@ -23,7 +23,6 @@ namespace observer ObAllVirtualLoadDataStat::ObAllVirtualLoadDataStat() : ObVirtualTableScannerIterator(), - addr_(), all_job_status_op_() { } @@ -35,7 +34,6 @@ ObAllVirtualLoadDataStat::~ObAllVirtualLoadDataStat() void ObAllVirtualLoadDataStat::reset() { - addr_.reset(); all_job_status_op_.reset(); ObVirtualTableScannerIterator::reset(); diff --git a/src/observer/virtual_table/ob_all_virtual_load_data_stat.h b/src/observer/virtual_table/ob_all_virtual_load_data_stat.h index 7cb7bd41b..2811209da 100644 --- a/src/observer/virtual_table/ob_all_virtual_load_data_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_load_data_stat.h @@ -38,10 +38,6 @@ class ObAllVirtualLoadDataStat : public common::ObVirtualTableScannerIterator virtual void reset(); virtual int inner_open(); virtual int inner_close(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: int64_t calc_remaining_time(sql::ObLoadDataStat &job_status) const; @@ -81,12 +77,9 @@ class ObAllVirtualLoadDataStat : public common::ObVirtualTableScannerIterator STORE_TRANS_STATUS, MESSAGE }; - common::ObAddr addr_; char ip_buf_[common::OB_IP_STR_BUFF]; sql::ObGetAllJobStatusOp all_job_status_op_; - TO_STRING_KV(K(addr_)); - private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualLoadDataStat); }; diff --git a/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.cpp b/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.cpp index 598166a1b..8c6a0eddd 100644 --- a/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.cpp @@ -43,57 +43,17 @@ int ObAllVirtualLockWaitStat::inner_get_next_row(ObNewRow *&row) int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - if (OB_ISNULL(allocator_)) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (!start_to_read_ && OB_FAIL(make_this_ready_to_read())) { - SERVER_LOG(WARN, "prepare_start_to_read_ error", K(ret), K(start_to_read_)); - } else if (OB_ISNULL(node_iter_ = share::g_mp->lock_wait_mgr() + if (!start_to_read_) { + start_to_read_ = true; + } + if (OB_ISNULL(node_iter_ = share::g_mp->lock_wait_mgr() ->next(node_iter_, &cur_node_))) { ret = OB_ITER_END; } else { - int type = 0; // 1-TR 2-TX 3-TM - get_lock_type(node_iter_->hash_, type); + const int type = get_lock_type(node_iter_->hash_); const int64_t col_count = output_column_ids_.count(); - // resolve compatibility problem - const int column_id_fix_offset = BLOCK_SESSION_ID; - ObString column_name; - bool exist = false; - bool need_align = false; - if (OB_ISNULL(table_schema_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "table_schema of all_virtual_lock_wait_stat is NULL", K(ret)); - } else { - table_schema_->get_column_name_by_column_id(column_id_fix_offset, column_name, exist); - if (!exist) { - // no need align - } else if (column_name == "holder_session_id" || column_name == "HOLDER_SESSION_ID") { - /* - * ... - * | session_id | - * | holder_session_id | <---- is here in the first version of 4.3.2 - * | block_session_id | - * ... - * - * ... - * | trans_id | - * | holder_trans_id | - * | holder_session_id | <---- is here in later version - * ... - * */ - need_align = true; - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { - uint64_t col_id = output_column_ids_.at(i); - if (need_align) { - if (col_id > column_id_fix_offset) { - col_id -= 1; - } else if (col_id == column_id_fix_offset) { - col_id = HOLDER_SESSION_ID; - } - } + const uint64_t col_id = output_column_ids_.at(i); switch (col_id) { case TABLET_ID: cur_row_.cells_[i].set_int(node_iter_->tablet_id_); @@ -144,41 +104,24 @@ int ObAllVirtualLockWaitStat::inner_get_next_row(ObNewRow *&row) cur_row_.cells_[i].set_int(type); break; case LMODE: { - // For compatibility, column type should be determined by schema before cluster is in upgrade mode. - const ObColumnSchemaV2 *tmp_column_schema = nullptr; - bool type_is_varchar = true; - if (OB_ISNULL(table_schema_) || - OB_ISNULL(tmp_column_schema = table_schema_->get_column_schema(col_id))) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "table or column schema is null", KR(ret), KP(table_schema_), KP(tmp_column_schema)); + if (type == 1 || type == 2) { + cur_row_.cells_[i].set_varchar("X"); + } else if (type == 3) { + char lock_mode_tmp[MAX_LOCK_MODE_BUF_LENGTH]; + if (OB_FAIL(lock_mode_to_string(node_iter_->lock_mode_, + lock_mode_tmp, + sizeof(lock_mode_tmp)))) { + SERVER_LOG(WARN, "get lock mode failed", K(ret), K(node_iter_)); + } else { + snprintf(lock_mode_, sizeof(lock_mode_), "%s", lock_mode_tmp); + cur_row_.cells_[i].set_varchar(lock_mode_); + } } else { - type_is_varchar = tmp_column_schema->get_meta_type().is_character_type(); + cur_row_.cells_[i].set_varchar("UNKNOWN"); } if (OB_SUCC(ret)) { - if (type_is_varchar) { - if (type == 1 || type == 2) { - cur_row_.cells_[i].set_varchar("X"); - } else if (type == 3) { - char lock_mode_tmp[MAX_LOCK_MODE_BUF_LENGTH]; - if (OB_FAIL(lock_mode_to_string(node_iter_->lock_mode_, - lock_mode_tmp, - sizeof(lock_mode_tmp)))) { - SERVER_LOG(WARN, "get lock mode failed", K(ret), - K(node_iter_)); - } else { - snprintf(lock_mode_, sizeof(lock_mode_), "%s", - lock_mode_tmp); - cur_row_.cells_[i].set_varchar(lock_mode_); - } - } - cur_row_.cells_[i].set_collation_type( - ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } else { - // this column is invalid when the - // version of observer is before 4.2 - cur_row_.cells_[i].set_int(0); - } + cur_row_.cells_[i].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); } break; } @@ -216,35 +159,6 @@ int ObAllVirtualLockWaitStat::inner_get_next_row(ObNewRow *&row) case HOLDER_SESSION_ID: cur_row_.cells_[i].set_int(node_iter_->holder_sessid_); break; - case ASSOC_SESS_ID: { - cur_row_.cells_[i].set_int(0); - break; - } - case WAIT_TIMEOUT: { - cur_row_.cells_[i].set_int(0); - break; - } - case TX_ACTIVE_TS: { - cur_row_.cells_[i].set_int(0); - break; - } - case NODE_ID: { - cur_row_.cells_[i].set_int(0); - break; - } - case NODE_TYPE: { - cur_row_.cells_[i].set_int(0); - break; - } - case REMTOE_ADDR: { - cur_row_.cells_[i].set_varchar("0.0.0.0"); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - } - case IS_PLACEHOLDER: { - cur_row_.cells_[i].set_int(0); - break; - } default: ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid col_id", K(ret), K(col_id)); @@ -258,10 +172,9 @@ int ObAllVirtualLockWaitStat::inner_get_next_row(ObNewRow *&row) return ret; } -int ObAllVirtualLockWaitStat::get_lock_type(int64_t hash, int &type) +int ObAllVirtualLockWaitStat::get_lock_type(int64_t hash) const { - int ret = OB_SUCCESS; - type = 0; // invalid type + int type = 0; if (LockHashHelper::is_rowkey_hash(hash)) { type = 1; } else if (LockHashHelper::is_trans_hash(hash)) { @@ -269,7 +182,7 @@ int ObAllVirtualLockWaitStat::get_lock_type(int64_t hash, int &type) } else if (LockHashHelper::is_table_lock_hash(hash)) { type = 3; } - return ret; + return type; } int ObAllVirtualLockWaitStat::get_rowkey_holder(int64_t hash, transaction::ObTransID &holder) @@ -285,21 +198,5 @@ int ObAllVirtualLockWaitStat::get_rowkey_holder(int64_t hash, transaction::ObTra return ret; } -int ObAllVirtualLockWaitStat::make_this_ready_to_read() -{ - int ret = OB_SUCCESS; - ObObj *cells = NULL; - if (OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "invalid allocator is NULL", K(allocator_), K(ret)); - } else if (OB_ISNULL(cells = cur_row_.cells_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); - } else { - start_to_read_ = true; - } - return ret; -} - }/* ns observer*/ }/* ns oceanbase */ diff --git a/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.h b/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.h index b4615c81a..d28ad77c8 100644 --- a/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_lock_wait_stat.h @@ -35,9 +35,8 @@ class ObAllVirtualLockWaitStat : public common::ObVirtualTableScannerIterator int inner_get_next_row(common::ObNewRow *&row) override; void reset() override; private: - int get_lock_type(int64_t hash, int &type); + int get_lock_type(int64_t hash) const; int get_rowkey_holder(int64_t hash, transaction::ObTransID &holder); - int make_this_ready_to_read(); private: #ifdef _WIN32 #pragma push_macro("WAIT_TIMEOUT") @@ -62,13 +61,6 @@ class ObAllVirtualLockWaitStat : public common::ObVirtualTableScannerIterator TRANS_ID, HOLDER_TRANS_ID, HOLDER_SESSION_ID, - ASSOC_SESS_ID, - WAIT_TIMEOUT, - TX_ACTIVE_TS, - NODE_ID, - NODE_TYPE, - REMTOE_ADDR, - IS_PLACEHOLDER, }; #ifdef _WIN32 #pragma pop_macro("WAIT_TIMEOUT") @@ -77,7 +69,6 @@ class ObAllVirtualLockWaitStat : public common::ObVirtualTableScannerIterator rpc::ObLockWaitNode cur_node_; char rowkey_[common::MAX_LOCK_ROWKEY_BUF_LENGTH]; char lock_mode_[common::MAX_LOCK_MODE_BUF_LENGTH]; - char remote_addr_[common::MAX_LOCK_REMOTE_ADDR_BUF_LENGTH]; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualLockWaitStat); diff --git a/src/observer/virtual_table/ob_all_virtual_log_stat.cpp b/src/observer/virtual_table/ob_all_virtual_log_stat.cpp index 315c2c10d..73b794e30 100644 --- a/src/observer/virtual_table/ob_all_virtual_log_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_log_stat.cpp @@ -22,10 +22,6 @@ namespace oceanbase { namespace observer { -ObAllVirtualPalfStat::ObAllVirtualPalfStat(omt::ObMultiTenant *omt) : omt_(omt) -{ -} - ObAllVirtualPalfStat::~ObAllVirtualPalfStat() { destroy(); @@ -39,44 +35,26 @@ int ObAllVirtualPalfStat::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { - auto func_iterate_palf = [&](const palf::PalfHandle &palf_handle) -> int { - int ret = OB_SUCCESS; - logservice::ObLogStat log_stat; - int64_t palf_id = -1; - palf_handle.get_palf_id(palf_id); - if (OB_FAIL(palf_handle.stat(log_stat.palf_stat_))) { - SERVER_LOG(WARN, "PalfHandle stat failed", K(ret), K(palf_id)); - } else if (OB_FAIL(insert_log_stat_(log_stat, &cur_row_))){ - SERVER_LOG(WARN, "ObAllVirtualPalfStat insert_log_stat_ failed", K(ret), K(palf_id), K(log_stat)); - } else { - SERVER_LOG(TRACE, "iterate this log_stream success", K(palf_id), K(log_stat)); - scanner_.add_row(cur_row_); - } - return ret; - }; + logservice::ObLogStat log_stat; logservice::ObLogService *log_service = share::g_mp->log_service(); if (NULL == log_service) { - SERVER_LOG(INFO, "tenant has no ObLogService"); - } else if (OB_FAIL(log_service->iterate_palf(func_iterate_palf))) { - SERVER_LOG(WARN, "ObLogService iterate_palf failed", K(ret)); + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "tenant has no ObLogService", K(ret)); + } else if (OB_FAIL(log_service->stat_palf(log_stat.palf_stat_))) { + SERVER_LOG(WARN, "ObLogService stat_palf failed", K(ret)); + } else if (OB_FAIL(insert_log_stat_(log_stat, &cur_row_))) { + SERVER_LOG(WARN, "ObAllVirtualPalfStat insert_log_stat_ failed", K(ret), K(log_stat)); } else { - SERVER_LOG(TRACE, "itearte this tenant success"); + SERVER_LOG(TRACE, "stat palf success", K(log_stat)); } if (OB_FAIL(ret)) { SERVER_LOG(WARN, "iterate log stat failed", K(ret)); } else { - scanner_it_ = scanner_.begin(); start_to_read_ = true; - } - } - if (OB_SUCC(ret) && true == start_to_read_) { - if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "failed to get_next_row", K(ret)); - } - } else { row = &cur_row_; } + } else { + ret = OB_ITER_END; } return ret; } @@ -90,33 +68,6 @@ int ObAllVirtualPalfStat::insert_log_stat_(const logservice::ObLogStat &log_stat uint64_t col_id = output_column_ids_.at(i); switch (col_id) { case OB_APP_MIN_COLUMN_ID: { - if (OB_FAIL(role_to_string(palf_stat.role_, role_str_, sizeof(role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(palf_stat)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 1: { - cur_row_.cells_[i].set_int(palf_stat.log_proposal_id_); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - break; - } - case OB_APP_MIN_COLUMN_ID + 2: { - if (0 >= palf_stat.config_version_.to_string(config_version_buf_, - palf::LogConfigVersion::CONFIG_VERSION_LEN)) { - SERVER_LOG(WARN, "config_version_ to_string failed", K(ret), K(palf_stat)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(config_version_buf_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 3: { if (OB_FAIL(palf::access_mode_to_string(palf_stat.access_mode_, access_mode_str_, sizeof(access_mode_str_)))) { SERVER_LOG(WARN, "access_mode_to_string failed", K(ret), K(palf_stat)); } else { @@ -126,147 +77,34 @@ int ObAllVirtualPalfStat::insert_log_stat_(const logservice::ObLogStat &log_stat } break; } - case OB_APP_MIN_COLUMN_ID + 4: { - if (OB_FAIL(member_list_to_string_(palf_stat.paxos_member_list_))) { - SERVER_LOG(WARN, "memberlist to_string failed", K(ret), K(palf_stat)); - } else { - cur_row_.cells_[i].set_varchar(member_list_buf_.string()); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 5: { - cur_row_.cells_[i].set_int(palf_stat.paxos_replica_num_); - break; - } - case OB_APP_MIN_COLUMN_ID + 6: { - const bool is_in_sync = log_stat.palf_stat_.is_in_sync_; - cur_row_.cells_[i].set_bool(is_in_sync); - break; - } - case OB_APP_MIN_COLUMN_ID + 7: { + case OB_APP_MIN_COLUMN_ID + 1: { cur_row_.cells_[i].set_uint64(palf_stat.base_lsn_.val_); break; } - case OB_APP_MIN_COLUMN_ID + 8: { + case OB_APP_MIN_COLUMN_ID + 2: { cur_row_.cells_[i].set_uint64(palf_stat.begin_lsn_.val_); break; } - case OB_APP_MIN_COLUMN_ID + 9: { + case OB_APP_MIN_COLUMN_ID + 3: { cur_row_.cells_[i].set_uint64(palf_stat.begin_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 10: { + case OB_APP_MIN_COLUMN_ID + 4: { cur_row_.cells_[i].set_uint64(palf_stat.end_lsn_.val_); break; } - case OB_APP_MIN_COLUMN_ID + 11: { + case OB_APP_MIN_COLUMN_ID + 5: { cur_row_.cells_[i].set_uint64(palf_stat.end_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 12: { + case OB_APP_MIN_COLUMN_ID + 6: { cur_row_.cells_[i].set_uint64(palf_stat.max_lsn_.val_); break; } - case OB_APP_MIN_COLUMN_ID + 13: { + case OB_APP_MIN_COLUMN_ID + 7: { cur_row_.cells_[i].set_uint64(palf_stat.max_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 14: { - const ObAddr arb_server = palf_stat.arbitration_member_.get_server(); - if (arb_server.is_valid() - && OB_FAIL(arb_server.ip_port_to_string(arbitration_member_buf_, MAX_SINGLE_MEMBER_LENGTH))) { - SERVER_LOG(WARN, "ip_port_to_string failed", K(ret), K(palf_stat)); - } else { - if (!arb_server.is_valid()) { - memset(arbitration_member_buf_, 0, MAX_SINGLE_MEMBER_LENGTH); - } - cur_row_.cells_[i].set_varchar(ObString::make_string(arbitration_member_buf_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 15: { - if (OB_FAIL(learner_list_to_string_(palf_stat.degraded_list_, degraded_list_buf_))) { - SERVER_LOG(WARN, "learner list to_string failed", K(ret), K(palf_stat)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(degraded_list_buf_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 16: { - if (OB_FAIL(learner_list_to_string_(palf_stat.learner_list_, learner_list_buf_))) { - SERVER_LOG(WARN, "learner list to_string failed", K(ret), K(palf_stat)); - } else { - ObString learner_list_str = ObString::make_string(learner_list_buf_); - cur_row_.cells_[i].set_lob_value(ObLongTextType, learner_list_str.ptr(), static_cast(learner_list_str.length())); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - } - } - return ret; -} - -int ObAllVirtualPalfStat::member_list_to_string_( - const common::ObMemberList &member_list) -{ - int ret = OB_SUCCESS; - char ip_port[MAX_IP_PORT_LENGTH]; - common::ObMember member; - if (OB_UNLIKELY(1 != member_list.get_member_number())) { - ret = OB_INVALID_ARGUMENT; - SERVER_LOG(WARN, "invalid member list count", KR(ret)); - } else if (OB_FAIL(member_list.get_member_by_index(0/*index*/, member))) { - SERVER_LOG(WARN, "fail to get member by index", KR(ret)); - } else if (OB_FAIL(member.get_server().ip_port_to_string(ip_port, sizeof(ip_port)))) { - SERVER_LOG(WARN, "convert server to string failed", KR(ret), K(member)); - } else if (OB_FAIL(member_list_buf_.append_fmt("%.*s:%ld", static_cast(sizeof(ip_port)), ip_port, member.get_timestamp()))) { - SERVER_LOG(WARN, "failed to append ip_port to string", KR(ret), K(member)); - } - return ret; -} - -int ObAllVirtualPalfStat::learner_list_to_string_( - const common::GlobalLearnerList &learner_list, - char *output_buf) -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - char buf[MAX_IP_PORT_LENGTH]; - memset(output_buf, 0, MAX_LEARNER_LIST_LENGTH); - const int64_t count = learner_list.get_member_number(); - ObMember tmp_learner; - for (int64_t i = 0; i < count && (OB_SUCCESS == ret); ++i) { - if (OB_FAIL(learner_list.get_learner(i, tmp_learner))) { - SERVER_LOG(WARN, "get_learner failed", KR(ret), K(i)); - } - if (0 != pos) { - if (pos + 1 < MAX_LEARNER_LIST_LENGTH) { - output_buf[pos++] = ','; - } else { - ret = OB_BUF_NOT_ENOUGH; - SERVER_LOG(WARN, "buffer not enough", KR(ret), K(pos)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tmp_learner.get_server().ip_port_to_string(buf, sizeof(buf)))) { - SERVER_LOG(WARN, "convert server to string failed", KR(ret), K(tmp_learner)); - } else { - int n = snprintf(output_buf + pos, MAX_LEARNER_LIST_LENGTH - pos, \ - "%s:%ld", buf, tmp_learner.get_timestamp()); - if (n < 0 || n >= MAX_LEARNER_LIST_LENGTH - pos) { - ret = OB_BUF_NOT_ENOUGH; - SERVER_LOG(WARN, "snprintf error or buf not enough", KR(ret), K(n), K(pos)); - } else { - pos += n; - } } } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_log_stat.h b/src/observer/virtual_table/ob_all_virtual_log_stat.h index a4fab0be3..f9f8cc786 100644 --- a/src/observer/virtual_table/ob_all_virtual_log_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_log_stat.h @@ -16,7 +16,6 @@ #ifndef OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_ALL_VIRTUAL_PALF_STAT_ #define OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_ALL_VIRTUAL_PALF_STAT_ -#include "observer/omt/ob_multi_tenant.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "common/row/ob_row.h" @@ -35,28 +34,16 @@ namespace observer class ObAllVirtualPalfStat: public common::ObVirtualTableScannerIterator { public: - explicit ObAllVirtualPalfStat(omt::ObMultiTenant *omt); + ObAllVirtualPalfStat() = default; virtual ~ObAllVirtualPalfStat(); public: virtual int inner_get_next_row(common::ObNewRow *&row); void destroy(); private: int insert_log_stat_(const logservice::ObLogStat &log_stat, common::ObNewRow *row); - int member_list_to_string_(const common::ObMemberList &member_list); - int learner_list_to_string_(const common::GlobalLearnerList &learner_list, char *output_buf); private: static const int64_t VARCHAR_32 = 32; - static const int64_t VARCHAR_64 = 64; - char role_str_[VARCHAR_32] = {'\0'}; char access_mode_str_[VARCHAR_32] = {'\0'}; - char ip_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; - ObSqlString member_list_buf_; - char arbitration_member_buf_[MAX_SINGLE_MEMBER_LENGTH] = {'\0'}; - char degraded_list_buf_[MAX_LEARNER_LIST_LENGTH] = {'\0'}; - char config_version_buf_[palf::LogConfigVersion::CONFIG_VERSION_LEN] = {'\0'}; - char replica_type_str_[VARCHAR_32] = {'\0'}; - char learner_list_buf_[MAX_LEARNER_LIST_LENGTH] = {'\0'}; - omt::ObMultiTenant *omt_; }; }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.cpp b/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.cpp deleted file mode 100644 index d206b3b8c..000000000 --- a/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_all_virtual_log_transport_dest_stat.h" -#include "logservice/ob_log_service.h" - -namespace oceanbase -{ -namespace observer -{ -ObAllVirtualLogTransportDestStat::ObAllVirtualLogTransportDestStat(omt::ObMultiTenant *omt) {} - -ObAllVirtualLogTransportDestStat::~ObAllVirtualLogTransportDestStat() -{ - destroy(); -} - -int ObAllVirtualLogTransportDestStat::inner_get_next_row(common::ObNewRow *&row) -{ - return OB_ITER_END; -} - -void ObAllVirtualLogTransportDestStat::destroy() {} -} -} diff --git a/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.h b/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.h deleted file mode 100644 index 74c5a21c8..000000000 --- a/src/observer/virtual_table/ob_all_virtual_log_transport_dest_stat.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OB_ALL_VIRTUAL_LOG_TRANSPORT_DEST_STAT_H_ -#define OB_ALL_VIRTUAL_LOG_TRANSPORT_DEST_STAT_H_ - -#include "observer/omt/ob_multi_tenant.h" -#include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "sql/ob_scanner.h" - -namespace oceanbase -{ - -namespace observer -{ -class ObAllVirtualLogTransportDestStat: public common::ObVirtualTableScannerIterator -{ -public: - explicit ObAllVirtualLogTransportDestStat(omt::ObMultiTenant *omt); - virtual ~ObAllVirtualLogTransportDestStat(); -public: - virtual int inner_get_next_row(common::ObNewRow *&row); - void destroy(); -}; -} -} - -#endif diff --git a/src/observer/virtual_table/ob_all_virtual_long_ops_status.cpp b/src/observer/virtual_table/ob_all_virtual_long_ops_status.cpp index 73e5bd2d6..afe688fc2 100644 --- a/src/observer/virtual_table/ob_all_virtual_long_ops_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_long_ops_status.cpp @@ -24,7 +24,7 @@ using namespace oceanbase::share; ObAllVirtualLongOpsStatus::ObAllVirtualLongOpsStatus() : ObVirtualTableScannerIterator(), - addr_(), longops_value_(), longops_iter_() + longops_value_(), longops_iter_() { } @@ -35,7 +35,6 @@ ObAllVirtualLongOpsStatus::~ObAllVirtualLongOpsStatus() void ObAllVirtualLongOpsStatus::reset() { - addr_.reset(); ObVirtualTableScannerIterator::reset(); } diff --git a/src/observer/virtual_table/ob_all_virtual_long_ops_status.h b/src/observer/virtual_table/ob_all_virtual_long_ops_status.h index a9e34f04f..ba6bb9168 100644 --- a/src/observer/virtual_table/ob_all_virtual_long_ops_status.h +++ b/src/observer/virtual_table/ob_all_virtual_long_ops_status.h @@ -32,15 +32,8 @@ class ObAllVirtualLongOpsStatus : public common::ObVirtualTableScannerIterator virtual ~ObAllVirtualLongOpsStatus(); virtual int inner_get_next_row(common::ObNewRow *&row) override; virtual void reset() override; - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } -private: - private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualLongOpsStatus); - common::ObAddr addr_; share::ObLongopsValue longops_value_; share::ObLongopsIterator longops_iter_; char ip_buf_[common::OB_IP_STR_BUFF]; diff --git a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp index 5949f8ab9..a68b42d58 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_ls_info.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -27,8 +28,7 @@ namespace observer ObAllVirtualLSInfo::ObAllVirtualLSInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_() + ls_(nullptr) { } @@ -39,33 +39,10 @@ ObAllVirtualLSInfo::~ObAllVirtualLSInfo() void ObAllVirtualLSInfo::reset() { - ls_iter_guard_.reset(); - addr_.reset(); + ls_ = nullptr; ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualLSInfo::next_ls_info_(ObLSVTInfo &ls_info) -{ - int ret = OB_SUCCESS; - ObLS *ls = nullptr; - do { - if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } else if (NULL == ls) { - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ls)); - // try another ls - ret = OB_EAGAIN; - } else if (OB_FAIL(ls->get_ls_info(ls_info))) { - SERVER_LOG(WARN, "get ls info failed", K(ret), KPC(ls)); - // try another ls - ret = OB_EAGAIN; - } - } while (OB_EAGAIN == ret); - return ret; -} - int ObAllVirtualLSInfo::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; @@ -73,96 +50,45 @@ int ObAllVirtualLSInfo::inner_get_next_row(ObNewRow *&row) if (NULL == allocator_) { ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (FALSE_IT(start_to_read_ = true)) { - } else if (ls_iter_guard_.get_ptr() == nullptr && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); - } else if (OB_FAIL(next_ls_info_(ls_info))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get next_ls_info failed", K(ret)); - } + } else if (start_to_read_) { + ret = OB_ITER_END; + } else if (OB_ISNULL(share::g_mp->ls_service())) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(ls_->get_ls_info(ls_info))) { + SERVER_LOG(WARN, "get log stream info failed", K(ret)); } else { + start_to_read_ = true; const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); switch (col_id) { - case OB_APP_MIN_COLUMN_ID: { - // replica_type - cur_row_.cells_[i].set_varchar(ObShareUtil::replica_type_to_string(ls_info.replica_type_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - } - case OB_APP_MIN_COLUMN_ID + 1: { - // ls_state - ObRole role; - int64_t unused_proposal_id = 0; - if (OB_FAIL(role_to_string(ls_info.ls_state_, - state_name_, - sizeof(state_name_)))) { - SERVER_LOG(WARN, "get state role name failed", K(ret), K(role)); - } else { - state_name_[MAX_LS_STATE_LENGTH - 1] = '\0'; - cur_row_.cells_[i].set_varchar(state_name_); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 2: + case OB_APP_MIN_COLUMN_ID: // tablet_count cur_row_.cells_[i].set_int(ls_info.tablet_count_); break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 1: // weak_read_timestamp cur_row_.cells_[i].set_uint64(ls_info.weak_read_scn_.get_val_for_inner_table_field()); break; - case OB_APP_MIN_COLUMN_ID + 4: - // need_rebuild - cur_row_.cells_[i].set_varchar(ls_info.need_rebuild_ ? "YES" : "NO"); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - case OB_APP_MIN_COLUMN_ID + 5: + case OB_APP_MIN_COLUMN_ID + 2: // clog_checkpoint_ts cur_row_.cells_[i].set_uint64(!ls_info.checkpoint_scn_.is_valid() ? 0 : ls_info.checkpoint_scn_.get_val_for_tx()); break; - case OB_APP_MIN_COLUMN_ID + 6: + case OB_APP_MIN_COLUMN_ID + 3: // clog_checkpoint_lsn cur_row_.cells_[i].set_uint64(ls_info.checkpoint_lsn_ < 0 ? 0 : ls_info.checkpoint_lsn_); break; - case OB_APP_MIN_COLUMN_ID + 7: - // migrate_status - cur_row_.cells_[i].set_int(ls_info.migrate_status_); - break; - case OB_APP_MIN_COLUMN_ID + 8: - // rebuild_seq - cur_row_.cells_[i].set_int(ls_info.rebuild_seq_); - break; - case OB_APP_MIN_COLUMN_ID + 9: + case OB_APP_MIN_COLUMN_ID + 4: // tablet_change_checkpoint_scn cur_row_.cells_[i].set_uint64(!ls_info.tablet_change_checkpoint_scn_.is_valid() ? 0 : ls_info.tablet_change_checkpoint_scn_.get_val_for_inner_table_field()); break; - case OB_APP_MIN_COLUMN_ID + 10: - // reserved_scn - cur_row_.cells_[i].set_uint64(!ls_info.reserved_scn_.is_valid() ? 0 : ls_info.reserved_scn_.get_val_for_inner_table_field()); - break; - case OB_APP_MIN_COLUMN_ID + 11: + case OB_APP_MIN_COLUMN_ID + 5: // tx blocked cur_row_.cells_[i].set_int(ls_info.tx_blocked_); break; - case OB_APP_MIN_COLUMN_ID + 12: - // required_data_disk_size - cur_row_.cells_[i].set_int(ls_info.required_data_disk_size_); - break; - case OB_APP_MIN_COLUMN_ID + 13: - // mv_major_merge_scn - cur_row_.cells_[i].set_uint64(!ls_info.mv_major_merge_scn_.is_valid() ? 0 : ls_info.mv_major_merge_scn_.get_val_for_inner_table_field()); - break; - case OB_APP_MIN_COLUMN_ID + 14: - // mv_publish_scn - cur_row_.cells_[i].set_uint64(!ls_info.mv_publish_scn_.is_valid() ? 0 : ls_info.mv_publish_scn_.get_val_for_inner_table_field()); - break; - case OB_APP_MIN_COLUMN_ID + 15: - // mv_publish_scn - cur_row_.cells_[i].set_uint64(!ls_info.mv_safe_scn_.is_valid() ? 0 : ls_info.mv_safe_scn_.get_val_for_inner_table_field()); - break; default: ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid col_id", K(ret), K(col_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_ls_info.h b/src/observer/virtual_table/ob_all_virtual_ls_info.h index 8ee8997b6..066461ed5 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_info.h +++ b/src/observer/virtual_table/ob_all_virtual_ls_info.h @@ -18,10 +18,14 @@ #define OB_ALL_VIRTUAL_OB_LS_INFO_H_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +struct ObLSVTInfo; +} namespace observer { @@ -33,18 +37,8 @@ class ObAllVirtualLSInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } -private: -private: - int next_ls_info_(ObLSVTInfo &ls_info); private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; - char state_name_[common::MAX_LS_STATE_LENGTH]; - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualLSInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp index 09aa4f1c4..9d1e26169 100644 --- a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp @@ -35,16 +35,6 @@ ObMallocSampleInfo::~ObMallocSampleInfo() reset(); } -int ObMallocSampleInfo::inner_open() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf_, sizeof(ip_buf_)) - == false)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ip_to_string() fail", K(ret)); - } - return ret; -} void ObMallocSampleInfo::reset() { col_count_ = 0; @@ -66,20 +56,13 @@ int ObMallocSampleInfo::inner_get_next_row(ObNewRow *&row) } } - for (; OB_SUCC(ret) && it_ != malloc_sample_map_.end(); ++it_) { - if (true || true) { - if (OB_FAIL(fill_row(row))) { - SERVER_LOG(WARN, "failed to fill row", K(ret)); - } - break; - } - } - if (OB_SUCC(ret)) { - if (it_ != malloc_sample_map_.end()) { - ++it_; - } else { + if (it_ == malloc_sample_map_.end()) { ret = OB_ITER_END; + } else if (OB_FAIL(fill_row(row))) { + SERVER_LOG(WARN, "failed to fill row", K(ret)); + } else { + ++it_; } } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h index 8baa82c39..acf10c479 100644 --- a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h +++ b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h @@ -30,7 +30,6 @@ class ObMallocSampleInfo : public common::ObVirtualTableScannerIterator public: ObMallocSampleInfo(); virtual ~ObMallocSampleInfo(); - virtual int inner_open(); virtual void reset(); virtual int inner_get_next_row(common::ObNewRow *&row); private: @@ -45,7 +44,6 @@ class ObMallocSampleInfo : public common::ObVirtualTableScannerIterator ALLOC_COUNT, ALLOC_BYTES, }; - char ip_buf_[common::OB_IP_STR_BUFF]; char bt_[lib::MAX_BACKTRACE_LENGTH]; lib::ObMallocSampleMap::const_iterator it_; lib::ObMallocSampleMap malloc_sample_map_; diff --git a/src/observer/virtual_table/ob_all_virtual_mds_event_history.cpp b/src/observer/virtual_table/ob_all_virtual_mds_event_history.cpp index 46fd3aa1a..8f72e1fe0 100644 --- a/src/observer/virtual_table/ob_all_virtual_mds_event_history.cpp +++ b/src/observer/virtual_table/ob_all_virtual_mds_event_history.cpp @@ -23,7 +23,6 @@ using namespace share; using namespace storage; using namespace storage::mds; using namespace common; -using namespace omt; namespace observer { @@ -53,21 +52,13 @@ int ObAllVirtualMdsEventHistory::range_scan_(char *temp_buffer, int64_t buf_len) MDS_LOG(INFO, "start range read", K(*this)); if (OB_FAIL(ObMdsEventBuffer::for_each([this, temp_buffer, buf_len](const MdsEventKey &key, const MdsEvent &event) -> int { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; if (judge_key_in_ranges_(key)) { - if (true ||// SYS tenant can see information of all tenants - true) {// Non-SYS tenant can only see information of its own tenant - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - - if (OB_SUCCESS == tmp_ret) { - if (OB_FAIL(convert_event_info_to_row_(key, event, temp_buffer, buf_len, cur_row_))) { - MDS_LOG(WARN, "failed to convert_node_info_to_row_", K(ret), K(*this)); - } else if (OB_FAIL(scanner_.add_row(cur_row_))) { - MDS_LOG(WARN, "fail to add_row to scanner_", K(*this)); - } else { - MDS_LOG(TRACE, "scan", K(key)); - } - } + if (OB_FAIL(convert_event_info_to_row_(key, event, temp_buffer, buf_len, cur_row_))) { + MDS_LOG(WARN, "failed to convert_node_info_to_row_", K(ret), K(*this)); + } else if (OB_FAIL(scanner_.add_row(cur_row_))) { + MDS_LOG(WARN, "fail to add_row to scanner_", K(*this)); + } else { + MDS_LOG(TRACE, "scan", K(key)); } } return ret; @@ -83,21 +74,13 @@ int ObAllVirtualMdsEventHistory::point_read_(char *temp_buffer, int64_t buf_len) MDS_LOG(INFO, "start point read", K(*this)); { for (int64_t idx = 0; idx < tablet_points_.count() && OB_SUCC(ret); ++idx) { - MdsEventKey key(share::SYS_LS, tablet_points_[idx]); + MdsEventKey key(tablet_points_[idx]); if (OB_FAIL(ObMdsEventBuffer::for_each(key, [&key, this, temp_buffer, buf_len](const MdsEvent &event) -> int { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - if (true ||// SYS tenant can see information of all tenants - true) {// Non-SYS tenant can only see information of its own tenant - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - - if (OB_SUCCESS == tmp_ret) { - if (OB_FAIL(convert_event_info_to_row_(key, event, temp_buffer, buf_len, cur_row_))) { - MDS_LOG(WARN, "failed to convert_node_info_to_row_", K(ret), K(*this)); - } else if (OB_FAIL(scanner_.add_row(cur_row_))) { - MDS_LOG(WARN, "fail to add_row to scanner_", K(*this)); - } - } + if (OB_FAIL(convert_event_info_to_row_(key, event, temp_buffer, buf_len, cur_row_))) { + MDS_LOG(WARN, "failed to convert_node_info_to_row_", K(ret), K(*this)); + } else if (OB_FAIL(scanner_.add_row(cur_row_))) { + MDS_LOG(WARN, "fail to add_row to scanner_", K(*this)); } return ret; }))) { @@ -169,11 +152,11 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k break; } - case OB_APP_MIN_COLUMN_ID + 3: {// tid + case OB_APP_MIN_COLUMN_ID + 1: {// tid cur_row_.cells_[i].set_int(event.tid_); break; } - case OB_APP_MIN_COLUMN_ID + 4: {// tname + case OB_APP_MIN_COLUMN_ID + 2: {// tname int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, "%s", event.tname_); buffer += pos; @@ -181,7 +164,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 5: {// trace + case OB_APP_MIN_COLUMN_ID + 3: {// trace int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, event.trace_id_); buffer += pos; @@ -189,11 +172,11 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 6: {// timestamp + case OB_APP_MIN_COLUMN_ID + 4: {// timestamp cur_row_.cells_[i].set_timestamp(event.timestamp_); break; } - case OB_APP_MIN_COLUMN_ID + 7: {// event + case OB_APP_MIN_COLUMN_ID + 5: {// event int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, "%s", event.event_); buffer += pos; @@ -201,7 +184,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 8: {// info + case OB_APP_MIN_COLUMN_ID + 6: {// info int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, event.info_str_); buffer += pos; @@ -209,7 +192,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 9: {// user_key + case OB_APP_MIN_COLUMN_ID + 7: {// user_key int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, event.key_str_); buffer += pos; @@ -217,7 +200,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 10: {// writer_type + case OB_APP_MIN_COLUMN_ID + 8: {// writer_type int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, "%s", mds::obj_to_string(event.writer_type_)); buffer += pos; @@ -225,27 +208,27 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 11: {// writer_id + case OB_APP_MIN_COLUMN_ID + 9: {// writer_id cur_row_.cells_[i].set_int(event.writer_id_); break; } - case OB_APP_MIN_COLUMN_ID + 12: {// seq_no + case OB_APP_MIN_COLUMN_ID + 10: {// seq_no cur_row_.cells_[i].set_int(event.seq_no_.get_seq()); break; } - case OB_APP_MIN_COLUMN_ID + 13: {// redo_scn + case OB_APP_MIN_COLUMN_ID + 11: {// redo_scn cur_row_.cells_[i].set_uint64(event.redo_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 14: {// end_scn + case OB_APP_MIN_COLUMN_ID + 12: {// end_scn cur_row_.cells_[i].set_uint64(event.end_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 15: {// trans_version + case OB_APP_MIN_COLUMN_ID + 13: {// trans_version cur_row_.cells_[i].set_uint64(event.trans_version_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 16: {// node_type + case OB_APP_MIN_COLUMN_ID + 14: {// node_type int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, "%s", mds::obj_to_string(event.node_type_)); buffer += pos; @@ -253,7 +236,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k cur_row_.cells_[i].set_string(ObLongTextType, ObString(pos, buffer - pos)); break; } - case OB_APP_MIN_COLUMN_ID + 17: {// state + case OB_APP_MIN_COLUMN_ID + 15: {// state int64_t pos = 0; databuff_printf(buffer, buffer_size, pos, "%s", mds::obj_to_string(event.state_)); buffer += pos; diff --git a/src/observer/virtual_table/ob_all_virtual_mds_event_history.h b/src/observer/virtual_table/ob_all_virtual_mds_event_history.h index 2bc3535b0..5f354c5a2 100644 --- a/src/observer/virtual_table/ob_all_virtual_mds_event_history.h +++ b/src/observer/virtual_table/ob_all_virtual_mds_event_history.h @@ -20,7 +20,6 @@ #include "lib/container/ob_tuple.h" #include "ob_tablet_id.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "observer/omt/ob_multi_tenant.h" #include "ob_mds_event_buffer.h" namespace oceanbase @@ -37,9 +36,8 @@ namespace observer class ObAllVirtualMdsEventHistory : public common::ObVirtualTableScannerIterator { - static constexpr int64_t IP_BUFFER_SIZE = 65; // >= MAX_IP_ADDR_LENGTH (e.g. INET6 on Windows) public: - explicit ObAllVirtualMdsEventHistory(omt::ObMultiTenant *omt) : omt_(omt) {} + ObAllVirtualMdsEventHistory() = default; virtual int inner_get_next_row(common::ObNewRow *&row) override; TO_STRING_KV(K_(tablet_ranges), K_(tablet_points)) private: @@ -53,8 +51,6 @@ class ObAllVirtualMdsEventHistory : public common::ObVirtualTableScannerIterator int range_scan_(char *temp_buffer, int64_t buf_len); int point_read_(char *temp_buffer, int64_t buf_len); DISALLOW_COPY_AND_ASSIGN(ObAllVirtualMdsEventHistory); - omt::ObMultiTenant *omt_; - char ip_buffer_[IP_BUFFER_SIZE]; ObArray> tablet_ranges_; ObArray tablet_points_; }; diff --git a/src/observer/virtual_table/ob_all_virtual_mds_node_stat.cpp b/src/observer/virtual_table/ob_all_virtual_mds_node_stat.cpp index 6a7b8d4e5..ef7b950b7 100644 --- a/src/observer/virtual_table/ob_all_virtual_mds_node_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_mds_node_stat.cpp @@ -15,7 +15,9 @@ */ #include "ob_all_virtual_mds_node_stat.h" +#include "share/rc/ob_module_provider.h" #include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -64,20 +66,6 @@ struct ApplyOnTabletOp { char *temp_buffer_; }; -struct ApplyOnLSOp { - ApplyOnLSOp(ObAllVirtualMdsNodeStat *table, ApplyOnTabletOp &apply_on_tablet_op) - : table_(table), - apply_on_tablet_op_(apply_on_tablet_op) {} - int operator()(ObLS &ls) { - int ret = OB_SUCCESS; - (void) table_->get_tablet_info_(ls, apply_on_tablet_op_); - return OB_SUCCESS; - } - ObAllVirtualMdsNodeStat *table_; - ApplyOnTabletOp &apply_on_tablet_op_; -}; - - int ObAllVirtualMdsNodeStat::get_mds_table_handle_(ObTablet &tablet, mds::MdsTableHandle &handle, const bool create_if_not_exist) @@ -101,9 +89,15 @@ int ObAllVirtualMdsNodeStat::inner_get_next_row(common::ObNewRow *&row) MDS_LOG(WARN, "fail to alloc buffer", K(*this)); } else { ApplyOnTabletOp apply_on_table_op(this, temp_buffer); - ApplyOnLSOp apply_on_ls_op(this, apply_on_table_op); - if (OB_FAIL(ObTenantMdsService::for_each_ls_in_tenant(apply_on_ls_op))) { - MDS_LOG(WARN, "failed to do for_each_ls_in_tenant", K(ret)); + ObLS *ls = nullptr; + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + MDS_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(get_tablet_info_(*ls, apply_on_table_op))) { + MDS_LOG(WARN, "iterate mds node failed", K(ret)); ret = OB_SUCCESS; } if (OB_FAIL(ret)) { diff --git a/src/observer/virtual_table/ob_all_virtual_mds_node_stat.h b/src/observer/virtual_table/ob_all_virtual_mds_node_stat.h index c31595fe6..f2639b56d 100644 --- a/src/observer/virtual_table/ob_all_virtual_mds_node_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_mds_node_stat.h @@ -20,7 +20,6 @@ #include "lib/container/ob_tuple.h" #include "ob_tablet_id.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "observer/omt/ob_multi_tenant.h" namespace oceanbase { namespace storage @@ -34,15 +33,12 @@ namespace observer { class ApplyOnTabletOp; -class ApplyOnLSOp; class ObAllVirtualMdsNodeStat : public common::ObVirtualTableScannerIterator { friend class ApplyOnTabletOp; - friend class ApplyOnLSOp; - static constexpr int64_t IP_BUFFER_SIZE = 65; // >= MAX_IP_ADDR_LENGTH (e.g. INET6 on Windows) public: - explicit ObAllVirtualMdsNodeStat(omt::ObMultiTenant *omt) : omt_(omt) {} + ObAllVirtualMdsNodeStat() = default; virtual int inner_get_next_row(common::ObNewRow *&row) override; TO_STRING_KV(K_(tablet_ranges), K_(tablet_points)) private: @@ -67,8 +63,6 @@ class ObAllVirtualMdsNodeStat : public common::ObVirtualTableScannerIterator mds::MdsTableHandle &handle, const bool create_if_not_exist); DISALLOW_COPY_AND_ASSIGN(ObAllVirtualMdsNodeStat); - omt::ObMultiTenant *omt_; - char ip_buffer_[IP_BUFFER_SIZE]; ObArray> tablet_ranges_; ObArray tablet_points_; }; diff --git a/src/observer/virtual_table/ob_all_virtual_memstore_info.cpp b/src/observer/virtual_table/ob_all_virtual_memstore_info.cpp index 1e858f018..995476ded 100644 --- a/src/observer/virtual_table/ob_all_virtual_memstore_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_memstore_info.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_memstore_info.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -28,9 +29,8 @@ namespace observer ObAllVirtualMemstoreInfo::ObAllVirtualMemstoreInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_(), - ls_tablet_iter_(ObMDSGetTabletMode::READ_ALL_COMMITED), + ls_(nullptr), + tablet_iter_(ObMDSGetTabletMode::READ_ALL_COMMITED), tables_handle_(), memtable_array_pos_(0) { @@ -43,68 +43,34 @@ ObAllVirtualMemstoreInfo::~ObAllVirtualMemstoreInfo() void ObAllVirtualMemstoreInfo::reset() { - addr_.reset(); - ls_tablet_iter_.reset(); - ls_iter_guard_.reset(); + tablet_iter_.reset(); + ls_ = nullptr; tables_handle_.reset(); memtable_array_pos_ = 0; memset(freeze_time_dist_, 0, OB_MAX_CHAR_LENGTH); - memset(compaction_info_buf_, 0, sizeof(compaction_info_buf_)); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualMemstoreInfo::get_next_ls(ObLS *&ls) +int ObAllVirtualMemstoreInfo::get_next_tablet(ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - - while (OB_SUCC(ret)) { - if (!ls_iter_guard_.get_ptr() - || OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to switch tenant", K(ret)); - } - // switch to next tenant - ret = OB_ITER_END; - } else if (OB_ISNULL(ls)) { + if (!tablet_iter_.is_valid()) { + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } else { - break; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(ls_->build_tablet_iter( + tablet_iter_, true /* except_ls_inner_tablet */))) { + SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); } } - - return ret; -} - -int ObAllVirtualMemstoreInfo::get_next_tablet(ObTabletHandle &tablet_handle) -{ - int ret = OB_SUCCESS; - - while (OB_SUCC(ret)) { - if (!ls_tablet_iter_.is_valid()) { - ObLS *ls = nullptr; - if (OB_FAIL(get_next_ls(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next ls", K(ret)); - } - } else if (OB_FAIL(ls->build_tablet_iter(ls_tablet_iter_, true /* except_ls_inner_tablet */))) { - SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls_tablet_iter_.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { - ls_tablet_iter_.reset(); - ret = OB_SUCCESS; - } else { - SERVER_LOG(WARN, "fail to get next tablet", K(ret)); - } - } else { - break; + if (OB_SUCC(ret) && OB_FAIL(tablet_iter_.get_next_tablet(tablet_handle))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get next tablet", K(ret)); } } - return ret; } @@ -177,8 +143,6 @@ int ObAllVirtualMemstoreInfo::inner_get_next_row(ObNewRow *&row) ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); } else if (FALSE_IT(start_to_read_ = true)) { - } else if (ls_iter_guard_.get_ptr() == nullptr && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); } else if (OB_FAIL(get_next_memtable(mt))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get_next_memtable failed", K(ret)); diff --git a/src/observer/virtual_table/ob_all_virtual_memstore_info.h b/src/observer/virtual_table/ob_all_virtual_memstore_info.h index 2d7e2e222..8c7ec7df5 100644 --- a/src/observer/virtual_table/ob_all_virtual_memstore_info.h +++ b/src/observer/virtual_table/ob_all_virtual_memstore_info.h @@ -24,10 +24,13 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "storage/tablet/ob_tablet_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace memtable { class ObMemtable; @@ -42,24 +45,16 @@ class ObAllVirtualMemstoreInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls(ObLS *&ls); int get_next_tablet(storage::ObTabletHandle &tablet_handle); int get_next_memtable(storage::ObITabletMemtable *&mt); void get_freeze_time_dist(const ObMtStat& mt_stat); private: - common::ObAddr addr_; - ObSharedGuard ls_iter_guard_; - storage::ObLSTabletIterator ls_tablet_iter_; - char ip_buf_[common::OB_IP_STR_BUFF]; + storage::ObLS *ls_; + storage::ObLSTabletIterator tablet_iter_; common::ObSEArray tables_handle_; int64_t memtable_array_pos_; char freeze_time_dist_[OB_MAX_CHAR_LENGTH]; - char compaction_info_buf_[common::OB_COMPACTION_INFO_LENGTH]; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualMemstoreInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.cpp b/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.cpp index 1a84324cc..2d2b7dd49 100644 --- a/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_minor_freeze_info.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -28,8 +29,7 @@ namespace observer ObAllVirtualMinorFreezeInfo::ObAllVirtualMinorFreezeInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_(), + ls_(nullptr), diagnose_info_(), memtables_info_() { @@ -42,38 +42,13 @@ ObAllVirtualMinorFreezeInfo::~ObAllVirtualMinorFreezeInfo() void ObAllVirtualMinorFreezeInfo::reset() { - addr_.reset(); - ls_iter_guard_.reset(); + ls_ = nullptr; diagnose_info_.reset(); memtables_info_.reset(); - memset(ip_buf_, 0, common::OB_IP_STR_BUFF); memset(memtables_info_string_, 0, OB_MAX_CHAR_LENGTH); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualMinorFreezeInfo::get_next_ls(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - while (OB_SUCC(ret)) { - if (!ls_iter_guard_.get_ptr() - || OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to switch tenant", K(ret)); - } - // switch to next tenant - ret = OB_ITER_END; - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } else { - break; - } - } - - return ret; -} - int ObAllVirtualMinorFreezeInfo::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; @@ -81,14 +56,14 @@ int ObAllVirtualMinorFreezeInfo::inner_get_next_row(ObNewRow *&row) if (NULL == allocator_) { ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (FALSE_IT(start_to_read_ = true)) { - } else if (ls_iter_guard_.get_ptr() == nullptr && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); + } else if (start_to_read_) { + ret = OB_ITER_END; } else if (OB_FAIL(get_next_freeze_stat(freeze_stat))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get_next_freeze_stat failed", K(ret)); } } else { + start_to_read_ = true; int64_t freeze_clock = 0; const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { @@ -183,29 +158,21 @@ int ObAllVirtualMinorFreezeInfo::inner_get_next_row(ObNewRow *&row) int ObAllVirtualMinorFreezeInfo::get_next_freeze_stat(ObFreezerStat &freeze_stat) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObFreezer *freezer = nullptr; - while (OB_SUCC(ret)) { - if (OB_FAIL(get_next_ls(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ret)); - } else if (FALSE_IT(freezer = ls->get_freezer())) { - } else if (OB_ISNULL(freezer)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ret)); - } else if (OB_FAIL(freezer->get_stat().deep_copy_to(freeze_stat))) { - SERVER_LOG(WARN, "fail to deep copy", K(ret)); - } else if (!(freeze_stat.is_valid())) { - SERVER_LOG(WARN, "freeze_stat is invalid", KP(ls), KP(freezer)); - } else { - // freeze_stat is valid - break; - } + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_ISNULL(freezer = ls_->get_freezer())) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "freezer is null", K(ret)); + } else if (OB_FAIL(freezer->get_stat().deep_copy_to(freeze_stat))) { + SERVER_LOG(WARN, "fail to deep copy", K(ret)); + } else if (!freeze_stat.is_valid()) { + ret = OB_ITER_END; } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.h b/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.h index e83fd660d..1c8a7da51 100644 --- a/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.h +++ b/src/observer/virtual_table/ob_all_virtual_minor_freeze_info.h @@ -24,11 +24,14 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "storage/tablet/ob_tablet_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" #include "storage/ls/ob_freezer.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { class ObAllVirtualMinorFreezeInfo : public common::ObVirtualTableScannerIterator @@ -39,19 +42,12 @@ class ObAllVirtualMinorFreezeInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls(ObLS *&ls); int generate_memtables_info(); int get_next_freeze_stat(ObFreezerStat &freeze_stat); void append_memtable_info_string(const char *name, const char *str, int64_t &size); private: - common::ObAddr addr_; - ObSharedGuard ls_iter_guard_; - char ip_buf_[common::OB_IP_STR_BUFF]; + storage::ObLS *ls_; ObStringHolder diagnose_info_; common::ObSArray memtables_info_; char memtables_info_string_[OB_MAX_CHAR_LENGTH]; diff --git a/src/observer/virtual_table/ob_all_virtual_obj_lock.cpp b/src/observer/virtual_table/ob_all_virtual_obj_lock.cpp index 733a5c9a7..1a20cad9e 100644 --- a/src/observer/virtual_table/ob_all_virtual_obj_lock.cpp +++ b/src/observer/virtual_table/ob_all_virtual_obj_lock.cpp @@ -16,8 +16,9 @@ #include "observer/virtual_table/ob_all_virtual_obj_lock.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" using namespace oceanbase::common; using namespace oceanbase::storage; @@ -28,11 +29,9 @@ namespace observer ObAllVirtualObjLock::ObAllVirtualObjLock() : ObVirtualTableScannerIterator(), - addr_(), ls_(nullptr), tx_ctx_(nullptr), - ls_iter_guard_(), - ls_tx_ctx_iter_(), + tx_ctx_iter_(), obj_lock_iter_(), lock_op_iter_(), prio_op_iter_(), @@ -50,66 +49,34 @@ void ObAllVirtualObjLock::reset() { ls_ = nullptr; if (OB_NOT_NULL(tx_ctx_)) { - ls_tx_ctx_iter_.revert_tx_ctx(tx_ctx_); + tx_ctx_iter_.revert_tx_ctx(tx_ctx_); tx_ctx_ = nullptr; } is_iter_tx_ = true; - ls_iter_guard_.reset(); - ls_tx_ctx_iter_.reset(); + tx_ctx_iter_.reset(); obj_lock_iter_.reset(); lock_op_iter_.reset(); prio_op_iter_.reset(); start_to_read_ = false; is_iter_priority_list_ = true; - addr_.reset(); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualObjLock::get_next_ls() -{ - int ret = OB_SUCCESS; - - if (!ls_iter_guard_.get_ptr() || OB_FAIL(ls_iter_guard_->get_next(ls_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to switch tenant", K(ret)); - } - // switch to next tenant - ret = OB_ITER_END; - SERVER_LOG(DEBUG, "finish iterate this tenant, switch to next tenant then", K(ret)); - } else if (OB_ISNULL(ls_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } else { - is_iter_tx_ = true; // iterate tx firstly, then iterate lock_memtable - ls_tx_ctx_iter_.reset(); - obj_lock_iter_.reset(); - lock_op_iter_.reset(); - prio_op_iter_.reset(); - is_iter_priority_list_ = true; - if (OB_NOT_NULL(tx_ctx_)) { - ls_tx_ctx_iter_.revert_tx_ctx(tx_ctx_); - tx_ctx_ = nullptr; - } - } - - return ret; -} - -int ObAllVirtualObjLock::get_next_tx_ctx(transaction::ObPartTransCtx *&tx_ctx) +int ObAllVirtualObjLock::get_next_tx_ctx(transaction::ObTxCtx *&tx_ctx) { int ret = OB_SUCCESS; while (OB_SUCC(ret)) { - if (!ls_tx_ctx_iter_.is_ready()) { + if (!tx_ctx_iter_.is_ready()) { if (OB_ISNULL(ls_)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "ls is null", K(ret)); - } else if (OB_FAIL(ls_->iterate_tx_ctx(ls_tx_ctx_iter_))) { + } else if (OB_FAIL(ls_->iterate_tx_ctx(tx_ctx_iter_))) { SERVER_LOG(WARN, "fail to get ls_tx_ctx_iter", K(ret)); } - } else if (OB_FAIL(ls_tx_ctx_iter_.get_next_tx_ctx(tx_ctx))) { + } else if (OB_FAIL(tx_ctx_iter_.get_next_tx_ctx(tx_ctx))) { if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "ls_tx_ctx_iter_.get_next_tx_ctx failed", K(ret)); + SERVER_LOG(WARN, "get next tx ctx failed", K(ret)); } } else { break; @@ -151,35 +118,40 @@ int ObAllVirtualObjLock::get_next_lock_op( // loop until get lock_op while (OB_SUCC(ret)) { + bool has_lock_op = false; if (!is_iter_priority_list_) { if (OB_FAIL(lock_op_iter_.get_next(lock_op))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next lock op", K(ret), K(is_iter_tx_)); + } else { + lock_op_iter_.reset(); // clean lock_op_iter to save memory } - lock_op_iter_.reset(); // clean lock_op_iter to save memory } else { priority = transaction::tablelock::ObTableLockPriority::NORMAL; + has_lock_op = true; } } else { transaction::tablelock::ObTableLockPrioOp prio_op; if (OB_FAIL(prio_op_iter_.get_next(prio_op))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next lock op", K(ret), K(is_iter_tx_)); + } else { + prio_op_iter_.reset(); // clean prio_op_iter to save memory } - prio_op_iter_.reset(); // clean lock_op_iter to save memory } else { lock_op = prio_op.lock_op_; priority = prio_op.priority_; + has_lock_op = true; } } - if (OB_FAIL(ret)) { + if (has_lock_op) { + break; + } else if (OB_ITER_END == ret) { if (OB_FAIL(get_next_lock_op_iter())) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next lock_op_iter", K(ret), K(is_iter_tx_)); } } - } else { - break; } } @@ -189,43 +161,31 @@ int ObAllVirtualObjLock::get_next_lock_op( int ObAllVirtualObjLock::get_next_lock_op_iter() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int64_t retry_times = 0; // use it to avoid unexpected error code, and try to iterate next one - // loop until get valid lock_op_iter, or has iterated all ls + // loop until get a valid lock_op_iter or both sources are exhausted while (OB_SUCC(ret)) { if (is_iter_tx_) { - if (OB_TMP_FAIL(get_next_lock_op_iter_from_tx_ctx())) { - if (OB_ITER_END != tmp_ret) { - retry_times++; - SERVER_LOG(WARN, "get next lock_op_iter from tx_ctx failed", K(tmp_ret), K(retry_times)); - } - if (OB_ITER_END == tmp_ret || retry_times >= MAX_RETRY_TIMES) { + if (OB_FAIL(get_next_lock_op_iter_from_tx_ctx())) { + if (OB_ITER_END == ret) { is_iter_tx_ = false; - ls_tx_ctx_iter_.reset(); // clean lx_tx_ctx_iter to save memory - SERVER_LOG(DEBUG, "iterate tx finish, iterate lock_memtable then", K(tmp_ret)); + tx_ctx_iter_.reset(); + SERVER_LOG(DEBUG, "iterate tx finish, iterate lock_memtable then", K(ret)); + ret = OB_SUCCESS; + } else { + SERVER_LOG(WARN, "get next lock_op_iter from tx_ctx failed", K(ret)); } + } else { + break; } } else { - if (OB_TMP_FAIL(get_next_lock_op_iter_from_lock_memtable())) { - if (OB_ITER_END != tmp_ret) { - retry_times++; - SERVER_LOG(WARN, "get next lock_op_iter from lock_memtable failed", K(tmp_ret), K(retry_times)); - } - if (OB_ITER_END == tmp_ret || retry_times >= MAX_RETRY_TIMES) { - if (OB_FAIL(get_next_ls())) { - // has iterated all ls - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get next ls failed", K(ret)); - } - } + if (OB_FAIL(get_next_lock_op_iter_from_lock_memtable())) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "get next lock_op_iter from lock_memtable failed", K(ret)); } + } else { + break; } } - // get valid lock_op_iter - if (OB_SUCCESS == tmp_ret) { - break; - } } return ret; @@ -270,7 +230,7 @@ int ObAllVirtualObjLock::get_next_lock_op_iter_from_tx_ctx() } if (need_revert_tx_ctx && OB_NOT_NULL(tx_ctx_)) { - ls_tx_ctx_iter_.revert_tx_ctx(tx_ctx_); + tx_ctx_iter_.revert_tx_ctx(tx_ctx_); tx_ctx_ = nullptr; } @@ -302,14 +262,15 @@ int ObAllVirtualObjLock::get_next_lock_op_iter_from_lock_memtable() int ObAllVirtualObjLock::prepare_start_to_read() { int ret = OB_SUCCESS; + ObLSService *ls_service = share::g_mp->ls_service(); if (OB_ISNULL(allocator_)) { ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (OB_ISNULL(ls_iter_guard_.get_ptr()) - && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "init ls_iter_guard_ failed", K(ret)); - } else if (OB_FAIL(get_next_ls())) { - SERVER_LOG(WARN, "init ls_ failed", K(ret)); + } else if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); } else if (OB_FAIL(get_next_lock_op_iter())) { SERVER_LOG(WARN, "init lock_op_iter_ failed", K(ret)); } else { @@ -325,12 +286,10 @@ int ObAllVirtualObjLock::inner_get_next_row(ObNewRow *&row) transaction::tablelock::ObTableLockPriority priority; if (!start_to_read_ && OB_FAIL(prepare_start_to_read())) { SERVER_LOG(WARN, "prepare start to read failed", K(ret)); - ret = OB_ITER_END; // to avoid throw error code to client } else if (OB_FAIL(get_next_lock_op(lock_op, priority))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get_next_lock_op failed", K(ret)); } - ret = OB_ITER_END; // to avoid throw error code to client } else { const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { diff --git a/src/observer/virtual_table/ob_all_virtual_obj_lock.h b/src/observer/virtual_table/ob_all_virtual_obj_lock.h index ab1fb13b0..c189fb091 100644 --- a/src/observer/virtual_table/ob_all_virtual_obj_lock.h +++ b/src/observer/virtual_table/ob_all_virtual_obj_lock.h @@ -19,10 +19,13 @@ #include "storage/tablelock/ob_obj_lock.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { @@ -34,13 +37,8 @@ class ObAllVirtualObjLock : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls(); - int get_next_tx_ctx(transaction::ObPartTransCtx *&tx_ctx); + int get_next_tx_ctx(transaction::ObTxCtx *&tx_ctx); int get_next_lock_id(ObLockID &lock_id); int get_next_lock_op(transaction::tablelock::ObTableLockOp &lock_op, transaction::tablelock::ObTableLockPriority &priority); @@ -50,7 +48,6 @@ class ObAllVirtualObjLock : public common::ObVirtualTableScannerIterator int prepare_start_to_read(); private: - static const int64_t MAX_RETRY_TIMES = 10; enum { LOCK_ID = OB_APP_MIN_COLUMN_ID, @@ -71,12 +68,10 @@ class ObAllVirtualObjLock : public common::ObVirtualTableScannerIterator WAIT_SEQ }; private: - common::ObAddr addr_; - ObLS *ls_; - transaction::ObPartTransCtx *tx_ctx_; - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; + transaction::ObTxCtx *tx_ctx_; // the tx_ctx of a ls - transaction::ObLSTxCtxIterator ls_tx_ctx_iter_; + transaction::ObLSTxCtxIterator tx_ctx_iter_; // the lock id of a ls ObLockIDIterator obj_lock_iter_; // the lock op of a obj lock diff --git a/src/observer/virtual_table/ob_all_virtual_plugin_info.cpp b/src/observer/virtual_table/ob_all_virtual_plugin_info.cpp index 04659a85e..fda55685f 100644 --- a/src/observer/virtual_table/ob_all_virtual_plugin_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_plugin_info.cpp @@ -50,7 +50,6 @@ ObAllVirtualPluginInfo::ObAllVirtualPluginInfo() {} ObAllVirtualPluginInfo::~ObAllVirtualPluginInfo() { reset(); } void ObAllVirtualPluginInfo::reset() { - addr_.reset(); plugin_entries_.reset(); iter_index_ = -1; } @@ -222,4 +221,4 @@ int ObAllVirtualPluginInfo::inner_get_next_row(ObNewRow *&row) } } // namespace observer -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_plugin_info.h b/src/observer/virtual_table/ob_all_virtual_plugin_info.h index 5f1364767..0873137e7 100644 --- a/src/observer/virtual_table/ob_all_virtual_plugin_info.h +++ b/src/observer/virtual_table/ob_all_virtual_plugin_info.h @@ -38,21 +38,12 @@ class ObAllVirtualPluginInfo final : public common::ObVirtualTableScannerIterato virtual void reset() override; virtual int inner_open() override; virtual int inner_close() override; - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; - ObArray plugin_entries_; int64_t iter_index_ = -1; - TO_STRING_KV(K(addr_)); - private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualPluginInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.cpp b/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.cpp index 064b92135..8d32e8564 100644 --- a/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.cpp +++ b/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.cpp @@ -30,27 +30,21 @@ int ObAllPxP2PDatahubTable::P2PMsgTraverseCall::operator() ( { int ret = OB_SUCCESS; if (OB_NOT_NULL(entry.second)) { - if (false && - false) { - /*do nothing*/ - } else { - P2PDatahubNode node; - node.p2p_datahub_id_ = entry.second->get_p2p_datahub_id(); - node.msg_type_ = (int64_t)entry.second->get_msg_type(); - node.start_time_ = entry.second->get_start_time(); - - node.trace_id_ = entry.second->get_trace_id(); - node.timeout_ts_ = entry.second->get_timeout_ts(); - if (OB_FAIL(node_array_.push_back(node))) { - SERVER_LOG(WARN, "fail to push back node", K(ret)); - } + P2PDatahubNode node; + node.p2p_datahub_id_ = entry.second->get_p2p_datahub_id(); + node.msg_type_ = (int64_t)entry.second->get_msg_type(); + node.start_time_ = entry.second->get_start_time(); + node.trace_id_ = entry.second->get_trace_id(); + node.timeout_ts_ = entry.second->get_timeout_ts(); + if (OB_FAIL(node_array_.push_back(node))) { + SERVER_LOG(WARN, "fail to push back node", K(ret)); } } return ret; } -ObAllPxP2PDatahubTable::ObAllPxP2PDatahubTable():addr_(NULL), start_to_read_(false), +ObAllPxP2PDatahubTable::ObAllPxP2PDatahubTable():start_to_read_(false), node_array_(), index_(0) { } @@ -60,7 +54,6 @@ ObAllPxP2PDatahubTable::~ObAllPxP2PDatahubTable() void ObAllPxP2PDatahubTable::reset() { - addr_ = NULL; start_to_read_ = false; node_array_.reset(); index_ = 0; @@ -80,9 +73,9 @@ int ObAllPxP2PDatahubTable::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; ObObj *cells = cur_row_.cells_; - if (OB_ISNULL(allocator_) || OB_ISNULL(addr_)) { + if (OB_ISNULL(allocator_)) { ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ or addr_ is null", K_(allocator), K_(addr), K(ret)); + SERVER_LOG(WARN, "allocator_ is null", K_(allocator), K(ret)); } else if (OB_ISNULL(cells)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "cur row cell is NULL", K(ret)); @@ -164,4 +157,3 @@ int ObAllPxP2PDatahubTable::inner_get_next_row(ObNewRow *&row) }/* ns observer*/ }/* ns oceanbase */ - diff --git a/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.h b/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.h index c9c850af2..186013a55 100644 --- a/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.h +++ b/src/observer/virtual_table/ob_all_virtual_px_p2p_datahub.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_PX_P2P_DATAHUB_TABLE_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "lib/net/ob_addr.h" #include "sql/engine/px/p2p_datahub/ob_p2p_dh_mgr.h" namespace oceanbase { @@ -57,11 +56,9 @@ class ObAllPxP2PDatahubTable : public common::ObVirtualTableScannerIterator virtual ~ObAllPxP2PDatahubTable(); virtual void reset(); virtual int inner_get_next_row(common::ObNewRow *&row); - inline void set_addr(common::ObAddr &addr) { addr_ = &addr; } private: int p2p_datahub_map_to_array(); private: - common::ObAddr *addr_; bool start_to_read_; char trace_id_[128]; common::ObArray node_array_; @@ -83,4 +80,3 @@ class ObAllPxP2PDatahubTable : public common::ObVirtualTableScannerIterator } // namespace observer } // namespace oceanbase #endif // OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_PX_P2P_DATAHUB_TABLE_ - diff --git a/src/observer/virtual_table/ob_all_virtual_px_worker_stat.cpp b/src/observer/virtual_table/ob_all_virtual_px_worker_stat.cpp index a058b3990..859bac89c 100644 --- a/src/observer/virtual_table/ob_all_virtual_px_worker_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_px_worker_stat.cpp @@ -24,7 +24,7 @@ using namespace sql; namespace observer { -ObAllPxWorkerStatTable::ObAllPxWorkerStatTable():addr_(NULL), start_to_read_(false), +ObAllPxWorkerStatTable::ObAllPxWorkerStatTable():start_to_read_(false), stat_array_(), index_(0) { } @@ -33,7 +33,6 @@ ObAllPxWorkerStatTable::~ObAllPxWorkerStatTable() } void ObAllPxWorkerStatTable::reset() { - addr_ = NULL; start_to_read_ = false; stat_array_.reset(); index_ = 0; @@ -42,9 +41,9 @@ int ObAllPxWorkerStatTable::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; ObObj *cells = cur_row_.cells_; - if (OB_ISNULL(allocator_) || OB_ISNULL(addr_)) { + if (OB_ISNULL(allocator_)) { ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ or addr_ is null", K_(allocator), K_(addr), K(ret)); + SERVER_LOG(WARN, "allocator_ is null", K_(allocator), K(ret)); } else if (OB_ISNULL(cells)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "cur row cell is NULL", K(ret)); @@ -121,4 +120,3 @@ int ObAllPxWorkerStatTable::inner_get_next_row(ObNewRow *&row) }/* ns observer*/ }/* ns oceanbase */ - diff --git a/src/observer/virtual_table/ob_all_virtual_px_worker_stat.h b/src/observer/virtual_table/ob_all_virtual_px_worker_stat.h index 82bf3c033..4807d817a 100644 --- a/src/observer/virtual_table/ob_all_virtual_px_worker_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_px_worker_stat.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_PX_WORKER_STAT_TABLE_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "lib/net/ob_addr.h" #include "sql/engine/px/ob_px_worker_stat.h" namespace oceanbase { @@ -37,9 +36,7 @@ class ObAllPxWorkerStatTable : public common::ObVirtualTableScannerIterator virtual ~ObAllPxWorkerStatTable(); virtual void reset(); virtual int inner_get_next_row(common::ObNewRow *&row); - inline void set_addr(common::ObAddr &addr) { addr_ = &addr; } private: - common::ObAddr *addr_; bool start_to_read_; common::ObArray stat_array_; int64_t index_; @@ -62,4 +59,3 @@ class ObAllPxWorkerStatTable : public common::ObVirtualTableScannerIterator } // namespace observer } // namespace oceanbase #endif // OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_PX_WORKER_STAT_TABLE_ - diff --git a/src/observer/virtual_table/ob_all_virtual_replay_stat.cpp b/src/observer/virtual_table/ob_all_virtual_replay_stat.cpp index 77eea1319..db02c4222 100644 --- a/src/observer/virtual_table/ob_all_virtual_replay_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_replay_stat.cpp @@ -26,44 +26,26 @@ int ObAllVirtualReplayStat::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { - auto func_iter_ls = [&](const logservice::ObReplayStatus &replay_status) -> int - { - int ret = OB_SUCCESS; - logservice::LSReplayStat replay_stat; - if (OB_FAIL(replay_status.stat(replay_stat))) { - SERVER_LOG(WARN, "replay_status stat failed", K(ret), K(replay_status)); - } else if (OB_FAIL(insert_stat_(replay_stat))) { - SERVER_LOG(WARN, "insert stat failed", K(ret)); - } else if (OB_FAIL(scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "iter replay stat failed", KR(ret), K(replay_stat)); - } else { - SERVER_LOG(INFO, "iter replay stat succ", K(replay_stat)); - } - return ret; - }; + logservice::LSReplayStat replay_stat; logservice::ObLogService *log_service = share::g_mp->log_service(); if (NULL == log_service) { - SERVER_LOG(INFO, "tenant has no ObLogService"); - } else if (OB_FAIL(log_service->iterate_replay(func_iter_ls))) { - SERVER_LOG(WARN, "iter ls failed", K(ret)); + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "tenant has no ObLogService", K(ret)); + } else if (OB_FAIL(log_service->stat_replay(replay_stat))) { + SERVER_LOG(WARN, "stat replay failed", K(ret)); + } else if (OB_FAIL(insert_stat_(replay_stat))) { + SERVER_LOG(WARN, "insert stat failed", K(ret)); } else { - SERVER_LOG(INFO, "iter ls succ", K(ret)); + SERVER_LOG(INFO, "stat replay success", K(replay_stat)); } if (OB_FAIL(ret)) { SERVER_LOG(WARN, "iter tenant failed", K(ret)); } else { - scanner_it_ = scanner_.begin(); start_to_read_ = true; - } - } - if (OB_SUCC(ret) && start_to_read_) { - if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get next row failed", K(ret)); - } - } else { row = &cur_row_; } + } else { + ret = OB_ITER_END; } return ret; } @@ -76,29 +58,20 @@ int ObAllVirtualReplayStat::insert_stat_(logservice::LSReplayStat &replay_stat) uint64_t col_id = output_column_ids_.at(i); switch (col_id) { case OB_APP_MIN_COLUMN_ID: - if (OB_FAIL(role_to_string(replay_stat.role_, role_str_, sizeof(role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K(replay_stat)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - case OB_APP_MIN_COLUMN_ID + 1: cur_row_.cells_[i].set_uint64(replay_stat.end_lsn_.val_); break; - case OB_APP_MIN_COLUMN_ID + 2: + case OB_APP_MIN_COLUMN_ID + 1: cur_row_.cells_[i].set_bool(replay_stat.enabled_); break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 2: //TODO:SCN cur_row_.cells_[i].set_uint64(replay_stat.unsubmitted_lsn_.val_); break; - case OB_APP_MIN_COLUMN_ID + 4: { + case OB_APP_MIN_COLUMN_ID + 3: { cur_row_.cells_[i].set_uint64(replay_stat.unsubmitted_scn_.get_val_for_inner_table_field()); break; } - case OB_APP_MIN_COLUMN_ID + 5: + case OB_APP_MIN_COLUMN_ID + 4: cur_row_.cells_[i].set_int(replay_stat.pending_cnt_); break; default: diff --git a/src/observer/virtual_table/ob_all_virtual_replay_stat.h b/src/observer/virtual_table/ob_all_virtual_replay_stat.h index c8ac7a53f..1eeeca4be 100644 --- a/src/observer/virtual_table/ob_all_virtual_replay_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_replay_stat.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_OB_ALL_VIRTUAL_REPLAY_STAT_H_ #include "common/row/ob_row.h" -#include "observer/omt/ob_multi_tenant.h" #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "logservice/replayservice/ob_replay_status.h" @@ -29,17 +28,10 @@ namespace observer { class ObAllVirtualReplayStat : public common::ObVirtualTableScannerIterator { -public: - explicit ObAllVirtualReplayStat(omt::ObMultiTenant *omt) : omt_(omt) {} public: virtual int inner_get_next_row(common::ObNewRow *&row); private: int insert_stat_(logservice::LSReplayStat &replay_stat); -private: - static const int64_t VARCHAR_32 = 32; - char role_str_[VARCHAR_32] = {'\0'}; - char ip_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; - omt::ObMultiTenant *omt_; }; } // namespace observer } // namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_schema_memory.cpp b/src/observer/virtual_table/ob_all_virtual_schema_memory.cpp index 57b24ed71..332d64002 100644 --- a/src/observer/virtual_table/ob_all_virtual_schema_memory.cpp +++ b/src/observer/virtual_table/ob_all_virtual_schema_memory.cpp @@ -16,48 +16,26 @@ # define USING_LOG_PREFIX SERVER #include "ob_all_virtual_schema_memory.h" -#include "observer/ob_server_struct.h" namespace oceanbase { namespace observer { -int ObAllVirtualSchemaMemory::inner_open() +int ObAllVirtualSchemaMemory::get_next_mem_info_(ObSchemaMemory &schema_mem) { int ret = OB_SUCCESS; - const ObAddr &addr = GCTX.self_addr(); - if (false == addr.ip_to_string(ip_buffer_, sizeof(ip_buffer_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to convert ip to string", KR(ret), K(addr)); - } - return ret; -} - -int ObAllVirtualSchemaMemory::get_next_tenant_mem_info(ObSchemaMemory &schema_mem) { - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - if (mem_idx_ >= schema_mem_infos_.count()) { - do { + if (OB_INVALID_INDEX == mem_idx_) { + if (OB_FAIL(schema_service_.get_tenant_mem_info(1UL, schema_mem_infos_))) { + LOG_WARN("fail to get tenant mem info", KR(ret)); schema_mem_infos_.reset(); - if (++t_loop_idx_ >= 1) { - ret = OB_ITER_END; - } else { - - if (OB_SUCCESS != (tmp_ret = schema_service_.get_tenant_mem_info(1UL, schema_mem_infos_))) { - LOG_WARN("fail to get tenant mem info", KR(tmp_ret), K(1UL)); - schema_mem_infos_.reset(); - } else { - mem_idx_ = 0; - } - } - } while (0 == schema_mem_infos_.count() && OB_SUCC(ret)); + } else { + mem_idx_ = 0; + } } if (OB_SUCC(ret)) { - if (OB_UNLIKELY(mem_idx_ < 0 || mem_idx_ >= schema_mem_infos_.count())) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("mem_idx_ out of range", KR(ret), K(mem_idx_)); + if (mem_idx_ >= schema_mem_infos_.count()) { + ret = OB_ITER_END; } else { schema_mem = schema_mem_infos_[mem_idx_++]; } @@ -70,7 +48,7 @@ int ObAllVirtualSchemaMemory::inner_get_next_row(common::ObNewRow *&row) int ret = OB_SUCCESS; ObSchemaMemory schema_mem; - if (OB_FAIL(get_next_tenant_mem_info(schema_mem))) { + if (OB_FAIL(get_next_mem_info_(schema_mem))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next tenant_mem_info", KR(ret)); } diff --git a/src/observer/virtual_table/ob_all_virtual_schema_memory.h b/src/observer/virtual_table/ob_all_virtual_schema_memory.h index bfc297dbc..b40b23c6d 100644 --- a/src/observer/virtual_table/ob_all_virtual_schema_memory.h +++ b/src/observer/virtual_table/ob_all_virtual_schema_memory.h @@ -41,19 +41,16 @@ class ObAllVirtualSchemaMemory: public common::ObVirtualTableScannerIterator }; public: explicit ObAllVirtualSchemaMemory(share::schema::ObMultiVersionSchemaService &schema_service) - : t_loop_idx_(OB_INVALID_INDEX), mem_idx_(0), + : mem_idx_(OB_INVALID_INDEX), schema_service_(schema_service) {} virtual ~ObAllVirtualSchemaMemory() {} public: - virtual int inner_open(); virtual int inner_get_next_row(common::ObNewRow *&row); - int get_next_tenant_mem_info(ObSchemaMemory &schema_mem); private: - int64_t t_loop_idx_; + int get_next_mem_info_(ObSchemaMemory &schema_mem); + int64_t mem_idx_; - const static int64_t DEFAULT_TENANT_NUM = 10; const static int64_t DEFAULT_ALLOCATOR_COUNT = 2; - char ip_buffer_[OB_MAX_SERVER_ADDR_SIZE]; share::schema::ObMultiVersionSchemaService &schema_service_; common::ObSEArray schema_mem_infos_; }; //class ObAllVirtualServerSchemaMem diff --git a/src/observer/virtual_table/ob_all_virtual_schema_slot.cpp b/src/observer/virtual_table/ob_all_virtual_schema_slot.cpp index 89079820e..bff32756f 100644 --- a/src/observer/virtual_table/ob_all_virtual_schema_slot.cpp +++ b/src/observer/virtual_table/ob_all_virtual_schema_slot.cpp @@ -16,68 +16,42 @@ # define USING_LOG_PREFIX SERVER #include "observer/virtual_table/ob_all_virtual_schema_slot.h" -#include "observer/ob_server_struct.h" namespace oceanbase { namespace observer { -void ObAllVirtualSchemaSlot::reset(common::ObIAllocator &allocator, common::ObIArray &tenant_slot_infos) +void ObAllVirtualSchemaSlot::reset_slot_infos_(common::ObIAllocator &allocator) { - const char *ptr = NULL; - common::ObString str; - int ret = OB_SUCCESS; - int len = tenant_slot_infos.count(); - - for (int64_t i = 0; i < len && OB_SUCC(ret); ++i) { - ptr = ((tenant_slot_infos.at(i)).get_mod_ref_infos()).ptr(); + for (int64_t i = 0; i < schema_slot_infos_.count(); ++i) { + const char *ptr = schema_slot_infos_.at(i).get_mod_ref_infos().ptr(); if (OB_NOT_NULL(ptr)) { allocator.free(const_cast(ptr)); } - (tenant_slot_infos.at(i)).reset(); + schema_slot_infos_.at(i).reset(); } - tenant_slot_infos.reset(); + schema_slot_infos_.reset(); } -int ObAllVirtualSchemaSlot::inner_open() +int ObAllVirtualSchemaSlot::get_next_slot_info_(ObSchemaSlot &schema_slot) { - const ObAddr &addr = GCTX.self_addr(); - int ret = OB_SUCCESS; - - if (false == addr.ip_to_string(ip_buffer_, sizeof(ip_buffer_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to convert ip to string", KR(ret), K(addr)); - } - return ret; -} - -int ObAllVirtualSchemaSlot::get_next_tenant_slot_info(ObSchemaSlot &schema_slot) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; if (OB_ISNULL(allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("allocator_ is null", KR(ret)); - } else if (slot_idx_ >= schema_slot_infos_.count()) { - do { - reset(*allocator_, schema_slot_infos_); - if (++t_loop_idx_ >= 1) { - ret = OB_ITER_END; - } else { - - if (OB_SUCCESS != (tmp_ret = schema_service_.get_tenant_slot_info(*allocator_, 1UL, schema_slot_infos_))) { - LOG_WARN("fail to get tenant slot info", KR(tmp_ret), K(1UL)); - reset(*allocator_, schema_slot_infos_); - } else { - slot_idx_ = 0; - } - } - } while (0 == schema_slot_infos_.count() && OB_SUCC(ret)); + } else if (OB_INVALID_INDEX == slot_idx_) { + if (OB_FAIL(schema_service_.get_tenant_slot_info(*allocator_, 1UL, schema_slot_infos_))) { + LOG_WARN("fail to get tenant slot info", KR(ret)); + reset_slot_infos_(*allocator_); + } else { + slot_idx_ = 0; + } } if (OB_SUCC(ret)) { - if (OB_UNLIKELY(slot_idx_ < 0 || slot_idx_ >= schema_slot_infos_.count())) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("slot_idx_ out of range", KR(ret), K(slot_idx_)); + if (slot_idx_ >= schema_slot_infos_.count()) { + reset_slot_infos_(*allocator_); + ret = OB_ITER_END; } else { schema_slot = schema_slot_infos_[slot_idx_++]; } @@ -90,7 +64,7 @@ int ObAllVirtualSchemaSlot::inner_get_next_row(common::ObNewRow *&row) int ret = OB_SUCCESS; ObSchemaSlot schema_slot; - if (OB_FAIL(get_next_tenant_slot_info(schema_slot))) { + if (OB_FAIL(get_next_slot_info_(schema_slot))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next tenant_info", KR(ret)); } diff --git a/src/observer/virtual_table/ob_all_virtual_schema_slot.h b/src/observer/virtual_table/ob_all_virtual_schema_slot.h index 7f68c78c9..f1407fb83 100644 --- a/src/observer/virtual_table/ob_all_virtual_schema_slot.h +++ b/src/observer/virtual_table/ob_all_virtual_schema_slot.h @@ -43,23 +43,19 @@ class ObAllVirtualSchemaSlot: public common::ObVirtualTableScannerIterator }; public: explicit ObAllVirtualSchemaSlot(share::schema::ObMultiVersionSchemaService &schema_service) - : t_loop_idx_(OB_INVALID_INDEX), slot_idx_(0), + : slot_idx_(OB_INVALID_INDEX), schema_service_(schema_service), schema_slot_infos_() {} virtual ~ObAllVirtualSchemaSlot() {} public: - virtual int inner_open(); virtual int inner_get_next_row(common::ObNewRow *&row); - void reset(common::ObIAllocator &allocator, common::ObIArray &schema_slot_infos); - int get_next_tenant_slot_info(ObSchemaSlot &schema_slot); - private: - int64_t t_loop_idx_;// tenant_iterator - int64_t slot_idx_;// slot iterator - const static int64_t DEFAULT_TENANT_NUM = 10; + void reset_slot_infos_(common::ObIAllocator &allocator); + int get_next_slot_info_(ObSchemaSlot &schema_slot); + + int64_t slot_idx_; const static int64_t DEFAULT_SLOT_NUM = 128; - char ip_buffer_[OB_MAX_SERVER_ADDR_SIZE]; share::schema::ObMultiVersionSchemaService &schema_service_; - common::ObSEArray schema_slot_infos_;// specified tenant's all slot info + common::ObSEArray schema_slot_infos_; }; //class ObAllVirtualSchemaSlot }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_server.cpp b/src/observer/virtual_table/ob_all_virtual_server.cpp index 475cc9812..cbe45ab00 100644 --- a/src/observer/virtual_table/ob_all_virtual_server.cpp +++ b/src/observer/virtual_table/ob_all_virtual_server.cpp @@ -24,7 +24,6 @@ #include "share/ob_all_tenant_info.h" // ObAllTenantInfoProxy #include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx_storage/ob_ls_handle.h" using namespace oceanbase; using namespace oceanbase::observer; @@ -133,18 +132,9 @@ int ObAllVirtualServer::inner_get_next_row(ObNewRow *&row) // Get sync_scn and readable_scn from LS in real-time uint64_t sync_scn_val = 0; uint64_t readable_scn_val = 0; - storage::ObLSHandle ls_handle; - share::ObLSID sys_ls_id(share::ObLSID::SYS_LS_ID); - storage::ObLSService *ls_svr = share::g_mp->ls_service(); - storage::ObLS *ls = NULL; - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(sys_ls_id, ls_handle, storage::ObLSGetMod::OBSERVER_MOD))){ + storage::ObLS *ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))){ SERVER_LOG(WARN, "get ls failed", K(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls is NULL", K(ret)); } else { share::SCN sync_scn; share::SCN readable_scn; diff --git a/src/observer/virtual_table/ob_all_virtual_server_object_pool.cpp b/src/observer/virtual_table/ob_all_virtual_server_object_pool.cpp index 5e731c28e..db2403fc4 100644 --- a/src/observer/virtual_table/ob_all_virtual_server_object_pool.cpp +++ b/src/observer/virtual_table/ob_all_virtual_server_object_pool.cpp @@ -21,7 +21,7 @@ using namespace oceanbase::common; using namespace oceanbase::observer; using namespace oceanbase::transaction; -ObAllVirtualServerObjectPool::ObAllVirtualServerObjectPool() : addr_(NULL) +ObAllVirtualServerObjectPool::ObAllVirtualServerObjectPool() { } diff --git a/src/observer/virtual_table/ob_all_virtual_server_object_pool.h b/src/observer/virtual_table/ob_all_virtual_server_object_pool.h index c3ed83f9f..a6a5796d2 100644 --- a/src/observer/virtual_table/ob_all_virtual_server_object_pool.h +++ b/src/observer/virtual_table/ob_all_virtual_server_object_pool.h @@ -34,11 +34,8 @@ class ObAllVirtualServerObjectPool: public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - void set_addr(common::ObAddr &addr) { addr_ = &addr; } private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualServerObjectPool); -private: - common::ObAddr *addr_; }; } diff --git a/src/observer/virtual_table/ob_all_virtual_server_schema_info.cpp b/src/observer/virtual_table/ob_all_virtual_server_schema_info.cpp index ea85e77b5..edfdfe088 100644 --- a/src/observer/virtual_table/ob_all_virtual_server_schema_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_server_schema_info.cpp @@ -22,24 +22,13 @@ namespace oceanbase { namespace observer { -int ObAllVirtualServerSchemaInfo::inner_open() -{ - int ret = OB_SUCCESS; - idx_ = 0; - share::schema::ObSchemaGetterGuard guard; - if (OB_FAIL(schema_service_.get_tenant_schema_guard(guard))) { - LOG_WARN("fail to get schema guard", K(ret)); - } - return ret; -} - int ObAllVirtualServerSchemaInfo::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; - if (idx_ >= 1) { + if (start_to_read_) { ret = OB_ITER_END; } else { - + start_to_read_ = true; int64_t refreshed_schema_version = OB_INVALID_VERSION; int64_t received_schema_version = OB_INVALID_VERSION; int64_t schema_count = OB_INVALID_ID; @@ -97,7 +86,6 @@ int ObAllVirtualServerSchemaInfo::inner_get_next_row(common::ObNewRow *&row) if (OB_SUCC(ret)) { row = &cur_row_; - idx_++; } } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_server_schema_info.h b/src/observer/virtual_table/ob_all_virtual_server_schema_info.h index 8fca40e0d..3be68d54f 100644 --- a/src/observer/virtual_table/ob_all_virtual_server_schema_info.h +++ b/src/observer/virtual_table/ob_all_virtual_server_schema_info.h @@ -31,16 +31,12 @@ class ObAllVirtualServerSchemaInfo: public common::ObVirtualTableScannerIterator { public: explicit ObAllVirtualServerSchemaInfo(share::schema::ObMultiVersionSchemaService &schema_service) - : schema_service_(schema_service), idx_(0) {} + : schema_service_(schema_service) {} virtual ~ObAllVirtualServerSchemaInfo() {} public: - virtual int inner_open(); virtual int inner_get_next_row(common::ObNewRow *&row); - void destroy(); private: - const static int64_t DEFAULT_TENANT_NUM = 10; share::schema::ObMultiVersionSchemaService &schema_service_; - int64_t idx_; }; //class ObAllVirtualServerSchemaInfo }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.cpp b/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.cpp index e28e73d8f..87e4ebe47 100644 --- a/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.cpp @@ -26,7 +26,6 @@ namespace observer { ObAllVirtualStorageMetaMemoryStatus::ObAllVirtualStorageMetaMemoryStatus() : ObVirtualTableScannerIterator(), - addr_(), pool_idx_(0) { } @@ -40,11 +39,10 @@ void ObAllVirtualStorageMetaMemoryStatus::reset() { pool_idx_ = 0; status_arr_.reset(); - addr_.reset(); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualStorageMetaMemoryStatus::init(ObIAllocator *allocator, common::ObAddr &addr) +int ObAllVirtualStorageMetaMemoryStatus::init(ObIAllocator *allocator) { int ret = OB_SUCCESS; if (OB_UNLIKELY(start_to_read_)) { @@ -55,7 +53,6 @@ int ObAllVirtualStorageMetaMemoryStatus::init(ObIAllocator *allocator, common::O SERVER_LOG(WARN, "invalid argument", K(ret)); } else { allocator_ = allocator; - addr_ = addr; start_to_read_ = true; } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.h b/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.h index e168f3a39..b3ba137a9 100644 --- a/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.h +++ b/src/observer/virtual_table/ob_all_virtual_storage_meta_memory_status.h @@ -51,19 +51,15 @@ class ObAllVirtualStorageMetaMemoryStatus : public common::ObVirtualTableScanner public: ObAllVirtualStorageMetaMemoryStatus(); virtual ~ObAllVirtualStorageMetaMemoryStatus(); - int init(common::ObIAllocator *allocator, common::ObAddr &addr); + int init(common::ObIAllocator *allocator); virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); private: private: - static const int64_t STRING_LEN = 128; static const int64_t POOL_NUM = 10; private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; - char address_[STRING_LEN]; int64_t pool_idx_; ObSEArray status_arr_; private: diff --git a/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp b/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp index d6f5f335e..cb733066c 100644 --- a/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp +++ b/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp @@ -26,7 +26,6 @@ using namespace observer; ObAllVirtualTableMgr::ObAllVirtualTableMgr() : ObVirtualTableScannerIterator(), - addr_(), tablet_iter_(nullptr), tablet_allocator_("VTTable"), tablet_handle_(), @@ -49,7 +48,6 @@ void ObAllVirtualTableMgr::reset() tablet_iter_ = nullptr; } tablet_allocator_.reset(); - addr_.reset(); if (OB_NOT_NULL(iter_buf_)) { allocator_->free(iter_buf_); diff --git a/src/observer/virtual_table/ob_all_virtual_table_mgr.h b/src/observer/virtual_table/ob_all_virtual_table_mgr.h index b4fc33cef..d89b96e01 100644 --- a/src/observer/virtual_table/ob_all_virtual_table_mgr.h +++ b/src/observer/virtual_table/ob_all_virtual_table_mgr.h @@ -64,16 +64,13 @@ class ObAllVirtualTableMgr : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) { addr_ = addr; } private: int get_next_tablet(); int get_next_table(storage::ObITable *&table); private: - common::ObAddr addr_; storage::ObTenantTabletIterator *tablet_iter_; common::ObArenaAllocator tablet_allocator_; ObTabletHandle tablet_handle_; - char ip_buf_[common::OB_IP_STR_BUFF]; storage::ObTableStoreIterator table_store_iter_; void *iter_buf_; private: diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_buffer_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_buffer_info.h index 5a698d8d9..f4d4514a2 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_buffer_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_buffer_info.h @@ -25,7 +25,6 @@ #include "share/rc/ob_tenant_base.h" #include "observer/omt/ob_multi_tenant.h" #include "storage/meta_mem/ob_tablet_handle.h" -#include "share/ob_ls_id.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_info.cpp index 924ab0205..1a535b93f 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_info.cpp @@ -70,9 +70,6 @@ int ObAllVirtualTabletCompactionInfo::inner_get_next_row(common::ObNewRow *&row) } else if (OB_ISNULL(tablet = tablet_handle_.get_obj())) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "tablet is null", K(ret), K(tablet_handle_)); - } else if (!tablet->get_tablet_meta().ha_status_.is_data_status_complete()) { - ret = OB_EAGAIN; - LOG_DEBUG("query all_virtual_tablet_compaction_info, tablet_data not complete", K(ret), K(tablet->get_tablet_id()), K(tablet->get_ls_id())); } // quit while, excepted errcode : OB_ITER_END, OB_SUCCESS } while(OB_EAGAIN == ret); diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp index b801af2d1..47e74ee61 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/ddl/ob_tablet_ddl_kv.h" @@ -28,9 +29,8 @@ namespace observer ObAllVirtualTabletDDLKVInfo::ObAllVirtualTabletDDLKVInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_(), - ls_tablet_iter_(ObMDSGetTabletMode::READ_ALL_COMMITED), + ls_(nullptr), + tablet_iter_(ObMDSGetTabletMode::READ_ALL_COMMITED), ddl_kvs_handle_(), curr_tablet_id_(), ddl_kv_idx_(-1) @@ -47,55 +47,27 @@ void ObAllVirtualTabletDDLKVInfo::reset() ddl_kv_idx_ = -1; ddl_kvs_handle_.reset(); curr_tablet_id_.reset(); - ls_tablet_iter_.reset(); - ls_iter_guard_.reset(); - addr_.reset(); + tablet_iter_.reset(); + ls_ = nullptr; ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualTabletDDLKVInfo::get_next_ls(ObLS *&ls) -{ - int ret = OB_SUCCESS; - if (nullptr == ls_iter_guard_.get_ptr() && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "fail to get ls iter", K(ret)); - } else if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next ls", K(ret)); - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } - return ret; -} - int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle) { int ret = OB_SUCCESS; - while (OB_SUCC(ret)) { - if (!ls_tablet_iter_.is_valid()) { - ObLS *ls = nullptr; - if (OB_FAIL(get_next_ls(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next ls", K(ret)); - } - } else if (OB_FAIL(ls->build_tablet_iter(ls_tablet_iter_))) { - SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); - } + if (!tablet_iter_.is_valid()) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(ls_->build_tablet_iter(tablet_iter_))) { + SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls_tablet_iter_.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { - if (OB_ITER_END == ret) { - ls_tablet_iter_.reset(); - ret = OB_SUCCESS; - } else { - SERVER_LOG(WARN, "fail to get next tablet", K(ret)); - } - } else { - curr_tablet_id_ = ddl_kv_mgr_handle.get_obj()->get_tablet_id(); - break; + } + if (OB_SUCC(ret) && OB_FAIL(tablet_iter_.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get next tablet", K(ret)); } + } else if (OB_SUCC(ret)) { + curr_tablet_id_ = ddl_kv_mgr_handle.get_obj()->get_tablet_id(); } return ret; } @@ -103,7 +75,6 @@ int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mg int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv(ObDDLKV *&ddl_kv) { int ret = OB_SUCCESS; - ObTabletHandle tablet_handle; while (OB_SUCC(ret)) { if (ddl_kv_idx_ < 0 || ddl_kv_idx_ >= ddl_kvs_handle_.count()) { ObDDLKvMgrHandle ddl_kv_mgr_handle; @@ -137,11 +108,7 @@ int ObAllVirtualTabletDDLKVInfo::inner_get_next_row(ObNewRow *&row) int ret = OB_SUCCESS; ObDDLKV *cur_kv = nullptr; const int64_t col_count = output_column_ids_.count(); - if (NULL == allocator_) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (FALSE_IT(start_to_read_ = true)) { - } else if (OB_FAIL(get_next_ddl_kv(cur_kv))) { + if (OB_FAIL(get_next_ddl_kv(cur_kv))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next tablet with ddl kv", K(ret)); } @@ -152,27 +119,27 @@ int ObAllVirtualTabletDDLKVInfo::inner_get_next_row(ObNewRow *&row) for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); switch (col_id) { - case OB_APP_MIN_COLUMN_ID + 2: + case OB_APP_MIN_COLUMN_ID + 0: // tablet_id cur_row_.cells_[i].set_int(curr_tablet_id_.id()); break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 1: // freeze_scn cur_row_.cells_[i].set_uint64(cur_kv->get_freeze_scn().get_val_for_inner_table_field()); break; - case OB_APP_MIN_COLUMN_ID + 4: + case OB_APP_MIN_COLUMN_ID + 2: // start_scn cur_row_.cells_[i].set_uint64(cur_kv->get_ddl_start_scn().get_val_for_inner_table_field()); break; - case OB_APP_MIN_COLUMN_ID + 5: + case OB_APP_MIN_COLUMN_ID + 3: // min_scn cur_row_.cells_[i].set_uint64(cur_kv->get_min_scn().get_val_for_inner_table_field()); break; - case OB_APP_MIN_COLUMN_ID + 6: + case OB_APP_MIN_COLUMN_ID + 4: // macro_block_cnt cur_row_.cells_[i].set_int(cur_kv->get_macro_block_cnt()); break; - case OB_APP_MIN_COLUMN_ID + 7: + case OB_APP_MIN_COLUMN_ID + 5: // ref_cnt cur_row_.cells_[i].set_int(cur_kv->get_ref()); break; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h index f10879a43..047c8a4b8 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h @@ -19,10 +19,13 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "storage/tablet/ob_tablet_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { class ObAllVirtualTabletDDLKVInfo : public common::ObVirtualTableScannerIterator @@ -33,22 +36,15 @@ class ObAllVirtualTabletDDLKVInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls(ObLS *&ls); int get_next_ddl_kv_mgr(storage::ObDDLKvMgrHandle &ddl_kv_mgr_handle); int get_next_ddl_kv(ObDDLKV *&ddl_kv); private: - common::ObAddr addr_; - ObSharedGuard ls_iter_guard_; - storage::ObLSTabletIterator ls_tablet_iter_; + storage::ObLS *ls_; + storage::ObLSTabletIterator tablet_iter_; ObArray ddl_kvs_handle_; common::ObTabletID curr_tablet_id_; int64_t ddl_kv_idx_; - char ip_buf_[common::OB_IP_STR_BUFF]; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTabletDDLKVInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp index fd03bf843..284c80d92 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_tablet_info.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/multi_data_source/runtime_utility/common_define.h" #include "storage/tx_storage/ob_ls_service.h" @@ -28,9 +29,8 @@ namespace observer ObAllVirtualTabletInfo::ObAllVirtualTabletInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_(), - ls_tablet_iter_(ObMDSGetTabletMode::READ_WITHOUT_CHECK) + ls_(nullptr), + tablet_iter_(ObMDSGetTabletMode::READ_WITHOUT_CHECK) { } @@ -41,64 +41,30 @@ ObAllVirtualTabletInfo::~ObAllVirtualTabletInfo() void ObAllVirtualTabletInfo::reset() { - addr_.reset(); - ls_tablet_iter_.reset(); - ls_iter_guard_.reset(); + tablet_iter_.reset(); + ls_ = nullptr; ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualTabletInfo::get_next_ls(ObLS *&ls) +int ObAllVirtualTabletInfo::get_next_tablet(ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - - while (OB_SUCC(ret)) { - if (!ls_iter_guard_.get_ptr() - || OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to switch tenant", K(ret)); - } - // switch to next tenant - ret = OB_ITER_END; - } else if (OB_ISNULL(ls)) { + if (!tablet_iter_.is_valid()) { + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } else { - break; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(ls_->build_tablet_iter(tablet_iter_))) { + SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); } } - - return ret; -} - -int ObAllVirtualTabletInfo::get_next_tablet(ObTabletHandle &tablet_handle) -{ - int ret = OB_SUCCESS; - - while (OB_SUCC(ret)) { - if (!ls_tablet_iter_.is_valid()) { - ObLS *ls = nullptr; - if (OB_FAIL(get_next_ls(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next ls", K(ret)); - } - } else if (OB_FAIL(ls->build_tablet_iter(ls_tablet_iter_))) { - SERVER_LOG(WARN, "fail to build tablet iter", K(ret)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls_tablet_iter_.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { - ls_tablet_iter_.reset(); - ret = OB_SUCCESS; - } else { - SERVER_LOG(WARN, "fail to get next tablet", K(ret)); - } - } else { - break; + if (OB_SUCC(ret) && OB_FAIL(tablet_iter_.get_next_tablet(tablet_handle))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get next tablet", K(ret)); } } - return ret; } @@ -116,8 +82,6 @@ int ObAllVirtualTabletInfo::inner_get_next_row(ObNewRow *&row) ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); } else if (FALSE_IT(start_to_read_ = true)) { - } else if (ls_iter_guard_.get_ptr() == nullptr && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); } else if (OB_FAIL(get_next_tablet(tablet_handle))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get_next_tablet failed", K(ret)); @@ -174,7 +138,7 @@ int ObAllVirtualTabletInfo::inner_get_next_row(ObNewRow *&row) case OB_APP_MIN_COLUMN_ID + 6: { // restore_status ObTabletRestoreStatus::STATUS restore_status; - if (OB_FAIL(tablet_meta.ha_status_.get_restore_status(restore_status))) { + if (OB_FAIL(tablet_meta.restore_state_.get_restore_status(restore_status))) { SERVER_LOG(WARN, "failed to get restore status", K(ret), K(tablet_meta)); } else { cur_row_.cells_[i].set_int(restore_status); diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_info.h index a40a709c8..98b683e1b 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_info.h @@ -24,10 +24,13 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "storage/tablet/ob_tablet_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { class ObAllVirtualTabletInfo : public common::ObVirtualTableScannerIterator @@ -38,19 +41,11 @@ class ObAllVirtualTabletInfo : public common::ObVirtualTableScannerIterator public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls(ObLS *&ls); int get_next_tablet(storage::ObTabletHandle &tablet_handle); private: - common::ObAddr addr_; - ObSharedGuard ls_iter_guard_; - storage::ObLSTabletIterator ls_tablet_iter_; - char ip_buf_[common::OB_IP_STR_BUFF]; - common::ObSEArray tables_handle_; + storage::ObLS *ls_; + storage::ObLSTabletIterator tablet_iter_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTabletInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.cpp index a71ac3002..62af618dd 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.cpp @@ -26,7 +26,6 @@ namespace observer { ObAllVirtualTabletPtr::ObAllVirtualTabletPtr() : ObVirtualTableScannerIterator(), - addr_(), tablet_iter_(nullptr), iter_buf_(nullptr) { @@ -43,7 +42,6 @@ void ObAllVirtualTabletPtr::reset() tablet_iter_->~ObTenantTabletPtrWithInMemObjIterator(); tablet_iter_ = nullptr; } - addr_.reset(); if (OB_NOT_NULL(iter_buf_)) { allocator_->free(iter_buf_); iter_buf_ = nullptr; @@ -51,7 +49,7 @@ void ObAllVirtualTabletPtr::reset() ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualTabletPtr::init(ObIAllocator *allocator, common::ObAddr &addr) +int ObAllVirtualTabletPtr::init(ObIAllocator *allocator) { int ret = OB_SUCCESS; if (OB_UNLIKELY(start_to_read_)) { @@ -65,7 +63,6 @@ int ObAllVirtualTabletPtr::init(ObIAllocator *allocator, common::ObAddr &addr) SERVER_LOG(WARN, "fail to alloc tablet iter buf", K(ret)); } else { allocator_ = allocator; - addr_ = addr; start_to_read_ = true; } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.h b/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.h index d0f06c206..8e689b96c 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_pointer_status.h @@ -56,7 +56,7 @@ class ObAllVirtualTabletPtr : public common::ObVirtualTableScannerIterator public: ObAllVirtualTabletPtr(); virtual ~ObAllVirtualTabletPtr(); - int init(common::ObIAllocator *allocator, common::ObAddr &addr); + int init(common::ObIAllocator *allocator); virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); private: @@ -69,8 +69,6 @@ class ObAllVirtualTabletPtr : public common::ObVirtualTableScannerIterator static const int64_t STR_LEN = 128; static const int64_t ADDR_STR_LEN = 256; private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; char address_[ADDR_STR_LEN]; char pointer_[STR_LEN]; char old_chain_[STR_LEN]; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp index a9bd7fd69..fc0969877 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp @@ -684,7 +684,7 @@ bool ObAllVirtualTabletSSTableMacroInfo::check_tenant_need_ignore() { // In this feature branch, rowkey of __all_virtual_tablet_sstable_macro_info is: // (tablet_id, end_log_scn, macro_idx_in_sstable) - // It doesn't contain tenant/svr_ip/svr_port/ls_id, so we cannot safely prune tenants by key_ranges_. + // It doesn't contain tenant or server columns, so key_ranges_ cannot safely prune tenants. return false; } diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.cpp index f9a43eeb5..048da3b58 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.cpp @@ -25,7 +25,6 @@ namespace observer ObAllVirtualTabletStoreStat::ObAllVirtualTabletStoreStat() : stat_(), stat_iter_(), is_inited_(false) { - memset(ip_buf_, 0, sizeof(ip_buf_)); memset(rowkey_prefix_info_, 0, sizeof(rowkey_prefix_info_)); } @@ -72,7 +71,6 @@ void ObAllVirtualTabletStoreStat::reset() ObVirtualTableScannerIterator::reset(); stat_.reset(); stat_iter_.reset(); - memset(ip_buf_, 0, sizeof(ip_buf_)); memset(rowkey_prefix_info_, 0, sizeof(rowkey_prefix_info_)); is_inited_ = false; } @@ -89,168 +87,168 @@ int ObAllVirtualTabletStoreStat::fill_cells(const ObTableStoreStat &stat) for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); switch (col_id) { - case OB_APP_MIN_COLUMN_ID + 2: + case OB_APP_MIN_COLUMN_ID + 0: //table_id cells[i].set_int(stat.table_id_); break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 1: //tablet_id cells[i].set_int(stat.tablet_id_.id()); break; - case OB_APP_MIN_COLUMN_ID + 4: + case OB_APP_MIN_COLUMN_ID + 2: //row_cache_hit_count cells[i].set_int(stat.row_cache_hit_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 5: + case OB_APP_MIN_COLUMN_ID + 3: //row_cache_miss_count cells[i].set_int(stat.row_cache_miss_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 6: + case OB_APP_MIN_COLUMN_ID + 4: //row_cache_put_count cells[i].set_int(stat.row_cache_put_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 7: + case OB_APP_MIN_COLUMN_ID + 5: //bf_filter_count cells[i].set_int(stat.bf_filter_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 8: + case OB_APP_MIN_COLUMN_ID + 6: //bf_empty_read_count cells[i].set_int(stat.bf_empty_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 9: + case OB_APP_MIN_COLUMN_ID + 7: //bf_access_count cells[i].set_int(stat.bf_access_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 10: + case OB_APP_MIN_COLUMN_ID + 8: //block_cache_hit_count cells[i].set_int(stat.block_cache_hit_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 11: + case OB_APP_MIN_COLUMN_ID + 9: //block_cache_miss_count cells[i].set_int(stat.block_cache_miss_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 12: + case OB_APP_MIN_COLUMN_ID + 10: //access_row_count cells[i].set_int(stat.access_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 13: + case OB_APP_MIN_COLUMN_ID + 11: //outout_row_count cells[i].set_int(stat.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 14: + case OB_APP_MIN_COLUMN_ID + 12: //fuse_row_cache_hit_count cells[i].set_int(stat.fuse_row_cache_hit_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 15: + case OB_APP_MIN_COLUMN_ID + 13: //fuse_row_cache_miss_count cells[i].set_int(stat.fuse_row_cache_miss_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 16: + case OB_APP_MIN_COLUMN_ID + 14: //fuse_row_cache_put_count cells[i].set_int(stat.fuse_row_cache_put_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 17: + case OB_APP_MIN_COLUMN_ID + 15: //single_get_call_count cells[i].set_int(stat.single_get_stat_.call_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 18: + case OB_APP_MIN_COLUMN_ID + 16: //single_get_output_row_count cells[i].set_int(stat.single_get_stat_.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 19: + case OB_APP_MIN_COLUMN_ID + 17: //multi_get_call_count cells[i].set_int(stat.multi_get_stat_.call_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 20: + case OB_APP_MIN_COLUMN_ID + 18: //multi_get_output_row_count cells[i].set_int(stat.multi_get_stat_.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 21: + case OB_APP_MIN_COLUMN_ID + 19: //index_back_call_count cells[i].set_int(stat.index_back_stat_.call_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 22: + case OB_APP_MIN_COLUMN_ID + 20: //index_back_output_row_count cells[i].set_int(stat.index_back_stat_.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 23: + case OB_APP_MIN_COLUMN_ID + 21: //single_scan_call_count cells[i].set_int(stat.single_scan_stat_.call_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 24: + case OB_APP_MIN_COLUMN_ID + 22: //single_scan_output_row_count cells[i].set_int(stat.single_scan_stat_.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 25: + case OB_APP_MIN_COLUMN_ID + 23: //multi_scan_call_count cells[i].set_int(stat.multi_scan_stat_.call_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 26: + case OB_APP_MIN_COLUMN_ID + 24: //multi_scan_output_row_count cells[i].set_int(stat.multi_scan_stat_.output_row_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 27: + case OB_APP_MIN_COLUMN_ID + 25: //exist_row and effect read cells[i].set_int(stat.exist_row_.effect_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 28: + case OB_APP_MIN_COLUMN_ID + 26: //exist_row and empty read cells[i].set_int(stat.exist_row_.empty_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 29: + case OB_APP_MIN_COLUMN_ID + 27: //get_row and effect read cells[i].set_int(stat.get_row_.effect_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 30: + case OB_APP_MIN_COLUMN_ID + 28: //get_row and empty read cells[i].set_int(stat.get_row_.empty_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 31: + case OB_APP_MIN_COLUMN_ID + 29: //scan_row and effect read cells[i].set_int(stat.scan_row_.effect_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 32: + case OB_APP_MIN_COLUMN_ID + 30: //scan_row and empty read cells[i].set_int(stat.scan_row_.empty_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 33: + case OB_APP_MIN_COLUMN_ID + 31: //scan macro block access count cells[i].set_int(stat.macro_access_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 34: + case OB_APP_MIN_COLUMN_ID + 32: //scan micro block access count cells[i].set_int(stat.micro_access_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 35: + case OB_APP_MIN_COLUMN_ID + 33: //pushdown scan micro block access count cells[i].set_int(stat.pushdown_micro_access_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 36: + case OB_APP_MIN_COLUMN_ID + 34: //pushdown scan row access count cells[i].set_int(stat.pushdown_row_access_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 37: + case OB_APP_MIN_COLUMN_ID + 35: //pushdown scan filtered row count cells[i].set_int(stat.pushdown_row_select_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 38: + case OB_APP_MIN_COLUMN_ID + 36: //rowkey prefix read info cells[i].set_varchar(rowkey_prefix_info_); cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; - case OB_APP_MIN_COLUMN_ID + 39: + case OB_APP_MIN_COLUMN_ID + 37: // index block cache hit count cells[i].set_int(stat.index_block_cache_hit_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 40: + case OB_APP_MIN_COLUMN_ID + 38: // index block cache miss count cells[i].set_int(stat.index_block_cache_miss_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 41: + case OB_APP_MIN_COLUMN_ID + 39: // logical read count cells[i].set_int(stat.logical_read_cnt_); break; - case OB_APP_MIN_COLUMN_ID + 42: + case OB_APP_MIN_COLUMN_ID + 40: // physical read count cells[i].set_int(stat.physical_read_cnt_); break; @@ -264,4 +262,3 @@ int ObAllVirtualTabletStoreStat::fill_cells(const ObTableStoreStat &stat) } } /* namespace observer */ } /* namespace oceanbase */ - diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.h b/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.h index f2411fde9..377d54baf 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_store_stat.h @@ -35,7 +35,6 @@ class ObAllVirtualTabletStoreStat : public common::ObVirtualTableScannerIterator protected: int fill_cells(const storage::ObTableStoreStat &stat); private: - char ip_buf_[common::OB_IP_STR_BUFF]; char rowkey_prefix_info_[common::COLUMN_DEFAULT_LENGTH]; //json format storage::ObTableStoreStat stat_; storage::ObTableStoreStatIterator stat_iter_; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.cpp index cc58b5f90..e86aa6093 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.cpp @@ -25,9 +25,7 @@ namespace observer { ObAllVirtualTenantMemstoreInfo::ObAllVirtualTenantMemstoreInfo() - : ObVirtualTableScannerIterator(), - current_pos_(0), - addr_() + : ObVirtualTableScannerIterator() { } @@ -38,92 +36,56 @@ ObAllVirtualTenantMemstoreInfo::~ObAllVirtualTenantMemstoreInfo() void ObAllVirtualTenantMemstoreInfo::reset() { - current_pos_ = 0; - addr_.reset(); ObVirtualTableScannerIterator::reset(); } int ObAllVirtualTenantMemstoreInfo::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; - if (NULL == allocator_) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (!start_to_read_) { - ObObj *cells = NULL; - // allocator_ is allocator of PageArena type, no need to free - if (NULL == (cells = cur_row_.cells_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); - } else { - - char ip_buf[common::OB_IP_STR_BUFF]; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_ISNULL(omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "omt is null", K(ret)); + if (start_to_read_) { + ret = OB_ITER_END; + } else { + start_to_read_ = true; + int64_t active_span = 0; + int64_t memstore_used = 0; + int64_t freeze_trigger = 0; + int64_t memstore_limit = 0; + int64_t freeze_cnt = 0; + MOD_SCOPE { + storage::ObTenantFreezer *freezer = share::g_mp->tenant_freezer(); + if (OB_FAIL(freezer->get_tenant_memstore_cond(active_span, + memstore_used, + freeze_trigger, + memstore_limit, + freeze_cnt))) { + SERVER_LOG(WARN, "fail to get memstore used", K(ret)); } - // does not check ret code, run for the single (sys) tenant. - { - int64_t active_span = 0; - int64_t memstore_used = 0; - int64_t freeze_trigger = 0; - int64_t memstore_limit = 0; - int64_t freeze_cnt = 0; - MOD_SCOPE { - storage::ObTenantFreezer *freezer = nullptr; - if (FALSE_IT(freezer = share::g_mp->tenant_freezer())) { - } else if (OB_FAIL(freezer->get_tenant_memstore_cond(active_span, - memstore_used, - freeze_trigger, - memstore_limit, - freeze_cnt))) { - SERVER_LOG(WARN, "fail to get memstore used", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); ++i) { - uint64_t col_id = output_column_ids_.at(i); - switch (col_id) { - case ACTIVE_SPAN: - cells[i].set_int(active_span); - break; - case FREEZE_TRIGGER: - cells[i].set_int(freeze_trigger); - break; - case FREEZE_CNT: - cells[i].set_int(freeze_cnt); - break; - case MEMSTORE_USED: - cells[i].set_int(memstore_used); - break; - case MEMSTORE_LIMIT: - cells[i].set_int(memstore_limit); - break; - default: - // abnormal column id - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected column id", K(ret)); - break; - } - } - if (OB_SUCCESS == ret - && OB_SUCCESS != (ret = scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "fail to add row", K(ret), K(cur_row_)); - } + for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); ++i) { + switch (output_column_ids_.at(i)) { + case ACTIVE_SPAN: + cur_row_.cells_[i].set_int(active_span); + break; + case FREEZE_TRIGGER: + cur_row_.cells_[i].set_int(freeze_trigger); + break; + case FREEZE_CNT: + cur_row_.cells_[i].set_int(freeze_cnt); + break; + case MEMSTORE_USED: + cur_row_.cells_[i].set_int(memstore_used); + break; + case MEMSTORE_LIMIT: + cur_row_.cells_[i].set_int(memstore_limit); + break; + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected column id", K(ret), K(output_column_ids_.at(i))); + break; } } - // always start to read, event it failed. - scanner_it_ = scanner_.begin(); - start_to_read_ = true; - } - } - // always get next row, if we have start to read. - if (start_to_read_) { - if (OB_SUCCESS != (ret = scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next row", K(ret)); + if (OB_SUCC(ret)) { + row = &cur_row_; } - } else { - row = &cur_row_; } } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.h b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.h index 569b510c2..c29813dc0 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_info.h @@ -23,10 +23,6 @@ #include "share/ob_define.h" namespace oceanbase { -namespace common -{ -class ObAddr; -} namespace observer { class ObAllVirtualTenantMemstoreInfo : public common::ObVirtualTableScannerIterator @@ -44,10 +40,6 @@ class ObAllVirtualTenantMemstoreInfo : public common::ObVirtualTableScannerItera public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) { addr_ = addr; } -private: - uint64_t current_pos_; - common::ObAddr addr_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTenantMemstoreInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.cpp index ecbac8b8d..c2792de2a 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.cpp @@ -15,7 +15,6 @@ */ #include "ob_all_virtual_tenant_vector_mem_info.h" -#include "observer/omt/ob_multi_tenant.h" // previously hidden behind the server_struct include chain, make the dependency explicit #include "share/rc/ob_module_provider.h" #include "lib/alloc/memory_dump.h" #include "observer/vector_index/ob_plugin_vector_index_service.h" @@ -29,33 +28,21 @@ namespace observer { ObAllVirtualTenantVectorMemInfo::ObAllVirtualTenantVectorMemInfo() - : ObVirtualTableScannerIterator(), - addr_(), - current_pos_(0) + : ObVirtualTableScannerIterator() { } ObAllVirtualTenantVectorMemInfo::~ObAllVirtualTenantVectorMemInfo() { - reset(); -} - -void ObAllVirtualTenantVectorMemInfo::reset() -{ - current_pos_ = 0; - addr_.reset(); - ObVirtualTableScannerIterator::reset(); } int64_t ObAllVirtualTenantVectorMemInfo::fill_glibc_used_info() { int64_t used_size = 0; for (it_ = malloc_sample_map_.begin(); it_ != malloc_sample_map_.end(); ++it_) { - { - if (0 == STRNCMP("VIndex", it_->first.label_, strlen("VIndex")) && - 0 == STRNCMP("GLIBC", get_global_ctx_info().get_ctx_name(it_->first.ctx_id_), strlen("GLIBC"))) { - used_size += it_->second.alloc_bytes_; - } + if (0 == STRNCMP("VIndex", it_->first.label_, strlen("VIndex")) && + 0 == STRNCMP("GLIBC", get_global_ctx_info().get_ctx_name(it_->first.ctx_id_), strlen("GLIBC"))) { + used_size += it_->second.alloc_bytes_; } } return used_size; @@ -64,110 +51,84 @@ int64_t ObAllVirtualTenantVectorMemInfo::fill_glibc_used_info() int ObAllVirtualTenantVectorMemInfo::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; - if (NULL == allocator_) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (!start_to_read_) { - ObObj *cells = NULL; - // allocator_ is allocator of PageArena type, no need to free - if (NULL == (cells = cur_row_.cells_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); + if (start_to_read_) { + ret = OB_ITER_END; + } else { + start_to_read_ = true; + ObObj *cells = cur_row_.cells_; + if (OB_FAIL(malloc_sample_map_.create(1000, "MallocInfoMap", "MallocInfoMap"))) { + SERVER_LOG(WARN, "create memory info map failed", K(ret)); + } else if (OB_FAIL(ObMemoryDump::get_instance().load_malloc_sample_map(malloc_sample_map_))) { + SERVER_LOG(WARN, "load memory info map failed", K(ret)); } else { - - char ip_buf[common::OB_IP_STR_BUFF]; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_ISNULL(omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "omt is null", K(ret)); - } else if (OB_FAIL(malloc_sample_map_.create(1000, "MallocInfoMap", "MallocInfoMap"))) { - SERVER_LOG(WARN, "create memory info map failed", K(ret)); - } else if (OB_FAIL(ObMemoryDump::get_instance().load_malloc_sample_map(malloc_sample_map_))) { - SERVER_LOG(WARN, "load memory info map failed", K(ret)); - } - // does not check ret code, run for the single (sys) tenant. - { - int64_t manage_used = 0; - int64_t vector_hold = 0; - int64_t vector_limit = 0; - int64_t freeze_cnt = 0; - MOD_SCOPE { - ObPluginVectorIndexService *service = share::g_mp->plugin_vector_index_service(); - ObSharedMemAllocMgr *shared_mem_mgr = share::g_mp->shared_mem_alloc_mgr(); - manage_used = service->get_allocator().used(); - vector_hold = shared_mem_mgr->vector_allocator().hold(); - int64_t rb_used = shared_mem_mgr->vector_allocator().get_rb_mem_used(); - int64_t vector_used = shared_mem_mgr->vector_allocator().used(); - int64_t pos = 0; - int64_t glibc_used = fill_glibc_used_info(); - MEMSET(vector_used_str_, 0, sizeof(vector_used_str_)); - complete_tablet_ids_.reset(); - partial_tablet_ids_.reset(); - cache_tablet_ids_.reset(); - if (OB_FAIL(service->get_snapshot_ids(complete_tablet_ids_, partial_tablet_ids_))) { - SERVER_LOG(WARN, "failed to get snapshot_ids", K(ret)); - } else if (OB_FAIL(service->get_cache_ids(cache_tablet_ids_))) { - SERVER_LOG(WARN, "failed to get cache_ids", K(ret)); - } else if (OB_FAIL(databuff_printf(vector_used_str_, OB_MAX_MYSQL_VARCHAR_LENGTH, pos, "{\"rb_used\":%lu", rb_used))) { - SERVER_LOG(WARN, "failed to print total vector mem usage", K(ret), K(vector_hold)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::get_mem_context_detail_info(service, complete_tablet_ids_, - partial_tablet_ids_, cache_tablet_ids_, vector_used_str_, OB_MAX_MYSQL_VARCHAR_LENGTH, pos))) { - SERVER_LOG(WARN, "failed to print vector mem usage detail", K(ret), K(vector_hold)); - } else if (OB_FAIL(databuff_printf(vector_used_str_, OB_MAX_MYSQL_VARCHAR_LENGTH, pos, "}"))) { - SERVER_LOG(WARN, "failed to print total vector mem usage", K(ret)); - } - vector_limit = shared_mem_mgr->share_resource_throttle_tool().get_resource_limit(); - int64_t tx_share_limit = shared_mem_mgr->share_resource_throttle_tool().get_resource_limit(); - for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); ++i) { - uint64_t col_id = output_column_ids_.at(i); - switch (col_id) { - case RAW_MALLOC_SIZE: - cells[i].set_int(glibc_used); - break; - case INDEX_METADATA_SIZE: - cells[i].set_int(manage_used); - break; - case VECTOR_MEM_HOLD: - cells[i].set_int(vector_hold); - break; - case VECTOR_MEM_USED: - cells[i].set_int(vector_used); - break; - case VECTOR_MEM_LIMIT: - cells[i].set_int(vector_limit); - break; - case TX_SHARE_LIMIT: - cells[i].set_int(tx_share_limit); - break; - case VECTOR_MEM_DETAIL_INFO: - cells[i].set_varchar(vector_used_str_); - cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - default: - // abnormal column id - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected column id", K(ret)); - break; - } - } - if (OB_SUCCESS == ret - && OB_SUCCESS != (ret = scanner_.add_row(cur_row_))) { - SERVER_LOG(WARN, "fail to add row", K(ret), K(cur_row_)); + int64_t manage_used = 0; + int64_t vector_hold = 0; + int64_t vector_limit = 0; + MOD_SCOPE { + ObPluginVectorIndexService *service = share::g_mp->plugin_vector_index_service(); + ObSharedMemAllocMgr *shared_mem_mgr = share::g_mp->shared_mem_alloc_mgr(); + manage_used = service->get_allocator().used(); + vector_hold = shared_mem_mgr->vector_allocator().hold(); + const int64_t rb_used = shared_mem_mgr->vector_allocator().get_rb_mem_used(); + const int64_t vector_used = shared_mem_mgr->vector_allocator().used(); + int64_t pos = 0; + const int64_t glibc_used = fill_glibc_used_info(); + MEMSET(vector_used_str_, 0, sizeof(vector_used_str_)); + complete_tablet_ids_.reset(); + partial_tablet_ids_.reset(); + cache_tablet_ids_.reset(); + if (OB_FAIL(service->get_snapshot_ids(complete_tablet_ids_, partial_tablet_ids_))) { + SERVER_LOG(WARN, "failed to get snapshot_ids", K(ret)); + } else if (OB_FAIL(service->get_cache_ids(cache_tablet_ids_))) { + SERVER_LOG(WARN, "failed to get cache_ids", K(ret)); + } else if (OB_FAIL(databuff_printf(vector_used_str_, OB_MAX_MYSQL_VARCHAR_LENGTH, pos, + "{\"rb_used\":%lu", rb_used))) { + SERVER_LOG(WARN, "failed to print total vector mem usage", K(ret), K(vector_hold)); + } else if (OB_FAIL(ObPluginVectorIndexUtils::get_mem_context_detail_info( + service, complete_tablet_ids_, partial_tablet_ids_, cache_tablet_ids_, vector_used_str_, + OB_MAX_MYSQL_VARCHAR_LENGTH, pos))) { + SERVER_LOG(WARN, "failed to print vector mem usage detail", K(ret), K(vector_hold)); + } else if (OB_FAIL(databuff_printf(vector_used_str_, OB_MAX_MYSQL_VARCHAR_LENGTH, pos, "}"))) { + SERVER_LOG(WARN, "failed to print total vector mem usage", K(ret)); + } + vector_limit = shared_mem_mgr->share_resource_throttle_tool() + .get_resource_limit(); + const int64_t tx_share_limit = + shared_mem_mgr->share_resource_throttle_tool().get_resource_limit(); + for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); ++i) { + const uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case RAW_MALLOC_SIZE: + cells[i].set_int(glibc_used); + break; + case INDEX_METADATA_SIZE: + cells[i].set_int(manage_used); + break; + case VECTOR_MEM_HOLD: + cells[i].set_int(vector_hold); + break; + case VECTOR_MEM_USED: + cells[i].set_int(vector_used); + break; + case VECTOR_MEM_LIMIT: + cells[i].set_int(vector_limit); + break; + case TX_SHARE_LIMIT: + cells[i].set_int(tx_share_limit); + break; + case VECTOR_MEM_DETAIL_INFO: + cells[i].set_varchar(vector_used_str_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected column id", K(ret)); + break; } } } - // always start to read, event it failed. - scanner_it_ = scanner_.begin(); - start_to_read_ = true; } - } - // always get next row, if we have start to read. - if (start_to_read_) { - if (OB_SUCCESS != (ret = scanner_it_.get_next_row(cur_row_))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next row", K(ret)); - } - } else { + if (OB_SUCC(ret)) { row = &cur_row_; } } diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.h b/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.h index a8ecab1fd..62b141f7b 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.h @@ -41,19 +41,15 @@ class ObAllVirtualTenantVectorMemInfo : public common::ObVirtualTableScannerIter ObAllVirtualTenantVectorMemInfo(); virtual ~ObAllVirtualTenantVectorMemInfo(); public: - inline void set_addr(common::ObAddr &addr) { addr_ = addr; } virtual int inner_get_next_row(common::ObNewRow *&row); - virtual void reset(); private: int64_t fill_glibc_used_info(); - common::ObAddr addr_; - uint64_t current_pos_; lib::ObMallocSampleMap::const_iterator it_; lib::ObMallocSampleMap malloc_sample_map_; char vector_used_str_[OB_MAX_MYSQL_VARCHAR_LENGTH]; - common::ObSEArray complete_tablet_ids_; - common::ObSEArray partial_tablet_ids_; - common::ObSEArray cache_tablet_ids_; + common::ObSEArray complete_tablet_ids_; + common::ObSEArray partial_tablet_ids_; + common::ObSEArray cache_tablet_ids_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTenantVectorMemInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_timestamp_service.cpp b/src/observer/virtual_table/ob_all_virtual_timestamp_service.cpp index 58d5e4006..83d07dc44 100644 --- a/src/observer/virtual_table/ob_all_virtual_timestamp_service.cpp +++ b/src/observer/virtual_table/ob_all_virtual_timestamp_service.cpp @@ -15,9 +15,8 @@ */ #include "observer/virtual_table/ob_all_virtual_timestamp_service.h" -#include "observer/omt/ob_multi_tenant.h" // previously hidden behind the server_struct include chain, make the dependency explicit #include "share/rc/ob_module_provider.h" -#include "storage/tx_storage/ob_ls_service.h" +#include "storage/tx/ob_timestamp_access.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -31,88 +30,23 @@ namespace observer void ObAllVirtualTimestampService::reset() { - init_ = false; - - ts_value_ = 0; - service_role_ = ObTimestampAccess::ServiceType::FOLLOWER; - is_primary_ = false; - role_ = common::ObRole::FOLLOWER; - service_epoch_ = 0; - done_ = false; ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualTimestampService::prepare_start_to_read_() +int ObAllVirtualTimestampService::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; - const int64_t execute_timeout = 10 * 1000 * 1000; // 10s - if (OB_FAIL(fill_ids_())) { - SERVER_LOG(WARN, "fail to fill tenant ids", K(ret)); - } else { - start_to_read_ = true; - } - return ret; -} -int ObAllVirtualTimestampService::get_next_info_() -{ - int ret = OB_SUCCESS; - if (done_) { + if (start_to_read_) { ret = OB_ITER_END; - } - if (OB_SUCC(ret)) { - + } else { + start_to_read_ = true; MOD_SCOPE { - bool exist = false; - if (OB_FAIL(share::g_mp->ls_service()->check_ls_exist(IDS_LS, exist))) { - SERVER_LOG(WARN, "check ls exist fail", K(ret)); - } else if (!exist) { - ret = OB_LS_NOT_EXIST; - done_ = true; - } else { - share::g_mp->timestamp_access()->get_virtual_info(ts_value_, service_role_, role_, service_epoch_); - is_primary_ = true; - done_ = true; - } - } else { - done_ = true; + share::g_mp->timestamp_access()->get_virtual_info(ts_value_); } } - - return ret; -} - -int ObAllVirtualTimestampService::fill_ids_() -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(GCTX.omt_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "failed to get multi tenant from GCTX", K(ret)); - } - - return ret; -} - -int ObAllVirtualTimestampService::inner_get_next_row(ObNewRow *&row) -{ - int ret = OB_SUCCESS; - - if (!start_to_read_ && OB_FAIL(prepare_start_to_read_())) { - SERVER_LOG(WARN, "prepare start to read error", K(ret), K(start_to_read_)); - } else { - do { - if (OB_FAIL(get_next_info_())) { - if (OB_ITER_END != ret && OB_LS_NOT_EXIST != ret) { - SERVER_LOG(WARN, "ObAllVirtualTimestampService iter error", K(ret)); - } - } - } while (OB_TENANT_NOT_IN_SERVER == ret || OB_LS_NOT_EXIST == ret); - } if (OB_SUCC(ret)) { - SERVER_LOG(INFO, "ObAllVirtualTimestampService iter success", K(*this)); - const ObAddr self = GCTX.self_addr(); const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); @@ -122,29 +56,10 @@ int ObAllVirtualTimestampService::inner_get_next_row(ObNewRow *&row) break; } case OB_APP_MIN_COLUMN_ID + 1: { // ts_type - cur_row_.cells_[i].set_varchar(ObTimestampAccess::ts_type_to_cstr(is_primary_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - } - case OB_APP_MIN_COLUMN_ID + 2: { // service_role - cur_row_.cells_[i].set_varchar((ObTimestampAccess::service_type_to_cstr(service_role_))); + cur_row_.cells_[i].set_varchar("GTS"); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; } - case OB_APP_MIN_COLUMN_ID + 3: { // role - if (OB_FAIL(role_to_string(role_, role_str_, sizeof(role_str_)))) { - SERVER_LOG(WARN, "role_to_string failed", K(ret), K_(role)); - } else { - cur_row_.cells_[i].set_varchar(ObString::make_string(role_str_)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } - break; - } - case OB_APP_MIN_COLUMN_ID + 4: { // service_epoch - cur_row_.cells_[i].set_int(service_epoch_); - break; - } default: { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid coloum_id", K(ret), K(col_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_timestamp_service.h b/src/observer/virtual_table/ob_all_virtual_timestamp_service.h index 40b066dde..03fe5ce65 100644 --- a/src/observer/virtual_table/ob_all_virtual_timestamp_service.h +++ b/src/observer/virtual_table/ob_all_virtual_timestamp_service.h @@ -18,50 +18,22 @@ #define OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_H_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "storage/tx/ob_timestamp_access.h" -#include "lib/container/ob_array.h" namespace oceanbase { -namespace share -{ -namespace schema -{ -class ObMultiVersionSchemaService; -class ObSchemaGetterGuard; -} -} - namespace observer { class ObAllVirtualTimestampService: public common::ObVirtualTableScannerIterator { public: explicit ObAllVirtualTimestampService() { reset(); } - virtual ~ObAllVirtualTimestampService() { destroy(); } + virtual ~ObAllVirtualTimestampService() = default; public: - virtual void reset(); - virtual void destroy() { reset(); } - virtual int inner_get_next_row(common::ObNewRow *&row); - TO_STRING_KV(K_(ts_value), - K_(service_role), K_(is_primary), - K_(role), K_(service_epoch)); -private: - int prepare_start_to_read_(); - int get_next_info_(); - int fill_ids_(); + void reset() override; + int inner_get_next_row(common::ObNewRow *&row) override; + TO_STRING_KV(K_(ts_value)); private: - bool init_; - bool done_; - int64_t expire_time_; - char ip_buf_[common::OB_IP_STR_BUFF]; - int64_t ts_value_; - transaction::ObTimestampAccess::ServiceType service_role_; - bool is_primary_; - common::ObRole role_; - char role_str_[32]; - int64_t service_epoch_; private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTimestampService); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tracepoint_info.cpp b/src/observer/virtual_table/ob_all_virtual_tracepoint_info.cpp index c08c60b9c..61debd525 100644 --- a/src/observer/virtual_table/ob_all_virtual_tracepoint_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tracepoint_info.cpp @@ -24,7 +24,7 @@ using namespace sql; namespace observer { -ObAllTracepointInfo::ObAllTracepointInfo(): addr_(NULL) +ObAllTracepointInfo::ObAllTracepointInfo() { } @@ -32,11 +32,6 @@ ObAllTracepointInfo::~ObAllTracepointInfo() { } -void ObAllTracepointInfo::reset() -{ - addr_ = NULL; -} - int ObAllTracepointInfo::get_rows_from_tracepoint_info_list() { int ret = OB_SUCCESS; @@ -128,9 +123,9 @@ int ObAllTracepointInfo::inner_get_next_row(ObNewRow *&row) int ObAllTracepointInfo::fill_scanner() { int ret = OB_SUCCESS; - if (OB_ISNULL(allocator_) || OB_ISNULL(addr_) || OB_ISNULL(session_)) { + if (OB_ISNULL(allocator_) || OB_ISNULL(session_)) { ret = OB_NOT_INIT; - SERVER_LOG(WARN, "allocator_ or addr_ is null", K_(allocator), K_(addr), K(ret)); + SERVER_LOG(WARN, "allocator_ or session_ is null", K_(allocator), K_(session), K(ret)); } else { if (OB_FAIL(get_rows_from_tracepoint_info_list())) { SERVER_LOG(WARN, "get rows from tracepoint_info_list failed", K(ret)); @@ -144,4 +139,3 @@ int ObAllTracepointInfo::fill_scanner() }/* ns observer*/ }/* ns oceanbase */ - diff --git a/src/observer/virtual_table/ob_all_virtual_tracepoint_info.h b/src/observer/virtual_table/ob_all_virtual_tracepoint_info.h index 8865ab10a..3cdd09988 100644 --- a/src/observer/virtual_table/ob_all_virtual_tracepoint_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tracepoint_info.h @@ -18,7 +18,6 @@ #define OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_TRACEPOINT_INFO_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "lib/net/ob_addr.h" #include "lib/utility/ob_tracepoint.h" namespace oceanbase @@ -31,11 +30,8 @@ class ObAllTracepointInfo : public common::ObVirtualTableScannerIterator public: ObAllTracepointInfo(); virtual ~ObAllTracepointInfo(); - virtual void reset(); virtual int inner_get_next_row(common::ObNewRow *&row) override; - inline void set_addr(common::ObAddr &addr) { addr_ = &addr; } private: - common::ObAddr *addr_; enum INSPECT_COLUMN { TP_NO = common::OB_APP_MIN_COLUMN_ID, @@ -56,4 +52,3 @@ class ObAllTracepointInfo : public common::ObVirtualTableScannerIterator } // namespace observer } // namespace oceanbase #endif // OCEANBASE_OBSERVER_VIRTUAL_TABLE_ALL_TRACEPOINT_INFO_ - diff --git a/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.cpp b/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.cpp index 1bd5af2c0..87f0ef373 100644 --- a/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.cpp +++ b/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_transaction_checkpoint.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -28,8 +29,7 @@ namespace observer ObAllVirtualTransCheckpointInfo::ObAllVirtualTransCheckpointInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_() + ls_(nullptr) { } @@ -40,49 +40,30 @@ ObAllVirtualTransCheckpointInfo::~ObAllVirtualTransCheckpointInfo() void ObAllVirtualTransCheckpointInfo::reset() { - ls_iter_guard_.reset(); + ls_ = nullptr; ob_common_checkpoint_iter_.reset(); - addr_.reset(); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualTransCheckpointInfo::get_next_ls_(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - if (ls_iter_guard_.get_ptr() == nullptr - && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); - } else if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } - - return ret; -} - int ObAllVirtualTransCheckpointInfo::prepare_to_read_() { int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObArray infos; ob_common_checkpoint_iter_.reset(); - if (OB_FAIL(get_next_ls_(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } else if (NULL == ls) { + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ret), K(ls)); + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); } else if (FALSE_IT(infos.reset())) { - } else if (OB_FAIL(ls->get_common_checkpoint_info(infos))) { - SERVER_LOG(WARN, "get commoncheckpoint info failed", K(ret), KPC(ls)); + } else if (OB_FAIL(ls_->get_common_checkpoint_info(infos))) { + SERVER_LOG(WARN, "get commoncheckpoint info failed", K(ret), KPC(ls_)); } else { int64_t idx = 0; for (; idx < infos.count() && OB_SUCC(ret); ++idx) { if (OB_FAIL(ob_common_checkpoint_iter_.push(infos.at(idx)))) { - SERVER_LOG(ERROR, "ob_common_checkpoint_iter push failed", K(ret), KPC(ls)); + SERVER_LOG(ERROR, "ob_common_checkpoint_iter push failed", K(ret), KPC(ls_)); } } } @@ -101,25 +82,12 @@ int ObAllVirtualTransCheckpointInfo::prepare_to_read_() int ObAllVirtualTransCheckpointInfo::get_next_(ObCommonCheckpointVTInfo &common_checkpoint) { int ret = OB_SUCCESS; - // ensure inner_get_next_row can get new data - bool need_retry = true; - while (need_retry) { - if (!ob_common_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { - if (OB_ITER_END == ret) { - SERVER_LOG(DEBUG, "iterate commoncheckpoint info iter end", K(ret)); - } else { - SERVER_LOG(WARN, "prepare data failed", K(ret)); - } - } else if (OB_FAIL(ob_common_checkpoint_iter_.get_next(common_checkpoint))) { - if (OB_ITER_END == ret) { - ob_common_checkpoint_iter_.reset(); - SERVER_LOG(DEBUG, "iterate commoncheckpoint info iter in the ls end", K(ret)); - continue; - } else { - SERVER_LOG(WARN, "get next commoncheckpoint info error.", K(ret)); - } + if (!ob_common_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { + SERVER_LOG(WARN, "prepare data failed", K(ret)); + } else if (OB_FAIL(ob_common_checkpoint_iter_.get_next(common_checkpoint))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "get next commoncheckpoint info error.", K(ret)); } - need_retry = false; } return ret; } diff --git a/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.h b/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.h index ec406a0a4..bf5b658fb 100644 --- a/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.h +++ b/src/observer/virtual_table/ob_all_virtual_transaction_checkpoint.h @@ -19,11 +19,13 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "storage/checkpoint/ob_common_checkpoint.h" -#include "storage/tx_storage/ob_ls_map.h" -#include "observer/omt/ob_multi_tenant.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { static constexpr const char OB_COMMON_CHECKPOINT[] = "ob_common_checkpoint"; @@ -39,19 +41,12 @@ class ObAllVirtualTransCheckpointInfo : public common::ObVirtualTableScannerIter public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls_(ObLS *&ls); int prepare_to_read_(); int get_next_(storage::checkpoint::ObCommonCheckpointVTInfo &common_checkpoint); private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; char checkpoint_type_buf_[common::MAX_CHECKPOINT_TYPE_BUF_LENGTH]; - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; ObCommonCheckpointVTIterator ob_common_checkpoint_iter_; private: diff --git a/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.cpp b/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.cpp index b50162ca0..46e13f1ae 100644 --- a/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.cpp +++ b/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -28,8 +29,7 @@ namespace observer ObAllVirtualFreezeCheckpointInfo::ObAllVirtualFreezeCheckpointInfo() : ObVirtualTableScannerIterator(), - addr_(), - ls_iter_guard_() + ls_(nullptr) { } @@ -40,49 +40,30 @@ ObAllVirtualFreezeCheckpointInfo::~ObAllVirtualFreezeCheckpointInfo() void ObAllVirtualFreezeCheckpointInfo::reset() { - ls_iter_guard_.reset(); + ls_ = nullptr; ob_freeze_checkpoint_iter_.reset(); - addr_.reset(); ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualFreezeCheckpointInfo::get_next_ls_(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - if (ls_iter_guard_.get_ptr() == nullptr - && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", K(ret)); - } else if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } - - return ret; -} - int ObAllVirtualFreezeCheckpointInfo::prepare_to_read_() { int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObArray infos; ob_freeze_checkpoint_iter_.reset(); - if (OB_FAIL(get_next_ls_(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "get_next_ls failed", K(ret)); - } - } else if (NULL == ls) { + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls shouldn't NULL here", K(ret), K(ls)); + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); } else if (FALSE_IT(infos.reset())) { - } else if (OB_FAIL(ls->get_freezecheckpoint_info(infos))) { - SERVER_LOG(WARN, "get freezecheckpoint info failed", K(ret), KPC(ls)); + } else if (OB_FAIL(ls_->get_freezecheckpoint_info(infos))) { + SERVER_LOG(WARN, "get freezecheckpoint info failed", K(ret), KPC(ls_)); } else { int64_t idx = 0; for (; idx < infos.count() && OB_SUCC(ret); ++idx) { if (OB_FAIL(ob_freeze_checkpoint_iter_.push(infos.at(idx)))) { - SERVER_LOG(ERROR, "ob_freeze_checkpoint_iter push failed", K(ret), KPC(ls)); + SERVER_LOG(ERROR, "ob_freeze_checkpoint_iter push failed", K(ret), KPC(ls_)); } } } @@ -101,25 +82,12 @@ int ObAllVirtualFreezeCheckpointInfo::prepare_to_read_() int ObAllVirtualFreezeCheckpointInfo::get_next_(ObFreezeCheckpointVTInfo &freeze_checkpoint) { int ret = OB_SUCCESS; - // ensure inner_get_next_row can get new data - bool need_retry = true; - while (need_retry) { - if (!ob_freeze_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { - if (OB_ITER_END == ret) { - SERVER_LOG(DEBUG, "iterate freezecheckpoint info iter end", K(ret)); - } else { - SERVER_LOG(WARN, "prepare data failed", K(ret)); - } - } else if (OB_FAIL(ob_freeze_checkpoint_iter_.get_next(freeze_checkpoint))) { - if (OB_ITER_END == ret) { - ob_freeze_checkpoint_iter_.reset(); - SERVER_LOG(DEBUG, "iterate freezecheckpoint info iter in the ls end", K(ret)); - continue; - } else { - SERVER_LOG(WARN, "get next freezecheckpoint info error.", K(ret)); - } + if (!ob_freeze_checkpoint_iter_.is_ready() && OB_FAIL(prepare_to_read_())) { + SERVER_LOG(WARN, "prepare data failed", K(ret)); + } else if (OB_FAIL(ob_freeze_checkpoint_iter_.get_next(freeze_checkpoint))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "get next freezecheckpoint info error.", K(ret)); } - need_retry = false; } return ret; } diff --git a/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.h b/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.h index a2231d02a..578b30b82 100644 --- a/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.h +++ b/src/observer/virtual_table/ob_all_virtual_transaction_freeze_checkpoint.h @@ -18,11 +18,13 @@ #define OB_ALL_VIRTUAL_TRANSACTION_FREEZE_CHECKPOINT_H_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" -#include "observer/omt/ob_multi_tenant.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { static constexpr const char OB_FREEZE_CHECKPOINT[] = "ob_freeze_checkpoint"; @@ -38,20 +40,13 @@ class ObAllVirtualFreezeCheckpointInfo : public common::ObVirtualTableScannerIte public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: - int get_next_ls_(ObLS *&ls); int prepare_to_read_(); int get_next_(storage::checkpoint::ObFreezeCheckpointVTInfo &freeze_checkpoint); private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; char freeze_checkpoint_location_buf_[common::MAX_FREEZE_CHECKPOINT_LOCATION_BUF_LENGTH]; - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; ObFreezeCheckpointVTIterator ob_freeze_checkpoint_iter_; private: diff --git a/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.cpp b/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.cpp index c678a3355..602d0cec5 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.cpp @@ -27,6 +27,8 @@ namespace observer void ObGVTxCtxMgrStat::reset() { memstore_version_buffer_[0] = '\0'; + tx_ctx_mgr_stat_.reset(); + is_stat_outputted_ = false; ObVirtualTableScannerIterator::reset(); } @@ -35,6 +37,8 @@ void ObGVTxCtxMgrStat::destroy() { trans_service_ = NULL; memset(memstore_version_buffer_, 0, common::MAX_VERSION_LENGTH); + tx_ctx_mgr_stat_.reset(); + is_stat_outputted_ = false; ObVirtualTableScannerIterator::reset(); } @@ -51,16 +55,10 @@ int ObGVTxCtxMgrStat::prepare_start_to_read_() } else if (NULL == (cells = cur_row_.cells_)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); - } else if (OB_FAIL(trans_service_->iterate_tx_ctx_mgr_stat(tx_ctx_mgr_stat_iter_))) { - SERVER_LOG(WARN, "iterate_tx_ctx_mgr_stat error", K(ret)); - if (OB_NOT_RUNNING == ret || OB_NOT_INIT == ret) { - ret = OB_SUCCESS; - } + } else if (OB_FAIL(trans_service_->get_tx_ctx_mgr_stat(tx_ctx_mgr_stat_))) { + SERVER_LOG(WARN, "get_tx_ctx_mgr_stat error", K(ret)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tx_ctx_mgr_stat_iter_.set_ready())) { - TRANS_LOG(WARN, "tx_ctx_mgr_stat_iter set ready error", KR(ret)); - } else { + if (OB_SUCC(ret)) { start_to_read_ = true; } @@ -70,22 +68,17 @@ int ObGVTxCtxMgrStat::prepare_start_to_read_() int ObGVTxCtxMgrStat::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; - ObLSTxCtxMgrStat ls_tx_ctx_mgr_stat; if (!start_to_read_ && OB_SUCCESS != (ret = prepare_start_to_read_())) { SERVER_LOG(WARN, "prepare_start_to_read_ error", K(ret), K(start_to_read_)); - } else if (OB_SUCCESS != (ret = tx_ctx_mgr_stat_iter_.get_next(ls_tx_ctx_mgr_stat))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "tx_ctx_mgr_stat_iter_ get next error", K(ret)); - } else { - SERVER_LOG(DEBUG, "tx_ctx_mgr_stat_iter_ end success"); - } + } else if (is_stat_outputted_) { + ret = OB_ITER_END; } else { // Column order after removing svr_ip and svr_port: - // OB_APP_MIN_COLUMN_ID (16): is_master - // OB_APP_MIN_COLUMN_ID + 1 (17): is_stopped - // OB_APP_MIN_COLUMN_ID + 2 (18): state - // OB_APP_MIN_COLUMN_ID + 3 (19): state_str + // OB_APP_MIN_COLUMN_ID (16): is_stopped + // OB_APP_MIN_COLUMN_ID + 1 (17): block_tx + // OB_APP_MIN_COLUMN_ID + 2 (18): block_normal_tx + // OB_APP_MIN_COLUMN_ID + 3 (19): block_all // OB_APP_MIN_COLUMN_ID + 4 (20): total_trans_ctx_count // OB_APP_MIN_COLUMN_ID + 5 (21): mgr_addr const int64_t col_count = output_column_ids_.count(); @@ -93,31 +86,24 @@ int ObGVTxCtxMgrStat::inner_get_next_row(ObNewRow *&row) uint64_t col_id = output_column_ids_.at(i); switch (col_id) { case OB_APP_MIN_COLUMN_ID: - // is_master_ - cur_row_.cells_[i].set_int(ls_tx_ctx_mgr_stat.is_master()? 1 : 0); + cur_row_.cells_[i].set_int(tx_ctx_mgr_stat_.is_stopped() ? 1 : 0); break; case OB_APP_MIN_COLUMN_ID + 1: - // is_stopped_ - cur_row_.cells_[i].set_int(ls_tx_ctx_mgr_stat.is_stopped()? 1 : 0); + cur_row_.cells_[i].set_int(tx_ctx_mgr_stat_.is_tx_blocked() ? 1 : 0); break; case OB_APP_MIN_COLUMN_ID + 2: - // state_ - cur_row_.cells_[i].set_int(ls_tx_ctx_mgr_stat.get_state()); + cur_row_.cells_[i].set_int( + tx_ctx_mgr_stat_.is_normal_tx_blocked() ? 1 : 0); break; - case OB_APP_MIN_COLUMN_ID + 3: { - // state_str - ObCStringHelper helper; - int64_t state = ls_tx_ctx_mgr_stat.get_state(); - cur_row_.cells_[i].set_varchar(helper.convert((ObTxLSStateMgr::TxLSStateContainer::StateVal*)&state)); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + case OB_APP_MIN_COLUMN_ID + 3: + cur_row_.cells_[i].set_int(tx_ctx_mgr_stat_.is_all_blocked() ? 1 : 0); break; - } case OB_APP_MIN_COLUMN_ID + 4: // total_tx_ctx_count - cur_row_.cells_[i].set_int(ls_tx_ctx_mgr_stat.get_total_tx_ctx_count()); + cur_row_.cells_[i].set_int(tx_ctx_mgr_stat_.get_total_tx_ctx_count()); break; case OB_APP_MIN_COLUMN_ID + 5: - cur_row_.cells_[i].set_int(ls_tx_ctx_mgr_stat.get_mgr_addr()); + cur_row_.cells_[i].set_int(tx_ctx_mgr_stat_.get_mgr_addr()); break; default: ret = OB_ERR_UNEXPECTED; @@ -127,6 +113,7 @@ int ObGVTxCtxMgrStat::inner_get_next_row(ObNewRow *&row) } } if (OB_SUCC(ret)) { + is_stat_outputted_ = true; row = &cur_row_; } diff --git a/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.h b/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.h index 6f06c6d7f..1a585d68a 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_ctx_mgr_stat.h @@ -20,9 +20,6 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" #include "common/row/ob_row.h" -#include "lib/container/ob_se_array.h" -#include "common/ob_simple_iterator.h" -#include "storage/tx/ob_tx_ls_log_writer.h" #include "storage/tx/ob_trans_ctx.h" #include "storage/tx/ob_trans_ctx_mgr_v4.h" @@ -39,7 +36,10 @@ class ObGVTxCtxMgrStat: public common::ObVirtualTableScannerIterator { public: explicit ObGVTxCtxMgrStat(transaction::ObTransService *trans_service) - : trans_service_(trans_service) { reset(); } + : trans_service_(trans_service), tx_ctx_mgr_stat_(), + is_stat_outputted_(false) { + reset(); + } virtual ~ObGVTxCtxMgrStat() { destroy(); } public: int inner_get_next_row(common::ObNewRow *&row); @@ -51,7 +51,8 @@ class ObGVTxCtxMgrStat: public common::ObVirtualTableScannerIterator char memstore_version_buffer_[common::MAX_VERSION_LENGTH]; private: transaction::ObTransService *trans_service_; - transaction::ObTxCtxMgrStatIterator tx_ctx_mgr_stat_iter_; + transaction::ObLSTxCtxMgrStat tx_ctx_mgr_stat_; + bool is_stat_outputted_; private: DISALLOW_COPY_AND_ASSIGN(ObGVTxCtxMgrStat); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data.cpp b/src/observer/virtual_table/ob_all_virtual_tx_data.cpp index 7879267c1..a1824d87b 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_data.cpp @@ -16,6 +16,7 @@ #include "ob_all_virtual_tx_data.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -157,22 +158,17 @@ int ObAllVirtualTxData::generate_virtual_tx_data_row_(VirtualTxDataRow &tx_data_ int ret = OB_SUCCESS; MOD_SCOPE { - ObLSHandle ls_handle; ObLS *ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); - if (OB_FAIL(ls_service->get_ls(share::SYS_LS, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - if (OB_LS_NOT_EXIST == ret) { - ret = OB_ITER_END; - } else { - SERVER_LOG(WARN, "get ls from ls service failed", KR(ret)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "get ls failed from ls handle", KR(ret), K(ls_handle)); + SERVER_LOG(WARN, "ls service is null", KR(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + SERVER_LOG(WARN, "get ls from ls service failed", KR(ret)); } else if (OB_FAIL(ls->generate_virtual_tx_data_row(tx_id_, tx_data_row))) { - SERVER_LOG(WARN, "ls genenrate virtual tx data row failed", KR(ret), K(ls_handle)); + SERVER_LOG(WARN, "ls genenrate virtual tx data row failed", KR(ret), K(ls)); } else { - SERVER_LOG(DEBUG, "generate tx data row succeed", KPC(ls), K(tx_data_row)); + SERVER_LOG(DEBUG, "generate tx data row succeed", K(tx_data_row)); } } diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data.h b/src/observer/virtual_table/ob_all_virtual_tx_data.h index 5b20fcf00..6b65df34e 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_data.h @@ -62,7 +62,7 @@ class ObAllVirtualTxData : public common::ObVirtualTableScannerIterator { public: - ObAllVirtualTxData() : addr_(), tx_id_(0) {} + ObAllVirtualTxData() : tx_id_(0) {} ~ObAllVirtualTxData() {} TO_STRING_KV(K_(tx_id)); @@ -71,13 +71,8 @@ class ObAllVirtualTxData : public common::ObVirtualTableScannerIterator { virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset() { - addr_.reset(); + ObVirtualTableScannerIterator::reset(); tx_id_.reset(); - memset(ip_buf_, 0, sizeof(ip_buf_)); - } - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; } private: @@ -87,9 +82,7 @@ class ObAllVirtualTxData : public common::ObVirtualTableScannerIterator { int fill_in_row_(const VirtualTxDataRow &row_data, common::ObNewRow *&row); private: - common::ObAddr addr_; transaction::ObTransID tx_id_; - char ip_buf_[common::OB_IP_STR_BUFF]; VirtualTxDataRow tx_data_row_; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data_table.cpp b/src/observer/virtual_table/ob_all_virtual_tx_data_table.cpp index 07c51785b..3033b54e4 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data_table.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_data_table.cpp @@ -16,6 +16,7 @@ #include "observer/virtual_table/ob_all_virtual_tx_data_table.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::common; @@ -26,10 +27,10 @@ namespace observer { ObAllVirtualTxDataTable::ObAllVirtualTxDataTable() : ObVirtualTableScannerIterator(), - addr_(), memtable_array_pos_(-1), sstable_array_pos_(-1), - ls_iter_guard_(), + ls_(nullptr), + tables_loaded_(false), tablet_handle_(), table_store_wrapper_(), mgr_handle_(), @@ -45,8 +46,10 @@ ObAllVirtualTxDataTable::~ObAllVirtualTxDataTable() void ObAllVirtualTxDataTable::reset() { mgr_handle_.reset(); - ls_iter_guard_.reset(); - addr_.reset(); + ls_ = nullptr; + tables_loaded_ = false; + memtable_array_pos_ = -1; + sstable_array_pos_ = -1; ObVirtualTableScannerIterator::reset(); } @@ -61,9 +64,6 @@ int ObAllVirtualTxDataTable::inner_get_next_row(common::ObNewRow *&row) ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be nullptr", K(allocator_), KR(ret)); } else if (FALSE_IT(start_to_read_ = true)) { - } else if (ls_iter_guard_.get_ptr() == nullptr && - OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "get_ls_iter fail", KR(ret)); } else if (OB_FAIL(get_next_tx_data_table_(tx_data_table))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get next tx data table failed", KR(ret)); @@ -119,11 +119,7 @@ int ObAllVirtualTxDataTable::get_next_tx_data_table_(ObITable *&tx_data_table) { int ret = OB_SUCCESS; - // memtable_array_pos_ < 0 && sstable_array_pos_ < 0 means all the tx data memtables of this logstream have been - // disposed or the first time get_next_tx_data_memtable() is invoked, when get_next_tx_data_table_ is invoked the - // first time, memtable_array_pos_ and memtable_array_.count() are both -1 - while (OB_SUCC(ret) && memtable_array_pos_ < 0 && sstable_array_pos_ < 0) { - ObLS *ls = nullptr; + if (!tables_loaded_) { ObTablet *tablet = nullptr; ObIMemtableMgr *memtable_mgr = nullptr; memtable_handles_.reset(); @@ -132,16 +128,21 @@ int ObAllVirtualTxDataTable::get_next_tx_data_table_(ObITable *&tx_data_table) mgr_handle_.reset(); table_store_wrapper_.reset(); - if (OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to get next logstream", KR(ret)); - } - } else if (OB_FAIL(ls->get_tablet_svr()->get_tx_data_memtable_mgr(mgr_handle_))) { + auto *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", KR(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", KR(ret)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ls_->get_tablet_svr()->get_tx_data_memtable_mgr(mgr_handle_))) { SERVER_LOG(WARN, "fail to get tx data memtable mgr.", KR(ret)); } else if (FALSE_IT(memtable_mgr = mgr_handle_.get_memtable_mgr())) { } else if (OB_FAIL(memtable_mgr->get_all_memtables(memtable_handles_))) { SERVER_LOG(WARN, "fail to get all memtables for log stream", KR(ret)); - } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet(LS_TX_DATA_TABLET, tablet_handle_))) { + } else if (OB_FAIL(ls_->get_tablet_svr()->get_tablet(LS_TX_DATA_TABLET, tablet_handle_))) { SERVER_LOG(WARN, "fail to get tx data tablet", KR(ret)); } else if (FALSE_IT(tablet = tablet_handle_.get_obj())) { } else if (OB_FAIL(tablet->fetch_table_store(table_store_wrapper_))) { @@ -163,17 +164,21 @@ int ObAllVirtualTxDataTable::get_next_tx_data_table_(ObITable *&tx_data_table) memtable_array_pos_ = memtable_handles_.count() - 1; // iterate from the newest sstable in sstable handles sstable_array_pos_ = sstable_handles_.count() - 1; + tables_loaded_ = true; if (memtable_array_pos_ < 0 && sstable_array_pos_ < 0) { SERVER_LOG(INFO, - "empty logstream. may be offlined", + "transaction data tables are empty", KR(ret), - K(addr_), K(memtable_array_pos_), K(sstable_array_pos_)); } } } + if (OB_SUCC(ret) && memtable_array_pos_ < 0 && sstable_array_pos_ < 0) { + ret = OB_ITER_END; + } + if (OB_FAIL(ret)) { } else if (memtable_array_pos_ >= 0) { tx_data_table = memtable_handles_[memtable_array_pos_--].get_table(); diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data_table.h b/src/observer/virtual_table/ob_all_virtual_tx_data_table.h index 59e2072aa..222650631 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data_table.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_data_table.h @@ -18,10 +18,13 @@ #define OB_ALL_VIRTUAL_TX_DATA_TABLE_H_ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { +namespace storage +{ +class ObLS; +} namespace observer { class ObAllVirtualTxDataTable : public common::ObVirtualTableScannerIterator { @@ -51,10 +54,6 @@ class ObAllVirtualTxDataTable : public common::ObVirtualTableScannerIterator { public: virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); - inline void set_addr(common::ObAddr &addr) - { - addr_ = addr; - } private: int get_next_tx_data_table_(ObITable *&tx_data_memtable); @@ -62,12 +61,10 @@ class ObAllVirtualTxDataTable : public common::ObVirtualTableScannerIterator { int prepare_row_data_(ObITable *tx_data_table, RowData &row_data); private: - common::ObAddr addr_; - char ip_buf_[common::OB_IP_STR_BUFF]; - int64_t memtable_array_pos_; int64_t sstable_array_pos_; - ObSharedGuard ls_iter_guard_; + storage::ObLS *ls_; + bool tables_loaded_; ObTabletHandle tablet_handle_; ObTabletMemberWrapper table_store_wrapper_; ObMemtableMgrHandle mgr_handle_; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.cpp b/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.cpp index 12a7d0103..d88bd4b6a 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.cpp @@ -16,8 +16,9 @@ #include "observer/virtual_table/ob_all_virtual_tx_lock_stat.h" #include "share/rc/ob_module_provider.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -28,110 +29,44 @@ namespace observer { ObGVTxLockStat::ObGVTxLockStat() : ObVirtualTableScannerIterator(), - ip_buffer_(), - tx_id_buffer_(), - proxy_session_id_buffer_(), memtable_key_buffer_(), - ls_iter_guard_(), - ls_id_(share::ObLSID::INVALID_LS_ID), ls_(nullptr), - ls_tx_ctx_iter_(), + tx_ctx_iter_(), tx_lock_stat_iter_() {} ObGVTxLockStat::~ObGVTxLockStat() { reset(); } void ObGVTxLockStat::reset() { - ip_buffer_[0] = '\0'; - tx_id_buffer_[0] = '\0'; - proxy_session_id_buffer_[0] = '\0'; memtable_key_buffer_[0] = '\0'; - ls_iter_guard_.reset(); - ls_id_ = share::ObLSID::INVALID_LS_ID; ls_ = nullptr; - ls_tx_ctx_iter_.reset(); + tx_ctx_iter_.reset(); tx_lock_stat_iter_.reset(); start_to_read_ = false; ObVirtualTableScannerIterator::reset(); } -int ObGVTxLockStat::get_next_ls_(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - if (!ls_iter_guard_.get_ptr() || OB_FAIL(ls_iter_guard_->get_next(ls))) { - if (OB_ITER_END != ret) { - SERVER_LOG(WARN, "fail to switch tenant", K(ret)); - } - // switch to next tenant - ret = OB_ITER_END; - SERVER_LOG(DEBUG, "finish iterate this tenant, switch to next tenant then", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "ls is null", K(ret)); - } else { - ls_id_ = ls->get_ls_id().id(); - ls_tx_ctx_iter_.reset(); - tx_lock_stat_iter_.reset(); - } - - return ret; -} - -int ObGVTxLockStat::get_next_tx_ctx_(transaction::ObPartTransCtx *&tx_ctx) -{ - int ret = OB_SUCCESS; - int64_t retry_times = 0; - - while (OB_SUCC(ret)) { - if (!ls_tx_ctx_iter_.is_ready()) { - if (OB_ISNULL(ls_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "ls is null", K(ret), K(ls_id_)); - } else if (OB_FAIL(ls_->iterate_tx_ctx(ls_tx_ctx_iter_))) { - SERVER_LOG(WARN, "fail to get ls_tx_ctx_iter", K(ret), K(ls_id_)); - } - } else if (OB_FAIL(ls_tx_ctx_iter_.get_next_tx_ctx(tx_ctx))) { - if (OB_ITER_END != ret) { - retry_times++; - SERVER_LOG(WARN, "ls_tx_ctx_iter_.get_next_tx_ctx failed", K(ret), K(retry_times)); - ret = OB_SUCCESS; // continue loop - } - if (OB_ITER_END == ret || retry_times >= MAX_RETRY_TIMES){ - ls_tx_ctx_iter_.reset(); - SERVER_LOG(DEBUG, "iterate this ls finished, iterate next ls then", K(ret), K(ls_id_)); - if (OB_FAIL(get_next_ls_(ls_))) { - SERVER_LOG(WARN, "get next ls failed", K(ret)); - } - } - } else { - break; - } - } - return ret; -} - int ObGVTxLockStat::get_next_tx_lock_stat_iter_(transaction::ObTxLockStatIterator &tx_lock_stat_iter) { int ret = OB_SUCCESS; - transaction::ObPartTransCtx *tx_ctx = nullptr; + transaction::ObTxCtx *tx_ctx = nullptr; - if (OB_FAIL(get_next_tx_ctx_(tx_ctx))) { + if (OB_FAIL(tx_ctx_iter_.get_next_tx_ctx(tx_ctx))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next tx_ctx", K(ret)); } } else if (OB_ISNULL(tx_ctx)) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "tx_ctx is null", K(ret), K(ls_id_)); + SERVER_LOG(WARN, "tx_ctx is null", K(ret)); } else { tx_lock_stat_iter.reset(); if (OB_FAIL(tx_ctx->iterate_tx_lock_stat(tx_lock_stat_iter))) { - SERVER_LOG(WARN, "fail to get lock op iter", K(ret), K(ls_id_)); + SERVER_LOG(WARN, "fail to get lock op iter", K(ret)); } else if (OB_FAIL(tx_lock_stat_iter.set_ready())) { - SERVER_LOG(WARN, "set lock_op_iter_ ready failed", K(ret), K(ls_id_)); + SERVER_LOG(WARN, "set lock_op_iter_ ready failed", K(ret)); } } if (OB_NOT_NULL(tx_ctx)) { - ls_tx_ctx_iter_.revert_tx_ctx(tx_ctx); + tx_ctx_iter_.revert_tx_ctx(tx_ctx); } return ret; } @@ -142,13 +77,14 @@ int ObGVTxLockStat::get_next_tx_lock_stat_(ObTxLockStat &tx_lock_stat) while (OB_SUCC(ret)) { if (OB_FAIL(tx_lock_stat_iter_.get_next(tx_lock_stat))) { - if (OB_ITER_END != ret) { - TRANS_LOG(WARN, "get next tx_lock_stat failed", K(ret), K(ls_id_)); - } - if (OB_FAIL(get_next_tx_lock_stat_iter_(tx_lock_stat_iter_))) { - if (OB_ITER_END != ret) { - TRANS_LOG(WARN, "get next tx_lock_stat_iter failed", K(ret), K(ls_id_)); + if (OB_ITER_END == ret) { + if (OB_FAIL(get_next_tx_lock_stat_iter_(tx_lock_stat_iter_))) { + if (OB_ITER_END != ret) { + TRANS_LOG(WARN, "get next tx_lock_stat_iter failed", K(ret)); + } } + } else { + TRANS_LOG(WARN, "get next tx_lock_stat failed", K(ret)); } } else { break; @@ -161,14 +97,17 @@ int ObGVTxLockStat::get_next_tx_lock_stat_(ObTxLockStat &tx_lock_stat) int ObGVTxLockStat::prepare_start_to_read_() { int ret = OB_SUCCESS; + ObLSService *ls_service = share::g_mp->ls_service(); if (OB_ISNULL(allocator_)) { ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); - } else if (OB_ISNULL(ls_iter_guard_.get_ptr()) - && OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard_, ObLSGetMod::OBSERVER_MOD))) { - SERVER_LOG(WARN, "init ls_iter_guard_ failed", K(ret)); - } else if (OB_FAIL(get_next_ls_(ls_))) { - SERVER_LOG(WARN, "init ls_ failed", K(ret)); + } else if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(ls_service->get_ls(ls_))) { + SERVER_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(ls_->iterate_tx_ctx(tx_ctx_iter_))) { + SERVER_LOG(WARN, "fail to get tx ctx iter", K(ret)); } else if (OB_FAIL(get_next_tx_lock_stat_iter_(tx_lock_stat_iter_))) { SERVER_LOG(WARN, "init tx_lock_stat_iter_ failed", K(ret)); } else { @@ -184,15 +123,12 @@ int ObGVTxLockStat::inner_get_next_row(ObNewRow *&row) if (!start_to_read_ && OB_FAIL(prepare_start_to_read_())) { SERVER_LOG(WARN, "prepare_start_to_read_ error", K(ret), K(start_to_read_)); - ret = OB_ITER_END; // to avoid throw error code to client } else if (OB_FAIL(get_next_tx_lock_stat_(tx_lock_stat))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "get_next_lock_op failed", K(ret)); } - ret = OB_ITER_END; // to avoid throw error code to client } else { const int64_t col_count = output_column_ids_.count(); - int ret = OB_SUCCESS; for (int64_t i = 0; i < col_count; i++) { uint64_t col_id = output_column_ids_.at(i); @@ -200,42 +136,14 @@ int ObGVTxLockStat::inner_get_next_row(ObNewRow *&row) case OB_APP_MIN_COLUMN_ID + 0: { // trans_id - // For compatibility, column type should be determined by schema before cluster is in upgrade mode. - const ObColumnSchemaV2 *tmp_column_schema = nullptr; - bool type_is_varchar = true; - if (OB_ISNULL(table_schema_) || - OB_ISNULL(tmp_column_schema = table_schema_->get_column_schema(col_id))) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "table or column schema is null", KR(ret), KP(table_schema_), KP(tmp_column_schema)); - } else { - type_is_varchar = tmp_column_schema->get_meta_type().is_character_type(); - } - if (OB_SUCC(ret)) { - // Before version 4.2, the type of trans_id in this table - // is varchar, e.g. 'tx_id: 10000'. It's incompatible with - // other virtual tables. So we change it to integer since 4.2. - if (type_is_varchar) { - (void)tx_lock_stat.get_tx_id().to_string(tx_id_buffer_, - OB_MIN_BUFFER_SIZE); - cur_row_.cells_[i].set_varchar(tx_id_buffer_); - cur_row_.cells_[i].set_collation_type( - ObCharset::get_default_collation( - ObCharset::get_default_charset())); - } else { - cur_row_.cells_[i].set_int(tx_lock_stat.get_tx_id().get_id()); - } - } + cur_row_.cells_[i].set_int(tx_lock_stat.get_tx_id().get_id()); break; } case OB_APP_MIN_COLUMN_ID + 1: - // table_id - cur_row_.cells_[i].set_int(0); - break; - case OB_APP_MIN_COLUMN_ID + 2: // tablet_id cur_row_.cells_[i].set_int(tx_lock_stat.get_memtable_key_info().get_tablet_id().id()); break; - case OB_APP_MIN_COLUMN_ID + 3: + case OB_APP_MIN_COLUMN_ID + 2: // rowkey snprintf(memtable_key_buffer_, OB_MEMTABLE_KEY_BUFFER_SIZE, "%s", tx_lock_stat.get_memtable_key_info().read_buf()); if ('\0' == memtable_key_buffer_[0]) { @@ -246,28 +154,23 @@ int ObGVTxLockStat::inner_get_next_row(ObNewRow *&row) } cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; - case OB_APP_MIN_COLUMN_ID + 4: + case OB_APP_MIN_COLUMN_ID + 3: // session_id cur_row_.cells_[i].set_int(tx_lock_stat.get_client_sid()); break; - case OB_APP_MIN_COLUMN_ID + 5: - // proxy_session_id (schema-positional; obproxy support removed, always NULL) - cur_row_.cells_[i].reset(); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; - case OB_APP_MIN_COLUMN_ID + 6: + case OB_APP_MIN_COLUMN_ID + 4: // tx_ctx_create_time cur_row_.cells_[i].set_timestamp(tx_lock_stat.get_tx_ctx_create_time()); break; - case OB_APP_MIN_COLUMN_ID + 7: + case OB_APP_MIN_COLUMN_ID + 5: // expired_time cur_row_.cells_[i].set_timestamp(tx_lock_stat.get_tx_expired_time()); break; - case OB_APP_MIN_COLUMN_ID + 8: + case OB_APP_MIN_COLUMN_ID + 6: // time_after_recv cur_row_.cells_[i].set_int(ObTimeUtility::current_time() - tx_lock_stat.get_tx_ctx_create_time()); break; - case OB_APP_MIN_COLUMN_ID + 9: + case OB_APP_MIN_COLUMN_ID + 7: // row_lock_addr cur_row_.cells_[i].set_uint64(uint64_t(tx_lock_stat.get_memtable_key_info().get_row_lock())); break; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.h b/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.h index a2b02ff2f..c05c400b1 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_lock_stat.h @@ -19,7 +19,6 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "sql/ob_scanner.h" -#include "sql/ob_scanner.h" #include "common/row/ob_row.h" #include "lib/container/ob_se_array.h" #include "storage/tx/ob_trans_ctx_mgr.h" @@ -27,14 +26,9 @@ namespace oceanbase { -namespace common -{ -template -class ObSharedGuard; -} namespace storage { -class ObLSIterator; +class ObLS; } namespace transaction { @@ -42,17 +36,6 @@ class ObTransService; class ObTxLockStat; } -// bool is_inited_; -// common::ObAddr addr_; -// uint64_t tenant_; -// share::ObLSID ls_id_; -// ObMemtableKeyInfo memtable_key_; -// uint32_t session_id_; -// uint64_t proxy_session_id_; -// ObTransID tx_id_; -// int64_t ctx_create_time_; -// int64_t expired_time_; - namespace observer { class ObGVTxLockStat : public common::ObVirtualTableScannerIterator @@ -65,23 +48,13 @@ class ObGVTxLockStat : public common::ObVirtualTableScannerIterator void reset() override; private: int prepare_start_to_read_(); - int get_next_ls_(ObLS *&ls); - int get_next_tx_ctx_(transaction::ObPartTransCtx *&tx_ctx); int get_next_tx_lock_stat_iter_(transaction::ObTxLockStatIterator &tx_lock_stat_iter); int get_next_tx_lock_stat_(transaction::ObTxLockStat &tx_lock_stat); - static const int64_t OB_MIN_BUFFER_SIZE = 128; static const int64_t OB_MEMTABLE_KEY_BUFFER_SIZE = 128; - char ip_buffer_[common::OB_IP_STR_BUFF]; - char tx_id_buffer_[OB_MIN_BUFFER_SIZE]; - char proxy_session_id_buffer_[OB_MIN_BUFFER_SIZE]; char memtable_key_buffer_[OB_MEMTABLE_KEY_BUFFER_SIZE]; - int output_row_(const transaction::ObTxLockStat& tx_lock_stat, ObNewRow *&row); private: - static const int64_t MAX_RETRY_TIMES = 10; - ObSharedGuard ls_iter_guard_; - share::ObLSID ls_id_; - ObLS *ls_; - transaction::ObLSTxCtxIterator ls_tx_ctx_iter_; + storage::ObLS *ls_; + transaction::ObLSTxCtxIterator tx_ctx_iter_; transaction::ObTxLockStatIterator tx_lock_stat_iter_; }; }//observer diff --git a/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.cpp b/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.cpp index efa730379..ea9f139ef 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.cpp @@ -72,11 +72,12 @@ int ObGVTxSchedulerStat::inner_get_next_row(common::ObNewRow *&row) SERVER_LOG(WARN, "allocator_ shouldn't be nullptr", K(allocator_), KR(ret)); } else if (FALSE_IT(start_to_read_ = true)) { } else if (!tx_scheduler_stat_iter_.is_ready()) { - if (OB_FAIL(share::g_mp->trans_service()->iterate_tx_scheduler_stat(tx_scheduler_stat_iter_))) { + transaction::ObTransService *txs = share::g_mp->trans_service(); + if (OB_ISNULL(txs)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "transaction service is null", KR(ret)); + } else if (OB_FAIL(txs->iterate_tx_scheduler_stat(tx_scheduler_stat_iter_))) { SERVER_LOG(WARN, "iterate transaction scheduler error", KR(ret)); - if (OB_NOT_RUNNING == ret || OB_NOT_INIT == ret) { - ret = OB_SUCCESS; - } } if (OB_FAIL(ret)) { } else if (OB_FAIL(tx_scheduler_stat_iter_.set_ready())) { @@ -107,11 +108,8 @@ int ObGVTxSchedulerStat::inner_get_next_row(common::ObNewRow *&row) case CLUSTER_ID: cur_row_.cells_[i].set_int(tx_scheduler_stat.cluster_id_); break; - case COORDINATOR: - cur_row_.cells_[i].set_int(tx_scheduler_stat.coord_id_.id()); - break; - case PARTICIPANTS: - if (0 < tx_scheduler_stat.parts_.count()) { + case WRITE_STATE: + if (tx_scheduler_stat.has_write_state_) { tx_scheduler_stat.get_parts_str(parts_buffer_, OB_MAX_BUFFER_SIZE); cur_row_.cells_[i].set_varchar(parts_buffer_); cur_row_.cells_[i].set_default_collation_type(); diff --git a/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.h b/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.h index 0e1a2c88d..1e931cad2 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_scheduler_stat.h @@ -50,11 +50,10 @@ class ObGVTxSchedulerStat: public common::ObVirtualTableScannerIterator enum { SESSION_ID = common::OB_APP_MIN_COLUMN_ID, - TX_ID, - STATE, - CLUSTER_ID, - COORDINATOR, - PARTICIPANTS, + TX_ID, + STATE, + CLUSTER_ID, + WRITE_STATE, ISOLATION_LEVEL, SNAPSHOT_VERSION, ACCESS_MODE, diff --git a/src/observer/virtual_table/ob_all_virtual_tx_stat.cpp b/src/observer/virtual_table/ob_all_virtual_tx_stat.cpp index 51fc043a4..8721b5c17 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_stat.cpp @@ -26,105 +26,50 @@ namespace oceanbase { namespace observer { +ObGVTxStat::ObGVTxStat() + : ObVirtualTableScannerIterator(), + ctx_addr_buffer_(), + tx_stat_iter_(), + xid_(), + cstring_helper_() +{} + void ObGVTxStat::reset() { - participants_buffer_[0] = '\0'; - scheduler_buffer_[0] = '\0'; ctx_addr_buffer_[0] = '\0'; - - ObVirtualTableScannerIterator::reset(); + tx_stat_iter_.reset(); xid_.reset(); - init_ = false; cstring_helper_.reset(); -} - -void ObGVTxStat::destroy() -{ - memset(participants_buffer_, 0, OB_MAX_BUFFER_SIZE); - memset(scheduler_buffer_, 0, MAX_IP_PORT_LENGTH + 8); - memset(ctx_addr_buffer_, 0, CTX_ADDR_BUFFER_SIZE); - ObVirtualTableScannerIterator::reset(); - xid_.reset(); - init_ = false; - cstring_helper_.reset(); } int ObGVTxStat::prepare_start_to_read_() { int ret = OB_SUCCESS; tx_stat_iter_.reset(); - if (NULL == allocator_) { - SERVER_LOG(WARN, "invalid argument, allocator_ or txs_ is null", "allocator", - OB_P(allocator_)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_SUCCESS != (ret = fill_ids_())) { - SERVER_LOG(WARN, "fail to fill tenant ids", K(ret)); + if (OB_ISNULL(allocator_)) { + ret = OB_NOT_INIT; + SERVER_LOG(WARN, "allocator is null", K(ret)); } else { - { - MOD_SCOPE { - transaction::ObTransService *txs = share::g_mp->trans_service(); - if (OB_SUCCESS != (ret = txs->iterate_all_observer_tx_stat(tx_stat_iter_))) { - SERVER_LOG(WARN, "iterate transaction stat error", K(ret)); - // when interate tenant failed, show all info collected, not need return error code - if (OB_NOT_RUNNING == ret || OB_NOT_INIT == ret) { - ret = OB_SUCCESS; - } - } + MOD_SCOPE { + transaction::ObTransService *txs = share::g_mp->trans_service(); + if (OB_ISNULL(txs)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "transaction service is null", K(ret)); + } else if (OB_FAIL(txs->iterate_all_observer_tx_stat(tx_stat_iter_))) { + SERVER_LOG(WARN, "iterate transaction stat error", K(ret)); } } } - if (OB_SUCCESS != ret) { - // SERVER_LOG(WARN, "fail to prepare start to read", K(ret)); - } else if (OB_SUCCESS != (ret = tx_stat_iter_.set_ready())) { // set ready for the first count + if (OB_SUCC(ret) && OB_FAIL(tx_stat_iter_.set_ready())) { SERVER_LOG(WARN, "ObTransStatIterator set ready error", K(ret)); - } else { + } else if (OB_SUCC(ret)) { start_to_read_ = true; } return ret; } -int ObGVTxStat::init() -{ - int ret = OB_SUCCESS; - if (init_) { - ret = OB_INIT_TWICE; - SERVER_LOG(WARN, "init twice", K(ret)); - } else { - init_ = true; - } - return ret; -} - -int ObGVTxStat::get_next_tx_info_(ObTxStat &tx_stat) -{ - ObTxStat tmp_tx_stat; - - int ret = tx_stat_iter_.get_next(tmp_tx_stat); - - if (OB_SUCC(ret)) { - tx_stat = tmp_tx_stat; - } - - return ret; -} - -int ObGVTxStat::fill_ids_() -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!init_)) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "not init", K(ret)); - } else if (OB_ISNULL(GCTX.omt_)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "fail to get multi tenant from GCTX", K(ret)); - } - - return ret; -} - int ObGVTxStat::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; @@ -132,7 +77,7 @@ int ObGVTxStat::inner_get_next_row(ObNewRow *&row) if (!start_to_read_ && OB_SUCCESS != (ret = prepare_start_to_read_())) { SERVER_LOG(WARN, "prepare start to read error", K(ret), K(start_to_read_)); - } else if (OB_SUCCESS != (ret = get_next_tx_info_(tx_stat))) { + } else if (OB_FAIL(tx_stat_iter_.get_next(tx_stat))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "ObGVTxStat iter error", K(ret)); } else { @@ -145,33 +90,18 @@ int ObGVTxStat::inner_get_next_row(ObNewRow *&row) for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { uint64_t col_id = output_column_ids_.at(i); switch (col_id) { - case TX_TYPE: - cur_row_.cells_[i].set_int(tx_stat.tx_type_); - break; case TX_ID: cur_row_.cells_[i].set_int(tx_stat.tx_id_.get_id()); break; case SESSION_ID: cur_row_.cells_[i].set_int(tx_stat.client_sid_); break; - case SCHEDULER_ADDR: - (void)tx_stat.scheduler_addr_.to_string(scheduler_buffer_, MAX_IP_PORT_LENGTH + 8); - cur_row_.cells_[i].set_varchar(scheduler_buffer_); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - break; case IS_DECIDED: cur_row_.cells_[i].set_bool(tx_stat.has_decided_); break; - case PARTICIPANTS: - // if participants' count is equal to 0, then its value is NULL - if (0 < tx_stat.participants_.count()) { - (void)tx_stat.participants_.to_string(participants_buffer_, OB_MAX_BUFFER_SIZE); - cur_row_.cells_[i].set_varchar(participants_buffer_); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } else { - cur_row_.cells_[i].reset(); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } + case WRITE_STATE: + cur_row_.cells_[i].set_varchar(tx_stat.has_write_state_ ? "true" : "false"); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; case TX_CTX_CREATE_TIME: if (is_valid_timestamp_(tx_stat.tx_ctx_create_time_)) { @@ -206,29 +136,19 @@ int ObGVTxStat::inner_get_next_row(ObNewRow *&row) break; case TX_CTX_ADDR: ctx_addr_buffer_[0] = 0; - snprintf(ctx_addr_buffer_, 18, "0x%lx", (uint64_t)tx_stat.tx_ctx_addr_); + snprintf(ctx_addr_buffer_, CTX_ADDR_BUFFER_SIZE, "0x%lx", (uint64_t)tx_stat.tx_ctx_addr_); cur_row_.cells_[i].set_varchar(ctx_addr_buffer_); cur_row_.cells_[i].set_default_collation_type(); break; - case MEM_CTX_ID: - //TODO shanyan.g removed schema - cur_row_.cells_[i].set_int(-1); - break; case PENDING_LOG_SIZE: cur_row_.cells_[i].set_int(tx_stat.pending_log_size_); break; case FLUSHED_LOG_SIZE: cur_row_.cells_[i].set_int(tx_stat.flushed_log_size_); break; - case ROLE_STATE: - cur_row_.cells_[i].set_int(tx_stat.role_state_); - break; case IS_EXITING: cur_row_.cells_[i].set_int(tx_stat.is_exiting_); break; - case COORD: - cur_row_.cells_[i].set_int(tx_stat.coord_.id()); - break; case LAST_REQUEST_TS: cur_row_.cells_[i].set_timestamp(tx_stat.last_request_ts_); break; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_stat.h b/src/observer/virtual_table/ob_all_virtual_tx_stat.h index 7c6829b9b..a75713ba6 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_stat.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_stat.h @@ -23,7 +23,6 @@ #include "lib/container/ob_se_array.h" #include "common/ob_simple_iterator.h" #include "storage/tx/ob_trans_ctx.h" -#include "storage/tx/ob_tx_ls_log_writer.h" #include "storage/tx/ob_trans_ctx_mgr_v4.h" #include "lib/time/ob_clock_generator.h" #include "storage/tx/ob_tx_stat.h" @@ -46,27 +45,21 @@ namespace observer class ObGVTxStat: public common::ObVirtualTableScannerIterator { public: - explicit ObGVTxStat() { reset(); } - virtual ~ObGVTxStat() { destroy(); } + ObGVTxStat(); + ~ObGVTxStat() override = default; public: - int init(); - virtual int inner_get_next_row(common::ObNewRow *&row); - virtual void reset(); - virtual void destroy(); + int inner_get_next_row(common::ObNewRow *&row) override; + void reset() override; private: int prepare_start_to_read_(); - int fill_ids_(); - int get_next_tx_info_(transaction::ObTxStat &tx_stat); bool is_valid_timestamp_(const int64_t timestamp) const; private: enum { - TX_TYPE = OB_APP_MIN_COLUMN_ID, - TX_ID, + TX_ID = OB_APP_MIN_COLUMN_ID, SESSION_ID, - SCHEDULER_ADDR, IS_DECIDED, - PARTICIPANTS, + WRITE_STATE, TX_CTX_CREATE_TIME, TX_EXPIRED_TIME, REF_CNT, @@ -75,12 +68,9 @@ class ObGVTxStat: public common::ObVirtualTableScannerIterator STATE, PART_TX_ACTION, TX_CTX_ADDR, - MEM_CTX_ID, PENDING_LOG_SIZE, FLUSHED_LOG_SIZE, - ROLE_STATE, IS_EXITING, - COORD, LAST_REQUEST_TS, GTRID, BQUAL, @@ -94,14 +84,9 @@ class ObGVTxStat: public common::ObVirtualTableScannerIterator CALLBACK_LIST_STATS }; - static const int64_t OB_MAX_BUFFER_SIZE = 1024; - static const int64_t OB_MIN_BUFFER_SIZE = 128; static const int64_t CTX_ADDR_BUFFER_SIZE = 20; - char participants_buffer_[OB_MAX_BUFFER_SIZE]; - char scheduler_buffer_[common::MAX_IP_PORT_LENGTH + 8]; - char ctx_addr_buffer_[20]; + char ctx_addr_buffer_[CTX_ADDR_BUFFER_SIZE]; private: - bool init_; transaction::ObTxStatIterator tx_stat_iter_; transaction::ObXATransID xid_; ObCStringHelper cstring_helper_; diff --git a/src/observer/virtual_table/ob_all_virtual_unit.cpp b/src/observer/virtual_table/ob_all_virtual_unit.cpp index f055dbf48..66bdd66dc 100644 --- a/src/observer/virtual_table/ob_all_virtual_unit.cpp +++ b/src/observer/virtual_table/ob_all_virtual_unit.cpp @@ -29,7 +29,6 @@ using namespace oceanbase::logservice; ObAllVirtualUnit::ObAllVirtualUnit() : ObVirtualTableScannerIterator(), - addr_(), tenant_meta_(), has_row_(false), consumed_(false) @@ -43,21 +42,19 @@ ObAllVirtualUnit::~ObAllVirtualUnit() void ObAllVirtualUnit::reset() { - addr_.reset(); tenant_meta_ = omt::ObTenantMeta(); has_row_ = false; consumed_ = false; ObVirtualTableScannerIterator::reset(); } -int ObAllVirtualUnit::init(common::ObAddr &addr) +int ObAllVirtualUnit::init() { int ret = OB_SUCCESS; if (start_to_read_) { ret = OB_INIT_TWICE; SERVER_LOG(WARN, "cannot init twice", K(ret)); } else { - addr_ = addr; start_to_read_ = true; } return ret; @@ -208,4 +205,3 @@ int ObAllVirtualUnit::get_clog_disk_used_size_(int64_t &log_used_size) // return OB_SUCCESS whatever. return OB_SUCCESS; } - diff --git a/src/observer/virtual_table/ob_all_virtual_unit.h b/src/observer/virtual_table/ob_all_virtual_unit.h index 2076529fa..051041a68 100644 --- a/src/observer/virtual_table/ob_all_virtual_unit.h +++ b/src/observer/virtual_table/ob_all_virtual_unit.h @@ -49,15 +49,13 @@ class ObAllVirtualUnit : public common::ObVirtualTableScannerIterator public: ObAllVirtualUnit(); virtual ~ObAllVirtualUnit(); - int init(common::ObAddr &addr); + int init(); virtual int inner_open(); virtual void reset(); virtual int inner_get_next_row(common::ObNewRow *&row); private: int get_clog_disk_used_size_(int64_t &log_used_size); private: - char ip_buf_[common::OB_IP_STR_BUFF]; - common::ObAddr addr_; omt::ObTenantMeta tenant_meta_; bool has_row_; bool consumed_; diff --git a/src/observer/virtual_table/ob_all_virtual_vector_index_info.cpp b/src/observer/virtual_table/ob_all_virtual_vector_index_info.cpp index 33d7bedea..bdc9863e9 100644 --- a/src/observer/virtual_table/ob_all_virtual_vector_index_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_vector_index_info.cpp @@ -59,30 +59,25 @@ int ObVectorIndexInfoIterator::get_next_info(ObVectorIndexInfo &info) ret = OB_ITER_END; } else if (cache_idx_ < cache_tablet_ids_.count()) { // fill cache info - ObLSID ls_id = cache_tablet_ids_.at(cache_idx_).ls_id_; ObTabletID tablet_id = cache_tablet_ids_.at(cache_idx_).tablet_id_; ObIvfCacheMgrGuard cache_mgr_guard; - if (OB_FAIL(share::g_mp->plugin_vector_index_service()->acquire_ivf_cache_mgr_guard(ls_id, tablet_id, cache_mgr_guard))) { - SERVER_LOG(WARN, "failed to get cache mgr guard", K(ls_id), K(tablet_id), KR(ret)); + if (OB_FAIL(share::g_mp->plugin_vector_index_service()->acquire_ivf_cache_mgr_guard(tablet_id, cache_mgr_guard))) { + SERVER_LOG(WARN, "failed to get cache mgr guard", K(tablet_id), KR(ret)); } else if (OB_FAIL(cache_mgr_guard.get_ivf_cache_mgr()->fill_cache_info(info))) { - SERVER_LOG(WARN, "failed to fill cache info", K(ret), K(ls_id), K(tablet_id)); + SERVER_LOG(WARN, "failed to fill cache info", K(ret), K(tablet_id)); } - info.ls_id_ = ls_id.id(); cache_idx_++; } else { // fill vector index info - ObLSID ls_id; ObTabletID tablet_id; if (index_idx_ < complete_tablet_ids_.count()) { - ls_id = complete_tablet_ids_.at(index_idx_).ls_id_; tablet_id = complete_tablet_ids_.at(index_idx_).tablet_id_; } else if (index_idx_ < complete_tablet_ids_.count() + partial_tablet_ids_.count()) { - ls_id = partial_tablet_ids_.at(index_idx_ - complete_tablet_ids_.count()).ls_id_; tablet_id = partial_tablet_ids_.at(index_idx_ - complete_tablet_ids_.count()).tablet_id_; } ObPluginVectorIndexAdapterGuard adapter_guard; - if (OB_FAIL(share::g_mp->plugin_vector_index_service()->get_adapter_inst_guard(ls_id, tablet_id, adapter_guard))) { + if (OB_FAIL(share::g_mp->plugin_vector_index_service()->get_adapter_inst_guard(tablet_id, adapter_guard))) { if (OB_HASH_NOT_EXIST != ret) { - SERVER_LOG(WARN, "failed to get adapter inst guard", K(ls_id), K(tablet_id), KR(ret)); + SERVER_LOG(WARN, "failed to get adapter inst guard", K(tablet_id), KR(ret)); } } else if (OB_HASH_EXIST == (ret = ptr_set_.exist_refactored(reinterpret_cast(adapter_guard.get_adatper())))) { ret = OB_HASH_NOT_EXIST; // set OB_HASH_NOT_EXIST to ignore this adapter @@ -91,9 +86,7 @@ int ObVectorIndexInfoIterator::get_next_info(ObVectorIndexInfo &info) if (OB_FAIL(ptr_set_.set_refactored(reinterpret_cast(adapter_guard.get_adatper())))) { SERVER_LOG(WARN, "failed to set adapter check set", K(ret)); } else if (OB_FAIL(adapter_guard.get_adatper()->fill_vector_index_info(info))) { - SERVER_LOG(WARN, "failed to fill vector index info", K(ret), K(ls_id), K(tablet_id)); - } else { - info.ls_id_ = ls_id.id(); + SERVER_LOG(WARN, "failed to fill vector index info", K(ret), K(tablet_id)); } } else { SERVER_LOG(WARN, "failed to check adapter ptr", K(ret)); diff --git a/src/observer/virtual_table/ob_all_virtual_vector_index_info.h b/src/observer/virtual_table/ob_all_virtual_vector_index_info.h index b738ee19a..0b240af7c 100644 --- a/src/observer/virtual_table/ob_all_virtual_vector_index_info.h +++ b/src/observer/virtual_table/ob_all_virtual_vector_index_info.h @@ -19,7 +19,6 @@ #include "observer/virtual_table/ob_virtual_table_scanner_iterator.h" #include "observer/vector_index/ob_plugin_vector_index_adaptor.h" #include "storage/tablet/ob_tablet_iterator.h" -#include "storage/tx_storage/ob_ls_map.h" namespace oceanbase { @@ -49,9 +48,9 @@ class ObVectorIndexInfoIterator private: static const int64_t MAX_PTR_SET_VALUES = 32; common::ObArenaAllocator allocator_; - common::ObSEArray complete_tablet_ids_; - common::ObSEArray partial_tablet_ids_; - common::ObSEArray cache_tablet_ids_; + common::ObSEArray complete_tablet_ids_; + common::ObSEArray partial_tablet_ids_; + common::ObSEArray cache_tablet_ids_; common::hash::ObHashSet ptr_set_; // only for check // can't use elements int64_t index_idx_; int64_t cache_idx_; diff --git a/src/observer/virtual_table/ob_mds_event_buffer.h b/src/observer/virtual_table/ob_mds_event_buffer.h index 7f18a2762..c9411d1b5 100644 --- a/src/observer/virtual_table/ob_mds_event_buffer.h +++ b/src/observer/virtual_table/ob_mds_event_buffer.h @@ -23,7 +23,6 @@ #include "lib/string/ob_string.h" #include "share/cache/ob_vtable_event_recycle_buffer.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/multi_data_source/runtime_utility/common_define.h" #include "util/easy_time.h" #include "share/ob_task_define.h" @@ -50,23 +49,20 @@ class ObAllVirtualMdsEventHistory; class ObMdsEventBuffer; struct MdsEventKey { MdsEventKey() = default; - MdsEventKey(share::ObLSID ls_id, common::ObTabletID tablet_id) - : ls_id_(ls_id), - tablet_id_(tablet_id) {} + explicit MdsEventKey(common::ObTabletID tablet_id) + : tablet_id_(tablet_id) {} bool operator<(const MdsEventKey &rhs) { - return ls_id_ < rhs.ls_id_ || (ls_id_ == rhs.ls_id_ && tablet_id_ < rhs.tablet_id_); + return tablet_id_ < rhs.tablet_id_; } bool operator==(const MdsEventKey &rhs) { - return true && ls_id_ == rhs.ls_id_ && tablet_id_ == rhs.tablet_id_; + return tablet_id_ == rhs.tablet_id_; } uint64_t hash() const { uint64_t hash = 0; hash = murmurhash(&tablet_id_, sizeof(tablet_id_), 0); - hash = murmurhash(&ls_id_, sizeof(ls_id_), hash); return hash; } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); - share::ObLSID ls_id_; + TO_STRING_KV(K_(tablet_id)); common::ObTabletID tablet_id_; }; diff --git a/src/observer/virtual_table/ob_mem_leak_checker_info.cpp b/src/observer/virtual_table/ob_mem_leak_checker_info.cpp index 080dad259..ad401ab22 100644 --- a/src/observer/virtual_table/ob_mem_leak_checker_info.cpp +++ b/src/observer/virtual_table/ob_mem_leak_checker_info.cpp @@ -25,8 +25,7 @@ namespace observer { ObMemLeakCheckerInfo::ObMemLeakCheckerInfo() : ObVirtualTableIterator(), - opened_(false), - addr_(NULL) + opened_(false) { leak_checker_ = &get_mem_leak_checker(); label_ = leak_checker_->get_str(); @@ -41,16 +40,15 @@ void ObMemLeakCheckerInfo::reset() { opened_ = false; leak_checker_ = NULL; - addr_ = NULL; label_ = nullptr; } int ObMemLeakCheckerInfo::sanity_check() { int ret = OB_SUCCESS; - if (NULL == leak_checker_ || NULL == addr_) { + if (NULL == leak_checker_) { ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "invalid argument", K_(leak_checker), K_(addr)); + SERVER_LOG(WARN, "invalid argument", K_(leak_checker)); } return ret; } diff --git a/src/observer/virtual_table/ob_mem_leak_checker_info.h b/src/observer/virtual_table/ob_mem_leak_checker_info.h index bfc787b05..bced7cc1a 100644 --- a/src/observer/virtual_table/ob_mem_leak_checker_info.h +++ b/src/observer/virtual_table/ob_mem_leak_checker_info.h @@ -18,7 +18,6 @@ #define _OB_MEM_LEAK_CHECKER_INFO_H_ #include "share/ob_define.h" -#include "lib/net/ob_addr.h" #include "lib/allocator/ob_mem_leak_checker.h" #include "observer/virtual_table/ob_virtual_table_iterator.h" @@ -39,9 +38,6 @@ class ObMemLeakCheckerInfo : public common::ObVirtualTableIterator public: ObMemLeakCheckerInfo(); virtual ~ObMemLeakCheckerInfo(); - - inline void set_addr(common::ObAddr &addr) {addr_ = &addr;} - virtual int inner_get_next_row(common::ObNewRow *&row); virtual void reset(); private: @@ -52,7 +48,6 @@ class ObMemLeakCheckerInfo : public common::ObVirtualTableIterator common::ObMemLeakChecker *leak_checker_; common::ObMemLeakChecker::mod_info_map_t::hashmap::const_iterator it_; common::ObMemLeakChecker::mod_info_map_t info_map_; - common::ObAddr *addr_; const char *label_; private: DISALLOW_COPY_AND_ASSIGN(ObMemLeakCheckerInfo); diff --git a/src/observer/virtual_table/ob_virtual_data_access_service.cpp b/src/observer/virtual_table/ob_virtual_data_access_service.cpp index d46180bda..819dca732 100644 --- a/src/observer/virtual_table/ob_virtual_data_access_service.cpp +++ b/src/observer/virtual_table/ob_virtual_data_access_service.cpp @@ -28,9 +28,7 @@ namespace observer int ObVirtualDataAccessService::table_scan(ObVTableScanParam ¶m, ObNewRowIterator *&result) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_read); - const share::ObLSID &ls_id = param.ls_id_; common::ObASHTabletIdSetterGuard ash_tablet_id_guard(param.index_id_); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, param.index_id_); int ret = OB_SUCCESS; ObVirtualTableIterator *vt_iter = NULL; diff --git a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp index 8c30c84ec..22b29e4bb 100644 --- a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp +++ b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp @@ -174,12 +174,10 @@ #include "observer/virtual_table/ob_all_virtual_vector_index_info.h" #include "observer/virtual_table/ob_all_virtual_tmp_file.h" #include "observer/virtual_table/ob_all_virtual_dml_stats.h" -#include "observer/virtual_table/ob_all_virtual_log_transport_dest_stat.h" #include "observer/virtual_table/ob_all_virtual_plugin_info.h" #include "observer/virtual_table/ob_all_virtual_ddl_diagnose_info.h" #include "observer/virtual_table/ob_all_virtual_change_stream_refresh_stat.h" #include "observer/virtual_table/ob_all_virtual_dynamic_partition_table.h" -#include "observer/virtual_table/ob_all_virtual_tenant_mview_running_job.h" #include "observer/virtual_table/ob_all_virtual_tenant_vector_mem_info.h" #include "observer/virtual_table/ob_all_virtual_ccl_status.h" #include "observer/virtual_table/ob_show_create_location.h" @@ -547,7 +545,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTenantMemstoreInfo, gv_tenant_memstore_info))) { SERVER_LOG(ERROR, "ObAllVirtualTenantMemstoreInfo construct failed", K(ret)); } else { - gv_tenant_memstore_info->set_addr(addr_); vt_iter = static_cast(gv_tenant_memstore_info); } break; @@ -574,7 +571,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMemstoreInfo, all_virtual_memstore_info))) { SERVER_LOG(ERROR, "ObAllVirtualMemstoreInfo construct failed", K(ret)); } else { - all_virtual_memstore_info->set_addr(addr_); vt_iter = static_cast(all_virtual_memstore_info); } break; @@ -584,7 +580,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMinorFreezeInfo, all_virtual_minor_freeze_info))) { SERVER_LOG(ERROR, "ObAllVirtualMinorFreezeInfo construct failed", K(ret)); } else { - all_virtual_minor_freeze_info->set_addr(addr_); vt_iter = static_cast(all_virtual_minor_freeze_info); } break; @@ -594,7 +589,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualLSInfo, all_virtual_ls_info))) { SERVER_LOG(ERROR, "ObAllVirtualLSInfo construct failed", K(ret)); } else { - all_virtual_ls_info->set_addr(addr_); vt_iter = static_cast(all_virtual_ls_info); } break; @@ -604,7 +598,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualObjLock, all_virtual_obj_lock))) { SERVER_LOG(ERROR, "ObAllVirtualObjLock construct failed", K(ret)); } else { - all_virtual_obj_lock->set_addr(addr_); vt_iter = static_cast(all_virtual_obj_lock); } break; @@ -614,7 +607,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTabletInfo, all_virtual_tablet_info))) { SERVER_LOG(ERROR, "ObAllVirtualTabletInfo construct failed", K(ret)); } else { - all_virtual_tablet_info->set_addr(addr_); vt_iter = static_cast(all_virtual_tablet_info); } break; @@ -624,7 +616,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTxData, all_virtual_tx_data))) { SERVER_LOG(ERROR, "ObAllVirtualMemstoreInfo construct failed", K(ret)); } else { - all_virtual_tx_data->set_addr(addr_); vt_iter = static_cast(all_virtual_tx_data); } break; @@ -634,7 +625,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTxDataTable, all_virtual_tx_data_table))) { SERVER_LOG(ERROR, "ObAllVirtualMemstoreInfo construct failed", K(ret)); } else { - all_virtual_tx_data_table->set_addr(addr_); vt_iter = static_cast(all_virtual_tx_data_table); } break; @@ -645,7 +635,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, all_virtual_transaction_freeze_checkpoint))) { SERVER_LOG(ERROR, "ObAllVirtualFreezeCheckpointInfo construct failed", K(ret)); } else { - all_virtual_transaction_freeze_checkpoint->set_addr(addr_); vt_iter = static_cast(all_virtual_transaction_freeze_checkpoint); } break; @@ -656,7 +645,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, all_virtual_trans_checkpoint))) { SERVER_LOG(ERROR, "ObAllVirtualTransCheckpointInfo construct failed", K(ret)); } else { - all_virtual_trans_checkpoint->set_addr(addr_); vt_iter = static_cast(all_virtual_trans_checkpoint); } break; @@ -667,7 +655,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, all_virtual_checkpoint))) { SERVER_LOG(ERROR, "ObAllVirtualCheckpointInfo construct failed", K(ret)); } else { - all_virtual_checkpoint->set_addr(addr_); vt_iter = static_cast(all_virtual_checkpoint); } break; @@ -679,7 +666,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } else if (OB_FAIL(table_mgr->init(&allocator))) { SERVER_LOG(WARN, "failed to init all virtual table mgr", K(ret)); } else { - table_mgr->set_addr(addr_); vt_iter = static_cast(table_mgr); } break; @@ -688,7 +674,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObAllVirtualStorageMetaMemoryStatus *mem_status = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualStorageMetaMemoryStatus, mem_status))) { SERVER_LOG(ERROR, "ObAllVirtualStorageMetaMemoryStatus construct failed", K(ret)); - } else if (OB_FAIL(mem_status->init(&allocator, addr_))) { + } else if (OB_FAIL(mem_status->init(&allocator))) { SERVER_LOG(WARN, "failed to init ObAllVirtualStorageMetaMemoryStatus", K(ret)); } else { vt_iter = static_cast(mem_status); @@ -699,7 +685,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObAllVirtualTabletPtr *tablet_ptr_status = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTabletPtr, tablet_ptr_status))) { SERVER_LOG(ERROR, "ObAllVirtualTabletPtr construct failed", K(ret)); - } else if (OB_FAIL(tablet_ptr_status->init(&allocator, addr_))) { + } else if (OB_FAIL(tablet_ptr_status->init(&allocator))) { SERVER_LOG(WARN, "failed to init ObAllVirtualTabletPtr", K(ret)); } else { vt_iter = static_cast(tablet_ptr_status); @@ -722,7 +708,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTabletDDLKVInfo, ddl_kv_info))) { SERVER_LOG(ERROR, "ObAllVirtualTabletDDLKVInfo construct failed", K(ret)); } else { - ddl_kv_info->set_addr(addr_); vt_iter = static_cast(ddl_kv_info); } break; @@ -734,8 +719,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObGVTxStat *gv_tx_stat = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObGVTxStat, gv_tx_stat))) { SERVER_LOG(ERROR, "ObGVTxStat construct failed", K(ret)); - } else if (OB_FAIL(gv_tx_stat->init())) { - SERVER_LOG(WARN, "fail to init all_virtual_trans_stat", K(ret)); } else { vt_iter = static_cast(gv_tx_stat); } @@ -1185,8 +1168,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObMemLeakCheckerInfo *leak_checker = NULL; if (OB_SUCC(NEW_VIRTUAL_TABLE(ObMemLeakCheckerInfo, leak_checker))) { leak_checker->set_allocator(&allocator); - - leak_checker->set_addr(addr_); vt_iter = static_cast(leak_checker); } break; @@ -1261,7 +1242,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, case OB_ALL_VIRTUAL_SERVER_OBJECT_POOL_TID: { ObAllVirtualServerObjectPool *server_object_pool = NULL; if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllVirtualServerObjectPool, server_object_pool))) { - server_object_pool->set_addr(addr_); vt_iter = static_cast(server_object_pool); } break; @@ -1276,11 +1256,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_LOG_STAT_TID: { ObAllVirtualPalfStat *palf_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualPalfStat, palf_stat, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualPalfStat, palf_stat))) { SERVER_LOG(ERROR, "ObAllVirtualPalfStat construct fail", K(ret)); } else { vt_iter = static_cast(palf_stat); @@ -1289,11 +1265,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_APPLY_STAT_TID: { ObAllVirtualApplyStat *apply_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualApplyStat, apply_stat, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualApplyStat, apply_stat))) { SERVER_LOG(ERROR, "ObAllVirtualApplyStat construct fail", K(ret)); } else { vt_iter = static_cast(apply_stat); @@ -1302,11 +1274,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_REPLAY_STAT_TID: { ObAllVirtualReplayStat *replay_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualReplayStat, replay_stat, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualReplayStat, replay_stat))) { SERVER_LOG(ERROR, "ObAllVirtualReplayStat construct fail", K(ret)); } else { vt_iter = static_cast(replay_stat); @@ -1315,11 +1283,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_HA_DIAGNOSE_TID: { ObAllVirtualHADiagnose *diagnose_info = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualHADiagnose, diagnose_info, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualHADiagnose, diagnose_info))) { SERVER_LOG(ERROR, "ObAllVirtualHADiagnose construct fail", K(ret)); } else { vt_iter = static_cast(diagnose_info); @@ -1441,7 +1405,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualLongOpsStatus, long_ops_status))) { SERVER_LOG(ERROR, "fail to placement new ObAllVirtualLongOpsStatus", K(ret)); } else { - long_ops_status->set_addr(addr_); vt_iter = static_cast(long_ops_status); } break; @@ -1514,7 +1477,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObAllPxWorkerStatTable *px_worker_stat = NULL; if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllPxWorkerStatTable, px_worker_stat))) { px_worker_stat->set_allocator(&allocator); - px_worker_stat->set_addr(addr_); vt_iter = static_cast(px_worker_stat); } break; @@ -1523,7 +1485,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObAllPxP2PDatahubTable *px_p2p_datahub = NULL; if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllPxP2PDatahubTable, px_p2p_datahub))) { px_p2p_datahub->set_allocator(&allocator); - px_p2p_datahub->set_addr(addr_); vt_iter = static_cast(px_p2p_datahub); } break; @@ -1628,7 +1589,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, ObAllVirtualUnit *unit = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualUnit, unit))) { SERVER_LOG(ERROR, "ObAllVirtualUnit construct failed", K(ret)); - } else if (OB_FAIL(unit->init(addr_))) { + } else if (OB_FAIL(unit->init())) { SERVER_LOG(WARN, "failed to init all_virtual_unit", K(ret)); } else { vt_iter = static_cast(unit); @@ -1904,7 +1865,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualLoadDataStat, load_data_stat))) { SERVER_LOG(ERROR, "failed to init ObAllVirtualLoadDataStat", K(ret)); } else { - load_data_stat->set_addr(addr_); vt_iter = static_cast(load_data_stat); } break; @@ -1924,7 +1884,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllTracepointInfo, tp_info))) { SERVER_LOG(ERROR, "failed to init ObAllTracepointInfo", K(ret)); } else { - tp_info->set_addr(addr_); vt_iter = static_cast(tp_info); } break; @@ -1949,11 +1908,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_MDS_NODE_STAT_TID: { ObAllVirtualMdsNodeStat *mds_node_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMdsNodeStat, mds_node_stat, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMdsNodeStat, mds_node_stat))) { SERVER_LOG(ERROR, "ObAllVirtualMdsNodeStat construct fail", K(ret)); } else { vt_iter = static_cast(mds_node_stat); @@ -1962,11 +1917,7 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } case OB_ALL_VIRTUAL_MDS_EVENT_HISTORY_TID: { ObAllVirtualMdsEventHistory *mds_node_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_UNLIKELY(NULL == omt)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get tenant fail", K(ret)); - } else if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMdsEventHistory, mds_node_stat, omt))) { + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualMdsEventHistory, mds_node_stat))) { SERVER_LOG(ERROR, "ObAllVirtualMdsEventHistory construct fail", K(ret)); } else { vt_iter = static_cast(mds_node_stat); @@ -2037,7 +1988,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualActivityMetric, all_virtual_activity_metrics))) { SERVER_LOG(ERROR, "ObAllVirtualActivityMetric construct failed", K(ret)); } else { - all_virtual_activity_metrics->set_addr(addr_); vt_iter = static_cast(all_virtual_activity_metrics); } break; @@ -2087,14 +2037,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } break; } - case OB_ALL_VIRTUAL_MVIEW_RUNNING_JOB_TID: - { - ObAllVirtualTenantMviewRunningJob *running_job = NULL; - if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllVirtualTenantMviewRunningJob, running_job))) { - vt_iter = static_cast(running_job); - } - break; - } case OB_ALL_VIRTUAL_SQLSTAT_TID: { ObAllVirtualSqlStat *all_virtual_sqlstat = NULL; if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllVirtualSqlStat, all_virtual_sqlstat))) { @@ -2113,19 +2055,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } break; } - case OB_ALL_VIRTUAL_LOG_TRANSPORT_DEST_STAT_TID: - { - ObAllVirtualLogTransportDestStat *all_virtual_log_transport_dest_stat = NULL; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualLogTransportDestStat, - all_virtual_log_transport_dest_stat, omt))) { - SERVER_LOG(ERROR, "ObAllVirtualLogTransportDestStat construct fail", K(ret)); - } else { - all_virtual_log_transport_dest_stat->set_allocator(&allocator); - vt_iter = all_virtual_log_transport_dest_stat; - } - break; - } case OB_ALL_VIRTUAL_VECTOR_INDEX_INFO_TID: { ObAllVirtualVectorIndexInfo *all_virtual_vector_index_info = NULL; @@ -2153,7 +2082,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualPluginInfo, plugin_info_table))) { SERVER_LOG(ERROR, "ObAllVirtualPluginInfo construct failed", K(ret)); } else { - plugin_info_table->set_addr(addr_); vt_iter = static_cast(plugin_info_table); } } break; @@ -2175,7 +2103,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTenantVectorMemInfo, gv_tenant_vector_mem_info))) { SERVER_LOG(ERROR, "ObAllVirtualTenantVectorMemInfo construct failed", K(ret)); } else { - gv_tenant_vector_mem_info->set_addr(addr_); vt_iter = static_cast(gv_tenant_vector_mem_info); } break; diff --git a/src/observer/virtual_table/ob_virtual_table_tablet_iter.cpp b/src/observer/virtual_table/ob_virtual_table_tablet_iter.cpp index 121b0fcd9..effea1ea7 100644 --- a/src/observer/virtual_table/ob_virtual_table_tablet_iter.cpp +++ b/src/observer/virtual_table/ob_virtual_table_tablet_iter.cpp @@ -29,7 +29,6 @@ ObVirtualTableTabletIter::ObVirtualTableTabletIter() tablet_iter_(nullptr), tablet_allocator_("VTTable"), tablet_handle_(), - ls_id_(share::ObLSID::INVALID_LS_ID), iter_buf_(nullptr) { } @@ -42,8 +41,6 @@ ObVirtualTableTabletIter::~ObVirtualTableTabletIter() void ObVirtualTableTabletIter::reset() { addr_.reset(); - ls_id_ = share::ObLSID::INVALID_LS_ID; - if (OB_NOT_NULL(tablet_iter_)) { tablet_iter_->~ObTenantTabletIterator(); tablet_iter_ = nullptr; @@ -101,10 +98,7 @@ int ObVirtualTableTabletIter::get_next_tablet() } else if (OB_UNLIKELY(!tablet_handle_.is_valid())) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "unexpected invalid tablet", K(ret), K(tablet_handle_)); - } else { - ls_id_ = tablet_handle_.get_obj()->get_tablet_meta().ls_id_.id(); } return ret; } - diff --git a/src/observer/virtual_table/ob_virtual_table_tablet_iter.h b/src/observer/virtual_table/ob_virtual_table_tablet_iter.h index 4a9abf723..4f68f0818 100644 --- a/src/observer/virtual_table/ob_virtual_table_tablet_iter.h +++ b/src/observer/virtual_table/ob_virtual_table_tablet_iter.h @@ -49,7 +49,6 @@ class ObVirtualTableTabletIter : public common::ObVirtualTableScannerIterator storage::ObTenantTabletIterator *tablet_iter_; common::ObArenaAllocator tablet_allocator_; ObTabletHandle tablet_handle_; - int64_t ls_id_; char ip_buf_[common::OB_IP_STR_BUFF]; void *iter_buf_; private: diff --git a/src/pl/sys_package/ob_dbms_space.cpp b/src/pl/sys_package/ob_dbms_space.cpp index ac9bcac8a..d8144eca5 100644 --- a/src/pl/sys_package/ob_dbms_space.cpp +++ b/src/pl/sys_package/ob_dbms_space.cpp @@ -939,22 +939,8 @@ int ObDbmsSpace::get_svr_info_from_schema(const ObTableSchema *table_schema, } else if (tablet_list.count() <= 0) { ret = OB_ERR_UNEXPECTED; SQL_ENG_LOG(WARN, "can't find tablet", K(ret)); - } else { - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - ObLSID dummy_ls_id; - ObAddr leader_addr; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_list.count(); i++) { - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, - tablet_list.at(i), - rpc_timeout, - dummy_ls_id, - leader_addr - ))) { - SQL_ENG_LOG(WARN, "fail to get tablet's leader_addr", K(ret)); - } else if (OB_FAIL(add_var_to_array_no_dup(addr_list, leader_addr))) { - SQL_ENG_LOG(WARN, "fail to add add to addr_list", K(ret)); - } - } + } else if (OB_FAIL(addr_list.push_back(GCTX.self_addr()))) { + SQL_ENG_LOG(WARN, "fail to add local address", K(ret)); } return ret; diff --git a/src/rootserver/ddl_task/ob_ddl_common_rs_impl.cpp b/src/rootserver/ddl_task/ob_ddl_common_rs_impl.cpp index bfab38491..af1438a40 100644 --- a/src/rootserver/ddl_task/ob_ddl_common_rs_impl.cpp +++ b/src/rootserver/ddl_task/ob_ddl_common_rs_impl.cpp @@ -21,7 +21,6 @@ #include "observer/ob_service.h" #include "share/ob_ddl_common.h" #include "observer/omt/ob_multi_tenant.h" // previously hidden behind the server_struct include chain, make the dependency explicit -#include "storage/mview/ob_mview_refresh_helper.h" #include "storage/ob_storage_rpc.h" #include "storage/ob_storage_rpc_arg.h" #include "share/ob_rpc_struct.h" @@ -44,7 +43,6 @@ #include "observer/vector_index/ob_plugin_vector_index_utils.h" #include "observer/vector_index/ob_vector_index_util.h" #include "sql/resolver/ddl/ob_fts_index_builder_util.h" -#include "share/location_cache/ob_location_service.h" #include "storage/tx/ob_ts_mgr.h" #include "storage/tablet/ob_tablet_binding_helper.h" #include "rootserver/ddl_task/ob_ddl_task.h" @@ -57,52 +55,13 @@ using namespace oceanbase::share::schema; using namespace oceanbase::obcall; using namespace oceanbase::sql; -int ObDDLUtil::get_sys_log_handler_role_and_proposal_id( - common::ObRole &role, - int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - role = FOLLOWER; - proposal_id = 0; - if (OB_ISNULL(GCTX.omt_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(GCTX.omt_)); - } else if (OB_UNLIKELY(!GCTX.omt_->has_tenant())) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("local server does not have SYS tenant resource", KR(ret)); - } else { - MOD_SCOPE { - ObLSService *ls_svr = share::g_mp->ls_service(); - ObLS *ls = NULL; - ObLSHandle handle; - logservice::ObLogHandler *log_handler = NULL; - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl ObLSService should not be null", KR(ret), KP(ls_svr)); - } else if (OB_FAIL(ls_svr->get_ls(SYS_LS, handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", KR(ret)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be null", KR(ret)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log_handler is null", KR(ret), KP(log_handler)); - } else if (OB_FAIL(log_handler->get_role(role, proposal_id))) { - LOG_WARN("fail to get role and epoch", KR(ret)); - } - } - } - return ret; -} - int ObDDLUtil::hold_snapshot( common::ObMySQLTransaction &trans, rootserver::ObDDLTask* task, const uint64_t table_id, const uint64_t target_table_id, rootserver::ObRootService *root_service, - const int64_t snapshot_version, - const common::ObIArray *extra_mv_tablet_ids) + const int64_t snapshot_version) { int ret = OB_SUCCESS; if (OB_ISNULL(task) || OB_ISNULL(root_service)) { @@ -157,12 +116,6 @@ int ObDDLUtil::hold_snapshot( if (OB_FAIL(ddl_service.get_snapshot_mgr().batch_acquire_snapshot( trans, SNAPSHOT_FOR_DDL, schema_version, snapshot_scn, nullptr, tablet_ids))) { LOG_WARN("batch acquire snapshot failed", K(ret), K(tablet_ids)); - } else if (OB_NOT_NULL(extra_mv_tablet_ids) && - !extra_mv_tablet_ids->empty() && - OB_FAIL(ddl_service.get_snapshot_mgr().batch_acquire_snapshot( - trans, SNAPSHOT_FOR_MAJOR_REFRESH_MV, schema_version, snapshot_scn, - nullptr, *extra_mv_tablet_ids))) { - LOG_WARN("batch acquire mv snapshot failed", K(ret), K(extra_mv_tablet_ids)); } } task->add_event_info("hold snapshot finish"); @@ -458,54 +411,6 @@ int ObDDLUtil::get_data_information(const uint64_t task_id, return ret; } -int ObDDLUtil::generate_mview_ddl_schema_hint_str(const uint64_t mview_table_id, - share::schema::ObSchemaGetterGuard &schema_guard, - const ObIArray &based_schema_object_infos, - ObSqlString &sql_string) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator("ObDDLTmp"); - ObString database_name; - ObString table_name; - for (int64_t i = 0; OB_SUCC(ret) && i < based_schema_object_infos.count(); ++i) { - const ObBasedSchemaObjectInfo &based_info = based_schema_object_infos.at(i); - const ObTableSchema *table_schema = nullptr; - const ObDatabaseSchema *database_schema = nullptr; - database_name.reset(); - table_name.reset(); - allocator.reuse(); - if (OB_FAIL(schema_guard.get_table_schema( based_info.schema_id_, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(based_info)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table not exist", KR(ret), K(based_info)); - } else if (OB_FAIL(schema_guard.get_database_schema( table_schema->get_database_id(), - database_schema))) { - LOG_WARN("fail to get database schema", KR(ret), - K(table_schema->get_database_id())); - } else if (OB_ISNULL(database_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, database schema must not be nullptr", KR(ret)); - } else if (OB_FAIL(sql::ObSQLUtils::generate_new_name_with_escape_character( - allocator, database_schema->get_database_name_str(), database_name))) { - LOG_WARN("fail to generate new name with escape character", KR(ret), - K(database_schema->get_database_name_str())); - } else if (OB_FAIL(sql::ObSQLUtils::generate_new_name_with_escape_character( - allocator, table_schema->get_table_name_str(), table_name))) { - LOG_WARN("fail to generate new name with escape character", KR(ret), - K(table_schema->get_table_name_str())); - } else { - if (OB_FAIL(sql_string.append_fmt("ob_ddl_schema_version(`%.*s`.`%.*s`, %ld) ", - static_cast(database_name.length()), database_name.ptr(), - static_cast(table_name.length()), table_name.ptr(), - based_info.schema_version_))) { - LOG_WARN("append sql string failed", KR(ret)); - } - } - } - return ret; -} - int ObDDLUtil::generate_build_replica_sql(const int64_t data_table_id, const int64_t dest_table_id, const int64_t schema_version, @@ -550,7 +455,7 @@ int ObDDLUtil::generate_build_replica_sql(const int64_t data_table_id, ObArray select_column_ids; ObArray order_column_ids; bool is_shadow_column = false; - const int64_t real_parallelism = ObDDLUtil::get_real_parallelism(parallelism, false/*is mv refresh*/); + const int64_t real_parallelism = ObDDLUtil::get_real_parallelism(parallelism); const bool is_rowkey_doc_aux_table = dest_table_schema->is_rowkey_doc_id(); uint64_t doc_id_col_id = OB_INVALID_ID; uint64_t ft_id_col_id = OB_INVALID_ID; @@ -799,136 +704,12 @@ int ObDDLUtil::generate_build_replica_sql(const int64_t data_table_id, return ret; } -int ObDDLUtil::generate_build_mview_replica_sql(const int64_t mview_table_id, - const int64_t container_table_id, - ObSchemaGetterGuard &schema_guard, - const int64_t snapshot_version, - const uint64_t mview_target_data_sync_scn, - const int64_t execution_id, - const int64_t task_id, - const int64_t parallelism, - const bool use_schema_version_hint_for_src_table, - const ObIArray &based_schema_object_infos, - const ObString &mview_select_sql, - ObSqlString &sql_string) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_ID == mview_table_id || - OB_INVALID_ID == container_table_id || snapshot_version <= 0 || - execution_id < 0 || task_id <= 0 || based_schema_object_infos.empty())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", KR(ret), K(mview_table_id), K(container_table_id), - K(snapshot_version), K(execution_id), K(task_id), K(based_schema_object_infos)); - } else { - const ObTableSchema *mview_table_schema = nullptr; - const ObTableSchema *container_table_schema = nullptr; - const ObDatabaseSchema *database_schema = nullptr; - if (OB_FAIL(schema_guard.get_table_schema( mview_table_id, mview_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(mview_table_id)); - } else if (OB_ISNULL(mview_table_schema)) { - ret = OB_ERR_MVIEW_NOT_EXIST; - LOG_WARN("fail to get mview table schema", KR(ret), K(mview_table_id)); - } else if (OB_FAIL(schema_guard.get_table_schema( container_table_id, - container_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(container_table_id)); - } else if (OB_ISNULL(container_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("fail to get table schema", KR(ret), K(container_table_id)); - } else if (OB_FAIL(schema_guard.get_database_schema( mview_table_schema->get_database_id(), database_schema))) { - LOG_WARN("fail to get database schema", KR(ret), - K(mview_table_schema->get_database_id())); - } else if (OB_ISNULL(database_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, database schema must not be nullptr", KR(ret)); - } else { - ObArenaAllocator allocator("ObDDLTmp"); - ObString database_name; - ObString container_table_name; - ObSqlString src_table_schema_version_hint; - ObSqlString rowkey_column_sql_string; - if (OB_FAIL(sql::ObSQLUtils::generate_new_name_with_escape_character( - allocator, database_schema->get_database_name_str(), database_name))) { - LOG_WARN("fail to generate new name with escape character", KR(ret), - K(database_schema->get_database_name_str())); - } else if (OB_FAIL(sql::ObSQLUtils::generate_new_name_with_escape_character( - allocator, container_table_schema->get_table_name_str(), container_table_name))) { - LOG_WARN("fail to generate new name with escape character", KR(ret), - K(container_table_schema->get_table_name_str())); - } else if (use_schema_version_hint_for_src_table) { - int64_t based_schema_version = OB_INVALID_VERSION; - for (int64_t i = 0; OB_SUCC(ret) && i < based_schema_object_infos.count(); ++i) { - const ObBasedSchemaObjectInfo &based_info = based_schema_object_infos.at(i); - const ObTableSchema *based_table_schema = nullptr; - if (OB_FAIL(schema_guard.get_table_schema( based_info.schema_id_, - based_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(based_info)); - } else if (OB_ISNULL(based_table_schema)) { - ret = OB_OLD_SCHEMA_VERSION; - LOG_WARN("based table is not exist", KR(ret), K(based_info)); - } else if (OB_UNLIKELY(based_table_schema->get_schema_version() != - based_info.schema_version_)) { - ret = OB_OLD_SCHEMA_VERSION; - LOG_WARN("based table schema version is changed", KR(ret), K(based_info), - KPC(based_table_schema)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(generate_mview_ddl_schema_hint_str(mview_table_id, schema_guard, based_schema_object_infos, - src_table_schema_version_hint))) { - LOG_WARN("failed to generated mview ddl schema hint", KR(ret)); - } - } - const bool nested_consistent_refresh = mview_target_data_sync_scn == OB_INVALID_SCN_VAL ? false : true; - const int64_t real_parallelism = ObDDLUtil::get_real_parallelism(parallelism, true/*is mv refresh*/); - if (OB_FAIL(ret)) { - } else if (!nested_consistent_refresh) { - const ObString &select_sql_string = mview_table_schema->get_view_schema().get_view_definition_str(); - if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ append monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) use_px */ INTO `%.*s`.`%.*s`" - " SELECT /*+ %.*s */ * from (%.*s) as of snapshot %ld %.*s;", - real_parallelism, execution_id, task_id, - static_cast(database_name.length()), database_name.ptr(), - static_cast(container_table_name.length()), container_table_name.ptr(), - static_cast(src_table_schema_version_hint.length()), src_table_schema_version_hint.ptr(), - static_cast(select_sql_string.length()), select_sql_string.ptr(), - snapshot_version, - static_cast(rowkey_column_sql_string.length()), rowkey_column_sql_string.ptr()))) { - LOG_WARN("fail to assign sql string", KR(ret)); - } - } else if (nested_consistent_refresh) { - std::string select_sql(mview_select_sql.ptr()); - std::string real_sql; - if (mview_select_sql.empty()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("nested sync refresh with empty sql string", K(mview_select_sql), K(mview_table_id)); - } else if (OB_FAIL(ObMViewRefreshHelper::replace_all_snapshot_zero( - select_sql, snapshot_version, real_sql))) { - LOG_WARN("fail to replace snapshot", K(ret)); - } else { - if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ append monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) " - " opt_param('ddl_task_id', %ld) use_px */ INTO `%.*s`.`%.*s`" - " SELECT /*+ %.*s */ * from (%.*s);", - real_parallelism, execution_id, task_id, - static_cast(database_name.length()), database_name.ptr(), - static_cast(container_table_name.length()), container_table_name.ptr(), - static_cast(src_table_schema_version_hint.length()), src_table_schema_version_hint.ptr(), - static_cast(real_sql.length()), real_sql.c_str()))) { - LOG_WARN("fail to assign sql string", KR(ret)); - } - } - } - LOG_INFO("execute sql", K(sql_string)); - } - } - return ret; -} - int ObDDLUtil::obtain_snapshot( const share::ObDDLTaskStatus next_task_status, const uint64_t table_id, const uint64_t target_table_id, int64_t &snapshot_version, - rootserver::ObDDLTask* task, - const common::ObIArray *extra_mv_tablet_ids) + rootserver::ObDDLTask* task) { int ret = OB_SUCCESS; rootserver::ObDDLWaitTransEndCtx* wait_trans_ctx = nullptr; @@ -979,7 +760,7 @@ int ObDDLUtil::obtain_snapshot( } else if (persisted_snapshot > 0) { // found a persisted snapshot, do not hold it again. FLOG_INFO("found a persisted snapshot in inner table", "task_id", task->get_task_id(), K(persisted_snapshot), K(new_fetched_snapshot)); - } else if (OB_FAIL(hold_snapshot(trans, task, table_id, target_table_id, GCTX.root_service_, new_fetched_snapshot, extra_mv_tablet_ids))) { + } else if (OB_FAIL(hold_snapshot(trans, task, table_id, target_table_id, GCTX.root_service_, new_fetched_snapshot))) { if (OB_SNAPSHOT_DISCARDED == ret) { wait_trans_ctx->reset(); } else { @@ -1090,7 +871,6 @@ int ObDDLUtil::calc_snapshot_with_gts( int ret = OB_SUCCESS; snapshot = 0; SCN curr_ts; - bool is_external_consistent = false; const int64_t timeout_us = ObDDLUtil::get_default_ddl_rpc_timeout(); ObFreezeInfoProxy freeze_info_proxy{}; ObFreezeInfo frozen_status; @@ -1103,13 +883,9 @@ int ObDDLUtil::calc_snapshot_with_gts( } else { { MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard); - // ignore return, MTL is only used in get_ts_sync, which will handle switch failure. - // for performance, everywhere calls get_ts_sync should ensure using correct tenant ctx tenant_guard.switch_to(); - if (OB_FAIL(OB_TS_MGR.get_ts_sync(timeout_us, - curr_ts, - is_external_consistent))) { - LOG_WARN("fail to get gts sync", K(ret), K(timeout_us), K(curr_ts), K(is_external_consistent)); + if (OB_FAIL(OB_TS_MGR.get_gts_sync(timeout_us, curr_ts))) { + LOG_WARN("fail to get gts sync", K(ret), K(timeout_us), K(curr_ts)); } } if (OB_SUCC(ret)) { @@ -1257,19 +1033,12 @@ int ObDDLUtil::check_and_cancel_single_replica_dag( { int ret = OB_SUCCESS; all_dag_exit = false; - const bool force_renew = true; - bool is_cache_hit = false; - const int64_t expire_renew_time = force_renew ? INT64_MAX : 0; - share::ObLocationService *location_service = GCTX.location_service_; if (OB_ISNULL(task)) { ret = OB_BAD_NULL_ERROR; LOG_WARN("invalid argument", K(ret)); } else if (OB_UNLIKELY(!task->is_inited())) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_ISNULL(location_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), KP(location_service)); } else if (OB_UNLIKELY(!check_dag_exit_tablets_map.created())) { const int64_t CHECK_DAG_EXIT_BUCKET_NUM = 64; common::ObArray src_tablet_ids; @@ -1292,31 +1061,17 @@ int ObDDLUtil::check_and_cancel_single_replica_dag( } if (OB_SUCC(ret)) { int saved_ret = OB_SUCCESS; - ObAddr unused_leader_addr; - const int64_t timeout_us = ObDDLUtil::get_default_ddl_rpc_timeout(); common::hash::ObHashMap ::const_iterator iter = check_dag_exit_tablets_map.begin(); ObArray dag_not_exist_tablets; for (; OB_SUCC(ret) && iter != check_dag_exit_tablets_map.end(); iter++) { - ObLSID src_ls_id; - ObLSID dst_ls_id; const common::ObTabletID &src_tablet_id = iter->first; const common::ObTabletID &dst_tablet_id = iter->second; - int64_t paxos_member_count = 0; - common::ObArray paxos_server_list; - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, src_tablet_id, timeout_us, src_ls_id, unused_leader_addr))) { - LOG_WARN("get src tablet leader addr failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, dst_tablet_id, timeout_us, dst_ls_id, unused_leader_addr))) { - LOG_WARN("get dst tablet leader addr failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_paxos_member_list(dst_tablet_id, paxos_server_list, paxos_member_count))) { - LOG_WARN("get tablet paxos member list failed", K(ret)); - } else { + { bool is_tablet_dag_exist = false; obcall::ObDDLBuildSingleReplicaRequestArg arg; - arg.ls_id_ = src_ls_id; - arg.dest_ls_id_ = dst_ls_id; @@ -1333,24 +1088,22 @@ int ObDDLUtil::check_and_cancel_single_replica_dag( arg.execution_id_ = 1; // to ensure arg valid only. arg.data_format_version_ = data_format_version; // to ensure arg valid only. arg.tablet_task_id_ = 1; // to ensure arg valid only. - for (int64_t j = 0; OB_SUCC(ret) && j < paxos_server_list.count(); j++) { - int tmp_ret = OB_SUCCESS; - obcall::Bool is_replica_dag_exist(true); - if (is_complement_data_dag && OB_TMP_FAIL(ex_rpc::sync_call(ObDDLUtil::get_default_ddl_rpc_timeout(), [&]() -> int { bool b = is_replica_dag_exist; int r = GCTX.ob_service_->check_and_cancel_ddl_complement_data_dag(arg, b); is_replica_dag_exist = b; return r; }))) { - // consider as dag does exist in this server. - saved_ret = OB_SUCC(saved_ret) ? tmp_ret : saved_ret; - is_tablet_dag_exist = true; - LOG_WARN("check and cancel ddl complement dag failed", K(ret), K(tmp_ret), K(arg)); - } else if (!is_complement_data_dag && OB_TMP_FAIL(ex_rpc::sync_call(ObDDLUtil::get_default_ddl_rpc_timeout(), [&]() -> int { bool b = is_replica_dag_exist; int r = GCTX.ob_service_->check_and_cancel_delete_lob_meta_row_dag(arg, b); is_replica_dag_exist = b; return r; }))) { - // consider as dag does exist in this server. - saved_ret = OB_SUCC(saved_ret) ? tmp_ret : saved_ret; - is_tablet_dag_exist = true; - LOG_WARN("check and cancel ddl complement dag failed", K(ret), K(tmp_ret), K(arg)); - } else if (is_replica_dag_exist) { - is_tablet_dag_exist = true; - if (REACH_COUNT_INTERVAL(1000L)) { - LOG_INFO("wait dag exist", "addr", paxos_server_list.at(j), K(arg)); - } + int tmp_ret = OB_SUCCESS; + bool is_replica_dag_exist = true; + if (is_complement_data_dag && OB_TMP_FAIL( + GCTX.ob_service_->check_and_cancel_ddl_complement_data_dag(arg, is_replica_dag_exist))) { + saved_ret = OB_SUCC(saved_ret) ? tmp_ret : saved_ret; + is_tablet_dag_exist = true; + LOG_WARN("check and cancel ddl complement dag failed", K(tmp_ret), K(arg)); + } else if (!is_complement_data_dag && OB_TMP_FAIL( + GCTX.ob_service_->check_and_cancel_delete_lob_meta_row_dag(arg, is_replica_dag_exist))) { + saved_ret = OB_SUCC(saved_ret) ? tmp_ret : saved_ret; + is_tablet_dag_exist = true; + LOG_WARN("check and cancel delete lob meta row dag failed", K(tmp_ret), K(arg)); + } else if (is_replica_dag_exist) { + is_tablet_dag_exist = true; + if (REACH_COUNT_INTERVAL(1000L)) { + LOG_INFO("wait local dag exit", K(arg)); } } if (OB_SUCC(ret) && !is_tablet_dag_exist) { @@ -1371,7 +1124,7 @@ int ObDDLUtil::check_and_cancel_single_replica_dag( } if (OB_SUCC(ret)) { all_dag_exit = check_dag_exit_tablets_map.empty() ? true : false; - task->set_delay_schedule_time(3000L * 1000L); // 3s, to avoid sending too many rpcs to the same replica frequently if retry. + task->set_delay_schedule_time(3000L * 1000L); } else if (OB_TABLE_NOT_EXIST == ret || OB_TENANT_NOT_EXIST == ret || (++check_dag_exit_retry_cnt >= 10 /*MAX RETRY COUNT IF FAILED*/)) { @@ -1610,78 +1363,53 @@ int ObDDLUtil::check_table_empty( return ret; } -int64_t ObDDLUtil::get_real_parallelism(const int64_t parallelism, const bool is_mv_refresh) +int64_t ObDDLUtil::get_real_parallelism(const int64_t parallelism) { - int64_t real_parallelism = 0L; - if (is_mv_refresh) { - real_parallelism = std::max(static_cast(2), parallelism); - } else { - real_parallelism = std::min(oceanbase::ObMacroDataSeq::MAX_PARALLEL_IDX + 1, std::max(static_cast(1), parallelism)); - } - return real_parallelism; + return std::min(oceanbase::ObMacroDataSeq::MAX_PARALLEL_IDX + 1, + std::max(static_cast(1), parallelism)); } int ObCheckTabletDataComplementOp::do_check_tablets_merge_status(const int64_t snapshot_version, const ObIArray &tablet_ids, - const ObLSID &ls_id, - hash::ObHashMap> &ip_tablets_map, hash::ObHashMap &tablets_commited_map, int64_t &tablet_build_succ_count) { int ret = OB_SUCCESS; - ip_tablets_map.reuse(); tablets_commited_map.reuse(); tablet_build_succ_count = 0; - if (OB_UNLIKELY(tablet_ids.count() < 0 || false || OB_INVALID_TIMESTAMP == snapshot_version)) { + if (OB_UNLIKELY(tablet_ids.count() <= 0 || OB_INVALID_TIMESTAMP == snapshot_version)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_ids.count()), K(snapshot_version)); } else { obcall::ObDDLCheckTabletMergeStatusArg arg; - arg.ls_id_ = ls_id; arg.snapshot_version_ = snapshot_version; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - const ObTabletID &tablet_id = tablet_ids.at(i); - if (OB_FAIL(construct_tablet_ip_map(tablet_id, ip_tablets_map))) { - LOG_WARN("fail to get tablet ip addr", K(ret), K(tablet_id)); - } - } - // Direct calls to local service (seekdb has no remote servers). - for (hash::ObHashMap>::const_iterator ip_iter = ip_tablets_map.begin(); - OB_SUCC(ret) && ip_iter != ip_tablets_map.end(); ++ip_iter) { - const ObAddr & dest_ip = ip_iter->first; - UNUSED(dest_ip); - const ObArray &tablet_array = ip_iter->second; - if (OB_FAIL(arg.tablet_ids_.assign(tablet_array))) { - LOG_WARN("fail to get tablet ip addr", K(ret), K(tablet_array)); - } else { - obcall::ObDDLCheckTabletMergeStatusResult cur_result; - int return_ret = GCTX.ob_service_->check_ddl_tablet_merge_status(arg, cur_result); - if (OB_SUCCESS == return_ret) { - common::ObSArray tablet_rsp_array; - common::ObArray tablet_req_array; - if (FALSE_IT(tablet_rsp_array = cur_result.merge_status_)) { - } else if (FALSE_IT(tablet_req_array = tablet_array)) { - } else if (tablet_req_array.count() != tablet_rsp_array.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet req count is not equal to tablet rsp count", K(ret), K(tablet_req_array), K(tablet_rsp_array)); - } else { - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_rsp_array.count(); ++idx) { - const common::ObTabletID &tablet_id = tablet_req_array.at(idx); - const bool tablet_status = tablet_rsp_array.at(idx); - if (OB_FAIL(update_replica_merge_status(tablet_id, tablet_status, tablets_commited_map))) { - LOG_WARN("fail to update replica merge status", K(ret), K(tablet_id), K(dest_ip)); - } else { - LOG_INFO("succ to update replica merge status", K(dest_ip), K(tablet_id), K(tablet_status)); - } + if (OB_FAIL(arg.tablet_ids_.assign(tablet_ids))) { + LOG_WARN("fail to assign tablet ids", K(ret), K(tablet_ids)); + } else { + obcall::ObDDLCheckTabletMergeStatusResult cur_result; + int return_ret = GCTX.ob_service_->check_ddl_tablet_merge_status(arg, cur_result); + if (OB_SUCCESS == return_ret) { + const common::ObSArray &tablet_rsp_array = cur_result.merge_status_; + if (tablet_ids.count() != tablet_rsp_array.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet req count is not equal to tablet rsp count", K(ret), K(tablet_ids), K(tablet_rsp_array)); + } else { + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_rsp_array.count(); ++idx) { + const common::ObTabletID &tablet_id = tablet_ids.at(idx); + const bool tablet_status = tablet_rsp_array.at(idx); + if (OB_FAIL(update_replica_merge_status(tablet_id, tablet_status, tablets_commited_map))) { + LOG_WARN("fail to update replica merge status", K(ret), K(tablet_id)); + } else { + LOG_INFO("succ to update replica merge status", K(tablet_id), K(tablet_status)); } } - } else { - LOG_WARN("check ddl tablet merge status failed.", K(return_ret)); } + } else { + LOG_WARN("check ddl tablet merge status failed.", K(return_ret)); } } // 3. check any commit tablet diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index d7185a76f..a82ef49c0 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -27,8 +27,6 @@ #include "storage/ob_partition_pre_split.h" #include "storage/ob_tablet_autoinc_seq_rpc_handler.h" #include "share/ob_ex_rpc.h" -#include "storage/mview/ob_mview_mds.h" -#include "storage/mview/ob_mview_refresh_helper.h" using namespace oceanbase::lib; using namespace oceanbase::common; @@ -49,21 +47,16 @@ ObDDLRedefinitionSSTableBuildTask::ObDDLRedefinitionSSTableBuildTask( const common::ObCurTraceId::TraceId &trace_id, const int64_t parallelism, const bool use_heap_table_ddl_plan, - const bool is_mview_complete_refresh, - const int64_t mview_table_id, ObRootService *root_service, const common::ObAddr &inner_sql_exec_addr, const int64_t data_format_version, - const bool is_retryable_ddl, - const uint64_t mview_target_data_sync_scn, - const ObString &mview_select_sql) + const bool is_retryable_ddl) : is_inited_(false), task_id_(task_id), data_table_id_(data_table_id), dest_table_id_(dest_table_id), schema_version_(schema_version), snapshot_version_(snapshot_version), execution_id_(execution_id), sql_mode_(sql_mode), trace_id_(trace_id), parallelism_(parallelism), use_heap_table_ddl_plan_(use_heap_table_ddl_plan), - is_mview_complete_refresh_(is_mview_complete_refresh), is_retryable_ddl_(is_retryable_ddl), mview_table_id_(mview_table_id), - root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr), data_format_version_(0), - mview_target_data_sync_scn_(mview_target_data_sync_scn), mview_select_sql_(mview_select_sql) + is_retryable_ddl_(is_retryable_ddl), root_service_(root_service), + inner_sql_exec_addr_(inner_sql_exec_addr), data_format_version_(data_format_version) { set_retry_times(0); // do not retry } @@ -72,8 +65,7 @@ int ObDDLRedefinitionSSTableBuildTask::init( const ObTableSchema &orig_table_schema, const ObTableSchema &hidden_table_schema, const AlterTableSchema &alter_table_schema, - const ObTimeZoneInfoWrap &tz_info_wrap, - const ObIArray &based_schema_object_infos) + const ObTimeZoneInfoWrap &tz_info_wrap) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -85,8 +77,6 @@ int ObDDLRedefinitionSSTableBuildTask::init( LOG_WARN("fail to copy time zone info wrap", K(ret), K(tz_info_wrap)); } else if (OB_FAIL(col_name_map_.init(orig_table_schema, hidden_table_schema, alter_table_schema))) { LOG_WARN("failed to init column name map", K(ret)); - } else if (OB_FAIL(based_schema_object_infos_.assign(based_schema_object_infos))) { - LOG_WARN("fail to assign based schema object infos", K(ret), K(based_schema_object_infos)); } else { is_inited_ = true; } @@ -106,7 +96,6 @@ int ObDDLRedefinitionSSTableBuildTask::process() ObSchemaGetterGuard schema_guard; ObDDLTaskKey task_key(dest_table_id_, schema_version_); ObDDLTaskInfo info; - bool need_exec_new_inner_sql = true; const ObTableSchema *data_table_schema = nullptr; if (OB_UNLIKELY(!is_inited_ || OB_ISNULL(GCTX.sql_proxy_))) { @@ -125,60 +114,32 @@ int ObDDLRedefinitionSSTableBuildTask::process() ret = OB_TABLE_NOT_EXIST; LOG_WARN("error unexpected, table schema must not be nullptr", K(ret), K(data_table_id_)); } else { - if (is_mview_complete_refresh_) { - LOG_INFO("print select sql", K(mview_select_sql_)); - if (mview_target_data_sync_scn_ != OB_INVALID_SCN_VAL && - OB_FAIL(ObMViewRefreshHelper::collect_deps_and_check_satisfy(mview_table_id_, mview_target_data_sync_scn_, - snapshot_version_, *GCTX.sql_proxy_, schema_guard))) { - LOG_WARN("fail to check satisfied", K(ret), - K(mview_table_id_), K(mview_target_data_sync_scn_), K(snapshot_version_)); - } else if (OB_FAIL(ObDDLUtil::generate_build_mview_replica_sql(mview_table_id_, - data_table_id_, - schema_guard, - snapshot_version_, - mview_target_data_sync_scn_, - execution_id_, - task_id_, - parallelism_, - true/*use_schema_version_hint_for_src_table*/, - based_schema_object_infos_, - mview_select_sql_, - sql_string))) { - LOG_WARN("fail to generate build mview replica sql", K(ret)); - } - } else { - ObString partition_names; - if (OB_FAIL(ObDDLUtil::generate_build_replica_sql(data_table_id_, - dest_table_id_, - data_table_schema->get_schema_version(), - snapshot_version_, - execution_id_, - task_id_, - parallelism_, - use_heap_table_ddl_plan_, - true/*use_schema_version_hint_for_src_table*/, - &col_name_map_, - partition_names, - sql_string))) { - LOG_WARN("fail to generate build replica sql", K(ret)); - } + ObString partition_names; + if (OB_FAIL(ObDDLUtil::generate_build_replica_sql(data_table_id_, + dest_table_id_, + data_table_schema->get_schema_version(), + snapshot_version_, + execution_id_, + task_id_, + parallelism_, + use_heap_table_ddl_plan_, + true/*use_schema_version_hint_for_src_table*/, + &col_name_map_, + partition_names, + sql_string))) { + LOG_WARN("fail to generate build replica sql", K(ret)); } if (OB_SUCC(ret)) { ObTimeoutCtx timeout_ctx; common::ObCommonSqlProxy *user_sql_proxy = nullptr; int64_t affected_rows = 0; - if (is_mview_complete_refresh_) { - sql_mode_ = SMO_STRICT_ALL_TABLES; - } ObSessionParam session_param; session_param.sql_mode_ = reinterpret_cast(&sql_mode_); session_param.tz_info_wrap_ = &tz_info_wrap_; - session_param.ddl_info_.set_is_ddl(!is_mview_complete_refresh_); + session_param.ddl_info_.set_is_ddl(true); session_param.ddl_info_.set_source_table_hidden(false); session_param.ddl_info_.set_dest_table_hidden(true); session_param.ddl_info_.set_heap_table_ddl(use_heap_table_ddl_plan_); - session_param.ddl_info_.set_mview_complete_refresh(is_mview_complete_refresh_); - session_param.ddl_info_.set_refreshing_mview(is_mview_complete_refresh_); session_param.ddl_info_.set_retryable_ddl(is_retryable_ddl_); session_param.use_external_session_ = true; // means session id dispatched by session mgr @@ -198,46 +159,14 @@ int ObDDLRedefinitionSSTableBuildTask::process() LOG_WARN("set timeout failed", K(ret)); } else { ObCompatibilityMode compat_mode = ObCompatibilityMode::MYSQL_MODE; - ObMySQLTransaction trans; - if (is_mview_complete_refresh_) { - ObMViewOpArg arg; - arg.table_id_ = mview_table_id_; - arg.parallel_ = parallelism_; - arg.session_id_ = 100; // fixed session id - arg.start_ts_ = ObTimeUtil::current_time(); - arg.mview_op_type_ = MVIEW_OP_TYPE::COMPLETE_REFRESH; - arg.read_snapshot_ = snapshot_version_; - if (mview_target_data_sync_scn_ != OB_INVALID_SCN_VAL && - OB_FAIL(arg.target_data_sync_scn_.convert_for_tx(mview_target_data_sync_scn_))) { - LOG_WARN("fail to conver to scn", K(ret), K(mview_target_data_sync_scn_), K(arg)); - } else if (OB_FAIL(trans.start(user_sql_proxy))) { - LOG_WARN("fail to start trans", K(ret)); - } else if (OB_FAIL(ObMViewMdsOpHelper::register_mview_mds(arg, trans))) { - LOG_WARN("register mview mds failed", K(ret), K(arg)); - } - DEBUG_SYNC(BEFORE_MV_LOAD_DATA); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(DDL_SIM(task_id_, REDEF_SSTABLE_BULD_TASK_PROCESS_FAILED))) { - LOG_WARN("ddl sim failure", K(ret), K(task_id_)); - } else if (OB_FAIL(user_sql_proxy->write(sql_string.ptr(), affected_rows, - compat_mode, &session_param, sql_exec_addr))) { - LOG_WARN("fail to execute build replica sql", K(ret)); - } - tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_ERROR("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } else { - if (OB_FAIL(DDL_SIM(task_id_, REDEF_SSTABLE_BULD_TASK_PROCESS_FAILED))) { - LOG_WARN("ddl sim failure", K(ret), K(task_id_)); - } else if (OB_FAIL(user_sql_proxy->write(sql_string.ptr(), affected_rows, - compat_mode, &session_param, sql_exec_addr))) { - LOG_WARN("fail to execute build replica sql", K(ret)); - } else if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(dest_table_id_, execution_id_, task_id_))) { - LOG_WARN("fail to check sstable checksum_report_finish", - K(ret), K(dest_table_id_), K(execution_id_), K(task_id_)); - } + if (OB_FAIL(DDL_SIM(task_id_, REDEF_SSTABLE_BULD_TASK_PROCESS_FAILED))) { + LOG_WARN("ddl sim failure", K(ret), K(task_id_)); + } else if (OB_FAIL(user_sql_proxy->write(sql_string.ptr(), affected_rows, + compat_mode, &session_param, sql_exec_addr))) { + LOG_WARN("fail to execute build replica sql", K(ret)); + } else if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(dest_table_id_, execution_id_, task_id_))) { + LOG_WARN("fail to check sstable checksum_report_finish", + K(ret), K(dest_table_id_), K(execution_id_), K(task_id_)); } } } @@ -283,20 +212,14 @@ ObAsyncTask *ObDDLRedefinitionSSTableBuildTask::deep_copy(char *buf, const int64 trace_id_, parallelism_, use_heap_table_ddl_plan_, - is_mview_complete_refresh_, - mview_table_id_, root_service_, inner_sql_exec_addr_, data_format_version_, - is_retryable_ddl_, - mview_target_data_sync_scn_, - mview_select_sql_); + is_retryable_ddl_); if (OB_FAIL(new_task->tz_info_wrap_.deep_copy(tz_info_wrap_))) { LOG_WARN("failed to copy tz info wrap", K(ret)); } else if (OB_FAIL(new_task->col_name_map_.assign(col_name_map_))) { LOG_WARN("failed to assign column name map", K(ret)); - } else if (OB_FAIL(new_task->based_schema_object_infos_.assign(based_schema_object_infos_))) { - LOG_WARN("failed to assign based schema object infos", K(ret)); } if (OB_FAIL(ret)) { LOG_WARN("fail to init new task", K(ret)); @@ -368,76 +291,10 @@ int ObDDLRedefinitionTask::check_table_empty(const ObDDLTaskStatus next_task_sta return ret; } -int ObDDLRedefinitionTask::add_table_tablets_for_snapshot_( - const uint64_t table_id, ObSchemaGetterGuard &schema_guard, - common::ObIArray &tablet_ids) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = NULL; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( table_id, table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(object_id_)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table not exist", K(ret), K(table_id)); - } else if (OB_FAIL(ObDDLUtil::get_tablets(table_id, tablet_ids))) { - LOG_WARN("failed to get data table snapshot", K(ret)); - } else if (table_schema->get_aux_lob_meta_tid() != OB_INVALID_ID && - OB_FAIL(ObDDLUtil::get_tablets(table_schema->get_aux_lob_meta_tid(), - tablet_ids))) { - LOG_WARN("failed to get data lob meta table snapshot", K(ret)); - } else if (table_schema->get_aux_lob_piece_tid() != OB_INVALID_ID && - OB_FAIL(ObDDLUtil::get_tablets(table_schema->get_aux_lob_piece_tid(), - tablet_ids))) { - LOG_WARN("failed to get data lob piece table snapshot", K(ret)); - } - - return ret; -} - -int ObDDLRedefinitionTask::prepare_tablets_for_major_refresh_mv_(common::ObIArray &tablet_ids) -{ - int ret = OB_SUCCESS; - ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance(); - ObSchemaGetterGuard schema_guard; - SCN snapshot_scn; - tablet_ids.reset(); - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); - } else if (OB_FAIL(schema_service.get_tenant_schema_guard(schema_guard))) { - LOG_WARN("get tenant schema guard failed", K(ret)); - } else if (OB_FAIL(add_table_tablets_for_snapshot_(target_object_id_, schema_guard, tablet_ids))) { - // target_table and src_table will be exchanged after loading data, which means target_table - // will become the container table of MV at last, so we should hold its snapshot. - LOG_WARN("failed to add table tablets for snapshot", K(ret), K(object_id_), - K(target_object_id_)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < alter_table_arg_.based_schema_object_infos_.count(); - ++i) { - const ObBasedSchemaObjectInfo &base_info = alter_table_arg_.based_schema_object_infos_.at(i); - if (ObSchemaType::TABLE_SCHEMA == base_info.schema_type_ && - OB_FAIL(add_table_tablets_for_snapshot_(base_info.schema_id_, schema_guard, tablet_ids))) { - LOG_WARN("failed to get base table snapshot", K(ret), K(base_info.schema_id_)); - } - } - } - - LOG_INFO("prepare tablets for major refresh mv", K(ret), K(tablet_ids)); - - return ret; -} - - // to hold snapshot, containing data in old table with new schema version. int ObDDLRedefinitionTask::obtain_snapshot(const ObDDLTaskStatus next_task_status) { int ret = OB_SUCCESS; - ObSEArray mv_tablet_ids; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); @@ -446,12 +303,8 @@ int ObDDLRedefinitionTask::obtain_snapshot(const ObDDLTaskStatus next_task_statu if (OB_FAIL(switch_status(next_task_status, true, ret))) { LOG_WARN("fail to switch task status", K(ret)); } - } else if (alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_ && - alter_table_arg_.alter_table_schema_.mv_major_refresh() && - OB_FAIL(prepare_tablets_for_major_refresh_mv_(mv_tablet_ids))) { - LOG_WARN("fail to prepare major refresh mv tablets", K(ret)); } else if (OB_FAIL(ObDDLUtil::obtain_snapshot(next_task_status, object_id_, target_object_id_, - snapshot_version_, this, &mv_tablet_ids))) { + snapshot_version_, this))) { LOG_WARN("fail to obtain_snapshot", K(ret), K(snapshot_version_)); } return ret; @@ -1492,8 +1345,7 @@ bool ObDDLRedefinitionTask::check_need_sync_stats() { // bugfix: // shouldn't sync stats if the ddl task is from load data's direct_load return ObDDLType::DDL_DIRECT_LOAD != task_type_ - && ObDDLType::DDL_DIRECT_LOAD_INSERT != task_type_ - && ObDDLType::DDL_MVIEW_COMPLETE_REFRESH != task_type_; + && ObDDLType::DDL_DIRECT_LOAD_INSERT != task_type_; } int ObDDLRedefinitionTask::sync_stats_info() @@ -2503,7 +2355,7 @@ int ObDDLRedefinitionTask::get_child_task_ids(char *buf, int64_t len) } ObSyncTabletAutoincSeqCtx::ObSyncTabletAutoincSeqCtx() - : is_inited_(false), is_synced_(false), need_renew_location_(false), orig_src_tablet_ids_(), + : is_inited_(false), is_synced_(false), orig_src_tablet_ids_(), src_tablet_ids_(), dest_tablet_ids_(), autoinc_params_() {} @@ -2551,55 +2403,6 @@ int ObSyncTabletAutoincSeqCtx::sync() is_synced_ = true; } } - if (share::ObIDDLTask::in_ddl_retry_white_list(ret)) { - need_renew_location_ = true; - } - return ret; -} - -int ObSyncTabletAutoincSeqCtx::build_ls_to_tablet_map( - share::ObLocationService *location_service, - const ObIArray &autoinc_params, - const int64_t timeout, - const bool force_renew, - const bool by_src_tablet, - ObHashMap> &map) -{ - int ret = OB_SUCCESS; - map.reuse(); - bool is_cache_hit = false; - const int64_t expire_renew_time = force_renew ? INT64_MAX : 0; - ObTimeoutCtx timeout_ctx; - if (nullptr == location_service || autoinc_params.count() <= 0 || timeout <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), KP(location_service), K(autoinc_params), K(timeout)); - } else if (OB_FAIL(timeout_ctx.set_trx_timeout_us(timeout))) { - LOG_WARN("set trx timeout failed", K(ret)); - } else if (OB_FAIL(timeout_ctx.set_timeout(timeout))) { - LOG_WARN("set timeout failed", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < autoinc_params.count(); i++) { - ObLSID ls_id; - const ObMigrateTabletAutoincSeqParam &autoinc_param = autoinc_params.at(i); - const ObTabletID &tablet_id = by_src_tablet ? autoinc_param.src_tablet_id_ : autoinc_param.dest_tablet_id_; - ObSEArray tmp_list; - if (OB_FAIL(location_service->get(tablet_id, expire_renew_time, is_cache_hit, ls_id))) { - LOG_WARN("fail to get log stream id", K(ret), K(tablet_id)); - } else if (OB_FAIL(map.get_refactored(ls_id, tmp_list))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get from map", K(ret), K(ls_id)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tmp_list.push_back(autoinc_param))) { - LOG_WARN("failed to push tablet id", K(ret), K(tablet_id)); - } else if (OB_FAIL(map.set_refactored(ls_id, tmp_list, 1/*overwrite*/))) { - LOG_WARN("failed to set map", K(ret), K(ls_id), K(tmp_list)); - } - } - } return ret; } @@ -2607,20 +2410,13 @@ int ObSyncTabletAutoincSeqCtx::call_and_process_all_tablet_autoinc_seqs(const bo { int ret = OB_SUCCESS; int64_t rpc_timeout = 0; - const int64_t tablet_count = src_tablet_ids_.count(); - share::ObLocationService *location_service = nullptr; - ObHashMap> ls_to_tablet_map; + const int64_t tablet_count = is_get ? src_tablet_ids_.count() : autoinc_params_.count(); + ObSEArray request_params; - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret)); - } else if (OB_FAIL(ls_to_tablet_map.create(MAP_BUCKET_NUM, "DDLRedefTmp"))) { - LOG_WARN("failed to create map", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tablet_count, rpc_timeout))) { + if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tablet_count, rpc_timeout))) { LOG_WARN("failed to get ddl rpc timeout", K(ret)); } else { if (is_get) { - ObSEArray tmp_autoinc_params; if (OB_UNLIKELY(src_tablet_ids_.count() != dest_tablet_ids_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet ids count", K(ret), K(src_tablet_ids_), K(dest_tablet_ids_)); @@ -2629,163 +2425,110 @@ int ObSyncTabletAutoincSeqCtx::call_and_process_all_tablet_autoinc_seqs(const bo ObMigrateTabletAutoincSeqParam autoinc_param; autoinc_param.src_tablet_id_ = src_tablet_ids_.at(i); autoinc_param.dest_tablet_id_ = dest_tablet_ids_.at(i); - if (OB_FAIL(tmp_autoinc_params.push_back(autoinc_param))) { + if (OB_FAIL(request_params.push_back(autoinc_param))) { LOG_WARN("failed to push back", K(ret), K(autoinc_param)); } } - if (OB_SUCC(ret) && OB_FAIL(build_ls_to_tablet_map(location_service, - tmp_autoinc_params, - rpc_timeout, - need_renew_location_, - true/*by src tablet*/, - ls_to_tablet_map))) { - LOG_ERROR("failed to build ls to tabmap", K(ret)); - } - } else { - if (OB_FAIL(build_ls_to_tablet_map(location_service, - autoinc_params_, - rpc_timeout, - need_renew_location_, - false/*by src tablet*/, - ls_to_tablet_map))) { - LOG_WARN("failed to build ls to tabmap", K(ret)); - } + } else if (OB_FAIL(request_params.assign(autoinc_params_))) { + LOG_WARN("failed to assign request params", K(ret), K(autoinc_params_)); } } - // dispatch each LS request directly to the local autoinc-seq handler and - // process results, replacing ObBatchGet/SetTabletAutoincSeqProxy + RPC transport. + // Dispatch one request to the local autoinc-seq handler. if (OB_SUCC(ret)) { storage::ObTabletAutoincSeqRpcHandler &handler = storage::ObTabletAutoincSeqRpcHandler::get_instance(); - common::ObArray tmp_ret_array; - // per-LS result params, parallel to tmp_ret_array (uniform for get/set); - // only populated when the handler call for that LS succeeds. - common::ObArray> result_params_array; - ObHashMap>::hashtable::const_iterator map_iter = ls_to_tablet_map.begin(); - for (; OB_SUCC(ret) && map_iter != ls_to_tablet_map.end(); ++map_iter) { - const ObLSID &ls_id = map_iter->first; - int rpc_ret_code = OB_SUCCESS; - common::ObSArray result_params; - if (is_get) { - obcall::ObBatchGetTabletAutoincSeqArg arg; - obcall::ObBatchGetTabletAutoincSeqRes res; - if (OB_FAIL(arg.init(ls_id, map_iter->second))) { - LOG_WARN("failed to init arg", K(ret)); - } else { - rpc_ret_code = ex_rpc::sync_call(rpc_timeout, [&]{ return handler.batch_get_tablet_autoinc_seq(arg, res); }); - if (OB_SUCCESS == rpc_ret_code && OB_FAIL(result_params.assign(res.autoinc_params_))) { - LOG_WARN("failed to assign result params", K(ret)); - } - } + int rpc_ret_code = OB_SUCCESS; + common::ObSArray result_params; + if (is_get) { + obcall::ObBatchGetTabletAutoincSeqArg arg; + obcall::ObBatchGetTabletAutoincSeqRes res; + if (OB_FAIL(arg.init(request_params))) { + LOG_WARN("failed to init arg", K(ret), K(request_params)); } else { - obcall::ObBatchSetTabletAutoincSeqArg arg; - obcall::ObBatchSetTabletAutoincSeqRes res; - if (OB_FAIL(arg.init(ls_id, map_iter->second))) { - LOG_WARN("failed to init arg", K(ret)); - } else { - rpc_ret_code = ex_rpc::sync_call(rpc_timeout, [&]{ return handler.batch_set_tablet_autoinc_seq(arg, res); }); - if (OB_SUCCESS == rpc_ret_code && OB_FAIL(result_params.assign(res.autoinc_params_))) { - LOG_WARN("failed to assign result params", K(ret)); - } + rpc_ret_code = ex_rpc::sync_call(rpc_timeout, [&]{ return handler.batch_get_tablet_autoinc_seq(arg, res); }); + if (OB_SUCCESS == rpc_ret_code && OB_FAIL(result_params.assign(res.autoinc_params_))) { + LOG_WARN("failed to assign result params", K(ret)); } } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tmp_ret_array.push_back(rpc_ret_code))) { - LOG_WARN("failed to push ret code", K(ret)); - } else if (OB_FAIL(result_params_array.push_back(result_params))) { - LOG_WARN("failed to push result params", K(ret)); + } else { + obcall::ObBatchSetTabletAutoincSeqArg arg; + obcall::ObBatchSetTabletAutoincSeqRes res; + if (OB_FAIL(arg.init(request_params))) { + LOG_WARN("failed to init arg", K(ret), K(request_params)); + } else { + rpc_ret_code = ex_rpc::sync_call(rpc_timeout, [&]{ return handler.batch_set_tablet_autoinc_seq(arg, res); }); + if (OB_SUCCESS == rpc_ret_code && OB_FAIL(result_params.assign(res.autoinc_params_))) { + LOG_WARN("failed to assign result params", K(ret)); + } } } - // process result int tmp_ret = OB_SUCCESS; - if (OB_FAIL(ret)) { - } else if (tmp_ret_array.count() != ls_to_tablet_map.size()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("result count not match", KR(ret), K(ls_to_tablet_map.size()), K(tmp_ret_array.count())); - } else { - ObHashMap>::hashtable::iterator map_iter = ls_to_tablet_map.begin(); + if (OB_SUCC(ret)) { int64_t new_params_cnt = 0; - // check and reserve first - for (int64_t i = 0; OB_SUCC(ret) && i < tmp_ret_array.count(); ++i, ++map_iter) { - const int rpc_ret_code = tmp_ret_array.at(i); - if (OB_UNLIKELY(map_iter == ls_to_tablet_map.end())) { + if (OB_SUCCESS == rpc_ret_code) { + if (OB_UNLIKELY(request_params.count() != result_params.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("map iter reach end unexpectedly", K(ret), K(i)); - } else if (OB_SUCCESS == rpc_ret_code) { - const ObIArray &cur_params = result_params_array.at(i); - if (OB_UNLIKELY(map_iter->second.count() != cur_params.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("result params count must be equal to request params count when rpc succ", - K(ret), K(map_iter->second.count()), K(cur_params.count())); - } - for (int64_t j = 0; OB_SUCC(ret) && j < cur_params.count(); j++) { - const ObMigrateTabletAutoincSeqParam &autoinc_param = cur_params.at(j); - if (OB_SUCCESS == autoinc_param.ret_code_) { - if (is_get) { - new_params_cnt++; - } + LOG_WARN("result params count must be equal to request params count when rpc succ", + K(ret), K(request_params.count()), K(result_params.count())); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < result_params.count(); j++) { + const ObMigrateTabletAutoincSeqParam &autoinc_param = result_params.at(j); + if (OB_SUCCESS == autoinc_param.ret_code_ && is_get) { + new_params_cnt++; } } } + } else { + for (int64_t j = 0; j < request_params.count(); j++) { + ObMigrateTabletAutoincSeqParam &autoinc_param = request_params.at(j); + autoinc_param.ret_code_ = rpc_ret_code; + } } if (OB_FAIL(ret)) { } else if (is_get && OB_FAIL(autoinc_params_.reserve(new_params_cnt))) { LOG_WARN("failed to reserve new param cnt", K(ret)); - } - - // the following won't fail - if (OB_SUCC(ret)) { + } else { if (is_get) { src_tablet_ids_.reuse(); dest_tablet_ids_.reuse(); } else { autoinc_params_.reuse(); } + const ObIArray ¶ms_to_process = + OB_SUCCESS == rpc_ret_code + ? static_cast &>(result_params) + : static_cast &>(request_params); tmp_ret = OB_SUCCESS; // last non-retry error or first retry error or success - map_iter = ls_to_tablet_map.begin(); - for (int64_t i = 0; OB_SUCC(ret) && i < tmp_ret_array.count(); ++i, ++map_iter) { - const int rpc_ret_code = tmp_ret_array.at(i); - const ObIArray *result_params = nullptr; - if (OB_SUCCESS == rpc_ret_code) { - result_params = &result_params_array.at(i); - } else { - for (int64_t j = 0; j < map_iter->second.count(); j++) { - ObMigrateTabletAutoincSeqParam &autoinc_param = map_iter->second.at(j); - autoinc_param.ret_code_ = rpc_ret_code; + for (int64_t j = 0; OB_SUCC(ret) && j < params_to_process.count(); j++) { + const ObMigrateTabletAutoincSeqParam &autoinc_param = params_to_process.at(j); + if (OB_SUCCESS == autoinc_param.ret_code_) { + if (is_get) { + if (OB_FAIL(autoinc_params_.push_back(autoinc_param))) { + LOG_WARN("failed to push autoinc param", K(ret), K(autoinc_param)); + } + } else { + // do nothing } - result_params = &map_iter->second; - } - for (int64_t j = 0; OB_SUCC(ret) && j < result_params->count(); j++) { - const ObMigrateTabletAutoincSeqParam &autoinc_param = result_params->at(j); - if (OB_SUCCESS == autoinc_param.ret_code_) { - if (is_get) { - if (OB_FAIL(autoinc_params_.push_back(autoinc_param))) { - LOG_WARN("failed to push autoinc param", K(ret), K(autoinc_param)); - } - } else { - // do nothing + } else { + // reclaim on failure + if (is_get) { + if (OB_FAIL(src_tablet_ids_.push_back(autoinc_param.src_tablet_id_))) { + LOG_WARN("failed to push src tablet id", K(ret), K(autoinc_param)); + } else if (OB_FAIL(dest_tablet_ids_.push_back(autoinc_param.dest_tablet_id_))) { + LOG_WARN("failed to push dest tablet id", K(ret), K(autoinc_param)); } } else { - // reclaim on failure - if (is_get) { - if (OB_FAIL(src_tablet_ids_.push_back(autoinc_param.src_tablet_id_))) { - LOG_WARN("failed to push src tablet id", K(ret), K(autoinc_param)); - } else if (OB_FAIL(dest_tablet_ids_.push_back(autoinc_param.dest_tablet_id_))) { - LOG_WARN("failed to push dest tablet id", K(ret), K(autoinc_param)); - } - } else { - if (OB_FAIL(autoinc_params_.push_back(autoinc_param))) { - LOG_WARN("failed to push autoinc param", K(ret)); - } + if (OB_FAIL(autoinc_params_.push_back(autoinc_param))) { + LOG_WARN("failed to push autoinc param", K(ret)); } - if (is_error_need_retry(autoinc_param.ret_code_)) { - if (tmp_ret == OB_SUCCESS) { - tmp_ret = autoinc_param.ret_code_; - } - } else { + } + if (is_error_need_retry(autoinc_param.ret_code_)) { + if (tmp_ret == OB_SUCCESS) { tmp_ret = autoinc_param.ret_code_; } + } else { + tmp_ret = autoinc_param.ret_code_; } } } @@ -2794,10 +2537,6 @@ int ObSyncTabletAutoincSeqCtx::call_and_process_all_tablet_autoinc_seqs(const bo } } } - - if (ls_to_tablet_map.created()) { - ls_to_tablet_map.destroy(); - } } if (OB_SUCC(ret) && is_get) { if (OB_UNLIKELY(orig_src_tablet_ids_.count() != autoinc_params_.count())) { diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h index 312c59b7c..877c3acc9 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h @@ -40,20 +40,15 @@ class ObDDLRedefinitionSSTableBuildTask : public share::ObAsyncTask const common::ObCurTraceId::TraceId &trace_id, const int64_t parallelism, const bool use_heap_table_ddl_plan, - const bool is_mview_complete_refresh, - const int64_t mview_table_id, ObRootService *root_service, const common::ObAddr &inner_sql_exec_addr, const int64_t data_format_version, - const bool is_retryable_ddl, - const uint64_t mview_target_data_sync_scn, - const ObString &mview_select_sql); + const bool is_retryable_ddl); int init( const ObTableSchema &orig_table_schema, const ObTableSchema &hidden_table_schema, const AlterTableSchema &alter_table_schema, - const ObTimeZoneInfoWrap &tz_info_wrap, - const common::ObIArray &based_schema_object_infos); + const ObTimeZoneInfoWrap &tz_info_wrap); ObDDLTaskID get_ddl_task_id() { return ObDDLTaskID(task_id_); } virtual ~ObDDLRedefinitionSSTableBuildTask() = default; virtual int process() override; @@ -74,15 +69,10 @@ class ObDDLRedefinitionSSTableBuildTask : public share::ObAsyncTask common::ObCurTraceId::TraceId trace_id_; int64_t parallelism_; bool use_heap_table_ddl_plan_; - bool is_mview_complete_refresh_; bool is_retryable_ddl_; - int64_t mview_table_id_; - common::ObArray based_schema_object_infos_; ObRootService *root_service_; common::ObAddr inner_sql_exec_addr_; int64_t data_format_version_; - uint64_t mview_target_data_sync_scn_; - ObString mview_select_sql_; }; class ObSyncTabletAutoincSeqCtx final @@ -98,13 +88,6 @@ class ObSyncTabletAutoincSeqCtx final TO_STRING_KV(K_(is_inited), K_(is_synced), K_(orig_src_tablet_ids), K_(src_tablet_ids), K_(dest_tablet_ids), K_(autoinc_params)); private: - int build_ls_to_tablet_map( - share::ObLocationService *location_service, - const common::ObIArray &tablet_ids, - const int64_t timeout, - const bool force_renew, - const bool by_src_tablet, - common::hash::ObHashMap> &map); int call_and_process_all_tablet_autoinc_seqs(const bool is_get); bool is_error_need_retry(const int ret_code) const { @@ -112,10 +95,8 @@ class ObSyncTabletAutoincSeqCtx final common::OB_EAGAIN == ret_code; } private: - static const int64_t MAP_BUCKET_NUM = 1024; bool is_inited_; bool is_synced_; - bool need_renew_location_; ObSEArray orig_src_tablet_ids_; ObSEArray src_tablet_ids_; ObSEArray dest_tablet_ids_; @@ -264,10 +245,6 @@ class ObDDLRedefinitionTask : public ObDDLTask obcall::ObAlterTableArg &alter_table_arg); int64_t get_build_replica_request_time(); private: - int add_table_tablets_for_snapshot_(const uint64_t table_id, ObSchemaGetterGuard &schema_guard, - common::ObIArray &tablet_ids); - int prepare_tablets_for_major_refresh_mv_(common::ObIArray &tablet_ids); - virtual int cleanup_impl() override; protected: static const int64_t MAP_BUCKET_NUM = 1024; diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp index afc49df7e..5b0023f9e 100755 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp @@ -22,9 +22,7 @@ #include "rootserver/ddl_task/ob_drop_fts_index_task.h" #include "rootserver/ddl_task/ob_drop_vec_index_task.h" #include "rootserver/ddl_task/ob_drop_lob_task.h" -#include "rootserver/ddl_task/ob_build_mview_task.h" #include "rootserver/ddl_task/ob_fts_index_build_task.h" -#include "rootserver/ddl_task/ob_build_mview_task.h" #include "rootserver/ddl_task/ob_drop_vec_ivf_index_task.h" #include "rootserver/ddl_task/ob_rebuild_index_task.h" #include "rootserver/fork_table/ob_fork_table_task.h" @@ -944,7 +942,7 @@ void ObDDLScheduler::mtl_wait(ObDDLScheduler *&ddl_scheduler) FLOG_INFO("finish mtl_wait for ddl scheduler", KR(ret)); } -int ObDDLScheduler::switch_to_leader() +int ObDDLScheduler::activate() { int ret = OB_SUCCESS; bool scan_timer_task_exist = false; @@ -954,7 +952,7 @@ int ObDDLScheduler::switch_to_leader() if (OB_UNLIKELY(!is_inited_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K_(is_inited)); - } else if (OB_FAIL(rootserver::ObTenantThreadHelper::switch_to_leader())) { + } else if (OB_FAIL(rootserver::ObTenantThreadHelper::activate())) { LOG_WARN("new ddl scheduler start thread failed", KR(ret)); } else if (OB_FAIL(ddl_builder_.start())) { LOG_WARN("fail to start new ddl builder", KR(ret)); @@ -993,40 +991,12 @@ int ObDDLScheduler::switch_to_leader() return ret; } -int ObDDLScheduler::resume_leader() +void ObDDLScheduler::deactivate() { - int ret = OB_SUCCESS; - FLOG_INFO("[SYS_DDL_SCHEDULER] ObDDLScheduler resume leader begin", - KR(ret), K_(is_inited), K_(is_stop)); - if (OB_FAIL(switch_to_leader())) { - LOG_WARN("resume leader failed", KR(ret)); - } - FLOG_INFO("[SYS_DDL_SCHEDULER] ObDDLScheduler resume leader finish", - KR(ret), K_(is_inited), K_(is_stop)); - return ret; -} - -void ObDDLScheduler::switch_to_follower_forcedly() -{ - switch_to_follower_gracefully(); -} - -int ObDDLScheduler::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - FLOG_INFO("[SYS_DDL_SCHEDULER] ObDDLScheduler switch follower begin", - KR(ret), K_(is_inited), K_(is_stop)); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("sys ddl scheduler is not inited", KR(ret), K(is_inited_)); - } else if (OB_FAIL(ObTenantThreadHelper::switch_to_follower_gracefully())) { - LOG_WARN("fail to switch to follower", KR(ret)); - } else { + if (is_inited_) { + ObTenantThreadHelper::deactivate(); stop(); } - FLOG_INFO("[SYS_DDL_SCHEDULER] ObDDLScheduler switch follower finish", - KR(ret), K_(is_inited), K_(is_stop)); - return ret; } int ObDDLScheduler::init() @@ -1210,7 +1180,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, const obcall::ObDropIndexArg *drop_index_arg = nullptr; const obcall::ObPartitionSplitArg *partition_split_arg = nullptr; const obcall::ObRebuildIndexArg *rebuild_index_arg = nullptr; - const obcall::ObMViewCompleteRefreshArg *mview_complete_refresh_arg = nullptr; const obcall::ObForkTableArg *fork_table_arg = nullptr; LOG_INFO("create ddl task", K(param)); if (OB_UNLIKELY(!is_inited_)) { @@ -1233,7 +1202,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, if (OB_SUCC(ret)) { switch (param.type_) { case DDL_CREATE_INDEX: - case DDL_CREATE_MLOG: case DDL_CREATE_PARTITIONED_LOCAL_INDEX: create_index_arg = static_cast(param.ddl_arg_); if (OB_FAIL(create_build_index_task(proxy, @@ -1305,7 +1273,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, } break; case DDL_DROP_INDEX: - case DDL_DROP_MLOG: // in this case, src_table_schema is data table, dest_table_schema is index table drop_index_arg = static_cast(param.ddl_arg_); if (OB_FAIL(create_drop_index_task(proxy, @@ -1394,7 +1361,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, case DDL_TABLE_REDEFINITION: case DDL_DIRECT_LOAD: case DDL_DIRECT_LOAD_INSERT: - case DDL_MVIEW_COMPLETE_REFRESH: case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION: if (OB_FAIL(create_table_redefinition_task(proxy, @@ -1414,7 +1380,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, } break; case DDL_REBUILD_INDEX: - case DDL_REPLACE_MLOG: rebuild_index_arg = static_cast(param.ddl_arg_); if (OB_FAIL(create_rebuild_index_task(proxy, param.type_, @@ -1429,18 +1394,6 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, LOG_WARN("fail to create rebuild index task", KR(ret)); } break; - case DDL_CREATE_MVIEW: - mview_complete_refresh_arg = static_cast(param.ddl_arg_); - if (OB_FAIL(create_build_mview_task(proxy, - param.src_table_schema_, - param.parallelism_, - param.parent_task_id_, - mview_complete_refresh_arg, - *param.allocator_, - task_record))) { - LOG_WARN("fail to create build mview task", K(ret)); - } - break; case DDL_DROP_PRIMARY_KEY: alter_table_arg = static_cast(param.ddl_arg_); if (OB_FAIL(create_drop_primary_key_task(proxy, @@ -1669,7 +1622,6 @@ int ObDDLScheduler::cache_auto_split_task(const obcall::ObAutoSplitTabletBatchAr const obcall::ObAutoSplitTabletArg &single_arg = single_arg_array.at(i); task.reset(); task.auto_split_tablet_size_ = single_arg.auto_split_tablet_size_; - task.ls_id_ = single_arg.ls_id_; task.tablet_id_ = single_arg.tablet_id_; @@ -1736,7 +1688,7 @@ int ObDDLScheduler::schedule_auto_split_task() tmp_ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(tmp_ret), K(task)); } else if (FALSE_IT(single_arg = new (buf) obcall::ObAlterTableArg())) { - } else if (OB_TMP_FAIL(split_helper.build_arg( task.ls_id_, task.tablet_id_, + } else if (OB_TMP_FAIL(split_helper.build_arg(task.tablet_id_, task.auto_split_tablet_size_, task.used_disk_space_, *single_arg))) { LOG_WARN("fail to build arg", K(tmp_ret), K(task)); } else if (!single_arg->is_auto_split_partition()) { @@ -3056,12 +3008,10 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) } else { switch (record.ddl_type_) { case ObDDLType::DDL_CREATE_INDEX: - case ObDDLType::DDL_CREATE_MLOG: case ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX: ret = schedule_build_index_task(record); break; case ObDDLType::DDL_DROP_INDEX: - case ObDDLType::DDL_DROP_MLOG: ret = schedule_drop_index_task(record); break; case ObDDLType::DDL_CREATE_FTS_INDEX: @@ -3094,8 +3044,8 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) ret = schedule_fork_table_task(record); break; case ObDDLType::DDL_REBUILD_INDEX: - case ObDDLType::DDL_REPLACE_MLOG: ret = schedule_rebuild_index_task(record); + break; case ObDDLType::DDL_DROP_LOB: ret = schedule_drop_lob_task(record); break; @@ -3110,14 +3060,10 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) case DDL_TABLE_REDEFINITION: case DDL_DIRECT_LOAD: case DDL_DIRECT_LOAD_INSERT: - case DDL_MVIEW_COMPLETE_REFRESH: case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION: ret = schedule_table_redefinition_task(record); break; - case DDL_CREATE_MVIEW: - ret = schedule_build_mview_task(record); - break; case DDL_DROP_COLUMN: case DDL_ADD_COLUMN_OFFLINE: case DDL_COLUMN_REDEFINITION: @@ -3298,9 +3244,9 @@ int ObDDLScheduler::create_rebuild_index_task( } else if (OB_ISNULL(index_schema) || OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(index_schema), KP(GCTX.sql_proxy_)); - } else if (!index_schema->is_vec_index() && !index_schema->is_mlog_table()) { // current only support vector index and mlog ddl rebuild task + } else if (!index_schema->is_vec_index()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("only vec index and mlog are supported", KR(ret), KPC(index_schema)); + LOG_WARN("only vec index is supported", KR(ret), KPC(index_schema)); } else if (index_schema->is_vec_index() && index_schema->is_built_in_vec_index()) { // Expecting the rebuild to be initiated by the visibility table (hnsw delta buffer table or ivf centroid table) ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected index schema", KR(ret), KPC(index_schema)); @@ -3710,73 +3656,6 @@ int ObDDLScheduler::schedule_fork_table_task( return ret; } -int ObDDLScheduler::create_build_mview_task( - common::ObISQLClient &proxy, - const ObTableSchema *mview_schema, - const int64_t parallelism, - const int64_t parent_task_id, - const obcall::ObMViewCompleteRefreshArg *mview_complete_refresh_arg, - ObIAllocator &allocator, - ObDDLTaskRecord &task_record) -{ - int ret = OB_SUCCESS; - int64_t task_id = 0; - SMART_VAR(ObBuildMViewTask, mview_task) { - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(mview_complete_refresh_arg) - || OB_ISNULL(mview_schema) - || OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(mview_complete_refresh_arg), - K(mview_schema), KP(GCTX.sql_proxy_)); - } else if (OB_FAIL(ObDDLTask::fetch_new_task_id(*GCTX.sql_proxy_, task_id))) { - LOG_WARN("failed to fetch new task id", KR(ret)); - } else if (OB_FAIL(mview_task.init(task_id, - mview_schema, - mview_schema->get_schema_version(), - parallelism, - *mview_complete_refresh_arg, - parent_task_id))) { - LOG_WARN("failed to init mview task", KR(ret)); - } else if (OB_FAIL(mview_task.set_trace_id(*ObCurTraceId::get_trace_id()))) { - LOG_WARN("failed to set trace id", KR(ret)); - } else if (OB_FAIL(insert_task_record(proxy, mview_task, allocator, task_record))) { - LOG_WARN("failed to insert task record", KR(ret)); - } - } - return ret; -} - -int ObDDLScheduler::schedule_build_mview_task(const ObDDLTaskRecord &task_record) -{ - int ret = OB_SUCCESS; - ObBuildMViewTask *build_mview_task = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_FAIL(alloc_ddl_task(build_mview_task))) { - LOG_WARN("failed to alloc dll task", KR(ret)); - } else { - if (OB_FAIL(build_mview_task->init(task_record))) { - LOG_WARN("failed to init build mview task", KR(ret), K(task_record)); - } else if (OB_FAIL(build_mview_task->set_trace_id(task_record.trace_id_))) { - LOG_WARN("failed to set trace id", KR(ret), K(task_record)); - } else if (OB_FAIL(inner_schedule_ddl_task(build_mview_task, task_record))) { - if (OB_ENTRY_EXIST != ret) { - LOG_WARN("failed to inner schedule task", KR(ret), KPC(build_mview_task)); - } - } - } - if (OB_FAIL(ret) && OB_NOT_NULL(build_mview_task)) { - build_mview_task->~ObBuildMViewTask(); - allocator_.free(build_mview_task); - build_mview_task = nullptr; - } - return ret; -} - int ObDDLScheduler::schedule_rebuild_index_task(const ObDDLTaskRecord &task_record) { int ret = OB_SUCCESS; @@ -4003,7 +3882,6 @@ int ObDDLScheduler::on_sstable_complement_job_reply( case ObDDLType::DDL_MODIFY_COLUMN: case ObDDLType::DDL_CONVERT_TO_CHARACTER: case ObDDLType::DDL_TABLE_REDEFINITION: - case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: case ObDDLType::DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION: if (OB_FAIL(static_cast(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) { @@ -4058,50 +3936,6 @@ int ObDDLScheduler::on_sstable_complement_job_reply( return ret; } -int ObDDLScheduler::on_ddl_task_prepare( - const ObDDLTaskID &parent_task_id, - const int64_t task_id, - const ObCurTraceId::TraceId &parent_task_trace_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObDDLScheduler has not been inited", K(ret)); - } else if (OB_UNLIKELY(!parent_task_id.is_valid()) || task_id <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(parent_task_id), K(task_id)); - } else { - ObDDLTask *ddl_task = nullptr; - struct Func { - Func(const int64_t task_id) :task_id_(task_id) {} - int operator()(ObDDLTask &task) { - ObBuildMViewTask *build_mv_task = static_cast(&task); - return build_mv_task->on_child_task_prepare(task_id_); - } - int64_t task_id_; - } func(task_id); - - ret = task_queue_.modify_task(parent_task_id, func); - if (OB_ENTRY_NOT_EXIST == ret) { - bool exist = false; - if (OB_FAIL(ObDDLTaskRecordOperator::check_task_id_exist(*GCTX.sql_proxy_, parent_task_id.task_id_, exist))) { - LOG_WARN("check task id exist fail", K(ret), K(parent_task_id)); - } else { - if (exist) { - ret = OB_EAGAIN; - LOG_INFO("entry exist, the ddl scheduler hasn't recovered the task yet", K(ret), K(parent_task_id)); - } else { - LOG_WARN("this task does not exist int hash table", K(ret), K(parent_task_id)); - } - } - } else if (OB_FAIL(ret)) { - LOG_WARN("failed to modify task", K(ret)); - } - } - LOG_INFO("ddl task on prepare", K(ret), "ddl_event_info", ObDDLEventInfo(), K(parent_task_id), K(task_id), K(parent_task_trace_id)); - return ret; -} - int ObDDLScheduler::on_ddl_task_finish( const ObDDLTaskID &parent_task_id, const ObDDLTaskKey &child_task_key, @@ -4148,7 +3982,6 @@ int ObDDLScheduler::notify_update_autoinc_end(const ObDDLTaskKey &task_key, case ObDDLType::DDL_TABLE_REDEFINITION: case ObDDLType::DDL_DIRECT_LOAD: case ObDDLType::DDL_DIRECT_LOAD_INSERT: - case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: if (OB_FAIL(static_cast(&task)->notify_update_autoinc_finish(autoinc_val, ret_code))) { LOG_WARN("update complete sstable job status", K(ret)); diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.h b/src/rootserver/ddl_task/ob_ddl_scheduler.h index 4a0d26bb6..4411c76a5 100755 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.h +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.h @@ -271,10 +271,8 @@ class ObDDLScheduler : public rootserver::ObTenantThreadHelper, return OB_SUCCESS; } // for role change - virtual void switch_to_follower_forcedly() override; - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override; - virtual int resume_leader() override; + void deactivate() override; + int activate() override; // mtl_functions static int mtl_init(ObDDLScheduler *&ddl_scheduler); @@ -313,11 +311,6 @@ class ObDDLScheduler : public rootserver::ObTenantThreadHelper, const int ret_code, const ObCurTraceId::TraceId &parent_task_trace_id); - int on_ddl_task_prepare( - const ObDDLTaskID &parent_task_id, - const int64_t task_id, - const ObCurTraceId::TraceId &parent_task_trace_id); - int notify_update_autoinc_end( const ObDDLTaskKey &task_key, const uint64_t autoinc_val, @@ -468,14 +461,6 @@ class ObDDLScheduler : public rootserver::ObTenantThreadHelper, const int32_t sub_task_trace_id, ObIAllocator &allocator, ObDDLTaskRecord &task_record); - int create_build_mview_task( - common::ObISQLClient &proxy, - const share::schema::ObTableSchema *mlog_schema, - const int64_t parallelism, - const int64_t parent_task_id, - const obcall::ObMViewCompleteRefreshArg *mview_complete_refresh_arg, - ObIAllocator &allocator, - ObDDLTaskRecord &task_record); int create_table_redefinition_task( common::ObISQLClient &proxy, const share::ObDDLType &type, @@ -629,7 +614,6 @@ int create_partition_split_task( const ObDDLTaskRecord &task_record); int schedule_build_index_task( const ObDDLTaskRecord &task_record); - int schedule_build_mview_task(const ObDDLTaskRecord &task_record); int schedule_drop_primary_key_task(const ObDDLTaskRecord &task_record); int schedule_table_redefinition_task(const ObDDLTaskRecord &task_record); int schedule_constraint_task(const ObDDLTaskRecord &task_record); diff --git a/src/rootserver/ddl_task/ob_ddl_single_replica_executor.cpp b/src/rootserver/ddl_task/ob_ddl_single_replica_executor.cpp index 96f2911d6..bfeaaa431 100644 --- a/src/rootserver/ddl_task/ob_ddl_single_replica_executor.cpp +++ b/src/rootserver/ddl_task/ob_ddl_single_replica_executor.cpp @@ -23,7 +23,6 @@ #include "rootserver/ob_root_service.h" #include "observer/ob_service.h" #include "share/ob_ddl_sim_point.h" -#include "share/location_cache/ob_location_service.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -292,29 +291,19 @@ int ObDDLReplicaBuildExecutor::check_build_end(const bool need_checksum, bool &i int ret = OB_SUCCESS; is_end = false; ret_code = OB_SUCCESS; - ObArray replica_tablet_ids; - ObArray replica_addrs; int64_t succ_cnt = 0; int64_t failed_cnt = 0; int64_t reschedule_cnt = 0; int64_t waiting_cnt = 0; int64_t total_cnt = 0; int64_t dest_table_id = OB_INVALID_ID; - const bool send_to_all_replicas = is_tablet_split(ddl_type_); - // keep get replica addr out of lock scope if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("replica build executor not init", K(ret)); - } else if (OB_FAIL(get_refreshed_replica_addrs(send_to_all_replicas, - replica_tablet_ids, replica_addrs))) { - LOG_WARN("failed to get refreshed replica addrs", K(ret)); } { // lock scope ObSpinLockGuard guard(lock_); - // first use refresh addrs to refresh build ctxs if (OB_FAIL(ret)) { - } else if (OB_FAIL(refresh_replica_build_ctxs(replica_tablet_ids, replica_addrs))) { - LOG_WARN("failed to refresh replica build ctxs", K(ret)); } else if (!replica_build_ctxs_.empty()) { dest_table_id = replica_build_ctxs_.at(0).dest_table_id_; } @@ -447,12 +436,6 @@ int ObDDLReplicaBuildExecutor::construct_rpc_arg( obcall::ObDDLBuildSingleReplicaRequestArg &arg) const { int ret = OB_SUCCESS; - ObLSID ls_id; - ObLSID dest_ls_id; - share::ObLocationService *location_service = GCTX.location_service_; - bool is_cache_hit = false; - const bool force_renew = true; - const int64_t expire_renew_time = force_renew ? INT64_MAX : 0; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("replica build executor not init", K(ret)); @@ -483,18 +466,6 @@ int ObDDLReplicaBuildExecutor::construct_rpc_arg( LOG_WARN("failed to assign to lob col idxs", K(ret)); } else if (OB_FAIL(arg.parallel_datum_rowkey_list_.assign(replica_build_ctx.parallel_datum_rowkey_list_))) { LOG_WARN("failed to assign split ranges", K(ret)); - } else if (OB_ISNULL(location_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("location service is nullptr", K(ret)); - } else if (OB_FAIL(location_service->get(arg.source_tablet_id_, - expire_renew_time, is_cache_hit, ls_id))) { - LOG_WARN("get ls failed", K(ret), K(arg.source_tablet_id_)); - } else if (OB_FAIL(location_service->get(arg.dest_tablet_id_, - expire_renew_time, is_cache_hit, dest_ls_id))) { - LOG_WARN("get dest ls failed", K(ret), K(arg)); - } else { - arg.ls_id_ = ls_id; - arg.dest_ls_id_ = dest_ls_id; } } return ret; @@ -568,23 +539,11 @@ int ObDDLReplicaBuildExecutor::construct_replica_build_ctxs( ObArray &replica_build_ctxs) const { int ret = OB_SUCCESS; - ObArray split_replica_addrs; replica_build_ctxs.reuse(); - const bool send_to_all_replicas = is_tablet_split(param.ddl_type_); - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); + const bool is_tablet_split_task = is_tablet_split(param.ddl_type_); if (!param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(param)); - } else if (send_to_all_replicas) { - ObLSID ls_id; - ObAddr unused_addr; - const ObTabletID &any_tablet_id = param.source_tablet_ids_.at(0); - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr( - GCTX.location_service_, any_tablet_id, rpc_timeout, ls_id, unused_addr))) { - LOG_WARN("get ls id failed", K(ret), K(any_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_split_replicas_addrs(ls_id, split_replica_addrs))) { - LOG_WARN("get split replica addrs failed", K(ret), K(ls_id)); - } } if (OB_SUCC(ret)) { for (int64_t i = 0; OB_SUCC(ret) && i < param.source_tablet_ids_.count(); ++i) { @@ -595,39 +554,24 @@ int ObDDLReplicaBuildExecutor::construct_replica_build_ctxs( const int64_t src_schema_version = param.source_schema_versions_.at(i); const int64_t dest_schema_version = param.dest_schema_versions_.at(i); int64_t tablet_task_id = i + 1; - int64_t compaction_scn = send_to_all_replicas ? param.compaction_scns_.at(i) : 0; - const bool can_reuse_macro_block = send_to_all_replicas ? param.can_reuse_macro_blocks_.at(i) : false; + int64_t compaction_scn = is_tablet_split_task ? param.compaction_scns_.at(i) : 0; + const bool can_reuse_macro_block = is_tablet_split_task ? param.can_reuse_macro_blocks_.at(i) : false; ObSEArray unused_empty_rowkey_list; // placeholder only. - const ObIArray ¶llel_datum_rowkey_list = send_to_all_replicas ? + const ObIArray ¶llel_datum_rowkey_list = is_tablet_split_task ? param.parallel_datum_rowkey_list_.at(i) : unused_empty_rowkey_list; - if (send_to_all_replicas) { - for (int64_t j = 0; OB_SUCC(ret) && j < split_replica_addrs.count(); ++j) { - ObSingleReplicaBuildCtx replica_build_ctx; - if (OB_FAIL(replica_build_ctx.init(split_replica_addrs.at(j), ddl_type_, - src_table_id, dest_table_id, src_schema_version, - dest_schema_version, tablet_task_id, compaction_scn, - src_tablet_id, dest_tablet_id, can_reuse_macro_block, parallel_datum_rowkey_list))) { - LOG_WARN("failed to init replica build ctx", K(ret)); - } else if (OB_FAIL(replica_build_ctxs.push_back(replica_build_ctx))) { - LOG_WARN("failed to push back replica build ctx", K(ret)); - } + if (is_tablet_split_task) { + ObSingleReplicaBuildCtx replica_build_ctx; + if (OB_FAIL(replica_build_ctx.init(GCTX.self_addr(), ddl_type_, + src_table_id, dest_table_id, src_schema_version, + dest_schema_version, tablet_task_id, compaction_scn, + src_tablet_id, dest_tablet_id, can_reuse_macro_block, parallel_datum_rowkey_list))) { + LOG_WARN("failed to init replica build ctx", K(ret)); + } else if (OB_FAIL(replica_build_ctxs.push_back(replica_build_ctx))) { + LOG_WARN("failed to push back replica build ctx", K(ret)); } - } else { // send to all leader only - ObLSID unused_ls_id; - ObAddr orig_leader_addr; - ObAddr dest_leader_addr; - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); + } else { ObSingleReplicaBuildCtx replica_build_ctx; - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, src_tablet_id, rpc_timeout, unused_ls_id, orig_leader_addr))) { - LOG_WARN("failed to get orig leader addr", K(ret), K(src_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, dest_tablet_id, rpc_timeout, unused_ls_id, - dest_leader_addr))) { - LOG_WARN("failed to get dest leader addr", K(ret), K(dest_tablet_id)); - } else if (orig_leader_addr != dest_leader_addr) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("orig leader addr not equal dest leader addr", K(ret), K(orig_leader_addr), - K(dest_leader_addr)); - } else if (OB_FAIL(replica_build_ctx.init(dest_leader_addr, ddl_type_, + if (OB_FAIL(replica_build_ctx.init(GCTX.self_addr(), ddl_type_, src_table_id, dest_table_id, src_schema_version, dest_schema_version, tablet_task_id, compaction_scn, src_tablet_id, dest_tablet_id, can_reuse_macro_block, parallel_datum_rowkey_list))) { LOG_WARN("failed to init replica build ctx", K(ret), K(src_tablet_id)); @@ -640,155 +584,8 @@ int ObDDLReplicaBuildExecutor::construct_replica_build_ctxs( return ret; } -int ObDDLReplicaBuildExecutor::get_refreshed_replica_addrs( - const bool send_to_all_replicas, - ObArray &replica_tablet_ids, - ObArray &replica_addrs) const -{ - int ret = OB_SUCCESS; - // TODO hanxuan optimization hold lock before refresh addrs (ObMemberListLockUtils) - replica_tablet_ids.reuse(); - replica_addrs.reuse(); - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("replica build executor not init", K(ret)); - } else if (OB_UNLIKELY(src_tablet_ids_.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected err", K(ret), KPC(this)); - } else if (send_to_all_replicas) { - ObLSID ls_id; - ObAddr unused_addr; - ObArray split_replica_addrs; - const ObTabletID &any_tablet_id = src_tablet_ids_.at(0); - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr( - GCTX.location_service_, any_tablet_id, rpc_timeout, ls_id, unused_addr))) { - LOG_WARN("get ls id failed", K(ret), K(any_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_split_replicas_addrs(ls_id, split_replica_addrs))) { - LOG_WARN("get split replica addrs failed", K(ret), K(ls_id)); - } else { // send to all replicas. - for (int64_t i = 0; OB_SUCC(ret) && i < src_tablet_ids_.count(); ++i) { - const ObTabletID &src_tablet_id = src_tablet_ids_.at(i); - for (int64_t j = 0; OB_SUCC(ret) && j < split_replica_addrs.count(); ++j) { - const ObAddr &addr = split_replica_addrs.at(j); - if (OB_FAIL(replica_tablet_ids.push_back(src_tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret)); - } else if (OB_FAIL(replica_addrs.push_back(addr))) { - LOG_WARN("failed to push back addr", K(ret)); - } - } - } - } - } else { // send to leader only. - for (int64_t i = 0; OB_SUCC(ret) && i < src_tablet_ids_.count(); ++i) { - ObLSID ls_id; - ObAddr orig_leader_addr; - ObAddr dest_leader_addr; - const ObTabletID &src_tablet_id = src_tablet_ids_.at(i); - const ObTabletID &dest_tablet_id = dest_tablet_ids_.at(i); - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, src_tablet_id, rpc_timeout, ls_id, orig_leader_addr))) { - LOG_WARN("failed to get orig leader addr", K(ret), K(src_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, dest_tablet_id, rpc_timeout, ls_id, - dest_leader_addr))) { - LOG_WARN("failed to get dest leader addr", K(ret), K(dest_tablet_id)); - } else if (orig_leader_addr != dest_leader_addr) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("orig leader addr not equal dest leader addr", K(ret), - K(orig_leader_addr), K(dest_leader_addr)); - } else { - if (OB_FAIL(replica_tablet_ids.push_back(src_tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret)); - } else if (OB_FAIL(replica_addrs.push_back(dest_leader_addr))) { - LOG_WARN("failed to push back addr", K(ret)); - } - } - } - } - return ret; -} - -/* for each replica, check if there exists build ctx whose tablet id && addr matches - * - if exists, copy it to new replica build ctx with its build stat - * - otherwise, init a new build ctx whose build stat is BUILD_INIT - * NOTE as caller, check_build_end() will hold lock_ - */ -int ObDDLReplicaBuildExecutor::refresh_replica_build_ctxs( - const ObArray &replica_tablet_ids, - const ObArray &replica_addrs) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("replica build executor not init", K(ret)); - } else if (replica_tablet_ids.empty() || replica_addrs.empty() || - replica_tablet_ids.count() != replica_addrs.count()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(replica_tablet_ids.count()), K(replica_addrs.count())); - } - ObArray new_replica_build_ctxs; - for (int64_t i = 0; OB_SUCC(ret) && i < replica_addrs.count(); ++i) { - const ObTabletID &tablet_id = replica_tablet_ids.at(i); - const ObAddr &replica_addr = replica_addrs.at(i); - bool is_found = false; - ObSingleReplicaBuildCtx *existing_ctx = nullptr; - if (OB_FAIL(get_replica_build_ctx(tablet_id, replica_addr, - existing_ctx, is_found))) { - LOG_WARN("failed to get replica build ctx", K(ret), - K(tablet_id), K(replica_addr)); - } else if (is_found) { - if (OB_FAIL(new_replica_build_ctxs.push_back(*existing_ctx))) { - LOG_WARN("failed to push back replica build ctx", K(ret)); - } - } else { // not found - ObSingleReplicaBuildCtx new_build_ctx; - bool ctx_partial_inited = false; - // 1. use existing replica build ctx to init new build ctx - // NOTE currently new build ctx's addr is an unrelated addr - for (int64_t j = 0; OB_SUCC(ret) && !ctx_partial_inited && - j < replica_build_ctxs_.count(); ++j) { - // find corresponding tablet - if (replica_build_ctxs_.at(j).src_tablet_id_ == tablet_id) { - if (OB_FAIL(new_build_ctx.assign(replica_build_ctxs_.at(j)))) { - LOG_WARN("failed to assign to replica build ctx", K(ret)); - } else { - // use correspoinding tablet's old replica ctx to init - ctx_partial_inited = true; - } - } - } - // 2. overwrite with correct addr && reset build stat - if (OB_FAIL(ret)) { - } else if (!ctx_partial_inited) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to setup new replica build ctx for refreshed replica addr", - K(ret), K(tablet_id), K(replica_addr)); - } else if (FALSE_IT(new_build_ctx.addr_ = replica_addr)) { - } else if (FALSE_IT(new_build_ctx.reset_build_stat())) { - } else if (OB_FAIL(new_replica_build_ctxs.push_back(new_build_ctx))) { - LOG_WARN("failed to push back replica build ctx", K(ret)); - } - if (OB_FAIL(ret)) { - LOG_WARN("failed to setup new replica build ctx for refreshed replica addr", - K(ret), K(tablet_id), K(replica_addr), K(new_build_ctx)); - } else { - LOG_INFO("setup new replica build ctx for refreshed replica addr", - K(tablet_id), K(replica_addr), K(new_build_ctx)); - } - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(replica_build_ctxs_.assign(new_replica_build_ctxs))) { - LOG_WARN("failed to assign to replica build ctxs", K(ret)); - } else if (new_replica_build_ctxs.count() != replica_tablet_ids.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the size of replica build ctxs doesn't equal to the size of replica_tablet_ids", K(ret), K(replica_build_ctxs_.count()), K(replica_tablet_ids.count())); - } - return ret; -} - // look up repica build ctx by tablet id && addr // NOTE as caller, update_build_progress() process_rpc_results() -// refresh_replica_build_ctxs() will hold lock int ObDDLReplicaBuildExecutor::get_replica_build_ctx( const ObTabletID &tablet_id, const ObAddr &addr, diff --git a/src/rootserver/ddl_task/ob_ddl_single_replica_executor.h b/src/rootserver/ddl_task/ob_ddl_single_replica_executor.h index e83f4d4c7..885325f39 100644 --- a/src/rootserver/ddl_task/ob_ddl_single_replica_executor.h +++ b/src/rootserver/ddl_task/ob_ddl_single_replica_executor.h @@ -234,13 +234,6 @@ class ObDDLReplicaBuildExecutor int construct_replica_build_ctxs( const ObDDLReplicaBuildExecutorParam ¶m, ObArray &replica_build_ctxs) const; - int get_refreshed_replica_addrs( - const bool send_to_all_replicas, - ObArray &replica_tablet_ids, - ObArray &replica_addrs) const; - int refresh_replica_build_ctxs( - const ObArray &replica_tablet_ids, - const ObArray &replica_addrs); int update_build_ctx( ObSingleReplicaBuildCtx &build_ctx, const obcall::ObDDLBuildSingleReplicaRequestResult *result, diff --git a/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.cpp b/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.cpp index c29e5e217..026a0e993 100644 --- a/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.cpp +++ b/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.cpp @@ -20,7 +20,6 @@ #include "rootserver/ob_ddl_service_launcher.h" // for ObDDLServiceLauncher #include "rootserver/ob_root_service.h" #include "share/ob_ddl_checksum.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" #include "src/observer/ob_inner_sql_connection.h" #include "observer/vector_index/ob_vector_index_util.h" #include "lib/utility/serialization.h" @@ -35,7 +34,10 @@ using namespace oceanbase::sql; ObDDLTabletScheduler::ObDDLTabletScheduler() : is_inited_(false), table_id_(OB_INVALID_ID), ref_data_table_id_(OB_INVALID_ID), - task_id_(OB_INVALID_ID), parallelism_(0), snapshot_version_(0), root_service_(nullptr) + task_id_(OB_INVALID_ID), parallelism_(0), snapshot_version_(0), trace_id_(), + lock_(), root_service_(nullptr), all_tablets_(), running_tablets_(), + running_execution_id_(-1), tablet_id_to_data_size_(), tablet_id_to_data_row_cnt_(), + tablet_id_to_execution_id_map_() { } @@ -56,7 +58,6 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, ObArenaAllocator arena("tblt_sched_init"); common::ObAddr inner_sql_exec_addr; common::ObArray running_sql_info; - common::ObArray ls_ids; common::ObArray ref_data_table_tablets; common::hash::ObHashMap tablet_finished_map; if (OB_UNLIKELY(is_inited_)) { @@ -86,27 +87,12 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(GCTX.sql_proxy_)); - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(*GCTX.sql_proxy_, tablets, ls_ids))) { - LOG_WARN("failed to batch get ls", K(ret), K(tablets), K(ls_ids)); - } else if (OB_UNLIKELY(tablets.count() != ls_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablets count is not equal to ls id count", K(ret), K(tablets.count()), K(ls_ids.count())); - } else if (OB_FAIL(all_ls_to_tablets_map_.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create lsid to tablet id map", K(ret), K(tablets.count())); - } else if (OB_FAIL(running_ls_to_tablets_map_.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create lsid to tablet id map", K(ret), K(tablets.count())); - } else if (OB_FAIL(ls_location_map_.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create lsid location map", K(ret), K(tablets.count())); - } else if (OB_FAIL(running_ls_to_execution_id_.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create lsid to execution id map", K(ret), K(tablets.count())); } else if (OB_FAIL(tablet_finished_map.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { LOG_WARN("fail to create column checksum map", K(ret), K(tablets.count())); } else if (OB_FAIL(tablet_id_to_data_size_.create(ref_data_table_tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { LOG_WARN("fail to create column checksum map", K(ret), K(ref_data_table_tablets.count())); } else if (OB_FAIL(tablet_id_to_data_row_cnt_.create(ref_data_table_tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { LOG_WARN("fail to create column checksum map", K(ret), K(ref_data_table_tablets.count())); - } else if (OB_FAIL(tablet_scheduled_times_statistic_.create(ref_data_table_tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create tablet scheduled count statistic map", K(ret), K(ref_data_table_tablets.count())); } else if (!tablet_id_to_execution_id_map_.created() && OB_FAIL(tablet_id_to_execution_id_map_.create(tablets.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { LOG_ERROR("fail to create tablet id to execution id map", K(ret), K(tablets.count())); } else if (OB_FAIL(ObDDLChecksumOperator::get_local_index_tablet_finish_status(ref_data_table_id, @@ -132,10 +118,10 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, tablet_data_row_cnt = 0; part_tablets.reuse(); partition_names.reuse(); - if (OB_FAIL(ObDDLUtil::get_tablet_data_size(ref_data_table_tablets.at(i), ls_ids.at(i), tablet_data_size))) { - LOG_WARN("fail to get tablet data size", K(ret), K(ref_data_table_tablets.at(i)), K(ls_ids.at(i)), K(tablet_data_size)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_data_row_cnt(ref_data_table_tablets.at(i), ls_ids.at(i), tablet_data_row_cnt))) { - LOG_WARN("fail to get tablet data size", K(ret), K(ref_data_table_tablets.at(i)), K(ls_ids.at(i)), K(tablet_data_row_cnt)); + if (OB_FAIL(ObDDLUtil::get_tablet_data_size(ref_data_table_tablets.at(i), tablet_data_size))) { + LOG_WARN("fail to get tablet data size", K(ret), K(ref_data_table_tablets.at(i)), K(tablet_data_size)); + } else if (OB_FAIL(ObDDLUtil::get_tablet_data_row_cnt(ref_data_table_tablets.at(i), tablet_data_row_cnt))) { + LOG_WARN("fail to get tablet row count", K(ret), K(ref_data_table_tablets.at(i)), K(tablet_data_row_cnt)); } else if (OB_FAIL(tablet_id_to_data_size_.set_refactored(ref_data_table_tablets.at(i).id(), tablet_data_size, true /* overwrite */))) { LOG_WARN("table id to data size map set fail", K(ret), K(ref_data_table_tablets.at(i).id()), K(tablet_data_size)); } else if (OB_FAIL(tablet_id_to_data_row_cnt_.set_refactored(ref_data_table_tablets.at(i).id(), tablet_data_row_cnt, true /* overwrite */))) { @@ -162,30 +148,14 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, if (!is_running_status && is_finished_status) { LOG_INFO("tablet has complemented data", K(ret), K(table_id), K(ref_data_table_id), K(tablets.at(i))); } else { - common::ObAddr leader_addr; - share::ObLocationService *location_service = nullptr; - int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - const int64_t retry_interval_us = 200 * 1000; // 200ms - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret), KP(location_service)); - } else if (OB_FAIL(location_service->get_leader_with_retry_until_timeout(GCONF.cluster_id, ls_ids.at(i), leader_addr, rpc_timeout, retry_interval_us))) { - LOG_WARN("fail to get ls locaiton leader", K(ret), K(ls_ids.at(i))); - } else if (OB_FAIL(ls_location_map_.set_refactored(ls_ids.at(i), leader_addr, true /* overwrite */))) { - LOG_WARN("ls location map set fail", K(ret), K(ls_ids.at(i)), K(leader_addr)); - } else if (is_running_status) { - if (OB_FAIL(ObDDLUtil::construct_ls_tablet_id_map(ls_ids.at(i), tablets.at(i), running_ls_to_tablets_map_))) { - LOG_WARN("fail to create running lsid to tablet id map", K(ret), K(ls_ids.at(i)), K(tablets.at(i)), K(table_id), K(ref_data_table_id), K(running_ls_to_tablets_map_.size())); - } else if (common::is_contain(running_task_ls_ids_before_, ls_ids.at(i))) { - } else if (OB_FAIL(running_task_ls_ids_before_.push_back(ls_ids.at(i)))) { - LOG_WARN("fail to push back", K(ret), K(table_id), K(ref_data_table_id), K(ls_ids.at(i)), K(is_finished_status)); + if (is_running_status) { + if (OB_FAIL(running_tablets_.push_back(tablets.at(i)))) { + LOG_WARN("fail to push running tablet", K(ret), K(table_id), K(ref_data_table_id), K(tablets.at(i)), K(is_finished_status)); } } if (OB_SUCC(ret)) { if (OB_FAIL(all_tablets_.push_back(tablets.at(i)))) { LOG_WARN("fail to push back", K(ret), K(tablets.at(i))); - } else if (OB_FAIL(ObDDLUtil::construct_ls_tablet_id_map(ls_ids.at(i), tablets.at(i), all_ls_to_tablets_map_))) { - LOG_WARN("fail to create lsid to tablet id map", K(ret), K(ls_ids.at(i)), K(tablets.at(i)), K(table_id), K(ref_data_table_id), K(all_ls_to_tablets_map_.size())); } } } @@ -201,9 +171,9 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, snapshot_version_ = snapshot_version; trace_id_ = trace_id; is_inited_ = true; - LOG_INFO("success to init", K(ret), K(table_id), K(ref_data_table_id), K(task_id), K(parallelism), K(snapshot_version), K(trace_id), K(tablets), K(all_ls_to_tablets_map_.size()), K(running_ls_to_tablets_map_.size()), K(running_task_ls_ids_before_.count())); + LOG_INFO("success to init", K(ret), K(table_id), K(ref_data_table_id), K(task_id), K(parallelism), K(snapshot_version), K(trace_id), K(tablets), K(all_tablets_.count()), K(running_tablets_.count())); } else { - LOG_INFO("fail to init", K(ret), K(table_id), K(ref_data_table_id), K(task_id), K(parallelism), K(snapshot_version), K(trace_id), K(tablets), K(all_ls_to_tablets_map_.size()), K(running_ls_to_tablets_map_.size()), K(running_task_ls_ids_before_.count())); + LOG_INFO("fail to init", K(ret), K(table_id), K(ref_data_table_id), K(task_id), K(parallelism), K(snapshot_version), K(trace_id), K(tablets), K(all_tablets_.count()), K(running_tablets_.count())); destroy(); } return ret; @@ -212,16 +182,12 @@ int ObDDLTabletScheduler::init(const uint64_t table_id, int ObDDLTabletScheduler::get_next_batch_tablets(const bool is_ddl_retryable, int64_t ¶llelism, int64_t &new_execution_id, - share::ObLSID &ls_id, - common::ObAddr &leader_addr, ObIArray &tablets) { int ret = OB_SUCCESS; bool need_send_task = false; parallelism = 0; new_execution_id = 0; - ls_id.reset(); - leader_addr.reset(); tablets.reset(); share::ObDDLType task_type = share::DDL_CREATE_PARTITIONED_LOCAL_INDEX; if (OB_UNLIKELY(!is_inited_)) { @@ -232,19 +198,9 @@ int ObDDLTabletScheduler::get_next_batch_tablets(const bool is_ddl_retryable, } else if (OB_FAIL(determine_if_need_to_send_new_task(need_send_task))) { LOG_WARN("fail to get status of if need to send new task", K(ret), K(need_send_task)); } else if (!need_send_task) { - if (!is_running_tasks_before_finished()) { - ObArray running_task_ls_ids_now; - ObArray potential_finished_ls_ids; - if (OB_FAIL(get_session_running_lsid(running_task_ls_ids_now))) { - LOG_WARN("fail to get session running lsid", K(ret), K(running_task_ls_ids_now)); - } else if (OB_FAIL(get_potential_finished_lsid(running_task_ls_ids_now, potential_finished_ls_ids))) { - LOG_WARN("fail to get potential finished lsid", K(ret), K(running_task_ls_ids_now), K(potential_finished_ls_ids)); - } else { - for (int64_t i = 0; i < potential_finished_ls_ids.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(check_target_ls_tasks_completion_status(potential_finished_ls_ids.at(i)))) { - LOG_WARN("fail to check target ls tasks completion status", K(ret), K(potential_finished_ls_ids.at(i))); - } - } + if (is_recovered_running_task()) { + if (OB_FAIL(check_running_task_completion_status())) { + LOG_WARN("fail to check running task completion status", K(ret)); } } if (OB_SUCC(ret)) { @@ -252,53 +208,41 @@ int ObDDLTabletScheduler::get_next_batch_tablets(const bool is_ddl_retryable, } } else if (OB_FAIL(get_next_parallelism(parallelism))) { LOG_WARN("fail to get next parallelism", K(ret), K(parallelism)); - } else if (OB_FAIL(get_unfinished_tablets(task_type, is_ddl_retryable, new_execution_id, ls_id, leader_addr, tablets))) { + } else if (OB_FAIL(get_unfinished_tablets(task_type, is_ddl_retryable, new_execution_id, tablets))) { LOG_WARN("failed to get unfinished tablets", K(ret), K(new_execution_id), K(tablets)); } return ret; } -int ObDDLTabletScheduler::confirm_batch_tablets_status(const int64_t execution_id, const bool finish_status, const ObLSID &ls_id, const ObIArray &tablets) +int ObDDLTabletScheduler::confirm_batch_tablets_status(const int64_t execution_id, const bool finish_status, const ObIArray &tablets) { int ret = OB_SUCCESS; - int64_t ls_execution_id = 0; TCWLockGuard guard(lock_); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(tablets.count() < 1 || !ls_id.is_valid())) { + } else if (OB_UNLIKELY(tablets.count() < 1)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablets array is null", K(ret), K(tablets.count()), K(ls_id)); - } else if (OB_UNLIKELY(all_ls_to_tablets_map_.size() == 0 || running_ls_to_tablets_map_.size() == 0)) { + LOG_WARN("tablets array is null", K(ret), K(tablets.count())); + } else if (OB_UNLIKELY(all_tablets_.empty() || running_tablets_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls to tablets map is null", K(ret), K(all_ls_to_tablets_map_.size()), K(running_ls_to_tablets_map_.size())); + LOG_WARN("tablet queue is null", K(ret), K(all_tablets_.count()), K(running_tablets_.count())); } else { if (execution_id != -1) { //execution_id == -1 indicates the task before switching to rs is confirming - if (OB_FAIL(running_ls_to_execution_id_.get_refactored(ls_id, ls_execution_id))) { - LOG_WARN("fail to get execution id", K(ret), K(ls_id), K(ls_execution_id)); - } else if (OB_UNLIKELY(execution_id != ls_execution_id)) { + if (OB_UNLIKELY(execution_id != running_execution_id_)) { ret = OB_TASK_EXPIRED; - LOG_WARN("receive a mismatch execution result", K(ret), K(execution_id), K(ls_execution_id), K(tablets), K(finish_status)); + LOG_WARN("receive a mismatch execution result", K(ret), K(execution_id), K_(running_execution_id), K(tablets), K(finish_status)); } } if (OB_SUCC(ret)) { - if (OB_FAIL(running_ls_to_tablets_map_.erase_refactored(ls_id))) { - LOG_WARN("failed to erase ls id", K(ret), K(ls_id)); - } else if (finish_status) { - ObTabletIdUpdater updater(tablets); - if (OB_FAIL(all_ls_to_tablets_map_.atomic_refactored(ls_id, updater))) { - LOG_WARN("fail to update tablet ids", K(ret), K(ls_id), K(tablets)); - } else { - bool is_erased = false; - HashMapEraseIfNull functor; - if (OB_FAIL(all_ls_to_tablets_map_.erase_if(ls_id, functor, is_erased))) { - LOG_WARN("fail to erase lsid", K(ret), K(ls_id), K(tablets)); - } - } + running_tablets_.reset(); + running_execution_id_ = -1; + if (finish_status && OB_FAIL(remove_finished_tablets_(tablets))) { + LOG_WARN("fail to remove finished tablets", K(ret), K(tablets)); } } } - LOG_INFO("confirm batch tablets status", K(ret), K(execution_id), K(finish_status), K(ls_id), K(tablets)); + LOG_INFO("confirm batch tablets status", K(ret), K(execution_id), K(finish_status), K(tablets)); return ret; } @@ -306,8 +250,7 @@ int ObDDLTabletScheduler::confirm_batch_tablets_status(const int64_t execution_i // so push_tablet_execution_id does the following: // 1.push tablet execution id, and check if the tablet execution id is pushed more than once // 2.push task execution id anyway, it is no need to check task execution id, we always push task execution id high to avoid different inner sql has same task execution id. -int ObDDLTabletScheduler::push_tablet_execution_id(share::ObDDLType task_type, - const bool ddl_can_retry, +int ObDDLTabletScheduler::push_tablet_execution_id(const bool ddl_can_retry, const common::ObIArray &tablets, int64_t &new_task_execution_id) { @@ -331,8 +274,8 @@ int ObDDLTabletScheduler::push_tablet_execution_id(share::ObDDLType task_type, tablet_execution_id = -1; ret = OB_SUCCESS; } - } else if (OB_FAIL(ObDDLTask::calc_next_execution_id(tablet_execution_id, task_type, ddl_can_retry, next_tablet_execution_id))) { - LOG_WARN("calc next execution id failed", K(ret), K(tablet_execution_id), K(task_type), K(ddl_can_retry)); + } else if (OB_FAIL(ObDDLTask::calc_next_execution_id(tablet_execution_id, ddl_can_retry, next_tablet_execution_id))) { + LOG_WARN("calc next execution id failed", K(ret), K(tablet_execution_id), K(ddl_can_retry)); } else if (OB_FAIL(tablet_id_to_execution_id_map_.set_refactored(tablet, next_tablet_execution_id, true))) { LOG_WARN("set tablet execution id failed", K(ret), K(tablet), K(next_tablet_execution_id)); } @@ -453,28 +396,6 @@ OB_DEF_SERIALIZE_SIZE(ObDDLTabletScheduler) return len; } -int ObDDLTabletScheduler::refresh_ls_location_map() { - int ret = OB_SUCCESS; - TCRLockGuard guard(lock_); - common::hash::ObHashMap::iterator iter; - for (iter = ls_location_map_.begin(); iter != ls_location_map_.end() && OB_SUCC(ret); ++iter) { - const ObLSID ls_id = iter->first; - common::ObAddr leader_addr; - share::ObLocationService *location_service = nullptr; - int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - const int64_t retry_interval_us = 200 * 1000; // 200ms - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret), KP(location_service)); - } else if (OB_FAIL(location_service->get_leader_with_retry_until_timeout(GCONF.cluster_id, ls_id, leader_addr, rpc_timeout, retry_interval_us))) { - LOG_WARN("fail to get ls locaiton leader", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_location_map_.set_refactored(ls_id, leader_addr, true /* overwrite */))) { - LOG_WARN("ls location map set fail", K(ret), K(ls_id), K(leader_addr)); - } - } - return ret; -} - int ObDDLTabletScheduler::get_next_parallelism(int64_t ¶llelism) { int ret = OB_SUCCESS; @@ -484,97 +405,63 @@ int ObDDLTabletScheduler::get_next_parallelism(int64_t ¶llelism) ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else { - int64_t ls_num = all_ls_to_tablets_map_.size(); - if (ls_num > 0) { // it is ensured that the value of ls_num is greater than 0 - parallelism = (parallelism_ + ls_num - 1) / ls_num; + if (all_tablets_.count() > 0) { + parallelism = parallelism_; } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("the all ls to tablet map size is less than 1", K(ret), K(all_ls_to_tablets_map_.size()), K(parallelism)); + LOG_WARN("tablet queue is empty", K(ret), K(all_tablets_.count()), K(parallelism)); } } return ret; } -int ObDDLTabletScheduler::get_unfinished_tablets(const share::ObDDLType task_type,const bool ddl_can_retry, int64_t &new_execution_id, share::ObLSID &ls_id, common::ObAddr &leader_addr, ObIArray &tablets) +int ObDDLTabletScheduler::get_unfinished_tablets(const share::ObDDLType task_type, const bool ddl_can_retry, + int64_t &new_execution_id, ObIArray &tablets) { int ret = OB_SUCCESS; new_execution_id = 0; tablets.reset(); - ls_id.reset(); - leader_addr.reset(); ObArray tablet_queue; uint64_t left_space_size = 0; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); - } else if (OB_FAIL(get_to_be_scheduled_tablets(ls_id, leader_addr, tablet_queue))) { + } else if (OB_FAIL(get_to_be_scheduled_tablets(tablet_queue))) { LOG_WARN("fail to get to be scheduled tablets", K(ret), K(tablet_queue)); } else if (OB_UNLIKELY(tablet_queue.count() < 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet queue is null", K(ret), K(tablet_queue.count())); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !leader_addr.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("got argument is error", K(ret), K(ls_id), K(leader_addr)); - } else if (OB_FAIL(ObDDLUtil::get_ls_host_left_disk_space(ls_id, leader_addr, left_space_size))) { - LOG_WARN("fail to get ls host left disk space", K(ret), K(ls_id), K(leader_addr), K(left_space_size)); + } else if (OB_FAIL(ObDDLUtil::get_ls_host_left_disk_space(left_space_size))) { + LOG_WARN("fail to get local disk free space", K(ret), K(left_space_size)); } else if (OB_FAIL(calculate_candidate_tablets(left_space_size, tablet_queue, tablets))) { LOG_WARN("fail to use strategy to get tablets", K(ret), K(left_space_size), K(tablet_queue), K(tablets)); - } else if (OB_FAIL(push_tablet_execution_id(task_type, ddl_can_retry, tablets, new_execution_id))) { + } else if (OB_FAIL(push_tablet_execution_id(ddl_can_retry, tablets, new_execution_id))) { LOG_WARN("failed to push tablet execution id", K(ret), K(task_type), K(ddl_can_retry), K(tablets), K(new_execution_id)); } else { TCWLockGuard guard(lock_); - if (OB_FAIL(running_ls_to_execution_id_.set_refactored(ls_id, new_execution_id, true /* overwrite */))) { - LOG_WARN("running ls to execution id map set fail", K(ret), K(ls_id), K(new_execution_id)); + if (OB_FAIL(running_tablets_.assign(tablets))) { + LOG_WARN("ObArray assign failed", K(ret), K(tablets)); } else { - ObArray running_tablet_queue; - if (OB_FAIL(running_tablet_queue.assign(tablets))) { - LOG_WARN("ObArray assign failed", K(ret), K(tablets)); - } else if (OB_FAIL(running_ls_to_tablets_map_.set_refactored(ls_id, running_tablet_queue, true /* overwrite */))) { - LOG_WARN("ls tablets map set fail", K(ret), K(ls_id), K(running_tablet_queue)); - } + running_execution_id_ = new_execution_id; } } return ret; } -int ObDDLTabletScheduler::get_to_be_scheduled_tablets(share::ObLSID &ls_id, common::ObAddr &leader_addr, ObIArray &tablets) +int ObDDLTabletScheduler::get_to_be_scheduled_tablets(ObIArray &tablets) { int ret = OB_SUCCESS; - ls_id.reset(); - leader_addr.reset(); tablets.reset(); TCWLockGuard guard(lock_); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(all_ls_to_tablets_map_.size() <= running_ls_to_tablets_map_.size())) { + } else if (OB_UNLIKELY(all_tablets_.empty() || !running_tablets_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("all ls tablets map is less than running ls tablets map", K(ret), K(all_ls_to_tablets_map_.size()), K(running_ls_to_tablets_map_.size())); - } else { - common::hash::ObHashMap>::iterator iter; - for (iter = all_ls_to_tablets_map_.begin(); iter != all_ls_to_tablets_map_.end() && OB_SUCC(ret); ++iter) { - ls_id = iter->first; - ObArray &tablet_queue = iter->second; - ObArray running_tablet_queue; - bool is_running_ls = true; - if (OB_FAIL(running_ls_to_tablets_map_.get_refactored(ls_id, running_tablet_queue))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) { - is_running_ls = false; - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get tablet queue from refactored", K(ret), K(ls_id)); - } - } - if (OB_SUCC(ret) && OB_LIKELY(!is_running_ls && tablet_queue.count() > 0)) { - if (OB_FAIL(ls_location_map_.get_refactored(ls_id, leader_addr))) { - LOG_WARN("fail to get leader addr from ls location map", K(ret), K(ls_id), K(leader_addr)); - } else if (OB_FAIL(tablets.assign(tablet_queue))) { - LOG_WARN("ObArray assign failed", K(ret), K(tablet_queue)); - } - break; - } - } + LOG_WARN("tablet queue state is invalid", K(ret), K(all_tablets_.count()), K(running_tablets_.count())); + } else if (OB_FAIL(tablets.assign(all_tablets_))) { + LOG_WARN("ObArray assign failed", K(ret), K_(all_tablets)); } return ret; } @@ -658,26 +545,6 @@ int ObDDLTabletScheduler::calculate_candidate_tablets(const uint64_t left_space_ pre_data_size = pre_data_size + tablet_data_size; pre_data_row_cnt = pre_data_row_cnt + tablet_data_row_cnt; } - if (OB_SUCC(ret)) { - int64_t has_scheduled_times = 0; - if (OB_FAIL(tablet_scheduled_times_statistic_.get_refactored(in_tablets.at(i).id(), has_scheduled_times))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) { - has_scheduled_times = 0; - ret = OB_SUCCESS; - } - } else { - LOG_WARN("tablet scheduled times exceed 1, need to refresh ls location map", K(ret), K(in_tablets.at(i).id()), K(has_scheduled_times)); - if (OB_FAIL(refresh_ls_location_map())) { - LOG_WARN("fail to refresh ls location map", K(ret)); - } - } - if (OB_SUCC(ret)) { - has_scheduled_times = has_scheduled_times + 1; - if (OB_FAIL(tablet_scheduled_times_statistic_.set_refactored(in_tablets.at(i).id(), has_scheduled_times, true /* overwrite */))) { - LOG_WARN("tablet scheduled times statistic map set fail", K(ret), K(in_tablets.at(i).id()), K(has_scheduled_times)); - } - } - } } else { break; } @@ -689,48 +556,105 @@ int ObDDLTabletScheduler::calculate_candidate_tablets(const uint64_t left_space_ return ret; } -int ObDDLTabletScheduler::get_session_running_lsid(ObIArray &running_ls_ids) +int ObDDLTabletScheduler::remove_finished_tablets_(const ObIArray &tablets) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < tablets.count(); ++i) { + bool found = false; + for (int64_t j = 0; OB_SUCC(ret) && j < all_tablets_.count(); ++j) { + if (tablets.at(i) == all_tablets_.at(j)) { + if (OB_FAIL(all_tablets_.remove(j))) { + LOG_WARN("failed to remove tablet id", K(ret), K(i), K(j), K_(all_tablets), K(tablets)); + } else { + found = true; + } + break; + } + } + if (OB_SUCC(ret) && !found) { + LOG_INFO("finished tablet is not in pending queue", K(tablets.at(i)), K_(all_tablets)); + } + } + LOG_INFO("remove finished tablets from pending queue", K(ret), K(tablets), K_(all_tablets)); + return ret; +} + +int ObDDLTabletScheduler::check_running_task_completion_status() { int ret = OB_SUCCESS; + ObArray running_tablet_queue; + common::hash::ObHashMap tablet_finished_map; ObArenaAllocator arena("tblt_sched_get"); common::ObAddr inner_sql_exec_addr; common::ObArray running_sql_info; - running_ls_ids.reset(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(GCTX.sql_proxy_)); - } else if (OB_FAIL(ObDDLTaskRecordOperator::get_running_tasks_inner_sql(*GCTX.sql_proxy_, trace_id_, task_id_, snapshot_version_, inner_sql_exec_addr, arena, running_sql_info))) { - LOG_WARN("get running tasks inner sql fail", K(ret), K(trace_id_), K(task_id_), K(snapshot_version_), K(inner_sql_exec_addr), K(running_sql_info)); } else { - TCRLockGuard guard(lock_); - common::hash::ObHashMap>::iterator iter; - for (iter = running_ls_to_tablets_map_.begin(); iter != running_ls_to_tablets_map_.end() && OB_SUCC(ret); ++iter) { - share::ObLSID &ls_id = iter->first; - ObArray &tablet_queue = iter->second; + { + TCRLockGuard guard(lock_); + if (OB_FAIL(running_tablet_queue.assign(running_tablets_))) { + LOG_WARN("ObArray assign failed", K(ret), K_(running_tablets)); + } + } + if (OB_FAIL(ret)) { + } else if (running_tablet_queue.empty()) { + // do nothing, the running task has finished and reported + } else if (OB_FAIL(ObDDLTaskRecordOperator::get_running_tasks_inner_sql( + *GCTX.sql_proxy_, trace_id_, task_id_, snapshot_version_, inner_sql_exec_addr, arena, running_sql_info))) { + LOG_WARN("get running tasks inner sql fail", K(ret), K(trace_id_), K(task_id_), K(snapshot_version_), K(inner_sql_exec_addr), K(running_sql_info)); + } else { ObArray partition_names; - if (OB_FAIL(ObDDLUtil::get_index_table_batch_partition_names(ref_data_table_id_, table_id_, tablet_queue, arena, partition_names))) { - LOG_WARN("fail to get index table batch partition names", K(ret), K(ref_data_table_id_), K(table_id_), K(tablet_queue), K(partition_names)); - } else { - bool is_running_status = false; - for (int64_t i = 0; i < partition_names.count() && OB_SUCC(ret); i++) { - is_running_status = false; - for (int64_t j = 0; j < running_sql_info.count() && OB_SUCC(ret); j++) { - if (OB_FAIL(ObDDLUtil::check_target_partition_is_running(running_sql_info.at(j), partition_names.at(i), arena, is_running_status))) { - LOG_WARN("fail to check target partition is running", K(ret), K(running_sql_info.at(j)), K(partition_names.at(i)), K(is_running_status)); - } else if (is_running_status) { - break; - } - } - if (is_running_status) { + bool is_running_status = false; + if (OB_FAIL(ObDDLUtil::get_index_table_batch_partition_names( + ref_data_table_id_, table_id_, running_tablet_queue, arena, partition_names))) { + LOG_WARN("fail to get index table batch partition names", K(ret), K(ref_data_table_id_), K(table_id_), K(running_tablet_queue), K(partition_names)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < partition_names.count(); i++) { + is_running_status = false; + for (int64_t j = 0; OB_SUCC(ret) && j < running_sql_info.count(); j++) { + if (OB_FAIL(ObDDLUtil::check_target_partition_is_running( + running_sql_info.at(j), partition_names.at(i), arena, is_running_status))) { + LOG_WARN("fail to check target partition is running", K(ret), K(running_sql_info.at(j)), K(partition_names.at(i)), K(is_running_status)); + } else if (is_running_status) { break; } } - if (is_running_status && OB_SUCC(ret)) { - if (OB_FAIL(running_ls_ids.push_back(ls_id))) { - LOG_WARN("ObArray assign failed", K(ret), K(ls_id)); + if (is_running_status) { + break; + } + } + if (OB_SUCC(ret) && !is_running_status) { + if (OB_FAIL(tablet_finished_map.create(running_tablet_queue.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { + LOG_WARN("fail to create tablet checksum status map", K(ret), K(running_tablet_queue.count())); + } else if (OB_FAIL(ObDDLChecksumOperator::get_local_index_tablet_finish_status(ref_data_table_id_, + table_id_, + task_id_, + running_tablet_queue, + *GCTX.sql_proxy_, + tablet_finished_map))) { + LOG_WARN("fail to get tablet checksum status", K(ret), K(table_id_), K(task_id_), K(running_tablet_queue)); + } else { + bool is_finished_status = true; + for (int64_t i = 0; i < running_tablet_queue.count() && OB_SUCC(ret); i++) { + if (OB_FAIL(tablet_finished_map.get_refactored(running_tablet_queue.at(i).id(), is_finished_status))) { + if (OB_HASH_NOT_EXIST == ret) { + LOG_ERROR("tablet checksum is not exist", K(ret), K(running_tablet_queue.at(i)), K(is_finished_status)); + is_finished_status = false; + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("fail to get refactored", K(ret), K(running_tablet_queue.at(i)), K(is_finished_status)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(confirm_batch_tablets_status(-1, is_finished_status, running_tablet_queue))) { + LOG_WARN("fail to confirm batch tablets status", K(ret), K(is_finished_status), K(running_tablet_queue)); + } } } } @@ -739,46 +663,6 @@ int ObDDLTabletScheduler::get_session_running_lsid(ObIArray &runn return ret; } -int ObDDLTabletScheduler::get_target_running_ls_tablets(const share::ObLSID &ls_id, ObIArray &tablets) -{ - int ret = OB_SUCCESS; - ObArray tablet_queue; - tablets.reset(); - TCRLockGuard guard(lock_); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(running_ls_to_tablets_map_.get_refactored(ls_id, tablet_queue))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) { // tasks in this ls have finished and reported reply - LOG_WARN("ls id is not exist in map", K(ret), K(ls_id), K(tablets)); - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get tablet queue from refactored", K(ret), K(ls_id), K(tablets)); - } - } else if (OB_FAIL(tablets.assign(tablet_queue))) { - LOG_WARN("ObArray assign failed", K(ret), K(tablet_queue)); - } - return ret; -} - -int ObDDLTabletScheduler::get_potential_finished_lsid(const ObIArray &running_ls_ids_now, ObIArray &potential_finished_ls_ids) -{ - int ret = OB_SUCCESS; - ObArray current_running_ls_ids; - TCWLockGuard guard(lock_); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(common::get_difference(running_task_ls_ids_before_, running_ls_ids_now, potential_finished_ls_ids))) { - LOG_WARN("get difference failed", K(ret), K(running_task_ls_ids_before_), K(running_ls_ids_now), K(potential_finished_ls_ids)); - } else if (OB_FAIL(common::get_difference(running_task_ls_ids_before_, potential_finished_ls_ids, current_running_ls_ids))) { - LOG_WARN("get difference failed", K(ret), K(running_task_ls_ids_before_), K(potential_finished_ls_ids), K(current_running_ls_ids)); - } else if (OB_FAIL(running_task_ls_ids_before_.assign(current_running_ls_ids))) { - LOG_WARN("ObArray assign failed", K(ret), K(current_running_ls_ids)); - } - return ret; -} - int ObDDLTabletScheduler::determine_if_need_to_send_new_task(bool &status) { int ret = OB_SUCCESS; @@ -787,78 +671,23 @@ int ObDDLTabletScheduler::determine_if_need_to_send_new_task(bool &status) if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (running_ls_to_tablets_map_.size() == all_ls_to_tablets_map_.size()) { - status = false; - } else if (running_ls_to_tablets_map_.size() < all_ls_to_tablets_map_.size()) { - status = true; } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sub map size bigger than map size", K(ret), K(status), K(running_ls_to_tablets_map_.size()), K(all_ls_to_tablets_map_.size())); + status = running_tablets_.empty() && !all_tablets_.empty(); } return ret; } -int ObDDLTabletScheduler::check_target_ls_tasks_completion_status(const share::ObLSID &ls_id) +bool ObDDLTabletScheduler::is_all_tasks_finished() { - int ret = OB_SUCCESS; - ObArray running_tablet_queue; - common::hash::ObHashMap tablet_finished_map; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if ((OB_UNLIKELY(!ls_id.is_valid()))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("the parameters is invalid", K(ret), K(ls_id)); - } else if (OB_FAIL(get_target_running_ls_tablets(ls_id, running_tablet_queue))) { - LOG_WARN("fail to get target running ls tablets", K(ret), K(ls_id),K(running_tablet_queue)); - } else if (OB_UNLIKELY(running_tablet_queue.count() < 1)) { - // do nothing, the ls tasks have finished and reported - } else if (OB_FAIL(tablet_finished_map.create(running_tablet_queue.count(), ObModIds::OB_SSTABLE_CREATE_INDEX))) { - LOG_WARN("fail to create tablet checksum status map", K(ret), K(running_tablet_queue.count())); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(GCTX.sql_proxy_)); - } else if (OB_FAIL(ObDDLChecksumOperator::get_local_index_tablet_finish_status(ref_data_table_id_, - table_id_, - task_id_, - running_tablet_queue, - *GCTX.sql_proxy_, - tablet_finished_map))) { - LOG_WARN("fail to get tablet checksum status", K(ret), K(table_id_), K(task_id_), K(running_tablet_queue)); - } else { - bool is_finished_status = true; - for (int64_t i = 0; i < running_tablet_queue.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(tablet_finished_map.get_refactored(running_tablet_queue.at(i).id(), is_finished_status))) { - if (OB_HASH_NOT_EXIST == ret) { - LOG_ERROR("tablet checksum is not exist", K(ret), K(running_tablet_queue.at(i)), K(is_finished_status)); - is_finished_status = false; - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("fail to get refactored", K(ret), K(running_tablet_queue.at(i)), K(is_finished_status)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(confirm_batch_tablets_status(-1, is_finished_status, ls_id, running_tablet_queue))) { - LOG_WARN("fail to confirm batch tablets status", K(ret), K(is_finished_status), K(running_tablet_queue)); - } - } - } - return ret; -} - - bool ObDDLTabletScheduler::is_all_tasks_finished() - { TCRLockGuard guard(lock_); - return all_ls_to_tablets_map_.size() < 1; - } + return all_tablets_.empty(); +} - bool ObDDLTabletScheduler::is_running_tasks_before_finished() - { +bool ObDDLTabletScheduler::is_recovered_running_task() +{ TCRLockGuard guard(lock_); - return running_task_ls_ids_before_.count() < 1; - } + return !running_tablets_.empty() && -1 == running_execution_id_; +} void ObDDLTabletScheduler::destroy() { @@ -870,32 +699,8 @@ void ObDDLTabletScheduler::destroy() snapshot_version_ = 0; trace_id_.reset(); all_tablets_.reset(); - running_task_ls_ids_before_.reset(); - all_ls_to_tablets_map_.destroy(); - running_ls_to_tablets_map_.destroy(); - ls_location_map_.destroy(); - running_ls_to_execution_id_.destroy(); + running_tablets_.reset(); + running_execution_id_ = -1; tablet_id_to_data_size_.destroy(); tablet_id_to_data_row_cnt_.destroy(); - tablet_scheduled_times_statistic_.destroy(); -} - -int ObTabletIdUpdater::operator() (common::hash::HashMapPair> &entry) { - int ret = OB_SUCCESS; - for (int64_t i = 0; i < tablets_->count() && OB_SUCC(ret); i++) { - for (int64_t j = 0; j < entry.second.count() && OB_SUCC(ret); j++) { - if (tablets_->at(i) == entry.second.at(j)) { - if (OB_FAIL(entry.second.remove(j))) { - LOG_WARN("failed to remove tablet id", K(ret), K(i), K(j), K(entry.second), KP(tablets_)); - } - break; - } - } - } - LOG_INFO("remove tablet ids from hash map", K(entry), KP(tablets_)); - return ret; -} - -bool HashMapEraseIfNull::operator() (common::hash::HashMapPair> &entry) { - return entry.second.count() == 0; } diff --git a/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.h b/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.h index bee37c377..01468f1b1 100644 --- a/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.h +++ b/src/rootserver/ddl_task/ob_ddl_tablet_scheduler.h @@ -39,29 +39,23 @@ class ObDDLTabletScheduler final int get_next_batch_tablets(const bool is_ddl_retryable, int64_t ¶llelism, int64_t &new_execution_id, - share::ObLSID &ls_id, - common::ObAddr &leader_addr, ObIArray &tablets); - int confirm_batch_tablets_status(const int64_t execution_id, const bool finish_status, const share::ObLSID &ls_id, const ObIArray &tablets); + int confirm_batch_tablets_status(const int64_t execution_id, const bool finish_status, const ObIArray &tablets); TO_STRING_KV(K_(is_inited), K_(table_id), K_(ref_data_table_id), - K_(task_id), K_(parallelism), K_(snapshot_version), K_(trace_id), K_(all_tablets), K_(running_task_ls_ids_before)); + K_(task_id), K_(parallelism), K_(snapshot_version), K_(trace_id), K_(all_tablets), K_(running_tablets)); private: int get_next_parallelism(int64_t ¶llelism); - int get_unfinished_tablets(const share::ObDDLType task_type, const bool ddl_can_retry, int64_t &new_execution_id, share::ObLSID &ls_id, common::ObAddr &leader_addr, ObIArray &tablets); - int get_to_be_scheduled_tablets(share::ObLSID &ls_id, common::ObAddr &leader_addr, ObIArray &tablets); + int get_unfinished_tablets(const share::ObDDLType task_type, const bool ddl_can_retry, int64_t &new_execution_id, ObIArray &tablets); + int get_to_be_scheduled_tablets(ObIArray &tablets); int calculate_candidate_tablets(const uint64_t left_space_size, const ObIArray &in_tablets, ObIArray &out_tablets); - int get_session_running_lsid(ObIArray &running_ls_ids); - int get_target_running_ls_tablets(const share::ObLSID &ls_id, ObIArray &tablets); - int get_potential_finished_lsid(const ObIArray &running_ls_ids_now, ObIArray &potential_finished_ls_ids); int determine_if_need_to_send_new_task(bool &status); - int check_target_ls_tasks_completion_status(const share::ObLSID &ls_id); + int check_running_task_completion_status(); + int remove_finished_tablets_(const ObIArray &tablets); bool is_all_tasks_finished(); - bool is_running_tasks_before_finished(); - int refresh_ls_location_map(); - int push_tablet_execution_id(share::ObDDLType task_type, - const bool ddl_can_retry, - const common::ObIArray &tablets, - int64_t &new_task_execution_id); + int push_tablet_execution_id(const bool ddl_can_retry, + const common::ObIArray &tablets, + int64_t &new_task_execution_id); + bool is_recovered_running_task(); int push_task_execution_id(int64_t &new_task_execution_id); void destroy(); private: @@ -72,40 +66,15 @@ class ObDDLTabletScheduler final int64_t parallelism_; int64_t snapshot_version_; common::ObCurTraceId::TraceId trace_id_; - common::TCRWLock lock_; // this lock is used to protect read and write operations of class members: running_task_ls_ids_before_、 all_ls_to_tablets_map_、 running_ls_to_tablets_map_、 running_ls_to_execution_id_, to avoid conflicts between ddl_builder task and ddl_scheduler task. + common::TCRWLock lock_; // protects pending/running tablet queues against ddl builder and scheduler races. ObRootService *root_service_; ObArray all_tablets_; - ObArray running_task_ls_ids_before_; // this is the used lsid array where the tablets is located when init ObDDLTabletScheduler; - common::hash::ObHashMap> all_ls_to_tablets_map_; - common::hash::ObHashMap> running_ls_to_tablets_map_; - common::hash::ObHashMap ls_location_map_; - common::hash::ObHashMap running_ls_to_execution_id_; + ObArray running_tablets_; + int64_t running_execution_id_; common::hash::ObHashMap tablet_id_to_data_size_; common::hash::ObHashMap tablet_id_to_data_row_cnt_; - common::hash::ObHashMap tablet_scheduled_times_statistic_; common::hash::ObHashMap tablet_id_to_execution_id_map_; }; - -class ObTabletIdUpdater final -{ -public: - ObTabletIdUpdater(const ObIArray &tablets) : tablets_(&tablets) {}; - ~ObTabletIdUpdater() {}; - int operator() (common::hash::HashMapPair> &entry); -private: - const ObIArray *tablets_; - DISALLOW_COPY_AND_ASSIGN(ObTabletIdUpdater); -}; - -class HashMapEraseIfNull final -{ -public: - HashMapEraseIfNull() {}; - ~HashMapEraseIfNull() {}; - bool operator() (common::hash::HashMapPair> &entry); -private: - DISALLOW_COPY_AND_ASSIGN(HashMapEraseIfNull); -}; } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index 20f89ebe9..bd4c7e504 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -393,18 +393,6 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st case DDL_DROP_VEC_IVFPQ_INDEX: ddl_type_str = "drop vec index"; break; - case DDL_MVIEW_COMPLETE_REFRESH: - ddl_type_str = "mview complete refresh"; - break; - case DDL_CREATE_MVIEW: - ddl_type_str = "create mview"; - break; - case DDL_CREATE_MLOG: - ddl_type_str = "create materialized view log"; - break; - case DDL_DROP_MLOG: - ddl_type_str = "drop materialized view log"; - break; case DDL_AUTO_SPLIT_BY_RANGE: ddl_type_str = "auto split by range"; break; @@ -423,9 +411,6 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION: ddl_type_str = "partition split recovery table redefinition"; break; - case DDL_REPLACE_MLOG: - ddl_type_str = "replace materialized view log"; - break; default: ret = OB_ERR_UNEXPECTED; } @@ -1055,10 +1040,10 @@ int64_t ObDDLTask::get_execution_id() const // calculate task execution id or tablet execution id // @param execution_id: task execution id or tablet execution id // @param next_execution_id: next task execution id or next tablet execution id -int ObDDLTask::calc_next_execution_id(int64_t execution_id, const ObDDLType ddl_type, const bool ddl_can_retry, int64_t &next_execution_id) +int ObDDLTask::calc_next_execution_id(int64_t execution_id, const bool ddl_can_retry, int64_t &next_execution_id) { int ret = OB_SUCCESS; - if (ObDDLUtil::use_idempotent_mode() || ObDDLUtil::is_mview_not_retryable(ddl_type)) { + if (ObDDLUtil::use_idempotent_mode()) { if (1 == execution_id && !ddl_can_retry) { ret = OB_TASK_EXPIRED; } else { @@ -1094,7 +1079,7 @@ int ObDDLTask::push_task_execution_id(const int64_t task_id, const ObDDLType ddl if (OB_FAIL(ObDDLTaskRecordOperator::select_for_update(trans, task_id, task_status, task_execution_id, ret_code, unused_snapshot_ver))) { LOG_WARN("select for update failed", K(ret), K(task_id)); } else { - if (OB_FAIL(calc_next_execution_id(task_execution_id, ddl_type, ddl_can_retry, new_task_execution_id))) { + if (OB_FAIL(calc_next_execution_id(task_execution_id, ddl_can_retry, new_task_execution_id))) { LOG_WARN("calc next execution id failed", K(ret), K(task_execution_id), K(ddl_type), K(ddl_can_retry), K(new_task_execution_id)); } else if (OB_FAIL(ObDDLTaskRecordOperator::update_execution_id(trans, task_id, new_task_execution_id))) { LOG_WARN("update execution id failed", K(ret)); @@ -1324,40 +1309,24 @@ void ObDDLWaitTransEndCtx::reset() struct SendItem final { public: - bool operator < (const SendItem &other) const { return leader_addr_ < other.leader_addr_; } - TO_STRING_KV(K_(leader_addr), K_(ls_id), K_(tablet_id), KP_(other_info)); + TO_STRING_KV(K_(tablet_id), KP_(other_info)); public: - ObAddr leader_addr_; - ObLSID ls_id_; ObTabletID tablet_id_; void *other_info_; }; -int group_tablets_leader_addr(const ObIArray &tablet_ids, ObLocationService *location_service, ObArray &group_items) +int build_send_items(const ObIArray &tablet_ids, ObArray &send_items) { int ret = OB_SUCCESS; - group_items.reuse(); - if (OB_UNLIKELY(nullptr == location_service)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_ids.count())); - } else { - int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - if (OB_FAIL(group_items.reserve(tablet_ids.count()))) { - LOG_WARN("reserve send array failed", K(ret), K(tablet_ids.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - const ObTabletID &tablet_id = tablet_ids.at(i); - SendItem item; - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, - tablet_id, - rpc_timeout, - item.ls_id_, - item.leader_addr_))) { - LOG_WARN("get tablet leader addr failed", K(ret)); - } else if (FALSE_IT(item.tablet_id_ = tablet_id)) { - } else if (OB_FAIL(group_items.push_back(item))) { - LOG_WARN("push back send item failed", K(ret), K(item)); - } + send_items.reuse(); + if (OB_FAIL(send_items.reserve(tablet_ids.count()))) { + LOG_WARN("reserve send array failed", K(ret), K(tablet_ids.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + SendItem item; + item.tablet_id_ = tablet_ids.at(i); + if (OB_FAIL(send_items.push_back(item))) { + LOG_WARN("push back send item failed", K(ret), K(item)); } } return ret; @@ -1374,7 +1343,7 @@ int check_trans_end(const ObArray &send_array, int ret = OB_SUCCESS; ret_array.reuse(); snapshot_array.reuse(); - hash::ObHashMap result_map; + hash::ObHashMap result_map; pending_tx_id.reset(); if (OB_UNLIKELY(send_array.empty())) { ret = OB_INVALID_ARGUMENT; @@ -1386,10 +1355,9 @@ int check_trans_end(const ObArray &send_array, arg.tablets_.reuse(); for (int64_t i = 0; OB_SUCC(ret) && i < send_array.count(); ++i) { const SendItem &send_item = send_array.at(i); - ObLSTabletPair ls_tablet_pair; - ls_tablet_pair.ls_id_ = send_item.ls_id_; - ls_tablet_pair.tablet_id_ = send_item.tablet_id_; - if (OB_FAIL(arg.tablets_.push_back(ls_tablet_pair))) { + ObTabletPair tablet_pair; + tablet_pair.tablet_id_ = send_item.tablet_id_; + if (OB_FAIL(arg.tablets_.push_back(tablet_pair))) { LOG_WARN("push back send item failed", K(ret), K(i), K(send_item)); } } @@ -1402,7 +1370,7 @@ int check_trans_end(const ObArray &send_array, LOG_WARN("result count mismatch", K(ret), K(arg.tablets_.count()), K(result.results_.count())); } else { for (int64_t j = 0; OB_SUCC(ret) && j < result.results_.count(); ++j) { - const ObLSTabletPair &send_item = arg.tablets_.at(j); + const ObTabletPair &send_item = arg.tablets_.at(j); const ObCheckTransElapsedResult &result_item = result.results_.at(j); if (OB_FAIL(result_map.set_refactored(send_item, result_item))) { LOG_WARN("insert into result map failed", K(ret)); @@ -1425,12 +1393,11 @@ int check_trans_end(const ObArray &send_array, LOG_WARN("reserve snapshot array failed", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < send_array.count(); ++i) { - const SendItem &send_item = send_array.at(i); - ObLSTabletPair ls_tablet_pair; - ls_tablet_pair.ls_id_ = send_item.ls_id_; - ls_tablet_pair.tablet_id_ = send_item.tablet_id_; - ObCheckTransElapsedResult result_item; - if (OB_FAIL(result_map.get_refactored(ls_tablet_pair, result_item))) { + const SendItem &send_item = send_array.at(i); + ObTabletPair tablet_pair; + tablet_pair.tablet_id_ = send_item.tablet_id_; + ObCheckTransElapsedResult result_item; + if (OB_FAIL(result_map.get_refactored(tablet_pair, result_item))) { LOG_WARN("get result failed", K(ret), K(send_item)); } else if (OB_FAIL(ret_array.push_back(result_item.ret_code_))) { LOG_WARN("push back return code failed", K(ret), K(send_item), K(result_item)); @@ -1450,7 +1417,6 @@ int ObDDLWaitTransEndCtx::check_schema_trans_end( const common::ObIArray &tablet_ids, common::ObIArray &ret_array, common::ObIArray &snapshot_array, - ObLocationService *location_service, const bool need_wait_trans_end, const bool need_write_defensive) { @@ -1458,11 +1424,11 @@ int ObDDLWaitTransEndCtx::check_schema_trans_end( ret_array.reset(); snapshot_array.reset(); ObArray send_array; - if (OB_UNLIKELY(schema_version <= 0 || tablet_ids.count() <= 0 || nullptr == location_service)) { + if (OB_UNLIKELY(schema_version <= 0 || tablet_ids.count() <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_version), K(tablet_ids.count()), KP(location_service)); - } else if (OB_FAIL(group_tablets_leader_addr(tablet_ids, location_service, send_array))) { - LOG_WARN("group tablet by leader addr failed", K(ret), K(tablet_ids.count())); + LOG_WARN("invalid argument", K(ret), K(schema_version), K(tablet_ids.count())); + } else if (OB_FAIL(build_send_items(tablet_ids, send_array))) { + LOG_WARN("build tablet send items failed", K(ret), K(tablet_ids.count())); } else if (need_write_defensive && !is_write_defensive_done_) { if (OB_FAIL(do_write_defensive(ddl_task_id_, ddl_task_status_, tablet_ids, schema_version))) { LOG_WARN("failed to do write defense", K(ret), K(ddl_task_id_), K(ddl_task_status_), K(tablet_ids.count())); @@ -1535,7 +1501,6 @@ int ObDDLWaitTransEndCtx::do_write_defensive(const int64_t ddl_task_id, int ObDDLWaitTransEndCtx::check_sstable_trans_end(const int64_t sstable_exist_ts, const common::ObIArray &tablet_ids, - ObLocationService *location_service, common::ObIArray &ret_array, common::ObIArray &snapshot_array) { @@ -1543,13 +1508,11 @@ int ObDDLWaitTransEndCtx::check_sstable_trans_end(const int64_t sstable_exist_ts ret_array.reset(); snapshot_array.reset(); ObArray send_array; - if (OB_UNLIKELY(sstable_exist_ts <= 0 || tablet_ids.count() <= 0 - || nullptr == location_service)) { + if (OB_UNLIKELY(sstable_exist_ts <= 0 || tablet_ids.count() <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(sstable_exist_ts), K(tablet_ids.count()), - KP(location_service)); - } else if (OB_FAIL(group_tablets_leader_addr(tablet_ids, location_service, send_array))) { - LOG_WARN("group tablet by leader addr failed", K(ret), K(tablet_ids.count())); + LOG_WARN("invalid argument", K(ret), K(sstable_exist_ts), K(tablet_ids.count())); + } else if (OB_FAIL(build_send_items(tablet_ids, send_array))) { + LOG_WARN("build tablet send items failed", K(ret), K(tablet_ids.count())); } else { obcall::ObCheckModifyTimeElapsedArg arg; @@ -1599,7 +1562,7 @@ int ObDDLWaitTransEndCtx::try_wait(bool &is_trans_end, int64_t &snapshot_version case WaitTransType::WAIT_SCHEMA_TRANS: { if (OB_FAIL(check_schema_trans_end( wait_version_, need_check_tablets, ret_codes, tmp_snapshots, - GCTX.location_service_, need_wait_trans_end, true/*need_write_defensive*/))) { + need_wait_trans_end, true/*need_write_defensive*/))) { LOG_WARN("check schema transactions elapsed failed", K(ret), K(wait_type_), K(wait_version_)); } break; @@ -1607,14 +1570,14 @@ int ObDDLWaitTransEndCtx::try_wait(bool &is_trans_end, int64_t &snapshot_version case WaitTransType::WAIT_SCHEMA_TRANS_WITHOUT_WRITE_DEFENSIVE: { if (OB_FAIL(check_schema_trans_end( wait_version_, need_check_tablets, ret_codes, tmp_snapshots, - GCTX.location_service_, need_wait_trans_end, false/*need_write_defensive*/))) { + need_wait_trans_end, false/*need_write_defensive*/))) { LOG_WARN("check schema transactions elapsed failed", K(ret), K(wait_type_), K(wait_version_)); } break; } case WaitTransType::WAIT_SSTABLE_TRANS: { if (OB_FAIL(check_sstable_trans_end(wait_version_, need_check_tablets, - GCTX.location_service_, ret_codes, tmp_snapshots))) { + ret_codes, tmp_snapshots))) { LOG_WARN("check sstable transactions elapsed failed", K(ret), K(wait_type_), K(wait_version_)); } break; @@ -1683,7 +1646,6 @@ int ObDDLWaitTransEndCtx::get_snapshot(int64_t &snapshot_version) { int ret = OB_SUCCESS; snapshot_version = 0; - bool is_external_consistent = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); @@ -1978,32 +1940,26 @@ int ObDDLWaitColumnChecksumCtx::refresh_zombie_task() return ret; } -int send_batch_calc_rpc(const ObAddr &leader_addr, - const ObCalcColumnChecksumRequestArg &arg, +int send_batch_calc_rpc(const ObCalcColumnChecksumRequestArg &arg, ObCalcColumnChecksumRequestRes &res, ObIArray &send_array, - const int64_t group_start_idx, - const int64_t group_end_idx, common::SpinRWLock &item_lock, int64_t &send_succ_count) { int ret = OB_SUCCESS; - int64_t rpc_timeout = 0; const int64_t tablet_count = arg.calc_items_.count(); - if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tablet_count, rpc_timeout))) { - LOG_WARN("get ddl rpc timeout failed", K(ret)); - } else if (OB_FAIL(GCTX.ob_service_->calc_column_checksum_request(arg, res))) { - LOG_WARN("send rpc failed", K(ret), K(arg), K(leader_addr)); - } else if (res.ret_codes_.count() != arg.calc_items_.count() || res.ret_codes_.count() != (group_end_idx - group_start_idx)) { + if (OB_FAIL(GCTX.ob_service_->calc_column_checksum_request(arg, res))) { + LOG_WARN("calculate column checksum request failed", K(ret), K(arg)); + } else if (res.ret_codes_.count() != tablet_count || send_array.count() != tablet_count) { ret = OB_ERR_UNEXPECTED; LOG_WARN("return codes count not match the argument", K(ret), K(arg.calc_items_.count()), - K(res.ret_codes_.count()), "group_count", group_end_idx - group_start_idx); + K(res.ret_codes_.count()), K(send_array.count())); } else { LOG_INFO("send checksum validation task", K(arg)); SpinWLockGuard guard(item_lock); - for (int64_t j = group_start_idx, k = 0; j < group_end_idx; ++j, ++k) { // ignore ret - PartitionColChecksumStat *item = reinterpret_cast(send_array.at(j).other_info_); - int ret_code = res.ret_codes_.at(k); + for (int64_t i = 0; i < tablet_count; ++i) { // ignore ret + PartitionColChecksumStat *item = reinterpret_cast(send_array.at(i).other_info_); + int ret_code = res.ret_codes_.at(i); if (OB_SUCCESS == ret_code) { item->snapshot_ = arg.snapshot_version_; item->col_checksum_stat_ = CCS_INVALID; @@ -2023,88 +1979,43 @@ int ObDDLWaitColumnChecksumCtx::send_calc_rpc(int64_t &send_succ_count) { int ret = OB_SUCCESS; send_succ_count = 0; - share::ObLocationService *location_service = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); - } else if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(location_service)); } else { - ObLSID ls_id; - int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); ObArray send_array; + ObCalcColumnChecksumRequestArg arg; + ObCalcColumnChecksumRequestRes res; + arg.task_id_ = task_id_; + arg.source_table_id_ = source_table_id_; + arg.target_table_id_ = target_table_id_; + arg.schema_version_ = schema_version_; + arg.execution_id_ = execution_id_; + arg.snapshot_version_ = snapshot_version_; + arg.user_parallelism_ = parallelism_; for (int64_t i = 0; OB_SUCC(ret) && i < stat_array_.count(); ++i) { PartitionColChecksumStat &item = stat_array_.at(i); - ObAddr leader_addr; if (!item.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("pkey invalid", K(ret), K(item)); } else if (item.snapshot_ <= 0 || CCS_NOT_MASTER == item.col_checksum_stat_) { - // only send rpc for the request not send or not master - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, item.tablet_id_, rpc_timeout, ls_id, leader_addr))) { - LOG_WARN("get tablet leader addr failed", K(ret)); - } else { - SendItem send_item; - send_item.leader_addr_ = leader_addr; - send_item.ls_id_ = ls_id; - send_item.tablet_id_ = item.tablet_id_; - send_item.other_info_ = reinterpret_cast(&item); - if (OB_FAIL(send_array.push_back(send_item))) { - LOG_WARN("push send array failed", K(ret)); - } + SendItem send_item; + send_item.tablet_id_ = item.tablet_id_; + send_item.other_info_ = reinterpret_cast(&item); + ObCalcColumnChecksumRequestArg::SingleItem calc_item; + calc_item.tablet_id_ = item.tablet_id_; + calc_item.calc_table_id_ = item.table_id_; + if (OB_FAIL(send_array.push_back(send_item))) { + LOG_WARN("push send array failed", K(ret)); + } else if (OB_FAIL(arg.calc_items_.push_back(calc_item))) { + LOG_WARN("push checksum item failed", K(ret), K(calc_item)); } } } - if (OB_SUCC(ret)) { - // group by leader addr and send batch rpc - lib::ob_sort(send_array.begin(), send_array.end()); - - ObAddr last_addr; - int64_t group_start_idx = 0; - ObCalcColumnChecksumRequestArg arg; - ObCalcColumnChecksumRequestRes res; - - arg.task_id_ = task_id_; - arg.source_table_id_ = source_table_id_; - arg.target_table_id_ = target_table_id_; - arg.schema_version_ = schema_version_; - arg.execution_id_ = execution_id_; - arg.snapshot_version_ = snapshot_version_; - arg.user_parallelism_ = parallelism_; - for (int64_t i = 0; OB_SUCC(ret) && i < send_array.count(); ++i) { - const SendItem &send_item = send_array.at(i); - if (send_item.leader_addr_ != last_addr) { - if (arg.calc_items_.count() > 0) { - if (OB_FAIL(send_batch_calc_rpc(last_addr, - arg, res, send_array, group_start_idx, i, lock_, send_succ_count))) { - LOG_WARN("send batch calc rpc failed", K(ret)); - } - } - if (OB_SUCC(ret)) { - arg.calc_items_.reuse(); - res.ret_codes_.reuse(); - last_addr = send_item.leader_addr_; - group_start_idx = i; - } - } - if (OB_SUCC(ret)) { - ObCalcColumnChecksumRequestArg::SingleItem calc_item; - calc_item.ls_id_ = send_item.ls_id_; - calc_item.tablet_id_ = send_item.tablet_id_; - calc_item.calc_table_id_ = reinterpret_cast(send_item.other_info_)->table_id_; - if (OB_FAIL(arg.calc_items_.push_back(calc_item))) { - LOG_WARN("push back send item failed", K(ret), K(i), K(send_item)); - } - } - } - if (OB_SUCC(ret) && arg.calc_items_.count() > 0) { - if (OB_FAIL(send_batch_calc_rpc(last_addr, - arg, res, send_array, group_start_idx, send_array.count(), lock_, send_succ_count))) { - LOG_WARN("send batch calc rpc failed", K(ret)); - } - } + if (OB_SUCC(ret) && !arg.calc_items_.empty() && + OB_FAIL(send_batch_calc_rpc(arg, res, send_array, lock_, send_succ_count))) { + LOG_WARN("calculate column checksum batch failed", K(ret)); } } return ret; @@ -3173,8 +3084,7 @@ int ObDDLTaskRecordOperator::check_has_conflict_ddl( } else if (task_record.task_id_ != task_id) { switch (ddl_type) { case ObDDLType::DDL_DROP_TABLE: { - if ((task_record.ddl_type_ == ObDDLType::DDL_DROP_INDEX - || task_record.ddl_type_ == ObDDLType::DDL_DROP_MLOG) + if (task_record.ddl_type_ == ObDDLType::DDL_DROP_INDEX && task_record.target_object_id_ != task_record.object_id_) { LOG_WARN("conflict with ddl", K(task_record)); has_conflict_ddl = true; @@ -3194,7 +3104,6 @@ int ObDDLTaskRecordOperator::check_has_conflict_ddl( case ObDDLType::DDL_TABLE_REDEFINITION: case ObDDLType::DDL_DIRECT_LOAD: case ObDDLType::DDL_DIRECT_LOAD_INSERT: - case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: case ObDDLType::DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION: has_conflict_ddl = true; break; @@ -3226,8 +3135,8 @@ int ObDDLTaskRecordOperator::check_rebuild_index_task_exist(const uint64_t data_ OB_INVALID_ID == index_table_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(index_table_id), K(data_table_id)); - } else if (OB_FAIL(sql_string.assign_fmt(" SELECT object_id, target_object_id, UNHEX(message) as message_unhex FROM %s WHERE ddl_type in (%d, %d)", - OB_ALL_DDL_TASK_STATUS_TNAME, ObDDLType::DDL_REBUILD_INDEX, ObDDLType::DDL_REPLACE_MLOG))) { + } else if (OB_FAIL(sql_string.assign_fmt(" SELECT object_id, target_object_id, UNHEX(message) as message_unhex FROM %s WHERE ddl_type = %d", + OB_ALL_DDL_TASK_STATUS_TNAME, ObDDLType::DDL_REBUILD_INDEX))) { LOG_WARN("assign sql string failed", K(ret)); } else { LOG_DEBUG("check_rebuild_index_task_exist target id", K(data_table_id), K(index_table_id)); @@ -3277,7 +3186,7 @@ int ObDDLTaskRecordOperator::check_rebuild_index_task_exist(const uint64_t data_ return ret; } -int ObDDLTaskRecordOperator::check_has_index_or_mlog_task( +int ObDDLTaskRecordOperator::check_has_index_task( common::ObISQLClient &proxy, const ObTableSchema &index_schema, const uint64_t data_table_id, @@ -3291,7 +3200,7 @@ int ObDDLTaskRecordOperator::check_has_index_or_mlog_task( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(data_table_id)); } else { - if (ObVecIndexBuildTask::is_rebuild_dense_vec_index_task(index_schema) || index_schema.is_mlog_table()) { + if (ObVecIndexBuildTask::is_rebuild_dense_vec_index_task(index_schema)) { ObArenaAllocator allocator(ObModIds::OB_SCHEMA); if (OB_FAIL(check_rebuild_index_task_exist(data_table_id, index_table_id, proxy, allocator, has_index_task))) { LOG_WARN("fail to check rebuild vec index task", K(ret), K(data_table_id), K(index_table_id)); @@ -3302,9 +3211,9 @@ int ObDDLTaskRecordOperator::check_has_index_or_mlog_task( ObSqlString sql_string; SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *result = NULL; - if (OB_FAIL(sql_string.assign_fmt("SELECT EXISTS(SELECT 1 FROM %s WHERE object_id = %lu AND target_object_id = %lu AND ddl_type IN (%d, %d, %d, %d, %d)) as has", - OB_ALL_DDL_TASK_STATUS_TNAME, data_table_id, index_table_id, ObDDLType::DDL_CREATE_INDEX, ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX, ObDDLType::DDL_DROP_INDEX, - ObDDLType::DDL_CREATE_MLOG, ObDDLType::DDL_DROP_MLOG))) { + if (OB_FAIL(sql_string.assign_fmt("SELECT EXISTS(SELECT 1 FROM %s WHERE object_id = %lu AND target_object_id = %lu AND ddl_type IN (%d, %d, %d)) as has", + OB_ALL_DDL_TASK_STATUS_TNAME, data_table_id, index_table_id, ObDDLType::DDL_CREATE_INDEX, + ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX, ObDDLType::DDL_DROP_INDEX))) { LOG_WARN("assign sql string failed", K(ret)); } else if (OB_FAIL(proxy.read(res, sql_string.ptr()))) { LOG_WARN("query ddl task record failed", K(ret), K(sql_string)); @@ -3322,7 +3231,7 @@ int ObDDLTaskRecordOperator::check_has_index_or_mlog_task( return ret; } -int ObDDLTaskRecordOperator::get_create_index_or_mlog_task_cnt( +int ObDDLTaskRecordOperator::get_create_index_task_cnt( common::ObISQLClient &proxy, const uint64_t data_table_id, int64_t &task_cnt) @@ -3335,9 +3244,9 @@ int ObDDLTaskRecordOperator::get_create_index_or_mlog_task_cnt( ObSqlString sql_string; SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *result = NULL; - if (OB_FAIL(sql_string.assign_fmt("SELECT COUNT(*) as cnt FROM %s WHERE object_id = %lu AND ddl_type IN (%d, %d, %d, %d, %d, %d)", - OB_ALL_DDL_TASK_STATUS_TNAME, data_table_id, - ObDDLType::DDL_CREATE_INDEX, ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX, ObDDLType::DDL_CREATE_MLOG, + if (OB_FAIL(sql_string.assign_fmt("SELECT COUNT(*) as cnt FROM %s WHERE object_id = %lu AND ddl_type IN (%d, %d, %d, %d, %d)", + OB_ALL_DDL_TASK_STATUS_TNAME, data_table_id, + ObDDLType::DDL_CREATE_INDEX, ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX, ObDDLType::DDL_CREATE_VEC_INDEX, ObDDLType::DDL_CREATE_MULTIVALUE_INDEX, ObDDLType::DDL_CREATE_FTS_INDEX))) { LOG_WARN("assign sql string failed", K(ret)); } else if (OB_FAIL(proxy.read(res, sql_string.ptr()))) { diff --git a/src/rootserver/ddl_task/ob_ddl_task.h b/src/rootserver/ddl_task/ob_ddl_task.h index 3451d6a24..8e257f793 100755 --- a/src/rootserver/ddl_task/ob_ddl_task.h +++ b/src/rootserver/ddl_task/ob_ddl_task.h @@ -114,13 +114,11 @@ struct ObDDLTaskInfo final public: ObDDLTaskInfo() : row_scanned_(0), row_inserted_(0), physical_row_count_(0) {} ~ObDDLTaskInfo() {} - TO_STRING_KV(K_(row_scanned), K_(row_inserted), K_(physical_row_count), K_(ls_id), K_(ls_leader_addr), K_(partition_ids)); + TO_STRING_KV(K_(row_scanned), K_(row_inserted), K_(physical_row_count), K_(partition_ids)); public: int64_t row_scanned_; int64_t row_inserted_; int64_t physical_row_count_; - share::ObLSID ls_id_; - common::ObAddr ls_leader_addr_; ObArray partition_ids_; }; @@ -450,13 +448,13 @@ class ObDDLTaskRecordOperator final const share::ObDDLType ddl_type, bool &has_conflict_ddl); - static int check_has_index_or_mlog_task( + static int check_has_index_task( common::ObISQLClient &proxy, const ObTableSchema &index_schema, const uint64_t data_table_id, bool &has_index_task); - static int get_create_index_or_mlog_task_cnt( + static int get_create_index_task_cnt( common::ObISQLClient &proxy, const uint64_t data_table_id, int64_t &task_cnt); @@ -558,14 +556,12 @@ class ObDDLWaitTransEndCtx const common::ObIArray &tablet_ids, common::ObIArray &ret_array, common::ObIArray &snapshot_array, - share::ObLocationService *location_service, const bool need_wait_trans_end, const bool need_write_defensive); // check if all transactions before a timestamp have ended int check_sstable_trans_end(const int64_t sstable_exist_ts, const common::ObIArray &tablet_ids, - share::ObLocationService *location_service, common::ObIArray &ret_array, common::ObIArray &snapshot_array); @@ -685,7 +681,6 @@ class ObDDLTask : public common::ObDLinkBase int64_t get_execution_id() const; static int calc_next_execution_id( int64_t execution_id, - const share::ObDDLType ddl_type, const bool ddl_can_retry, int64_t &next_execution_id); static int push_task_execution_id( diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index d8b23061f..460f850d8 100755 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -25,7 +25,6 @@ #include "rootserver/ddl_task/ob_sys_ddl_util.h" // for ObSysDDLSchedulerUtil #include "rootserver/ob_ddl_service_launcher.h" // for ObDDLServiceLauncher #include "rootserver/ob_root_service.h" -#include "share/schema/ob_mlog_info.h" using namespace oceanbase::rootserver; using namespace oceanbase::common; @@ -56,15 +55,12 @@ int ObIndexSSTableBuildTask::set_nls_format(const ObString &nls_date_format, return ret; } -int ObIndexSSTableBuildTask::set_addition_info(const share::ObLSID &ls_id, const common::ObAddr &ls_leader_addr, const ObIArray &index_partition_ids) +int ObIndexSSTableBuildTask::set_addition_info(const ObIArray &index_partition_ids) { int ret = OB_SUCCESS; if (index_partition_ids.count() > 0) { if (OB_FAIL(addition_info_.partition_ids_.assign(index_partition_ids))) { LOG_WARN("ObArray assign failed", K(ret), K(index_partition_ids)); - } else { - addition_info_.ls_id_ = ls_id; - addition_info_.ls_leader_addr_ = ls_leader_addr; } } return ret; @@ -178,11 +174,6 @@ int ObIndexSSTableBuildTask::process() session_param.nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP_TZ] = nls_timestamp_tz_format_; session_param.use_external_session_ = true; // means session id dispatched by session mgr - common::ObAddr *sql_exec_addr = nullptr; - if (inner_sql_exec_addr_.is_valid()) { - sql_exec_addr = &inner_sql_exec_addr_; - LOG_INFO("inner sql execute addr" , K(*sql_exec_addr)); - } int tmp_ret = OB_SUCCESS; user_sql_proxy = GCTX.ddl_sql_proxy_; DEBUG_SYNC(BEFORE_INDEX_SSTABLE_BUILD_TASK_SEND_SQL); @@ -198,7 +189,7 @@ int ObIndexSSTableBuildTask::process() } else if (OB_FAIL(DDL_SIM(task_id_, CREATE_INDEX_BUILD_SSTABLE_FAILED))) { LOG_WARN("ddl sim failure: create index build sstable failed", K(ret), K(task_id_)); } else if (OB_FAIL(user_sql_proxy->write(sql_string.ptr(), affected_rows, - ObCompatibilityMode::MYSQL_MODE, &session_param, sql_exec_addr))) { + ObCompatibilityMode::MYSQL_MODE, &session_param, nullptr))) { LOG_WARN("fail to execute build replica sql", K(ret)); } @@ -241,8 +232,7 @@ void ObIndexSSTableBuildTask::add_event_info(const int ret, const ObString &ddl_ "ret", ret, K_(trace_id), K_(task_id), - "table_id", table_id_buffer, - "sql_exec_addr", inner_sql_exec_addr_); + "table_id", table_id_buffer); } ObAsyncTask *ObIndexSSTableBuildTask::deep_copy(char *buf, const int64_t buf_size) const @@ -262,13 +252,12 @@ ObAsyncTask *ObIndexSSTableBuildTask::deep_copy(char *buf, const int64_t buf_siz parallelism_, is_partitioned_local_index_task_, root_service_, - inner_sql_exec_addr_, is_retryable_ddl_); if (OB_SUCCESS != (task->set_nls_format(nls_date_format_, nls_timestamp_format_, nls_timestamp_tz_format_))) { task->~ObIndexSSTableBuildTask(); task = nullptr; LOG_WARN_RET(OB_ALLOCATE_MEMORY_FAILED, "failed to set nls format"); - } else if (OB_SUCCESS != (task->set_addition_info(addition_info_.ls_id_, addition_info_.ls_leader_addr_, addition_info_.partition_ids_))) { + } else if (OB_SUCCESS != (task->set_addition_info(addition_info_.partition_ids_))) { task->~ObIndexSSTableBuildTask(); task = nullptr; LOG_WARN_RET(OB_ALLOCATE_MEMORY_FAILED, "failed to set index partition ids"); @@ -409,11 +398,6 @@ int ObIndexBuildTask::init( } else if (OB_ISNULL(index_schema)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("fail to get table schema", K(ret)); - } else if (OB_UNLIKELY((ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_) - && (!index_schema->is_mlog_table()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("index action is add_mlog but index schema is not mlog", - KR(ret), K(create_index_arg_.index_action_type_), K(index_schema->get_table_type())); } else { set_gmt_create(ObTimeUtility::current_time()); task_type_ = task_type; @@ -489,24 +473,6 @@ int ObIndexBuildTask::init(const ObDDLTaskRecord &task_record) LOG_WARN("ddl sim failure", K(task_record.task_id_)); } else if (OB_FAIL(deserialize_params_from_message(task_record.message_.ptr(), task_record.message_.length(), pos))) { LOG_WARN("deserialize params from message failed", KR(ret)); - } else if (ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_) { - ObSchemaGetterGuard schema_guard; - const ObTableSchema *index_schema = nullptr; - if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard( - schema_guard, schema_version))) { - LOG_WARN("fail to get schema guard", KR(ret), K(index_table_id), K(schema_version)); - } else if (OB_FAIL(schema_guard.check_formal_guard())) { - LOG_WARN("schema_guard is not formal", KR(ret), K(index_table_id)); - } else if (OB_FAIL(schema_guard.get_table_schema( index_table_id, index_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(index_table_id)); - } else if (OB_ISNULL(index_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("fail to get table schema", KR(ret)); - } else if (OB_UNLIKELY((!index_schema->is_mlog_table()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("index action is add_mlog but index schema is not mlog", - KR(ret), K(create_index_arg_.index_action_type_), K(index_schema->get_table_type())); - } } if (OB_SUCC(ret)) { task_type_ = task_record.ddl_type_; @@ -517,7 +483,7 @@ int ObIndexBuildTask::init(const ObDDLTaskRecord &task_record) snapshot_version_ = task_record.snapshot_version_; execution_id_ = task_record.execution_id_; task_status_ = static_cast(task_record.task_status_); - task_type_ = task_record.ddl_type_; // could be create index / mlog + task_type_ = task_record.ddl_type_; consensus_schema_version_ = task_record.consensus_schema_version_; if (ObDDLTaskStatus::VALIDATE_CHECKSUM == task_status_) { sstable_complete_ts_ = ObTimeUtility::current_time(); @@ -718,10 +684,7 @@ int ObIndexBuildTask::wait_trans_end() } if (snapshot_version_ > 0 || OB_FAIL(ret)) { - // a newly-created mlog is empty - ObDDLTaskStatus next_status = (ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_) ? - ObDDLTaskStatus::TAKE_EFFECT : ObDDLTaskStatus::REDEFINITION; - (void)switch_status(next_status, true, ret); + (void)switch_status(ObDDLTaskStatus::REDEFINITION, true, ret); LOG_INFO("wait_trans_end finished", K(ret), K(*this)); } return ret; @@ -888,11 +851,13 @@ int ObIndexBuildTask::reap_old_replica_build_task(bool &need_exec_new_inner_sql) } // construct ObIndexSSTableBuildTask build task -int ObIndexBuildTask::send_build_single_replica_request(const bool &is_partitioned_local_index_task, const int64_t ¶llelism, const int64_t &task_execution_id, const share::ObLSID &ls_id, const common::ObAddr &leader_addr, const ObIArray &index_partition_ids) +int ObIndexBuildTask::send_build_single_replica_request(const bool &is_partitioned_local_index_task, + const int64_t ¶llelism, + const int64_t &task_execution_id, + const ObIArray &index_partition_ids) { int ret = OB_SUCCESS; int64_t new_task_execution_id = task_execution_id; - common::ObAddr ls_leader_addr = leader_addr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObIndexBuildTask has not been inited", K(ret)); @@ -902,15 +867,9 @@ int ObIndexBuildTask::send_build_single_replica_request(const bool &is_partition ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(root_service_)); } else { - if (OB_FAIL(ObDDLUtil::get_sys_ls_leader_addr(GCONF.cluster_id, create_index_arg_.inner_sql_exec_addr_))) { - LOG_WARN("get sys ls leader addr fail", K(ret)); - ret = OB_SUCCESS; // ingore ret - } if (!is_partitioned_local_index_task) { if (OB_FAIL(ObDDLTask::push_task_execution_id(task_id_, task_type_, is_retryable_ddl_, new_task_execution_id))) { LOG_WARN("failed to fetch new execution id", K(ret), K(task_id_), K(task_type_), K(new_task_execution_id)); - } else { - ls_leader_addr = create_index_arg_.inner_sql_exec_addr_; } } else if (OB_UNLIKELY(index_partition_ids.count() < 1)) { ret = OB_INVALID_ARGUMENT; @@ -929,15 +888,12 @@ int ObIndexBuildTask::send_build_single_replica_request(const bool &is_partition parallelism, is_partitioned_local_index_task, root_service_, - ls_leader_addr, is_retryable_ddl_); - if (OB_FAIL(set_sql_exec_addr(ls_leader_addr))) { - LOG_WARN("failed to set sql execute addr", K(ret), K(ls_leader_addr)); - } else if (OB_FAIL(task.set_nls_format(create_index_arg_.nls_date_format_, + if (OB_FAIL(task.set_nls_format(create_index_arg_.nls_date_format_, create_index_arg_.nls_timestamp_format_, create_index_arg_.nls_timestamp_tz_format_))) { LOG_WARN("failed to set nls format", K(ret), K(create_index_arg_)); - } else if (OB_FAIL(task.set_addition_info(ls_id, ls_leader_addr, index_partition_ids))) { + } else if (OB_FAIL(task.set_addition_info(index_partition_ids))) { LOG_WARN("failed to set partition ids", K(ret), K(index_partition_ids)); } else if (OB_FAIL(root_service_->submit_ddl_single_replica_build_task(task))) { LOG_WARN("fail to submit task", K(ret), KPC(this)); @@ -955,8 +911,6 @@ int ObIndexBuildTask::wait_and_send_single_partition_replica_task(bool &state_fi int ret = OB_SUCCESS; int64_t parallelism = 0; int64_t execution_id = 0; - share::ObLSID ls_id; - common::ObAddr leader_addr; ObArray tablets; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -964,7 +918,7 @@ int ObIndexBuildTask::wait_and_send_single_partition_replica_task(bool &state_fi } else if (ObDDLTaskStatus::REDEFINITION != task_status_) { ret = OB_STATE_NOT_MATCH; LOG_WARN("task status not match", K(ret), K(task_status_)); - } else if (OB_FAIL(tablet_scheduler_.get_next_batch_tablets(is_retryable_ddl_, parallelism, execution_id, ls_id, leader_addr, tablets))) { + } else if (OB_FAIL(tablet_scheduler_.get_next_batch_tablets(is_retryable_ddl_, parallelism, execution_id, tablets))) { if (OB_UNLIKELY(ret == OB_EAGAIN)) { ret = OB_SUCCESS; } else if (OB_UNLIKELY(ret == OB_ITER_END)) { @@ -978,7 +932,7 @@ int ObIndexBuildTask::wait_and_send_single_partition_replica_task(bool &state_fi } } else if (OB_FAIL(serialize_and_update_message())) { LOG_WARN("fail to serialize and update message", K(ret)); - } else if (OB_FAIL(send_build_single_replica_request(true, parallelism, execution_id, ls_id, leader_addr, tablets))) { + } else if (OB_FAIL(send_build_single_replica_request(true, parallelism, execution_id, tablets))) { LOG_WARN("fail to send build single partition replica request", K(ret), K(parallelism), K(execution_id), K(tablets)); } return ret; @@ -1056,8 +1010,6 @@ int ObIndexBuildTask::wait_data_complement() int ret = OB_SUCCESS; bool state_finished = false; bool is_request_end = false; - share::ObLSID ls_id; - common::ObAddr leader_addr; ObArray index_partition_ids; if (OB_UNLIKELY(!is_inited_) || OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_NOT_INIT; @@ -1080,7 +1032,7 @@ int ObIndexBuildTask::wait_data_complement() } } else if (!need_exec_new_inner_sql) { state_finished = true; - } else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, ls_id, leader_addr, index_partition_ids))) { + } else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, index_partition_ids))) { LOG_WARN("fail to send build single replica request", K(ret), K(parallelism_), K(index_partition_ids)); } } @@ -1459,13 +1411,11 @@ int ObIndexBuildTask::update_complete_sstable_job_status( LOG_WARN("snapshot version not match", K(ret), K(addr), K(snapshot_version), K(snapshot_version_)); } else { if (is_create_partitioned_local_index()) { - if (OB_UNLIKELY(addition_info.partition_ids_.count() < 1 || !addition_info.ls_id_.is_valid() || !addition_info.ls_leader_addr_.is_valid())) { + if (OB_UNLIKELY(addition_info.partition_ids_.count() < 1)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(addition_info), K(ret_code)); - } else if (OB_FAIL(tablet_scheduler_.confirm_batch_tablets_status(execution_id, OB_SUCCESS == ret_code, addition_info.ls_id_, addition_info.partition_ids_))) { + } else if (OB_FAIL(tablet_scheduler_.confirm_batch_tablets_status(execution_id, OB_SUCCESS == ret_code, addition_info.partition_ids_))) { LOG_WARN("fail to confirm batch tablets status", K(ret), K(execution_id), K(ret_code), K(addition_info)); - } else if (OB_FAIL(remove_sql_exec_addr(addition_info.ls_leader_addr_))) { - LOG_WARN("failed to remove sql execute addr", K(ret), K(addition_info)); } } else if (OB_UNLIKELY(execution_id < execution_id_)) { ret = OB_TASK_EXPIRED; @@ -1538,9 +1488,6 @@ int ObIndexBuildTask::enable_index() ret = OB_ERR_UNEXPECTED; LOG_WARN("index status not match", K(ret), K(index_table_id_), K(index_status)); } - } else if ((ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_) - && OB_FAIL(update_mlog_last_purge_scn())) { - LOG_WARN("failed to update mlog last purge scn", KR(ret)); } else if (OB_FAIL(update_index_status_in_schema(*index_schema, INDEX_STATUS_AVAILABLE, schema_guard))) { LOG_WARN("fail to try notify index take effect", K(ret), K(index_table_id_)); } else { @@ -1683,7 +1630,7 @@ int ObIndexBuildTask::clean_on_failed() } else if (!wait_trans_ctx_.is_inited() && OB_FAIL(wait_trans_ctx_.init( task_id_, task_status_, object_id_, ObDDLWaitTransEndCtx::WaitTransType::WAIT_SCHEMA_TRANS_WITHOUT_WRITE_DEFENSIVE, index_schema->get_schema_version()))) { // Reasons for no need write densive: - // 1. setting write defensive mds may race with concurrent tablet status changes + // 1. setting write defensive mds may be concurrent with tablet state changes // 2. writes to index still concurrent with dropping index, because the write defensive is done on data tablets LOG_WARN("init wait_trans_ctx failed", K(ret), K(object_id_), K(index_table_id_)); } else if (OB_FAIL(wait_trans_ctx_.try_wait(is_trans_end, tmp_snapshot_version))) { @@ -1710,17 +1657,6 @@ int ObIndexBuildTask::clean_on_failed() } else if (index_schema->is_in_recyclebin()) { // index is already in recyclebin, skip get index name, use a fake one, this is just to pass IndexArg validity check index_name = "__fake"; - } else if (ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_) { - const ObString &data_table_name = data_table_schema->get_table_name_str(); - if (OB_FAIL(index_schema->get_mlog_name(index_name))) { - LOG_WARN("failed to get mlog name", KR(ret)); - } else if (OB_FALSE_IT(index_action_type = obcall::ObIndexArg::DROP_MLOG)) { - } else if ((0 == parent_task_id_) && create_index_arg_.ddl_stmt_str_.empty()) { - if (OB_FAIL(drop_index_sql.append_fmt("drop materialized view log on %.*s", - data_table_name.length(), data_table_name.ptr()))) { - LOG_WARN("failed to generate drop mlog sql", KR(ret), K(data_table_name)); - } - } } else if (OB_FAIL(index_schema->get_index_name(index_name))) { LOG_WARN("get index name failed", K(ret)); } else if (0 == parent_task_id_) { @@ -1979,42 +1915,6 @@ int64_t ObIndexBuildTask::get_serialize_param_size() const + tablet_scheduler_.get_serialize_size(); } -int ObIndexBuildTask::update_mlog_last_purge_scn() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - ObMySQLTransaction trans; - ObMLogInfo mlog_info; - if (OB_FAIL(trans.start(GCTX.sql_proxy_))) { - LOG_WARN("failed to start trans", KR(ret)); - } else if (OB_FAIL(ObMLogInfo::fetch_mlog_info(trans, index_table_id_, mlog_info, true/*for_update*/))) { - LOG_WARN("failed to fetch mlog info", KR(ret)); - } else { - mlog_info.set_last_purge_scn(snapshot_version_); - mlog_info.set_last_purge_date(ObTimeUtility::current_time()); - mlog_info.set_last_purge_time(0); - mlog_info.set_last_purge_rows(0); - char trace_id_buf[OB_MAX_TRACE_ID_BUFFER_SIZE] = {'\0'}; - if (OB_FAIL(mlog_info.set_last_purge_trace_id(ObCurTraceId::get_trace_id_str(trace_id_buf, sizeof(trace_id_buf))))) { - LOG_WARN("failed to set last purge trace id", KR(ret)); - } else if (OB_FAIL(ObMLogInfo::update_mlog_last_purge_info(trans, mlog_info))) { - LOG_WARN("failed to update mlog last purge info", KR(ret), K(mlog_info)); - } - } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_ERROR("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - } - return ret; -} - int ObIndexBuildTask::serialize_and_update_message() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ddl_task/ob_index_build_task.h b/src/rootserver/ddl_task/ob_index_build_task.h index 580bd3395..add7afb4e 100644 --- a/src/rootserver/ddl_task/ob_index_build_task.h +++ b/src/rootserver/ddl_task/ob_index_build_task.h @@ -39,12 +39,11 @@ class ObIndexSSTableBuildTask : public share::ObAsyncTask const int64_t parallelism, const bool is_partitioned_local_index_task, ObRootService *root_service, - const common::ObAddr &inner_sql_exec_addr, const bool is_retryable_ddl) : task_id_(task_id), data_table_id_(data_table_id), dest_table_id_(dest_table_id), schema_version_(schema_version), snapshot_version_(snapshot_version), execution_id_(execution_id), trace_id_(trace_id), parallelism_(parallelism), is_partitioned_local_index_task_(is_partitioned_local_index_task), - allocator_("IdxSSTBuildTask"), root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr), is_retryable_ddl_(is_retryable_ddl) + allocator_("IdxSSTBuildTask"), root_service_(root_service), is_retryable_ddl_(is_retryable_ddl) { set_retry_times(0); } @@ -53,7 +52,7 @@ class ObIndexSSTableBuildTask : public share::ObAsyncTask int set_nls_format(const ObString &nls_date_format, const ObString &nls_timestamp_format, const ObString &nls_timestamp_tz_format); - int set_addition_info(const share::ObLSID &ls_id, const common::ObAddr &ls_leader_addr, const ObIArray &index_partition_ids); + int set_addition_info(const ObIArray &index_partition_ids); ObDDLTaskID get_ddl_task_id() { return ObDDLTaskID(task_id_); } virtual int process() override; virtual int64_t get_deep_copy_size() const override { return sizeof(*this); } @@ -79,7 +78,6 @@ class ObIndexSSTableBuildTask : public share::ObAsyncTask ObString nls_timestamp_format_; ObString nls_timestamp_tz_format_; ObRootService *root_service_; - common::ObAddr inner_sql_exec_addr_; ObDDLTaskInfo addition_info_; bool is_retryable_ddl_; @@ -154,15 +152,12 @@ class ObIndexBuildTask : public ObDDLTask int send_build_single_replica_request(const bool &is_partitioned_local_index_task, const int64_t ¶llelism, const int64_t &task_execution_id, - const share::ObLSID &ls_id, - const common::ObAddr &leader_addr, const ObIArray &index_partition_ids); int wait_and_send_single_partition_replica_task(bool &state_finished); int check_build_single_replica(bool &is_end); int check_build_local_index_single_replica(bool &is_end); int check_need_verify_checksum(bool &need_verify); bool is_sstable_complete_task_submitted(); - int update_mlog_last_purge_scn(); bool is_create_partitioned_local_index(); int serialize_and_update_message(); private: diff --git a/src/rootserver/ddl_task/ob_partition_split_task.cpp b/src/rootserver/ddl_task/ob_partition_split_task.cpp index 2cceeebdf..1d353f473 100644 --- a/src/rootserver/ddl_task/ob_partition_split_task.cpp +++ b/src/rootserver/ddl_task/ob_partition_split_task.cpp @@ -24,7 +24,6 @@ #include "rootserver/ob_ddl_service_launcher.h" // for ObDDLServiceLauncher #include "rootserver/ob_root_service.h" #include "observer/ob_service.h" -#include "src/storage/tx_storage/ob_ls_map.h" #include "share/ob_tablet_reorganize_history_table_operator.h" using namespace oceanbase::rootserver; @@ -629,32 +628,21 @@ int ObPartitionSplitTask::check_freeze_progress( int ret = OB_SUCCESS; is_end = false; int64_t num_tablet_finished = 0; - ObLocationService *location_service = nullptr; ObArray request_tablet_ids; - const int64_t rpc_timeout = max(GCONF.rpc_timeout, static_cast(1000L * 1000L * 9L)); for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - share::ObLSID ls_id; - ObAddr leader_addr; - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, tablet_id, rpc_timeout, ls_id, leader_addr))) { - LOG_WARN("get tablet leader addr failed", K(ret)); - } else { + { ObCheckProgressStatus freeze_status; if (OB_FAIL(freeze_progress_map_.get_refactored( tablet_id, freeze_status))) { - LOG_WARN("failed to get addr freeze status", K(ret), K(leader_addr)); + LOG_WARN("failed to get freeze status", K(ret), K(tablet_id)); } else if (freeze_status == ObCheckProgressStatus::NOT_STARTED || freeze_status == ObCheckProgressStatus::ONGOING) { // Direct call (seekdb: all tablets are local). - UNUSED(leader_addr); obcall::ObCheckMemtableCntArg arg; obcall::ObCheckMemtableCntResult cur_result; - arg.ls_id_ = ls_id; arg.tablet_id_ = tablet_id; int call_ret = GCTX.ob_service_->check_memtable_cnt(arg, cur_result); UNUSED(call_ret); @@ -662,7 +650,7 @@ int ObPartitionSplitTask::check_freeze_progress( TCWLockGuard guard(lock_); if (OB_FAIL(freeze_progress_map_.set_refactored(tablet_id, ObCheckProgressStatus::ONGOING, true/*overwrite*/))) { - LOG_WARN("failed to update addr freeze status", K(ret), K(leader_addr)); + LOG_WARN("failed to update freeze status", K(ret), K(tablet_id)); } else if (OB_FAIL(request_tablet_ids.push_back(tablet_id))) { LOG_WARN("failed to push back tablet id", K(ret)); } else if (cur_result.memtable_cnt_ == 0) { @@ -689,9 +677,7 @@ int ObPartitionSplitTask::wait_compaction_end( int ret = OB_SUCCESS; int64_t num_tablet_finished = 0; // TODO hanxuan grab mem lock - ObLSID ls_id; - ObAddr unused_addr; - ObArray split_replica_addrs; + ObAddr split_replica_addr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -705,18 +691,11 @@ int ObPartitionSplitTask::wait_compaction_end( } else if (OB_FAIL(init_compaction_scn_map())) { LOG_WARN("failed to setup compaction scn map", K(ret)); } else { // check tablet compaction progress - const ObTabletID &any_tablet_id = all_src_tablet_ids_.at(0); - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr( - GCTX.location_service_, any_tablet_id, rpc_timeout, ls_id, unused_addr))) { - LOG_WARN("get ls id failed", K(ret), K(any_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_split_replicas_addrs(ls_id, split_replica_addrs))) { - LOG_WARN("get split replica addrs failed", K(ret), K(ls_id)); - } + split_replica_addr = GCTX.self_addr(); for (int64_t i = 0; OB_SUCC(ret) && i < all_src_tablet_ids_.count(); ++i) { const ObTabletID &tablet_id = all_src_tablet_ids_.at(i); bool current_tablet_finished = false; - if (OB_FAIL(check_compaction_progress(ls_id, tablet_id, split_replica_addrs, current_tablet_finished))) { + if (OB_FAIL(check_compaction_progress(tablet_id, split_replica_addr, current_tablet_finished))) { LOG_WARN("failed to check compaction progress", K(ret)); } else if (current_tablet_finished) { ++num_tablet_finished; @@ -733,20 +712,7 @@ int ObPartitionSplitTask::wait_compaction_end( if (OB_FAIL(ret)) { } else if (num_tablet_finished == all_src_tablet_ids_.count()) { DEBUG_SYNC(PARTITION_SPLIT_WAIT_COMPACTION_END); - ObArray double_check_replica_addrs; - ObArray different_replica_addrs; - // double check. - if (OB_FAIL(ObDDLUtil::get_split_replicas_addrs(ls_id, double_check_replica_addrs))) { - LOG_WARN("get split replica addrs failed", K(ret), K(ls_id)); - } else if (double_check_replica_addrs.count() != split_replica_addrs.count()) { - ret = OB_EAGAIN; - LOG_INFO("need retry when double check failed", K(ret), K(split_replica_addrs), K(double_check_replica_addrs)); - } else if OB_FAIL(get_difference(split_replica_addrs, double_check_replica_addrs, different_replica_addrs)) { - LOG_WARN("get difference failed", K(ret)); - } else if (!different_replica_addrs.empty()) { - ret = OB_EAGAIN; - LOG_INFO("different replica addrs occur, need retry", K(ret), K(split_replica_addrs), K(double_check_replica_addrs)); - } else if (OB_FAIL(serialize_compaction_scn_to_task_record())) { + if (OB_FAIL(serialize_compaction_scn_to_task_record())) { LOG_WARN("failed to serialize compaction scn to ddl task record", K(ret)); } else if (OB_FAIL(switch_status(next_task_status, true, ret))) { LOG_WARN("failed to switch task status", K(ret)); @@ -757,86 +723,78 @@ int ObPartitionSplitTask::wait_compaction_end( } int ObPartitionSplitTask::check_compaction_progress( - const ObLSID &ls_id, const ObTabletID &tablet_id, - const ObIArray &split_replica_addrs, + const ObAddr &split_replica_addr, bool& is_end) { int ret = OB_SUCCESS; is_end = false; - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || split_replica_addrs.empty())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || !split_replica_addr.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(tablet_id), K(split_replica_addrs)); + LOG_WARN("invalid arg", K(ret), K(tablet_id), K(split_replica_addr)); } else if (OB_UNLIKELY(all_src_tablet_ids_.empty())) { ret = OB_ERR_SYS; LOG_WARN("unexpected null tablet ids", K(ret), KPC(this)); } else if (!compaction_progress_map_.created() && OB_FAIL(compaction_progress_map_.create(all_src_tablet_ids_.count(), lib::ObLabel("DDLWaitCompact")))) { LOG_ERROR("failed to create compaction progress map", K(ret)); } else { - int64_t compaction_end_number = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < split_replica_addrs.count(); i++) { - ObCheckProgressStatus compaction_status; - const common::ObAddr &replica_addr = split_replica_addrs.at(i); - const ObCheckProgressKey check_progress_key(replica_addr, tablet_id); - if (OB_FAIL(compaction_progress_map_.get_refactored(check_progress_key, compaction_status))) { - if (OB_HASH_NOT_EXIST == ret) { - // override ret. - TCWLockGuard guard(lock_); - if (OB_FAIL(compaction_progress_map_.set_refactored(check_progress_key, ObCheckProgressStatus::NOT_STARTED))) { - LOG_WARN("failed to set compaction progress", K(ret)); - } else { - compaction_status = ObCheckProgressStatus::NOT_STARTED; - } + ObCheckProgressStatus compaction_status; + const ObCheckProgressKey check_progress_key(split_replica_addr, tablet_id); + if (OB_FAIL(compaction_progress_map_.get_refactored(check_progress_key, compaction_status))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + TCWLockGuard guard(lock_); + if (OB_FAIL(compaction_progress_map_.set_refactored(check_progress_key, ObCheckProgressStatus::NOT_STARTED))) { + LOG_WARN("failed to set compaction progress", K(ret)); } else { - LOG_WARN("failed to get from map", K(ret)); + compaction_status = ObCheckProgressStatus::NOT_STARTED; } + } else { + LOG_WARN("failed to get from map", K(ret)); } + } - if (OB_FAIL(ret)) { - } else if (ObCheckProgressStatus::DONE != compaction_status) { - // check compaction status - obcall::ObCheckMediumCompactionInfoListArg arg; - - arg.ls_id_ = ls_id; - arg.tablet_id_ = tablet_id; - // Direct call (seekdb: all replicas are local). - UNUSED(replica_addr); - obcall::ObCheckMediumCompactionInfoListResult result; - int call_ret = GCTX.ob_service_->check_medium_compaction_info_list_cnt(arg, result); - if (OB_FAIL(call_ret)) { - LOG_WARN("check medium compaction failed", K(call_ret), K(arg)); - } else { - TCWLockGuard guard(lock_); - if (result.info_list_cnt_ > 0) { - if (OB_FAIL(compaction_progress_map_.set_refactored(check_progress_key, ObCheckProgressStatus::ONGOING, true/*overwrite*/))) { - LOG_WARN("failed to update compaction progress", K(ret), K(replica_addr)); - } - } else if (result.info_list_cnt_ == 0) { - int64_t existed_compaction_scn; - if (OB_FAIL(tablet_compaction_scn_map_.get_refactored(tablet_id, existed_compaction_scn))) { - if (OB_HASH_NOT_EXIST == ret) { - if (OB_FAIL(tablet_compaction_scn_map_.set_refactored(tablet_id, result.primary_compaction_scn_))) { - LOG_WARN("failed to set tablet primary compaction scn", K(ret), K(tablet_id)); - } - } else { - LOG_WARN("get compaction scn from map failed", K(ret), K(tablet_id)); + if (OB_FAIL(ret)) { + } else if (ObCheckProgressStatus::DONE != compaction_status) { + obcall::ObCheckMediumCompactionInfoListArg arg; + arg.tablet_id_ = tablet_id; + obcall::ObCheckMediumCompactionInfoListResult result; + int call_ret = GCTX.ob_service_->check_medium_compaction_info_list_cnt(arg, result); + if (OB_FAIL(call_ret)) { + LOG_WARN("check medium compaction failed", K(call_ret), K(arg)); + } else { + TCWLockGuard guard(lock_); + if (result.info_list_cnt_ > 0) { + if (OB_FAIL(compaction_progress_map_.set_refactored( + check_progress_key, ObCheckProgressStatus::ONGOING, true/*overwrite*/))) { + LOG_WARN("failed to update compaction progress", K(ret), K(split_replica_addr)); + } + } else if (result.info_list_cnt_ == 0) { + int64_t existed_compaction_scn; + if (OB_FAIL(tablet_compaction_scn_map_.get_refactored(tablet_id, existed_compaction_scn))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + if (OB_FAIL(tablet_compaction_scn_map_.set_refactored(tablet_id, result.primary_compaction_scn_))) { + LOG_WARN("failed to set tablet primary compaction scn", K(ret), K(tablet_id)); } - } else if (OB_UNLIKELY(existed_compaction_scn != result.primary_compaction_scn_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unequal compaction scn between replicas", K(ret), K(tablet_id), K(replica_addr), K(existed_compaction_scn), K(result.primary_compaction_scn_)); - } else if (OB_FAIL(compaction_progress_map_.set_refactored(check_progress_key, ObCheckProgressStatus::DONE, true/*overwrite*/))) { - LOG_WARN("failed to update compaction progress", K(ret), K(replica_addr)); } else { - compaction_end_number++; + LOG_WARN("get compaction scn from map failed", K(ret), K(tablet_id)); } + } else if (OB_UNLIKELY(existed_compaction_scn != result.primary_compaction_scn_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected compaction scn change", K(ret), K(tablet_id), + K(split_replica_addr), K(existed_compaction_scn), K(result.primary_compaction_scn_)); + } else if (OB_FAIL(compaction_progress_map_.set_refactored( + check_progress_key, ObCheckProgressStatus::DONE, true/*overwrite*/))) { + LOG_WARN("failed to update compaction progress", K(ret), K(split_replica_addr)); + } else { + is_end = true; } } - } else { - compaction_end_number++; // compact done. } + } else { + is_end = true; } - is_end = OB_SUCC(ret) && (compaction_end_number == split_replica_addrs.count()) ? true : false; } return ret; } @@ -1368,19 +1326,12 @@ int ObPartitionSplitTask::setup_split_finish_items( int ret = OB_SUCCESS; leader_addr.reset(); split_info_array.reset(); - ObLSID ls_id; ObArray tmp_split_info_array; - ObLocationService *location_service = nullptr; - const int64_t rpc_timeout = max(GCONF.rpc_timeout, static_cast(1000L * 1000L * 9L)); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, partition_split_arg_.src_tablet_id_, rpc_timeout, ls_id, leader_addr))) { - LOG_WARN("get tablet leader addr failed", K(ret), "tablet_id", partition_split_arg_.src_tablet_id_); - } else if (OB_FAIL(prepare_tablet_split_infos(ls_id, leader_addr, tmp_split_info_array))) { + } else if (FALSE_IT(leader_addr = GCTX.self_addr())) { + } else if (OB_FAIL(prepare_tablet_split_infos(tmp_split_info_array))) { LOG_WARN("prepare tablet split infos failed", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < tmp_split_info_array.count(); ++i) { @@ -2496,9 +2447,6 @@ int ObPartitionSplitTask::prepare_tablet_split_ranges( { int ret = OB_SUCCESS; parallel_datum_rowkey_list.reset(); - ObLSID ls_id; - ObAddr leader_addr; - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); const int64_t source_index_tablets_cnt = partition_split_arg_.src_local_index_tablet_ids_.count(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -2518,8 +2466,6 @@ int ObPartitionSplitTask::prepare_tablet_split_ranges( LOG_WARN("unexpected err, empty parallel rowkey list under a mismatched status", K(ret), K(task_status_)); } else if (OB_FAIL(index_tablet_parallel_rowkey_list_.prepare_allocate(source_index_tablets_cnt))) { LOG_WARN("prepare alloc failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_leader_addr(GCTX.location_service_, partition_split_arg_.src_tablet_id_, rpc_timeout, ls_id, leader_addr))) { - LOG_WARN("failed to get orig leader addr", K(ret), "tablet_id", partition_split_arg_.src_tablet_id_); } else { for (int64_t tablet_idx = 0; OB_SUCC(ret) && tablet_idx < 1 /*data_tablet*/ + source_index_tablets_cnt; tablet_idx++) { const ObTabletID &src_tablet_id = tablet_idx == 0 ? partition_split_arg_.src_tablet_id_ : partition_split_arg_.src_local_index_tablet_ids_.at(tablet_idx - 1); @@ -2527,7 +2473,6 @@ int ObPartitionSplitTask::prepare_tablet_split_ranges( data_tablet_parallel_rowkey_list_ : index_tablet_parallel_rowkey_list_.at(tablet_idx - 1); obcall::ObPrepareSplitRangesArg arg; obcall::ObPrepareSplitRangesRes result; - arg.ls_id_ = ls_id; arg.tablet_id_ = src_tablet_id; arg.user_parallelism_ = parallelism_; // parallelism_; arg.schema_tablet_size_ = std::max(tablet_size_, static_cast(128 * 1024 * 1024L)/*128MB*/); @@ -2583,8 +2528,6 @@ int ObPartitionSplitTask::prepare_tablet_split_ranges( } int ObPartitionSplitTask::prepare_tablet_split_infos( - const share::ObLSID &ls_id, - const ObAddr &leader_addr, ObIArray &split_info_array) { int ret = OB_SUCCESS; @@ -2594,7 +2537,6 @@ int ObPartitionSplitTask::prepare_tablet_split_infos( ObSArray can_reuse_macro_blocks; ObSchemaGetterGuard schema_guard; ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance(); - const int64_t rpc_timeout = max(GCONF.rpc_timeout, static_cast(1000L * 1000L * 9L)); if (all_src_tablet_ids_.empty() && OB_FAIL(setup_src_tablet_ids_array())) { LOG_WARN("failed to setup all src tablet ids", K(ret)); } else if (OB_FAIL(setup_lob_idxs_arr(lob_col_idxs))) { @@ -2622,7 +2564,6 @@ int ObPartitionSplitTask::prepare_tablet_split_infos( LOG_WARN("failed to get compaction scn for tablet", K(ret), K(tablet_id)); } else { ObTabletSplitArg split_info; - split_info.ls_id_ = ls_id; split_info.table_id_ = i < lob_tablet_start_idx ? table_ids.at(i) : object_id_; split_info.lob_table_id_ = i < lob_tablet_start_idx ? OB_INVALID_ID : table_ids.at(i); split_info.schema_version_ = schema_version_; diff --git a/src/rootserver/ddl_task/ob_partition_split_task.h b/src/rootserver/ddl_task/ob_partition_split_task.h index babed3037..f18b6545f 100644 --- a/src/rootserver/ddl_task/ob_partition_split_task.h +++ b/src/rootserver/ddl_task/ob_partition_split_task.h @@ -195,9 +195,8 @@ class ObPartitionSplitTask final : public ObDDLTask const ObSArray &tablet_ids, bool &is_end); int check_compaction_progress( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, - const ObIArray &split_replica_addrs, + const ObAddr &split_replica_addr, bool &is_end); int send_split_request( const ObPartitionSplitReplicaType replica_type); @@ -232,8 +231,6 @@ class ObPartitionSplitTask final : public ObDDLTask int prepare_tablet_split_ranges( ObSEArray, 8> ¶llel_datum_rowkey_list); int prepare_tablet_split_infos( - const share::ObLSID &ls_id, - const ObAddr &leader_addr, ObIArray &split_info_array); int update_task_message(); private: diff --git a/src/rootserver/ddl_task/ob_sys_ddl_util.h b/src/rootserver/ddl_task/ob_sys_ddl_util.h index afbfc506d..c90df9629 100644 --- a/src/rootserver/ddl_task/ob_sys_ddl_util.h +++ b/src/rootserver/ddl_task/ob_sys_ddl_util.h @@ -35,7 +35,7 @@ namespace rootserver } else if (OB_UNLIKELY(!GCTX.omt_->has_tenant())) { \ ret = OB_TENANT_NOT_EXIST; \ LOG_WARN("local server does not have SYS tenant resource", KR(ret)); \ - } else if (OB_FAIL(ObDDLUtil::check_local_is_sys_leader())) { \ + } else if (OB_FAIL(ObDDLUtil::check_local_sys_tenant())) { \ LOG_WARN("local is not sys tenant leader", KR(ret)); \ } else { \ MOD_SCOPE { \ @@ -65,7 +65,6 @@ class ObSysDDLSchedulerUtil SYS_DDL_SCHEDULER_FUNC(modify_redef_task); SYS_DDL_SCHEDULER_FUNC(on_column_checksum_calc_reply); SYS_DDL_SCHEDULER_FUNC(on_ddl_task_finish); - SYS_DDL_SCHEDULER_FUNC(on_ddl_task_prepare); SYS_DDL_SCHEDULER_FUNC(on_sstable_complement_job_reply); SYS_DDL_SCHEDULER_FUNC(prepare_alter_table_arg); SYS_DDL_SCHEDULER_FUNC(recover_task); diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp index 0d15ebc89..36a10774f 100755 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp @@ -258,17 +258,14 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql() if (!modify_autoinc) { sql_mode = sql_mode | SMO_NO_AUTO_VALUE_ON_ZERO; } - // get execute inner sql addr - if (OB_FAIL(ObDDLUtil::get_sys_ls_leader_addr(GCONF.cluster_id, alter_table_arg_.inner_sql_exec_addr_))) { - LOG_WARN("get sys ls leader addr fail", K(ret)); - ret = OB_SUCCESS; // ignore ret - } else if (OB_FAIL(set_sql_exec_addr(alter_table_arg_.inner_sql_exec_addr_))) { + alter_table_arg_.inner_sql_exec_addr_ = GCTX.self_addr(); + if (OB_FAIL(set_sql_exec_addr(alter_table_arg_.inner_sql_exec_addr_))) { LOG_WARN("failed to set sql execute addr", K(ret), K(alter_table_arg_.inner_sql_exec_addr_)); - } - ObSchemaGetterGuard schema_guard; - const ObTableSchema *orig_table_schema = nullptr; - const ObTableSchema *hidden_table_schema = nullptr; - ObDDLRedefinitionSSTableBuildTask task( + } else { + ObSchemaGetterGuard schema_guard; + const ObTableSchema *orig_table_schema = nullptr; + const ObTableSchema *hidden_table_schema = nullptr; + ObDDLRedefinitionSSTableBuildTask task( task_id_, object_id_, target_object_id_, @@ -279,24 +276,21 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql() trace_id_, parallelism_, use_heap_table_ddl_plan_, - alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_, - alter_table_arg_.mview_refresh_info_.mview_table_id_, GCTX.root_service_, alter_table_arg_.inner_sql_exec_addr_, data_format_version_, - is_ddl_retryable_, - alter_table_arg_.mview_refresh_info_.mview_target_data_sync_scn_.get_val_for_gts(), - alter_table_arg_.mview_refresh_info_.select_sql_); - if (OB_FAIL(root_service->get_ddl_service().get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( object_id_, orig_table_schema))) { - LOG_WARN("failed to get orig table schema", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( target_object_id_, hidden_table_schema))) { - LOG_WARN("fail to get table schema", K(ret), K(target_object_id_)); - } else if (OB_FAIL(task.init(*orig_table_schema, *hidden_table_schema, alter_table_arg_.alter_table_schema_, alter_table_arg_.tz_info_wrap_, alter_table_arg_.based_schema_object_infos_))) { - LOG_WARN("fail to init table redefinition sstable build task", K(ret)); - } else if (OB_FAIL(root_service->submit_ddl_single_replica_build_task(task))) { - LOG_WARN("fail to submit ddl build single replica", K(ret)); + is_ddl_retryable_); + if (OB_FAIL(root_service->get_ddl_service().get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { + LOG_WARN("get schema guard failed", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(object_id_, orig_table_schema))) { + LOG_WARN("failed to get orig table schema", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(target_object_id_, hidden_table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(target_object_id_)); + } else if (OB_FAIL(task.init(*orig_table_schema, *hidden_table_schema, alter_table_arg_.alter_table_schema_, alter_table_arg_.tz_info_wrap_))) { + LOG_WARN("fail to init table redefinition sstable build task", K(ret)); + } else if (OB_FAIL(root_service->submit_ddl_single_replica_build_task(task))) { + LOG_WARN("fail to submit ddl build single replica", K(ret)); + } } } return ret; @@ -335,8 +329,6 @@ int ObTableRedefinitionTask::check_ddl_can_retry(const bool ddl_need_retry_at_ex LOG_WARN("invalid arguments", K(ret), KP(table_schema)); } else if (OB_FAIL(check_use_heap_table_ddl_plan(table_schema))) { LOG_WARN("check use heap table ddl plan failed", K(ret)); - } else if (ObDDLUtil::is_mview_not_retryable(task_type_)) { - is_ddl_retryable_ = false; } else { if (ObDDLUtil::use_idempotent_mode()) { if (use_heap_table_ddl_plan_) { @@ -361,8 +353,7 @@ int ObTableRedefinitionTask::check_use_heap_table_ddl_plan(const ObTableSchema * } else if (OB_FAIL(DDL_SIM(task_id_, TABLE_REDEF_TASK_CHECK_USE_HEAP_PLAN_FAILED))) { LOG_WARN("ddl sim failure", K(task_id_)); } else if (target_table_schema->is_table_with_hidden_pk_column() && - (DDL_ALTER_PARTITION_BY == task_type_ || DDL_DROP_PRIMARY_KEY == task_type_ || - DDL_MVIEW_COMPLETE_REFRESH == task_type_)) { + (DDL_ALTER_PARTITION_BY == task_type_ || DDL_DROP_PRIMARY_KEY == task_type_)) { use_heap_table_ddl_plan_ = true; } return ret; @@ -434,8 +425,7 @@ int ObTableRedefinitionTask::replica_end_check(const int ret_code) int ret = OB_SUCCESS; switch(task_type_) { case DDL_DIRECT_LOAD : - case DDL_DIRECT_LOAD_INSERT : - case DDL_MVIEW_COMPLETE_REFRESH: { + case DDL_DIRECT_LOAD_INSERT : { break; } default : { @@ -468,8 +458,8 @@ int ObTableRedefinitionTask::copy_table_indexes() int64_t active_task_cnt = 0; // check if has rebuild index if (has_rebuild_index_) { - } else if (OB_FAIL(ObDDLTaskRecordOperator::get_create_index_or_mlog_task_cnt(*GCTX.sql_proxy_, target_object_id_, active_task_cnt))) { - LOG_WARN("failed to check index or mlog task cnt", K(ret)); + } else if (OB_FAIL(ObDDLTaskRecordOperator::get_create_index_task_cnt(*GCTX.sql_proxy_, target_object_id_, active_task_cnt))) { + LOG_WARN("failed to check index task cnt", K(ret)); } else if (active_task_cnt >= MAX_ACTIVE_TASK_CNT) { ret = OB_EAGAIN; } else { @@ -488,19 +478,13 @@ int ObTableRedefinitionTask::copy_table_indexes() LOG_WARN("error unexpected, table schema must not be nullptr", K(ret), K(target_object_id_)); } else { const common::ObIArray &index_infos = table_schema->get_simple_index_infos(); - if ((index_infos.count() > 0) || (table_schema->mv_container_table() - && table_schema->has_mlog_table())) { + if (index_infos.count() > 0) { // if there is indexes in new tables, if so, the indexes is already rebuilt in new table for (int64_t i = 0; OB_SUCC(ret) && i < index_infos.count(); ++i) { if (OB_FAIL(index_ids.push_back(index_infos.at(i).table_id_))) { LOG_WARN("push back index id failed", K(ret)); } } - if (OB_SUCC(ret) && table_schema->has_mlog_table()) { - if (OB_FAIL(index_ids.push_back(table_schema->get_mlog_tid()))) { - LOG_WARN("failed to push back mlog tid", KR(ret), K(table_schema->get_mlog_tid())); - } - } LOG_INFO("indexes schema are already built", K(index_ids)); } else { int64_t ddl_rpc_timeout = 0; @@ -561,13 +545,7 @@ int ObTableRedefinitionTask::copy_table_indexes() } else if (active_task_cnt >= MAX_ACTIVE_TASK_CNT) { ret = OB_EAGAIN; } else { - ObDDLType ddl_type = ObDDLType::DDL_CREATE_INDEX; - if (index_schema->is_mlog_table()) { - ddl_type = ObDDLType::DDL_CREATE_MLOG; - create_index_arg.index_action_type_ = ObIndexArg::ADD_MLOG; - } else { - ddl_type = get_create_index_type(data_format_version_, *index_schema); - } + ObDDLType ddl_type = get_create_index_type(data_format_version_, *index_schema); create_index_arg.index_type_ = index_schema->get_index_type(); if (index_schema->is_vec_index() || index_schema->is_fts_index() || index_schema->is_multivalue_index()) { has_rebuild_domain_indexes_ = true; @@ -898,9 +876,6 @@ int ObTableRedefinitionTask::take_effect(const ObDDLTaskStatus next_task_status) } } else if (OB_FAIL(sync_stats_info())) { LOG_WARN("fail to sync stats info", K(ret), K(object_id_), K(target_object_id_)); - } else if (alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_ && - OB_FAIL(alter_table_arg_.mview_refresh_info_.refresh_scn_.convert_for_inner_table_field(snapshot_version_))) { - LOG_WARN("fail to convert scn", K(ret), K(snapshot_version_)); } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout_by_table(target_object_id_, ddl_rpc_timeout))) { LOG_WARN("get ddl rpc timeout fail", K(ret)); } else if (OB_FAIL(rootserver::serial_call([&]{ return GCTX.root_service_-> execute_ddl_task(alter_table_arg_, objs); }))) { @@ -1009,16 +984,8 @@ int ObTableRedefinitionTask::process() } else { switch(task_status_) { case ObDDLTaskStatus::PREPARE: - if (alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_ && parent_task_id_ > 0) { - const ObDDLTaskID parent_task_id(parent_task_id_); - if (OB_FAIL(ObSysDDLSchedulerUtil::on_ddl_task_prepare(parent_task_id, task_id_, trace_id_))) { - LOG_WARN("fail to do parent task callback", KR(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(prepare(ObDDLTaskStatus::WAIT_TRANS_END))) { - LOG_WARN("fail to prepare table redefinition task", K(ret)); - } + if (OB_FAIL(prepare(ObDDLTaskStatus::WAIT_TRANS_END))) { + LOG_WARN("fail to prepare table redefinition task", K(ret)); } break; case ObDDLTaskStatus::WAIT_TRANS_END: diff --git a/src/rootserver/fork_table/ob_fork_table_helper.cpp b/src/rootserver/fork_table/ob_fork_table_helper.cpp index 49bb64d8f..a629123f6 100644 --- a/src/rootserver/fork_table/ob_fork_table_helper.cpp +++ b/src/rootserver/fork_table/ob_fork_table_helper.cpp @@ -27,11 +27,12 @@ #include "rootserver/fork_table/ob_fork_table_util.h" #include "observer/vector_index/ob_vector_index_util.h" #include "share/schema/ob_schema_utils.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "storage/tablet/ob_tablet_fork_mds_helper.h" #include "storage/truncate_info/ob_truncate_info.h" #include "storage/truncate_info/ob_truncate_info_array.h" #include "storage/tx/ob_ts_mgr.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -146,19 +147,6 @@ int check_fork_table_supported(const ObTableSchema &src_table_schema, ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; LOG_WARN("can fork table from table in recyclebin", K(ret), K(src_table_schema)); - } else if (src_table_schema.has_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("fork table on table with materialized view log is not supported", - KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "fork table on table with materialized view log is"); - } else if (src_table_schema.table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN( - "fork table on table required by materialized view is not supported", - KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "fork table on table required by materialized view is"); } else if (OB_FAIL(check_table_index_features( src_table_schema, schema_guard, has_semantic_index, has_ivf_index, has_spatial_index, has_global_index, @@ -301,7 +289,6 @@ int ObForkTableHelper::copy_tablet_autoinc_seq_info_() obcall::ObBatchSetTabletAutoincSeqArg arg; - arg.ls_id_ = SYS_LS; arg.is_tablet_creating_ = true; for (int64_t i = 0; OB_SUCC(ret) && i < src_tablet_ids_.count(); ++i) { @@ -334,15 +321,14 @@ int ObForkTableHelper::copy_tablet_autoinc_seq_info_() if (OB_SUCC(ret)) { storage::ObTabletForkMdsArg fork_mds_arg; - fork_mds_arg.ls_id_ = SYS_LS; if (OB_FAIL(fork_mds_arg.set_autoinc_seq_arg(arg))) { LOG_WARN("failed to set autoinc seq arg", K(ret), K(arg)); } else if (OB_FAIL(storage::ObTabletForkMdsHelper::register_mds( fork_mds_arg, false, trans_))) { - LOG_WARN("failed to register fork mds", K(ret), K(SYS_LS)); + LOG_WARN("failed to register fork mds", K(ret)); } else { LOG_INFO("fork table: successfully registered fork mds for autoinc seq", - K(SYS_LS), K(arg.autoinc_params_.count())); + K(arg.autoinc_params_.count())); } } } @@ -365,7 +351,7 @@ int ObForkTableHelper::copy_tablet_truncate_info_() int64_t empty_cnt = 0; int64_t registered_cnt = 0; - if (OB_FAIL(OB_TS_MGR.get_ts_sync(GCONF.rpc_timeout, + if (OB_FAIL(OB_TS_MGR.get_gts_sync(GCONF.rpc_timeout, max_readable_scn))) { LOG_WARN("failed to get gts", K(ret)); } @@ -402,9 +388,7 @@ int ObForkTableHelper::copy_tablet_truncate_info_() } else { storage::ObTabletForkMdsArg fork_mds_arg; - fork_mds_arg.ls_id_ = SYS_LS; rootserver::ObTruncateTabletArg truncate_arg; - truncate_arg.ls_id_ = SYS_LS; truncate_arg.index_tablet_id_ = dst_tablet_id; if (OB_FAIL(truncate_arg.truncate_info_.assign( allocator, *latest_truncate_info))) { @@ -418,7 +402,7 @@ int ObForkTableHelper::copy_tablet_truncate_info_() } else if (OB_FAIL(storage::ObTabletForkMdsHelper::register_mds( fork_mds_arg, false /*need_flush_redo*/, trans_))) { LOG_WARN("failed to register fork mds for truncate info", K(ret), - K(SYS_LS), K(dst_tablet_id)); + K(dst_tablet_id)); } else { ++registered_cnt; LOG_DEBUG( @@ -647,21 +631,11 @@ int ObForkTableHelper::get_tablet_handle_( storage::ObTabletHandle &tablet_handle) const { int ret = OB_SUCCESS; - ObLSService *ls_service = nullptr; ObLS *ls = nullptr; - ObLSHandle ls_handle; MOD_SCOPE { - if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls service is null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(SYS_LS, ls_handle, - ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(SYS_LS)); - } else if (FALSE_IT(ls = ls_handle.get_ls())) { - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(SYS_LS)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } diff --git a/src/rootserver/fork_table/ob_fork_table_task.cpp b/src/rootserver/fork_table/ob_fork_table_task.cpp index 361c4a9be..919d3bfd0 100644 --- a/src/rootserver/fork_table/ob_fork_table_task.cpp +++ b/src/rootserver/fork_table/ob_fork_table_task.cpp @@ -22,7 +22,6 @@ #include "rootserver/ob_ddl_operator.h" #include "share/ob_ddl_common.h" #include "share/schema/ob_schema_getter_guard.h" -#include "share/location_cache/ob_location_service.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/ob_ddl_task_executor.h" #include "share/ob_ddl_sim_point.h" @@ -32,7 +31,7 @@ #include "common/mysqlclient/ob_mysql_transaction.h" #include "share/ob_share_util.h" #include "common/ob_timeout_ctx.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "rootserver/ddl_task/ob_sys_ddl_util.h" #include "storage/ddl/ob_tablet_fork_task.h" #include "storage/compaction/ob_schedule_dag_func.h" @@ -351,8 +350,8 @@ int ObForkTableTask::wait_freeze_end(const ObDDLTaskStatus next_task_status) LOG_WARN("fail to get schema guard", K(ret)); } else if (OB_FAIL(ObForkTableUtil::collect_tablet_ids_from_table(schema_guard, object_id_, src_tablet_ids))) { LOG_WARN("fail to get src tablet ids", K(ret)); - } else if (OB_FAIL(storage::ObTabletForkUtil::freeze_tablets(SYS_LS, src_tablet_ids))) { - LOG_WARN("fail to freeze tablets", K(ret), K(SYS_LS), K(src_tablet_ids)); + } else if (OB_FAIL(storage::ObTabletForkUtil::freeze_tablets(src_tablet_ids))) { + LOG_WARN("fail to freeze tablets", K(ret), K(src_tablet_ids)); } if (OB_SUCC(ret)) { @@ -363,14 +362,13 @@ int ObForkTableTask::wait_freeze_end(const ObDDLTaskStatus next_task_status) } else if (OB_UNLIKELY(!freeze_log.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid fork freeze log", K(ret), K(freeze_log)); - } else if (OB_FAIL(ObDDLRedoLogWriter::write_auto_fork_log(SYS_LS, - ObDDLClogType::DDL_TABLE_FORK_FREEZE_LOG, + } else if (OB_FAIL(ObDDLRedoLogWriter::write_auto_fork_log(ObDDLClogType::DDL_TABLE_FORK_FREEZE_LOG, logservice::ObReplayBarrierType::NO_NEED_BARRIER, freeze_log, free_log_scn))) { LOG_WARN("fail to write table fork freeze log", K(ret), K(freeze_log)); } else { - LOG_INFO("fork table freeze stage done", K(task_id_), K(SYS_LS), K(free_log_scn), + LOG_INFO("fork table freeze stage done", K(task_id_), K(free_log_scn), "src_table_id", object_id_, "tablet_cnt", src_tablet_ids.count(), "cost_us", ObTimeUtility::current_time() - start_ts); LOG_DEBUG("fork table freeze log detail", K(task_id_), K(freeze_log)); @@ -411,14 +409,13 @@ int ObForkTableTask::build_data(const ObDDLTaskStatus next_task_status) LOG_WARN("fail to assign fork info", K(ret)); } else if (OB_FAIL(storage::ObTabletForkUtil::try_schedule_fork_dags(fork_info))) { LOG_WARN("fail to try schedule fork dags", K(ret)); - } else if (OB_FAIL(ObDDLRedoLogWriter::write_auto_fork_log(SYS_LS, - ObDDLClogType::DDL_TABLE_FORK_START_LOG, + } else if (OB_FAIL(ObDDLRedoLogWriter::write_auto_fork_log(ObDDLClogType::DDL_TABLE_FORK_START_LOG, logservice::ObReplayBarrierType::STRICT_BARRIER, start_log, start_log_scn))) { LOG_WARN("fail to write table fork start log", K(ret), K(start_log)); } else { - LOG_INFO("fork table build_data stage started", K(task_id_), K(SYS_LS), K(start_log_scn), + LOG_INFO("fork table build_data stage started", K(task_id_), K(start_log_scn), "src_table_id", object_id_, "dst_table_id", target_object_id_, "tablet_cnt", src_tablet_ids.count(), "cost_us", ObTimeUtility::current_time() - start_ts); @@ -503,14 +500,13 @@ int ObForkTableTask::wait_data_complement(const ObDDLTaskStatus next_task_status finish_log.fork_info_ = fork_info; SCN scn; if (OB_FAIL(ObDDLRedoLogWriter::write_auto_fork_log( - SYS_LS, ObDDLClogType::DDL_TABLE_FORK_FINISH_LOG, logservice::ObReplayBarrierType::STRICT_BARRIER, finish_log, scn))) { LOG_WARN("fail to write table fork finish log", K(ret), K(finish_log)); } else { - LOG_INFO("fork table finish log written", K(task_id_), K(SYS_LS), K(scn), + LOG_INFO("fork table finish log written", K(task_id_), K(scn), "src_table_id", object_id_, "dst_table_id", target_object_id_, "tablet_cnt", dst_tablet_ids.count()); LOG_DEBUG("fork table finish log detail", K(task_id_), K(finish_log)); @@ -702,7 +698,6 @@ int ObForkTableTask::build_fork_info( LOG_WARN("src and dst tablet count mismatch", K(ret), K(src_tablet_ids.count()), K(dst_tablet_ids.count())); } else { fork_info = storage::ObTableForkInfo( - SYS_LS, object_id_, schema_version_, task_id_, @@ -722,4 +717,3 @@ int ObForkTableTask::build_fork_info( } // namespace rootserver } // namespace oceanbase - diff --git a/src/rootserver/freeze/ob_checksum_validator.cpp b/src/rootserver/freeze/ob_checksum_validator.cpp index 0014a46eb..015b7174f 100755 --- a/src/rootserver/freeze/ob_checksum_validator.cpp +++ b/src/rootserver/freeze/ob_checksum_validator.cpp @@ -44,7 +44,7 @@ int ObChecksumValidator::init( LOG_WARN("failed to init ckm array", KR(ret)); } else { is_primary_service_ = is_primary_service; - cur_tablet_ls_pair_array_.set_attr(ObMemAttr("RSCompPairs")); + cur_tablet_ids_.set_attr(ObMemAttr("RSCompTabs")); sql_proxy_ = &sql_proxy; is_inited_ = true; } @@ -73,8 +73,7 @@ int ObChecksumValidator::deal_with_special_table_at_last(bool &finish_validate) int ret = OB_SUCCESS; finish_validate = false; ObSchemaGetterGuard schema_guard(ObSchemaMgrItem::MOD_RS_MAJOR_CHECK); - cur_tablet_ls_pair_array_.reuse(); - ObSEArray tmp_tablet_ids; + cur_tablet_ids_.reuse(); if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard( schema_guard, OB_INVALID_VERSION, OB_INVALID_VERSION, ObMultiVersionSchemaService::RefreshSchemaMode::FORCE_LAZY))) { @@ -89,23 +88,19 @@ int ObChecksumValidator::deal_with_special_table_at_last(bool &finish_validate) } else if (OB_ISNULL(simple_schema_)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("table schema is null", KR(ret), K_(table_id)); - } else if (OB_FAIL(simple_schema_->get_tablet_ids(tmp_tablet_ids))) { + } else if (OB_FAIL(simple_schema_->get_tablet_ids(cur_tablet_ids_))) { LOG_WARN("fail to get tablet_ids from simple table schema", KR(ret), KPC_(simple_schema)); - } else if (OB_FAIL(tablet_ls_pair_cache_.get_tablet_ls_pairs( - table_id_, tmp_tablet_ids, cur_tablet_ls_pair_array_))) { - LOG_WARN("failed to get tablet ls pairs from cache", KR(ret)); - } else if (OB_UNLIKELY(cur_tablet_ls_pair_array_.empty())) { + } else if (OB_UNLIKELY(cur_tablet_ids_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet_ls pairs of current table schema", KR(ret), K_(table_id), - K_(cur_tablet_ls_pair_array)); + LOG_WARN("fail to get tablet ids of current table schema", KR(ret), K_(table_id), K_(cur_tablet_ids)); } else if (OB_FAIL(get_tablet_replica_checksum_and_validate(true /*include_larger_than*/))) { if (OB_ITEM_NOT_MATCH == ret) { (void) uncompact_info_.add_skip_verify_table(table_id_); - LOG_TRACE("mismatch checksum cnt when deal with special table", KR(ret), K_(cur_tablet_ls_pair_array)); + LOG_TRACE("mismatch checksum cnt when deal with special table", KR(ret), K_(cur_tablet_ids)); ret = OB_SUCCESS; } else { LOG_ERROR("fail to validate tablet replica checksum", KR(ret), "compaction_scn", get_compaction_scn(), K_(table_id), - KPC(simple_schema_), K_(cur_tablet_ls_pair_array)); + KPC(simple_schema_), K_(cur_tablet_ids)); } } else if (FALSE_IT(table_compaction_info_.set_index_ckm_verified())) { } else if (OB_FAIL(validate_cross_cluster_checksum())) { @@ -182,17 +177,16 @@ void ObChecksumValidator::clear_cached_info() schema_guard_ = nullptr; simple_schema_ = nullptr; table_compaction_info_.reset(); - cur_tablet_ls_pair_array_.reuse(); - finish_tablet_ls_pair_array_.reuse(); + cur_tablet_ids_.reuse(); + finish_tablet_ids_.reuse(); replica_ckm_items_.reset(); last_table_ckm_items_.clear(); } -int ObChecksumValidator::get_tablet_ls_pairs( +int ObChecksumValidator::get_tablet_ids( const share::schema::ObSimpleTableSchemaV2 &simple_schema) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; SMART_VAR(ObArray, tablet_ids) { if (OB_UNLIKELY(!simple_schema.has_tablet())) { ret = OB_ERR_UNEXPECTED; @@ -202,12 +196,12 @@ int ObChecksumValidator::get_tablet_ls_pairs( } else if (OB_UNLIKELY(tablet_ids.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get tablet_ids of current table schema", KR(ret), K(simple_schema)); - } else if (OB_FAIL(cur_tablet_ls_pair_array_.reserve(tablet_ids.count()))) { + } else if (OB_FAIL(cur_tablet_ids_.reserve(tablet_ids.count()))) { LOG_WARN("failed to reserve tablet array", KR(ret), K(tablet_ids.count())); - } else if (OB_FAIL(finish_tablet_ls_pair_array_.reserve(tablet_ids.count()))) { + } else if (OB_FAIL(finish_tablet_ids_.reserve(tablet_ids.count()))) { LOG_WARN("failed to reserve finish tablet array", KR(ret)); - } else if (OB_FAIL(tablet_ls_pair_cache_.get_tablet_ls_pairs(table_id_, tablet_ids, cur_tablet_ls_pair_array_))) { - LOG_WARN("failed to tablet ls pair", KR(ret), K(tablet_ids)); + } else if (OB_FAIL(cur_tablet_ids_.assign(tablet_ids))) { + LOG_WARN("failed to assign tablet ids", KR(ret), K(tablet_ids)); } else { #ifdef ERRSIM static int64_t enter_cnt = 0; @@ -216,14 +210,14 @@ int ObChecksumValidator::get_tablet_ls_pairs( if (OB_FAIL(ret)) { if (enter_cnt++ == 0) { ret = OB_ITEM_NOT_MATCH; - STORAGE_LOG(INFO, "ERRSIM EN_GET_TABLET_LS_PAIR_IN_RS", K(ret), K(simple_schema), K_(cur_tablet_ls_pair_array)); + STORAGE_LOG(INFO, "ERRSIM EN_GET_TABLET_LS_PAIR_IN_RS", K(ret), K(simple_schema), K_(cur_tablet_ids)); } else { ret = OB_SUCCESS; } } } #endif - LOG_TRACE("success to get tablet ls pairs", KR(ret), K_(cur_tablet_ls_pair_array)); + LOG_TRACE("success to get tablet ids", KR(ret), K_(cur_tablet_ids)); } } return ret; @@ -254,21 +248,21 @@ int ObChecksumValidator::validate_checksum( || !simple_schema_->has_tablet())) { // like VIEW, it does not have tablet, treat it as compaction finished and can skip verifying table_compaction_info_.set_can_skip_verifying(); - } else if (OB_FAIL(get_tablet_ls_pairs(*simple_schema_))) { + } else if (OB_FAIL(get_tablet_ids(*simple_schema_))) { if (OB_ITEM_NOT_MATCH == ret) { ret = OB_SUCCESS; table_compaction_info_.set_can_skip_verifying(); (void) uncompact_info_.add_skip_verify_table(table_id_); } else { - LOG_WARN("failed to get table pairs", K(ret), KPC_(simple_schema)); + LOG_WARN("failed to get tablet ids", K(ret), KPC_(simple_schema)); } - } else if (OB_UNLIKELY(cur_tablet_ls_pair_array_.empty())) { + } else if (OB_UNLIKELY(cur_tablet_ids_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet ls pair array is unexpected empty", KR(ret), KPC_(simple_schema), K_(cur_tablet_ls_pair_array)); + LOG_WARN("tablet id array is unexpected empty", KR(ret), KPC_(simple_schema), K_(cur_tablet_ids)); } else { if (OB_FAIL(validate_tablet_replica_checksum())) { LOG_WARN("fail to validate tablet replica checksum", KR(ret), "compaction_scn", get_compaction_scn(), K_(table_id), - KPC(simple_schema_), K_(cur_tablet_ls_pair_array)); + KPC(simple_schema_), K_(cur_tablet_ids)); } else if (OB_FAIL(validate_index_checksum())) { LOG_WARN("failed to validate index checksum", K(ret)); } else if (OB_FAIL(validate_cross_cluster_checksum())) { @@ -282,8 +276,8 @@ int ObChecksumValidator::validate_checksum( } else if (replica_ckm_items_.count() > 0) { int tmp_ret = OB_SUCCESS; last_table_ckm_items_.clear(); - if (OB_TMP_FAIL(last_table_ckm_items_.build(*schema_guard_, *simple_schema_, cur_tablet_ls_pair_array_, replica_ckm_items_))) { - LOG_WARN("failed to build table ckm items", KR(tmp_ret), K_(table_id), K_(cur_tablet_ls_pair_array), + if (OB_TMP_FAIL(last_table_ckm_items_.build(*schema_guard_, *simple_schema_, cur_tablet_ids_, replica_ckm_items_))) { + LOG_WARN("failed to build table ckm items", KR(tmp_ret), K_(table_id), K_(cur_tablet_ids), K_(replica_ckm_items)); } else { LOG_DEBUG("success to build ckm item", KR(tmp_ret), K(last_table_ckm_items_), K_(table_compaction_info)); @@ -292,7 +286,7 @@ int ObChecksumValidator::validate_checksum( last_table_ckm_items_.clear(); } } - cur_tablet_ls_pair_array_.reuse(); // need reuse array when get_tablet_ls_pairs failed + cur_tablet_ids_.reuse(); // need reuse array when get_tablet_ids failed if (FAILEDx(table_compaction_map_.set_refactored(table_id_, table_compaction_info_, true /*overwrite*/))) { LOG_WARN("fail to set refactored", KR(ret), K_(table_id), K_(table_compaction_info)); @@ -337,18 +331,18 @@ int ObChecksumValidator::update_table_compaction_info_by_tablet() int ret = OB_SUCCESS; // iterate all tablets to check 'compacted/finished status' or not. int64_t idx = 0; - const int64_t end_idx = cur_tablet_ls_pair_array_.count(); + const int64_t end_idx = cur_tablet_ids_.count(); for ( ; OB_SUCC(ret) && (idx < end_idx); ++idx) { - const ObTabletID &tablet_id = cur_tablet_ls_pair_array_.at(idx).get_tablet_id(); + const ObTabletID &tablet_id = cur_tablet_ids_.at(idx); ObTabletCompactionStatusEnum tablet_status = ObTabletCompactionStatusEnum::INITIAL; if (OB_FAIL(tablet_status_map_.get_refactored(tablet_id, tablet_status))) { // if tablet not finish compaction, it won't be added into this map if (OB_HASH_NOT_EXIST == ret) { ret = OB_SUCCESS; table_compaction_info_.set_uncompacted(); - (void) uncompact_info_.add_tablet(cur_tablet_ls_pair_array_.at(idx).get_ls_id(), tablet_id); + (void) uncompact_info_.add_tablet(tablet_id); LOG_TRACE("tablet not exist in tablet status map", KR(ret), K(tablet_id), - K_(cur_tablet_ls_pair_array), K_(table_compaction_info)); + K_(cur_tablet_ids), K_(table_compaction_info)); #ifdef ERRSIM ret = OB_E(EventTable::EN_SKIP_INDEX_MAJOR) ret; if (OB_FAIL(ret)) { @@ -373,7 +367,7 @@ int ObChecksumValidator::update_table_compaction_info_by_tablet() } // end of for if (OB_SUCC(ret)) { if (idx == end_idx) { // loop finish - table_compaction_info_.tablet_cnt_ = cur_tablet_ls_pair_array_.count(); + table_compaction_info_.tablet_cnt_ = cur_tablet_ids_.count(); table_compaction_info_.set_compacted(); } LOG_TRACE("update_table_compaction_info_by_tablet", KR(ret), K_(table_compaction_info)); @@ -388,13 +382,13 @@ int ObChecksumValidator::get_tablet_replica_checksum_and_validate(const bool inc FREEZE_TIME_GUARD; if (OB_FAIL(get_replica_ckm(include_larger_than))) { LOG_WARN("fail to check major tablet_replica checksum", KR(ret), - K_(cur_tablet_ls_pair_array), "compaction_scn", get_compaction_scn(), K_(table_compaction_info)); - } else if (OB_UNLIKELY(replica_ckm_items_.get_tablet_cnt() != cur_tablet_ls_pair_array_.count())) { + K_(cur_tablet_ids), "compaction_scn", get_compaction_scn(), K_(table_compaction_info)); + } else if (OB_UNLIKELY(replica_ckm_items_.get_tablet_cnt() != cur_tablet_ids_.count())) { ret = OB_ITEM_NOT_MATCH; replica_ckm_items_.reset(); (void) uncompact_info_.add_skip_verify_table(table_id_); - LOG_TRACE("checksum cnt is not equal to tablet_ls pairs cnt", KR(ret), - K_(cur_tablet_ls_pair_array), "compaction_scn", get_compaction_scn(), K_(table_compaction_info), K(replica_ckm_items_)); + LOG_TRACE("checksum cnt is not equal to tablet id cnt", KR(ret), + K_(cur_tablet_ids), "compaction_scn", get_compaction_scn(), K_(table_compaction_info), K(replica_ckm_items_)); } return ret; } @@ -423,8 +417,8 @@ int ObChecksumValidator::validate_cross_cluster_checksum() } ret = OB_ITEM_NOT_MATCH == ret ? OB_SUCCESS : ret; // clear errno if (OB_SUCC(ret)) { - if (OB_FAIL(push_finish_tablet_ls_pairs_with_update(cur_tablet_ls_pair_array_))) { - LOG_WARN("failed to push back tablet_ls_pair", KR(ret)); + if (OB_FAIL(push_finish_tablet_ids_with_update(table_id_, cur_tablet_ids_))) { + LOG_WARN("failed to push back tablet ids", KR(ret)); } else { table_compaction_info_.set_verified(); LOG_TRACE("after cross cluster validate table checksum", K(ret), K_(table_compaction_info)); @@ -470,18 +464,18 @@ int ObChecksumValidator::batch_write_tablet_ckm() int ObChecksumValidator::batch_update_report_scn() { int ret = OB_SUCCESS; - if (finish_tablet_ls_pair_array_.empty()) { + if (finish_tablet_ids_.empty()) { } else if (OB_FAIL(ObTabletMetaTableCompactionOperator::batch_update_report_scn( get_compaction_scn_val(), - finish_tablet_ls_pair_array_, + finish_tablet_ids_, ObTabletReplica::ScnStatus::SCN_STATUS_ERROR /*except_status*/))) { LOG_WARN("fail to batch update report_scn", KR(ret), - K_(finish_tablet_ls_pair_array)); + K_(finish_tablet_ids)); } else { ++statistics_.update_report_scn_sql_cnt_; LOG_INFO("success to batch update report_scn", KR(ret), - "table_cnt", finish_tablet_ls_pair_array_.count()); - finish_tablet_ls_pair_array_.reuse(); + "table_cnt", finish_tablet_ids_.count()); + finish_tablet_ids_.reuse(); } return ret; } @@ -494,8 +488,8 @@ int ObChecksumValidator::check_tablet_checksum_sync_finish(const bool force_chec // 1) force check when first init // 2) ckm not sync finish in standby service if (!force_check && (is_primary_service_ || cross_cluster_ckm_sync_finish_)) { - } else if (OB_FAIL(ObTabletChecksumOperator::is_first_tablet_in_sys_ls_exist(*sql_proxy_, get_compaction_scn(), is_exist))) { - LOG_WARN("fail to check is first tablet in first ls exist", KR(ret), "compaction_scn", get_compaction_scn()); + } else if (OB_FAIL(ObTabletChecksumOperator::is_first_tablet_checksum_exist(*sql_proxy_, get_compaction_scn(), is_exist))) { + LOG_WARN("fail to check first tablet checksum", KR(ret), "compaction_scn", get_compaction_scn()); } else if (is_exist) { cross_cluster_ckm_sync_finish_ = true; } else if (is_primary_service_) { @@ -503,8 +497,8 @@ int ObChecksumValidator::check_tablet_checksum_sync_finish(const bool force_chec } else { cross_cluster_ckm_sync_finish_ = check_waiting_tablet_checksum_timeout(); if (TC_REACH_TIME_INTERVAL(PRINT_CROSS_CLUSTER_LOG_INVERVAL)) { - LOG_ERROR("can not check cross-cluster checksum now, please wait until first tablet" - "in sys ls exists", "compaction_scn", get_compaction_scn(), K_(major_merge_start_us), + LOG_ERROR("can not check cross-cluster checksum until the first tablet checksum exists", + "compaction_scn", get_compaction_scn(), K_(major_merge_start_us), "fast_current_time_us", ObTimeUtil::fast_current_time(), K(is_exist), K_(is_primary_service)); } } @@ -519,7 +513,7 @@ int ObChecksumValidator::validate_replica_and_tablet_checksum() if (replica_ckm_items_.empty() && OB_FAIL(get_replica_ckm())) { LOG_ERROR("fail to batch get tablet replica checksum items", KR(ret), "compaction_scn", get_compaction_scn()); } else if (OB_FAIL(ObTabletChecksumOperator::load_tablet_checksum_items(*sql_proxy_, - cur_tablet_ls_pair_array_, get_compaction_scn(), tablet_checksum_items))) { + cur_tablet_ids_, get_compaction_scn(), tablet_checksum_items))) { LOG_WARN("fail to batch get tablet checksum items", KR(ret), "compaction_scn", get_compaction_scn()); } else if (replica_ckm_items_.empty() || tablet_checksum_items.empty() || replica_ckm_items_.get_tablet_cnt() != tablet_checksum_items.count()) { @@ -587,26 +581,27 @@ int ObChecksumValidator::try_update_tablet_checksum_items() const bool include_lager_than = (table_id_ == SPECIAL_TABLE_ID ? true : false); if (replica_ckm_items_.empty() && OB_FAIL(get_replica_ckm(include_lager_than))) { LOG_ERROR("fail to batch get tablet replica checksum items", KR(ret), "compaction_scn", get_compaction_scn()); - } else if (replica_ckm_items_.get_tablet_cnt() < cur_tablet_ls_pair_array_.count()) { + } else if (replica_ckm_items_.get_tablet_cnt() < cur_tablet_ids_.count()) { ret = OB_ITEM_NOT_MATCH; (void) uncompact_info_.add_skip_verify_table(table_id_); LOG_WARN("fail to get tablet replica checksum items", KR(ret), "compaction_scn", get_compaction_scn(), - K_(cur_tablet_ls_pair_array), K(replica_ckm_items_)); + K_(cur_tablet_ids), K(replica_ckm_items_)); } else if (OB_FAIL(push_tablet_ckm_items_with_update(replica_ckm_items_.get_array()))) { LOG_WARN("fail to push tablet checksum items", KR(ret)); } return ret; } -int ObChecksumValidator::push_finish_tablet_ls_pairs_with_update( - const common::ObIArray &tablet_ls_pairs) +int ObChecksumValidator::push_finish_tablet_ids_with_update( + const uint64_t table_id, + const common::ObIArray &tablet_ids) { int ret = OB_SUCCESS; - if (OB_FAIL(finish_tablet_ls_pair_array_.push_back(tablet_ls_pairs))) { - LOG_WARN("failed to push back tablet_ls_pairs", KR(ret), K(tablet_ls_pairs)); + if (OB_FAIL(finish_tablet_ids_.push_back(tablet_ids))) { + LOG_WARN("failed to push back tablet ids", KR(ret), K(table_id), K(tablet_ids)); } else { - bool need_update_report_scn = (finish_tablet_ls_pair_array_.count() >= MAX_BATCH_INSERT_COUNT) - || table_id_ == SPECIAL_TABLE_ID; + bool need_update_report_scn = (finish_tablet_ids_.count() >= MAX_BATCH_INSERT_COUNT) + || table_id == SPECIAL_TABLE_ID; #ifdef ERRSIM need_update_report_scn = true; #endif @@ -797,11 +792,11 @@ int ObChecksumValidator::verify_table_index( const uint64_t data_table_id = index_simple_schema.get_data_table_id(); if (replica_ckm_items_.empty() && OB_FAIL(get_replica_ckm())) { LOG_ERROR("fail to batch get tablet replica checksum items", KR(ret), "compaction_scn", get_compaction_scn()); - } else if (replica_ckm_items_.get_tablet_cnt() < cur_tablet_ls_pair_array_.count()) { + } else if (replica_ckm_items_.get_tablet_cnt() < cur_tablet_ids_.count()) { ret = OB_ITEM_NOT_MATCH; (void) uncompact_info_.add_skip_verify_table(table_id_); LOG_WARN("fail to get tablet replica checksum items", KR(ret), "compaction_scn", get_compaction_scn(), - K_(cur_tablet_ls_pair_array), K(replica_ckm_items_)); + K_(cur_tablet_ids), K(replica_ckm_items_)); } else { ObTableCkmItems data_table_ckm{}; ObTableCkmItems *data_table_ckm_ptr = nullptr; @@ -818,13 +813,12 @@ int ObChecksumValidator::verify_table_index( if (nullptr != data_table_ckm_ptr || OB_FAIL(ret)) { } else if (FALSE_IT(data_table_ckm_ptr = &data_table_ckm)) { } else if (OB_FAIL(data_table_ckm.build(data_table_id, get_compaction_scn(), - *sql_proxy_, *schema_guard_, - tablet_ls_pair_cache_))) { + *sql_proxy_, *schema_guard_))) { LOG_WARN("failed to get checksum items", K(ret), K(data_table_id), "compaction_scn", get_compaction_scn()); } else { ++statistics_.query_ckm_sql_cnt_; } - if (FAILEDx(index_table_ckm.build(*schema_guard_, index_simple_schema, cur_tablet_ls_pair_array_, + if (FAILEDx(index_table_ckm.build(*schema_guard_, index_simple_schema, cur_tablet_ids_, replica_ckm_items_))) { LOG_WARN("failed to assgin checksum items", K(ret), K(replica_ckm_items_)); } else { @@ -836,7 +830,7 @@ int ObChecksumValidator::verify_table_index( index_table_ckm))) { LOG_WARN("failed to validate checksum", K(ret), K(data_table_id), K(index_table_id), KPC(data_table_ckm_ptr), K(index_table_ckm), - K_(replica_ckm_items), K_(cur_tablet_ls_pair_array), K_(last_table_ckm_items)); + K_(replica_ckm_items), K_(cur_tablet_ids), K_(last_table_ckm_items)); } } } @@ -854,10 +848,14 @@ int ObChecksumValidator::verify_table_index( int ObChecksumValidator::get_replica_ckm(const bool include_larger_than/* = false*/) { + int ret = OB_SUCCESS; ++statistics_.query_ckm_sql_cnt_; - return ObTabletReplicaChecksumOperator::batch_get( - cur_tablet_ls_pair_array_, get_compaction_scn(), *sql_proxy_, - replica_ckm_items_, include_larger_than, 0); + if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_get( + cur_tablet_ids_, get_compaction_scn(), *sql_proxy_, + replica_ckm_items_, include_larger_than, 0))) { + LOG_WARN("failed to batch get tablet replica checksum items", KR(ret), K_(table_id), K_(cur_tablet_ids)); + } + return ret; } /***************************************** FTS Checksum Section ******************************************/ @@ -874,8 +872,7 @@ int ObChecksumValidator::build_ckm_item_for_fts(const int64_t table_id, } else if (OB_UNLIKELY(!table_compaction_info.is_compacted())) { LOG_WARN("exist special status table", KR(ret), K(table_compaction_info)); skip_verify = true; - } else if (OB_FAIL(ckm_item.build(table_id, get_compaction_scn(), *sql_proxy_, - *schema_guard_, tablet_ls_pair_cache_))) { + } else if (OB_FAIL(ckm_item.build(table_id, get_compaction_scn(), *sql_proxy_, *schema_guard_))) { if (OB_TABLE_NOT_EXIST == ret || OB_STATE_NOT_MATCH == ret || OB_ITEM_NOT_MATCH == ret) { skip_verify = true; ret = OB_SUCCESS; diff --git a/src/rootserver/freeze/ob_checksum_validator.h b/src/rootserver/freeze/ob_checksum_validator.h index 868eb7a90..f33523bf8 100644 --- a/src/rootserver/freeze/ob_checksum_validator.h +++ b/src/rootserver/freeze/ob_checksum_validator.h @@ -61,12 +61,11 @@ class ObChecksumValidator public: ObChecksumValidator( volatile bool &stop, - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache, const compaction::ObTabletStatusMap &tablet_status_map, compaction::ObTableCompactionInfoMap &table_compaction_map, compaction::ObIndexCkmValidatePairArray &idx_ckm_validate_array, compaction::ObCkmValidatorStatistics &statistics, - ObArray &finish_tablet_ls_pair_array, + ObArray &finish_tablet_ids, ObArray &finish_tablet_ckm_array, compaction::ObUncompactInfo &uncompact_info, ObFTSGroupArray &fts_group_array) @@ -81,11 +80,10 @@ class ObChecksumValidator major_merge_start_us_(0), statistics_(statistics), sql_proxy_(nullptr), - tablet_ls_pair_cache_(tablet_ls_pair_cache), tablet_status_map_(tablet_status_map), table_compaction_map_(table_compaction_map), idx_ckm_validate_array_(idx_ckm_validate_array), - finish_tablet_ls_pair_array_(finish_tablet_ls_pair_array), + finish_tablet_ids_(finish_tablet_ids), finish_tablet_ckm_array_(finish_tablet_ckm_array), uncompact_info_(uncompact_info), fts_group_array_(fts_group_array), @@ -117,8 +115,9 @@ class ObChecksumValidator } int push_tablet_ckm_items_with_update( const ObIArray &replica_ckm_items); - int push_finish_tablet_ls_pairs_with_update( - const common::ObIArray &tablet_ls_pairs); + int push_finish_tablet_ids_with_update( + const uint64_t table_id, + const common::ObIArray &tablet_ids); int batch_write_tablet_ckm(); int batch_update_report_scn(); static int check_column_checksum( @@ -135,7 +134,7 @@ class ObChecksumValidator int check_inner_status(); int get_table_compaction_info(const uint64_t table_id, compaction::ObTableCompactionInfo &table_compaction_info); int set_need_validate(); - int get_tablet_ls_pairs(const share::schema::ObSimpleTableSchemaV2 &simple_schema); + int get_tablet_ids(const share::schema::ObSimpleTableSchemaV2 &simple_schema); int get_replica_ckm(const bool include_larger_than = false); /* Tablet Replica Checksum Section */ int validate_tablet_replica_checksum(); @@ -185,11 +184,10 @@ class ObChecksumValidator compaction::ObCkmValidatorStatistics &statistics_; common::ObMySQLProxy *sql_proxy_; /* reference to obj in PorgressChecker */ - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache_; const compaction::ObTabletStatusMap &tablet_status_map_; compaction::ObTableCompactionInfoMap &table_compaction_map_; compaction::ObIndexCkmValidatePairArray &idx_ckm_validate_array_; - ObArray &finish_tablet_ls_pair_array_; + ObArray &finish_tablet_ids_; ObArray &finish_tablet_ckm_array_; compaction::ObUncompactInfo &uncompact_info_; ObFTSGroupArray &fts_group_array_; @@ -198,7 +196,7 @@ class ObChecksumValidator share::schema::ObSchemaGetterGuard *schema_guard_; const share::schema::ObSimpleTableSchemaV2 *simple_schema_; compaction::ObTableCompactionInfo table_compaction_info_; - ObArray cur_tablet_ls_pair_array_; + ObArray cur_tablet_ids_; share::ObReplicaCkmArray replica_ckm_items_; compaction::ObTableCkmItems last_table_ckm_items_; // only cached last data table with index }; diff --git a/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp b/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp index e550f0384..56c16968b 100644 --- a/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp +++ b/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp @@ -248,7 +248,7 @@ int ObDailyMajorFreezeLauncher::try_gc_freeze_info() int ObDailyMajorFreezeLauncher::try_gc_tablet_checksum() { int ret = OB_SUCCESS; - // keep 30 days for tablet_checksum whose (tablet_id, ls_id) is (1, 1) + // Keep 30 days of tablet checksums. const int64_t MAX_KEEP_INTERVAL_NS = 30LL * 24 * 60 * 60 * 1000 * 1000 * 1000; // 30 day const int64_t MIN_RESERVED_COUNT = 8; SCN cur_gts_scn; diff --git a/src/rootserver/freeze/ob_freeze_info_detector.cpp b/src/rootserver/freeze/ob_freeze_info_detector.cpp index 16821705e..08c1f7880 100644 --- a/src/rootserver/freeze/ob_freeze_info_detector.cpp +++ b/src/rootserver/freeze/ob_freeze_info_detector.cpp @@ -116,15 +116,8 @@ void ObMajorMergeInfoDetector::runTimerTask() bool can_work = false; bool skip_refresh_zone_info = false; - int64_t proposal_id = 0; - ObRole role = ObRole::INVALID_ROLE; - - if (OB_FAIL(obtain_proposal_id_from_ls(is_primary_service_, proposal_id, role))) { - LOG_WARN("fail to obtain proposal_id from ls", KR(ret)); - } else if (ObRole::LEADER != role) { - LOG_INFO("follower should not run freeze_info_detector", K(role), - K_(is_primary_service)); - } else if (OB_FAIL(can_start_work(can_work))) { + + if (OB_FAIL(can_start_work(can_work))) { LOG_WARN("fail to judge can start work", KR(ret)); } else if (can_work) { if (is_primary_service()) { @@ -424,29 +417,6 @@ bool ObMajorMergeInfoDetector::need_check_snapshot_gc_scn(const int64_t start_ti return (ObTimeUtility::current_time() - start_time_us) > START_CHECK_INTERVAL_US; } -int ObMajorMergeInfoDetector::obtain_proposal_id_from_ls( - const bool is_primary_service, - int64_t &proposal_id, - ObRole &role) -{ - int ret = OB_SUCCESS; - storage::ObLSHandle ls_handle; - logservice::ObLogHandler *handler = nullptr; - if (OB_ISNULL(share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls service is null", KR(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(SYS_LS, ls_handle, ObLSGetMod::RS_MOD))) { - LOG_WARN("fail to get ls", KR(ret)); - } else if (OB_ISNULL(ls_handle.get_ls()) - || OB_ISNULL(handler = ls_handle.get_ls()->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should not null", KR(ret), K(is_primary_service)); - } else if (OB_FAIL(handler->get_role(role, proposal_id))) { - LOG_WARN("fail to get role", KR(ret), K(is_primary_service)); - } - return ret; -} - void ObMajorMergeInfoDetector::update_last_run_timestamp_() { last_run_timestamp_ = ObTimeUtility::current_time(); diff --git a/src/rootserver/freeze/ob_freeze_info_detector.h b/src/rootserver/freeze/ob_freeze_info_detector.h index f5a4b93b0..67aeae8a7 100644 --- a/src/rootserver/freeze/ob_freeze_info_detector.h +++ b/src/rootserver/freeze/ob_freeze_info_detector.h @@ -68,9 +68,6 @@ class ObMajorMergeInfoDetector : public common::ObTimerTask // adjust global_merge_info in memory to avoid useless major freezes on restore major_freeze_service int try_adjust_global_merge_info(); int check_global_merge_info(bool &is_initial) const; - int obtain_proposal_id_from_ls(const bool is_primary_service, - int64_t &proposal_id, - common::ObRole &role); void update_last_run_timestamp_(); // For backup-restore tenant that switchover to primary tenant, FreezeInfoDetector is not able to // has write access immediately when it starts. Thus, FreezeInfoDetector can not renew diff --git a/src/rootserver/freeze/ob_freeze_reentrant_thread.cpp b/src/rootserver/freeze/ob_freeze_reentrant_thread.cpp index 5acf4592e..68c1e541b 100644 --- a/src/rootserver/freeze/ob_freeze_reentrant_thread.cpp +++ b/src/rootserver/freeze/ob_freeze_reentrant_thread.cpp @@ -79,26 +79,5 @@ int ObFreezeReentrantThread::try_idle( return ret; } -int ObFreezeReentrantThread::obtain_proposal_id_from_ls( - const bool is_primary_service, - int64_t &proposal_id, - ObRole &role) -{ - int ret = OB_SUCCESS; - - storage::ObLSHandle ls_handle; - logservice::ObLogHandler *handler = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(SYS_LS, ls_handle, ObLSGetMod::RS_MOD))) { - LOG_WARN("fail to get ls", KR(ret)); - } else if (OB_ISNULL(ls_handle.get_ls()) - || OB_ISNULL(handler = ls_handle.get_ls()->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should not null", KR(ret), K(is_primary_service)); - } else if (OB_FAIL(handler->get_role(role, proposal_id))) { - LOG_WARN("fail to get role", KR(ret), K(is_primary_service)); - } - return ret; -} - } // rootserver } // oceanbase diff --git a/src/rootserver/freeze/ob_freeze_reentrant_thread.h b/src/rootserver/freeze/ob_freeze_reentrant_thread.h index 54865ad99..0e8db9d74 100644 --- a/src/rootserver/freeze/ob_freeze_reentrant_thread.h +++ b/src/rootserver/freeze/ob_freeze_reentrant_thread.h @@ -44,8 +44,6 @@ class ObFreezeReentrantThread : public ObRsReentrantThread protected: virtual int try_idle(const int64_t idle_time_us, const int exe_ret); - int obtain_proposal_id_from_ls(const bool is_primary_service, int64_t &proposal_id, common::ObRole &role); - protected: common::ObMySQLProxy *sql_proxy_; diff --git a/src/rootserver/freeze/ob_major_freeze_helper.cpp b/src/rootserver/freeze/ob_major_freeze_helper.cpp index 340033233..ea5a05f12 100644 --- a/src/rootserver/freeze/ob_major_freeze_helper.cpp +++ b/src/rootserver/freeze/ob_major_freeze_helper.cpp @@ -19,7 +19,6 @@ #include "rootserver/freeze/ob_major_freeze_helper.h" #include "share/rc/ob_module_provider.h" #include "share/ob_freeze_info_proxy.h" -#include "share/location_cache/ob_location_service.h" #include "src/observer/ob_srv_network_frame.h" #include "share/ob_ex_rpc.h" #include "rootserver/freeze/ob_major_freeze_service.h" @@ -78,9 +77,6 @@ int ObMajorFreezeHelper::tablet_major_freeze(const ObTabletMajorFreezeParam &par if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(param)); - } else if (OB_UNLIKELY(nullptr == GCTX.location_service_ || nullptr == GCTX.net_frame_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid GCTX", KR(ret)); } else if (!GCONF.enable_major_freeze) { ret = OB_MAJOR_FREEZE_NOT_ALLOW; LOG_WARN("enable_major_freeze is off, refuse to to major_freeze", K(param), KR(ret)); @@ -88,29 +84,14 @@ int ObMajorFreezeHelper::tablet_major_freeze(const ObTabletMajorFreezeParam &par LOG_INFO("tablet major freeze", K(ret), K(param)); const int64_t start_time = ObTimeUtility::fast_current_time(); - share::ObLSID ls_id; - bool is_cache_hit = false; - - if (OB_FAIL(GCTX.location_service_->get( - param.tablet_id_, MAX_PROCESS_TIME_US, is_cache_hit, ls_id))) { - LOG_WARN("failed to get ls_id", K(ret), K(param.tablet_id_)); - } else { - // RPC removed: leader is self on single replica; dispatch in-process. - // Mirrors ObService::tablet_major_freeze local logic. - ret = ex_rpc::sync_call([&]() -> int { - int ret = OB_SUCCESS; - MOD_SCOPE { - if (OB_FAIL(share::g_mp->tenant_tablet_scheduler()->user_request_schedule_medium_merge( - ls_id, param.tablet_id_))) { - LOG_WARN("failed to try schedule tablet major freeze", K(ret), - K(ls_id), K(param)); - } - } - return ret; - }); + MOD_SCOPE { + if (OB_FAIL(share::g_mp->tenant_tablet_scheduler()->user_request_schedule_medium_merge( + param.tablet_id_))) { + LOG_WARN("failed to try schedule tablet major freeze", K(ret), K(param)); + } } const int64_t cost_time = ObTimeUtility::current_time() - start_time; - LOG_INFO("do tenant major freeze", KR(ret), K(ls_id), K(param), K(cost_time)); + LOG_INFO("do tenant major freeze", KR(ret), K(param), K(cost_time)); } return ret; } diff --git a/src/rootserver/freeze/ob_major_freeze_rpc_define.cpp b/src/rootserver/freeze/ob_major_freeze_rpc_define.cpp index 7dcf04a12..22b9e35b2 100644 --- a/src/rootserver/freeze/ob_major_freeze_rpc_define.cpp +++ b/src/rootserver/freeze/ob_major_freeze_rpc_define.cpp @@ -32,6 +32,5 @@ OB_SERIALIZE_MEMBER(ObTenantAdminMergeRequest, type_); OB_SERIALIZE_MEMBER(ObTenantAdminMergeResponse, err_code_); - } // namespace obcall } // namespace oceanbase diff --git a/src/rootserver/freeze/ob_major_freeze_service.cpp b/src/rootserver/freeze/ob_major_freeze_service.cpp index 0e153f5d9..d2c21715f 100644 --- a/src/rootserver/freeze/ob_major_freeze_service.cpp +++ b/src/rootserver/freeze/ob_major_freeze_service.cpp @@ -42,7 +42,7 @@ ObMajorFreezeService::~ObMajorFreezeService() ob_delete(tenant_major_freeze_); } -int ObMajorFreezeService::switch_to_leader() +int ObMajorFreezeService::activate() { ObRecursiveMutexGuard switch_guard(switch_lock_); int64_t start_time_us = ObTimeUtility::current_time(); @@ -66,20 +66,10 @@ int ObMajorFreezeService::switch_to_leader() return ret; } -int ObMajorFreezeService::switch_to_follower_gracefully() +void ObMajorFreezeService::deactivate() { int ret = OB_SUCCESS; - LOG_INFO("switch_to_follower_gracefully"); - if (OB_FAIL(inner_switch_to_follower())) { - LOG_WARN("fail to switch to follower", KR(ret)); - } - return ret; -} - -void ObMajorFreezeService::switch_to_follower_forcedly() -{ - int ret = OB_SUCCESS; - LOG_INFO("switch_to_follower_forcedly"); + LOG_INFO("deactivate major freeze service"); if (OB_FAIL(inner_switch_to_follower())) { LOG_WARN("fail to switch to follower", KR(ret)); } diff --git a/src/rootserver/freeze/ob_major_freeze_service.h b/src/rootserver/freeze/ob_major_freeze_service.h index 5d20c034e..2f1b8dc47 100644 --- a/src/rootserver/freeze/ob_major_freeze_service.h +++ b/src/rootserver/freeze/ob_major_freeze_service.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_ROOTSERVER_FREEZE_OB_MAJOR_FREEZE_SERVICE_ #define OCEANBASE_ROOTSERVER_FREEZE_OB_MAJOR_FREEZE_SERVICE_ -#include "share/ob_ls_id.h" #include "logservice/ob_log_base_type.h" #include "share/scn.h" #include "lib/lock/ob_recursive_mutex.h" @@ -38,7 +37,7 @@ enum ObMajorFreezeServiceType : uint8_t { class ObMajorFreezeService : public logservice::ObIReplaySubHandler, public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler + public logservice::ObILocalLogHandler { public: ObMajorFreezeService() @@ -73,11 +72,9 @@ class ObMajorFreezeService : public logservice::ObIReplaySubHandler, } // switch leader - void switch_to_follower_forcedly(); - int switch_to_leader(); + void deactivate() override; + int activate() override; - int switch_to_follower_gracefully(); - int resume_leader() { return switch_to_leader(); } int launch_major_freeze(const ObMajorFreezeReason freeze_reason); int suspend_merge(); diff --git a/src/rootserver/freeze/ob_major_merge_info_manager.cpp b/src/rootserver/freeze/ob_major_merge_info_manager.cpp index 77d88e7cf..21bfac24e 100644 --- a/src/rootserver/freeze/ob_major_merge_info_manager.cpp +++ b/src/rootserver/freeze/ob_major_merge_info_manager.cpp @@ -491,14 +491,10 @@ int ObMajorMergeInfoManager::adjust_global_merge_info() int ObMajorMergeInfoManager::get_gts(SCN >s_scn) const { int ret = OB_SUCCESS; - bool is_external_consistent = true; const int64_t timeout_us = 10 * 1000 * 1000; - if (OB_FAIL(OB_TS_MGR.get_ts_sync(timeout_us, gts_scn, is_external_consistent))) { + if (OB_FAIL(OB_TS_MGR.get_gts_sync(timeout_us, gts_scn))) { LOG_WARN("fail to get ts sync", K(ret), K(timeout_us)); - } else if (!is_external_consistent) { // only suppport gts in 4.0 - ret = OB_NOT_SUPPORTED; - LOG_ERROR("cannot freeze without gts", KR(ret)); } return ret; } diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index 2d94cde0a..1f9c75a40 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -45,9 +45,9 @@ ObMajorMergeProgressChecker::ObMajorMergeProgressChecker( freeze_info_(), sql_proxy_(nullptr), schema_service_(nullptr), progress_(), tablet_status_map_(), table_compaction_map_(), fts_group_array_(), - ckm_validator_(stop_, tablet_ls_pair_cache_, tablet_status_map_, + ckm_validator_(stop_, tablet_status_map_, table_compaction_map_, idx_ckm_validate_array_, validator_statistics_, - finish_tablet_ls_pair_array_, finish_tablet_ckm_array_, uncompact_info_, fts_group_array_), + finish_tablet_ids_, finish_tablet_ckm_array_, uncompact_info_, fts_group_array_), uncompact_info_(), total_time_guard_(), validator_statistics_(), batch_size_mgr_() {} int ObMajorMergeProgressChecker::init( @@ -68,7 +68,7 @@ int ObMajorMergeProgressChecker::init( LOG_WARN("fail to init checksum validator", KR(ret)); } else { idx_ckm_validate_array_.set_attr(ObMemAttr("RSCompCkmPair")); - finish_tablet_ls_pair_array_.set_attr(ObMemAttr("RSCompTabPair")); + finish_tablet_ids_.set_attr(ObMemAttr("RSCompTabIds")); finish_tablet_ckm_array_.set_attr(ObMemAttr("RSCompCkmArray")); sql_proxy_ = &sql_proxy; schema_service_ = &schema_service; @@ -127,12 +127,11 @@ int ObMajorMergeProgressChecker::clear_cached_info() first_loop_in_cur_round_ = true; table_ids_.reset(); idx_ckm_validate_array_.reset(); - finish_tablet_ls_pair_array_.reset(); + finish_tablet_ids_.reset(); finish_tablet_ckm_array_.reset(); progress_.reset(); ckm_validator_.clear_cached_info(); loop_cnt_ = 0; - tablet_ls_pair_cache_.reuse(); reset_uncompacted_tablets(); } return ret; @@ -216,7 +215,7 @@ int ObMajorMergeProgressChecker::check_table_merge_progress( int64_t idx = table_ids_.batch_start_idx_; int64_t table_cnt = 0; - SMART_VAR(ObArray, cur_tablet_ls_pair_array) { + SMART_VAR(ObArray, cur_tablet_ids) { for ( ; OB_SUCC(ret) && !stop_ && idx < table_ids_.count(); ++idx) { const uint64_t table_id = table_ids_.at(idx); const ObSimpleTableSchemaV2 *simple_schema = nullptr; @@ -243,14 +242,14 @@ int ObMajorMergeProgressChecker::check_table_merge_progress( } else if (OB_UNLIKELY(nullptr == simple_schema || !simple_schema->has_tablet())) { // like VIEW, it does not have tablet, treat it as compaction finished and can skip verifying table_compaction_info.set_can_skip_verifying(); - } else if (OB_FAIL(get_tablet_ls_pairs(table_id, *simple_schema, cur_tablet_ls_pair_array))) { + } else if (OB_FAIL(get_tablet_ids(*simple_schema, cur_tablet_ids))) { if (OB_ITEM_NOT_MATCH == ret) { ret = OB_SUCCESS; table_compaction_info.set_can_skip_verifying(); } else { - LOG_WARN("failed to get table pairs", K(ret), KPC(simple_schema)); + LOG_WARN("failed to get tablet ids", K(ret), KPC(simple_schema)); } - } else if (table_compaction_info.is_uncompacted() && OB_FAIL(update_table_compaction_info_by_tablet(cur_tablet_ls_pair_array, table_compaction_info))) { + } else if (table_compaction_info.is_uncompacted() && OB_FAIL(update_table_compaction_info_by_tablet(cur_tablet_ids, table_compaction_info))) { LOG_ERROR("failed to check table compaction finish", K(ret)); } else if (table_compaction_info.is_compacted()) { table_compaction_info.set_verified(); @@ -261,21 +260,21 @@ int ObMajorMergeProgressChecker::check_table_merge_progress( } else if (FALSE_IT(progress_.update_table_cnt(table_compaction_info.status_))) { } else if (table_compaction_info.finish_verified()) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(finish_tablet_ls_pair_array_.push_back(cur_tablet_ls_pair_array))) { - LOG_WARN("failed to push back tablet ls pair array", KR(tmp_ret), K(cur_tablet_ls_pair_array)); - } else if (finish_tablet_ls_pair_array_.count() < MAX_BATCH_INSERT_COUNT) { + if (OB_TMP_FAIL(finish_tablet_ids_.push_back(cur_tablet_ids))) { + LOG_WARN("failed to push back tablet ids", KR(tmp_ret), K(cur_tablet_ids)); + } else if (finish_tablet_ids_.count() < MAX_BATCH_INSERT_COUNT) { } else if (OB_TMP_FAIL(ObTabletMetaTableCompactionOperator::batch_update_report_scn(get_compaction_scn_val(), - finish_tablet_ls_pair_array_, + finish_tablet_ids_, ObTabletReplica::ScnStatus::SCN_STATUS_ERROR /*except_status*/))) { - LOG_WARN("fail to batch update report_scn", KR(tmp_ret), K_(finish_tablet_ls_pair_array)); + LOG_WARN("fail to batch update report_scn", KR(tmp_ret), K_(finish_tablet_ids)); } else { - LOG_INFO("success to batch update report_scn", "table_cnt", finish_tablet_ls_pair_array_.count()); - finish_tablet_ls_pair_array_.reuse(); + LOG_INFO("success to batch update report_scn", "table_cnt", finish_tablet_ids_.count()); + finish_tablet_ids_.reuse(); } } else if (OB_FAIL(unfinish_table_id_array.push_back(table_id))) { LOG_WARN("failed to push table_id into finish_array", KR(ret), K(table_compaction_info)); } - cur_tablet_ls_pair_array.reuse(); + cur_tablet_ids.reuse(); if (++table_cnt >= table_batch_size) { break; } @@ -291,20 +290,20 @@ int ObMajorMergeProgressChecker::check_table_merge_progress( } int ObMajorMergeProgressChecker::update_table_compaction_info_by_tablet( - const ObIArray &cur_tablet_ls_pair_array, + const ObIArray &cur_tablet_ids, ObTableCompactionInfo &table_compaction_info) { int ret = OB_SUCCESS; int64_t idx = 0; - for ( ; OB_SUCC(ret) && idx < cur_tablet_ls_pair_array.count(); ++idx) { - const ObTabletID &tablet_id = cur_tablet_ls_pair_array.at(idx).get_tablet_id(); + for ( ; OB_SUCC(ret) && idx < cur_tablet_ids.count(); ++idx) { + const ObTabletID &tablet_id = cur_tablet_ids.at(idx); ObTabletCompactionStatusEnum tablet_status = ObTabletCompactionStatusEnum::INITIAL; if (OB_FAIL(tablet_status_map_.get_refactored(tablet_id, tablet_status))) { // if tablet not finish compaction, it won't be added into this map if (OB_HASH_NOT_EXIST == ret) { ret = OB_SUCCESS; table_compaction_info.set_uncompacted(); - LOG_TRACE("tablet not exist in tablet status map", KR(ret), K(tablet_id), K(cur_tablet_ls_pair_array), K(table_compaction_info)); + LOG_TRACE("tablet not exist in tablet status map", KR(ret), K(tablet_id), K(cur_tablet_ids), K(table_compaction_info)); #ifdef ERRSIM ret = OB_E(EventTable::EN_SKIP_INDEX_MAJOR) ret; if (OB_FAIL(ret)) { @@ -329,8 +328,8 @@ int ObMajorMergeProgressChecker::update_table_compaction_info_by_tablet( } if (OB_SUCC(ret)) { - if (idx == cur_tablet_ls_pair_array.count()) { // loop finish - table_compaction_info.tablet_cnt_ = cur_tablet_ls_pair_array.count(); + if (idx == cur_tablet_ids.count()) { // loop finish + table_compaction_info.tablet_cnt_ = cur_tablet_ids.count(); table_compaction_info.set_compacted(); } LOG_TRACE("update table compaction info by tablet", KR(ret), K(table_compaction_info)); @@ -339,10 +338,9 @@ int ObMajorMergeProgressChecker::update_table_compaction_info_by_tablet( } -int ObMajorMergeProgressChecker::get_tablet_ls_pairs( - const uint64_t table_id, +int ObMajorMergeProgressChecker::get_tablet_ids( const share::schema::ObSimpleTableSchemaV2 &simple_schema, - ObIArray &cur_tablet_ls_pair_array) + ObIArray &cur_tablet_ids) { int ret = OB_SUCCESS; SMART_VAR(ObArray, tablet_ids) { @@ -351,13 +349,13 @@ int ObMajorMergeProgressChecker::get_tablet_ls_pairs( } else if (OB_UNLIKELY(tablet_ids.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get tablet_ids of current table schema", KR(ret), K(simple_schema)); - } else if (OB_FAIL(cur_tablet_ls_pair_array.reserve(tablet_ids.count()))) { + } else if (OB_FAIL(cur_tablet_ids.reserve(tablet_ids.count()))) { LOG_WARN("failed to reserve tablet array", KR(ret), K(tablet_ids.count())); - } else if (OB_FAIL(tablet_ls_pair_cache_.get_tablet_ls_pairs(table_id, tablet_ids, cur_tablet_ls_pair_array))) { - LOG_WARN("failed to tablet ls pair", KR(ret), K(table_id), K(tablet_ids)); - } else if (OB_UNLIKELY(cur_tablet_ls_pair_array.empty())) { + } else if (OB_FAIL(cur_tablet_ids.assign(tablet_ids))) { + LOG_WARN("failed to assign tablet ids", KR(ret), K(tablet_ids)); + } else if (OB_UNLIKELY(cur_tablet_ids.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet ls pair array is unexpected empty", KR(ret), K(simple_schema), K(cur_tablet_ls_pair_array)); + LOG_WARN("tablet id array is unexpected empty", KR(ret), K(simple_schema), K(cur_tablet_ids)); } else { #ifdef ERRSIM static int64_t enter_cnt = 0; @@ -366,14 +364,14 @@ int ObMajorMergeProgressChecker::get_tablet_ls_pairs( if (OB_FAIL(ret)) { if (enter_cnt++ == 0) { ret = OB_ITEM_NOT_MATCH; - STORAGE_LOG(INFO, "ERRSIM EN_GET_TABLET_LS_PAIR_IN_RS", K(ret), K(simple_schema), K(cur_tablet_ls_pair_array)); + STORAGE_LOG(INFO, "ERRSIM EN_GET_TABLET_LS_PAIR_IN_RS", K(ret), K(simple_schema), K(cur_tablet_ids)); } else { ret = OB_SUCCESS; } } } #endif - LOG_TRACE("success to get tablet ls pairs", KR(ret), K(cur_tablet_ls_pair_array)); + LOG_TRACE("success to get tablet ids", KR(ret), K(cur_tablet_ids)); } } return ret; @@ -618,10 +616,8 @@ int ObMajorMergeProgressChecker::prepare_check_progress( } if (FAILEDx(rebuild_tablet_status_map())) { LOG_WARN("failed to rebuild tablet status map by tablet cnt", K(ret)); - } else if (OB_FAIL(tablet_ls_pair_cache_.try_refresh(first_loop_in_cur_round_ /*force_refresh*/))) { - LOG_WARN("failed to refresh tablet ls pair", K(ret)); } else { - tmp_time_guard.click(ObRSCompactionTimeGuard::GET_TABLET_LS_PAIRS); + tmp_time_guard.click(ObRSCompactionTimeGuard::GET_TABLET_IDS); first_loop_in_cur_round_ = false; exist_uncompacted_table = progress_.exist_uncompacted_table(); progress_.clear_before_each_loop(); @@ -656,7 +652,7 @@ int ObMajorMergeProgressChecker::check_index_and_rest_table() } else #endif if (OB_FAIL(ckm_validator_.deal_with_special_table_at_last(finish_validate))) { - LOG_WARN("fail to handle table with first tablet in sys ls", KR(ret), + LOG_WARN("fail to handle first tablet checksum", KR(ret), "compaction_scn", get_compaction_scn()); } else if (finish_validate) { progress_.deal_with_special_tablet(); @@ -752,7 +748,7 @@ int ObMajorMergeProgressChecker::check_progress() const int64_t cost_us = ObTimeUtility::fast_current_time() - start_time; ++loop_cnt_; if (OB_SUCCESS == last_errno_) { - DEL_SUSPECT_INFO(compaction::MAJOR_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, share::ObDiagnoseTabletType::TYPE_RS_MAJOR_MERGE); + DEL_SUSPECT_INFO(compaction::MAJOR_MERGE, UNKNOW_TABLET_ID, share::ObDiagnoseTabletType::TYPE_RS_MAJOR_MERGE); } print_unfinish_info(cost_us); @@ -910,9 +906,9 @@ int ObMajorMergeProgressChecker::deal_with_validated_table( } if (OB_SUCC(ret) && validate_finish) { if (table_ckm.is_inited() ) { - if (OB_TMP_FAIL(ckm_validator_.push_finish_tablet_ls_pairs_with_update( - table_ckm.get_tablet_ls_pairs()))) { - LOG_WARN("failed to push back tablet_ls_pair", KR(tmp_ret)); + if (OB_TMP_FAIL(ckm_validator_.push_finish_tablet_ids_with_update( + table_id, table_ckm.get_tablet_ids()))) { + LOG_WARN("failed to push back tablet ids", KR(tmp_ret), K(table_id)); } if (OB_TMP_FAIL(ckm_validator_.push_tablet_ckm_items_with_update( table_ckm.get_ckm_items()))) { @@ -949,7 +945,7 @@ int ObMajorMergeProgressChecker::loop_index_ckm_validate_array() data_table_ckm.clear(); prev_data_table_id = data_table_id; if (OB_FAIL(data_table_ckm.build(data_table_id, get_compaction_scn(), - *sql_proxy_, schema_guard, get_tablet_ls_pair_cache()))) { + *sql_proxy_, schema_guard))) { LOG_WARN("fail to prepare schema checksum items", KR(ret), K(data_table_id)); } else { ++validator_statistics_.query_ckm_sql_cnt_; @@ -991,8 +987,7 @@ int ObMajorMergeProgressChecker::get_idx_ckm_and_validate( // only for case : check special index table first if (should_handle_index_table) { if (OB_FAIL(index_table_ckm.build(index_table_id, get_compaction_scn(), - *sql_proxy_, schema_guard, - get_tablet_ls_pair_cache()))) { + *sql_proxy_, schema_guard))) { LOG_WARN("failed to get checksum items", KR(ret), K(index_table_id), "compaction_scn", get_compaction_scn()); } else if (OB_UNLIKELY(index_table_ckm.get_table_schema()->should_not_validate_data_index_ckm())) { ret = OB_ERR_UNEXPECTED; @@ -1084,27 +1079,23 @@ int ObMajorMergeProgressChecker::generate_tablet_status_map() ObTabletCompactionStatusEnum status = ObTabletCompactionStatusEnum::COMPACTED; SCN replica_snapshot_scn; SCN report_scn; - const ObLSID &ls_id = tablet_info.get_ls_id(); - FOREACH_CNT_X(replica, tablet_info.get_replicas(), OB_SUCC(ret)) { - filter = false; - if (OB_FAIL(replica_snapshot_scn.convert_for_tx(replica->get_snapshot_version()))) { - LOG_WARN("fail to convert val to SCN", KR(ret), KPC(replica)); - } else if (OB_UNLIKELY(ObTabletReplica::ScnStatus::SCN_STATUS_ERROR == replica->get_status())) { - ret = OB_CHECKSUM_ERROR; - LOG_ERROR("ERROR! ERROR! ERROR! find error status tablet replica", KR(ret), K(tablet_info)); - } else if (replica_snapshot_scn < get_compaction_scn()) { - status = ObTabletCompactionStatusEnum::INITIAL; - (void) uncompact_info_.add_tablet(*replica); - LOG_TRACE("unfinish tablet", KR(ret), KPC(replica), K(replica_snapshot_scn), "compaction_scn", get_compaction_scn()); - break; - } else if (OB_FAIL(report_scn.convert_for_tx(replica->get_report_scn()))) { // check report_scn - LOG_WARN("fail to convert val to SCN", KR(ret), KPC(replica)); - } else if (report_scn >= get_compaction_scn() - || replica_snapshot_scn > get_compaction_scn()) { - status = ObTabletCompactionStatusEnum::CAN_SKIP_VERIFYING; - break; - } - } // end of FOREACH + const ObTabletReplica &replica = tablet_info.get_replica(); + filter = false; + if (OB_FAIL(replica_snapshot_scn.convert_for_tx(replica.get_snapshot_version()))) { + LOG_WARN("fail to convert val to SCN", KR(ret), K(replica)); + } else if (OB_UNLIKELY(ObTabletReplica::ScnStatus::SCN_STATUS_ERROR == replica.get_status())) { + ret = OB_CHECKSUM_ERROR; + LOG_ERROR("ERROR! ERROR! ERROR! find error status tablet replica", KR(ret), K(tablet_info)); + } else if (replica_snapshot_scn < get_compaction_scn()) { + status = ObTabletCompactionStatusEnum::INITIAL; + (void) uncompact_info_.add_tablet(replica); + LOG_TRACE("unfinish tablet", KR(ret), K(replica), K(replica_snapshot_scn), "compaction_scn", get_compaction_scn()); + } else if (OB_FAIL(report_scn.convert_for_tx(replica.get_report_scn()))) { // check report_scn + LOG_WARN("fail to convert val to SCN", KR(ret), K(replica)); + } else if (report_scn >= get_compaction_scn() + || replica_snapshot_scn > get_compaction_scn()) { + status = ObTabletCompactionStatusEnum::CAN_SKIP_VERIFYING; + } if (OB_SUCC(ret) && ObTabletCompactionStatusEnum::INITIAL != status) { ++progress_.merged_tablet_cnt_; if (OB_FAIL(tablet_status_map_.set_refactored(tablet_info.get_tablet_id(), status, 1/*overwrite*/))) { diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.h b/src/rootserver/freeze/ob_major_merge_progress_checker.h index 0041f6412..1cd067415 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.h +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.h @@ -100,7 +100,6 @@ class ObMajorMergeProgressChecker : public ObBasicMergeProgressChecker OB_INLINE virtual void reset_uncompacted_tablets() override { uncompact_info_.reset(); } virtual int check_progress() override; const compaction::ObBasicMergeProgress &get_merge_progress() const override { return progress_; } - const compaction::ObTabletLSPairCache &get_tablet_ls_pair_cache() const { return tablet_ls_pair_cache_; } private: int set_table_compaction_info_status(const uint64_t table_id, const compaction::ObTableCompactionInfo::Status status); @@ -122,12 +121,11 @@ class ObMajorMergeProgressChecker : public ObBasicMergeProgressChecker int check_table_merge_progress( share::schema::ObSchemaGetterGuard &schema_guard, ObIArray &unfinish_table_id_array); - int get_tablet_ls_pairs( - const uint64_t table_id, + int get_tablet_ids( const share::schema::ObSimpleTableSchemaV2 &simple_schema, - ObIArray &cur_tablet_ls_pair_array); + ObIArray &cur_tablet_ids); int update_table_compaction_info_by_tablet( - const ObIArray &cur_tablet_ls_pair_array, + const ObIArray &cur_tablet_ids, compaction::ObTableCompactionInfo &table_compaction_info); int prepare_unfinish_table_ids(); int check_schema_version(); @@ -192,13 +190,12 @@ class ObMajorMergeProgressChecker : public ObBasicMergeProgressChecker ObFTSGroupArray fts_group_array_; ObChecksumValidator ckm_validator_; compaction::ObUncompactInfo uncompact_info_; - // cache of ls_infos in __all_ls_meta_table + // cache of log stream meta info // statistics section compaction::ObRSCompactionTimeGuard total_time_guard_; compaction::ObCkmValidatorStatistics validator_statistics_; - compaction::ObTabletLSPairCache tablet_ls_pair_cache_; compaction::ObScheduleBatchSizeMgr batch_size_mgr_; - ObArray finish_tablet_ls_pair_array_; + ObArray finish_tablet_ids_; ObArray finish_tablet_ckm_array_; DISALLOW_COPY_AND_ASSIGN(ObMajorMergeProgressChecker); }; diff --git a/src/rootserver/freeze/ob_major_merge_progress_util.cpp b/src/rootserver/freeze/ob_major_merge_progress_util.cpp index ccefedeb1..38ed3e57a 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_util.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_util.cpp @@ -15,8 +15,6 @@ */ #define USING_LOG_PREFIX RS_COMPACTION #include "rootserver/freeze/ob_major_merge_progress_util.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" -#include "share/compaction/ob_schedule_batch_size_mgr.h" #include "src/share/ob_tablet_replica_checksum_operator.h" namespace oceanbase @@ -25,8 +23,6 @@ using namespace share; using namespace common; namespace compaction { -ERRSIM_POINT_DEF(EN_COMPACTION_SKIP_CREATE_MAP); - ObTableCompactionInfo &ObTableCompactionInfo::operator=(const ObTableCompactionInfo &other) { table_id_ = other.table_id_; @@ -88,188 +84,6 @@ int64_t ObMergeProgress::to_string(char *buf, const int64_t buf_len) const return pos; } -/** - * -------------------------------------------------------------------ObTabletLSPairCache------------------------------------------------------------------- - */ -const int64_t ObTabletLSPairCache::TABLET_LS_MAP_BUCKET_CNT; -const int64_t ObTabletLSPairCache::RANGE_SIZE; -ObTabletLSPairCache::ObTabletLSPairCache() - : last_refresh_ts_(0) -{ -} - -ObTabletLSPairCache::~ObTabletLSPairCache() -{ - destroy(); -} - -void ObTabletLSPairCache::reuse() -{ - last_refresh_ts_ = 0; - map_.reuse(); -} - -void ObTabletLSPairCache::destroy() -{ - last_refresh_ts_ = 0; - if (map_.created()) { - map_.destroy(); - } -} - -int ObTabletLSPairCache::refresh() -{ - int ret = OB_SUCCESS; - ObTabletID start_tablet_id; - int64_t cost_ts = ObTimeUtility::fast_current_time(); - if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is unexpected null", KR(ret)); - } else { - map_.reuse(); - } - SMART_VAR(ObArray, tablet_ls_pair_array) { - tablet_ls_pair_array.set_attr(ObMemAttr("RSCompPairCache")); - if (OB_FAIL(tablet_ls_pair_array.reserve(RANGE_SIZE))) { - LOG_WARN("failed to reserve array", KR(ret)); - } - while (OB_SUCC(ret)) { - tablet_ls_pair_array.reuse(); - if (OB_FAIL(ObTabletToLSTableOperator::range_get_tablet( - *GCTX.sql_proxy_, - start_tablet_id, - RANGE_SIZE, - tablet_ls_pair_array))) { - LOG_WARN("fail to get a range of tablet through tablet_to_ls_table_operator", - KR(ret), K(start_tablet_id), K(RANGE_SIZE), - K(tablet_ls_pair_array)); - } else if (tablet_ls_pair_array.empty()) { - break; -#ifdef ERRSIM - } else if (OB_UNLIKELY(EN_COMPACTION_SKIP_CREATE_MAP)) { - LOG_INFO("ERRSIM EN_COMPACTION_SKIP_CREATE_MAP, skip set map"); // tablet_ls_pair_array is empty - break; -#endif - } else { - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_ls_pair_array.count(); ++idx) { - ObTabletLSPair &pair = tablet_ls_pair_array.at(idx); - if (OB_FAIL(map_.set_refactored(pair.get_tablet_id(), pair.get_ls_id()))) { - LOG_WARN("fail set tablet ls map", KR(ret)); - } - } // end of for - if (OB_SUCC(ret)) { - start_tablet_id = tablet_ls_pair_array.at(tablet_ls_pair_array.count() - 1).get_tablet_id(); - } - } - } // end of while - } - if (OB_SUCC(ret)) { - last_refresh_ts_ = ObTimeUtility::fast_current_time(); - cost_ts = last_refresh_ts_ - cost_ts; - LOG_INFO("success to refresh tablet ls pair cache", KR(ret), K(cost_ts), "map_item_cnt", map_.size(), K(map_.bucket_count())); - } - return ret; -} - -int ObTabletLSPairCache::rebuild_map_by_tablet_cnt() -{ - int ret = OB_SUCCESS; - int64_t tablet_cnt = 0; - if (map_.empty()) { - if (OB_FAIL(ObTabletToLSTableOperator::get_tablet_ls_pairs_cnt(*GCTX.sql_proxy_, tablet_cnt))) { - LOG_WARN("failed to get tablet_ls pair cnt", KR(ret)); - } -#ifdef ERRSIM - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(EN_COMPACTION_SKIP_CREATE_MAP)) { - tablet_cnt = 0; - LOG_INFO("ERRSIM EN_COMPACTION_SKIP_CREATE_MAP, set tablet_cnt to 0", K(tablet_cnt)); - } -#endif - } else { - tablet_cnt = map_.size(); - } - if (OB_FAIL(ret) || tablet_cnt == 0) { - } else { - int64_t recommend_map_bucked_cnt = 0; - const int64_t cur_bucket_cnt = map_.created() ? map_.bucket_count() : 0; - bool rebuild_map_flag = ObScheduleBatchSizeMgr::need_rebuild_map( - TABLET_LS_MAP_BUCKET_CNT, tablet_cnt, cur_bucket_cnt, recommend_map_bucked_cnt); - if (rebuild_map_flag) { - if (map_.created()) { - map_.destroy(); - } - if (OB_FAIL(map_.create(recommend_map_bucked_cnt, "RSCompPairCache", "RSCompPairCache"))) { - LOG_WARN("fail to create tablet ls pair map", KR(ret), K(recommend_map_bucked_cnt)); - } else { - LOG_INFO("success to rebuild or create map", KR(ret), K(tablet_cnt), K(map_.bucket_count())); - } - } - } - return ret; -} - -int ObTabletLSPairCache::try_refresh(const bool force_refresh/* = false*/) -{ - int ret = OB_SUCCESS; - if ((force_refresh || !map_.created()) && OB_FAIL(rebuild_map_by_tablet_cnt())) { - LOG_ERROR("failed to rebuild map by tablet cnt", KR(ret), K(force_refresh)); - } else if (force_refresh) { - if (OB_FAIL(refresh())) { - LOG_WARN("failed to refresh", KR(ret)); - } - } - return ret; -} - -int ObTabletLSPairCache::get_tablet_ls_pairs( - const uint64_t table_id, - const ObIArray &tablet_ids, - ObIArray &pairs) const -{ - int ret = OB_SUCCESS; - if (true || is_sys_table(table_id)) { - ObLSID tmp_ls_id(ObLSID::SYS_LS_ID); - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_ids.count(); ++idx) { - if (OB_FAIL(pairs.push_back(ObTabletLSPair(tablet_ids.at(idx), tmp_ls_id)))) { - LOG_WARN("fail to push back pair", KR(ret), K(table_id)); - } - } // end of for - } else { - ObLSID tmp_ls_id; - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_ids.count(); ++idx) { - if (OB_FAIL(map_.get_refactored(tablet_ids.at(idx), tmp_ls_id))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ITEM_NOT_MATCH; - } else { - LOG_WARN("failed to get ls id", KR(ret), K(idx), K(tablet_ids)); - } - } else if (OB_FAIL(pairs.push_back(ObTabletLSPair(tablet_ids.at(idx), tmp_ls_id)))) { - LOG_WARN("fail to push back pair", KR(ret), K(table_id)); - } - } // end of for - } - return ret; -} - -int ObTabletLSPairCache::get_tablet_ls_id( - const uint64_t table_id, - const ObTabletID tablet_id, - share::ObLSID &ls_id) const -{ - int ret = OB_SUCCESS; - if (true || is_sys_table(table_id)) { - ls_id = ObLSID(ObLSID::SYS_LS_ID); - } else if (OB_FAIL(map_.get_refactored(tablet_id, ls_id))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ITEM_NOT_MATCH; - } else { - LOG_WARN("failed to get ls id", KR(ret), K(table_id), K(tablet_id)); - } - } - return ret; -} - /** * -------------------------------------------------------------------ObUncompactInfo------------------------------------------------------------------- */ @@ -322,13 +136,11 @@ void ObUncompactInfo::add_tablet(const share::ObTabletReplica &replica) } } -void ObUncompactInfo::add_tablet( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +void ObUncompactInfo::add_tablet(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; ObTabletReplica fake_replica; - fake_replica.fake_for_diagnose( ls_id, tablet_id); + fake_replica.fake_for_diagnose(tablet_id); SpinWLockGuard w_guard(diagnose_rw_lock_); if (tablets_.count() < DEBUG_INFO_CNT && OB_FAIL(tablets_.push_back(fake_replica))) { diff --git a/src/rootserver/freeze/ob_major_merge_progress_util.h b/src/rootserver/freeze/ob_major_merge_progress_util.h index bad54a3e9..a8909c3aa 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_util.h +++ b/src/rootserver/freeze/ob_major_merge_progress_util.h @@ -17,7 +17,6 @@ #define OB_ROOTSERVER_FREEZE_MAJOR_MERGE_PROGRESS_UTIL_H_ #include "share/compaction/ob_compaction_time_guard.h" #include "share/ob_delegate.h" -#include "share/ob_ls_id.h" #include "share/tablet/ob_tablet_info.h" namespace oceanbase { @@ -231,7 +230,6 @@ struct ObUnfinishTableIds }; typedef hash::ObHashMap ObTabletStatusMap; -typedef common::ObArray ObTabletLSPairArray; typedef hash::ObHashMap ObTableCompactionInfoMap; struct ObCkmValidatorStatistics @@ -254,36 +252,6 @@ struct ObCkmValidatorStatistics int64_t checker_validate_idx_cnt_; }; -// single thread operation -struct ObTabletLSPairCache -{ -public: - ObTabletLSPairCache(); - ~ObTabletLSPairCache(); - - void reuse(); - void destroy(); - int try_refresh(const bool force_refresh = false); - int get_tablet_ls_pairs( - const uint64_t table_id, - const ObIArray &tablet_ids, - ObIArray &pairs) const; - int get_tablet_ls_id( - const uint64_t table_id, - const ObTabletID tablet_id, - share::ObLSID &ls_id) const; - TO_STRING_KV(K_(last_refresh_ts), "map_cnt", map_.size()); -private: - int refresh(); - int rebuild_map_by_tablet_cnt(); - const static int64_t RANGE_SIZE = 1000; - const static int64_t REFRESH_CACHE_TIME_INTERVAL = 60 * 1000 * 1000; // 1m - const static int64_t TABLET_LS_MAP_BUCKET_CNT = 3000; - const static int64_t TABLET_LS_MAP_BUCKET_MAX_CNT = 300000; - int64_t last_refresh_ts_; - hash::ObHashMap map_; -}; - struct ObUncompactInfo { public: @@ -293,9 +261,7 @@ struct ObUncompactInfo void add_table(const uint64_t table_id); void add_skip_verify_table(const uint64_t table_id); void add_tablet(const share::ObTabletReplica &replica); - void add_tablet( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id); + void add_tablet(const common::ObTabletID &tablet_id); int get_uncompact_info( common::ObIArray &input_tablets, common::ObIArray &input_table_ids) const; diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index f60993415..6d37a0321 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -348,7 +348,6 @@ int ObBootstrap::prepare_create_partitions( LOG_WARN("check_inner_stat failed", KR(ret)); } else if (tschema.has_partition()) { common::ObArray table_schema_ptrs; - common::ObArray ls_id_array; common::ObArray need_create_empty_majors; ObArray index_tids; uint64_t lob_meta_table_id = 0; @@ -392,17 +391,11 @@ int ObBootstrap::prepare_create_partitions( } } - for (int64_t j = 0; OB_SUCC(ret) && j < tschema.get_all_part_num(); ++j) { - if (OB_FAIL(ls_id_array.push_back(share::ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back ls id", KR(ret)); - } - } } } if (OB_SUCC(ret) && OB_FAIL(creator.add_create_tablets_of_tables_arg( table_schema_ptrs, - ls_id_array, DATA_CURRENT_VERSION, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_ERROR("fail to add create tablet arg", KR(ret)); @@ -436,7 +429,6 @@ int ObBootstrap::prepare_create_partition( } else if (tschema.has_partition()) { common::ObArray table_schema_array; common::ObArray table_schema_ptrs; - common::ObArray ls_id_array; common::ObArray need_create_empty_majors; if (OB_FAIL(generate_table_schema_array_for_create_partition(tschema, table_schema_array))) { LOG_WARN("fail to generate table schema array", KR(ret)); @@ -453,18 +445,11 @@ int ObBootstrap::prepare_create_partition( LOG_WARN("fail to push back", KR(ret), K(table_schema_array)); } } - - for (int i = 0; i < tschema.get_all_part_num() && OB_SUCC(ret); ++i) { - if (OB_FAIL(ls_id_array.push_back(share::ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back", KR(ret)); - } - } } if (OB_FAIL(ret)) { } else if (OB_FAIL(creator.add_create_tablets_of_tables_arg( table_schema_ptrs, - ls_id_array, DATA_CURRENT_VERSION, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_WARN("fail to add create tablet arg", KR(ret)); diff --git a/src/rootserver/ob_create_index_on_empty_table_helper.cpp b/src/rootserver/ob_create_index_on_empty_table_helper.cpp index b891d32a6..388d6a22e 100644 --- a/src/rootserver/ob_create_index_on_empty_table_helper.cpp +++ b/src/rootserver/ob_create_index_on_empty_table_helper.cpp @@ -68,14 +68,11 @@ int ObCreateIndexOnEmptyTableHelper::check_create_index_on_empty_table_opt( int ObCreateIndexOnEmptyTableHelper::get_major_frozen_scn(share::SCN &major_frozen_scn) { int ret = OB_SUCCESS; - bool is_external_consistent = false; ObTimeoutCtx ctx; if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { LOG_WARN("fail to set timeout ctx", KR(ret)); - } else if (OB_FAIL(OB_TS_MGR.get_ts_sync(ctx.get_timeout(), - major_frozen_scn, - is_external_consistent))) { - LOG_WARN("fail to get gts sync", KR(ret), K(ctx.get_timeout()), K(major_frozen_scn), K(is_external_consistent)); + } else if (OB_FAIL(OB_TS_MGR.get_gts_sync(ctx.get_timeout(), major_frozen_scn))) { + LOG_WARN("fail to get gts sync", KR(ret), K(ctx.get_timeout()), K(major_frozen_scn)); } return ret; } diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index edaf7e596..6f358866a 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -32,7 +32,6 @@ #include "share/ob_zone_merge_table_operator.h" #include "share/ob_zone_merge_info.h" #include "sql/optimizer/stat/ob_dbms_stats_maintenance_window.h" -#include "storage/mview/ob_mview_sched_job_utils.h" #include "pl/ob_pl_persistent.h" #include "pl/pl_cache/ob_pl_cache_mgr.h" #include "pl/pl_recompile/ob_pl_recompile_task_helper.h" @@ -76,8 +75,6 @@ ObSysStat::ObSysStat() ob_max_used_unit_group_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_UNIT_GROUP_ID_TYPE)), ob_max_used_normal_rowid_table_tablet_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_NORMAL_ROWID_TABLE_TABLET_ID_TYPE)), ob_max_used_extended_rowid_table_tablet_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_EXTENDED_ROWID_TABLE_TABLET_ID_TYPE)), - ob_max_used_ls_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_LS_ID_TYPE)), - ob_max_used_ls_group_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_LS_GROUP_ID_TYPE)), ob_max_used_sys_pl_object_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_SYS_PL_OBJECT_ID_TYPE)), ob_max_used_object_id_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_OBJECT_ID_TYPE)), ob_max_used_rewrite_rule_version_(item_list_, MAX_ID_NAME_INFO(OB_MAX_USED_REWRITE_RULE_VERSION_TYPE)) @@ -100,8 +97,6 @@ int ObSysStat::set_initial_values() if (OB_SUCC(ret)) { ob_max_used_normal_rowid_table_tablet_id_.value_.set_int(ObTabletID::MIN_USER_NORMAL_ROWID_TABLE_TABLET_ID); ob_max_used_extended_rowid_table_tablet_id_.value_.set_int(ObTabletID::MIN_USER_EXTENDED_ROWID_TABLE_TABLET_ID); - ob_max_used_ls_id_.value_.set_int(ObLSID::MIN_USER_LS_ID); - ob_max_used_ls_group_id_.value_.set_int(ObLSID::MIN_USER_LS_GROUP_ID); ob_max_used_sys_pl_object_id_.value_.set_int(OB_MIN_SYS_PL_OBJECT_ID); // Use OB_INITIAL_TEST_DATABASE_ID to avoid confict when create tenant with initial user schema objects. ob_max_used_object_id_.value_.set_int(OB_INITIAL_TEST_DATABASE_ID); @@ -302,7 +297,7 @@ int ObDDLOperator::drop_database(const ObDatabaseSchema &db_schema, } else if (table->is_in_recyclebin()) { // already been dropped before } else { - bool is_delete_first = table->is_aux_table() || table->is_mlog_table(); + bool is_delete_first = table->is_aux_table(); if ((0 == cycle ? is_delete_first : !is_delete_first)) { // drop triggers before drop table if (OB_FAIL(ObPLDDLOperator::drop_trigger_cascade(*table, trans, *this))) { @@ -3723,51 +3718,6 @@ int ObDDLOperator::update_indexes_type(const ObTableSchema &data_table_schema, return ret; } -int ObDDLOperator::switch_mlog_status(const share::schema::ObTableSchema &data_table_schema, - const uint64_t old_mlog_id, - const uint64_t new_mlog_id, - ObSchemaGetterGuard &schema_guard, - common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObSchemaService *schema_service = schema_service_.get_schema_service(); - int64_t new_schema_version = OB_INVALID_VERSION; - const ObTableSchema *old_mlog_schema = nullptr; - const ObTableSchema *new_mlog_schema = nullptr; - - uint64_t data_table_id = data_table_schema.get_table_id(); - - if (OB_ISNULL(schema_service)) { - ret = OB_ERR_SYS; - LOG_WARN("schema service should not be NULL"); - } else if (OB_FAIL(schema_guard.get_table_schema( old_mlog_id, old_mlog_schema))) { - LOG_WARN("get table schema failed", K(ret)); - } else if (nullptr == old_mlog_schema) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, table schema must not be nullptr", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( new_mlog_id, new_mlog_schema))) { - LOG_WARN("get table schema failed", K(ret)); - } else if (nullptr == new_mlog_schema) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, table schema must not be nullptr", K(ret)); - } else if (OB_FAIL(schema_service_.gen_new_schema_version(new_schema_version))) { - LOG_WARN("fail to gen new schema_version", K(ret)); - } else if (OB_FAIL(schema_service->get_table_sql_service().update_mlog_status( - data_table_schema, new_mlog_id, old_mlog_schema->get_table_name(), new_schema_version, trans))) { - LOG_WARN("update mlog type failed", K(ret), K(data_table_schema)); - } else if (OB_FAIL(schema_service_.gen_new_schema_version(new_schema_version))) { - LOG_WARN("fail to gen new schema_version", K(ret)); - } else if (OB_FAIL(schema_service->get_table_sql_service().update_mlog_status( - data_table_schema, old_mlog_id, new_mlog_schema->get_table_name(), new_schema_version, trans))) { - LOG_WARN("update mlog type failed", K(ret), K(data_table_schema)); - } else if (OB_FAIL(schema_service->get_table_sql_service().update_data_table_schema_version( - trans, data_table_id, - data_table_schema.get_in_offline_ddl_white_list()))) { - LOG_WARN("update data table schema version failed", K(ret)); - } - return ret; -} - int ObDDLOperator::update_table_attribute(ObTableSchema &new_table_schema, common::ObMySQLTransaction &trans, const ObSchemaOperationType operation_type, @@ -4074,7 +4024,7 @@ int ObDDLOperator::drop_table( } if (OB_FAIL(ret)) { - } else if ((table_schema.is_aux_table() || table_schema.is_mlog_table()) + } else if (table_schema.is_aux_table() && !is_inner_table(table_schema.get_table_id())) { ObSnapshotInfoManager snapshot_mgr; ObArray tablet_ids; @@ -4101,19 +4051,7 @@ int ObDDLOperator::drop_table( if (OB_SUCC(ret)) { const uint64_t table_id = table_schema.get_table_id(); - if (table_schema.is_materialized_view()) { - if (OB_FAIL(ObMViewSchedJobUtils::remove_mview_refresh_job( - trans, table_id))) { - LOG_WARN("failed to remove mview refresh job", - KR(ret), K(table_id)); - } - } else if (table_schema.is_mlog_table()) { - if (OB_FAIL(ObMViewSchedJobUtils::remove_mlog_purge_job( - trans, table_id))) { - LOG_WARN("failed to remove mlog purge job", - KR(ret), K(table_id)); - } - } else if ((table_schema.is_vec_delta_buffer_type() || table_schema.is_hybrid_vec_index_log_type()) && + if ((table_schema.is_vec_delta_buffer_type() || table_schema.is_hybrid_vec_index_log_type()) && OB_FAIL(ObVectorIndexUtil::remove_dbms_vector_jobs(trans, table_schema.get_table_id()))) { LOG_WARN("failed to remove dbms vector jobs", K(ret), K(table_schema.get_table_id())); } @@ -4169,10 +4107,9 @@ int ObDDLOperator::drop_table_for_not_dropped_schema( &schema_guard, drop_table_set))) { LOG_WARN("schema_service_impl drop_table failed", K(table_schema), K(ret)); - } else if (OB_FAIL(sync_version_for_cascade_table(table_schema.get_base_table_ids(), trans)) - || OB_FAIL(sync_version_for_cascade_table(table_schema.get_depend_table_ids(), trans))) { + } else if (OB_FAIL(sync_version_for_cascade_table(table_schema.get_depend_table_ids(), trans))) { LOG_WARN("fail to sync versin for cascade tables", K(ret), - K(table_schema.get_base_table_ids()), K(table_schema.get_depend_table_ids())); + K(table_schema.get_depend_table_ids())); } else if (OB_FAIL(sync_version_for_cascade_mock_fk_parent_table(table_schema.get_depend_mock_fk_parent_table_ids(), trans))) { LOG_WARN("fail to sync cascade depend_mock_fk_parent_table_ids table", K(ret)); } @@ -4198,9 +4135,7 @@ int ObDDLOperator::cleanup_autoinc_cache(const ObTableSchema &table_schema) uint64_t autoinc_column_id = table_schema.get_autoinc_column_id(); LOG_INFO("begin to clear all auto-increment cache", K(table_id), K(autoinc_column_id)); - if (OB_FAIL(autoinc_service.clear_autoinc_cache_all(table_id, - autoinc_column_id, - table_schema.is_order_auto_increment_mode()))) { + if (OB_FAIL(autoinc_service.clear_autoinc_cache(table_id, autoinc_column_id))) { LOG_WARN("failed to clear auto-increment cache", K(table_id)); } @@ -4266,19 +4201,7 @@ int ObDDLOperator::drop_table_to_recyclebin(const ObTableSchema &table_schema, int64_t new_schema_version = OB_INVALID_VERSION; bool recycle_db_exist = false; - // materialized view will not be dropped into recyclebin - if (table_schema.get_table_type() == MATERIALIZED_VIEW) { - LOG_WARN("bypass recyclebin for materialized view"); - } else if (OB_UNLIKELY(table_schema.has_mlog_table())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table with materialized view log should not come to recyclebin", KR(ret)); - } else if (OB_UNLIKELY(table_schema.table_referenced_by_fast_lsm_mv())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table required by materialized view should not come to recyclebin", KR(ret)); - } else if (OB_UNLIKELY(table_schema.get_table_type() == MATERIALIZED_VIEW_LOG)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("materialized view log should not come to recyclebin", KR(ret)); - } else if (OB_ISNULL(schema_service_impl)) { + if (OB_ISNULL(schema_service_impl)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("schema_service_impl must not null", K(ret)); } else if (OB_FAIL(schema_guard.check_database_exist(OB_RECYCLEBIN_SCHEMA_ID, diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 40b0b0135..c3c19225a 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -53,25 +53,17 @@ #include "ob_lob_meta_builder.h" #include "ob_lob_piece_builder.h" #include "storage/ddl/ob_ddl_lock.h" -#include "share/schema/ob_mlog_info.h" -#include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" #include "storage/compaction/ob_compaction_schedule_util.h" -#include "share/schema/ob_mview_info.h" #include "storage/vector_index/ob_vector_index_sched_job_utils.h" #include "rootserver/ob_split_partition_helper.h" -#include "rootserver/mview/ob_mview_dependency_service.h" #include "rootserver/parallel_ddl/ob_ddl_helper.h" #include "storage/ddl/ob_ddl_alter_auto_part_attr.h" #include "sql/resolver/ddl/ob_vec_index_builder_util.h" #include "rootserver/direct_load/ob_direct_load_partition_exchange.h" #include "rootserver/truncate_info/ob_truncate_info_service.h" #include "share/truncate_info/ob_truncate_info_util.h" -#include "share/schema/ob_mview_info.h" -#include "sql/resolver/mv/ob_mv_provider.h" -#include "sql/resolver/mv/ob_mv_dep_utils.h" -#include "rootserver/mview/ob_mview_alter_service.h" #include "storage/tablet/ob_tablet_binding_helper.h" #include "rootserver/ob_dynamic_partition_manager.h" #include "sql/resolver/ddl/ob_storage_cache_ddl_util.h" @@ -79,8 +71,7 @@ #include "rootserver/ob_alter_table_constraint_checker.h" #include "rootserver/ob_domain_index_builder_util.h" #include "rootserver/ob_objpriv_mysql_ddl_service.h" -#include "share/location_cache/ob_location_service.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/ob_autoincrement_service.h" #include "storage/ob_micro_block_format_version_helper.h" #include "rootserver/ob_create_index_on_empty_table_helper.h" @@ -398,7 +389,7 @@ int ObDDLService::create_index_table( // For create index operation, generate ddl_stmt_str when index enables, but // for alter table add index operation or create index on empty table, keep generating ddl_stmt_str same as 3.x while generating index schema. const ObString *ddl_stmt_str = is_table_empty ? &arg.ddl_stmt_str_ : nullptr; - if (OB_FAIL(create_index_or_mlog_table_in_trans(table_schema, + if (OB_FAIL(create_index_table_in_trans(table_schema, ddl_stmt_str, &sql_trans, schema_guard, true/*need_check_tablet_cnt*/, tenant_data_version, is_table_empty))) { LOG_WARN("create_table_in_trans failed", KR(ret), K(arg), K(table_schema)); } @@ -408,187 +399,6 @@ ddl_stmt_str, &sql_trans, schema_guard, true/*need_check_tablet_cnt*/, tenant_da return ret; } -int ObDDLService::create_mlog_table( - ObMySQLTransaction &sql_trans, - const obcall::ObCreateMLogArg &arg, - const uint64_t tenant_data_version, - ObSchemaGetterGuard &schema_guard, - ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t new_table_id = table_schema.get_table_id(); - - ObSchemaService *schema_service = nullptr; - const ObDatabaseSchema *database_schema = nullptr; - ObArray conflict_schema_types; - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", KR(ret)); - } else if (OB_FAIL(check_table_exist(table_schema))) { - if (OB_ERR_TABLE_EXIST != ret) { - LOG_WARN("failed to check table exist", KR(ret), K(table_schema)); - } else { - ret = OB_ERR_MLOG_EXIST; - LOG_WARN("a materialized view log already exists on table", - KR(ret), K(arg.table_name_)); - ObCStringHelper helper; - LOG_USER_ERROR(OB_ERR_MLOG_EXIST, helper.convert(arg.table_name_)); - } - } else if (OB_FAIL(schema_guard.get_database_schema(table_schema.get_database_id(), database_schema))) { - LOG_WARN("failed to get database schema", - KR(ret), K(table_schema.get_database_id())); - } else if (OB_ISNULL(database_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("database schema should not be null", - KR(ret), K(table_schema.get_database_id())); - } else if (database_schema->is_in_recyclebin()) { - ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; - LOG_WARN("cannot create mlog in recyclebin", KR(ret)); - } else if (OB_ISNULL(schema_service = schema_service_->get_schema_service())){ - ret = OB_ERR_SYS; - LOG_ERROR("schema sevice cannot be null", KR(ret)); - } else if (OB_FAIL(schema_service->fetch_new_table_id(new_table_id))) { - LOG_WARN("failed to fetch new table id", KR(ret)); - } else if (OB_FALSE_IT(table_schema.set_table_id(new_table_id))) { - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(create_index_or_mlog_table_in_trans( - table_schema, - &arg.ddl_stmt_str_, - &sql_trans, - schema_guard, - true /*need_check_tablet_cnt*/, - tenant_data_version, - false /*is_table_empty*/))) { - LOG_WARN("failed to create index or mlog table in trans", KR(ret), K(arg.ddl_stmt_str_), K(table_schema)); - } else if (OB_FAIL(add_mlog(sql_trans, arg, schema_guard, table_schema))) { - LOG_WARN("failed to add mlog", KR(ret)); - } - } - return ret; -} - -int ObDDLService::create_mlog_tablet( - ObMySQLTransaction &trans, - ObSchemaGetterGuard &schema_guard, - const ObTableSchema &mlog_schema, - const bool need_check_tablet_cnt, - const uint64_t tenant_data_version) -{ - int ret = OB_SUCCESS; - - SCN frozen_scn; - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("check_inner_stat error", K(is_inited()), KR(ret)); - } else if (!mlog_schema.is_mlog_table() || tenant_data_version <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("arg must be materialized view log table", KR(ret), K(tenant_data_version), K(mlog_schema)); - } else if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(frozen_scn))) { - LOG_WARN("failed to get frozen status for create tablet", KR(ret)); - } else { - ObTableCreator table_creator(frozen_scn, trans); - common::ObArray ls_id_array; - if (OB_FAIL(table_creator.init(need_check_tablet_cnt))) { - LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); - } else { - const ObTableSchema *data_table_schema = NULL; - const uint64_t data_table_id = mlog_schema.get_data_table_id(); - ObSEArray schemas; - ObSEArray need_create_empty_majors; - if (OB_FAIL(schema_guard.get_table_schema( data_table_id, data_table_schema))) { - LOG_WARN("failed to get table schema", KR(ret), K(data_table_id)); - } else if (OB_ISNULL(data_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("data table schema not exists", KR(ret), K(data_table_id)); - } else if (OB_FAIL(schemas.push_back(&mlog_schema)) - || OB_FAIL(need_create_empty_majors.push_back(true))) { - LOG_WARN("failed to push back mlog schema", KR(ret), K(mlog_schema)); - } else if (OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( - schemas, data_table_schema, ls_id_array, - tenant_data_version, - need_create_empty_majors))) { - LOG_WARN("failed to add create tablets of local aux tables arg", KR(ret)); - } else if (OB_FAIL(table_creator.execute())) { - LOG_WARN("failed to execute create tablet", KR(ret)); - } - } - } - return ret; -} - -int ObDDLService::add_mlog( - ObMySQLTransaction &trans, - const obcall::ObCreateMLogArg &arg, - ObSchemaGetterGuard &schema_guard, - const ObTableSchema &mlog_schema) -{ - int ret = OB_SUCCESS; - - - const uint64_t mlog_table_id = mlog_schema.get_table_id(); - const obcall::ObCreateMLogArg::PurgeOptions &purge_options = arg.purge_options_; - ObMLogPurgeMode purge_mode = purge_options.purge_mode_; - int64_t purge_start = purge_options.start_datetime_expr_.get_timestamp(); - ObString purge_next = purge_options.next_datetime_expr_; - ObString purge_job; - ObArenaAllocator allocator("mlog_sched"); - - if (!purge_options.start_datetime_expr_.is_null() - || !purge_options.next_datetime_expr_.empty()) { - const ObDatabaseSchema *database = NULL; - // job_name is generated as "job_prefix+job_id" - if (OB_FAIL(schema_guard.get_database_schema( mlog_schema.get_database_id(), database))) { - LOG_WARN("failed to get database_schema", KR(ret), - "database_id", mlog_schema.get_database_id()); - } else if (OB_ISNULL(database)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("database_schema is null", KR(ret), - "database_id", mlog_schema.get_database_id()); - } else { - const ObString &db_name = database->get_database_name_str(); - const ObString &table_name = arg.table_name_; - // job_action is generated as "mlog_purge_func('db_name.table_name')" - if (OB_FAIL(ObMViewSchedJobUtils::create_mlog_scheduler_job( - trans, - mlog_table_id, - db_name, - table_name, - purge_options.start_datetime_expr_, - purge_next, - arg.purge_options_.exec_env_, - allocator, - purge_job))) { - LOG_WARN("failed to add purge mlog scheduler job", KR(ret), K(mlog_table_id), K(db_name), K(table_name), - K(purge_options.start_datetime_expr_), K(purge_next)); - } else { - LOG_INFO("succeed to add purge mlog scheduler job", K(mlog_table_id), - K(purge_options.start_datetime_expr_), K(purge_next), K(purge_job)); - } - } - } - - if (OB_SUCC(ret)) { - ObMLogInfo mlog_info; - - mlog_info.set_mlog_id(mlog_table_id); - mlog_info.set_purge_mode(purge_options.purge_mode_); - if (!purge_options.start_datetime_expr_.is_null()) { - mlog_info.set_purge_start(purge_options.start_datetime_expr_.get_timestamp()); - } - mlog_info.set_schema_version(mlog_schema.get_schema_version()); - if (OB_FAIL(mlog_info.set_purge_next(purge_next))) { - LOG_WARN("fail to set purge next", KR(ret)); - } else if (OB_FAIL(mlog_info.set_purge_job(purge_job))) { - LOG_WARN("fail to set purge job", KR(ret)); - } else if (OB_FAIL(ObMLogInfo::insert_mlog_info(trans, mlog_info))) { - LOG_WARN("fail to insert mlog info", KR(ret), K(mlog_info)); - } - } - return ret; -} - int ObDDLService::check_create_with_db_id(ObDatabaseSchema &schema) { int ret = OB_SUCCESS; @@ -1648,9 +1458,8 @@ int ObDDLService::print_view_expanded_definition( ret = OB_ERR_UNEXPECTED; LOG_WARN("database not exist", K(ret), K(database_id)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, - "CREATE%s %sVIEW `%s`.`%s` AS %.*s", + "CREATE%s VIEW `%s`.`%s` AS %.*s", if_not_exist ? " OR REPLACE" : "", - table_schema.is_materialized_view() ? "MATERIALIZED " : "", database_schema->get_database_name(), table_schema.get_table_name(), table_schema.get_view_schema().get_view_definition_str().length(), @@ -1689,84 +1498,6 @@ int ObDDLService::get_obj_privs_ora(const uint64_t obj_id, return ret; } -int ObDDLService::create_tablets_in_trans_for_mv_(ObIArray &table_schemas, - ObDDLOperator &ddl_operator, - ObMySQLTransaction &trans, - ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_data_version) -{ - int ret = OB_SUCCESS; - SCN frozen_scn; - if (table_schemas.count() < 2) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("table_schemas count should not be smaller than 2", KR(ret), K(table_schemas.count())); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(frozen_scn.convert_for_tx( - compaction::ObBasicMergeScheduler::INIT_COMPACTION_SCN))) { - LOG_WARN("failed to get frozen status for create tablet", KR(ret)); - } else { - ObTableCreator table_creator( - frozen_scn, - trans); - common::ObArray ls_id_array; - if (OB_FAIL(table_creator.init(true/*need_tablet_cnt_check*/))) { - LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); - } - - ObArray tablet_schemas; - ObArray need_create_empty_majors; - int64_t last_schema_version = OB_INVALID_VERSION; - // create tablets for container table and lob tables - for (int64_t i = 0; OB_SUCC(ret) && (i < table_schemas.count()); ++i) { - const ObTableSchema &this_table = table_schemas.at(i); - const int64_t table_id = this_table.get_table_id(); - if (OB_INVALID_VERSION == last_schema_version - && OB_FAIL(get_last_schema_version(last_schema_version))) { - LOG_WARN("get last schema version failed", KR(ret)); - } else if (OB_FAIL(ddl_operator.insert_ori_schema_version( - trans, table_id, last_schema_version))) { - LOG_WARN("failed to insert_ori_schema_version!", - KR(ret), K(table_id), K(last_schema_version)); - } else if (!this_table.is_materialized_view()) { - // no need to create tablets for mv because its data is stored in container table - if (OB_FAIL(tablet_schemas.push_back(&this_table))) { - LOG_WARN("failed to push back", KR(ret), K(this_table)); - } else if (OB_FAIL(need_create_empty_majors.push_back(true))) { - LOG_WARN("failed to push_back", KR(ret), K(this_table)); - } - } - } - if (OB_SUCC(ret)) { - const ObTableSchema &container_table = table_schemas.at(1); - const ObTablegroupSchema *container_tablegroup_schema = NULL; // keep NULL if no tablegroup - if (OB_INVALID_ID != container_table.get_tablegroup_id()) { - if (OB_FAIL(schema_guard.get_tablegroup_schema( - container_table.get_tablegroup_id(), - container_tablegroup_schema))) { - LOG_WARN("get tablegroup schema failed", KR(ret), K(container_table)); - } else if (OB_ISNULL(container_tablegroup_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("container_tablegroup_schema is null", KR(ret), K(container_table)); - } - } - if (FAILEDx(table_creator.add_create_tablets_of_tables_arg( - tablet_schemas, - ls_id_array, - tenant_data_version, - need_create_empty_majors/*need_create_empty_major_sstable*/))) { - LOG_WARN("create table partitions failed", KR(ret), K(tablet_schemas)); - } else if (OB_FAIL(table_creator.execute())) { - LOG_WARN("execute create partition failed", KR(ret)); - } - } - } - return ret; -} - int ObDDLService::create_tablets_in_trans_(ObIArray &table_schemas, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, @@ -1794,12 +1525,9 @@ int ObDDLService::create_tablets_in_trans_(ObIArray &table_schema LOG_WARN("failed to get frozen status for create tablet", KR(ret)); } else { ObTableCreator table_creator(frozen_scn, trans); - common::ObArray ls_id_array; const ObTablegroupSchema *data_tablegroup_schema = NULL; // keep NULL if no tablegroup - if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); - } else if (fork_table_info.is_valid()) { + if (fork_table_info.is_valid()) { common::ObArray dest_table_ids; for (int64_t i = 0; OB_SUCC(ret) && i < table_schemas.count(); ++i) { const uint64_t table_id = table_schemas.at(i).get_table_id(); @@ -1849,7 +1577,6 @@ int ObDDLService::create_tablets_in_trans_(ObIArray &table_schema } else { if (OB_FAIL(table_creator.add_create_tablets_of_table_arg( this_table, - ls_id_array, tenant_data_version, need_create_empty_major /*need_create_empty_major_sstable*/, &schema_guard))) { @@ -1873,7 +1600,6 @@ int ObDDLService::create_tablets_in_trans_(ObIArray &table_schema // virtual tablet and view skip } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg( schemas, - ls_id_array, tenant_data_version, need_create_empty_majors, &schema_guard))) { @@ -1921,9 +1647,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, const ObTableSchema *old_view_schema = NULL; int64_t refreshed_schema_version = 0; - ObDDLTaskRecord task_record; ddl_task_id = 0; - ObMViewInfo mview_info; if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { LOG_WARN("fail to get schema guard with version in inner table", K(ret)); } else if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { @@ -2042,25 +1766,6 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, } } } - - if (OB_SUCC(ret) && (0 == i) && table_schema.is_materialized_view()) { - const uint64_t db_id = table_schema.get_database_id(); - const ObSimpleDatabaseSchema *db_schema = NULL; - if (OB_FAIL(schema_guard.get_database_schema( db_id, db_schema))) { - LOG_WARN("failed to get database schema", KR(ret), K(db_id)); - } else if (OB_ISNULL(db_schema)) { - ret = OB_ERR_BAD_DATABASE; - LOG_WARN("db schema is NULL", KR(ret), K(db_id)); - } else if (OB_FAIL(ObMViewSchedJobUtils::add_mview_info_and_refresh_job(trans, - table_schema.get_table_id(), - db_schema->get_database_name_str(), - table_schema.get_table_name_str(), - table_schema.get_view_schema().get_mv_refresh_info(), - table_schema.get_schema_version(), - mview_info))) { - LOG_WARN("fail to start mview refresh job", KR(ret)); - } - } } // add error info for create force view @@ -2076,36 +1781,9 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, LOG_WARN("fail to deal_with_mock_fk_parent_tables", K(ret)); } - if (OB_SUCC(ret)) { - if (first_table->is_materialized_view()) { - if (OB_FAIL(create_tablets_in_trans_for_mv_(table_schemas, ddl_operator, trans, schema_guard, tenant_data_version))) { - LOG_WARN("fail to create tablets in trans for mv", KR(ret)); - } - } else { - if (OB_FAIL(create_tablets_in_trans_(table_schemas, ddl_operator, trans, schema_guard, tenant_data_version))) { - LOG_WARN("fail to create tablets in trans", KR(ret)); - } - } - if (OB_SUCC(ret) - && first_table->is_materialized_view() - && table_schemas.count() >= 2) { - ObTableSchema &mview_schema = table_schemas.at(0); - ObTableSchema &container_table_schema = table_schemas.at(1); - if (OB_FAIL(start_mview_complete_refresh_task(trans, - schema_guard, - mview_schema, - container_table_schema, - dep_infos, - allocator, - tenant_data_version, - mview_info, - ddl_stmt_str, - task_record))) { - LOG_WARN("failed to start mview complete refresh task", KR(ret)); - } else { - ddl_task_id = task_record.task_id_; - } - } + if (OB_SUCC(ret) && OB_FAIL(create_tablets_in_trans_( + table_schemas, ddl_operator, trans, schema_guard, tenant_data_version))) { + LOG_WARN("fail to create tablets in trans", KR(ret)); } RS_TRACE(operator_create_table_end); } @@ -2135,110 +1813,14 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, RS_TRACE(public_schema_end); } } - if (OB_SUCC(ret) - && first_table->is_materialized_view()) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ObSysDDLSchedulerUtil::schedule_ddl_task(task_record))) { - LOG_WARN("fail to schedule ddl task", KR(tmp_ret), K(task_record)); - } - } } RS_TRACE(create_tables_in_trans_end); return ret; } -int ObDDLService::start_mview_complete_refresh_task( - ObMySQLTransaction &trans, - ObSchemaGetterGuard &schema_guard, - const ObTableSchema &mview_schema, - const ObTableSchema &container_table_schema, - const ObIArray *dep_infos, - common::ObIAllocator &allocator, - const uint64_t tenant_data_version, - const ObMViewInfo &mview_info, - const ObString &ddl_stmt_str, - ObDDLTaskRecord &task_record) -{ - int ret = OB_SUCCESS; - int64_t max_dependency_version = 0; - - const ObMVRefreshInfo *mv_refresh_info = mview_schema.get_view_schema().get_mv_refresh_info(); - ObFixedLengthString time_zone; - obcall::ObMViewCompleteRefreshArg arg; - - arg.table_id_ = mview_schema.get_table_id(); - arg.session_id_ = 100;// FIXME - - if (OB_UNLIKELY(nullptr == dep_infos || nullptr == mv_refresh_info)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("dep_infos is nullptr", KR(ret) , KP(dep_infos), KP(mv_refresh_info)); - } else if (OB_FAIL(arg.last_refresh_scn_.convert_for_inner_table_field(mview_info.get_last_refresh_scn()))) { - LOG_WARN("fail to covert for inner table field", KR(ret), K(mview_info)); - } else if (OB_FAIL(share::ObSysTimeZoneUtil::get_tenant_sys_time_zone_wrap(time_zone, - arg.tz_info_wrap_))) { - LOG_WARN("failed to get tenant sys timezoen wrap", K(ret)); - } else { - arg.parallelism_ = mv_refresh_info->parallel_; - arg.tz_info_ = arg.tz_info_wrap_.get_tz_info_offset(); - arg.nls_formats_[ObNLSFormatEnum::NLS_DATE] = ObTimeConverter::COMPAT_OLD_NLS_DATE_FORMAT; - arg.nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP] = ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_FORMAT; - arg.nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP_TZ] = ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_TZ_FORMAT; - arg.ddl_stmt_str_ = ddl_stmt_str; - } - for (int64_t i = 0; OB_SUCC(ret) && i < dep_infos->count(); ++i) { - const ObDependencyInfo &dep = dep_infos->at(i); - const ObSimpleTableSchemaV2 *base_table_schema = nullptr; - if (OB_UNLIKELY((ObObjectType::TABLE != dep.get_ref_obj_type()) - && ObObjectType::VIEW != dep.get_ref_obj_type())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("ref obj type is not table or mview, not supported", KR(ret), K(dep)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "the ref obj type of materialized view not user table or mview is"); - } else if (OB_FAIL( - schema_guard.get_simple_table_schema( dep.get_ref_obj_id(), base_table_schema))) { - LOG_WARN("fail to get table schema", K(ret), K(dep)); - } else if (OB_ISNULL(base_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("base table schema is nullptr", K(ret)); - } else { - ObBasedSchemaObjectInfo base_info; - base_info.schema_id_ = dep.get_ref_obj_id(); - base_info.schema_type_ = ObSchemaType::TABLE_SCHEMA; - base_info.schema_version_ = base_table_schema->get_schema_version(); - - max_dependency_version = MAX(max_dependency_version, base_table_schema->get_schema_version()); - if (OB_FAIL(arg.based_schema_object_infos_.push_back(base_info))) { - LOG_WARN("fail to push back base info", KR(ret)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(max_dependency_version > mview_schema.get_max_dependency_version())) { - ret = OB_OLD_SCHEMA_VERSION; - LOG_WARN("base table schema version is old", - KR(ret), K(max_dependency_version), KPC(dep_infos), - "old_max_dependency_version", mview_schema.get_max_dependency_version()); - } else { - ObCreateDDLTaskParam param(ObDDLType::DDL_CREATE_MVIEW, - &mview_schema, - &container_table_schema, - 0/*object_id*/, - container_table_schema.get_schema_version(), - arg.parallelism_, - &allocator, - &arg); - param.tenant_data_version_ = tenant_data_version; - if (OB_FAIL(ObSysDDLSchedulerUtil::create_ddl_task(param, trans, task_record))) { - LOG_WARN("submit create index ddl task failed", K(ret)); - } - } - } - - return ret; -} - // Create table information is written to the internal table within a transaction. // If sql_trans is NULL, it need to create a transaction inside the function. -int ObDDLService::create_index_or_mlog_table_in_trans( +int ObDDLService::create_index_table_in_trans( ObTableSchema &table_schema, const ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans, @@ -2282,14 +1864,10 @@ int ObDDLService::create_index_or_mlog_table_in_trans( } } if (OB_SUCC(ret) && table_schema.has_tablet()) { - if (table_schema.is_mlog_table()) { - if (OB_FAIL(create_mlog_tablet(trans, schema_guard, table_schema, need_check_tablet_cnt, tenant_data_version))) { - LOG_WARN("failed to create_mlog_tablet", KR(ret), K(table_schema)); - } - } else { - if (OB_FAIL(create_index_tablet(table_schema, trans, schema_guard, need_check_tablet_cnt, tenant_data_version, is_table_empty))) { - LOG_WARN("fail to create_index_tablet", KR(ret), K(table_schema)); - } + if (OB_FAIL(create_index_tablet(table_schema, trans, schema_guard, + need_check_tablet_cnt, tenant_data_version, + is_table_empty))) { + LOG_WARN("fail to create_index_tablet", KR(ret), K(table_schema)); } } if (OB_ISNULL(sql_trans) && trans.is_started()) { @@ -4148,9 +3726,6 @@ int ObDDLService::check_alter_table_index(const obcall::ObAlterTableArg &alter_t } else { const share::schema::ObTableSchema *index_schema = nullptr; const int64_t index_table_id = drop_index_arg->index_table_id_; - if (table_schema->is_materialized_view()) { - data_table_id = table_schema->get_data_table_id(); - } if (OB_FAIL(get_index_schema_by_name(data_table_id, table_schema->get_database_id(), *drop_index_arg, @@ -5384,9 +4959,8 @@ int ObDDLService::delete_auto_increment_attribute( LOG_INFO("begin to clear all auto-increment cache", K(table_id), K(part_num), K(orig_table_schema->get_autoinc_column_id())); - if (OB_FAIL(autoinc_service.clear_autoinc_cache_all(table_id, - orig_table_schema->get_autoinc_column_id(), - orig_table_schema->is_order_auto_increment_mode()))) { + if (OB_FAIL(autoinc_service.clear_autoinc_cache( + table_id, orig_table_schema->get_autoinc_column_id()))) { LOG_WARN("failed to clear auto-increment cache", K(table_id)); } } @@ -5396,7 +4970,7 @@ int ObDDLService::delete_auto_increment_attribute( if (new_table_schema.get_autoinc_column_id() > 0 && auto_increment > 0) { LOG_INFO("begin to sync auto_increment", K(table_id), K(part_num), K(auto_increment)); - if (OB_FAIL(autoinc_service.sync_auto_increment_all( + if (OB_FAIL(autoinc_service.refresh_auto_increment_cache( table_id, new_table_schema.get_autoinc_column_id(), 0 == auto_increment ? 0 : auto_increment - 1))) { @@ -5770,81 +5344,6 @@ int ObDDLService::switch_index_name_and_status_for_vec_index_table(obcall::ObAlt return ret; } -int ObDDLService::switch_index_name_and_status_for_mlog_table(obcall::ObAlterTableArg &alter_table_arg) -{ - int ret = OB_SUCCESS; - const int64_t old_index_id = alter_table_arg.table_id_; - const int64_t new_index_id = alter_table_arg.hidden_table_id_; - - - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", K(ret)); - } else if (old_index_id == OB_INVALID_ID || new_index_id == OB_INVALID_ID) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(old_index_id), K(new_index_id)); - } else { - ObDDLSQLTransaction trans(schema_service_); - const ObTableSchema *old_mlog_schema = nullptr; - const ObTableSchema *new_mlog_schema = nullptr; - const ObTableSchema *base_table_schema = nullptr; - ObSchemaGetterGuard schema_guard; - ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); - schema_guard.set_session_id(alter_table_arg.session_id_); - int64_t refreshed_schema_version = 0; - if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { - LOG_WARN("fail to get schema guard with version in inner table", K(ret)); - } else if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { - LOG_WARN("failed to get tenant schema version", KR(ret)); - } else if (OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { - LOG_WARN("fail to start trans, ", KR(ret), K(refreshed_schema_version)); - } else if (OB_FAIL(schema_guard.get_table_schema( old_index_id, old_mlog_schema))) { - LOG_WARN("fail to get old mlog schema", K(ret), K(old_index_id)); - } else if (OB_ISNULL(old_mlog_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpectd null pointer", K(ret)); - } else if (!old_mlog_schema->is_mlog_table()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old index table is not a mlog table", K(ret), KPC(old_mlog_schema)); - } else if (old_mlog_schema->is_tmp_mlog_table()) { - LOG_INFO("mlog has already been switched", KR(ret), K(alter_table_arg)); - } else if (OB_FAIL(schema_guard.get_table_schema( new_index_id, new_mlog_schema))) { - LOG_WARN("fail to get new mlog schema", K(ret), K(new_index_id)); - } else if (OB_ISNULL(new_mlog_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpectd null pointer", K(ret)); - } else if (!new_mlog_schema->is_tmp_mlog_table()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new index table is not a tmp mlog table", K(ret), KPC(new_mlog_schema)); - } else if (old_mlog_schema->get_database_id() != new_mlog_schema->get_database_id()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old table and new table should in the same database", K(ret)); - } else if (old_mlog_schema->get_data_table_id() != new_mlog_schema->get_data_table_id()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old table and new table should have the same data table", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( old_mlog_schema->get_data_table_id(), base_table_schema))) { - LOG_WARN("failed to get base table schema", K(ret), K(new_index_id)); - } else if (OB_ISNULL(base_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpectd null pointer", K(ret)); - } else if (OB_FAIL(ddl_operator.switch_mlog_status(*base_table_schema, old_index_id, new_index_id, schema_guard, trans))) { - LOG_WARN("failed to switch mlog schemas", KR(ret), K(old_index_id), K(new_index_id)); - } - if (trans.is_started()) { - int temp_ret = OB_SUCCESS; - if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { - LOG_ERROR("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret)); - ret = (OB_SUCC(ret)) ? temp_ret : ret; - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(publish_schema())) { - LOG_WARN("publish_schema failed", K(ret)); - } - } - - return ret; -} - int ObDDLService::update_autoinc_schema(obcall::ObAlterTableArg &alter_table_arg) { int ret = OB_SUCCESS; @@ -5968,7 +5467,6 @@ int ObDDLService::lock_partitions(ObMySQLTransaction &trans, // skip those type table for lock table if (!table_schema.has_tablet() || table_schema.is_aux_table() - || table_schema.is_mlog_table() || table_schema.is_sys_table()) { } else if (OB_FAIL(table_schema.get_tablet_ids(tablet_ids))) { LOG_WARN("failed to get tablet ids", KR(ret), K(table_schema)); @@ -6017,7 +5515,6 @@ int ObDDLService::lock_table(ObMySQLTransaction &trans, // skip those type table for lock table if (!table_schema.has_tablet() || table_schema.is_aux_table() - || table_schema.is_mlog_table() || table_schema.is_sys_table()) { } else if (OB_ISNULL(conn = dynamic_cast (trans.get_connection()))) { @@ -6038,35 +5535,6 @@ int ObDDLService::lock_table(ObMySQLTransaction &trans, return ret; } -int ObDDLService::lock_mview(ObMySQLTransaction &trans, const ObSimpleTableSchemaV2 &table_schema) -{ - int ret = OB_SUCCESS; - - const uint64_t mview_id = table_schema.get_table_id(); - observer::ObInnerSQLConnection *conn = nullptr; - if (OB_UNLIKELY(!table_schema.is_materialized_view())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(table_schema)); - } else if (OB_ISNULL(conn = dynamic_cast - (trans.get_connection()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("conn_ is NULL", KR(ret)); - } else { - ObLockObjRequest lock_arg; - lock_arg.obj_type_ = ObLockOBJType::OBJ_TYPE_MATERIALIZED_VIEW; - lock_arg.obj_id_ = mview_id; - lock_arg.owner_id_ = ObTableLockOwnerID::default_owner(); - lock_arg.lock_mode_ = EXCLUSIVE; - lock_arg.op_type_ = ObTableLockOpType::IN_TRANS_COMMON_LOCK; - lock_arg.timeout_us_ = 0; - LOG_INFO("lock mview", KR(ret), K(mview_id), K(lock_arg), KPC(conn)); - if (OB_FAIL(ObInnerConnectionLockUtil::lock_obj(lock_arg, conn))) { - LOG_WARN("fail to lock mview obj", KR(ret), K(lock_arg), KPC(conn)); - } - } - return ret; -} - // this interface only used for drop and purge database int ObDDLService::lock_tables_of_database_for_drop( const ObDatabaseSchema &database_schema, @@ -6087,21 +5555,6 @@ int ObDDLService::lock_tables_of_database_for_drop( LOG_WARN("fail to get table ids in database", K(1UL), K(database_id), K(ret)); } else { const ObSimpleTableSchemaV2 *table_schema = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < table_schemas.count(); i++) { - table_schema = table_schemas.at(i); - // when drop or purge database, would lock all tables first, - // stop mview refresh job to avoid block drop database - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", K(ret)); - } else if (table_schema->is_materialized_view()) { - if (OB_FAIL(ObMViewSchedJobUtils::disable_and_stop_job(table_schema->get_table_id()))) { - LOG_WARN("fail to disable and stop job", K(ret)); - } else if (OB_FAIL(lock_mview(trans, *table_schema))) { - LOG_WARN("fail to lock mview", KR(ret), KPC(table_schema)); - } - } - } for (int64_t i = 0; OB_SUCC(ret) && i < table_schemas.count(); i++) { table_schema = table_schemas.at(i); if (OB_ISNULL(table_schema)) { @@ -6538,22 +5991,6 @@ int ObDDLService::lock_tables_in_recyclebin(const ObDatabaseSchema &database_sch } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(schema_guard))) { LOG_WARN("fail to get schema guard", KR(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < recycle_objs.count(); ++i) { - const ObRecycleObject &recycle_obj = recycle_objs.at(i); - const ObSimpleTableSchemaV2* table_schema = NULL; - if (OB_FAIL(schema_guard.get_simple_table_schema( - recycle_obj.get_table_id(), table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(recycle_obj)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table is not exist", K(ret), K(recycle_obj)); - ObCStringHelper helper; - LOG_USER_ERROR(OB_TABLE_NOT_EXIST, helper.convert(database_schema.get_database_name_str()), - helper.convert(recycle_obj.get_object_name())); - } else if (table_schema->is_materialized_view() && OB_FAIL(lock_mview(trans, *table_schema))) { - LOG_WARN("fail to lock mview", KR(ret), KPC(table_schema)); - } - } for (int64_t i = 0; OB_SUCC(ret) && i < recycle_objs.count(); ++i) { const ObRecycleObject &recycle_obj = recycle_objs.at(i); const ObSimpleTableSchemaV2* table_schema = NULL; @@ -6600,7 +6037,6 @@ int ObDDLService::create_index_tablet(const ObTableSchema &index_schema, ObTableCreator table_creator( frozen_scn, trans); - common::ObArray ls_id_array; const uint64_t data_table_id = index_schema.get_data_table_id(); const ObTableSchema *data_table_schema = NULL; const ObTablegroupSchema *data_tablegroup_schema = NULL; // keep NULL if no tablegroup @@ -6608,8 +6044,6 @@ int ObDDLService::create_index_tablet(const ObTableSchema &index_schema, if (OB_FAIL(table_creator.init(need_check_tablet_cnt))) { LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); } else if (OB_FAIL(schema_guard.get_table_schema( data_table_id, data_table_schema))) { LOG_WARN("failed to get table schema", KR(ret), K(data_table_id)); } else if (OB_ISNULL(data_table_schema)) { @@ -6640,7 +6074,6 @@ int ObDDLService::create_index_tablet(const ObTableSchema &index_schema, } else if (OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( schemas, data_table_schema, - ls_id_array, tenant_data_version, need_create_empty_majors /* add_index, need_create_empty_major_sstable*/))) { LOG_WARN("create table tablet failed", KR(ret), K(index_schema)); @@ -6648,7 +6081,6 @@ int ObDDLService::create_index_tablet(const ObTableSchema &index_schema, } else { if (OB_FAIL(table_creator.add_create_tablets_of_table_arg( index_schema, - ls_id_array, tenant_data_version, is_table_empty /*need_create_empty_major_sstable*/))) { LOG_WARN("create table tablet failed", KR(ret), K(index_schema)); @@ -6761,11 +6193,6 @@ int ObDDLService::alter_table_index(obcall::ObAlterTableArg &alter_table_arg, continue; } } - if (OB_SUCC(ret) && new_table_schema.mv_major_refresh()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not support to add index on mv", K(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "add index on major refresh materialized view is"); - } if (OB_FAIL(ret)) { } else if (create_index_arg->index_name_.empty()) { if (OB_FAIL(generate_index_name(*create_index_arg, @@ -7755,24 +7182,20 @@ int ObDDLService::get_index_schema_by_name( ObArenaAllocator allocator(ObModIds::OB_SCHEMA); const ObString &index_name = drop_index_arg.index_name_; - const bool is_mlog = (obcall::ObIndexArg::DROP_MLOG == drop_index_arg.index_action_type_); index_table_schema = nullptr; //build index name and get index schema - if (is_mlog) { - index_table_name = index_name; - } else if (OB_FAIL(ObTableSchema::build_index_table_name(allocator, + if (OB_FAIL(ObTableSchema::build_index_table_name(allocator, data_table_id, index_name, index_table_name))) { LOG_WARN("build_index_table_name failed", K(ret), K(data_table_id), K(index_name)); } if (OB_SUCC(ret)) { - const bool is_index = !is_mlog; if (OB_FAIL(schema_guard.get_table_schema( database_id, index_table_name, - is_index, + true /* is_index */, index_table_schema))) { LOG_WARN("fail to get table schema", K(ret), K(database_id), K(index_table_schema)); } else if (nullptr == index_table_schema) { @@ -7780,7 +7203,7 @@ int ObDDLService::get_index_schema_by_name( LOG_USER_ERROR(OB_ERR_CANT_DROP_FIELD_OR_KEY, index_name.length(), index_name.ptr()); LOG_WARN("get index table schema failed", K(database_id), K(index_table_name), K(ret)); - } else if (is_index && data_table_id != index_table_schema->get_data_table_id()) { + } else if (data_table_id != index_table_schema->get_data_table_id()) { ret = OB_ERR_CANT_DROP_FIELD_OR_KEY; LOG_USER_ERROR(OB_ERR_CANT_DROP_FIELD_OR_KEY, index_name.length(), index_name.ptr()); LOG_WARN("get index table schema failed", K(ret), @@ -9614,7 +9037,6 @@ int ObDDLService::check_new_column_for_index( // wrapper for alter column effects // if column is in // 1. update index if modified column is in index -// 2. update materialized view if modified column is in materialized view // but 2 is disabled for now int ObDDLService::alter_table_update_index_and_view_column( const ObTableSchema &new_table_schema, @@ -10990,11 +10412,8 @@ int ObDDLService::create_aux_lob_table_if_need(ObTableSchema &data_table_schema, } } if (OB_SUCC(ret)) { - common::ObArray ls_id_array; - if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); - } else if (OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( - schemas, &data_table_schema, ls_id_array, tenant_data_version, + if (OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( + schemas, &data_table_schema, tenant_data_version, need_create_empty_majors /*online_ddl, need_create_empty_major_sstable*/))) { LOG_WARN("create table partitions failed", KR(ret), K(last_schema_version)); } else if (OB_FAIL(table_creator.execute())) { @@ -11212,9 +10631,7 @@ int ObDDLService::alter_table_auto_increment( // column. all new auto_inc request will be based on the auto_increment value in the table // schema. if (OB_FAIL(ret)) { - } else if (OB_FAIL(auto_inc_service.clear_autoinc_cache_all(table_id, - column_id, is_order_mode, - false /* ignore_rpc_errors */))) { + } else if (OB_FAIL(auto_inc_service.clear_autoinc_cache(table_id, column_id))) { LOG_WARN("fail to clear autoinc cache all", K(ret)); } else if (OB_FAIL(ddl_operator.reinit_autoinc_row(new_table_schema, trans))) { LOG_WARN("fail to reinit autoinc row", K(ret)); @@ -11886,14 +11303,6 @@ const char* ObDDLService::ddl_type_str(const ObDDLType ddl_type) str = "convert to character"; } else if (DDL_CHANGE_COLUMN_NAME == ddl_type) { str = "change column name"; - } else if (DDL_MVIEW_COMPLETE_REFRESH == ddl_type) { - str = "mview complete refresh"; - } else if (DDL_CREATE_MVIEW == ddl_type) { - str = "create mview"; - } else if (DDL_CREATE_MLOG == ddl_type) { - str = "create materialized view log"; - } else if (DDL_DROP_MLOG == ddl_type) { - str = "drop materialized view log"; } else if (DDL_AUTO_SPLIT_BY_RANGE == ddl_type) { str = "auto split by range"; } else if (DDL_AUTO_SPLIT_NON_RANGE == ddl_type) { @@ -11985,7 +11394,7 @@ int ObDDLService::make_index_unusable_(common::ObIAllocator &allocator, obcall::ObDropIndexArg *drop_index_arg = nullptr; typedef common::ObSEArray TableSchemaArray; SMART_VAR(TableSchemaArray, drop_index_schemas) { - if (OB_FAIL(ObDDLTaskRecordOperator::check_has_index_or_mlog_task( + if (OB_FAIL(ObDDLTaskRecordOperator::check_has_index_task( trans, index_table_schema, orig_table_schema.get_table_id(), has_index_task))) { LOG_WARN("fail to check has index task", KR(ret), @@ -12111,7 +11520,7 @@ int ObDDLService::drop_and_create_index_schema_(obcall::ObAlterTableArg &arg, bool has_index_task = false; bool is_inner_and_domain_index = index_table_schema.is_fts_or_multivalue_index(); ObSArray &index_arg_list = arg.index_arg_list_; - if (OB_FAIL(ObDDLTaskRecordOperator::check_has_index_or_mlog_task( + if (OB_FAIL(ObDDLTaskRecordOperator::check_has_index_task( trans, index_table_schema, orig_table_schema.get_table_id(), has_index_task))) { LOG_WARN("fail to check has index task", KR(ret), K(orig_table_schema.get_table_id()), K(index_table_schema.get_table_id())); @@ -12141,7 +11550,7 @@ int ObDDLService::drop_and_create_index_schema_(obcall::ObAlterTableArg &arg, new_index_table_schema.set_index_status(INDEX_STATUS_UNAVAILABLE); if (OB_FAIL(generate_tablet_id(new_index_table_schema))) { LOG_WARN("failed to generate tablet id", KR(ret)); - } else if (OB_FAIL(create_index_or_mlog_table_in_trans(new_index_table_schema, + } else if (OB_FAIL(create_index_table_in_trans(new_index_table_schema, nullptr /* ddl_stmt */, &trans, schema_guard, false/*need_check_tablet_cnt*/, tenant_data_version, false /*is_table_empty*/))) { LOG_WARN("create_index_in_trans failed", KR(ret), K(new_index_table_schema)); @@ -12520,8 +11929,7 @@ int ObDDLService::generate_tables_array(const obcall::ObAlterTableArg &alter_tab const ObString new_part_name = inc_table_schema.get_new_part_name(); if (!orig_table_schema.has_tablet() || orig_table_schema.is_index_local_storage() - || orig_table_schema.is_aux_lob_table() - || orig_table_schema.is_mlog_table()) { + || orig_table_schema.is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("table_schema must be data table or global indexes", KR(ret), K(orig_table_schema)); } else if (OB_FAIL(orig_table_schemas.push_back(&orig_table_schema))) { @@ -12570,13 +11978,6 @@ int ObDDLService::generate_tables_array(const obcall::ObAlterTableArg &alter_tab LOG_WARN("fail to push back lob piece tid", KR(ret), K(ptid)); } } - if (OB_SUCC(ret)) { - uint64_t mlog_tid = orig_table_schema.get_mlog_tid(); - if ((OB_INVALID_ID != mlog_tid) - && OB_FAIL(aux_table_ids.push_back(mlog_tid))) { - LOG_WARN("failed to push back materialized view log tid", KR(ret), K(mlog_tid)); - } - } } for (int64_t i = 0; OB_SUCC(ret) && i < aux_table_ids.count(); ++i) { @@ -12588,8 +11989,7 @@ int ObDDLService::generate_tables_array(const obcall::ObAlterTableArg &alter_tab ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema should not be null", KR(ret)); } else if (orig_aux_table_schema->is_index_local_storage() - || orig_aux_table_schema->is_aux_lob_table() - || orig_aux_table_schema->is_mlog_table()) { + || orig_aux_table_schema->is_aux_lob_table()) { ObTableSchema *new_aux_table_schema = NULL; AlterTableSchema *inc_aux_table_schema = NULL; void *new_schema_ptr = allocator.alloc(sizeof(ObTableSchema)); @@ -13203,14 +12603,7 @@ int ObDDLService::alter_table_sess_active_time_in_trans(obcall::ObAlterTableArg LOG_WARN("failed to do offline ddl in trans", K(ret), K(alter_table_arg));; } } else { - if ((alter_table_arg.is_alter_mlog_attributes_ || - alter_table_arg.is_alter_mview_attributes_) && - OB_FAIL(ObMviewAlterService::alter_mview_or_mlog_in_trans( - alter_table_arg, res, schema_guard, schema_service_, sql_proxy_, - tenant_data_version))) { - LOG_WARN("alter_mview_or_mlog_in_trans failed", KR(ret)); - } else if (OB_FAIL( - alter_table_in_trans(alter_table_arg, res, tenant_data_version))) { + if (OB_FAIL(alter_table_in_trans(alter_table_arg, res, tenant_data_version))) { LOG_WARN("alter_table_in_trans failed", K(ret)); } } @@ -13403,17 +12796,6 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, } else if (OB_UNLIKELY(NULL == tenant_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tenant schema is null", K(ret), KP(tenant_schema)); - } else if (orig_table_schema->is_materialized_view()) { - const ObTableSchema *mv_orig_table_schema = nullptr; - if (OB_FAIL(schema_guard.get_table_schema( - orig_table_schema->get_data_table_id(), mv_orig_table_schema))) { - LOG_WARN("failed to get mv container table schema", KR(ret)); - } else if (OB_ISNULL(mv_orig_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mv container table schema is null", KR(ret)); - } else { - orig_table_schema = mv_orig_table_schema; - } } if (OB_FAIL(ret)) { @@ -13883,7 +13265,6 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, } else { ObTableCreator table_creator(frozen_scn, trans); - common::ObArray ls_id_array; const ObTableSchema *tmp_table_schema = inc_table_schema_ptrs.at(0); const ObTablegroupSchema *tablegroup_schema = NULL; // keep NULL if no tablegroup if (OB_ISNULL(tmp_table_schema)) { @@ -13891,8 +13272,6 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, LOG_WARN("table schema is null", KR(ret), K(inc_table_schema_ptrs)); } else if (OB_FAIL(table_creator.init(need_check_tablet_cnt))) { LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); } else if (OB_INVALID_ID != tmp_table_schema->get_tablegroup_id()) { if (OB_FAIL(schema_guard.get_tablegroup_schema( tmp_table_schema->get_tablegroup_id(), @@ -13906,7 +13285,6 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, if (OB_FAIL(ret)) { } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg( inc_table_schema_ptrs, - ls_id_array, tenant_data_version, need_create_empty_majors/*need_create_empty_majors*/))) { LOG_WARN("create table partitions failed", KR(ret), K(alter_table_schema), @@ -13945,9 +13323,8 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, alter_table_schema.alter_option_bitset_.has_member(ObAlterTableArg::INCREMENT_MODE) && 0 != orig_table_schema->get_autoinc_column_id()) { ObAutoincrementService &autoinc_service = ObAutoincrementService::get_instance(); - if (OB_FAIL(autoinc_service.clear_autoinc_cache_all(orig_table_schema->get_table_id(), - orig_table_schema->get_autoinc_column_id(), - orig_table_schema->is_order_auto_increment_mode()))) { + if (OB_FAIL(autoinc_service.clear_autoinc_cache(orig_table_schema->get_table_id(), + orig_table_schema->get_autoinc_column_id()))) { LOG_WARN("fail to clear autoinc cache", K(ret)); } } @@ -14211,29 +13588,6 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, return ret; } -int ObDDLService::check_long_run_ddl_table_type_(const ObTableSchema &orig_table_schema) -{ - int ret = OB_SUCCESS; - if (orig_table_schema.has_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("double table long running ddl on table with materialized view log is not supported", - KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "double table long running ddl on table with materialized view log is"); - } else if (orig_table_schema.table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("double table long running ddl on table required by materialized view is not supported", - KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "double table long running ddl on table required by materialized view is"); - } else if (orig_table_schema.is_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("double table long running ddl on materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "double table long running ddl on materialized view log is"); - } - return ret; -} - int ObDDLService::check_long_run_ddl_has_index_(const ObTableSchema *orig_table_schema, obcall::ObAlterTableArg &alter_table_arg, ObSchemaGetterGuard &schema_guard) @@ -14428,9 +13782,7 @@ int ObDDLService::check_is_offline_ddl(ObAlterTableArg &alter_table_arg, if (OB_SUCC(ret) && is_double_table_long_running_ddl(ddl_type)) { bool is_adding_constraint = false; uint64_t table_id = alter_table_arg.alter_table_schema_.get_table_id(); - if (OB_FAIL(check_long_run_ddl_table_type_(*orig_table_schema))) { - LOG_WARN("fail to check long run ddl table_type", KR(ret)); - } else if (OB_FAIL(check_long_run_ddl_has_index_(orig_table_schema, alter_table_arg, schema_guard))) { + if (OB_FAIL(check_long_run_ddl_has_index_(orig_table_schema, alter_table_arg, schema_guard))) { LOG_WARN("fail to check long run ddl has index", KR(ret)); } else if (OB_FAIL(check_is_adding_constraint(table_id, is_adding_constraint))) { LOG_WARN("failed to call check_is_adding_constraint", K(ret)); @@ -15178,7 +14530,6 @@ int ObDDLService::create_hidden_table( int64_t refreshed_schema_version = 0; new_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_OFFLINE_DDL); - new_table_schema.set_table_referenced_by_mv(ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV); if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { LOG_WARN("failed to get tenant schema version", KR(ret)); } else if (OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { @@ -15257,10 +14608,7 @@ int ObDDLService::create_hidden_table( if (OB_FAIL(ObSysDDLSchedulerUtil::create_ddl_task(param, trans, task_record))) { LOG_WARN("submit ddl task failed", K(ret)); } else { - - res.table_id_ = table_id; - res.dest_table_id_ = task_record.target_object_id_; res.schema_version_ = task_record.schema_version_; res.trace_id_ = task_record.trace_id_; @@ -15292,187 +14640,6 @@ int ObDDLService::create_hidden_table( return ret; } -int ObDDLService::mview_complete_refresh( - const obcall::ObMViewCompleteRefreshArg &arg, - obcall::ObMViewCompleteRefreshRes &res, - share::schema::ObSchemaGetterGuard &schema_guard) -{ - int ret = OB_SUCCESS; - - int64_t refreshed_schema_version = 0; - common::ObArenaAllocator allocator("MVRef"); - ObDDLTaskRecord task_record; - uint64_t tenant_data_version = 0; - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", KR(ret)); - } else if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { - LOG_WARN("failed to get tenant schema version", KR(ret)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_data_version))) { - LOG_WARN("get min data version failed", K(ret)); - } else { - ObDDLSQLTransaction trans(schema_service_); - if (OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { - LOG_WARN("start transaction failed", KR(ret), K(refreshed_schema_version)); - } else if (OB_FAIL(mview_complete_refresh_in_trans(arg, res, trans, allocator, schema_guard, tenant_data_version, task_record))) { - LOG_WARN("failed to do mview complete refresh in trans", KR(ret), K(arg)); - } - if (trans.is_started()) { - int temp_ret = OB_SUCCESS; - if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { - LOG_ERROR("trans end failed", "is_commit", OB_SUCCESS == ret, KR(temp_ret)); - ret = COVER_SUCC(temp_ret); - } - } - } - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(publish_schema())) { - LOG_WARN("publish_schema failed", KR(ret)); - } else if (OB_TMP_FAIL(ObSysDDLSchedulerUtil::schedule_ddl_task(task_record))) { - LOG_WARN("fail to schedule ddl task", KR(tmp_ret), K(task_record)); - } else { - LOG_INFO("schedule ddl task success", K(task_record)); - } - } - return ret; -} - -int ObDDLService::mview_complete_refresh_in_trans( - const obcall::ObMViewCompleteRefreshArg &arg, - obcall::ObMViewCompleteRefreshRes &res, - ObDDLSQLTransaction &trans, - common::ObIAllocator &allocator, - share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_data_version, - ObDDLTaskRecord &task_record) -{ - int ret = OB_SUCCESS; - - const int64_t mview_table_id = arg.table_id_; - const ObTableSchema *mview_table_schema = nullptr; - const ObDatabaseSchema *database_schema = nullptr; - const ObTableSchema *container_table_schema = nullptr; - int64_t refreshed_schema_version = 0; - int64_t task_id = 0; - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(arg)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(GCTX.sql_proxy_)); - } else if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", KR(ret)); - } else if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { - LOG_WARN("failed to get tenant schema version", KR(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( mview_table_id, mview_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(mview_table_id)); - } else if (OB_ISNULL(mview_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("mview table schema is nullptr", KR(ret), K(mview_table_id)); - } else if (OB_UNLIKELY(!mview_table_schema->is_materialized_view())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected not materialized view", KR(ret), KPC(mview_table_schema)); - } else if (OB_FAIL(schema_guard.get_database_schema( mview_table_schema->get_database_id(), database_schema))) { - LOG_WARN("fail to get database schema", KR(ret), "database_id", mview_table_schema->get_database_id()); - } else if (OB_ISNULL(database_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("database schema is nullptr", KR(ret), "database_id", mview_table_schema->get_database_id()); - } else if (OB_FAIL(schema_guard.get_table_schema( mview_table_schema->get_data_table_id(), container_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), "container_table_id", mview_table_schema->get_data_table_id()); - } else if (OB_ISNULL(container_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("container table schema is nullptr", KR(ret), "container_table_id", mview_table_schema->get_data_table_id()); - } else { - HEAP_VAR(ObTableSchema, new_container_table_schema) { - if (OB_FAIL(new_container_table_schema.assign(*container_table_schema))) { - LOG_WARN("fail to assign schema", KR(ret)); - } else { - const bool bind_tablets = true; - ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); - new_container_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_OFFLINE_DDL); - ObTableLockOwnerID owner_id; - if (OB_FAIL(ObDDLTask::fetch_new_task_id(*GCTX.sql_proxy_, task_id))) { - LOG_WARN("fetch new task id failed", KR(ret)); - } else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, - task_id))) { - LOG_WARN("failed to get owner id", K(ret), K(task_id)); - } else if (OB_FAIL(ObDDLLock::lock_for_offline_ddl(*container_table_schema, - nullptr, - owner_id, - trans))) { - LOG_WARN("failed to lock ddl lock", KR(ret)); - } else if (OB_FAIL(create_user_hidden_table(*container_table_schema, - new_container_table_schema, - nullptr/*sequence_ddl_arg*/, - bind_tablets, - schema_guard, - schema_guard, - ddl_operator, - trans, - allocator, - tenant_data_version))) { - LOG_WARN("fail to create hidden table", KR(ret)); - } else { - LOG_INFO("create hidden table success!", K(mview_table_id), "container_table_id", container_table_schema->get_table_id(), - K(new_container_table_schema)); - } - } - if (OB_SUCC(ret)) { - HEAP_VAR(obcall::ObAlterTableArg, alter_table_arg) { - ObString empty_ddl_stmt_str; - ObPrepareAlterTableArgParam prepare_param; - ObArray dependency_infos; - if (OB_FAIL(prepare_param.init(arg.session_id_, - arg.sql_mode_, - empty_ddl_stmt_str, - container_table_schema->get_table_name_str(), - database_schema->get_database_name_str(), - database_schema->get_database_name_str(), - arg.tz_info_, - arg.tz_info_wrap_, - arg.nls_formats_, - true/*foreign key checks*/))) { - LOG_WARN("prepare param init failed", KR(ret)); - } else if (OB_FAIL(ObSysDDLSchedulerUtil::prepare_alter_table_arg(prepare_param, &new_container_table_schema, alter_table_arg))) { - LOG_WARN("prepare alter table arg fail", KR(ret)); - } else if (OB_FAIL(alter_table_arg.based_schema_object_infos_.assign(arg.based_schema_object_infos_))) { - LOG_WARN("fail to assign based schema object infos", KR(ret)); - } else { - alter_table_arg.mview_refresh_info_.is_mview_complete_refresh_ = true; - alter_table_arg.mview_refresh_info_.mview_table_id_ = mview_table_id; - alter_table_arg.mview_refresh_info_.last_refresh_scn_ = arg.last_refresh_scn_; - alter_table_arg.mview_refresh_info_.start_time_ = ObTimeUtil::current_time(); - alter_table_arg.mview_refresh_info_.mview_target_data_sync_scn_ = arg.target_data_sync_scn_; - alter_table_arg.mview_refresh_info_.select_sql_ = arg.select_sql_; - LOG_DEBUG("alter table arg preparation complete!", K(alter_table_arg)); - ObCreateDDLTaskParam param(DDL_MVIEW_COMPLETE_REFRESH, - container_table_schema, - &new_container_table_schema, - container_table_schema->get_table_id(), - refreshed_schema_version, - arg.parallelism_, - &allocator, - &alter_table_arg, - arg.parent_task_id_, - task_id); - param.tenant_data_version_ = tenant_data_version; - if (OB_FAIL(ObSysDDLSchedulerUtil::create_ddl_task(param, trans, task_record))) { - LOG_WARN("submit ddl task failed", KR(ret)); - } else { - res.trace_id_ = task_record.trace_id_; - res.task_id_ = task_record.task_id_; - } - } - } - } - } - } - return ret; -} - int ObDDLService::get_and_check_table_schema( const obcall::ObAlterTableArg &alter_table_arg, ObSchemaGetterGuard &schema_guard, @@ -15513,49 +14680,12 @@ int ObDDLService::get_and_check_table_schema( LOG_WARN("can not alter table in recyclebin", K(ret), K(alter_table_arg)); } else if (!alter_table_arg.skip_sys_table_check_ && OB_FAIL(check_enable_sys_table_ddl(*orig_table_schema, OB_DDL_ALTER_TABLE))) { - LOG_WARN("ddl is not allowed on system table", K(ret)); - } else if (!alter_table_arg.is_inner_ - && (orig_table_schema->is_in_recyclebin() || is_db_in_recyclebin)) { - ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; - LOG_WARN("can not alter table in recyclebin", - K(ret), K(alter_table_arg), K(is_db_in_recyclebin)); - } else if (orig_table_schema->is_materialized_view()) { - bool allow_alter_mview = false; - if (alter_table_arg.is_alter_indexs_) { - bool is_alter_pk = false; - for (int64_t i = 0; OB_SUCC(ret) && (i < alter_table_arg.index_arg_list_.count()); ++i) { - const ObIndexArg *index_arg = alter_table_arg.index_arg_list_.at(i); - if (OB_ISNULL(index_arg)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("index arg is null", KR(ret)); - } else if ((ObIndexArg::ADD_PRIMARY_KEY == index_arg->index_action_type_) - || (ObIndexArg::DROP_PRIMARY_KEY == index_arg->index_action_type_) - || (ObIndexArg::ALTER_PRIMARY_KEY == index_arg->index_action_type_)) { - is_alter_pk = true; - break; - } - } - if (OB_SUCC(ret) && !is_alter_pk) { - allow_alter_mview = true; - } - } else { - // only allow for alter tablegroup - if (!alter_table_arg.is_alter_indexs_ && !alter_table_arg.is_alter_columns_ && !alter_table_arg.is_alter_partitions_ && - !alter_table_arg.is_update_global_indexes_ && !alter_table_arg.is_convert_to_character_ && alter_table_arg.is_alter_options_) { - if (1 == alter_table_arg.alter_table_schema_.alter_option_bitset_.num_members() && - alter_table_arg.alter_table_schema_.alter_option_bitset_.has_member(ObAlterTableArg::TABLEGROUP_NAME)) { - allow_alter_mview = true; - } - } - } - if (OB_SUCC(ret) && (alter_table_arg.is_alter_mview_attributes_ || alter_table_arg.is_alter_mlog_attributes_)) { - allow_alter_mview = true; - } - if (OB_SUCC(ret) && !allow_alter_mview) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("alter materialized view is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter materialized view is"); - } + LOG_WARN("ddl is not allowed on system table", K(ret)); + } else if (!alter_table_arg.is_inner_ + && (orig_table_schema->is_in_recyclebin() || is_db_in_recyclebin)) { + ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; + LOG_WARN("can not alter table in recyclebin", + K(ret), K(alter_table_arg), K(is_db_in_recyclebin)); } else if (OB_FAIL(alter_table_arg.is_alter_comment(is_alter_comment))) { LOG_WARN("failed to get is alter comment", K(ret)); } else if (!orig_table_schema->is_user_table() @@ -15587,7 +14717,7 @@ int ObDDLService::get_and_check_table_schema( * whether the merge has started according to whether the __all_zone version numbers are consistent * to standalone, It is impossible to make the DDL of the primary cluster and the merge of standalone library * to be mutually exclusive, so this solution is only to reduce the probability of bad cases. - * When the merge checksum is inconsistent in the standalone cluster, it can be corrected by drop replica + * When the merge checksum is inconsistent in the standalone cluster, it needs manual tablet metadata repair. * >= 3.3, allow drop column when doing major freeze */ int ObDDLService::check_can_alter_column( @@ -15756,10 +14886,6 @@ int ObDDLService::check_alter_partitions(const ObTableSchema &orig_table_schema, LOG_USER_ERROR(OB_OP_NOT_ALLOW, "partition maintenance during upgrade"); } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { LOG_WARN("fail to get schema guard with version in inner table", K(ret)); - } else if (orig_table_schema.is_materialized_view()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("alter partition of materialized view is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter partition of materialized view is"); } else if (orig_table_schema.is_interval_part()) { if (PARTITION_LEVEL_TWO == orig_table_schema.get_part_level() && !orig_table_schema.has_sub_part_template_def()) { @@ -17053,11 +16179,7 @@ int ObDDLService::alter_table(obcall::ObAlterTableArg &alter_table_arg, LOG_WARN("failed to do offline ddl in trans", K(ret), K(alter_table_arg), K(ddl_type)); } } else { - if ((alter_table_arg.is_alter_mlog_attributes_ || - alter_table_arg.is_alter_mview_attributes_) && - OB_FAIL(ObMviewAlterService::alter_mview_or_mlog_in_trans(alter_table_arg, res, schema_guard, schema_service_, sql_proxy_, data_version))) { - LOG_WARN("alter_mview_or_mlog_in_trans failed", KR(ret)); - } else if (OB_FAIL(alter_table_in_trans(alter_table_arg, res, data_version))) { + if (OB_FAIL(alter_table_in_trans(alter_table_arg, res, data_version))) { LOG_WARN("alter_table_in_trans failed", K(ret)); } } @@ -17174,12 +16296,6 @@ int ObDDLService::rename_table(const obcall::ObRenameTableArg &rename_table_arg) if (OB_FAIL(ret)) { } else if (OB_FAIL(full_rename_items.push_back(rename_item))) { LOG_WARN("failed to add rename_item", KR(ret), K(rename_item)); - } else if (OB_NOT_NULL(table_schema) && table_schema->is_materialized_view() && - OB_FAIL(construct_rename_table_items_for_mview( - table_schema, schema_guard, rename_item, - allocator, full_rename_items))) { - // for mv, not only should we rename the mv itself, we should also rename its container table and its mlog table - LOG_WARN("failed to add mview related rename item", KR(ret), K(rename_item)); } } const int64_t rename_items_count = full_rename_items.count(); @@ -17526,86 +16642,11 @@ int ObDDLService::validate_rename_table_args(const ObTableSchema *table_schema) if (OB_UNLIKELY(OB_ISNULL(table_schema))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", KR(ret)); - } else if (table_schema->is_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("rename materialized view log is not supported", KR(ret), - K(table_schema->get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename materialized view log is"); - } else if (table_schema->has_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("rename table with materialized view log is not supported", KR(ret), - K(table_schema->get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename table with materialized view log is"); - } else if (table_schema->table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("rename table required by materialized view is not supported", KR(ret), - K(table_schema->get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename table required by materialized view is"); - } - - return ret; -} - -int ObDDLService::construct_rename_table_items_for_mview(const ObTableSchema *table_schema, - share::schema::ObSchemaGetterGuard &schema_guard, - const obcall::ObRenameTableItem &rename_item, - common::ObArenaAllocator &allocator, - ObIArray &full_rename_items) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(OB_ISNULL(table_schema) || !table_schema->is_materialized_view())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is invalid", KR(ret)); - } else { - const ObTableSchema *container_table_schema = NULL; - if (OB_FAIL(schema_guard.get_table_schema( table_schema->get_data_table_id(), - container_table_schema))) { - LOG_WARN("failed to get table schema", KR(ret)); - } else if (OB_ISNULL(container_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("container table schema should not be null", KR(ret)); - } else if (rename_item.origin_db_name_ != rename_item.new_db_name_) { - // when the database name of the view is changed, the database name of the container table - // should also be changed, and the table name of the container table remains the same - ObRenameTableItem container_rename_item; - container_rename_item.origin_db_name_ = rename_item.origin_db_name_; - container_rename_item.new_db_name_ = rename_item.new_db_name_; - container_rename_item.origin_table_name_ = container_table_schema->get_table_name(); - container_rename_item.new_table_name_ = container_table_schema->get_table_name(); - if (OB_FAIL(full_rename_items.push_back(container_rename_item))) { - LOG_WARN("failed to add related container item", KR(ret), K(container_rename_item)); - } - } - if (OB_FAIL(ret)) { - } else if (container_table_schema->has_mlog_table()) { - const ObTableSchema *mlog_schema = NULL; - ObString new_mlog_name; - if (OB_FAIL(schema_guard.get_table_schema( container_table_schema->get_mlog_tid(), mlog_schema))) { - LOG_WARN("failed to get table schema", KR(ret)); - } else if (OB_ISNULL(mlog_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mlog schema is null", KR(ret)); - } else if (OB_FAIL(ObTableSchema::build_mlog_table_name( - allocator, rename_item.new_table_name_, new_mlog_name))) { - LOG_WARN("failed to build mlog table name", KR(ret), K(rename_item.new_table_name_)); - } else { - ObRenameTableItem mlog_rename_item; - mlog_rename_item.origin_db_name_ = rename_item.origin_db_name_; - mlog_rename_item.new_db_name_ = rename_item.new_db_name_; - mlog_rename_item.origin_table_name_ = mlog_schema->get_table_name(); - mlog_rename_item.new_table_name_ = new_mlog_name; - if (OB_FAIL(full_rename_items.push_back(mlog_rename_item))) { - LOG_WARN("failed to add related mlog item", KR(ret), K(mlog_rename_item)); - } - } - } } return ret; } - int ObDDLService::build_single_table_rw_defensive_(const ObArray &tablet_ids, const int64_t schema_version, ObDDLSQLTransaction &trans) @@ -17931,14 +16972,11 @@ int ObDDLService::truncate_table_in_trans(const obcall::ObTruncateTableArg &arg, } else { ObTableCreator table_creator(frozen_scn, trans); - common::ObArray ls_id_array; const ObTablegroupSchema *data_tablegroup_schema = NULL; // keep NULL if no tablegroup const ObTableSchema &data_table_schema = table_schemas.at(0); if (OB_FAIL(table_creator.init(false/*need_check_tablet_cnt*/))) { LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); } else if (OB_INVALID_ID != data_table_schema.get_tablegroup_id()) { if (OB_FAIL(schema_guard.get_tablegroup_schema( data_table_schema.get_tablegroup_id(), @@ -17965,7 +17003,6 @@ int ObDDLService::truncate_table_in_trans(const obcall::ObTruncateTableArg &arg, } else { if (OB_FAIL(table_creator.add_create_tablets_of_table_arg( this_table, - ls_id_array, tenant_data_version, true /*need_create_empty_major_sstable*/))) { LOG_WARN("create table partitions failed", KR(ret), K(this_table)); @@ -17988,7 +17025,6 @@ int ObDDLService::truncate_table_in_trans(const obcall::ObTruncateTableArg &arg, else if (schemas.count() <= 0) { } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg( schemas, - ls_id_array, tenant_data_version, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_WARN("create table partitions failed", KR(ret), K(orig_table_schema), @@ -18581,12 +17617,9 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem ObTableCreator table_creator( frozen_scn, trans); - common::ObArray ls_id_array; const ObTablegroupSchema *tablegroup_schema = NULL; // keep NULL if no tablegroup if (OB_FAIL(table_creator.init(false/*need_tablet_cnt_check*/))) { LOG_WARN("fail to init table creator", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); } else if (OB_INVALID_ID != hidden_table_schema.get_tablegroup_id()) { if (OB_FAIL(dst_tenant_schema_guard.get_tablegroup_schema( hidden_table_schema.get_tablegroup_id(), @@ -18636,21 +17669,18 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem if (OB_SUCC(ret) && hidden_table_schema.has_tablet()) { if (!bind_tablets && OB_FAIL(table_creator.add_create_tablets_of_tables_arg( schemas, - ls_id_array, tenant_data_version, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_WARN("create table tablets failed", K(ret), K(hidden_table_schema)); } else if (bind_tablets && OB_FAIL(table_creator.add_create_bind_tablets_of_hidden_table_arg( orig_table_schema, hidden_table_schema, - ls_id_array, tenant_data_version))) { LOG_WARN("failed to add arg", K(ret), K(hidden_table_schema)); } else if (bind_tablets && schemas.count() > 0 && OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( schemas, &hidden_table_schema, - ls_id_array, tenant_data_version, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_WARN("failed to add arg", K(ret), K(aux_table_schemas), K(hidden_table_schema)); @@ -19115,9 +18145,9 @@ int ObDDLService::add_new_index_schema(obcall::ObAlterTableArg &alter_table_arg, } else { bool is_exist = false; ObString final_index_name; - if (OB_FAIL(build_hidden_table_index_and_mlog_schema_( + if (OB_FAIL(build_hidden_table_index_schema_( hidden_table_schema, index_schema))) { - LOG_WARN("failed to build hidden table index and mlog schema", + LOG_WARN("failed to build hidden table index schema", KR(ret), K(hidden_table_schema), K(index_schema)); } else if (OB_FAIL(index_schema.get_index_name(final_index_name))) { LOG_WARN("get index name failed", K(ret)); @@ -19176,7 +18206,7 @@ int ObDDLService::add_new_index_schema(obcall::ObAlterTableArg &alter_table_arg, return ret; } -int ObDDLService::build_hidden_table_index_and_mlog_schema_( +int ObDDLService::build_hidden_table_index_schema_( const ObTableSchema &hidden_table_schema, ObTableSchema &index_schema) { @@ -19471,7 +18501,7 @@ int ObDDLService::reconstruct_index_schema(obcall::ObAlterTableArg &alter_table_ return ret; } -int ObDDLService::rebuild_hidden_table_index_and_mlog(ObSchemaGetterGuard &schema_guard, +int ObDDLService::rebuild_hidden_table_index(ObSchemaGetterGuard &schema_guard, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, ObSArray &new_table_schemas) @@ -19502,11 +18532,6 @@ int ObDDLService::rebuild_hidden_table_index_and_mlog(ObSchemaGetterGuard &schem has_tablet = is_system_table(table_id); } if (!has_tablet) { - } else if (this_table.is_mlog_table()) { - if (OB_FAIL(create_mlog_tablet(trans, schema_guard, this_table, - false/*need_check_tablet_cnt*/, tenant_data_version))) { - LOG_WARN("failed to create mlog tablet", KR(ret), K(this_table)); - } } else if (OB_FAIL(create_index_tablet(this_table, trans, schema_guard, false/*need_check_tablet_cnt*/, tenant_data_version, false /*is_table_empty*/))) { LOG_WARN("create table tablets failed", K(ret), K(this_table)); } else {} @@ -19526,73 +18551,6 @@ int ObDDLService::rebuild_hidden_table_index_and_mlog(ObSchemaGetterGuard &schem return ret; } -int ObDDLService::add_new_mlog_schema(const share::schema::ObTableSchema &orig_table_schema, - const share::schema::ObTableSchema &hidden_table_schema, - share::schema::ObSchemaGetterGuard &schema_guard, - share::schema::ObSchemaGetterGuard &dest_schema_guard, - common::ObMySQLTransaction &trans, - ObIAllocator &allocator, - ObSArray &new_table_schemas, - ObSArray &index_ids) -{ - int ret = OB_SUCCESS; - ObSchemaService *schema_service = nullptr; - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", K(ret)); - } else if (OB_ISNULL(schema_service = schema_service_->get_schema_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema service is null", K(ret), KP(schema_service)); - } else { - // generate mlog schema - const ObTableSchema *orig_mlog_schema = nullptr; - ObTableSchema new_mlog_schema; - const uint64_t orig_mlog_tid = orig_table_schema.get_mlog_tid(); - ObString new_mlog_name; - bool is_exist = false; - - if (OB_FAIL(schema_guard.get_table_schema(orig_mlog_tid, orig_mlog_schema))) { - LOG_WARN("failed to get table schema", KR(ret), "table id", orig_mlog_tid); - } else if (OB_ISNULL(orig_mlog_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", KR(ret)); - } else if (OB_FAIL(new_mlog_schema.assign(*orig_mlog_schema))) { - LOG_WARN("failed to assign schema", KR(ret)); - } else if (OB_FAIL(ObTableSchema::build_mlog_table_name(allocator, - hidden_table_schema.get_table_name_str(), new_mlog_name))) { - LOG_WARN("failed to build mlog table name", KR(ret), K(hidden_table_schema.get_table_name_str())); - } else if (OB_FAIL(new_mlog_schema.set_table_name(new_mlog_name))) { - LOG_WARN("failed to set table name", KR(ret), K(new_mlog_name)); - } else if (OB_FAIL(build_hidden_table_index_and_mlog_schema_( - hidden_table_schema, new_mlog_schema))) { - LOG_WARN("failed to build hidden table index and mlog schema", - KR(ret), K(hidden_table_schema), K(new_mlog_schema)); - } else if (OB_FAIL(dest_schema_guard.check_table_exist(new_mlog_schema.get_database_id(), - new_mlog_schema.get_table_name_str(), - false/*is_index*/, - ObSchemaGetterGuard::USER_HIDDEN_TABLE_TYPE/*check_type*/, - is_exist))) { - LOG_WARN("failed to check table exist", KR(ret), K(new_mlog_schema)); - } else if (is_exist) { - LOG_INFO("mlog already rebuilt, skip", - K(new_mlog_schema.get_table_id()), K(new_mlog_schema.get_table_name_str())); - } else if (OB_FAIL(new_table_schemas.push_back(new_mlog_schema))) { - LOG_WARN("failed to add table schema!", KR(ret), K(new_mlog_schema)); - } else if (OB_FAIL(index_ids.push_back(new_mlog_schema.get_table_id()))) { - LOG_WARN("failed to add new index id!", KR(ret), K(new_mlog_schema)); - } else { // insert new mlog_info to all_mlog - ObMLogInfo mlog_info; - uint64_t new_mlog_tid = new_mlog_schema.get_table_id(); - if (OB_FAIL(ObMLogInfo::fetch_mlog_info(trans, orig_mlog_tid, mlog_info, false/*for_update*/))) { - LOG_WARN("failed to fetch mlog info", KR(ret), K(orig_mlog_tid)); - } else if (OB_FALSE_IT(mlog_info.set_mlog_id(new_mlog_tid))) { - } else if (OB_FAIL(ObMLogInfo::insert_mlog_info(trans, mlog_info))) { - LOG_WARN("fail to insert mlog info", KR(ret), K(mlog_info)); - } - } - } - return ret; -} - int ObDDLService::rebuild_hidden_table_index_in_trans(obcall::ObAlterTableArg &alter_table_arg, ObSArray &index_ids) { @@ -19620,7 +18578,6 @@ int ObDDLService::rebuild_hidden_table_index_in_trans(obcall::ObAlterTableArg &a const ObTableSchema *hidden_table_schema = nullptr; int64_t refreshed_schema_version = 0; ObArray drop_cols_id_arr; - bool has_mlog_table = false; if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table( hold_buf_src_tenant_schema_guard, hold_buf_dst_tenant_schema_guard, src_tenant_schema_guard, dst_tenant_schema_guard))) { @@ -19636,9 +18593,7 @@ int ObDDLService::rebuild_hidden_table_index_in_trans(obcall::ObAlterTableArg &a orig_table_schema, hidden_table_schema))) { LOG_WARN("failed to get orig and hidden table schema", K(ret)); - } else if (OB_FALSE_IT(has_mlog_table = orig_table_schema->has_mlog_table())) { - } else if (hidden_table_schema->get_simple_index_infos().count() > 0 - || hidden_table_schema->has_mlog_table()) { + } else if (hidden_table_schema->get_simple_index_infos().count() > 0) { // if there is any index in the hidden table, all indexes are already rebuilt because the rebuild is wrap in a transaction const common::ObIArray &index_infos = hidden_table_schema->get_simple_index_infos(); for (int64_t i = 0; OB_SUCC(ret) && i < index_infos.count(); ++i) { @@ -19646,11 +18601,6 @@ int ObDDLService::rebuild_hidden_table_index_in_trans(obcall::ObAlterTableArg &a LOG_WARN("push back index id failed", K(ret)); } } - if (OB_SUCC(ret) && hidden_table_schema->has_mlog_table()) { - if (OB_FAIL(index_ids.push_back(hidden_table_schema->get_mlog_tid()))) { - LOG_WARN("failed to push back mlog tid", KR(ret), K(hidden_table_schema->get_mlog_tid())); - } - } } else { ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); if (OB_FAIL(col_name_map.init(*orig_table_schema, *hidden_table_schema, alter_table_schema))) { @@ -19681,20 +18631,11 @@ int ObDDLService::rebuild_hidden_table_index_in_trans(obcall::ObAlterTableArg &a new_table_schemas, index_ids))) { LOG_WARN("failed to add new index schema", K(ret)); - } else if (has_mlog_table && OB_FAIL(add_new_mlog_schema(*orig_table_schema, - *hidden_table_schema, - *src_tenant_schema_guard, - *dst_tenant_schema_guard, - trans, - alter_table_arg.allocator_, - new_table_schemas, - index_ids))) { - LOG_WARN("failed to add new mlog schema", KR(ret)); - } else if (OB_FAIL(rebuild_hidden_table_index_and_mlog(*dst_tenant_schema_guard, + } else if (OB_FAIL(rebuild_hidden_table_index(*dst_tenant_schema_guard, ddl_operator, trans, new_table_schemas))) { - LOG_WARN("failed to rebuild hidden table index and mlog", K(ret)); + LOG_WARN("failed to rebuild hidden table index", K(ret)); } } if (trans.is_started()) { @@ -20518,68 +19459,22 @@ int ObDDLService::get_orig_and_hidden_table_schema( int ObDDLService::get_tablets( const ObArray &tablet_ids, - ObIArray &tablets, + ObIArray &tablets, ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - ObArray ls_ids; - if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("failed to batch get ls", K(ret)); - } else if (OB_UNLIKELY(tablet_ids.count() != ls_ids.count())) { + ObArray tablet_infos; + if (OB_FAIL(ObTabletMappingTableOperator::batch_get(trans, tablet_ids, tablet_infos))) { + LOG_WARN("failed to batch get tablet info", K(ret), K(tablet_ids)); + } else if (OB_UNLIKELY(tablet_ids.count() != tablet_infos.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet ids ls ids", K(ret)); + LOG_WARN("invalid tablet info count", K(ret), K(tablet_ids), K(tablet_infos)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(tablets.push_back({ls_ids[i], tablet_ids[i]}))) { - LOG_WARN("failed to push back tablet id and ls id", K(ret)); - } - } - return ret; -} - -int ObDDLService::build_modify_tablet_binding_args(const ObArray &tablet_ids, - const bool is_hidden_tablets, - const int64_t schema_version, - ObIArray &modify_args, - ObDDLSQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObArray tablets; - if (OB_FAIL(get_tablets(tablet_ids, tablets, trans))) { - LOG_WARN("failed to get tablet ids of orig table", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablets.count(); i++) { - const ObLSID &ls_id = tablets[i].first; - int64_t j = 0; - for (; j < modify_args.count(); j++) { - if (modify_args.at(j).ls_id_ == ls_id) { - break; - } - } - if (j == modify_args.count()) { - ObBatchUnbindTabletArg modify_arg; - - modify_arg.ls_id_ = ls_id; - modify_arg.schema_version_ = schema_version; - if (OB_FAIL(modify_args.push_back(modify_arg))) { - LOG_WARN("failed to push back modify arg", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObBatchUnbindTabletArg &modify_arg = modify_args.at(j); - const ObTabletID &tablet_id = tablets[i].second; - if (is_hidden_tablets) { - if (OB_FAIL(modify_arg.hidden_tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back", K(ret)); - } - } else { - if (OB_FAIL(modify_arg.orig_tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back", K(ret)); - } - } + if (OB_FAIL(tablets.push_back(tablet_ids.at(i)))) { + LOG_WARN("failed to push back tablet id", K(ret)); } } - LOG_DEBUG("build modify tablet binding args", K(ret), K(modify_args)); return ret; } @@ -20614,50 +19509,32 @@ int ObDDLService::write_ddl_barrier( int ret = OB_SUCCESS; ObArray tablet_ids; - ObArray hidden_tablets; + ObArray hidden_tablets; if (OB_FAIL(hidden_table_schema.get_tablet_ids(tablet_ids))) { LOG_WARN("failed to get tablets", K(ret)); } else if (OB_FAIL(get_tablets(tablet_ids, hidden_tablets, trans))) { LOG_WARN("failed to get tablet", K(ret)); } else { ObArenaAllocator allocator("DDLSrvBarrier"); - ObArray logs; + ObDDLBarrierLog log; for (int64_t i = 0; OB_SUCC(ret) && i < hidden_tablets.count(); i++) { - const ObLSID &ls_id = hidden_tablets[i].first; - const ObTabletID &tablet_id = hidden_tablets[i].second; - int64_t j = 0; - for (; j < logs.count(); j++) { - if (logs[j].ls_id_ == ls_id) { - break; - } - } - if (j == logs.count()) { - ObDDLBarrierLog log; - log.ls_id_ = ls_id; - if (OB_FAIL(logs.push_back(log))) { - LOG_WARN("failed to push back log", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(logs[j].hidden_tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back hidden tablet", K(ret)); - } + if (OB_FAIL(log.hidden_tablet_ids_.push_back(hidden_tablets.at(i)))) { + LOG_WARN("failed to push back hidden tablet", K(ret)); } } - for (int64_t i = 0; OB_SUCC(ret) && i < logs.count(); i++) { + if (OB_SUCC(ret) && !hidden_tablets.empty()) { int64_t pos = 0; - int64_t size = logs[i].get_serialize_size(); + int64_t size = log.get_serialize_size(); char *buf = nullptr; - allocator.reuse(); - if (OB_UNLIKELY(!logs[i].is_valid())) { + if (OB_UNLIKELY(!log.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ddl barrier log", K(ret), K(logs[i])); + LOG_WARN("invalid ddl barrier log", K(ret), K(log)); } else if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate", K(ret)); - } else if (OB_FAIL(logs[i].serialize(buf, size, pos))) { + } else if (OB_FAIL(log.serialize(buf, size, pos))) { LOG_WARN("failed to serialize arg", K(ret)); - } else if (OB_FAIL(trans.register_tx_data(logs[i].ls_id_, transaction::ObTxDataSourceType::DDL_BARRIER, buf, pos))) { + } else if (OB_FAIL(trans.register_tx_data(transaction::ObTxDataSourceType::DDL_BARRIER, buf, pos))) { LOG_WARN("failed to register tx data", K(ret)); } } @@ -20665,8 +19542,7 @@ int ObDDLService::write_ddl_barrier( return ret; } -int ObDDLSQLTransaction::register_tx_data(const share::ObLSID &ls_id, - const transaction::ObTxDataSourceType &type, +int ObDDLSQLTransaction::register_tx_data(const transaction::ObTxDataSourceType &type, const char *buf, const int64_t buf_len) { @@ -20677,7 +19553,7 @@ int ObDDLSQLTransaction::register_tx_data(const share::ObLSID &ls_id, LOG_WARN("invalid connection", K(ret)); } else { observer::ObInnerSQLConnection *conn = static_cast(isql_conn); - if (OB_FAIL(conn->register_multi_data_source(ls_id, type, buf, buf_len))) { + if (OB_FAIL(conn->register_multi_data_source(type, buf, buf_len))) { LOG_WARN("failed to register tx data", K(ret)); } } @@ -20725,10 +19601,6 @@ int ObDDLService::swap_orig_and_hidden_table_state(obcall::ObAlterTableArg &alte if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init", K(ret)); - } else if (alter_table_arg.mview_refresh_info_.is_mview_complete_refresh_ && - OB_UNLIKELY(!alter_table_arg.mview_refresh_info_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid mview refresh info", K(ret), K(alter_table_arg.mview_refresh_info_)); } else { ObDDLSQLTransaction trans(schema_service_); const ObTableSchema *orig_table_schema = NULL; @@ -20832,17 +19704,6 @@ int ObDDLService::swap_orig_and_hidden_table_state(obcall::ObAlterTableArg &alte } } } - // materialized view log - if (OB_SUCC(ret) && alter_table_arg.mview_refresh_info_.is_mview_complete_refresh_) { - if (orig_table_schema->has_mlog_table() && hidden_table_schema->has_mlog_table()) { - if (OB_FAIL(swap_orig_and_hidden_table_mlog_state_(schema_guard, - orig_table_schema->get_mlog_tid(), - hidden_table_schema->get_mlog_tid(), - table_schemas))) { - LOG_WARN("failed to swap orig and hidden table mlog state", KR(ret)); - } - } - } } if (OB_SUCC(ret)) { new_orig_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL); @@ -20884,11 +19745,7 @@ int ObDDLService::swap_orig_and_hidden_table_state(obcall::ObAlterTableArg &alte } if (OB_SUCC(ret)) { int64_t schema_version = table_schemas[table_schemas.count()-1].get_schema_version(); - if (orig_table_schema->mv_major_refresh()) { - // for major refresh mv skip modify ddl_info - // because of we read majoar mv data by specified snapshot - // last_refresh_scn may less then ddl snapshot - } else if (OB_FAIL(unbind_hidden_tablets(*orig_table_schema, *hidden_table_schema, + if (OB_FAIL(unbind_hidden_tablets(*orig_table_schema, *hidden_table_schema, schema_version, trans))) { LOG_WARN("failed to unbind hidden tablets", K(ret)); } @@ -20898,133 +19755,6 @@ int ObDDLService::swap_orig_and_hidden_table_state(obcall::ObAlterTableArg &alte LOG_WARN("failed to write ddl barrier", K(ret)); } } - if (OB_SUCC(ret) && alter_table_arg.mview_refresh_info_.is_mview_complete_refresh_) { - // update mview table_schema - const uint64_t mview_table_id = alter_table_arg.mview_refresh_info_.mview_table_id_; - const ObTableSchema *mview_table_schema = nullptr; - bool is_based_schema_version_consistent = true; - int64_t based_schema_version = OB_INVALID_VERSION; - int64_t max_dependency_version = 0; - if (OB_FAIL(schema_guard.get_table_schema( mview_table_id, mview_table_schema))) { - LOG_WARN("failed to get mview table schema", KR(ret), K(mview_table_id)); - } else if (OB_ISNULL(mview_table_schema)) { - ret = OB_ERR_MVIEW_NOT_EXIST; - LOG_WARN("fail to get mview table schema", KR(ret), K(mview_table_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && is_based_schema_version_consistent && i < alter_table_arg.based_schema_object_infos_.count(); ++i) { - const ObBasedSchemaObjectInfo &based_info = alter_table_arg.based_schema_object_infos_.at(i); - const ObTableSchema *based_table_schema = nullptr; - if (OB_FAIL(schema_guard.get_table_schema( based_info.schema_id_, - based_table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(based_info)); - } else if (OB_ISNULL(based_table_schema)) { - // ignore ret - LOG_WARN("based table is not exist", KR(ret), K(based_info)); - is_based_schema_version_consistent = false; - } else if (OB_UNLIKELY(based_table_schema->get_schema_version() != - based_info.schema_version_)) { - // ignore ret - LOG_WARN("based table schema version is changed", KR(ret), K(based_info), - KPC(based_table_schema)); - is_based_schema_version_consistent = false; - } else { - max_dependency_version = MAX(max_dependency_version, based_info.schema_version_); - } - } - if (OB_SUCC(ret) && !is_based_schema_version_consistent) { - ret = OB_ERR_MVIEW_BASE_TABLE_ALTERED; - LOG_WARN("base tables are altered during creation or complete refresh of materialized views", KR(ret)); - } - if (OB_SUCC(ret)) { - HEAP_VAR(ObTableSchema, new_mview_table_schema) { - if (OB_FAIL(new_mview_table_schema.assign(*mview_table_schema))) { - LOG_WARN("fail to assign table schema", KR(ret)); - } else { - new_mview_table_schema.set_data_table_id(alter_table_arg.hidden_table_id_); - if (is_based_schema_version_consistent) { - new_mview_table_schema.set_object_status(ObObjectStatus::VALID); - new_mview_table_schema.set_max_dependency_version(max_dependency_version); - } else { - new_mview_table_schema.set_object_status(ObObjectStatus::INVALID); - new_mview_table_schema.set_max_dependency_version(OB_INVALID_VERSION); - } - } - if (OB_SUCC(ret)) { - ObSchemaOperationType operation_type = OB_DDL_ALTER_TABLE; - if (OB_FAIL(ddl_operator.update_table_attribute(new_mview_table_schema, trans, operation_type))) { - LOG_WARN("failed to update mview table schema attribute", KR(ret)); - } - } - } - } - // update dependency - if (OB_SUCC(ret) && is_based_schema_version_consistent) { - uint64_t property = 0; - ObString dep_attrs, dep_reason; - ObArray deps; - int64_t new_schema_version = OB_INVALID_VERSION; - if (OB_FAIL(schema_service_->gen_new_schema_version(new_schema_version))) { - LOG_WARN("fail to gen new schema version", KR(ret)); - } else if (OB_FAIL(ObDependencyInfo::collect_dep_infos( - alter_table_arg.based_schema_object_infos_, deps, - ObObjectType::VIEW, mview_table_id, mview_table_id, property, dep_attrs, - dep_reason, new_schema_version))) { - LOG_WARN("fail to collect dep infos", KR(ret), K(alter_table_arg)); - } else if (OB_FAIL(ObDependencyInfo::delete_schema_object_dependency( - trans, mview_table_id, new_schema_version, - ObObjectType::VIEW))) { - LOG_WARN("fail to delete schema object dependency", KR(ret), K(mview_table_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < deps.count(); ++i) { - ObDependencyInfo &dep = deps.at(i); - if (OB_FAIL(dep.insert_schema_object_dependency(trans))) { - LOG_WARN("fail to insert schema object dependency", KR(ret), K(dep)); - } - } - - if (OB_SUCC(ret)) { - ObMViewDependencyService mv_dep_service(*schema_service_); - if (OB_FAIL(mv_dep_service.update_mview_dep_infos( - trans, schema_guard, mview_table_id, deps))) { - LOG_WARN("failed to update mview dep infos", KR(ret)); - } - } - } - // update mview_info - if (OB_SUCC(ret)) { - const uint64_t last_refresh_scn_val = - alter_table_arg.mview_refresh_info_.last_refresh_scn_.get_val_for_inner_table_field(); - const uint64_t refresh_scn_val = - alter_table_arg.mview_refresh_info_.refresh_scn_.get_val_for_inner_table_field(); - const uint64_t target_data_sync_scn_val = - alter_table_arg.mview_refresh_info_.mview_target_data_sync_scn_.get_val_for_inner_table_field(); - const int64_t start_time = alter_table_arg.mview_refresh_info_.start_time_; - ObMViewInfo mview_info; - if (OB_FAIL(ObMViewInfo::fetch_mview_info(trans, mview_table_id, mview_info, - true /*for_update*/, true /*nowait*/))) { - LOG_WARN("fail to fetch mview info", KR(ret), K(mview_table_id)); - } else if (OB_UNLIKELY(mview_info.get_last_refresh_scn() != last_refresh_scn_val)) { - ret = OB_VERSION_NOT_MATCH; - LOG_WARN("mview version is old", KR(ret), K(alter_table_arg.mview_refresh_info_), - K(mview_info)); - } else { - DEBUG_SYNC(BEFORE_MV_FINISH_COMPLETE_REFRESH); - mview_info.set_last_refresh_scn(refresh_scn_val); - mview_info.set_last_refresh_type(share::schema::ObMVRefreshType::COMPLETE); - mview_info.set_last_refresh_date(start_time); - mview_info.set_last_refresh_time((ObTimeUtil::current_time() - start_time) / 1000 / 1000); - if (OB_FAIL(mview_info.set_last_refresh_trace_id(ObCurTraceId::get_trace_id_str()))) { - LOG_WARN("fail to set last refresh trace id", KR(ret)); - } else if (OB_FAIL(sql::ObMVDepUtils::update_mview_data_attr(trans, - refresh_scn_val, target_data_sync_scn_val, mview_info))) { - LOG_WARN("fail to update mview data scn", KR(ret), K(mview_info), - K(refresh_scn_val)); - } else if (OB_FAIL(ObMViewInfo::update_mview_last_refresh_info(trans, mview_info))) { - LOG_WARN("fail to update mview last refresh info", KR(ret), K(mview_info)); - } - } - } - } if (OB_SUCC(ret)) { if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, alter_table_arg.task_id_))) { LOG_WARN("failed to get owner id", K(ret), K(alter_table_arg.task_id_)); @@ -21186,43 +19916,6 @@ int ObDDLService::swap_orig_and_hidden_table_partitions(obcall::ObAlterTableArg return ret; } -int ObDDLService::swap_orig_and_hidden_table_mlog_state_( - ObSchemaGetterGuard &schema_guard, - const uint64_t orig_table_mlog_tid, - const uint64_t hidden_table_mlog_tid, - ObIArray &table_schemas) -{ - int ret = OB_SUCCESS; - const ObTableSchema *orig_table_mlog_schema = nullptr; - const ObTableSchema *hidden_table_mlog_schema = nullptr; - HEAP_VARS_2((ObTableSchema, new_orig_table_mlog_schema), - (ObTableSchema, new_hidden_table_mlog_schema)) { - if (OB_FAIL(schema_guard.get_table_schema( orig_table_mlog_tid, orig_table_mlog_schema))) { - LOG_WARN("failed to get orig table mlog schema", K(ret), K(orig_table_mlog_tid)); - } else if (OB_FAIL(schema_guard.get_table_schema( hidden_table_mlog_tid, hidden_table_mlog_schema))) { - LOG_WARN("failed to get hidden table mlog schema", K(ret), K(hidden_table_mlog_tid)); - } else if (OB_ISNULL(orig_table_mlog_schema) || OB_ISNULL(hidden_table_mlog_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mlog table schema should not be null", - KR(ret), KP(orig_table_mlog_schema), KP(hidden_table_mlog_schema)); - } else if (OB_FAIL(new_orig_table_mlog_schema.assign(*orig_table_mlog_schema))) { - LOG_WARN("failed to assign schema", KR(ret)); - } else if (OB_FAIL(new_hidden_table_mlog_schema.assign(*hidden_table_mlog_schema))) { - LOG_WARN("failed to assign schema", KR(ret)); - } else { - new_orig_table_mlog_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL); - new_hidden_table_mlog_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_NORMAL); - new_orig_table_mlog_schema.set_table_name(hidden_table_mlog_schema->get_table_name_str()); - new_hidden_table_mlog_schema.set_table_name(orig_table_mlog_schema->get_table_name_str()); - if (OB_FAIL(table_schemas.push_back(new_orig_table_mlog_schema)) - || OB_FAIL(table_schemas.push_back(new_hidden_table_mlog_schema))) { - LOG_WARN("failed to add table schemas", KR(ret)); - } - } - } - return ret; -} - int ObDDLService::check_and_replace_default_index_name_on_demand( common::ObIAllocator &allocator, const ObTableSchema &hidden_data_schema/*without target table name*/, @@ -22253,7 +20946,6 @@ int ObDDLService::inner_drop_and_create_tablet_(const int64_t &schema_version, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - common::ObArray orig_ls_id_array; const ObTableSchema *tmp_table_schema = NULL; int64_t del_tablet_count = orig_table_schemas.count(); int64_t create_table_count = new_table_schemas.count(); @@ -22282,12 +20974,9 @@ int ObDDLService::inner_drop_and_create_tablet_(const int64_t &schema_version, // drop tablet if (FAILEDx(tablet_drop.init())) { LOG_WARN("fail to init tablet_drop", KR(ret)); - // get old ls - } else if (OB_FAIL(tablet_drop.get_ls_from_table(*orig_table_schemas.at(0), false /*is_include_hidden*/, orig_ls_id_array))) { - LOG_WARN("failed to get drop tablet's ls", KR(ret)); } else if (OB_FAIL(tablet_drop.add_drop_tablets_of_table_arg(orig_table_schemas))) { LOG_WARN("fail to add drop tablets table args", KR(ret)); - // delete all_tablet_to_ls/insert into all_tablet_to_table_history + // delete tablet mapping / insert into all_tablet_to_table_history } else if (OB_FAIL(tablet_drop.execute())) { LOG_WARN("failed to execute tablet drop", KR(ret)); } @@ -22319,11 +21008,11 @@ int ObDDLService::inner_drop_and_create_tablet_(const int64_t &schema_version, if (OB_FAIL(table_creator.init(true/*need_check_tablet_cnt*/))) { LOG_WARN("table_creator init failed", KR(ret)); } else if (1 == create_table_count && create_table_schema_ptrs.at(0)->is_global_index_table()) { - if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(*create_table_schema_ptrs.at(0), orig_ls_id_array, + if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(*create_table_schema_ptrs.at(0), tenant_data_version, true/*need_create_empty_major_sstable*/))) { LOG_WARN("fail to add_create_tablets_of_tables_arg", KR(ret)); } - } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg(create_table_schema_ptrs, orig_ls_id_array, + } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg(create_table_schema_ptrs, tenant_data_version, need_create_empty_majors/*need_create_empty_major_sstable*/))) { LOG_WARN("fail to add_create_tablets_of_tables_arg", KR(ret)); } @@ -22771,17 +21460,7 @@ int ObDDLService::check_table_schema_is_legal(const obcall::ObTruncateTableArg & LOG_WARN("can not truncate table in recyclebin", KR(ret), K(table_name), K(table_id), K(database_name)); } else if (table_schema.is_user_table() || table_schema.is_mysql_tmp_table()) { - if (table_schema.has_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table with materialized view log is not supported", KR(ret), - K(table_schema), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table with materialized view log is"); - } else if (table_schema.table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table required by materialized view is not supported", KR(ret), - K(table_schema), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table required by materialized view is"); - } else if (check_foreign_key && + if (check_foreign_key && OB_FAIL(check_is_foreign_key_parent_table(table_schema, trans))) { LOG_WARN("failed to check table is foreign key's parent table", KR(ret), K(table_name), K(table_id)); } @@ -22794,21 +21473,6 @@ int ObDDLService::check_table_schema_is_legal(const obcall::ObTruncateTableArg & } else if (table_schema.is_index_table() || table_schema.is_aux_vp_table() || table_schema.is_aux_lob_table()) { ret = OB_NOT_SUPPORTED; LOG_WARN("truncate table is not supported on index or aux vp table", KR(ret) ,K(table_name), K(table_id)); - } else if (table_schema.is_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view log is not supported", - KR(ret), K(table_name), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view log is"); - } else if (table_schema.is_materialized_view()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view is not supported", - KR(ret), K(table_name), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view is"); - } else if (table_schema.mv_container_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view container table is not supported", - KR(ret), K(table_name), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view container table is"); } else { ret = OB_TABLE_NOT_EXIST; LOG_WARN("truncate table not exist", KR(ret), K(table_name), K(table_id), K(database_name)); @@ -22996,20 +21660,6 @@ int ObDDLService::truncate_table(const ObTruncateTableArg &arg, } else if(orig_table_schema->is_index_table() || orig_table_schema->is_aux_vp_table() || orig_table_schema->is_aux_lob_table()) { ret = OB_NOT_SUPPORTED; LOG_WARN("truncate table is not supported on index or aux vp table", K(ret)); - } else if (orig_table_schema->is_materialized_view()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view is not supported", - KR(ret), K(*orig_table_schema)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view is"); - } else if (orig_table_schema->mv_container_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view container table is not supported", - KR(ret), K(*orig_table_schema)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view container table is"); - } else if (orig_table_schema->is_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view log is"); } else if (!orig_table_schema->is_user_table() && !orig_table_schema->is_tmp_table()) { if (orig_table_schema->is_sys_table()) { ret = OB_NOT_SUPPORTED; @@ -23021,16 +21671,6 @@ int ObDDLService::truncate_table(const ObTruncateTableArg &arg, } } else if (OB_FAIL(check_enable_sys_table_ddl(*orig_table_schema, OB_DDL_TRUNCATE_TABLE_CREATE))) { LOG_WARN("ddl is not allowed on system table", K(ret)); - } else if (orig_table_schema->has_mlog_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table with materialized view log is not supported", - KR(ret), KPC(orig_table_schema)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table with materialized view log is"); - } else if (orig_table_schema->table_referenced_by_fast_lsm_mv()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table required by materialized view is not supported", - KR(ret), KPC(orig_table_schema)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table required by materialized view is"); } else if (!orig_table_schema->check_can_do_ddl()) { ret = OB_NOT_SUPPORTED; LOG_WARN("offline ddl is being executed, other ddl operations are not allowed", @@ -23672,46 +22312,10 @@ int ObDDLService::drop_table_in_trans( } else if (OB_ISNULL(sql_trans) && OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { LOG_ERROR("start transaction failed", KR(ret), K(refreshed_schema_version)); - } else if (table_schema.is_materialized_view()) { - // drop mv container table's index - uint64_t container_table_id = table_schema.get_data_table_id(); - const ObTableSchema *container_table_schema = NULL; - if (OB_FAIL(schema_guard.get_table_schema( container_table_id, container_table_schema))) { - LOG_WARN("failed to get table schema", KR(ret), K(container_table_id)); - } else if (OB_ISNULL(container_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("container table schema should not be null", - KR(ret), K(container_table_id)); - } else if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, - *container_table_schema, USER_INDEX, false)))) { - LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); - } else if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, - *container_table_schema, AUX_LOB_META, false)))) { - LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); - } else if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, - *container_table_schema, AUX_LOB_PIECE, false)))) { - LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); - } else if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, // drop mv container table - table_schema, MATERIALIZED_VIEW, false)))) { - LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); - } - - if (OB_SUCC(ret)) { - const uint64_t mview_table_id = table_schema.get_table_id(); - ObMViewDependencyService mv_dep_service(*schema_service_); - if (OB_FAIL(mv_dep_service.remove_mview_dep_infos( - trans, schema_guard, mview_table_id))) { - LOG_WARN("failed to remove mview dep infos", KR(ret)); - } - } } else if (!table_schema.is_aux_table()) { if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, table_schema, USER_INDEX, to_recyclebin)))) { LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); - } else if (table_schema.mv_container_table() - && OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, - table_schema, MATERIALIZED_VIEW_LOG, to_recyclebin)))) { - LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); } else if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, table_schema, AUX_VERTIAL_PARTITION_TABLE, to_recyclebin)))) { LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); @@ -23734,9 +22338,7 @@ int ObDDLService::drop_table_in_trans( if (to_recyclebin && !table_schema.is_index_table() && !is_inner_table(table_schema.get_table_id()) && !table_schema.is_aux_lob_table() - && !table_schema.is_aux_vp_table() - && !table_schema.is_materialized_view() - && !table_schema.is_mlog_table()) { //index/aux_vp/aux_lob/mview table and inner table will drop directly + && !table_schema.is_aux_vp_table()) { // index/aux_vp/aux_lob and inner tables will drop directly if (OB_FAIL(ddl_operator.drop_table_to_recyclebin(table_schema, schema_guard, trans, @@ -23866,14 +22468,6 @@ int ObDDLService::drop_aux_table_in_drop_table( if (OB_FAIL(table_schema.get_aux_vp_tid_array(aux_tid_array))) { LOG_WARN("get_aux_vp_tid_array failed", K(ret)); } - } else if (MATERIALIZED_VIEW == table_type) { - if (OB_FAIL(aux_tid_array.push_back(table_schema.get_data_table_id()))) { - LOG_WARN("push container table id failed", KR(ret)); - } - } else if (MATERIALIZED_VIEW_LOG == table_type) { - if (table_schema.has_mlog_table() && OB_FAIL(aux_tid_array.push_back(table_schema.get_mlog_tid()))) { - LOG_WARN("push materialized view log table id failed", KR(ret)); - } } else { ret = OB_INVALID_ARGUMENT; LOG_WARN("invaid aux table type", K(ret), K(table_type)); @@ -24295,10 +22889,9 @@ int ObDDLService::restore_table_from_recyclebin(const ObRecyclebinRestoreTableAr if (OB_SUCC(ret)) { if (table_schema->is_index_table() || table_schema->is_aux_vp_table() - || table_schema->is_aux_lob_table() - || table_schema->is_mlog_table()) { + || table_schema->is_aux_lob_table()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("restore index or materialized view log table is not supported now", K(ret)); + LOG_WARN("restore index or auxiliary table is not supported now", K(ret)); } else if (OB_FAIL(restore_table_from_recyclebin_in_trans(*table_schema, new_db_id, arg.new_table_name_, @@ -24434,13 +23027,7 @@ int ObDDLService::purge_table( } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_shema is null", K(ret), K(database_id), K(arg.table_name_), K(object_name)); - } else if (OB_UNLIKELY(table_schema->is_materialized_view() || table_schema->mv_container_table())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected table type", KR(ret), K(arg), KPC(table_schema)); } - } else if (OB_UNLIKELY(table_schema->is_materialized_view() || table_schema->mv_container_table())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected table type", KR(ret), K(arg), KPC(table_schema)); } else if (OB_FAIL(check_object_name_matches_db_name(arg.table_name_, database_id, is_match))) { @@ -24815,7 +23402,7 @@ int ObDDLService::check_table_exists(const ObTableItem &table_item, if (OB_SUCC(ret)) { bool is_view = false; uint64_t org_sess_id = guard.get_session_id(); - if (USER_VIEW == expected_table_type || MATERIALIZED_VIEW == expected_table_type) { + if (USER_VIEW == expected_table_type) { is_view = true; guard.set_session_id(0); } else { @@ -24856,11 +23443,6 @@ int ObDDLService::check_table_exists(const ObTableItem &table_item, ret = OB_TABLE_NOT_EXIST; LOG_WARN("Table type not equal!", K(expected_table_type), K(table_item), K(ret)); } else { /*maybe SYS_TABLE or VIRTUAL TABLE */ } - } else if (MATERIALIZED_VIEW == expected_table_type) { - if (!tmp_table_schema->is_materialized_view()) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("Table type not equal!", K(expected_table_type), K(table_item), K(ret)); - } } else if (is_view) { if (SYSTEM_VIEW == tmp_table_schema->get_table_type()) { // let it go, for case compatible @@ -25065,11 +23647,7 @@ int ObDDLService::collect_temporary_tables_in_session(const ObDropTableArg &cons return ret; } -//same api for drop table, drop index, drop view, drop materialized view -// -// mv rule: -// If the deleted table has mv, you must first delete mv to delete this table -// If it is mv, you need to update the version of the base/depend table, and mv does not enter the recycle bin +// Same API for drop table, drop index and drop view. // drop table process (after add temporary table featrue) // drop table t1; The temporary table is matched first, and the ordinary table cannot be found to be matched again; // drop temporary table t1; must by temp table; @@ -25143,24 +23721,6 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: LOG_WARN("set table_id to hash set failed", K(table_schema->get_table_id()), K(ret)); } else if (OB_FAIL(lock_table(trans, *table_schema))) { LOG_WARN("fail to lock_table", KR(ret), KPC(table_schema)); - } else if (table_schema->is_materialized_view()) { - // first disable and stop old dbms job to avoid lock blocked - // lock mv and mv container table - uint64_t container_table_id = table_schema->get_data_table_id(); - const ObTableSchema *container_table_schema = NULL; - if (OB_FAIL(ObMViewSchedJobUtils::disable_and_stop_job(table_schema->get_table_id()))) { - LOG_WARN("fail to disable and stop job", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( container_table_id, container_table_schema))) { - LOG_WARN("failed to get table schema", KR(ret), K(container_table_id)); - } else if (OB_ISNULL(container_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("container table schema should not be null", - KR(ret), K(container_table_id)); - } else if (OB_FAIL(lock_mview(trans, *table_schema))) { - LOG_WARN("fail to lock mview", KR(ret), KPC(table_schema)); - } else if (OB_FAIL(lock_table(trans, *container_table_schema))) { - LOG_WARN("fail to lock_table", KR(ret), KPC(container_table_schema)); - } } } } @@ -25175,8 +23735,7 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: const ObTableSchema *data_table_schema = nullptr; tmp_table_schema.reset(); is_db_in_recyclebin = false; - const bool is_drop_index_or_mlog = ObTableSchema::is_mlog_table(drop_table_arg.table_type_) - || (USER_INDEX == drop_table_arg.table_type_); + const bool is_drop_index = USER_INDEX == drop_table_arg.table_type_; //ensure use the newest schema of each table if (OB_FAIL(check_table_exists(table_item, drop_table_arg.table_type_, @@ -25192,19 +23751,6 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: } else if (!drop_table_arg.force_drop_ && table_schema->is_in_recyclebin()) { ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; LOG_WARN("can not drop table in recyclebin, use purge instead", K(ret), K(table_item)); - } else if (table_schema->is_materialized_view() && OB_FAIL(schema_guard.get_table_schema( table_schema->get_data_table_id(), data_table_schema))) { - LOG_WARN("failed to get mview container table schema", KR(ret), KPC(table_schema)); - } else if (table_schema->is_materialized_view() && OB_ISNULL(data_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("data_table_schema should not be null", KR(ret)); - } else if (table_schema->has_mlog_table() || (table_schema->is_materialized_view() && data_table_schema->has_mlog_table())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("drop table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "drop table with materialized view log is"); - } else if (table_schema->table_referenced_by_fast_lsm_mv() && !table_schema->is_index_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("drop table required by materialized view is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "drop table required by materialized view is"); } else if (OB_FAIL(tmp_table_schema.assign(*table_schema))) { LOG_WARN("fail to assign table schema", K(ret)); } else if (OB_FAIL(schema_guard.check_database_in_recyclebin(table_schema->get_database_id(), is_db_in_recyclebin))) { @@ -25214,9 +23760,9 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: LOG_WARN("can not drop table in recyclebin", K(ret), K(*table_schema)); } else if (OB_FAIL(check_enable_sys_table_ddl(*table_schema, OB_DDL_DROP_TABLE))) { LOG_WARN("ddl is not allowed on sys table", K(ret)); - } else if (is_drop_index_or_mlog && OB_FAIL(schema_guard.get_table_schema( tmp_table_schema.get_data_table_id(), data_table_schema))) { + } else if (is_drop_index && OB_FAIL(schema_guard.get_table_schema( tmp_table_schema.get_data_table_id(), data_table_schema))) { LOG_WARN("failed to get data table schema", K(ret)); - } else if (is_drop_index_or_mlog && OB_ISNULL(data_table_schema)) { + } else if (is_drop_index && OB_ISNULL(data_table_schema)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("data table not found", K(ret), K(tmp_table_schema.get_data_table_id())); } else if (FALSE_IT(tmp_table_schema.set_in_offline_ddl_white_list(table_item.is_hidden_ || @@ -25234,7 +23780,7 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: // do not record drop table stmt for recover restore table. const bool is_offline_ddl_hidden_data_table = ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL == table_schema->get_table_state_flag(); - const bool use_drop_table_stmt_in_arg = (is_drop_index_or_mlog || is_offline_ddl_hidden_data_table); + const bool use_drop_table_stmt_in_arg = (is_drop_index || is_offline_ddl_hidden_data_table); bool is_cascade_constrains = false; lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL; if (OB_FAIL(ret)) { @@ -25300,9 +23846,7 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: { bool to_recyclebin = drop_table_arg.to_recyclebin_; bool has_conflict_ddl = false; - if (table_schema->is_materialized_view() - || table_schema->is_mlog_table() - || table_schema->is_tmp_table()) { + if (table_schema->is_tmp_table()) { to_recyclebin = false; } if (drop_table_arg.table_type_ == USER_TABLE && OB_FAIL(ObDDLTaskRecordOperator::check_has_conflict_ddl( @@ -25325,7 +23869,7 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: &drop_table_set, mock_fk_parent_table_ptr /* will use it when drop a fk_parent_table */))) { LOG_WARN("ddl_service_ drop_table failed", K(table_item), K(ret)); - } else if (drop_table_arg.task_id_ != 0 && is_drop_index_or_mlog) { + } else if (drop_table_arg.task_id_ != 0 && is_drop_index) { ObTableLockOwnerID owner_id; if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, drop_table_arg.task_id_))) { @@ -25354,28 +23898,15 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: fail_for_fk_cons = true; ret = OB_SUCCESS; } else if (OB_TABLE_NOT_EXIST == ret || OB_ERR_BAD_DATABASE == ret) { - if (MATERIALIZED_VIEW == drop_table_arg.table_type_) { - if (!drop_table_arg.if_exist_) { - // OB_ERR_MVIEW_NOT_EXIST can only print one mview, report an error and exit the loop when encountering the first non-existent mview - ret = OB_ERR_MVIEW_NOT_EXIST; - ObCStringHelper helper; - LOG_USER_ERROR(OB_ERR_MVIEW_NOT_EXIST, helper.convert(table_item.database_name_), helper.convert(table_item.table_name_)); - } else { - ObCStringHelper helper; - LOG_USER_NOTE(OB_ERR_MVIEW_NOT_EXIST, helper.convert(table_item.database_name_), helper.convert(table_item.table_name_)); - ret = OB_SUCCESS; - } + int tmp_ret = OB_SUCCESS; + ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = log_drop_warn_or_err_msg(table_item, + drop_table_arg.if_exist_, + err_table_list))) { + ret = tmp_ret; + LOG_WARN("log_drop_warn_or_err_msg failed", K(ret)); } else { - int tmp_ret = OB_SUCCESS; ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = log_drop_warn_or_err_msg(table_item, - drop_table_arg.if_exist_, - err_table_list))) { - ret = tmp_ret; - LOG_WARN("log_drop_warn_or_err_msg failed", K(ret)); - } else { - ret = OB_SUCCESS; - } } } } @@ -25405,16 +23936,6 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: LOG_WARN("failed to drop index table", K(err_table_list), K(ret)); break; } - case MATERIALIZED_VIEW: { - if (fail_for_fk_cons) { - ret = OB_ERR_TABLE_IS_REFERENCED; - LOG_WARN("Cannot drop table that is referenced by foreign key", K(ret)); - } else { - ret = OB_ERR_MVIEW_NOT_EXIST; - LOG_WARN("failed to drop mview ", K(ret), K(drop_table_arg)); - } - break; - } default: { ret = OB_ERR_UNEXPECTED; SQL_RESV_LOG(WARN, "Unknown table type", K(drop_table_arg), K(ret)); @@ -25427,7 +23948,6 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obcall: // drop table and update ddl task status should be done in single trans. if (OB_FAIL(ret)) { } else if (drop_table_arg.task_id_ > 0 && drop_table_arg.table_type_ != USER_INDEX - && !ObTableSchema::is_mlog_table(drop_table_arg.table_type_) && OB_FAIL(ObDDLRetryTask::update_task_status_wait_child_task_finish(trans, drop_table_arg.task_id_))) { LOG_WARN("update task status of drop table failed", K(ret)); } @@ -26030,7 +24550,7 @@ int ObDDLService::drop_directly_and_create_index_schema_( if (OB_FAIL(ret)) { } else if (OB_FAIL(generate_tablet_id(index_schema))) { LOG_WARN("failed to generate tablet id", K(ret)); - } else if (OB_FAIL(create_index_or_mlog_table_in_trans(index_schema, + } else if (OB_FAIL(create_index_table_in_trans(index_schema, ddl_stmt_str, &trans, schema_guard, false/*need_check_tablet_cnt*/, tenant_data_version, false/*is_table_empty*/))) { LOG_WARN("create_table_in_trans failed", K(index_schema), KR(ret), K(ddl_stmt_str)); @@ -26138,77 +24658,6 @@ int ObDDLService::update_index_status(const obcall::ObUpdateIndexStatusArg &arg) return ret; } -int ObDDLService::update_mview_status(const obcall::ObUpdateMViewStatusArg &arg) -{ - int ret = OB_SUCCESS; - const ObTableSchema *mview_schema = nullptr; - ObSchemaGetterGuard schema_guard; - enum ObMVAvailableFlag mv_available_flag = arg.mv_available_flag_; - const uint64_t mview_table_id = arg.mview_table_id_; - - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init", KR(ret)); - } else if (OB_INVALID_ID == mview_table_id - || (ObMVAvailableFlag::IS_MV_UNAVAILABLE != mv_available_flag - && ObMVAvailableFlag::IS_MV_AVAILABLE != mv_available_flag)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ObUpdateMViewStatusArg", KR(ret), K(arg)); - } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { - LOG_WARN("failed to get schema guard with version in inner table", KR(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema( mview_table_id, mview_schema))) { - LOG_WARN("failed to get table schema", KR(ret), KT(mview_table_id)); - } else if (OB_ISNULL(mview_schema)) { - // maybe table has already been deleted, do-nothing - } else { - ObDDLSQLTransaction trans(schema_service_); - int64_t refreshed_schema_version = 0; - int64_t new_schema_version = OB_INVALID_VERSION; - ObSchemaService *schema_service = schema_service_->get_schema_service(); - if (schema_service == nullptr) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("schema_service is null", KR(ret), KP(schema_service)); - } else if (OB_FAIL(schema_guard.get_schema_version(refreshed_schema_version))) { - LOG_WARN("failed to get tenant schema version", KR(ret)); - } else if (OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { - LOG_WARN("failed to start transaction", KR(ret), K(refreshed_schema_version)); - } else if (OB_FAIL(schema_service_->gen_new_schema_version(new_schema_version))) { - LOG_WARN("fail to gen new schema_version", KR(ret)); - } else { - SMART_VAR(ObTableSchema, new_mview_schema) { - if (OB_FAIL(new_mview_schema.assign(*mview_schema))) { - LOG_WARN("fail to assign mview schema", KR(ret)); - } - if (OB_SUCC(ret)) { - new_mview_schema.set_table_id(mview_table_id); - new_mview_schema.set_mv_available(mv_available_flag); - new_mview_schema.set_schema_version(new_schema_version); - new_mview_schema.set_in_offline_ddl_white_list(arg.in_offline_ddl_white_list_); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(schema_service->get_table_sql_service().update_mview_status( - new_mview_schema, trans))) { - LOG_WARN("failed to update mview status", KR(ret), K(mview_table_id), K(mv_available_flag)); - } - } - } - } - - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) { - LOG_WARN("end transaction failed", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(publish_schema())) { - LOG_WARN("failed to publish schema", KR(ret)); - } - } - } - return ret; -} - int ObDDLService::add_table_schema( ObTableSchema &table_schema, share::schema::ObSchemaGetterGuard &schema_guard) @@ -26220,7 +24669,7 @@ int ObDDLService::add_table_schema( LOG_WARN("variable is not init", KR(ret)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_data_version))) { LOG_WARN("get min data version failed", K(ret), K(table_schema)); - } else if (OB_FAIL(create_index_or_mlog_table_in_trans(table_schema, NULL, NULL, schema_guard, + } else if (OB_FAIL(create_index_table_in_trans(table_schema, NULL, NULL, schema_guard, false/*need_check_tablet_cnt*/, tenant_data_version, false/*is_table_empty*/))) { LOG_WARN("create_table_in_trans failed", KR(ret), K(table_schema)); } @@ -30991,8 +29440,7 @@ int ObDDLSQLTransaction::serialize_inc_schemas_( // Upstream serializes full incremental schema dict here (via ObDataDictStorage), // but SeekDB only needs the MDS type signal, not the data content. char dummy = '\0'; - if (OB_FAIL(register_tx_data(SYS_LS, - transaction::ObTxDataSourceType::DDL_TRANS, + if (OB_FAIL(register_tx_data(transaction::ObTxDataSourceType::DDL_TRANS, &dummy, sizeof(dummy)))) { LOG_WARN("register DDL_TRANS MDS failed", KR(ret)); @@ -31095,7 +29543,7 @@ int ObDDLSQLTransaction::lock_all_ddl_operation( if (OB_FAIL(schema_service_->get_ddl_epoch_mgr().get_ddl_epoch(ddl_epoch_tmp))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; - if (OB_FAIL(ObDDLUtil::check_local_is_sys_leader())) { + if (OB_FAIL(ObDDLUtil::check_local_sys_tenant())) { LOG_WARN("fail to check whether local is sys tenant leader", KR(ret)); } else if (OB_FAIL(schema_service_->get_ddl_epoch_mgr().promote_ddl_epoch( ctx.get_timeout(), ddl_epoch_tmp))) { LOG_WARN("promote epoch fail", K(ret)); @@ -33055,7 +31503,7 @@ int ObDDLService::drop_index_to_scheduler_(ObMySQLTransaction &trans, if (OB_EAGAIN != ret) { LOG_WARN("failed to check vec index ", K(ret)); } - } else if (!drop_index_arg->is_inner_ && !index_table_schema->can_read_index() && OB_FAIL(ObDDLTaskRecordOperator::check_has_index_or_mlog_task( + } else if (!drop_index_arg->is_inner_ && !index_table_schema->can_read_index() && OB_FAIL(ObDDLTaskRecordOperator::check_has_index_task( trans, *index_table_schema, orig_table_schema.get_table_id(), has_index_task))) { LOG_WARN("failed to check ddl conflict", KR(ret)); } else if (has_index_task) { @@ -33156,67 +31604,36 @@ int ObDDLService:: get_tablets_with_table_id_(const ObArray &tab return ret; } -int ObDDLService::build_unbind_lob_args(const ObArray &tablet_ids, - ObIArray &args, - ObDDLSQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObArray tablets; - if (OB_FAIL(get_tablets(tablet_ids, tablets, trans))) { - LOG_WARN("failed to get tablet ids of orig table", KR(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablets.count(); i++) { - const ObLSID &ls_id = tablets[i].first; - int64_t j = 0; - for (; j < args.count(); j++) { - if (args.at(j).ls_id_ == ls_id) { - break; - } - } - if (j == args.count()) { - ObBatchUnbindLobTabletArg arg; - - arg.ls_id_ = ls_id; - if (OB_FAIL(args.push_back(arg))) { - LOG_WARN("failed to push back modify arg", KR(ret)); - } - } - if (OB_SUCC(ret)) { - ObBatchUnbindLobTabletArg &arg = args.at(j); - const ObTabletID &tablet_id = tablets[i].second; - if (OB_FAIL(arg.data_tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back", KR(ret)); - } - } - } - LOG_DEBUG("build modify tablet binding args", KR(ret), K(args)); - return ret; -} - int ObDDLService::unbind_lob_tablets( const ObTableSchema &data_table_schema, ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - ObArray data_tablet_ids; - ObArray args; + ObArray tablets; if (OB_FAIL(data_table_schema.get_tablet_ids(data_tablet_ids))) { LOG_WARN("get tablet ids failed", K(ret)); - } else if (OB_FAIL(build_unbind_lob_args(data_tablet_ids, args, trans))) { - LOG_WARN("failed to build reuse index args", KR(ret)); + } else if (OB_FAIL(get_tablets(data_tablet_ids, tablets, trans))) { + LOG_WARN("failed to get tablet ids of data table", KR(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < args.count(); i++) { - int64_t pos = 0; - int64_t size = args[i].get_serialize_size(); - ObArenaAllocator allocator("DropLob"); - char *buf = nullptr; - if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate", K(ret)); - LOG_WARN("failed to serialize arg", KR(ret)); - } else if (OB_FAIL(trans.register_tx_data(args[i].ls_id_, transaction::ObTxDataSourceType::UNBIND_LOB_TABLET, buf, pos))) { - LOG_WARN("failed to register tx data", KR(ret)); + + if (OB_SUCC(ret) && !tablets.empty()) { + ObBatchUnbindLobTabletArg arg; + if (OB_FAIL(arg.data_tablet_ids_.assign(tablets))) { + LOG_WARN("failed to assign data tablet ids", KR(ret), K(tablets)); + } else { + ObArenaAllocator allocator("DropLob"); + int64_t pos = 0; + int64_t size = arg.get_serialize_size(); + char *buf = nullptr; + if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate", K(ret)); + } else if (OB_FAIL(arg.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize arg", KR(ret)); + } else if (OB_FAIL(trans.register_tx_data(transaction::ObTxDataSourceType::UNBIND_LOB_TABLET, buf, pos))) { + LOG_WARN("failed to register tx data", KR(ret)); + } } } return ret; @@ -33329,22 +31746,15 @@ namespace oceanbase namespace share { -int ObDDLUtil::check_local_is_sys_leader() +int ObDDLUtil::check_local_sys_tenant() { int ret = OB_SUCCESS; - common::ObRole role; - int64_t proposal_id = 0; if (OB_ISNULL(GCTX.omt_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(GCTX.omt_)); } else if (OB_UNLIKELY(!GCTX.omt_->has_tenant())) { ret = OB_TENANT_NOT_EXIST; LOG_WARN("local server does not have SYS tenant resource", KR(ret)); - } else if (OB_FAIL(get_sys_log_handler_role_and_proposal_id(role, proposal_id))) { - LOG_WARN("fail to get role and proposal id", KR(ret), K(role), K(proposal_id)); - } else if (!is_strong_leader(role)) { - ret = OB_LS_NOT_LEADER; - LOG_WARN("current server not leader of SYS tenant", KR(ret), K(role)); } return ret; } diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index d359dd8ea..d388e6563 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -31,7 +31,6 @@ #include "rootserver/ddl_task/ob_ddl_task.h" #include "common/mysqlclient/ob_mysql_transaction.h" #include "lib/container/ob_iarray.h" -#include "storage/tablet/ob_tablet_binding_helper.h" #include "storage/ddl/ob_ddl_clog.h" #include "share/ob_lonely_table_clean_rpc_struct.h" // for ObForceDropLonelyLobAuxTableArg #include "share/ob_freeze_info_proxy.h" @@ -55,7 +54,6 @@ class ObMySQLTransaction; namespace obcall { class ObAccountArg; -class ObCreateMLogArg; } namespace share @@ -72,7 +70,6 @@ class ObTableSchema; class ObMultiVersionSchemaService; class ObNeedPriv; class ObSchemaMgr; -class ObMViewInfo; } } @@ -95,8 +92,6 @@ struct ObTruncateInfoService; class ObPLDDLService; class ObDDLService { -public: - typedef std::pair LSTabletID; public: friend class ObTableGroupHelp; friend class ObStandbyClusterSchemaProcessor; @@ -122,23 +117,6 @@ class ObDDLService share::schema::ObTableSchema &table_schema, ObMySQLTransaction &sql_trans); - virtual int create_mlog_table(ObMySQLTransaction &sql_trans, - const obcall::ObCreateMLogArg &arg, - const uint64_t tenant_data_version, - share::schema::ObSchemaGetterGuard &schema_guard, - share::schema::ObTableSchema &table_schema); - - virtual int create_mlog_tablet(ObMySQLTransaction &trans, - share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema &mlog_schema, - const bool need_check_tablet_cnt, - const uint64_t tenant_data_version); - - virtual int add_mlog(ObMySQLTransaction &trans, - const obcall::ObCreateMLogArg &arg, - ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema &mlog_schema); - int rebuild_vec_index(const obcall::ObRebuildIndexArg &arg, obcall::ObAlterTableRes &res); @@ -211,16 +189,6 @@ class ObDDLService const share::schema::ObTableSchema **table_schema); int create_hidden_table(const obcall::ObCreateHiddenTableArg &create_hidden_table_arg, obcall::ObCreateHiddenTableRes &res); - int mview_complete_refresh(const obcall::ObMViewCompleteRefreshArg &arg, - obcall::ObMViewCompleteRefreshRes &res, - share::schema::ObSchemaGetterGuard &schema_guard); - int mview_complete_refresh_in_trans(const obcall::ObMViewCompleteRefreshArg &arg, - obcall::ObMViewCompleteRefreshRes &res, - ObDDLSQLTransaction &trans, - common::ObIAllocator &allocator, - share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_data_version, - ObDDLTaskRecord &task_record); /** * For recover restore table ddl task, it is a cross-tenant task, including, * 1. Create a hidden table under different tenant but associated with the source table; @@ -231,7 +199,6 @@ class ObDDLService const common::ObIArray &foreign_key_infos, bool &have_index); virtual int update_index_status(const obcall::ObUpdateIndexStatusArg &arg); - virtual int update_mview_status(const obcall::ObUpdateMViewStatusArg &arg); virtual int add_table_schema(share::schema::ObTableSchema &table_schema, share::schema::ObSchemaGetterGuard &schema_guard); @@ -473,12 +440,7 @@ class ObDDLService int get_tablets( const ObArray &tablet_ids, - common::ObIArray &tablets, - ObDDLSQLTransaction &trans); - int build_modify_tablet_binding_args(const ObArray &tablet_ids, - const bool is_hidden_tablets, - const int64_t schema_version, - common::ObIArray &args, + common::ObIArray &tablets, ObDDLSQLTransaction &trans); int unbind_hidden_tablets( const share::schema::ObTableSchema &orig_table_schema, @@ -500,7 +462,6 @@ class ObDDLService * All these index status and name will change in the same trans */ int switch_index_name_and_status_for_vec_index_table(obcall::ObAlterTableArg &alter_table_arg); - int switch_index_name_and_status_for_mlog_table(obcall::ObAlterTableArg &alter_table_arg); /** * This function is called by the storage layer in the three stage of offline ddl. @@ -969,9 +930,6 @@ static int get_tenant_schema_guard_with_version_in_inner_table(share::schema::Ob const transaction::tablelock::ObTableLockPriority lock_priority = transaction::tablelock::ObTableLockPriority::NORMAL, const int64_t timeout_us = 0); - // lock mview object, unlock when ddl trans end - // Must before locking the container table - int lock_mview(ObMySQLTransaction &trans, const ObSimpleTableSchemaV2 &table_schema); int recompile_view(const ObTableSchema &view_schema, const bool reset_view_column_infos, ObDDLSQLTransaction &trans); int recompile_all_views_batch(const common::ObIArray &view_ids); int try_check_and_set_table_schema_in_tablegroup( @@ -1172,7 +1130,7 @@ int check_will_be_having_domain_index_operation( share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *&index_table_schema); int set_tablegroup_id(share::schema::ObTableSchema &table_schema); - int create_index_or_mlog_table_in_trans(share::schema::ObTableSchema &table_schema, + int create_index_table_in_trans(share::schema::ObTableSchema &table_schema, const common::ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans, share::schema::ObSchemaGetterGuard &schema_guard, @@ -1185,11 +1143,6 @@ int check_will_be_having_domain_index_operation( share::schema::ObSchemaGetterGuard &schema_guard, const uint64_t tenant_data_version, const share::ObForkTableInfo &fork_table_info = share::ObForkTableInfo()); - int create_tablets_in_trans_for_mv_(common::ObIArray &table_schemas, - ObDDLOperator &ddl_operator, - ObMySQLTransaction &trans, - share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_data_version); int set_new_database_options(const obcall::ObAlterDatabaseArg &arg, share::schema::ObDatabaseSchema &new_database_schema); int set_new_table_options( @@ -1417,7 +1370,6 @@ int check_will_be_having_domain_index_operation( bool &is_exist); int check_alter_unused_column(const share::schema::ObSchemaOperationType &operation_type, const share::schema::ObColumnSchemaV2 *orig_column_schema); - int check_long_run_ddl_table_type_(const ObTableSchema &orig_table_schema); int check_long_run_ddl_has_index_(const ObTableSchema *orig_table_schema, obcall::ObAlterTableArg &alter_table_arg, ObSchemaGetterGuard &schema_guard); @@ -1549,7 +1501,7 @@ int check_will_be_having_domain_index_operation( const share::schema::ObTableSchema &hidden_table_schema, common::ObMySQLTransaction &trans, common::ObSArray &cst_ids); - int rebuild_hidden_table_index_and_mlog(share::schema::ObSchemaGetterGuard &schema_guard, + int rebuild_hidden_table_index(share::schema::ObSchemaGetterGuard &schema_guard, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans, common::ObSArray &new_table_schemas); @@ -1563,14 +1515,6 @@ int check_will_be_having_domain_index_operation( common::ObMySQLTransaction &trans, ObSArray &new_table_schemas, ObSArray &index_ids); - int add_new_mlog_schema(const share::schema::ObTableSchema &orig_table_schema, - const share::schema::ObTableSchema &hidden_table_schema, - share::schema::ObSchemaGetterGuard &schema_guard, - share::schema::ObSchemaGetterGuard &dest_schema_guard, - common::ObMySQLTransaction &trans, - common::ObIAllocator &allocator, - common::ObSArray &new_table_schemas, - common::ObSArray &index_ids); int check_index_table_need_rebuild( const share::schema::ObTableSchema &index_table_schema, const common::ObIArray &drop_cols_id_arr, @@ -1779,9 +1723,6 @@ int check_will_be_having_domain_index_operation( int check_new_column_for_index( ObIArray &idx_schemas, const share::schema::ObColumnSchemaV2 &new_column_schema); - // - // materialized view related - // int alter_table_update_index_and_view_column( const share::schema::ObTableSchema &new_table_schema, const share::schema::ObColumnSchemaV2 &new_column_schema, @@ -1956,11 +1897,6 @@ int check_will_be_having_domain_index_operation( const ObTableSchema &orig_table_schema, obcall::ObIndexArg *index_arg); int validate_rename_table_args(const ObTableSchema *table_schema); - int construct_rename_table_items_for_mview(const ObTableSchema *table_schema, - share::schema::ObSchemaGetterGuard &schema_guard, - const obcall::ObRenameTableItem &rename_item, - common::ObArenaAllocator &allocator, - ObIArray &full_rename_items); public: int check_restore_point_allow(const share::schema::ObTableSchema &table_schema); @@ -1977,9 +1913,6 @@ int check_will_be_having_domain_index_operation( int drop_lob(const obcall::ObDropLobArg &arg); int force_drop_lonely_lob_aux_table(const obcall::ObForceDropLonelyLobAuxTableArg &arg); - int build_unbind_lob_args(const common::ObArray &tablet_ids, - common::ObIArray &args, - ObDDLSQLTransaction &trans); int unbind_lob_tablets(const share::schema::ObTableSchema &data_table_schema, ObDDLSQLTransaction &trans); int submit_drop_lob_task_(ObMySQLTransaction &trans, @@ -2181,18 +2114,6 @@ int check_will_be_having_domain_index_operation( const int table_id, ObArray &tablet_ids); public: - - int start_mview_complete_refresh_task( - ObMySQLTransaction &trans, - ObSchemaGetterGuard &schema_guard, - const ObTableSchema &mview_schema, - const ObTableSchema &container_table_schema, - const ObIArray *dep_infos, - common::ObIAllocator &allocator, - const uint64_t tenant_data_version, - const share::schema::ObMViewInfo &mview_info, - const ObString &ddl_stmt_str, - ObDDLTaskRecord &task_record); //not check belong to the same table int check_same_partition(const ObPartition &l, const ObPartition &r, const ObPartitionFuncType part_type, bool &is_matched) const; @@ -2374,12 +2295,7 @@ int check_will_be_having_domain_index_operation( } bool need_modify_dep_obj_status(const obcall::ObAlterTableArg &alter_table_arg) const; - int swap_orig_and_hidden_table_mlog_state_( - ObSchemaGetterGuard &schema_guard, - const uint64_t orig_table_mlog_tid, - const uint64_t hidden_table_mlog_tid, - ObIArray &table_schemas); - int build_hidden_table_index_and_mlog_schema_( + int build_hidden_table_index_schema_( const share::schema::ObTableSchema &hidden_table_schema, share::schema::ObTableSchema &index_schema); @@ -2441,8 +2357,7 @@ class ObDDLSQLTransaction : public common::ObMySQLTransaction int lock_all_ddl_operation( common::ObMySQLTransaction &trans, const bool enable_ddl_parallel); - int register_tx_data(const share::ObLSID &ls_id, - const transaction::ObTxDataSourceType &type, + int register_tx_data(const transaction::ObTxDataSourceType &type, const char *buf, const int64_t buf_len); void disable_serialize_inc_schemas() { trans_start_schema_version_ = 0; } diff --git a/src/rootserver/ob_ddl_service_launcher.cpp b/src/rootserver/ob_ddl_service_launcher.cpp index de338617f..655565db4 100644 --- a/src/rootserver/ob_ddl_service_launcher.cpp +++ b/src/rootserver/ob_ddl_service_launcher.cpp @@ -83,7 +83,7 @@ void ObDDLServiceLauncher::destroy() K(duration_time)); } -int ObDDLServiceLauncher::switch_to_leader() +int ObDDLServiceLauncher::activate() { int ret = OB_SUCCESS; int64_t start_time = ObTimeUtility::current_time(); @@ -107,27 +107,9 @@ int ObDDLServiceLauncher::get_sys_palf_role_and_epoch( common::ObRole &role, int64_t &proposal_id) { - int ret = OB_SUCCESS; - role = FOLLOWER; - proposal_id = 0; - if (OB_ISNULL(GCTX.omt_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(GCTX.omt_)); - } else if (OB_UNLIKELY(!GCTX.omt_->has_tenant())) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("local server does not have SYS tenant resource", KR(ret)); - } else { - MOD_SCOPE { - logservice::ObLogService *log_service = share::g_mp->log_service(); - if (OB_ISNULL(log_service)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("mtl ObLogService should not be null", KR(ret), KP(log_service)); - } else if (OB_FAIL(log_service->get_palf_role(SYS_LS, role, proposal_id))) { - LOG_WARN("failed to get role from palf", KR(ret)); - } - } - } - return ret; + role = LEADER; + proposal_id = 1; + return OB_SUCCESS; } int ObDDLServiceLauncher::start_ddl_service_with_old_logic( @@ -154,46 +136,15 @@ int ObDDLServiceLauncher::start_ddl_service_with_old_logic( return ret; } -int ObDDLServiceLauncher::resume_leader() -{ - int ret = OB_SUCCESS; - FLOG_INFO("[DDL_SERVICE_LAUNCHER] begin resume_leader for ddl_service_launcher"); - if (OB_FAIL(switch_to_leader())) { - LOG_WARN("fail to switch to leader", KR(ret)); - } - FLOG_INFO("[DDL_SERVICE_LAUNCHER] finish resume_leader for ddl_service_launcher", KR(ret)); - return ret; -} - -void ObDDLServiceLauncher::switch_to_follower_forcedly() +void ObDDLServiceLauncher::deactivate() { - FLOG_INFO("[DDL_SERVICE_LAUNCHER] begin switch_to_follower_forcedly for ddl_service_launcher"); - switch_to_follower_gracefully(); - FLOG_INFO("[DDL_SERVICE_LAUNCHER] finish switch_to_follower_forcedly for ddl_service_launcher"); -} - -int ObDDLServiceLauncher::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - int64_t start_time = ObTimeUtility::current_time(); - FLOG_INFO("[DDL_SERVICE_LAUNCHER] begin switch_to_follower_gracefully for ddl_service_launcher"); - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ddl service launcher is not inited", KR(ret), K_(inited)); - } else { + if (inited_) { SpinWLockGuard guard(rw_lock_); if (OB_NOT_NULL(GCTX.root_service_)) { - // try reset cache for schema refresh GCTX.root_service_->get_ddl_service().get_index_name_checker().reset_all_cache(); - FLOG_INFO("reset index name checker success"); } ATOMIC_SET(&is_ddl_service_started_, false); } - // rw_lock_ is released now - int64_t duration_time = ObTimeUtility::current_time() - start_time; - FLOG_INFO("[DDL_SERVICE_LAUNCHER] finish switch_to_follower_gracefully for ddl_service_launcher", KR(ret), - K(duration_time)); - return ret; } int ObDDLServiceLauncher::init_sequence_id_( diff --git a/src/rootserver/ob_ddl_service_launcher.h b/src/rootserver/ob_ddl_service_launcher.h index 19b2ff04d..55ddc8430 100644 --- a/src/rootserver/ob_ddl_service_launcher.h +++ b/src/rootserver/ob_ddl_service_launcher.h @@ -20,7 +20,7 @@ #include "common/ob_role.h" // for ObRole #include "lib/utility/ob_macro_utils.h" // for DISALLOW_COPY_AND_ASSIGN //#include "lib/lock/ob_spin_rwlock.h" // for SpinRWLock -#include "logservice/ob_log_base_type.h" // for ObIRoleChangeSubHandler etc. +#include "logservice/ob_log_base_type.h" // for ObILocalLogHandler etc. #include "share/scn.h" // for SCN #include "rootserver/ob_tenant_thread_helper.h" // for DEFINE_MTL_FUNC @@ -32,7 +32,7 @@ class SpinRWLock; } namespace rootserver { -class ObDDLServiceLauncher : public logservice::ObIRoleChangeSubHandler, +class ObDDLServiceLauncher : public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler, public logservice::ObIReplaySubHandler { @@ -46,11 +46,9 @@ class ObDDLServiceLauncher : public logservice::ObIRoleChangeSubHandler, bool is_inited() const { return inited_; } static bool is_ddl_service_started() { return ATOMIC_LOAD(&is_ddl_service_started_); } - // for ObIRoleChangeSubHandler - virtual int switch_to_leader() override; - virtual void switch_to_follower_forcedly() override; - virtual int switch_to_follower_gracefully() override; - virtual int resume_leader() override; + // for ObILocalLogHandler + int activate() override; + void deactivate() override; int start_ddl_service_with_old_logic( const int64_t new_rs_epoch, diff --git a/src/rootserver/ob_load_inner_table_schema_executor.cpp b/src/rootserver/ob_load_inner_table_schema_executor.cpp index 68a9ed399..2d8279fc1 100644 --- a/src/rootserver/ob_load_inner_table_schema_executor.cpp +++ b/src/rootserver/ob_load_inner_table_schema_executor.cpp @@ -21,7 +21,6 @@ #include "share/inner_table/ob_load_inner_table_schema.h" #include "lib/utility/utility.h" #include "share/ob_server_struct.h" -#include "share/location_cache/ob_location_service.h" #include "share/ob_global_stat_proxy.h" #include "share/inner_table/ob_dump_inner_table_schema.h" diff --git a/src/rootserver/ob_lonely_table_cleaner.cpp b/src/rootserver/ob_lonely_table_cleaner.cpp index fe1d2f8b0..234801545 100644 --- a/src/rootserver/ob_lonely_table_cleaner.cpp +++ b/src/rootserver/ob_lonely_table_cleaner.cpp @@ -18,77 +18,33 @@ #include "ob_ddl_service.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" -#include "storage/tx_storage/ob_ls_service.h" +#include "share/schema/ob_schema_getter_guard.h" namespace oceanbase { namespace rootserver { -static int get_ls_from_table(const share::schema::ObTableSchema &table_schema, - ObDDLSQLTransaction &trans, common::ObIArray &ls_ids) +static int check_tenant_not_active() { int ret = OB_SUCCESS; - ObArray tablet_ids; - int64_t all_part_num = 0; - if (-1 == (all_part_num = table_schema.get_all_part_num())) { + ObSchemaGetterGuard schema_guard; + const ObSimpleTenantSchema *tenant_schema = nullptr; + if (OB_ISNULL(GCTX.schema_service_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet num", K(table_schema), KR(ret)); - } else if (OB_FAIL(ls_ids.reserve(all_part_num))) { - LOG_WARN("fail to reserve array", KR(ret), K(all_part_num)); - } else if (OB_FAIL(table_schema.get_tablet_ids(tablet_ids))) { - LOG_WARN("fail to get tablet ids", KR(ret), K(table_schema)); - } else { - // use sys_ls if is sys tablet firstly - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - const ObTabletID &tablet_id = tablet_ids.at(i); - if (tablet_id.belong_to_sys_ls()) { - if (OB_FAIL(ls_ids.push_back(share::SYS_LS))) { - LOG_WARN("push back ls id fail", KR(ret), K(i), K(tablet_id)); - } - } - } - if(OB_FAIL(ret)) { - } else if (ls_ids.count() == 0) { - // no sys tablet, get ls id from system table - if (OB_FAIL(share::ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("fail to batch_get_ls", KR(ret), K(tablet_ids), K(ls_ids), K(table_schema)); - } - } - } - if (OB_FAIL(ret)) { - } else if (ls_ids.count() != all_part_num) { + LOG_WARN("schema service is null", K(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_schema))) { + LOG_WARN("get tenant schema failed", K(ret)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant does not exist", K(ret)); + } else if (tenant_schema->is_normal() || tenant_schema->is_dropping()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls of tablet is not equal partition num", KR(ret), K(ls_ids.count()), K(all_part_num), - K(tablet_ids), K(ls_ids), K(table_schema)); - } - return ret; -} - -static int check_ls_not_exist(const common::ObArray &ls_ids) -{ - int ret = OB_SUCCESS; - int64_t not_exist_ls_cnt = 0; - for (int i = 0; OB_SUCC(ret) && i < ls_ids.count(); ++i){ - const ObLSID ls_id = ls_ids.at(i); - ObLSExistState state; - if (OB_FAIL(ObLocationService::check_ls_exist(ls_id, state))) { - LOG_WARN("failed to check ls exist", KR(ret), K(i), K(ls_id)); - } else if (state.is_deleted()) { - ++not_exist_ls_cnt; - LOG_WARN("ls is deleted", KR(ret), K(i), K(ls_id), K(state)); - } - } - if (OB_SUCC(ret)) { - if (0 == not_exist_ls_cnt) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("all ls exist, unexcepted situation", KR(ret), K(not_exist_ls_cnt), K(ls_ids.count()), K(ls_ids)); - } else if (not_exist_ls_cnt != ls_ids.count()) { - LOG_ERROR("there are some ls exist, some not exist", KR(ret), K(not_exist_ls_cnt), K(ls_ids.count()), K(ls_ids)); - } else { - LOG_ERROR("all ls not exist", KR(ret), K(not_exist_ls_cnt), K(ls_ids.count()), K(ls_ids)); - } + LOG_ERROR("tenant is still active", K(ret)); + } else { + LOG_INFO("tenant is not active", K(ret)); } return ret; } @@ -115,8 +71,6 @@ int ObDDLService::force_drop_lonely_lob_aux_table(const obcall::ObForceDropLonel bool exist = false; bool ignore_ls_not_exist_for_lob_meta = false; bool ignore_ls_not_exist_for_lob_piece = false; - common::ObArray lob_meta_ls_ids; - common::ObArray lob_piece_ls_ids; HEAP_VAR(ObTableSchema, tmp_lob_table_schema) { if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { @@ -145,15 +99,13 @@ int ObDDLService::force_drop_lonely_lob_aux_table(const obcall::ObForceDropLonel // 4. drop lob meta table } else if (OB_FAIL(tmp_lob_table_schema.assign(*lob_meta_table_schema_ptr))) { LOG_WARN("fail to assign lob meta table schema", KR(ret)); - } else if (OB_FAIL(get_ls_from_table(tmp_lob_table_schema, trans, lob_meta_ls_ids))) { - LOG_WARN("get ls of lob meta table fail", KR(ret), K(arg), K(tmp_lob_table_schema)); } else if (OB_FAIL(ddl_operator.drop_table(tmp_lob_table_schema, trans, nullptr/*ddl_stmt_str*/, false/*is_truncate_table*/, nullptr/*drop_table_set*/, false/*is_drop_db*/, true/*delete_priv*/, true/*is_force_drop_lonely_lob_aux_table*/))) { LOG_ERROR("fail to drop lob meta table", KR(ret), K(tmp_lob_table_schema)); if (OB_LS_NOT_EXIST == ret || OB_LS_IS_DELETED == ret) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(check_ls_not_exist(lob_meta_ls_ids))) { - LOG_WARN("check_ls_not_exist fail", KR(tmp_ret)); + if (OB_TMP_FAIL(check_tenant_not_active())) { + LOG_WARN("check tenant state failed", KR(tmp_ret)); } else { LOG_ERROR("ls not exist, ignore this when drop lob meta aux table", KR(ret), K(tmp_lob_table_schema)); ret = OB_SUCCESS; @@ -167,15 +119,13 @@ int ObDDLService::force_drop_lonely_lob_aux_table(const obcall::ObForceDropLonel } else if (FALSE_IT(tmp_lob_table_schema.reset())) { } else if (OB_FAIL(tmp_lob_table_schema.assign(*lob_piece_table_schema_ptr))) { LOG_WARN("fail to assign lob piece table schema", KR(ret)); - } else if (OB_FAIL(get_ls_from_table(tmp_lob_table_schema, trans, lob_piece_ls_ids))) { - LOG_WARN("get ls of lob piece table fail", KR(ret), K(arg), K(tmp_lob_table_schema)); } else if (OB_FAIL(ddl_operator.drop_table(tmp_lob_table_schema, trans, nullptr/*ddl_stmt_str*/, false/*is_truncate_table*/, nullptr/*drop_table_set*/, false/*is_drop_db*/, true/*delete_priv*/, true/*is_force_drop_lonely_lob_aux_table*/))) { LOG_WARN("fail to drop lob piece table", KR(ret), K(tmp_lob_table_schema)); if (OB_LS_NOT_EXIST == ret || OB_LS_IS_DELETED == ret) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(check_ls_not_exist(lob_piece_ls_ids))) { - LOG_WARN("check_ls_not_exist fail", KR(tmp_ret)); + if (OB_TMP_FAIL(check_tenant_not_active())) { + LOG_WARN("check tenant state failed", KR(tmp_ret)); } else { LOG_ERROR("ls not exist, ignore this when drop lob piece aux table", KR(ret), K(tmp_lob_table_schema)); ret = OB_SUCCESS; diff --git a/src/rootserver/ob_partition_exchange.cpp b/src/rootserver/ob_partition_exchange.cpp index 7822d7483..a7ffa2481 100644 --- a/src/rootserver/ob_partition_exchange.cpp +++ b/src/rootserver/ob_partition_exchange.cpp @@ -20,7 +20,8 @@ #include "ob_root_service.h" #include "share/ob_sql_client_decorator.h" // ObSQLClientRetryWeak #include "share/tablet/ob_tablet_to_table_history_operator.h" // ObTabletToTableHistoryOperator -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" +#include "storage/tablet/ob_tablet_binding_helper.h" #include "sql/resolver/ddl/ob_ddl_resolver.h" namespace oceanbase @@ -1794,7 +1795,7 @@ int ObPartitionExchange::update_exchange_table_non_schema_attributes_(const ObTa ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(old_partition_ids), K(new_partition_ids), K(old_table_schema), K(new_table_id), K(old_tablet_ids)); } else { - // modify inner table __all_tablet_to_ls, __all_sequence_value or __all_sequence_value, __all_table_stat, __all_column_stat, __all_histogram_stat, __all_monitor_modified + // modify tablet mapping, __all_sequence_value, __all_table_stat, __all_column_stat, __all_histogram_stat, __all_monitor_modified if (OB_FAIL(update_table_to_tablet_ids_mapping_( new_table_id, old_tablet_ids, trans))) { LOG_WARN("fail to update table to tablet id mapping", K(ret), K(new_table_id), K(old_tablet_ids)); } else if (!old_table_schema.is_aux_table()) { @@ -1898,7 +1899,7 @@ int ObPartitionExchange::update_table_to_tablet_ids_mapping_(const uint64_t tabl LOG_WARN("invalid argument", K(ret), K(table_id), K(tablet_ids.count())); } else { for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(trans, table_id, tablet_ids.at(i)))) { + if (OB_FAIL(ObTabletMappingTableOperator::update_table_to_tablet_id_mapping(trans, table_id, tablet_ids.at(i)))) { LOG_WARN("fail to update table to tablet id mapping", K(ret), K(table_id), K(tablet_ids.at(i))); } } @@ -2271,74 +2272,6 @@ int ObPartitionExchange::build_single_table_rw_defensive_(const ObIArray &tablet_ids, - const int64_t schema_version, - ObIArray &modify_args, - ObDDLSQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObArray tablets; - if (OB_FAIL(get_tablets_( tablet_ids, tablets, trans))) { - LOG_WARN("failed to get tablet ids of orig table", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablets.count(); i++) { - const ObLSID &ls_id = tablets[i].first; - int64_t j = 0; - for (; j < modify_args.count(); j++) { - if (modify_args.at(j).ls_id_ == ls_id) { - break; - } - } - if (j == modify_args.count()) { - storage::ObBatchUnbindTabletArg modify_arg; - - modify_arg.ls_id_ = ls_id; - modify_arg.schema_version_ = schema_version; - if (OB_FAIL(modify_args.push_back(modify_arg))) { - LOG_WARN("failed to push back modify arg", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (0 <= j && j < modify_args.count()) { - storage::ObBatchUnbindTabletArg &modify_arg = modify_args.at(j); - const ObTabletID &tablet_id = tablets[i].second; - if (OB_FAIL(modify_arg.hidden_tablet_ids_.push_back(tablet_id))) { - LOG_WARN("failed to push back", K(ret)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Array idx out of bounds", K(ret), K(j), K(modify_args.count())); - } - } - } - LOG_DEBUG("build modify tablet binding args", K(ret), K(modify_args)); - return ret; -} - -int ObPartitionExchange::get_tablets_(const ObIArray &tablet_ids, - ObIArray &tablets, - ObDDLSQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObArray ls_ids; - tablets.reset(); - if (OB_UNLIKELY(tablet_ids.count() < 1)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_ids)); - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("failed to batch get ls", K(ret)); - } else if (OB_UNLIKELY(tablet_ids.count() != ls_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet ids ls ids", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(tablets.push_back({ls_ids.at(i), tablet_ids.at(i)}))) { - LOG_WARN("failed to push back tablet id and ls id", K(ret)); - } - } - return ret; -} - int ObPartitionExchange::adapting_cdc_changes_in_exchange_partition_(const uint64_t partitioned_table_id, const uint64_t non_partitioned_table_id, ObDDLSQLTransaction &trans) @@ -2354,9 +2287,6 @@ int ObPartitionExchange::adapting_cdc_changes_in_exchange_partition_(const uint6 LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(partitioned_table_id), K(non_partitioned_table_id)); } else { ObChangeTabletToTableArg arg; - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); - - arg.ls_id_ = ls_id; arg.base_table_id_ = partitioned_table_id; arg.inc_table_id_ = non_partitioned_table_id; common::hash::ObHashMap::iterator iter_table; @@ -2402,7 +2332,7 @@ int ObPartitionExchange::adapting_cdc_changes_in_exchange_partition_(const uint6 LOG_WARN("failed to allocate", K(ret)); } else if (OB_FAIL(arg.serialize(buf, size, pos))) { LOG_WARN("failed to serialize arg", K(ret)); - } else if (OB_FAIL(trans.register_tx_data(arg.ls_id_, transaction::ObTxDataSourceType::CHANGE_TABLET_TO_TABLE_MDS, buf, pos))) { + } else if (OB_FAIL(trans.register_tx_data(transaction::ObTxDataSourceType::CHANGE_TABLET_TO_TABLE_MDS, buf, pos))) { LOG_WARN("failed to register tx data", K(ret)); } } @@ -2803,7 +2733,7 @@ int ObPartitionExchange::ddl_exchange_table_partitions( -OB_SERIALIZE_MEMBER(ObChangeTabletToTableArg, ls_id_, base_table_id_, inc_table_id_, table_ids_, tablet_ids_); +OB_SERIALIZE_MEMBER(ObChangeTabletToTableArg, base_table_id_, inc_table_id_, table_ids_, tablet_ids_); }//end namespace rootserver }//end namespace oceanbase diff --git a/src/rootserver/ob_partition_exchange.h b/src/rootserver/ob_partition_exchange.h index ad6bae3a0..8eb89d8e2 100644 --- a/src/rootserver/ob_partition_exchange.h +++ b/src/rootserver/ob_partition_exchange.h @@ -21,7 +21,6 @@ #include "share/ob_ddl_task_executor.h" #include "share/ob_rpc_struct.h" #include "share/schema/ob_schema_struct.h" -#include "storage/tablet/ob_tablet_binding_helper.h" #include "sql/optimizer/stat/ob_stat_define.h" namespace oceanbase @@ -31,11 +30,26 @@ namespace share class AutoincParam; namespace schema { +struct AlterTableSchema; +class ObColumnGroupSchema; +class ObColumnSchemaV2; +class ObPartition; +class ObPartitionSchema; +class ObSubPartition; class ObTableSchema; } } namespace rootserver { +using share::schema::AlterTableSchema; +using share::schema::ObSchemaGetterGuard; +using share::schema::ObColumnGroupSchema; +using share::schema::ObColumnSchemaV2; +using share::schema::ObPartition; +using share::schema::ObPartitionSchema; +using share::schema::ObSubPartition; +using share::schema::ObTableSchema; + class ObDDLService; class ObDDLSQLTransaction; class ObDDLOperator; @@ -84,7 +98,6 @@ static inline bool is_partition_exchange_between_subpart_and_part(const ObPartit class ObPartitionExchange { public: - typedef std::pair LSTabletID; // 'is_part_id_exchanged = false' only happens in partition-level direct load explicit ObPartitionExchange(ObDDLService &ddl_service, const bool is_part_id_exchanged = true); @@ -235,13 +248,6 @@ class ObPartitionExchange int build_single_table_rw_defensive_(const ObIArray &tablet_ids, const int64_t schema_version, ObDDLSQLTransaction &trans); - int build_modify_tablet_binding_args_v1_(const ObIArray &tablet_ids, - const int64_t schema_version, - ObIArray &modify_args, - ObDDLSQLTransaction &trans); - int get_tablets_(const ObIArray &tablet_ids, - ObIArray &tablets, - ObDDLSQLTransaction &trans); int adapting_cdc_changes_in_exchange_partition_(const uint64_t partitioned_table_id, const uint64_t non_partitioned_table_id, ObDDLSQLTransaction &trans); @@ -352,16 +358,15 @@ struct ObChangeTabletToTableArg final { OB_UNIS_VERSION_V(1); public: - ObChangeTabletToTableArg() : ls_id_(), base_table_id_(OB_INVALID_ID), inc_table_id_(OB_INVALID_ID), tablet_ids_(), table_ids_() {} + ObChangeTabletToTableArg() : base_table_id_(OB_INVALID_ID), inc_table_id_(OB_INVALID_ID), tablet_ids_(), table_ids_() {} ~ObChangeTabletToTableArg() {} - share::ObLSID ls_id_; uint64_t base_table_id_; // PT table, always with the large amount of data. uint64_t inc_table_id_; // NT table, with the incremental data. // tablet ids of data table, index table, lob meta/piece table. common::ObSArray tablet_ids_; // the table ids corresponding to the tablet ids. common::ObSArray table_ids_; - TO_STRING_KV(K_(ls_id), K_(base_table_id), K_(inc_table_id), K_(tablet_ids), K_(table_ids)); + TO_STRING_KV(K_(base_table_id), K_(inc_table_id), K_(tablet_ids), K_(table_ids)); }; }//end namespace rootserver diff --git a/src/rootserver/ob_root_minor_freeze.cpp b/src/rootserver/ob_root_minor_freeze.cpp index 0158dd24f..c179caddf 100644 --- a/src/rootserver/ob_root_minor_freeze.cpp +++ b/src/rootserver/ob_root_minor_freeze.cpp @@ -1,4 +1,3 @@ -#include "share/ob_ex_rpc.h" /* * Copyright (c) 2025 OceanBase. * @@ -18,25 +17,20 @@ #define USING_LOG_PREFIX RS #include "ob_root_minor_freeze.h" -#include "share/ob_server_struct.h" -#include "observer/ob_service.h" // for ObService - -#include "share/location_cache/ob_location_service.h" +#include "observer/ob_service.h" namespace oceanbase { using namespace common; using namespace obcall; using namespace share; -using namespace share::schema; namespace rootserver { + ObRootMinorFreeze::ObRootMinorFreeze() - :inited_(false), - stopped_(false) -{ -} + : inited_(false), stopped_(false) +{} ObRootMinorFreeze::~ObRootMinorFreeze() { @@ -56,7 +50,6 @@ int ObRootMinorFreeze::init() stopped_ = false; inited_ = true; } - return ret; } @@ -72,12 +65,10 @@ void ObRootMinorFreeze::stop() int ObRootMinorFreeze::destroy() { - int ret = OB_SUCCESS; inited_ = false; - return ret; + return OB_SUCCESS; } -inline int ObRootMinorFreeze::check_cancel() const { int ret = OB_SUCCESS; @@ -91,198 +82,23 @@ int ObRootMinorFreeze::check_cancel() const return ret; } -inline -bool ObRootMinorFreeze::is_server_alive(const ObAddr &server) const -{ - return true; -} - -int ObRootMinorFreeze::try_minor_freeze(const obcall::ObRootMinorFreezeArg &arg) const +int ObRootMinorFreeze::try_minor_freeze(const ObRootMinorFreezeArg &arg) const { int ret = OB_SUCCESS; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("ObRootMinorFreeze not init", K(ret)); + ObMinorFreezeArg freeze_arg; + Int64 result; + if (OB_FAIL(check_cancel())) { + LOG_WARN("minor freeze canceled", K(ret)); } else { - ParamsContainer params; - if ((arg.ls_id_.is_valid() && arg.ls_id_.id() > 0) || arg.tablet_id_.is_valid()) { - if (OB_FAIL(init_params_by_ls_or_tablet( arg.ls_id_, arg.tablet_id_, params))) { - LOG_WARN("fail to init param by tablet_id"); - } - } else if (arg.server_list_.count() == 0 && arg.zone_.size() > 0) { - if (OB_FAIL(init_params_by_zone(arg.zone_, params))) { - LOG_WARN("fail to init param by zone, ", K(ret), K(arg)); - } - } else if (arg.server_list_.count() > 0) { - if (OB_FAIL(init_params_by_server(arg.server_list_, params))) { - LOG_WARN("fail to init param by server, ", K(ret), K(arg)); - } - } else { - if (OB_FAIL(init_params_by_tenant(params))) { - LOG_WARN("fail to init param by tenant, ", K(ret), K(arg)); - } + if (arg.tablet_id_.is_valid()) { + freeze_arg.tablet_id_ = arg.tablet_id_; } - - if (OB_SUCC(ret) && !params.is_empty()) { - if (OB_FAIL(do_minor_freeze(params))) { - LOG_WARN("fail to do minor freeze, ", K(ret)); - } - } - } - - return ret; -} - -int ObRootMinorFreeze::do_minor_freeze(const ParamsContainer ¶ms) const -{ - int ret = OB_SUCCESS; - int64_t failure_cnt = 0; - LOG_INFO("do minor freeze", K(params)); - - for (int64_t i = 0; OB_SUCC(ret) && i < params.get_params().count(); ++i) { - const MinorFreezeParam ¶m = params.get_params().at(i); - if (OB_FAIL(check_cancel())) { - LOG_WARN("fail to check cancel", KR(ret)); - } else { - obcall::Int64 result; - int rpc_ret = GCTX.ob_service_->minor_freeze(param.arg, result); - if (OB_SUCCESS != rpc_ret || OB_SUCCESS != static_cast(result)) { - LOG_WARN("minor freeze failed", KR(rpc_ret), "dest addr", param.server, "param", param.arg); - failure_cnt++; - } + if (OB_FAIL(GCTX.ob_service_->minor_freeze(freeze_arg, result))) { + LOG_WARN("local minor freeze failed", K(ret), K(freeze_arg)); + } else if (OB_FAIL(static_cast(result))) { + LOG_WARN("local minor freeze returned error", K(ret), K(freeze_arg)); } } - - if (OB_FAIL(ret)) { - } else if (0 != failure_cnt) { - ret = OB_PARTIAL_FAILED; - LOG_WARN("minor freeze partial failed", KR(ret), K(failure_cnt)); - } - - return ret; -} - -int ObRootMinorFreeze::is_server_belongs_to_zone(const ObAddr &addr, - const ObZone &zone, - bool &server_in_zone) const -{ - int ret = OB_SUCCESS; - server_in_zone = true; - return ret; -} - -int ObRootMinorFreeze::init_params_by_ls_or_tablet(share::ObLSID ls_id, - const common::ObTabletID &tablet_id, - ParamsContainer ¶ms) const -{ - int ret = OB_SUCCESS; - - const int64_t expire_renew_time = INT64_MAX; - share::ObLSLocation location; - bool is_cache_hit = false; - if (OB_UNLIKELY(OB_ISNULL(GCTX.location_service_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("location service ptr is null", KR(ret)); - } else if (tablet_id.is_valid() && !ls_id.is_valid()) { - // get ls id by tablet_id - if (tablet_id.is_ls_inner_tablet()) { - ret = OB_NOT_SUPPORTED; - LOG_ERROR("can not minor freeze inner tablet without specifying ls id", K(ls_id), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->get(tablet_id, expire_renew_time, is_cache_hit, ls_id))) { - LOG_WARN("fail to get ls id according to tablet_id", K(ret), K(tablet_id)); - } - } - - if (OB_FAIL(ret)) { - } else if (ls_id.is_valid()) { - // get ls location by ls_id - if (OB_FAIL(GCTX.location_service_->get( - GCONF.cluster_id, ls_id, expire_renew_time, is_cache_hit, location))) { - LOG_WARN("get ls location failed", KR(ret), K(ls_id), K(tablet_id)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls_id or tablet_id", KR(ret), K(ls_id), K(tablet_id)); - } - - if (OB_FAIL(ret)) { - } else { - const ObIArray &ls_locations = location.get_replica_locations(); - for (int i = 0; i < ls_locations.count() && OB_SUCC(ret); ++i) { - const ObAddr &server = ls_locations.at(i).get_server(); - if (is_server_alive(server)) { - if (OB_FAIL(params.push_back_param(server, ls_id, tablet_id))) { - LOG_WARN("fail to add tenant & server, ", K(ret), K(ls_id), K(tablet_id)); - } - } else { - int tmp_ret = OB_SERVER_NOT_ACTIVE; - LOG_WARN("server not alive or invalid", "server", server, K(tmp_ret), K(ls_id), K(tablet_id)); - } - } - } - - return ret; -} - -int ObRootMinorFreeze::init_params_by_tenant(ParamsContainer ¶ms) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(params.push_back_param(GCTX.self_addr()))) { - LOG_WARN("fail to add tenant & server", KR(ret)); - } - return ret; -} - -int ObRootMinorFreeze::init_params_by_zone(const ObZone &zone, - ParamsContainer ¶ms) const -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(0 == zone.size())) { - ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(params.push_back_param(GCTX.self_addr()))) { - LOG_WARN("fail to add server", K(ret)); - } - return ret; -} - -int ObRootMinorFreeze::init_params_by_server(const ObIArray &server_list, - ParamsContainer ¶ms) const -{ - int ret = OB_SUCCESS; - if (server_list.count() > 0) { - for (int i = 0; i < server_list.count() && OB_SUCC(ret); ++i) { - if (is_server_alive(server_list.at(i))) { - if (OB_FAIL(params.push_back_param(server_list.at(i)))) { - LOG_WARN("fail to add server, ", K(ret)); - } - } else { - ret = OB_SERVER_NOT_ACTIVE; - LOG_WARN("server not alive or invalid", "server", server_list.at(i), K(ret)); - } - } - } else if (OB_FAIL(params.push_back_param(GCTX.self_addr()))) { - LOG_WARN("fail to add server", K(ret)); - } - - return ret; -} - -int ObRootMinorFreeze::ParamsContainer::push_back_param(const common::ObAddr &server, - share::ObLSID ls_id, - const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - - MinorFreezeParam param; - param.server = server; - param.arg.ls_id_ = ls_id; - param.arg.tablet_id_ = tablet_id; - - if (OB_FAIL(params_.push_back(param))) { - LOG_WARN("fail to push server, ", K(ret)); - } - return ret; } diff --git a/src/rootserver/ob_root_minor_freeze.h b/src/rootserver/ob_root_minor_freeze.h index 8ae280c53..e7263d858 100644 --- a/src/rootserver/ob_root_minor_freeze.h +++ b/src/rootserver/ob_root_minor_freeze.h @@ -18,20 +18,10 @@ #define OCEANBASE_ROOTSERVER_OB_ROOT_MINOR_FREEZE_H_ #include "share/ob_define.h" -#include "lib/net/ob_addr.h" #include "share/ob_rpc_struct.h" namespace oceanbase { -namespace common -{ - template class ObIArray; -} - -namespace obcall -{ -}//end namespace obcall - namespace rootserver { @@ -47,53 +37,7 @@ class ObRootMinorFreeze int destroy(); int try_minor_freeze(const obcall::ObRootMinorFreezeArg &arg) const; private: - typedef struct MinorFreezeParam - { - common::ObAddr server; - obcall::ObMinorFreezeArg arg; - - TO_STRING_KV(K(server), K(arg)); - } MinorFreezeParam; - - class ParamsContainer - { - public: - void reset() { params_.reset(); } - bool is_empty() const { return params_.count() <= 0; } - const common::ObIArray &get_params() const { return params_; } - - int push_back_param(const common::ObAddr &server, - share::ObLSID ls_id = share::INVALID_LS, - const common::ObTabletID &tablet_id = ObTabletID(ObTabletID::INVALID_TABLET_ID)); - - TO_STRING_KV(K_(params)); - private: - common::ObSEArray params_; - }; - - static const int64_t MAX_FREEZE_OP_RETRY_CNT = 5; - static const int64_t MINOR_FREEZE_TIMEOUT = (1000 * 30 + 1000) * 1000; // copy from major freeze - - int is_server_belongs_to_zone(const common::ObAddr &addr, - const common::ObZone &zone, - bool &server_in_zone) const; - - int init_params_by_ls_or_tablet(share::ObLSID ls_id, - const common::ObTabletID &tablet_id, - ParamsContainer ¶ms) const; - int init_params_by_tenant(ParamsContainer ¶ms) const; - - int init_params_by_zone(const common::ObZone &zone, - ParamsContainer ¶ms) const; - - int init_params_by_server(const common::ObIArray &server_list, - ParamsContainer ¶ms) const; - - int do_minor_freeze(const ParamsContainer ¶ms) const; - int check_cancel() const; - bool is_server_alive(const common::ObAddr &server) const; - bool inited_; bool stopped_; diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index 8285b02c1..234f2b4cf 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -36,7 +36,6 @@ #include "rootserver/ob_partition_exchange.h" #include "rootserver/ob_schema2ddl_sql.h" #include "rootserver/ob_index_builder.h" -#include "rootserver/ob_mlog_builder.h" #include "rootserver/ob_ddl_sql_generator.h" #include "rootserver/ddl_task/ob_ddl_task.h" #include "rootserver/ddl_task/ob_constraint_task.h" @@ -56,7 +55,6 @@ #include "parallel_ddl/ob_create_table_helper.h" // ObCreateTableHelper #include "parallel_ddl/ob_create_table_like_helper.h" // ObCreateTableLikeHelper #include "rootserver/parallel_ddl/ob_create_view_helper.h" // ObCreateViewHelper -#include "rootserver/parallel_ddl/ob_create_materialized_view_helper.h" // ObCreateMaterializedViewHelper #include "parallel_ddl/ob_set_comment_helper.h" //ObCommentHelper #include "parallel_ddl/ob_create_index_helper.h" // ObCreateIndexHelper #include "parallel_ddl/ob_update_index_status_helper.h" // ObUpdateIndexStatusHelper @@ -1158,20 +1156,11 @@ int ObRootService::parallel_create_table(const ObCreateTableArg &arg, ObCreateTa } else if (OB_FAIL(parallel_ddl_pre_check_())) { LOG_WARN("pre check failed before parallel ddl execute", KR(ret)); } else if (arg.schema_.is_view_table()) { - if (arg.schema_.is_materialized_view()) { - ObCreateMaterializedViewHelper create_mv_helper(schema_service_, arg, res, nullptr /*external trans*/,is_parallel); - if (OB_FAIL(create_mv_helper.init(ddl_service_))) { - LOG_WARN("fail to init create materialized view helper", KR(ret)); - } else if (OB_FAIL(create_mv_helper.execute())) { - LOG_WARN("fail to execute create materialized view", KR(ret)); - } - } else { - ObCreateViewHelper create_view_helper(schema_service_, arg, res, nullptr /*external trans*/,is_parallel); - if (OB_FAIL(create_view_helper.init(ddl_service_))) { - LOG_WARN("fail to init create view helper", KR(ret)); - } else if (OB_FAIL(create_view_helper.execute())) { - LOG_WARN("fail to execute create view", KR(ret)); - } + ObCreateViewHelper create_view_helper(schema_service_, arg, res, nullptr /*external trans*/,is_parallel); + if (OB_FAIL(create_view_helper.init(ddl_service_))) { + LOG_WARN("fail to init create view helper", KR(ret)); + } else if (OB_FAIL(create_view_helper.execute())) { + LOG_WARN("fail to execute create view", KR(ret)); } } else { ObCreateTableHelper create_table_helper(schema_service_, arg, res); @@ -1317,30 +1306,6 @@ int ObRootService::maintain_obj_dependency_info(const obcall::ObDependencyObjDDL return ret; } -int ObRootService::mview_complete_refresh(const obcall::ObMViewCompleteRefreshArg &arg, - obcall::ObMViewCompleteRefreshRes &res) -{ - LOG_DEBUG("receive mview complete refresh arg", K(arg)); - int ret = OB_SUCCESS; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", KR(ret), K(arg)); - } else { - ObSchemaGetterGuard schema_guard; - if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(schema_guard))) { - LOG_WARN("get schema guard in inner table failed", KR(ret)); - } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) { - LOG_WARN("check parallel ddl conflict failed", KR(ret), K(arg)); - } else if (OB_FAIL(ddl_service_.mview_complete_refresh(arg, res, schema_guard))) { - LOG_WARN("failed to mview complete refresh", KR(ret), K(arg)); - } - } - return ret; -} - int ObRootService::execute_ddl_task(const obcall::ObAlterTableArg &arg, common::ObSArray &obj_ids) { @@ -1440,12 +1405,6 @@ int ObRootService::execute_ddl_task(const obcall::ObAlterTableArg &arg, } break; } - case share::SWITCH_MLOG_NAME_TASK: { - if (OB_FAIL(ddl_service_.switch_index_name_and_status_for_mlog_table(const_cast(arg)))) { - LOG_WARN("failed to switch index name and status for mlog table", K(ret), K(arg)); - } - break; - } default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unknown ddl task type", K(ret), K(arg.ddl_task_type_)); @@ -1955,32 +1914,6 @@ int ObRootService::create_index(const ObCreateIndexArg &arg, obcall::ObAlterTabl return ret; } -int ObRootService::create_mlog(const obcall::ObCreateMLogArg &arg, obcall::ObCreateMLogRes &res) -{ - int ret = OB_SUCCESS; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", KR(ret), K(arg)); - } else { - ObSchemaGetterGuard schema_guard; - ObMLogBuilder mlog_builder(ddl_service_); - if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table( - schema_guard))) { - LOG_WARN("get schema guard in inner table failed", K(ret)); - } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) { - LOG_WARN("check parallel ddl conflict failed", K(ret)); - } else if (OB_FAIL(mlog_builder.init())) { - LOG_WARN("failed to init mlog builder", KR(ret)); - } else if (OB_FAIL(mlog_builder.create_or_replace_mlog(schema_guard, arg, res))) { - LOG_WARN("failed to create mlog", KR(ret), K(arg)); - } - } - return ret; -} - int ObRootService::parallel_create_index(const ObCreateIndexArg &arg, obcall::ObAlterTableRes &res) { LOG_TRACE("receive parallel create index arg", K(arg)); @@ -2756,21 +2689,6 @@ int ObRootService::update_index_status(const obcall::ObUpdateIndexStatusArg &arg return ret; } -int ObRootService::update_mview_status(const obcall::ObUpdateMViewStatusArg &arg) -{ - int ret = OB_SUCCESS; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", KR(ret), K(arg)); - } else if (OB_FAIL(ddl_service_.update_mview_status(arg))) { - LOG_WARN("update mview table status failed", KR(ret), K(arg)); - } - return ret; -} - int ObRootService::parallel_update_index_status(const obcall::ObUpdateIndexStatusArg &arg, obcall::ObParallelDDLRes &res) { LOG_TRACE("receive update index status arg", K(arg)); @@ -3024,72 +2942,6 @@ int ObRootService::check_parallel_ddl_conflict( return ddl_service_.check_parallel_ddl_conflict(schema_guard, arg); } -int ObRootService::increase_rs_epoch_and_get_proposal_id_( - int64_t &new_rs_epoch, - int64_t &proposal_id_to_check) -{ - int ret = OB_SUCCESS; - ObMySQLTransaction trans; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema_service is null", KR(ret), K(schema_service_)); - } else if (OB_FAIL(trans.start(&sql_proxy_))) { - LOG_WARN("trans start failed", K(ret)); - } else { - ObGlobalStatProxy proxy(trans); - ObSchemaService *schema_service = schema_service_->get_schema_service(); - int64_t schema_version = OB_INVALID_VERSION; - ObRefreshSchemaInfo schema_info; - common::ObRole role = FOLLOWER; - int64_t proposal_id_double_check = 0; - // 1. get role and proposal id from PALF to make sure local is leader - // ATTENTION: - // start_ddl_service will check ObDDLServiceLauncher::is_ddl_service_started_ - // to decide whether start ddl service with old logic - // we can ensure that RS try start ddl service after __all_core_table be readable - // because operations like unit_manager_.load() can make sure sys leader's - // switch_to_leader() successfully called. - // In other words, sys leader's switch_to_leader() must before RS start_ddl_service() - // Based on this reason, we can make sure RS can start with old logic by checking - // ObDDLServiceLauncher::is_ddl_service_started_ - // So we have to check log handle leader here - if (OB_FAIL(ObDDLUtil::get_sys_log_handler_role_and_proposal_id( - role, proposal_id_to_check))) { - LOG_WARN("fail to get sys log handler role and proposal id", KR(ret)); - } else if (OB_UNLIKELY(!is_strong_leader(role))) { - ret = OB_LS_NOT_LEADER; - LOG_WARN("local is not sys tenant leader", KR(ret), K(role), K(proposal_id_to_check)); - // 2. increase rootservice_epoch in __all_core_table and make sure it is valid - } else if (OB_FAIL(proxy.inc_rootservice_epoch())) { - LOG_WARN("fail to increase rootservice_epoch", KR(ret)); - } else if (OB_FAIL(proxy.get_rootservice_epoch(new_rs_epoch))) { - LOG_WARN("fail to get rootservice start times", KR(ret), K(new_rs_epoch)); - } else if (new_rs_epoch <= 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid rootservice_epoch", KR(ret), K(new_rs_epoch)); - // 3. double check local is still leader and proposal id not changed before commit - // it's ok to remove double check here, we just want to let it fail as soon as possible - } else if (OB_FAIL(ObDDLUtil::get_sys_log_handler_role_and_proposal_id( - role, proposal_id_double_check))) { - LOG_WARN("fail to get sys log handler role and proposal id", KR(ret)); - } else if (OB_UNLIKELY(!is_strong_leader(role)) - || OB_UNLIKELY(proposal_id_double_check != proposal_id_to_check)) { - ret = OB_LS_NOT_LEADER; - LOG_WARN("local is not sys tenant leader now", KR(ret), K(role), K(proposal_id_double_check)); - } - // 4. commit transation - int temp_ret = OB_SUCCESS; - if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCCESS == ret))) { - LOG_ERROR("trans end failed", "commit", OB_SUCCESS == ret, K(temp_ret)); - ret = (OB_SUCCESS == ret) ? temp_ret : ret; - } - } - return ret; -} - ERRSIM_POINT_DEF(ERROR_EVENT_TABLE_CLEAR_INTERVAL); int ObRootService::start_timer_tasks() { @@ -4008,7 +3860,7 @@ int ObRootService::admin_set_tracepoint(const obcall::ObAdminSetTPArg &arg) } else { ObAdminSetTP admin_util(ctx, arg); if (OB_FAIL(admin_util.execute(arg))) { - LOG_WARN("execute report replica failed", K(arg), K(ret)); + LOG_WARN("execute set tracepoint failed", K(arg), K(ret)); } } } @@ -4117,11 +3969,6 @@ int ObRootService::table_allow_ddl_operation(const obcall::ObAlterTableArg &arg) LOG_WARN("try to alter invisible table schema", K(schema->get_session_id()), K(arg)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "try to alter invisible table"); } - } else if ((schema->required_by_mview_refresh() || schema->is_mlog_table()) && - !arg.is_alter_mlog_attributes_) { - if (OB_FAIL(ObResolverUtils::check_allowed_alter_operations_for_mlog(arg, *schema))) { - LOG_WARN("failed to check allowed alter operation for mlog", KR(ret), K(arg)); - } } return ret; } @@ -4862,7 +4709,7 @@ int ObRootService::start_ddl_service_() if (OB_ISNULL(ddl_service_launcher)) { ret = OB_ERR_UNEXPECTED; FLOG_WARN("ddl service is null", KR(ret), KP(ddl_service_launcher)); - } else if (OB_FAIL(ddl_service_launcher->switch_to_leader())) { + } else if (OB_FAIL(ddl_service_launcher->activate())) { FLOG_WARN("fail to start ddl service", KR(ret)); } else { FLOG_INFO("success to start ddl service", KR(ret)); diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 3b3f2e4b7..65a97ca25 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -24,7 +24,6 @@ #include "share/object_storage/ob_object_storage_struct.h" #include "share/ob_schema_version_info.h" #include "share/ob_unit_replica_counter.h" -#include "share/ob_ls_id.h" #include "share/ob_max_id_cache.h" #include "rpc/ob_packet.h" @@ -202,7 +201,6 @@ class ObRootService int cancel_ddl_task(const obcall::ObCancelDDLTaskArg &arg); int alter_tablegroup(const obcall::ObAlterTablegroupArg &arg); int maintain_obj_dependency_info(const obcall::ObDependencyObjDDLArg &arg); - int mview_complete_refresh(const obcall::ObMViewCompleteRefreshArg &arg, obcall::ObMViewCompleteRefreshRes &res); int rename_table(const obcall::ObRenameTableArg &arg); int fork_table(const obcall::ObForkTableArg &arg, obcall::ObDDLRes &res); int fork_database(const obcall::ObForkDatabaseArg &arg, obcall::ObDDLRes &res); @@ -219,7 +217,6 @@ class ObRootService int drop_database(const obcall::ObDropDatabaseArg &arg, obcall::ObDropDatabaseRes &drop_database_res); int drop_tablegroup(const obcall::ObDropTablegroupArg &arg); int drop_index(const obcall::ObDropIndexArg &arg, obcall::ObDropIndexRes &res); - int create_mlog(const obcall::ObCreateMLogArg &arg, obcall::ObCreateMLogRes &res); int drop_lob(const obcall::ObDropLobArg &arg); int force_drop_lonely_lob_aux_table(const obcall::ObForceDropLonelyLobAuxTableArg &drop_table_arg); int rebuild_vec_index(const obcall::ObRebuildIndexArg &arg, obcall::ObAlterTableRes &res); @@ -233,7 +230,6 @@ class ObRootService int parallel_create_table_like(const obcall::ObCreateTableLikeArg &arg, obcall::ObCreateTableRes &res); int root_minor_freeze(const obcall::ObRootMinorFreezeArg &arg); int update_index_status(const obcall::ObUpdateIndexStatusArg &arg); - int update_mview_status(const obcall::ObUpdateMViewStatusArg &arg); int parallel_update_index_status(const obcall::ObUpdateIndexStatusArg &arg, obcall::ObParallelDDLRes &res); int purge_table(const obcall::ObPurgeTableArg &arg); int restore_table_from_recyclebin(const obcall::ObRecyclebinRestoreTableArg &arg); @@ -397,9 +393,6 @@ class ObRootService int check_parallel_ddl_conflict( share::schema::ObSchemaGetterGuard &schema_guard, const obcall::ObDDLArg &arg); - int increase_rs_epoch_and_get_proposal_id_( - int64_t &new_rs_epoch, - int64_t &proposal_id_to_check); // create system table in mysql backend for debugging mode. int init_debug_database(); int do_restart(); diff --git a/src/rootserver/ob_split_partition_helper.cpp b/src/rootserver/ob_split_partition_helper.cpp index d09436173..108080875 100644 --- a/src/rootserver/ob_split_partition_helper.cpp +++ b/src/rootserver/ob_split_partition_helper.cpp @@ -1,4 +1,3 @@ -#include "share/ob_ex_rpc.h" #include "observer/ob_service.h" #include "share/rc/ob_module_provider.h" /* @@ -25,9 +24,10 @@ #include "share/tablet/ob_tablet_to_table_history_operator.h" #include "observer/scheduler/ob_partition_auto_split_helper.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "storage/ddl/ob_ddl_lock.h" #include "src/rootserver/ob_root_service.h" +#include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/ob_tablet_autoinc_seq_rpc_handler.h" #include "observer/ob_service.h" @@ -62,8 +62,6 @@ int ObSplitPartitionHelper::execute(ObDDLTaskRecord &task_record) } else if (OB_FAIL(prepare_start_args_(new_table_schemas_, upd_table_schemas_, inc_table_schemas_, - ls_id_, - leader_addr_, src_tablet_ids_, dst_tablet_ids_, start_src_arg_, @@ -73,8 +71,6 @@ int ObSplitPartitionHelper::execute(ObDDLTaskRecord &task_record) trans_))) { LOG_WARN("failed to split start src", KR(ret)); } else if (OB_FAIL(prepare_dst_tablet_creator_(tenant_data_version_, - ls_id_, - leader_addr_, src_tablet_ids_, dst_tablet_ids_, inc_table_schemas_, @@ -84,33 +80,26 @@ int ObSplitPartitionHelper::execute(ObDDLTaskRecord &task_record) tablet_creator_, trans_))) { LOG_WARN("failed to prepare dst tablet creator", K(ret)); - } else if (OB_FAIL(insert_dst_tablet_to_ls_and_table_history_(ls_id_, - dst_tablet_ids_, - inc_table_schemas_, - trans_))) { - LOG_WARN("failed to insert dst tablet to ls and table history", K(ret)); + } else if (OB_FAIL(insert_dst_tablet_mapping_and_table_history_(dst_tablet_ids_, + inc_table_schemas_, + trans_))) { + LOG_WARN("failed to insert dst tablet mapping and table history", K(ret)); } else if (OB_FAIL(create_ddl_task_(task_id_, split_type_, inc_table_schemas_, parallelism_, - ls_id_, tenant_data_version_, allocator_, task_record, trans_))) { LOG_WARN("failed to create ddl task", K(ret)); - } else if (OB_FAIL(start_src_(ls_id_, - leader_addr_, - src_tablet_ids_, + } else if (OB_FAIL(start_src_(src_tablet_ids_, start_src_arg_, data_end_scn_, end_autoinc_seqs_, trans_))) { LOG_WARN("failed to split wait src end", KR(ret)); - } else if (OB_FAIL(start_dst_(tenant_data_version_, - ls_id_, - leader_addr_, - inc_table_schemas_, + } else if (OB_FAIL(start_dst_(inc_table_schemas_, dst_tablet_ids_, data_end_scn_, end_autoinc_seqs_, @@ -162,56 +151,33 @@ int ObSplitPartitionHelper::check_allow_split( return ret; } -int ObSplitPartitionHelper::freeze_split_src_tablet(const ObFreezeSplitSrcTabletArg &arg, - ObFreezeSplitSrcTabletRes &res, - const int64_t abs_timeout_us) +int ObSplitPartitionHelper::freeze_split_src_tablet_(const ObIArray &tablet_ids, + SCN &data_end_scn, + const int64_t abs_timeout_us) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!arg.is_valid())) { + if (OB_UNLIKELY(tablet_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(arg)); + LOG_WARN("empty split source tablets", K(ret)); } else { MOD_SCOPE { ObLSService *ls_service = share::g_mp->ls_service(); - logservice::ObLogService *log_service = share::g_mp->log_service(); - compaction::ObTenantTabletScheduler *tenant_tablet_scheduler = share::g_mp->tenant_tablet_scheduler(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - ObRole role = INVALID_ROLE; - int64_t proposal_id = -1; bool has_active_memtable = false; - const ObIArray &tablet_ids = arg.tablet_ids_; - if (OB_ISNULL(ls_service) || OB_ISNULL(log_service) || OB_ISNULL(tenant_tablet_scheduler)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls_service or log_service", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { - ret = OB_NOT_MASTER; - LOG_WARN("ls not leader", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->tablet_freeze(tablet_ids, true/*is_sync*/, abs_timeout_us, - false/*need_rewrite_meta*/, ObFreezeSourceFlag::TABLET_SPLIT))) { - LOG_WARN("batch tablet freeze failed", K(ret), K(arg)); - } else if (OB_FAIL(ls->check_tablet_no_active_memtable(tablet_ids, has_active_memtable))) { - // safer with this check, non-mandatory - LOG_WARN("check tablet has active memtable failed", K(ret), K(arg)); - } else if (has_active_memtable) { - ret = OB_EAGAIN; - LOG_WARN("tablet has active memtable need retry", K(ret), K(arg)); - } - - // the followings are workarounds, still INCORRECT in some leader switch corner cases - // 1. wait write end for medium - // 2. wait write end for table autoinc seq will be done in batch_get_tablet_autoinc_seq - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls->get_log_handler()->get_max_scn(res.data_end_scn_))) { - LOG_WARN("log_handler get_max_scn failed", K(ret), K(arg)); + if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else { + if (OB_FAIL(ls->tablet_freeze(tablet_ids, true/*is_sync*/, abs_timeout_us, + false/*need_rewrite_meta*/, ObFreezeSourceFlag::TABLET_SPLIT))) { + LOG_WARN("batch tablet freeze failed", K(ret), K(tablet_ids)); + } else if (OB_FAIL(ls->check_tablet_no_active_memtable(tablet_ids, has_active_memtable))) { + LOG_WARN("check tablet has active memtable failed", K(ret), K(tablet_ids)); + } else if (has_active_memtable) { + ret = OB_EAGAIN; + LOG_WARN("tablet has active memtable need retry", K(ret), K(tablet_ids)); + } else if (OB_FAIL(ls->get_log_handler()->get_max_scn(data_end_scn))) { + LOG_WARN("log_handler get_max_scn failed", K(ret)); + } } } } @@ -288,8 +254,6 @@ int ObSplitPartitionHelper::check_enable_global_index_auto_split( int ObSplitPartitionHelper::prepare_start_args_(const ObIArray &new_table_schemas, const ObIArray &upd_table_schemas, const ObIArray &inc_table_schemas, - ObLSID &ls_id, - ObAddr &leader_addr, ObIArray &src_tablet_ids, ObIArray> &dst_tablet_ids, ObTabletSplitMdsArg &start_src_arg, @@ -299,11 +263,7 @@ int ObSplitPartitionHelper::prepare_start_args_(const ObIArray ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - ObLocationService *location_service = nullptr; - ObArray ls_ids; ObArray dst_high_bound_vals; - ls_id.reset(); - leader_addr.reset(); src_tablet_ids.reset(); dst_tablet_ids.reset(); start_src_arg.reset(); @@ -332,7 +292,7 @@ int ObSplitPartitionHelper::prepare_start_args_(const ObIArray LOG_WARN("verify tablet checksum error", K(ret), K(src_tablet_ids)); } - // lock and get ls_id + // lock source and destination tablets if (OB_FAIL(ret)) { } else if (OB_FAIL(src_data_tablet_id.push_back(src_tablet_ids.at(0)))) { LOG_WARN("failed to push back src data tablet id", K(ret)); @@ -340,48 +300,20 @@ int ObSplitPartitionHelper::prepare_start_args_(const ObIArray LOG_WARN("fetch new task id failed", K(ret)); } else if (OB_FALSE_IT(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, task_id))) { - } else if (OB_FAIL(ObDDLLock::lock_for_split_partition(*upd_table_schemas.at(0), nullptr/*ls_id*/, &src_data_tablet_id, dst_tablet_ids.at(0), owner_id, trans))) { + } else if (OB_FAIL(ObDDLLock::lock_src_and_dst_for_split_partition(*upd_table_schemas.at(0), src_data_tablet_id, dst_tablet_ids.at(0), owner_id, trans))) { LOG_WARN("failed to lock for split src partition", K(ret), K(src_tablet_ids), K(task_id)); - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, src_tablet_ids, ls_ids))) { - LOG_WARN("failed to batch get ls", KR(ret)); - } else if (OB_UNLIKELY(src_tablet_ids.count() != ls_ids.count() || ls_ids.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet ids ls ids", KR(ret), K(src_tablet_ids), K(ls_ids)); - } else { - ls_id = ls_ids.at(0); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids.count(); i++) { - if (OB_UNLIKELY(ls_id != ls_ids.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("src tablet ls mismatched", KR(ret), K(ls_id), K(ls_ids)); - } - } } - // prepare mds args if (OB_FAIL(ret)) { - } else if (OB_FAIL(start_src_arg.init_split_start_src(ls_id, new_table_schemas, upd_table_schemas, src_tablet_ids, dst_tablet_ids))) { + } else if (OB_FAIL(start_src_arg.init_split_start_src(new_table_schemas, upd_table_schemas, src_tablet_ids, dst_tablet_ids))) { LOG_WARN("failed to init split start src", KR(ret)); - } else if (OB_FAIL(start_dst_arg.init_split_start_dst(ls_id, inc_table_schemas, src_tablet_ids, dst_tablet_ids, dst_high_bound_vals))) { + } else if (OB_FAIL(start_dst_arg.init_split_start_dst(inc_table_schemas, src_tablet_ids, dst_tablet_ids, dst_high_bound_vals))) { LOG_WARN("failed to init split start dst", KR(ret)); } - - // prepare leader addr, as late as possible - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", KR(ret), KP(location_service)); - } else if (OB_FAIL(location_service->get_leader(GCONF.cluster_id, - ls_id, - true, - leader_addr))) { - LOG_WARN("get leader failed", KR(ret), K(ls_id)); - } return ret; } int ObSplitPartitionHelper::prepare_dst_tablet_creator_(const uint64_t tenant_data_version, - const ObLSID &ls_id, - const ObAddr &leader_addr, const ObIArray &src_tablet_ids, const ObIArray> &dst_tablet_ids, const ObIArray &inc_table_schemas, @@ -410,16 +342,13 @@ int ObSplitPartitionHelper::prepare_dst_tablet_creator_(const uint64_t tenant_da // fetch split src tablet size and create_commit_versions if (OB_SUCC(ret)) { - // FIXME: timeout ctx - const int64_t timeout_us = THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : GCONF.rpc_timeout; int64_t data_tablet_size = OB_INVALID_SIZE; obcall::ObFetchSplitTabletInfoArg arg; obcall::ObFetchSplitTabletInfoRes res; - arg.ls_id_ = ls_id; if (OB_FAIL(arg.tablet_ids_.assign(src_tablet_ids))) { - } else if (OB_FAIL(ex_rpc::sync_call([&]{ return GCTX.ob_service_->fetch_split_tablet_info(arg, res, 30000000); }))) { - LOG_WARN("failed to freeze src tablet", KR(ret), K(leader_addr)); + } else if (OB_FAIL(GCTX.ob_service_->fetch_split_tablet_info(arg, res, 30000000))) { + LOG_WARN("failed to fetch split tablet info", KR(ret)); } else if (OB_FAIL(create_commit_versions.assign(res.create_commit_versions_))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FALSE_IT(data_tablet_size = res.tablet_sizes_.at(0))) { @@ -478,7 +407,6 @@ int ObSplitPartitionHelper::prepare_dst_tablet_creator_(const uint64_t tenant_da } if (OB_FAIL(ret)) { } else if (OB_FAIL(arg.init(tablet_ids, - ls_id, data_tablet_id, inc_table_schemas, compat_mode, @@ -497,13 +425,13 @@ int ObSplitPartitionHelper::prepare_dst_tablet_creator_(const uint64_t tenant_da return ret; } -int ObSplitPartitionHelper::insert_dst_tablet_to_ls_and_table_history_(const ObLSID &ls_id, +int ObSplitPartitionHelper::insert_dst_tablet_mapping_and_table_history_( const ObIArray> &dst_tablet_ids, const ObIArray &inc_table_schemas, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - ObArray tablet_infos; + ObArray tablet_infos; if (OB_UNLIKELY(dst_tablet_ids.empty() || dst_tablet_ids.count() != inc_table_schemas.count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(dst_tablet_ids), K(inc_table_schemas.count())); @@ -519,7 +447,7 @@ int ObSplitPartitionHelper::insert_dst_tablet_to_ls_and_table_history_(const ObL ObArray tablet_table_pairs; for (int64_t i = 0; OB_SUCC(ret) && i < dst_tablet_ids.at(table_idx).count(); i++) { const ObTabletID &tablet_id = dst_tablet_ids.at(table_idx).at(i); - const ObTabletToLSInfo tablet_info(tablet_id, ls_id, table_id); + const ObTabletTablePair tablet_info(tablet_id, table_id); if (OB_FAIL(tablet_table_pairs.push_back(ObTabletTablePair(tablet_id, table_id)))) { LOG_WARN("failed to push back", K(ret)); } else if (OB_FAIL(tablet_infos.push_back(tablet_info))) { @@ -534,19 +462,19 @@ int ObSplitPartitionHelper::insert_dst_tablet_to_ls_and_table_history_(const ObL } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_update(trans, tablet_infos))) { + } else if (OB_FAIL(ObTabletMappingTableOperator::batch_update(trans, tablet_infos))) { LOG_WARN("fail to batch update tablet info", K(ret), K(tablet_infos)); } return ret; } -int ObSplitPartitionHelper::delete_src_tablet_to_ls_and_table_history_(const ObIArray &src_tablet_ids, +int ObSplitPartitionHelper::delete_src_tablet_mapping_and_table_history_(const ObIArray &src_tablet_ids, const int64_t new_schema_version, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - if (OB_FAIL(share::ObTabletToLSTableOperator::batch_remove(trans, src_tablet_ids))) { - LOG_WARN("failed to batch remove tablet to ls table", KR(ret)); + if (OB_FAIL(share::ObTabletMappingTableOperator::batch_remove(trans, src_tablet_ids))) { + LOG_WARN("failed to batch remove tablet mapping table", KR(ret)); } else if (OB_FAIL(share::ObTabletToTableHistoryOperator::drop_tablet_to_table_history( trans, new_schema_version, src_tablet_ids))) { LOG_WARN("failed to drop tablet to table history", KR(ret), K(new_schema_version)); @@ -558,7 +486,6 @@ int ObSplitPartitionHelper::create_ddl_task_(const int64_t task_id, const ObDDLType split_type, const ObIArray &inc_table_schemas, const int64_t parallelism, - const share::ObLSID &ls_id, const uint64_t tenant_data_version, ObIAllocator &allocator, ObDDLTaskRecord &task_record, @@ -586,7 +513,6 @@ int ObSplitPartitionHelper::create_ddl_task_(const int64_t task_id, LOG_WARN("global index should not have aux tables", K(ret), KPC(split_table)); } else { split_arg.task_type_ = split_type; - split_arg.src_ls_id_ = ls_id; ObPartition** table_parts = split_table->get_part_array(); // for main table or global_index. for (int64_t i = 0; OB_SUCC(ret) && i < split_part_num; i++) { @@ -692,9 +618,7 @@ int ObSplitPartitionHelper::create_ddl_task_(const int64_t task_id, return ret; } -int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, - const ObAddr &leader_addr, - const ObIArray &src_tablet_ids, +int ObSplitPartitionHelper::start_src_(const ObIArray &src_tablet_ids, const ObTabletSplitMdsArg &start_src_arg, share::SCN &data_end_scn, ObIArray> &end_autoinc_seqs, @@ -705,10 +629,9 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, int64_t start_time = finish_time; data_end_scn.reset(); end_autoinc_seqs.reset(); - if (OB_UNLIKELY(!ls_id.is_valid() || !leader_addr.is_valid() - || src_tablet_ids.empty() || !start_src_arg.is_valid())) { + if (OB_UNLIKELY(src_tablet_ids.empty() || !start_src_arg.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ls id", KR(ret), K(ls_id), K(leader_addr), K(src_tablet_ids), K(start_src_arg)); + LOG_WARN("invalid split source", KR(ret), K(src_tablet_ids), K(start_src_arg)); } // start src @@ -723,17 +646,9 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, // wait src freeze end if (OB_SUCC(ret)) { - // FIXME: timeout ctx - const int64_t timeout_us = THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : GCONF.rpc_timeout; - obcall::ObFreezeSplitSrcTabletArg arg; - obcall::ObFreezeSplitSrcTabletRes res; - - arg.ls_id_ = ls_id; - if (OB_FAIL(arg.tablet_ids_.assign(src_tablet_ids))) { - } else if (OB_FAIL(ex_rpc::sync_call([&]{ return rootserver::ObSplitPartitionHelper::freeze_split_src_tablet(arg, res, 30000000); }))) { - LOG_WARN("failed to freeze src tablet", KR(ret), K(leader_addr)); - } else { - // data_end_scn = res.data_end_scn_; + SCN unused_data_end_scn; + if (OB_FAIL(freeze_split_src_tablet_(src_tablet_ids, unused_data_end_scn, 30000000))) { + LOG_WARN("failed to freeze src tablet", KR(ret)); } finish_time = ObTimeUtility::current_time(); LOG_INFO("finish freeze_split_src_tablet", KR(ret), "cost_ts", finish_time - start_time, K(data_end_scn)); @@ -744,7 +659,7 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, if (OB_SUCC(ret)) { const bool need_flush_redo = true; // flush redo so that the following data_end_scn will be >= this redo scn ObTabletSplitMdsArg set_src_freeze_flag_arg; - if (OB_FAIL(set_src_freeze_flag_arg.init_set_freeze_flag(ls_id, src_tablet_ids))) { + if (OB_FAIL(set_src_freeze_flag_arg.init_set_freeze_flag(src_tablet_ids))) { LOG_WARN("failed to init split start src", KR(ret)); } else if (OB_FAIL(ObTabletSplitMdsHelper::register_mds(set_src_freeze_flag_arg, need_flush_redo, trans))) { LOG_WARN("failed to register mds", KR(ret)); @@ -759,7 +674,6 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, obcall::ObBatchGetTabletAutoincSeqArg arg; obcall::ObBatchGetTabletAutoincSeqRes result; - arg.ls_id_ = ls_id; if (OB_FAIL(arg.src_tablet_ids_.assign(src_tablet_ids))) { LOG_WARN("failed to assign", KR(ret)); } else if (OB_FAIL(arg.dest_tablet_ids_.assign(src_tablet_ids))) { @@ -789,17 +703,8 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, // get data end scn finally to guarantee both mds_checkpoint_scn and clog_checkpoint_scn are less than data_end_scn if (OB_SUCC(ret)) { - const int64_t timeout_us = THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : GCONF.rpc_timeout; - obcall::ObFreezeSplitSrcTabletArg arg; - obcall::ObFreezeSplitSrcTabletRes res; - - arg.ls_id_ = ls_id; - if (OB_FAIL(arg.tablet_ids_.assign(src_tablet_ids))) { - LOG_WARN("failed to assign", KR(ret)); - } else if (OB_FAIL(ex_rpc::sync_call([&]{ return rootserver::ObSplitPartitionHelper::freeze_split_src_tablet(arg, res, 30000000); }))) { - LOG_WARN("failed to freeze src tablet", KR(ret), K(leader_addr)); - } else { - data_end_scn = res.data_end_scn_; + if (OB_FAIL(freeze_split_src_tablet_(src_tablet_ids, data_end_scn, 30000000))) { + LOG_WARN("failed to freeze src tablet", KR(ret)); } finish_time = ObTimeUtility::current_time(); LOG_INFO("finish get data end scn", KR(ret), "cost_ts", finish_time - start_time, K(data_end_scn)); @@ -808,10 +713,7 @@ int ObSplitPartitionHelper::start_src_(const ObLSID &ls_id, return ret; } -int ObSplitPartitionHelper::start_dst_(const uint64_t tenant_data_version, - const share::ObLSID &ls_id, - const ObAddr &leader_addr, - const ObIArray &inc_table_schemas, +int ObSplitPartitionHelper::start_dst_(const ObIArray &inc_table_schemas, const ObIArray> &dst_tablet_ids, const share::SCN &data_end_scn, const ObIArray> &end_autoinc_seqs, @@ -823,10 +725,10 @@ int ObSplitPartitionHelper::start_dst_(const uint64_t tenant_data_version, int ret = OB_SUCCESS; int64_t finish_time = ObTimeUtility::current_time(); int64_t start_time = finish_time; - if (OB_UNLIKELY(!ls_id.is_valid() || !leader_addr.is_valid() - || !start_dst_arg.is_valid() || !data_end_scn.is_valid() || inc_table_schemas.empty() || OB_ISNULL(tablet_creator))) { + if (OB_UNLIKELY(!start_dst_arg.is_valid() || !data_end_scn.is_valid() + || inc_table_schemas.empty() || OB_ISNULL(tablet_creator))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(leader_addr), K(start_dst_arg), K(data_end_scn), K(inc_table_schemas.count()), KP(tablet_creator)); + LOG_WARN("invalid argument", KR(ret), K(start_dst_arg), K(data_end_scn), K(inc_table_schemas.count()), KP(tablet_creator)); } // create dst tablets @@ -847,7 +749,7 @@ int ObSplitPartitionHelper::start_dst_(const uint64_t tenant_data_version, } else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, task_id))) { LOG_WARN("get owner id failed", K(ret), K(task_id)); - } else if (OB_FAIL(ObDDLLock::lock_for_split_partition(*inc_table_schemas.at(0), &ls_id, nullptr/*src_tablet_ids*/, dst_tablet_ids.at(0), owner_id, trans))) { + } else if (OB_FAIL(ObDDLLock::lock_dst_for_split_partition(*inc_table_schemas.at(0), dst_tablet_ids.at(0), owner_id, trans))) { LOG_WARN("failed to lock for split src partition", K(ret), K(dst_tablet_ids), K(task_id)); } @@ -860,7 +762,6 @@ int ObSplitPartitionHelper::start_dst_(const uint64_t tenant_data_version, } else if (!end_autoinc_seqs.empty()) { obcall::ObBatchSetTabletAutoincSeqArg arg; - arg.ls_id_ = ls_id; arg.is_tablet_creating_ = true; share::ObMigrateTabletAutoincSeqParam param; for (int64_t i = 0; OB_SUCC(ret) && i < end_autoinc_seqs.count(); i++) { @@ -928,41 +829,22 @@ int ObSplitPartitionHelper::clean_split_src_and_dst_tablet( ObTabletSplitMdsArg end_src_arg; ObTabletSplitMdsArg end_dst_arg; ObArray src_tablet_ids; - ObArray ls_ids; - ObLSID ls_id; if (OB_FAIL(src_tablet_ids.push_back(arg.src_table_tablet_id_))) { LOG_WARN("failed to push back src tablet id", KR(ret)); } else if (OB_FAIL(append(src_tablet_ids, arg.src_local_index_tablet_ids_))) { LOG_WARN("failed to append", KR(ret)); } else if (OB_FAIL(append(src_tablet_ids, arg.src_lob_tablet_ids_))) { LOG_WARN("failed to append", KR(ret)); - } else if (OB_FAIL(delete_src_tablet_to_ls_and_table_history_(src_tablet_ids, new_schema_version, trans))) { - LOG_WARN("failed to delete src tablet to ls and table history", K(ret)); + } else if (OB_FAIL(delete_src_tablet_mapping_and_table_history_(src_tablet_ids, new_schema_version, trans))) { + LOG_WARN("failed to delete src tablet mapping and table history", K(ret)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, arg.dest_tablet_ids_, ls_ids))) { - LOG_WARN("failed to batch get ls", KR(ret)); - } else if (OB_UNLIKELY(arg.dest_tablet_ids_.count() != ls_ids.count() || ls_ids.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet ids ls ids", KR(ret), K(ls_ids), K(arg.dest_tablet_ids_)); - } else { - ls_id = ls_ids.at(0); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids.count(); i++) { - if (OB_UNLIKELY(ls_id != ls_ids.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("src tablet ls mismatched", KR(ret), K(ls_id), K(ls_ids)); - } - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(end_src_arg.init_split_end_src(ls_id, - arg.src_table_tablet_id_, + } else if (OB_FAIL(end_src_arg.init_split_end_src(arg.src_table_tablet_id_, arg.src_local_index_tablet_ids_, arg.src_lob_tablet_ids_))) { LOG_WARN("failed to init split end src", K(ret)); - } else if (OB_FAIL(end_dst_arg.init_split_end_dst(ls_id, - auto_part_size, + } else if (OB_FAIL(end_dst_arg.init_split_end_dst(auto_part_size, arg.dest_tablet_ids_, arg.dest_local_index_tablet_ids_, arg.dest_lob_tablet_ids_))) { diff --git a/src/rootserver/ob_split_partition_helper.h b/src/rootserver/ob_split_partition_helper.h index 3f4f7021a..ae39edf97 100644 --- a/src/rootserver/ob_split_partition_helper.h +++ b/src/rootserver/ob_split_partition_helper.h @@ -22,12 +22,6 @@ namespace oceanbase { -namespace obcall -{ -struct ObFreezeSplitSrcTabletArg; -struct ObFreezeSplitSrcTabletRes; -} - namespace rootserver { @@ -46,7 +40,7 @@ class ObSplitPartitionHelper const int64_t parallelism) : trans_(trans), schema_guard_(schema_guard), allocator_(allocator), tenant_data_version_(tenant_data_version), split_type_(split_type), new_table_schemas_(new_table_schemas), upd_table_schemas_(upd_table_schemas), inc_table_schemas_(inc_table_schemas), parallelism_(parallelism), - ls_id_(), leader_addr_(), src_tablet_ids_(), dst_tablet_ids_(), start_src_arg_(), start_dst_arg_(), task_id_(0), + src_tablet_ids_(), dst_tablet_ids_(), start_src_arg_(), start_dst_arg_(), task_id_(0), tablet_creator_(), data_end_scn_(), end_autoinc_seqs_() {} ~ObSplitPartitionHelper(); @@ -59,19 +53,16 @@ class ObSplitPartitionHelper static int check_allow_split( share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema &table_schema); - static int freeze_split_src_tablet( - const obcall::ObFreezeSplitSrcTabletArg &arg, - obcall::ObFreezeSplitSrcTabletRes &res, - const int64_t abs_timeout_us); static int get_split_src_tablet_id_if_any(const share::schema::ObTableSchema &table_schema, ObTabletID &tablet_id); static int check_enable_global_index_auto_split(const share::schema::ObTableSchema &data_table_schema, bool &enable_auto_split, int64_t &auto_part_size); private: + static int freeze_split_src_tablet_(const ObIArray &tablet_ids, + share::SCN &data_end_scn, + const int64_t abs_timeout_us); static int prepare_start_args_(const common::ObIArray &new_table_schemas, const common::ObIArray &upd_table_schemas, const common::ObIArray &inc_table_schemas, - share::ObLSID &ls_id, - ObAddr &leader_addr, ObIArray &src_tablet_ids, ObIArray> &dst_tablet_ids, storage::ObTabletSplitMdsArg &start_src_arg, @@ -80,8 +71,6 @@ class ObSplitPartitionHelper ObIAllocator &allocator, ObMySQLTransaction &trans); static int prepare_dst_tablet_creator_(const uint64_t tenant_data_version, - const share::ObLSID &ls_id, - const ObAddr &leader_addr, const ObIArray &src_tablet_ids, const ObIArray> &dst_tablet_ids, const ObIArray &inc_table_schemas, @@ -90,33 +79,27 @@ class ObSplitPartitionHelper ObIAllocator &allocator, ObTabletCreator *&tablet_creator, ObMySQLTransaction &trans); - static int insert_dst_tablet_to_ls_and_table_history_(const share::ObLSID &ls_id, + static int insert_dst_tablet_mapping_and_table_history_( const ObIArray> &dst_tablet_ids, const ObIArray &inc_table_schemas, ObMySQLTransaction &trans); - static int delete_src_tablet_to_ls_and_table_history_(const ObIArray &dst_tablet_ids, + static int delete_src_tablet_mapping_and_table_history_(const ObIArray &dst_tablet_ids, const int64_t new_schema_version, ObMySQLTransaction &trans); static int create_ddl_task_(const int64_t task_id, const share::ObDDLType split_type, const ObIArray &inc_table_schemas, const int64_t parallelism, - const share::ObLSID &ls_id, const uint64_t tenant_data_version, ObIAllocator &allocator, ObDDLTaskRecord &task_record, ObMySQLTransaction &trans); - static int start_src_(const share::ObLSID &ls_id, - const ObAddr &leader_addr, - const ObIArray &src_tablet_ids, + static int start_src_(const ObIArray &src_tablet_ids, const storage::ObTabletSplitMdsArg &start_src_arg, share::SCN &data_end_scn, ObIArray> &end_autoinc_seqs, ObMySQLTransaction &trans); - static int start_dst_(const uint64_t tenant_data_version, - const share::ObLSID &ls_id, - const ObAddr &leader_addr, - const ObIArray &inc_table_schemas, + static int start_dst_(const ObIArray &inc_table_schemas, const ObIArray> &dst_tablet_ids, const share::SCN &data_end_scn, const ObIArray> &end_autoinc_seqs, @@ -141,8 +124,6 @@ class ObSplitPartitionHelper int64_t parallelism_; // prepared before start_src - ObLSID ls_id_; - ObAddr leader_addr_; ObArray src_tablet_ids_; ObArray> dst_tablet_ids_; ObTabletSplitMdsArg start_src_arg_; diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 4d3e1fe70..29fa4021a 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -845,7 +845,7 @@ int ObAdminSetTP::execute(const obcall::ObAdminSetTPArg &arg) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), KR(ret)); } else if (OB_FAIL(call_all(arg))) { - LOG_WARN("execute report replica failed", KR(ret), K(arg)); + LOG_WARN("execute set tracepoint failed", KR(ret), K(arg)); } LOG_INFO("end execute set_tp request", K(arg)); return ret; @@ -861,7 +861,7 @@ int ObAdminSetTP::call_server(const ObAddr &server) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid server", K(server), KR(ret)); } else if (OB_FAIL(GCTX.ob_service_->set_tracepoint(arg_))) { - LOG_WARN("request server report replica failed", KR(ret), K(server)); + LOG_WARN("request server set tracepoint failed", KR(ret), K(server)); } return ret; } diff --git a/src/rootserver/ob_system_admin_util.h b/src/rootserver/ob_system_admin_util.h index 9a4c876e4..1036d1983 100644 --- a/src/rootserver/ob_system_admin_util.h +++ b/src/rootserver/ob_system_admin_util.h @@ -38,8 +38,6 @@ class ObConfigManager; namespace obcall { -struct ObAdminChangeReplicaArg; -struct ObAdminMigrateReplicaArg; struct ObServerZoneArg; struct ObAdminMergeArg; struct ObAdminClearRoottableArg; diff --git a/src/rootserver/ob_table_creator.cpp b/src/rootserver/ob_table_creator.cpp index 0276f54dc..d086a1d53 100644 --- a/src/rootserver/ob_table_creator.cpp +++ b/src/rootserver/ob_table_creator.cpp @@ -70,7 +70,7 @@ int ObTableCreator::execute() if (OB_FAIL(tablet_creator_.execute())) { LOG_WARN("fail to execute tablet creator", KR(ret)); } else if (tablet_infos_.count() > 0 - && OB_FAIL(share::ObTabletToLSTableOperator::batch_update(trans_, tablet_infos_))) { + && OB_FAIL(share::ObTabletMappingTableOperator::batch_update(trans_, tablet_infos_))) { LOG_ERROR("fail to batch update tablet info", KR(ret)); } return ret; @@ -79,7 +79,6 @@ int ObTableCreator::execute() int ObTableCreator::add_create_tablets_of_local_aux_tables_arg( const common::ObIArray &schemas, const share::schema::ObTableSchema *data_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors) { @@ -89,8 +88,7 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg( LOG_WARN("data_table_schema must be null, when table_schema is not local index", KR(ret)); } else if (!data_table_schema->has_tablet() || data_table_schema->is_index_table() || - data_table_schema->is_aux_lob_table() || - data_table_schema->is_mlog_table()) { + data_table_schema->is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("data_table_schema must be data table", KR(ret), KPC(data_table_schema)); } else if (OB_UNLIKELY(tenant_data_version <= 0 || need_create_empty_majors.count() != schemas.count())) { @@ -104,15 +102,14 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg( ret = OB_INVALID_ARGUMENT; LOG_WARN("ptr is null", KR(ret), K(schemas)); } else if (!aux_schema->is_index_local_storage() - && !aux_schema->is_aux_lob_table() - && !aux_schema->is_mlog_table()) { + && !aux_schema->is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("aux_schema must be local aux table", KR(ret), K(schemas), KPC(aux_schema)); } } if (OB_FAIL(ret)) { } else if (OB_FAIL(add_create_tablets_of_tables_arg_( - schemas, data_table_schema, ls_id_array, tenant_data_version, need_create_empty_majors))) { + schemas, data_table_schema, tenant_data_version, need_create_empty_majors))) { LOG_WARN("fail to add_create_tablets_of_tables_arg_", KR(ret), K(schemas)); } return ret; @@ -121,7 +118,6 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg( int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg( const share::schema::ObTableSchema &orig_table_schema, const share::schema::ObTableSchema &hidden_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version) { int ret = OB_SUCCESS; @@ -130,15 +126,13 @@ int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg( if (OB_UNLIKELY(!orig_table_schema.has_tablet() || orig_table_schema.is_index_table() || hidden_table_schema.is_index_table() - || orig_table_schema.is_mlog_table() - || hidden_table_schema.is_mlog_table() || !hidden_table_schema.is_user_hidden_table())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("both orig and hidden table must be data table", K(ret), K(orig_table_schema), K(hidden_table_schema)); } else if (OB_FAIL(schemas.push_back(&hidden_table_schema)) || OB_FAIL(need_create_empty_majors.push_back(false))) { LOG_WARN("failed to push back hidden table schema", K(ret)); } else if (OB_FAIL(add_create_tablets_of_tables_arg_( - schemas, &orig_table_schema, ls_id_array, tenant_data_version, need_create_empty_majors))) { + schemas, &orig_table_schema, tenant_data_version, need_create_empty_majors))) { LOG_WARN("failed to add arg", K(ret), K(schemas)); } return ret; @@ -146,7 +140,6 @@ int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg( int ObTableCreator::add_create_tablets_of_table_arg( const share::schema::ObTableSchema &table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const bool need_create_empty_major_sstable, share::schema::ObSchemaGetterGuard *schema_guard) @@ -155,13 +148,13 @@ int ObTableCreator::add_create_tablets_of_table_arg( ObSEArray schemas; ObSEArray need_create_empty_majors; if (!table_schema.has_tablet() || table_schema.is_index_local_storage() || table_schema.is_aux_lob_table() - || table_schema.is_mlog_table() || tenant_data_version <= 0) { + || tenant_data_version <= 0) { LOG_WARN("table_schema must be data table or global indexes", KR(ret), K(table_schema), K(tenant_data_version)); } else if (OB_FAIL(schemas.push_back(&table_schema)) || OB_FAIL(need_create_empty_majors.push_back(need_create_empty_major_sstable))) { LOG_WARN("failed to push_back", KR(ret), K(table_schema), K(need_create_empty_major_sstable)); } else if (OB_FAIL(add_create_tablets_of_tables_arg_( - schemas, NULL, ls_id_array, tenant_data_version, need_create_empty_majors, schema_guard))) { + schemas, NULL, tenant_data_version, need_create_empty_majors, schema_guard))) { LOG_WARN("failed to add create tablet arg", KR(ret), K(table_schema)); } return ret; @@ -169,7 +162,6 @@ int ObTableCreator::add_create_tablets_of_table_arg( int ObTableCreator::add_create_tablets_of_tables_arg( const common::ObIArray &schemas, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors, share::schema::ObSchemaGetterGuard *schema_guard) @@ -189,15 +181,13 @@ int ObTableCreator::add_create_tablets_of_tables_arg( } else if (0 == i) { if (!table_schema->has_tablet() || table_schema->is_index_table() - || table_schema->is_aux_lob_table() - || table_schema->is_mlog_table()) { + || table_schema->is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("data_table_schema must be data table", KR(ret), KPC(table_schema)); } } else { if (!table_schema->is_index_local_storage() - && !table_schema->is_aux_lob_table() - && !table_schema->is_mlog_table()) { + && !table_schema->is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("table_schema must be local index", KR(ret), K(schemas), KPC(table_schema)); } @@ -205,7 +195,7 @@ int ObTableCreator::add_create_tablets_of_tables_arg( } if (OB_FAIL(ret)) { } else if (OB_FAIL(add_create_tablets_of_tables_arg_( - schemas, NULL, ls_id_array, tenant_data_version, need_create_empty_majors, schema_guard))) { + schemas, NULL, tenant_data_version, need_create_empty_majors, schema_guard))) { LOG_WARN("fail to add_create_tablets_of_tables_arg_", KR(ret), K(schemas)); } return ret; @@ -217,7 +207,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg( int ObTableCreator::add_create_tablets_of_tables_arg_( const common::ObIArray &schemas, const share::schema::ObTableSchema *data_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors, share::schema::ObSchemaGetterGuard *schema_guard) @@ -238,8 +227,7 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( common::ObArray pairs; bool is_create_bind_hidden_tablets = false; if (table_schema.is_index_local_storage() - || table_schema.is_aux_lob_table() - || table_schema.is_mlog_table()) { + || table_schema.is_aux_lob_table()) { if (OB_ISNULL(data_table_schema)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("data_table_schema is NULL when create local_index", KR(ret)); @@ -268,10 +256,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( if (FAILEDx(pairs.reserve(all_part_num * schema_cnt))) { LOG_WARN("fail to reserve array", KR(ret), K(all_part_num), K(schema_cnt)); - } else if (OB_FAIL(ls_id_array_.reserve(all_part_num))) { - LOG_WARN("fail to reserve array", KR(ret), K(all_part_num)); - } else if (OB_FAIL(ls_id_array_.assign(ls_id_array))) { - LOG_WARN("fail to assign ls id array", KR(ret)); } else { for (int64_t i = 1; OB_SUCC(ret) && i < schema_cnt; ++i) { if (OB_ISNULL(schemas.at(i))) { @@ -288,7 +272,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( } if (OB_SUCC(ret)) { - int64_t ls_idx = 0; ObPartitionLevel part_level = table_schema.get_part_level(); lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL; if (part_level >= PARTITION_LEVEL_MAX) { @@ -299,7 +282,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( schemas, *data_table_schema, compat_mode, - ObLSID(SYS_LS), pairs, OB_INVALID_INDEX, OB_INVALID_INDEX, @@ -326,7 +308,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( schemas, *data_table_schema, compat_mode, - ObLSID(SYS_LS), pairs, i, OB_INVALID_INDEX, @@ -353,7 +334,6 @@ int ObTableCreator::add_create_tablets_of_tables_arg_( schemas, *data_table_schema, compat_mode, - ObLSID(SYS_LS), pairs, i, j, @@ -395,7 +375,6 @@ int ObTableCreator::generate_create_tablet_arg_( const common::ObIArray &schemas, const ObTableSchema &data_table_schema, const lib::Worker::CompatMode &mode, - const share::ObLSID &ls_id, common::ObIArray &pairs, const int64_t part_idx, const int64_t subpart_idx, @@ -444,7 +423,7 @@ int ObTableCreator::generate_create_tablet_arg_( tablet_id = part->get_tablet_id(); } share::ObTabletTablePair pair; - share::ObTabletToLSInfo tablet_info(tablet_id, ls_id, table_schema_ptr->get_table_id()); + share::ObTabletTablePair tablet_info(tablet_id, table_schema_ptr->get_table_id()); if (OB_FAIL(ret)) { } else if (OB_FAIL(tablet_id_array.push_back(tablet_id))) { LOG_WARN("failed to assign table schema point", KR(ret)); @@ -471,13 +450,12 @@ int ObTableCreator::generate_create_tablet_arg_( } } if (OB_SUCC(ret) && fork_tablet_infos.count() > 0) { - LOG_INFO("generate fork tablet infos finished", "fork_tablet_cnt", fork_tablet_infos.count(), K(ls_id)); + LOG_INFO("generate fork tablet infos finished", "fork_tablet_cnt", fork_tablet_infos.count()); } if (OB_SUCC(ret)) { if (OB_FAIL(create_tablet_arg.init( tablet_id_array, - ls_id, data_tablet_id, schemas, mode, diff --git a/src/rootserver/ob_table_creator.h b/src/rootserver/ob_table_creator.h index 749209136..9d5e74c2e 100644 --- a/src/rootserver/ob_table_creator.h +++ b/src/rootserver/ob_table_creator.h @@ -21,7 +21,7 @@ #include "rootserver/fork_table/ob_fork_table_info_builder.h" #include "share/tablet/ob_tablet_info.h" // ObTabletTablePair -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/ob_ddl_common.h" // ObForkTabletInfo namespace oceanbase @@ -40,7 +40,6 @@ class ObTableCreator ObMySQLTransaction &trans) : tablet_creator_(frozen_scn, trans), trans_(trans), - ls_id_array_(), inited_(false), fork_table_info_builder_{} {} @@ -59,7 +58,6 @@ class ObTableCreator // @param [in] schemas, tables schema for creating tablets, the first is data table, others are its local indexes int add_create_tablets_of_tables_arg( const common::ObIArray &schemas, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors, share::schema::ObSchemaGetterGuard *schema_guard = nullptr); @@ -71,7 +69,6 @@ class ObTableCreator int add_create_tablets_of_local_aux_tables_arg( const common::ObIArray &schemas, const share::schema::ObTableSchema *data_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors); @@ -79,7 +76,6 @@ class ObTableCreator int add_create_bind_tablets_of_hidden_table_arg( const share::schema::ObTableSchema &orig_table_schema, const share::schema::ObTableSchema &hidden_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version); // create tablets in a table @@ -87,7 +83,6 @@ class ObTableCreator // @param [in] table_schema, table schema for creating tablets int add_create_tablets_of_table_arg( const share::schema::ObTableSchema &table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const bool need_create_empty_major_sstable, share::schema::ObSchemaGetterGuard *schema_guard = nullptr); @@ -95,7 +90,6 @@ class ObTableCreator int add_create_tablets_of_tables_arg_( const common::ObIArray &schemas, const share::schema::ObTableSchema *data_table_schema, - const common::ObIArray &ls_id_array, const uint64_t tenant_data_version, const common::ObIArray &need_create_empty_majors, share::schema::ObSchemaGetterGuard *schema_guard = nullptr); @@ -103,7 +97,6 @@ class ObTableCreator const common::ObIArray &schemas, const ObTableSchema &data_table_schema, const lib::Worker::CompatMode &mode, - const share::ObLSID &ls_id, common::ObIArray &pairs, const int64_t part_idx, const int64_t subpart_idx, @@ -118,8 +111,7 @@ class ObTableCreator private: ObTabletCreator tablet_creator_; ObMySQLTransaction &trans_; - common::ObArray ls_id_array_; - common::ObArray tablet_infos_; + common::ObArray tablet_infos_; bool inited_; ObForkTableInfoBuilder fork_table_info_builder_; }; diff --git a/src/rootserver/ob_tablet_creator.cpp b/src/rootserver/ob_tablet_creator.cpp index 502d93ceb..a98f706bd 100644 --- a/src/rootserver/ob_tablet_creator.cpp +++ b/src/rootserver/ob_tablet_creator.cpp @@ -31,8 +31,7 @@ namespace rootserver bool ObTabletCreatorArg::is_valid() const { - bool is_valid = ls_key_.is_valid() - && table_schemas_.count() > 0 + bool is_valid = table_schemas_.count() > 0 && table_schemas_.count() == tablet_ids_.count() && lib::Worker::CompatMode::INVALID != compat_mode_ && tenant_data_version_ > 0 @@ -48,7 +47,6 @@ bool ObTabletCreatorArg::is_valid() const int ObTabletCreatorArg::init( const ObIArray &tablet_ids, - const share::ObLSID &ls_key, const ObTabletID data_tablet_id, const ObIArray &table_schemas, const lib::Worker::CompatMode &mode, @@ -59,7 +57,7 @@ int ObTabletCreatorArg::init( const ObIArray &fork_tablet_infos) { int ret = OB_SUCCESS; - bool is_valid = ls_key.is_valid() && table_schemas.count() > 0 + bool is_valid = table_schemas.count() > 0 && table_schemas.count() == tablet_ids.count() && tenant_data_version > 0 && need_create_empty_majors.count() == table_schemas.count() @@ -70,7 +68,7 @@ int ObTabletCreatorArg::init( if (OB_UNLIKELY(!is_valid)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tablet_ids), - "count", table_schemas.count(), K(tablet_ids), K(ls_key), + "count", table_schemas.count(), K(tablet_ids), K(tenant_data_version), "count_to_create_empty_major", need_create_empty_majors.count(), "fork_tablet_infos_count", fork_tablet_infos.count()); } else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) { @@ -85,7 +83,6 @@ int ObTabletCreatorArg::init( LOG_WARN("failed to assign fork tablet infos", KR(ret), K(fork_tablet_infos)); } else { data_tablet_id_ = data_tablet_id; - ls_key_ = ls_key; compat_mode_ = mode; is_create_bind_hidden_tablets_ = is_create_bind_hidden_tablets; tenant_data_version_ = tenant_data_version; @@ -95,7 +92,6 @@ int ObTabletCreatorArg::init( int ObTabletCreatorArg::init( const ObIArray &tablet_ids, - const share::ObLSID &ls_key, const ObTabletID data_tablet_id, const ObIArray &table_schemas, const lib::Worker::CompatMode &mode, @@ -105,7 +101,7 @@ int ObTabletCreatorArg::init( const ObIArray &create_commit_versions) { ObArray empty_fork_tablet_infos; - return init(tablet_ids, ls_key, data_tablet_id, table_schemas, mode, + return init(tablet_ids, data_tablet_id, table_schemas, mode, is_create_bind_hidden_tablets, tenant_data_version, need_create_empty_majors, create_commit_versions, empty_fork_tablet_infos); @@ -114,7 +110,7 @@ int ObTabletCreatorArg::init( DEF_TO_STRING(ObTabletCreatorArg) { int64_t pos = 0; - J_KV(K_(compat_mode), K_(tablet_ids), K_(data_tablet_id), K_(ls_key), K_(table_schemas), K_(is_create_bind_hidden_tablets), + J_KV(K_(compat_mode), K_(tablet_ids), K_(data_tablet_id), K_(table_schemas), K_(is_create_bind_hidden_tablets), K_(tenant_data_version), K_(need_create_empty_majors), K_(create_commit_versions), K_(fork_tablet_infos)); return pos; } @@ -122,18 +118,14 @@ DEF_TO_STRING(ObTabletCreatorArg) ///////////////////////////////////////////////////////// int ObBatchCreateTabletHelper::init( - const share::ObLSID &ls_key, const SCN &major_frozen_scn, const bool need_check_tablet_cnt) { int ret = OB_SUCCESS; const int64_t bucket_count = hash::cal_next_prime(100); auto_part_size_arr_.reset(); - if (OB_UNLIKELY(!ls_key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_key)); - } else if (OB_FAIL(batch_arg_.init_create_tablet(ls_key, major_frozen_scn, need_check_tablet_cnt))) { - LOG_WARN("failed to init create tablet", KR(ret), K(ls_key), K(major_frozen_scn)); + if (OB_FAIL(batch_arg_.init_create_tablet(major_frozen_scn, need_check_tablet_cnt))) { + LOG_WARN("failed to init create tablet", KR(ret), K(major_frozen_scn)); } else if (OB_FAIL(table_schemas_map_.create(bucket_count, "CreateTablet", "CreateTablet"))) { LOG_WARN("failed to create hashmap", KR(ret)); } @@ -277,15 +269,13 @@ ObTabletCreator::~ObTabletCreator() void ObTabletCreator::reset() { - FOREACH(iter, args_map_) { - ObBatchCreateTabletHelper *batch_arg = iter->second; - while (OB_NOT_NULL(batch_arg)) { - ObBatchCreateTabletHelper *tmp = batch_arg; - batch_arg = batch_arg->next_; - tmp->~ObBatchCreateTabletHelper(); - } + ObBatchCreateTabletHelper *batch_arg = single_batch_arg_; + while (OB_NOT_NULL(batch_arg)) { + ObBatchCreateTabletHelper *tmp = batch_arg; + batch_arg = batch_arg->next_; + tmp->~ObBatchCreateTabletHelper(); } - args_map_.clear(); + single_batch_arg_ = NULL; need_check_tablet_cnt_ = false; } @@ -295,8 +285,6 @@ int ObTabletCreator::init(const bool need_check_tablet_cnt) if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ObTabletCreator init twice", KR(ret)); - } else if (OB_FAIL(args_map_.create(MAP_BUCKET_NUM, "TabletCtr"))) { - LOG_WARN("fail to create args map", KR(ret)); } else { need_check_tablet_cnt_ = need_check_tablet_cnt; inited_ = true; @@ -314,24 +302,20 @@ int ObTabletCreator::add_create_tablet_arg(const ObTabletCreatorArg &arg) } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("arg is invalid", KR(ret), K(arg)); - } else if (OB_SUCC(args_map_.get_refactored(arg.ls_key_, batch_arg))) { - //already exist - } else if (OB_HASH_NOT_EXIST == ret) { - //create new arg + } else if (OB_ISNULL(single_batch_arg_)) { void *arg_buf = allocator_.alloc(sizeof(ObBatchCreateTabletHelper)); if (OB_ISNULL(arg_buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate new arg", KR(ret), KP(batch_arg)); } else if (FALSE_IT(batch_arg = new (arg_buf)ObBatchCreateTabletHelper())) { - } else if (OB_FAIL(batch_arg->init(arg.ls_key_, major_frozen_scn_, need_check_tablet_cnt_))) { + } else if (OB_FAIL(batch_arg->init(major_frozen_scn_, need_check_tablet_cnt_))) { LOG_WARN("failed to init batch arg helper", KR(ret), K(arg)); - } else if (OB_FAIL(args_map_.set_refactored(arg.ls_key_, batch_arg, 0/*not overwrite*/))) { - LOG_WARN("fail to set refactored", KR(ret), K(arg)); } else { - LOG_INFO("new log stream", "ls_key", arg.ls_key_); + single_batch_arg_ = batch_arg; + LOG_INFO("new single log stream tablet create batch", K(arg)); } } else { - LOG_WARN("failed to get batch arg", KR(ret), K(arg)); + batch_arg = single_batch_arg_; } if (OB_FAIL(ret)) { @@ -343,12 +327,11 @@ int ObTabletCreator::add_create_tablet_arg(const ObTabletCreatorArg &arg) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate new arg", KR(ret)); } else if (FALSE_IT(new_arg = new (arg_buf)ObBatchCreateTabletHelper())) { - } else if (OB_FAIL(new_arg->init(arg.ls_key_, major_frozen_scn_, need_check_tablet_cnt_))) { + } else if (OB_FAIL(new_arg->init(major_frozen_scn_, need_check_tablet_cnt_))) { LOG_WARN("failed to init batch arg helper", KR(ret), K(arg)); - } else if (FALSE_IT(new_arg->next_ = batch_arg)) { - } else if (OB_FAIL(args_map_.set_refactored(arg.ls_key_, new_arg, 1/*overwrite*/))) { - LOG_WARN("fail to set refactored", KR(ret), K(arg)); } else { + new_arg->next_ = batch_arg; + single_batch_arg_ = new_arg; batch_arg = new_arg; } } @@ -367,27 +350,20 @@ int ObTabletCreator::modify_batch_args( const bool clear_auto_part_size) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(0 >= args_map_.size())) { + if (OB_ISNULL(single_batch_arg_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("batch arg count is invalid", KR(ret)); } else { - FOREACH_X(iter, args_map_, OB_SUCC(ret)) { - ObBatchCreateTabletHelper *batch_arg = iter->second; - if (OB_ISNULL(batch_arg)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("batch arg not be null", KR(ret)); - } else { - while (OB_SUCC(ret) && OB_NOT_NULL(batch_arg)) { - batch_arg->batch_arg_.clog_checkpoint_scn_ = clog_checkpoint_scn; - batch_arg->batch_arg_.mds_checkpoint_scn_ = mds_checkpoint_scn; - batch_arg->batch_arg_.create_type_ = create_type; - if (clear_auto_part_size) { - batch_arg->auto_part_size_arr_.reset(); - } - batch_arg = batch_arg->next_; - } // end while + ObBatchCreateTabletHelper *batch_arg = single_batch_arg_; + while (OB_SUCC(ret) && OB_NOT_NULL(batch_arg)) { + batch_arg->batch_arg_.clog_checkpoint_scn_ = clog_checkpoint_scn; + batch_arg->batch_arg_.mds_checkpoint_scn_ = mds_checkpoint_scn; + batch_arg->batch_arg_.create_type_ = create_type; + if (clear_auto_part_size) { + batch_arg->auto_part_size_arr_.reset(); } - } // end for + batch_arg = batch_arg->next_; + } } return ret; } @@ -397,7 +373,6 @@ int ObTabletCreator::execute() int ret = OB_SUCCESS; ObTimeoutCtx ctx; const int64_t default_timeout_ts = GCONF.rpc_timeout; - const int64_t SLEEP_INTERVAL = 100 * 1000L; // 100ms observer::ObInnerSQLConnection *conn = NULL; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; @@ -406,78 +381,58 @@ int ObTabletCreator::execute() (trans_.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("conn_ is NULL", KR(ret)); - } else if (OB_UNLIKELY(0 >= args_map_.size())) { + } else if (OB_ISNULL(single_batch_arg_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("batch arg count is invalid", KR(ret)); } else { - FOREACH_X(iter, args_map_, OB_SUCC(ret)) { - ObBatchCreateTabletHelper *batch_arg = iter->second; - if (OB_ISNULL(batch_arg)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("batch arg not be null", KR(ret)); + ObBatchCreateTabletHelper *batch_arg = single_batch_arg_; + while (OB_SUCC(ret) && OB_NOT_NULL(batch_arg)) { + int64_t buf_len = batch_arg->batch_arg_.get_serialize_size(); + int64_t pos = 0; + char *buf = (char*)allocator_.alloc(buf_len); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail alloc memory", KR(ret)); + } else if (OB_FAIL(batch_arg->batch_arg_.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), K(batch_arg->batch_arg_)); + } else if (OB_FAIL(share::ObShareUtil::set_default_timeout_ctx(ctx, default_timeout_ts))) { + LOG_WARN("fail to set timeout ctx", KR(ret), K(default_timeout_ts)); } else { - while (OB_SUCC(ret) && OB_NOT_NULL(batch_arg)) { - int64_t buf_len = batch_arg->batch_arg_.get_serialize_size(); - int64_t pos = 0; - char *buf = (char*)allocator_.alloc(buf_len); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail alloc memory", KR(ret)); - } else if (OB_FAIL(batch_arg->batch_arg_.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize", KR(ret), K(batch_arg->batch_arg_)); - } else if (OB_FAIL(share::ObShareUtil::set_default_timeout_ctx(ctx, default_timeout_ts))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(default_timeout_ts)); - } else { - do { - int64_t start_time = ObTimeUtility::current_time(); - if (ctx.is_timeouted()) { - ret = OB_TIMEOUT; - LOG_WARN("already timeout", KR(ret), K(ctx)); - } else if (OB_FAIL(conn->register_multi_data_source(iter->first, - transaction::ObTxDataSourceType::CREATE_TABLET_NEW_MDS, buf, buf_len))) { - if (need_retry(ret)) { - LOG_INFO("fail to register_tx_data, try again", KR(ret), K(batch_arg->batch_arg_)); - ob_usleep(SLEEP_INTERVAL); - } else { - LOG_WARN("fail to register_tx_data", KR(ret), K(batch_arg->batch_arg_), K(buf), K(buf_len)); - } - } - int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("generate create arg", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), - K(batch_arg->batch_arg_), "cost_ts", end_time - start_time); - } while (need_retry(ret)); - if (OB_SUCC(ret) && batch_arg->batch_arg_.set_binding_info_outside_create()) { - const int64_t start_time = ObTimeUtility::current_time(); - if (OB_FAIL(ObTabletBindingMdsHelper::modify_tablet_binding_for_create(batch_arg->batch_arg_, ctx.get_abs_timeout(), trans_))) { - LOG_WARN("failed to modify tablet binding for create", K(ret)); - } - const int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("modify binding for create", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), - "cost_ts", end_time - start_time); - } - if (OB_SUCC(ret)) { - const int64_t start_time = ObTimeUtility::current_time(); - if (OB_FAIL(ObTabletSplitMdsHelper::set_auto_part_size_for_create(batch_arg->batch_arg_, batch_arg->auto_part_size_arr_, ctx.get_abs_timeout(), trans_))) { - LOG_WARN("failed to set auto part size for create", K(ret)); - } - const int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("set auto part size for create", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), - "cost_ts", end_time - start_time); - } + int64_t start_time = ObTimeUtility::current_time(); + if (ctx.is_timeouted()) { + ret = OB_TIMEOUT; + LOG_WARN("already timeout", KR(ret), K(ctx)); + } else if (OB_FAIL(conn->register_multi_data_source(transaction::ObTxDataSourceType::CREATE_TABLET_NEW_MDS, buf, buf_len))) { + LOG_WARN("fail to register_tx_data", KR(ret), K(batch_arg->batch_arg_), K(buf), K(buf_len)); + } + int64_t end_time = ObTimeUtility::current_time(); + LOG_INFO("generate create arg", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), + K(batch_arg->batch_arg_), "cost_ts", end_time - start_time); + if (OB_SUCC(ret) && batch_arg->batch_arg_.set_binding_info_outside_create()) { + const int64_t start_time = ObTimeUtility::current_time(); + if (OB_FAIL(ObTabletBindingMdsHelper::modify_tablet_binding_for_create(batch_arg->batch_arg_, ctx.get_abs_timeout(), trans_))) { + LOG_WARN("failed to modify tablet binding for create", K(ret)); + } + const int64_t end_time = ObTimeUtility::current_time(); + LOG_INFO("modify binding for create", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), + "cost_ts", end_time - start_time); + } + if (OB_SUCC(ret)) { + const int64_t start_time = ObTimeUtility::current_time(); + if (OB_FAIL(ObTabletSplitMdsHelper::set_auto_part_size_for_create(batch_arg->batch_arg_, batch_arg->auto_part_size_arr_, ctx.get_abs_timeout(), trans_))) { + LOG_WARN("failed to set auto part size for create", K(ret)); } - batch_arg = batch_arg->next_; - } // end while + const int64_t end_time = ObTimeUtility::current_time(); + LOG_INFO("set auto part size for create", KR(ret), K(buf_len), K(batch_arg->batch_arg_.tablets_.count()), + "cost_ts", end_time - start_time); + } } - } // end for + batch_arg = batch_arg->next_; + } } reset(); return ret; } -bool ObTabletCreator::need_retry(int ret) -{ - return is_location_service_renew_error(ret) || OB_NOT_MASTER == ret; -} - } // rootserver } // oceanbase diff --git a/src/rootserver/ob_tablet_creator.h b/src/rootserver/ob_tablet_creator.h index 4ca9212ae..c765f0ede 100644 --- a/src/rootserver/ob_tablet_creator.h +++ b/src/rootserver/ob_tablet_creator.h @@ -24,7 +24,6 @@ #include "lib/allocator/ob_malloc.h" #include "common/ob_tablet_id.h"//ObTabletID #include "share/ob_rpc_struct.h"//ObBatchCreateTabletArg -#include "share/ob_ls_id.h"//share::ObLSID #include "share/ob_ddl_common.h" // ObForkTabletInfo #include "common/mysqlclient/ob_mysql_transaction.h" #include "share/ob_define.h" @@ -43,7 +42,6 @@ struct ObTabletCreatorArg ObTabletCreatorArg() : tablet_ids_(), data_tablet_id_(), - ls_key_(), table_schemas_(), compat_mode_(lib::Worker::CompatMode::INVALID), is_create_bind_hidden_tablets_(false), @@ -53,7 +51,6 @@ struct ObTabletCreatorArg virtual ~ObTabletCreatorArg() {} bool is_valid() const; int init(const ObIArray &tablet_ids, - const share::ObLSID &ls_key, const common::ObTabletID data_tablet_id, const ObIArray &table_schemas, const lib::Worker::CompatMode &mode, @@ -63,7 +60,6 @@ struct ObTabletCreatorArg const ObIArray &create_commit_versions, const ObIArray &fork_tablet_infos); int init(const ObIArray &tablet_ids, - const share::ObLSID &ls_key, const common::ObTabletID data_tablet_id, const ObIArray &table_schemas, const lib::Worker::CompatMode &mode, @@ -75,7 +71,6 @@ struct ObTabletCreatorArg DECLARE_TO_STRING; common::ObArray tablet_ids_; common::ObTabletID data_tablet_id_; - share::ObLSID ls_key_; common::ObArray table_schemas_; lib::Worker::CompatMode compat_mode_; bool is_create_bind_hidden_tablets_; @@ -97,8 +92,7 @@ struct ObBatchCreateTabletHelper result_(common::OB_NOT_MASTER), next_(NULL) {} - int init(const share::ObLSID &ls_key, - const share::SCN &major_frozen_scn, + int init(const share::SCN &major_frozen_scn, const bool need_check_tablet_cnt); int try_add_table_schema(const share::schema::ObTableSchema *table_schema, const uint64_t tenant_data_version, @@ -143,7 +137,7 @@ const static int64_t BATCH_ARG_SIZE = 1024 * 1024; // 1M ObMySQLTransaction &trans) : major_frozen_scn_(major_frozen_scn), allocator_("TbtCret"), - args_map_(), + single_batch_arg_(NULL), trans_(trans), need_check_tablet_cnt_(false), inited_(false) {} @@ -151,7 +145,6 @@ const static int64_t BATCH_ARG_SIZE = 1024 * 1024; // 1M virtual ~ObTabletCreator(); int init(const bool need_check_tablet_cnt); int execute(); - bool need_retry(int ret); int add_create_tablet_arg(const ObTabletCreatorArg &arg); int modify_batch_args(const storage::ObTabletMdsUserDataType &create_type, const share::SCN &clog_checkpoint_scn, @@ -159,14 +152,9 @@ const static int64_t BATCH_ARG_SIZE = 1024 * 1024; // 1M const bool clear_auto_part_size); void reset(); private: - int find_leader_of_ls(const share::ObLSID &id, ObAddr &addr); -private: - const int64_t MAP_BUCKET_NUM = 1024; -private: - const share::SCN major_frozen_scn_; ObArenaAllocator allocator_; - common::hash::ObHashMap args_map_; + ObBatchCreateTabletHelper *single_batch_arg_; ObMySQLTransaction &trans_; bool need_check_tablet_cnt_; bool inited_; diff --git a/src/rootserver/ob_tablet_drop.cpp b/src/rootserver/ob_tablet_drop.cpp index 28754277e..43d474138 100644 --- a/src/rootserver/ob_tablet_drop.cpp +++ b/src/rootserver/ob_tablet_drop.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX RS #include "ob_tablet_drop.h" #include "share/tablet/ob_tablet_to_table_history_operator.h" // ObTabletToTableHistoryOperator -#include "share/tablet/ob_tablet_to_ls_operator.h" #include "observer/ob_inner_sql_connection.h" namespace oceanbase @@ -27,10 +26,8 @@ namespace rootserver ObTabletDrop::~ObTabletDrop() { - FOREACH(iter, args_map_) { - if (OB_NOT_NULL(iter->second) - && FALSE_IT(iter->second->~ObIArray())) { - } + if (OB_NOT_NULL(tablet_ids_) + && FALSE_IT(tablet_ids_->~ObIArray())) { } } @@ -41,8 +38,6 @@ int ObTabletDrop::init() if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ObTabletDrop init twice", KR(ret)); - } else if (OB_FAIL(args_map_.create(MAP_BUCKET_NUM, "TabletCtr"))) { - LOG_WARN("fail to args map", KR(ret)); } else { inited_ = true; } @@ -53,8 +48,6 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( const common::ObIArray &schemas) { int ret = OB_SUCCESS; - common::ObArray ls_ids; - int64_t all_part_num = 0; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObTabletCreator not init", KR(ret)); @@ -72,8 +65,7 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( } else if (schemas.count() > 1) { int64_t data_table_id = OB_INVALID_ID; if (table_schema.is_index_local_storage() - || table_schema.is_aux_lob_table() - || table_schema.is_mlog_table()) { + || table_schema.is_aux_lob_table()) { data_table_id = table_schema.get_data_table_id(); } else { data_table_id = table_schema.get_table_id(); @@ -87,8 +79,7 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( ret = OB_INVALID_ARGUMENT; LOG_WARN("sys table cannot drop", KPC(aux_table_schema), KR(ret)); } else if (!aux_table_schema->is_index_local_storage() - && !aux_table_schema->is_aux_lob_table() - && !aux_table_schema->is_mlog_table()) { + && !aux_table_schema->is_aux_lob_table()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("aux_table_schema must be local index or aux lob table", KR(ret), K(schemas), KPC(aux_table_schema)); } else if (data_table_id != aux_table_schema->get_data_table_id()) { @@ -98,14 +89,10 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(get_ls_from_table(table_schema, true/*is_include_hidden*/, ls_ids))) { - LOG_WARN("fail to get ls from table", KR(ret)); } else { - int64_t ls_idx = 0; ObPartitionLevel part_level = table_schema.get_part_level(); if (PARTITION_LEVEL_ZERO == part_level) { - if (OB_FAIL(drop_tablet_(schemas, ls_ids.at(ls_idx++), - OB_INVALID_INDEX, OB_INVALID_INDEX, false/*is_hidden*/))) { + if (OB_FAIL(drop_tablet_(schemas, OB_INVALID_INDEX, OB_INVALID_INDEX, false/*is_hidden*/))) { LOG_WARN("fail to drop tablet", K(table_schema), KR(ret)); } } else { @@ -135,11 +122,9 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ptr", K(i), K(table_schema), KR(ret)); } else if (PARTITION_LEVEL_ONE == part_level) { - if (is_hidden && OB_FAIL(drop_tablet_(schemas, ls_ids.at(ls_idx++), i - part_num, - OB_INVALID_INDEX, is_hidden))) { + if (is_hidden && OB_FAIL(drop_tablet_(schemas, i - part_num, OB_INVALID_INDEX, is_hidden))) { LOG_ERROR("fail to drop tablet", K(table_schema), KR(ret)); - } else if (!is_hidden && OB_FAIL(drop_tablet_(schemas, ls_ids.at(ls_idx++), i, - OB_INVALID_INDEX, is_hidden))) { + } else if (!is_hidden && OB_FAIL(drop_tablet_(schemas, i, OB_INVALID_INDEX, is_hidden))) { LOG_ERROR("fail to drop tablet", K(table_schema), KR(ret)); } } else if (PARTITION_LEVEL_TWO == part_level) { @@ -154,7 +139,7 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ptr", K(j), K(table_schema), KR(ret)); } else { - if (OB_FAIL(drop_tablet_(schemas, ls_ids.at(ls_idx++), i, j, false/*is_hidden*/))) { + if (OB_FAIL(drop_tablet_(schemas, i, j, false/*is_hidden*/))) { LOG_WARN("fail to drop tablet", K(table_schema), KR(ret)); } } @@ -172,74 +157,30 @@ int ObTabletDrop::add_drop_tablets_of_table_arg( return ret; } -int ObTabletDrop::get_ls_from_table(const share::schema::ObTableSchema &table_schema, - const bool is_include_hidden, - common::ObIArray &assign_ls_id_array) -{ - int ret = OB_SUCCESS; - int64_t all_part_num = table_schema.get_all_part_num(); - if (is_include_hidden - && 0 < table_schema.get_hidden_partition_num() - && PARTITION_LEVEL_ONE == table_schema.get_part_level()) { - all_part_num += table_schema.get_hidden_partition_num(); - } - if (-1 == all_part_num) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet num", K(table_schema), KR(ret)); - } else if (OB_FAIL(assign_ls_id_array.reserve(all_part_num))) { - LOG_WARN("fail to reserve array", KR(ret), K(all_part_num)); - } else { - for (int64_t i = 0; i < all_part_num && OB_SUCC(ret); i++) { - if (OB_FAIL(assign_ls_id_array.push_back(share::SYS_LS))) { - LOG_WARN("failed to push_back", KR(ret)); - } - } - } - - if (OB_FAIL(ret)) { - } else if (assign_ls_id_array.count() != all_part_num) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls of tablet is not equal partition num", - KR(ret), K(assign_ls_id_array.count()), K(all_part_num)); - } - return ret; -} - int ObTabletDrop::drop_tablet_( const common::ObIArray &table_schema_ptr_array, - const share::ObLSID &ls_id, const int64_t part_idx, const int64_t subpart_idx, const bool is_hidden) { int ret = OB_SUCCESS; - common::ObIArray *tablet_id_array = NULL; if (table_schema_ptr_array.count() < 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_schema_ptr_array is empty", K(table_schema_ptr_array), KR(ret)); - } else if (OB_SUCC(args_map_.get_refactored(ls_id, tablet_id_array))) { - //already exist - } else if (OB_HASH_NOT_EXIST == ret) { - //create new tablet_id_array + } else if (OB_ISNULL(tablet_ids_)) { void *ptr1 = allocator_.alloc(sizeof(ObArray)); - void *ptr2 = allocator_.alloc(sizeof(ObArray)); - if (OB_ISNULL(ptr1) || OB_ISNULL(ptr2)) { + if (OB_ISNULL(ptr1)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail alloc memory", KR(ret)); } else { - tablet_id_array = new (ptr1)ObArray( + tablet_ids_ = new (ptr1)ObArray( OB_MALLOC_NORMAL_BLOCK_SIZE, allocator_); - if (OB_FAIL(args_map_.set_refactored(ls_id, tablet_id_array, 0/*not overwrite*/))) { - LOG_WARN("fail to set refactored", KR(ret), K(ls_id)); - } } - } else { - LOG_WARN("fail to get element from args_map", KR(ret), K(ls_id)); } if (OB_FAIL(ret)) { - } else if (OB_ISNULL(tablet_id_array)) { + } else if (OB_ISNULL(tablet_ids_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ptr", K(table_schema_ptr_array), KR(ret)); } else if (OB_ISNULL(table_schema_ptr_array.at(0))) { @@ -262,7 +203,7 @@ int ObTabletDrop::drop_tablet_( LOG_WARN("hidden tablet only support partition-levle-one", K(table_schema_ptr), KR(ret), K(part_idx)); } else if (PARTITION_LEVEL_ZERO == table_schema_ptr->get_part_level()) { ObTabletID tablet_id = table_schema_ptr->get_tablet_id(); - if (OB_FAIL(tablet_id_array->push_back(tablet_id))) { + if (OB_FAIL(tablet_ids_->push_back(tablet_id))) { LOG_WARN("failed to assign table schema point", KR(ret), KPC(table_schema_ptr)); } } else if(is_hidden && OB_FALSE_IT(part = table_schema_ptr->get_hidden_part_array()[part_idx])) { @@ -281,7 +222,7 @@ int ObTabletDrop::drop_tablet_( } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(tablet_id_array->push_back(part->get_tablet_id()))) { + } else if (OB_FAIL(tablet_ids_->push_back(part->get_tablet_id()))) { LOG_WARN("failed to assign table schema point", KR(ret), K(part_idx), K(subpart_idx)); } } @@ -303,53 +244,43 @@ int ObTabletDrop::execute() (trans_.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("conn_ is NULL", KR(ret)); - } else if (OB_UNLIKELY(0 >= args_map_.size())) { + } else if (OB_ISNULL(tablet_ids_) || OB_UNLIKELY(tablet_ids_->count() < 1)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("batch arg count is invalid", KR(ret)); } else { - FOREACH_X(iter, args_map_, OB_SUCC(ret)) { - common::ObIArray *tablet_ids = iter->second; - obcall::ObBatchRemoveTabletArg arg; - if (OB_ISNULL(tablet_ids)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet_ids is NULL ptr", KR(ret)); - } else if (tablet_ids->count() < 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet_ids is empty", KR(ret)); - } else if (OB_FAIL(share::ObTabletToLSTableOperator::batch_remove(trans_, *tablet_ids))) { - LOG_WARN("tablet_ids count less than 1", KR(ret)); - } else if (OB_FAIL(share::ObTabletToTableHistoryOperator::drop_tablet_to_table_history( - trans_, schema_version_, *tablet_ids))) { - LOG_WARN("fail to create tablet to table history", KR(ret), K(schema_version_),KPC(tablet_ids)); - } else if (OB_FAIL(arg.init(*(tablet_ids), iter->first))) { - LOG_WARN("failed to find leader", KR(ret), K(iter->first), KPC(tablet_ids)); + obcall::ObBatchRemoveTabletArg arg; + if (OB_FAIL(share::ObTabletMappingTableOperator::batch_remove(trans_, *tablet_ids_))) { + LOG_WARN("tablet_ids count less than 1", KR(ret)); + } else if (OB_FAIL(share::ObTabletToTableHistoryOperator::drop_tablet_to_table_history( + trans_, schema_version_, *tablet_ids_))) { + LOG_WARN("fail to create tablet to table history", KR(ret), K(schema_version_), KPC(tablet_ids_)); + } else if (OB_FAIL(arg.init(*tablet_ids_))) { + LOG_WARN("failed to init remove tablet arg", KR(ret), KPC(tablet_ids_)); + } else { + LOG_INFO("generate remove arg", K(arg), K(lbt()), KPC(tablet_ids_)); + int64_t buf_len = arg.get_serialize_size(); + int64_t pos = 0; + char *buf = (char*)allocator_.alloc(buf_len); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail alloc memory", KR(ret)); + } else if (OB_FAIL(arg.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), K(arg)); + } else if (OB_FAIL(share::ObShareUtil::set_default_timeout_ctx(ctx, default_timeout_ts))) { + LOG_WARN("fail to set timeout ctx", KR(ret), K(default_timeout_ts)); } else { - LOG_INFO("generate remove arg", K(arg), K(lbt()), KPC(tablet_ids)); - int64_t buf_len = arg.get_serialize_size(); - int64_t pos = 0; - char *buf = (char*)allocator_.alloc(buf_len); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail alloc memory", KR(ret)); - } else if (OB_FAIL(arg.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize", KR(ret), K(arg)); - } else if (OB_FAIL(share::ObShareUtil::set_default_timeout_ctx(ctx, default_timeout_ts))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(default_timeout_ts)); - } else { - do { - if (ctx.is_timeouted()) { - ret = OB_TIMEOUT; - LOG_WARN("already timeout", KR(ret), K(ctx)); - } else if (OB_FAIL(conn->register_multi_data_source(iter->first, - transaction::ObTxDataSourceType::DELETE_TABLET_NEW_MDS, buf, buf_len))) { - LOG_WARN("fail to register_tx_data", KR(ret), K(arg), K(buf), K(buf_len)); - if (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret) { - LOG_INFO("fail to find leader, try again", K(arg)); - ob_usleep(SLEEP_INTERVAL); - } + do { + if (ctx.is_timeouted()) { + ret = OB_TIMEOUT; + LOG_WARN("already timeout", KR(ret), K(ctx)); + } else if (OB_FAIL(conn->register_multi_data_source(transaction::ObTxDataSourceType::DELETE_TABLET_NEW_MDS, buf, buf_len))) { + LOG_WARN("fail to register_tx_data", KR(ret), K(arg), K(buf), K(buf_len)); + if (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret) { + LOG_INFO("fail to find leader, try again", K(arg)); + ob_usleep(SLEEP_INTERVAL); } - } while (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret); - } + } + } while (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret); } } } diff --git a/src/rootserver/ob_tablet_drop.h b/src/rootserver/ob_tablet_drop.h index 08dbdfe5d..24b153509 100644 --- a/src/rootserver/ob_tablet_drop.h +++ b/src/rootserver/ob_tablet_drop.h @@ -23,7 +23,6 @@ #include "common/mysqlclient/ob_mysql_transaction.h" #include "share/ob_define.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -34,7 +33,6 @@ namespace schema class ObTableSchema; } -class ObLSID; } namespace rpc @@ -52,6 +50,7 @@ class ObTabletDrop int64_t schema_version) : trans_(trans), allocator_("TbtDrop"), + tablet_ids_(NULL), schema_version_(schema_version), inited_(false) {} virtual ~ObTabletDrop(); @@ -66,23 +65,18 @@ class ObTabletDrop // 2. or all are local indexes of a table int add_drop_tablets_of_table_arg( const common::ObIArray &schemas); - int get_ls_from_table(const share::schema::ObTableSchema &table_schema, - const bool is_include_hidden, - common::ObIArray &assign_ls_id_array); private: int drop_tablet_( const common::ObIArray &table_schema_ptr_array, - const share::ObLSID &ls_key, const int64_t i, const int64_t j, const bool is_hidden); private: ObMySQLTransaction &trans_; ObArenaAllocator allocator_; - common::hash::ObHashMap*> args_map_; + common::ObIArray *tablet_ids_; int64_t schema_version_; bool inited_; - const int64_t MAP_BUCKET_NUM = 1024; }; } } diff --git a/src/rootserver/ob_tenant_ddl_service.cpp b/src/rootserver/ob_tenant_ddl_service.cpp index 3fb1c5029..14ec610c3 100644 --- a/src/rootserver/ob_tenant_ddl_service.cpp +++ b/src/rootserver/ob_tenant_ddl_service.cpp @@ -22,7 +22,6 @@ #include "rootserver/ob_tenant_thread_helper.h" #include "rootserver/ob_ddl_service.h" #include "rootserver/ob_root_service.h" -#include "share/location_cache/ob_location_service.h" #include "rootserver/ob_table_creator.h" #include "share/ob_global_stat_proxy.h" #include "share/ob_schema_status_proxy.h" diff --git a/src/rootserver/ob_tenant_ddl_service.h b/src/rootserver/ob_tenant_ddl_service.h index 8ce2a6aac..ae8ffa4c0 100644 --- a/src/rootserver/ob_tenant_ddl_service.h +++ b/src/rootserver/ob_tenant_ddl_service.h @@ -75,8 +75,6 @@ struct ObSysStat // all tenant own Item ob_max_used_normal_rowid_table_tablet_id_; Item ob_max_used_extended_rowid_table_tablet_id_; - Item ob_max_used_ls_id_; - Item ob_max_used_ls_group_id_; Item ob_max_used_sys_pl_object_id_; Item ob_max_used_object_id_; Item ob_max_used_rewrite_rule_version_; diff --git a/src/rootserver/ob_tenant_event_def.h b/src/rootserver/ob_tenant_event_def.h index e198b1f50..69742df66 100644 --- a/src/rootserver/ob_tenant_event_def.h +++ b/src/rootserver/ob_tenant_event_def.h @@ -55,47 +55,6 @@ */ #ifdef DEF_MODULE #ifdef DEF_EVENT - /** - * @description: - * Log events related to tenant role change - * failover to primary/switchover to primary/switchover to standby - */ - class TENANT_ROLE_CHANGE { - public: - DEF_MODULE(TENANT_ROLE_CHANGE, "TENANT ROLE CHANGE"); - DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_START, "SWITCHOVER TO PRIMARY START", - STMT_STR, - TENANT_INFO); - DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_END, "SWITCHOVER TO PRIMARY END", - STMT_STR, - TENANT_INFO, - SWITCHOVER_SCN, - COST_DETAIL, - ALL_LS); - DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_START, "SWITCHOVER TO STANDBY START", - STMT_STR, - TENANT_INFO); - DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_END, "SWITCHOVER TO STANDBY END", - STMT_STR, - TENANT_INFO, - SWITCHOVER_SCN, - COST_DETAIL, - ALL_LS); - DEF_EVENT(TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_START, "FAILOVER TO PRIMARY START", - STMT_STR, - TENANT_INFO); - DEF_EVENT(TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_END, "FAILOVER TO PRIMARY END", - STMT_STR, - TENANT_INFO, - FAILOVER_SCN, - COST_DETAIL, - ALL_LS); - DEF_EVENT(TENANT_ROLE_CHANGE, WAIT_LOG_SYNC, "WAIT LOG SYNC", - IS_SYS_LS_SYNCED, - IS_ALL_LS_SYNCED, - NON_SYNC_INFO); - }; - class SERVICE_NAME { public: DEF_MODULE(SERVICE_NAME, "SERVICE NAME"); diff --git a/src/rootserver/ob_tenant_thread_helper.cpp b/src/rootserver/ob_tenant_thread_helper.cpp index 0ce0b3661..ee933ecf0 100755 --- a/src/rootserver/ob_tenant_thread_helper.cpp +++ b/src/rootserver/ob_tenant_thread_helper.cpp @@ -143,11 +143,11 @@ void ObTenantThreadHelper::destroy() LOG_INFO("[TENANT THREAD] thread destory finish", K(thread_cnt_), K(thread_name_)); } -void ObTenantThreadHelper::switch_to_follower_forcedly() +void ObTenantThreadHelper::deactivate() { stop(); } -int ObTenantThreadHelper::switch_to_leader() +int ObTenantThreadHelper::activate() { int ret = OB_SUCCESS; LOG_INFO("[TENANT THREAD] thread start", K(thread_cnt_), K(thread_name_)); diff --git a/src/rootserver/ob_tenant_thread_helper.h b/src/rootserver/ob_tenant_thread_helper.h index d104b3413..cff6379fb 100644 --- a/src/rootserver/ob_tenant_thread_helper.h +++ b/src/rootserver/ob_tenant_thread_helper.h @@ -49,7 +49,7 @@ class ObLogHandler; namespace rootserver { class ObTenantThreadHelper : public share::ObReentrantThread, - public logservice::ObIRoleChangeSubHandler + public logservice::ObILocalLogHandler { public: ObTenantThreadHelper() @@ -74,18 +74,8 @@ class ObTenantThreadHelper : public share::ObReentrantThread, void idle(const int64_t idle_time_us); void wakeup(); public: - virtual void switch_to_follower_forcedly() override; - - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override - { - stop(); - return OB_SUCCESS; - } - virtual int resume_leader() override - { - return OB_SUCCESS; - } + void deactivate() override; + int activate() override; public: #define DEFINE_MTL_FUNC(TYPE)\ static int mtl_init(TYPE *&ka) {\ diff --git a/src/rootserver/parallel_ddl/ob_create_index_helper.cpp b/src/rootserver/parallel_ddl/ob_create_index_helper.cpp index 6ca427876..92b6a082e 100644 --- a/src/rootserver/parallel_ddl/ob_create_index_helper.cpp +++ b/src/rootserver/parallel_ddl/ob_create_index_helper.cpp @@ -596,14 +596,11 @@ int ObCreateIndexHelper::create_tablets_() } else { ObTableSchema &index_schema = index_schemas_.at(0); ObTableCreator table_creator(frozen_scn, get_trans_()); - common::ObArray ls_id_array; const ObTablegroupSchema *data_tablegroup_schema = NULL; // keep NULL if no tablegroup ObSEArray need_create_empty_majors; uint64_t tenant_data_version = 0; if (OB_FAIL(table_creator.init(true/*need_tablet_cnt_check*/))) { LOG_WARN("fail to init table craetor", KR(ret)); - } else if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_data_version))) { LOG_WARN("fail to get data version", KR(ret)); } else if (OB_INVALID_ID != orig_data_table_schema_->get_tablegroup_id()) { @@ -625,13 +622,12 @@ int ObCreateIndexHelper::create_tablets_() } else if (OB_FAIL(table_creator.add_create_tablets_of_local_aux_tables_arg( schemas, orig_data_table_schema_, - ls_id_array, tenant_data_version, need_create_empty_majors))) { - LOG_WARN("create table tablet failed", KR(ret), K(index_schema), K(ls_id_array)); + LOG_WARN("create table tablet failed", KR(ret), K(index_schema)); } } else { - if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(index_schema, ls_id_array, + if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(index_schema, tenant_data_version, create_index_on_empty_table_opt_/*need create major sstable*/))) { LOG_WARN("create table tablet failed", KR(ret), K(index_schema)); } diff --git a/src/rootserver/parallel_ddl/ob_table_helper.cpp b/src/rootserver/parallel_ddl/ob_table_helper.cpp index 9e58b0568..d7153ea80 100644 --- a/src/rootserver/parallel_ddl/ob_table_helper.cpp +++ b/src/rootserver/parallel_ddl/ob_table_helper.cpp @@ -413,12 +413,8 @@ int ObTableHelper::create_tablets_() if (OB_FAIL(ret)) { } else { ObArray schemas; - common::ObArray ls_id_array; ObArray need_create_empty_majors; ObArray table_ids; - if (OB_FAIL(ls_id_array.push_back(ObLSID(SYS_LS)))) { - LOG_WARN("fail to push back sys ls", KR(ret)); - } for (int64_t i = 0; OB_SUCC(ret) && i < new_tables_.count(); i++) { const ObTableSchema &new_table = new_tables_.at(i); const uint64_t table_id = new_table.get_table_id(); @@ -432,7 +428,7 @@ int ObTableHelper::create_tablets_() } } else { if (OB_FAIL(table_creator.add_create_tablets_of_table_arg( - new_table, ls_id_array, tenant_data_version, true/*need create major sstable*/))) { + new_table, tenant_data_version, true/*need create major sstable*/))) { LOG_WARN("create table partitions failed", KR(ret), K(new_table)); } } @@ -447,7 +443,7 @@ int ObTableHelper::create_tablets_() K(last_schema_version)); } else if (schemas.count() > 0) { if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg( - schemas, ls_id_array, tenant_data_version, need_create_empty_majors /*need create major sstable*/))) { + schemas, tenant_data_version, need_create_empty_majors /*need create major sstable*/))) { LOG_WARN("create table partitions failed", KR(ret), K(data_table)); } else if (OB_FAIL(table_creator.execute())) { LOG_WARN("execute create partition failed", KR(ret)); diff --git a/src/rootserver/truncate_info/ob_truncate_info_service.cpp b/src/rootserver/truncate_info/ob_truncate_info_service.cpp index baa646ab3..60d432707 100644 --- a/src/rootserver/truncate_info/ob_truncate_info_service.cpp +++ b/src/rootserver/truncate_info/ob_truncate_info_service.cpp @@ -17,7 +17,7 @@ #include "rootserver/truncate_info/ob_truncate_info_service.h" #include "share/ob_rpc_struct.h" #include "share/schema/ob_schema_struct.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "observer/ob_inner_sql_connection.h" #include "rootserver/ob_ddl_service.h" #include "share/ob_debug_sync_point.h" @@ -42,14 +42,14 @@ ERRSIM_POINT_DEF(EN_TRUNCATE_TABLET_TRANS); int ObTruncateTabletArg::serialize(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, info_, ls_id_, index_tablet_id_, truncate_info_); + LST_DO_CODE(OB_UNIS_ENCODE, info_, index_tablet_id_, truncate_info_); return ret; } int64_t ObTruncateTabletArg::get_serialize_size() const { int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, info_, ls_id_, index_tablet_id_, truncate_info_); + LST_DO_CODE(OB_UNIS_ADD_LEN, info_, index_tablet_id_, truncate_info_); return len; } @@ -60,7 +60,7 @@ int ObTruncateTabletArg::deserialize( int64_t &pos) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, info_, ls_id_, index_tablet_id_); + LST_DO_CODE(OB_UNIS_DECODE, info_, index_tablet_id_); if (FAILEDx(truncate_info_.deserialize(allocator, buf, data_len, pos))) { LOG_WARN("failed to deserialize truncate arg", KR(ret)); } @@ -426,7 +426,6 @@ ObTruncateInfoService::ObTruncateInfoService( arg_(arg), data_table_schema_(data_table_schema), index_tablet_array_(), - ls_id_array_(), part_key_info_(allocator_), ddl_task_id_(0), is_inited_(false) @@ -510,8 +509,8 @@ int ObTruncateInfoService::execute(ObMySQLTransaction &trans, ObTableSchema &index_table_schema) { int ret = OB_SUCCESS; ObInnerSQLConnection *conn = nullptr; + ObArray tablet_infos; index_tablet_array_.reuse(); - ls_id_array_.reuse(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTruncateInfoService is not inited", K(ret)); @@ -524,8 +523,13 @@ int ObTruncateInfoService::execute(ObMySQLTransaction &trans, LOG_WARN("failed to get tablet id from index schema", KR(ret), K(index_table_schema)); } else if (index_tablet_array_.empty()) { // do nothing - } else if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, index_tablet_array_, ls_id_array_))) { - LOG_WARN("failed to get ls id array", KR(ret), K(1UL), K_(index_tablet_array)); + } else if (OB_FAIL(ObTabletMappingTableOperator::batch_get(trans, index_tablet_array_, tablet_infos))) { + LOG_WARN("failed to get tablet info array", KR(ret), K(1UL), K_(index_tablet_array)); + } else if (OB_UNLIKELY(index_tablet_array_.count() != tablet_infos.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tablet info count", KR(ret), K_(index_tablet_array), K(tablet_infos)); + } + if (OB_FAIL(ret)) { } else if (OB_ISNULL(conn = static_cast(trans.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("conn is NULL", KR(ret)); @@ -864,19 +868,13 @@ int ObTruncateInfoService::loop_index_tablet_id_to_register_( ObTruncateTabletArg &truncate_arg) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(ls_id_array_.count() != index_tablet_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet id array and ls id array is not equal", KR(ret), K_(ls_id_array), K_(index_tablet_array)); - } else { - truncate_arg.truncate_info_.key_.tx_id_ = ddl_task_id_; - } + truncate_arg.truncate_info_.key_.tx_id_ = ddl_task_id_; for (int64_t j = 0; OB_SUCC(ret) && j < index_tablet_array_.count(); ++j) { // for same index tablet, different truncate key have different inc_seq - truncate_arg.ls_id_ = ls_id_array_.at(j); truncate_arg.index_tablet_id_ = index_tablet_array_.at(j); if (OB_FAIL(register_mds_(conn, truncate_arg))) { LOG_WARN("failed to register mds", KR(ret), K(1UL), K(j), - K_(index_tablet_array), K_(ls_id_array), K(truncate_arg)); + K_(index_tablet_array), K(truncate_arg)); } } // for return ret; @@ -895,44 +893,26 @@ int ObTruncateInfoService::register_mds_( LOG_WARN("fail alloc memory", KR(ret), K(buf_len)); } else if (OB_FAIL(arg.serialize(buf, buf_len, pos))) { LOG_WARN("fail to serialize", KR(ret), K(arg)); - } else if (OB_FAIL(retry_register_mds_(conn, arg, buf, buf_len))) { + } else if (OB_FAIL(register_mds_(conn, arg, buf, buf_len))) { LOG_WARN("fail to register mds", KR(ret), K(1UL), K(arg)); } return ret; } -int ObTruncateInfoService::retry_register_mds_( +int ObTruncateInfoService::register_mds_( ObInnerSQLConnection &conn, const ObTruncateTabletArg &arg, const char *buf, const int64_t buf_len) { int ret = OB_SUCCESS; - ObTimeoutCtx ctx; - const int64_t default_timeout_ts = GCONF.rpc_timeout; - if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, default_timeout_ts))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(default_timeout_ts)); + const int64_t start_time = ObTimeUtility::current_time(); + if (OB_FAIL(conn.register_multi_data_source(transaction::ObTxDataSourceType::SYNC_TRUNCATE_INFO, + buf, buf_len))) { + LOG_WARN("fail to register_tx_data", KR(ret), K(arg), K(buf), K(buf_len)); } else { - const int64_t start_time = ObTimeUtility::current_time(); - do { - if (ctx.is_timeouted()) { - ret = OB_TIMEOUT; - LOG_WARN("already timeout", KR(ret), K(ctx)); - } else if (OB_FAIL(conn.register_multi_data_source(arg.ls_id_, - transaction::ObTxDataSourceType::SYNC_TRUNCATE_INFO, - buf, buf_len))) { - if (need_retry_errno(ret)) { - LOG_INFO("fail to register_tx_data, try again", KR(ret), K(1UL), K(arg)); - ob_usleep(SLEEP_INTERVAL); - } else { - LOG_WARN("fail to register_tx_data", KR(ret), K(arg), K(buf), K(buf_len)); - } - } - } while (need_retry_errno(ret)); - if (OB_SUCC(ret)) { - LOG_INFO("[TRUNCATE_INFO] success to register mds", KR(ret), K(buf_len), K(arg), - "cost_ts", ObTimeUtility::current_time() - start_time); - } + LOG_INFO("[TRUNCATE_INFO] success to register mds", KR(ret), K(buf_len), K(arg), + "cost_ts", ObTimeUtility::current_time() - start_time); } return ret; } @@ -999,10 +979,5 @@ int ObTruncateInfoService::loop_subpart_to_register_mds_( return ret; } -bool ObTruncateInfoService::need_retry_errno(int ret) -{ - return is_location_service_renew_error(ret) || OB_NOT_MASTER == ret; -} - } // namespace rootserver } // namespace oceanbase diff --git a/src/rootserver/truncate_info/ob_truncate_info_service.h b/src/rootserver/truncate_info/ob_truncate_info_service.h index 91585dc18..d1e62854c 100644 --- a/src/rootserver/truncate_info/ob_truncate_info_service.h +++ b/src/rootserver/truncate_info/ob_truncate_info_service.h @@ -139,20 +139,17 @@ struct ObTruncateInfoService final int register_mds_( observer::ObInnerSQLConnection &conn, const ObTruncateTabletArg &arg); - int retry_register_mds_( - observer::ObInnerSQLConnection &conn, - const ObTruncateTabletArg &arg, - const char *buf, - const int64_t buf_len); - static bool need_retry_errno(const int ret); - static const int64_t SLEEP_INTERVAL = 100 * 1000L; // 100ms + int register_mds_( + observer::ObInnerSQLConnection &conn, + const ObTruncateTabletArg &arg, + const char *buf, + const int64_t buf_len); private: ObArenaAllocator allocator_; // for part_key_info_, only init once ObArenaAllocator loop_allocator_; // for loop index tablets const obcall::ObAlterTableArg &arg_; const share::schema::ObTableSchema &data_table_schema_; ObSEArray index_tablet_array_; - ObSEArray ls_id_array_; ObTruncatePartKeyInfo part_key_info_; int64_t ddl_task_id_; bool is_inited_; diff --git a/src/rootserver/truncate_info/ob_truncate_tablet_arg.h b/src/rootserver/truncate_info/ob_truncate_tablet_arg.h index cceb18e14..f5b8fcc4c 100644 --- a/src/rootserver/truncate_info/ob_truncate_tablet_arg.h +++ b/src/rootserver/truncate_info/ob_truncate_tablet_arg.h @@ -16,7 +16,6 @@ #ifndef OB_ROOTSERVER_TRUNCATE_INFO_TRUNCATE_TABLET_ARG_H_ #define OB_ROOTSERVER_TRUNCATE_INFO_TRUNCATE_TABLET_ARG_H_ -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "storage/truncate_info/ob_truncate_info.h" @@ -29,19 +28,17 @@ struct ObTruncateTabletArg { public: ObTruncateTabletArg() - : version_(TRUNCATE_INFO_ARG_VERSION_V1), + : version_(TRUNCATE_INFO_ARG_VERSION_V2), reserved_(0), - ls_id_(), index_tablet_id_(), truncate_info_() {} ~ObTruncateTabletArg() { destroy(); } void destroy() { - ls_id_.reset(); index_tablet_id_.reset(); truncate_info_.destroy(); } - bool is_valid() const { return ls_id_.is_valid() && index_tablet_id_.is_valid() && truncate_info_.is_valid(); } + bool is_valid() const { return index_tablet_id_.is_valid() && truncate_info_.is_valid(); } int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize( @@ -50,8 +47,8 @@ struct ObTruncateTabletArg const int64_t data_len, int64_t &pos); int64_t get_serialize_size() const; - TO_STRING_KV(K_(version), K_(ls_id), K_(index_tablet_id), K_(truncate_info)); - static const int64_t TRUNCATE_INFO_ARG_VERSION_V1 = 1; + TO_STRING_KV(K_(version), K_(index_tablet_id), K_(truncate_info)); + static const int64_t TRUNCATE_INFO_ARG_VERSION_V2 = 2; static const int32_t TIA_ONE_BYTE = 8; static const int32_t TIA_RESERVED_BITS = 56; union { @@ -62,7 +59,6 @@ struct ObTruncateTabletArg uint64_t reserved_ : TIA_RESERVED_BITS; }; }; - share::ObLSID ls_id_; ObTabletID index_tablet_id_; storage::ObTruncateInfo truncate_info_; private: @@ -73,4 +69,3 @@ struct ObTruncateTabletArg } // namespace oceanbase #endif // OB_ROOTSERVER_TRUNCATE_INFO_TRUNCATE_TABLET_ARG_H_ - diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 0fd85f87c..cae895624 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -56,7 +56,6 @@ ob_set_subtarget(ob_share ALONE ob_tenant_mgr.cpp parameter/ob_parameter_attr.cpp ob_ddl_args.cpp - ob_mview_args.cpp ob_lonely_table_clean_rpc_struct.cpp ) @@ -110,9 +109,7 @@ ob_set_subtarget(ob_share common ob_event_history_table_operator.cpp ob_schema_version_info.cpp ob_freeze_info_proxy.cpp - ob_gais_client.cpp ob_gais_msg.cpp - ob_gais_rpc.cpp session/ob_local_session_var.cpp ob_global_autoinc_service.cpp ob_global_merge_table_operator.cpp @@ -125,7 +122,6 @@ ob_set_subtarget(ob_share common ob_list_parser.cpp ob_local_device.cpp ob_locality_info.cpp - ob_ls_id.cpp ob_table_range.cpp ob_max_id_fetcher.cpp ob_max_id_cache.cpp @@ -163,7 +159,6 @@ ob_set_subtarget(ob_share common ob_admin_dump_helper.cpp ob_inner_kv_table_operator.cpp ob_inner_table_operator.cpp - ob_standby_upgrade.cpp ob_common_id.cpp ob_cluster_event_history_table_operator.cpp scn.cpp @@ -182,7 +177,6 @@ ob_set_subtarget(ob_share common_mixed diagnosis/ob_sql_monitor_statname.cpp interrupt/ob_global_interrupt_call.cpp interrupt/ob_interrupt_message.cpp - location_cache/ob_location_service.cpp ls/ob_ls_restore_status.cpp ob_sys_time_zone_util.cpp ls/ob_restore_status.cpp @@ -228,7 +222,6 @@ ob_set_subtarget(ob_share common_mixed storage/ob_rootservice_job_table_storage.cpp storage_cache_policy/ob_storage_cache_common.cpp storage_cache_policy/ob_storage_cache_partition_sql_helper.cpp - partition_table/ob_partition_location.cpp longops_mgr/ob_i_longops.cpp longops_mgr/ob_longops_mgr.cpp ob_module_data_arg.cpp @@ -237,13 +230,11 @@ ob_set_subtarget(ob_share common_mixed ob_set_subtarget(ob_share tablet tablet/ob_tablet_info.cpp tablet/ob_tablet_table_operator.cpp - tablet/ob_tablet_to_ls_operator.cpp + tablet/ob_tablet_mapping_operator.cpp tablet/ob_tablet_table_iterator.cpp tablet/ob_tablet_meta_table_storage.cpp - tablet/ob_tablet_to_ls_iterator.cpp tablet/ob_tablet_filter.cpp tablet/ob_tablet_to_table_history_operator.cpp - tablet/ob_tenant_tablet_to_ls_map.cpp ) ob_set_subtarget(ob_share config @@ -327,10 +318,6 @@ ob_set_subtarget(ob_share schema schema/ob_server_schema_service.cpp schema/ob_table_schema.cpp schema/ob_latest_schema_guard.cpp - schema/ob_mlog_info.cpp - schema/ob_mview_info.cpp - schema/ob_mview_refresh_stats_params.cpp - schema/ob_mview_refresh_stats.cpp schema/ob_list_row_values.cpp schema/ob_catalog_schema_struct.cpp schema/ob_catalog_mgr.cpp diff --git a/src/share/compaction/ob_compaction_time_guard.cpp b/src/share/compaction/ob_compaction_time_guard.cpp index 4f13f32bc..a4458e546 100644 --- a/src/share/compaction/ob_compaction_time_guard.cpp +++ b/src/share/compaction/ob_compaction_time_guard.cpp @@ -140,7 +140,7 @@ uint16_t ObCompactionTimeGuard::get_max_event_count(const ObCompactionTimeGuardT */ const char *ObRSCompactionTimeGuard::CompactionEventStr[] = { "PREPARE_UNFINISH_TABLE_IDS", - "GET_TABLET_LS_PAIRS", + "GET_TABLET_IDS", "GET_TABLET_META_TABLE", "CKM_VERIFICATION" }; diff --git a/src/share/compaction/ob_compaction_time_guard.h b/src/share/compaction/ob_compaction_time_guard.h index 6ae941407..01f14219e 100644 --- a/src/share/compaction/ob_compaction_time_guard.h +++ b/src/share/compaction/ob_compaction_time_guard.h @@ -112,7 +112,7 @@ struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard virtual ~ObRSCompactionTimeGuard() {} enum CompactionEvent : uint16_t { PREPARE_UNFINISH_TABLE_IDS = 0, - GET_TABLET_LS_PAIRS, + GET_TABLET_IDS, GET_TABLET_META_TABLE, CKM_VERIFICATION, COMPACTION_EVENT_MAX, @@ -193,7 +193,7 @@ struct ObSSCompactionTimeGuard : public ObCompactionTimeGuard {} virtual ~ObSSCompactionTimeGuard() {} enum CompactionEvent : uint16_t { - // ls merge scheduler + // tablet merge scheduler GET_SCHEDULE_TABLET, PREPARE_CLOG, UPDATE_TABLET_OBJ, diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 63fff5633..8d28a11ea 100755 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -544,11 +544,9 @@ ('lob_inrow_threshold', 'int', 'false', 'OB_DEFAULT_LOB_INROW_THRESHOLD'), ('auto_increment_cache_size', 'int', 'false', '0'), ('external_properties', 'varbinary:OB_MAX_VARCHAR_LENGTH', 'true'), - ('local_session_vars', 'longtext', 'true'), ('duplicate_read_consistency', 'int', 'false', '0'), ('index_params', 'varchar:OB_MAX_INDEX_PARAMS_LENGTH', 'false', ''), ('micro_index_clustered', 'bool', 'false', 'false'), - ('mv_mode', 'int', 'false', '0'), ('parser_properties', 'longtext', 'false', ''), ('enable_macro_block_bloom_filter', 'bool', 'false', 'false'), ('storage_cache_policy', 'varchar:OB_MAX_VARCHAR_LENGTH', 'false', r'{\"GLOBAL\":\"AUTO\"}'), @@ -2471,11 +2469,11 @@ ] ) -# 342: __all_ls_meta_table (abandoned) +# 342: legacy ls meta table (abandoned) def_table_schema( owner = 'yanmu.ztl', - table_name = '__all_tablet_to_ls', + table_name = '__all_tablet_to_table', table_id = '343', table_type = 'SYSTEM_TABLE', gm_columns = ['gmt_create', 'gmt_modified'], @@ -2492,16 +2490,16 @@ # 344: __all_tablet_meta_table # migrated to SQLite, see gen_sqlite_table_def above # Placeholder - original definition removed, using SQLite version -# 345: __all_ls_status (abandoned) +# 345: legacy ls status table (abandoned) # 346: __all_zone_v2 # abandoned in 4.0 # 348: __all_log_archive_progress # abandoned # 349: __all_log_archive_history # abandoned # 350: __all_log_archive_piece_files # abandoned -# 351: __all_ls_log_archive_progress # abandoned +# 351: legacy ls log archive progress table # abandoned -# 352: __all_ls (abandoned) +# 352: legacy ls table (abandoned) # 353: abandoned # 354: __all_backup_storage_info # abandoned # 357: __all_backup_job # abandoned @@ -2535,7 +2533,7 @@ ] ) -# 370: __all_ls_recovery_stat (abandoned) +# 370: legacy ls recovery stat table (abandoned) # 371: __all_backup_ls_task_info_history # abandoned # 372: __all_tablet_replica_checksum # migrated to SQLite, see gen_sqlite_table_def above @@ -2562,7 +2560,7 @@ ] ) -# 374: __all_ls_replica_task (abandoned) +# 374: legacy ls replica task table (abandoned) def_table_schema( owner = 'lixinze.lxz', @@ -2579,7 +2577,6 @@ ('gtrid', 'varbinary:128'), ('bqual', 'varbinary:128'), ('format_id', 'int', 'false', '1'), - ('coordinator', 'int'), ('scheduler_ip', 'varchar:OB_MAX_SERVER_ADDR_SIZE'), ('scheduler_port', 'int'), ('state', 'int'), @@ -2750,7 +2747,7 @@ def_table_schema(**gen_history_table_def(382, all_context_def)) # 383: __all_global_context_value (abandoned) -# 385: __all_ls_election_reference_info (abandoned) +# 385: legacy ls election reference info table (abandoned) # backup clean inner table # 386: __all_backup_delete_job # abandoned @@ -2786,8 +2783,8 @@ # 398:__all_spm_config abandoned # 399: abandoned # 400:__all_backup_parameter abandoned -# 401: __all_ls_restore_progress (abandoned) -# 402: __all_ls_restore_history (abandoned) +# 401: legacy ls restore progress table (abandoned) +# 402: legacy ls restore history table (abandoned) # 403: __all_backup_storage_info_history (abandoned) # 404: __all_backup_delete_policy (abandoned) @@ -2878,7 +2875,7 @@ # 419-422: abandoned metadata tables # 429: __all_arbitration_service (abandoned) -# 430: __all_ls_arb_replica_task (abandoned) +# 430: legacy ls arb replica task table (abandoned) def_table_schema( owner = 'bohou.ws', @@ -2897,7 +2894,7 @@ is_cluster_private = False ) -# 432: __all_ls_arb_replica_task_history (abandoned) +# 432: legacy ls arb replica task history table (abandoned) def_table_schema( owner = 'luofan.zp', @@ -2932,7 +2929,7 @@ # Placeholder - original definition removed, using SQLite version # 445: __all_cluster_event_history # migrated to SQLite, see gen_sqlite_table_def above -# 447 : __all_ls_log_restore_stat +# 447 : legacy ls log restore stat table # 449 : __all_wait_for_partition_split_tablet # 450: __all_external_table_file # abandoned in seekdb @@ -3009,196 +3006,6 @@ # 461: __all_tenant_snapshot_ls (abandoned) # 462: __all_tenant_snapshot_ls_replica (abandoned) -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mlog', - table_id = '463', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('mlog_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('purge_mode', 'int'), - ('purge_start', 'timestamp', 'true'), - ('purge_next', 'varchar:OB_MAX_FUNC_EXPR_LENGTH', 'true'), - ('purge_job', 'varchar:OB_MAX_SCHEDULER_JOB_NAME_LENGTH', 'true'), - ('last_purge_scn', 'uint', 'true'), - ('last_purge_date', 'timestamp', 'true'), - ('last_purge_time', 'int', 'true'), - ('last_purge_rows', 'int', 'true'), - ('last_purge_trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'true'), - ('schema_version', 'int') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview', - table_id = '464', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('mview_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('build_mode', 'int'), - ('refresh_mode', 'int'), - ('refresh_method', 'int'), - ('refresh_start', 'timestamp', 'true'), - ('refresh_next', 'varchar:OB_MAX_FUNC_EXPR_LENGTH', 'true'), - ('refresh_job', 'varchar:OB_MAX_SCHEDULER_JOB_NAME_LENGTH', 'true'), - ('last_refresh_scn', 'uint', 'true'), - ('last_refresh_type', 'int', 'true'), - ('last_refresh_date', 'timestamp', 'true'), - ('last_refresh_time', 'int', 'true'), - ('last_refresh_trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'true'), - ('schema_version', 'int'), - ('refresh_dop', 'int', 'false', '0'), - ('data_sync_scn', 'uint', 'false', '0'), - ('is_synced', 'bool', 'false', '0'), - ('nested_refresh_mode', 'int', 'false', '0') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_stats_sys_defaults', - table_id = '465', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('collection_level', 'int'), - ('retention_period', 'int') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_stats_params', - table_id = '466', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('mview_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('collection_level', 'int'), - ('retention_period', 'int') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_run_stats', - table_id = '467', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('refresh_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('run_user_id', 'int'), - ('num_mvs_total', 'int'), - ('num_mvs_current', 'int'), - ('mviews', 'varchar:4000'), - ('base_tables', 'varchar:4000', 'true'), - ('method', 'varchar:4000', 'true'), - ('rollback_seg', 'varchar:4000', 'true'), - ('push_deferred_rpc', 'bool'), - ('refresh_after_errors', 'bool'), - ('purge_option', 'int'), - ('parallelism', 'int'), - ('heap_size', 'int'), - ('atomic_refresh', 'bool'), - ('nested', 'bool'), - ('out_of_place', 'bool'), - ('number_of_failures', 'int'), - ('start_time', 'timestamp'), - ('end_time', 'timestamp'), - ('elapsed_time', 'int'), - ('log_purge_time', 'int'), - ('complete_stats_avaliable', 'bool'), - ('trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'true') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_stats', - table_id = '468', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('refresh_id', 'int'), - ('mview_id', 'int'), - ('retry_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('refresh_type', 'int'), - ('start_time', 'timestamp'), - ('end_time', 'timestamp'), - ('elapsed_time', 'int'), - ('log_purge_time', 'int'), - ('initial_num_rows', 'int'), - ('final_num_rows', 'int'), - ('num_steps', 'int'), - ('result', 'int') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_change_stats', - table_id = '469', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('refresh_id', 'int'), - ('mview_id', 'int'), - ('retry_id', 'int'), - ('detail_table_id', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('num_rows_ins', 'int', 'true'), - ('num_rows_upd', 'int', 'true'), - ('num_rows_del', 'int', 'true'), - ('num_rows', 'int', 'true') - ] -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = '__all_mview_refresh_stmt_stats', - table_id = '470', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('refresh_id', 'int'), - ('mview_id', 'int'), - ('retry_id', 'int'), - ('step', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('sqlid', 'varchar:OB_MAX_SQL_ID_LENGTH', 'true'), - ('stmt', 'longtext'), - ('execution_time', 'int'), - ('execution_plan', 'longtext', 'true'), - ('result', 'int') - ] -) - def_table_schema( owner = 'yangyifei.yyf', table_name = '__all_dbms_lock_allocated', @@ -3445,32 +3252,13 @@ def_table_schema(**gen_history_table_def(506, all_column_privilege_def)) # 507: __all_tenant_snapshot_ls_replica_history (abandoned) -# 508: __all_ls_replica_task_history (abandoned) -# 509 : __all_ls_compaction_status +# 508: legacy ls replica task history table (abandoned) +# 509 : legacy ls compaction status table # 510 : __all_tablet_compaction_status # 511 : __all_tablet_checksum_error_info (abandoned) # 516 : __all_service (abandoned) # 517: __all_storage_io_usage (abandoned) -def_table_schema( - owner = 'yuya.yu', - table_name = '__all_mview_dep', - table_id = '518', - table_type = 'SYSTEM_TABLE', - gm_columns = ['gmt_create', 'gmt_modified'], - rowkey_columns = [ - ('mview_id', 'int'), - ('p_order', 'int') - ], - in_tenant_space = True, - normal_columns = [ - ('p_obj', 'int'), - ('p_type', 'int'), - ('qbcid', 'int'), - ('flags', 'int') - ] -) - def_table_schema( owner = 'fyy280124', table_name = '__all_scheduler_job_run_detail_v2', @@ -4670,12 +4458,10 @@ in_tenant_space = True, normal_columns = [ - ('trans_type', 'int'), ('trans_id', 'int'), ('session_id', 'int'), - ('scheduler_addr', 'varchar:64'), ('is_decided', 'bool'), - ('participants', 'varchar:1024'), + ('write_state', 'varchar:1024'), ('ctx_create_time', 'timestamp', 'true'), ('expired_time', 'timestamp', 'true'), ('ref_cnt', 'int'), @@ -4684,12 +4470,9 @@ ('state', 'int'), ('part_trans_action', 'int'), ('trans_ctx_addr', 'varchar:20'), - ('mem_ctx_id', 'int'), ('pending_log_size', 'int'), ('flushed_log_size', 'int'), - ('role', 'int'), ('is_exiting', 'int'), - ('coordinator', 'int'), ('last_request_time', 'timestamp', 'true'), ('gtrid', 'varbinary:128'), ('bqual', 'varbinary:128'), @@ -4714,10 +4497,10 @@ ], normal_columns = [ - ('is_master', 'int'), ('is_stopped', 'int'), - ('state', 'int'), - ('state_str', 'varchar:64'), + ('block_tx', 'int'), + ('block_normal_tx', 'int'), + ('block_all', 'int'), ('total_trans_ctx_count', 'int'), ('mgr_addr', 'bigint:20') ], vtable_route_policy = 'local' @@ -4734,11 +4517,10 @@ normal_columns = [ ('session_id', 'int'), - ('trans_id', 'int'), - ('state', 'int'), - ('cluster_id', 'int'), - ('coordinator', 'int'), - ('participants', 'varchar:1024', 'true'), + ('trans_id', 'int'), + ('state', 'int'), + ('cluster_id', 'int'), + ('write_state', 'varchar:1024', 'true'), ('isolation_level', 'int'), ('snapshot_version', 'uint', 'true'), ('access_mode', 'int'), @@ -4971,7 +4753,7 @@ # 11069: __all_virtual_leader_stat # abandoned in 4.0 -# 11070: __all_virtual_partition_migration_status # abandoned in 4.0 +# 11070: abandoned in 4.0 def_table_schema( @@ -5112,11 +4894,9 @@ in_tenant_space = True, normal_columns = [ ('trans_id', 'int'), - ('table_id', 'int'), ('tablet_id', 'int'), ('rowkey', 'varchar:512', 'true'), ('session_id', 'int'), - ('proxy_session_id', 'varchar:512'), ('ctx_create_time', 'timestamp', 'true'), ('expired_time', 'timestamp', 'true'), ('time_after_recv', 'int'), @@ -5803,20 +5583,13 @@ ('total_update_cnt', 'int'), ('trans_id', 'int'), ('holder_trans_id', 'int'), - ('holder_session_id', 'int'), - ('assoc_session_id', 'int'), - ('wait_timeout', 'int'), - ('tx_active_ts', 'int'), - ('node_id', 'int'), - ('node_type', 'int'), - ('remote_addr', 'varchar:MAX_LOCK_REMOTE_ADDR_BUF_LENGTH'), - ('is_placeholder', 'int') + ('holder_session_id', 'int') ], vtable_route_policy = 'local' ) # 12014: __all_virtual_partition_item # abandoned in 4.0 -# 12015: __all_virtual_replica_task # abandoned in 4.0 +# 12015: abandoned in 4.0 # 12016: __all_virtual_partition_location # abandoned in 4.0 # 12030: proc # abandoned in 4.2.5.1, replaced by 21628 @@ -6436,8 +6209,7 @@ ('rec_log_scn', 'uint'), ('latest_log_scn', 'uint'), ('pre_allocated_range', 'int'), - ('submit_log_ts', 'int'), - ('is_master', 'bool') + ('submit_log_ts', 'int') ], vtable_route_policy = 'local' ) @@ -6779,7 +6551,7 @@ table_name = '__all_virtual_tablet_meta_table', keywords = all_def_keywords['__all_tablet_meta_table'])) -# 12241: __all_virtual_tablet_to_ls # removed (single-tenant: iterate VT mechanism deleted) +# 12241: __all_virtual_tablet_to_table # removed (single-tenant: iterate VT mechanism deleted) def_table_schema( owner = 'yuya.yu', @@ -6844,23 +6616,14 @@ ], normal_columns = [ - ('role', 'varchar:32'), - ('proposal_id', 'int'), - ('config_version', 'varchar:128'), ('access_mode', 'varchar:32'), - ('paxos_member_list', 'varchar:1024'), - ('paxos_replica_num', 'int'), - ('in_sync', 'bool'), ('base_lsn', 'uint'), ('begin_lsn', 'uint'), ('begin_scn', 'uint'), ('end_lsn', 'uint'), ('end_scn', 'uint'), ('max_lsn', 'uint'), - ('max_scn', 'uint'), - ('arbitration_member', 'varchar:128'), - ('degraded_list', 'varchar:1024'), - ('learner_list', 'longtext') + ('max_scn', 'uint') ], vtable_route_policy = 'local' ) @@ -7015,9 +6778,7 @@ ], normal_columns = [ - ('role', 'varchar:32'), ('end_lsn', 'uint'), - ('proposal_id', 'int'), ('pending_cnt', 'int') ], vtable_route_policy = 'local' ) @@ -7033,7 +6794,6 @@ ], normal_columns = [ - ('role', 'varchar:32'), ('end_lsn', 'uint'), ('enabled', 'bool'), ('unsubmitted_lsn', 'uint'), @@ -7061,22 +6821,12 @@ in_tenant_space = True, normal_columns = [ - ('replica_type', 'varchar:MAX_REPLICA_TYPE_LENGTH'), - ('ls_state', 'varchar:MAX_LS_STATE_LENGTH'), ('tablet_count', 'int'), ('weak_read_scn', 'uint'), - ('need_rebuild', 'varchar:MAX_COLUMN_YES_NO_LENGTH'), ('checkpoint_scn', 'uint'), ('checkpoint_lsn', 'uint'), - ('migrate_status', 'int'), - ('rebuild_seq', 'int'), ('tablet_change_checkpoint_scn', 'uint'), - ('reserved_scn', 'uint'), - ('tx_blocked', 'int'), - ('required_data_disk_size', 'int', 'false', 0), - ('mv_major_merge_scn', 'uint', 'false', 0), - ('mv_publish_scn', 'uint', 'false', 0), - ('mv_safe_scn', 'uint', 'false', 0) + ('tx_blocked', 'int') ], vtable_route_policy = 'local' ) @@ -7512,33 +7262,14 @@ ], normal_columns = [ - ('election_role', 'varchar:32'), - ('election_epoch', 'int'), - ('palf_role', 'varchar:32'), ('palf_state', 'varchar:32'), - ('palf_proposal_id', 'int'), - ('log_handler_role', 'varchar:32'), - ('log_handler_proposal_id', 'int'), - ('log_handler_takeover_state', 'varchar:32'), - ('log_handler_takeover_log_type', 'varchar:32'), ('max_applied_scn', 'uint'), ('max_replayed_lsn', 'uint'), ('max_replayed_scn', 'uint'), ('replay_diagnose_info', 'varchar:1024'), - ('gc_state', 'varchar:32'), - ('gc_start_ts', 'int'), - ('archive_scn', 'uint'), ('checkpoint_scn', 'uint'), ('min_rec_scn', 'uint'), ('min_rec_scn_log_type', 'varchar:32'), - ('restore_handler_role', 'varchar:32'), - ('restore_proposal_id', 'int'), - ('restore_context_info', 'varchar:1024'), - ('restore_err_context_info', 'varchar:1024'), - ('enable_sync', 'bool'), - ('enable_vote', 'bool'), - ('arb_srv_info', 'varchar:1024'), - ('parent', 'varchar:1024'), ('readonly_tx', 'varchar:1024') ], vtable_route_policy = 'local' ) @@ -7631,8 +7362,8 @@ ], vtable_route_policy = 'local',) -# 12364: __all_ls_arb_replica_task (abandoned) -# 12365: __all_ls_arb_replica_task_history (abandoned) +# 12364: legacy ls arb replica task table (abandoned) +# 12365: legacy ls arb replica task history table (abandoned) # 12366: __all_virtual_archive_dest_status (removed: backup/restore/log-archive deleted) @@ -7819,10 +7550,7 @@ rowkey_columns = [], normal_columns = [ ('ts_value', 'int'), - ('ts_type', 'varchar:100'), - ('service_role', 'varchar:100'), - ('role', 'varchar:100'), - ('service_epoch', 'int') + ('ts_type', 'varchar:100') ], vtable_route_policy = 'local' ) @@ -7872,22 +7600,6 @@ ] ) -# 12406: __all_virtual_mlog # removed (single-tenant: iterate VT mechanism deleted) - -# 12407: __all_virtual_mview # removed (single-tenant: iterate VT mechanism deleted) - -# 12408: __all_virtual_mview_refresh_stats_sys_defaults # removed (single-tenant: iterate VT mechanism deleted) - -# 12409: __all_virtual_mview_refresh_stats_params # removed (single-tenant: iterate VT mechanism deleted) - -# 12410: __all_virtual_mview_refresh_run_stats # removed (single-tenant: iterate VT mechanism deleted) - -# 12411: __all_virtual_mview_refresh_stats # removed (single-tenant: iterate VT mechanism deleted) - -# 12412: __all_virtual_mview_refresh_change_stats # removed (single-tenant: iterate VT mechanism deleted) - -# 12413: __all_virtual_mview_refresh_stmt_stats # removed (single-tenant: iterate VT mechanism deleted) - # 12414: __all_virtual_wr_control # removed # 12415: __all_virtual_event_history - migrated to SQLite, see gen_sqlite_virtual_table_def above @@ -8246,35 +7958,6 @@ # 12489: __all_virtual_deadlock_detector_stat # 12490: __all_virtual_spatial_reference_systems # removed (single-tenant: iterate VT mechanism deleted) -def_table_schema( - owner = 'wenyue.zxl', - table_name = '__all_virtual_log_transport_dest_stat', - table_id = '12491', - table_type = 'VIRTUAL_TABLE', - gm_columns = [], - rowkey_columns = [ - ], - - in_tenant_space = True, - normal_columns = [ - ('client_ip', 'varchar:MAX_IP_ADDR_LENGTH'), - ('client_pid', 'int'), - ('client_type', 'int'), - ('start_serve_time', 'timestamp'), - ('last_serve_time', 'timestamp'), - ('last_read_source', 'int'), - ('last_request_type', 'int'), - ('last_request_log_lsn', 'uint'), - ('last_request_log_scn', 'uint'), - ('last_failed_request', 'longtext'), - ('avg_request_process_time', 'int'), - ('avg_request_queue_time', 'int'), - ('avg_request_read_log_time', 'int'), - ('avg_request_read_log_size', 'int'), - ('avg_log_transport_bandwidth', 'int') - ], vtable_route_policy = 'local' - ) - # 12492: __all_virtual_ss_local_cache_info abandoned # 12493: __all_virtual_kv_group_commit_status abandoned @@ -8509,26 +8192,6 @@ ], vtable_route_policy = 'local' ) -def_table_schema( - owner = 'zg410411', - table_name = '__all_virtual_mview_running_job', - table_id = '12535', - table_type = 'VIRTUAL_TABLE', - gm_columns = [], - rowkey_columns = [], - in_tenant_space = True, - - normal_columns = [ - ('table_id', 'int'), - ('job_type', 'uint'), - ('session_id', 'uint'), - ('read_snapshot', 'int'), - ('parallel', 'int'), - ('job_start_time', 'timestamp'), - ('target_data_sync_scn', 'uint') - ], vtable_route_policy = 'local' - ) - def_table_schema( owner = 'zhaoziqian.zzq', table_name = '__all_virtual_dynamic_partition_table', @@ -9050,7 +8713,7 @@ when a.table_type = 4 then 'VIEW' when a.table_type = 14 then 'EXTERNAL TABLE' else 'BASE TABLE' end as char(64)) as TABLE_TYPE, - cast(case when a.table_type in (0,3,5,6,7,11,12,13,15) then 'InnoDB' + cast(case when a.table_type in (0,3,5,6,11,12,13) then 'InnoDB' else 'MEMORY' end as char(64)) as ENGINE, cast(NULL as unsigned) as VERSION, cast(a.store_format as char(10)) as ROW_FORMAT, @@ -9130,7 +8793,7 @@ WHERE e.index_type in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 41) and e.table_type = 5 group by data_table_id ) idx_stat on idx_stat.data_table_id = a.table_id - where a.table_type in (0, 1, 2, 3, 4, 14, 15) + where a.table_type in (0, 1, 2, 3, 4, 14) and b.database_name != '__recyclebin' and b.in_recyclebin = 0 and 0 = sys_privilege_check('table_acc', 1, b.database_name, a.table_name) @@ -10807,37 +10470,6 @@ # 21168: DBA_OB_LS_LOCATIONS (abandoned) # 21169: CDB_OB_LS_LOCATIONS (abandoned) -def_table_schema( - owner = 'yanmu.ztl', - table_name = 'DBA_OB_TABLET_TO_LS', - table_id = '21170', - table_type = 'SYSTEM_VIEW', - gm_columns = [], - rowkey_columns = [], - normal_columns = [], - in_tenant_space = True, - view_definition = - """ - ( - SELECT CAST(TABLE_ID AS SIGNED) AS TABLET_ID - FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE - ) - UNION ALL - ( - SELECT CAST(TABLE_ID AS SIGNED) AS TABLET_ID - FROM OCEANBASE.__ALL_TABLE - WHERE ((TABLE_ID > 0 AND TABLE_ID < 10000) - OR (TABLE_ID > 50000 AND TABLE_ID < 70000) - OR (TABLE_ID > 100000 AND TABLE_ID < 200000)) - ) - UNION ALL - ( - SELECT TABLET_ID - FROM OCEANBASE.__ALL_TABLET_TO_LS - ) - """.replace("\n", " ") - ) - def_table_schema( owner = 'yanmu.ztl', table_name = 'DBA_OB_TABLET_REPLICAS', @@ -11135,10 +10767,8 @@ WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' - WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' - WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE - ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' + ,CAST(CASE WHEN TABLE_TYPE IN (5) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' @@ -11383,7 +11013,7 @@ in_tenant_space = True, view_definition = """ SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) OWNER, - CAST(TB.NEW_TABLE_NAME AS CHAR(128)) TABLE_NAME, + CAST(TB.TABLE_NAME AS CHAR(128)) TABLE_NAME, CAST((CASE TB.PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' @@ -11459,34 +11089,7 @@ CAST(NULL AS CHAR(1000)) DEF_INMEMORY_SERVICE_NAME, CAST('NO' AS CHAR(3)) AUTO FROM - (( - SELECT - mv_table.table_name AS new_table_name, - container_table.* - FROM - oceanbase.__all_table AS mv_table, - ( - SELECT * FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 1 << 24 - ) AS container_table - WHERE - mv_table.data_table_id = container_table.table_id - and mv_table.table_type = 7 - ) - - UNION ALL - - ( - SELECT - table_name as new_table_name, - * - FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 0 - )) TB + oceanbase.__all_table TB JOIN OCEANBASE.__ALL_DATABASE DB ON TB.DATABASE_ID = DB.DATABASE_ID JOIN @@ -11705,38 +11308,11 @@ FROM (SELECT DB.DATABASE_NAME, DB.DATABASE_ID, TB.TABLE_ID, - TB.NEW_TABLE_NAME AS TABLE_NAME, + TB.TABLE_NAME AS TABLE_NAME, TB.B_TRANSITION_POINT, TB.PART_LEVEL FROM - (( - SELECT - mv_table.table_name AS new_table_name, - container_table.* - FROM - oceanbase.__all_table AS mv_table, - ( - SELECT * FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 1 << 24 - ) AS container_table - WHERE - mv_table.data_table_id = container_table.table_id - and mv_table.table_type = 7 - ) - - UNION ALL - - ( - SELECT - table_name as new_table_name, - * - FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 0 - )) TB, + oceanbase.__all_table TB, OCEANBASE.__ALL_DATABASE DB WHERE TB.DATABASE_ID = DB.DATABASE_ID AND TB.TABLE_TYPE in (3, 6) @@ -11833,36 +11409,9 @@ (SELECT DB.DATABASE_NAME, DB.DATABASE_ID, TB.TABLE_ID, - TB.NEW_TABLE_NAME AS TABLE_NAME + TB.TABLE_NAME AS TABLE_NAME FROM - (( - SELECT - mv_table.table_name AS new_table_name, - container_table.* - FROM - oceanbase.__all_table AS mv_table, - ( - SELECT * FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 1 << 24 - ) AS container_table - WHERE - mv_table.data_table_id = container_table.table_id - and mv_table.table_type = 7 - ) - - UNION ALL - - ( - SELECT - table_name as new_table_name, - * - FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 0 - )) TB, + oceanbase.__all_table TB, OCEANBASE.__ALL_DATABASE DB WHERE TB.DATABASE_ID = DB.DATABASE_ID AND TB.TABLE_MODE >> 12 & 15 in (0,1) @@ -12034,7 +11583,7 @@ I.PART_FUNC_TYPE, I.PART_NUM, I.SUB_PART_FUNC_TYPE, - T.NEW_TABLE_NAME AS TABLE_NAME, + T.TABLE_NAME AS TABLE_NAME, T.SUB_PART_NUM, T.SUB_PART_TEMPLATE_FLAGS, T.TABLESPACE_ID, @@ -12057,34 +11606,7 @@ ELSE I.TABLE_ID END) AS JOIN_TABLE_ID FROM OCEANBASE.__ALL_TABLE I JOIN - (( - SELECT - mv_table.table_name AS new_table_name, - container_table.* - FROM - oceanbase.__all_table AS mv_table, - ( - SELECT * FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 1 << 24 - ) AS container_table - WHERE - mv_table.data_table_id = container_table.table_id - and mv_table.table_type = 7 - ) - - UNION ALL - - ( - SELECT - table_name as new_table_name, - * - FROM - oceanbase.__all_table - WHERE - (table_mode & 1 << 24) = 0 - )) T + oceanbase.__all_table T ON I.DATA_TABLE_ID = T.TABLE_ID JOIN OCEANBASE.__ALL_DATABASE D ON T.DATABASE_ID = D.DATABASE_ID @@ -12519,11 +12041,11 @@ # 21224: V$OB_KVCACHE # removed (single-tenant GV/V collapse; use oceanbase.__all_virtual_kvcache_info) -# 21225: GV$OB_TRANSACTION_PARTICIPANTS # removed (single-tenant GV/V collapse; folded into V$OB_TRANSACTION_PARTICIPANTS) +# 21225: GV$OB_TRANSACTION_WRITE_STATE # removed (single-tenant GV/V collapse; folded into V$OB_TRANSACTION_WRITE_STATE) def_table_schema( owner = 'gjw228474', - table_name = 'V$OB_TRANSACTION_PARTICIPANTS', + table_name = 'V$OB_TRANSACTION_WRITE_STATE', table_id = '21226', table_type = 'SYSTEM_VIEW', rowkey_columns = [], @@ -12532,20 +12054,8 @@ in_tenant_space = True, view_definition = """SELECT session_id AS SESSION_ID, - scheduler_addr AS SCHEDULER_ADDR, - CASE - WHEN part_trans_action >= 3 AND trans_type = 0 - THEN 'LOCAL' - WHEN part_trans_action >= 3 AND trans_type = 2 - THEN 'DISTRIBUTED' - WHEN trans_type = 0 and state = 10 - THEN 'UNDECIDED' - WHEN trans_type = 0 - THEN 'LOCAL' - ELSE 'DISTRIBUTED' - END AS TX_TYPE, trans_id AS TX_ID, - participants AS PARTICIPANTS, + write_state AS WRITE_STATE, ctx_create_time AS CTX_CREATE_TIME, expired_time AS TX_EXPIRED_TIME, CASE @@ -12570,11 +12080,6 @@ END AS CHAR(10)) AS ACTION, pending_log_size AS PENDING_LOG_SIZE, flushed_log_size AS FLUSHED_LOG_SIZE, - CASE - WHEN role = 0 THEN 'LEADER' - ELSE 'FOLLOWER' - END AS ROLE, - COORDINATOR AS COORD, LAST_REQUEST_TIME, FORMAT_ID AS FORMATID, HEX(GTRID) AS GLOBALID, @@ -12750,7 +12255,7 @@ 'TABLE' AS OBJECT_TYPE FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE - WHERE TABLE_TYPE IN (0,2,3,6,14,15) + WHERE TABLE_TYPE IN (0,2,3,6,14) UNION ALL SELECT DATABASE_ID, TABLE_ID, @@ -12763,7 +12268,7 @@ 'TABLE' AS OBJECT_TYPE FROM oceanbase.__all_table T - WHERE T.TABLE_TYPE IN (0,2,3,6,14,15) + WHERE T.TABLE_TYPE IN (0,2,3,6,14) AND T.TABLE_MODE >> 12 & 15 in (0,1) AND T.INDEX_ATTRIBUTES_SET & 16 = 0) UNION ALL @@ -12781,7 +12286,7 @@ JOIN oceanbase.__all_part P ON T.TABLE_ID = P.TABLE_ID - WHERE T.TABLE_TYPE IN (0,2,3,6,14,15) + WHERE T.TABLE_TYPE IN (0,2,3,6,14) AND T.TABLE_MODE >> 12 & 15 in (0,1) AND (P.PARTITION_TYPE = 0 OR P.PARTITION_TYPE IS NULL) AND T.INDEX_ATTRIBUTES_SET & 16 = 0 @@ -12804,7 +12309,7 @@ oceanbase.__all_sub_part SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID - WHERE T.TABLE_TYPE IN (0,2,3,6,14,15) + WHERE T.TABLE_TYPE IN (0,2,3,6,14) AND T.TABLE_MODE >> 12 & 15 in (0,1) AND (P.PARTITION_TYPE = 0 OR P.PARTITION_TYPE IS NULL) AND (SP.PARTITION_TYPE = 0 OR SP.PARTITION_TYPE IS NULL) @@ -13697,28 +13202,16 @@ rowkey_columns = [], normal_columns = [], in_tenant_space = True, - view_definition = """ + view_definition = """ SELECT - ROLE, - PROPOSAL_ID, - CONFIG_VERSION, ACCESS_MODE, - PAXOS_MEMBER_LIST, - PAXOS_REPLICA_NUM, - CASE in_sync - WHEN 1 THEN 'YES' - ELSE 'NO' END - AS IN_SYNC, BASE_LSN, BEGIN_LSN, BEGIN_SCN, END_LSN, END_SCN, MAX_LSN, - MAX_SCN, - ARBITRATION_MEMBER, - DEGRADED_LIST, - LEARNER_LIST + MAX_SCN FROM oceanbase.__all_virtual_log_stat """.replace("\n", " ") ) @@ -15069,20 +14562,13 @@ WHEN state = 5 THEN 'IN_TERMINATE' WHEN state = 6 THEN 'ABORTED' WHEN state = 7 THEN 'ROLLED_BACK' - WHEN state = 8 THEN 'COMMIT_TIMEOUT' - WHEN state = 9 THEN 'COMMIT_UNKNOWN' - WHEN state = 10 THEN 'COMMITTED' - WHEN state = 11 THEN 'SUB_PREPARING' - WHEN state = 12 THEN 'SUB_PREPARED' - WHEN state = 13 THEN 'SUB_COMMITTING' - WHEN state = 14 THEN 'SUB_COMMITTED' - WHEN state = 15 THEN 'SUB_ROLLBACKING' - WHEN state = 16 THEN 'SUB_ROLLBACKED' - ELSE 'UNKNOWN' - END AS STATE, - cluster_id AS CLUSTER_ID, - coordinator AS COORDINATOR, - participants AS PARTICIPANTS, + WHEN state = 8 THEN 'COMMIT_TIMEOUT' + WHEN state = 9 THEN 'COMMIT_UNKNOWN' + WHEN state = 10 THEN 'COMMITTED' + ELSE 'UNKNOWN' + END AS STATE, + cluster_id AS CLUSTER_ID, + write_state AS WRITE_STATE, CASE WHEN isolation_level = -1 THEN 'INVALID' WHEN isolation_level = 0 THEN 'READ UNCOMMITTED' @@ -15310,7 +14796,7 @@ FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END -WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) +WHERE T.TABLE_TYPE IN (3,6,8,9,14) AND (P.PARTITION_TYPE = 0 OR P.PARTITION_TYPE is NULL) AND (SP.PARTITION_TYPE = 0 OR SP.PARTITION_TYPE is NULL) AND (0 = sys_privilege_check('table_acc', 1) @@ -15575,13 +15061,13 @@ SELECT OBJ_LOCK.CREATE_TRANS_ID AS TRANS_ID, - TRX_PART.SESSION_ID AS SESSION_ID, + TRX_WRITE_PART.SESSION_ID AS SESSION_ID, CASE WHEN OBJ_LOCK.OBJ_TYPE IN ('TABLE', 'TABLET') THEN 'TM' WHEN OBJ_LOCK.OBJ_TYPE = 'DBMS_LOCK' THEN 'UL' ELSE 'UNKONWN' END AS TYPE, OBJ_LOCK.CREATE_TRANS_ID AS ID1, - TRX_PART.SESSION_ID AS ID2, + TRX_WRITE_PART.SESSION_ID AS ID2, OBJ_LOCK.OBJ_ID AS ID3, OBJ_LOCK.LOCK_MODE AS LMODE, 'NONE' AS REQUEST, @@ -15590,9 +15076,9 @@ FROM oceanbase.__ALL_VIRTUAL_OBJ_LOCK AS OBJ_LOCK LEFT JOIN - oceanbase.V$OB_TRANSACTION_PARTICIPANTS TRX_PART + oceanbase.V$OB_TRANSACTION_WRITE_STATE TRX_WRITE_PART ON - TRX_PART.TX_ID = OBJ_LOCK.CREATE_TRANS_ID + TRX_WRITE_PART.TX_ID = OBJ_LOCK.CREATE_TRANS_ID WHERE OBJ_LOCK.OBJ_TYPE IN ('TABLE', 'TABLET', 'DBMS_LOCK') AND OBJ_LOCK.EXTRA_INFO LIKE '%tx_ctx%' @@ -15618,22 +15104,9 @@ TS_VALUE FROM oceanbase.__all_virtual_timestamp_service as a - WHERE - ROLE = 'LEADER' AND SERVICE_EPOCH = - (SELECT MAX(SERVICE_EPOCH) FROM - oceanbase.__all_virtual_timestamp_service) - GROUP BY TS_TYPE, TS_VALUE """.replace("\n", " ") ) -# 21405: DBA_OB_BALANCE_JOBS (abandoned) -# 21406: CDB_OB_BALANCE_JOBS (abandoned) -# 21407: DBA_OB_BALANCE_JOB_HISTORY (abandoned) -# 21408: CDB_OB_BALANCE_JOB_HISTORY (abandoned) -# 21409: DBA_OB_BALANCE_TASKS (abandoned) -# 21410: CDB_OB_BALANCE_TASKS (abandoned) -# 21411: DBA_OB_BALANCE_TASK_HISTORY (abandoned) -# 21412: CDB_OB_BALANCE_TASK_HISTORY (abandoned) # 21417: DBA_OB_EXTERNAL_TABLE_FILES # abandoned in seekdb @@ -15840,23 +15313,6 @@ # 21426: DBA_DB_LINKS # abandoned in seekdb -# 21427: CDB_OB_MLOGS # abandoned in 4.3 -# 21428: CDB_OB_MVIEWS # abandoned in 4.3 -# 21429: CDB_OB_MVIEW_REFRESH_STATS_SYS_DEFAULTS # abandoned in 4.3 -# 21430: CDB_OB_MVIEW_REFRESH_STATS_PARAMS # abandoned in 4.3 -# 21431: CDB_OB_MVIEW_REFRESH_RUN_STATS # abandoned in 4.3 -# 21432: CDB_OB_MVIEW_REFRESH_STATS # abandoned in 4.3 -# 21433: CDB_OB_MVIEW_REFRESH_CHANGE_STATS # abandoned in 4.3 -# 21434: CDB_OB_MVIEW_REFRESH_STMT_STATS # abandoned in 4.3 -# 21435: DBA_OB_MLOGS # abandoned in 4.3 -# 21436: DBA_OB_MVIEWS # abandoned in 4.3 -# 21437: DBA_OB_MVIEW_REFRESH_STATS_SYS_DEFAULTS # abandoned in 4.3 -# 21438: DBA_OB_MVIEW_REFRESH_STATS_PARAMS # abandoned in 4.3 -# 21439: DBA_OB_MVIEW_REFRESH_RUN_STATS # abandoned in 4.3 -# 21440: DBA_OB_MVIEW_REFRESH_STATS # abandoned in 4.3 -# 21441: DBA_OB_MVIEW_REFRESH_CHANGE_STATS # abandoned in 4.3 -# 21442: DBA_OB_MVIEW_REFRESH_STMT_STATS # abandoned in 4.3 - # 21443: DBA_WR_CONTROL # removed # 21444: CDB_WR_CONTROL # 21444: CDB_WR_CONTROL # removed @@ -16273,426 +15729,6 @@ # 21524: CDB_OB_LS_REPLICA_TASK_HISTORY (abandoned) # 21522: CDB_UNUSED_COL_TABS -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVIEW_LOGS', - table_id = '21526', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(A.DATABASE_NAME AS CHAR(128)) AS LOG_OWNER, - CAST(D.TABLE_NAME AS CHAR(128)) AS MASTER, - CAST(B.TABLE_NAME AS CHAR(128)) AS LOG_TABLE, - CAST(NULL AS CHAR(128)) AS LOG_TRIGGER, - CAST(IF(D.TABLE_MODE & 66048 = 66048, 'YES', 'NO') AS CHAR(3)) AS ROWIDS, - CAST(IF(D.TABLE_MODE & 66048 = 0, 'YES', 'NO') AS CHAR(3)) AS PRIMARY_KEY, - CAST('NO' AS CHAR(3)) AS OBJECT_ID, - CAST( - IF(( - SELECT COUNT(*) - FROM oceanbase.__all_column C1, - oceanbase.__all_column C2 - WHERE - B.TABLE_ID = C1.TABLE_ID - AND C1.COLUMN_ID >= 16 - AND C1.COLUMN_ID < 65520 - AND D.TABLE_ID = C2.TABLE_ID - AND C2.ROWKEY_POSITION != 0 - AND C1.COLUMN_ID != C2.COLUMN_ID - ) = 0, 'NO', 'YES') AS CHAR(3) - ) AS FILTER_COLUMNS, - CAST('YES' AS CHAR(3)) AS SEQUENCE, - CAST('YES' AS CHAR(3)) AS INCLUDE_NEW_VALUES, - CAST(IF(C.PURGE_MODE = 1, 'YES', 'NO') AS CHAR(3)) AS PURGE_ASYNCHRONOUS, - CAST(IF(C.PURGE_MODE = 2, 'YES', 'NO') AS CHAR(3)) AS PURGE_DEFERRED, - CAST(C.PURGE_START AS DATETIME) AS PURGE_START, - CAST(C.PURGE_NEXT AS CHAR(200)) AS PURGE_INTERVAL, - CAST(C.LAST_PURGE_DATE AS DATETIME) AS LAST_PURGE_DATE, - CAST(0 AS SIGNED) AS LAST_PURGE_STATUS, - C.LAST_PURGE_ROWS AS NUM_ROWS_PURGED, - CAST('YES' AS CHAR(3)) AS COMMIT_SCN_BASED, - CAST('NO' AS CHAR(3)) AS STAGING_LOG, - B.DOP AS PURGE_DOP, - C.LAST_PURGE_TIME AS LAST_PURGE_TIME - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - oceanbase.__all_mlog C, - oceanbase.__all_table D - WHERE - A.DATABASE_ID = B.DATABASE_ID - AND B.TABLE_ID = C.MLOG_ID - AND B.TABLE_TYPE = 15 - AND B.DATA_TABLE_ID = D.TABLE_ID -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVIEWS', - table_id = '21528', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, - CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, - CAST(D.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, - B.VIEW_DEFINITION AS QUERY, - CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, - CAST('N' AS CHAR(1)) AS UPDATABLE, - CAST(NULL AS CHAR(128)) AS UPDATE_LOG, - CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, - CAST(NULL AS CHAR(128)) AS MASTER_LINK, - CAST( - CASE ((B.TABLE_MODE >> 27) & 1) - WHEN 0 THEN 'N' - WHEN 1 THEN 'Y' - ELSE NULL - END AS CHAR(1) - ) AS REWRITE_ENABLED, - CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, - CAST( - CASE C.REFRESH_MODE - WHEN 0 THEN 'NEVER' - WHEN 1 THEN 'DEMAND' - WHEN 2 THEN 'COMMIT' - WHEN 3 THEN 'STATEMENT' - WHEN 4 THEN 'MAJOR_COMPACTION' - ELSE NULL - END AS CHAR(32) - ) AS REFRESH_MODE, - CAST( - CASE C.REFRESH_METHOD - WHEN 0 THEN 'NEVER' - WHEN 1 THEN 'COMPLETE' - WHEN 2 THEN 'FAST' - WHEN 3 THEN 'FORCE' - ELSE NULL - END AS CHAR(8) - ) AS REFRESH_METHOD, - CAST( - CASE C.BUILD_MODE - WHEN 0 THEN 'IMMEDIATE' - WHEN 1 THEN 'DEFERRED' - WHEN 2 THEN 'PERBUILT' - ELSE NULL - END AS CHAR(9) - ) AS BUILD_MODE, - CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, - CAST( - CASE C.LAST_REFRESH_TYPE - WHEN 0 THEN 'COMPLETE' - WHEN 1 THEN 'FAST' - ELSE 'NA' - END AS CHAR(8) - ) AS LAST_REFRESH_TYPE, - CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, - CAST(DATE_ADD(C.LAST_REFRESH_DATE, INTERVAL C.LAST_REFRESH_TIME SECOND) AS DATETIME) AS LAST_REFRESH_END_TIME, - CAST(NULL AS CHAR(19)) AS STALENESS, - CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, - CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, - CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, - CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, - CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, - CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, - CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, - CAST(NULL AS CHAR(19)) AS COMPILE_STATE, - CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, - CAST(NULL AS DATETIME) AS STALE_SINCE, - CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, - CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, - CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, - CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, - CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, - CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, - CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, - CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, - CAST( - CASE ((B.TABLE_MODE >> 28) & 1) - WHEN 0 THEN 'N' - WHEN 1 THEN 'Y' - ELSE NULL - END AS CHAR(1) - ) AS ON_QUERY_COMPUTATION, - C.REFRESH_DOP AS REFRESH_DOP, - C.data_sync_scn AS DATA_SYNC_SCN, - CAST( - CASE C.data_sync_scn - WHEN 0 THEN 'NOT AVAILABLE' - ELSE TIMESTAMPDIFF(SECOND, SCN_TO_TIMESTAMP(C.data_sync_scn), NOW()) - END AS CHAR(128) - ) AS DATA_SYNC_DELAY - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - oceanbase.__all_mview C, - oceanbase.__all_table D - WHERE A.DATABASE_ID = B.DATABASE_ID - AND B.TABLE_ID = C.MVIEW_ID - AND B.TABLE_TYPE = 7 - AND B.DATA_TABLE_ID = D.TABLE_ID -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_STATS_SYS_DEFAULTS', - table_id = '21530', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, - CAST(VALUE AS CHAR(40)) AS VALUE - FROM - ( - /* COLLECTION_LEVEL */ - SELECT - 'COLLECTION_LEVEL' PARAMETER_NAME, - CASE IFNULL(MAX(COLLECTION_LEVEL), 1) - WHEN 0 THEN 'NONE' - WHEN 1 THEN 'TYPICAL' - WHEN 2 THEN 'ADVANCED' - ELSE NULL - END VALUE - FROM - oceanbase.__all_mview_refresh_stats_sys_defaults - - UNION ALL - - /* RETENTION_PERIOD */ - SELECT - 'RETENTION_PERIOD' PARAMETER_NAME, - CAST(IFNULL(MAX(RETENTION_PERIOD), 31) AS CHAR) VALUE - FROM - oceanbase.__all_mview_refresh_stats_sys_defaults - ) -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_STATS_PARAMS', - table_id = '21532', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(MV_OWNER AS CHAR(128)) AS MV_OWNER, - CAST(MV_NAME AS CHAR(128)) AS MV_NAME, - CAST( - CASE COLLECTION_LEVEL - WHEN 0 THEN 'NONE' - WHEN 1 THEN 'TYPICAL' - WHEN 2 THEN 'ADVANCED' - ELSE NULL - END AS CHAR(8) - ) AS COLLECTION_LEVEL, - RETENTION_PERIOD - FROM - ( - WITH DEFVALS AS - ( - SELECT - IFNULL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, - IFNULL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD - FROM - oceanbase.__all_mview_refresh_stats_sys_defaults - ) - - SELECT - A.DATABASE_NAME MV_OWNER, - B.TABLE_NAME MV_NAME, - IFNULL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, - IFNULL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - ( - SELECT MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM oceanbase.__all_mview_refresh_stats_params - RIGHT OUTER JOIN - ( - SELECT MVIEW_ID FROM oceanbase.__all_mview - ) - USING (MVIEW_ID) - ) C, - DEFVALS D - WHERE A.DATABASE_ID = B.DATABASE_ID - AND B.TABLE_ID = C.MVIEW_ID - AND B.TABLE_TYPE = 7 - ) -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_RUN_STATS', - table_id = '21534', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(A.USER_NAME AS CHAR(128)) AS RUN_OWNER, - B.REFRESH_ID AS REFRESH_ID, - B.NUM_MVS_TOTAL AS NUM_MVS, - CAST(B.MVIEWS AS CHAR(4000)) AS MVIEWS, - CAST(B.BASE_TABLES AS CHAR(4000)) AS BASE_TABLES, - CAST(B.METHOD AS CHAR(4000)) AS METHOD, - CAST(B.ROLLBACK_SEG AS CHAR(4000)) AS ROLLBACK_SEG, - CAST(IF(B.PUSH_DEFERRED_RPC = 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, - CAST(IF(B.REFRESH_AFTER_ERRORS = 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, - B.PURGE_OPTION AS PURGE_OPTION, - B.PARALLELISM AS PARALLELISM, - B.HEAP_SIZE AS HEAP_SIZE, - CAST(IF(B.ATOMIC_REFRESH = 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, - CAST(IF(B.NESTED = 1, 'Y', 'N') AS CHAR(1)) AS NESTED, - CAST(IF(B.OUT_OF_PLACE = 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, - B.NUMBER_OF_FAILURES AS NUMBER_OF_FAILURES, - CAST(B.START_TIME AS DATETIME) AS START_TIME, - CAST(B.END_TIME AS DATETIME) AS END_TIME, - B.ELAPSED_TIME AS ELAPSED_TIME, - CAST(0 AS SIGNED) AS LOG_SETUP_TIME, - B.LOG_PURGE_TIME AS LOG_PURGE_TIME, - CAST(IF(B.COMPLETE_STATS_AVALIABLE = 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE - FROM - oceanbase.__all_user A, - oceanbase.__all_mview_refresh_run_stats B, - ( - SELECT - C1.REFRESH_ID AS REFRESH_ID - FROM - oceanbase.__all_mview_refresh_stats C1, - oceanbase.__all_table C2 - WHERE C1.MVIEW_ID = C2.TABLE_ID - GROUP BY REFRESH_ID - ) C - WHERE A.USER_ID = B.RUN_USER_ID - AND B.REFRESH_ID = C.REFRESH_ID -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_STATS', - table_id = '21536', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, - CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, - C.REFRESH_ID AS REFRESH_ID, - CAST( - CASE C.REFRESH_TYPE - WHEN 0 THEN 'COMPLETE' - WHEN 1 THEN 'FAST' - ELSE NULL - END AS CHAR(30) - ) AS REFRESH_METHOD, - CAST(NULL AS CHAR(4000)) AS REFRESH_OPTIMIZATIONS, - CAST(NULL AS CHAR(4000)) AS ADDITIONAL_EXECUTIONS, - CAST(C.START_TIME AS DATETIME) AS START_TIME, - CAST(C.END_TIME AS DATETIME) AS END_TIME, - C.ELAPSED_TIME AS ELAPSED_TIME, - CAST(0 AS SIGNED) AS LOG_SETUP_TIME, - C.LOG_PURGE_TIME AS LOG_PURGE_TIME, - C.INITIAL_NUM_ROWS AS INITIAL_NUM_ROWS, - C.FINAL_NUM_ROWS AS FINAL_NUM_ROWS, - C.RESULT AS RESULT - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - oceanbase.__all_mview_refresh_stats C - WHERE A.DATABASE_ID = B.DATABASE_ID - AND B.TABLE_ID = C.MVIEW_ID - AND B.TABLE_TYPE = 7 -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_CHANGE_STATS', - table_id = '21538', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(C.DATABASE_NAME AS CHAR(128)) AS TBL_OWNER, - CAST(D.TABLE_NAME AS CHAR(128)) AS TBL_NAME, - CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, - CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, - E.REFRESH_ID AS REFRESH_ID, - E.NUM_ROWS_INS AS NUM_ROWS_INS, - E.NUM_ROWS_UPD AS NUM_ROWS_UPD, - E.NUM_ROWS_DEL AS NUM_ROWS_DEL, - CAST(0 AS SIGNED) AS NUM_ROWS_DL_INS, - CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, - CAST(NULL AS CHAR(4000)) AS PMOP_DETAILS, - E.NUM_ROWS AS NUM_ROWS - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - oceanbase.__all_database C, - oceanbase.__all_table D, - oceanbase.__all_mview_refresh_change_stats E - WHERE A.DATABASE_ID = B.DATABASE_ID - AND C.DATABASE_ID = D.DATABASE_ID - AND E.MVIEW_ID = B.TABLE_ID - AND E.DETAIL_TABLE_ID = D.TABLE_ID -""".replace("\n", " ") -) - -def_table_schema( - owner = 'suzhi.yt', - table_name = 'DBA_MVREF_STMT_STATS', - table_id = '21540', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, - CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, - C.REFRESH_ID AS REFRESH_ID, - C.STEP AS STEP, - CAST(C.SQLID AS CHAR(32)) AS SQLID, - C.STMT AS STMT, - C.EXECUTION_TIME AS EXECUTION_TIME, - C.EXECUTION_PLAN AS EXECUTION_PLAN - FROM - oceanbase.__all_database A, - oceanbase.__all_table B, - oceanbase.__all_mview_refresh_stmt_stats C - WHERE A.DATABASE_ID = B.DATABASE_ID - AND B.TABLE_ID = C.MVIEW_ID -""".replace("\n", " ") -) - # 21541: GV$OB_SESSION_PS_INFO # removed (single-tenant GV/V collapse; use oceanbase.__all_virtual_session_ps_info) # 21542: V$OB_SESSION_PS_INFO # removed (single-tenant GV/V collapse; use oceanbase.__all_virtual_session_ps_info) @@ -17741,7 +16777,7 @@ sum(avtps.required_size) as REQUIRED_SIZE from oceanbase.__all_virtual_tablet_pointer_status avtps - INNER JOIN oceanbase.__all_tablet_to_ls attl + INNER JOIN oceanbase.__all_tablet_to_table attl ON attl.tablet_id = avtps.tablet_id INNER JOIN oceanbase.__all_table at ON at.table_id = attl.table_id @@ -17756,60 +16792,6 @@ """.replace("\n", " ") ) -# 21597: GV$OB_LOG_TRANSPORT_DEST_STAT # removed (single-tenant GV/V collapse; folded into V$OB_LOG_TRANSPORT_DEST_STAT) - -def_table_schema( - owner = 'wenyue.zxl', - table_name = 'V$OB_LOG_TRANSPORT_DEST_STAT', - table_id = '21598', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - CLIENT_IP, - CLIENT_PID, - CASE CLIENT_TYPE - WHEN 1 THEN 'CDC' - WHEN 2 THEN 'STANDBY' - ELSE 'UNKNOWN' - END AS CLIENT_TYPE, - START_SERVE_TIME, - LAST_SERVE_TIME, - CASE LAST_READ_SOURCE - WHEN 1 THEN 'ONLINE' - WHEN 2 THEN 'ARCHIVE' - ELSE 'UNKNOWN' - END AS LAST_READ_SOURCE, - CASE LAST_REQUEST_TYPE - WHEN 0 THEN 'SEQUENTIAL_READ_SERIAL' - WHEN 1 THEN 'SEQUENTIAL_READ_PARALLEL' - WHEN 2 THEN 'SCATTERED_READ' - ELSE 'UNKNOWN' - END AS LAST_REQUEST_TYPE, - LAST_REQUEST_LOG_LSN, - LAST_REQUEST_LOG_SCN, - LAST_FAILED_REQUEST, - AVG_REQUEST_PROCESS_TIME, - AVG_REQUEST_QUEUE_TIME, - AVG_REQUEST_READ_LOG_TIME, - AVG_REQUEST_READ_LOG_SIZE, - CASE - WHEN AVG_LOG_TRANSPORT_BANDWIDTH >= 1024 * 1024 * 1024 THEN - CONCAT(ROUND(AVG_LOG_TRANSPORT_BANDWIDTH/1024/1024/1024, 2), 'GB/S') - WHEN AVG_LOG_TRANSPORT_BANDWIDTH >= 1024 * 1024 THEN - CONCAT(ROUND(AVG_LOG_TRANSPORT_BANDWIDTH/1024/1024, 2), 'MB/S') - WHEN AVG_LOG_TRANSPORT_BANDWIDTH >= 1024 THEN - CONCAT(ROUND(AVG_LOG_TRANSPORT_BANDWIDTH/1024, 2), 'KB/S') - ELSE - CONCAT(AVG_LOG_TRANSPORT_BANDWIDTH, 'B/s') - END AS AVG_LOG_TRANSPORT_BANDWIDTH - FROM OCEANBASE.__ALL_VIRTUAL_LOG_TRANSPORT_DEST_STAT -""".replace("\n", " ") -) - # 21599: GV$OB_SS_LOCAL_CACHE abandoned # 21600: V$OB_SS_LOCAL_CACHE abandoned @@ -18148,75 +17130,6 @@ ) -def_table_schema( - owner = 'zg410411', - table_name = 'DBA_MVIEW_RUNNING_JOBS', - table_id = '21652', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - B.TABLE_NAME AS TABLE_NAME, - CAST ( - CASE A.JOB_TYPE - WHEN 0 THEN 'INVALID' - WHEN 1 THEN 'COMPLETE REFRESH' - WHEN 2 THEN 'FAST REFRESH' - WHEN 3 THEN 'PURGE MLOG' - ELSE NULL - END AS CHAR(64) - ) AS JOB_TYPE, - A.SESSION_ID AS SESSION_ID, - A.READ_SNAPSHOT AS READ_SNAPSHOT, - A.PARALLEL AS PARALLEL, - A.JOB_START_TIME AS JOB_START_TIME - FROM oceanbase.__all_virtual_mview_running_job A, - oceanbase.__all_table B - WHERE A.table_id = B.table_id -""".replace("\n", " ") -) - -def_table_schema( - owner = 'zg410411', - table_name = 'DBA_MVIEW_DEPS', - table_id = '21654', - table_type = 'SYSTEM_VIEW', - rowkey_columns = [], - normal_columns = [], - gm_columns = [], - in_tenant_space = True, - view_definition = """ - SELECT - D.DATABASE_NAME AS MVIEW_OWNER, - B.TABLE_NAME AS MVIEW_NAME, - E.DATABASE_NAME AS DEP_OWNER, - C.TABLE_NAME AS DEP_NAME, - CAST ( - CASE C.TABLE_TYPE - WHEN 3 THEN 'TABLE' - WHEN 4 THEN 'VIEW' - WHEN 7 THEN 'MV' - WHEN 14 THEN 'EXTERNAL TABLE' - ELSE 'INVALID TYPE' - END AS CHAR(64) - ) AS DEP_TYPE - FROM oceanbase.__all_mview_dep A, - oceanbase.__all_table B, - oceanbase.__all_table C, - oceanbase.__all_database D, - oceanbase.__all_database E - WHERE A.mview_id = B.table_id - AND A.p_obj = C.table_id - AND B.database_id = D.database_id - AND C.database_id = E.database_id - AND (C.table_mode >> 24 & 1 ) = 0 -""".replace("\n", " ") -) - - def_table_schema( owner = 'zhaoziqian.zzq', table_name = 'DBA_OB_DYNAMIC_PARTITION_TABLES', @@ -18863,7 +17776,7 @@ index_columns = ['table_id'], index_using_type = 'USING_BTREE', index_type = 'INDEX_TYPE_NORMAL_LOCAL', - keywords = all_def_keywords['__all_tablet_to_ls']) + keywords = all_def_keywords['__all_tablet_to_table']) def_sys_index_table( index_name = 'idx_pending_tx_id', @@ -18977,30 +17890,6 @@ # 101093: idx_kv_ttl_task_table_id (abandoned) # 101094: idx_kv_ttl_task_history_upd_time (abandoned) -def_sys_index_table( - index_name = 'idx_mview_refresh_run_stats_num_mvs_current', - index_table_id = 101095, - index_columns = ['num_mvs_current'], - index_using_type = 'USING_BTREE', - index_type = 'INDEX_TYPE_NORMAL_LOCAL', - keywords = all_def_keywords['__all_mview_refresh_run_stats']) - -def_sys_index_table( - index_name = 'idx_mview_refresh_stats_end_time', - index_table_id = 101096, - index_columns = ['end_time'], - index_using_type = 'USING_BTREE', - index_type = 'INDEX_TYPE_NORMAL_LOCAL', - keywords = all_def_keywords['__all_mview_refresh_stats']) - -def_sys_index_table( - index_name = 'idx_mview_refresh_stats_mview_end_time', - index_table_id = 101097, - index_columns = ['mview_id', 'end_time'], - index_using_type = 'USING_BTREE', - index_type = 'INDEX_TYPE_NORMAL_LOCAL', - keywords = all_def_keywords['__all_mview_refresh_stats']) - def_sys_index_table( index_name = 'idx_client_to_server_session_info_client_session_id', index_table_id = 101099, diff --git a/src/share/io/ob_backup_io_adapter.cpp b/src/share/io/ob_backup_io_adapter.cpp index 24ae77133..19307842e 100644 --- a/src/share/io/ob_backup_io_adapter.cpp +++ b/src/share/io/ob_backup_io_adapter.cpp @@ -1601,56 +1601,5 @@ int ObDirPrefixEntryNameFilter::init( return ret; } -//*************ObDirPrefixLSIDFilter************* -int ObDirPrefixLSIDFilter::init(const char *filter_str, const int32_t filter_str_len) -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - if (is_inited_) { - ret = OB_INIT_TWICE; - OB_LOG(WARN, "init twice", KR(ret)); - } else if (OB_ISNULL(filter_str) || OB_UNLIKELY(filter_str_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "invalid argument", KR(ret), KP(filter_str), K(filter_str_len)); - } else if (OB_UNLIKELY(filter_str_len > (sizeof(filter_str_) - 1))) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "the length of dir prefix too long", KR(ret), K(filter_str_len)); - } else if (OB_FAIL(databuff_printf(filter_str_, sizeof(filter_str_), "%.*s", filter_str_len, filter_str))) { - OB_LOG(WARN, "failed to init filter_str", KR(ret), K(filter_str), K(filter_str_len)); - } else if (OB_FAIL(common::databuff_printf(format_buffer_, sizeof(format_buffer_), pos, // "logstream_%ld" - "%s_%%ld", share::OB_STR_LS))) { - OB_LOG(WARN, "Failed to construct sscanf format string or buffer too small", - KR(ret), "pos", pos, "buffer_size", sizeof(format_buffer_)); - } else { - is_inited_ = true; - } - return ret; -} - -int ObDirPrefixLSIDFilter::func(const dirent *entry) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - OB_LOG(WARN, "dir prefix filter not init", KR(ret)); - } else if (OB_ISNULL(entry) || OB_ISNULL(entry->d_name)) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "invalid argument, entry or d_name is null", KR(ret), K(entry)); - } else if (STRLEN(entry->d_name) < STRLEN(filter_str_)) { - // do nothing, entry name is too short to match filter_str_ - } else if (0 == STRNCMP(entry->d_name, filter_str_, STRLEN(filter_str_))) { - int64_t ls_stream_id; - const int sscanf_ret = sscanf(entry->d_name, format_buffer_, &ls_stream_id); - if (1 != sscanf_ret) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "failed to parse logstream dir, sscanf returned unexpected count", - KR(ret), K(sscanf_ret), "entry", entry->d_name, K(format_buffer_), "expected_count", 1); - } else if (OB_FAIL(d_entrys_.push_back(share::ObLSID(ls_stream_id)))) { - OB_LOG(WARN, "fail to push back directory entry", KR(ret), K(ls_stream_id), K_(filter_str)); - } - } - return ret; -} - } } diff --git a/src/share/io/ob_backup_io_adapter.h b/src/share/io/ob_backup_io_adapter.h index 7a629ad8e..473c8ebd1 100644 --- a/src/share/io/ob_backup_io_adapter.h +++ b/src/share/io/ob_backup_io_adapter.h @@ -22,7 +22,6 @@ #include "lib/container/ob_array.h" #include "share/io/ob_backup_storage_info.h" #include "share/io/ob_io_define.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -317,27 +316,6 @@ class ObDirPrefixEntryNameFilter : public ObBaseDirEntryOperator DISALLOW_COPY_AND_ASSIGN(ObDirPrefixEntryNameFilter); }; -class ObDirPrefixLSIDFilter final : public ObBaseDirEntryOperator -{ - public: - ObDirPrefixLSIDFilter(common::ObIArray &d_entrys) - : is_inited_(false), - d_entrys_(d_entrys) - { - filter_str_[0] = '\0'; - format_buffer_[0] = '\0'; - } - virtual ~ObDirPrefixLSIDFilter() = default; - int init(const char *filter_str, const int32_t filter_str_len); - virtual int func(const dirent *entry) override; -private: - bool is_inited_; - char filter_str_[common::MAX_PATH_SIZE]; - char format_buffer_[share::OB_BACKUP_LS_DIR_NAME_LENGTH]; - common::ObIArray &d_entrys_; - DISALLOW_COPY_AND_ASSIGN(ObDirPrefixLSIDFilter); -}; - } } diff --git a/src/share/leak_checker/obj_leak_checker.h b/src/share/leak_checker/obj_leak_checker.h index 00e92011e..981ac873a 100644 --- a/src/share/leak_checker/obj_leak_checker.h +++ b/src/share/leak_checker/obj_leak_checker.h @@ -31,7 +31,6 @@ namespace share enum ObLeakCheckObjType { LEAK_CHECK_OBJ_TABLET_HANDLE, - LEAK_CHECK_OBJ_LS_HANDLE, LEAK_CHECK_OBJ_MAX_NUM }; diff --git a/src/share/location_cache/ob_location_service.cpp b/src/share/location_cache/ob_location_service.cpp deleted file mode 100644 index c17dfeff7..000000000 --- a/src/share/location_cache/ob_location_service.cpp +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE_LOCATION - -#include "share/location_cache/ob_location_service.h" -#include "lib/string/ob_sql_string.h" -#include "share/schema/ob_schema_struct.h" -#include "share/schema/ob_schema_getter_guard.h" -#include "share/inner_table/ob_inner_table_schema_constants.h" - -namespace oceanbase -{ -using namespace common; - -namespace share -{ -ObLocationService::ObLocationService() - : inited_(false), - stopped_(false) -{ -} - -int ObLocationService::get( - const int64_t cluster_id, - const ObLSID &ls_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObLSLocation &location) -{ - is_cache_hit = true; - location.reset(); - return nonblock_get(cluster_id, ls_id, location); -} - -int ObLocationService::get_leader( - const int64_t cluster_id, - const ObLSID &ls_id, - const bool force_renew, - common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - leader = GCTX.self_addr(); - } - return ret; -} - -int ObLocationService::get_leader_with_retry_until_timeout( - const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader, - const int64_t abs_retry_timeout, - const int64_t retry_interval) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), KCSTRING(lbt())); - } else { - leader = GCTX.self_addr(); - } - return ret; -} - -int ObLocationService::nonblock_get( - const int64_t cluster_id, - const ObLSID &ls_id, - ObLSLocation &location) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(GCTX.config_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(GCTX.config_)); - } else if (OB_FAIL(location.init(cluster_id, ls_id, ObTimeUtility::current_time()))) { - LOG_WARN("location init error", KR(ret), K(cluster_id), K(ls_id)); - } else { - ObLSReplicaLocation replica_location; - common::ObReplicaProperty property; - if(OB_FAIL(replica_location.init( - GCTX.self_addr(), LEADER/*role*/, GCTX.config_->mysql_port/*sql_port*/, - REPLICA_TYPE_FULL/*replica_type*/, property/*not_used*/, - ObLSRestoreStatus(ObLSRestoreStatus::Status::NONE)/*restore_status*/, 1/*proposal_id*/))) { - LOG_WARN("fail to init", KR(ret)); - } else if (OB_FAIL(location.add_replica_location(replica_location))) { - LOG_WARN("fail to add replica locaiton", KR(ret), K(replica_location)); - } - } - return ret; -} - -int ObLocationService::nonblock_get_leader( - const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - leader = GCTX.self_addr(); - } - return ret; -} - -int ObLocationService::get( - const ObTabletID &tablet_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - is_cache_hit = true; - ls_id = ObLSID::SYS_LS_ID; - } - return ret; -} - -int ObLocationService::nonblock_get( - const ObTabletID &tablet_id, - ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - ls_id = ObLSID::SYS_LS_ID; - } - return ret; -} - -int ObLocationService::vtable_get(const uint64_t table_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObIArray &locations) -{ - int ret = OB_SUCCESS; - locations.reset(); - is_cache_hit = true; - if (OB_FAIL(locations.push_back(GCTX.self_addr()))) { - LOG_WARN("fail to get location for virtual table", KR(ret)); - } - return ret; -} - -int ObLocationService::init( - schema::ObMultiVersionSchemaService &schema_service, - common::ObMySQLProxy &sql_proxy) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("location service init twice", KR(ret)); - } else { - inited_ = true; - } - return ret; -} - -int ObLocationService::start() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("location service not init", KR(ret)); - } - return ret; -} - -void ObLocationService::stop() -{ -} - -void ObLocationService::wait() -{ -} - -int ObLocationService::destroy() -{ - int ret = OB_SUCCESS; - stopped_ = true; - inited_ = false; - return ret; -} - -ERRSIM_POINT_DEF(EN_CHECK_LS_EXIST_WITH_TENANT_NOT_NORMAL); - -int ObLocationService::check_ls_exist( - const ObLSID &ls_id, - ObLSExistState &state) -{ - int ret = OB_SUCCESS; - state.reset(); - ObSqlString sql; - if (OB_UNLIKELY(!ls_id.is_valid_with_tenant())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(ls_id)); - } else if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("GCTX has null ptr", KR(ret), KP(GCTX.schema_service_), KP(GCTX.sql_proxy_)); - } else { - schema::ObSchemaGetterGuard schema_guard; - const schema::ObSimpleTenantSchema *tenant_schema = NULL; - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(schema_guard))) { - LOG_WARN("fail to get tenant schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_schema))) { - LOG_WARN("fail to get tenant schema", KR(ret)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant does not exist", KR(ret)); - } else if ((true) - && (tenant_schema->is_normal() || tenant_schema->is_dropping())) { - // sys and meta tenants only have sys ls. If tenant is in normal or dropping status, sys ls exists. - state.set_existing(); - } - } // release schema_guard as soon as possible - - // errsim for test - if (EN_CHECK_LS_EXIST_WITH_TENANT_NOT_NORMAL) { - state.reset(); - } - return ret; -} - -} // end namespace share -} // end namespace oceanbase diff --git a/src/share/location_cache/ob_location_service.h b/src/share/location_cache/ob_location_service.h deleted file mode 100644 index 1b739eca4..000000000 --- a/src/share/location_cache/ob_location_service.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_OB_LOCATION_SERVICE -#define OCEANBASE_SHARE_OB_LOCATION_SERVICE - -#include "share/location_cache/ob_location_struct.h" - -namespace oceanbase -{ -namespace share -{ -namespace schema -{ -class ObMultiVersionSchemaService; -} -// This class provides interfaces to get/renew location -// in block and nonblock ways for log stream and tablet. -class ObLocationService -{ -public: - ObLocationService(); - virtual ~ObLocationService() {} - // ------------------------- Interfaces for log stream location ------------------------- - // Gets log stream location synchronously. - // - // @param [in] cluster_id: target cluster which the ls belongs to - // @param [in] tenant: target tenant which the ls belongs to - // @param [in] ls_id: target ls - // @param [in] expire_renew_time: The oldest renew time of cache which the user can tolerate. - // Setting it to 0 means that the cache will not be renewed. - // Setting it to INT64_MAX means renewing cache synchronously. - // @param [out] is_cache_hit: If hit in location cache. - // @param [out] location: Include all replicas' addresses of a log stream. - // @return OB_LS_LOCATION_NOT_EXIST if no records in sys table. - // OB_GET_LOCATION_TIME_OUT if get location by inner sql timeout. - int get( - const int64_t cluster_id, - const ObLSID &ls_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObLSLocation &location); - - // Gets leader address of a log stream synchronously. - // - // @param [in] force_renew: whether to renew location synchronously. - // @param [out] leader: leader address of the log stream. - // @return OB_LS_LOCATION_NOT_EXIST if no records in sys table, - // OB_LS_LEADER_NOT_EXIST if no leader in location. - // OB_GET_LOCATION_TIME_OUT if get location by inner sql timeout. - int get_leader( - const int64_t cluster_id, - const ObLSID &ls_id, - const bool force_renew, - common::ObAddr &leader); - - // Get leader of ls. If not hit in cache or leader not exist, - // it will renew location and try to get leader again until abs_retry_timeout. - // - // @param [in] cluster_id: target cluster which the ls belongs to - // @param [in] tenant: target tenant which the ls belongs to - // @param [in] ls_id: target ls - // @param [out] leader: leader address of the log stream. - // @param [in] abs_retry_timeout: absolute timestamp for retry timeout. - // (default use remain timeout of ObTimeoutCtx or THIS_WORKER) - // @param [in] retry_interval: interval between each retry. (default 100ms) - // @return OB_LS_LOCATION_NOT_EXIST if no records in sys table, - // OB_LS_LEADER_NOT_EXIST if no leader in location. - int get_leader_with_retry_until_timeout( - const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader, - const int64_t abs_retry_timeout = 0, - const int64_t retry_interval = 100000/*100ms*/); - - // Nonblock way to get log stream location. - // - // @param [out] location: include all replicas' addresses of a log stream - // @return OB_LS_LOCATION_NOT_EXIST if no records in sys table. - int nonblock_get( - const int64_t cluster_id, - const ObLSID &ls_id, - ObLSLocation &location); - - // Nonblock way to get leader address of the log stream. - // - // @param [out] leader: leader address of the log stream - // @return OB_LS_LOCATION_NOT_EXIST if no records in sys table - // OB_LS_LEADER_NOT_EXIST if no leader in location. - int nonblock_get_leader( - const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader); - - // ------------------------ End interfaces for log stream location ------------------------ - - // ------------- Interfaces for tablet to log stream (just for local cluster) ------------- - // Gets the mapping between the tablet and log stream synchronously. - // - // @param [in] tenant: target tenant which the tablet belongs to - // @param [in] expire_renew_time: The oldest renew time of cache which the user can tolerate. - // Setting it to 0 means that the cache will not be renewed. - // Setting it to INT64_MAX means renewing cache synchronously. - // @param [out] is_cache_hit: If hit in location cache. - // @return OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST if no records in sys table. - // OB_GET_LOCATION_TIME_OUT if get location by inner sql timeout. - int get( - const ObTabletID &tablet_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObLSID &ls_id); - - // Noblock way to get the mapping between the tablet and log stream. - // - // @return OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST if no records in sys table. - int nonblock_get( - const ObTabletID &tablet_id, - ObLSID &ls_id); - - // ----------------------- End interfaces for tablet to log stream ----------------------- - - // ----------------------- Interfaces for virtual table location ------------------------- - // Gets location for virtual table synchronously. - // - // @param [in] tenant: Tenant for virtual table. - // @param [in] table_id: Virtual table id. - // @param [in] expire_renew_time: The oldest renew time of cache which the user can tolerate. - // Setting it to 0 means that the cache will not be renewed. - // Setting it to INT64_MAX means renewing cache synchronously. - // @param [out] is_cache_hit: If hit in cache. - // @param [out] locations: Array of virtual table locations. - // @return OB_LOCATION_NOT_EXIST if can not find location. - int vtable_get(const uint64_t table_id, - const int64_t expire_renew_time, - bool &is_cache_hit, - ObIArray &locations); - - // Nonblock way to renew the virtual table location - // - // @param [in] tenant: Tenant for virtual table. - // @param [in] table_id: Virtual table id. - // --------------------- End interfaces for virtual table location ----------------------- - - /* check if the ls exists by querying __all_ls_status and __all_tenant_info - * - * @param[in] ls_id: target ls_id - * @param[out] state: EXISTING/DELETED/UNCREATED - * @return - * - OB_SUCCESS: check successfully - * - OB_TENANT_NOT_EXIST: tenant not exist - * - OB_INVALID_ARGUMENT: invalid ls_id - * - other: other failures - */ - static int check_ls_exist(const ObLSID &ls_id, ObLSExistState &state); - - int init( - schema::ObMultiVersionSchemaService &schema_service, - common::ObMySQLProxy &sql_proxy); - int start(); - void stop(); - void wait(); - int destroy(); - -private: - bool inited_; - bool stopped_; -}; - -} // end namespace share -} // end namespace oceanbase -#endif diff --git a/src/share/location_cache/ob_location_struct.cpp b/src/share/location_cache/ob_location_struct.cpp index a58d16e25..7a8a506a5 100644 --- a/src/share/location_cache/ob_location_struct.cpp +++ b/src/share/location_cache/ob_location_struct.cpp @@ -18,9 +18,6 @@ #include "lib/stat/ob_diagnostic_info_guard.h" #include "share/location_cache/ob_location_struct.h" -#include "share/config/ob_server_config.h" // GCONF -#include "share/ob_server_struct.h" -#include "lib/stat/ob_diagnostic_info_guard.h" #include "lib/statistic_event/ob_stat_event.h" namespace oceanbase @@ -37,25 +34,6 @@ OB_SERIALIZE_MEMBER(ObLSReplicaLocation, restore_status_, proposal_id_); -OB_SERIALIZE_MEMBER(ObLSLocationCacheKey, - cluster_id_, - ls_id_); - -OB_SERIALIZE_MEMBER(ObLSLocation, - cache_key_, - replica_locations_, - renew_time_); - -OB_SERIALIZE_MEMBER(ObTabletLocation, - - tablet_id_, - replica_locations_, - renew_time_); - -OB_SERIALIZE_MEMBER(ObTabletLSKey, - - tablet_id_); - ObLSReplicaLocation::ObLSReplicaLocation() : server_(), role_(FOLLOWER), @@ -169,616 +147,6 @@ int ObLSReplicaLocation::init_without_check( return ret; } -ObLSLocation::ObLSLocation() - : ObLink(), - cache_key_(), - renew_time_(0), - last_access_ts_(0), - replica_locations_() -{ -} - -ObLSLocation::ObLSLocation(common::ObIAllocator &allocator) - : ObLink(), - cache_key_(), - renew_time_(0), - last_access_ts_(0), - replica_locations_( - common::OB_MALLOC_NORMAL_BLOCK_SIZE, - common::ModulePageAllocator(allocator)) -{ -} - -ObLSLocation::~ObLSLocation() -{ -} - - -int ObLSLocation::deep_copy(const ObLSLocation &ls_location) -{ - int ret = OB_SUCCESS; - if (this != &ls_location) { - this->cache_key_ = ls_location.cache_key_; - this->renew_time_ = ls_location.renew_time_; - if (OB_FAIL(this->replica_locations_.assign(ls_location.replica_locations_))) { - LOG_WARN("ls location deep copy error", K(ret), K(*this), K(ls_location)); - } - } - return ret; -} - -int ObLSLocation::init( - const int64_t cluster_id, - const ObLSID &ls_id, - const int64_t renew_time) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid_with_tenant() || renew_time <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", KR(ret), K(ls_id), K(renew_time)); - } else if (OB_FAIL(cache_key_.init(cluster_id, ls_id))) { - LOG_WARN("cache key init error", K(ret), K(ls_id)); - } else { - renew_time_ = renew_time; - last_access_ts_ = 0; - replica_locations_.reset(); - ObLink::reset(); - } - return ret; -} - -int ObLSLocation::init_fake_location() -{ - int ret = OB_SUCCESS; - reset(); - ObLSReplicaLocation replica_location; - const char *VIRTUAL_IP = GCONF.use_ipv6 ? "::" : "0.0.0.0"; - uint32_t VIRTUAL_PORT = 0; - ObAddr server; - ObReplicaProperty property; - ObLSRestoreStatus restore_status; - if (OB_UNLIKELY(!server.set_ip_addr(VIRTUAL_IP, VIRTUAL_PORT))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to set ip addr", KR(ret), K(VIRTUAL_IP), K(VIRTUAL_PORT)); - } else if (OB_FAIL(replica_location.init_without_check( - server, - FOLLOWER, - VIRTUAL_PORT, - REPLICA_TYPE_FULL, - property, - restore_status, - 0 /*proposal_id*/))) { - LOG_WARN("fail to init replica location", KR(ret), K(server), - K(FOLLOWER), K(VIRTUAL_PORT), K(REPLICA_TYPE_FULL), K(property), K(restore_status)); - } else if (OB_FAIL(replica_locations_.push_back(replica_location))) { - LOG_WARN("fail to add replica location", KR(ret), K(replica_location)); - } else { - LOG_INFO("success to init fake location", KPC(this)); - } - return ret; -} - -void ObLSLocation::reset() -{ - cache_key_.reset(); - renew_time_ = 0; - last_access_ts_ = 0; - replica_locations_.reset(); - ObLink::reset(); -} - -int ObLSLocation::assign(const ObLSLocation &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - cache_key_ = other.cache_key_; - renew_time_ = other.renew_time_; - if (OB_FAIL(replica_locations_.assign(other.replica_locations_))) { - LOG_WARN("Failed to assign replica locations", KR(ret), K(other)); - } - } - return ret; -} - -bool ObLSLocation::is_valid() const -{ - return cache_key_.is_valid() - && renew_time_ > 0; -} - -bool ObLSLocation::is_same_with(const ObLSLocation &other) const -{ - bool bret = true; - if (cache_key_ != other.cache_key_ - || replica_locations_.count() != other.replica_locations_.count()) { - bret = false; - } else { - ARRAY_FOREACH_X(replica_locations_, idx, cnt, bret) { - if (replica_locations_.at(idx) != other.replica_locations_.at(idx)) { - bret = false; - } - } - } - return bret; -} - -bool ObLSLocation::operator==(const ObLSLocation &other) const -{ - return is_same_with(other) - && (last_access_ts_ == other.last_access_ts_) - && (renew_time_ == other.renew_time_); -} - - -int ObLSLocation::get_replica_count(int64_t &full_replica_cnt, int64_t &non_paxos_replica_cnt) -{ - int ret = OB_SUCCESS; - full_replica_cnt = 0; - non_paxos_replica_cnt = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < replica_locations_.count(); ++i) { - const ObLSReplicaLocation &replica = replica_locations_.at(i); - if (REPLICA_TYPE_FULL == replica.get_replica_type()) { - full_replica_cnt++; - } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica.get_replica_type())) { - non_paxos_replica_cnt++; - } - } - return ret; -} - -int ObLSLocation::get_leader(common::ObAddr &leader) const -{ - int ret = OB_LS_LOCATION_LEADER_NOT_EXIST; - leader.reset(); - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), "self", *this); - } else { - ARRAY_FOREACH_X(replica_locations_, i, cnt, OB_LS_LOCATION_LEADER_NOT_EXIST == ret) { - if (replica_locations_.at(i).is_strong_leader()) { - leader = replica_locations_.at(i).get_server(); - ret = OB_SUCCESS; - } - } - } - return ret; -} - -int ObLSLocation::get_leader(ObLSReplicaLocation &leader) const -{ - int ret = OB_LS_LOCATION_LEADER_NOT_EXIST; - leader.reset(); - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), "self", *this); - } else { - ARRAY_FOREACH_X(replica_locations_, i, cnt, OB_LS_LOCATION_LEADER_NOT_EXIST == ret) { - if (replica_locations_.at(i).is_strong_leader()) { - if(OB_FAIL(leader.assign(replica_locations_.at(i)))) { - LOG_WARN("fail to assign leader", KR(ret), "leader", replica_locations_.at(i)); - } - } - } - } - return ret; -} - - -int ObLSLocation::add_replica_location(const ObLSReplicaLocation &replica_location) -{ - int ret = OB_SUCCESS; - if (!replica_location.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid replica location", KR(ret), K(replica_location)); - } else if (OB_FAIL(replica_locations_.push_back(replica_location))) { - LOG_WARN("fail to do add replica location", KR(ret), K(replica_location)); - } - return ret; -} - -int ObLSLocation::merge_leader_from(const ObLSLocation &new_location) -{ - int ret = OB_SUCCESS; - ObLSReplicaLocation new_leader; - int tmp_ret = new_location.get_leader(new_leader); - if (OB_LS_LOCATION_LEADER_NOT_EXIST != tmp_ret && OB_SUCCESS != tmp_ret) { - ret = tmp_ret; - LOG_WARN("fail to get leader from new location", KR(ret), K(new_location)); - } else if (OB_LS_LOCATION_LEADER_NOT_EXIST == tmp_ret) { - // no valid leader exist, just skip - } else { - ObLSReplicaLocation *old_leader = NULL; - ObLSReplicaLocation *exist_replica = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < replica_locations_.count(); i++) { - ObLSReplicaLocation &replica = replica_locations_.at(i); - if (replica.is_strong_leader()) { - if (OB_NOT_NULL(old_leader)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("duplicate leader exist", KR(ret), K(replica), KPC(old_leader)); - } else { - old_leader = &replica; - } - } - if (OB_SUCC(ret) && replica.get_server() == new_leader.get_server()) { - if (OB_NOT_NULL(exist_replica)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("duplidate replica exist", KR(ret), K(replica), KPC(exist_replica)); - } else { - exist_replica = &replica; - } - } - } // end for - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(old_leader) || old_leader->get_proposal_id() < new_leader.get_proposal_id()) { - // megre leader when: - // 1. leader not exist - // 2. old leader has smaller proposal id - FLOG_INFO("[LS_LOCATION] leader will be changed", - K(new_leader), KPC(old_leader), KPC(exist_replica), "old_location", *this); - if (OB_ISNULL(exist_replica)) { - if (OB_FAIL(add_replica_location(new_leader))) { - LOG_WARN("fail to add replica", KR(ret), K(new_leader)); - } - } else if (OB_FAIL(exist_replica->assign(new_leader))) { - LOG_WARN("fail to assign new leader", KR(ret), K(new_leader)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < replica_locations_.count(); i++) { - ObLSReplicaLocation &replica = replica_locations_.at(i); - if (replica.get_server() != new_leader.get_server()) { - replica.set_role(FOLLOWER); - replica.set_proposal_id(0); - } - } // end for - if (OB_SUCC(ret) && get_renew_time() < new_location.get_renew_time()) { - (void) set_renew_time(new_location.get_renew_time()); - } - } - } - return ret; -} - -ObTabletLocation::ObTabletLocation() - : tablet_id_(), - renew_time_(0), - replica_locations_() -{ -} - -ObTabletLocation::ObTabletLocation(common::ObIAllocator &allocator) - : tablet_id_(), - renew_time_(0), - replica_locations_( - common::OB_MALLOC_NORMAL_BLOCK_SIZE, - common::ModulePageAllocator(allocator)) -{ -} - -ObTabletLocation::~ObTabletLocation() -{ -} - - -int64_t ObTabletLocation::size() const -{ - return sizeof(*this) + replica_locations_.get_data_size(); -} - -void ObTabletLocation::reset() -{ - - renew_time_ = 0; - tablet_id_.reset(); - replica_locations_.reset(); -} - -int ObTabletLocation::assign(const ObTabletLocation &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - - renew_time_ = other.renew_time_; - tablet_id_ = other.tablet_id_; - if (OB_FAIL(replica_locations_.assign(other.replica_locations_))) { - LOG_WARN("Failed to assign replica locations", KR(ret), K(other)); - } - } - return ret; -} - -bool ObTabletLocation::is_valid() const -{ - return true - && tablet_id_.is_valid() - && renew_time_ > 0; -} - -bool ObTabletLocation::operator==(const ObTabletLocation &other) const -{ - bool equal = true; - if (!is_valid() || !other.is_valid()) { - equal = false; - LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid argument", "self", *this, K(other)); - } else if (replica_locations_.count() != other.replica_locations_.count()) { - equal = false; - } else { - for (int64_t i = 0; i < replica_locations_.count(); ++i) { - if (replica_locations_.at(i) != other.replica_locations_.at(i)) { - equal = false; - break; - } - } - equal = equal - && (true) - && (tablet_id_ == other.tablet_id_) - && (renew_time_ == other.renew_time_); - } - return equal; -} - - - -int ObTabletLocation::get_leader(ObLSReplicaLocation &leader) const -{ - int ret = OB_LS_LOCATION_LEADER_NOT_EXIST; - leader.reset(); - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), "self", *this); - } else { - ARRAY_FOREACH_X(replica_locations_, i, cnt, OB_LS_LOCATION_LEADER_NOT_EXIST == ret) { - if (replica_locations_.at(i).is_strong_leader()) { - if(OB_FAIL(leader.assign(replica_locations_.at(i)))) { - LOG_WARN("fail to assign leader", KR(ret), "leader", replica_locations_.at(i)); - } - } - } - } - return ret; -} - -int ObTabletLocation::add_replica_location(const ObLSReplicaLocation &replica_location) -{ - int ret = OB_SUCCESS; - if (!replica_location.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid replica location", KR(ret), K(replica_location)); - } else if (OB_FAIL(replica_locations_.push_back(replica_location))) { - LOG_WARN("fail to do add replica location", KR(ret), K(replica_location)); - } - return ret; -} - - - -int ObTabletLocation::deep_copy( - char *buf, - const int64_t buf_len, - ObIKVCacheValue *&value) const -{ - int ret = OB_SUCCESS; - if (NULL == buf || buf_len < size()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(buf), K(buf_len), "size", size()); - } else { - ObTabletLocation *pvalue = new (buf) ObTabletLocation(); - if (OB_FAIL(pvalue->assign(*this))) { - LOG_WARN("fail to assign ObTabletLocation", KR(ret), KP(this), KP(pvalue)); - } else { - value = pvalue; - } - } - return ret; -} - -ObTabletLSCache::ObTabletLSCache() - : cache_key_(), - ls_id_(), - renew_time_(0) -{ -} - - -void ObTabletLSCache::reset() -{ - cache_key_.reset(); - ls_id_.reset(); - renew_time_ = 0; -} - -int ObTabletLSCache::assign(const ObTabletLSCache &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - cache_key_ = other.cache_key_; - ls_id_ = other.ls_id_; - renew_time_ = other.renew_time_; - } - return ret; -} - -bool ObTabletLSCache::is_valid() const -{ - return cache_key_.is_valid() - && ls_id_.is_valid() - && renew_time_ > 0; -} - -bool ObTabletLSCache::operator==(const ObTabletLSCache &other) const -{ - return cache_key_ == other.cache_key_ - && ls_id_ == other.ls_id_ - && renew_time_ == other.renew_time_; -} - - -int ObTabletLSCache::init( - const ObTabletID &tablet_id, - const ObLSID &ls_id, - const int64_t renew_time) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(cache_key_.init(tablet_id))) { - LOG_WARN("fail to init ObTabletLSCache", KR(ret), K(tablet_id)); - } else { - ls_id_ = ls_id; - renew_time_ = renew_time; - ObLink::reset(); - } - return ret; -} - -ObLSLocationCacheKey::ObLSLocationCacheKey() - : cluster_id_(OB_INVALID_CLUSTER_ID), - ls_id_() -{ -} - -ObLSLocationCacheKey::ObLSLocationCacheKey( - const int64_t cluster_id, - const ObLSID ls_id) - : cluster_id_(cluster_id), ls_id_(ls_id) -{ -} - -int ObLSLocationCacheKey::init( - const int64_t cluster_id, - const ObLSID ls_id) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!ls_id.is_valid_with_tenant() - || cluster_id == OB_INVALID_CLUSTER_ID - || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls location cache key init error", K(ret), K(cluster_id), K(ls_id)); - } else { - cluster_id_ = cluster_id; - ls_id_ = ls_id; - } - return ret; -} - -int ObLSLocationCacheKey::assign(const ObLSLocationCacheKey &other) -{ - int ret = OB_SUCCESS; - cluster_id_ = other.cluster_id_; - ls_id_ = other.ls_id_; - return ret; -} - -void ObLSLocationCacheKey::reset() -{ - cluster_id_ = OB_INVALID_CLUSTER_ID; - ls_id_.reset(); -} - -bool ObLSLocationCacheKey::operator ==(const ObLSLocationCacheKey &other) const -{ - return cluster_id_ == other.cluster_id_ - && ls_id_ == other.ls_id_; -} - -bool ObLSLocationCacheKey::operator !=(const ObLSLocationCacheKey &other) const -{ - return !(*this == other); -} - -bool ObLSLocationCacheKey::is_valid() const -{ - return OB_INVALID_CLUSTER_ID != cluster_id_ - && true - && ls_id_.is_valid(); -} - -uint64_t ObLSLocationCacheKey::hash() const -{ - uint64_t hash_val = 0; - hash_val = murmurhash(&cluster_id_, sizeof(cluster_id_), hash_val); - hash_val = murmurhash(&ls_id_, sizeof(ls_id_), hash_val); - return hash_val; -} - -int ObTabletLSKey::init(const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!tablet_id.is_valid_with_tenant())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tablet_id", KR(ret), K(tablet_id)); - } else { - - tablet_id_ = tablet_id; - } - return ret; -} - -void ObTabletLSKey::reset() -{ - - tablet_id_.reset(); -} - -bool ObTabletLSKey::is_valid() const -{ - return true - && tablet_id_.is_valid(); -} - -uint64_t ObTabletLSKey::hash() const -{ - uint64_t hash_val = 0; - hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); - return hash_val; -} - -bool ObTabletLSKey::operator ==(const ObTabletLSKey &other) const -{ - return tablet_id_ == other.tablet_id_; -} - -bool ObTabletLSCacheKey::operator ==(const ObIKVCacheKey &other) const -{ - const ObTabletLSCacheKey &other_key - = reinterpret_cast(other); - return true - && tablet_id_ == other_key.tablet_id_; -} - -bool ObTabletLSCacheKey::operator !=(const ObIKVCacheKey &other) const -{ - return !(*this == other); -} - -bool ObTabletLSCacheKey::is_valid() const -{ - return true - && tablet_id_.is_valid(); -} - -uint64_t ObTabletLSCacheKey::hash() const -{ - uint64_t hash_val = 0; - hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), 0); - return hash_val; -} - -int ObTabletLSCacheKey::deep_copy( - char *buf, - const int64_t buf_len, - ObIKVCacheKey *&key) const -{ - int ret = OB_SUCCESS; - if (NULL == buf || buf_len < size()) { - ret = common::OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(buf), K(buf_len), "size", size()); - } else { - ObTabletLSCacheKey *pkey = new (buf) ObTabletLSCacheKey(); - *pkey = *this; - key = pkey; - } - return ret; -} - ObLocationSem::ObLocationSem() : cur_count_(0), max_count_(0), cond_() { cond_.init(ObWaitEventIds::LOCATION_CACHE_COND_WAIT); diff --git a/src/share/location_cache/ob_location_struct.h b/src/share/location_cache/ob_location_struct.h index fdfde91e5..ab7b1db3c 100644 --- a/src/share/location_cache/ob_location_struct.h +++ b/src/share/location_cache/ob_location_struct.h @@ -20,10 +20,7 @@ #include "lib/ob_replica_define.h" #include "common/ob_role.h" -#include "common/ob_tablet_id.h" #include "share/ob_define.h" -#include "share/cache/ob_kv_storecache.h" -#include "share/ob_ls_id.h" #include "share/ob_share_util.h" // for ObShareUtil #include "share/ls/ob_ls_restore_status.h" #include "lib/lock/ob_thread_cond.h" @@ -37,14 +34,6 @@ class ObReplicaProperty; } namespace share { -static inline bool is_location_service_renew_error(const int err) -{ - return err == OB_LOCATION_NOT_EXIST - || err == OB_LS_LOCATION_NOT_EXIST - || err == OB_LS_LOCATION_LEADER_NOT_EXIST - || err == OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST; -} - class ObLSReplicaLocation { OB_UNIS_VERSION(1); @@ -107,181 +96,6 @@ class ObLSReplicaLocation int64_t proposal_id_; // only leader's proposal_id_ is useful }; -class ObLSLocationCacheKey -{ - OB_UNIS_VERSION(1); -public: - ObLSLocationCacheKey(); - ObLSLocationCacheKey( - const int64_t cluster_id, - const ObLSID ls_id); - virtual ~ObLSLocationCacheKey() {} - int init( - const int64_t cluster_id, - const ObLSID ls_id); - int assign(const ObLSLocationCacheKey &other); - void reset(); - bool operator ==(const ObLSLocationCacheKey &other) const; - bool operator !=(const ObLSLocationCacheKey &other) const; - bool is_valid() const; - uint64_t hash() const; - int64_t size() const { return sizeof(*this); } - - inline ObLSID get_ls_id() const { return ls_id_; } - inline int64_t get_cluster_id() const { return cluster_id_; } - TO_STRING_KV(K_(ls_id), K_(cluster_id)); -private: - int64_t cluster_id_; - ObLSID ls_id_; -}; - -class ObLSLocation : public common::ObLink -{ - OB_UNIS_VERSION(1); -public: - typedef common::ObSEArray ObLSReplicaLocations; - ObLSLocation(); - explicit ObLSLocation(common::ObIAllocator &allocator); - ~ObLSLocation(); - int deep_copy(const ObLSLocation &ls_location); - int init(const int64_t cluster_id, const ObLSID &ls_id, const int64_t renew_time); - int init_fake_location(); // make fake location for virtual table in __all_virtual_proxy_schema - void reset(); - int assign(const ObLSLocation &ls_location); - bool is_valid() const; - uint64_t hash() const { return cache_key_.hash(); } - // compare key and locations with other, ignoring the timestamp - bool is_same_with(const ObLSLocation &other) const; - // compare all private members with other - bool operator==(const ObLSLocation &other) const; - int add_replica_location(const ObLSReplicaLocation &replica_location); - - inline ObLSID get_ls_id() const { return cache_key_.get_ls_id(); } - const ObLSLocationCacheKey &get_cache_key() const { return cache_key_; } - int get_replica_count(int64_t &full_replica_cnt, int64_t &non_paxos_replica_cnt); - inline const common::ObIArray &get_replica_locations() const - { - return replica_locations_; - } - inline int64_t get_renew_time() const { return renew_time_; } - inline void set_renew_time(const int64_t renew_time) { renew_time_ = renew_time; }; - int get_leader(common::ObAddr &leader) const; - int get_leader(ObLSReplicaLocation &leader) const; - void set_last_access_ts(const int64_t ts) { last_access_ts_ = ts; } - int64_t get_last_access_ts() const { return last_access_ts_; } - int merge_leader_from(const ObLSLocation &new_location); - static const int64_t OB_MAX_LOCATION_SERIALIZATION_SIZE = common::OB_MALLOC_BIG_BLOCK_SIZE; - TO_STRING_KV(K_(cache_key), K_(renew_time), K_(replica_locations)); -protected: - ObLSLocationCacheKey cache_key_; - int64_t renew_time_; // renew location by sql/rpc - int64_t last_access_ts_; - ObLSReplicaLocations replica_locations_; -}; - -class ObTabletLocation : public common::ObIKVCacheValue -{ - OB_UNIS_VERSION(1); -public: - typedef common::ObSEArray ObLSReplicaLocations; - ObTabletLocation(); - explicit ObTabletLocation(common::ObIAllocator &allocator); - virtual ~ObTabletLocation(); - virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; - virtual int64_t size() const; - int init_fake_location(); // make fake location for virtual table in __all_virtual_proxy_schema - void reset(); - int assign(const ObTabletLocation &ls_location); - bool is_valid() const; - bool operator==(const ObTabletLocation &other) const; - - - inline ObTabletID get_tablet_id() const { return tablet_id_; } - inline void set_tablet_id(const ObTabletID &tablet_id) - { - tablet_id_ = tablet_id; - } - inline void set_renew_time(const int64_t renew_time) { renew_time_ = renew_time; }; - inline int64_t get_renew_time() const { return renew_time_; } - int get_leader(ObLSReplicaLocation &leader) const; - int add_replica_location(const ObLSReplicaLocation &replica_location); - TO_STRING_KV(K_(tablet_id), K_(renew_time), K_(replica_locations)); -private: - - ObTabletID tablet_id_; - int64_t renew_time_; - ObLSReplicaLocations replica_locations_; -}; - -class ObTabletLSKey -{ - OB_UNIS_VERSION(1); -public: - ObTabletLSKey() : tablet_id_() {} - ObTabletLSKey(const ObTabletID &tablet_id) - : tablet_id_(tablet_id) {} - ~ObTabletLSKey() {} - int init(const ObTabletID &tablet_id); - void reset(); - bool is_valid() const; - uint64_t hash() const; - bool operator ==(const ObTabletLSKey &other) const; - int64_t size() const { return sizeof(*this); } - - inline ObTabletID get_tablet_id() const { return tablet_id_; } - TO_STRING_KV(K_(tablet_id)); -private: - - ObTabletID tablet_id_; -}; - -class ObTabletLSCache : public common::ObLink -{ - OB_UNIS_VERSION(1); -public: - ObTabletLSCache(); - virtual ~ObTabletLSCache() {} - void reset(); - int assign(const ObTabletLSCache &other); - bool is_valid() const; - bool operator==(const ObTabletLSCache &other) const; - - inline ObTabletID get_tablet_id() const { return cache_key_.get_tablet_id(); } - inline ObLSID get_ls_id() const { return ls_id_; } - inline int64_t get_renew_time() const { return renew_time_; } - const ObTabletLSKey &get_cache_key() const { return cache_key_; } - int init( - const ObTabletID &tablet_id, - const ObLSID &ls_id, - const int64_t renew_time); - TO_STRING_KV(K_(cache_key), K_(ls_id), K_(renew_time)); -private: - ObTabletLSKey cache_key_; - ObLSID ls_id_; - int64_t renew_time_; // renew by sql -}; - -// Reserved compatibility cache key. -class ObTabletLSCacheKey : public common::ObIKVCacheKey -{ -public: - ObTabletLSCacheKey() : tablet_id_() {} - ObTabletLSCacheKey(const ObTabletID tablet_id) - : tablet_id_(tablet_id) {} - virtual ~ObTabletLSCacheKey() {} - virtual bool operator ==(const ObIKVCacheKey &other) const; - virtual bool operator !=(const ObIKVCacheKey &other) const; - virtual bool is_valid() const; - virtual uint64_t hash() const; - virtual int64_t size() const { return sizeof(*this); } - virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const; - - inline ObTabletID get_tablet_id() const { return tablet_id_; } - TO_STRING_KV(K_(tablet_id)); -private: - ObTabletID tablet_id_; -}; - class ObLocationSem { public: diff --git a/src/share/ob_all_tenant_info.h b/src/share/ob_all_tenant_info.h index c15d08b40..6dea9aa1f 100644 --- a/src/share/ob_all_tenant_info.h +++ b/src/share/ob_all_tenant_info.h @@ -33,7 +33,6 @@ class ObISQLClient; namespace share { -// Simplified ObAllTenantInfo for single tenant/single LS scenario // tenant_role is persisted via config parameter (tenant_role), not in KV storage struct ObAllTenantInfo { @@ -79,7 +78,6 @@ struct ObAllTenantInfo OB_UNIS_VERSION(1); }; -// Simplified ObAllTenantInfoProxy for single tenant/single LS scenario // Uses KV storage instead of database table class ObAllTenantInfoProxy { diff --git a/src/share/ob_autoincrement_service.cpp b/src/share/ob_autoincrement_service.cpp index 844fd65e6..4f5c65e54 100644 --- a/src/share/ob_autoincrement_service.cpp +++ b/src/share/ob_autoincrement_service.cpp @@ -18,9 +18,9 @@ #include "lib/stat/ob_diagnostic_info_guard.h" #include "share/ob_autoincrement_service.h" -#include "share/location_cache/ob_location_service.h" // previously hidden behind a transitive include(free within share) +#include "share/ob_global_autoinc_service.h" #include "share/ob_sql_client_decorator.h" -#include "lib/stat/ob_diagnostic_info_guard.h" +#include "share/rc/ob_module_provider.h" #include "lib/wait_event/ob_inner_sql_wait_type.h" using namespace oceanbase::obcall; @@ -44,11 +44,6 @@ namespace share #define UINT24_MAX (16777215U) #endif -static bool is_rpc_error(int error_code) -{ - return (is_timeout_err(error_code) || is_server_down_error(error_code)); -} - int CacheNode::combine_cache_node(CacheNode &new_node) { int ret = OB_SUCCESS; @@ -241,12 +236,10 @@ ObAutoincrementService &ObAutoincrementService::get_instance() return autoinc_service; } -int ObAutoincrementService::init(ObAddr &addr, - ObMySQLProxy *mysql_proxy, +int ObAutoincrementService::init(ObMySQLProxy *mysql_proxy, ObMultiVersionSchemaService *schema_service) { int ret = OB_SUCCESS; - my_addr_ = addr; mysql_proxy_ = mysql_proxy; schema_service_ = schema_service; @@ -254,8 +247,6 @@ int ObAutoincrementService::init(ObAddr &addr, SET_USE_500(attr); if (OB_FAIL(distributed_autoinc_service_.init(mysql_proxy))) { LOG_WARN("fail init distributed_autoinc_service_ service", K(ret)); - } else if (OB_FAIL(global_autoinc_service_.init(my_addr_))) { - LOG_WARN("fail init auto inc global service", K(ret)); } else if (OB_FAIL(node_allocator_.init(sizeof(TableNode), attr))) { LOG_WARN("failed to init table node allocator", K(ret)); } else if (OB_FAIL(handle_allocator_.init(sizeof(CacheHandle), attr))) { @@ -585,21 +576,18 @@ void ObAutoincrementService::release_handle(CacheHandle *&handle) handle = NULL; } -int ObAutoincrementService::refresh_sync_value(const obcall::ObAutoincSyncArg &arg) +int ObAutoincrementService::refresh_sync_value_(const uint64_t table_id, + const uint64_t column_id, + const uint64_t sync_value) { int ret = OB_SUCCESS; - LOG_DEBUG("begin to get global sync", K(arg)); + LOG_DEBUG("begin to refresh local sync value", K(table_id), K(column_id), K(sync_value)); TableNode *table_node = NULL; - AutoincKey key; - - - key.table_id_ = arg.table_id_; - key.column_id_ = arg.column_id_; + const AutoincKey key(table_id, column_id); if (OB_ENTRY_NOT_EXIST == (ret = node_map_.get(key, table_node))) { - LOG_TRACE("there is no cache here", K(arg)); + LOG_TRACE("there is no local cache", K(key)); ret = OB_SUCCESS; } else if (OB_SUCC(ret)) { - const uint64_t sync_value = arg.sync_value_; lib::ObMutexGuard guard(table_node->sync_mutex_); atomic_update(table_node->local_sync_, sync_value); atomic_update(table_node->last_refresh_ts_, ObTimeUtility::current_time()); @@ -788,61 +776,16 @@ int ObAutoincrementService::calculate_idempotent_autoinc_val_for_ddl( return ret; } -int ObAutoincrementService::clear_autoinc_cache_all(const uint64_t table_id, - const uint64_t column_id, - const bool autoinc_is_order, - const bool ignore_rpc_errors /*true*/) +int ObAutoincrementService::clear_autoinc_cache(const uint64_t table_id, + const uint64_t column_id) { int ret = OB_SUCCESS; - if (OB_SUCC(ret)) { - // sync observer where table partitions exist - LOG_INFO("begin to clear all sever's auto-increment cache", - K(table_id), K(column_id), K(autoinc_is_order)); - ObAutoincSyncArg arg; - - - arg.table_id_ = table_id; - arg.column_id_ = column_id; - ObHashSet server_set; - if (OB_FAIL(server_set.create(PARTITION_LOCATION_SET_BUCKET_NUM))) { - LOG_WARN("failed to create hash set", K(ret)); - } else if (OB_FAIL(get_server_set(table_id, server_set, true))) { - SHARE_LOG(WARN, "failed to get table partitions server set", K(ret)); - } - if (OB_SUCC(ret)) { - const int64_t sync_timeout = SYNC_OP_TIMEOUT + TIME_SKEW; - ObHashSet::iterator iter; - for(iter = server_set.begin(); OB_SUCC(ret) && iter != server_set.end(); ++iter) { - LOG_INFO("send rpc call to other observers", "server", iter->first, K(table_id)); - if (OB_FAIL(ObAutoincrementService::get_instance().clear_autoinc_cache(arg))) { - if (ignore_rpc_errors && (is_rpc_error(ret) || autoinc_is_order)) { - // ignore time out and clear ordered auto increment cache error, go on - LOG_WARN("rpc call time out, ignore the error", "server", iter->first, - K(table_id), K(autoinc_is_order), K(ret)); - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to send rpc call", K(table_id), K(ret)); - } - } - } - } - } - return ret; -} - -int ObAutoincrementService::clear_autoinc_cache(const obcall::ObAutoincSyncArg &arg) -{ - int ret = OB_SUCCESS; - LOG_INFO("begin to clear local auto-increment cache", K(arg)); - // auto-increment key - AutoincKey key; - - key.table_id_ = arg.table_id_; - key.column_id_ = arg.column_id_; + LOG_INFO("begin to clear local auto-increment cache", K(table_id), K(column_id)); + const AutoincKey key(table_id, column_id); map_mutex_.lock(); if (OB_FAIL(node_map_.del(key))) { - LOG_WARN("failed to erase table node", K(arg), K(ret)); + LOG_WARN("failed to erase table node", K(key), K(ret)); } map_mutex_.unlock(); @@ -1104,88 +1047,6 @@ int ObAutoincrementService::set_pre_op_timeout(common::ObTimeoutCtx &ctx) return ret; } -//FIXME: use location_service instead -// TODO shengle -int ObAutoincrementService::get_server_set(const uint64_t table_id, - common::hash::ObHashSet &server_set, - const bool get_follower) -{ - int ret = OB_SUCCESS; - const int64_t expire_renew_time = INT64_MAX; // means must renew location - bool is_cache_hit = false; - ObSchemaGetterGuard schema_guard; - bool is_found = false; - ObLSID ls_id; - ObLSLocation ls_loc; - ObSEArray tablet_ids; - ObSEArray ls_locations; - const ObSimpleTableSchemaV2 *table_schema = nullptr; - - if (OB_ISNULL(schema_service_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(schema_service_)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(schema_guard))) { - LOG_WARN("failed to get tenant schema guard", K(ret)); - } else if (OB_FAIL(schema_guard.get_simple_table_schema( table_id, table_schema))) { - LOG_WARN("fail to get table schema", K(ret),K(table_id)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_SCHEMA_EAGAIN; - LOG_WARN("fail to get table schema", KR(ret)); - } else if (OB_FAIL(table_schema->get_tablet_ids(tablet_ids))) { - LOG_WARN("fail to get tablet ids", K(ret)); - } else { - ObLSLocation ls_loc; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(GCTX.location_service_->get(tablet_ids.at(i), - expire_renew_time, - is_cache_hit, - ls_id))) { - LOG_WARN("fail to get ls id", K(ret), K(tablet_ids.at(i))); - } else if (OB_FAIL(GCTX.location_service_->get(GCONF.cluster_id, - ls_id, - 0, /*not force to renew*/ - is_cache_hit, - ls_loc))) { - LOG_WARN("failed to get all partitions' locations", K(table_id), K(ret)); - } - OZ(ls_locations.push_back(ls_loc)); - } - } - - if (OB_SUCC(ret)) { - LOG_TRACE("get partition locations", K(ls_locations), K(ret)); - ObLSReplicaLocation location; - for(int64_t i = 0; OB_SUCC(ret) && i < ls_locations.count(); ++i) { - if (!get_follower) { - location.reset(); - if (OB_FAIL(ls_locations.at(i).get_leader(location))) { - LOG_WARN("failed to get partition leader", "location", ls_locations.at(i), K(ret)); - } else { - int err = OB_SUCCESS; - err = server_set.set_refactored(location.get_server()); - if (OB_SUCCESS != err && OB_HASH_EXIST != err) { - ret = err; - LOG_WARN("failed to add element to set", "server", location.get_server(), K(ret)); - } - } - } else { - const ObIArray &replica_locations = ls_locations.at(i).get_replica_locations(); - for (int j = 0; j < replica_locations.count(); ++j) { - int err = OB_SUCCESS; - err = server_set.set_refactored(replica_locations.at(j).get_server()); - if (OB_SUCCESS != err && OB_HASH_EXIST != err) { - ret = err; - LOG_WARN("failed to add element to set", - "server", replica_locations.at(j).get_server(), K(ret)); - } - } - } - } - } - LOG_DEBUG("sync server_set", K(server_set), K(ret)); - return ret; -} - int ObAutoincrementService::sync_insert_value_order(AutoincParam ¶m, CacheHandle *&cache_handle, const uint64_t insert_value) @@ -1289,9 +1150,7 @@ int ObAutoincrementService::sync_insert_value_noorder(AutoincParam ¶m, atomic_update(table_node->last_refresh_ts_, ObTimeUtility::current_time()); } else { atomic_update(table_node->local_sync_, value_to_sync); - if (OB_FAIL(sync_value_to_other_servers(param, value_to_sync))) { - LOG_WARN("fail sync value to other servers", K(ret)); - } + atomic_update(table_node->last_refresh_ts_, ObTimeUtility::current_time()); } table_node->alloc_mutex_.unlock(); } @@ -1363,72 +1222,6 @@ int ObAutoincrementService::sync_insert_value_noorder(AutoincParam ¶m, return ret; } -int ObAutoincrementService::sync_value_to_other_servers( - AutoincParam ¶m, - uint64_t insert_value) -{ - int ret = OB_SUCCESS; - // sync observer where table partitions exist - LOG_DEBUG("begin to sync other servers", K(param)); - ObAutoincSyncArg arg; - - - arg.table_id_ = param.autoinc_table_id_; - arg.column_id_ = param.autoinc_col_id_; - arg.sync_value_ = insert_value; - ObHashSet server_set; - if (OB_FAIL(server_set.create(PARTITION_LOCATION_SET_BUCKET_NUM))) { - LOG_WARN("failed to create hash set", K(param), K(ret)); - } else { - if (OB_FAIL(get_server_set(param.autoinc_table_id_, server_set))) { - SHARE_LOG(WARN, "failed to get table partitions server set", K(ret)); - } - - if (OB_SUCC(ret)) { - LOG_DEBUG("server_set after remove self", K(server_set), K(ret)); - ObHashSet::iterator iter; - // this operation succeeds in two case: - // 1. sync to all servers (without timeout) - // 2. reach SYNC_TIMEOUT (without worker timeout) - // other cases won't succeed: - // 1. worker timeout - // 2. rpc calls return failure (not OB_TIMEOUT) - int64_t sync_us = 0; - const int64_t start_us = ObTimeUtility::current_time(); - const int64_t sync_timeout = SYNC_OP_TIMEOUT + TIME_SKEW; - for(iter = server_set.begin(); OB_SUCC(ret) && iter != server_set.end(); ++iter) { - LOG_INFO("send rpc call to other observers", "server", iter->first); - sync_us = THIS_WORKER.get_timeout_remain(); - if (sync_us > sync_timeout) { - sync_us = sync_timeout; - } - // TODO: send sync_value to other servers - if (OB_FAIL(ObAutoincrementService::get_instance().refresh_sync_value(arg))) { - if (is_rpc_error(ret)) { - LOG_WARN("sync rpc call time out", "server", iter->first, K(sync_us), K(param), K(ret)); - if (!THIS_WORKER.is_timeout()) { - // reach SYNC_TIMEOUT, go on - // ignore time out - ret = OB_SUCCESS; - break; - } - } else { - LOG_WARN("failed to send rpc call", K(param), K(ret)); - } - } else if (ObTimeUtility::current_time() - start_us >= sync_timeout) { - // reach SYNC_TIMEOUT, go on - LOG_INFO("reach SYNC_TIMEOUT, go on", "server", iter->first, K(param), K(ret)); - break; - } else if (THIS_WORKER.is_timeout()) { - ret = OB_TIMEOUT; - LOG_WARN("failed to sync insert value; worker timeout", K(param), K(ret)); - break; - } - } - } - } - return ret; -} // sync last user specified value first(compatible with MySQL) int ObAutoincrementService::sync_insert_value_global(AutoincParam ¶m) { @@ -1469,59 +1262,12 @@ int ObAutoincrementService::sync_insert_value_local(AutoincParam ¶m) return ret; } -int ObAutoincrementService::sync_auto_increment_all(const uint64_t table_id, - const uint64_t column_id, - const uint64_t sync_value) +int ObAutoincrementService::refresh_auto_increment_cache(const uint64_t table_id, + const uint64_t column_id, + const uint64_t sync_value) { - int ret = OB_SUCCESS; - LOG_INFO("begin to sync auto_increment value", K(sync_value), K(table_id)); - - // if (OB_SUCC(ret)) { - // AutoincKey key(sys tenant, table_id, column_id); - // if(OB_FAIL(sync_table_auto_increment(sys tenant, key, sync_value))) { - // LOG_WARN("fail sync table auto increment value", K(ret)); - // } - // } - - if (OB_SUCC(ret)) { - // sync observer where table partitions exist - ObAutoincSyncArg arg; - - arg.table_id_ = table_id; - arg.column_id_ = column_id; - arg.sync_value_ = sync_value; - ObHashSet server_set; - if (OB_FAIL(server_set.create(PARTITION_LOCATION_SET_BUCKET_NUM))) { - LOG_WARN("failed to create hash set", K(ret)); - } else { - if (OB_FAIL(get_server_set(table_id, server_set, true))) { - SHARE_LOG(WARN, "failed to get table partitions server set", K(ret)); - } - LOG_DEBUG("sync server_set", K(server_set), K(ret)); - if (OB_SUCC(ret)) { - const int64_t sync_timeout = SYNC_OP_TIMEOUT + TIME_SKEW; - ObHashSet::iterator iter; - for(iter = server_set.begin(); OB_SUCC(ret) && iter != server_set.end(); ++iter) { - LOG_DEBUG("send rpc call to other observers", "server", iter->first); - if (OB_FAIL(ObAutoincrementService::get_instance().refresh_sync_value(arg))) { - if (is_rpc_error(ret)) { - // ignore time out, go on - LOG_WARN("rpc call time out", "server", iter->first, K(ret)); - if (!THIS_WORKER.is_timeout()) { - // reach SYNC_TIMEOUT, go on - // ignore time out - ret = OB_SUCCESS; - break; - } - } else { - LOG_WARN("failed to send rpc call", K(ret)); - } - } - } - } - } - } - return ret; + LOG_INFO("begin to refresh auto_increment cache", K(sync_value), K(table_id), K(column_id)); + return refresh_sync_value_(table_id, column_id, sync_value); } @@ -1760,24 +1506,6 @@ int ObInnerTableGlobalAutoIncrementService::local_sync_with_global_value( return inner_table_proxy_.get_autoinc_value(key, autoinc_version, seq_value, sync_value); } -int ObCallGlobalAutoIncrementService::init( - const common::ObAddr &addr) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - LOG_WARN("ObCallGlobalAutoIncrementService inited twice", KR(ret)); - } else if (!addr.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(addr)); - } else if (OB_FAIL(gais_request_rpc_.init(addr))) { - LOG_WARN("response rpc init failed", K(ret), K(addr)); - } else if (OB_FAIL(gais_client_.init(addr, &gais_request_rpc_))) { - LOG_WARN("init client failed", K(ret)); - } - return ret; -} - int ObCallGlobalAutoIncrementService::get_value( const AutoincKey &key, const uint64_t offset, @@ -1791,31 +1519,79 @@ int ObCallGlobalAutoIncrementService::get_value( uint64_t &start_inclusive, uint64_t &end_inclusive) { - return gais_client_.get_value(key, offset, increment, max_value, table_auto_increment, - desired_count, cache_size, autoinc_version,sync_value, - start_inclusive, end_inclusive); + int ret = OB_SUCCESS; + ObGAISNextAutoIncValReq request; + obcall::ObGAISNextValResult result; + if (OB_FAIL(request.init(key, offset, increment, table_auto_increment, max_value, + desired_count, cache_size, autoinc_version))) { + LOG_WARN("fail to init auto increment request", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_next_autoinc_request( + request, result))) { + LOG_WARN("fail to allocate auto increment value", KR(ret), K(request)); + } else { + start_inclusive = result.start_inclusive_; + end_inclusive = result.end_inclusive_; + sync_value = result.sync_value_; + } + return ret; } int ObCallGlobalAutoIncrementService::get_sequence_next_value( const ObSequenceSchema &schema, ObSequenceValue &nextval) { - return gais_client_.get_sequence_next_value(schema, nextval); + int ret = OB_SUCCESS; + ObGAISNextSequenceValReq request; + obcall::ObGAISNextSequenceValResult result; + if (OB_FAIL(request.init(schema))) { + LOG_WARN("fail to init sequence request", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_next_sequence_request( + request, result))) { + LOG_WARN("fail to allocate sequence value", KR(ret)); + } else { + nextval.assign(result.nextval_); + } + return ret; } int ObCallGlobalAutoIncrementService::get_sequence_value(const AutoincKey &key, const int64_t &autoinc_version, uint64_t &sequence_value) { - return gais_client_.get_sequence_value(key, autoinc_version, sequence_value); + int ret = OB_SUCCESS; + ObGAISAutoIncKeyArg request; + obcall::ObGAISCurrValResult result; + if (OB_FAIL(request.init(key, autoinc_version))) { + LOG_WARN("fail to init auto increment key", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_curr_autoinc_request( + request, result))) { + LOG_WARN("fail to get auto increment value", KR(ret), K(key)); + } else { + sequence_value = result.sequence_value_; + } + return ret; } int ObCallGlobalAutoIncrementService::get_auto_increment_values( const common::ObIArray &autoinc_keys, const common::ObIArray &autoinc_versions, - common::hash::ObHashMap &seq_valuesm) + common::hash::ObHashMap &seq_values) { - return gais_client_.get_auto_increment_values(autoinc_keys, autoinc_versions, seq_valuesm); + int ret = OB_SUCCESS; + ObGAISAutoIncKeyArg request; + obcall::ObGAISCurrValResult result; + for (int64_t i = 0; OB_SUCC(ret) && i < autoinc_keys.count(); ++i) { + result.reset(); + if (OB_FAIL(request.init(autoinc_keys.at(i), autoinc_versions.at(i)))) { + LOG_WARN("fail to init auto increment key", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_curr_autoinc_request( + request, result))) { + LOG_WARN("fail to get auto increment value", KR(ret), K(request)); + } else if (OB_FAIL(seq_values.set_refactored(autoinc_keys.at(i), result.sequence_value_))) { + LOG_WARN("fail to save auto increment value", KR(ret)); + } + } + return ret; } int ObCallGlobalAutoIncrementService::local_push_to_global_value( @@ -1826,9 +1602,15 @@ int ObCallGlobalAutoIncrementService::local_push_to_global_value( const int64_t cache_size, uint64_t &global_sync_value) { - return gais_client_.local_push_to_global_value(key, max_value, value, autoinc_version, - cache_size, - global_sync_value); + int ret = OB_SUCCESS; + ObGAISPushAutoIncValReq request; + if (OB_FAIL(request.init(key, value, max_value, autoinc_version, cache_size))) { + LOG_WARN("fail to init auto increment sync request", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_push_autoinc_request( + request, global_sync_value))) { + LOG_WARN("fail to sync auto increment value", KR(ret), K(request)); + } + return ret; } int ObCallGlobalAutoIncrementService::local_sync_with_global_value( @@ -1836,12 +1618,31 @@ int ObCallGlobalAutoIncrementService::local_sync_with_global_value( const int64_t &autoinc_version, uint64_t &value) { - return gais_client_.local_sync_with_global_value(key, autoinc_version, value); + int ret = OB_SUCCESS; + ObGAISAutoIncKeyArg request; + obcall::ObGAISCurrValResult result; + if (OB_FAIL(request.init(key, autoinc_version))) { + LOG_WARN("fail to init auto increment key", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_curr_autoinc_request( + request, result))) { + LOG_WARN("fail to get auto increment value", KR(ret), K(request)); + } else { + value = result.sync_value_; + } + return ret; } int ObCallGlobalAutoIncrementService::clear_global_autoinc_cache(const AutoincKey &key) { - return gais_client_.clear_global_autoinc_cache(key); + int ret = OB_SUCCESS; + ObGAISAutoIncKeyArg request; + if (OB_FAIL(request.init(key, OB_INVALID_VERSION))) { + LOG_WARN("fail to init auto increment key", KR(ret)); + } else if (OB_FAIL(share::g_mp->global_auto_inc_service()->handle_clear_autoinc_cache_request( + request))) { + LOG_WARN("fail to clear auto increment cache", KR(ret), K(request)); + } + return ret; } int ObAutoIncInnerTableProxy::check_inner_autoinc_version(const int64_t &request_autoinc_version, diff --git a/src/share/ob_autoincrement_service.h b/src/share/ob_autoincrement_service.h index 7311ca86e..afa463256 100644 --- a/src/share/ob_autoincrement_service.h +++ b/src/share/ob_autoincrement_service.h @@ -25,7 +25,6 @@ #include "common/mysqlclient/ob_mysql_proxy.h" #include "common/ob_timeout_ctx.h" #include "share/ob_autoincrement_param.h" -#include "share/ob_gais_client.h" #include "share/ob_i_global_autoincrement_service.h" #include "share/ob_rpc_struct.h" #include "share/schema/ob_multi_version_schema_service.h" @@ -38,11 +37,8 @@ namespace schema { class ObTableSchema; } -static const int64_t TIME_SKEW = 100 * 1000; // 100ms, time skew static const int64_t PRE_OP_TIMEOUT = 500 * 1000; // 500ms, for prefetch or presync -static const int64_t SYNC_OP_TIMEOUT = 1000 * 1000; // 1000ms, for first sync static const int PRE_OP_THRESHOLD = 4; // for prefetch or presync -static const int64_t PARTITION_LOCATION_SET_BUCKET_NUM = 3; static const int64_t FETCH_SEQ_NUM_ONCE = 1000; static const uint64_t AUTO_INC_DEFAULT_NB_MAX_BITS = 16; // from MySQL static const uint64_t AUTO_INC_DEFAULT_NB_MAX = (1 << AUTO_INC_DEFAULT_NB_MAX_BITS) - 1; // from MySQL @@ -296,11 +292,9 @@ class ObInnerTableGlobalAutoIncrementService : public ObIGlobalAutoIncrementServ class ObCallGlobalAutoIncrementService : public ObIGlobalAutoIncrementService { public: - ObCallGlobalAutoIncrementService() : is_inited_(false) {} + ObCallGlobalAutoIncrementService() = default; virtual ~ObCallGlobalAutoIncrementService() = default; - int init(const common::ObAddr &addr); - virtual int get_value( const AutoincKey &key, const uint64_t offset, @@ -339,14 +333,9 @@ class ObCallGlobalAutoIncrementService : public ObIGlobalAutoIncrementService uint64_t &value) override final; int clear_global_autoinc_cache(const AutoincKey &key); - ObGAISRequestRpc *get_gais_request_rpc() { return &gais_request_rpc_; } int get_sequence_next_value(const schema::ObSequenceSchema &schema, ObSequenceValue &nextval); -private: - bool is_inited_; - ObGAISClient gais_client_; - ObGAISRequestRpc gais_request_rpc_; }; class ObAutoincrementService @@ -359,8 +348,7 @@ class ObAutoincrementService ObAutoincrementService(); ~ObAutoincrementService(); static ObAutoincrementService &get_instance(); - int init(common::ObAddr &addr, - common::ObMySQLProxy *mysql_proxy, + int init(common::ObMySQLProxy *mysql_proxy, share::schema::ObMultiVersionSchemaService *schema_service); int get_handle(AutoincParam ¶m, CacheHandle *&handle); int get_handle(const schema::ObSequenceSchema &schema, ObSequenceValue &nextval); @@ -370,20 +358,11 @@ class ObAutoincrementService int sync_insert_value_local(AutoincParam ¶m); - int sync_auto_increment_all(const uint64_t table_id, - const uint64_t column_id, - const uint64_t sync_value); - // int sync_table_auto_increment( - // uint64_t tenant, - // AutoincKey &key, - // uint64_t auto_increment); - int refresh_sync_value(const obcall::ObAutoincSyncArg &arg); - - int clear_autoinc_cache_all(const uint64_t table_id, - const uint64_t column_id, - const bool autoinc_mode_is_order, - const bool ignore_rpc_errors = true); - int clear_autoinc_cache(const obcall::ObAutoincSyncArg &arg); + int refresh_auto_increment_cache(const uint64_t table_id, + const uint64_t column_id, + const uint64_t sync_value); + int clear_autoinc_cache(const uint64_t table_id, + const uint64_t column_id); int get_sequence_value(const uint64_t table_id, const uint64_t column_id, @@ -428,8 +407,6 @@ class ObAutoincrementService const uint64_t offset, const uint64_t increment, uint64_t &prev_value); - ObGAISRequestRpc* get_gais_request_rpc() - { return global_autoinc_service_.get_gais_request_rpc(); } static uint64_t get_max_value(const common::ObObjType type); private: @@ -449,12 +426,9 @@ class ObAutoincrementService const uint64_t column_id, TableNode &table_node, const bool sync_presync = false); - int get_server_set(const uint64_t table_id, - common::hash::ObHashSet &server_set, - const bool get_follower = false); - int sync_value_to_other_servers( - AutoincParam ¶m, - uint64_t insert_value); + int refresh_sync_value_(const uint64_t table_id, + const uint64_t column_id, + const uint64_t sync_value); int try_periodic_refresh_global_sync_value( uint64_t table_id, @@ -473,7 +447,6 @@ class ObAutoincrementService private: common::ObSmallAllocator node_allocator_; common::ObSmallAllocator handle_allocator_; - common::ObAddr my_addr_; common::ObMySQLProxy *mysql_proxy_; share::schema::ObMultiVersionSchemaService *schema_service_; ObCallGlobalAutoIncrementService global_autoinc_service_; // for order increment mode diff --git a/src/share/ob_ddl_common.cpp b/src/share/ob_ddl_common.cpp index a78c4c50a..459c9fd47 100644 --- a/src/share/ob_ddl_common.cpp +++ b/src/share/ob_ddl_common.cpp @@ -20,7 +20,6 @@ #include "ob_ddl_common.h" #include "common/datum/ob_datum.h" // ObDatum complete type(previously hidden behind the block_sstable_struct include chain) #include "share/ob_rpc_struct.h" -#include "share/location_cache/ob_location_service.h" #include "share/system_variable/ob_system_variable_alias.h" #include "lib/worker.h" #include "share/ob_ddl_checksum.h" @@ -69,12 +68,6 @@ const char *oceanbase::share::get_ddl_type(ObDDLType ddl_type) case ObDDLType::DDL_CREATE_FTS_INDEX: ret_name = "DDL_CREATE_FTS_INDEX"; break; - case ObDDLType::DDL_CREATE_MLOG: - ret_name = "DDL_CREATE_MLOG"; - break; - case ObDDLType::DDL_DROP_MLOG: - ret_name = "DDL_DROP_MLOG"; - break; case ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX: ret_name = "DDL_CREATE_PARTITIONED_LOCAL_INDEX"; break; @@ -179,12 +172,6 @@ const char *oceanbase::share::get_ddl_type(ObDDLType ddl_type) case ObDDLType::DDL_DIRECT_LOAD_INSERT: ret_name = "DDL_DIRECT_LOAD_INSERT"; break; - case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: - ret_name = "DDL_MVIEW_COMPLETE_REFRESH"; - break; - case ObDDLType::DDL_CREATE_MVIEW: - ret_name = "DDL_CREATE_MVIEW"; - break; case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: ret_name = "DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION"; break; @@ -777,71 +764,6 @@ int ObDDLUtil::append_multivalue_extra_column(const ObTableSchema &dest_table_sc } return ret; } - - - - -int ObDDLUtil::generate_order_by_str_for_mview(const ObTableSchema &container_table_schema, - ObSqlString &rowkey_column_sql_string) -{ - int ret = OB_SUCCESS; - rowkey_column_sql_string.reset(); - const ObRowkeyInfo &rowkey_info = container_table_schema.get_rowkey_info(); - if (container_table_schema.is_table_without_pk()) { - /* do nothing */ - } else if (OB_UNLIKELY(rowkey_info.get_size() < 1)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpcted rowkey size", K(ret), K(rowkey_info.get_size())); - } else if (OB_FAIL(rowkey_column_sql_string.append(" order by "))) { - LOG_WARN("append failed", KR(ret)); - } else { - uint64_t column_id = OB_INVALID_ID; - int64_t sel_pos = 0; - for (int col_idx = 0; OB_SUCC(ret) && col_idx < rowkey_info.get_size(); ++col_idx) { - if (OB_FAIL(rowkey_info.get_column_id(col_idx, column_id))) { - LOG_WARN("Failed to get column id", K(ret)); - } else if (OB_UNLIKELY(1 > (sel_pos = column_id - OB_APP_MIN_COLUMN_ID + 1))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpcted select position", K(ret), K(sel_pos), K(column_id)); - } else if (OB_FAIL(rowkey_column_sql_string.append_fmt("%s %ld", 0 != col_idx ? ",": "", sel_pos))) { - LOG_WARN("append fmt failed", K(ret)); - } - } - } - return ret; -} - -int ObDDLUtil::get_tablet_leader_addr( - share::ObLocationService *location_service, - const ObTabletID &tablet_id, - const int64_t timeout, - ObLSID &ls_id, - ObAddr &leader_addr) -{ - int ret = OB_SUCCESS; - const bool force_renew = true; - bool is_cache_hit = false; - leader_addr.reset(); - const int64_t expire_renew_time = force_renew ? INT64_MAX : 0; - ObTimeoutCtx timeout_ctx; - if (nullptr == location_service || !tablet_id.is_valid() || timeout <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), KP(location_service), K(tablet_id), K(timeout)); - } else if (OB_FAIL(timeout_ctx.set_trx_timeout_us(timeout))) { - LOG_WARN("set trx timeout failed", K(ret)); - } else if (OB_FAIL(timeout_ctx.set_timeout(timeout))) { - LOG_WARN("set timeout failed", K(ret)); - } else if (OB_FAIL(location_service->get(tablet_id, expire_renew_time, is_cache_hit, ls_id))) { - LOG_WARN("fail to get log stream id", K(ret), K(tablet_id)); - } else if (OB_FAIL(location_service->get_leader(GCONF.cluster_id, - ls_id, - force_renew, - leader_addr))) { - LOG_WARN("get leader failed", K(ret), K(ls_id)); - } - return ret; -} - // Used in offline ddl to delete all checksum record in __all_ddl_checksum // DELETE FROM __all_ddl_checksum WHERE @@ -996,143 +918,6 @@ int ObDDLUtil::get_table_lob_col_idx(const ObTableSchema &table_schema, ObIArray -bool ObDDLUtil::need_remote_write(const int ret_code) -{ - return ObTenantRole::PRIMARY_TENANT == MTL_GET_TENANT_ROLE_CACHE() - && (OB_NOT_MASTER == ret_code - || OB_NOT_RUNNING == ret_code - || OB_LS_LOCATION_LEADER_NOT_EXIST == ret_code - || OB_EAGAIN == ret_code); -} - - - -int ObDDLUtil::get_tablet_paxos_member_list(const common::ObTabletID &tablet_id, - common::ObIArray &paxos_server_list, - int64_t &paxos_member_count) -{ - int ret = OB_SUCCESS; - ObLSLocation location; - paxos_member_count = 0; - share::ObLSID unused_ls_id; - if (false || !tablet_id.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet replica location, invalid id", K(ret), K(tablet_id)); - } else if (OB_FAIL(get_tablet_replica_location(tablet_id, unused_ls_id, location))) { - LOG_WARN("fail to get tablet replica location", K(tablet_id), K(ret)); - } else { - const ObIArray &ls_locations = location.get_replica_locations(); - for (int64_t i = 0; i < ls_locations.count() && OB_SUCC(ret); ++i) { - common::ObReplicaType replica_type = ls_locations.at(i).get_replica_type(); - if (REPLICA_TYPE_FULL != replica_type && REPLICA_TYPE_LOGONLY != replica_type) { - continue; - } - paxos_member_count++; - if (REPLICA_TYPE_FULL == replica_type) { // paxos replica - const ObAddr &server = ls_locations.at(i).get_server(); - if (!has_exist_in_array(paxos_server_list, server) && OB_FAIL(paxos_server_list.push_back(server))) { - LOG_WARN("fail to push back addr", K(ret), K(server)); - } - } - } - } - return ret; -} - -int ObDDLUtil::get_tablet_replica_location(const common::ObTabletID &tablet_id, - ObLSID &ls_id, - ObLSLocation &location) -{ - int ret = OB_SUCCESS; - const int64_t cluster_id = GCONF.cluster_id; - int64_t expire_renew_time = INT64_MAX; - bool is_cache_hit = false; - ls_id.reset(); - if (OB_UNLIKELY(nullptr == GCTX.location_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("location service ptr is null", K(ret)); - } else if (false || !tablet_id.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tablet replica location, invalid id", K(ret), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->get(tablet_id, - INT64_MAX, - is_cache_hit, - ls_id))) { - LOG_WARN("fail to get ls id according to tablet_id", K(ret), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->get(cluster_id, - ls_id, - expire_renew_time, - is_cache_hit, - location))) { - LOG_WARN("fail to get ls location", K(ret), K(cluster_id), K(ls_id), K(tablet_id)); - } - return ret; -} - -// filter offline replica and arbitration one. -int ObDDLUtil::get_split_replicas_addrs(const share::ObLSID &ls_id, - ObIArray &member_addrs_array, - ObIArray &learner_addrs_array) -{ - int ret = OB_SUCCESS; - member_addrs_array.reset(); - learner_addrs_array.reset(); - if (OB_UNLIKELY(false || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id)); - } else if (OB_FAIL(member_addrs_array.push_back(GCTX.self_addr()))) { - LOG_WARN("fail to push back addr", KR(ret)); - } - return ret; -} - -int ObDDLUtil::get_split_replicas_addrs(const share::ObLSID &ls_id, - ObIArray &replica_addr_array) -{ - int ret = OB_SUCCESS; - replica_addr_array.reset(); - ObArray member_addrs_array; - ObArray learners_addr_array; - if (OB_FAIL(get_split_replicas_addrs(ls_id, member_addrs_array, learners_addr_array))) { - LOG_WARN("get addrs failed", K(ret), K(ls_id)); - } else if (OB_FAIL(append(replica_addr_array/*dst*/, member_addrs_array/*src*/))) { - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(append(replica_addr_array/*dst*/, learners_addr_array/*src*/))) { - LOG_WARN("append failed", K(ret)); - } - return ret; -} - -int ObDDLUtil::construct_ls_tablet_id_map( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - hash::ObHashMap> &ls_tablet_id_map) -{ - int ret = OB_SUCCESS; - bool is_cache_hit = false; - ObArray tablet_id_array; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls_tablet_id_map.get_refactored(ls_id, tablet_id_array))) { - if (OB_HASH_NOT_EXIST == ret) { // first time - ret = OB_SUCCESS; - if (OB_FAIL(tablet_id_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id), K(tablet_id_array)); - } else if (OB_FAIL(ls_tablet_id_map.set_refactored(ls_id, tablet_id_array, true /* overwrite */))) { - LOG_WARN("ls tablets map set fail", K(ret), K(ls_id), K(tablet_id_array)); - } - } else { - LOG_WARN("ls tablets map get fail", K(ret), K(ls_id), K(tablet_id_array)); - } - } else if (OB_FAIL(tablet_id_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id_array), K(tablet_id)); - } else if (OB_FAIL(ls_tablet_id_map.set_refactored(ls_id, tablet_id_array, true /* overwrite */))) { - LOG_WARN("ls tablets map set fail", K(ret), K(ls_id), K(tablet_id_array)); - } - return ret; -} - int ObDDLUtil::get_index_table_batch_partition_names( const int64_t &data_table_id, const int64_t &index_table_id, @@ -1218,12 +1003,11 @@ int ObDDLUtil::get_index_table_batch_partition_names( int ObDDLUtil::get_tablet_data_size( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, int64_t &data_size) { int ret = OB_SUCCESS; data_size = 0; - if (!tablet_id.is_valid() || !ls_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(GCTX.meta_db_pool_)) { @@ -1233,8 +1017,8 @@ int ObDDLUtil::get_tablet_data_size( ObTabletMetaTableStorage storage; if (OB_FAIL(storage.init(GCTX.meta_db_pool_))) { LOG_WARN("failed to init storage", K(ret)); - } else if (OB_FAIL(storage.get_max_data_size(tablet_id, ls_id, data_size))) { - LOG_WARN("failed to get max data size", K(ret), K(tablet_id), K(ls_id)); + } else if (OB_FAIL(storage.get_max_data_size(tablet_id, data_size))) { + LOG_WARN("failed to get max data size", K(ret), K(tablet_id)); } } return ret; @@ -1242,12 +1026,11 @@ int ObDDLUtil::get_tablet_data_size( int ObDDLUtil::get_tablet_data_row_cnt( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, int64_t &data_row_cnt) { int ret = OB_SUCCESS; data_row_cnt = 0; - if (!tablet_id.is_valid() || !ls_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(GCTX.meta_db_pool_)) { @@ -1257,47 +1040,36 @@ int ObDDLUtil::get_tablet_data_row_cnt( ObTabletReplicaChecksumTableStorage storage; if (OB_FAIL(storage.init(GCTX.meta_db_pool_))) { LOG_WARN("failed to init storage", K(ret)); - } else if (OB_FAIL(storage.get_max_row_count(tablet_id, ls_id, data_row_cnt))) { - LOG_WARN("failed to get max row count from storage", K(ret), K(tablet_id), K(ls_id)); + } else if (OB_FAIL(storage.get_max_row_count(tablet_id, data_row_cnt))) { + LOG_WARN("failed to get max row count from storage", K(ret), K(tablet_id)); } } return ret; } -int ObDDLUtil::get_ls_host_left_disk_space( - const share::ObLSID &ls_id, - const common::ObAddr &leader_addr, - uint64_t &left_space_size) +int ObDDLUtil::get_ls_host_left_disk_space(uint64_t &left_space_size) { int ret = OB_SUCCESS; left_space_size = 0; - if (!ls_id.is_valid() || !leader_addr.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(leader_addr)); - } else { - UNUSED(leader_addr); // vtable is local, no need to filter by server - { - SMART_VAR(ObMySQLProxy::MySQLResult, res) { - ObSqlString query_string; - sqlclient::ObMySQLResult *result = NULL; - if (OB_FAIL(query_string.assign_fmt("SELECT free_size FROM %s LIMIT 1", - OB_ALL_VIRTUAL_DISK_STAT_TNAME))) { - LOG_WARN("assign sql string failed", K(ret), K(OB_ALL_VIRTUAL_DISK_STAT_TNAME)); - } else if (OB_FAIL(GCTX.sql_proxy_->read(res, query_string.ptr()))) { - LOG_WARN("read record failed", K(ret), K(ls_id), K(leader_addr), K(query_string)); - } else if (OB_ISNULL(result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get sql result", K(ret), K(ls_id), K(leader_addr), K(query_string)); - } else if (OB_FAIL(result->next())) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get next", K(ret), K(ls_id), K(leader_addr), K(query_string)); - } - } else { - EXTRACT_INT_FIELD_MYSQL(*result, "free_size", left_space_size, uint64_t); - } + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString query_string; + sqlclient::ObMySQLResult *result = NULL; + if (OB_FAIL(query_string.assign_fmt("SELECT free_size FROM %s LIMIT 1", + OB_ALL_VIRTUAL_DISK_STAT_TNAME))) { + LOG_WARN("assign sql string failed", K(ret), K(OB_ALL_VIRTUAL_DISK_STAT_TNAME)); + } else if (OB_FAIL(GCTX.sql_proxy_->read(res, query_string.ptr()))) { + LOG_WARN("read record failed", K(ret), K(query_string)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get sql result", K(ret), K(query_string)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get next", K(ret), K(query_string)); } + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "free_size", left_space_size, uint64_t); } } return ret; @@ -1305,32 +1077,6 @@ int ObDDLUtil::get_ls_host_left_disk_space( -int ObDDLUtil::get_sys_ls_leader_addr( - const uint64_t cluster_id, - common::ObAddr &leader_addr) -{ - int ret = OB_SUCCESS; - bool force_renew = false; - share::ObLocationService *location_service = nullptr; - share::ObLSID ls_id(share::ObLSID::SYS_LS_ID); - - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to check and wait old completement, null pointer. ", K(ret)); - } else if (OB_FAIL(location_service->get_leader(cluster_id, - ls_id, - force_renew, - leader_addr))) { - LOG_WARN("failed to get ls_leader", K(ret)); - } else if (OB_UNLIKELY(!leader_addr.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("leader addr is invalid", K(ret), K(leader_addr), K(cluster_id)); - } else { - LOG_INFO("succ to get ls leader addr", K(cluster_id), K(leader_addr)); - } - return ret; -} - int ObDDLUtil::check_table_exist( const uint64_t table_id, ObSchemaGetterGuard &schema_guard) @@ -1720,11 +1466,6 @@ bool ObDDLUtil::use_idempotent_mode() // int64_t ObDDLUtil::get_parallel_idx moved definition to storage/ddl/ob_ddl_common_storage_impl.cpp(accesses blocksstable members) -bool ObDDLUtil::is_mview_not_retryable(const share::ObDDLType task_type) -{ - return (task_type == DDL_MVIEW_COMPLETE_REFRESH); -} - #ifdef OB_BUILD_SHARED_STORAGE int ObDDLUtil::upload_block_for_ss(const char *buf, const int64_t len, const blocksstable::MacroBlockId ¯o_block_id) @@ -2554,7 +2295,7 @@ int ObDDLDiagnoseInfo::generate_session_longops_message_v1(ObDDLTaskStatInfo &st MAX_LONG_OPS_MESSAGE_LENGTH, pos, "STATUS: REPLICA BUILD, PARALLELISM: %ld, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED: %ld", - ObDDLUtil::get_real_parallelism(parallelism_, false/*is mv refresh*/), + ObDDLUtil::get_real_parallelism(parallelism_), row_scanned_, row_sorted_ + row_merge_sorted_, row_inserted_file_))) { @@ -2703,92 +2444,11 @@ int ObCheckTabletDataComplementOp::update_replica_merge_status( } -// only get un-merge tablet replica ip addr -int ObCheckTabletDataComplementOp::construct_tablet_ip_map(const ObTabletID &tablet_id, - hash::ObHashMap> &ip_tablets_map) -{ - int ret = OB_SUCCESS; - common::ObArray paxos_server_list; - common::ObArray unfinished_replica_addrs; - common::ObArray tablet_array; - int64_t paxos_member_count; // unused, but need - - if (!tablet_id.is_valid() || false) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObDDLUtil::get_tablet_paxos_member_list(tablet_id, paxos_server_list, paxos_member_count))) { - LOG_WARN("fail to get tablet replica location!", K(ret), K(tablet_id)); - } else { - // classify un-merge tablet addr and tablet ids - for (int64_t i = 0; OB_SUCC(ret) && i < paxos_server_list.count(); ++i) { - tablet_array.reset(); - const ObAddr & addr = paxos_server_list.at(i); - if (OB_FAIL(ip_tablets_map.get_refactored(addr, tablet_array))) { - if (OB_HASH_NOT_EXIST == ret) { // first time - ret = OB_SUCCESS; - if (OB_FAIL(tablet_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id)); - } else if (OB_FAIL(ip_tablets_map.set_refactored(addr, tablet_array, true/* overwrite */))) { - LOG_WARN("set ip tablet map fail.", K(ret), K(tablet_id), K(addr)); - } - } else { - LOG_WARN("get ip tablet from map fail.", K(ret), K(tablet_id), K(addr)); - } - } else if (OB_FAIL(tablet_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id)); - } else if (OB_FAIL(ip_tablets_map.set_refactored(addr, tablet_array, true/* overwrite */))) { - LOG_WARN("set ip tablet map fail.", K(ret), K(tablet_id), K(addr)); - } - } - } - - return ret; -} - -int ObCheckTabletDataComplementOp::construct_ls_tablet_map(const common::ObTabletID &tablet_id, - hash::ObHashMap> &ls_tablets_map) -{ - int ret = OB_SUCCESS; - bool is_cache_hit = false; - share::ObLSID ls_id; - common::ObArray tablet_array; - - if (!tablet_id.is_valid() || false) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->get(tablet_id, - INT64_MAX, - is_cache_hit, /*is_cache_hit*/ - ls_id))) { - LOG_WARN("fail to get ls id according to tablet_id", K(ret), K(tablet_id)); - } else if (OB_FAIL(ls_tablets_map.get_refactored(ls_id, tablet_array))) { - if (OB_HASH_NOT_EXIST == ret) { // first time - ret = OB_SUCCESS; - if (OB_FAIL(tablet_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id)); - } else if (OB_FAIL(ls_tablets_map.set_refactored(ls_id, tablet_array, false))) { - LOG_WARN("ls_tablets_map set fail", K(ret), K(tablet_id), K(ls_id)); - } - } else { - LOG_WARN("ls_tablets_map get fail", K(ret), K(tablet_id), K(ls_id)); - } - } else if (OB_FAIL(tablet_array.push_back(tablet_id))) { - LOG_WARN("fail to push back to array", K(ret), K(tablet_id)); - } else if (OB_FAIL(ls_tablets_map.set_refactored(ls_id, tablet_array, true /* overwrite */))) { - LOG_WARN("ls_tablets_map set fail", K(ret), K(tablet_id), K(ls_id)); - } - - return ret; -} - int ObCheckTabletDataComplementOp::calculate_build_finish(const common::ObIArray &tablet_ids, hash::ObHashMap &tablets_commited_map, int64_t &build_succ_count) { int ret = OB_SUCCESS; - common::ObArray paxos_server_list; // unused - int64_t paxos_member_count = 0; - build_succ_count = 0; if (OB_UNLIKELY(false)) { @@ -2800,18 +2460,9 @@ int ObCheckTabletDataComplementOp::calculate_build_finish(const common::ObIArray int commited_count = 0; for (int64_t tablet_idx = 0; OB_SUCC(ret) && tablet_idx < tablet_ids.count(); ++tablet_idx) { common::ObTabletID tablet_id = tablet_ids.at(tablet_idx); - if (OB_FAIL(ObDDLUtil::get_tablet_paxos_member_list(tablet_id, - paxos_server_list, - paxos_member_count))) { - LOG_WARN("fail to get tablet paxos member list.", - K(ret), K(tablet_id), K(paxos_server_list), K(paxos_member_count)); - } else if (paxos_member_count == 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to check task tablet, unexpected!", - K(ret), K(paxos_member_count), K(tablet_id)); - } else if (OB_FAIL(tablets_commited_map.get_refactored(tablet_id, commited_count))){ + if (OB_FAIL(tablets_commited_map.get_refactored(tablet_id, commited_count))){ LOG_WARN("fail to get tablet commited map, unexpected!", K(ret), K(tablet_id)); - } else if (commited_count < ((paxos_member_count >> 1) + 1)) { // not finished majority + } else if (commited_count < 1) { // do nothing } else { build_succ_count++; @@ -2832,47 +2483,33 @@ int ObCheckTabletDataComplementOp::check_tablet_merge_status(const ObIArray> ip_tablets_map; // use for classify tablet replica addr - hash::ObHashMap> ls_tablets_map; // use for classify tablet ls hash::ObHashMap tablets_commited_map; - const static int64_t max_map_hash_bucket = tablet_ids.count(); + const int64_t max_map_hash_bucket = tablet_ids.count(); if (OB_UNLIKELY( tablet_ids.count() <= 0 || OB_INVALID_TIMESTAMP == snapshot_version)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_ids.count()), K(snapshot_version)); - } else if (OB_FAIL(ip_tablets_map.create(max_map_hash_bucket, "DdlTablet"))) { - LOG_WARN("fail to create ip_tablets_map", K(ret)); - } else if (OB_FAIL(ls_tablets_map.create(max_map_hash_bucket, "DdlTablet"))) { - LOG_WARN("fail to create ls_tablets_map", K(ret)); } else if (OB_FAIL(tablets_commited_map.create(max_map_hash_bucket, "DdlTablet"))){ LOG_WARN("fail to create tablets_commited_map", K(ret)); } else { const static int64_t batch_size = 100; // batch tablet number int64_t total_build_succ_count = 0; int64_t one_batch_build_succ_count = 0; + ObArray batch_tablet_ids; for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - if (OB_FAIL(construct_ls_tablet_map(tablet_id, ls_tablets_map))) { - LOG_WARN("construct_tablet_ls_map fail", K(ret), K(tablet_id)); - } else { - if ((i != 0 && i % batch_size == 0) /* reach batch size */ || i == tablet_ids.count() - 1 /* reach end */) { - for (hash::ObHashMap>::const_iterator ls_iter = ls_tablets_map.begin(); - ls_iter != ls_tablets_map.end() && OB_SUCC(ret); ++ls_iter) { - const ObLSID &ls_id = ls_iter->first; - const ObArray &tablet_array = ls_iter->second; - if (OB_FAIL(do_check_tablets_merge_status(snapshot_version, - tablet_array, - ls_id, - ip_tablets_map, - tablets_commited_map, - one_batch_build_succ_count))) { - LOG_WARN("do check tablets merge status fail", K(ret)); - } else { - total_build_succ_count += one_batch_build_succ_count; - } - } - ls_tablets_map.reuse(); // reuse map + if (OB_FAIL(batch_tablet_ids.push_back(tablet_id))) { + LOG_WARN("fail to push back tablet id", K(ret), K(tablet_id)); + } else if (batch_tablet_ids.count() >= batch_size || i == tablet_ids.count() - 1) { + if (OB_FAIL(do_check_tablets_merge_status(snapshot_version, + batch_tablet_ids, + tablets_commited_map, + one_batch_build_succ_count))) { + LOG_WARN("do check tablets merge status fail", K(ret), K(batch_tablet_ids)); + } else { + total_build_succ_count += one_batch_build_succ_count; + batch_tablet_ids.reuse(); } } } @@ -2885,8 +2522,6 @@ int ObCheckTabletDataComplementOp::check_tablet_merge_status(const ObIArray &based_schema_object_infos, - const ObString &mview_select_sql, - ObSqlString &sql_string); - - static int get_tablet_leader_addr( - share::ObLocationService *location_service, - const common::ObTabletID &tablet_id, - const int64_t timeout, - share::ObLSID &ls_id, - common::ObAddr &leader_addr); static int refresh_alter_table_arg(const int64_t orig_table_id, const uint64_t foreign_key_id, @@ -1170,16 +1128,8 @@ class ObDDLUtil const int64_t schema_version, ObSqlString &sql_string); - static int generate_mview_ddl_schema_hint_str(const uint64_t mview_table_id, - share::schema::ObSchemaGetterGuard &schema_guard, - const common::ObIArray &based_schema_object_infos, - ObSqlString &sql_string); - - static int generate_order_by_str_for_mview(const schema::ObTableSchema &container_table_schema, - ObSqlString &rowkey_column_sql_string); - static int ddl_get_tablet( - const storage::ObLSHandle &ls_handle, + storage::ObLS *ls, const ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle, const storage::ObMDSGetTabletMode mode = storage::ObMDSGetTabletMode::READ_WITHOUT_CHECK); @@ -1192,34 +1142,12 @@ class ObDDLUtil || OB_EAGAIN == ret || OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_TRANS_CTX_NOT_EXIST == ret; } - static bool need_remote_write(const int ret_code); - static int check_can_convert_character(const ObObjMeta &obj_meta, const bool is_domain_index, const bool is_string_lob) { return (obj_meta.is_string_type() || obj_meta.is_enum_or_set()) && (is_string_lob || (CS_TYPE_BINARY != obj_meta.get_collation_type() && !is_domain_index)); } - static int get_sys_ls_leader_addr( - const uint64_t cluster_id, - common::ObAddr &leader_addr); - - static int get_tablet_paxos_member_list(const common::ObTabletID &tablet_id, - common::ObIArray &paxos_server_list, - int64_t &paxos_member_count); - - static int get_tablet_replica_location(const common::ObTabletID &tablet_id, - ObLSID &ls_id, - ObLSLocation &location); - static int get_split_replicas_addrs(const share::ObLSID &ls_id, - ObIArray &member_addrs_array, - ObIArray &learner_addrs_array); - static int get_split_replicas_addrs(const share::ObLSID &ls_id, - ObIArray &replica_addr_array); - static int construct_ls_tablet_id_map( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - hash::ObHashMap> &ls_tablet_id_map); static int get_index_table_batch_partition_names( const int64_t &data_table_id, const int64_t &index_table_id, @@ -1228,15 +1156,11 @@ class ObDDLUtil ObIArray &partition_names); static int get_tablet_data_size( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, int64_t &data_size); static int get_tablet_data_row_cnt( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, int64_t &data_row_cnt); static int get_ls_host_left_disk_space( - const share::ObLSID &ls_id, - const common::ObAddr &leader_addr, uint64_t &left_space_size); static int generate_partition_names( const ObIArray &partition_names_array, @@ -1315,7 +1239,6 @@ class ObDDLUtil share::schema::ObSchemaGetterGuard *&src_tenant_schema_guard, share::schema::ObSchemaGetterGuard *&dst_tenant_schema_guard); static int get_tablet_physical_row_cnt( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool calc_sstable, const bool calc_memtable, @@ -1328,12 +1251,12 @@ class ObDDLUtil bool &is_table_empty); static int check_schema_version_refreshed(const int64_t target_schema_version); static bool reach_time_interval(const int64_t i, volatile int64_t &last_time); - static int is_major_exist(const ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &is_exist); + static int is_major_exist(const common::ObTabletID &tablet_id, bool &is_exist); #ifdef OB_BUILD_SHARED_STORAGE static int upload_block_for_ss(const char* buf, const int64_t len, const blocksstable::MacroBlockId ¯o_block_id); static int update_tablet_gc_info(const ObTabletID &tablet_id, const int64_t pre_snapshot_version, const int64_t new_snapshot_version); #endif - static int set_tablet_autoinc_seq(const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t seq_value); + static int set_tablet_autoinc_seq(const ObTabletID &tablet_id, const int64_t seq_value); static int check_table_compaction_checksum_error(const uint64_t table_id); static int get_temp_store_compress_type(const ObCompressorType schema_compr_type, const int64_t parallel, @@ -1357,10 +1280,7 @@ class ObDDLUtil case DDL_TABLE_REDEFINITION: case DDL_DIRECT_LOAD: case DDL_DIRECT_LOAD_INSERT: - case DDL_MVIEW_COMPLETE_REFRESH: - case DDL_CREATE_MVIEW: case DDL_CREATE_INDEX: - case DDL_CREATE_MLOG: case DDL_CREATE_FTS_INDEX: case DDL_CREATE_VEC_INDEX: case DDL_CREATE_PARTITIONED_LOCAL_INDEX: @@ -1380,8 +1300,7 @@ class ObDDLUtil } static int get_global_index_table_ids(const schema::ObTableSchema &table_schema, ObIArray &global_index_table_ids, share::schema::ObSchemaGetterGuard &schema_guard); static bool use_idempotent_mode(); - static bool is_mview_not_retryable(const share::ObDDLType task_type); - static int64_t get_real_parallelism(const int64_t parallelism, const bool is_mv_refresh); + static int64_t get_real_parallelism(const int64_t parallelism); static int get_tablet_ids( const int64_t table_id, const int64_t target_table_id, @@ -1391,8 +1310,7 @@ class ObDDLUtil const uint64_t table_id, const uint64_t target_table_id, int64_t &snapshot_version, - rootserver::ObDDLTask* task, - const common::ObIArray *extra_mv_tablet_ids = NULL); + rootserver::ObDDLTask* task); static int release_snapshot( rootserver::ObDDLTask* task, const uint64_t table_id, @@ -1465,14 +1383,12 @@ class ObDDLUtil static bool need_reshape(const ObObjMeta &col_type); static int report_ddl_checksum_from_major_sstable( - const ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t table_id, const int64_t execution_id, const int64_t ddl_task_id, const int64_t tenant_data_version); static int report_ddl_sstable_checksum( - const ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t target_table_id, const int64_t execution_id, @@ -1526,7 +1442,6 @@ class ObDDLUtil ObWriteMacroParam ¶m); static int get_task_ranges( const int64_t task_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t tablet_size, const int64_t hint_parallelism, @@ -1543,7 +1458,6 @@ class ObDDLUtil const int64_t range_interval, const int64_t slice_row_idx); - static int is_ls_leader(ObLS &ls, bool &is_leader); static int alloc_storage_macro_block_writer( const ObWriteMacroParam ¶m, ObIAllocator &allocator, @@ -1570,8 +1484,7 @@ class ObDDLUtil const uint64_t table_id, const uint64_t target_table_id, rootserver::ObRootService *root_service, - const int64_t snapshot_version, - const common::ObIArray *extra_mv_tablet_ids); + const int64_t snapshot_version); static int check_table_column_checksum_error(const int64_t table_id); @@ -1626,8 +1539,6 @@ class ObCheckTabletDataComplementOp static int do_check_tablets_merge_status(const int64_t snapshot_version, const ObIArray &tablet_ids, - const ObLSID &ls_id, - hash::ObHashMap> &ip_tablets_map, hash::ObHashMap &tablets_commited_map, int64_t &tablet_commit_count); @@ -1645,11 +1556,6 @@ class ObCheckTabletDataComplementOp hash::ObHashMap &tablets_commited_map, int64_t &commit_succ_count); - static int construct_ls_tablet_map(const common::ObTabletID &tablet_id, - hash::ObHashMap> &ls_tablets_map); - - static int construct_tablet_ip_map(const ObTabletID &tablet_id, - hash::ObHashMap> &ip_tablets_map); }; class ObSplitUtil diff --git a/src/share/ob_ddl_sim_point.h b/src/share/ob_ddl_sim_point.h index 2bc1bc71a..271989388 100644 --- a/src/share/ob_ddl_sim_point.h +++ b/src/share/ob_ddl_sim_point.h @@ -33,7 +33,7 @@ enum ObSimType { SIM_TYPE_ALL = 0, SIM_TYPE_DDL = 1, - SIM_TYPE_RESERVED_2 = 2, + SIM_TYPE_RESERVED = 2, }; struct ObDDLSimAction diff --git a/src/share/ob_ddl_task_executor.h b/src/share/ob_ddl_task_executor.h index 47df2ab6a..e54886c3f 100644 --- a/src/share/ob_ddl_task_executor.h +++ b/src/share/ob_ddl_task_executor.h @@ -63,8 +63,7 @@ class ObIDDLTask : public common::ObDLinkBase || is_stop_state(ret_code) || is_not_exist(ret_code) || is_retry(ret_code) - || is_timeout(ret_code) - || is_location_service_renew_error(ret_code); + || is_timeout(ret_code); } static bool in_ddl_retry_black_list(const int ret_code) { diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index f29c3b5fb..164db234c 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -29,8 +29,6 @@ class ObString; ACT(INVALID_DEBUG_SYNC_POINT, = 0) \ ACT(NOW,) \ ACT(MAJOR_FREEZE_BEFORE_SYS_COORDINATE_COMMIT,) \ - ACT(BEFORE_REBALANCE_TASK_EXECUTE,) \ - ACT(REBALANCE_TASK_MGR_BEFORE_EXECUTE_OVER,) \ ACT(UNIT_BALANCE_BEFORE_PARTITION_BALANCE,) \ ACT(BEFORE_UNIT_MANAGER_LOAD,) \ ACT(BEFORE_INNER_SQL_COMMIT,) \ @@ -85,10 +83,6 @@ class ObString; ACT(STOP_BALANCE_EXECUTOR,) \ ACT(BEFORE_STOP_RS,) \ ACT(BEFORE_ANSWER_RS_STATUS, )\ - ACT(DELAY_CHANGE_REPLICA_CALLBACK, )\ - ACT(AFTER_POST_REMOVE_REPLICA_MC_MSG, )\ - ACT(AFTER_POST_ADD_REPLICA_MC_MSG, )\ - ACT(DELAY_REMOVE_REPLICA, )\ ACT(BEFORE_CHECK_ALL_PARTITION_MERGED, )\ ACT(BEFORE_UPDATE_INDEX_STATUS, )\ ACT(AFTER_MIGRATION_MARK, )\ @@ -254,7 +248,6 @@ class ObString; ACT(MIGRATE_AFTER_MARK_TASK_STATUS,)\ ACT(MIGRATE_AFTER_DELETE_TASK_FILE,)\ ACT(MIGRATE_AFTER_DELETE_PG_LOCK_DIR,)\ - ACT(MIGRATE_AFTER_REMOVE_REPLICA_SUB,)\ ACT(MIGRATE_AFTER_CREATE_MIGRATE_OUT,)\ ACT(FAST_RECOVER_BEFORE_PREPROCESS,)\ ACT(FAST_RECOVER_AFTER_PREPROCESS,)\ @@ -384,7 +377,6 @@ class ObString; ACT(BEFORE_RESTORE_AUX_TENANT,)\ ACT(BEFORE_BACKUP_SYS_TABLETS,)\ ACT(BEFORE_WRITE_DDL_PREPARE_LOG,)\ - ACT(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG,)\ ACT(BEFORE_INDEX_SSTABLE_BUILD_TASK_SEND_SQL,)\ ACT(BEFORE_CHECK_FK_DATA_VALID_SEND_SQL,)\ ACT(BEFORE_CHECK_CONSTRAINT_VALID_SEND_SQL,)\ @@ -436,24 +428,13 @@ class ObString; ACT(BEFORE_LOCK_SERVICE_UNLOCK,)\ ACT(DDL_CHECK_TABLET_MERGE_STATUS,)\ ACT(BEFORE_LOCK_LS_WHEN_CREATE_TABLE,)\ - ACT(AFTER_BLOCK_TABLET_IN_WHEN_LS_MERGE,)\ ACT(TABLE_LOCK_AFTER_LOCK_TABLE_BEFORE_LOCK_TABLET,)\ - ACT(BEFORE_TENANT_BALANCE_SERVICE,)\ - ACT(BEFORE_TENANT_BALANCE_SERVICE_EXECUTE,)\ - ACT(BEFORE_PROCESS_BALANCE_TASK_INIT,)\ - ACT(BEFORE_PROCESS_BALANCE_TASK_CREATE_LS,)\ - ACT(BEFORE_PROCESS_BALANCE_TASK_ALTER_LS,)\ - ACT(BEFORE_PROCESS_BALANCE_TASK_SET_MERGE,)\ - ACT(BEFORE_PROCESS_BALANCE_TASK_DROP_LS,)\ ACT(MODIFY_HIDDEN_TABLE_NOT_NULL_COLUMN_STATE_BEFORE_PUBLISH_SCHEMA,)\ ACT(AFTER_MIGRATION_FETCH_TABLET_INFO,)\ ACT(AFTER_LOCK_LS_AND_BEFORE_CHANGE_LS_FLAG,)\ - ACT(AFTER_LOCK_ALL_BALANCE_JOB,)\ - ACT(BEFORE_DROPPING_LS_IN_BALANCE_MERGE_TASK,)\ ACT(BEFORE_WAIT_LOG_SYNC,)\ ACT(BEFORE_WAIT_LOG_REPLAY_SYNC,)\ ACT(BEFORE_FOREIGN_KEY_CONSTRAINT_CHECK,)\ - ACT(BEFORE_EXECUTE_ARB_REPLICA_TASK,)\ ACT(ARCHIVE_SENDER_HANDLE_TASK_DONE,)\ ACT(BEFORE_SET_LS_MEMBER_LIST,)\ ACT(BEFORE_MIGRATION_FETCH_TABLET_INFO,)\ @@ -482,7 +463,6 @@ class ObString; ACT(AFTER_RESTORE_TABLET_TASK,)\ ACT(BEFORE_TABLET_GC,)\ ACT(AFTER_CHECKPOINT_GET_CURSOR,)\ - ACT(BEFORE_EXECUTE_BALANCE_TASK,)\ ACT(BEFORE_CHANGE_BACKUP_TURN,)\ ACT(BEFORE_PROCESS_AFTER_HAS_MEMBER_LIST,)\ ACT(END_DELETE_SERVER_BEFORE_CHECK_META_TABLE,)\ @@ -492,7 +472,6 @@ class ObString; ACT(BEFORE_UNIQ_TASK_RUN,)\ ACT(BEFORE_PARELLEL_TRUNCATE,)\ ACT(END_DDL_IN_PX_SUBCOORD,)\ - ACT(BEFORE_SEND_ADD_REPLICA_DRTASK,)\ ACT(BETWEEN_INSERT_LOCK_INFO_AND_TRY_LOCK_CONFIG_CHANGE,)\ ACT(BEFORE_CHECK_SHRINK_RESOURCE_POOL,)\ ACT(STOP_RECOVERY_LS_THREAD0,)\ @@ -514,12 +493,11 @@ class ObString; ACT(BEFORE_FETCH_SIMPLE_TABLES,)\ ACT(BEFORE_SEND_PARALLEL_CREATE_TABLE,)\ ACT(BEFORE_DROP_TENANT,)\ - ACT(BEFORE_WAIT_SYS_LS_END_SCN,)\ ACT(BEFORE_CREATE_CLONE_TENANT_END,)\ ACT(BEFORE_CALC_CONSISTENT_SCN,)\ ACT(RENAME_TABLE_BEFORE_PUBLISH_SCHEMA,)\ ACT(BEFORE_SET_NEW_SCHEMA_VERSION,)\ - ACT(REPLAY_SWITCH_TO_FOLLOWER_BEFORE_PUSH_SUBMIT_TASK,)\ + ACT(REPLAY_ENABLE_LOCAL_BEFORE_PUSH_SUBMIT_TASK,)\ ACT(BEFORE_CHECK_TTL_TASK_FINISH,)\ ACT(BEFORE_TTL_SCHEDULER_RUN,)\ ACT(BEFORE_MERGE_BACKUP_META_INFO,)\ @@ -551,7 +529,6 @@ class ObString; ACT(BEFORE_ADD_SERVER_TRANS,)\ ACT(AFTER_MEMBERLIST_CHANGED,)\ ACT(AFTER_FIRST_CLONE_CHECK_FOR_STANDBY,)\ - ACT(BEFORE_REMOVE_BALANCE_TASK_HELPER,)\ ACT(BEFORE_CHOOSE_SOURCE,)\ ACT(AFTER_CHECK_LOG_NEED_REBUILD,)\ ACT(REBUILD_VEC_INDEX_PREPARE,)\ @@ -606,7 +583,6 @@ class ObString; ACT(BEFOR_PREPARE_CREATE_TFS_INDEX_DOC_WORD,)\ ACT(BEFOR_EXECUTE_CREATE_TABLE_WITH_FTS_INDEX,)\ ACT(AFTER_JOIN_LEARNER_LIST_FOR_SPECIFIED_SERVER,)\ - ACT(BEFORE_MV_FINISH_COMPLETE_REFRESH,)\ ACT(BEFORE_MIGRATION_CREATE_TABLE_STORE,)\ ACT(BEFORE_FILL_AUTO_SPLIT_PARAMS,)\ ACT(BEFORE_UPDATE_TABLET_HA_STATUS,)\ @@ -645,15 +621,9 @@ class ObString; ACT(BEFORE_MANAGE_DYNAMIC_PARTITION,)\ ACT(BEFORE_MANAGE_DYNAMIC_PARTITION_ON_TABLE,)\ ACT(BEFORE_SWAP_ORIG_AND_HIDDEN_TABLE_STATE,)\ - ACT(BEFORE_MV_FINISH_RUNNING_JOB,)\ - ACT(BEFORE_MV_LOAD_DATA,)\ ACT(BEFORE_ALTER_TABLE_IN_TRANS,)\ - ACT(AFTER_LOCK_MVIEW_IN_REFRESH,)\ - ACT(REBUILD_INDEX_WAIT_PURGE_OLD_MLOG,)\ ACT(REBUILD_INDEX_WAIT_CREATE_TASK_FINISH,)\ ACT(REBUILD_INDEX_WAIT_DROP_TASK_FINISH,)\ - ACT(BEFORE_NESTED_MV_GET_SCN,)\ - ACT(BEFORE_NESTED_MV_REFRESH,)\ ACT(LOAD_INNER_TABLE_SCHEMA,)\ ACT(CANCEL_VEC_TASK_ADD_SNAP_INDEX,)\ ACT(BEFORE_SCHEDULE_REPLACE_LS,)\ diff --git a/src/share/ob_define.h b/src/share/ob_define.h index ee6077466..fc4f493bb 100644 --- a/src/share/ob_define.h +++ b/src/share/ob_define.h @@ -24,7 +24,6 @@ #include "common/ob_tablet_id.h" #include "share/ob_errno.h" #include "lib/worker.h" -#include "share/ob_ls_id.h" #include "cmath" #ifdef __linux__ #include diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index d53641c1c..8d5d52878 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -3653,15 +3653,6 @@ static const _error _error_OB_SERVER_MIGRATE_IN_DENIED = { .str_error = "server migrate in denied", .str_user_error = "server migrate in denied" }; -static const _error _error_OB_REBALANCE_TASK_CANT_EXEC = { - .error_name = "OB_REBALANCE_TASK_CANT_EXEC", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "rebalance task can not executing now", - .str_user_error = "rebalance task can not executing now" -}; static const _error _error_OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT = { .error_name = "OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT", .error_cause = "Internal Error", @@ -3671,15 +3662,6 @@ static const _error _error_OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT = { .str_error = "rootserver can not hold more partition", .str_user_error = "rootserver can not hold more partition" }; -static const _error _error_OB_REBALANCE_TASK_NOT_IN_PROGRESS = { - .error_name = "OB_REBALANCE_TASK_NOT_IN_PROGRESS", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "rebalance task not in progress on seekdb", - .str_user_error = "rebalance task not in progress on seekdb" -}; static const _error _error_OB_DATA_SOURCE_NOT_EXIST = { .error_name = "OB_DATA_SOURCE_NOT_EXIST", .error_cause = "Internal Error", @@ -4598,15 +4580,6 @@ static const _error _error_OB_LEADER_COORDINATOR_NEED_RETRY = { .str_error = "leader coordinator needs retry", .str_user_error = "leader coordinator needs retry" }; -static const _error _error_OB_REBALANCE_TASK_NEED_RETRY = { - .error_name = "OB_REBALANCE_TASK_NEED_RETRY", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "rebalance task needs retry", - .str_user_error = "rebalance task needs retry" -}; static const _error _error_OB_LS_NOT_EXIST = { .error_name = "OB_LS_NOT_EXIST", .error_cause = "Internal Error", @@ -4616,15 +4589,6 @@ static const _error _error_OB_LS_NOT_EXIST = { .str_error = "log stream does not exist", .str_user_error = "log stream does not exist" }; -static const _error _error_OB_TOO_MANY_TENANT_LS = { - .error_name = "OB_TOO_MANY_TENANT_LS", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "too many log streams were defined for this tenant", - .str_user_error = "too many log streams were defined for this tenant" -}; static const _error _error_OB_LS_LOCATION_NOT_EXIST = { .error_name = "OB_LS_LOCATION_NOT_EXIST", .error_cause = "Internal Error", @@ -4796,15 +4760,6 @@ static const _error _error_OB_DEVICE_NOT_EXIST = { .str_error = "Device does not exist", .str_user_error = "Device does not exist" }; -static const _error _error_OB_LS_REPLICA_TASK_RESULT_UNCERTAIN = { - .error_name = "OB_LS_REPLICA_TASK_RESULT_UNCERTAIN", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "log stream replica task result is uncertain", - .str_user_error = "log stream replica task result is uncertain" -}; static const _error _error_OB_WAIT_REPLAY_TIMEOUT = { .error_name = "OB_WAIT_REPLAY_TIMEOUT", .error_cause = "Internal Error", @@ -14435,15 +14390,6 @@ static const _error _error_OB_TRANS_COMMIT_TOO_MUCH_TIME = { .str_error = "Transaction commit cost too much", .str_user_error = "Transaction commit cost too much" }; -static const _error _error_OB_TRANS_TOO_MANY_PARTICIPANTS = { - .error_name = "OB_TRANS_TOO_MANY_PARTICIPANTS", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = 6002, - .sqlstate = "40000", - .str_error = "too many transaction participants", - .str_user_error = "too many transaction participants" -}; static const _error _error_OB_LOG_ALREADY_SPLIT = { .error_name = "OB_LOG_ALREADY_SPLIT", .error_cause = "Internal Error", @@ -15155,15 +15101,6 @@ static const _error _error_OB_PARTITION_ALREADY_BALANCED = { .str_error = "partitions are already balanced", .str_user_error = "partitions are already balanced, %s" }; -static const _error _error_NEW_MV_MAJOR_VERSION_NOT_MATCH = { - .error_name = "NEW_MV_MAJOR_VERSION_NOT_MATCH", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "new mv major version is not match", - .str_user_error = "new mv major version is not match" -}; static const _error _error_OB_ERR_GIS_DIFFERENT_SRIDS = { .error_name = "OB_ERR_GIS_DIFFERENT_SRIDS", .error_cause = "Internal Error", @@ -16802,15 +16739,6 @@ static const _error _error_OB_INVALID_LOG_ARCHIVE_STATUS = { .str_error = "log archive status is not valid", .str_user_error = "log archive status is not valid" }; -static const _error _error_OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST = { - .error_name = "OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "Cannot add replica during set member list in restore", - .str_user_error = "Cannot add replica during set member list in restore" -}; static const _error _error_OB_LOG_ARCHIVE_LEADER_CHANGED = { .error_name = "OB_LOG_ARCHIVE_LEADER_CHANGED", .error_cause = "Internal Error", @@ -17648,15 +17576,6 @@ static const _error _error_OB_TMP_FILE_EXCEED_DISK_QUOTA = { .str_error = "tmp file exceeds disk quota", .str_user_error = "tmp file exceeds disk quota" }; -static const _error _error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = { - .error_name = "OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled", - .str_user_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled" -}; static const _error _error_OB_ERR_DUPLICATE_INDEX = { .error_name = "OB_ERR_DUPLICATE_INDEX", .error_cause = "Internal Error", @@ -20024,69 +19943,6 @@ static const _error _error_OB_ERR_DEBUG_ID_NOT_EXIST = { .str_error = "debug_session_id does not exist", .str_user_error = "debug_session_id = %u does not exist" }; -static const _error _error_OB_ERR_TABLE_NO_MLOG = { - .error_name = "OB_ERR_TABLE_NO_MLOG", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "table does not have a materialized view log", - .str_user_error = "table `%s`.`%s` does not have a materialized view log" -}; -static const _error _error_OB_ERR_MLOG_EXIST = { - .error_name = "OB_ERR_MLOG_EXIST", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "a materialized view log already exists on table", - .str_user_error = "a materialized view log already exists on table `%s`" -}; -static const _error _error_OB_ERR_MVIEW_NOT_EXIST = { - .error_name = "OB_ERR_MVIEW_NOT_EXIST", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "materialized view does not exist", - .str_user_error = "materialized view `%s`.`%s` does not exist" -}; -static const _error _error_OB_ERR_MVIEW_EXIST = { - .error_name = "OB_ERR_MVIEW_EXIST", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "materialized view already exists", - .str_user_error = "materialized view `%s`.`%s` already exists" -}; -static const _error _error_OB_ERR_MLOG_IS_YOUNGER = { - .error_name = "OB_ERR_MLOG_IS_YOUNGER", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "materialized view log younger than last refresh", - .str_user_error = "materialized view log on `%s`.`%s` younger than last refresh" -}; -static const _error _error_OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH = { - .error_name = "OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "cannot fast refresh materialized view", - .str_user_error = "cannot fast refresh materialized view %s: %s" -}; -static const _error _error_OB_ERR_MVIEW_NEVER_REFRESH = { - .error_name = "OB_ERR_MVIEW_NEVER_REFRESH", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "cannot explicitly refresh a NEVER REFRESH materialized view", - .str_user_error = "cannot explicitly refresh a NEVER REFRESH materialized view (`%s`)" -}; static const _error _error_OB_ERR_CLIENT_LOCAL_FILES_DISABLED = { .error_name = "OB_ERR_CLIENT_LOCAL_FILES_DISABLED", .error_cause = "Internal Error", @@ -20276,15 +20132,6 @@ static const _error _error_OB_ERR_INVALID_CHARACTER = { .str_error = "invalid character", .str_user_error = "invalid character" }; -static const _error _error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = { - .error_name = "OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "cannot ENABLE ON QUERY COMPUTATION for the materialized view", - .str_user_error = "cannot ENABLE ON QUERY COMPUTATION for the materialized view %s: %s" -}; static const _error _error_OB_ERR_CURSOR_ATTR_APPLY = { .error_name = "OB_ERR_CURSOR_ATTR_APPLY", .error_cause = "Internal Error", @@ -20501,33 +20348,6 @@ static const _error _error_OB_ERR_INVALID_PLSQL_UNIT = { .str_error = "input to DBMS_DDL.WRAP is not a legal PL/SQL unit", .str_user_error = "input to DBMS_DDL.WRAP is not a legal PL/SQL unit" }; -static const _error _error_OB_ERR_MVIEW_INVALID_TABLE_TYPE = { - .error_name = "OB_ERR_MVIEW_INVALID_TABLE_TYPE", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "Table type is not valid, the definition of materialized view can only reference user tables or other materialized views", - .str_user_error = "Table type is not valid, the definition of materialized view can only reference user tables or other materialized views" -}; -static const _error _error_OB_ERR_MVIEW_BASE_TABLE_ALTERED = { - .error_name = "OB_ERR_MVIEW_BASE_TABLE_ALTERED", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "Base tables are altered during the creation or complete refresh of materialized views, please try again when base tables have no ongoing DDL operations", - .str_user_error = "Base tables are altered during the creation or complete refresh of materialized views, please try again when base tables have no ongoing DDL operations" -}; -static const _error _error_OB_ERR_MVIEW_MISSING_DEPENDENCE = { - .error_name = "OB_ERR_MVIEW_MISSING_DEPENDENCE", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "Materialized view %s.%s has invalid dependency info, please perform a complete refresh to recover.", - .str_user_error = "Materialized view %s.%s has invalid dependency info, please perform a complete refresh to recover." -}; static const _error _error_OB_ERR_INVALID_NESTED_CURSOR = { .error_name = "OB_ERR_INVALID_NESTED_CURSOR", .error_cause = "Internal Error", @@ -20537,15 +20357,6 @@ static const _error _error_OB_ERR_INVALID_NESTED_CURSOR = { .str_error = "Cursor Variable in record, object, or collection is not supported by this release", .str_user_error = "Cursor Variable in record, object, or collection is not supported by this release" }; -static const _error _error_OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH = { - .error_name = "OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH", - .error_cause = "Internal Error", - .error_solution = "Contact OceanBase Support", - .mysql_errno = -1, - .sqlstate = "HY000", - .str_error = "materialized view can not nested consistent refresh", - .str_user_error = "materialized view can not nested consistent refresh" -}; static const _error _error_OB_TTL_NOT_ENABLE = { .error_name = "OB_TTL_NOT_ENABLE", .error_cause = "Internal Error", @@ -21981,9 +21792,7 @@ struct ObStrErrorInit _errors[-OB_ROOTSERVICE_EXIST] = &_error_OB_ROOTSERVICE_EXIST; _errors[-OB_RS_SHUTDOWN] = &_error_OB_RS_SHUTDOWN; _errors[-OB_SERVER_MIGRATE_IN_DENIED] = &_error_OB_SERVER_MIGRATE_IN_DENIED; - _errors[-OB_REBALANCE_TASK_CANT_EXEC] = &_error_OB_REBALANCE_TASK_CANT_EXEC; _errors[-OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT] = &_error_OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT; - _errors[-OB_REBALANCE_TASK_NOT_IN_PROGRESS] = &_error_OB_REBALANCE_TASK_NOT_IN_PROGRESS; _errors[-OB_DATA_SOURCE_NOT_EXIST] = &_error_OB_DATA_SOURCE_NOT_EXIST; _errors[-OB_DATA_SOURCE_TABLE_NOT_EXIST] = &_error_OB_DATA_SOURCE_TABLE_NOT_EXIST; _errors[-OB_DATA_SOURCE_RANGE_NOT_EXIST] = &_error_OB_DATA_SOURCE_RANGE_NOT_EXIST; @@ -22086,9 +21895,7 @@ struct ObStrErrorInit _errors[-OB_ERR_DROP_TRUNCATE_PARTITION_REBUILD_INDEX] = &_error_OB_ERR_DROP_TRUNCATE_PARTITION_REBUILD_INDEX; _errors[-OB_ERR_ATLER_TABLE_ILLEGAL_FK] = &_error_OB_ERR_ATLER_TABLE_ILLEGAL_FK; _errors[-OB_LEADER_COORDINATOR_NEED_RETRY] = &_error_OB_LEADER_COORDINATOR_NEED_RETRY; - _errors[-OB_REBALANCE_TASK_NEED_RETRY] = &_error_OB_REBALANCE_TASK_NEED_RETRY; _errors[-OB_LS_NOT_EXIST] = &_error_OB_LS_NOT_EXIST; - _errors[-OB_TOO_MANY_TENANT_LS] = &_error_OB_TOO_MANY_TENANT_LS; _errors[-OB_LS_LOCATION_NOT_EXIST] = &_error_OB_LS_LOCATION_NOT_EXIST; _errors[-OB_LS_LOCATION_LEADER_NOT_EXIST] = &_error_OB_LS_LOCATION_LEADER_NOT_EXIST; _errors[-OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST] = &_error_OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST; @@ -22108,7 +21915,6 @@ struct ObStrErrorInit _errors[-OB_LS_EXIST] = &_error_OB_LS_EXIST; _errors[-OB_DEVICE_EXIST] = &_error_OB_DEVICE_EXIST; _errors[-OB_DEVICE_NOT_EXIST] = &_error_OB_DEVICE_NOT_EXIST; - _errors[-OB_LS_REPLICA_TASK_RESULT_UNCERTAIN] = &_error_OB_LS_REPLICA_TASK_RESULT_UNCERTAIN; _errors[-OB_WAIT_REPLAY_TIMEOUT] = &_error_OB_WAIT_REPLAY_TIMEOUT; _errors[-OB_WAIT_TABLET_READY_TIMEOUT] = &_error_OB_WAIT_TABLET_READY_TIMEOUT; _errors[-OB_FREEZE_SERVICE_EPOCH_MISMATCH] = &_error_OB_FREEZE_SERVICE_EPOCH_MISMATCH; @@ -23179,7 +22985,6 @@ struct ObStrErrorInit _errors[-OB_TRANS_FREE_ROUTE_NOT_SUPPORTED] = &_error_OB_TRANS_FREE_ROUTE_NOT_SUPPORTED; _errors[-OB_TRANS_LIVE_TOO_MUCH_TIME] = &_error_OB_TRANS_LIVE_TOO_MUCH_TIME; _errors[-OB_TRANS_COMMIT_TOO_MUCH_TIME] = &_error_OB_TRANS_COMMIT_TOO_MUCH_TIME; - _errors[-OB_TRANS_TOO_MANY_PARTICIPANTS] = &_error_OB_TRANS_TOO_MANY_PARTICIPANTS; _errors[-OB_LOG_ALREADY_SPLIT] = &_error_OB_LOG_ALREADY_SPLIT; _errors[-OB_ERR_UNSUPPROTED_REF_IN_JSON_SCHEMA] = &_error_OB_ERR_UNSUPPROTED_REF_IN_JSON_SCHEMA; _errors[-OB_ERR_TYPE_OF_JSON_SCHEMA] = &_error_OB_ERR_TYPE_OF_JSON_SCHEMA; @@ -23259,7 +23064,6 @@ struct ObStrErrorInit _errors[-OB_SEQUENCE_NOT_MATCH] = &_error_OB_SEQUENCE_NOT_MATCH; _errors[-OB_SEQUENCE_TOO_SMALL] = &_error_OB_SEQUENCE_TOO_SMALL; _errors[-OB_PARTITION_ALREADY_BALANCED] = &_error_OB_PARTITION_ALREADY_BALANCED; - _errors[-NEW_MV_MAJOR_VERSION_NOT_MATCH] = &_error_NEW_MV_MAJOR_VERSION_NOT_MATCH; _errors[-OB_ERR_GIS_DIFFERENT_SRIDS] = &_error_OB_ERR_GIS_DIFFERENT_SRIDS; _errors[-OB_ERR_GIS_UNSUPPORTED_ARGUMENT] = &_error_OB_ERR_GIS_UNSUPPORTED_ARGUMENT; _errors[-OB_ERR_GIS_UNKNOWN_ERROR] = &_error_OB_ERR_GIS_UNKNOWN_ERROR; @@ -23442,7 +23246,6 @@ struct ObStrErrorInit _errors[-OB_RESTORE_INDEX_FAILED] = &_error_OB_RESTORE_INDEX_FAILED; _errors[-OB_BACKUP_IN_PROGRESS] = &_error_OB_BACKUP_IN_PROGRESS; _errors[-OB_INVALID_LOG_ARCHIVE_STATUS] = &_error_OB_INVALID_LOG_ARCHIVE_STATUS; - _errors[-OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST] = &_error_OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST; _errors[-OB_LOG_ARCHIVE_LEADER_CHANGED] = &_error_OB_LOG_ARCHIVE_LEADER_CHANGED; _errors[-OB_BACKUP_CAN_NOT_START] = &_error_OB_BACKUP_CAN_NOT_START; _errors[-OB_CANCEL_BACKUP_NOT_ALLOWED] = &_error_OB_CANCEL_BACKUP_NOT_ALLOWED; @@ -23536,7 +23339,6 @@ struct ObStrErrorInit _errors[-OB_BACKUP_ZONE_IDC_REGION_INVALID] = &_error_OB_BACKUP_ZONE_IDC_REGION_INVALID; _errors[-OB_ERR_TMP_FILE_ALREADY_SEALED] = &_error_OB_ERR_TMP_FILE_ALREADY_SEALED; _errors[-OB_TMP_FILE_EXCEED_DISK_QUOTA] = &_error_OB_TMP_FILE_EXCEED_DISK_QUOTA; - _errors[-OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE] = &_error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE; _errors[-OB_ERR_DUPLICATE_INDEX] = &_error_OB_ERR_DUPLICATE_INDEX; _errors[-OB_NO_LS_REPLICA_CAN_DO_BACKUP] = &_error_OB_NO_LS_REPLICA_CAN_DO_BACKUP; _errors[-OB_INVALID_KMS_DEST] = &_error_OB_INVALID_KMS_DEST; @@ -23800,13 +23602,6 @@ struct ObStrErrorInit _errors[-OB_ERR_FORALL_ON_REMOTE_TABLE] = &_error_OB_ERR_FORALL_ON_REMOTE_TABLE; _errors[-OB_ERR_SEQUENCE_NOT_DEFINE] = &_error_OB_ERR_SEQUENCE_NOT_DEFINE; _errors[-OB_ERR_DEBUG_ID_NOT_EXIST] = &_error_OB_ERR_DEBUG_ID_NOT_EXIST; - _errors[-OB_ERR_TABLE_NO_MLOG] = &_error_OB_ERR_TABLE_NO_MLOG; - _errors[-OB_ERR_MLOG_EXIST] = &_error_OB_ERR_MLOG_EXIST; - _errors[-OB_ERR_MVIEW_NOT_EXIST] = &_error_OB_ERR_MVIEW_NOT_EXIST; - _errors[-OB_ERR_MVIEW_EXIST] = &_error_OB_ERR_MVIEW_EXIST; - _errors[-OB_ERR_MLOG_IS_YOUNGER] = &_error_OB_ERR_MLOG_IS_YOUNGER; - _errors[-OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH] = &_error_OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; - _errors[-OB_ERR_MVIEW_NEVER_REFRESH] = &_error_OB_ERR_MVIEW_NEVER_REFRESH; _errors[-OB_ERR_CLIENT_LOCAL_FILES_DISABLED] = &_error_OB_ERR_CLIENT_LOCAL_FILES_DISABLED; _errors[-OB_ERR_OUT_PARAM_NOT_BIND_VAR] = &_error_OB_ERR_OUT_PARAM_NOT_BIND_VAR; _errors[-OB_ERR_TIME_EARLIER_THAN_SYSDATE] = &_error_OB_ERR_TIME_EARLIER_THAN_SYSDATE; @@ -23828,7 +23623,6 @@ struct ObStrErrorInit _errors[-OB_ERR_EVENT_RECURSION_FORBIDDEN] = &_error_OB_ERR_EVENT_RECURSION_FORBIDDEN; _errors[-OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR] = &_error_OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR; _errors[-OB_ERR_INVALID_CHARACTER] = &_error_OB_ERR_INVALID_CHARACTER; - _errors[-OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE] = &_error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; _errors[-OB_ERR_CURSOR_ATTR_APPLY] = &_error_OB_ERR_CURSOR_ATTR_APPLY; _errors[-OB_UTL_TCP_BUFFER_TOO_SMALL] = &_error_OB_UTL_TCP_BUFFER_TOO_SMALL; _errors[-OB_UTL_TCP_END_OF_INPUT] = &_error_OB_UTL_TCP_END_OF_INPUT; @@ -23853,11 +23647,7 @@ struct ObStrErrorInit _errors[-OB_ERR_LOGOFF_TRIGGER] = &_error_OB_ERR_LOGOFF_TRIGGER; _errors[-OB_ERR_MALFORMED_WRAPPED_UNIT] = &_error_OB_ERR_MALFORMED_WRAPPED_UNIT; _errors[-OB_ERR_INVALID_PLSQL_UNIT] = &_error_OB_ERR_INVALID_PLSQL_UNIT; - _errors[-OB_ERR_MVIEW_INVALID_TABLE_TYPE] = &_error_OB_ERR_MVIEW_INVALID_TABLE_TYPE; - _errors[-OB_ERR_MVIEW_BASE_TABLE_ALTERED] = &_error_OB_ERR_MVIEW_BASE_TABLE_ALTERED; - _errors[-OB_ERR_MVIEW_MISSING_DEPENDENCE] = &_error_OB_ERR_MVIEW_MISSING_DEPENDENCE; _errors[-OB_ERR_INVALID_NESTED_CURSOR] = &_error_OB_ERR_INVALID_NESTED_CURSOR; - _errors[-OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH] = &_error_OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH; _errors[-OB_TTL_NOT_ENABLE] = &_error_OB_TTL_NOT_ENABLE; _errors[-OB_TTL_COLUMN_NOT_EXIST] = &_error_OB_TTL_COLUMN_NOT_EXIST; _errors[-OB_TTL_COLUMN_TYPE_NOT_SUPPORTED] = &_error_OB_TTL_COLUMN_TYPE_NOT_SUPPORTED; @@ -23996,7 +23786,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2408] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4404, -4405, -4406, -4407, -4408, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4708, -4709, -4711, -4712, -4713, -4714, -4716, -4717, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -4785, -4786, -4787, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5523, -5524, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6286, -6288, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7125, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -7605, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9128, -9129, -9130, -9131, -9132, -9133, -9134, -9135, -9136, -9137, -9138, -9139, -9140, -9141, -9142, -9143, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -9784, -9785, -9786, -9787, -9788, -9789, -9790, -9791, -9792, -9793, -9794, -9795, -9796, -9797, -9798, -9799, -9800, -9801, -9802, -9803, -9804, -9805, -9806, -9807, -9808, -9809, -9810, -9811, -9812, -10501, -10502, -10503, -10504, -10505, -10506, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -11053, -11054, -11055, -11056, -11057, -11058, -11059, -11060, -11061, -11062, -11063, -11064, -11065, -11066, -11067, -11068, -11069, -11070, -11071, -11072, -11073, -11074, -11075, -11076, -11077, -11078, -11079, -11080, -11081, -11082, -11083, -11084, -11085, -11086, -11087, -11088, -11089, -11090, -11091, -11092, -11112, -11113, -11114, -11115, -11116, -11117, -11118, -11119, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2378] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4404, -4405, -4406, -4407, -4408, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4708, -4709, -4711, -4712, -4713, -4714, -4716, -4717, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -4785, -4786, -4787, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5523, -5524, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6286, -6288, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7104, -7105, -7111, -7117, -7118, -7120, -7121, -7122, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -7605, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9128, -9129, -9130, -9131, -9132, -9133, -9134, -9135, -9137, -9138, -9139, -9140, -9141, -9142, -9143, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9784, -9785, -9786, -9787, -9788, -9789, -9790, -9791, -9792, -9793, -9794, -9795, -9796, -9797, -9798, -9799, -9800, -9801, -9802, -9803, -9804, -9805, -9806, -9807, -9811, -10501, -10502, -10503, -10504, -10505, -10506, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -11053, -11054, -11055, -11056, -11057, -11058, -11059, -11060, -11061, -11062, -11063, -11064, -11065, -11066, -11067, -11068, -11069, -11070, -11071, -11072, -11073, -11074, -11075, -11076, -11077, -11078, -11079, -11080, -11081, -11082, -11083, -11084, -11085, -11086, -11087, -11088, -11089, -11090, -11091, -11092, -11112, -11113, -11114, -11115, -11116, -11117, -11118, -11119, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index e3c4ba538..f50d25e74 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -489,9 +489,7 @@ DEFINE_ERROR(OB_MERGE_ALREADY_STARTED, -4548, -1, "HY000", "merge already starte DEFINE_ERROR(OB_ROOTSERVICE_EXIST, -4549, -1, "HY000", "rootservice already exist"); DEFINE_ERROR(OB_RS_SHUTDOWN, -4550, -1, "HY000", "rootservice is shutdown"); DEFINE_ERROR(OB_SERVER_MIGRATE_IN_DENIED, -4551, -1, "HY000", "server migrate in denied"); -DEFINE_ERROR(OB_REBALANCE_TASK_CANT_EXEC, -4552, -1, "HY000", "rebalance task can not executing now"); DEFINE_ERROR(OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT, -4553, -1, "HY000", "rootserver can not hold more partition"); -DEFINE_ERROR(OB_REBALANCE_TASK_NOT_IN_PROGRESS, -4554, -1, "HY000", "rebalance task not in progress on seekdb"); DEFINE_ERROR(OB_DATA_SOURCE_NOT_EXIST, -4600, -1, "HY000", "Data source not exist"); DEFINE_ERROR(OB_DATA_SOURCE_TABLE_NOT_EXIST, -4601, -1, "HY000", "Data source table not exist"); DEFINE_ERROR(OB_DATA_SOURCE_RANGE_NOT_EXIST, -4602, -1, "HY000", "Data source range not exist"); @@ -595,11 +593,9 @@ DEFINE_ERROR_EXT(OB_TENANT_RESOURCE_UNIT_EXIST, -4712, -1, "HY000", "tenant alre DEFINE_ERROR_EXT(OB_ERR_DROP_TRUNCATE_PARTITION_REBUILD_INDEX, -4713, -1, "HY000", "rebuild global index failed when drop/truncate partitions", "rebuild global index:'%.*s' failed when drop/truncate partitions"); DEFINE_ERROR(OB_ERR_ATLER_TABLE_ILLEGAL_FK, -4714, -1, "HY000", "unique/primary keys in table referenced by enabled foreign keys"); DEFINE_ERROR(OB_LEADER_COORDINATOR_NEED_RETRY, -4716, -1, "HY000", "leader coordinator needs retry"); -DEFINE_ERROR(OB_REBALANCE_TASK_NEED_RETRY, -4717, -1, "HY000", "rebalance task needs retry"); // for log stream and tablet DEFINE_ERROR(OB_LS_NOT_EXIST, -4719, -1, "HY000", "log stream does not exist"); -DEFINE_ERROR(OB_TOO_MANY_TENANT_LS, -4720, -1, "HY000", "too many log streams were defined for this tenant"); DEFINE_ERROR(OB_LS_LOCATION_NOT_EXIST, -4721, -1, "HY000", "log stream location does not exist"); DEFINE_ERROR(OB_LS_LOCATION_LEADER_NOT_EXIST, -4722, -1, "HY000", "log stream location leader does not exist"); DEFINE_ERROR(OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST, -4723, -1, "HY000", "mapping between tablet and log stream does not exist"); @@ -620,7 +616,6 @@ DEFINE_ERROR(OB_RESOURCE_UNIT_VALUE_INVALID, -4736, -1, "HY000", "resource unit DEFINE_ERROR(OB_LS_EXIST, -4737, -1, "HY000", "log stream already exist"); DEFINE_ERROR(OB_DEVICE_EXIST, -4738, -1, "HY000", "Unexpected device existence"); DEFINE_ERROR(OB_DEVICE_NOT_EXIST, -4739, -1, "HY000", "Device does not exist"); -DEFINE_ERROR(OB_LS_REPLICA_TASK_RESULT_UNCERTAIN, -4740, -1, "HY000", "log stream replica task result is uncertain"); DEFINE_ERROR(OB_WAIT_REPLAY_TIMEOUT, -4741, -1, "HY000", "log stream wait log replay timeout"); DEFINE_ERROR(OB_WAIT_TABLET_READY_TIMEOUT, -4742, -1, "HY000", "log stream wait tablet ready timeout"); DEFINE_ERROR(OB_FREEZE_SERVICE_EPOCH_MISMATCH, -4743, -1, "HY000", "freeze service epoch is not expected"); @@ -1778,7 +1773,6 @@ DEFINE_ERROR(OB_TRANS_FREE_ROUTE_NOT_SUPPORTED, -6279, 6279, "HY000", "Query is DEFINE_ERROR(OB_TRANS_LIVE_TOO_MUCH_TIME, -6280, -1, "HY000", "Transaction cost too much without commit or rollback"); DEFINE_ERROR(OB_TRANS_COMMIT_TOO_MUCH_TIME, -6281, -1, "HY000", "Transaction commit cost too much"); -DEFINE_ERROR(OB_TRANS_TOO_MANY_PARTICIPANTS, -6282, 6002, "40000", "too many transaction participants"); DEFINE_ERROR(OB_LOG_ALREADY_SPLIT, -6283, -1, "HY000", "The big log entry has been split into multiple part"); // for json schema @@ -1902,7 +1896,6 @@ DEFINE_ERROR(OB_TABLET_GC_LOCK_CONFLICT, -7120, -1, "HY000", "tablet GC lock con DEFINE_ERROR(OB_SEQUENCE_NOT_MATCH, -7121, -1, "HY000", "compare sequence not match"); DEFINE_ERROR(OB_SEQUENCE_TOO_SMALL, -7122, -1, "HY000", "compare sequence too small"); DEFINE_ERROR_EXT_DEP(OB_PARTITION_ALREADY_BALANCED, -7124, -1, "HY000", "partitions are already balanced", "partitions are already balanced, %s"); -DEFINE_ERROR(NEW_MV_MAJOR_VERSION_NOT_MATCH, -7125, -1, "HY000", "new mv major version is not match"); // 余留位置 //////////////////////////////////////////////////////////////// @@ -2178,7 +2171,6 @@ DEFINE_ERROR(OB_LOG_ARCHIVE_ALREADY_STOPPED, -9034, -1, "HY000", "log archive al DEFINE_ERROR(OB_RESTORE_INDEX_FAILED, -9035, -1, "HY000", "restore index failed"); DEFINE_ERROR(OB_BACKUP_IN_PROGRESS, -9036, -1, "HY000", "Backup is in progress"); DEFINE_ERROR(OB_INVALID_LOG_ARCHIVE_STATUS, -9037, -1, "HY000", "log archive status is not valid"); -DEFINE_ERROR(OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST, -9038, -1, "HY000", "Cannot add replica during set member list in restore"); DEFINE_ERROR(OB_LOG_ARCHIVE_LEADER_CHANGED, -9039, -1, "HY000", "pg log archive leader changed"); DEFINE_ERROR_EXT(OB_BACKUP_CAN_NOT_START, -9040, -1, "HY000", "backup can not start", "backup can not start, because %s"); DEFINE_ERROR(OB_CANCEL_BACKUP_NOT_ALLOWED, -9041, -1, "HY000", "cancel backup do not allow"); @@ -2289,7 +2281,6 @@ DEFINE_ERROR_EXT_DEP(OB_BACKUP_ZONE_IDC_REGION_INVALID, -9133, -1, "HY000", "bac DEFINE_ERROR_DEP(OB_ERR_TMP_FILE_ALREADY_SEALED, -9134, -1, "HY000", "tmp file has already sealed"); DEFINE_ERROR_DEP(OB_TMP_FILE_EXCEED_DISK_QUOTA, -9135, -1, "HY000", "tmp file exceeds disk quota"); -DEFINE_ERROR(OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE, -9136, -1, "HY000", "Backup missing mview dependent tablet sstable, sstable maybe recycled"); DEFINE_ERROR_DEP(OB_ERR_DUPLICATE_INDEX, -9137, ER_DUP_INDEX, "HY000", "Duplicate index \'%s\' defined on the table \'%s.%s\'"); DEFINE_ERROR(OB_NO_LS_REPLICA_CAN_DO_BACKUP, -9138, -1, "HY000", "No suitable replica found to perform backup"); DEFINE_ERROR_EXT_DEP(OB_INVALID_KMS_DEST, -9139, -1, "HY000", "kms destination is not valid", "%s is not a valid uri"); @@ -2585,13 +2576,6 @@ DEFINE_ERROR_EXT(OB_ERR_WRONG_USAGE, -9751, ER_WRONG_USAGE, "HY000", "Incorrect DEFINE_ERROR(OB_ERR_FORALL_ON_REMOTE_TABLE, -9752, -1, "HY000", "FORALL INSERT/UPDATE/DELETE not support on remote tables"); DEFINE_ERROR(OB_ERR_SEQUENCE_NOT_DEFINE, -9753, -1, "HY000", "sequence is not yet defined in this session"); DEFINE_ERROR_EXT(OB_ERR_DEBUG_ID_NOT_EXIST, -9754, -1, "HY000", "debug_session_id does not exist", "debug_session_id = %u does not exist"); -DEFINE_ERROR_EXT(OB_ERR_TABLE_NO_MLOG, -9755, -1, "HY000", "table does not have a materialized view log", "table `%s`.`%s` does not have a materialized view log"); -DEFINE_ERROR_EXT(OB_ERR_MLOG_EXIST, -9756, -1, "HY000", "a materialized view log already exists on table", "a materialized view log already exists on table `%s`"); -DEFINE_ERROR_EXT(OB_ERR_MVIEW_NOT_EXIST, -9757, -1, "HY000", "materialized view does not exist", "materialized view `%s`.`%s` does not exist"); -DEFINE_ERROR_EXT(OB_ERR_MVIEW_EXIST, -9758, -1, "HY000", "materialized view already exists", "materialized view `%s`.`%s` already exists"); -DEFINE_ERROR_EXT(OB_ERR_MLOG_IS_YOUNGER, -9759, -1, "HY000", "materialized view log younger than last refresh", "materialized view log on `%s`.`%s` younger than last refresh"); -DEFINE_ERROR_EXT(OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH, -9760, -1, "HY000", "cannot fast refresh materialized view", "cannot fast refresh materialized view %s: %s"); -DEFINE_ERROR_EXT(OB_ERR_MVIEW_NEVER_REFRESH, -9761, -1, "HY000", "cannot explicitly refresh a NEVER REFRESH materialized view", "cannot explicitly refresh a NEVER REFRESH materialized view (`%s`)"); DEFINE_ERROR(OB_ERR_CLIENT_LOCAL_FILES_DISABLED, -9762, ER_CLIENT_LOCAL_FILES_DISABLED, "42000", "Loading local data is disabled; this must be enabled on both the client and server sides"); DEFINE_ERROR(OB_ERR_OUT_PARAM_NOT_BIND_VAR, -9763, -1, "HY000", "output parameter not a bind variable"); DEFINE_ERROR_EXT(OB_ERR_TIME_EARLIER_THAN_SYSDATE, -9764, -1, "HY000", "the parameter must evaluate to a time in the future", "the parameter %s must evaluate to a time in the future"); @@ -2614,7 +2598,6 @@ DEFINE_ERROR(OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST, -9779, ER_EVENT_CANNOT_ALTER DEFINE_ERROR(OB_ERR_EVENT_RECURSION_FORBIDDEN, -9780, ER_EVENT_RECURSION_FORBIDDEN, "HY000", "Recursion of EVENT DDL statements is forbidden when body is present"); DEFINE_ERROR(OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR, -9781, ER_NO_PARTITION_FOR_GIVEN_VALUE, "HY000", "Table has no partition for value"); DEFINE_ERROR(OB_ERR_INVALID_CHARACTER, -9782, -1, "HY000", "invalid character"); -DEFINE_ERROR_EXT(OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE, -9783, -1, "HY000", "cannot ENABLE ON QUERY COMPUTATION for the materialized view", "cannot ENABLE ON QUERY COMPUTATION for the materialized view %s: %s"); DEFINE_ERROR_EXT(OB_ERR_CURSOR_ATTR_APPLY, -9784, -1, "HY000", "cursor attribute may not be applied to non-cursor", "cursor attribute may not be applied to non-cursor %.*s"); DEFINE_ERROR(OB_UTL_TCP_BUFFER_TOO_SMALL, -9785, -1, "HY000", "buffer too small"); DEFINE_ERROR(OB_UTL_TCP_END_OF_INPUT, -9786, -1, "HY000", "end-of-input reached"); @@ -2639,11 +2622,7 @@ DEFINE_ERROR(OB_ERR_LOGON_TRIGGER, -9804, -1, "HY000", "client logon triggers ca DEFINE_ERROR(OB_ERR_LOGOFF_TRIGGER, -9805, -1, "HY000", "client logoff triggers cannot have AFTER type"); DEFINE_ERROR(OB_ERR_MALFORMED_WRAPPED_UNIT, -9806, -1, "HY000", "malformed or corrupted wrapped unit"); DEFINE_ERROR(OB_ERR_INVALID_PLSQL_UNIT, -9807, -1, "HY000", "input to DBMS_DDL.WRAP is not a legal PL/SQL unit"); -DEFINE_ERROR(OB_ERR_MVIEW_INVALID_TABLE_TYPE, -9808, -1, "HY000", "Table type is not valid, the definition of materialized view can only reference user tables or other materialized views"); -DEFINE_ERROR(OB_ERR_MVIEW_BASE_TABLE_ALTERED, -9809, -1, "HY000", "Base tables are altered during the creation or complete refresh of materialized views, please try again when base tables have no ongoing DDL operations"); -DEFINE_ERROR(OB_ERR_MVIEW_MISSING_DEPENDENCE, -9810, -1, "HY000", "Materialized view %s.%s has invalid dependency info, please perform a complete refresh to recover."); DEFINE_ERROR(OB_ERR_INVALID_NESTED_CURSOR, -9811, -1, "HY000", "Cursor Variable in record, object, or collection is not supported by this release"); -DEFINE_ERROR(OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH, -9812, -1, "HY000", "materialized view can not nested consistent refresh"); // 余留位置 //////////////////////////////////////////////////////////////// // PL/SQL错误码值域 [-9500, -10000) diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index fe9081710..5a896ba4f 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -312,9 +312,7 @@ constexpr int OB_MERGE_ALREADY_STARTED = -4548; constexpr int OB_ROOTSERVICE_EXIST = -4549; constexpr int OB_RS_SHUTDOWN = -4550; constexpr int OB_SERVER_MIGRATE_IN_DENIED = -4551; -constexpr int OB_REBALANCE_TASK_CANT_EXEC = -4552; constexpr int OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT = -4553; -constexpr int OB_REBALANCE_TASK_NOT_IN_PROGRESS = -4554; constexpr int OB_DATA_SOURCE_NOT_EXIST = -4600; constexpr int OB_DATA_SOURCE_TABLE_NOT_EXIST = -4601; constexpr int OB_DATA_SOURCE_RANGE_NOT_EXIST = -4602; @@ -417,9 +415,7 @@ constexpr int OB_TENANT_RESOURCE_UNIT_EXIST = -4712; constexpr int OB_ERR_DROP_TRUNCATE_PARTITION_REBUILD_INDEX = -4713; constexpr int OB_ERR_ATLER_TABLE_ILLEGAL_FK = -4714; constexpr int OB_LEADER_COORDINATOR_NEED_RETRY = -4716; -constexpr int OB_REBALANCE_TASK_NEED_RETRY = -4717; constexpr int OB_LS_NOT_EXIST = -4719; -constexpr int OB_TOO_MANY_TENANT_LS = -4720; constexpr int OB_LS_LOCATION_NOT_EXIST = -4721; constexpr int OB_LS_LOCATION_LEADER_NOT_EXIST = -4722; constexpr int OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST = -4723; @@ -437,7 +433,6 @@ constexpr int OB_RESOURCE_UNIT_VALUE_INVALID = -4736; constexpr int OB_LS_EXIST = -4737; constexpr int OB_DEVICE_EXIST = -4738; constexpr int OB_DEVICE_NOT_EXIST = -4739; -constexpr int OB_LS_REPLICA_TASK_RESULT_UNCERTAIN = -4740; constexpr int OB_WAIT_REPLAY_TIMEOUT = -4741; constexpr int OB_WAIT_TABLET_READY_TIMEOUT = -4742; constexpr int OB_FREEZE_SERVICE_EPOCH_MISMATCH = -4743; @@ -1375,7 +1370,6 @@ constexpr int OB_TRANS_IDLE_TIMEOUT = -6278; constexpr int OB_TRANS_FREE_ROUTE_NOT_SUPPORTED = -6279; constexpr int OB_TRANS_LIVE_TOO_MUCH_TIME = -6280; constexpr int OB_TRANS_COMMIT_TOO_MUCH_TIME = -6281; -constexpr int OB_TRANS_TOO_MANY_PARTICIPANTS = -6282; constexpr int OB_LOG_ALREADY_SPLIT = -6283; constexpr int INCORRECT_ARGUMENTS_TO_URL_DECODE = -6286; constexpr int OB_TX_PENDING_LOG_OVERFLOW = -6288; @@ -1452,7 +1446,6 @@ constexpr int OB_OBSOLETE_CLOG_NEED_SKIP = -7118; constexpr int OB_TABLET_GC_LOCK_CONFLICT = -7120; constexpr int OB_SEQUENCE_NOT_MATCH = -7121; constexpr int OB_SEQUENCE_TOO_SMALL = -7122; -constexpr int NEW_MV_MAJOR_VERSION_NOT_MATCH = -7125; constexpr int OB_ERR_DIMENSION_NUMBER_IS_OUT_OF_RANGE = -7290; constexpr int OB_ERR_INVALID_SRID_IN_SDO_GEOMETRY = -7292; constexpr int OB_ERR_INVALID_GTYPE_FOR_POINT_OBJECT = -7293; @@ -1523,7 +1516,6 @@ constexpr int OB_LOG_ARCHIVE_ALREADY_STOPPED = -9034; constexpr int OB_RESTORE_INDEX_FAILED = -9035; constexpr int OB_BACKUP_IN_PROGRESS = -9036; constexpr int OB_INVALID_LOG_ARCHIVE_STATUS = -9037; -constexpr int OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST = -9038; constexpr int OB_LOG_ARCHIVE_LEADER_CHANGED = -9039; constexpr int OB_BACKUP_CAN_NOT_START = -9040; constexpr int OB_CANCEL_BACKUP_NOT_ALLOWED = -9041; @@ -1576,7 +1568,6 @@ constexpr int OB_STORAGE_DEST_NOT_CONNECT = -9115; constexpr int OB_TABLET_IS_SPLIT_SRC = -9123; constexpr int OB_TABLET_STATUS_NO_NEED_TO_SPLIT = -9127; constexpr int OB_FILE_DELETE_FAILED = -9128; -constexpr int OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = -9136; constexpr int OB_NO_LS_REPLICA_CAN_DO_BACKUP = -9138; constexpr int OB_RESOURCE_RELEASED = -9141; constexpr int OB_ERR_RESIZE_FILE_TO_SMALLER = -9200; @@ -1833,13 +1824,6 @@ constexpr int OB_ERR_WRONG_USAGE = -9751; constexpr int OB_ERR_FORALL_ON_REMOTE_TABLE = -9752; constexpr int OB_ERR_SEQUENCE_NOT_DEFINE = -9753; constexpr int OB_ERR_DEBUG_ID_NOT_EXIST = -9754; -constexpr int OB_ERR_TABLE_NO_MLOG = -9755; -constexpr int OB_ERR_MLOG_EXIST = -9756; -constexpr int OB_ERR_MVIEW_NOT_EXIST = -9757; -constexpr int OB_ERR_MVIEW_EXIST = -9758; -constexpr int OB_ERR_MLOG_IS_YOUNGER = -9759; -constexpr int OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH = -9760; -constexpr int OB_ERR_MVIEW_NEVER_REFRESH = -9761; constexpr int OB_ERR_CLIENT_LOCAL_FILES_DISABLED = -9762; constexpr int OB_ERR_OUT_PARAM_NOT_BIND_VAR = -9763; constexpr int OB_ERR_TIME_EARLIER_THAN_SYSDATE = -9764; @@ -1861,7 +1845,6 @@ constexpr int OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST = -9779; constexpr int OB_ERR_EVENT_RECURSION_FORBIDDEN = -9780; constexpr int OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR = -9781; constexpr int OB_ERR_INVALID_CHARACTER = -9782; -constexpr int OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = -9783; constexpr int OB_ERR_CURSOR_ATTR_APPLY = -9784; constexpr int OB_UTL_TCP_BUFFER_TOO_SMALL = -9785; constexpr int OB_UTL_TCP_END_OF_INPUT = -9786; @@ -1886,11 +1869,7 @@ constexpr int OB_ERR_LOGON_TRIGGER = -9804; constexpr int OB_ERR_LOGOFF_TRIGGER = -9805; constexpr int OB_ERR_MALFORMED_WRAPPED_UNIT = -9806; constexpr int OB_ERR_INVALID_PLSQL_UNIT = -9807; -constexpr int OB_ERR_MVIEW_INVALID_TABLE_TYPE = -9808; -constexpr int OB_ERR_MVIEW_BASE_TABLE_ALTERED = -9809; -constexpr int OB_ERR_MVIEW_MISSING_DEPENDENCE = -9810; constexpr int OB_ERR_INVALID_NESTED_CURSOR = -9811; -constexpr int OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH = -9812; constexpr int OB_TTL_NOT_ENABLE = -10501; constexpr int OB_TTL_COLUMN_NOT_EXIST = -10502; constexpr int OB_TTL_COLUMN_TYPE_NOT_SUPPORTED = -10503; @@ -1969,6 +1948,7 @@ constexpr int OB_ERR_SPECIFIED_ROW_NO_LONGER_EXISTS = -38105; constexpr int OB_ERR_DATA_TOO_LONG_MSG_FMT_V2 = -5167; constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; + #define OB_SUCCESS__USER_ERROR_MSG "Success" #define OB_ERROR__USER_ERROR_MSG "Common error" #define OB_OBJ_TYPE_ERROR__USER_ERROR_MSG "Object type error" @@ -2370,9 +2350,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ROOTSERVICE_EXIST__USER_ERROR_MSG "rootservice already exist" #define OB_RS_SHUTDOWN__USER_ERROR_MSG "rootservice is shutdown" #define OB_SERVER_MIGRATE_IN_DENIED__USER_ERROR_MSG "server migrate in denied" -#define OB_REBALANCE_TASK_CANT_EXEC__USER_ERROR_MSG "rebalance task can not executing now" #define OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT__USER_ERROR_MSG "rootserver can not hold more partition" -#define OB_REBALANCE_TASK_NOT_IN_PROGRESS__USER_ERROR_MSG "rebalance task not in progress on seekdb" #define OB_DATA_SOURCE_NOT_EXIST__USER_ERROR_MSG "Data source not exist" #define OB_DATA_SOURCE_TABLE_NOT_EXIST__USER_ERROR_MSG "Data source table not exist" #define OB_DATA_SOURCE_RANGE_NOT_EXIST__USER_ERROR_MSG "Data source range not exist" @@ -2475,9 +2453,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DROP_TRUNCATE_PARTITION_REBUILD_INDEX__USER_ERROR_MSG "rebuild global index:'%.*s' failed when drop/truncate partitions" #define OB_ERR_ATLER_TABLE_ILLEGAL_FK__USER_ERROR_MSG "unique/primary keys in table referenced by enabled foreign keys" #define OB_LEADER_COORDINATOR_NEED_RETRY__USER_ERROR_MSG "leader coordinator needs retry" -#define OB_REBALANCE_TASK_NEED_RETRY__USER_ERROR_MSG "rebalance task needs retry" #define OB_LS_NOT_EXIST__USER_ERROR_MSG "log stream does not exist" -#define OB_TOO_MANY_TENANT_LS__USER_ERROR_MSG "too many log streams were defined for this tenant" #define OB_LS_LOCATION_NOT_EXIST__USER_ERROR_MSG "log stream location does not exist" #define OB_LS_LOCATION_LEADER_NOT_EXIST__USER_ERROR_MSG "log stream location leader does not exist" #define OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST__USER_ERROR_MSG "mapping between tablet and log stream does not exist" @@ -2497,7 +2473,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_LS_EXIST__USER_ERROR_MSG "log stream already exist" #define OB_DEVICE_EXIST__USER_ERROR_MSG "Unexpected device existence" #define OB_DEVICE_NOT_EXIST__USER_ERROR_MSG "Device does not exist" -#define OB_LS_REPLICA_TASK_RESULT_UNCERTAIN__USER_ERROR_MSG "log stream replica task result is uncertain" #define OB_WAIT_REPLAY_TIMEOUT__USER_ERROR_MSG "log stream wait log replay timeout" #define OB_WAIT_TABLET_READY_TIMEOUT__USER_ERROR_MSG "log stream wait tablet ready timeout" #define OB_FREEZE_SERVICE_EPOCH_MISMATCH__USER_ERROR_MSG "freeze service epoch is not expected" @@ -3568,7 +3543,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_TRANS_FREE_ROUTE_NOT_SUPPORTED__USER_ERROR_MSG "Query is not supported to be executed on txn temporary node" #define OB_TRANS_LIVE_TOO_MUCH_TIME__USER_ERROR_MSG "Transaction cost too much without commit or rollback" #define OB_TRANS_COMMIT_TOO_MUCH_TIME__USER_ERROR_MSG "Transaction commit cost too much" -#define OB_TRANS_TOO_MANY_PARTICIPANTS__USER_ERROR_MSG "too many transaction participants" #define OB_LOG_ALREADY_SPLIT__USER_ERROR_MSG "The big log entry has been split into multiple part" #define OB_ERR_UNSUPPROTED_REF_IN_JSON_SCHEMA__USER_ERROR_MSG "This version doesn't yet support 'references in JSON Schema." #define OB_ERR_TYPE_OF_JSON_SCHEMA__USER_ERROR_MSG "Invalid JSON type in argument, should be object." @@ -3648,7 +3622,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_SEQUENCE_NOT_MATCH__USER_ERROR_MSG "compare sequence not match" #define OB_SEQUENCE_TOO_SMALL__USER_ERROR_MSG "compare sequence too small" #define OB_PARTITION_ALREADY_BALANCED__USER_ERROR_MSG "partitions are already balanced, %s" -#define NEW_MV_MAJOR_VERSION_NOT_MATCH__USER_ERROR_MSG "new mv major version is not match" #define OB_ERR_GIS_DIFFERENT_SRIDS__USER_ERROR_MSG "Binary geometry function %s given two geometries of different srids: %u and %u, which should have been identical." #define OB_ERR_GIS_UNSUPPORTED_ARGUMENT__USER_ERROR_MSG "Calling geometry function %s with unsupported types of arguments." #define OB_ERR_GIS_UNKNOWN_ERROR__USER_ERROR_MSG "Unknown GIS error occurred in function %s." @@ -3831,7 +3804,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_RESTORE_INDEX_FAILED__USER_ERROR_MSG "restore index failed" #define OB_BACKUP_IN_PROGRESS__USER_ERROR_MSG "Backup is in progress" #define OB_INVALID_LOG_ARCHIVE_STATUS__USER_ERROR_MSG "log archive status is not valid" -#define OB_CANNOT_ADD_REPLICA_DURING_SET_MEMBER_LIST__USER_ERROR_MSG "Cannot add replica during set member list in restore" #define OB_LOG_ARCHIVE_LEADER_CHANGED__USER_ERROR_MSG "pg log archive leader changed" #define OB_BACKUP_CAN_NOT_START__USER_ERROR_MSG "backup can not start, because %s" #define OB_CANCEL_BACKUP_NOT_ALLOWED__USER_ERROR_MSG "cancel backup do not allow" @@ -3925,7 +3897,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_BACKUP_ZONE_IDC_REGION_INVALID__USER_ERROR_MSG "%s" #define OB_ERR_TMP_FILE_ALREADY_SEALED__USER_ERROR_MSG "tmp file has already sealed" #define OB_TMP_FILE_EXCEED_DISK_QUOTA__USER_ERROR_MSG "tmp file exceeds disk quota" -#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__USER_ERROR_MSG "Backup missing mview dependent tablet sstable, sstable maybe recycled" #define OB_ERR_DUPLICATE_INDEX__USER_ERROR_MSG "Duplicate index \'%s\' defined on the table \'%s.%s\'" #define OB_NO_LS_REPLICA_CAN_DO_BACKUP__USER_ERROR_MSG "No suitable replica found to perform backup" #define OB_INVALID_KMS_DEST__USER_ERROR_MSG "%s is not a valid uri" @@ -4189,13 +4160,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_FORALL_ON_REMOTE_TABLE__USER_ERROR_MSG "FORALL INSERT/UPDATE/DELETE not support on remote tables" #define OB_ERR_SEQUENCE_NOT_DEFINE__USER_ERROR_MSG "sequence is not yet defined in this session" #define OB_ERR_DEBUG_ID_NOT_EXIST__USER_ERROR_MSG "debug_session_id = %u does not exist" -#define OB_ERR_TABLE_NO_MLOG__USER_ERROR_MSG "table `%s`.`%s` does not have a materialized view log" -#define OB_ERR_MLOG_EXIST__USER_ERROR_MSG "a materialized view log already exists on table `%s`" -#define OB_ERR_MVIEW_NOT_EXIST__USER_ERROR_MSG "materialized view `%s`.`%s` does not exist" -#define OB_ERR_MVIEW_EXIST__USER_ERROR_MSG "materialized view `%s`.`%s` already exists" -#define OB_ERR_MLOG_IS_YOUNGER__USER_ERROR_MSG "materialized view log on `%s`.`%s` younger than last refresh" -#define OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH__USER_ERROR_MSG "cannot fast refresh materialized view %s: %s" -#define OB_ERR_MVIEW_NEVER_REFRESH__USER_ERROR_MSG "cannot explicitly refresh a NEVER REFRESH materialized view (`%s`)" #define OB_ERR_CLIENT_LOCAL_FILES_DISABLED__USER_ERROR_MSG "Loading local data is disabled; this must be enabled on both the client and server sides" #define OB_ERR_OUT_PARAM_NOT_BIND_VAR__USER_ERROR_MSG "output parameter not a bind variable" #define OB_ERR_TIME_EARLIER_THAN_SYSDATE__USER_ERROR_MSG "the parameter %s must evaluate to a time in the future" @@ -4217,7 +4181,6 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_EVENT_RECURSION_FORBIDDEN__USER_ERROR_MSG "Recursion of EVENT DDL statements is forbidden when body is present" #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__USER_ERROR_MSG "Table has no partition for value" #define OB_ERR_INVALID_CHARACTER__USER_ERROR_MSG "invalid character" -#define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__USER_ERROR_MSG "cannot ENABLE ON QUERY COMPUTATION for the materialized view %s: %s" #define OB_ERR_CURSOR_ATTR_APPLY__USER_ERROR_MSG "cursor attribute may not be applied to non-cursor %.*s" #define OB_UTL_TCP_BUFFER_TOO_SMALL__USER_ERROR_MSG "buffer too small" #define OB_UTL_TCP_END_OF_INPUT__USER_ERROR_MSG "end-of-input reached" @@ -4242,11 +4205,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_LOGOFF_TRIGGER__USER_ERROR_MSG "client logoff triggers cannot have AFTER type" #define OB_ERR_MALFORMED_WRAPPED_UNIT__USER_ERROR_MSG "malformed or corrupted wrapped unit" #define OB_ERR_INVALID_PLSQL_UNIT__USER_ERROR_MSG "input to DBMS_DDL.WRAP is not a legal PL/SQL unit" -#define OB_ERR_MVIEW_INVALID_TABLE_TYPE__USER_ERROR_MSG "Table type is not valid, the definition of materialized view can only reference user tables or other materialized views" -#define OB_ERR_MVIEW_BASE_TABLE_ALTERED__USER_ERROR_MSG "Base tables are altered during the creation or complete refresh of materialized views, please try again when base tables have no ongoing DDL operations" -#define OB_ERR_MVIEW_MISSING_DEPENDENCE__USER_ERROR_MSG "Materialized view %s.%s has invalid dependency info, please perform a complete refresh to recover." #define OB_ERR_INVALID_NESTED_CURSOR__USER_ERROR_MSG "Cursor Variable in record, object, or collection is not supported by this release" -#define OB_ERR_MVIEW_CAN_NOT_NESTED_CONSISTENT_REFRESH__USER_ERROR_MSG "materialized view can not nested consistent refresh" #define OB_TTL_NOT_ENABLE__USER_ERROR_MSG "TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__USER_ERROR_MSG "TTL column '%.*s' not exists" #define OB_TTL_COLUMN_TYPE_NOT_SUPPORTED__USER_ERROR_MSG "Column type of '%.*s' is not supported for TTL definition" @@ -4364,7 +4323,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DATA_TOO_LONG_MSG_FMT_V2__USER_ERROR_MSG "Data too long for column '%.*s' at row %ld" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__USER_ERROR_MSG "Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2408]; +extern int g_all_ob_errnos[2378]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); @@ -4380,6 +4339,7 @@ extern int g_all_ob_errnos[2408]; const char *ob_errpkt_strerror(const int oberr); const char *ob_errpkt_str_user_error(const int oberr); + } // end namespace common } // end namespace oceanbase diff --git a/src/share/ob_gais_client.cpp b/src/share/ob_gais_client.cpp deleted file mode 100644 index 216453069..000000000 --- a/src/share/ob_gais_client.cpp +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE - -#include "share/ob_gais_client.h" - -namespace oceanbase -{ -using namespace oceanbase::common; -using namespace oceanbase::obcall; -using namespace oceanbase::observer; -using namespace oceanbase::share; -using namespace oceanbase::transaction; - -namespace share -{ -int ObGAISClient::init(const ObAddr &self, ObGAISRequestRpc *gais_request_rpc) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", KR(ret)); - } else if (OB_UNLIKELY(!self.is_valid()) || OB_ISNULL(gais_request_rpc)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(self), KP(gais_request_rpc)); - } else { - self_ = self; - gais_request_rpc_ = gais_request_rpc; - is_inited_ = true; - LOG_INFO("gais client init success", K(self), KP(this)); - } - return ret; -} - - -int ObGAISClient::get_value(const AutoincKey &key, - const uint64_t offset, - const uint64_t increment, - const uint64_t max_value, - const uint64_t table_auto_increment, - const uint64_t desired_count, - const uint64_t cache_size, - const int64_t &autoinc_version, - uint64_t &sync_value, - uint64_t &start_inclusive, - uint64_t &end_inclusive) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISNextAutoIncValReq msg; - ObGAISNextValRpcResult rpc_result; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(key, offset, increment, table_auto_increment, max_value, - desired_count, cache_size, self_, autoinc_version))) { - LOG_WARN("fail to init request msg", K(ret)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->next_autoinc_val(leader, msg, rpc_result))) { - LOG_WARN("handle gais request failed", K(ret), K(msg), K(rpc_result)); - } else if (!rpc_result.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("rpc result is unexpected", K(ret), K(rpc_result)); - } else { - start_inclusive = rpc_result.start_inclusive_; - end_inclusive = rpc_result.end_inclusive_; - sync_value = rpc_result.sync_value_; - LOG_DEBUG("handle gais success", K(rpc_result)); - } - } - return ret; -} - -int ObGAISClient::get_sequence_value(const AutoincKey &key, const int64_t &autoinc_version, uint64_t &sequence_value) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISAutoIncKeyArg msg; - ObGAISCurrValRpcResult rpc_result; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(key, self_, autoinc_version))) { - LOG_WARN("fail to init request msg", KR(ret), K(key), K(autoinc_version)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->curr_autoinc_val(leader, msg, rpc_result))) { - LOG_WARN("handle gais request failed", K(ret), K(msg), K(rpc_result)); - } else { - sequence_value = rpc_result.sequence_value_; - LOG_DEBUG("handle gais success", K(rpc_result)); - } - } - - return ret; -} - -int ObGAISClient::get_auto_increment_values( - const common::ObIArray &autoinc_keys, - const common::ObIArray &autoinc_versions, - common::hash::ObHashMap &seq_values) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(autoinc_keys.empty())) { - // do nothing, return empty seq_values - } else { - // all keys share the same tenant id - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISAutoIncKeyArg msg; - ObGAISCurrValRpcResult rpc_result; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < autoinc_keys.count(); ++i) { - rpc_result.reset(); - AutoincKey key = autoinc_keys.at(i); - int64_t autoinc_version = autoinc_versions.at(i); - if (OB_FAIL(msg.init(key, self_, autoinc_version))) { - LOG_WARN("fail to init request msg", KR(ret), K(key), K(autoinc_version)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->curr_autoinc_val(leader, msg, rpc_result))) { - LOG_WARN("handle gais request failed", K(ret), K(msg), K(rpc_result)); - } else if (OB_FAIL(seq_values.set_refactored(key, rpc_result.sequence_value_))) { - LOG_WARN("fail to get int_value.", K(ret)); - } else { - LOG_DEBUG("handle gais success", K(rpc_result)); - } - } - } - } - } - - return ret; -} - -int ObGAISClient::local_push_to_global_value(const AutoincKey &key, - const uint64_t max_value, - const uint64_t local_sync_value, - const int64_t &autoinc_version, - const int64_t cache_size, - uint64_t &global_sync_value) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISPushAutoIncValReq msg; - uint64_t new_sync_value = 0; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(key, local_sync_value, max_value, self_, autoinc_version, - cache_size))) { - LOG_WARN("fail to init request msg", KR(ret), K(key), K(autoinc_version)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->push_autoinc_val(leader, msg, new_sync_value))) { - LOG_WARN("handle gais request failed", K(ret), K(msg)); - } else { - global_sync_value = new_sync_value; - LOG_DEBUG("handle gais success", K(global_sync_value)); - } - } - return ret; -} - -int ObGAISClient::local_sync_with_global_value(const AutoincKey &key, const int64_t &autoinc_version, uint64_t &global_sync_value) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISAutoIncKeyArg msg; - ObGAISCurrValRpcResult rpc_result; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(key, self_, autoinc_version))) { - LOG_WARN("fail to init request msg", KR(ret), K(key), K(autoinc_version)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->curr_autoinc_val(leader, msg, rpc_result))) { - LOG_WARN("handle gais request failed", K(ret), K(msg)); - } else if (!rpc_result.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("rpc result is unexpected", K(ret), K(rpc_result)); - } else { - global_sync_value = rpc_result.sync_value_; - LOG_DEBUG("handle gais success", K(global_sync_value)); - } - } - return ret; -} - -int ObGAISClient::clear_global_autoinc_cache(const AutoincKey &key) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISAutoIncKeyArg msg; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(key, self_, OB_INVALID_VERSION))) { - LOG_WARN("fail to init request msg", KR(ret), K(key)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_FAIL(gais_request_rpc_->clear_autoinc_cache(leader, msg))) { - LOG_WARN("handle gais request failed", K(ret), K(msg)); - } else { - LOG_DEBUG("clear global autoinc cache success", K(msg)); - } - } - return ret; -} - -int ObGAISClient::get_sequence_next_value(const schema::ObSequenceSchema &schema, - ObSequenceValue &nextval) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - ObGAISNextSequenceValReq msg; - ObGAISNextSequenceValRpcResult rpc_result; - ObAddr leader; - if (OB_FAIL(get_leader_(leader))) { - LOG_WARN("get leader fail", K(ret)); - } else if (OB_FAIL(msg.init(schema, self_))) { - LOG_WARN("fail to init request msg", K(ret)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_UNLIKELY(OB_ISNULL(gais_request_rpc_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("gais request rpc is null", K(ret)); - }else if (OB_FAIL(gais_request_rpc_->next_sequence_val(leader, msg, rpc_result))) { - LOG_WARN("handle gais request failed", K(ret), K(msg), K(rpc_result)); - } else { - nextval.assign(rpc_result.nextval_); - LOG_DEBUG("handle gais success", K(rpc_result)); - } - } - return ret; -} - -int ObGAISClient::get_leader_(ObAddr &leader) -{ - int ret = OB_SUCCESS; - if (gais_cache_leader_valid_) { - leader = gais_cache_leader_; - } else { - ret = OB_HASH_NOT_EXIST; - } - if (OB_SUCC(ret)) { - } else if (ret == OB_HASH_NOT_EXIST) { - ret = OB_SUCCESS; - const int64_t cluster_id = GCONF.cluster_id; - if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_NOT_INIT; - LOG_WARN("location cache is NULL", K(ret)); - } else if (OB_FAIL(GCTX.location_service_->nonblock_get_leader( - cluster_id, GAIS_LS, leader))) { - LOG_WARN("gais nonblock get leader failed", K(ret), K(GAIS_LS)); - } else if (OB_UNLIKELY(!leader.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invaild lear from location adapter", K(ret), K(leader)); - } else { - gais_cache_leader_ = leader; - gais_cache_leader_valid_ = true; - LOG_INFO("succ to refresh leader", K(cluster_id), K(leader)); - } - } else { - LOG_WARN("fail get cache from hash map", K(ret)); - } - return ret; -} - -} // share -} // oceanbase diff --git a/src/share/ob_gais_client.h b/src/share/ob_gais_client.h deleted file mode 100644 index e14979cc8..000000000 --- a/src/share/ob_gais_client.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _OB_SHARE_OB_GAIS_CLIENT_H_ -#define _OB_SHARE_OB_GAIS_CLIENT_H_ - -#include "lib/net/ob_addr.h" -#include "share/ob_gais_msg.h" -#include "share/ob_gais_rpc.h" - -namespace oceanbase -{ -namespace share -{ -class ObGAISClient -{ -public: - ObGAISClient() : is_inited_(false), self_(), gais_request_rpc_(nullptr), gais_cache_leader_(), gais_cache_leader_valid_(false) - { } - ~ObGAISClient() { } - int init(const common::ObAddr &self, share::ObGAISRequestRpc *gais_request_rpc); - TO_STRING_KV(K_(self), K_(gais_cache_leader_valid)); - -public: - int get_value(const AutoincKey &key, - const uint64_t offset, - const uint64_t increment, - const uint64_t max_value, - const uint64_t table_auto_increment, - const uint64_t desired_count, - const uint64_t cache_size, - const int64_t &autoinc_version, - uint64_t &sync_value, - uint64_t &start_inclusive, - uint64_t &end_inclusive); - int get_sequence_value(const AutoincKey &key, const int64_t &autoinc_version, uint64_t &sequence_value); - int get_auto_increment_values(const common::ObIArray &autoinc_keys, - const common::ObIArray &autoinc_versions, - common::hash::ObHashMap &seq_values); - int local_push_to_global_value(const AutoincKey &key, - const uint64_t max_value, - const uint64_t local_sync_value, - const int64_t &autoinc_version, - const int64_t cache_size, - uint64_t &global_sync_value); - int local_sync_with_global_value(const AutoincKey &key, const int64_t &autoinc_version, uint64_t &global_sync_value); - int clear_global_autoinc_cache(const AutoincKey &key); - - int get_sequence_next_value(const schema::ObSequenceSchema &schema, ObSequenceValue &nextval); - -private: - int get_leader_(common::ObAddr &leader); - -private: - bool is_inited_; - common::ObAddr self_; - share::ObGAISRequestRpc *gais_request_rpc_; - common::ObAddr gais_cache_leader_; - bool gais_cache_leader_valid_; -}; - -} // share -} // oceanbase - -#endif // _OB_SHARE_OB_GAIS_CLIENT_H_ diff --git a/src/share/ob_gais_msg.cpp b/src/share/ob_gais_msg.cpp index ba21da89b..cab045dee 100644 --- a/src/share/ob_gais_msg.cpp +++ b/src/share/ob_gais_msg.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX SHARE #include "ob_gais_msg.h" -#include "share/ob_gais_rpc.h" namespace oceanbase { @@ -25,55 +24,41 @@ using namespace oceanbase::common; using namespace oceanbase::obcall; using namespace oceanbase::share; -namespace share +namespace obcall { -// ObGAISNextAutoIncValReq -OB_SERIALIZE_MEMBER(ObGAISNextAutoIncValReq, - autoinc_key_, offset_, increment_, base_value_, max_value_, desired_cnt_, - cache_size_, sender_, autoinc_version_); - -// ObGAISAutoIncKeyArg -OB_SERIALIZE_MEMBER(ObGAISAutoIncKeyArg, autoinc_key_, sender_, autoinc_version_); - -// ObGAISPushAutoIncValReq -OB_SERIALIZE_MEMBER(ObGAISPushAutoIncValReq, autoinc_key_, base_value_, max_value_, sender_, - autoinc_version_, cache_size_); -OB_DEF_SERIALIZE(ObGAISBroadcastAutoIncCacheReq) +int ObGAISNextValResult::init(const uint64_t start_inclusive, + const uint64_t end_inclusive, + const uint64_t sync_value) { int ret = OB_SUCCESS; - OB_UNIS_ENCODE(buf_size_); - if (OB_SUCC(ret)) { - if (pos + buf_size_ > buf_len) { - ret = OB_BUF_NOT_ENOUGH; - } else { - MEMCPY(buf + pos, buf_, buf_size_); - pos += buf_size_; - } + if (start_inclusive <= 0 || end_inclusive <= 0 || start_inclusive > end_inclusive + || sync_value > end_inclusive) { + ret = OB_INVALID_ARGUMENT; + } else { + start_inclusive_ = start_inclusive; + end_inclusive_ = end_inclusive; + sync_value_ = sync_value; } return ret; } -OB_DEF_DESERIALIZE(ObGAISBroadcastAutoIncCacheReq) +int ObGAISCurrValResult::init(const uint64_t sequence_value, const uint64_t sync_value) { int ret = OB_SUCCESS; - OB_UNIS_DECODE(buf_size_); - if (OB_SUCC(ret)) { - buf_ = buf + pos; - pos += buf_size_; + if (sequence_value < sync_value) { + ret = OB_INVALID_ARGUMENT; + } else { + sequence_value_ = sequence_value; + sync_value_ = sync_value; } return ret; } -OB_DEF_SERIALIZE_SIZE(ObGAISBroadcastAutoIncCacheReq) -{ - int64_t len = 0; - OB_UNIS_ADD_LEN(buf_size_); - len += buf_size_; - return len; -} -OB_SERIALIZE_MEMBER(ObGAISNextSequenceValReq, schema_, sender_); +} // namespace obcall +namespace share +{ int ObGAISNextAutoIncValReq::init(const AutoincKey &autoinc_key, const uint64_t offset, const uint64_t increment, @@ -81,16 +66,15 @@ int ObGAISNextAutoIncValReq::init(const AutoincKey &autoinc_key, const uint64_t max_value, const uint64_t desired_cnt, const uint64_t cache_size, - const common::ObAddr &sender, const int64_t &autoinc_version) { int ret = OB_SUCCESS; if (max_value <= 0 || cache_size <= 0 || offset < 1 || increment < 1 || base_value > max_value || - desired_cnt <= 0 || !sender.is_valid()) { + desired_cnt <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(autoinc_key), K(offset), K(increment), K(base_value), - K(max_value), K(desired_cnt), K(cache_size), K(sender)); + K(max_value), K(desired_cnt), K(cache_size)); } else { autoinc_key_ = autoinc_key; offset_ = offset; @@ -99,59 +83,43 @@ int ObGAISNextAutoIncValReq::init(const AutoincKey &autoinc_key, max_value_ = max_value; desired_cnt_ = desired_cnt; cache_size_ = cache_size; - sender_ = sender; autoinc_version_ = get_modify_autoinc_version(autoinc_version); } return ret; } -int ObGAISAutoIncKeyArg::init(const AutoincKey &autoinc_key, const common::ObAddr &sender, const int64_t autoinc_version) +int ObGAISAutoIncKeyArg::init(const AutoincKey &autoinc_key, const int64_t autoinc_version) { - int ret = OB_SUCCESS; - if (!sender.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(autoinc_key), K(sender)); - } else { - autoinc_key_ = autoinc_key; - sender_ = sender; - autoinc_version_ = get_modify_autoinc_version(autoinc_version); - } - return ret; + autoinc_key_ = autoinc_key; + autoinc_version_ = get_modify_autoinc_version(autoinc_version); + return OB_SUCCESS; } int ObGAISPushAutoIncValReq::init(const AutoincKey &autoinc_key, const uint64_t base_value, const uint64_t max_value, - const common::ObAddr &sender, const int64_t &autoinc_version, const int64_t cache_size) { int ret = OB_SUCCESS; - if (max_value <= 0 || base_value > max_value || !sender.is_valid()) { + if (max_value <= 0 || base_value > max_value) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(autoinc_key), K(base_value), K(max_value), K(sender)); + LOG_WARN("invalid argument", K(ret), K(autoinc_key), K(base_value), K(max_value)); } else { autoinc_key_ = autoinc_key; base_value_ = base_value; max_value_ = max_value; - sender_ = sender; autoinc_version_ = get_modify_autoinc_version(autoinc_version); cache_size_ = cache_size; } return ret; } -int ObGAISNextSequenceValReq::init(const schema::ObSequenceSchema &schema, - const common::ObAddr &sender) +int ObGAISNextSequenceValReq::init(const schema::ObSequenceSchema &schema) { int ret = OB_SUCCESS; - if (!sender.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema), K(sender)); - } else if (OB_FAIL(schema_.assign(schema))){ + if (OB_FAIL(schema_.assign(schema))){ LOG_WARN("fail to init schemar_", K(ret)); - } else { - sender_ = sender; } return ret; } diff --git a/src/share/ob_gais_msg.h b/src/share/ob_gais_msg.h index e104d440a..8db9804de 100644 --- a/src/share/ob_gais_msg.h +++ b/src/share/ob_gais_msg.h @@ -17,17 +17,55 @@ #ifndef _OB_SHARE_OB_GAIS_MSG_H_ #define _OB_SHARE_OB_GAIS_MSG_H_ -#include "lib/utility/ob_unify_serialize.h" -#include "lib/net/ob_addr.h" #include "share/ob_autoincrement_param.h" #include "share/ob_define.h" +#include "share/sequence/ob_sequence_option.h" namespace oceanbase { namespace obcall { -struct ObGAISNextValRpcResult; -struct ObGAISCurrValRpcResult; +struct ObGAISNextValResult +{ + ObGAISNextValResult() : start_inclusive_(0), end_inclusive_(0), sync_value_(0) {} + int init(const uint64_t start_inclusive, const uint64_t end_inclusive, const uint64_t sync_value); + bool is_valid() const + { + return start_inclusive_ > 0 && end_inclusive_ > 0 && start_inclusive_ <= end_inclusive_ + && sync_value_ <= end_inclusive_; + } + TO_STRING_KV(K_(start_inclusive), K_(end_inclusive), K_(sync_value)); + + uint64_t start_inclusive_; + uint64_t end_inclusive_; + uint64_t sync_value_; +}; + +struct ObGAISCurrValResult +{ + ObGAISCurrValResult() : sequence_value_(0), sync_value_(0) {} + int init(const uint64_t sequence_value, const uint64_t sync_value); + bool is_valid() const + { + return sequence_value_ > 0 && sequence_value_ >= sync_value_; + } + void reset() + { + sequence_value_ = 0; + sync_value_ = 0; + } + TO_STRING_KV(K_(sequence_value), K_(sync_value)); + + uint64_t sequence_value_; + uint64_t sync_value_; +}; + +struct ObGAISNextSequenceValResult +{ + ObGAISNextSequenceValResult() : nextval_() {} + TO_STRING_KV(K_(nextval)); + share::ObSequenceValue nextval_; +}; } namespace share { @@ -35,11 +73,9 @@ namespace share /* Request for get next auto increment value */ struct ObGAISNextAutoIncValReq { - OB_UNIS_VERSION(1); - public: ObGAISNextAutoIncValReq() : autoinc_key_(), offset_(0), increment_(0), base_value_(0), - max_value_(0), desired_cnt_(0), cache_size_(0), sender_(), autoinc_version_(OB_INVALID_VERSION) {} + max_value_(0), desired_cnt_(0), cache_size_(0), autoinc_version_(OB_INVALID_VERSION) {} int init(const AutoincKey &autoinc_key, const uint64_t offset, const uint64_t increment, @@ -47,16 +83,15 @@ struct ObGAISNextAutoIncValReq const uint64_t max_value, const uint64_t desired_cnt, const uint64_t cache_size, - const common::ObAddr &sender, const int64_t &autoinc_version); bool is_valid() const { return offset_ > 0 && increment_ > 0 && - max_value_ > 0 && desired_cnt_ > 0 && cache_size_ > 0 && sender_.is_valid() + max_value_ > 0 && desired_cnt_ > 0 && cache_size_ > 0 && autoinc_version_ >= OB_INVALID_VERSION; } TO_STRING_KV(K_(autoinc_key), K_(offset), K_(increment), K_(base_value), K_(max_value), - K_(desired_cnt), K_(cache_size), K_(sender), K_(autoinc_version)); + K_(desired_cnt), K_(cache_size), K_(autoinc_version)); AutoincKey autoinc_key_; uint64_t offset_; @@ -65,103 +100,65 @@ struct ObGAISNextAutoIncValReq uint64_t max_value_; uint64_t desired_cnt_; uint64_t cache_size_; - common::ObAddr sender_; int64_t autoinc_version_; }; /* GAIS autoinc key rpc argument */ struct ObGAISAutoIncKeyArg { - OB_UNIS_VERSION(1); - public: - ObGAISAutoIncKeyArg() : autoinc_key_(), sender_(), autoinc_version_(OB_INVALID_VERSION) {} - int init(const AutoincKey &autoinc_key, const common::ObAddr &sender, const int64_t autoinc_version); + ObGAISAutoIncKeyArg() : autoinc_key_(), autoinc_version_(OB_INVALID_VERSION) {} + int init(const AutoincKey &autoinc_key, const int64_t autoinc_version); bool is_valid() const { - return sender_.is_valid() && autoinc_version_ >= OB_INVALID_VERSION; + return autoinc_version_ >= OB_INVALID_VERSION; } - TO_STRING_KV(K_(autoinc_key), K_(sender), K_(autoinc_version)); + TO_STRING_KV(K_(autoinc_key), K_(autoinc_version)); AutoincKey autoinc_key_; - common::ObAddr sender_; int64_t autoinc_version_; }; /* Request for push local sync value to global */ struct ObGAISPushAutoIncValReq { - OB_UNIS_VERSION(1); - public: - ObGAISPushAutoIncValReq() : autoinc_key_(), base_value_(0), max_value_(0), sender_(), + ObGAISPushAutoIncValReq() : autoinc_key_(), base_value_(0), max_value_(0), autoinc_version_(OB_INVALID_VERSION), cache_size_(0) {} int init(const AutoincKey &autoinc_key, const uint64_t base_value, const uint64_t max_value, - const common::ObAddr &sender, const int64_t &autoinc_version, const int64_t cache_size); bool is_valid() const { return max_value_ > 0 && base_value_ <= max_value_ - && sender_.is_valid() && autoinc_version_ >= OB_INVALID_VERSION && cache_size_ >= 0; + && autoinc_version_ >= OB_INVALID_VERSION && cache_size_ >= 0; } - TO_STRING_KV(K_(autoinc_key), K_(base_value), K_(max_value), K_(sender), K_(autoinc_version), + TO_STRING_KV(K_(autoinc_key), K_(base_value), K_(max_value), K_(autoinc_version), K_(cache_size)); AutoincKey autoinc_key_; uint64_t base_value_; uint64_t max_value_; - common::ObAddr sender_; int64_t autoinc_version_; int64_t cache_size_; }; -struct ObGAISBroadcastAutoIncCacheReq -{ - OB_UNIS_VERSION(1); - -public: - ObGAISBroadcastAutoIncCacheReq() : buf_(NULL), buf_size_(0) {} - - int init(const char *buf, const int64_t size) - { - - buf_ = buf; - buf_size_ = size; - return common::OB_SUCCESS; - } - - bool is_valid() const - { - return NULL != buf_ && buf_size_ > 0; - } - - TO_STRING_KV(KP_(buf), K_(buf_size)); - - const char *buf_; - int64_t buf_size_; -}; - /* Request for get next sequence value */ struct ObGAISNextSequenceValReq { - OB_UNIS_VERSION(1); - public: - ObGAISNextSequenceValReq() : schema_(), sender_() - {} - int init(const schema::ObSequenceSchema &schema, const common::ObAddr &sender); + ObGAISNextSequenceValReq() : schema_() {} + int init(const schema::ObSequenceSchema &schema); bool is_valid() const { return schema_.get_sequence_id() != OB_INVALID_ID - && schema_.get_cache_size() > static_cast(0) && sender_.is_valid(); + && schema_.get_cache_size() > static_cast(0); } - TO_STRING_KV(K_(schema), K_(sender)); + TO_STRING_KV(K_(schema)); schema::ObSequenceSchema schema_; - common::ObAddr sender_; }; } // share diff --git a/src/share/ob_gais_rpc.cpp b/src/share/ob_gais_rpc.cpp deleted file mode 100644 index 779cdb86c..000000000 --- a/src/share/ob_gais_rpc.cpp +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE - -#include "ob_gais_rpc.h" -#include "share/rc/ob_module_provider.h" -#include "share/ob_global_autoinc_service.h" -#include "share/ob_ex_rpc.h" - -namespace oceanbase -{ -using namespace oceanbase::common; -using namespace oceanbase::obcall; -using namespace oceanbase::observer; -using namespace oceanbase::share; -using namespace oceanbase::transaction; - -namespace obcall -{ - -OB_SERIALIZE_MEMBER(ObGAISNextValRpcResult, start_inclusive_, end_inclusive_, sync_value_); - -OB_SERIALIZE_MEMBER(ObGAISCurrValRpcResult, sequence_value_, sync_value_); - -OB_DEF_SERIALIZE(ObGAISNextSequenceValRpcResult) -{ - int ret = OB_SUCCESS; - OB_UNIS_ENCODE(nextval_); - return ret; -} - -OB_DEF_DESERIALIZE(ObGAISNextSequenceValRpcResult) -{ - int ret = OB_SUCCESS; - share::ObSequenceValue nextval; - OB_UNIS_DECODE(nextval); - // deep copy is needed to ensure that the memory of nextval_ will not be reclaimed - if (OB_SUCC(ret) && OB_FAIL(nextval_.assign(nextval))) { - LOG_WARN("fail to assign nextval", K(ret)); - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObGAISNextSequenceValRpcResult) -{ - int64_t len = 0; - OB_UNIS_ADD_LEN(nextval_); - return len; -} - -int ObGAISNextValRpcResult::init(const uint64_t start_inclusive, const uint64_t end_inclusive, - const uint64_t sync_value) -{ - int ret = OB_SUCCESS; - if (start_inclusive <= 0 || end_inclusive <= 0 || start_inclusive > end_inclusive || - sync_value_ > end_inclusive) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(start_inclusive), K(end_inclusive), K(sync_value)); - } else { - start_inclusive_ = start_inclusive; - end_inclusive_ = end_inclusive; - sync_value_ = sync_value; - } - return ret; -} - -int ObGAISCurrValRpcResult::init(const uint64_t sequence_value, const uint64_t sync_value) -{ - int ret = OB_SUCCESS; - if (sequence_value < sync_value) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(sequence_value), K(sync_value)); - } else { - sequence_value_ = sequence_value; - sync_value_ = sync_value; - } - return ret; -} - -} // obcall - -namespace share -{ -int ObGAISRequestRpc::init(const ObAddr &self) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - LOG_WARN("gais request rpc inited twice", KR(ret)); - } else if (!self.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(self)); - } else { - self_ = self; - is_inited_ = true; - LOG_INFO("gais request rpc inited success", KP(this), K(self)); - } - return ret; -} - -void ObGAISRequestRpc::destroy() -{ - int tmp_ret = OB_SUCCESS; - if (is_inited_) { - is_inited_ = false; - self_.reset(); - LOG_INFO("gais request rpc destroy"); - } -} - -int ObGAISRequestRpc::next_autoinc_val(const ObAddr &server, - const ObGAISNextAutoIncValReq &msg, - ObGAISNextValRpcResult &rpc_result) -{ - int ret = OB_SUCCESS; - const uint64_t timeout = THIS_WORKER.get_timeout_remain(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server), K(msg)); - } else { - ObGlobalAutoIncService *gais = nullptr; - - MOD_SCOPE { - if (OB_ISNULL(gais = share::g_mp->global_auto_inc_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("global autoinc service is null", K(ret)); - } else if (OB_FAIL(gais->handle_next_autoinc_request(msg, rpc_result))) { - LOG_WARN("post local gais require autoinc request failed", KR(ret), K(server), K(msg)); - } else if (!rpc_result.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("post local gais require autoinc and gais_rpc_result is invalid", KR(ret), K(server), - K(msg), K(rpc_result)); - } else { - LOG_TRACE("post local require autoinc request success", K(msg), K(rpc_result)); - } - } - } - return ret; -} - -int ObGAISRequestRpc::curr_autoinc_val(const ObAddr &server, - const ObGAISAutoIncKeyArg &msg, - ObGAISCurrValRpcResult &rpc_result) -{ - int ret = OB_SUCCESS; - const uint64_t timeout = THIS_WORKER.get_timeout_remain(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server), K(msg)); - } else { - ObGlobalAutoIncService *gais = nullptr; - - MOD_SCOPE { - if (OB_ISNULL(gais = share::g_mp->global_auto_inc_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("global autoinc service is null", K(ret)); - } else if (OB_FAIL(gais->handle_curr_autoinc_request(msg, rpc_result))) { - LOG_WARN("post local gais get autoinc request failed", KR(ret), K(server), K(msg)); - } else { - LOG_TRACE("post local get autoinc request success", K(msg), K(rpc_result)); - } - } - } - return ret; -} - -int ObGAISRequestRpc::push_autoinc_val(const ObAddr &server, - const ObGAISPushAutoIncValReq &msg, - uint64_t &sync_value) -{ - int ret = OB_SUCCESS; - const uint64_t timeout = THIS_WORKER.get_timeout_remain(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server), K(msg)); - } else { - ObGlobalAutoIncService *gais = nullptr; - - MOD_SCOPE { - if (OB_ISNULL(gais = share::g_mp->global_auto_inc_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("global autoinc service is null", K(ret)); - } else if (OB_FAIL(gais->handle_push_autoinc_request(msg, sync_value))) { - LOG_WARN("post local gais push global request failed", KR(ret), K(server), K(msg)); - } else { - LOG_TRACE("post local gais push global request request success", K(msg), K(sync_value)); - } - } - } - return ret; -} - -int ObGAISRequestRpc::clear_autoinc_cache(const ObAddr &server, const ObGAISAutoIncKeyArg &msg) -{ - int ret = OB_SUCCESS; - const uint64_t timeout = THIS_WORKER.get_timeout_remain(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server), K(msg)); - } else { - ObGlobalAutoIncService *gais = nullptr; - - MOD_SCOPE { - if (OB_ISNULL(gais = share::g_mp->global_auto_inc_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("global autoinc service is null", K(ret)); - } else if (OB_FAIL(gais->handle_clear_autoinc_cache_request(msg))) { - LOG_WARN("post local gais clear autoinc cache failed", KR(ret), K(server), K(msg)); - } else { - LOG_TRACE("clear autoinc cache success", K(server), K(msg)); - } - } - } - return ret; -} - -int ObGAISRequestRpc::broadcast_global_autoinc_cache(const ObGAISBroadcastAutoIncCacheReq &msg) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(msg)); - } else { - // single-replica: broadcast target is always self; nothing to do - } - return ret; -} - -int ObGAISRequestRpc::next_sequence_val(const common::ObAddr &server, - const ObGAISNextSequenceValReq &msg, - ObGAISNextSequenceValRpcResult &rpc_result) -{ - int ret = OB_SUCCESS; - const uint64_t timeout = THIS_WORKER.get_timeout_remain(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("gais request rpc not inited", KR(ret)); - } else if (!server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server), K(msg)); - } else { - ObGlobalAutoIncService *gais = nullptr; - - MOD_SCOPE { - if (OB_ISNULL(gais = share::g_mp->global_auto_inc_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("global autoinc service is null", K(ret)); - } else if (OB_FAIL(gais->handle_next_sequence_request(msg, rpc_result))) { - LOG_WARN("post local gais require autoinc request failed", KR(ret), K(server), K(msg)); - } else { - LOG_TRACE("post local require autoinc request success", K(msg), K(rpc_result)); - } - } - } - return ret; -} - -} // share -} // oceanbase diff --git a/src/share/ob_gais_rpc.h b/src/share/ob_gais_rpc.h deleted file mode 100644 index 9949cc41f..000000000 --- a/src/share/ob_gais_rpc.h +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _OB_SHARE_OB_GAIS_RPC_H_ -#define _OB_SHARE_OB_GAIS_RPC_H_ - -#include "config/ob_server_config.h" -#include "lib/utility/ob_unify_serialize.h" -#include "lib/utility/utility.h" -#include "share/ob_server_struct.h" -#include "share/ob_define.h" -#include "share/ob_rpc_struct.h" -#include "share/ob_gais_msg.h" -#include "rpc/frame/ob_result_code.h" - -namespace oceanbase -{ -namespace obcall -{ -struct ObGAISNextValRpcResult -{ - ObGAISNextValRpcResult() : start_inclusive_(0), end_inclusive_(0), sync_value_(0) {} - int init(const uint64_t start_inclusive, const uint64_t end_inclusive, const uint64_t sync_value); - bool is_valid() const - { - return start_inclusive_ > 0 && end_inclusive_ > 0 && start_inclusive_ <= end_inclusive_ - && sync_value_ <= end_inclusive_; - } - TO_STRING_KV(K_(start_inclusive), K_(end_inclusive), K_(sync_value)); - - uint64_t start_inclusive_; - uint64_t end_inclusive_; - uint64_t sync_value_; - - OB_UNIS_VERSION(1); -}; - -struct ObGAISCurrValRpcResult -{ - ObGAISCurrValRpcResult() : sequence_value_(0), sync_value_(0) {} - int init(const uint64_t sequence_value, const uint64_t sync_value); - bool is_valid() const - { - return sequence_value_ > 0 && sequence_value_ >= sync_value_; - } - void reset() - { - sequence_value_ = 0; - sync_value_ = 0; - } - TO_STRING_KV(K_(sequence_value), K_(sync_value)); - - uint64_t sequence_value_; - uint64_t sync_value_; - - OB_UNIS_VERSION(1); -}; - -struct ObGAISNextSequenceValRpcResult -{ - ObGAISNextSequenceValRpcResult() : nextval_() {} - TO_STRING_KV(K_(nextval)); - share::ObSequenceValue nextval_; - - OB_UNIS_VERSION(1); -}; - - -} // obcall - -namespace share -{ - -class ObGAISRequestRpc -{ -public: - ObGAISRequestRpc() : is_inited_(false) {} - ~ObGAISRequestRpc() { destroy(); } - int init(const common::ObAddr &self); - void destroy(); -public: - /* - * Returns the next (batch) auto-increment value of specified key, - * and changes the current auto-increment value. - */ - int next_autoinc_val(const common::ObAddr &server, - const ObGAISNextAutoIncValReq &msg, - obcall::ObGAISNextValRpcResult &rpc_result); - - /* - * Returns the next sequence value of specified key, - * and changes the current sequence value. - */ - int next_sequence_val(const common::ObAddr &server, - const ObGAISNextSequenceValReq &msg, - obcall::ObGAISNextSequenceValRpcResult &rpc_result); - /* - * Returns the current auto-increment value of specified key. - */ - int curr_autoinc_val(const common::ObAddr &server, - const ObGAISAutoIncKeyArg &msg, - obcall::ObGAISCurrValRpcResult &rpc_result); - /* - * Push local sync value to global auto-increment service. This function may - * change global sync value and current auto-increment value, and return - * updated latest sync value. - */ - int push_autoinc_val(const common::ObAddr &server, - const ObGAISPushAutoIncValReq &msg, - uint64_t &sync_value); - - int clear_autoinc_cache(const common::ObAddr &server, - const ObGAISAutoIncKeyArg &msg); - - int broadcast_global_autoinc_cache(const ObGAISBroadcastAutoIncCacheReq &msg); - -private: - bool is_inited_; - common::ObAddr self_; -}; - -} // share -} // oceanbase - -#endif // _OB_SHARE_OB_GAIS_RPC_H_ diff --git a/src/share/ob_global_autoinc_service.cpp b/src/share/ob_global_autoinc_service.cpp index 546b53150..559dd78ff 100644 --- a/src/share/ob_global_autoinc_service.cpp +++ b/src/share/ob_global_autoinc_service.cpp @@ -18,6 +18,7 @@ #include "lib/lock/ob_mutex.h" #include "share/ob_global_autoinc_service.h" +#include "share/ob_server_struct.h" #include "share/sequence/ob_sequence_cache.h" namespace oceanbase @@ -28,9 +29,6 @@ using namespace oceanbase::share; namespace share { -OB_SERIALIZE_MEMBER(ObAutoIncCacheNode, // FARM COMPAT WHITELIST - start_, end_, sync_value_, autoinc_version_); - int ObAutoIncCacheNode::init(const uint64_t start, const uint64_t end, const uint64_t sync_value, @@ -96,7 +94,7 @@ int ObAutoIncCacheNode::with_new_end(const uint64_t new_end) } -int ObGlobalAutoIncService::init(const ObAddr &addr, ObMySQLProxy *mysql_proxy) +int ObGlobalAutoIncService::init(ObMySQLProxy *mysql_proxy) { int ret = OB_SUCCESS; ObMemAttr attr(ObModIds::OB_AUTOINCREMENT); @@ -109,15 +107,10 @@ int ObGlobalAutoIncService::init(const ObAddr &addr, ObMySQLProxy *mysql_proxy) attr, attr))) { LOG_WARN("init autoinc_map_ failed", K(ret)); - } else if (OB_ISNULL(gais_request_rpc_ = - ObAutoincrementService::get_instance().get_gais_request_rpc())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("gais request rpc is null", K(ret), KP(gais_request_rpc_)); } else { for (int64_t i = 0; i < MUTEX_NUM; ++i) { op_mutex_[i].set_latch_id(common::ObLatchIds::AUTO_INCREMENT_GAIS_LOCK); } - self_ = addr; is_inited_ = true; } return ret; @@ -127,8 +120,7 @@ int ObGlobalAutoIncService::mtl_init(ObGlobalAutoIncService *&gais) { int ret = OB_SUCCESS; ObMySQLProxy *mysql_proxy = GCTX.sql_proxy_; - const ObAddr &self = GCTX.self_addr(); - ret = gais->init(self, mysql_proxy); + ret = gais->init(mysql_proxy); return ret; } @@ -136,11 +128,6 @@ void ObGlobalAutoIncService::destroy() { autoinc_map_.destroy(); inner_table_proxy_.reset(); - ObSpinLockGuard lock(cache_ls_lock_); - cache_ls_ = NULL; - is_leader_ = false; - gais_request_rpc_ = NULL; - is_switching_ = false; is_inited_ = false; } @@ -155,24 +142,15 @@ int ObGlobalAutoIncService::clear() int ObGlobalAutoIncService::handle_next_autoinc_request( const ObGAISNextAutoIncValReq &request, - obcall::ObGAISNextValRpcResult &result) + obcall::ObGAISNextValResult &result) { int ret = OB_SUCCESS; const AutoincKey &key = request.autoinc_key_; const uint64_t desired_count = request.desired_cnt_; - bool is_leader = false; lib::ObMutex &mutex = op_mutex_[key.hash() % MUTEX_NUM]; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("global service is not init", K(ret)); - } else if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_FAIL(check_leader_( is_leader))) { - LOG_WARN("check leader failed", K(ret), K(request.sender_), K(self_)); - } else if (OB_UNLIKELY(!is_leader)) { - ret = OB_NOT_MASTER; - LOG_WARN("gais service is not leader", K(ret)); } else if (OB_FAIL(mutex.lock())) { LOG_WARN("fail to get lock", K(ret)); } else { @@ -184,8 +162,6 @@ int ObGlobalAutoIncService::handle_next_autoinc_request( if (OB_UNLIKELY(OB_SUCCESS != err && OB_HASH_NOT_EXIST != err)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get seq value", K(ret), K(key)); - } else if (OB_UNLIKELY(cache_node.is_received())) { - ret = read_and_push_inner_table(key, request.max_value_, cache_node); } if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(!cache_node.is_valid() @@ -248,22 +224,16 @@ int ObGlobalAutoIncService::handle_next_autoinc_request( } int ObGlobalAutoIncService::handle_curr_autoinc_request(const ObGAISAutoIncKeyArg &request, - obcall::ObGAISCurrValRpcResult &result) + obcall::ObGAISCurrValResult &result) { int ret = OB_SUCCESS; const AutoincKey &key = request.autoinc_key_; uint64_t sequence_value = 0; uint64_t sync_value = 0; - bool is_leader = false; lib::ObMutex &mutex = op_mutex_[key.hash() % MUTEX_NUM]; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("global service is not init", K(ret)); - } else if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_FAIL(check_leader_( is_leader))) { - LOG_WARN("check leader failed", K(ret), K(request.sender_), K(self_)); } else if (OB_FAIL(mutex.lock())) { LOG_WARN("fail to get lock", K(ret)); } else { @@ -275,20 +245,16 @@ int ObGlobalAutoIncService::handle_curr_autoinc_request(const ObGAISAutoIncKeyAr if (OB_UNLIKELY(OB_SUCCESS != err && OB_HASH_NOT_EXIST != err)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get seq value", K(ret), K(key)); - } else if (is_leader - && OB_LIKELY(cache_node.is_valid() && !cache_node.is_received()) + } else if (OB_LIKELY(cache_node.is_valid()) && request_version == cache_node.autoinc_version_) { // get autoinc values from cache sequence_value = cache_node.start_; sync_value = cache_node.sync_value_; - // hash not exist, cache node is non-valid or service is not leader, + // hash not exist or cache node is non-valid, // read value from inner table } else if (OB_FAIL(read_value_from_inner_table_(key, request_version, sequence_value, sync_value))) { LOG_WARN("fail to read value from inner table", KR(ret), K_(key.table_id)); - } else if (OB_UNLIKELY(cache_node.is_received()) && (sequence_value - 1 == cache_node.end_)) { - sequence_value = cache_node.start_; - sync_value = cache_node.sync_value_; } if (OB_SUCC(ret)) { if (OB_FAIL(result.init(sequence_value, sync_value))) { @@ -307,19 +273,10 @@ int ObGlobalAutoIncService::handle_push_autoinc_request( { int ret = OB_SUCCESS; const AutoincKey &key = request.autoinc_key_; - bool is_leader = false; lib::ObMutex &mutex = op_mutex_[key.hash() % MUTEX_NUM]; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("global service is not init", K(ret)); - } else if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_FAIL(check_leader_( is_leader))) { - LOG_WARN("check leader failed", K(ret), K(request.sender_), K(self_)); - } else if (OB_UNLIKELY(!is_leader)) { - ret = OB_NOT_MASTER; - LOG_WARN("gais service is not leader", K(ret)); } else if (OB_FAIL(mutex.lock())) { LOG_WARN("fail to get lock", K(ret)); } else { @@ -353,10 +310,7 @@ int ObGlobalAutoIncService::handle_push_autoinc_request( // insert value is too small and no need to update node } else { sync_value = MAX(MAX(insert_value, cache_node.sync_value_), cache_node.start_ - 1); - if (OB_UNLIKELY(cache_node.is_received()) && - OB_FAIL(read_and_push_inner_table(key, request.max_value_, cache_node))) { - LOG_WARN("fail to read and push inner table", K(ret), K(key), K(cache_node)); - } else if (cache_node.is_valid()) { + if (cache_node.is_valid()) { cache_node.start_ = sync_value + 1; cache_node.sync_value_ = sync_value; } @@ -374,19 +328,10 @@ int ObGlobalAutoIncService::handle_clear_autoinc_cache_request(const ObGAISAutoI { int ret = OB_SUCCESS; const AutoincKey &key = request.autoinc_key_; - bool is_leader = false; lib::ObMutex &mutex = op_mutex_[key.hash() % MUTEX_NUM]; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("global service is not init", K(ret)); - } else if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_FAIL(check_leader_( is_leader))) { - LOG_WARN("check leader failed", K(ret), K(request.sender_), K(self_)); - } else if (OB_UNLIKELY(!is_leader)) { - ret = OB_NOT_MASTER; - LOG_WARN("gais service is not leader", K(ret)); } else if (OB_FAIL(mutex.lock())) { LOG_WARN("fail to get lock", K(ret)); } else { @@ -404,7 +349,7 @@ int ObGlobalAutoIncService::handle_clear_autoinc_cache_request(const ObGAISAutoI int ObGlobalAutoIncService::handle_next_sequence_request( const ObGAISNextSequenceValReq &request, - obcall::ObGAISNextSequenceValRpcResult &result) + obcall::ObGAISNextSequenceValResult &result) { int ret = OB_SUCCESS; ObSequenceCache *sequence_cache = &share::ObSequenceCache::get_instance(); @@ -490,190 +435,5 @@ int ObGlobalAutoIncService::sync_value_to_inner_table_( return ret; } -int ObGlobalAutoIncService::inner_switch_to_follower() -{ - int ret = OB_SUCCESS; - const int64_t start_time_us = ObTimeUtility::current_time(); - lib::ObMutexGuard lock(switching_mutex_); - LOG_INFO("start to switch to follower", KP(this), K(*this)); - ATOMIC_STORE(&is_switching_, true); - ATOMIC_STORE(&is_leader_, false); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("global service is not init", K(ret)); - } else if (OB_FAIL(broadcast_global_autoinc_cache())) { - LOG_WARN("fail to broadcast global autoinc cache", K(ret)); - } - if (OB_SUCC(ret)) { - // If the broadcast is successful, all cache nodes will be updated to is_received, - // otherwise the map will be cleared directly. - } else { - // ob failed - int tmp_ret = clear(); - if (tmp_ret != OB_SUCCESS) { - LOG_WARN("fail to clear auto inc map", K(ret), K(tmp_ret), K(autoinc_map_.size())); - } - } - ATOMIC_STORE(&is_switching_, false); - const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; - LOG_INFO("global_autoinc service: switch_to_follower", K(*this), K(cost_us)); - return ret; -} - -int ObGlobalAutoIncService::broadcast_global_autoinc_cache() -{ - int ret = OB_SUCCESS; - if (autoinc_map_.size() > 0) { - if (OB_FAIL(wait_all_requests_to_finish())) { - LOG_WARN("fail to wait all requests to finish", K(ret)); - } else if (autoinc_map_.size() > 0) { - const int64_t size = serialize_size_autoinc_cache(); - - ObMemAttr attr(ObModIds::OB_AUTOINCREMENT); - char *buffer = NULL; - int64_t pos = 0; - ObGAISBroadcastAutoIncCacheReq msg; - if (OB_ISNULL(buffer = static_cast(ob_malloc(size, attr)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc cache buffer", K(ret), K(size)); - } else if (OB_FAIL(serialize_autoinc_cache(buffer, size, pos))) { - LOG_WARN("fail to serialize global autoinc cache", K(ret)); - } else if (OB_FAIL(msg.init(buffer, pos))) { - LOG_WARN("fail to init msg", K(ret), K(buffer), K(pos)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(msg)); - } else if (OB_ISNULL(gais_request_rpc_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("global service request rpc is not init", K(ret)); - } else if (OB_FAIL(gais_request_rpc_->broadcast_global_autoinc_cache(msg))) { - LOG_WARN("broadcast gais request failed", K(ret), K(msg)); - } else { - LOG_INFO("succ to broadcast global autoinc cache", K(msg)); - } - if (NULL != buffer) { - ob_free(buffer); - buffer = NULL; - } - } - } - return ret; -} - -int ObGlobalAutoIncService::receive_global_autoinc_cache( - const ObGAISBroadcastAutoIncCacheReq &request) -{ - int ret = OB_SUCCESS; - lib::ObMutexGuard lock(switching_mutex_); - ATOMIC_STORE(&is_switching_, true); - ATOMIC_STORE(&is_leader_, false); - int64_t pos = 0; - if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_UNLIKELY(false)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(request)); - } else if (OB_FAIL(wait_all_requests_to_finish())) { - LOG_WARN("fail to wait all requests to finish", K(ret)); - } else if (OB_FAIL(deserialize_autoinc_cache(request.buf_, request.buf_size_, pos))) { - LOG_WARN("fail to deserialize global auto inc cache", K(ret), K(request)); - } - ATOMIC_STORE(&is_switching_, false); - return ret; -} - -int64_t ObGlobalAutoIncService::serialize_size_autoinc_cache() -{ - int64_t len = 0; - common::hash::ObHashMap::iterator iter = autoinc_map_.begin(); - const uint64_t count = autoinc_map_.size(); - OB_UNIS_ADD_LEN(count); - for (; iter != autoinc_map_.end(); ++iter) { - const uint64_t &table_id = iter->first; - const ObAutoIncCacheNode &cache_node = iter->second; - OB_UNIS_ADD_LEN(table_id); - OB_UNIS_ADD_LEN(cache_node); - } - return len; -} - -int ObGlobalAutoIncService::serialize_autoinc_cache(SERIAL_PARAMS) -{ - int ret = OB_SUCCESS; - common::hash::ObHashMap::iterator iter = autoinc_map_.begin(); - const uint64_t count = autoinc_map_.size(); - OB_UNIS_ENCODE(count); - for (; OB_SUCC(ret) && iter != autoinc_map_.end(); ++iter) { - const uint64_t &table_id = iter->first; - ObAutoIncCacheNode &cache_node = iter->second; - OB_UNIS_ENCODE(table_id); - OB_UNIS_ENCODE(cache_node); - cache_node.is_received_ = true; // mark local cache as received - } - return ret; -} - -int ObGlobalAutoIncService::deserialize_autoinc_cache(DESERIAL_PARAMS) -{ - int ret = OB_SUCCESS; - uint64_t count = 0; - uint64_t table_id = 0; - ObAutoIncCacheNode cache_node; - ret = autoinc_map_.clear(); - OB_UNIS_DECODE(count); - for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { - cache_node.reset(); - OB_UNIS_DECODE(table_id); - OB_UNIS_DECODE(cache_node); - cache_node.is_received_ = true; - if (OB_SUCC(ret) && OB_FAIL(autoinc_map_.set_refactored(table_id, cache_node, 1))) { - LOG_WARN("fail to set map", K(ret), K(table_id), K(cache_node)); - } - } - return ret; -} - -int ObGlobalAutoIncService::wait_all_requests_to_finish() -{ - int ret = OB_SUCCESS; - const int64_t abs_timeout_us = ObTimeUtility::current_time() + BROADCAST_OP_TIMEOUT; - for (int64_t i = 0; OB_SUCC(ret) && i < MUTEX_NUM; i++) { - // wait for all working threads to finish - if (OB_FAIL(op_mutex_[i].lock(abs_timeout_us))) { - LOG_WARN("fail to lock mutex", K(ret), K(i)); - } else { - op_mutex_[i].unlock(); - } - } - return ret; -} - -int ObGlobalAutoIncService::read_and_push_inner_table(const AutoincKey &key, - const uint64_t max_value, - ObAutoIncCacheNode &received_node) -{ - int ret = OB_SUCCESS; - if (received_node.is_valid()) { - bool is_valid = false; - uint64_t new_end = received_node.end_; - if (OB_FAIL(inner_table_proxy_.read_and_push_inner_table(key, - max_value, - received_node.end_, - received_node.autoinc_version_, - is_valid, - new_end))) { - LOG_WARN("fail to read and push inner table", K(ret), K(key), K(received_node)); - } else if (!is_valid) { - received_node.reset(); - } else if (OB_FAIL(received_node.with_new_end(new_end))) { - LOG_WARN("fail to update node end", K(ret), K(new_end), K(received_node)); - } else { - received_node.is_received_ = false; - } - } - return ret; -} - } // share } // oceanbase diff --git a/src/share/ob_global_autoinc_service.h b/src/share/ob_global_autoinc_service.h index cf0539b6d..75f1b3fc4 100644 --- a/src/share/ob_global_autoinc_service.h +++ b/src/share/ob_global_autoinc_service.h @@ -21,34 +21,21 @@ #include "lib/hash/ob_link_hashmap.h" #include "lib/lock/ob_spin_lock.h" #include "common/mysqlclient/ob_mysql_proxy.h" -#include "logservice/ob_log_base_type.h" -#include "share/scn.h" #include "share/ob_autoincrement_param.h" #include "share/ob_autoincrement_service.h" #include "share/ob_gais_msg.h" -#include "share/ob_gais_rpc.h" namespace oceanbase { -namespace obcall -{ -struct ObGAISReqRpcResult; -struct ObGAISGetRpcResult; -} - namespace share { struct ObGAISNextAutoIncValReq; struct ObGAISAutoIncKeyArg; struct ObGAISPushAutoIncValReq; -struct ObGAISBroadcastAutoIncCacheReq; - struct ObAutoIncCacheNode { - OB_UNIS_VERSION(1); public: - ObAutoIncCacheNode() : start_(0), end_(0), sync_value_(0), autoinc_version_(OB_INVALID_VERSION), - is_received_(false) {} + ObAutoIncCacheNode() : start_(0), end_(0), sync_value_(0), autoinc_version_(OB_INVALID_VERSION) {} int init(const uint64_t start, const uint64_t end, const uint64_t sync_value, @@ -64,7 +51,6 @@ struct ObAutoIncCacheNode { return new_sync_value > sync_value_ && new_sync_value > end_; } - inline bool is_received() const { return is_received_; } int with_new_start(const uint64_t new_start); int with_new_end(const uint64_t new_end); void reset() { @@ -72,33 +58,24 @@ struct ObAutoIncCacheNode end_ = 0; sync_value_ = 0; autoinc_version_ = OB_INVALID_VERSION; - is_received_ = false; } - TO_STRING_KV(K_(start), K_(end), K_(sync_value), K_(autoinc_version), K_(is_received)); + TO_STRING_KV(K_(start), K_(end), K_(sync_value), K_(autoinc_version)); uint64_t start_; // next auto_increment value can be used uint64_t end_; // last available value in the cache(included) uint64_t sync_value_; int64_t autoinc_version_; - bool is_received_; }; -class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, - public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler +class ObGlobalAutoIncService { public: - ObGlobalAutoIncService() : is_inited_(false), is_leader_(false), - cache_ls_lock_(common::ObLatchIds::AUTO_INCREMENT_LEADER_LOCK), cache_ls_(NULL), - gais_request_rpc_(NULL), is_switching_(false), - switching_mutex_(common::ObLatchIds::AUTO_INCREMENT_LEADER_LOCK) - {} + ObGlobalAutoIncService() : is_inited_(false) {} virtual ~ObGlobalAutoIncService() {} const static int MUTEX_NUM = 1024; const static int INIT_HASHMAP_SIZE = 1000; - const static int64_t BROADCAST_OP_TIMEOUT = 1000 * 1000; // 1000ms, for broadcast auto increment cache - int init(const common::ObAddr &addr, common::ObMySQLProxy *mysql_proxy); + int init(common::ObMySQLProxy *mysql_proxy); static int mtl_init(ObGlobalAutoIncService *&gais); void destroy(); int clear(); @@ -110,7 +87,7 @@ class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, * and then consume the auto-increment in the cache. */ int handle_next_autoinc_request(const ObGAISNextAutoIncValReq &request, - obcall::ObGAISNextValRpcResult &result); + obcall::ObGAISNextValResult &result); /* * This method handles the request for getting current auto-increment value. If it exists @@ -118,7 +95,7 @@ class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, * Note: the cache will not be updated during this method. */ int handle_curr_autoinc_request(const ObGAISAutoIncKeyArg &request, - obcall::ObGAISCurrValRpcResult &result); + obcall::ObGAISCurrValResult &result); /* * This method handles the request for push local sync value to global. If the local sync @@ -129,8 +106,6 @@ class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, uint64_t &sync_value); int handle_clear_autoinc_cache_request(const ObGAISAutoIncKeyArg &request); - int receive_global_autoinc_cache(const ObGAISBroadcastAutoIncCacheReq &request); - /* * This method handles the request for getting next (batch) sequence value. * If the cache can satisfy the request, use the sequence in the cache to return, @@ -138,63 +113,11 @@ class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, * and then consume the sequence in the cache. */ int handle_next_sequence_request(const ObGAISNextSequenceValReq &request, - obcall::ObGAISNextSequenceValRpcResult &result); - -public: - void switch_to_follower_forcedly() - { - inner_switch_to_follower(); - } - int switch_to_follower_gracefully() - { - return inner_switch_to_follower(); - } - int resume_leader() { - ATOMIC_STORE(&is_leader_, true); - return common::OB_SUCCESS; - } - int switch_to_leader() { - ATOMIC_STORE(&is_leader_, true); - return common::OB_SUCCESS; - } - - // for replay, do nothing - int replay(const void *buffer, - const int64_t nbytes, - const palf::LSN &lsn, - const SCN &scn) override final - { - int ret = OB_SUCCESS; - UNUSED(buffer); - UNUSED(nbytes); - UNUSED(lsn); - UNUSED(scn); - return ret; - } + obcall::ObGAISNextSequenceValResult &result); - // for checkpoint, do nothing - SCN get_rec_scn() override final - { - return share::SCN::max_scn();; - } - - int flush(share::SCN &scn) override final - { - int ret = OB_SUCCESS; - UNUSED(scn); - return ret; - } - - void set_cache_ls(storage::ObLS * ls_ptr) { - ObSpinLockGuard lock(cache_ls_lock_); - cache_ls_ = ls_ptr; - } - - TO_STRING_KV(K_(is_inited), K_(is_leader), K_(self), K(autoinc_map_.size()), KP_(cache_ls), - K_(is_switching)); + TO_STRING_KV(K_(is_inited), K(autoinc_map_.size())); private: - int check_leader_(bool &is_leader); int fetch_next_node_(const ObGAISNextAutoIncValReq &request, ObAutoIncCacheNode &node); int read_value_from_inner_table_(const share::AutoincKey &key, const int64_t &inner_autoinc_version, @@ -215,33 +138,11 @@ class ObGlobalAutoIncService : public logservice::ObIReplaySubHandler, } return next_cache_boundary; } - int inner_switch_to_follower(); - int broadcast_global_autoinc_cache(); - int64_t serialize_size_autoinc_cache(); - int serialize_autoinc_cache(SERIAL_PARAMS); - int deserialize_autoinc_cache(DESERIAL_PARAMS); - int wait_all_requests_to_finish(); - /* - * The function will check whether the data of the received node and inner table are consistent. - * If they are consistent, the inner table will be pushed up by increasing seq_value. - * Otherwise, the node will be set as invalid. - */ - int read_and_push_inner_table(const AutoincKey &key, - const uint64_t max_value, - ObAutoIncCacheNode &received_node); - private: bool is_inited_; - bool is_leader_; - common::ObAddr self_; share::ObAutoIncInnerTableProxy inner_table_proxy_; common::hash::ObHashMap autoinc_map_; // table_id -> node - common::ObSpinLock cache_ls_lock_; - storage::ObLS *cache_ls_; lib::ObMutex op_mutex_[MUTEX_NUM]; - ObGAISRequestRpc* gais_request_rpc_; - bool is_switching_; - lib::ObMutex switching_mutex_; }; } // share diff --git a/src/share/ob_i_tablet_scan.h b/src/share/ob_i_tablet_scan.h index 78dc8c46a..54535b7ed 100644 --- a/src/share/ob_i_tablet_scan.h +++ b/src/share/ob_i_tablet_scan.h @@ -26,13 +26,11 @@ #include "lib/container/ob_se_array.h" #include "share/geo/ob_s2adapter.h" #include "share/ob_i_sql_expression.h" -#include "share/ob_ls_id.h" #include "share/schema/ob_schema_getter_guard.h" namespace oceanbase { namespace share { -class ObLSID; class ObExternalObjectCtx; } namespace sql @@ -423,8 +421,6 @@ ObVTableScanParam() : // data tablet id ObTabletID tablet_id_; - // log stream id - share::ObLSID ls_id_; // columns to output ObColumnIdArray column_ids_; // output column(s) //ObColDescArray column_descs_; @@ -486,7 +482,6 @@ ObVTableScanParam() : bool is_estimate_valid() const { return (tablet_id_.is_valid() - && ls_id_.is_valid() && OB_INVALID_ID != index_id_ && schema_version_ >= 0); } @@ -573,18 +568,16 @@ class ObITabletScan } virtual int get_multi_ranges_cost( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const common::ObIArray &ranges, int64_t &total_size) { - UNUSEDx(ls_id, tablet_id, timeout_us, ranges, total_size); + UNUSEDx(tablet_id, timeout_us, ranges, total_size); return OB_SUCCESS; } virtual int split_multi_ranges( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const common::ObIArray &ranges, @@ -592,7 +585,7 @@ class ObITabletScan ObIAllocator &allocator, ObArrayArray &multi_range_split_array) { - UNUSEDx(ls_id, tablet_id, timeout_us, ranges, expected_task_count, allocator, multi_range_split_array); + UNUSEDx(tablet_id, timeout_us, ranges, expected_task_count, allocator, multi_range_split_array); return OB_SUCCESS; } }; diff --git a/src/share/ob_internal_table_change_notifier.cpp b/src/share/ob_internal_table_change_notifier.cpp index 61db50f24..5d46a0d4a 100644 --- a/src/share/ob_internal_table_change_notifier.cpp +++ b/src/share/ob_internal_table_change_notifier.cpp @@ -102,11 +102,11 @@ int ObInternalTableChangeNotifier::notify( return ret; } -void ObInternalTableChangeNotifier::switch_to_follower_forcedly() +void ObInternalTableChangeNotifier::deactivate() { } -int ObInternalTableChangeNotifier::switch_to_leader() +int ObInternalTableChangeNotifier::activate() { int ret = OB_SUCCESS; diff --git a/src/share/ob_internal_table_change_notifier.h b/src/share/ob_internal_table_change_notifier.h index afcab8d3a..ae7133b48 100644 --- a/src/share/ob_internal_table_change_notifier.h +++ b/src/share/ob_internal_table_change_notifier.h @@ -27,7 +27,7 @@ namespace oceanbase namespace share { -class ObInternalTableChangeNotifier : public logservice::ObIRoleChangeSubHandler +class ObInternalTableChangeNotifier : public logservice::ObILocalLogHandler { public: using ModuleCallback = common::ObFunction; @@ -44,11 +44,9 @@ class ObInternalTableChangeNotifier : public logservice::ObIRoleChangeSubHandler // switch_to_leader. Returns immediately — the actual work is async. int notify(table::ObModuleDataArg::ObExecModule module); - // ObIRoleChangeSubHandler — called by ObRoleChangeHandler when LS switches role. - void switch_to_follower_forcedly() override; - int switch_to_leader() override; - int switch_to_follower_gracefully() override { return OB_SUCCESS; } - int resume_leader() override { return OB_SUCCESS; } + // ObILocalLogHandler — called by ObLocalLogHandlerSet when LS switches role. + void deactivate() override; + int activate() override; private: ObInternalTableChangeNotifier(); diff --git a/src/share/ob_ls_id.cpp b/src/share/ob_ls_id.cpp deleted file mode 100644 index 77c935f57..000000000 --- a/src/share/ob_ls_id.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX COMMON - -#include "ob_ls_id.h" - - -namespace oceanbase -{ -using namespace common; -namespace share -{ - -uint64_t ObLSID::hash() const -{ - OB_ASSERT(id_ >= 0); - return id_; -} - -int ObLSID::hash(uint64_t &hash_val) const -{ - hash_val = hash(); - return OB_SUCCESS; -} - -int ObLSID::serialize(char* buf, const int64_t buf_len, int64_t& pos) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(buf), K(buf_len)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, id_))) { - LOG_WARN("serialize LS ID failed", KR(ret), KP(buf), K(buf_len), K(pos)); - } - return ret; -} - -int ObLSID::deserialize(const char* buf, const int64_t data_len, int64_t& pos) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(buf), K(data_len)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &id_))) { - LOG_WARN("deserialize LS ID failed", KR(ret), KP(buf), K(data_len), K(pos)); - } - return ret; -} - -int64_t ObLSID::get_serialize_size() const -{ - int64_t size = 0; - size += serialization::encoded_length_i64(id_); - return size; -} -} // end namespace share -} // end namespace oceanbase diff --git a/src/share/ob_ls_id.h b/src/share/ob_ls_id.h deleted file mode 100644 index 8c1483a3e..000000000 --- a/src/share/ob_ls_id.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_OB_LS_ID -#define OCEANBASE_SHARE_OB_LS_ID -#include -#include "lib/ob_define.h" // is_sys_tenant/is_meta_tenant/is_user_tenant -#include "lib/container/ob_se_array.h" // ObSEArray -#include "lib/utility/ob_print_utils.h" // TO_STRING_KV - -namespace oceanbase -{ -namespace share -{ -class ObLSID -{ -public: - // constant LS ID - static const int64_t INVALID_LS_ID = -1; // INVALID LS - static const int64_t SYS_LS_ID = 1; // SYS LS for every Tenant - static const int64_t VT_LS_ID = SYS_LS_ID; // LS for virtual table - static const int64_t IDS_LS_ID = SYS_LS_ID; // LS for Trans GTS service - static const int64_t LOCK_SERVICE_LS_ID = SYS_LS_ID; // LS for Lock Service - static const int64_t GAIS_LS_ID = SYS_LS_ID; // LS for Global AutoInc Service - static const int64_t DAS_ID_LS_ID = SYS_LS_ID; // LS for DAS Id service - static const int64_t WRS_LS_ID = SYS_LS_ID; // LS for Weak Read Service - static const int64_t SCHEDULER_LS_ID = INT64_MAX; // LS for Trans scheduler - static const int64_t MAJOR_FREEZE_LS_ID = SYS_LS_ID; // LS for Major Freeze Service - - static const int64_t MIN_USER_LS_ID = 1000; - static const int64_t MIN_USER_LS_GROUP_ID = 1000; - -public: - ObLSID() : id_(INVALID_LS_ID) {} - ObLSID(const ObLSID &other) : id_(other.id_) {} - explicit ObLSID(const int64_t id) : id_(id) {} - ~ObLSID() { reset(); } - -public: - int64_t id() const { return id_; } - void reset() { id_ = INVALID_LS_ID; } - - // assignment - ObLSID &operator=(const int64_t id) { id_ = id; return *this; } - ObLSID &operator=(const ObLSID &other) { id_ = other.id_; return *this; } - - // LS attribute interface - bool is_sys_ls() const { return SYS_LS_ID == id_; } - bool is_user_ls() const { return id_ > MIN_USER_LS_ID && SCHEDULER_LS_ID != id_; } - bool is_scheduler_ls() const { return SCHEDULER_LS_ID == id_; } - bool is_valid() const { return INVALID_LS_ID != id_; } - bool is_valid_with_tenant() const - { - // 1. User tenant have SYS LS and User LS - // 2. SYS tenant and Meta tenant only have SYS LS - return true && is_sys_ls(); - } - - // compare operator - bool operator == (const ObLSID &other) const { return id_ == other.id_; } - bool operator > (const ObLSID &other) const { return id_ > other.id_; } - bool operator >= (const ObLSID &other) const { return id_ >= other.id_; } - bool operator != (const ObLSID &other) const { return id_ != other.id_; } - bool operator < (const ObLSID &other) const { return id_ < other.id_; } - bool operator <= (const ObLSID &other) const { return id_ <= other.id_; } - int compare(const ObLSID &other) const - { - if (id_ == other.id_) { - return 0; - } else if (id_ < other.id_) { - return -1; - } else { - return 1; - } - } - - uint64_t hash() const; - int hash(uint64_t &hash_val) const; - NEED_SERIALIZE_AND_DESERIALIZE; - TO_STRING_KV(K_(id)); - -private: - int64_t id_; -}; - -static const ObLSID INVALID_LS(ObLSID::INVALID_LS_ID); -static const ObLSID SYS_LS(ObLSID::SYS_LS_ID); -static const ObLSID IDS_LS(ObLSID::IDS_LS_ID); -static const ObLSID GTS_LS(ObLSID::IDS_LS_ID); -static const ObLSID GTI_LS(ObLSID::IDS_LS_ID); -static const ObLSID GAIS_LS(ObLSID::GAIS_LS_ID); -static const ObLSID SCHEDULER_LS(ObLSID::SCHEDULER_LS_ID); -static const ObLSID LOCK_SERVICE_LS(ObLSID::LOCK_SERVICE_LS_ID); -static const ObLSID DAS_ID_LS(ObLSID::DAS_ID_LS_ID); -static const ObLSID MAJOR_FREEZE_LS(ObLSID::MAJOR_FREEZE_LS_ID); -static const ObLSID WRS_LS_ID(ObLSID::WRS_LS_ID); - -static const int64_t OB_DEFAULT_LS_COUNT = 3; -typedef common::ObSEArray ObLSArray; -} // end namespace share -} // end namespace oceanbase - -#endif // OCEANBASE_SHARE_OB_LS_ID diff --git a/src/share/ob_max_id_fetcher.cpp b/src/share/ob_max_id_fetcher.cpp index 6acf24499..2a4c4c6d7 100755 --- a/src/share/ob_max_id_fetcher.cpp +++ b/src/share/ob_max_id_fetcher.cpp @@ -40,8 +40,7 @@ const char *ObMaxIdFetcher::max_id_name_info_[OB_MAX_ID_TYPE][2] = { { "ob_max_used_unit_group_id", "max used unit group id"}, { "ob_max_used_non_primary_key_table_tablet_id", "ob max used non primary key table tablet id"}, { "ob_max_used_primary_key_table_tablet_id", "ob max used primary key table tablet id"}, - { "ob_max_used_logstrema_id", "max used log stream id"}, - { "ob_max_used_logstrema_group_id", "max used log stream group id"}, + { "ob_reserved_max_used_id_10", "reserved max used id slot"}, { "ob_max_used_sys_pl_object_id", "max used sys pl object id"}, { "ob_max_used_object_id", "max used object id"}, { "ob_max_used_lock_owner_id", "max used lock owner id"}, @@ -123,8 +122,6 @@ int ObMaxIdFetcher::convert_id_type( case OB_MAX_USED_UNIT_GROUP_ID_TYPE: case OB_MAX_USED_NORMAL_ROWID_TABLE_TABLET_ID_TYPE: case OB_MAX_USED_EXTENDED_ROWID_TABLE_TABLET_ID_TYPE: - case OB_MAX_USED_LS_ID_TYPE: - case OB_MAX_USED_LS_GROUP_ID_TYPE: case OB_MAX_USED_SYS_PL_OBJECT_ID_TYPE: case OB_MAX_USED_OBJECT_ID_TYPE: case OB_MAX_USED_LOCK_OWNER_ID_TYPE: @@ -562,8 +559,6 @@ int ObMaxIdFetcher::check_id_valid(const ObMaxIdType &max_id_type, const uint64_ case OB_MAX_USED_STORAGE_OP_ID_TYPE: case OB_MAX_USED_UNIT_GROUP_ID_TYPE: case OB_MAX_USED_LOCK_OWNER_ID_TYPE: - case OB_MAX_USED_LS_ID_TYPE: - case OB_MAX_USED_LS_GROUP_ID_TYPE: case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: case OB_MAX_USED_SERVICE_NAME_ID_TYPE: case OB_MAX_USED_AI_MODEL_ID_TYPE: diff --git a/src/share/ob_max_id_fetcher.h b/src/share/ob_max_id_fetcher.h index 0623e1471..95e5bdcc6 100755 --- a/src/share/ob_max_id_fetcher.h +++ b/src/share/ob_max_id_fetcher.h @@ -41,8 +41,7 @@ enum ObMaxIdType OB_MAX_USED_UNIT_GROUP_ID_TYPE, OB_MAX_USED_NORMAL_ROWID_TABLE_TABLET_ID_TYPE, /* used for tablet_id */ OB_MAX_USED_EXTENDED_ROWID_TABLE_TABLET_ID_TYPE, /* used for tablet_id */ - OB_MAX_USED_LS_ID_TYPE, - OB_MAX_USED_LS_GROUP_ID_TYPE, + OB_MAX_USED_RESERVED_10_TYPE, OB_MAX_USED_SYS_PL_OBJECT_ID_TYPE, /* used for sys package object id */ OB_MAX_USED_OBJECT_ID_TYPE, /* used for all kinds of user schema objects */ OB_MAX_USED_LOCK_OWNER_ID_TYPE, diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 4cd25d755..0df41fe35 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -917,7 +917,6 @@ int ObCreateTableArg::assign(const ObCreateTableArg &other) OX(is_alter_view_ = other.is_alter_view_); OZ(sequence_ddl_arg_.assign(other.sequence_ddl_arg_)); OZ(dep_infos_.assign(other.dep_infos_)); - OZ(mv_ainfo_.assign(other.mv_ainfo_)); return ret; } @@ -956,8 +955,7 @@ OB_SERIALIZE_MEMBER((ObCreateTableArg, ObDDLArg), error_info_, is_alter_view_, sequence_ddl_arg_, - dep_infos_, - mv_ainfo_); + dep_infos_); bool ObCreateTableArg::is_allow_when_upgrade() const { @@ -1526,7 +1524,6 @@ OB_DEF_SERIALIZE(ObAlterTableArg) is_add_to_scheduler_, inner_sql_exec_addr_, local_session_var_, - mview_refresh_info_, alter_algorithm_, alter_auto_partition_attr_, rebuild_index_arg_list_, @@ -1542,10 +1539,6 @@ OB_DEF_SERIALIZE(ObAlterTableArg) } LST_DO_CODE(OB_UNIS_ENCODE, - is_alter_mview_attributes_, - alter_mview_arg_, - is_alter_mlog_attributes_, - alter_mlog_arg_, part_storage_cache_policy_, data_version_); @@ -1638,7 +1631,6 @@ OB_DEF_DESERIALIZE(ObAlterTableArg) is_add_to_scheduler_, inner_sql_exec_addr_, local_session_var_, - mview_refresh_info_, alter_algorithm_, alter_auto_partition_attr_, rebuild_index_arg_list_, @@ -1653,10 +1645,6 @@ OB_DEF_DESERIALIZE(ObAlterTableArg) } } LST_DO_CODE(OB_UNIS_DECODE, - is_alter_mview_attributes_, - alter_mview_arg_, - is_alter_mlog_attributes_, - alter_mlog_arg_, part_storage_cache_policy_, data_version_); return ret; @@ -1703,7 +1691,6 @@ OB_DEF_SERIALIZE_SIZE(ObAlterTableArg) is_add_to_scheduler_, inner_sql_exec_addr_, local_session_var_, - mview_refresh_info_, alter_algorithm_, alter_auto_partition_attr_, rebuild_index_arg_list_, @@ -1711,10 +1698,6 @@ OB_DEF_SERIALIZE_SIZE(ObAlterTableArg) client_session_create_ts_, lock_priority_, is_direct_load_partition_, - is_alter_mview_attributes_, - alter_mview_arg_, - is_alter_mlog_attributes_, - alter_mlog_arg_, part_storage_cache_policy_, data_version_); } @@ -2417,17 +2400,13 @@ DEF_TO_STRING(ObRebuildIndexArg) { K_(database_name), K_(index_action_type), K_(index_table_id), - K_(vidx_refresh_info), - K_(rebuild_index_type), - K_(create_mlog_arg)); + K_(vidx_refresh_info)); J_OBJ_END(); return pos; } OB_SERIALIZE_MEMBER((ObRebuildIndexArg, ObIndexArg), index_table_id_, - vidx_refresh_info_, - rebuild_index_type_, - create_mlog_arg_); + vidx_refresh_info_); DEF_TO_STRING(ObAlterIndexParallelArg) @@ -2639,16 +2618,15 @@ bool ObCheckFrozenScnArg::is_valid() const -OB_SERIALIZE_MEMBER(ObCalcColumnChecksumRequestArg::SingleItem, ls_id_, tablet_id_, calc_table_id_); +OB_SERIALIZE_MEMBER(ObCalcColumnChecksumRequestArg::SingleItem, tablet_id_, calc_table_id_); bool ObCalcColumnChecksumRequestArg::SingleItem::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid() && OB_INVALID_ID != calc_table_id_; + return tablet_id_.is_valid() && OB_INVALID_ID != calc_table_id_; } void ObCalcColumnChecksumRequestArg::SingleItem::reset() { - ls_id_.reset(); tablet_id_.reset(); calc_table_id_ = OB_INVALID_ID; } @@ -2656,7 +2634,6 @@ void ObCalcColumnChecksumRequestArg::SingleItem::reset() int ObCalcColumnChecksumRequestArg::SingleItem::assign(const SingleItem &other) { int ret = OB_SUCCESS; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; calc_table_id_ = other.calc_table_id_; return ret; @@ -2746,7 +2723,7 @@ OB_SERIALIZE_MEMBER(ObSwitchSchemaArg, schema_info_, force_refresh_, is_async_); -OB_SERIALIZE_MEMBER(ObLSTabletPair, ls_id_, tablet_id_); +OB_SERIALIZE_MEMBER(ObTabletPair, tablet_id_); OB_SERIALIZE_MEMBER(ObCheckSchemaVersionElapsedArg, schema_version_, need_wait_trans_end_, tablets_, ddl_task_id_); bool ObCheckSchemaVersionElapsedArg::is_valid() const @@ -2773,14 +2750,12 @@ int ObDDLCheckTabletMergeStatusArg::assign(const ObDDLCheckTabletMergeStatusArg if (OB_FAIL(tablet_ids_.assign(other.tablet_ids_))) { LOG_WARN("assign tablet_ids_ failed", K(ret), K(other.tablet_ids_)); } else { - - ls_id_ = other.ls_id_; snapshot_version_ = other.snapshot_version_; } return ret; } -OB_SERIALIZE_MEMBER(ObDDLCheckTabletMergeStatusArg, ls_id_, tablet_ids_, snapshot_version_); +OB_SERIALIZE_MEMBER(ObDDLCheckTabletMergeStatusArg, tablet_ids_, snapshot_version_); OB_SERIALIZE_MEMBER(ObCheckModifyTimeElapsedArg, sstable_exist_ts_, tablets_, ddl_task_id_); @@ -3348,14 +3323,6 @@ OB_SERIALIZE_MEMBER((ObCreateRoleArg, ObDDLArg), //----End of structs for managing privileges---- -OB_SERIALIZE_MEMBER(ObAdminMigrateReplicaArg, force_cmd_); - - - - - - - OB_SERIALIZE_MEMBER(ObServerZoneArg, server_, zone_); @@ -3414,11 +3381,6 @@ OB_SERIALIZE_MEMBER(ObAdminSetConfigArg, items_, is_inner_, is_backup_config_); -OB_SERIALIZE_MEMBER(ObAutoincSyncArg, - table_id_, column_id_, table_part_num_, auto_increment_, sync_value_); - -OB_SERIALIZE_MEMBER(ObAdminChangeReplicaArg, force_cmd_); - bool ObUpdateIndexStatusArg::is_allow_when_disable_ddl() const { bool bret = false; @@ -3436,13 +3398,6 @@ bool ObUpdateIndexStatusArg::is_valid() const -bool ObUpdateMViewStatusArg::is_valid() const -{ - return (OB_INVALID_ID != mview_table_id_) - && (ObMVAvailableFlag::IS_MV_UNAVAILABLE == mv_available_flag_ - || ObMVAvailableFlag::IS_MV_AVAILABLE == mv_available_flag_); -} - OB_SERIALIZE_MEMBER((ObUpdateIndexStatusArg, ObDDLArg), index_table_id_, status_, @@ -3453,12 +3408,6 @@ OB_SERIALIZE_MEMBER((ObUpdateIndexStatusArg, ObDDLArg), task_id_, error_code_); -OB_SERIALIZE_MEMBER((ObUpdateMViewStatusArg, ObDDLArg), - mview_table_id_, - mv_available_flag_, - convert_status_, - in_offline_ddl_white_list_); - OB_SERIALIZE_MEMBER(ObMergeFinishArg, server_, frozen_version_); OB_SERIALIZE_MEMBER(ObDebugSyncActionArg, reset_, clear_, action_); @@ -3470,27 +3419,20 @@ OB_SERIALIZE_MEMBER(ObDebugSyncActionArg, reset_, clear_, action_); OB_SERIALIZE_MEMBER(ObMinorFreezeArg, - tablet_id_, - ls_id_); + tablet_id_); int ObMinorFreezeArg::assign(const ObMinorFreezeArg &other) { int ret = OB_SUCCESS; tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; return ret; } OB_SERIALIZE_MEMBER(ObRootMinorFreezeArg, - server_list_, - zone_, - tablet_id_, - ls_id_); + tablet_id_); OB_SERIALIZE_MEMBER(ObTabletMajorFreezeArg, - - ls_id_, tablet_id_); @@ -3654,47 +3596,11 @@ OB_SERIALIZE_MEMBER((ObAlterTriggerArg, ObDDLArg), trigger_database_, OB_SERIALIZE_MEMBER(ObCancelTaskArg, task_id_); -OB_SERIALIZE_MEMBER(ObReportSingleReplicaArg, ls_id_); -DEF_TO_STRING(ObForceSetServerListArg) -{ - int64_t pos = 0; - J_KV(K(server_list_), K(replica_num_)); - return pos; -} - -OB_SERIALIZE_MEMBER(ObForceSetServerListArg, server_list_, replica_num_); - -OB_SERIALIZE_MEMBER(ObForceSetServerListResult::LSFailedInfo, ls_id_, failed_ret_code_, failed_reason_); - -OB_SERIALIZE_MEMBER(ObForceSetServerListResult::ResultInfo, successful_ls_, failed_ls_info_); - -int ObForceSetServerListResult::ResultInfo::add_ls_info(const share::ObLSID ls_id, const int failed_ret) -{ - int ret = OB_SUCCESS; - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(failed_ret)); - } else if (OB_SUCCESS == failed_ret) { - if (OB_FAIL(successful_ls_.push_back(ls_id))) { - LOG_WARN("push_back failed", K(ret), K(ls_id)); - } - } else { - const common::ObString failed_reason = ob_error_name(failed_ret); - LSFailedInfo failed_info(ls_id, failed_ret, failed_reason); - if (OB_FAIL(failed_ls_info_.push_back(failed_info))) { - LOG_WARN("insert_and_get failed for failed_ls_info_", K(ret), K(ls_id), K(failed_ret)); - } - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObForceSetServerListResult, ret_, result_list_); - DEF_TO_STRING(ObForceCreateSysTableArg) { @@ -3749,7 +3655,6 @@ int64_t ObEstPartArgElement::get_serialize_size(void) const OB_UNIS_ADD_LEN(range_columns_count_); OB_UNIS_ADD_LEN(batch_); OB_UNIS_ADD_LEN(tablet_id_); - OB_UNIS_ADD_LEN(ls_id_); OB_UNIS_ADD_LEN(tx_id_); return len; @@ -3765,7 +3670,6 @@ int ObEstPartArgElement::serialize(char *buf, OB_UNIS_ENCODE(range_columns_count_); OB_UNIS_ENCODE(batch_); OB_UNIS_ENCODE(tablet_id_); - OB_UNIS_ENCODE(ls_id_); OB_UNIS_ENCODE(tx_id_); return ret; @@ -3786,7 +3690,6 @@ int ObEstPartArgElement::deserialize(common::ObIAllocator &allocator, } } OB_UNIS_DECODE(tablet_id_); - OB_UNIS_DECODE(ls_id_); OB_UNIS_DECODE(tx_id_); return ret; } @@ -3838,27 +3741,6 @@ OB_SERIALIZE_MEMBER(ObEstPartResElement, logical_row_count_, OB_SERIALIZE_MEMBER(ObEstPartRes, index_param_res_); -int ObForceSetLSAsSingleReplicaArg::init(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(false - || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id)); - } else { - - ls_id_ = ls_id; - } - return ret; -} - -bool ObForceSetLSAsSingleReplicaArg::is_valid() const -{ - return true && ls_id_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObForceSetLSAsSingleReplicaArg, ls_id_); - OB_SERIALIZE_MEMBER((ObDDLNopOpreatorArg, ObDDLArg), @@ -4011,9 +3893,9 @@ OB_SERIALIZE_MEMBER(ObRefreshTimezoneArg); -OB_SERIALIZE_MEMBER(ObDDLBuildSingleReplicaResponseArg, ls_id_, tablet_id_, +OB_SERIALIZE_MEMBER(ObDDLBuildSingleReplicaResponseArg, tablet_id_, source_table_id_, dest_schema_id_, ret_code_, snapshot_version_, schema_version_, - task_id_, execution_id_, row_scanned_, row_inserted_, dest_ls_id_, dest_schema_version_, + task_id_, execution_id_, row_scanned_, row_inserted_, dest_schema_version_, server_addr_, physical_row_count_); @@ -4034,11 +3916,6 @@ OB_SERIALIZE_MEMBER(ObDDLBuildSingleReplicaResponseArg, ls_id_, tablet_id_, -OB_SERIALIZE_MEMBER(ObFreezeSplitSrcTabletArg, ls_id_, tablet_ids_); - - -OB_SERIALIZE_MEMBER(ObFreezeSplitSrcTabletRes, data_end_scn_); - int ObAutoSplitTabletArg::assign(const ObAutoSplitTabletArg &other) { int ret = OB_SUCCESS; @@ -4046,7 +3923,6 @@ int ObAutoSplitTabletArg::assign(const ObAutoSplitTabletArg &other) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(other)); } else { - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; auto_split_tablet_size_ = other.auto_split_tablet_size_; @@ -4055,7 +3931,7 @@ int ObAutoSplitTabletArg::assign(const ObAutoSplitTabletArg &other) return ret; } -OB_SERIALIZE_MEMBER(ObAutoSplitTabletArg, ls_id_, tablet_id_, +OB_SERIALIZE_MEMBER(ObAutoSplitTabletArg, tablet_id_, auto_split_tablet_size_, used_disk_space_); @@ -4079,7 +3955,7 @@ bool ObAutoSplitTabletBatchRes::is_valid() const OB_SERIALIZE_MEMBER(ObAutoSplitTabletBatchRes, rets_, suggested_next_valid_time_); -OB_SERIALIZE_MEMBER(ObFetchSplitTabletInfoArg, ls_id_, tablet_ids_); +OB_SERIALIZE_MEMBER(ObFetchSplitTabletInfoArg, tablet_ids_); OB_SERIALIZE_MEMBER(ObFetchSplitTabletInfoRes, tablet_sizes_, create_commit_versions_); @@ -4094,7 +3970,7 @@ OB_SERIALIZE_MEMBER((ObDropDirectoryArg, ObDDLArg), directory_name_); bool ObBatchRemoveTabletArg::is_valid() const { - bool is_valid = id_.is_valid(); + bool is_valid = !tablet_ids_.empty(); for (int64_t i = 0; i < tablet_ids_.count() && is_valid; i++) { is_valid = tablet_ids_.at(i).is_valid(); } @@ -4104,8 +3980,6 @@ bool ObBatchRemoveTabletArg::is_valid() const void ObBatchRemoveTabletArg::reset() { tablet_ids_.reset(); - id_.reset(); - is_old_mds_ = false; } int ObBatchRemoveTabletArg::assign(const ObBatchRemoveTabletArg &arg) @@ -4113,102 +3987,44 @@ int ObBatchRemoveTabletArg::assign(const ObBatchRemoveTabletArg &arg) int ret = OB_SUCCESS; if (OB_FAIL(tablet_ids_.assign(arg.tablet_ids_))) { LOG_WARN("failed to assign table ids", KR(ret), K(arg)); - } else { - id_ = arg.id_; - is_old_mds_ = arg.is_old_mds_; } return ret; } -int ObBatchRemoveTabletArg::init(const ObIArray &tablet_ids, - const share::ObLSID id) +int ObBatchRemoveTabletArg::init(const ObIArray &tablet_ids) { int ret = OB_SUCCESS; - bool is_valid = id.is_valid(); + bool is_valid = !tablet_ids.empty(); for (int64_t i = 0; i < tablet_ids.count() && is_valid; i++) { is_valid = tablet_ids.at(i).is_valid(); } if (OB_UNLIKELY(!is_valid)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_ids), K(id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_ids)); } else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign table schema index", KR(ret), K(tablet_ids)); - } else { - id_ = id; - } - return ret; -} - -int ObBatchRemoveTabletArg::skip_array_len(const char *buf, - int64_t data_len, - int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t count = 0; - if (pos > data_len) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos)); - } else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { - TRANS_LOG(WARN, "failed to decode array count", K(ret), KP(buf), K(data_len)); - } else if (count <= 0) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos), K(count)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - ObTabletID tablet_id; - OB_UNIS_DECODE(tablet_id); - } } return ret; } -int ObBatchRemoveTabletArg::is_old_mds(const char *buf, - int64_t data_len, - bool &is_old_mds) -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - is_old_mds = false; - int64_t version = 0; - int64_t len = 0; - share::ObLSID id; - - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, version, len); - if (OB_FAIL(ret)) { - } - // tablets array - else if (OB_FAIL(skip_array_len(buf, data_len, pos))) { - TRANS_LOG(WARN, "failed to skip_unis_array_len", K(ret), KP(buf), K(data_len), K(pos), K(version), K(len), K(id)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, id, is_old_mds); - } - } - - return ret; -} - DEF_TO_STRING(ObBatchRemoveTabletArg) { int64_t pos = 0; - J_KV(K_(id), K_(tablet_ids), K_(is_old_mds)); + J_KV(K_(tablet_ids)); return pos; } OB_DEF_SERIALIZE(ObBatchRemoveTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, tablet_ids_, id_, is_old_mds_); + LST_DO_CODE(OB_UNIS_ENCODE, tablet_ids_); return ret; } OB_DEF_SERIALIZE_SIZE(ObBatchRemoveTabletArg) { int len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, tablet_ids_, id_, is_old_mds_); + LST_DO_CODE(OB_UNIS_ADD_LEN, tablet_ids_); return len; } @@ -4224,8 +4040,7 @@ OB_SERIALIZE_MEMBER((ObPartitionSplitArg, ObDDLArg), lob_schema_versions_, src_lob_tablet_ids_, dest_lob_tablet_ids_, - task_type_, - src_ls_id_); + task_type_); OB_SERIALIZE_MEMBER(ObCleanSplittedTabletArg, @@ -4245,14 +4060,11 @@ int ObCheckMemtableCntArg::assign(const ObCheckMemtableCntArg &other) { int ret = OB_SUCCESS; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; return ret; } OB_SERIALIZE_MEMBER(ObCheckMemtableCntArg, - - ls_id_, tablet_id_); @@ -4263,14 +4075,11 @@ int ObCheckMediumCompactionInfoListArg::assign(const ObCheckMediumCompactionInfo { int ret = OB_SUCCESS; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; return ret; } OB_SERIALIZE_MEMBER(ObCheckMediumCompactionInfoListArg, - - ls_id_, tablet_id_); OB_SERIALIZE_MEMBER(ObCheckMediumCompactionInfoListResult, @@ -4280,14 +4089,7 @@ OB_SERIALIZE_MEMBER(ObCheckMediumCompactionInfoListResult, OB_DEF_DESERIALIZE(ObBatchRemoveTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, tablet_ids_, id_); - if (OB_SUCC(ret)) { - if (pos == data_len) { - is_old_mds_ = true; - } else { - LST_DO_CODE(OB_UNIS_DECODE, is_old_mds_); - } - } + LST_DO_CODE(OB_UNIS_DECODE, tablet_ids_); return ret; } @@ -4476,11 +4278,11 @@ bool ObFetchTabletSeqArg::is_valid() const DEF_TO_STRING(ObFetchTabletSeqArg) { int64_t pos = 0; - J_KV(K_(tablet_id), K_(ls_id)); + J_KV(K_(tablet_id)); return pos; } -OB_SERIALIZE_MEMBER(ObFetchTabletSeqArg, cache_size_, tablet_id_, ls_id_); +OB_SERIALIZE_MEMBER(ObFetchTabletSeqArg, cache_size_, tablet_id_); bool ObFetchTabletSeqRes::is_valid() const { @@ -4567,11 +4369,10 @@ int ObEstBlockArgElement::assign(const ObEstBlockArgElement &other) int ret = OB_SUCCESS; tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; return ret; } -OB_SERIALIZE_MEMBER(ObEstBlockArgElement, tablet_id_, ls_id_); +OB_SERIALIZE_MEMBER(ObEstBlockArgElement, tablet_id_); @@ -4593,13 +4394,12 @@ OB_SERIALIZE_MEMBER(ObEstBlockResElement, macro_block_count_, micro_block_count_ OB_SERIALIZE_MEMBER(ObEstBlockRes, tablet_params_res_); -OB_SERIALIZE_MEMBER(ObBatchGetTabletAutoincSeqArg, ls_id_, src_tablet_ids_, dest_tablet_ids_); +OB_SERIALIZE_MEMBER(ObBatchGetTabletAutoincSeqArg, src_tablet_ids_, dest_tablet_ids_); int ObBatchGetTabletAutoincSeqArg::assign(const ObBatchGetTabletAutoincSeqArg &other) { int ret = OB_SUCCESS; - ls_id_ = other.ls_id_; if (OB_FAIL(src_tablet_ids_.assign(other.src_tablet_ids_))) { LOG_WARN("failed to assign src tablet ids", K(ret), K(other)); } else if (OB_FAIL(dest_tablet_ids_.assign(other.dest_tablet_ids_))) { @@ -4608,11 +4408,10 @@ int ObBatchGetTabletAutoincSeqArg::assign(const ObBatchGetTabletAutoincSeqArg &o return ret; } -int ObBatchGetTabletAutoincSeqArg::init(const share::ObLSID &ls_id, const ObIArray ¶ms) +int ObBatchGetTabletAutoincSeqArg::init(const ObIArray ¶ms) { int ret = OB_SUCCESS; - ls_id_ = ls_id; src_tablet_ids_.reset(); dest_tablet_ids_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); i++) { @@ -4633,13 +4432,12 @@ int ObBatchGetTabletAutoincSeqArg::init(const share::ObLSID &ls_id, const ObIArr OB_SERIALIZE_MEMBER(ObBatchGetTabletAutoincSeqRes, autoinc_params_); -OB_SERIALIZE_MEMBER(ObBatchSetTabletAutoincSeqArg, ls_id_, autoinc_params_, is_tablet_creating_); +OB_SERIALIZE_MEMBER(ObBatchSetTabletAutoincSeqArg, autoinc_params_, is_tablet_creating_); int ObBatchSetTabletAutoincSeqArg::assign(const ObBatchSetTabletAutoincSeqArg &other) { int ret = OB_SUCCESS; - ls_id_ = other.ls_id_; is_tablet_creating_ = other.is_tablet_creating_; if (OB_FAIL(autoinc_params_.assign(other.autoinc_params_))) { LOG_WARN("failed to assign autoinc params", K(ret), K(other)); @@ -4647,11 +4445,10 @@ int ObBatchSetTabletAutoincSeqArg::assign(const ObBatchSetTabletAutoincSeqArg &o return ret; } -int ObBatchSetTabletAutoincSeqArg::init(const share::ObLSID &ls_id, const ObIArray ¶ms) +int ObBatchSetTabletAutoincSeqArg::init(const ObIArray ¶ms) { int ret = OB_SUCCESS; - ls_id_ = ls_id; autoinc_params_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); i++) { const ObMigrateTabletAutoincSeqParam ¶m = params.at(i); @@ -4669,7 +4466,6 @@ int ObBatchSetTabletAutoincSeqArg::init(const share::ObLSID &ls_id, const ObIArr void ObBatchSetTabletAutoincSeqArg::reset() { - ls_id_.reset(); autoinc_params_.reset(); is_tablet_creating_ = false; return; @@ -4678,14 +4474,13 @@ void ObBatchSetTabletAutoincSeqArg::reset() OB_SERIALIZE_MEMBER(ObBatchSetTabletAutoincSeqRes, autoinc_params_); -OB_SERIALIZE_MEMBER(ObBatchGetTabletBindingArg, ls_id_, tablet_ids_, check_committed_); +OB_SERIALIZE_MEMBER(ObBatchGetTabletBindingArg, tablet_ids_, check_committed_); -int ObBatchGetTabletBindingArg::init(const share::ObLSID &ls_id, const ObIArray &tablet_ids, const bool check_committed) +int ObBatchGetTabletBindingArg::init(const ObIArray &tablet_ids, const bool check_committed) { int ret = OB_SUCCESS; - ls_id_ = ls_id; check_committed_ = check_committed; if (OB_FAIL(tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign", K(ret)); @@ -4699,20 +4494,18 @@ int ObBatchGetTabletBindingArg::init(const share::ObLSID &ls_id, const ObIArray< -OB_SERIALIZE_MEMBER(ObBatchGetTabletSplitArg, ls_id_, tablet_ids_, check_committed_); +OB_SERIALIZE_MEMBER(ObBatchGetTabletSplitArg, tablet_ids_, check_committed_); -int ObBatchGetTabletSplitArg::init(const share::ObLSID &ls_id, const ObIArray &tablet_ids, const bool check_committed) +int ObBatchGetTabletSplitArg::init(const ObIArray &tablet_ids, const bool check_committed) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || tablet_ids.empty())) { + if (OB_UNLIKELY(tablet_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_ids), K(check_committed)); + LOG_WARN("invalid argument", K(ret), K(tablet_ids), K(check_committed)); } else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign", K(ret)); } else { - - ls_id_ = ls_id; check_committed_ = check_committed; } if (OB_SUCC(ret) && OB_UNLIKELY(!is_valid())) { @@ -5009,9 +4802,6 @@ OB_SERIALIZE_MEMBER(ObCancelGatherStatsArg, task_id_); -OB_SERIALIZE_MEMBER(ObCheckNestedMViewMdsArg, mview_id_, refresh_id_, target_data_sync_scn_); -OB_SERIALIZE_MEMBER(ObCheckNestedMViewMdsRes, target_data_sync_scn_, ret_); - OB_SERIALIZE_MEMBER((ObCreateTableGroupRes, ObParallelDDLRes), tablegroup_id_); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 3a69cd6eb..8968f6091 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -53,7 +53,6 @@ #include "storage/tablet/ob_tablet_create_delete_mds_user_data.h" // ObTabletMdsUserDataType #include "storage/tablelock/ob_table_lock_priority.h" // conf L2 #include "storage/tx/ob_trans_id.h" // conf L2 -#include "share/ob_ls_id.h" #include "share/ob_tablet_autoincrement_param.h" #include "share/unit/ob_unit_info.h" //ObUnit* #include "logservice/palf/palf_base_info.h"//PalfBaseInfo @@ -68,7 +67,6 @@ #include "share/schema/ob_catalog_schema_struct.h" #include "share/schema/ob_ccl_schema_struct.h" #include "ob_ddl_args.h" -#include "ob_mview_args.h" #include "share/inner_table/ob_load_inner_table_schema.h" #include "share/ai_service/ob_ai_model_info.h" #include "share/schema/ob_location_schema_struct.h" @@ -100,7 +98,6 @@ using share::schema::ObTableSchema; using share::schema::ObPartitionLevel; using share::schema::ObSimpleTableSchemaV2; using share::schema::PARTITION_LEVEL_MAX; -using share::schema::ObMVAvailableFlag; typedef common::ObSArray ObServerList; static const int64_t MAX_COUNT = 128; static const int64_t OB_DEFAULT_ARRAY_SIZE = 8; @@ -256,7 +253,7 @@ struct ObCreateTenantArg : public ObDDLArg common::ObSArray sys_var_list_; common::ObNameCaseMode name_case_mode_; bool is_restore_; - //for restore tenant sys ls + // Palf base info for restoring the tenant log stream. palf::PalfBaseInfo palf_base_info_; //for restore tenant, from backuped meta file uint64_t compatible_version_; @@ -675,10 +672,7 @@ struct ObIndexArg : public ObDDLArg REBUILD_INDEX, ALTER_PRIMARY_KEY, ADD_PRIMARY_KEY, - DROP_PRIMARY_KEY, - ADD_MLOG, - DROP_MLOG, - RENAME_MLOG + DROP_PRIMARY_KEY }; static const char *to_type_str(const IndexActionType type) @@ -704,12 +698,6 @@ struct ObIndexArg : public ObDDLArg str = "add primary key"; } else if (DROP_PRIMARY_KEY == type) { str = "drop primary key"; - } else if (ADD_MLOG == type) { - str = "add materialized view log"; - } else if (DROP_MLOG == type) { - str = "drop materialized view log"; - } else if (RENAME_MLOG == type) { - str = "rename materialized view log"; } return str; } @@ -941,16 +929,11 @@ struct ObRebuildIndexArg: public ObIndexArg OB_UNIS_VERSION(1); //if add new member,should add to_string and serialize function public: - enum RebuildIndexType { - REBUILD_INDEX_TYPE_VEC = 0, - REBUILD_INDEX_TYPE_MLOG = 1 - }; ObRebuildIndexArg() : ObIndexArg(), vidx_refresh_info_() { index_action_type_ = REBUILD_INDEX; index_table_id_ = common::OB_INVALID_ID; - rebuild_index_type_ = REBUILD_INDEX_TYPE_VEC; } virtual ~ObRebuildIndexArg() {} @@ -958,12 +941,9 @@ struct ObRebuildIndexArg: public ObIndexArg int ret = common::OB_SUCCESS; if (OB_FAIL(ObIndexArg::assign(other))) { SHARE_LOG(WARN, "fail to assign base", K(ret)); - } else if (OB_FAIL(create_mlog_arg_.assign(other.create_mlog_arg_))) { - SHARE_LOG(WARN, "fail to assign create mlog arg", K(ret)); } else { index_table_id_ = other.index_table_id_; vidx_refresh_info_ = other.vidx_refresh_info_; - rebuild_index_type_ = other.rebuild_index_type_; } return ret; } @@ -973,19 +953,10 @@ struct ObRebuildIndexArg: public ObIndexArg ObIndexArg::reset(); index_action_type_ = REBUILD_INDEX; vidx_refresh_info_.reset(); - // in 4.3.5.3, we add rebuild_index_type_ field to support mlog, - // before that, the rebuild index task was only used for vec index. - // to ensure compatibility, the default value is set to vec index. - rebuild_index_type_ = REBUILD_INDEX_TYPE_VEC; - create_mlog_arg_.reset(); - } - bool is_rebuild_mlog() const { return REBUILD_INDEX_TYPE_MLOG == rebuild_index_type_; } - bool is_rebuild_vec_index() const { return REBUILD_INDEX_TYPE_VEC == rebuild_index_type_; } + } bool is_valid() const { return ObIndexArg::is_valid(); } uint64_t index_table_id_; share::schema::ObVectorIndexRefreshInfo vidx_refresh_info_; - RebuildIndexType rebuild_index_type_; - ObCreateMLogArg create_mlog_arg_; DECLARE_VIRTUAL_TO_STRING; }; @@ -1729,7 +1700,6 @@ struct ObAlterTableArg : public ObDDLArg is_add_to_scheduler_(false), inner_sql_exec_addr_(), local_session_var_(&allocator_), - mview_refresh_info_(), alter_algorithm_(INPLACE), alter_auto_partition_attr_(false), rebuild_index_arg_list_(), @@ -1737,10 +1707,6 @@ struct ObAlterTableArg : public ObDDLArg client_session_create_ts_(0), lock_priority_(transaction::tablelock::ObTableLockPriority::NORMAL), is_direct_load_partition_(false), - is_alter_mview_attributes_(false), - alter_mview_arg_(), - is_alter_mlog_attributes_(false), - alter_mlog_arg_(), part_storage_cache_policy_(), data_version_(0) { @@ -1853,7 +1819,6 @@ struct ObAlterTableArg : public ObDDLArg K_(hidden_table_id), K_(inner_sql_exec_addr), K_(local_session_var), - K_(mview_refresh_info), K_(alter_algorithm), K_(alter_auto_partition_attr), K_(rebuild_index_arg_list), @@ -1861,10 +1826,6 @@ struct ObAlterTableArg : public ObDDLArg K_(client_session_create_ts), K_(lock_priority), K_(is_direct_load_partition), - K_(is_alter_mview_attributes), - K_(alter_mview_arg), - K_(is_alter_mlog_attributes), - K_(alter_mlog_arg), K_(part_storage_cache_policy), K_(data_version)); private: @@ -1899,7 +1860,6 @@ struct ObAlterTableArg : public ObDDLArg bool is_add_to_scheduler_; common::ObAddr inner_sql_exec_addr_; share::ObLocalSessionVar local_session_var_; - ObMViewRefreshInfo mview_refresh_info_; AlterAlgorithm alter_algorithm_; bool alter_auto_partition_attr_; common::ObSArray rebuild_index_arg_list_; // pre split @@ -1907,10 +1867,6 @@ struct ObAlterTableArg : public ObDDLArg int64_t client_session_create_ts_; transaction::tablelock::ObTableLockPriority lock_priority_; bool is_direct_load_partition_; - bool is_alter_mview_attributes_; - ObAlterMViewArg alter_mview_arg_; - bool is_alter_mlog_attributes_; - ObAlterMLogArg alter_mlog_arg_; common::ObString part_storage_cache_policy_; uint64_t data_version_; int serialize_index_args(char *buf, const int64_t data_len, int64_t &pos) const; @@ -2492,7 +2448,7 @@ typedef ObCreateIndexArg ObAlterPrimaryArg; struct ObPartitionSplitArg : public ObDDLArg { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObPartitionSplitArg() : ObDDLArg(), @@ -2506,8 +2462,7 @@ struct ObPartitionSplitArg : public ObDDLArg lob_schema_versions_(), src_lob_tablet_ids_(), dest_lob_tablet_ids_(), - task_type_(share::ObDDLType::DDL_INVALID), - src_ls_id_() + task_type_(share::ObDDLType::DDL_INVALID) {} virtual ~ObPartitionSplitArg() {} bool is_valid() const @@ -2536,14 +2491,13 @@ struct ObPartitionSplitArg : public ObDDLArg dest_lob_tablet_ids_.reset(); task_type_ = share::ObDDLType::DDL_INVALID; ObDDLArg::reset(); - src_ls_id_.reset(); } INHERIT_TO_STRING_KV("ObDDLArg", ObDDLArg, K(src_tablet_id_), K(dest_tablet_ids_), K(local_index_table_ids_), K(local_index_schema_versions_), K(src_local_index_tablet_ids_), K(dest_local_index_tablet_ids_), K(lob_table_ids_), K(lob_schema_versions_), K(src_lob_tablet_ids_), K(dest_lob_tablet_ids_), - K(task_type_), K(src_ls_id_)); + K(task_type_)); public: ObTabletID src_tablet_id_; ObSArray dest_tablet_ids_; @@ -2556,7 +2510,6 @@ struct ObPartitionSplitArg : public ObDDLArg ObSArray src_lob_tablet_ids_; ObSArray> dest_lob_tablet_ids_; share::ObDDLType task_type_; - share::ObLSID src_ls_id_; // valid when data_version >= 4.3.5 }; struct ObCleanSplittedTabletArg final @@ -2642,24 +2595,20 @@ struct ObCleanSplittedTabletArg final struct ObCheckMemtableCntArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObCheckMemtableCntArg() - : ls_id_(), - tablet_id_(common::ObTabletID::INVALID_TABLET_ID) + : tablet_id_(common::ObTabletID::INVALID_TABLET_ID) {} ~ObCheckMemtableCntArg() {} bool is_valid() const { - return OB_INVALID_ID != 1UL && - ls_id_.is_valid() && - tablet_id_.is_valid(); + return tablet_id_.is_valid(); } int assign(const ObCheckMemtableCntArg &other); - TO_STRING_KV(K(1UL), K(ls_id_), K(tablet_id_)); + TO_STRING_KV(K(tablet_id_)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; }; @@ -2677,24 +2626,20 @@ struct ObCheckMemtableCntResult final struct ObCheckMediumCompactionInfoListArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObCheckMediumCompactionInfoListArg() - : ls_id_(), - tablet_id_(common::ObTabletID::INVALID_TABLET_ID) + : tablet_id_(common::ObTabletID::INVALID_TABLET_ID) {} ~ObCheckMediumCompactionInfoListArg() {} bool is_valid() const { - return OB_INVALID_ID != 1UL && - ls_id_.is_valid() && - tablet_id_.is_valid(); + return tablet_id_.is_valid(); } int assign(const ObCheckMediumCompactionInfoListArg &other); - TO_STRING_KV(K(1UL), K(ls_id_), K(tablet_id_)); + TO_STRING_KV(K(tablet_id_)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; }; @@ -2887,8 +2832,7 @@ struct ObCreateTableArg : ObDDLArg error_info_(), is_alter_view_(false), sequence_ddl_arg_(), - dep_infos_(), - mv_ainfo_() + dep_infos_() {} bool is_valid() const; virtual bool is_allow_when_upgrade() const; @@ -2909,7 +2853,6 @@ struct ObCreateTableArg : ObDDLArg bool is_alter_view_; ObSequenceDDLArg sequence_ddl_arg_; common::ObSArray dep_infos_; - common::ObSArray mv_ainfo_; }; @@ -2988,28 +2931,18 @@ struct ObBatchCreateTabletArg; struct ObBatchRemoveTabletArg { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchRemoveTabletArg() { reset(); } ~ObBatchRemoveTabletArg() {} bool is_valid() const; - bool is_inited() const; void reset(); int assign (const ObBatchRemoveTabletArg &arg); - int init(const ObIArray &tablet_ids, - const share::ObLSID id); - static int is_old_mds(const char *buf, - int64_t data_len, - bool &is_old_mds); - static int skip_array_len(const char *buf, - int64_t data_len, - int64_t &pos); + int init(const ObIArray &tablet_ids); DECLARE_TO_STRING; public: - share::ObLSID id_; common::ObSArray tablet_ids_; - bool is_old_mds_; }; @@ -3018,9 +2951,9 @@ struct ObBatchRemoveTabletArg struct ObFetchTabletSeqArg { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: - ObFetchTabletSeqArg() : cache_size_(0), tablet_id_(), ls_id_() {} + ObFetchTabletSeqArg() : cache_size_(0), tablet_id_() {} ~ObFetchTabletSeqArg() {} bool is_valid() const; public: @@ -3029,7 +2962,6 @@ struct ObFetchTabletSeqArg uint64_t cache_size_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; private: DISALLOW_COPY_AND_ASSIGN(ObFetchTabletSeqArg); @@ -3084,15 +3016,14 @@ struct ObCalcColumnChecksumRequestArg final K_(snapshot_version), K_(source_table_id), K_(task_id), K_(calc_items), K_(user_parallelism)); struct SingleItem final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: SingleItem() { reset(); } ~SingleItem() = default; bool is_valid() const; void reset(); int assign(const SingleItem &other); - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(calc_table_id)); - share::ObLSID ls_id_; + TO_STRING_KV(K_(tablet_id), K_(calc_table_id)); common::ObTabletID tablet_id_; int64_t calc_table_id_; }; @@ -3157,14 +3088,6 @@ struct ObCalcColumnChecksumResponseArg -enum ObReplicaMovingType : int8_t -{ - REPLICA_MOVING_TYPE_INVALID = 0, - REPLICA_MOVING_TYPE_ADD_REPLICA, - REPLICA_MOVING_TYPE_DROP_REPLICA, - REPLICA_MOVING_TYPE_MAX -}; - //----Structs for managing privileges---- @@ -3193,17 +3116,16 @@ struct ObSwitchSchemaArg bool is_async_; }; -struct ObLSTabletPair final +struct ObTabletPair final { OB_UNIS_VERSION(1); public: - bool is_valid() const { return ls_id_.is_valid() && tablet_id_.is_valid(); } - uint64_t hash() const { return ls_id_.hash() + tablet_id_.hash(); } + bool is_valid() const { return tablet_id_.is_valid(); } + uint64_t hash() const { return tablet_id_.hash(); } int hash(uint64_t &hash_val) const { hash_val = hash(); return OB_SUCCESS; } - bool operator == (const ObLSTabletPair &other) const { return ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_; } - bool operator < (const ObLSTabletPair &other) const { return ls_id_ != other.ls_id_ ? ls_id_ < other.ls_id_ : tablet_id_ < other.tablet_id_; } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); - share::ObLSID ls_id_; + bool operator == (const ObTabletPair &other) const { return tablet_id_ == other.tablet_id_; } + bool operator < (const ObTabletPair &other) const { return tablet_id_ < other.tablet_id_; } + TO_STRING_KV(K_(tablet_id)); common::ObTabletID tablet_id_; }; @@ -3220,33 +3142,29 @@ struct ObCheckSchemaVersionElapsedArg final int64_t schema_version_; bool need_wait_trans_end_; - ObSEArray tablets_; + ObSEArray tablets_; int64_t ddl_task_id_; }; struct ObDDLCheckTabletMergeStatusArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObDDLCheckTabletMergeStatusArg() - : ls_id_(), tablet_ids_(), snapshot_version_() + : tablet_ids_(), snapshot_version_() {} ~ObDDLCheckTabletMergeStatusArg() = default; bool is_valid() const { - return true && - common::OB_INVALID_TIMESTAMP != snapshot_version_ && - ls_id_.is_valid() && + return common::OB_INVALID_TIMESTAMP != snapshot_version_ && tablet_ids_.count() > 0; } int assign(const ObDDLCheckTabletMergeStatusArg &other); void reset() { - ls_id_.reset(); tablet_ids_.reset(); } public: - TO_STRING_KV(K_(ls_id), K_(tablet_ids), K_(snapshot_version)); + TO_STRING_KV(K_(tablet_ids), K_(snapshot_version)); - share::ObLSID ls_id_; common::ObSArray tablet_ids_; int64_t snapshot_version_; }; @@ -3260,7 +3178,7 @@ struct ObCheckModifyTimeElapsedArg final TO_STRING_KV(K_(sstable_exist_ts), K_(tablets), K_(ddl_task_id)); int64_t sstable_exist_ts_; - ObSEArray tablets_; + ObSEArray tablets_; int64_t ddl_task_id_; }; @@ -3837,20 +3755,6 @@ struct ObCreateRoleArg : public ObDDLArg // system admin (alter system ...) rpc argument define -// TODO: @wanhong.wwh to implement change replica stmt -struct ObAdminChangeReplicaArg -{ - OB_UNIS_VERSION(1); - -public: - ObAdminChangeReplicaArg() : force_cmd_(false) {} - ~ObAdminChangeReplicaArg() {} - - bool is_valid() const { return true; } - TO_STRING_KV(K_(force_cmd)); - bool force_cmd_; -}; - struct ObAdminAddDiskArg { OB_UNIS_VERSION(1); @@ -3887,22 +3791,6 @@ struct ObAdminDropDiskArg common::ObZone zone_; }; -struct ObAdminMigrateReplicaArg -{ - OB_UNIS_VERSION(1); - -public: - ObAdminMigrateReplicaArg() - : force_cmd_(false) - {} - ~ObAdminMigrateReplicaArg() {} - - bool is_valid() const { return true; } - TO_STRING_KV(K_(force_cmd)); - bool force_cmd_; -}; - - struct ObServerZoneArg { OB_UNIS_VERSION(1); @@ -4082,33 +3970,6 @@ struct ObAdminFlushCacheArg -struct ObAutoincSyncArg -{ - OB_UNIS_VERSION(1); - -public: - ObAutoincSyncArg() - : table_id_(common::OB_INVALID_ID), - column_id_(common::OB_INVALID_ID), - sync_value_(0), - table_part_num_(0), - auto_increment_(0) - {} - TO_STRING_KV(K_(table_id), K_(column_id), K_(sync_value), K_(table_part_num), K_(auto_increment)); - - - uint64_t table_id_; - uint64_t column_id_; - uint64_t sync_value_; - //TODO may need add first_part_num. As table_part_num not see usefull. - //Not add first_part_num now. - uint64_t table_part_num_; - uint64_t auto_increment_; // only for sync table option auto_increment -}; - - - - struct ObUpdateIndexStatusArg : public ObDDLArg { OB_UNIS_VERSION(1); @@ -4140,30 +4001,6 @@ struct ObUpdateIndexStatusArg : public ObDDLArg int error_code_; }; -struct ObUpdateMViewStatusArg : public ObDDLArg -{ - OB_UNIS_VERSION(1); -public: - ObUpdateMViewStatusArg(): - ObDDLArg(), - mview_table_id_(common::OB_INVALID_ID), - mv_available_flag_(ObMVAvailableFlag::IS_MV_UNAVAILABLE), - convert_status_(true), - in_offline_ddl_white_list_(false) - {} - bool is_valid() const; - virtual bool is_allow_when_disable_ddl() const { return false; }; - virtual bool is_allow_when_upgrade() const { return true; } - virtual bool is_in_offline_ddl_white_list() const { return in_offline_ddl_white_list_; } - TO_STRING_KV(K_(mview_table_id), K_(mv_available_flag), K_(convert_status), K_(in_offline_ddl_white_list)); - -public: - uint64_t mview_table_id_; - enum ObMVAvailableFlag mv_available_flag_; - bool convert_status_; - bool in_offline_ddl_white_list_; -}; - struct ObMergeFinishArg { OB_UNIS_VERSION(1); @@ -4206,7 +4043,6 @@ struct ObMinorFreezeArg void reset() { tablet_id_.reset(); - ls_id_.reset(); } bool is_valid() const @@ -4214,24 +4050,20 @@ struct ObMinorFreezeArg return true; } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); common::ObTabletID tablet_id_; - share::ObLSID ls_id_; }; struct ObRootMinorFreezeArg { - OB_UNIS_VERSION(2); + OB_UNIS_VERSION(1); public: ObRootMinorFreezeArg() {} void reset() { - server_list_.reset(); - zone_.reset(); tablet_id_.reset(); - ls_id_.reset(); } bool is_valid() const @@ -4239,30 +4071,25 @@ struct ObRootMinorFreezeArg return true; } - TO_STRING_KV(K_(server_list), K_(zone), K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(tablet_id)); - common::ObSArray server_list_; - common::ObZone zone_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; }; struct ObTabletMajorFreezeArg { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObTabletMajorFreezeArg() - : ls_id_(), - tablet_id_() + : tablet_id_() {} ~ObTabletMajorFreezeArg() = default; bool is_valid() const { - return true && ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; @@ -4703,76 +4530,6 @@ struct ObCancelTaskArg : public ObServerZoneArg share::ObTaskId task_id_; }; -struct ObReportSingleReplicaArg -{ - OB_UNIS_VERSION(1); -public: - ObReportSingleReplicaArg() : ls_id_() {} - bool is_valid() const { return true && ls_id_.is_valid(); } - - TO_STRING_KV(K_(ls_id)); - - - share::ObLSID ls_id_; -}; - - - - - -struct ObForceSetServerListArg -{ - OB_UNIS_VERSION(1); -public: - ObForceSetServerListArg() : server_list_(), replica_num_(0) {} - ~ObForceSetServerListArg() {} - - DECLARE_TO_STRING; - obcall::ObServerList server_list_; - int64_t replica_num_; -}; - -struct ObForceSetServerListResult -{ - OB_UNIS_VERSION(1); -public: - ObForceSetServerListResult() : ret_(OB_SUCCESS), result_list_() {} - ~ObForceSetServerListResult() {} - int init(); - DISALLOW_COPY_AND_ASSIGN(ObForceSetServerListResult); - - struct LSFailedInfo - { - OB_UNIS_VERSION(1); - public: - LSFailedInfo() : ls_id_(), failed_ret_code_(OB_SUCCESS), failed_reason_() {} - LSFailedInfo(const share::ObLSID &ls_id, - const int failed_ret_code, - const common::ObString failed_reason) - : ls_id_(ls_id), failed_ret_code_(failed_ret_code), failed_reason_(failed_reason) {} - ~LSFailedInfo() {} - TO_STRING_KV(K_(ls_id), K_(failed_ret_code), K_(failed_reason)); - share::ObLSID ls_id_; - int failed_ret_code_; - common::ObString failed_reason_; - }; - struct ResultInfo - { - OB_UNIS_VERSION(1); - public: - ResultInfo() : successful_ls_(), failed_ls_info_() {} - ~ResultInfo() {} - int add_ls_info(const share::ObLSID ls_id, const int failed_ret = OB_SUCCESS); - TO_STRING_KV(K_(successful_ls), K_(failed_ls_info)); - - common::ObSArray successful_ls_; - common::ObSArray failed_ls_info_; - }; - TO_STRING_KV(K_(ret), K_(result_list)); - int ret_; - common::ObSArray result_list_; -}; - struct ObForceCreateSysTableArg { OB_UNIS_VERSION(1); @@ -4803,28 +4560,6 @@ struct ObSplitPartitionArg -struct ObForceSetLSAsSingleReplicaArg -{ - OB_UNIS_VERSION(1); -public: - ObForceSetLSAsSingleReplicaArg(): ls_id_() {} - ~ObForceSetLSAsSingleReplicaArg() {} - bool is_valid() const; - int init(const share::ObLSID &ls_id); - TO_STRING_KV(K_(ls_id)); - - - share::ObLSID get_ls_id() const - { - return ls_id_; - } -private: - DISALLOW_COPY_AND_ASSIGN(ObForceSetLSAsSingleReplicaArg); -private: - - share::ObLSID ls_id_; -}; - @@ -4854,7 +4589,7 @@ struct ObDDLNopOpreatorArg : public ObDDLArg struct ObEstPartArgElement { ObEstPartArgElement() : batch_(), scan_flag_(), - index_id_(common::OB_INVALID_ID), range_columns_count_(0), tablet_id_(), ls_id_(), tx_id_() + index_id_(common::OB_INVALID_ID), range_columns_count_(0), tablet_id_(), tx_id_() {} // Essentially, we can use ObIArray here // For compatibility reason, we still use ObSimpleBatch @@ -4863,7 +4598,6 @@ struct ObEstPartArgElement int64_t index_id_; int64_t range_columns_count_; ObTabletID tablet_id_; - share::ObLSID ls_id_; transaction::ObTransID tx_id_; @@ -4873,7 +4607,6 @@ struct ObEstPartArgElement K(batch_), K(range_columns_count_), K(tablet_id_), - K(ls_id_), K(tx_id_)); int64_t get_serialize_size(void) const; int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; @@ -5386,11 +5119,10 @@ struct ObRefreshTimezoneArg struct ObDDLBuildSingleReplicaResponseArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObDDLBuildSingleReplicaResponseArg() - : ls_id_(), - tablet_id_(), + : tablet_id_(), source_table_id_(OB_INVALID_ID), dest_schema_id_(OB_INVALID_ID), ret_code_(OB_SUCCESS), @@ -5400,16 +5132,13 @@ struct ObDDLBuildSingleReplicaResponseArg final execution_id_(-1), row_scanned_(0), row_inserted_(0), - dest_ls_id_(), dest_schema_version_(0), server_addr_(), physical_row_count_(0) {} ~ObDDLBuildSingleReplicaResponseArg() = default; bool is_valid() const { - return ls_id_.is_valid() && - dest_ls_id_.is_valid() && - tablet_id_.is_valid() && + return tablet_id_.is_valid() && OB_INVALID_ID != source_table_id_ && OB_INVALID_ID != dest_schema_id_ && snapshot_version_ > 0 && @@ -5419,13 +5148,11 @@ struct ObDDLBuildSingleReplicaResponseArg final execution_id_ >= 0 && server_addr_.is_valid(); } - TO_STRING_KV(K_(ls_id), K_(dest_ls_id), - K_(tablet_id), K_(source_table_id), K_(dest_schema_id), K_(ret_code), + TO_STRING_KV(K_(tablet_id), K_(source_table_id), K_(dest_schema_id), K_(ret_code), K_(snapshot_version), K_(schema_version), K_(dest_schema_version), K_(task_id), K_(execution_id), K_(row_scanned), K_(row_inserted), K_(server_addr), K_(physical_row_count)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; int64_t source_table_id_; int64_t dest_schema_id_; @@ -5437,7 +5164,6 @@ struct ObDDLBuildSingleReplicaResponseArg final int64_t row_scanned_; int64_t row_inserted_; - share::ObLSID dest_ls_id_; int64_t dest_schema_version_; common::ObAddr server_addr_; int64_t physical_row_count_; @@ -5451,50 +5177,23 @@ struct ObDDLBuildSingleReplicaResponseArg final -struct ObFreezeSplitSrcTabletArg final -{ - OB_UNIS_VERSION(1); -public: - ObFreezeSplitSrcTabletArg() : ls_id_(), tablet_ids_() {} - ~ObFreezeSplitSrcTabletArg() = default; - bool is_valid() const { return true && ls_id_.is_valid() && !tablet_ids_.empty(); } - TO_STRING_KV(K_(ls_id), K_(tablet_ids)); -public: - - share::ObLSID ls_id_; - common::ObSArray tablet_ids_; -}; - -struct ObFreezeSplitSrcTabletRes final -{ - OB_UNIS_VERSION(1); -public: - ObFreezeSplitSrcTabletRes() : data_end_scn_() {} - ~ObFreezeSplitSrcTabletRes() = default; - bool is_valid() const { return data_end_scn_.is_valid_and_not_min(); } - TO_STRING_KV(K_(data_end_scn)); -public: - share::SCN data_end_scn_; -}; - struct ObAutoSplitTabletArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObAutoSplitTabletArg() - : ls_id_(), tablet_id_(), + : tablet_id_(), auto_split_tablet_size_(OB_INVALID_SIZE), used_disk_space_(OB_INVALID_SIZE) {} ~ObAutoSplitTabletArg() = default; int assign(const ObAutoSplitTabletArg &other); bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid() + return tablet_id_.is_valid() && OB_INVALID_SIZE != auto_split_tablet_size_ && OB_INVALID_SIZE != used_disk_space_; }; - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(auto_split_tablet_size), K_(used_disk_space)); + TO_STRING_KV(K_(tablet_id), K_(auto_split_tablet_size), K_(used_disk_space)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; int64_t auto_split_tablet_size_; @@ -5532,15 +5231,13 @@ struct ObAutoSplitTabletBatchRes final struct ObFetchSplitTabletInfoArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: - ObFetchSplitTabletInfoArg() : ls_id_(), tablet_ids_() {} + ObFetchSplitTabletInfoArg() : tablet_ids_() {} ~ObFetchSplitTabletInfoArg() = default; - bool is_valid() const { return true && ls_id_.is_valid() && !tablet_ids_.empty(); } - TO_STRING_KV(K_(ls_id), K_(tablet_ids)); + bool is_valid() const { return !tablet_ids_.empty(); } + TO_STRING_KV(K_(tablet_ids)); public: - - share::ObLSID ls_id_; common::ObSArray tablet_ids_; }; @@ -5572,7 +5269,6 @@ struct ObFetchSplitTabletInfoRes final -// ObRegisterTxDataResult moved to storage/tx/ob_tx_result_struct.h enum LogMemberGCStat @@ -5714,13 +5410,12 @@ struct ObEstBlockArgElement { OB_UNIS_VERSION(1); public: - ObEstBlockArgElement() : tablet_id_(), ls_id_() {} - bool is_valid() const { return tablet_id_.is_valid() && ls_id_.is_valid(); } + ObEstBlockArgElement() : tablet_id_() {} + bool is_valid() const { return tablet_id_.is_valid(); } int assign(const ObEstBlockArgElement &other); ObTabletID tablet_id_; - share::ObLSID ls_id_; - TO_STRING_KV(K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(tablet_id)); }; struct ObEstBlockArg @@ -5760,24 +5455,22 @@ struct ObEstBlockRes struct ObBatchGetTabletAutoincSeqArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchGetTabletAutoincSeqArg() - : ls_id_(), src_tablet_ids_(), dest_tablet_ids_() + : src_tablet_ids_(), dest_tablet_ids_() {} ~ObBatchGetTabletAutoincSeqArg() {} public: int assign(const ObBatchGetTabletAutoincSeqArg &other); bool is_valid() const { - return true && ls_id_.is_valid() && src_tablet_ids_.count() > 0 + return src_tablet_ids_.count() > 0 && src_tablet_ids_.count() == dest_tablet_ids_.count(); } - int init(const share::ObLSID &ls_id, const ObIArray ¶ms); - TO_STRING_KV(K_(ls_id), K_(src_tablet_ids), K_(dest_tablet_ids)); + int init(const ObIArray ¶ms); + TO_STRING_KV(K_(src_tablet_ids), K_(dest_tablet_ids)); public: - - share::ObLSID ls_id_; common::ObSEArray src_tablet_ids_; common::ObSEArray dest_tablet_ids_; }; @@ -5797,21 +5490,19 @@ struct ObBatchGetTabletAutoincSeqRes final struct ObBatchSetTabletAutoincSeqArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchSetTabletAutoincSeqArg() - : ls_id_(), autoinc_params_(), is_tablet_creating_(false) + : autoinc_params_(), is_tablet_creating_(false) {} ~ObBatchSetTabletAutoincSeqArg() {} public: int assign(const ObBatchSetTabletAutoincSeqArg &other); - bool is_valid() const { return true && ls_id_.is_valid() && autoinc_params_.count() > 0; } - int init(const share::ObLSID &ls_id, const ObIArray ¶ms); + bool is_valid() const { return autoinc_params_.count() > 0; } + int init(const ObIArray ¶ms); void reset(); - TO_STRING_KV(K_(ls_id), K_(autoinc_params), K_(is_tablet_creating)); + TO_STRING_KV(K_(autoinc_params), K_(is_tablet_creating)); public: - - share::ObLSID ls_id_; common::ObSEArray autoinc_params_; bool is_tablet_creating_; }; @@ -5831,19 +5522,17 @@ struct ObBatchSetTabletAutoincSeqRes final struct ObBatchGetTabletBindingArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchGetTabletBindingArg() - : ls_id_(), tablet_ids_(), check_committed_(false) + : tablet_ids_(), check_committed_(false) {} ~ObBatchGetTabletBindingArg() {} public: - bool is_valid() const { return true && ls_id_.is_valid() && tablet_ids_.count() > 0; } - int init(const share::ObLSID &ls_id, const common::ObIArray &tablet_ids, const bool check_committed); - TO_STRING_KV(K_(ls_id), K_(tablet_ids), K_(check_committed)); + bool is_valid() const { return tablet_ids_.count() > 0; } + int init(const common::ObIArray &tablet_ids, const bool check_committed); + TO_STRING_KV(K_(tablet_ids), K_(check_committed)); public: - - share::ObLSID ls_id_; common::ObSArray tablet_ids_; bool check_committed_; }; @@ -5851,19 +5540,17 @@ struct ObBatchGetTabletBindingArg final struct ObBatchGetTabletSplitArg final { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchGetTabletSplitArg() - : ls_id_(), tablet_ids_(), check_committed_(false) + : tablet_ids_(), check_committed_(false) {} ~ObBatchGetTabletSplitArg() {} public: - bool is_valid() const { return true && ls_id_.is_valid() && tablet_ids_.count() > 0; } - int init(const share::ObLSID &ls_id, const common::ObIArray &tablet_ids, const bool check_committed); - TO_STRING_KV(K_(ls_id), K_(tablet_ids), K_(check_committed)); + bool is_valid() const { return tablet_ids_.count() > 0; } + int init(const common::ObIArray &tablet_ids, const bool check_committed); + TO_STRING_KV(K_(tablet_ids), K_(check_committed)); public: - - share::ObLSID ls_id_; common::ObSArray tablet_ids_; bool check_committed_; }; @@ -6329,38 +6016,6 @@ struct ObCancelGatherStatsArg -struct ObCheckNestedMViewMdsArg final -{ - OB_UNIS_VERSION(1); -public: - ObCheckNestedMViewMdsArg() : mview_id_(common::OB_INVALID_ID), - refresh_id_(common::OB_INVALID_ID), - target_data_sync_scn_() - {} - ~ObCheckNestedMViewMdsArg() {} - bool is_valid() { - return 1UL != OB_INVALID_TENANT_ID && mview_id_ != OB_INVALID_ID; - } - TO_STRING_KV(K_(mview_id), K_(refresh_id), K(target_data_sync_scn_)); -public: - - uint64_t mview_id_; - uint64_t refresh_id_; - share::SCN target_data_sync_scn_; -}; - -struct ObCheckNestedMViewMdsRes final -{ - OB_UNIS_VERSION(1); -public: - ObCheckNestedMViewMdsRes() : target_data_sync_scn_(), ret_(OB_SUCCESS) {} - ~ObCheckNestedMViewMdsRes() {} - TO_STRING_KV(K_(target_data_sync_scn), K_(ret)); -public: - share::SCN target_data_sync_scn_; - int ret_; -}; - struct ObCreateTableGroupRes : ObParallelDDLRes { OB_UNIS_VERSION(1); diff --git a/src/share/ob_server_struct.h b/src/share/ob_server_struct.h index 879bac8ce..eb1852955 100644 --- a/src/share/ob_server_struct.h +++ b/src/share/ob_server_struct.h @@ -108,7 +108,6 @@ namespace share class ObTabletTableOperator; class ObSQLiteConnectionPool; class ObRsMgr; -class ObLocationService; class ObSchemaStatusProxy; class ObKVStorage; @@ -140,7 +139,6 @@ struct ObGlobalContext pl::ObPL *pl_engine_; omt::ObMultiTenant *omt_; observer::ObVTIterCreator *vt_iter_creator_; - share::ObLocationService *location_service_; int64_t start_time_; int64_t *warm_up_start_time_; ObServiceStatus status_; diff --git a/src/share/ob_share_util.cpp b/src/share/ob_share_util.cpp index 03882d585..e9833f691 100644 --- a/src/share/ob_share_util.cpp +++ b/src/share/ob_share_util.cpp @@ -391,13 +391,12 @@ ObReplicaType ObShareUtil::string_to_replica_type(const ObString &str) return replica_type; } -// get_sys_ls_readable_scn moved definition to storage/tx_storage/ob_ls_service.cpp(real user ObLSService/ObLS complete type, previously hidden behind a removed include chain) +// The LS readable SCN helper is implemented by storage/tx_storage/ob_ls_service.cpp. // Note: master tenant-elim changed the original body(MTL(ObLSService*) -> share::g_mp->ls_service()), HOST must be synced (see routing item) // check_clog_disk_full_or_hang moved definition to logservice/ob_log_service.cpp(removes share→logservice dependency, declaration remains in this class header) // Note: master tenant-elim changed the original body(MTL(logservice::ObLogService*) -> share::g_mp->log_service()), HOST must be synced (see routing item) -// get_tenant_gts moved definition to storage/tx/ob_ts_mgr.cpp(removes share→storage dependency, declaration remains in this class header) // Note: master tenant-elim changed the original body(removed the tenant_id parameter, OB_TS_MGR.get_gts removed the tenant_id argument), HOST must be synced (see routing item) int ObShareUtil::gen_sys_unit(ObUnit &unit) diff --git a/src/share/ob_share_util.h b/src/share/ob_share_util.h index 1a2d3f4e1..0a6d93040 100644 --- a/src/share/ob_share_util.h +++ b/src/share/ob_share_util.h @@ -110,10 +110,7 @@ class ObShareUtil static inline uint64_t compute_server_index(uint64_t server_id) { return server_id % (MAX_SERVER_COUNT + 1); } - // get_sys_ls_readable_scn has been demoted to storage::free function(see end of file storage ns) // check_clog_disk_full_or_hang has been demoted to logservice::free function - // get_tenant_gts has been demoted to storage::free function(transaction::get_tenant_gts, see end of file storage/transaction ns and ob_ts_mgr.cpp) - // Note: master still keeps ObShareUtil::get_tenant_gts(SCN&) shape and still has callers, changed to transaction::get_tenant_gts (see routing item) static int gen_sys_unit(ObUnit &unit); static int gen_sys_resource_pool(ObResourcePool &resource_pool); static int gen_default_sys_tenant_schema(schema::ObTenantSchema &tenant_schema); diff --git a/src/share/ob_standby_upgrade.cpp b/src/share/ob_standby_upgrade.cpp deleted file mode 100644 index da4901619..000000000 --- a/src/share/ob_standby_upgrade.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE - -#include "ob_standby_upgrade.h" - -using namespace oceanbase; -using namespace oceanbase::common; -namespace oceanbase -{ -namespace share -{ - -OB_SERIALIZE_MEMBER(ObStandbyUpgrade, data_version_); - -int ObUpgradeDataVersionMDSHelper::on_register( - const char* buf, - const int64_t len, - storage::mds::BufferCtx &ctx) -{ - UNUSEDx(buf, len, ctx); - return OB_SUCCESS; -} - -int ObUpgradeDataVersionMDSHelper::on_replay( - const char* buf, - const int64_t len, - const share::SCN &scn, - storage::mds::BufferCtx &ctx) -{ - UNUSEDx(buf, len, scn, ctx); - return OB_SUCCESS; -} -} -} - diff --git a/src/share/ob_standby_upgrade.h b/src/share/ob_standby_upgrade.h deleted file mode 100644 index 3e5ba9860..000000000 --- a/src/share/ob_standby_upgrade.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_ -#define OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_ - -#include "lib/utility/ob_print_utils.h" // Print* -#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION -#include "share/ob_cluster_version.h" -#include "share/scn.h" -namespace oceanbase { namespace storage { namespace mds { class BufferCtx; } } } - -namespace oceanbase -{ - -namespace share -{ - -struct ObStandbyUpgrade -{ - OB_UNIS_VERSION(1); - public: - ObStandbyUpgrade(): data_version_(0) {} - ObStandbyUpgrade(const uint64_t data_version): data_version_(data_version) {} - ~ObStandbyUpgrade() {} - bool is_valid() const - { - return true; - } - uint64_t get_data_version() const - { - return data_version_; - } - - TO_STRING_KV(K_(data_version)); -private: - uint64_t data_version_; -}; - -class ObUpgradeDataVersionMDSHelper -{ -public: - static int on_register( - const char* buf, - const int64_t len, - storage::mds::BufferCtx &ctx); - static int on_replay( - const char* buf, - const int64_t len, - const share::SCN &scn, - storage::mds::BufferCtx &ctx); -}; - -} -} - -#endif /* !OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_ */ diff --git a/src/share/ob_table_access_helper.h b/src/share/ob_table_access_helper.h index ae3e16002..83484a4d5 100644 --- a/src/share/ob_table_access_helper.h +++ b/src/share/ob_table_access_helper.h @@ -29,7 +29,6 @@ #include "share/ob_define.h" #include "lib/net/ob_addr.h" #include "share/ob_errno.h" -#include "share/ob_ls_id.h" #include "share/rc/ob_tenant_base.h" #include "share/ob_server_struct.h" #include @@ -584,21 +583,6 @@ class ObTableAccessHelper return ret; #undef PRINT_WRAPPER } - static int get_signle_column_from_signle_row_(common::sqlclient::ObMySQLResult *row, - const char *column, - share::ObLSID &ls_id) - { - TIMEGUARD_INIT(OCCAM, 1_s); - int ret = common::OB_SUCCESS; - int64_t value = 0; - if (CLICK_FAIL(row->get_int(column, value))) { - OB_LOG(WARN, "get_column_from_signle_row failed", KR(ret), K(column)); - } else { - ls_id = share::ObLSID(value); - OB_LOG(TRACE, "get_column_from_signle_row success", KR(ret), K(column), K(value)); - } - return ret; - } static int get_signle_column_from_signle_row_(common::sqlclient::ObMySQLResult *row, const char *column, common::ObTabletID &tablet_id) diff --git a/src/share/ob_tablet_checksum_iterator.cpp b/src/share/ob_tablet_checksum_iterator.cpp index bd573d23f..40801c1b1 100644 --- a/src/share/ob_tablet_checksum_iterator.cpp +++ b/src/share/ob_tablet_checksum_iterator.cpp @@ -47,20 +47,20 @@ int ObTabletChecksumIterator::fetch_next_batch() ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else { - ObTabletLSPair start_pair; + ObTabletID start_tablet_id; if (checksum_items_.count() > 0) { ObTabletChecksumItem tmp_item; if (OB_FAIL(checksum_items_.at(checksum_items_.count() - 1, tmp_item))) { LOG_WARN("fail to fetch last checksum item", KR(ret), K_(checksum_items)); - } else if (OB_FAIL(start_pair.init(tmp_item.tablet_id_, tmp_item.ls_id_))) { - LOG_WARN("fail to init start tablet_ls_pair", KR(ret), K(tmp_item)); + } else { + start_tablet_id = tmp_item.tablet_id_; } } if (OB_SUCC(ret)) { checksum_items_.reuse(); - if (OB_FAIL(ObTabletChecksumOperator::load_tablet_checksum_items(*sql_proxy_, start_pair, + if (OB_FAIL(ObTabletChecksumOperator::load_tablet_checksum_items(*sql_proxy_, start_tablet_id, BATCH_FETCH_COUNT, compaction_scn_, checksum_items_))) { - LOG_WARN("fail to load tablet checksums", KR(ret), K(start_pair), + LOG_WARN("fail to load tablet checksums", KR(ret), K(start_tablet_id), K_(compaction_scn)); } else if (OB_UNLIKELY(0 == checksum_items_.count())) { ret = OB_ITER_END; diff --git a/src/share/ob_tablet_checksum_iterator.h b/src/share/ob_tablet_checksum_iterator.h index 120506917..2793fa913 100644 --- a/src/share/ob_tablet_checksum_iterator.h +++ b/src/share/ob_tablet_checksum_iterator.h @@ -30,8 +30,6 @@ class ObISQLClient; } namespace share { -class ObTabletLSPair; - class ObTabletChecksumIterator { diff --git a/src/share/ob_tablet_checksum_operator.cpp b/src/share/ob_tablet_checksum_operator.cpp index 32a9b06a5..23ad785a7 100644 --- a/src/share/ob_tablet_checksum_operator.cpp +++ b/src/share/ob_tablet_checksum_operator.cpp @@ -33,9 +33,8 @@ using namespace schema; void ObTabletChecksumItem::reset() { - + tablet_id_.reset(); - ls_id_.reset(); data_checksum_ = -1; row_count_ = 0; compaction_scn_.reset(); @@ -44,14 +43,13 @@ void ObTabletChecksumItem::reset() bool ObTabletChecksumItem::is_valid() const { - return (tablet_id_.is_valid()) && (ls_id_.is_valid()); + return tablet_id_.is_valid(); } ObTabletChecksumItem &ObTabletChecksumItem::operator=(const ObTabletChecksumItem &other) { - + tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; data_checksum_ = other.data_checksum_; row_count_ = other.row_count_; compaction_scn_ = other.compaction_scn_; @@ -65,8 +63,7 @@ int ObTabletChecksumItem::verify_tablet_column_checksum(const ObTabletReplicaChe { int ret = OB_SUCCESS; - if ((tablet_id_ != replica_item.tablet_id_) - || (ls_id_ != replica_item.ls_id_)) { + if (tablet_id_ != replica_item.tablet_id_) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_item), K(*this)); } else { @@ -80,7 +77,7 @@ int ObTabletChecksumItem::verify_tablet_column_checksum(const ObTabletReplicaChe ret = OB_CHECKSUM_ERROR; LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "fatal checksum error", KR(ret), K(is_same_column_checksum), K(replica_item), K(*this)); } - } + } } return ret; } @@ -92,9 +89,8 @@ int ObTabletChecksumItem::assign(const ObTabletReplicaChecksumItem &replica_item ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_item)); } else { - + tablet_id_ = replica_item.tablet_id_; - ls_id_ = replica_item.ls_id_; data_checksum_ = replica_item.data_checksum_; row_count_ = replica_item.row_count_; compaction_scn_ = replica_item.compaction_scn_; @@ -113,9 +109,8 @@ int ObTabletChecksumItem::assign(const ObTabletChecksumItem &other) LOG_WARN("invalid argument", KR(ret), K(other)); } else if (this != &other) { reset(); - + tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; data_checksum_ = other.data_checksum_; row_count_ = other.row_count_; compaction_scn_ = other.compaction_scn_; @@ -130,7 +125,7 @@ int ObTabletChecksumItem::assign(const ObTabletChecksumItem &other) int ObTabletChecksumOperator::load_tablet_checksum_items( ObISQLClient &sql_client, - const ObTabletLSPair &start_pair, + const ObTabletID &start_tablet_id, const int64_t batch_cnt, const SCN &compaction_scn, ObIArray &items) @@ -149,25 +144,23 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( const int64_t limit_cnt = ((remain_cnt >= MAX_BATCH_COUNT) ? MAX_BATCH_COUNT : remain_cnt); ori_items_cnt = items.count(); - ObTabletLSPair last_pair; + ObTabletID last_tablet_id; if (remain_cnt == batch_cnt) { - last_pair = start_pair; + last_tablet_id = start_tablet_id; } else { const int64_t items_cnt = items.count(); if (items_cnt > 0) { const ObTabletChecksumItem &last_item = items.at(items_cnt - 1); - if (OB_FAIL(last_pair.init(last_item.tablet_id_, last_item.ls_id_))) { - LOG_WARN("fail to init last tablet_ls_pair", KR(ret), K(last_item)); - } + last_tablet_id = last_item.tablet_id_; } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected, about tablet items count", KR(ret), K(start_pair), + LOG_WARN("err unexpected, about tablet items count", KR(ret), K(start_tablet_id), K(batch_cnt), K(remain_cnt), K(items_cnt)); } } - if (FAILEDx(construct_load_sql_str_(last_pair, limit_cnt, compaction_scn, sql))) { - LOG_WARN("fail to construct load sql", KR(ret), K(last_pair), K(limit_cnt), + if (FAILEDx(construct_load_sql_str_(last_tablet_id, limit_cnt, compaction_scn, sql))) { + LOG_WARN("fail to construct load sql", KR(ret), K(last_tablet_id), K(limit_cnt), K(compaction_scn)); } else if (OB_FAIL(load_tablet_checksum_items(sql_client, sql, items))) { LOG_WARN("fail to load tablet checksum items", KR(ret), K(sql)); @@ -185,25 +178,25 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( int ObTabletChecksumOperator::load_tablet_checksum_items( ObISQLClient &sql_client, - const ObIArray &pairs, + const ObIArray &tablet_ids, const SCN &compaction_scn, ObIArray &items) { int ret = OB_SUCCESS; - const int64_t pairs_cnt = pairs.count(); + const int64_t tablet_cnt = tablet_ids.count(); int64_t start_idx = 0; - int64_t end_idx = min(MAX_BATCH_COUNT, pairs_cnt); + int64_t end_idx = min(MAX_BATCH_COUNT, tablet_cnt); ObSqlString sql; while (OB_SUCC(ret) && (start_idx < end_idx)) { sql.reuse(); - if (OB_FAIL(construct_load_sql_str_(pairs, start_idx, end_idx, compaction_scn, sql))) { - LOG_WARN("fail to construct load sql", KR(ret), K(pairs_cnt), + if (OB_FAIL(construct_load_sql_str_(tablet_ids, start_idx, end_idx, compaction_scn, sql))) { + LOG_WARN("fail to construct load sql", KR(ret), K(tablet_cnt), K(start_idx), K(end_idx)); } else if (OB_FAIL(load_tablet_checksum_items(sql_client, sql, items))) { LOG_WARN("fail to load tablet checksum items", KR(ret), K(sql)); } else { start_idx = end_idx; - end_idx = min(start_idx + MAX_BATCH_COUNT, pairs_cnt); + end_idx = min(start_idx + MAX_BATCH_COUNT, tablet_cnt); } } return ret; @@ -233,8 +226,6 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( LOG_WARN("fail to get next row", KR(ret)); } } else { - int64_t ls_id = ObLSID::SYS_LS_ID; - int64_t tablet_id = -1; uint64_t compaction_scn_val = 0; ObString column_meta_str; @@ -248,7 +239,6 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( LOG_WARN("fail to convert val to SCN", KR(ret), K(compaction_scn_val)); } else { item.tablet_id_ = (uint64_t)tablet_id; - item.ls_id_ = ls_id; if (OB_FAIL(item.column_meta_.set_with_str(compaction_scn_val, column_meta_str))) { LOG_WARN("fail to set column meta", KR(ret), K(compaction_scn_val)); } @@ -279,13 +269,13 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( return ret; } -int ObTabletChecksumOperator::construct_load_sql_str_(const ObTabletLSPair &start_pair, +int ObTabletChecksumOperator::construct_load_sql_str_(const ObTabletID &start_tablet_id, const int64_t batch_cnt, const SCN &compaction_scn, ObSqlString &sql) { int ret = OB_SUCCESS; - + if ((batch_cnt < 1) || (!compaction_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(batch_cnt), K(compaction_scn)); @@ -293,15 +283,15 @@ int ObTabletChecksumOperator::construct_load_sql_str_(const ObTabletLSPair &star if (compaction_scn > SCN::min_scn()) { if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE compaction_scn = %lu " "and tablet_id > '%lu' ", OB_ALL_TABLET_CHECKSUM_TNAME, - compaction_scn.get_val_for_inner_table_field(), start_pair.get_tablet_id().id()))) { - LOG_WARN("fail to assign sql", KR(ret), K(start_pair), K(compaction_scn)); + compaction_scn.get_val_for_inner_table_field(), start_tablet_id.id()))) { + LOG_WARN("fail to assign sql", KR(ret), K(start_tablet_id), K(compaction_scn)); } else if (OB_FAIL(sql.append_fmt(" ORDER BY tablet_id limit %ld", batch_cnt))) { LOG_WARN("fail to assign sql string", KR(ret), K(batch_cnt), K(compaction_scn)); } } else { // compaction_scn == 0: get records with all compaction_scn - if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE tablet_id > '%lu' ", - OB_ALL_TABLET_CHECKSUM_TNAME, start_pair.get_tablet_id().id()))) { - LOG_WARN("fail to assign sql", KR(ret), K(start_pair), K(compaction_scn)); + if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE tablet_id > '%lu' ", + OB_ALL_TABLET_CHECKSUM_TNAME, start_tablet_id.id()))) { + LOG_WARN("fail to assign sql", KR(ret), K(start_tablet_id), K(compaction_scn)); } else if (OB_FAIL(sql.append_fmt(" ORDER BY tablet_id, compaction_scn limit %ld", batch_cnt))) { LOG_WARN("fail to assign sql string", KR(ret), K(batch_cnt), K(compaction_scn)); @@ -311,38 +301,38 @@ int ObTabletChecksumOperator::construct_load_sql_str_(const ObTabletLSPair &star return ret; } -int ObTabletChecksumOperator::construct_load_sql_str_(const common::ObIArray &pairs, +int ObTabletChecksumOperator::construct_load_sql_str_(const common::ObIArray &tablet_ids, const int64_t start_idx, const int64_t end_idx, const SCN &compaction_scn, common::ObSqlString &sql) { int ret = OB_SUCCESS; - - const int64_t pairs_cnt = pairs.count(); - if ((start_idx < 0) || (end_idx > pairs_cnt) || - (start_idx > end_idx) || (pairs_cnt < 1)) { + + const int64_t tablet_cnt = tablet_ids.count(); + if ((start_idx < 0) || (end_idx > tablet_cnt) || + (start_idx > end_idx) || (tablet_cnt < 1)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(start_idx), K(end_idx), K(pairs_cnt)); + LOG_WARN("invalid argument", KR(ret), K(start_idx), K(end_idx), K(tablet_cnt)); } else if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE compaction_scn = " "%lu and tablet_id IN (", OB_ALL_TABLET_CHECKSUM_TNAME, compaction_scn.get_val_for_inner_table_field()))) { LOG_WARN("fail to assign sql", KR(ret)); } else { for (int64_t idx = start_idx; OB_SUCC(ret) && (idx < end_idx); ++idx) { - const ObTabletLSPair &pair = pairs.at(idx); - if (OB_UNLIKELY(!pair.is_valid())) { + const ObTabletID &tablet_id = tablet_ids.at(idx); + if (OB_UNLIKELY(!tablet_id.is_valid_with_tenant())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tablet_ls_pair", KR(ret), K(pair), K(idx)); + LOG_WARN("invalid tablet id", KR(ret), K(tablet_id), K(idx)); } else if (OB_FAIL(sql.append_fmt( "%ld%s", - pair.get_tablet_id().id(), + tablet_id.id(), ((idx == end_idx - 1) ? ")" : ", ")))) { - LOG_WARN("fail to assign sql", KR(ret), K(pair)); + LOG_WARN("fail to assign sql", KR(ret), K(tablet_id)); } } if (FAILEDx(sql.append_fmt(" ORDER BY tablet_id"))) { - SHARE_LOG(WARN, "fail to assign sql string", KR(ret), K(compaction_scn), K(pairs_cnt)); + SHARE_LOG(WARN, "fail to assign sql string", KR(ret), K(compaction_scn), K(tablet_cnt)); } } return ret; @@ -364,7 +354,7 @@ int ObTabletChecksumOperator::insert_or_update_tablet_checksum_items_( int ret = OB_SUCCESS; ObSqlString sql; ObMySQLTransaction trans; - + const int64_t item_cnt = items.count(); if (OB_UNLIKELY(item_cnt < 1)) { ret = OB_INVALID_ARGUMENT; @@ -372,7 +362,7 @@ int ObTabletChecksumOperator::insert_or_update_tablet_checksum_items_( } else if (OB_FAIL(trans.start(&sql_client))) { LOG_WARN("failed to start transaction", KR(ret)); } else { - int64_t remain_cnt = item_cnt; + int64_t remain_cnt = item_cnt; int64_t report_idx = 0; while (OB_SUCC(ret) && (remain_cnt > 0)) { sql.reuse(); @@ -459,7 +449,7 @@ int ObTabletChecksumOperator::delete_tablet_checksum_items( int ret = OB_SUCCESS; ObSqlString sql; affected_rows = 0; - + const uint64_t gc_scn_val = gc_compaction_scn.is_valid() ? gc_compaction_scn.get_val_for_inner_table_field() : 0; if (OB_UNLIKELY(!gc_compaction_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -483,7 +473,7 @@ int ObTabletChecksumOperator::delete_special_tablet_checksum_items( int ret = OB_SUCCESS; ObSqlString sql; int64_t affected_rows = 0; - + const uint64_t gc_scn_val = gc_compaction_scn.is_valid() ? gc_compaction_scn.get_val_for_inner_table_field() : 0; if (OB_UNLIKELY(!gc_compaction_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -561,7 +551,7 @@ int ObTabletChecksumOperator::load_all_compaction_scn( return ret; } -int ObTabletChecksumOperator::is_first_tablet_in_sys_ls_exist( +int ObTabletChecksumOperator::is_first_tablet_checksum_exist( common::ObISQLClient &sql_client, const SCN &compaction_scn, bool &is_exist) @@ -573,7 +563,7 @@ int ObTabletChecksumOperator::is_first_tablet_in_sys_ls_exist( } if (OB_SUCC(ret)) { is_exist = false; - + ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult, res) { ObMySQLResult *result = nullptr; @@ -599,7 +589,7 @@ int ObTabletChecksumOperator::is_first_tablet_in_sys_ls_exist( is_exist = false; } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected count of first tablet in sys ls", KR(ret), K(sql), K(cnt)); + LOG_WARN("unexpected first tablet checksum count", KR(ret), K(sql), K(cnt)); } } } diff --git a/src/share/ob_tablet_checksum_operator.h b/src/share/ob_tablet_checksum_operator.h index c249e7efa..47a8e2f31 100644 --- a/src/share/ob_tablet_checksum_operator.h +++ b/src/share/ob_tablet_checksum_operator.h @@ -37,7 +37,7 @@ struct ObTabletChecksumItem { public: ObTabletChecksumItem() - : tablet_id_(), ls_id_(), data_checksum_(-1), + : tablet_id_(), data_checksum_(-1), row_count_(0), compaction_scn_(), replica_type_(0), column_meta_() {} virtual ~ObTabletChecksumItem() = default; @@ -48,12 +48,11 @@ struct ObTabletChecksumItem int assign(const ObTabletChecksumItem &other); ObTabletChecksumItem &operator =(const ObTabletChecksumItem &other); common::ObTabletID get_tablet_id() const { return tablet_id_; } - TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(data_checksum), K_(row_count), + TO_STRING_KV(K_(tablet_id), K_(data_checksum), K_(row_count), K_(compaction_scn), K_(replica_type), K_(column_meta)); common::ObTabletID tablet_id_; - ObLSID ls_id_; int64_t data_checksum_; int64_t row_count_; SCN compaction_scn_; @@ -72,14 +71,14 @@ class ObTabletChecksumOperator // else, invalid argument static int load_tablet_checksum_items( common::ObISQLClient &sql_client, - const ObTabletLSPair &start_pair, + const common::ObTabletID &start_tablet_id, const int64_t batch_cnt, const SCN &compaction_scn, common::ObIArray &items); // multi get tablet checksum static int load_tablet_checksum_items( common::ObISQLClient &sql_client, - const common::ObIArray &pairs, + const common::ObIArray &tablet_ids, const SCN &compaction_scn, common::ObIArray &items); static int load_tablet_checksum_items( @@ -89,12 +88,12 @@ class ObTabletChecksumOperator static int update_tablet_checksum_items( common::ObISQLClient &sql_client, common::ObIArray &items); - // delete records whose compaction_scn <= @gc_compaction_scn and (tablet_id, ls_id) is (1, 1) + // delete records whose compaction_scn <= @gc_compaction_scn for the special tablet static int delete_special_tablet_checksum_items( common::ObISQLClient &sql_client, const SCN &gc_compaction_scn); // delete limited records whose compaction_scn <= @gc_compaction_scn - // , while the record of whose (tablet_id, ls_id) is (1, 1) can't be deleted. + // while the special tablet record can't be deleted. static int delete_tablet_checksum_items( common::ObISQLClient &sql_client, const SCN &gc_compaction_scn, @@ -103,17 +102,17 @@ class ObTabletChecksumOperator static int load_all_compaction_scn( common::ObISQLClient &sql_client, common::ObIArray &compaction_scn_arr); - static int is_first_tablet_in_sys_ls_exist( + static int is_first_tablet_checksum_exist( common::ObISQLClient &sql_client, const SCN &compaction_scn, bool &is_exist); private: - static int construct_load_sql_str_(const ObTabletLSPair &start_pair, + static int construct_load_sql_str_(const common::ObTabletID &start_tablet_id, const int64_t batch_cnt, const SCN &compaction_scn, common::ObSqlString &sql); - static int construct_load_sql_str_(const common::ObIArray &pairs, + static int construct_load_sql_str_(const common::ObIArray &tablet_ids, const int64_t start_idx, const int64_t end_idx, const SCN &compaction_scn, diff --git a/src/share/ob_tablet_meta_table_compaction_operator.cpp b/src/share/ob_tablet_meta_table_compaction_operator.cpp index 37c2b8f1e..58febf1e0 100644 --- a/src/share/ob_tablet_meta_table_compaction_operator.cpp +++ b/src/share/ob_tablet_meta_table_compaction_operator.cpp @@ -27,7 +27,7 @@ using namespace compaction; namespace share { -int ObTabletMetaTableCompactionOperator::batch_set_info_status(const ObIArray &error_pairs, +int ObTabletMetaTableCompactionOperator::batch_set_info_status(const ObIArray &error_tablets, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -40,17 +40,17 @@ int ObTabletMetaTableCompactionOperator::batch_set_info_status(const ObIArray tablet_ls_pairs; + ObArray tablet_ids; ObArray compaction_scns; - for (int64_t i = 0; OB_SUCC(ret) && i < error_pairs.count(); ++i) { - if (OB_FAIL(tablet_ls_pairs.push_back(error_pairs.at(i).tablet_info_))) { - LOG_WARN("failed to push back tablet_ls_pair", K(ret)); - } else if (OB_FAIL(compaction_scns.push_back(error_pairs.at(i).compaction_scn_))) { + for (int64_t i = 0; OB_SUCC(ret) && i < error_tablets.count(); ++i) { + if (OB_FAIL(tablet_ids.push_back(error_tablets.at(i).tablet_info_))) { + LOG_WARN("failed to push back tablet id", K(ret)); + } else if (OB_FAIL(compaction_scns.push_back(error_tablets.at(i).compaction_scn_))) { LOG_WARN("failed to push back compaction_scn", K(ret)); } } if (OB_SUCC(ret)) { - if (OB_FAIL(storage.batch_update_status(tablet_ls_pairs, + if (OB_FAIL(storage.batch_update_status(tablet_ids, compaction_scns, (int64_t)ObTabletReplica::ScnStatus::SCN_STATUS_ERROR, affected_rows))) { @@ -84,14 +84,13 @@ int ObTabletMetaTableCompactionOperator::get_status( int64_t report_scn = 0; int64_t status = 0; if (OB_FAIL(storage.get_max_report_scn_and_status(common::ObTabletID(input_info.tablet_id_), - ObLSID(input_info.ls_id_), report_scn, status))) { if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("failed to get max report_scn and status", KR(ret), K(input_info)); } } else { - ret_info = input_info; // assign ls_id / tablet_id + ret_info = input_info; ret_info.report_scn_ = report_scn; ret_info.status_ = ObTabletReplica::ScnStatus(status); LOG_TRACE("success to get medium snapshot info", K(ret_info)); @@ -101,7 +100,7 @@ int ObTabletMetaTableCompactionOperator::get_status( return ret; } -int ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet(const share::ObLSID &ls_id, +int ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet( const int64_t major_frozen_scn, const common::ObIArray &input_tablet_id_array) { @@ -110,7 +109,7 @@ int ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet( ret = OB_NOT_INIT; LOG_WARN("meta_db_pool_ is not initialized", K(ret)); } else { - LOG_INFO("start to update unequal tablet id array", KR(ret), K(ls_id), K(major_frozen_scn), + LOG_INFO("start to update unequal tablet id array", KR(ret), K(major_frozen_scn), "input_tablet_id_array_cnt", input_tablet_id_array.count()); ObTabletMetaTableStorage storage; if (OB_FAIL(storage.init(GCTX.meta_db_pool_))) { @@ -129,15 +128,15 @@ int ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet( } } if (OB_SUCC(ret)) { - if (OB_FAIL(storage.get_distinct_tablet_ids_with_conditions(ls_id, batch_tablet_ids, major_frozen_scn, unequal_tablet_id_array))) { + if (OB_FAIL(storage.get_distinct_tablet_ids_with_conditions(batch_tablet_ids, major_frozen_scn, unequal_tablet_id_array))) { LOG_WARN("failed to get distinct tablet_ids with conditions", K(ret)); } else if (unequal_tablet_id_array.count() > 0) { LOG_TRACE("success to get unequal tablet_id array", K(ret), K(unequal_tablet_id_array)); int64_t tmp_affected_rows = 0; - if (OB_FAIL(storage.batch_update_report_scn_unequal(ls_id, unequal_tablet_id_array, major_frozen_scn, tmp_affected_rows))) { + if (OB_FAIL(storage.batch_update_report_scn_unequal(unequal_tablet_id_array, major_frozen_scn, tmp_affected_rows))) { LOG_WARN("fail to update unequal tablet id array", KR(ret)); } else { - LOG_INFO("success to update unequal report_scn", K(ret), K(ls_id), K(tmp_affected_rows)); + LOG_INFO("success to update unequal report_scn", K(ret), K(tmp_affected_rows)); } unequal_tablet_id_array.reuse(); } @@ -232,37 +231,6 @@ int ObTabletMetaTableCompactionOperator::append_tablet_id_array(const common::Ob return ret; } -int ObTabletMetaTableCompactionOperator::inner_batch_update_unequal_report_scn_tablet(const share::ObLSID &ls_id, - const int64_t major_frozen_scn, - const common::ObIArray &unequal_tablet_id_array) -{ - int ret = OB_SUCCESS; - int64_t affected_rows = 0; -#ifdef ERRSIM - ret = OB_E(EventTable::EN_COMPACTION_UPDATE_REPORT_SCN) ret; - if (OB_FAIL(ret)) { - LOG_INFO("ERRSIM EN_COMPACTION_UPDATE_REPORT_SCN", K(ret)); - } -#endif - if (OB_FAIL(ret)) { - // ERRSIM error, skip - } else if (OB_ISNULL(GCTX.meta_db_pool_)) { - ret = OB_NOT_INIT; - LOG_WARN("meta_db_pool_ is not initialized", K(ret)); - } else { - ObTabletMetaTableStorage storage; - if (OB_FAIL(storage.init(GCTX.meta_db_pool_))) { - LOG_WARN("failed to init storage", K(ret)); - } else if (OB_FAIL(storage.batch_update_report_scn_unequal(ls_id, unequal_tablet_id_array, major_frozen_scn, affected_rows))) { - LOG_WARN("fail to update unequal report_scn", KR(ret), K(ls_id)); - } else if (affected_rows > 0) { - LOG_INFO("success to update unequal report_scn", K(ret), K(ls_id), K(unequal_tablet_id_array.count()), K(affected_rows)); - } - } - return ret; -} - - int ObTabletMetaTableCompactionOperator::batch_update_report_scn( const uint64_t global_broadcast_scn_val, const ObTabletReplica::ScnStatus &except_status, @@ -424,15 +392,15 @@ int ObTabletMetaTableCompactionOperator::get_tablet_replica_cnt(int64_t &tablet_ int ObTabletMetaTableCompactionOperator::batch_update_report_scn( const uint64_t global_broadcast_scn_val, - const ObIArray &tablet_pairs, + const ObIArray &tablet_ids, const ObTabletReplica::ScnStatus &except_status) { int ret = OB_SUCCESS; int64_t affected_rows = 0; - const int64_t all_pair_cnt = tablet_pairs.count(); - if (OB_UNLIKELY(all_pair_cnt < 1)) { + const int64_t all_tablet_cnt = tablet_ids.count(); + if (OB_UNLIKELY(all_tablet_cnt < 1)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(all_pair_cnt)); + LOG_WARN("invalid argument", KR(ret), K(all_tablet_cnt)); } else if (OB_ISNULL(GCTX.meta_db_pool_)) { ret = OB_NOT_INIT; LOG_WARN("meta_db_pool_ is not initialized", K(ret)); @@ -441,18 +409,18 @@ int ObTabletMetaTableCompactionOperator::batch_update_report_scn( if (OB_FAIL(storage.init(GCTX.meta_db_pool_))) { LOG_WARN("failed to init storage", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && (i < all_pair_cnt); i += MAX_BATCH_COUNT) { - const int64_t cur_end_idx = MIN(i + MAX_BATCH_COUNT, all_pair_cnt); - ObArray batch_pairs; + for (int64_t i = 0; OB_SUCC(ret) && (i < all_tablet_cnt); i += MAX_BATCH_COUNT) { + const int64_t cur_end_idx = MIN(i + MAX_BATCH_COUNT, all_tablet_cnt); + ObArray batch_tablet_ids; for (int64_t idx = i; OB_SUCC(ret) && (idx < cur_end_idx); ++idx) { - if (OB_FAIL(batch_pairs.push_back(tablet_pairs.at(idx)))) { - LOG_WARN("failed to push back tablet_ls_pair", K(ret)); + if (OB_FAIL(batch_tablet_ids.push_back(tablet_ids.at(idx)))) { + LOG_WARN("failed to push back tablet id", K(ret)); } } if (OB_SUCC(ret)) { int64_t tmp_affected_rows = 0; if (OB_FAIL(storage.batch_update_report_scn( - batch_pairs, + batch_tablet_ids, global_broadcast_scn_val, global_broadcast_scn_val, (int64_t)except_status, @@ -460,7 +428,7 @@ int ObTabletMetaTableCompactionOperator::batch_update_report_scn( LOG_WARN("fail to batch update report_scn", KR(ret), K(global_broadcast_scn_val)); } else { affected_rows += tmp_affected_rows; - LOG_TRACE("success to update report_scn", KR(ret), K(batch_pairs), K(tmp_affected_rows)); + LOG_TRACE("success to update report_scn", KR(ret), K(batch_tablet_ids), K(tmp_affected_rows)); } } } diff --git a/src/share/ob_tablet_meta_table_compaction_operator.h b/src/share/ob_tablet_meta_table_compaction_operator.h index de0b61169..1f5d801fe 100644 --- a/src/share/ob_tablet_meta_table_compaction_operator.h +++ b/src/share/ob_tablet_meta_table_compaction_operator.h @@ -21,7 +21,6 @@ #include "common/mysqlclient/ob_isql_client.h" #include "common/ob_zone.h" #include "common/mysqlclient/ob_mysql_transaction.h" -#include "share/ob_ls_id.h" #include "share/tablet/ob_tablet_info.h" #include "share/ob_server_struct.h" #include "storage/compaction/ob_ckm_error_tablet_info.h" @@ -42,25 +41,22 @@ struct ObTabletCompactionScnInfo { public: ObTabletCompactionScnInfo() - : ls_id_(0), - tablet_id_(0), + : tablet_id_(0), compaction_scn_(0), report_scn_(0), status_(ObTabletReplica::SCN_STATUS_MAX) {} ObTabletCompactionScnInfo( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObTabletReplica::ScnStatus status) - : ls_id_(ls_id.id()), - tablet_id_(tablet_id.id()), + : tablet_id_(tablet_id.id()), compaction_scn_(0), report_scn_(0), status_(status) {} bool is_valid() const { - return true && ls_id_ > 0 && tablet_id_ > 0 && report_scn_ >= 0; + return tablet_id_ > 0 && report_scn_ >= 0; } // only check when last compaction type is major bool could_schedule_next_round(const int64_t major_frozen_scn) @@ -69,15 +65,13 @@ struct ObTabletCompactionScnInfo } void reset() { - ls_id_ = 0; tablet_id_ = 0; compaction_scn_ = 0; report_scn_ = 0; status_ = ObTabletReplica::SCN_STATUS_MAX; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(compaction_scn), K_(report_scn), K_(status)); + TO_STRING_KV(K_(tablet_id), K_(compaction_scn), K_(report_scn), K_(status)); public: - int64_t ls_id_; int64_t tablet_id_; int64_t compaction_scn_; int64_t report_scn_; @@ -88,15 +82,15 @@ struct ObTabletCompactionScnInfo class ObTabletMetaTableCompactionOperator { public: - static int batch_set_info_status(const ObIArray &tablet_ls_pairs, + static int batch_set_info_status(const ObIArray &tablet_infos, int64_t &affected_rows); static int get_status( const ObTabletCompactionScnInfo &input_info, ObTabletCompactionScnInfo &ret_info); - // update report_scn of all tablets which belong to @tablet_pairs + // update report_scn of all tablets in @tablet_ids static int batch_update_report_scn( const uint64_t global_broadcast_scn_val, - const common::ObIArray &tablet_pairs, + const common::ObIArray &tablet_ids, const ObTabletReplica::ScnStatus &except_status); // after major_freeze, update all tablets' report_scn to global_broadcast_scn_val static int batch_update_report_scn( @@ -105,8 +99,7 @@ class ObTabletMetaTableCompactionOperator const volatile bool &stop); // designed for 'clear merge error'. it updates all tablets' status to SCN_STATUS_IDLE static int batch_update_status(); - static int batch_update_unequal_report_scn_tablet(const share::ObLSID &ls_id, - const int64_t major_frozen_scn, + static int batch_update_unequal_report_scn_tablet(const int64_t major_frozen_scn, const common::ObIArray &input_tablet_id_array); static int get_min_compaction_scn(SCN &min_compaction_scn); static int range_scan_for_compaction(const int64_t compaction_scn, @@ -131,9 +124,6 @@ class ObTabletMetaTableCompactionOperator common::ObMySQLTransaction &trans, const bool is_update_finish_scn, const common::ObIArray &replicas); - static int inner_batch_update_unequal_report_scn_tablet(const share::ObLSID &ls_id, - const int64_t major_frozen_scn, - const common::ObIArray &unequal_tablet_id_array); static int append_tablet_id_array(const common::ObIArray &input_tablet_id_array, const int64_t start_idx, const int64_t end_idx, diff --git a/src/share/ob_tablet_reorganize_history_table_operator.cpp b/src/share/ob_tablet_reorganize_history_table_operator.cpp index 218a5313b..554245b00 100644 --- a/src/share/ob_tablet_reorganize_history_table_operator.cpp +++ b/src/share/ob_tablet_reorganize_history_table_operator.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX SHARE #include "share/ob_tablet_reorganize_history_table_operator.h" #include "share/ob_server_struct.h" -#include "share/location_cache/ob_location_service.h" namespace oceanbase { @@ -27,11 +26,9 @@ namespace share int ObTabletReorganizeHistoryTableOperator::check_tablet_has_reorganized( common::ObMySQLProxy &proxy, const common::ObTabletID &tablet_id, - share::ObLSID &ls_id, bool &reorganized) { int ret = OB_SUCCESS; - ls_id.reset(); reorganized = false; ObSqlString sql; @@ -59,8 +56,6 @@ int ObTabletReorganizeHistoryTableOperator::check_tablet_has_reorganized( LOG_WARN("failed to get next row", K(ret), K(sql)); } } else { - int64_t tmp_ls_id = 0; - ls_id = ObLSID::SYS_LS_ID; reorganized = true; } } @@ -70,17 +65,13 @@ int ObTabletReorganizeHistoryTableOperator::check_tablet_has_reorganized( int ObTabletReorganizeHistoryTableOperator::get_all_split_tablet_pairs( ObMySQLProxy &sql_proxy, - const ObLSID &ls_id, ObIArray &tablet_pairs) { int ret = OB_SUCCESS; common::ObSqlString sql_string; // clear tablet pairs array tablet_pairs.reuse(); - if (false || !ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else { + { SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *result = NULL; @@ -121,7 +112,6 @@ int ObTabletReorganizeHistoryTableOperator::get_all_split_tablet_pairs( int ObTabletReorganizeHistoryTableOperator::get_split_tablet_pairs_by_src( ObMySQLProxy &sql_proxy, - const ObLSID &ls_id, const common::ObTabletID &tablet_id, ObIArray &tablet_pairs) { @@ -129,10 +119,7 @@ int ObTabletReorganizeHistoryTableOperator::get_split_tablet_pairs_by_src( common::ObSqlString sql_string; // clear tablet pairs array tablet_pairs.reuse(); - if (false || !ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else { + { SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *result = NULL; @@ -224,8 +211,7 @@ int ObTabletReorganizeHistoryTableOperator::insert_( LOG_WARN("invalid argument", K(ret), K(dest_tablet_ids)); } else { common::ObSqlString insert_sql; - ObTabletReorganizeRecord complete_record(incomplete_record.ls_id_, - incomplete_record.src_tablet_id_, + ObTabletReorganizeRecord complete_record(incomplete_record.src_tablet_id_, incomplete_record.dest_tablet_id_, incomplete_record.type_, incomplete_record.create_time_, @@ -275,7 +261,7 @@ int ObTabletReorganizeHistoryTableOperator::batch_insert( if (OB_FAIL(trans.start(&sql_proxy))) { LOG_WARN("failed to start trans", K(ret)); } else { - if (OB_FAIL(inner_batch_insert_(trans, split_arg.src_ls_id_, split_arg.src_tablet_id_, split_arg.dest_tablet_ids_, start_time, finish_time))) { + if (OB_FAIL(inner_batch_insert_(trans, split_arg.src_tablet_id_, split_arg.dest_tablet_ids_, start_time, finish_time))) { LOG_WARN("failed to inner batch insert", K(ret)); } else if (split_arg.src_local_index_tablet_ids_.count() != split_arg.dest_local_index_tablet_ids_.count()) { ret = OB_ERR_UNEXPECTED; @@ -284,7 +270,7 @@ int ObTabletReorganizeHistoryTableOperator::batch_insert( ARRAY_FOREACH_X(split_arg.dest_local_index_tablet_ids_, idx, cnt, OB_SUCC(ret)) { const ObTabletID &src_tablet_id = split_arg.src_local_index_tablet_ids_.at(idx); const ObSArray &dest_tablet_ids = split_arg.dest_local_index_tablet_ids_.at(idx); - if (OB_FAIL(inner_batch_insert_(trans, split_arg.src_ls_id_, src_tablet_id, dest_tablet_ids, start_time, finish_time))) { + if (OB_FAIL(inner_batch_insert_(trans, src_tablet_id, dest_tablet_ids, start_time, finish_time))) { LOG_WARN("failed to inner batch insert", K(ret)); } } @@ -297,7 +283,7 @@ int ObTabletReorganizeHistoryTableOperator::batch_insert( ARRAY_FOREACH_X(split_arg.dest_lob_tablet_ids_, idx, cnt, OB_SUCC(ret)) { const ObTabletID &src_tablet_id = split_arg.src_lob_tablet_ids_.at(idx); const ObSArray &dest_tablet_ids = split_arg.dest_lob_tablet_ids_.at(idx); - if (OB_FAIL(inner_batch_insert_(trans, split_arg.src_ls_id_, src_tablet_id, dest_tablet_ids, start_time, finish_time))) { + if (OB_FAIL(inner_batch_insert_(trans, src_tablet_id, dest_tablet_ids, start_time, finish_time))) { LOG_WARN("failed to inner batch insert", K(ret)); } } @@ -314,7 +300,6 @@ int ObTabletReorganizeHistoryTableOperator::batch_insert( int ObTabletReorganizeHistoryTableOperator::inner_batch_insert_( ObISQLClient &sql_proxy, - const share::ObLSID src_ls_id, const ObTabletID &tablet_id, const ObSArray &dest_tablet_ids, const int64 start_time, @@ -322,28 +307,14 @@ int ObTabletReorganizeHistoryTableOperator::inner_batch_insert_( { int ret = OB_SUCCESS; ObArray records; - share::ObLSID ls_id = src_ls_id; - ObAddr leader_addr; - ObLocationService *location_service = nullptr; - const int64_t rpc_timeout = ObDDLUtil::get_default_ddl_rpc_timeout(); - // it's guranteed that all dest_tablets_ids are located at the same server with src - // so all the records can share one single record instance - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret)); - } else if (!ls_id.is_valid() && OB_FAIL(ObDDLUtil::get_tablet_leader_addr(location_service, tablet_id, rpc_timeout, ls_id, leader_addr))) { - LOG_WARN("get tablet leader addr failed", K(ret)); - } ObTabletID invalid_des_tablet_id; - ObTabletReorganizeRecord incomplete_record = ObTabletReorganizeRecord(ls_id, - tablet_id, + ObTabletReorganizeRecord incomplete_record = ObTabletReorganizeRecord(tablet_id, invalid_des_tablet_id, ObTabletReorganizeType::SPLIT, start_time/*create_time*/, finish_time/*finish_time*/); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(insert_(sql_proxy, incomplete_record, dest_tablet_ids))) { + if (OB_FAIL(insert_(sql_proxy, incomplete_record, dest_tablet_ids))) { LOG_WARN("failed to insert record", K(ret), K(records)); } return ret; diff --git a/src/share/ob_tablet_reorganize_history_table_operator.h b/src/share/ob_tablet_reorganize_history_table_operator.h index ddb8ec90a..accc19664 100644 --- a/src/share/ob_tablet_reorganize_history_table_operator.h +++ b/src/share/ob_tablet_reorganize_history_table_operator.h @@ -14,7 +14,6 @@ * limitations under the License. */ -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" #include "common/mysqlclient/ob_mysql_proxy.h" @@ -39,14 +38,12 @@ class ObTabletReorganizeRecord final public: ObTabletReorganizeRecord() = default; ObTabletReorganizeRecord( - const ObLSID &ls_id, const ObTabletID &src_tablet_id, const ObTabletID &dest_tablet_id, const ObTabletReorganizeType type, const uint64_t create_time, const uint64_t finish_time) - : ls_id_(ls_id), - src_tablet_id_(src_tablet_id), + : src_tablet_id_(src_tablet_id), dest_tablet_id_(dest_tablet_id), type_(type), create_time_(create_time), @@ -54,16 +51,14 @@ class ObTabletReorganizeRecord final {} bool is_valid() const { - return ls_id_.is_valid() && - src_tablet_id_.is_valid() && + return src_tablet_id_.is_valid() && dest_tablet_id_.is_valid() && type_ > INVALID_REORGANIZE_TYPE && type_ < MAX_REORGANIZE_TYPE; } - TO_STRING_KV(K(ls_id_), K(src_tablet_id_), K(dest_tablet_id_), + TO_STRING_KV(K(src_tablet_id_), K(dest_tablet_id_), K(type_), K(create_time_), K(finish_time_)); void reset() { - ls_id_.reset(); src_tablet_id_.reset(); dest_tablet_id_.reset(); type_ = INVALID_REORGANIZE_TYPE; @@ -71,7 +66,6 @@ class ObTabletReorganizeRecord final finish_time_ = 0; } public: - ObLSID ls_id_; ObTabletID src_tablet_id_; ObTabletID dest_tablet_id_; ObTabletReorganizeType type_; @@ -104,15 +98,12 @@ class ObTabletReorganizeHistoryTableOperator final static int check_tablet_has_reorganized( common::ObMySQLProxy &sql_proxy, const common::ObTabletID &tablet_id, - share::ObLSID &ls_id, bool &reorganized); static int get_all_split_tablet_pairs( ObMySQLProxy &sql_proxy, - const ObLSID &ls_id, ObIArray &tablet_pairs); static int get_split_tablet_pairs_by_src( ObMySQLProxy &sql_proxy, - const ObLSID &ls_id, const common::ObTabletID &tablet_id, ObIArray &tablet_pairs); static int get_split_tablet_pairs_by_dest( @@ -132,7 +123,6 @@ class ObTabletReorganizeHistoryTableOperator final private: static int inner_batch_insert_( ObISQLClient &sql_proxy, - const share::ObLSID src_ls_id, const ObTabletID &tablet_id, const ObSArray &dest_tablet_ids, const int64 start_time, diff --git a/src/share/ob_tablet_replica_checksum_operator.cpp b/src/share/ob_tablet_replica_checksum_operator.cpp index 1d98fb25a..553d27e80 100644 --- a/src/share/ob_tablet_replica_checksum_operator.cpp +++ b/src/share/ob_tablet_replica_checksum_operator.cpp @@ -417,8 +417,7 @@ int ObTabletReplicaReportColumnMeta::get_serialize_str( /****************************** ObTabletReplicaChecksumItem ******************************/ ObTabletReplicaChecksumItem::ObTabletReplicaChecksumItem() - : ls_id_(), - tablet_id_(), + : tablet_id_(), server_(), row_count_(0), compaction_scn_(), @@ -429,7 +428,6 @@ ObTabletReplicaChecksumItem::ObTabletReplicaChecksumItem() void ObTabletReplicaChecksumItem::reset() { - ls_id_.reset(); tablet_id_.reset(); server_.reset(); row_count_ = 0; @@ -441,8 +439,7 @@ void ObTabletReplicaChecksumItem::reset() bool ObTabletReplicaChecksumItem::is_key_valid() const { - return ls_id_.is_valid_with_tenant() - && tablet_id_.is_valid_with_tenant() + return tablet_id_.is_valid_with_tenant() && server_.is_valid(); } @@ -457,8 +454,6 @@ bool ObTabletReplicaChecksumItem::is_same_tablet(const ObTabletReplicaChecksumIt { return is_key_valid() && other.is_key_valid() - && true - && ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_; } @@ -496,7 +491,6 @@ int ObTabletReplicaChecksumItem::assign(const ObTabletReplicaChecksumItem &other LOG_WARN("fail to assign column meta", KR(ret), K(other)); } else { tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; server_ = other.server_; row_count_ = other.row_count_; compaction_scn_ = other.compaction_scn_; @@ -615,16 +609,16 @@ int ObTabletReplicaChecksumOperator::batch_update_with_trans( int ObTabletReplicaChecksumOperator::batch_remove_with_trans( ObSQLiteConnection *conn, - const common::ObIArray &tablet_replicas) + const common::ObIArray &tablet_meta_rows) { int ret = OB_SUCCESS; - const int64_t replicas_count = tablet_replicas.count(); + const int64_t tablet_meta_row_count = tablet_meta_rows.count(); if (OB_ISNULL(conn)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid connection", K(ret)); - } else if (OB_UNLIKELY(false || replicas_count <= 0)) { + } else if (OB_UNLIKELY(tablet_meta_row_count <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), "tablet_replica cnt", replicas_count); + LOG_WARN("invalid argument", KR(ret), K(tablet_meta_row_count)); } else if (!storage_.is_inited()) { ret = OB_NOT_INIT; LOG_WARN("storage not initialized", K(ret)); @@ -637,8 +631,8 @@ int ObTabletReplicaChecksumOperator::batch_remove_with_trans( if (OB_FAIL(conn->prepare_execute(delete_sql, stmt))) { LOG_WARN("failed to prepare execute", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < replicas_count; ++i) { - const ObTabletReplica &replica = tablet_replicas.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_meta_row_count; ++i) { + const ObTabletReplica &replica = tablet_meta_rows.at(i); if (replica.primary_keys_are_valid()) { auto binder = [&](ObSQLiteBinder &b) -> int { b.bind_int64(replica.get_tablet_id().id()); @@ -684,32 +678,32 @@ int ObTabletReplicaChecksumOperator::remove_residual_checksum( return ret; } -int ObTabletReplicaChecksumOperator::get_tablets_replica_checksum(const ObIArray &pairs, +int ObTabletReplicaChecksumOperator::get_tablets_replica_checksum( + const ObIArray &tablet_check_infos, ObReplicaCkmArray &tablet_replica_checksum_items) { int ret = OB_SUCCESS; - const int64_t pairs_cnt = pairs.count(); - if (OB_UNLIKELY(!true || pairs_cnt <= 0)) { + const int64_t tablet_check_info_cnt = tablet_check_infos.count(); + if (OB_UNLIKELY(tablet_check_info_cnt <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", KR(ret), K(pairs)); + LOG_WARN("invalid arguments", KR(ret), K(tablet_check_infos)); } else if (!storage_.is_inited()) { ret = OB_NOT_INIT; LOG_WARN("storage not initialized", K(ret)); } else { - // Convert ObTabletCheckInfo to ObTabletLSPair - ObSEArray tablet_ls_pairs; - for (int64_t i = 0; OB_SUCC(ret) && i < pairs_cnt; ++i) { - const compaction::ObTabletCheckInfo &check_info = pairs.at(i); - if (OB_FAIL(tablet_ls_pairs.push_back(ObTabletLSPair(check_info.get_tablet_id(), check_info.get_ls_id())))) { - LOG_WARN("failed to push back tablet ls pair", K(ret)); + ObSEArray tablet_ids; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_check_info_cnt; ++i) { + const compaction::ObTabletCheckInfo &check_info = tablet_check_infos.at(i); + if (OB_FAIL(tablet_ids.push_back(check_info.get_tablet_id()))) { + LOG_WARN("failed to push back tablet id", K(ret), K(check_info)); } } if (OB_SUCC(ret)) { - ret = storage_.batch_get(tablet_ls_pairs, SCN(), tablet_replica_checksum_items, false); + ret = storage_.batch_get(tablet_ids, SCN(), tablet_replica_checksum_items, false); if (OB_FAIL(ret)) { LOG_WARN("failed to batch get from storage", K(ret)); } else { - LOG_TRACE("success to get tablet replica checksum items", KR(ret), K(pairs_cnt)); + LOG_TRACE("success to get tablet replica checksum items", KR(ret), K(tablet_check_info_cnt)); } } } @@ -717,7 +711,7 @@ int ObTabletReplicaChecksumOperator::get_tablets_replica_checksum(const ObIArray } int ObTabletReplicaChecksumOperator::batch_get( - const ObIArray &pairs, + const ObIArray &tablet_ids, const SCN &compaction_scn, ObISQLClient &sql_proxy, ObReplicaCkmArray &items, @@ -726,15 +720,15 @@ int ObTabletReplicaChecksumOperator::batch_get( { int ret = OB_SUCCESS; items.reset(); - const int64_t pairs_cnt = pairs.count(); - if (OB_UNLIKELY(pairs_cnt < 1 || false || group_id < 0)) { + const int64_t tablet_cnt = tablet_ids.count(); + if (OB_UNLIKELY(tablet_cnt < 1 || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(pairs_cnt), K(group_id)); + LOG_WARN("invalid argument", KR(ret), K(tablet_cnt), K(group_id)); } else if (!storage_.is_inited()) { ret = OB_NOT_INIT; LOG_WARN("storage not initialized", K(ret)); } else { - ret = storage_.batch_get(pairs, compaction_scn, items, include_larger_than); + ret = storage_.batch_get(tablet_ids, compaction_scn, items, include_larger_than); if (OB_FAIL(ret)) { LOG_WARN("failed to batch get from storage", K(ret)); } @@ -839,7 +833,6 @@ int ObTabletReplicaChecksumOperator::construct_tablet_replica_checksum_item_( LOG_WARN("fail to convert val to SCN", KR(ret), K(compaction_scn_val)); } else { item.tablet_id_ = (uint64_t)int_tablet_id; - item.ls_id_ = ObLSID::SYS_LS_ID; item.server_ = GCTX.self_addr(); if (OB_FAIL(item.column_meta_.set_with_str(item.data_checksum_type_, b_column_meta_str))) { LOG_WARN("fail to set column meta", KR(ret), K(compaction_scn_val), K(b_column_meta_str)); @@ -870,19 +863,19 @@ int ObTabletReplicaChecksumOperator::construct_tablet_replica_checksum_item_( int ObTabletReplicaChecksumOperator::get_tablet_replica_checksum_items(ObMySQLProxy &sql_proxy, const SCN &compaction_scn, - const ObIArray &tablet_pairs, + const ObIArray &tablet_ids, ObReplicaCkmArray &items) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!true)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret)); - } else if (OB_FAIL(batch_get(tablet_pairs, compaction_scn, + } else if (OB_FAIL(batch_get(tablet_ids, compaction_scn, sql_proxy, items, false/*include_larger_than*/, 0))) { LOG_WARN("fail to batch get tablet checksum item", KR(ret), K(compaction_scn), - "pairs_count", tablet_pairs.count()); - } else if (items.get_tablet_cnt() < tablet_pairs.count()) { + "tablet_count", tablet_ids.count()); + } else if (items.get_tablet_cnt() < tablet_ids.count()) { ret = OB_ITEM_NOT_MATCH; LOG_WARN("fail to get tablet replica checksum items", KR(ret), K(compaction_scn), K(items)); @@ -1021,12 +1014,6 @@ int ObTabletReplicaChecksumOperator::multi_get( ret = OB_NOT_INIT; LOG_WARN("storage not initialized", K(ret)); } else { - // Convert tablet_ids to tablet-ls pairs (we need ls_id, but multi_get only provides tablet_id) - // For now, we'll query all replicas for these tablets and filter by compaction_scn - ObSEArray tablet_ls_pairs; - // Since we don't have ls_id, we'll need to query all and filter - // For efficiency, we'll use range_get with a large range and filter - // But a better approach is to query by tablet_id list const char *select_sql = "SELECT tablet_id, compaction_scn, " " row_count, data_checksum, column_checksums, b_column_checksums, " @@ -1071,7 +1058,6 @@ int ObTabletReplicaChecksumOperator::multi_get( item.tablet_id_ = ObTabletID(tablet_id_val); - item.ls_id_ = ObLSID::SYS_LS_ID; item.server_ = GCTX.self_addr(); item.compaction_scn_.convert_for_inner_table_field(compaction_scn_val); item.row_count_ = row_count; @@ -1084,7 +1070,7 @@ int ObTabletReplicaChecksumOperator::multi_get( int tmp_ret = item.column_meta_.set_with_str(item.data_checksum_type_, b_column_checksums_obstr); if (OB_SUCCESS != tmp_ret) { LOG_WARN("failed to set column meta with b_column_checksums blob, skip invalid data", - K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_), K(item.ls_id_)); + K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_)); item.column_meta_.reset(); } } diff --git a/src/share/ob_tablet_replica_checksum_operator.h b/src/share/ob_tablet_replica_checksum_operator.h index c0c20b529..2908c8788 100644 --- a/src/share/ob_tablet_replica_checksum_operator.h +++ b/src/share/ob_tablet_replica_checksum_operator.h @@ -22,7 +22,6 @@ #include "lib/hash/ob_hashmap.h" #include "lib/string/ob_sql_string.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "share/schema/ob_table_schema.h" #include "share/tablet/ob_tablet_info.h" #include "share/ob_column_checksum_error_operator.h" @@ -117,11 +116,10 @@ struct ObTabletReplicaChecksumItem void set_data_checksum_type(); common::ObTabletID get_tablet_id() const { return tablet_id_; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(server), K_(row_count), + TO_STRING_KV(K_(tablet_id), K_(server), K_(row_count), K_(compaction_scn), K_(data_checksum), K_(column_meta), K_(data_checksum_type)); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; common::ObAddr server_; int64_t row_count_; @@ -143,7 +141,7 @@ class ObTabletReplicaChecksumOperator // Default: checksum_items' compaction_scn = @compaction_scn // If include_larger_than = true: checksum_items' compaction_scn >= @compaction_scn static int batch_get( - const common::ObIArray &pairs, + const common::ObIArray &tablet_ids, const SCN &compaction_scn, common::ObISQLClient &sql_proxy, ObReplicaCkmArray &items, @@ -156,13 +154,13 @@ class ObTabletReplicaChecksumOperator // Remove checksum items within a SQLite transaction static int batch_remove_with_trans( share::ObSQLiteConnection *conn, - const common::ObIArray &tablet_replicas); + const common::ObIArray &tablet_meta_rows); static int remove_residual_checksum( common::ObISQLClient &sql_client, const ObAddr &server, const int64_t limit, int64_t &affected_rows); - static int get_tablets_replica_checksum(const ObIArray &pairs, + static int get_tablets_replica_checksum(const ObIArray &tablet_check_infos, ObReplicaCkmArray &tablet_replica_checksum_items); static int get_visible_column_meta( const ObTabletReplicaReportColumnMeta &column_meta, @@ -213,7 +211,7 @@ class ObTabletReplicaChecksumOperator // KV of @column_ckm_map is: static int get_tablet_replica_checksum_items(common::ObMySQLProxy &mysql_proxy, const SCN &compaction_scn, - const common::ObIArray &tablet_pairs, + const common::ObIArray &tablet_ids, ObReplicaCkmArray &items); static int construct_tablet_id_list(const ObIArray &tablet_ids, ObSqlString &sql); private: diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 43d40a289..b440944d2 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -766,7 +766,6 @@ DEF_PARAM(resource_hard_limit, INT, OB_CLUSTER_PARAMETER, "100", "[100, 10000]", "system utilization should not be large than resource_hard_limit", ObParameterAttr(Section::LOAD_BALANCE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); // abandoned in 4.x -//DEF_PARAM(__balance_controller, OB_CLUSTER_PARAMETER, "", // "specifies whether the balance events are turned on or turned off.", // ObParameterAttr(Section::LOAD_BALANCE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(__min_full_resource_pool_memory, INT, OB_CLUSTER_PARAMETER, "1073741824", "[1073741824,)", @@ -824,9 +823,9 @@ DEF_PARAM(merger_check_interval, TIME, OB_CLUSTER_PARAMETER, "10m", "[10s, 60m]" "that checks on the progress of MERGE for each zone. Range: [10s, 60m]", ObParameterAttr(Section::DAILY_MERGE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); //// transaction config -DEF_PARAM(trx_2pc_retry_interval, TIME, OB_CLUSTER_PARAMETER, "100ms", "[1ms, 5000ms]", +DEF_PARAM(trx_commit_retry_interval, TIME, OB_CLUSTER_PARAMETER, "100ms", "[1ms, 5000ms]", "the time interval between the retries in case of failure " - "during a transaction\\'s two-phase commit phase. Range: [1ms,5000ms]", + "during a transaction commit. Range: [1ms,5000ms]", ObParameterAttr(Section::TRANS, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(clog_sync_time_warn_threshold, TIME, OB_CLUSTER_PARAMETER, "100ms", "[1ms, 10000ms]", "the time given to the commit log synchronization between a leader and its followers " @@ -847,9 +846,6 @@ DEF_PARAM(_rpc_checksum, STR_WITH_CHECKER, OB_CLUSTER_PARAMETER, "Force", common ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE), "Force, Optional, Disable"); -DEF_PARAM(_ob_trans_rpc_timeout, TIME, OB_CLUSTER_PARAMETER, "3s", "[0s, 3600s]", - "transaction rpc timeout(s). Range: [0s, 3600s]", - ObParameterAttr(Section::TRANS, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(enable_early_lock_release, BOOL, OB_CLUSTER_PARAMETER, "True", "enable early lock release", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); @@ -934,8 +930,6 @@ DEF_PARAM(_auto_broadcast_tablet_location_rate_limit, INT, OB_CLUSTER_PARAMETER, ObParameterAttr(Section::LOCATION_CACHE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); //// cache config -DEF_PARAM(tablet_ls_cache_priority, INT, OB_CLUSTER_PARAMETER, "1000", "[1,)", "tablet ls cache priority. Range:[1, )", - ObParameterAttr(Section::CACHE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(opt_tab_stat_cache_priority, INT, OB_CLUSTER_PARAMETER, "1", "[1,)", "tab stat cache priority. Range:[1, )", ObParameterAttr(Section::CACHE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(index_block_cache_priority, INT, OB_CLUSTER_PARAMETER, "10", "[1,)", "index cache priority. Range:[1, )", @@ -1123,12 +1117,6 @@ DEF_PARAM(_cache_wash_interval, TIME, OB_CLUSTER_PARAMETER, "200ms", "[1ms, 3s]" "specify interval of cache background wash", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_PARAM(_max_ls_cnt_per_server, INT, OB_CLUSTER_PARAMETER, "0", "[0, 1024]", - "specify max ls count of one tenant on one observer." - "WARNING: Modifying this can potentially destabilize the cluster. It is strongly advised to avoid making such changes as they are unlikely to be necessary." - "0: the cluster will adapt the max ls number according to the memory size of tenant itself", - ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); - DEF_PARAM(_io_read_batch_size, CAP, OB_CLUSTER_PARAMETER, "0K", "[0K,16M]", "Maximum batch size in one read io request. Range:[0K,16M]", ObParameterAttr(Section::SSTABLE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(_io_read_redundant_limit_percentage, INT, OB_CLUSTER_PARAMETER, "0", "[0, 99]", @@ -1474,9 +1462,6 @@ DEF_PARAM(_ob_immediate_row_conflict_check, BOOL, OB_CLUSTER_PARAMETER, "False", "When the switch is turned off, " "it will only check whether the final state of a batch of data after the update satisfies the unique constraint.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_PARAM(_enable_insertup_replace_gts_opt, BOOL, OB_CLUSTER_PARAMETER, "True", - "By default, insert/replace ... values statement can use gts optimization", - ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); // Add a config to enable use das if the sql statement has variable assignment DEF_PARAM(_enable_var_assign_use_das, BOOL, OB_CLUSTER_PARAMETER, "False", "enable use das if the sql statement has variable assignment", @@ -1540,7 +1525,7 @@ DEF_PARAM(_display_non_session_cursor, BOOL, OB_CLUSTER_PARAMETER, "True", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_PARAM(dump_data_dictionary_to_log_interval, TIME, OB_CLUSTER_PARAMETER, "24h", "(0s,]", - "data dictionary dump to log(SYS LS) interval" + "data dictionary dump to log interval" "Range: (0s,+∞)", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); @@ -1788,12 +1773,6 @@ DEF_PARAM(sql_protocol_min_tls_version, STR_WITH_CHECKER, OB_CLUSTER_PARAMETER, DEF_PARAM(shared_log_retention, TIME, OB_CLUSTER_PARAMETER, "1d", "[0s,7d]", "Retention time of log files on shared storage", ObParameterAttr(Section::TRANS, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_PARAM(_standby_max_replay_gap_time, TIME, OB_CLUSTER_PARAMETER, "900s", "[10s,)", - "The difference in replayable_scn between log streams on standby tenants is not greater than " - "_standby_max_replay_gap_time, and the gap between sync_scn and replayable_scn of each log stream " - "is kept reasonably small. Range: [10s, )", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); - DEF_PARAM(_enable_optimizer_qualify_filter, BOOL, OB_CLUSTER_PARAMETER, "True", "Enable extracting qualify filters for window function", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/partition_table/ob_partition_location.cpp b/src/share/partition_table/ob_partition_location.cpp deleted file mode 100644 index 40485cf7c..000000000 --- a/src/share/partition_table/ob_partition_location.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE_PT - -#include "ob_partition_location.h" - - -namespace oceanbase -{ -using namespace common; -namespace share -{ - -OB_SERIALIZE_MEMBER(ObPartitionLocation, - table_id_, - partition_id_, - partition_cnt_, - replica_locations_, - renew_time_, - is_mark_fail_); - -OB_SERIALIZE_MEMBER(ObReplicaLocation, - replica_type_, - property_); - -OB_SERIALIZE_MEMBER(ObPartitionReplicaLocation, - table_id_, - partition_id_, - partition_cnt_, - replica_location_, - renew_time_); - -ObReplicaLocation::ObReplicaLocation() -{ - reset(); -} - -void ObReplicaLocation::reset() -{ - replica_type_ = REPLICA_TYPE_INVALID; - property_.reset(); -} - -ObPartitionLocation::ObPartitionLocation() - :replica_locations_(ObModIds::OB_MS_LOCATION_CACHE, OB_MALLOC_NORMAL_BLOCK_SIZE) -{ - reset(); -} - -ObPartitionLocation::ObPartitionLocation(common::ObIAllocator &allocator) - :replica_locations_(common::OB_MALLOC_NORMAL_BLOCK_SIZE, - common::ModulePageAllocator(allocator)) -{ - reset(); -} - -ObPartitionLocation::~ObPartitionLocation() -{ - reset(); -} - -void ObPartitionLocation::reset() -{ - table_id_ = OB_INVALID_ID; - partition_id_ = OB_INVALID_INDEX; - partition_cnt_ = 0; - replica_locations_.reset(); - renew_time_ = 0; - sql_renew_time_ = 0; - is_mark_fail_ = false; -} - -int ObPartitionLocation::assign(const ObPartitionLocation &other) -{ - int ret = OB_SUCCESS; - - if (this != &other) { - table_id_ = other.table_id_; - partition_id_ = other.partition_id_; - partition_cnt_ = other.partition_cnt_; - renew_time_ = other.renew_time_; - sql_renew_time_ = other.sql_renew_time_; - is_mark_fail_ = other.is_mark_fail_; - if (OB_FAIL(replica_locations_.assign(other.replica_locations_))) { - LOG_WARN("Failed to assign replica locations", K(ret)); - } - } - - return ret; -} - - - -bool ObPartitionLocation::is_valid() const -{ - return OB_INVALID_ID != table_id_ && OB_INVALID_INDEX != partition_id_ - && renew_time_ >= 0 && sql_renew_time_ >= 0; -} - - -int ObPartitionLocation::add(const ObReplicaLocation &replica_location) -{ - int ret = OB_SUCCESS; - // Replica location may be added before table_id/partition_id set, - // can not check self validity here. - if (!replica_location.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid replica location", K(replica_location), K(ret)); - } else if (OB_FAIL(add_with_no_check(replica_location))) { - LOG_WARN("fail to do add replica location", K(replica_location), K(ret)); - } - return ret; -} - -int ObPartitionLocation::add_with_no_check(const ObReplicaLocation &replica_location) -{ - int ret = OB_SUCCESS; - int64_t idx = OB_INVALID_INDEX; - if (OB_LIKELY(OB_SUCCESS != (ret = (find(replica_location.get_server(), idx))))) { - if (OB_LIKELY(OB_ENTRY_NOT_EXIST == ret)) { - if (OB_FAIL(replica_locations_.push_back(replica_location))) { - LOG_WARN("push back replica location failed", K(ret)); - } - } else { - LOG_WARN("find server location failed", K(ret), "server", replica_location.get_server()); - } - } else { - ret = OB_ERR_ALREADY_EXISTS; - LOG_WARN("replica location already exist, can not add", K(replica_location), K(ret)); - } - return ret; -} - - - -int ObPartitionLocation::get_strong_leader(ObReplicaLocation &replica_location, int64_t &replica_idx) const -{ - int ret = OB_LOCATION_LEADER_NOT_EXIST; - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), "self", *this); - } else { - ARRAY_FOREACH_X(replica_locations_, i, cnt, OB_LOCATION_LEADER_NOT_EXIST == ret) { - if (replica_locations_.at(i).is_strong_leader()) { - replica_location = replica_locations_.at(i); - replica_idx = i; - ret = OB_SUCCESS; - } - } - } - return ret; -} - -int ObPartitionLocation::get_strong_leader(ObReplicaLocation &replica_location) const -{ - int64_t replica_idx = OB_INVALID_INDEX; - return get_strong_leader(replica_location, replica_idx); -} - - - -int ObPartitionLocation::find(const ObAddr &server, int64_t &idx) const -{ - int ret = OB_ENTRY_NOT_EXIST; - idx = OB_INVALID_INDEX; - // May be used before table_id/partition_id set, can not check self validity here. - // server(ObAddr) is checked by caller, no need check again. - for (int64_t i = 0; OB_ENTRY_NOT_EXIST == ret && i < replica_locations_.count(); ++i) { - if (replica_locations_.at(i).get_server() == server) { - idx = i; - ret = OB_SUCCESS; - } - } - return ret; -} - - - - -ObPartitionReplicaLocation::ObPartitionReplicaLocation() -{ - reset(); -} - - - -void ObPartitionReplicaLocation::reset() -{ - table_id_ = OB_INVALID_ID; - partition_id_ = OB_INVALID_INDEX; - partition_cnt_ = 0; - replica_location_.reset(); - renew_time_ = 0; -} - -int ObPartitionReplicaLocation::assign(const ObPartitionReplicaLocation &other) -{ - int ret = OB_SUCCESS; - table_id_ = other.table_id_; - partition_id_ = other.partition_id_; - partition_cnt_ = other.partition_cnt_; - replica_location_ = other.replica_location_; - renew_time_ = other.renew_time_; - return ret; -} - - - -bool ObPartitionReplicaLocation::is_valid() const -{ - return OB_INVALID_ID != table_id_ && OB_INVALID_INDEX != partition_id_ && renew_time_ >= 0; -} - -bool ObPartitionReplicaLocation::operator==(const ObPartitionReplicaLocation &other) const -{ - bool equal = true; - if (!is_valid() || !other.is_valid()) { - equal = false; - LOG_WARN_RET(common::OB_INVALID_ARGUMENT, "invalid argument", "self", *this, K(other)); - } else if (replica_location_ != other.replica_location_) { - equal = false; - } else { - equal = (table_id_ == other.table_id_) - && (partition_id_ == other.partition_id_) - && (partition_cnt_ == other.partition_cnt_) - && (renew_time_ == other.renew_time_); - } - return equal; -} - -}//end namespace share -}//end namespace oceanbase diff --git a/src/share/partition_table/ob_partition_location.h b/src/share/partition_table/ob_partition_location.h deleted file mode 100644 index 041d48808..000000000 --- a/src/share/partition_table/ob_partition_location.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_PARTITION_TABLE_OB_PARTITION_LOCATION_H_ -#define OCEANBASE_SHARE_PARTITION_TABLE_OB_PARTITION_LOCATION_H_ - -#include "share/ob_define.h" -#include "lib/net/ob_addr.h" -#include "lib/container/ob_se_array.h" -#include "lib/ob_replica_define.h" -#include "common/ob_role.h" -#include "share/ob_server_struct.h" - -namespace oceanbase -{ -namespace sql -{ -class ObOptTabletLoc; -} -namespace share -{ - -struct ObPidAddrPair { -public: - int64_t pid_; - common::ObAddr addr_; - - ObPidAddrPair(int64_t pid, common::ObAddr addr) - : pid_(pid), addr_(addr) - { - } - - bool operator==(const ObPidAddrPair &other) const - { - return pid_ == other.pid_ && addr_ == other.addr_; - } - - TO_STRING_KV(K_(pid), K_(addr)); -}; - -class ObPartitionReplicaLocation; -struct ObReplicaLocation //TODO(xiumin): delete it -{ - OB_UNIS_VERSION(1); -public: - common::ObReplicaType replica_type_; - common::ObReplicaProperty property_; // memstore_percent is used - - ObReplicaLocation(); - void reset(); - const common::ObAddr &get_server() const { return GCTX.self_addr(); } - void set_server(const common::ObAddr &) {} - common::ObRole get_role() const { return LEADER; } - void set_role(common::ObRole) {} - int64_t get_sql_port() const { return GCTX.self_addr().get_port(); } - void set_sql_port(int64_t) {} - inline bool is_valid() const; - inline bool operator==(const ObReplicaLocation &other) const; - inline bool operator!=(const ObReplicaLocation &other) const; - bool is_leader_like() const { return common::is_leader_like(get_role()); } - bool is_leader_by_election() const { return common::is_leader_by_election(get_role()); } - bool is_strong_leader() const { return common::is_strong_leader(get_role()); } - bool is_standby_leader() const { return common::is_standby_leader(get_role()); } - bool is_follower() const { return common::is_follower(get_role()); } - TO_STRING_KV("server", get_server(), "role", get_role(), "sql_port", get_sql_port(), - K_(replica_type), K_(property)); -}; - -inline bool ObReplicaLocation::is_valid() const -{ - return REPLICA_TYPE_INVALID != replica_type_; -} - -bool ObReplicaLocation::operator==(const ObReplicaLocation &other) const -{ - return replica_type_ == other.replica_type_ - && property_ == other.property_; -} - -bool ObReplicaLocation::operator!=(const ObReplicaLocation &other) const -{ - return !(*this == other); -} - -class ObPartitionLocation -{ - OB_UNIS_VERSION(1); - friend class ObPartitionReplicaLocation; - friend class sql::ObOptTabletLoc; -public: - typedef common::ObSEArray ObReplicaLocationArray; - - ObPartitionLocation(); - explicit ObPartitionLocation(common::ObIAllocator &allocator); - virtual ~ObPartitionLocation(); - - void reset(); - int assign(const ObPartitionLocation &partition_location); - - bool is_valid() const; - int add(const ObReplicaLocation &replica_location); - int add_with_no_check(const ObReplicaLocation &replica_location); - // return OB_LOCATION_LEADER_NOT_EXIST for leader not exist. - int get_strong_leader(ObReplicaLocation &replica_location, int64_t &replica_idx) const; - int get_strong_leader(ObReplicaLocation &replica_location) const; - // return OB_LOCATION_LEADER_NOT_EXIST for leader not exist. - - int64_t size() const { return replica_locations_.count(); } - - inline uint64_t get_table_id() const { return table_id_; } - inline void set_table_id(const uint64_t table_id) { table_id_ = table_id; } - - inline int64_t get_partition_id() const { return partition_id_; } - inline void set_partition_id(const int64_t partition_id) { partition_id_ = partition_id; } - - inline int64_t get_partition_cnt() const { return partition_cnt_; } - inline void set_partition_cnt(const int64_t partition_cnt) { partition_cnt_ = partition_cnt; } - - inline int64_t get_renew_time() const { return renew_time_; } - inline void set_renew_time(const int64_t renew_time) { renew_time_ = renew_time; } - - inline int64_t get_sql_renew_time() const { return sql_renew_time_; } - inline void set_sql_renew_time(const int64_t sql_renew_time) { sql_renew_time_ = sql_renew_time; } - - inline const common::ObIArray &get_replica_locations() const { return replica_locations_; } - inline void mark_fail() { is_mark_fail_ = true; } - inline void unmark_fail() { is_mark_fail_ = false; } - inline bool is_mark_fail() const { return is_mark_fail_; } - - TO_STRING_KV(KT_(table_id), K_(partition_id), K_(partition_cnt), - K_(replica_locations), K_(renew_time), K_(sql_renew_time), K_(is_mark_fail)); - -private: - // return OB_ENTRY_NOT_EXIST for not found. - int find(const common::ObAddr &server, int64_t &idx) const; - -private: - uint64_t table_id_; - int64_t partition_id_; - int64_t partition_cnt_; - ObReplicaLocationArray replica_locations_; - int64_t renew_time_; // renew time when location_cache is renewed successfully. - int64_t sql_renew_time_; // renew time when location_cache is renewed successfully by SQL. - bool is_mark_fail_; -}; - -class ObPartitionReplicaLocation final -{ - OB_UNIS_VERSION(1); - friend class ObPartitionLocation; -public: -public: - ObPartitionReplicaLocation(); - - void reset(); - int assign(const ObPartitionReplicaLocation &partition_location); - - bool is_valid() const; - bool operator==(const ObPartitionReplicaLocation &other) const; - - inline uint64_t get_table_id() const { return table_id_; } - inline void set_table_id(const uint64_t table_id) { table_id_ = table_id; } - - inline int64_t get_partition_id() const { return partition_id_; } - inline void set_partition_id(const int64_t partition_id) { partition_id_ = partition_id; } - - inline int64_t get_partition_cnt() const { return partition_cnt_; } - inline void set_partition_cnt(const int64_t partition_cnt) { partition_cnt_ = partition_cnt; } - - inline int64_t get_renew_time() const { return renew_time_; } - inline void set_renew_time(const int64_t renew_time) { renew_time_ = renew_time; } - - inline const ObReplicaLocation &get_replica_location() const { return replica_location_; } - inline void set_replica_location(const ObReplicaLocation &replica_location) { replica_location_ = replica_location; } - - TO_STRING_KV(KT_(table_id), K_(partition_id), K_(partition_cnt), - K_(replica_location), K_(renew_time)); - -private: - uint64_t table_id_; - int64_t partition_id_; - int64_t partition_cnt_; - ObReplicaLocation replica_location_; - int64_t renew_time_; -}; - -}//end namespace share -}//end namespace oceanbase - -#endif diff --git a/src/share/rc/ob_module_provider.h b/src/share/rc/ob_module_provider.h index e155d73ca..884da467a 100644 --- a/src/share/rc/ob_module_provider.h +++ b/src/share/rc/ob_module_provider.h @@ -43,7 +43,7 @@ class ObIModuleProvider virtual oceanbase::sql::ObTenantSQLSessionMgr * tenant_sql_session_mgr() { return nullptr; } virtual storage::ObTenantMetaMemMgr * tenant_meta_mem_mgr() { return nullptr; } virtual storage::ObTenantFTPluginMgr * tenant_ft_plugin_mgr() { return nullptr; } - virtual ObPartTransCtxObjPool * part_trans_ctx_obj_pool() { return nullptr; } + virtual ObTxCtxObjPool * part_trans_ctx_obj_pool() { return nullptr; } virtual ObTableScanIteratorObjPool * table_scan_iterator_obj_pool() { return nullptr; } virtual common::ObTenantIOManager * tenant_io_manager() { return nullptr; } virtual storage::mds::ObTenantMdsService * tenant_mds_service() { return nullptr; } @@ -88,7 +88,6 @@ class ObIModuleProvider virtual sql::dtl::ObDTLIntermResultManager * dtl_interm_result_manager() { return nullptr; } virtual sql::ObPlanMonitorNodeList * plan_monitor_node_list() { return nullptr; } virtual sql::ObDataAccessService * data_access_service() { return nullptr; } - virtual sql::ObDASIDService * dasid_service() { return nullptr; } virtual share::schema::ObTenantSchemaService * tenant_schema_service() { return nullptr; } virtual storage::ObTenantFreezer * tenant_freezer() { return nullptr; } virtual storage::checkpoint::ObCheckPointService * check_point_service() { return nullptr; } @@ -117,7 +116,6 @@ class ObIModuleProvider virtual table::ObTableObjectPoolMgr * table_object_pool_mgr() { return nullptr; } virtual share::ObIndexUsageInfoMgr * index_usage_info_mgr() { return nullptr; } virtual storage::ObTabletMemtableMgrPool * tablet_memtable_mgr_pool() { return nullptr; } - virtual rootserver::ObMViewMaintenanceService * m_view_maintenance_service() { return nullptr; } virtual share::ObResourceLimitCalculator * resource_limit_calculator() { return nullptr; } virtual storage::ObGlobalIteratorPool * global_iterator_pool() { return nullptr; } virtual common::ObRbMemMgr * rb_mem_mgr() { return nullptr; } @@ -153,7 +151,7 @@ template <> inline transaction::tablelock::ObTableLockService *mtl_checkedtable_lock_service(); } template common::ObServerObjectPool *mtl_obj_pool(); -template <> inline common::ObServerObjectPool *mtl_obj_pool() +template <> inline common::ObServerObjectPool *mtl_obj_pool() { return g_mp->part_trans_ctx_obj_pool(); } template <> inline common::ObServerObjectPool *mtl_obj_pool() { return g_mp->table_scan_iterator_obj_pool(); } diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index f98a9da34..49b75545d 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -56,7 +56,6 @@ namespace sql { class ObPlanMonitorNodeList; class ObPlanBaselineMgr; class ObDataAccessService; - class ObDASIDService; class ObPlanCache; class ObPsCache; class ObSQLCCLRuleManager; @@ -104,7 +103,7 @@ namespace transaction { class ObTransIDService; class ObUniqueIDService; class ObTxLoopWorker; - class ObPartTransCtx; + class ObTxCtx; namespace tablelock { class ObTableLockService; } @@ -139,8 +138,6 @@ namespace compaction class ObTenantMediumChecker; class ObTenantTabletScheduler; class ObTenantCompactionObjMgr; - class ObTenantLSMergeScheduler; - class ObTenantLSMergeChecker; } namespace memtable { @@ -151,7 +148,6 @@ namespace rootserver class ObPrimaryMajorFreezeService; class ObRestoreMajorFreezeService; class ObDBMSSchedService; - class ObMViewMaintenanceService; class ObDDLScheduler; class ObDDLServiceLauncher; class ObSysTenantLoadSysPackageService; @@ -218,14 +214,12 @@ namespace detector #define SSMicroCachePrewarmService #define SSMicroCache #define TenantCompactionObjMgr -#define TenantLSMergeScheduler -#define TenantLSMergeChecker #define PublicBlockGCService #define StorageCachePolicyService // List the types of tenant-local variables that need to be added here, the tenant will create an instance for each type. // The lifecycle of each instance is driven by ObServer's explicit obs_* routines. // Use the MTL interface to obtain an instance. -using ObPartTransCtxObjPool = common::ObServerObjectPool; +using ObTxCtxObjPool = common::ObServerObjectPool; using ObTableScanIteratorObjPool = common::ObServerObjectPool; #define MTL_MEMBERS \ MTL_LIST( \ @@ -233,7 +227,7 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPool &pairs, + const ObIArray &tablet_ids, const SCN &compaction_scn, ObReplicaCkmArray &items, const bool include_larger_than) @@ -279,7 +279,7 @@ int ObTabletReplicaChecksumTableStorage::batch_get( if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (pairs.empty()) { + } else if (tablet_ids.empty()) { // do nothing } else { // Build SQL with IN clause @@ -292,12 +292,12 @@ int ObTabletReplicaChecksumTableStorage::batch_get( "WHERE tablet_id IN ("))) { LOG_WARN("failed to append sql", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < pairs.count(); ++i) { - const ObTabletLSPair &pair = pairs.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); if (OB_FAIL(sql.append_fmt( "%s %ld", i == 0 ? "" : ",", - pair.get_tablet_id().id()))) { + tablet_id.id()))) { LOG_WARN("failed to append sql", K(ret)); } } @@ -328,7 +328,6 @@ int ObTabletReplicaChecksumTableStorage::batch_get( UNUSED(column_checksums_str); item.tablet_id_ = ObTabletID(tablet_id_val); - item.ls_id_ = ObLSID::SYS_LS_ID; item.server_ = GCTX.self_addr(); item.compaction_scn_.convert_for_inner_table_field(compaction_scn_val); item.row_count_ = row_count; @@ -342,7 +341,7 @@ int ObTabletReplicaChecksumTableStorage::batch_get( int tmp_ret = item.column_meta_.set_with_str(item.data_checksum_type_, b_column_checksums_obstr); if (OB_SUCCESS != tmp_ret) { LOG_WARN("failed to set column meta with b_column_checksums blob, skip invalid data", - K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_), K(item.ls_id_)); + K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_)); item.column_meta_.reset(); } } @@ -417,7 +416,6 @@ int ObTabletReplicaChecksumTableStorage::range_get(const common::ObTabletID &sta UNUSED(column_checksums_str); item.tablet_id_ = ObTabletID(tablet_id_val); - item.ls_id_ = ObLSID::SYS_LS_ID; item.server_ = GCTX.self_addr(); item.compaction_scn_.convert_for_inner_table_field(compaction_scn_val); item.row_count_ = row_count; @@ -431,7 +429,7 @@ int ObTabletReplicaChecksumTableStorage::range_get(const common::ObTabletID &sta int tmp_ret = item.column_meta_.set_with_str(item.data_checksum_type_, b_column_checksums_obstr); if (OB_SUCCESS != tmp_ret) { LOG_WARN("failed to set column meta with b_column_checksums blob, skip invalid data", - K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_), K(item.ls_id_)); + K(tmp_ret), K(b_column_checksums_obstr), K(item.tablet_id_)); item.column_meta_.reset(); } } @@ -506,7 +504,6 @@ int ObTabletReplicaChecksumTableStorage::get_min_compaction_scn(uint64_t &min_co } int ObTabletReplicaChecksumTableStorage::get_max_row_count(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &max_row_count) { int ret = OB_SUCCESS; diff --git a/src/share/storage/ob_tablet_replica_checksum_table_storage.h b/src/share/storage/ob_tablet_replica_checksum_table_storage.h index a6bbca446..938508040 100644 --- a/src/share/storage/ob_tablet_replica_checksum_table_storage.h +++ b/src/share/storage/ob_tablet_replica_checksum_table_storage.h @@ -34,7 +34,6 @@ struct ObTabletReplicaChecksumEntry { int64_t tablet_id_; common::ObAddr svr_addr_; - int64_t ls_id_; uint64_t compaction_scn_; int64_t row_count_; int64_t data_checksum_; @@ -45,7 +44,6 @@ struct ObTabletReplicaChecksumEntry ObTabletReplicaChecksumEntry() : tablet_id_(0), svr_addr_(), - ls_id_(0), compaction_scn_(0), row_count_(0), data_checksum_(0), @@ -58,7 +56,6 @@ struct ObTabletReplicaChecksumEntry { tablet_id_ = 0; svr_addr_.reset(); - ls_id_ = 0; compaction_scn_ = 0; row_count_ = 0; data_checksum_ = 0; @@ -91,9 +88,9 @@ class ObTabletReplicaChecksumTableStorage const int64_t limit, int64_t &affected_rows); - // Get checksum items for tablet-ls pairs + // Get checksum items for tablets int batch_get( - const ObIArray &pairs, + const ObIArray &tablet_ids, const SCN &compaction_scn, ObReplicaCkmArray &items, const bool include_larger_than = false); @@ -109,7 +106,6 @@ class ObTabletReplicaChecksumTableStorage // Get max row_count for a tablet-ls pair int get_max_row_count(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &max_row_count); // Batch check tablet checksum for multiple tablets diff --git a/src/share/table/ob_table_load_define.cpp b/src/share/table/ob_table_load_define.cpp index b8dea740d..e986b7cf4 100644 --- a/src/share/table/ob_table_load_define.cpp +++ b/src/share/table/ob_table_load_define.cpp @@ -42,8 +42,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadPartitionId, partition_id_, tablet_id_); -OB_SERIALIZE_MEMBER(ObTableLoadLSIdAndPartitionId, - ls_id_, +OB_SERIALIZE_MEMBER(ObTableLoadTabletId, part_tablet_id_); OB_SERIALIZE_MEMBER(ObTableLoadResultInfo, diff --git a/src/share/table/ob_table_load_define.h b/src/share/table/ob_table_load_define.h index 41c70fa9e..6806122ed 100644 --- a/src/share/table/ob_table_load_define.h +++ b/src/share/table/ob_table_load_define.h @@ -19,7 +19,6 @@ #include "common/ob_tablet_id.h" #include "lib/utility/ob_print_utils.h" #include "lib/utility/ob_unify_serialize.h" -#include "share/ob_ls_id.h" #include "lib/oblog/ob_log_module.h" #include "lib/utility/ob_print_utils.h" #include "share/rc/ob_tenant_base.h" @@ -117,56 +116,29 @@ struct ObTableLoadPartitionId TO_STRING_KV(K_(partition_id), K_(tablet_id)); }; -struct ObTableLoadLSIdAndPartitionId +struct ObTableLoadTabletId { OB_UNIS_VERSION(1); public: - ObTableLoadLSIdAndPartitionId() {} - ObTableLoadLSIdAndPartitionId(const share::ObLSID &ls_id, const ObTableLoadPartitionId &partition_id) - : ls_id_(ls_id), part_tablet_id_(partition_id) + ObTableLoadTabletId() {} + explicit ObTableLoadTabletId(const ObTableLoadPartitionId &partition_id) + : part_tablet_id_(partition_id) { } - share::ObLSID ls_id_; ObTableLoadPartitionId part_tablet_id_; - ObTableLoadLSIdAndPartitionId &operator=(const ObTableLoadLSIdAndPartitionId &other) + ObTableLoadTabletId &operator=(const ObTableLoadTabletId &other) { - ls_id_ = other.ls_id_; part_tablet_id_ = other.part_tablet_id_; return *this; } bool is_valid() const { - return ls_id_.is_valid() && part_tablet_id_.is_valid(); + return part_tablet_id_.is_valid(); } - TO_STRING_KV(K_(ls_id), K_(part_tablet_id)); -}; - -struct ObTableLoadLSTabletID -{ - OB_UNIS_VERSION(1); -public: - ObTableLoadLSTabletID() {} - ObTableLoadLSTabletID(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) - : ls_id_(ls_id), tablet_id_(tablet_id) - { - } - ObTableLoadLSTabletID &operator=(const ObTableLoadLSTabletID &other) - { - ls_id_ = other.ls_id_; - tablet_id_ = other.tablet_id_; - return *this; - } - bool is_valid() const - { - return ls_id_.is_valid() && tablet_id_.is_valid(); - } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); -public: - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; + TO_STRING_KV(K_(part_tablet_id)); }; enum class ObTableLoadStatusType : int64_t diff --git a/src/share/table/ob_ttl_util.cpp b/src/share/table/ob_ttl_util.cpp index ed97af185..7d3b7c79e 100644 --- a/src/share/table/ob_ttl_util.cpp +++ b/src/share/table/ob_ttl_util.cpp @@ -19,7 +19,6 @@ #include "share/ob_server_struct.h" #include "share/table/ob_ttl_util.h" #include "share/ob_tenant_timezone_mgr.h" -#include "share/location_cache/ob_location_service.h" #include "lib/json/ob_json.h" #include "share/schema/ob_dependency_info.h" #include "share/ob_ex_rpc.h" @@ -892,65 +891,8 @@ int ObTTLUtil::dispatch_one_tenant_ttl(obcall::ObTTLRequestArg::TTLRequestType t ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ttl_info)); } else { - const int64_t launch_start_time = ObTimeUtility::current_time(); - ObAddr leader; - obcall::ObTTLRequestArg req; - obcall::ObTTLResponseArg resp; - - - - req.cmd_code_ = type; - req.trigger_type_ = TRIGGER_TYPE::USER_TRIGGER; - if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid GCTX", KR(ret)); - } else { - const int64_t MAX_RETRY_COUNT = 5; - bool ttl_done = false; - static const int64_t MAX_PROCESS_TIME_US = 10 * 1000 * 1000L; - for (int64_t i = 0; OB_SUCC(ret) && (!ttl_done) && (i < MAX_RETRY_COUNT); ++i) { - if (OB_FAIL(GCTX.location_service_->get_leader_with_retry_until_timeout(GCONF.cluster_id, share::SYS_LS, leader))) { - LOG_WARN("fail to get ls locaiton leader", KR(ret), K(1UL)); - } else if (OB_FAIL(ex_rpc::sync_call([&]() -> int { - // table_api removed from build: ObTTLService is gone, TTL task launch unsupported - int ret = OB_NOT_SUPPORTED; - resp.err_code_ = ret; - return ret; - }))) { - LOG_WARN("tenant ttl rpc failed", KR(ret), K(1UL), K(leader), K(ttl_info)); - } else { - ret = resp.err_code_; - } - - if (OB_FAIL(ret)) { - if (OB_LEADER_NOT_EXIST == ret || OB_EAGAIN == ret) { - const int64_t RESERVED_TIME_US = 600 * 1000; // 600 ms - const int64_t timeout_remain_us = THIS_WORKER.get_timeout_remain(); - const int64_t idle_time_us = 200 * 1000 * (i + 1); - if (timeout_remain_us - idle_time_us > RESERVED_TIME_US) { - LOG_WARN("leader may switch or ddl confilict, will retry", KR(ret), K(ttl_info), - "ori_leader", leader, K(timeout_remain_us), K(idle_time_us), K(RESERVED_TIME_US)); - ob_throttle_usleep((const useconds_t)idle_time_us, ret); - ret = OB_SUCCESS; - } else { - LOG_WARN("leader may switch or ddl confilict, will not retry cuz timeout_remain is " - "not enough", KR(ret), K(ttl_info), "ori_leader", leader, - K(timeout_remain_us), K(idle_time_us), K(RESERVED_TIME_US)); - } - } - } else { - ttl_done = true; - } - } - - if (OB_SUCC(ret) && !ttl_done) { - ret = OB_EAGAIN; - LOG_WARN("fail to retry ttl cuz switching role", KR(ret), K(MAX_RETRY_COUNT)); - } - } - - const int64_t launch_cost_time = ObTimeUtility::current_time() - launch_start_time; - LOG_INFO("do tenant ttl", KR(ret), K(leader), K(ttl_info), K(launch_cost_time)); + UNUSED(type); + ret = OB_NOT_SUPPORTED; } return ret; } diff --git a/src/share/tablet/ob_tablet_info.cpp b/src/share/tablet/ob_tablet_info.cpp index 15ba53bc7..1aaf2358a 100644 --- a/src/share/tablet/ob_tablet_info.cpp +++ b/src/share/tablet/ob_tablet_info.cpp @@ -32,7 +32,6 @@ const static char * ObDataChecksumTypeStr[] = { ObTabletReplica::ObTabletReplica() : tablet_id_(), - ls_id_(), server_(), snapshot_version_(0), data_size_(0), @@ -50,7 +49,6 @@ ObTabletReplica::~ObTabletReplica() void ObTabletReplica::reset() { tablet_id_.reset(); - ls_id_.reset(); server_.reset(); snapshot_version_ = 0; data_size_ = 0; @@ -64,7 +62,6 @@ int ObTabletReplica::assign(const ObTabletReplica &other) int ret = OB_SUCCESS; if (this != &other) { tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; server_ = other.server_; snapshot_version_ = other.snapshot_version_; data_size_ = other.data_size_; @@ -77,7 +74,6 @@ int ObTabletReplica::assign(const ObTabletReplica &other) int ObTabletReplica::init( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const common::ObAddr &server, const int64_t snapshot_version, const int64_t data_size, @@ -88,7 +84,6 @@ int ObTabletReplica::init( int ret = OB_SUCCESS; if (OB_UNLIKELY( !tablet_id.is_valid_with_tenant() - || !ls_id.is_valid_with_tenant() || !server.is_valid() || snapshot_version < 0 || data_size < 0 @@ -96,11 +91,10 @@ int ObTabletReplica::init( || report_scn < 0 || !is_status_valid(status))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("init with invalid arguments", KR(ret), K(tablet_id), K(ls_id), + LOG_WARN("init with invalid arguments", KR(ret), K(tablet_id), K(server), K(snapshot_version), K(data_size), K(required_size), K(report_scn), K(status)); } else { tablet_id_ = tablet_id; - ls_id_ = ls_id; server_ = server; snapshot_version_ = snapshot_version; data_size_ = data_size; @@ -118,7 +112,6 @@ bool ObTabletReplica::is_equal_for_report(const ObTabletReplica &other) const is_equal = true; } else if (true && tablet_id_ == other.tablet_id_ - && ls_id_ == other.ls_id_ && server_ == other.server_ && snapshot_version_ == other.snapshot_version_ && data_size_ == other.data_size_ @@ -128,29 +121,34 @@ bool ObTabletReplica::is_equal_for_report(const ObTabletReplica &other) const return is_equal; } -void ObTabletReplica::fake_for_diagnose(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +void ObTabletReplica::fake_for_diagnose(const common::ObTabletID &tablet_id) { reset(); - ls_id_ = ls_id; tablet_id_ = tablet_id; } ObTabletInfo::ObTabletInfo() : tablet_id_(), - ls_id_(), - replicas_() + has_replica_(false), + replica_() +{ +} + +ObTabletInfo::ObTabletInfo(const common::ObTabletID &tablet_id) + : tablet_id_(tablet_id), + has_replica_(false), + replica_() { } ObTabletInfo::ObTabletInfo( const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const ObArray &replicas) - : tablet_id_(tablet_id), - ls_id_(ls_id), - replicas_(replicas) + const ObTabletReplica &replica) + : tablet_id_(), + has_replica_(false), + replica_() { + (void)init(tablet_id, replica); } ObTabletInfo::~ObTabletInfo() @@ -161,8 +159,8 @@ ObTabletInfo::~ObTabletInfo() void ObTabletInfo::reset() { tablet_id_.reset(); - ls_id_.reset(); - replicas_.reset(); + has_replica_ = false; + replica_.reset(); } int ObTabletInfo::assign(const ObTabletInfo &other) @@ -170,31 +168,43 @@ int ObTabletInfo::assign(const ObTabletInfo &other) int ret = OB_SUCCESS; if (this != &other) { tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; - if (OB_FAIL(replicas_.assign(other.replicas_))) { - LOG_WARN("fail to assign replicas", KR(ret), K_(tablet_id), K_(replicas)); + has_replica_ = other.has_replica_; + if (OB_FAIL(replica_.assign(other.replica_))) { + LOG_WARN("fail to assign replica", KR(ret), K_(tablet_id), K_(replica)); } } return ret; } +int ObTabletInfo::init_empty(const common::ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_UNLIKELY(!tablet_id.is_valid_with_tenant())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("init empty tablet info with invalid arguments", KR(ret), K(tablet_id)); + } else { + tablet_id_ = tablet_id; + } + return ret; +} + int ObTabletInfo::init( const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObIArray &replicas) + const ObTabletReplica &replica) { int ret = OB_SUCCESS; + reset(); if (OB_UNLIKELY(!tablet_id.is_valid_with_tenant() - || !ls_id.is_valid_with_tenant() - || replicas.count() < 1)) { + || !replica.is_valid() + || tablet_id != replica.get_tablet_id())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("init with invalid arguments", KR(ret), - K(tablet_id), K(ls_id), K(replicas)); - } else if (OB_FAIL(replicas_.assign(replicas))) { - LOG_WARN("fail to assign replicas", KR(ret), K(replicas)); + LOG_WARN("init with invalid arguments", KR(ret), K(tablet_id), K(replica)); + } else if (OB_FAIL(replica_.assign(replica))) { + LOG_WARN("fail to assign replica", KR(ret), K(replica)); } else { tablet_id_ = tablet_id; - ls_id_ = ls_id; + has_replica_ = true; } return ret; } @@ -203,133 +213,57 @@ int ObTabletInfo::init_by_replica(const ObTabletReplica &replica) { int ret = OB_SUCCESS; reset(); - ObSEArray replicas; if (OB_UNLIKELY(!replica.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid replica", KR(ret), K(replica)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("fail to push back replica", KR(ret), K(replica)); - } else if (OB_FAIL(init( - replica.get_tablet_id(), - replica.get_ls_id(), - replicas))) { - LOG_WARN("fail to init tablet_info", KR(ret), K(replica), K(replicas)); + } else if (OB_FAIL(init(replica.get_tablet_id(), replica))) { + LOG_WARN("fail to init tablet_info", KR(ret), K(replica)); } return ret; } -int ObTabletInfo::add_replica(const ObTabletReplica &replica) +int ObTabletInfo::set_replica(const ObTabletReplica &replica) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid() || !replica.is_valid())) { + if (OB_UNLIKELY(!tablet_id_.is_valid_with_tenant() + || !replica.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KPC(this), K(replica)); - } else if (OB_UNLIKELY(tablet_id_ != replica.get_tablet_id()) - || ls_id_ != replica.get_ls_id()) { + } else if (OB_UNLIKELY(tablet_id_ != replica.get_tablet_id())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("replica not belong to this tablet", - KR(ret), K_(tablet_id), K_(ls_id), K(replica)); + KR(ret), K_(tablet_id), K(replica)); + } else if (OB_FAIL(replica_.assign(replica))) { + LOG_WARN("fail to assign replica", KR(ret), K(replica)); } else { - int64_t idx = OB_INVALID_INDEX; - ret = find_replica_idx_(replica, idx); - if (OB_SUCC(ret)) { // repeated replica - if (OB_UNLIKELY(idx < 0 || idx >= replicas_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid index", KR(ret), K(idx), "replica_count", replicas_.count()); - } else if (OB_FAIL(replicas_.at(idx).assign(replica))) { - LOG_WARN("fail to assign replicas_.at(idx)", KR(ret), K(idx), K(replica)); - } - } else if (OB_ENTRY_NOT_EXIST == ret) { - if (OB_FAIL(replicas_.push_back(replica))) { - LOG_WARN("insert replica failed", KR(ret), K(replica)); - } - } else { - LOG_WARN("find replica index failed", KR(ret), K(replica)); - } + has_replica_ = true; } return ret; } bool ObTabletInfo::is_self_replica(const ObTabletReplica &replica) const { - return true - && replica.get_tablet_id() == tablet_id_ - && replica.get_ls_id() == ls_id_; + return replica.get_tablet_id() == tablet_id_; } int ObTabletInfo::filter(const ObTabletReplicaFilter &filter) { int ret = OB_SUCCESS; - if (!is_valid()) { + if (!tablet_id_.is_valid_with_tenant()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), "tablet_info", *this); - } - for (int64_t i = replicas_.count() - 1; OB_SUCC(ret) && i >= 0; i--) { + } else if (has_replica_) { bool pass = true; - if (OB_FAIL(filter.check(replicas_.at(i), pass))) { - LOG_WARN("filter replica failed", K(ret), "replica", replicas_.at(i)); - } else { - if (!pass) { - if (OB_FAIL(replicas_.remove(i))) { - LOG_WARN("remove replica failed", K(ret), "idx", i); - } - } + if (OB_FAIL(filter.check(replica_, pass))) { + LOG_WARN("filter tablet meta row failed", K(ret), "tablet_meta_row", replica_); + } else if (!pass) { + has_replica_ = false; + replica_.reset(); } } return ret; } -int ObTabletInfo::find_replica_idx_(const ObTabletReplica &replica, int64_t &idx) const -{ - int ret = OB_ENTRY_NOT_EXIST; - idx = OB_INVALID_INDEX; - if (OB_UNLIKELY(!is_valid() - || !replica.is_valid() - || !is_self_replica(replica))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KPC(this), K(replica)); - } else { - ARRAY_FOREACH_X(replicas_, i, cnt, OB_ENTRY_NOT_EXIST == ret) { - if (replica.get_server() == replicas_.at(i).get_server()) { - idx = i; - ret = OB_SUCCESS; - } - } - } - return ret; -} - -int ObTabletToLSInfo::init( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const uint64_t table_id) { - int ret = OB_SUCCESS; - if (OB_UNLIKELY( - !tablet_id.is_valid() - || !ls_id.is_valid() - || OB_INVALID_ID == table_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("init with invalid argument", - KR(ret), K(tablet_id), K(ls_id), K(table_id)); - } else { - tablet_id_ = tablet_id; - ls_id_ = ls_id; - table_id_ = table_id; - } - return ret; -} - -int ObTabletToLSInfo::assign(const ObTabletToLSInfo &other) -{ - int ret = OB_SUCCESS; - if (&other != this) { - tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; - table_id_ = other.table_id_; - } - return ret; -} - ObTabletTablePair::ObTabletTablePair() : tablet_id_(), table_id_(OB_INVALID_ID) {} @@ -343,70 +277,40 @@ ObTabletTablePair::ObTabletTablePair( ObTabletTablePair::~ObTabletTablePair() {} -int ObTabletTablePair::init( - const common::ObTabletID &tablet_id, - const uint64_t table_id) -{ - int ret = OB_SUCCESS; - tablet_id_ = tablet_id; - table_id_ = table_id; - return ret; -} - - -bool ObTabletTablePair::is_valid() const +void ObTabletTablePair::reset() { - return tablet_id_.is_valid() && OB_INVALID_ID != table_id_; + tablet_id_.reset(); + table_id_ = OB_INVALID_ID; } -int ObTabletLSPair::init( +int ObTabletTablePair::init( const common::ObTabletID &tablet_id, - const ObLSID &ls_id) + const uint64_t table_id) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!tablet_id.is_valid() || !ls_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || OB_INVALID_ID == table_id)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arugments", KR(ret), K(tablet_id), K(ls_id)); + LOG_WARN("init with invalid argument", KR(ret), K(tablet_id), K(table_id)); } else { tablet_id_ = tablet_id; - ls_id_ = ls_id; + table_id_ = table_id; } return ret; } -int ObTabletLSPair::assign(const ObTabletLSPair &other) +int ObTabletTablePair::assign(const ObTabletTablePair &other) { int ret = OB_SUCCESS; - if (this != &other) { + if (&other != this) { tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; + table_id_ = other.table_id_; } return ret; } -void ObTabletLSPair::reset() -{ - tablet_id_.reset(); - ls_id_.reset(); -} - -bool ObTabletLSPair::is_valid() const -{ - return tablet_id_.is_valid() && ls_id_.is_valid(); -} - -bool ObTabletLSPair::operator==(const ObTabletLSPair &other) const -{ - return tablet_id_ == other.tablet_id_ - && ls_id_ == other.ls_id_; -} - -uint64_t ObTabletLSPair::hash() const +bool ObTabletTablePair::is_valid() const { - uint64_t hash_val = 0; - hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); - hash_val = murmurhash(&ls_id_, sizeof(ls_id_), hash_val); - return hash_val; + return tablet_id_.is_valid() && OB_INVALID_ID != table_id_; } } // end namespace share diff --git a/src/share/tablet/ob_tablet_info.h b/src/share/tablet/ob_tablet_info.h index 5260d61dd..7158243fc 100644 --- a/src/share/tablet/ob_tablet_info.h +++ b/src/share/tablet/ob_tablet_info.h @@ -19,7 +19,6 @@ #include "common/ob_tablet_id.h" // ObTabletID #include "lib/net/ob_addr.h" // ObAddr -#include "share/ob_ls_id.h" // ObLSID namespace oceanbase { @@ -62,7 +61,6 @@ class ObTabletReplica inline bool is_valid() const { return tablet_id_.is_valid_with_tenant() - && ls_id_.is_valid_with_tenant() && server_.is_valid() && snapshot_version_ >= 0 && data_size_ >= 0 @@ -73,14 +71,12 @@ class ObTabletReplica inline bool primary_keys_are_valid() const { return tablet_id_.is_valid_with_tenant() - && server_.is_valid() - && ls_id_.is_valid_with_tenant(); + && server_.is_valid(); } int assign(const ObTabletReplica &other); inline const common::ObTabletID &get_tablet_id() const { return tablet_id_; } inline const common::ObAddr &get_server() const { return server_; } - inline const share::ObLSID &get_ls_id() const { return ls_id_; } inline int64_t get_snapshot_version() const { return snapshot_version_; } inline int64_t get_data_size() const { return data_size_; } inline int64_t get_required_size() const { return required_size_; } @@ -88,15 +84,13 @@ class ObTabletReplica inline ScnStatus get_status() const { return status_; } int init( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const common::ObAddr &server, const int64_t snapshot_version, const int64_t data_size, const int64_t required_size, const int64_t report_scn, const ScnStatus status); - void fake_for_diagnose(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id); + void fake_for_diagnose(const common::ObTabletID &tablet_id); bool is_equal_for_report(const ObTabletReplica &other) const; static bool is_status_valid(const ScnStatus status) { @@ -104,7 +98,6 @@ class ObTabletReplica } TO_STRING_KV( K_(tablet_id), - K_(ls_id), K_(server), K_(snapshot_version), K_(data_size), @@ -113,7 +106,6 @@ class ObTabletReplica K_(status)); private: common::ObTabletID tablet_id_; - share::ObLSID ls_id_; common::ObAddr server_; int64_t snapshot_version_; int64_t data_size_; // load balancing releated @@ -127,90 +119,40 @@ class ObTabletInfo { public: ObTabletInfo(); + explicit ObTabletInfo(const common::ObTabletID &tablet_id); explicit ObTabletInfo( const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const ObArray &replicas); + const ObTabletReplica &replica); virtual ~ObTabletInfo(); void reset(); inline bool is_valid() const { return true && tablet_id_.is_valid_with_tenant() - && ls_id_.is_valid_with_tenant() - && replicas_.count() > 0; + && has_replica_ + && replica_.is_valid(); } int assign(const ObTabletInfo &other); inline const common::ObTabletID &get_tablet_id() const { return tablet_id_; } - inline const ObLSID &get_ls_id() const { return ls_id_; } - inline const common::ObArray &get_replicas() const { return replicas_; } - int64_t replica_count() const { return replicas_.count(); } + inline bool has_replica() const { return has_replica_; } + inline const ObTabletReplica &get_replica() const { return replica_; } + int64_t replica_count() const { return has_replica_ ? 1 : 0; } + int init_empty(const common::ObTabletID &tablet_id); int init(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObIArray &replicas); + const ObTabletReplica &replica); int init_by_replica(const ObTabletReplica &replica); - int add_replica(const ObTabletReplica &replica); + int set_replica(const ObTabletReplica &replica); bool is_self_replica(const ObTabletReplica &replica) const; int filter(const ObTabletReplicaFilter &filter); - TO_STRING_KV(K_(tablet_id), K_(replicas)); + TO_STRING_KV(K_(tablet_id), K_(has_replica), K_(replica)); private: - int find_replica_idx_(const ObTabletReplica &replica, int64_t &idx) const; - common::ObTabletID tablet_id_; - ObLSID ls_id_; - common::ObArray replicas_; + bool has_replica_; + ObTabletReplica replica_; DISALLOW_COPY_AND_ASSIGN(ObTabletInfo); }; -// ObTabletToLSInfo is used to store info for __all_tablet_to_ls. -// Structure: -class ObTabletToLSInfo -{ -public: - ObTabletToLSInfo() - : tablet_id_(), ls_id_(), table_id_(OB_INVALID_ID) {} - explicit ObTabletToLSInfo( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const uint64_t table_id) - : tablet_id_(tablet_id), ls_id_(ls_id), table_id_(table_id) - { - } - ~ObTabletToLSInfo() { reset(); } - inline void reset() - { - tablet_id_.reset(); - ls_id_.reset(); - table_id_ = OB_INVALID_ID; - } - inline bool is_valid() const - { - return tablet_id_.is_valid() - && ls_id_.is_valid() - && OB_INVALID_ID != table_id_; - } - inline bool operator==(const ObTabletToLSInfo &other) const - { - return other.tablet_id_ == tablet_id_ - && other.ls_id_ == ls_id_ - && other.table_id_ == table_id_; - } - inline const common::ObTabletID &get_tablet_id() const { return tablet_id_; } - inline const ObLSID &get_ls_id() const { return ls_id_; } - inline uint64_t get_table_id() const { return table_id_; } - int init( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const uint64_t table_id); - int assign(const ObTabletToLSInfo &other); - TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(table_id)); -private: - common::ObTabletID tablet_id_; - ObLSID ls_id_; - uint64_t table_id_; -}; - class ObTabletTablePair { public: @@ -218,7 +160,9 @@ class ObTabletTablePair ObTabletTablePair(const common::ObTabletID &tablet_id, const uint64_t table_id); ~ObTabletTablePair(); + void reset(); int init(const ObTabletID &tablet_id, const uint64_t table_id); + int assign(const ObTabletTablePair &other); bool is_valid() const; const ObTabletID &get_tablet_id() const { return tablet_id_; } uint64_t get_table_id() const { return table_id_; } @@ -228,30 +172,6 @@ class ObTabletTablePair uint64_t table_id_; }; -class ObTabletLSPair -{ -public: - ObTabletLSPair() {} - explicit ObTabletLSPair(const common::ObTabletID &tablet_id, const ObLSID &ls_id) - : tablet_id_(tablet_id), ls_id_(ls_id) {} - explicit ObTabletLSPair(const int64_t tablet_id, const int64_t ls_id) - : tablet_id_(tablet_id), ls_id_(ls_id) {} - ~ObTabletLSPair() {} - int init(const ObTabletID &tablet_id, const ObLSID &ls_id); - int assign(const ObTabletLSPair &other); - void reset(); - bool is_valid() const; - uint64_t hash() const; - int hash(uint64_t &hash_val) const { hash_val = hash(); return OB_SUCCESS; } - const ObTabletID &get_tablet_id() const { return tablet_id_; } - const ObLSID &get_ls_id() const { return ls_id_; } - bool operator==(const ObTabletLSPair &other) const; - TO_STRING_KV(K_(tablet_id), K_(ls_id)); -private: - common::ObTabletID tablet_id_; - ObLSID ls_id_; -}; - } // end namespace share } // end namespace oceanbase #endif diff --git a/src/share/tablet/ob_tablet_to_ls_operator.cpp b/src/share/tablet/ob_tablet_mapping_operator.cpp similarity index 54% rename from src/share/tablet/ob_tablet_to_ls_operator.cpp rename to src/share/tablet/ob_tablet_mapping_operator.cpp index d1ea8894f..8474d6872 100644 --- a/src/share/tablet/ob_tablet_to_ls_operator.cpp +++ b/src/share/tablet/ob_tablet_mapping_operator.cpp @@ -16,12 +16,10 @@ #define USING_LOG_PREFIX SHARE -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/inner_table/ob_inner_table_schema_constants.h" #include "share/ob_dml_sql_splicer.h" // ObDMLSqlSplicer #include "share/ob_sql_client_decorator.h" // ObSQLClientRetryWeak -#include "lib/stat/ob_diagnostic_info_guard.h" // ObASHSetInnerSqlWaitGuard, previously hidden behind the medium_checker include chain, make the dependency explicit -#include "lib/stat/ob_diagnostic_info_guard.h" namespace oceanbase { @@ -29,23 +27,19 @@ using namespace common; namespace share { -#define RANGE_GET(sql_proxy, ls_white_list, start_tablet_id, range_size, tablets) \ +#define RANGE_GET(sql_proxy, start_tablet_id, range_size, tablets) \ do { \ - ObSqlString subsql; \ ObSqlString sql; \ if (OB_FAIL(ret)) { \ } else if (OB_UNLIKELY(range_size <= 0)) { /* do not check start_tablet_id */ \ ret = OB_INVALID_ARGUMENT; \ LOG_WARN("invalid argument", KR(ret), K(range_size)); \ - } else if (OB_FAIL(construct_ls_white_list_where_sql_(ls_white_list, subsql))) { \ - LOG_WARN("construct sub sql string for LS white list fail", KR(ret), K(ls_white_list)); \ } else if (OB_FAIL(sql.append_fmt( \ - "SELECT * FROM %s WHERE tablet_id > %lu %s ORDER BY tablet_id LIMIT %ld", \ - OB_ALL_TABLET_TO_LS_TNAME, \ + "SELECT * FROM %s WHERE tablet_id > %lu ORDER BY tablet_id LIMIT %ld", \ + OB_ALL_TABLET_TO_TABLE_TNAME, \ start_tablet_id.id(), \ - subsql.empty() ? "" : subsql.ptr(), \ range_size))) { \ - LOG_WARN("fail to assign sql", KR(ret), K(sql), K(subsql), K(start_tablet_id)); \ + LOG_WARN("fail to assign sql", KR(ret), K(sql), K(start_tablet_id)); \ } else { \ SMART_VAR(ObISQLClient::ReadResult, result) { \ if (OB_FAIL(sql_proxy.read(result, sql.ptr()))) { \ @@ -79,7 +73,7 @@ namespace share ObSQLClientRetryWeak sql_client_retry_weak( \ &sql_proxy, \ false,/*did_use_retry*/ \ - OB_ALL_TABLET_TO_LS_TID); \ + OB_ALL_TABLET_TO_TABLE_TID); \ ObSqlString sql; \ ObSqlString tablet_list; \ for (int64_t idx = start_idx; OB_SUCC(ret) && (idx < end_idx); ++idx) { \ @@ -97,7 +91,7 @@ namespace share if (FAILEDx(sql.append_fmt( \ "SELECT %s FROM %s WHERE tablet_id IN (", \ query_column_str, \ - OB_ALL_TABLET_TO_LS_TNAME))) { \ + OB_ALL_TABLET_TO_TABLE_TNAME))) { \ LOG_WARN("fail to assign sql", KR(ret), K(sql)); \ } else if (OB_FAIL(sql.append(tablet_list.string()))) { \ LOG_WARN("fail to assign sql", KR(ret), K(sql), K(tablet_list)); \ @@ -117,7 +111,7 @@ namespace share ret = OB_ERR_UNEXPECTED; \ LOG_WARN("get mysql result failed", KR(ret)); \ } else if (OB_FAIL(construct_results_(*result.get_result(), results))) { \ - LOG_WARN("construct log stream info failed", KR(ret), K(results)); \ + LOG_WARN("construct tablet mapping info failed", KR(ret), K(results)); \ } \ } \ } \ @@ -150,94 +144,20 @@ namespace share } \ } while(0) -int ObTabletToLSTableOperator::range_get_tablet( +int ObTabletMappingTableOperator::range_get_tablet_table_pairs( common::ObISQLClient &sql_proxy, const ObTabletID &start_tablet_id, const int64_t range_size, - common::ObIArray &tablet_ls_pairs) + common::ObIArray &tablets) { int ret = OB_SUCCESS; - const ObArray ls_white_list; - RANGE_GET(sql_proxy, ls_white_list, start_tablet_id, range_size, tablet_ls_pairs); + RANGE_GET(sql_proxy, start_tablet_id, range_size, tablets); return ret; } -int ObTabletToLSTableOperator::range_get_tablet_info( +int ObTabletMappingTableOperator::batch_update( common::ObISQLClient &sql_proxy, - const ObIArray &ls_white_list, - const ObTabletID &start_tablet_id, - const int64_t range_size, - common::ObIArray &tablets) -{ - int ret = OB_SUCCESS; - RANGE_GET(sql_proxy, ls_white_list, start_tablet_id, range_size, tablets); - return ret; -} - -int ObTabletToLSTableOperator::construct_ls_white_list_where_sql_( - const ObIArray &ls_white_list, - ObSqlString &subsql) -{ - int ret = OB_SUCCESS; - return ret; -} - -int ObTabletToLSTableOperator::batch_get_ls( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - ObIArray &ls_ids) -{ - int ret = OB_SUCCESS; - BATCH_GET(sql_proxy, tablet_ids, ls_ids); - if (OB_SUCC(ret) && OB_UNLIKELY(ls_ids.count() != tablet_ids.count())) { - ret = OB_ITEM_NOT_MATCH; - LOG_WARN("count of ls_ids and tablet_ids do not match," - " there may be duplicates or nonexistent values in tablet_ids", - KR(ret), "tablet_ids count", tablet_ids.count(), "ls_ids count", ls_ids.count(), - K(tablet_ids), K(ls_ids)); - } - return ret; -} - -int ObTabletToLSTableOperator::inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - ObIArray &ls_ids) -{ - int ret = OB_SUCCESS; - const char *query_column_str = "tablet_id"; - const bool keep_order = true; - INNER_BATCH_GET(sql_proxy, tablet_ids, start_idx, end_idx, - query_column_str, keep_order, ls_ids); - return ret; -} - -int ObTabletToLSTableOperator::construct_results_( - common::sqlclient::ObMySQLResult &res, - ObIArray &ls_ids) -{ - int ret = OB_SUCCESS; - while (OB_SUCC(ret) && OB_SUCC(res.next())) { - if (OB_FAIL(ls_ids.push_back(SYS_LS))) { - LOG_WARN("fail to push back", KR(ret), K(ls_ids)); - } - } - if (OB_ITER_END != ret) { - if (OB_UNLIKELY(OB_SUCCESS == ret)) { - ret = OB_ERR_UNEXPECTED; - } - LOG_WARN("fail to get next row to the end", KR(ret)); - } else { - ret = OB_SUCCESS; - } - return ret; -} - -int ObTabletToLSTableOperator::batch_update( - common::ObISQLClient &sql_proxy, - const ObIArray &infos) + const ObIArray &infos) { int ret = OB_SUCCESS; if (OB_UNLIKELY(infos.empty())) { @@ -256,12 +176,12 @@ int ObTabletToLSTableOperator::batch_update( } } if (OB_SUCC(ret)) { - LOG_TRACE("batch update tablet_to_ls success", K(infos)); + LOG_TRACE("batch update tablet-table mapping success", K(infos)); } } return ret; } -int ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(common::ObISQLClient &sql_proxy, +int ObTabletMappingTableOperator::update_table_to_tablet_id_mapping(common::ObISQLClient &sql_proxy, const uint64_t table_id, const common::ObTabletID &tablet_id) { @@ -276,7 +196,7 @@ int ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(common::ObISQLC if (OB_FAIL(dml_splicer.add_pk_column("tablet_id", tablet_id.id())) || OB_FAIL(dml_splicer.add_column("table_id", table_id))) { LOG_WARN("fail to add column", K(ret), K(tablet_id), K(table_id)); - } else if (OB_FAIL(dml_splicer.splice_update_sql(OB_ALL_TABLET_TO_LS_TNAME, sql))) { + } else if (OB_FAIL(dml_splicer.splice_update_sql(OB_ALL_TABLET_TO_TABLE_TNAME, sql))) { LOG_WARN("fail to splice batch insert update sql", K(ret), K(sql)); } else if (OB_FAIL(sql_proxy.write(sql.ptr(), affected_rows))) { LOG_WARN("fail to write sql", K(ret), K(sql), K(affected_rows)); @@ -284,15 +204,15 @@ int ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(common::ObISQLC ret = OB_ERR_UNEXPECTED; LOG_WARN("expect one row", K(ret), K(sql), K(affected_rows)); } else { - LOG_TRACE("update tablet_to_ls success", K(affected_rows)); + LOG_TRACE("update tablet-table mapping success", K(affected_rows)); } } return ret; } -int ObTabletToLSTableOperator::inner_batch_update_by_sql_( +int ObTabletMappingTableOperator::inner_batch_update_by_sql_( common::ObISQLClient &sql_proxy, - const ObIArray &infos, + const ObIArray &infos, const int64_t start_idx, const int64_t end_idx) { @@ -308,10 +228,10 @@ int ObTabletToLSTableOperator::inner_batch_update_by_sql_( ObDMLSqlSplicer dml_splicer; int64_t affected_rows = 0; for (int64_t idx = start_idx; OB_SUCC(ret) && (idx < end_idx); ++idx) { - const ObTabletToLSInfo &info = infos.at(idx); + const ObTabletTablePair &info = infos.at(idx); if (OB_UNLIKELY(!info.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid TabletToLSInfo", KR(ret), K(info)); + LOG_WARN("invalid tablet-table pair", KR(ret), K(info)); } else if (OB_FAIL(dml_splicer.add_pk_column("tablet_id", info.get_tablet_id().id())) || OB_FAIL(dml_splicer.add_column("table_id", info.get_table_id()))) { LOG_WARN("fail to add column", KR(ret), K(info)); @@ -319,20 +239,20 @@ int ObTabletToLSTableOperator::inner_batch_update_by_sql_( LOG_WARN("fail to finish row", KR(ret)); } } - if (FAILEDx(dml_splicer.splice_batch_insert_update_sql(OB_ALL_TABLET_TO_LS_TNAME, sql))) { + if (FAILEDx(dml_splicer.splice_batch_insert_update_sql(OB_ALL_TABLET_TO_TABLE_TNAME, sql))) { LOG_WARN("fail to splice batch insert update sql", KR(ret), K(sql)); } else if (OB_FAIL(sql_proxy.write(sql.ptr(), affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(sql), K(affected_rows), K(infos), K(start_idx), K(end_idx)); } else { - LOG_TRACE("update tablet_to_ls success", + LOG_TRACE("update tablet-table mapping success", K(affected_rows), K(start_idx), K(end_idx)); } } return ret; } -int ObTabletToLSTableOperator::batch_remove( +int ObTabletMappingTableOperator::batch_remove( common::ObISQLClient &sql_proxy, const ObIArray &tablet_ids) { @@ -360,7 +280,7 @@ int ObTabletToLSTableOperator::batch_remove( return ret; } -int ObTabletToLSTableOperator::inner_batch_remove_by_sql_( +int ObTabletMappingTableOperator::inner_batch_remove_by_sql_( common::ObISQLClient &sql_proxy, const ObIArray &tablet_ids, const int64_t start_idx, @@ -379,7 +299,7 @@ int ObTabletToLSTableOperator::inner_batch_remove_by_sql_( K(tablet_ids), K(start_idx), K(end_idx)); } else if (OB_FAIL(sql.append_fmt( "DELETE FROM %s WHERE tablet_id IN (", - OB_ALL_TABLET_TO_LS_TNAME))) { + OB_ALL_TABLET_TO_TABLE_TNAME))) { LOG_WARN("fail to assign sql", KR(ret)); } else { int64_t affected_rows = 0; @@ -402,10 +322,10 @@ int ObTabletToLSTableOperator::inner_batch_remove_by_sql_( } -int ObTabletToLSTableOperator::batch_get( +int ObTabletMappingTableOperator::batch_get( common::ObISQLClient &sql_proxy, const ObIArray &tablet_ids, - ObIArray &infos) + ObIArray &infos) { int ret = OB_SUCCESS; BATCH_GET(sql_proxy, tablet_ids, infos); @@ -419,12 +339,12 @@ int ObTabletToLSTableOperator::batch_get( return ret; } -int ObTabletToLSTableOperator::inner_batch_get_( +int ObTabletMappingTableOperator::inner_batch_get_( common::ObISQLClient &sql_proxy, const ObIArray &tablet_ids, const int64_t start_idx, const int64_t end_idx, - ObIArray &infos) + ObIArray &infos) { int ret = OB_SUCCESS; const char *query_column_str = "*"; @@ -434,22 +354,21 @@ int ObTabletToLSTableOperator::inner_batch_get_( return ret; } -int ObTabletToLSTableOperator::construct_results_( +int ObTabletMappingTableOperator::construct_results_( common::sqlclient::ObMySQLResult &res, - ObIArray &infos) + ObIArray &infos) { int ret = OB_SUCCESS; while (OB_SUCC(ret) && OB_SUCC(res.next())) { int64_t tablet_id = ObTabletID::INVALID_TABLET_ID; - int64_t ls_id = ObLSID::SYS_LS_ID; uint64_t table_id = OB_INVALID_ID; - ObTabletToLSInfo info; + ObTabletTablePair info; EXTRACT_INT_FIELD_MYSQL(res, "tablet_id", tablet_id, int64_t); EXTRACT_INT_FIELD_MYSQL(res, "table_id", table_id, uint64_t); - if (FAILEDx(info.init(ObTabletID(tablet_id), ObLSID(ls_id), table_id))) { - LOG_WARN("init failed", KR(ret), K(tablet_id), K(ls_id), K(table_id)); + if (FAILEDx(info.init(ObTabletID(tablet_id), table_id))) { + LOG_WARN("init failed", KR(ret), K(tablet_id), K(table_id)); } else if (OB_FAIL(infos.push_back(info))) { LOG_WARN("fail to push back", KR(ret), K(info)); } @@ -465,190 +384,5 @@ int ObTabletToLSTableOperator::construct_results_( return ret; } -int ObTabletToLSTableOperator::construct_results_( - common::sqlclient::ObMySQLResult &res, - ObIArray &pairs) -{ - int ret = OB_SUCCESS; - while (OB_SUCC(ret) && OB_SUCC(res.next())) { - int64_t tablet_id = ObTabletID::INVALID_TABLET_ID; - int64_t ls_id = ObLSID::SYS_LS_ID; - - EXTRACT_INT_FIELD_MYSQL(res, "tablet_id", tablet_id, int64_t); - - if (FAILEDx(pairs.push_back(ObTabletLSPair(ObTabletID(tablet_id), ObLSID(ls_id))))) { - LOG_WARN("fail to push back", KR(ret), K(tablet_id), K(ls_id)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - } - LOG_WARN("construct results failed", KR(ret), K(pairs)); - } - return ret; -} - -int ObTabletToLSTableOperator::get_ls_by_tablet( - common::ObISQLClient &sql_proxy, - const common::ObTabletID &tablet_id, - ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ls_id.reset(); - ObSEArray tablet_ids; - ObSEArray ls_ids; - if (OB_UNLIKELY(!tablet_id.is_valid_with_tenant())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_id)); - } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { - LOG_WARN("push back failed", KR(ret), K(tablet_id)); - } else { - int64_t start_idx = 0; - int64_t end_idx = 1; - if (OB_FAIL(inner_batch_get_( - sql_proxy, - tablet_ids, - start_idx, - end_idx, - ls_ids))) { - LOG_WARN("fail to inner batch get by sql", - KR(ret), K(tablet_ids), K(start_idx), K(end_idx)); - } else if (ls_ids.empty()) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("tablet not found", KR(ret), K(tablet_id)); - } else if (1 != ls_ids.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get too much ls_ids", KR(ret), K(tablet_id), K(ls_ids)); - } else { - ls_id = ls_ids.at(0); - } - } - return ret; -} - -int ObTabletToLSTableOperator::batch_get_tablet_ls_cache( - common::ObISQLClient &sql_proxy, - const common::ObIArray &tablet_ids, - common::ObIArray &tablet_ls_caches) -{ - ObASHSetInnerSqlWaitGuard ash_inner_sql_guard(ObInnerSqlWaitTypeId::GET_TABLET_LOCATION); - int ret = OB_SUCCESS; - BATCH_GET(sql_proxy, tablet_ids, tablet_ls_caches); - return ret; -} - -int ObTabletToLSTableOperator::inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - common::ObIArray &tablet_ls_caches) -{ - int ret = OB_SUCCESS; - const char *query_column_str = "*"; - const bool keep_order = false; - INNER_BATCH_GET(sql_proxy, tablet_ids, start_idx, end_idx, - query_column_str, keep_order, tablet_ls_caches); - return ret; -} - -int ObTabletToLSTableOperator::construct_results_( - common::sqlclient::ObMySQLResult &res, - common::ObIArray &tablet_ls_caches) -{ - int ret = OB_SUCCESS; - ObTabletLSCache tablet_ls_cache; - while (OB_SUCC(ret) && OB_SUCC(res.next())) { - tablet_ls_cache.reset(); - uint64_t tablet_id = ObTabletID::INVALID_TABLET_ID; - int64_t ls_id = ObLSID::SYS_LS_ID; - EXTRACT_INT_FIELD_MYSQL(res, "tablet_id", tablet_id, uint64_t); - const int64_t now = ObTimeUtility::fast_current_time(); - if (FAILEDx(tablet_ls_cache.init( - ObTabletID(tablet_id), - ObLSID(ls_id), - now))) { - LOG_WARN("init tablet_ls_cache failed", KR(ret), - K(tablet_id), K(ls_id), K(now)); - } else if (OB_FAIL(tablet_ls_caches.push_back(tablet_ls_cache))) { - LOG_WARN("fail to push back", KR(ret), K(tablet_ls_cache)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - } - LOG_WARN("construct results failed", KR(ret), K(tablet_ls_caches)); - } - return ret; -} - -int ObTabletToLSTableOperator::batch_get_tablet_ls_pairs( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - ObIArray &tablet_ls_pairs) -{ - int ret = OB_SUCCESS; - BATCH_GET(sql_proxy, tablet_ids, tablet_ls_pairs); - if (OB_SUCC(ret) && OB_UNLIKELY(tablet_ls_pairs.count() != tablet_ids.count())) { - ret = OB_ITEM_NOT_MATCH; - LOG_WARN("count of tablet_ls_pairs and tablet_ids do not match," - " there may be duplicates or nonexistent values in tablet_ids", - KR(ret), "tablet_ids count", tablet_ids.count(), "tablet_ls_pairs count", - tablet_ls_pairs.count(), K(tablet_ids), K(tablet_ls_pairs)); - } - return ret; -} - -int ObTabletToLSTableOperator::inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - ObIArray &tablet_ls_pairs) -{ - int ret = OB_SUCCESS; - const char *query_column_str = "tablet_id"; - const bool keep_order = false; - INNER_BATCH_GET(sql_proxy, tablet_ids, start_idx, end_idx, - query_column_str, keep_order, tablet_ls_pairs); - return ret; -} - -int ObTabletToLSTableOperator::get_tablet_ls_pairs_cnt( - common::ObISQLClient &sql_proxy, - int64_t &input_cnt) -{ - int ret = OB_SUCCESS; - ObSqlString sql; - input_cnt = 0; - if (OB_FAIL(sql.append_fmt( - "select count(*) as cnt from %s", - OB_ALL_TABLET_TO_LS_TNAME))) { - LOG_WARN("failed to append fmt", K(ret)); - } else { - common::sqlclient::ObMySQLResult *result = nullptr; - int64_t cnt = 0; - SMART_VAR(ObISQLClient::ReadResult, res) { - if (OB_FAIL(sql_proxy.read(res, sql.ptr()))) { - LOG_WARN("fail to do read", KR(ret), K(sql)); - } else if (OB_ISNULL(result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get result", KR(ret), K(sql)); - } else if (OB_FAIL(result->get_int("cnt", cnt))) { - LOG_WARN("failed to get int", KR(ret), K(cnt)); - } else { - input_cnt = cnt; - } - } - } - return ret; -} - } // end namespace share } // end namespace oceanbase diff --git a/src/share/tablet/ob_tablet_mapping_operator.h b/src/share/tablet/ob_tablet_mapping_operator.h new file mode 100644 index 000000000..348c91a5a --- /dev/null +++ b/src/share/tablet/ob_tablet_mapping_operator.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_SHARE_OB_TABLET_MAPPING_OPERATOR +#define OCEANBASE_SHARE_OB_TABLET_MAPPING_OPERATOR + +#include "lib/container/ob_iarray.h" // ObIArray +#include "share/tablet/ob_tablet_info.h" // ObTabletTablePair + +namespace oceanbase +{ +namespace common +{ +class ObISQLClient; + +namespace sqlclient +{ +class ObMySQLResult; +} +} // end nampspace common + +namespace share +{ +// This operator keeps the tablet-to-table mapping in the legacy inner table. +class ObTabletMappingTableOperator +{ +public: + ObTabletMappingTableOperator() {} + virtual ~ObTabletMappingTableOperator() {} + // Get tablets sequentially by range + // + static int range_get_tablet_table_pairs( + common::ObISQLClient &sql_proxy, + const ObTabletID &start_tablet_id, + const int64_t range_size, + common::ObIArray &tablets); + + // Updates tablet-to-table pairs in the legacy tablet mapping table. + // + // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction + // @param [in] infos, tablet-to-table pairs for updating + // @return OB_SUCCESS if success + static int batch_update( + common::ObISQLClient &sql_proxy, + const ObIArray &infos); + // Removes tablet_id from the legacy tablet mapping table. + // + // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction + // @param [in] tablet_ids, ObTabletIDs for removing + // @return OB_SUCCESS if success + static int batch_remove( + common::ObISQLClient &sql_proxy, + const ObIArray &tablet_ids); + static int update_table_to_tablet_id_mapping( + common::ObISQLClient &sql_proxy, + const uint64_t table_id, + const common::ObTabletID &tablet_id); + // Get tablet-to-table mapping rows according to ObTabletIDs. + // + // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction + // @param [in] tablet_ids, ObTabletIDs for query + // (should exist in the legacy tablet mapping table and have no duplicate values) + // @param [out] infos, ObTabletTablePair corresponding to tablet_ids (not same order) + // not same order, not same order, not same order + // @return OB_SUCCESS if success; + // OB_ITEM_NOT_MATCH if tablet_ids have duplicates or nonexistent tablets; + // Other error according to unexpected situation + static int batch_get( + common::ObISQLClient &sql_proxy, + const ObIArray &tablet_ids, + ObIArray &infos); + + const static int64_t MAX_BATCH_COUNT = 200; +private: + static int inner_batch_get_( + common::ObISQLClient &sql_proxy, + const ObIArray &tablet_ids, + const int64_t start_idx, + const int64_t end_idx, + ObIArray &infos); + static int inner_batch_update_by_sql_( + common::ObISQLClient &sql_proxy, + const ObIArray &infos, + const int64_t start_idx, + const int64_t end_idx); + static int inner_batch_remove_by_sql_( + common::ObISQLClient &sql_proxy, + const ObIArray &tablet_ids, + const int64_t start_idx, + const int64_t end_idx); + + static int construct_results_( + common::sqlclient::ObMySQLResult &res, + ObIArray &infos); +}; + +} // end namespace share +} // end namespace oceanbase +#endif // OCEANBASE_SHARE_OB_TABLET_MAPPING_OPERATOR diff --git a/src/share/tablet/ob_tablet_meta_table_storage.cpp b/src/share/tablet/ob_tablet_meta_table_storage.cpp index ba2aec834..c3cf5b20a 100644 --- a/src/share/tablet/ob_tablet_meta_table_storage.cpp +++ b/src/share/tablet/ob_tablet_meta_table_storage.cpp @@ -53,7 +53,6 @@ int ObTabletReplicaConstructor::operator()(share::ObSQLiteRowReader &reader, ObT LOG_WARN("invalid status", K(ret), K(status)); } else if (OB_FAIL(replica.init( ObTabletID(tablet_id), - ObLSID(ObLSID::SYS_LS_ID), server, compaction_scn, data_size, @@ -110,101 +109,8 @@ int ObTabletMetaTableStorage::create_table_if_not_exists() return ret; } -int ObTabletMetaTableStorage::get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObAddr &addr, - ObTabletReplica &tablet_replica) -{ - int ret = OB_SUCCESS; - tablet_replica.reset(); - if (!is_inited()) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - UNUSED(addr); - const char *select_sql = - "SELECT tablet_id, " - " compaction_scn, data_size, required_size, report_scn, status " - "FROM __all_tablet_meta_table " - "WHERE tablet_id = ?;"; - - auto binder = [&](share::ObSQLiteBinder &b) -> int { - b.bind_int64(tablet_id.id()); - return OB_SUCCESS; - }; - - ObTabletReplicaConstructor constructor; - auto row_processor = [&](share::ObSQLiteRowReader &reader) -> int { - return constructor(reader, tablet_replica); - }; - - ObSQLiteConnectionGuard guard(pool_); - if (!guard) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to acquire connection", K(ret)); - } else { - ret = guard->query(select_sql, binder, row_processor); - } - } - return ret; -} - -int ObTabletMetaTableStorage::get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - ObTabletInfo &tablet_info) -{ - int ret = OB_SUCCESS; - tablet_info.reset(); - if (!is_inited()) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - const char *select_sql = - "SELECT tablet_id, " - " compaction_scn, data_size, required_size, report_scn, status " - "FROM __all_tablet_meta_table " - "WHERE tablet_id = ?;"; - - auto binder = [&](share::ObSQLiteBinder &b) -> int { - b.bind_int64(tablet_id.id()); - return OB_SUCCESS; - }; - - ObArray replicas; - ObTabletReplicaConstructor constructor; - auto row_processor = [&](share::ObSQLiteRowReader &reader) -> int { - ObTabletReplica replica; - if (OB_FAIL(constructor(reader, replica))) { - LOG_WARN("failed to construct tablet replica", K(ret)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); - } - return ret; - }; - - ObSQLiteConnectionGuard guard(pool_); - if (!guard) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to acquire connection", K(ret)); - } else if (OB_FAIL(guard->query(select_sql, binder, row_processor))) { - if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("failed to query", K(ret)); - } else { - ret = OB_SUCCESS; // No rows is acceptable - } - } else if (replicas.count() > 0) { - if (OB_FAIL(tablet_info.init(tablet_id, ObLSID(ObLSID::SYS_LS_ID), replicas))) { - LOG_WARN("failed to init tablet info", K(ret)); - } - } - } - return ret; -} - int ObTabletMetaTableStorage::batch_get( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, ObIArray &tablet_infos) { int ret = OB_SUCCESS; @@ -212,9 +118,9 @@ int ObTabletMetaTableStorage::batch_get( if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (tablet_ls_pairs.count() <= 0) { + } else if (tablet_ids.count() <= 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), "count", tablet_ls_pairs.count()); + LOG_WARN("invalid argument", K(ret), "count", tablet_ids.count()); } else { // Build SQL with IN clause ObSqlString sql; @@ -225,12 +131,12 @@ int ObTabletMetaTableStorage::batch_get( "WHERE tablet_id IN ("))) { LOG_WARN("failed to append sql", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ls_pairs.count(); ++i) { - const ObTabletLSPair &pair = tablet_ls_pairs.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); if (OB_FAIL(sql.append_fmt( "%s %ld", i == 0 ? "" : ",", - pair.get_tablet_id().id()))) { + tablet_id.id()))) { LOG_WARN("failed to append sql", K(ret)); } } @@ -242,14 +148,14 @@ int ObTabletMetaTableStorage::batch_get( } if (OB_SUCC(ret)) { - ObArray replicas; + ObArray tablet_meta_rows; ObTabletReplicaConstructor constructor; auto row_processor = [&](share::ObSQLiteRowReader &reader) -> int { ObTabletReplica replica; if (OB_FAIL(constructor(reader, replica))) { LOG_WARN("failed to construct tablet replica", K(ret)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); + } else if (OB_FAIL(tablet_meta_rows.push_back(replica))) { + LOG_WARN("failed to push back tablet meta row", K(ret)); } return ret; }; @@ -264,8 +170,8 @@ int ObTabletMetaTableStorage::batch_get( } else { ret = OB_SUCCESS; // No rows is acceptable } - } else if (OB_FAIL(group_replicas_to_tablet_infos(replicas, tablet_infos))) { - LOG_WARN("failed to group replicas to tablet infos", K(ret)); + } else if (OB_FAIL(build_tablet_infos_from_rows(tablet_meta_rows, tablet_infos))) { + LOG_WARN("failed to build tablet infos from tablet meta rows", K(ret)); } } } @@ -299,14 +205,14 @@ int ObTabletMetaTableStorage::range_get(const common::ObTabletID &start_tablet_i return OB_SUCCESS; }; - ObArray replicas; + ObArray tablet_meta_rows; ObTabletReplicaConstructor constructor; auto row_processor = [&](share::ObSQLiteRowReader &reader) -> int { ObTabletReplica replica; if (OB_FAIL(constructor(reader, replica))) { LOG_WARN("failed to construct tablet replica", K(ret)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); + } else if (OB_FAIL(tablet_meta_rows.push_back(replica))) { + LOG_WARN("failed to push back tablet meta row", K(ret)); } return ret; }; @@ -321,8 +227,8 @@ int ObTabletMetaTableStorage::range_get(const common::ObTabletID &start_tablet_i } else { ret = OB_SUCCESS; // No rows is acceptable } - } else if (OB_FAIL(group_replicas_to_tablet_infos(replicas, tablet_infos))) { - LOG_WARN("failed to group replicas to tablet infos", K(ret)); + } else if (OB_FAIL(build_tablet_infos_from_rows(tablet_meta_rows, tablet_infos))) { + LOG_WARN("failed to build tablet infos from tablet meta rows", K(ret)); } } return ret; @@ -533,7 +439,6 @@ int ObTabletMetaTableStorage::remove_residual_tablet( } int ObTabletMetaTableStorage::get_max_data_size(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &data_size) { int ret = OB_SUCCESS; @@ -576,7 +481,6 @@ int ObTabletMetaTableStorage::get_max_data_size(const common::ObTabletID &tablet } int ObTabletMetaTableStorage::get_max_report_scn_and_status(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &report_scn, int64_t &status) { @@ -688,7 +592,7 @@ int ObTabletMetaTableStorage::get_tablet_replica_cnt(int64_t &tablet_replica_cnt return ret; } -int ObTabletMetaTableStorage::batch_update_status(const ObIArray &tablet_ls_pairs, +int ObTabletMetaTableStorage::batch_update_status(const ObIArray &tablet_ids, const ObIArray &compaction_scns, const int64_t status, int64_t &affected_rows) @@ -698,9 +602,9 @@ int ObTabletMetaTableStorage::batch_update_status(const ObIArray if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (tablet_ls_pairs.count() != compaction_scns.count()) { + } else if (tablet_ids.count() != compaction_scns.count()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), "pairs count", tablet_ls_pairs.count(), "scns count", compaction_scns.count()); + LOG_WARN("invalid argument", K(ret), "tablet count", tablet_ids.count(), "scns count", compaction_scns.count()); } else { ObSQLiteConnectionGuard guard(pool_); if (!guard) { @@ -709,8 +613,8 @@ int ObTabletMetaTableStorage::batch_update_status(const ObIArray } else if (OB_FAIL(guard->begin_transaction())) { LOG_WARN("failed to begin transaction", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ls_pairs.count(); ++i) { - const ObTabletLSPair &pair = tablet_ls_pairs.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); const int64_t compaction_scn = compaction_scns.at(i); const char *update_sql = "UPDATE __all_tablet_meta_table " @@ -718,7 +622,7 @@ int ObTabletMetaTableStorage::batch_update_status(const ObIArray "WHERE tablet_id = ? AND compaction_scn = ?;"; auto binder = [&](share::ObSQLiteBinder &b) -> int { - b.bind_int64(status); b.bind_int64(pair.get_tablet_id().id()); + b.bind_int64(status); b.bind_int64(tablet_id.id()); b.bind_int64(compaction_scn); return OB_SUCCESS; }; @@ -746,7 +650,7 @@ int ObTabletMetaTableStorage::batch_update_status(const ObIArray } int ObTabletMetaTableStorage::batch_update_report_scn( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, const uint64_t report_scn, const uint64_t compaction_scn_min, const int64_t except_status, @@ -757,9 +661,9 @@ int ObTabletMetaTableStorage::batch_update_report_scn( if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (tablet_ls_pairs.count() <= 0) { + } else if (tablet_ids.count() <= 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), "pairs count", tablet_ls_pairs.count()); + LOG_WARN("invalid argument", K(ret), "tablet count", tablet_ids.count()); } else { ObSQLiteConnectionGuard guard(pool_); if (!guard) { @@ -777,12 +681,12 @@ int ObTabletMetaTableStorage::batch_update_report_scn( report_scn))) { LOG_WARN("failed to append sql", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ls_pairs.count(); ++i) { - const ObTabletLSPair &pair = tablet_ls_pairs.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); if (OB_FAIL(sql.append_fmt( "%s %ld", i == 0 ? "" : ",", - pair.get_tablet_id().id()))) { + tablet_id.id()))) { LOG_WARN("failed to append sql", K(ret)); } } @@ -812,7 +716,7 @@ int ObTabletMetaTableStorage::batch_update_report_scn( return ret; } -int ObTabletMetaTableStorage::batch_update_report_scn_unequal(const ObLSID &ls_id, +int ObTabletMetaTableStorage::batch_update_report_scn_unequal( const ObIArray &tablet_ids, const uint64_t major_frozen_scn, int64_t &affected_rows) @@ -983,7 +887,7 @@ int ObTabletMetaTableStorage::get_distinct_tablet_ids(const common::ObTabletID & return ret; } -int ObTabletMetaTableStorage::get_distinct_tablet_ids_with_conditions(const ObLSID &ls_id, +int ObTabletMetaTableStorage::get_distinct_tablet_ids_with_conditions( const ObIArray &tablet_ids, const uint64_t report_scn_max, ObIArray &result_tablet_ids) @@ -1113,14 +1017,14 @@ int ObTabletMetaTableStorage::range_scan_for_compaction(const common::ObTabletID } if (OB_SUCC(ret)) { - ObArray replicas; + ObArray tablet_meta_rows; ObTabletReplicaConstructor constructor; auto row_processor = [&](share::ObSQLiteRowReader &reader) -> int { ObTabletReplica replica; if (OB_FAIL(constructor(reader, replica))) { LOG_WARN("failed to construct tablet replica", K(ret)); - } else if (OB_FAIL(replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); + } else if (OB_FAIL(tablet_meta_rows.push_back(replica))) { + LOG_WARN("failed to push back tablet meta row", K(ret)); } return ret; }; @@ -1135,63 +1039,44 @@ int ObTabletMetaTableStorage::range_scan_for_compaction(const common::ObTabletID } else { ret = OB_SUCCESS; // No rows is acceptable } - } else if (OB_FAIL(group_replicas_to_tablet_infos(replicas, tablet_infos))) { - LOG_WARN("failed to group replicas to tablet infos", K(ret)); + } else if (OB_FAIL(build_tablet_infos_from_rows(tablet_meta_rows, tablet_infos))) { + LOG_WARN("failed to build tablet infos from tablet meta rows", K(ret)); } } } return ret; } -int ObTabletMetaTableStorage::group_replicas_to_tablet_infos( - const ObIArray &replicas, +int ObTabletMetaTableStorage::build_tablet_infos_from_rows( + const ObIArray &tablet_meta_rows, ObIArray &tablet_infos) { int ret = OB_SUCCESS; tablet_infos.reset(); - if (replicas.count() <= 0) { + if (tablet_meta_rows.count() <= 0) { // Empty result, return empty array } else { ObTabletInfo current_tablet_info; - ObArray current_replicas; - for (int64_t i = 0; OB_SUCC(ret) && i < replicas.count(); ++i) { - const ObTabletReplica &replica = replicas.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_meta_rows.count(); ++i) { + const ObTabletReplica &replica = tablet_meta_rows.at(i); if (!current_tablet_info.is_valid()) { - // First replica, start new tablet info - current_replicas.reset(); - if (OB_FAIL(current_replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); - } else if (OB_FAIL(current_tablet_info.init( - replica.get_tablet_id(), - replica.get_ls_id(), - current_replicas))) { + if (OB_FAIL(current_tablet_info.init_by_replica(replica))) { LOG_WARN("failed to init tablet info", K(ret)); } - } else if (current_tablet_info.get_tablet_id() == replica.get_tablet_id() && - current_tablet_info.get_ls_id() == replica.get_ls_id()) { - // Same tablet, add replica - if (OB_FAIL(current_tablet_info.add_replica(replica))) { - LOG_WARN("failed to add replica", K(ret)); - } + } else if (current_tablet_info.get_tablet_id() == replica.get_tablet_id()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet meta table has more than one local row", K(ret), K(replica), K(current_tablet_info)); } else { // Different tablet, save current and start new if (OB_FAIL(tablet_infos.push_back(current_tablet_info))) { LOG_WARN("failed to push back tablet info", K(ret)); } current_tablet_info.reset(); - current_replicas.reset(); - if (OB_SUCC(ret)) { - if (OB_FAIL(current_replicas.push_back(replica))) { - LOG_WARN("failed to push back replica", K(ret)); - } else if (OB_FAIL(current_tablet_info.init( - replica.get_tablet_id(), - replica.get_ls_id(), - current_replicas))) { - LOG_WARN("failed to init tablet info", K(ret)); - } + if (OB_SUCC(ret) && OB_FAIL(current_tablet_info.init_by_replica(replica))) { + LOG_WARN("failed to init tablet info", K(ret), K(replica)); } } } diff --git a/src/share/tablet/ob_tablet_meta_table_storage.h b/src/share/tablet/ob_tablet_meta_table_storage.h index ae5f37e6b..ae0ad7bd1 100644 --- a/src/share/tablet/ob_tablet_meta_table_storage.h +++ b/src/share/tablet/ob_tablet_meta_table_storage.h @@ -20,7 +20,6 @@ #include "share/tablet/ob_tablet_info.h" #include "lib/container/ob_iarray.h" #include "share/storage/ob_sqlite_connection_pool.h" -#include namespace oceanbase { @@ -45,22 +44,9 @@ class ObTabletMetaTableStorage bool is_inited() const { return nullptr != pool_; } - // Get tablet replica by primary keys - int get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObAddr &addr, - ObTabletReplica &tablet_replica); - - // Get tablet info (all replicas for a tablet) - int get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - ObTabletInfo &tablet_info); - // Batch get tablet infos int batch_get( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, ObIArray &tablet_infos); // Range get tablet infos @@ -72,16 +58,16 @@ class ObTabletMetaTableStorage int batch_update( const ObIArray &replicas); - // Batch update replicas within an external transaction + // Batch update tablet meta rows within an external transaction int batch_update( ObSQLiteConnection *conn, const ObIArray &replicas); - // Batch remove replicas + // Batch remove tablet meta rows int batch_remove( const ObIArray &replicas); - // Batch remove replicas within an external transaction + // Batch remove tablet meta rows within an external transaction int batch_remove( ObSQLiteConnection *conn, const ObIArray &replicas); @@ -92,14 +78,12 @@ class ObTabletMetaTableStorage const int64_t limit, int64_t &affected_rows); - // Get max data_size for a tablet-ls pair + // Get max data_size for a tablet int get_max_data_size(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &data_size); - // Get max report_scn and max status for a tablet-ls pair + // Get max report_scn and max status for a tablet int get_max_report_scn_and_status(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, int64_t &report_scn, int64_t &status); @@ -109,15 +93,15 @@ class ObTabletMetaTableStorage // Get tablet replica count for a tenant int get_tablet_replica_cnt(int64_t &tablet_replica_cnt); - // Batch update status for specific tablet-ls pairs with compaction_scn - int batch_update_status(const ObIArray &tablet_ls_pairs, + // Batch update status for specific tablets with compaction_scn + int batch_update_status(const ObIArray &tablet_ids, const ObIArray &compaction_scns, const int64_t status, int64_t &affected_rows); // Batch update report_scn for tablets int batch_update_report_scn( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, const uint64_t report_scn, const uint64_t compaction_scn_min, const int64_t except_status, @@ -143,8 +127,8 @@ class ObTabletMetaTableStorage const int64_t limit, ObIArray &tablet_ids); - // Get distinct tablet_ids for a tenant-ls with conditions - int get_distinct_tablet_ids_with_conditions(const ObLSID &ls_id, + // Get distinct tablet_ids with conditions + int get_distinct_tablet_ids_with_conditions( const ObIArray &tablet_ids, const uint64_t report_scn_max, ObIArray &result_tablet_ids); @@ -162,7 +146,7 @@ class ObTabletMetaTableStorage ObIArray &tablet_infos); // Batch update report_scn for tablets with conditions (for unequal report_scn update) - int batch_update_report_scn_unequal(const ObLSID &ls_id, + int batch_update_report_scn_unequal( const ObIArray &tablet_ids, const uint64_t major_frozen_scn, int64_t &affected_rows); @@ -171,9 +155,9 @@ class ObTabletMetaTableStorage // Create table if not exists int create_table_if_not_exists(); - // Helper to construct tablet infos from replicas (grouping logic) - int group_replicas_to_tablet_infos( - const ObIArray &replicas, + // Helper to construct tablet infos from local tablet meta rows. + int build_tablet_infos_from_rows( + const ObIArray &tablet_meta_rows, ObIArray &tablet_infos); ObSQLiteConnectionPool *pool_; @@ -183,4 +167,3 @@ class ObTabletMetaTableStorage } // namespace oceanbase #endif // OCEANBASE_SHARE_TABLET_OB_TABLET_META_TABLE_STORAGE_H_ - diff --git a/src/share/tablet/ob_tablet_table_iterator.cpp b/src/share/tablet/ob_tablet_table_iterator.cpp index ffbc4f821..aefc0594c 100644 --- a/src/share/tablet/ob_tablet_table_iterator.cpp +++ b/src/share/tablet/ob_tablet_table_iterator.cpp @@ -17,7 +17,7 @@ #define USING_LOG_PREFIX SHARE #include "share/tablet/ob_tablet_table_iterator.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "share/ob_tablet_meta_table_compaction_operator.h" namespace oceanbase @@ -70,7 +70,7 @@ int ObTabletMetaIterator::next(ObTabletInfo &tablet_info) tablet_info.reset(); if (OB_FAIL(tablet_info.assign(prefetched_tablets_.at(prefetch_tablet_idx_)))) { LOG_WARN("fail to assign tablet_info", KR(ret), K_(prefetch_tablet_idx)); - } else if (tablet_info.replica_count() > 0) { + } else if (tablet_info.has_replica()) { // if (OB_FAIL(tablet_info.filter(filters_))) { LOG_WARN("fail to filter tablet_info", KR(ret), K(tablet_info)); @@ -121,13 +121,9 @@ int ObCompactionTabletMetaIterator::next(ObTabletInfo &tablet_info) if (OB_ITER_END != ret) { LOG_WARN("fail to get next tablet info", KR(ret)); } - } else if (!tablet_info.is_valid()) { - if (tablet_info.get_replicas().empty()) { - // ObTabletMetaIterator::next may fillter some replica members and make tablet_info invalid, skip and fetch next one - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet_info is invalid", KR(ret), K(tablet_info)); - } + } else if (!tablet_info.is_valid() && tablet_info.has_replica()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet_info is invalid", KR(ret), K(tablet_info)); } } while (OB_SUCC(ret) && !tablet_info.is_valid()); return ret; @@ -181,8 +177,7 @@ int ObCompactionTabletMetaIterator::prefetch() ObTenantTabletMetaIterator::ObTenantTabletMetaIterator() : ObTabletMetaIterator(), first_prefetch_(true), - sql_proxy_(nullptr), - valid_tablet_ls_pairs_idx_(0) + sql_proxy_(nullptr) { } @@ -200,8 +195,7 @@ int ObTenantTabletMetaIterator::init( LOG_WARN("fail to init tablet table operator", KR(ret)); } else { sql_proxy_ = &sql_proxy; - valid_tablet_ls_pairs_.reuse(); - valid_tablet_ls_pairs_idx_ = 0; + valid_tablet_ids_.reuse(); is_inited_ = true; if (OB_FAIL(prefetch())) { // need to prefetch a batch of tablet_info if (OB_ITER_END != ret) { @@ -217,8 +211,7 @@ void ObTenantTabletMetaIterator::reset() ObTabletMetaIterator::reset(); first_prefetch_ = true; sql_proxy_ = nullptr; - valid_tablet_ls_pairs_idx_ = -1; - valid_tablet_ls_pairs_.reset(); + valid_tablet_ids_.reset(); tablet_table_operator_.reset(); } @@ -228,7 +221,7 @@ int ObTenantTabletMetaIterator::prefetch() if (OB_FAIL(prefetch_valid_tablet_ids())) { if (OB_ITER_END != ret) { LOG_WARN("fail to prefetch valid tablet ids", KR(ret), - K_(valid_tablet_ls_pairs_idx), K_(valid_tablet_ls_pairs)); + K_(valid_tablet_ids)); } } else if (OB_FAIL(prefetch_tablets())) { LOG_WARN("fail to prefetch tablets", KR(ret), @@ -243,8 +236,7 @@ int ObTenantTabletMetaIterator::prefetch_valid_tablet_ids() if (OB_UNLIKELY(prefetch_tablet_idx_ != prefetched_tablets_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("prefetched valid tablet ids have not been iterated to end", - KR(ret), K_(valid_tablet_ls_pairs_idx), "tablet ls pair count", - valid_tablet_ls_pairs_.count()); + KR(ret), "tablet id count", valid_tablet_ids_.count()); } else { // for any tenant, firstly, get its sys tables' tablet_ids, then get user // tables' tablet_ids if it has user tables. @@ -274,11 +266,10 @@ int ObTenantTabletMetaIterator::prefetch_tablets() } else { prefetch_tablet_idx_ = 0; prefetched_tablets_.reuse(); - - if (OB_FAIL(tablet_table_operator_.batch_get(valid_tablet_ls_pairs_, - prefetched_tablets_))) { + if (OB_FAIL(tablet_table_operator_.batch_get(valid_tablet_ids_, + prefetched_tablets_))) { LOG_WARN("fail to do batch_get through tablet_table_operator", KR(ret), - K_(valid_tablet_ls_pairs), K_(prefetched_tablets)); + K_(valid_tablet_ids), K_(prefetched_tablets)); } } return ret; @@ -294,7 +285,6 @@ int ObTenantTabletMetaIterator::prefetch_sys_table_tablet_ids() } else if (OB_FAIL(schema_guard.get_table_schemas_in_tenant(table_schemas))) { LOG_WARN("fail to get tenant table schemas", KR(ret)); } else { - ObTabletLSPair pair; for (int64_t i = 0; (i < table_schemas.count()) && OB_SUCC(ret); ++i) { const ObSimpleTableSchemaV2 *simple_schema = table_schemas.at(i); if (OB_ISNULL(simple_schema)) { @@ -305,11 +295,8 @@ int ObTenantTabletMetaIterator::prefetch_sys_table_tablet_ids() if (is_sys_table(tid)) { // sys-table only has one tablet in 4.0 version const ObTabletID &tablet_id = simple_schema->get_tablet_id(); - pair.reset(); - if (OB_FAIL(pair.init(tablet_id, SYS_LS))) { - LOG_WARN("fail to init tablet_ls_pair", KR(ret), K(tablet_id)); - } else if (OB_FAIL(valid_tablet_ls_pairs_.push_back(pair))) { - LOG_WARN("fail to push back tablet_ls_pair", KR(ret), K(pair)); + if (OB_FAIL(valid_tablet_ids_.push_back(tablet_id))) { + LOG_WARN("fail to push back tablet id", KR(ret), K(tablet_id)); } } } @@ -322,23 +309,31 @@ int ObTenantTabletMetaIterator::prefetch_user_table_tablet_ids() { int ret = OB_SUCCESS; ObTabletID start_tablet_id; - if (valid_tablet_ls_pairs_.count() <= 0) { - // start with default invalid ObTabletLSPair(0,-1) in the first time + if (valid_tablet_ids_.count() <= 0) { + // start with default invalid tablet id in the first time } else { - const int64_t last_idx = valid_tablet_ls_pairs_.count() - 1; - start_tablet_id = valid_tablet_ls_pairs_.at(last_idx).get_tablet_id(); + const int64_t last_idx = valid_tablet_ids_.count() - 1; + start_tablet_id = valid_tablet_ids_.at(last_idx); } const int64_t range_count = GCONF.tablet_meta_table_scan_batch_count; - valid_tablet_ls_pairs_.reuse(); - if (OB_FAIL(ObTabletToLSTableOperator::range_get_tablet( + valid_tablet_ids_.reuse(); + ObArray tablet_infos; + if (OB_FAIL(ObTabletMappingTableOperator::range_get_tablet_table_pairs( *sql_proxy_, start_tablet_id, range_count, - valid_tablet_ls_pairs_))) { - LOG_WARN("fail to get a range of tablet through tablet_to_ls_table_operator", + tablet_infos))) { + LOG_WARN("fail to get a range of tablet through tablet mapping operator", KR(ret), K(start_tablet_id), K(range_count), - K_(valid_tablet_ls_pairs)); - } else if (valid_tablet_ls_pairs_.count() <= 0) { + K_(valid_tablet_ids)); + } + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_infos.count(); ++idx) { + if (OB_FAIL(valid_tablet_ids_.push_back(tablet_infos.at(idx).get_tablet_id()))) { + LOG_WARN("fail to push tablet id", KR(ret), K(idx), K(tablet_infos)); + } + } + if (OB_FAIL(ret)) { + } else if (valid_tablet_ids_.count() <= 0) { ret = OB_ITER_END; } return ret; diff --git a/src/share/tablet/ob_tablet_table_iterator.h b/src/share/tablet/ob_tablet_table_iterator.h index 1e1a4d30b..43b9430c6 100644 --- a/src/share/tablet/ob_tablet_table_iterator.h +++ b/src/share/tablet/ob_tablet_table_iterator.h @@ -28,7 +28,7 @@ class ObISQLClient; } namespace share { -class ObTabletToLSTableOperator; +class ObTabletMappingTableOperator; class ObTabletTableOperator; class ObTabletInfo; @@ -92,8 +92,7 @@ class ObTenantTabletMetaIterator : public ObTabletMetaIterator private: bool first_prefetch_; common::ObISQLClient *sql_proxy_; - int64_t valid_tablet_ls_pairs_idx_; - common::ObArray valid_tablet_ls_pairs_; + common::ObArray valid_tablet_ids_; ObTabletTableOperator tablet_table_operator_; private: DISALLOW_COPY_AND_ASSIGN(ObTenantTabletMetaIterator); diff --git a/src/share/tablet/ob_tablet_table_operator.cpp b/src/share/tablet/ob_tablet_table_operator.cpp index de922b774..f0738aee6 100644 --- a/src/share/tablet/ob_tablet_table_operator.cpp +++ b/src/share/tablet/ob_tablet_table_operator.cpp @@ -101,51 +101,9 @@ void ObTabletTableOperator::reset() batch_size_ = 0; } -int ObTabletTableOperator::get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObAddr &addr, - ObTabletReplica &tablet_replica) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - ret = storage_.get(tablet_id, ls_id, addr, tablet_replica); - if (OB_FAIL(ret)) { - LOG_WARN("failed to get tablet replica from storage", K(ret), K(tablet_id), K(ls_id), K(addr)); - } - } - return ret; -} - -// will fill empty tablet_info when tablet not exist -int ObTabletTableOperator::get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - ObTabletInfo &tablet_info) -{ - int ret = OB_SUCCESS; - tablet_info.reset(); - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - ret = storage_.get(tablet_id, ls_id, tablet_info); - if (OB_FAIL(ret) && OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("fail to get tablet info from storage", KR(ret), K(tablet_id), K(ls_id)); - } else if (OB_ENTRY_NOT_EXIST == ret) { - // Return empty tablet_info when not exist - ret = OB_SUCCESS; - } - } - return ret; -} - int ObTabletTableOperator::batch_get_tablet_info( common::ObISQLClient *sql_proxy, - const ObIArray &tablet_ls_infos, + const ObIArray &tablet_check_infos, const int32_t group_id, ObArrayWithMap &tablet_infos) { @@ -155,17 +113,16 @@ int ObTabletTableOperator::batch_get_tablet_info( if (OB_FAIL(get_shared_storage(storage))) { LOG_WARN("failed to get shared storage", K(ret)); } else { - // Convert ObTabletCheckInfo to ObTabletLSPair - ObSEArray tablet_ls_pairs; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ls_infos.count(); ++i) { - const compaction::ObTabletCheckInfo &check_info = tablet_ls_infos.at(i); - if (OB_FAIL(tablet_ls_pairs.push_back(ObTabletLSPair(check_info.get_tablet_id(), check_info.get_ls_id())))) { - LOG_WARN("failed to push back tablet ls pair", K(ret)); + ObSEArray tablet_ids; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_check_infos.count(); ++i) { + const compaction::ObTabletCheckInfo &check_info = tablet_check_infos.at(i); + if (OB_FAIL(tablet_ids.push_back(check_info.get_tablet_id()))) { + LOG_WARN("failed to push back tablet id", K(ret), K(check_info)); } } if (OB_SUCC(ret)) { ObSEArray infos; - if (OB_FAIL(storage->batch_get(tablet_ls_pairs, infos))) { + if (OB_FAIL(storage->batch_get(tablet_ids, infos))) { LOG_WARN("failed to batch get from storage", K(ret)); } else { // Convert to ObArrayWithMap @@ -181,74 +138,69 @@ int ObTabletTableOperator::batch_get_tablet_info( } int ObTabletTableOperator::batch_get( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, ObIArray &tablet_infos) { int ret = OB_SUCCESS; tablet_infos.reset(); - const int64_t pairs_cnt = tablet_ls_pairs.count(); - hash::ObHashMap pairs_map; + const int64_t tablet_cnt = tablet_ids.count(); + hash::ObHashMap tablet_map; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(pairs_cnt < 1)) { + } else if (OB_UNLIKELY(tablet_cnt < 1)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(pairs_cnt)); + LOG_WARN("invalid argument", KR(ret), K(tablet_cnt)); } // Step 1: check duplicates by hash map - if (FAILEDx(pairs_map.create( - hash::cal_next_prime(pairs_cnt * 2), + if (FAILEDx(tablet_map.create( + hash::cal_next_prime(tablet_cnt * 2), ObModIds::OB_HASH_BUCKET))) { - LOG_WARN("fail to create pairs_map", KR(ret), K(pairs_cnt)); + LOG_WARN("fail to create tablet_map", KR(ret), K(tablet_cnt)); } else { - ARRAY_FOREACH_N(tablet_ls_pairs, idx, cnt) { - // if same talet_id exist, return error - if (OB_FAIL(pairs_map.set_refactored(tablet_ls_pairs.at(idx), false))) { + ARRAY_FOREACH_N(tablet_ids, idx, cnt) { + // if same tablet_id exists, return error + if (OB_FAIL(tablet_map.set_refactored(tablet_ids.at(idx), false))) { if (OB_HASH_EXIST == ret) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablet_ls_pairs have duplicates", KR(ret), K(tablet_ls_pairs), K(idx)); + LOG_WARN("tablet ids have duplicates", KR(ret), K(tablet_ids), K(idx)); } else { - LOG_WARN("fail to set refactored", KR(ret), K(tablet_ls_pairs), K(idx)); + LOG_WARN("fail to set refactored", KR(ret), K(tablet_ids), K(idx)); } } } // end for if (OB_FAIL(ret)) { - } else if (pairs_map.size() != pairs_cnt) { + } else if (tablet_map.size() != tablet_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid pairs_map size", "size", pairs_map.size(), K(pairs_cnt)); + LOG_WARN("invalid tablet_map size", "size", tablet_map.size(), K(tablet_cnt)); } } // Step 2: get from SQLite storage if (OB_SUCC(ret)) { - if (OB_FAIL(storage_.batch_get(tablet_ls_pairs, tablet_infos))) { - LOG_WARN("fail to batch get from storage", KR(ret), K(tablet_ls_pairs)); + if (OB_FAIL(storage_.batch_get(tablet_ids, tablet_infos))) { + LOG_WARN("fail to batch get from storage", KR(ret), K(tablet_ids)); } } // Step 3: check tablet_infos and push back empty tablet_info for tablets not exist - if (OB_SUCC(ret) && (tablet_infos.count() < pairs_cnt)) { + if (OB_SUCC(ret) && (tablet_infos.count() < tablet_cnt)) { // check tablet infos and set flag in map int overwrite_flag = 1; ARRAY_FOREACH_N(tablet_infos, idx, cnt) { const ObTabletID &tablet_id = tablet_infos.at(idx).get_tablet_id(); - const ObLSID &ls_id = tablet_infos.at(idx).get_ls_id(); - if (OB_FAIL(pairs_map.set_refactored(ObTabletLSPair(tablet_id, ls_id), true, overwrite_flag))) { - LOG_WARN("fail to set_fefactored", KR(ret), K(tablet_id), K(ls_id)); + if (OB_FAIL(tablet_map.set_refactored(tablet_id, true, overwrite_flag))) { + LOG_WARN("fail to set_fefactored", KR(ret), K(tablet_id)); } } // push back empty tablet_info if (OB_SUCC(ret)) { - FOREACH_X(iter, pairs_map, OB_SUCC(ret)) { + FOREACH_X(iter, tablet_map, OB_SUCC(ret)) { if (!iter->second) { - ObArray replica; // empty replica - ObTabletInfo tablet_info( - iter->first.get_tablet_id(), - iter->first.get_ls_id(), - replica); + ObTabletInfo tablet_info(iter->first); if (OB_FAIL(tablet_infos.push_back(tablet_info))) { LOG_WARN("fail to push back tablet info", KR(ret), K(tablet_info)); } LOG_TRACE("tablet not exist in meta table", - KR(ret), "tablet_id", iter->first); + KR(ret), K(iter->first)); } } } @@ -256,111 +208,6 @@ int ObTabletTableOperator::batch_get( return ret; } -int ObTabletTableOperator::construct_tablet_infos( - sqlclient::ObMySQLResult &res, - std::function &&push_tablet) -{ - int ret = OB_SUCCESS; - ObTabletInfo tablet_info; - ObTabletReplica replica; - while (OB_SUCC(ret)) { - if (OB_FAIL(res.next())) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("get next result failed", KR(ret)); - } - break; - } else { - replica.reset(); - if (OB_FAIL(construct_tablet_replica_(res, replica))) { - LOG_WARN("fail to construct tablet replica", KR(ret)); - } else if (OB_UNLIKELY(!replica.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("construct invalid replica", KR(ret), K(replica)); - } else if (tablet_info.is_self_replica(replica)) { - if (OB_FAIL(tablet_info.add_replica(replica))) { - LOG_WARN("fail to add replica", KR(ret), K(replica)); - } - } else { - if (tablet_info.is_valid()) { - if (OB_FAIL(push_tablet(tablet_info))) { - LOG_WARN("fail to push back", KR(ret), K(tablet_info)); - } - } - tablet_info.reset(); - if (FAILEDx(tablet_info.init_by_replica(replica))) { - LOG_WARN("fail to init tablet_info by replica", KR(ret), K(replica)); - } - } - } - } // end while - if (OB_SUCC(ret) && tablet_info.is_valid()) { - // last tablet info - if (OB_FAIL(push_tablet(tablet_info))) { - LOG_WARN("fail to push back", KR(ret), K(tablet_info)); - } - } - return ret; -} - -int ObTabletTableOperator::construct_tablet_replica_( - sqlclient::ObMySQLResult &res, - ObTabletReplica &replica) -{ - int ret = OB_SUCCESS; - int64_t tablet_id = ObTabletID::INVALID_TABLET_ID; - common::ObAddr server; - ObString ip; - int64_t port = OB_INVALID_INDEX; - int64_t ls_id = OB_INVALID_ID; - uint64_t uint_compaction_scn = 0; - int64_t compaction_scn = 0; - int64_t data_size = 0; - int64_t required_size = 0; - uint64_t uint_report_scn = 0; - int64_t status_in_table = 0; - ObTabletReplica::ScnStatus status = ObTabletReplica::SCN_STATUS_IDLE; - bool skip_null_error = false; - bool skip_column_error = true; - - (void) GET_COL_IGNORE_NULL(res.get_int, "tablet_id", tablet_id); - (void) GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void) GET_COL_IGNORE_NULL(res.get_varchar, "svr_ip", ip); - (void) GET_COL_IGNORE_NULL(res.get_int, "svr_port", port); - (void) GET_COL_IGNORE_NULL(res.get_uint, "compaction_scn", uint_compaction_scn); - (void) GET_COL_IGNORE_NULL(res.get_int, "data_size", data_size); - (void) GET_COL_IGNORE_NULL(res.get_int, "required_size", required_size); - - EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "report_scn", uint_report_scn, uint64_t, skip_null_error, skip_column_error, 0); - EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "status", status_in_table, int64_t, skip_null_error, skip_column_error, ObTabletReplica::SCN_STATUS_IDLE); - - status = (ObTabletReplica::ScnStatus)status_in_table; - compaction_scn = static_cast(uint_compaction_scn); - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(!server.set_ip_addr(ip, static_cast(port)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid server address", KR(ret), K(ip), K(port)); - } else if (OB_UNLIKELY(!ObTabletReplica::is_status_valid(status))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid status", K(ret), K(status_in_table)); - } else if (OB_FAIL( - replica.init( - ObTabletID(tablet_id), - share::ObLSID(ls_id), - server, - compaction_scn, - data_size, - required_size, - (int64_t)uint_report_scn, - status))) { - LOG_WARN("fail to init replica", KR(ret), - K(tablet_id), K(server), K(ls_id), K(data_size), K(required_size)); - } - LOG_TRACE("construct tablet replica", KR(ret), K(replica)); - return ret; -} - int ObTabletTableOperator::batch_update( const ObIArray &replicas) diff --git a/src/share/tablet/ob_tablet_table_operator.h b/src/share/tablet/ob_tablet_table_operator.h index fcd2a7f05..f38e5a4e7 100644 --- a/src/share/tablet/ob_tablet_table_operator.h +++ b/src/share/tablet/ob_tablet_table_operator.h @@ -27,21 +27,13 @@ namespace oceanbase { namespace common { -class ObMySQLProxy; class ObISQLClient; class ObAddr; class ObTabletID; - -namespace sqlclient -{ -class ObMySQLResult; -} } namespace share { class ObDMLSqlSplicer; -class ObTabletLSPair; -class ObLSID; // Operator for __all_tablet_meta_table. // Providing (batch-)get, (batch-)update, remove capabilities by sql. @@ -58,33 +50,16 @@ class ObTabletTableOperator int init(share::ObSQLiteConnectionPool *pool); void reset(); void set_batch_size(int64_t batch_size) {batch_size_ = batch_size;} - int get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - const common::ObAddr &addr, - ObTabletReplica &tablet_replica); - // get ObTabletInfo according to tablet_id - // - // @param [in] tenant, tenant for query - // @param [in] tablet_id, tablet id for query - // @param [in] ls_id, the ls which tablet belongs to - // @param [out] tablet_info, tablet info get from __all_tablet_meta_table. - // return empty tablet_info if tablet does not exist in meta table. - int get( - const common::ObTabletID &tablet_id, - const ObLSID &ls_id, - ObTabletInfo &tablet_info); // update ObTabletReplica into __all_tablet_meta_table // // @param [in] replica, ObTabletReplica for update - // batch get ObTabletInfos according to tenant, ls_id and tablet_ids + // batch get ObTabletInfos according to tablet_ids // - // @param [in] tenant, tenant for query - // @param [in] tablet_ls_pairs, tablet_id with ls_id + // @param [in] tablet_ids, tablet ids for query // @param [out] tablet_infos, array of tablet infos from __all_tablet_meta_table. // @return empty tablet_info if tablet does not exist in meta table. int batch_get( - const ObIArray &tablet_ls_pairs, + const ObIArray &tablet_ids, ObIArray &tablet_infos); // range get tablet infos from start_tablet_id // @@ -97,31 +72,30 @@ class ObTabletTableOperator int range_get(const common::ObTabletID &start_tablet_id, const int64_t range_size, ObIArray &tablet_infos); - // batch update replicas into __all_tablet_meta_table + // batch update tablet meta rows into __all_tablet_meta_table // // @param [in] tenant, tenant for query // @param [in] replicas, ObTabletReplicas for updating(should belong to the same tenant!) int batch_update( const ObIArray &replicas); - // batch update replicas within an external SQLite transaction + // batch update tablet meta rows within an external SQLite transaction int batch_update( ObSQLiteConnection *conn, const ObIArray &replicas); - // batch remove replicas from __all_tablet_meta_table + // batch remove tablet meta rows from __all_tablet_meta_table // // @param [in] tenant, target tenant // @param [in] replicas, ObTabletReplicas for removing(should belong to the same tenant!) - // (only tenant, tablet_id, ls_id, server are used in this interface) - // Legacy method for backward compatibility (will use SQLite internally) + // (only tablet_id and server are used in this interface) int batch_remove( const ObIArray &replicas); - // batch remove replicas within an external SQLite transaction + // batch remove tablet meta rows within an external SQLite transaction int batch_remove( ObSQLiteConnection *conn, const ObIArray &replicas); // remove residual tablet in __all_tablet_meta_table for ObServerMetaTableChecker // - // @param [in] sql_client, client for executing query (legacy, will use SQLite internally) + // @param [in] sql_client, client for executing query // @param [in] tenant, tenant for query // @param [in] server, target ObAddr // @param [in] limit, limit number for delete sql @@ -131,19 +105,13 @@ class ObTabletTableOperator const ObAddr &server, const int64_t limit, int64_t &affected_rows); - static int construct_tablet_infos( - common::sqlclient::ObMySQLResult &res, - std::function &&push_tablet); public: static int batch_get_tablet_info( common::ObISQLClient *sql_proxy, - const ObIArray &tablet_ls_infos, + const ObIArray &tablet_check_infos, const int32_t group_id, ObArrayWithMap &tablet_infos); private: - static int construct_tablet_replica_( - common::sqlclient::ObMySQLResult &res, - ObTabletReplica &replica); const static int64_t MAX_BATCH_COUNT = 100; bool inited_; ObTabletMetaTableStorage storage_; diff --git a/src/share/tablet/ob_tablet_to_ls_iterator.cpp b/src/share/tablet/ob_tablet_to_ls_iterator.cpp deleted file mode 100644 index feb4249c1..000000000 --- a/src/share/tablet/ob_tablet_to_ls_iterator.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE - -#include "share/tablet/ob_tablet_to_ls_iterator.h" -#include "share/ob_rpc_struct.h" - -namespace oceanbase -{ -namespace share -{ -ObTenantTabletToLSIterator::ObTenantTabletToLSIterator() - : inited_(false), - inner_idx_(0), - ls_white_list_(), - inner_tablet_infos_(), - sql_proxy_(NULL) -{ -} - -int ObTenantTabletToLSIterator::init( - common::ObISQLClient &sql_proxy) -{ - const ObArray ls_white_list; - return init(sql_proxy, ls_white_list); -} - -int ObTenantTabletToLSIterator::init( - common::ObISQLClient &sql_proxy, - const common::ObIArray &ls_white_list) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", KR(ret)); - } else if (OB_FAIL(ls_white_list_.assign(ls_white_list))) { - LOG_WARN("assign LS white list fail", KR(ret), K(ls_white_list)); - } else { - sql_proxy_ = &sql_proxy; - inited_ = true; - } - return ret; -} - -int ObTenantTabletToLSIterator::next(ObTabletLSPair &pair) -{ - int ret = OB_SUCCESS; - ObTabletToLSInfo info; - if (OB_FAIL(next(info))) { - if (OB_ITER_END != ret) { - LOG_WARN("next tablet to LS info fail", KR(ret)); - } - } else if (OB_FAIL(pair.init(info.get_tablet_id(), info.get_ls_id()))) { - LOG_WARN("init ObTabletLSPair fail", KR(ret), K(info)); - } - return ret; -} - -int ObTenantTabletToLSIterator::next(ObTabletToLSInfo &info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(inner_idx_ < 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("inner_idx_ can't be smaller than 0", KR(ret), K_(inner_idx)); - } else { - info.reset(); - if (inner_idx_ >= inner_tablet_infos_.count()) { - if (OB_FAIL(prefetch_())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to prfetch", KR(ret)); - } - } else { - inner_idx_ = 0; - } - } - if (FAILEDx(info.assign(inner_tablet_infos_[inner_idx_]))) { - LOG_WARN("failed to assign tablet to ls info", - KR(ret), K_(inner_idx), K(inner_tablet_infos_[inner_idx_])); - } else { - ++inner_idx_; - } - } - return ret; -} - -int ObTenantTabletToLSIterator::prefetch_() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - ObTabletID last_tablet_id; // start with INVALID_TABLET_ID = 0 - if (inner_tablet_infos_.count() > 0) { - const int64_t last_idx = inner_tablet_infos_.count() - 1; - last_tablet_id = inner_tablet_infos_.at(last_idx).get_tablet_id(); - } - inner_tablet_infos_.reset(); - const int64_t range_size = GCONF.tablet_meta_table_scan_batch_count; - if (OB_FAIL(ObTabletToLSTableOperator::range_get_tablet_info( - *sql_proxy_, - ls_white_list_, - last_tablet_id, - range_size, - inner_tablet_infos_))) { - LOG_WARN("fail to range get by operator", KR(ret), - K_(ls_white_list), K(last_tablet_id), K(range_size), K(inner_tablet_infos_)); - } else if (inner_tablet_infos_.count() <= 0) { - ret = OB_ITER_END; - } - } - return ret; -} - -} // end namespace share -} // end namespace oceanbase diff --git a/src/share/tablet/ob_tablet_to_ls_iterator.h b/src/share/tablet/ob_tablet_to_ls_iterator.h deleted file mode 100644 index ca39ff8e4..000000000 --- a/src/share/tablet/ob_tablet_to_ls_iterator.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_TABLET_OB_TABLET_TO_LS_ITERATOR_H -#define OCEANBASE_SHARE_TABLET_OB_TABLET_TO_LS_ITERATOR_H - -#include "share/tablet/ob_tablet_to_ls_operator.h" - -namespace oceanbase -{ -namespace common -{ -class ObISQLClient; -} -namespace share -{ -class ObTabletToLSTableOperator; - -// ObTenantTabletToLSIterator is used to iterate __all_tablet_to_ls according to target tenant. -class ObTenantTabletToLSIterator -{ -public: - ObTenantTabletToLSIterator(); - virtual ~ObTenantTabletToLSIterator() {} - int init( - common::ObISQLClient &sql_proxy); - // init with LS white list - // ls_white_list: LS white list that only output tablets on the LS white list. - // If list is empty, it means ALL LS are in white list. - int init( - common::ObISQLClient &sql_proxy, - const common::ObIArray &ls_white_list); - int next(ObTabletLSPair &pair); - int next(ObTabletToLSInfo &info); -private: - int prefetch_(); - - bool inited_; - int64_t inner_idx_; - common::ObSEArray ls_white_list_; - common::ObArray inner_tablet_infos_; - common::ObISQLClient *sql_proxy_; -}; - -} // end namespace -} // end namespace oceanbase - -#endif // OCEANBASE_SHARE_TABLET_OB_TABLET_TO_LS_ITERATOR_H diff --git a/src/share/tablet/ob_tablet_to_ls_operator.h b/src/share/tablet/ob_tablet_to_ls_operator.h deleted file mode 100644 index 3ae57b173..000000000 --- a/src/share/tablet/ob_tablet_to_ls_operator.h +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_OB_TABLET_TO_LS_OPERATOR -#define OCEANBASE_SHARE_OB_TABLET_TO_LS_OPERATOR - -#include "lib/container/ob_iarray.h" // ObIArray -#include "share/tablet/ob_tablet_info.h" // ObTabletToLSInfo -#include "share/location_cache/ob_location_struct.h" // ObTabletLSCache - -namespace oceanbase -{ -namespace common -{ -class ObISQLClient; - -namespace sqlclient -{ -class ObMySQLResult; -} -} // end nampspace common - -namespace share -{ -// This operator is used to manipulate inner table __all_tablet_to_ls. -class ObTabletToLSTableOperator -{ -public: - ObTabletToLSTableOperator() {} - virtual ~ObTabletToLSTableOperator() {} - // Get tablets sequentially by range - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] start_tablet_id, starting point of the range (not included in output!) - // Usually start from 0. - // @param [in] range_size, range size of the query - // @param [out] tablet_ls_pairs, sequential tablets' info in __all_tablet_to_ls - // @return OB_SUCCESS if success - static int range_get_tablet( - common::ObISQLClient &sql_proxy, - const ObTabletID &start_tablet_id, - const int64_t range_size, - common::ObIArray &tablet_ls_pairs); - - // Get tablets sequentially by range - // - // Same function as the previous one, except that: - // 1. you can specify a list of LS whitelists to get the tables on the specified LS - // 2. you can get ObTabletToLSInfo instead of ObTabletLSPair - // - // @param [in] ls_white_list LS whitelist, empty means ALL LS in white list - static int range_get_tablet_info( - common::ObISQLClient &sql_proxy, - const common::ObIArray &ls_white_list, - const ObTabletID &start_tablet_id, - const int64_t range_size, - common::ObIArray &tablets); - - // Gets ObLSIDs according to ObTableIDs - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] tablet_ids, ObTabletIDs for query - // (should exist in __all_tablet_to_ls and have no duplicate values) - // @param [out] ls_ids, ObLSIDs corresponding to tablet_ids (same order) - // @return OB_SUCCESS if success; - // OB_ITEM_NOT_MATCH if tablet_ids have duplicates or - // tablet_id which is not recorded in __all_tablet_to_ls; - // Other error according to unexpected situation - static int batch_get_ls( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - ObIArray &ls_ids); - // Updates ObTabletToLSInfos to __all_tablet_to_ls - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for updating - // @param [in] infos, ObTabletToLSInfos for updating - // @return OB_SUCCESS if success - static int batch_update( - common::ObISQLClient &sql_proxy, - const ObIArray &infos); - // Removes tablet_id from __all_tablet_to_ls - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for removing - // @param [in] tablet_ids, ObTabletIDs for removing - // @return OB_SUCCESS if success - static int batch_remove( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids); - static int update_table_to_tablet_id_mapping( - common::ObISQLClient &sql_proxy, - const uint64_t table_id, - const common::ObTabletID &tablet_id); - // Get rows from __all_tablet_to_ls according to ObTableIDs - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] tablet_ids, ObTabletIDs for query - // (should exist in __all_tablet_to_ls and have no duplicate values) - // @param [out] infos, ObTabletToLSInfo corresponding to tablet_ids (not same order) - // not same order, not same order, not same order - // @return OB_SUCCESS if success; - // OB_ITEM_NOT_MATCH if tablet_ids have duplicates or nonexistent tablets; - // Other error according to unexpected situation - static int batch_get( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - ObIArray &infos); - // Get ls_id by tablet_id - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] tablet_id, target tablet_id - // @param [out] ls_id, ls_id which the tablet belongs to - // @return OB_SUCCESS if success; - // OB_ENTRY_NOT_EXIST if tablet_id not exist - // Other error according to unexpected situation - static int get_ls_by_tablet( - common::ObISQLClient &sql_proxy, - const common::ObTabletID &tablet_id, - ObLSID &ls_id); - // Batch get ObTabletLSCache for location_service - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] tablet_id, target tablet_id - // @param [out] tablet_ls_cache, ObTabletLSCache array - // @return OB_SUCCESS if success - static int batch_get_tablet_ls_cache( - common::ObISQLClient &sql_proxy, - const common::ObIArray &tablet_ids, - common::ObIArray &tablet_ls_caches); - - // Gets ObTabletLSPair according to ObTableIDs - // - // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction - // @param [in] tenant, tenant for query - // @param [in] tablet_ids, ObTabletIDs for query (no duplicate values) - // @param [out] tablet_ls_pairs, array of - // @return OB_SUCCESS if success; - // OB_ITEM_NOT_MATCH if tablet_ids have duplicates or - // tablet_id which is not recorded in __all_tablet_to_ls; - // Other error according to unexpected situation - static int batch_get_tablet_ls_pairs( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - ObIArray &tablet_ls_pairs); - static int get_tablet_ls_pairs_cnt( - common::ObISQLClient &sql_proxy, - int64_t &input_cnt); - const static int64_t MAX_BATCH_COUNT = 200; -private: - static int inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - ObIArray &ls_ids); - static int inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - ObIArray &infos); - static int inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - common::ObIArray &tablet_ls_caches); - static int inner_batch_get_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx, - ObIArray &tablet_ls_pairs); - - static int inner_batch_update_by_sql_( - common::ObISQLClient &sql_proxy, - const ObIArray &infos, - const int64_t start_idx, - const int64_t end_idx); - static int inner_batch_remove_by_sql_( - common::ObISQLClient &sql_proxy, - const ObIArray &tablet_ids, - const int64_t start_idx, - const int64_t end_idx); - - static int construct_results_( - common::sqlclient::ObMySQLResult &res, - ObIArray &ls_ids); - static int construct_results_( - common::sqlclient::ObMySQLResult &res, - ObIArray &infos); - static int construct_results_( - common::sqlclient::ObMySQLResult &res, - ObIArray &pairs); - static int construct_results_( - common::sqlclient::ObMySQLResult &res, - common::ObIArray &tablet_ls_caches); - static int construct_ls_white_list_where_sql_( - const ObIArray &ls_white_list, - ObSqlString &subsql); -}; - -} // end namespace share -} // end namespace oceanbase -#endif diff --git a/src/share/tablet/ob_tenant_tablet_to_ls_map.cpp b/src/share/tablet/ob_tenant_tablet_to_ls_map.cpp deleted file mode 100644 index a344428ce..000000000 --- a/src/share/tablet/ob_tenant_tablet_to_ls_map.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SHARE - -#include "share/tablet/ob_tablet_to_ls_iterator.h" // ObTenantTabletToLSIterator - -#include "share/tablet/ob_tenant_tablet_to_ls_map.h" - -namespace oceanbase -{ -namespace share -{ - -int ObTenantTabletToLSMap::build(common::ObMySQLProxy &sql_proxy) -{ - int ret = OB_SUCCESS; - - ObTenantTabletToLSIterator iter; - if (OB_FAIL(iter.init(sql_proxy))) { - LOG_WARN("init iter fail", KR(ret)); - } else { - ObTabletLSPair tablet_ls_pair; - while (OB_SUCC(ret)) { - if (OB_FAIL(iter.next(tablet_ls_pair))) { - if (OB_ITER_END != ret) { - LOG_WARN("iter next fail", KR(ret)); - } else { - ret = OB_SUCCESS; - break; - } - } else if (OB_FAIL(map_.set_refactored(tablet_ls_pair.get_tablet_id(), tablet_ls_pair.get_ls_id()))) { - LOG_WARN("tablet_to_ls map set fail", KR(ret), K(tablet_ls_pair)); - } - } - } - - return ret; -} - - -} // end namespace share -} // end namespace oceanbase diff --git a/src/share/tablet/ob_tenant_tablet_to_ls_map.h b/src/share/tablet/ob_tenant_tablet_to_ls_map.h deleted file mode 100644 index 0c1d8d942..000000000 --- a/src/share/tablet/ob_tenant_tablet_to_ls_map.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SHARE_TABLET_OB_TENANT_TABLET_TO_LS_MAP_H -#define OCEANBASE_SHARE_TABLET_OB_TENANT_TABLET_TO_LS_MAP_H - -#include "lib/alloc/alloc_struct.h" // ObLabel -#include "lib/hash/ob_hashmap.h" // ObHashMap -#include "share/ob_ls_id.h" // ObLSID - -namespace oceanbase -{ -namespace common -{ -class ObISQLClient; -class ObTabletID; -class ObMySQLProxy; -} -namespace share -{ -class ObTabletToLSTableOperator; - -typedef common::hash::ObHashMap ObTabletToLSMap; - -// Tenant all tablet to LS info -// It will build map in build() function. -class ObTenantTabletToLSMap final -{ -public: - ObTenantTabletToLSMap() : map_() {} - ~ObTenantTabletToLSMap() {} - - int init(const int64_t bucket_num = 4096, - const lib::ObLabel label = lib::ObLabel("TenantTabletToLSMap")) - { - return map_.create(bucket_num, label, ObModIds::OB_HASH_NODE); - } - void destroy() { map_.destroy(); } - - int build(common::ObMySQLProxy &sql_proxy); - - int clear() { return map_.clear(); } - - int get(const common::ObTabletID &tablet_id, ObLSID &ls_id) const - { - return map_.get_refactored(tablet_id, ls_id); - } - int64_t size() const { return map_.size(); } - -private: - ObTabletToLSMap map_; -}; - -} // end namespace share -} // end namespace oceanbase - -#endif // OCEANBASE_SHARE_TABLET_OB_TENANT_TABLET_TO_LS_MAP_H diff --git a/src/share/unit/ob_unit_resource.h b/src/share/unit/ob_unit_resource.h index 8bfd66b48..c2a9018c9 100644 --- a/src/share/unit/ob_unit_resource.h +++ b/src/share/unit/ob_unit_resource.h @@ -75,18 +75,15 @@ class ObUnitResource static const int64_t UNIT_SAFE_MIN_MEMORY = 2 * META_TENANT_SAFE_MIN_MEMORY; ////////////////////////// LOG DISK //////////////////////////// - // Unit LOG DISK SIZE is limited by LS replica. Every LS replica need 512M. - // - // META_TENANT: 512M -> only SYS LS on Meta tenant - // USER_TENANT: 512M * 3 = 1.5G -> 3 LS at least for every USER tenant unit + // Each tenant log stream needs at least 512M of log disk. static const int64_t META_TENANT_MIN_LOG_DISK_SIZE = 512LL * MB; - static const int64_t USER_TENANT_MIN_LOG_DISK_SIZE = 3LL * 512LL * MB; // 1.5G + static const int64_t USER_TENANT_MIN_LOG_DISK_SIZE = 512LL * MB; // unit_min_log_disk_size and mem_to_log_disk_default_factor are different in SS and SN mode, // decide which to use. // default factor of mapping MEMORY_SIZE to LOG_DISK_SIZE // MEMORY_SIZE * FACTOR = LOG_DISK_SIZE // shared-nothing mode: - static const int64_t UNIT_MIN_LOG_DISK_SIZE_SN = META_TENANT_MIN_LOG_DISK_SIZE + USER_TENANT_MIN_LOG_DISK_SIZE; // 2G + static const int64_t UNIT_MIN_LOG_DISK_SIZE_SN = META_TENANT_MIN_LOG_DISK_SIZE + USER_TENANT_MIN_LOG_DISK_SIZE; static const int64_t MEMORY_TO_LOG_DISK_FACTOR_SN = 3; // shared-storage mode: static const int64_t UNIT_MIN_LOG_DISK_SIZE_SS = 3LL * GB; // 3G diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index bd5ea43f0..c4ec086fc 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -30,17 +30,6 @@ ob_set_subtarget(ob_sql common ob_set_subtarget(ob_sql common_mixed privilege_check/ob_privilege_check.cpp privilege_check/ob_ai_model_priv_util.cpp - resolver/mv/ob_mv_checker.cpp - resolver/mv/ob_mv_provider.cpp - resolver/mv/ob_mv_printer.cpp - resolver/mv/ob_simple_mav_printer.cpp - resolver/mv/ob_simple_mjv_printer.cpp - resolver/mv/ob_simple_join_mav_printer.cpp - resolver/mv/ob_outer_join_mjv_printer.cpp - resolver/mv/ob_union_all_mv_printer.cpp - resolver/mv/ob_major_refresh_mjv_printer.cpp - resolver/mv/ob_mv_dep_utils.cpp - resolver/mv/ob_alter_mview_utils.cpp ) ob_set_subtarget(ob_sql hybrid_search @@ -192,9 +181,6 @@ ob_set_subtarget(ob_sql das das/ob_das_update_op.cpp das/ob_data_access_service.cpp das/ob_das_utils.cpp - das/ob_das_id_service.cpp - das/ob_das_id_rpc.cpp - das/ob_das_id_cache.cpp das/ob_das_ir_define.cpp das/ob_das_vec_define.cpp das/ob_das_task_result.cpp @@ -422,7 +408,6 @@ ob_set_subtarget(ob_sql engine_cmd engine/cmd/ob_empty_query_executor.cpp engine/cmd/ob_get_diagnostics_executor.cpp engine/cmd/ob_index_executor.cpp - engine/cmd/ob_mlog_executor.cpp engine/cmd/ob_kill_executor.cpp engine/cmd/ob_kill_session_arg.cpp engine/cmd/ob_load_data_direct_impl.cpp @@ -688,7 +673,6 @@ ob_set_subtarget(ob_sql engine_expr engine/expr/ob_expr_extract_cert_expired_time.cpp engine/expr/ob_expr_transaction_id.cpp engine/expr/ob_expr_inner_row_cmp_val.cpp - engine/expr/ob_expr_last_refresh_scn.cpp engine/expr/ob_expr_json_utils.cpp engine/expr/ob_expr_array.cpp engine/expr/ob_expr_vector.cpp @@ -1194,7 +1178,6 @@ ob_set_subtarget(ob_sql optimizer optimizer/ob_help_log_plan.cpp optimizer/ob_index_info_cache.cpp optimizer/ob_insert_log_plan.cpp - optimizer/ob_intersect_route_policy.cpp optimizer/ob_join_order.cpp optimizer/ob_log_del_upd.cpp optimizer/ob_log_delete.cpp @@ -1244,8 +1227,6 @@ ob_set_subtarget(ob_sql optimizer optimizer/ob_raw_expr_add_to_context.cpp optimizer/ob_raw_expr_check_dep.cpp optimizer/ob_raw_expr_get_hash_value.cpp - optimizer/ob_replica_compare.cpp - optimizer/ob_route_policy.cpp optimizer/ob_sel_estimator.cpp optimizer/ob_select_log_plan.cpp optimizer/ob_sharding_info.cpp @@ -1401,8 +1382,6 @@ ob_set_subtarget(ob_sql resolver_ddl resolver/ddl/ob_create_index_stmt.cpp resolver/ddl/ob_create_location_resolver.cpp resolver/ddl/ob_create_location_stmt.cpp - resolver/ddl/ob_create_mlog_resolver.cpp - resolver/ddl/ob_create_mlog_stmt.cpp resolver/ddl/ob_create_outline_resolver.cpp resolver/ddl/ob_create_package_resolver.cpp resolver/ddl/ob_create_routine_resolver.cpp @@ -1425,8 +1404,6 @@ ob_set_subtarget(ob_sql resolver_ddl resolver/ddl/ob_drop_index_stmt.cpp resolver/ddl/ob_drop_location_resolver.cpp resolver/ddl/ob_drop_location_stmt.cpp - resolver/ddl/ob_drop_mlog_resolver.cpp - resolver/ddl/ob_drop_mlog_stmt.cpp resolver/ddl/ob_drop_outline_resolver.cpp resolver/ddl/ob_drop_package_resolver.cpp resolver/ddl/ob_drop_routine_resolver.cpp @@ -1566,11 +1543,9 @@ ob_set_subtarget(ob_sql rewrite rewrite/ob_transform_expr_pullup.cpp rewrite/ob_transformer_impl.cpp rewrite/ob_transform_conditional_aggr_coalesce.cpp - rewrite/ob_transform_mv_rewrite.cpp rewrite/ob_union_find.cpp rewrite/ob_transform_decorrelate.cpp rewrite/ob_transform_late_materialization.cpp - rewrite/ob_transform_mv_rewrite_prepare.cpp rewrite/ob_transform_distinct_aggregate.cpp ) diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index c06a0ac65..bbf72d06a 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -119,7 +119,6 @@ #include "sql/engine/set/ob_hash_intersect_vec_op.h" #include "sql/engine/set/ob_hash_except_vec_op.h" #include "sql/engine/join/ob_merge_join_vec_op.h" -#include "sql/resolver/mv/ob_mv_provider.h" #include "sql/engine/set/ob_merge_set_vec_op.h" #include "sql/engine/set/ob_merge_union_vec_op.h" #include "sql/engine/set/ob_merge_intersect_vec_op.h" @@ -753,11 +752,6 @@ int ObStaticEngineCG::generate_spec_basic(ObLogicalOperator &op, if (OB_SUCC(ret) && need_check_output_datum) { OZ(add_output_datum_check_flag(spec)); } - if (OB_SUCC(ret)) { - if (OB_FAIL(extract_all_mview_ids(cur_op_exprs_))) { - LOG_WARN("fail to extract all mview ids", K(ret)); - } - } if (OB_SUCC(ret)) { CK (OB_NOT_NULL(op.get_plan()) && OB_NOT_NULL(op.get_plan()->get_stmt()) @@ -812,37 +806,6 @@ int ObStaticEngineCG::get_query_compress_type(const ObLogPlan &log_plan, return ret; } -int ObStaticEngineCG::extract_all_mview_ids(const ObIArray &exprs) -{ - int ret = OB_SUCCESS; - for (int i = 0; OB_SUCC(ret) && i < exprs.count(); ++i) { - if (OB_FAIL(extract_all_mview_ids(exprs.at(i)))) { - LOG_WARN("extract all mview ids failed", K(ret), K(i), KPC(exprs.at(i))); - } - } - return ret; -} - -int ObStaticEngineCG::extract_all_mview_ids(const ObRawExpr *expr) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr is null", K(ret), K(expr)); - } else if (ObItemType::T_FUN_SYS_LAST_REFRESH_SCN == expr->get_expr_type()) { - if (OB_FAIL(add_var_to_array_no_dup(mview_ids_, static_cast(expr)->get_mview_id()))) { - LOG_WARN("failed to add var to array no dup", K(ret), K(mview_ids_.count())); - } - } else { - for (int i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { - if (OB_FAIL(SMART_CALL(extract_all_mview_ids(expr->get_param_expr(i))))) { - LOG_WARN("extract all mview ids failed", K(ret), KPC(expr->get_param_expr(i))); - } - } - } - return ret; -} - int ObStaticEngineCG::generate_calc_exprs( const ObIArray &dep_exprs, const ObIArray &cur_exprs, @@ -2853,7 +2816,7 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableReplaceSpec &spec, c { int ret = OB_SUCCESS; UNUSED(in_root_job); - bool can_do_gts_opt = false; + bool can_use_snapshot_opt = false; bool has_unique_index = false; bool has_partition_index = false; const ObIArray &insert_dml_infos = op.get_index_dml_infos();; @@ -2862,19 +2825,19 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableReplaceSpec &spec, c if (NULL == primary_dml_info) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); - } else if (OB_FAIL(op.is_insertup_or_replace_values(can_do_gts_opt))) { + } else if (OB_FAIL(op.is_insertup_or_replace_values(can_use_snapshot_opt))) { LOG_WARN("fail to check is plain insert", K(ret)); - } else if (!can_do_gts_opt) { + } else if (!can_use_snapshot_opt) { // do nothing - LOG_TRACE("can't do insert_up gts opt", K(op.get_insert_up_index_dml_infos())); + LOG_TRACE("can't do insert_up snapshot opt", K(op.get_insert_up_index_dml_infos())); } else if (OB_FAIL(check_has_global_partiton_index(op.get_plan(), primary_dml_info->ref_table_id_, has_partition_index))) { LOG_WARN("check has global partition index failed", K(ret), K(primary_dml_info->ref_table_id_)); } else if (has_partition_index) { - LOG_TRACE("has partition index, can't support gts opt"); + LOG_TRACE("has partition index, can't support snapshot opt"); } else { - spec.plan_->set_insertup_can_do_gts_opt(can_do_gts_opt); + spec.plan_->set_insertup_can_use_snapshot_opt(can_use_snapshot_opt); if (OB_FAIL(check_has_global_unique_index(op.get_plan(), primary_dml_info->ref_table_id_, has_unique_index))) { LOG_WARN("check has global unique index", K(ret), K(primary_dml_info->ref_table_id_)); } else { @@ -3288,16 +3251,16 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec, } if (OB_SUCC(ret)) { - bool can_do_gts_opt = false; + bool can_use_snapshot_opt = false; bool has_unique_index = false; bool update_part_key = false; bool has_partition_index = false; const IndexDMLInfo *ins_pri_dml_info = op.get_index_dml_infos().at(0); - if (OB_FAIL(op.is_insertup_or_replace_values(can_do_gts_opt))) { + if (OB_FAIL(op.is_insertup_or_replace_values(can_use_snapshot_opt))) { LOG_WARN("fail to check is plain insert", K(ret)); - } else if (!can_do_gts_opt) { + } else if (!can_use_snapshot_opt) { // do nothing - LOG_TRACE("can't do insert_up gts opt", K(op.get_insert_up_index_dml_infos())); + LOG_TRACE("can't do insert_up snapshot opt", K(op.get_insert_up_index_dml_infos())); } else if (OB_FAIL(check_has_update_part_key(op.get_insert_up_index_dml_infos(), update_part_key))) { LOG_WARN("fail to check has update part key", K(ret), K(op.get_insert_up_index_dml_infos())); } else if (update_part_key) { @@ -3308,9 +3271,9 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec, has_partition_index))) { LOG_WARN("check has global partition index failed", K(ins_pri_dml_info->ref_table_id_)); } else if (has_partition_index) { - LOG_TRACE("has partition index, can't support gts opt"); + LOG_TRACE("has partition index, can't support snapshot opt"); } else { - spec.plan_->set_insertup_can_do_gts_opt(can_do_gts_opt); + spec.plan_->set_insertup_can_use_snapshot_opt(can_use_snapshot_opt); if (OB_FAIL(check_has_global_unique_index(op.get_plan(), ins_pri_dml_info->ref_table_id_, has_unique_index))) { LOG_WARN("check has global unique index", K(ret), K(ins_pri_dml_info->ref_table_id_)); } else { @@ -5125,8 +5088,7 @@ int ObStaticEngineCG::generate_normal_tsc(ObLogTableScan &op, ObTableScanSpec &s if (OB_SUCC(ret)) { if (opt_ctx_->is_online_ddl() && - stmt::T_INSERT == opt_ctx_->get_session_info()->get_stmt_type() && - !opt_ctx_->get_session_info()->get_ddl_info().is_mview_complete_refresh()) { + stmt::T_INSERT == opt_ctx_->get_session_info()->get_stmt_type()) { spec.report_col_checksum_ = true; } } @@ -8157,9 +8119,6 @@ int ObStaticEngineCG::set_properties_post(const ObLogPlan &log_plan, ObPhysicalP || OB_ISNULL(exec_ctx->get_stmt_factory()->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid query_ctx", K(ret)); - } else if (my_session->get_ddl_info().is_refreshing_mview() - && OB_FAIL(check_refreshing_mview_session_var(*schema_guard, *my_session, log_plan.get_stmt()))) { - LOG_WARN("failed to check refreshing mview session var", K(ret)); } else { ret = phy_plan.set_params_info(*(log_plan.get_optimizer_context().get_params())); } @@ -8200,8 +8159,6 @@ int ObStaticEngineCG::set_properties_post(const ObLogPlan &log_plan, ObPhysicalP LOG_WARN("set expected worker map", K(ret)); } else if (OB_FAIL(phy_plan.set_minimal_worker_map(log_plan.get_optimizer_context().get_minimal_worker_map()))) { LOG_WARN("set minimal worker map", K(ret)); - } else if (OB_FAIL(phy_plan.set_mview_ids(mview_ids_))) { - LOG_WARN("failed to set set mview_ids", K(ret), K(mview_ids_.count())); } else { if (log_plan.get_optimizer_context().is_online_ddl()) { if (log_plan.get_stmt()->get_table_items().count() > 0) { @@ -8219,17 +8176,6 @@ int ObStaticEngineCG::set_properties_post(const ObLogPlan &log_plan, ObPhysicalP } } } - if (log_plan.get_optimizer_context().get_session_info()->get_ddl_info().is_mview_complete_refresh()) { - if (log_plan.get_stmt()->get_table_items().count() > 0) { - const TableItem *insert_table_item = log_plan.get_stmt()->get_table_item(0); - if (nullptr != insert_table_item) { - int64_t ddl_task_id = 0; - const ObOptParamHint *opt_params = &log_plan.get_stmt()->get_query_ctx()->get_global_hint().opt_params_; - OZ(opt_params->get_integer_opt_param(ObOptParamHint::DDL_TASK_ID, ddl_task_id)); - phy_plan.set_ddl_task_id(ddl_task_id); - } - } - } } ObParamOption param_opt = log_plan.get_optimizer_context().get_global_hint().param_option_; bool is_exact_mode = my_session->get_enable_exact_mode(); @@ -8457,33 +8403,6 @@ int ObStaticEngineCG::set_properties_post(const ObLogPlan &log_plan, ObPhysicalP return ret; } -int ObStaticEngineCG::check_refreshing_mview_session_var(ObSchemaGetterGuard &schema_guard, - ObSQLSessionInfo &session, - const ObDMLStmt *dml_stmt) -{ - int ret = OB_SUCCESS; - bool is_same = false; - uint64_t mview_id = OB_INVALID_ID; - const share::schema::ObTableSchema *mview_schema = NULL; - const ObDelUpdStmt *del_up_stmt = dynamic_cast(dml_stmt); - bool is_vars_matched = false; - if (!session.get_ddl_info().is_refreshing_mview() || OB_ISNULL(del_up_stmt)) { - /* do nothing */ - } else if (OB_FAIL(del_up_stmt->get_modified_materialized_view_id(mview_id))) { - LOG_WARN("fail to get modified mview_id", K(ret), K(mview_id)); - } else if (OB_INVALID_ID == mview_id) { - /* do nothing */ - } else if (OB_FAIL(schema_guard.get_table_schema( mview_id, mview_schema))) { - LOG_WARN("fail to get mview schema", K(ret), K(mview_id)); - } else if (OB_ISNULL(mview_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("fail to get mview schema", K(ret), K(mview_id)); - } else if (OB_FAIL(ObMVProvider::check_mview_dep_session_vars(*mview_schema, session, true, is_vars_matched))) { - LOG_WARN("failed to check mview dep session vars", K(ret)); - } - return ret; -} - // FIXME bin.lb: We should split the big switch case into logical operator class. int ObStaticEngineCG::get_phy_op_type(ObLogicalOperator &log_op, ObPhyOperatorType &type, diff --git a/src/sql/das/iter/ob_das_domain_id_merge_iter.cpp b/src/sql/das/iter/ob_das_domain_id_merge_iter.cpp index 92f2621a8..7da967769 100644 --- a/src/sql/das/iter/ob_das_domain_id_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_domain_id_merge_iter.cpp @@ -32,7 +32,6 @@ namespace sql ObDASDomainIdMergeIterParam::ObDASDomainIdMergeIterParam() : ObDASIterParam(DAS_ITER_DOMAIN_ID_MERGE), - rowkey_domain_ls_id_(), rowkey_domain_tablet_ids_(), data_table_iter_(nullptr), data_table_ctdef_(nullptr), @@ -50,8 +49,7 @@ ObDASDomainIdMergeIterParam::~ObDASDomainIdMergeIterParam() bool ObDASDomainIdMergeIterParam::is_valid() const { bool bret = true; - bret = rowkey_domain_ls_id_.is_valid() && - data_table_iter_ != nullptr && + bret = data_table_iter_ != nullptr && data_table_ctdef_ != nullptr && data_table_rtdef_ != nullptr && snapshot_ != nullptr && @@ -79,7 +77,6 @@ ObDASDomainIdMergeIter::ObDASDomainIdMergeIter() rowkey_domain_rtdefs_(), data_table_rtdef_(nullptr), rowkey_domain_tablet_ids_(), - rowkey_domain_ls_id_(), merge_memctx_(), is_need_multi_get_(false) {} @@ -123,7 +120,6 @@ int ObDASDomainIdMergeIter::rescan() LOG_WARN("fail to rescan data table iter", K(ret), KPC(data_table_iter_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_domain_iters_.count(); i++) { - rowkey_domain_scan_params_.at(i)->ls_id_ = rowkey_domain_ls_id_; rowkey_domain_scan_params_.at(i)->tablet_id_ = rowkey_domain_tablet_ids_.at(i); if (OB_ISNULL(rowkey_domain_iters_.at(i))) { ret = OB_ERR_UNEXPECTED; @@ -183,7 +179,6 @@ int ObDASDomainIdMergeIter::inner_init(ObDASIterParam ¶m) LOG_WARN("failed to new rowkey scan param", K(sizeof(ObTableScanParam)), K(ret)); } else if (OB_FAIL(init_rowkey_domain_scan_param( merge_param.rowkey_domain_tablet_ids_.at(i), - merge_param.rowkey_domain_ls_id_, merge_param.rowkey_domain_ctdefs_.at(i), merge_param.rowkey_domain_rtdefs_.at(i), merge_param.trans_desc_, merge_param.snapshot_, @@ -207,7 +202,6 @@ int ObDASDomainIdMergeIter::inner_init(ObDASIterParam ¶m) data_table_iter_ = merge_param.data_table_iter_; data_table_ctdef_ = merge_param.data_table_ctdef_; data_table_rtdef_ = merge_param.data_table_rtdef_; - rowkey_domain_ls_id_ = merge_param.rowkey_domain_ls_id_; need_filter_rowkey_domain_ = true; } } @@ -215,15 +209,13 @@ int ObDASDomainIdMergeIter::inner_init(ObDASIterParam ¶m) } int ObDASDomainIdMergeIter::set_domain_id_merge_related_ids( - const ObDASRelatedTabletID &tablet_ids, - const share::ObLSID &ls_id) + const ObDASRelatedTabletID &tablet_ids) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || tablet_ids.domain_tablet_ids_.count() != rowkey_domain_tablet_ids_.count())) { + if (OB_UNLIKELY(tablet_ids.domain_tablet_ids_.count() != rowkey_domain_tablet_ids_.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ls id", K(ret), K(ls_id), K(tablet_ids.domain_tablet_ids_), K(rowkey_domain_tablet_ids_)); + LOG_WARN("unexpected domain tablet count", K(ret), K(tablet_ids.domain_tablet_ids_), K(rowkey_domain_tablet_ids_)); } else { - rowkey_domain_ls_id_ = ls_id; for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.domain_tablet_ids_.count(); i++) { if (OB_UNLIKELY(!tablet_ids.domain_tablet_ids_.at(i).is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -459,7 +451,6 @@ int ObDASDomainIdMergeIter::build_rowkey_domain_range() } if (OB_SUCC(ret)) { scan_param.tablet_id_ = rowkey_domain_tablet_ids_.at(k); - scan_param.ls_id_ = rowkey_domain_ls_id_; scan_param.sample_info_ = data_table_iter_->get_scan_param().sample_info_; scan_param.scan_flag_.scan_order_ = data_table_iter_->get_scan_param().scan_flag_.scan_order_; scan_param.enable_new_false_range_ = data_table_iter_->get_scan_param().enable_new_false_range_; @@ -486,7 +477,6 @@ int ObDASDomainIdMergeIter::build_rowkey_domain_range() int ObDASDomainIdMergeIter::init_rowkey_domain_scan_param( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, transaction::ObTxDesc *trans_desc, @@ -498,15 +488,14 @@ int ObDASDomainIdMergeIter::init_rowkey_domain_scan_param( scan_param.key_ranges_.set_attr(ObMemAttr("SParamKR")); scan_param.ss_key_ranges_.set_attr(ObMemAttr("SParamSSKR")); - if (OB_UNLIKELY(!tablet_id.is_valid() || !ls_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(ls_id)); + LOG_WARN("invalid tablet id", K(ret), K(tablet_id)); } else if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr ctdef or rtdef", K(ret), KPC(ctdef), KPC(rtdef)); } else { scan_param.tablet_id_ = tablet_id; - scan_param.ls_id_ = ls_id; scan_param.scan_allocator_ = &rtdef->scan_allocator_; scan_param.allocator_ = &rtdef->stmt_allocator_; scan_param.tx_lock_timeout_ = rtdef->tx_lock_timeout_; @@ -1521,7 +1510,6 @@ int ObDASDomainIdMergeIter::reset_rowkey_domain_iter_scan_range(int64_t iter_idx } if (OB_SUCC(ret)) { scan_param.tablet_id_ = rowkey_domain_tablet_ids_.at(iter_idx); - scan_param.ls_id_ = rowkey_domain_ls_id_; scan_param.sample_info_ = data_table_iter_->get_scan_param().sample_info_; scan_param.scan_flag_.scan_order_ = data_table_iter_->get_scan_param().scan_flag_.scan_order_; if (!data_table_iter_->get_scan_param().need_switch_param_) { diff --git a/src/sql/das/iter/ob_das_domain_id_merge_iter.h b/src/sql/das/iter/ob_das_domain_id_merge_iter.h index f94c9638c..73ce0f50d 100644 --- a/src/sql/das/iter/ob_das_domain_id_merge_iter.h +++ b/src/sql/das/iter/ob_das_domain_id_merge_iter.h @@ -20,7 +20,6 @@ #include "sql/das/iter/ob_das_iter.h" #include "sql/das/iter/ob_das_scan_iter.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/access/ob_dml_param.h" #include "sql/das/ob_domain_id.h" #include "observer/vector_index/ob_vector_index_util.h" @@ -43,7 +42,6 @@ class ObDASDomainIdMergeIterParam final : public ObDASIterParam virtual bool is_valid() const override; INHERIT_TO_STRING_KV("ObDASIterParam", ObDASIterParam, - K(rowkey_domain_ls_id_), K(rowkey_domain_tablet_ids_), KP(data_table_iter_), KP(data_table_ctdef_), @@ -54,7 +52,6 @@ class ObDASDomainIdMergeIterParam final : public ObDASIterParam KPC(trans_desc_), KPC(snapshot_)); public: - share::ObLSID rowkey_domain_ls_id_; // all domain index should be in one ls (local index) common::ObArray rowkey_domain_tablet_ids_; ObDASScanIter* data_table_iter_; ObDASScanCtDef* data_table_ctdef_; @@ -95,7 +92,7 @@ class ObDASDomainIdMergeIter final : public ObDASIter ObIAllocator *alloc, int64_t group_id) override; ObDASScanIter *get_data_table_iter() { return data_table_iter_; } - int set_domain_id_merge_related_ids(const ObDASRelatedTabletID &tablet_ids, const share::ObLSID &ls_id); + int set_domain_id_merge_related_ids(const ObDASRelatedTabletID &tablet_ids); INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K(rowkey_domain_scan_params_), @@ -110,7 +107,6 @@ class ObDASDomainIdMergeIter final : public ObDASIter common::ObArenaAllocator &get_arena_allocator() { return merge_memctx_->get_arena_allocator(); } int init_rowkey_domain_scan_param( const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, transaction::ObTxDesc *trans_desc, @@ -177,7 +173,6 @@ class ObDASDomainIdMergeIter final : public ObDASIter ObArray rowkey_domain_rtdefs_; ObDASScanRtDef *data_table_rtdef_; ObArray rowkey_domain_tablet_ids_; - share::ObLSID rowkey_domain_ls_id_; lib::MemoryContext merge_memctx_; bool is_need_multi_get_; }; diff --git a/src/sql/das/iter/ob_das_func_data_iter.cpp b/src/sql/das/iter/ob_das_func_data_iter.cpp index c6f8504e7..d38bd4ae7 100644 --- a/src/sql/das/iter/ob_das_func_data_iter.cpp +++ b/src/sql/das/iter/ob_das_func_data_iter.cpp @@ -53,7 +53,6 @@ ObDASFuncDataIter::ObDASFuncDataIter() main_lookup_rtdef_(nullptr), main_lookup_iter_(nullptr), main_lookup_tablet_id_(0), - main_lookup_ls_id_(0), main_lookup_param_(), merge_memctx_(), doc_ids_(), @@ -76,12 +75,11 @@ int ObDASFuncDataIter::do_table_scan() LOG_WARN("fail to build rowkey doc range", K(ret)); } else { if (nullptr != main_lookup_iter_) { - if (OB_UNLIKELY(!main_lookup_tablet_id_.is_valid() || !main_lookup_ls_id_.is_valid())) { + if (OB_UNLIKELY(!main_lookup_tablet_id_.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, main lookup tablet id or ls id is invalid", K(ret), K(main_lookup_tablet_id_), K(main_lookup_ls_id_)); + LOG_WARN("unexpected error, main lookup tablet id is invalid", K(ret), K(main_lookup_tablet_id_)); } else { main_lookup_param_.tablet_id_ = main_lookup_tablet_id_; - main_lookup_param_.ls_id_ = main_lookup_ls_id_; if (OB_FAIL(main_lookup_iter_->do_table_scan())) { LOG_WARN("fail to do table scan for main lookup table", K(ret), KPC(main_lookup_iter_)); } @@ -213,7 +211,6 @@ int ObDASFuncDataIter::inner_reuse() main_lookup_param_.need_switch_param_ = main_lookup_param_.need_switch_param_ || ((old_tablet_id.is_valid() && old_tablet_id != main_lookup_tablet_id_) ? true : false); main_lookup_param_.tablet_id_ = main_lookup_tablet_id_; - main_lookup_param_.ls_id_ = main_lookup_ls_id_; if (!main_lookup_param_.key_ranges_.empty()) { main_lookup_param_.key_ranges_.reuse(); } diff --git a/src/sql/das/iter/ob_das_func_data_iter.h b/src/sql/das/iter/ob_das_func_data_iter.h index 80591bc89..55f6fb50d 100644 --- a/src/sql/das/iter/ob_das_func_data_iter.h +++ b/src/sql/das/iter/ob_das_func_data_iter.h @@ -21,7 +21,6 @@ #include "sql/das/iter/ob_das_scan_iter.h" #include "sql/das/iter/ob_das_text_retrieval_merge_iter.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/access/ob_dml_param.h" namespace oceanbase @@ -95,7 +94,6 @@ class ObDASFuncDataIter final : public ObDASIter } void set_tablet_id(const ObTabletID &tablet_id) { main_lookup_tablet_id_ = tablet_id; } - void set_ls_id(const share::ObLSID &ls_id) { main_lookup_ls_id_ = ls_id; } bool has_main_lookup_iter() const { return nullptr != main_lookup_iter_; } ObTableScanParam &get_main_lookup_scan_param() { return main_lookup_param_; } const ObDASScanCtDef *get_main_lookup_ctdef() { return main_lookup_ctdef_; } @@ -148,7 +146,6 @@ class ObDASFuncDataIter final : public ObDASIter ObDASScanRtDef *main_lookup_rtdef_; ObDASIter *main_lookup_iter_; ObTabletID main_lookup_tablet_id_; - share::ObLSID main_lookup_ls_id_; storage::ObTableScanParam main_lookup_param_; lib::MemoryContext merge_memctx_; ObSEArray, 4> doc_ids_; diff --git a/src/sql/das/iter/ob_das_hnsw_scan_iter.cpp b/src/sql/das/iter/ob_das_hnsw_scan_iter.cpp index 7af6ae75b..d1f56757b 100644 --- a/src/sql/das/iter/ob_das_hnsw_scan_iter.cpp +++ b/src/sql/das/iter/ob_das_hnsw_scan_iter.cpp @@ -168,7 +168,6 @@ int ObDASHNSWScanIter::inner_init(ObDASIterParam ¶m) LOG_WARN("invalid das iter param type for hnsw scan iter", K(ret), K(param)); } else { ObDASHNSWScanIterParam &hnsw_scan_param = static_cast(param); - ls_id_ = hnsw_scan_param.ls_id_; tx_desc_ = hnsw_scan_param.tx_desc_; snapshot_ = hnsw_scan_param.snapshot_; use_vid_ = hnsw_scan_param.use_vid_; @@ -300,7 +299,7 @@ int ObDASHNSWScanIter::reuse_func_lookup_iter() int tmp_ret = OB_SUCCESS; if (func_lookup_iter_->get_type() == ObDASIterType::DAS_ITER_FUNC_LOOKUP) { ObDASFuncLookupIter* func_iter = static_cast(func_lookup_iter_); - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, func_iter->get_index_scan_iter(), func_filter_scan_param_, rowkey_doc_tablet_id_))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(func_iter->get_index_scan_iter(), func_filter_scan_param_, rowkey_doc_tablet_id_))) { LOG_WARN("failed to set lookup key", K(ret)); tmp_ret = tmp_ret == OB_SUCCESS ? ret : tmp_ret; ret = OB_SUCCESS; @@ -968,8 +967,8 @@ int ObDASHNSWScanIter::process_adaptor_state_hnsw(ObIAllocator &allocator, bool index_ctx.data_tablet_id_ = com_aux_vec_tablet_id_; bool ls_leader = true; - if (OB_FAIL(vec_index_service->acquire_adapter_guard(ls_id_, index_ctx, adaptor_guard, &vec_index_param_, dim_))) { - LOG_WARN("failed to get ObPluginVectorIndexAdapter", K(ret), K(ls_id_), K(index_ctx)); + if (OB_FAIL(vec_index_service->acquire_adapter_guard(index_ctx, adaptor_guard, &vec_index_param_, dim_))) { + LOG_WARN("failed to get ObPluginVectorIndexAdapter", K(ret), K(index_ctx)); } else { share::ObPluginVectorIndexAdaptor* adaptor = adaptor_guard.get_adatper(); if (OB_ISNULL(adaptor)) { @@ -979,8 +978,8 @@ int ObDASHNSWScanIter::process_adaptor_state_hnsw(ObIAllocator &allocator, bool LOG_WARN("failed to set query condition.", K(ret)); } else if (vec_aux_ctdef_->relevance_col_cnt_ > 0 && OB_FAIL(init_rel_map(adaptor))) { LOG_WARN("failed to init rel map", K(ret)); - } else if (OB_FAIL(ObPluginVectorIndexUtils::get_ls_leader_flag(ls_id_, ls_leader))) { - LOG_WARN("fail to get ls leader flag", K(ret), K(ls_id_)); + } else if (OB_FAIL(ObPluginVectorIndexUtils::get_leader_flag(ls_leader))) { + LOG_WARN("fail to get ls leader flag", K(ret)); } else if (OB_FALSE_IT(ada_ctx.set_ls_leader(ls_leader))) { } else if (OB_FALSE_IT(ada_ctx.set_sparse_vector(is_ipivf()))) { } else { @@ -1538,10 +1537,10 @@ int ObDASHNSWScanIter::process_adaptor_state_pre_filter( if (OB_FAIL(ret)) { } else if (OB_NOT_NULL(snapshot_iter_) && OB_FALSE_IT(query_cond_.row_iter_ = snapshot_iter_->get_output_result_iter())) { - } else if (OB_FAIL(adaptor->query_result(ls_id_, ada_ctx, &query_cond_, adaptor_vid_iter_))) { + } else if (OB_FAIL(adaptor->query_result(ada_ctx, &query_cond_, adaptor_vid_iter_))) { LOG_WARN("failed to query result.", K(ret)); } else if (PVQ_REFRESH == ada_ctx->get_status()) { - if (OB_FAIL(ObPluginVectorIndexUtils::query_need_refresh_memdata(adaptor, ls_id_))) { + if (OB_FAIL(ObPluginVectorIndexUtils::query_need_refresh_memdata(adaptor))) { if (ret != OB_SCHEMA_EAGAIN) { LOG_WARN("fail to refresh memdata in query", K(ret)); } @@ -2649,7 +2648,7 @@ int ObDASHNSWScanIter::filter_func_lookup_iter(bool is_vectorized, if (func_lookup_first_scan_) { func_filter_scan_param_.need_switch_param_ = false; - if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, rowkey_doc_tablet_id_, + if (OB_FAIL(ObDasVecScanUtils::init_scan_param(rowkey_doc_tablet_id_, static_cast(func_iter->index_ctdef_), static_cast(func_iter->index_rtdef_), tx_desc_, snapshot_, func_filter_scan_param_))) { LOG_WARN("failed to init scan param", K(ret)); @@ -3111,7 +3110,7 @@ int ObDASHNSWScanIter::call_pva_interface(const ObVidAdaLookupStatus& cur_state, if (!index_id_scan_param_.snapshot_.is_valid() || !index_id_scan_param_.snapshot_.core_.version_.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get index id scan param invalid.", K(ret)); - } else if (OB_FAIL(adaptor.check_index_id_table_readnext_status(&ada_ctx, real_index_id_iter, index_id_scan_param_.snapshot_.core_.version_, skip_delta_buffer_, ls_id_))) { + } else if (OB_FAIL(adaptor.check_index_id_table_readnext_status(&ada_ctx, real_index_id_iter, index_id_scan_param_.snapshot_.core_.version_, skip_delta_buffer_))) { LOG_WARN("failed to check index id table readnext status.", K(ret)); } break; @@ -3126,13 +3125,13 @@ int ObDASHNSWScanIter::call_pva_interface(const ObVidAdaLookupStatus& cur_state, if (OB_NOT_NULL(snapshot_iter_) && OB_FALSE_IT(query_cond_.row_iter_ = snapshot_iter_->get_output_result_iter())) { } else if (!ada_ctx.get_ls_leader() && OB_FAIL(prepare_follower_query_cond(query_cond_))) { LOG_WARN("fail to prepare query cond of follower", K(ret)); - } else if (OB_FAIL(adaptor.query_result(ls_id_, &ada_ctx, &query_cond_, is_iter_filter() ? tmp_adaptor_vid_iter_ : adaptor_vid_iter_))) { + } else if (OB_FAIL(adaptor.query_result(&ada_ctx, &query_cond_, is_iter_filter() ? tmp_adaptor_vid_iter_ : adaptor_vid_iter_))) { LOG_WARN("failed to query result.", K(ret)); } break; } case ObVidAdaLookupStatus::STATES_REFRESH: { // refresh - if (OB_FAIL(ObPluginVectorIndexUtils::query_need_refresh_memdata(&adaptor, ls_id_))) { + if (OB_FAIL(ObPluginVectorIndexUtils::query_need_refresh_memdata(&adaptor))) { if (ret != OB_SCHEMA_EAGAIN) { LOG_WARN("fail to refresh memdata in query", K(ret)); } @@ -3718,7 +3717,7 @@ int ObDASHNSWScanIter::do_com_aux_vec_table_scan() const ObDASScanCtDef *com_aux_tbl_ctdef = vec_aux_ctdef_->get_vec_aux_tbl_ctdef( vec_aux_ctdef_->get_com_aux_tbl_idx(), ObTSCIRScanType::OB_VEC_COM_AUX_SCAN); ObDASScanRtDef *com_aux_tbl_rtdef = vec_aux_rtdef_->get_vec_aux_tbl_rtdef(vec_aux_ctdef_->get_com_aux_tbl_idx()); - if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param(ls_id_, + if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param( com_aux_vec_tablet_id_, com_aux_tbl_ctdef, com_aux_tbl_rtdef, @@ -3778,11 +3777,11 @@ int ObDASHNSWScanIter::do_rowkey_vid_table_scan() rowkey_vid_scan_param_.need_switch_param_ = false; ObDASScanIter *inv_idx_scan_iter = static_cast(inv_idx_scan_iter_); if (!is_primary_pre_with_rowkey_with_filter_ && - OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, rowkey_vid_tablet_id_, rowkey_vid_ctdef, rowkey_vid_rtdef, + OB_FAIL(ObDasVecScanUtils::init_scan_param(rowkey_vid_tablet_id_, rowkey_vid_ctdef, rowkey_vid_rtdef, tx_desc_, snapshot_, rowkey_vid_scan_param_))) { LOG_WARN("failed to init scan param", K(ret)); } else if (is_primary_pre_with_rowkey_with_filter_ && - OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, rowkey_vid_tablet_id_, rowkey_vid_ctdef, rowkey_vid_rtdef, + OB_FAIL(ObDasVecScanUtils::init_scan_param(rowkey_vid_tablet_id_, rowkey_vid_ctdef, rowkey_vid_rtdef, tx_desc_, snapshot_, rowkey_vid_scan_param_, inv_idx_scan_iter->get_scan_param().is_get_))) { LOG_WARN("failed to init scan param", K(ret)); @@ -3812,7 +3811,7 @@ int ObDASHNSWScanIter::do_aux_table_scan_need_reuse(bool &first_scan, ObNewRange scan_range; if (first_scan) { - if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param(ls_id_, tablet_id, ctdef, rtdef,tx_desc_, snapshot_, scan_param, is_get))) { + if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param(tablet_id, ctdef, rtdef,tx_desc_, snapshot_, scan_param, is_get))) { LOG_WARN("failed to init scan param", K(ret)); } else if (OB_FALSE_IT(ObDasVecScanUtils::set_whole_range(scan_range, ctdef->ref_table_id_))) { LOG_WARN("failed to generate init scan range", K(ret)); @@ -3825,7 +3824,7 @@ int ObDASHNSWScanIter::do_aux_table_scan_need_reuse(bool &first_scan, first_scan = false; } } else { - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, iter, scan_param, tablet_id))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(iter, scan_param, tablet_id))) { LOG_WARN("failed to reuse scan iterator", K(ret)); } else if (OB_FALSE_IT(ObDasVecScanUtils::set_whole_range(scan_range, ctdef->ref_table_id_))) { } else if (OB_FAIL(scan_param.key_ranges_.push_back(scan_range))) { @@ -3849,7 +3848,7 @@ int ObDASHNSWScanIter::do_aux_table_scan(bool &first_scan, if (first_scan) { scan_param.need_switch_param_ = false; if (OB_FAIL( - ObDasVecScanUtils::init_scan_param(ls_id_, tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param))) { + ObDasVecScanUtils::init_scan_param(tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param))) { LOG_WARN("failed to init scan param", K(ret)); } else if (OB_FALSE_IT(iter->set_scan_param(scan_param))) { } else if (OB_FAIL(iter->do_table_scan())) { diff --git a/src/sql/das/iter/ob_das_hnsw_scan_iter.h b/src/sql/das/iter/ob_das_hnsw_scan_iter.h index 85fab4baf..031679e13 100644 --- a/src/sql/das/iter/ob_das_hnsw_scan_iter.h +++ b/src/sql/das/iter/ob_das_hnsw_scan_iter.h @@ -54,7 +54,6 @@ struct ObDASHNSWScanIterParam : public ObDASIterParam public: ObDASHNSWScanIterParam() : ObDASIterParam(ObDASIterType::DAS_ITER_HNSW_SCAN), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), inv_idx_scan_iter_(nullptr), @@ -91,8 +90,7 @@ struct ObDASHNSWScanIterParam : public ObDASIterParam // skip_delta_buffer_: HNSW+heap+async has no delta_buffer table, delta_buf_iter_ is not created. // When skip_delta_buffer_=false, delta_buf_iter_ must be initialized. bool delta_buf_ready = skip_delta_buffer_ ? true : (nullptr != delta_buf_iter_); - bool valid = ls_id_.is_valid() && - nullptr != tx_desc_ && + bool valid = nullptr != tx_desc_ && nullptr != snapshot_ && delta_buf_ready && nullptr != index_id_iter_ && @@ -124,7 +122,6 @@ struct ObDASHNSWScanIterParam : public ObDASIterParam K_(can_extract_range), K_(is_primary_index)); - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; @@ -241,7 +238,6 @@ class ObDASHNSWScanIter : public ObDASIter mem_context_(nullptr), vec_op_alloc_("HNSW", OB_MALLOC_NORMAL_BLOCK_SIZE), hnsw_iter_alloc_("HNSWITER", OB_MALLOC_NORMAL_BLOCK_SIZE), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), inv_idx_scan_iter_(nullptr), @@ -339,7 +335,6 @@ class ObDASHNSWScanIter : public ObDASIter rowkey_doc_tablet_id_ = related_tablet_ids.rowkey_doc_tablet_id_; } - void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } uint64_t adjust_batch_count(bool is_vectored, uint64_t batch_count); bool enable_using_simplified_scan() { return need_save_distance_result(); } inline bool has_func_lookup() {return OB_NOT_NULL(func_lookup_iter_) && OB_NOT_NULL(func_lookup_ctdef_) && OB_NOT_NULL(func_lookup_rtdef_);} @@ -426,11 +421,11 @@ class ObDASHNSWScanIter : public ObDASIter ObTabletID &tablet_id, bool is_get = false); int do_aux_table_scan(bool &first_scan, ObTableScanParam &scan_param, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, ObDASScanIter *iter, ObTabletID &tablet_id); - int reuse_vid_rowkey_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, vid_rowkey_iter_, vid_rowkey_scan_param_, vid_rowkey_tablet_id_); }; - int reuse_rowkey_vid_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, rowkey_vid_iter_, rowkey_vid_scan_param_, rowkey_vid_tablet_id_); }; - int reuse_com_aux_vec_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, com_aux_vec_iter_, com_aux_vec_scan_param_, com_aux_vec_tablet_id_); }; - int reuse_embedded_table_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, embedded_table_iter_, embedded_table_scan_param_, embedded_tablet_id_); }; - int reuse_filter_data_table_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, data_filter_iter_, data_filter_scan_param_, com_aux_vec_tablet_id_); }; + int reuse_vid_rowkey_iter() { return ObDasVecScanUtils::reuse_iter(vid_rowkey_iter_, vid_rowkey_scan_param_, vid_rowkey_tablet_id_); }; + int reuse_rowkey_vid_iter() { return ObDasVecScanUtils::reuse_iter(rowkey_vid_iter_, rowkey_vid_scan_param_, rowkey_vid_tablet_id_); }; + int reuse_com_aux_vec_iter() { return ObDasVecScanUtils::reuse_iter(com_aux_vec_iter_, com_aux_vec_scan_param_, com_aux_vec_tablet_id_); }; + int reuse_embedded_table_iter() { return ObDasVecScanUtils::reuse_iter(embedded_table_iter_, embedded_table_scan_param_, embedded_tablet_id_); }; + int reuse_filter_data_table_iter() { return ObDasVecScanUtils::reuse_iter(data_filter_iter_, data_filter_scan_param_, com_aux_vec_tablet_id_); }; int reuse_func_lookup_iter(); int get_ctdef_with_rowkey_exprs(const ObDASScanCtDef *&ctdef, ObDASScanRtDef *&rtdef); @@ -516,7 +511,6 @@ class ObDASHNSWScanIter : public ObDASIter lib::MemoryContext mem_context_; ObArenaAllocator vec_op_alloc_; ObArenaAllocator hnsw_iter_alloc_; // lifetime is same with hnsw iter - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; diff --git a/src/sql/das/iter/ob_das_index_merge_iter.cpp b/src/sql/das/iter/ob_das_index_merge_iter.cpp index 30dd10388..e1a9652e8 100644 --- a/src/sql/das/iter/ob_das_index_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_index_merge_iter.cpp @@ -264,8 +264,7 @@ int ObDASIndexMergeIter::inner_init(ObDASIterParam ¶m) return ret; } -int ObDASIndexMergeIter::init_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObDASIndexMergeIter::init_scan_param(const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, sql::ObDASScanRtDef *rtdef, ObTableScanParam &scan_param) @@ -273,7 +272,7 @@ int ObDASIndexMergeIter::init_scan_param(const share::ObLSID &ls_id, int ret = OB_SUCCESS; if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(tablet_id)); } else { @@ -303,7 +302,6 @@ int ObDASIndexMergeIter::init_scan_param(const share::ObLSID &ls_id, scan_param.need_scn_ = rtdef->need_scn_; scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; scan_param.fb_snapshot_ = rtdef->fb_snapshot_; - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) { scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; @@ -425,7 +423,7 @@ int ObDASIndexMergeIter::do_table_scan() if (OB_ISNULL(scan_ctdef) || OB_ISNULL(scan_param)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr child scan info", K(scan_ctdef), K(scan_param), K(ret)); - } else if (OB_FAIL(init_scan_param(ls_id_, child_tablet_ids_.at(i), scan_ctdef, scan_rtdef, *scan_param))) { + } else if (OB_FAIL(init_scan_param(child_tablet_ids_.at(i), scan_ctdef, scan_rtdef, *scan_param))) { LOG_WARN("failed to init child scan param", K(ret)); } else if (OB_FAIL(iter->do_table_scan())) { LOG_WARN("child iter failed to do table scan", K(ret)); @@ -454,7 +452,6 @@ int ObDASIndexMergeIter::rescan() LOG_WARN("unexpected nullptr child scan info", K(scan_ctdef), K(scan_param), K(ret)); } else { scan_param->tablet_id_ = child_tablet_ids_.at(scan_ctdef->index_merge_idx_); - scan_param->ls_id_ = ls_id_; if (OB_FAIL(prepare_scan_ranges(*scan_param, scan_rtdef))) { LOG_WARN("failed to prepare scan ranges", K(ret)); } else if (OB_FAIL(iter->rescan())) { @@ -477,10 +474,9 @@ void ObDASIndexMergeIter::clear_evaluated_flag() } } -int ObDASIndexMergeIter::set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids) +int ObDASIndexMergeIter::set_tablet_ids(const ObDASRelatedTabletID &related_tablet_ids) { int ret = OB_SUCCESS; - ls_id_ = ls_id; const ObIArray &index_merge_tablet_ids = related_tablet_ids.index_merge_tablet_ids_; for (int64_t i = 0; OB_SUCC(ret) && i < child_scan_rtdefs_.count(); ++i) { ObDASScanRtDef *scan_rtdef = child_scan_rtdefs_.at(i); diff --git a/src/sql/das/iter/ob_das_index_merge_iter.h b/src/sql/das/iter/ob_das_index_merge_iter.h index a4792745b..3f356a98c 100644 --- a/src/sql/das/iter/ob_das_index_merge_iter.h +++ b/src/sql/das/iter/ob_das_index_merge_iter.h @@ -166,7 +166,6 @@ class ObDASIndexMergeIter : public ObDASIter mem_ctx_(), get_next_row_(nullptr), get_next_rows_(nullptr), - ls_id_(), merge_ctdef_(nullptr), merge_rtdef_(nullptr), tx_desc_(nullptr), @@ -182,7 +181,7 @@ class ObDASIndexMergeIter : public ObDASIter virtual int do_table_scan() override; virtual int rescan() override; virtual void clear_evaluated_flag() override; - int set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids); + int set_tablet_ids(const ObDASRelatedTabletID &related_tablet_ids); ObTableScanParam *get_child_scan_param(int64_t idx) { return child_scan_params_.at(idx); } protected: @@ -198,8 +197,7 @@ class ObDASIndexMergeIter : public ObDASIter int intersect_get_next_rows(int64_t &count, int64_t capacity); int union_get_next_row(); int union_get_next_rows(int64_t &count, int64_t capacity); - int init_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + int init_scan_param(const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, sql::ObDASScanRtDef *rtdef, ObTableScanParam &scan_param); @@ -224,7 +222,6 @@ class ObDASIndexMergeIter : public ObDASIter lib::MemoryContext mem_ctx_; int (ObDASIndexMergeIter::*get_next_row_)(); int (ObDASIndexMergeIter::*get_next_rows_)(int64_t&, int64_t); - ObLSID ls_id_; const ObDASIndexMergeCtDef *merge_ctdef_; ObDASIndexMergeRtDef *merge_rtdef_; transaction::ObTxDesc *tx_desc_; diff --git a/src/sql/das/iter/ob_das_iter_utils.cpp b/src/sql/das/iter/ob_das_iter_utils.cpp index aca0e3973..9af2d9cc3 100644 --- a/src/sql/das/iter/ob_das_iter_utils.cpp +++ b/src/sql/das/iter/ob_das_iter_utils.cpp @@ -324,7 +324,6 @@ int ObDASIterUtils::create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef, int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter) { int ret = OB_SUCCESS; @@ -341,7 +340,6 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASLocalLookupIter *lookup_iter = static_cast(root_iter); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); need_set_child = true; } @@ -353,7 +351,6 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASCacheLookupIter *lookup_iter = static_cast(root_iter); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); need_set_child = true; } @@ -365,7 +362,6 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASLocalLookupIter *aux_lookup_iter = static_cast(root_iter); - aux_lookup_iter->set_ls_id(ls_id); aux_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); need_set_child = true; } @@ -406,7 +402,7 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ ObDASTRMergeIter *tr_merge_iter = static_cast(root_iter); need_set_child = false; const int64_t fts_idx = static_cast(attach_rtdef)->fts_idx_; - if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids.fts_tablet_ids_.at(fts_idx)))) { + if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(related_tablet_ids.fts_tablet_ids_.at(fts_idx)))) { LOG_WARN("failed to set related tablet ids", K(ret)); } } @@ -429,7 +425,6 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ attach_ctdef->children_[i], attach_rtdef->children_[i], related_tablet_ids, - ls_id, root_iter->get_children()[i]))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } @@ -442,7 +437,6 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter) { int ret = OB_SUCCESS; @@ -459,7 +453,6 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASLocalLookupIter *lookup_iter = static_cast(root_iter); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); need_set_child = true; } @@ -471,7 +464,6 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASCacheLookupIter *lookup_iter = static_cast(root_iter); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); need_set_child = true; } @@ -483,7 +475,7 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASIndexMergeIter *merge_iter = static_cast(root_iter); - if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) { + if (OB_FAIL(merge_iter->set_tablet_ids(related_tablet_ids))) { LOG_WARN("failed to set related tablet ids", K(ret)); } need_set_child = false; @@ -496,7 +488,6 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASLocalLookupIter *aux_lookup_iter = static_cast(root_iter); - aux_lookup_iter->set_ls_id(ls_id); aux_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); need_set_child = true; } @@ -518,7 +509,7 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd } else { ObDASTRMergeIter *tr_merge_iter = static_cast(root_iter); int64_t fts_index_idx = static_cast(attach_rtdef)->fts_idx_; - if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids.fts_tablet_ids_.at(fts_index_idx)))) { + if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(related_tablet_ids.fts_tablet_ids_.at(fts_index_idx)))) { LOG_WARN("failed to set related tablet ids", K(ret)); } need_set_child = false; @@ -558,7 +549,6 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd if (OB_FAIL(set_index_merge_related_ids(child_attach_ctdef, child_attach_rtdef, related_tablet_ids, - ls_id, root_iter->get_children()[i]))) { LOG_WARN("failed to set index merge related ids", K(ret)); } @@ -571,7 +561,6 @@ int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctd int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter) { int ret = OB_SUCCESS; @@ -589,7 +578,6 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac } else { ObDASCacheLookupIter *local_lookup_iter = static_cast(root_iter); local_lookup_iter->set_tablet_id(related_tablet_ids.rowkey_doc_tablet_id_); - local_lookup_iter->set_ls_id(ls_id); need_set_child = true; } break; @@ -610,14 +598,12 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac } if (func_lookup_ctdef->has_main_table_lookup()) { merge_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); - merge_iter->set_ls_id(ls_id); } for (int64_t i = 0; OB_SUCC(ret) && i < func_lookup_cnt; ++i) { if (OB_FAIL(set_func_lookup_iter_related_ids( func_lookup_ctdef->get_func_lookup_scan_ctdef(i), func_lookup_rtdef->get_func_lookup_scan_rtdef(i), related_tablet_ids, - ls_id, merge_iter->get_children()[i]))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } @@ -632,7 +618,6 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASLocalLookupIter *aux_lookup_iter = static_cast(root_iter); - aux_lookup_iter->set_ls_id(ls_id); aux_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); need_set_child = true; } @@ -654,7 +639,7 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac } else { ObDASTRMergeIter *tr_merge_iter = static_cast(root_iter); int64_t fts_index_idx = static_cast(attach_rtdef)->fts_idx_; - if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids.fts_tablet_ids_.at(fts_index_idx)))) { + if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(related_tablet_ids.fts_tablet_ids_.at(fts_index_idx)))) { LOG_WARN("failed to set related tablet ids", K(ret)); } need_set_child = false; @@ -676,7 +661,7 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); } else { ObDASIndexMergeIter *merge_iter = static_cast(root_iter); - if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) { + if (OB_FAIL(merge_iter->set_tablet_ids(related_tablet_ids))) { LOG_WARN("failed to set related tablet ids", K(ret)); } need_set_child = true; @@ -707,7 +692,6 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac child_attach_ctdef, child_attach_rtdef, related_tablet_ids, - ls_id, root_iter->get_children()[i]))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } @@ -720,8 +704,7 @@ int ObDASIterUtils::set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attac int ObDASIterUtils::set_vec_pre_filter_related_ids(const ObDASVecAuxScanCtDef *vec_aux_ctdef, ObDASVecAuxScanRtDef *vec_aux_rtdef, ObDASIter *pre_filter_iter, - const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id) + const ObDASRelatedTabletID &related_tablet_ids) { int ret = OB_SUCCESS; if (OB_ISNULL(pre_filter_iter) || OB_ISNULL(vec_aux_ctdef) || OB_ISNULL(vec_aux_rtdef)) { @@ -741,7 +724,7 @@ int ObDASIterUtils::set_vec_pre_filter_related_ids(const ObDASVecAuxScanCtDef *v ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(hnsw_scan_iter->get_pre_filter_iter())); } else if (OB_FAIL(ObDASIterUtils::set_index_merge_related_ids( - inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, ls_id, hnsw_scan_iter->get_pre_filter_iter()))) { + inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, hnsw_scan_iter->get_pre_filter_iter()))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } } else if (ObDASUtils::is_func_lookup(inv_idx_ctdef)) { @@ -749,7 +732,7 @@ int ObDASIterUtils::set_vec_pre_filter_related_ids(const ObDASVecAuxScanCtDef *v ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(hnsw_scan_iter->get_pre_filter_iter())); } else if (OB_FAIL(ObDASIterUtils::set_func_lookup_iter_related_ids( - inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, ls_id, hnsw_scan_iter->get_pre_filter_iter()))) { + inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, hnsw_scan_iter->get_pre_filter_iter()))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } } else if (ObDASUtils::is_fts_idx_scan(inv_idx_ctdef)) { @@ -757,19 +740,17 @@ int ObDASIterUtils::set_vec_pre_filter_related_ids(const ObDASVecAuxScanCtDef *v ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(hnsw_scan_iter->get_pre_filter_iter())); } else if (OB_FAIL(ObDASIterUtils::set_text_retrieval_related_ids( - inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, ls_id, hnsw_scan_iter->get_pre_filter_iter()))) { + inv_idx_ctdef, inv_idx_rtdef, related_tablet_ids, hnsw_scan_iter->get_pre_filter_iter()))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } } if (OB_FAIL(ret)) { } else if (ObDASIterType::DAS_ITER_LOCAL_LOOKUP == hnsw_scan_iter->get_inv_idx_scan_iter()->get_type()) { ObDASLocalLookupIter *lookup_iter = static_cast(hnsw_scan_iter->get_inv_idx_scan_iter()); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); } } else if (ObDASIterType::DAS_ITER_LOCAL_LOOKUP == pre_filter_iter->get_type()) { ObDASLocalLookupIter *lookup_iter = static_cast(pre_filter_iter); - lookup_iter->set_ls_id(ls_id); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); } } @@ -780,7 +761,6 @@ int ObDASIterUtils::set_hnsw_lookup_related_ids(const ObDASVecAuxScanCtDef *vec_ ObDASVecAuxScanRtDef *vec_aux_rtdef, ObDASHNSWScanIter *hnsw_scan_iter, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter) { int ret = OB_SUCCESS; @@ -788,16 +768,15 @@ int ObDASIterUtils::set_hnsw_lookup_related_ids(const ObDASVecAuxScanCtDef *vec_ ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr", K(ret), KP(vec_aux_ctdef), KP(vec_aux_rtdef), KP(hnsw_scan_iter)); } else { - hnsw_scan_iter->set_ls_id(ls_id); hnsw_scan_iter->set_related_tablet_ids(related_tablet_ids); if (OB_FAIL(set_vec_pre_filter_related_ids(vec_aux_ctdef, vec_aux_rtdef, - hnsw_scan_iter, related_tablet_ids, ls_id))) { + hnsw_scan_iter, related_tablet_ids))) { LOG_WARN("failed to set vec pre filter related ids", K(ret)); } else if (hnsw_scan_iter->has_func_lookup()) { const ObDASBaseCtDef *func_lookup_ctdef = vec_aux_ctdef->get_functional_lookup_ctdef(); ObDASBaseRtDef *func_lookup_rtdef = vec_aux_rtdef->get_functional_lookup_rtdef(); if (OB_FAIL(ObDASIterUtils::set_func_lookup_iter_related_ids( - func_lookup_ctdef, func_lookup_rtdef, related_tablet_ids, ls_id, hnsw_scan_iter->get_func_lookup_scan_iter()))) { + func_lookup_ctdef, func_lookup_rtdef, related_tablet_ids, hnsw_scan_iter->get_func_lookup_scan_iter()))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } } @@ -808,7 +787,6 @@ int ObDASIterUtils::set_hnsw_lookup_related_ids(const ObDASVecAuxScanCtDef *vec_ int ObDASIterUtils::set_vec_lookup_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter) { int ret = OB_SUCCESS; @@ -826,7 +804,6 @@ int ObDASIterUtils::set_vec_lookup_related_ids(const ObDASBaseCtDef *attach_ctde ObDASIter *iter = root_iter->get_children()[0]; if (nullptr != iter && ObDASIterType::DAS_ITER_LOCAL_LOOKUP == iter->get_type()) { ObDASLocalLookupIter *local_lookup_iter = static_cast(iter); - local_lookup_iter->set_ls_id(ls_id); if (is_shortcut_scan) { local_lookup_iter->set_tablet_id(related_tablet_ids.vid_rowkey_tablet_id_); } else { @@ -836,63 +813,57 @@ int ObDASIterUtils::set_vec_lookup_related_ids(const ObDASBaseCtDef *attach_ctde } else if (is_shortcut_scan) { if (root_iter->get_type() == ObDASIterType::DAS_ITER_HNSW_SCAN) { ObDASHNSWScanIter *hnsw_scan_iter = static_cast(root_iter); - if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, ls_id, root_iter))) { + if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, root_iter))) { LOG_WARN("failed to set hnsw lookup related ids", K(ret)); } } else { ObDASLocalLookupIter *aux_lookup_iter = static_cast(root_iter); - aux_lookup_iter->set_ls_id(ls_id); aux_lookup_iter->set_tablet_id(related_tablet_ids.vid_rowkey_tablet_id_); if (ObDASIterType::DAS_ITER_HNSW_SCAN == aux_lookup_iter->get_children()[0]->get_type()) { ObDASHNSWScanIter *hnsw_scan_iter = static_cast(aux_lookup_iter->get_children()[0]); - if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, ls_id, root_iter))) { + if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, root_iter))) { LOG_WARN("failed to set hnsw lookup related ids", K(ret)); } } } } else if (ObDASIterType::DAS_ITER_LOCAL_LOOKUP == root_iter->get_type()) { ObDASLocalLookupIter *local_lookup_iter = static_cast(root_iter); - local_lookup_iter->set_ls_id(ls_id); local_lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); if (ObDASIterType::DAS_ITER_LOCAL_LOOKUP == local_lookup_iter->get_children()[0]->get_type()) { ObDASLocalLookupIter *aux_lookup_iter = static_cast(local_lookup_iter->get_children()[0]); - aux_lookup_iter->set_ls_id(ls_id); if (ObDASIterType::DAS_ITER_HNSW_SCAN == aux_lookup_iter->get_children()[0]->get_type()) { aux_lookup_iter->set_tablet_id(related_tablet_ids.vid_rowkey_tablet_id_); ObDASHNSWScanIter *hnsw_scan_iter = static_cast(aux_lookup_iter->get_children()[0]); - if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, ls_id, root_iter))) { + if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, root_iter))) { LOG_WARN("failed to set hnsw lookup related ids", K(ret)); } } else if (ObDASIterType::DAS_ITER_SPIV_MERGE == aux_lookup_iter->get_children()[0]->get_type()) { aux_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); ObDASSPIVMergeIter *spiv_merge_iter = static_cast(aux_lookup_iter->get_children()[0]); - spiv_merge_iter->set_ls_id(ls_id); spiv_merge_iter->set_related_tablet_ids(related_tablet_ids); if (OB_FAIL(set_vec_pre_filter_related_ids(vec_aux_ctdef, vec_aux_rtdef, - spiv_merge_iter->get_inv_idx_scan_iter(), related_tablet_ids, ls_id))) { + spiv_merge_iter->get_inv_idx_scan_iter(), related_tablet_ids))) { LOG_WARN("failed to set vec pre filter related ids", K(ret)); } } } else if (ObDASIterType::DAS_ITER_IVF_SCAN == local_lookup_iter->get_children()[0]->get_type()) { ObDASIvfBaseScanIter *ivf_scan_iter = static_cast(local_lookup_iter->get_children()[0]); - ivf_scan_iter->set_ls_id(ls_id); ivf_scan_iter->set_related_tablet_ids(related_tablet_ids); if (OB_FAIL(set_vec_pre_filter_related_ids(vec_aux_ctdef, vec_aux_rtdef, - ivf_scan_iter->get_inv_idx_scan_iter(), related_tablet_ids, ls_id))) { + ivf_scan_iter->get_inv_idx_scan_iter(), related_tablet_ids))) { LOG_WARN("failed to set vec pre filter related ids", K(ret)); } } else if (ObDASIterType::DAS_ITER_HNSW_SCAN == local_lookup_iter->get_children()[0]->get_type()) { ObDASHNSWScanIter *hnsw_scan_iter = static_cast(local_lookup_iter->get_children()[0]); - if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, ls_id, root_iter))) { + if (OB_FAIL(set_hnsw_lookup_related_ids(vec_aux_ctdef, vec_aux_rtdef, hnsw_scan_iter, related_tablet_ids, root_iter))) { LOG_WARN("failed to set hnsw lookup related ids", K(ret)); } } else if (ObDASIterType::DAS_ITER_SPIV_MERGE == local_lookup_iter->get_children()[0]->get_type()) { ObDASSPIVMergeIter *spiv_merge_iter = static_cast(local_lookup_iter->get_children()[0]); - spiv_merge_iter->set_ls_id(ls_id); spiv_merge_iter->set_related_tablet_ids(related_tablet_ids); if (OB_FAIL(set_vec_pre_filter_related_ids(vec_aux_ctdef, vec_aux_rtdef, - spiv_merge_iter->get_inv_idx_scan_iter(), related_tablet_ids, ls_id))) { + spiv_merge_iter->get_inv_idx_scan_iter(), related_tablet_ids))) { LOG_WARN("failed to set vec pre filter related ids", K(ret)); } } @@ -1110,7 +1081,6 @@ int ObDASIterUtils::create_local_lookup_sub_tree(ObTableScanParam &scan_param, lookup_iter->get_children()[1] = data_table_sub_tree; data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); lookup_iter->set_tablet_id(lookup_tablet_id); - lookup_iter->set_ls_id(scan_param.ls_id_); iter_tree = lookup_iter; } } @@ -1182,7 +1152,6 @@ int ObDASIterUtils::create_text_retrieval_tree(ObTableScanParam &scan_param, ir_scan_rtdef))) { LOG_WARN("fail to find ir scan definition", K(ret)); } else if (OB_FAIL(create_text_retrieval_sub_tree( - scan_param.ls_id_, alloc, ir_scan_ctdef, ir_scan_rtdef, @@ -1279,7 +1248,6 @@ int ObDASIterUtils::create_text_retrieval_tree(ObTableScanParam &scan_param, LOG_WARN("failed to create cache lookup sub tree", K(ret)); } else if (has_lookup && OB_FAIL(create_domain_lookup_sub_tree( scan_param, - scan_param.ls_id_, alloc, table_lookup_ctdef, table_lookup_rtdef, @@ -1420,7 +1388,6 @@ int ObDASIterUtils::create_match_iter_tree(ObTableScanParam &scan_param, main_lookup_keep_order = true; if (OB_FAIL(create_domain_lookup_sub_tree( scan_param, - scan_param.ls_id_, alloc, table_lookup_ctdef, table_lookup_rtdef, @@ -1551,8 +1518,7 @@ int ObDASIterUtils::create_match_part_score_sub_tree(ObTableScanParam &scan_para } else if (FALSE_IT(ir_scan_rtdef = static_cast(es_match_rtdef->children_[i]))) { } else if (FALSE_IT(ir_scan_rtdef->minimum_should_match_ = minimum_should_match)) { } else if (OB_ISNULL(domain_id_expr) && FALSE_IT(domain_id_expr = ir_scan_ctdef->inv_scan_domain_id_col_)) { - } else if (OB_FAIL(create_text_retrieval_sub_tree(scan_param.ls_id_, - alloc, + } else if (OB_FAIL(create_text_retrieval_sub_tree(alloc, ir_scan_ctdef, ir_scan_rtdef, related_tablet_ids.fts_tablet_ids_.at(ir_scan_rtdef->fts_idx_), @@ -1614,7 +1580,6 @@ int ObDASIterUtils::create_match_part_score_sub_tree(ObTableScanParam &scan_para } int ObDASIterUtils::create_text_retrieval_sub_tree( - const ObLSID &ls_id, common::ObIAllocator &alloc, const ObDASIRScanCtDef *ir_scan_ctdef, ObDASIRScanRtDef *ir_scan_rtdef, @@ -1736,7 +1701,7 @@ int ObDASIterUtils::create_text_retrieval_sub_tree( for (int64_t i = 0; i < iters.count(); ++i) { tr_merge_children[i] = iters.at(i); } - tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids); + tr_merge_iter->set_related_tablet_ids(related_tablet_ids); retrieval_result = tr_merge_iter; } @@ -1814,7 +1779,6 @@ int ObDASIterUtils::create_doc_id_scan_sub_tree( } if (OB_FAIL(ret)) { } else { - domain_id_merge_param.rowkey_domain_ls_id_ = scan_param.ls_id_; domain_id_merge_param.data_table_iter_ = data_table_iter; domain_id_merge_param.data_table_ctdef_ = data_ctdef; domain_id_merge_param.data_table_rtdef_ = data_rtdef; @@ -1909,7 +1873,6 @@ int ObDASIterUtils::create_vid_scan_sub_tree( } if (OB_FAIL(ret)) { } else { - domain_id_merge_param.rowkey_domain_ls_id_ = scan_param.ls_id_; domain_id_merge_param.data_table_iter_ = data_table_iter; domain_id_merge_param.data_table_ctdef_ = data_ctdef; domain_id_merge_param.data_table_rtdef_ = data_rtdef; @@ -1980,7 +1943,6 @@ int ObDASIterUtils::create_domain_id_scan_sub_tree( } } if (OB_SUCC(ret)) { - domain_id_merge_param.rowkey_domain_ls_id_ = scan_param.ls_id_; domain_id_merge_param.data_table_iter_ = data_table_iter; domain_id_merge_param.data_table_ctdef_ = data_ctdef; domain_id_merge_param.data_table_rtdef_ = data_rtdef; @@ -2005,7 +1967,6 @@ int ObDASIterUtils::create_domain_id_scan_sub_tree( } int ObDASIterUtils::create_domain_lookup_sub_tree(ObTableScanParam &scan_param, - const ObLSID &ls_id, common::ObIAllocator &alloc, const ObDASTableLookupCtDef *table_lookup_ctdef, ObDASTableLookupRtDef *table_lookup_rtdef, @@ -2152,7 +2113,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, ir_scan_rtdef))) { LOG_WARN("fail to find ir scan definition", K(ret)); } else if (OB_FAIL(create_text_retrieval_sub_tree( - scan_param.ls_id_, alloc, ir_scan_ctdef, ir_scan_rtdef, @@ -2220,7 +2180,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, doc_id_lookup_iter->get_children()[1] = docid_rowkey_table_iter; docid_rowkey_table_iter->set_scan_param(doc_id_lookup_iter->get_lookup_param()); doc_id_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); - doc_id_lookup_iter->set_ls_id(scan_param.ls_id_); rowkey_scan_iter = doc_id_lookup_iter; } } @@ -2230,7 +2189,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, ObDASIRScanRtDef *ir_scan_rtdef = static_cast(rowkey_scan_rtdef); ObDASIter *text_retrieval_result = nullptr; if (OB_FAIL(create_text_retrieval_sub_tree( - scan_param.ls_id_, alloc, ir_scan_ctdef, ir_scan_rtdef, @@ -2284,7 +2242,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, const ObDASIRScanCtDef *ir_scan_ctdef = static_cast(child_ctdef); ObDASIRScanRtDef *ir_scan_rtdef = static_cast(child_rtdef); if (OB_FAIL(create_text_retrieval_sub_tree( - scan_param.ls_id_, alloc, ir_scan_ctdef, ir_scan_rtdef, @@ -2426,7 +2383,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, rowkey_exprs.push_back(func_lookup_ctdef->lookup_domain_id_expr_); if (FAILEDx(create_functional_lookup_sub_tree( scan_param, - scan_param.ls_id_, alloc, func_lookup_ctdef, func_lookup_rtdef, @@ -2475,7 +2431,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, static_cast(func_lookup_result)->set_index_scan_param(root_lookup_iter->get_lookup_param()); } root_lookup_iter->set_tablet_id(related_tablet_ids.rowkey_doc_tablet_id_); - root_lookup_iter->set_ls_id(scan_param.ls_id_); iter_tree = root_lookup_iter; } } @@ -2483,7 +2438,6 @@ int ObDASIterUtils::create_function_lookup_tree(ObTableScanParam &scan_param, } int ObDASIterUtils::create_functional_lookup_sub_tree(ObTableScanParam &scan_param, - const ObLSID &ls_id, common::ObIAllocator &alloc, const ObDASFuncLookupCtDef *func_lookup_ctdef, ObDASFuncLookupRtDef *func_lookup_rtdef, @@ -2521,8 +2475,7 @@ int ObDASIterUtils::create_functional_lookup_sub_tree(ObTableScanParam &scan_par data_table_iters[i] = nullptr; const ObDASIRScanCtDef *ir_ctdef = static_cast(func_lookup_ctdef->get_func_lookup_scan_ctdef(i)); ObDASIRScanRtDef *ir_rtdef = static_cast(func_lookup_rtdef->get_func_lookup_scan_rtdef(i)); - if (OB_FAIL(create_text_retrieval_sub_tree(scan_param.ls_id_, - alloc, + if (OB_FAIL(create_text_retrieval_sub_tree(alloc, ir_ctdef, ir_rtdef, related_tablet_ids.fts_tablet_ids_.at(ir_rtdef->fts_idx_), @@ -2586,7 +2539,6 @@ int ObDASIterUtils::create_functional_lookup_sub_tree(ObTableScanParam &scan_par main_lookup_table_iter->set_scan_param(fts_merge_iter->get_main_lookup_scan_param()); } fts_merge_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); // for main_lookup - fts_merge_iter->set_ls_id(ls_id); } // create function lookup iter @@ -2743,7 +2695,6 @@ int ObDASIterUtils::create_mvi_lookup_tree(ObTableScanParam &scan_param, index_table_iter->set_scan_param(scan_param); docid_rowkey_table_iter->set_scan_param(mvi_lookup_iter->get_lookup_param()); mvi_lookup_iter->set_tablet_id(related_tablet_ids.doc_rowkey_tablet_id_); - mvi_lookup_iter->set_ls_id(scan_param.ls_id_); } } @@ -3194,7 +3145,6 @@ int ObDASIterUtils::create_local_lookup_sub_tree(ObTableScanParam &scan_param, lookup_iter->get_children()[1] = data_table_iter; data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); lookup_iter->set_tablet_id(lookup_tablet_id); - lookup_iter->set_ls_id(scan_param.ls_id_); } } return ret; @@ -3328,7 +3278,6 @@ int ObDASIterUtils::create_cache_lookup_sub_tree(ObTableScanParam &scan_param, lookup_iter->get_children()[1] = data_table_sub_tree; data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); lookup_iter->set_tablet_id(proj_lookup ? related_tablet_ids.lookup_tablet_id_ : related_tablet_ids.doc_rowkey_tablet_id_); - lookup_iter->set_ls_id(scan_param.ls_id_); } } @@ -3420,8 +3369,7 @@ if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef) || ctdef->op_type_ != DAS_OP_INDEX_MERG if (OB_ISNULL(ir_ctdef) || OB_ISNULL(ir_rtdef)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null ir ctdef or rtdef", K(ret)); - } else if (OB_FAIL(create_text_retrieval_sub_tree(scan_param.ls_id_, - alloc, + } else if (OB_FAIL(create_text_retrieval_sub_tree(alloc, ir_ctdef, ir_rtdef, related_tablet_ids.fts_tablet_ids_.at(ir_rtdef->fts_idx_), @@ -3525,7 +3473,7 @@ if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef) || ctdef->op_type_ != DAS_OP_INDEX_MERG merge_param.rowkey_exprs_ = &merge_ctdef->rowkey_exprs_; if (OB_FAIL(create_das_iter(alloc, merge_param, merge_iter))) { LOG_WARN("failed to create das index merge iter", K(ret)); - } else if (OB_FAIL(merge_iter->set_ls_tablet_ids(scan_param.ls_id_, related_tablet_ids))) { + } else if (OB_FAIL(merge_iter->set_tablet_ids(related_tablet_ids))) { LOG_WARN("failed to set ls tablet ids", K(ret)); } else if (OB_FAIL(create_iter_children_array(children_cnt, alloc, merge_iter))) { LOG_WARN("failed to create iter children array", K(ret)); @@ -3741,7 +3689,6 @@ int ObDASIterUtils::create_vec_spiv_lookup_tree(ObTableScanParam &scan_param, spiv_merge_param.block_max_scan_rtdef_ = block_max_scan_rtdef; spiv_merge_param.vec_aux_ctdef_ = vec_aux_ctdef; spiv_merge_param.vec_aux_rtdef_ = vec_aux_rtdef; - spiv_merge_param.ls_id_ = scan_param.ls_id_; spiv_merge_param.tx_desc_ = trans_desc; spiv_merge_param.snapshot_ = snapshot; uint64_t batch_count = spiv_merge_param.max_size_; @@ -3951,7 +3898,6 @@ int ObDASIterUtils::create_vec_func_indexback_sub_tree(ObTableScanParam &scan_pa lookup_iter->get_children()[1] = data_table_sub_tree; data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); - lookup_iter->set_ls_id(scan_param.ls_id_); } } @@ -4099,7 +4045,7 @@ int ObDASIterUtils::create_vec_hnsw_lookup_tree(ObTableScanParam &scan_param, && OB_FAIL(create_das_scan_iter(alloc, data_table_ctdef, data_table_rtdef, data_filter_iter))) { LOG_WARN("failed to create data filter scan iter", K(ret)); } else if (OB_NOT_NULL(func_lookup_ctdef) && OB_NOT_NULL(func_lookup_rtdef) && - OB_FAIL(create_functional_lookup_sub_tree(scan_param, scan_param.ls_id_, alloc, + OB_FAIL(create_functional_lookup_sub_tree(scan_param, alloc, func_lookup_ctdef, func_lookup_rtdef, related_tablet_ids, true, trans_desc, snapshot, func_lookup_iter))) { LOG_WARN("failed to create func lookup scan iter", K(ret)); } @@ -4143,7 +4089,6 @@ int ObDASIterUtils::create_vec_hnsw_lookup_tree(ObTableScanParam &scan_param, hnsw_scan_param.vid_rowkey_rtdef_ = vid_rowkey_rtdef; hnsw_scan_param.sort_ctdef_ = sort_ctdef; hnsw_scan_param.sort_rtdef_ = sort_rtdef; - hnsw_scan_param.ls_id_ = scan_param.ls_id_; hnsw_scan_param.tx_desc_ = trans_desc; hnsw_scan_param.snapshot_ = snapshot; hnsw_scan_param.pre_scan_param_ = &scan_param; @@ -4166,7 +4111,7 @@ int ObDASIterUtils::create_vec_hnsw_lookup_tree(ObTableScanParam &scan_param, } if (OB_FAIL(create_das_iter(alloc, hnsw_scan_param, hnsw_scan_iter))) { - LOG_WARN("failed to create hnsw scan iter", K(ret), K(hnsw_scan_param.ls_id_), K(hnsw_scan_param.tx_desc_), + LOG_WARN("failed to create hnsw scan iter", K(ret), K(hnsw_scan_param.tx_desc_), K(hnsw_scan_param.snapshot_), K(hnsw_scan_param.delta_buf_iter_), K(hnsw_scan_param.index_id_iter_), K(hnsw_scan_param.snapshot_iter_), K(hnsw_scan_param.com_aux_vec_iter_), K(hnsw_scan_param.rowkey_vid_iter_), K(hnsw_scan_param.vec_aux_ctdef_), K(hnsw_scan_param.vec_aux_rtdef_), K(hnsw_scan_param.vid_rowkey_iter_), @@ -4387,7 +4332,6 @@ int ObDASIterUtils::create_vec_ivf_lookup_tree(ObTableScanParam &scan_param, ivf_scan_param.vec_aux_rtdef_ = vec_aux_rtdef; ivf_scan_param.sort_ctdef_ = sort_ctdef; ivf_scan_param.sort_rtdef_ = sort_rtdef; - ivf_scan_param.ls_id_ = scan_param.ls_id_; ivf_scan_param.tx_desc_ = trans_desc; ivf_scan_param.snapshot_ = snapshot; ivf_scan_param.brute_iter_ = brute_iter; @@ -4426,4 +4370,3 @@ int ObDASIterUtils::create_vec_ivf_lookup_tree(ObTableScanParam &scan_param, } // namespace sql } // namespace oceanbase - diff --git a/src/sql/das/iter/ob_das_iter_utils.h b/src/sql/das/iter/ob_das_iter_utils.h index 430df8c28..01da1bfcf 100644 --- a/src/sql/das/iter/ob_das_iter_utils.h +++ b/src/sql/das/iter/ob_das_iter_utils.h @@ -85,40 +85,33 @@ class ObDASIterUtils static int set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter); static int set_func_lookup_iter_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter); static int set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter); static int set_hnsw_lookup_related_ids(const ObDASVecAuxScanCtDef *vec_aux_ctdef, ObDASVecAuxScanRtDef *vec_aux_rtdef, ObDASHNSWScanIter *hnsw_scan_iter, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter); static int set_vec_lookup_related_ids(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef, const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id, ObDASIter *root_iter); static int set_vec_pre_filter_related_ids(const ObDASVecAuxScanCtDef *vec_aux_ctdef, ObDASVecAuxScanRtDef *vec_aux_rtdef, ObDASIter *pre_filter_iter, - const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id); + const ObDASRelatedTabletID &related_tablet_ids); static int set_vec_pre_local_lookup_related_ids(const ObDASVecAuxScanCtDef *vec_aux_ctdef, ObDASVecAuxScanRtDef *vec_aux_rtdef, ObDASIter *pre_filter_iter, - const ObDASRelatedTabletID &related_tablet_ids, - const ObLSID &ls_id); + const ObDASRelatedTabletID &related_tablet_ids); static bool is_vec_ivf_scan(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef); static bool is_vec_spiv_scan(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef); static bool is_vec_hnsw_scan(const ObDASBaseCtDef *attach_ctdef, ObDASBaseRtDef *attach_rtdef); @@ -248,7 +241,6 @@ class ObDASIterUtils ObDASIter *&iter_tree); static int create_domain_lookup_sub_tree(ObTableScanParam &scan_param, - const ObLSID &ls_id, common::ObIAllocator &alloc, const ObDASTableLookupCtDef *table_lookup_ctdef, ObDASTableLookupRtDef *table_lookup_rtdef, @@ -338,8 +330,7 @@ class ObDASIterUtils transaction::ObTxDesc *trans_desc, transaction::ObTxReadSnapshot *snapshot, ObDASIter *&iter_tree); - static int create_text_retrieval_sub_tree(const ObLSID &ls_id, - common::ObIAllocator &alloc, + static int create_text_retrieval_sub_tree(common::ObIAllocator &alloc, const ObDASIRScanCtDef *ir_scan_ctdef, ObDASIRScanRtDef *ir_scan_rtdef, const ObDASFTSTabletID &related_tablet_ids, @@ -410,7 +401,6 @@ class ObDASIterUtils transaction::ObTxReadSnapshot *snapshot, ObDASIter *&iter); static int create_functional_lookup_sub_tree(ObTableScanParam &scan_param, - const ObLSID &ls_id, common::ObIAllocator &alloc, const ObDASFuncLookupCtDef *table_lookup_ctdef, ObDASFuncLookupRtDef *table_lookup_rtdef, diff --git a/src/sql/das/iter/ob_das_ivf_scan_iter.cpp b/src/sql/das/iter/ob_das_ivf_scan_iter.cpp index 37a4e5f21..2927b78b6 100644 --- a/src/sql/das/iter/ob_das_ivf_scan_iter.cpp +++ b/src/sql/das/iter/ob_das_ivf_scan_iter.cpp @@ -106,7 +106,7 @@ int ObDASIvfBaseScanIter::do_table_full_scan(bool is_vectorized, if (first_scan) { ObNewRange scan_range; - if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param, + if (OB_FAIL(ObDasVecScanUtils::init_scan_param(tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param, false /*is_get*/, &mem_context_->get_arena_allocator()))) { LOG_WARN("failed to generate init vec aux scan param", K(ret)); } else if (OB_FALSE_IT(ObDasVecScanUtils::set_whole_range(scan_range, ctdef->ref_table_id_))) { @@ -123,7 +123,6 @@ int ObDASIvfBaseScanIter::do_table_full_scan(bool is_vectorized, scan_param.need_switch_param_ = scan_param.need_switch_param_ || (scan_tablet_id.is_valid() && (tablet_id != scan_tablet_id)); scan_param.tablet_id_ = tablet_id; - scan_param.ls_id_ = ls_id_; ObNewRange scan_range; if (OB_FAIL(iter->reuse())) { @@ -150,7 +149,7 @@ int ObDASIvfBaseScanIter::do_aux_table_scan(bool &first_scan, if (first_scan) { scan_param.need_switch_param_ = false; if (OB_FAIL(ObDasVecScanUtils::init_scan_param( - ls_id_, tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param, false/*is_get*/, &mem_context_->get_arena_allocator()))) { + tablet_id, ctdef, rtdef, tx_desc_, snapshot_, scan_param, false/*is_get*/, &mem_context_->get_arena_allocator()))) { LOG_WARN("failed to init scan param", K(ret)); } else if (OB_FALSE_IT(iter->set_scan_param(scan_param))) { } else if (OB_FAIL(iter->do_table_scan())) { @@ -163,7 +162,6 @@ int ObDASIvfBaseScanIter::do_aux_table_scan(bool &first_scan, scan_param.need_switch_param_ = scan_param.need_switch_param_ || (scan_tablet_id.is_valid() && (tablet_id != scan_tablet_id)); scan_param.tablet_id_ = tablet_id; - scan_param.ls_id_ = ls_id_; if (OB_FAIL(iter->rescan())) { LOG_WARN("fail to rescan scan iterator.", K(ret)); } @@ -178,17 +176,17 @@ int ObDASIvfBaseScanIter::inner_reuse() if (OB_NOT_NULL(inv_idx_scan_iter_) && OB_FAIL(inv_idx_scan_iter_->reuse())) { LOG_WARN("failed to reuse inv idx scan iter", K(ret)); } else if (!centroid_iter_first_scan_ && OB_FAIL(ObDasVecScanUtils::reuse_iter( - ls_id_, centroid_iter_, centroid_scan_param_, centroid_tablet_id_))) { + centroid_iter_, centroid_scan_param_, centroid_tablet_id_))) { LOG_WARN("failed to reuse com aux vec iter", K(ret)); } else if (!cid_vec_iter_first_scan_ && - OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, cid_vec_iter_, cid_vec_scan_param_, cid_vec_tablet_id_))) { + OB_FAIL(ObDasVecScanUtils::reuse_iter(cid_vec_iter_, cid_vec_scan_param_, cid_vec_tablet_id_))) { LOG_WARN("failed to reuse rowkey vid iter", K(ret)); } else if (!rowkey_cid_iter_first_scan_ && OB_FAIL(ObDasVecScanUtils::reuse_iter( - ls_id_, rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { + rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { LOG_WARN("failed to reuse vid rowkey iter", K(ret)); } else if (!brute_first_scan_ && OB_FAIL(ObDasVecScanUtils::reuse_iter( - ls_id_, brute_iter_, brute_scan_param_, brute_tablet_id_))) { + brute_iter_, brute_scan_param_, brute_tablet_id_))) { LOG_WARN("failed to reuse iter", K(ret)); } @@ -216,7 +214,6 @@ int ObDASIvfBaseScanIter::inner_init(ObDASIterParam ¶m) LOG_WARN("invalid das iter param type for ivf scan iter", K(ret), K(param)); } else { ObDASIvfScanIterParam &ivf_scan_param = static_cast(param); - ls_id_ = ivf_scan_param.ls_id_; tx_desc_ = ivf_scan_param.tx_desc_; snapshot_ = ivf_scan_param.snapshot_; @@ -512,7 +509,7 @@ int ObDASIvfBaseScanIter::do_rowkey_cid_table_scan() vec_aux_ctdef_->get_ivf_rowkey_cid_tbl_idx(), ObTSCIRScanType::OB_VEC_IVF_ROWKEY_CID_SCAN); ObDASScanRtDef *rowkey_cid_rtdef = vec_aux_rtdef_->get_vec_aux_tbl_rtdef(vec_aux_ctdef_->get_ivf_rowkey_cid_tbl_idx()); - if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, + if (OB_FAIL(ObDasVecScanUtils::init_scan_param( rowkey_cid_tablet_id_, rowkey_cid_ctdef, rowkey_cid_rtdef, @@ -533,7 +530,6 @@ int ObDASIvfBaseScanIter::do_rowkey_cid_table_scan() rowkey_cid_scan_param_.need_switch_param_ = rowkey_cid_scan_param_.need_switch_param_ || (scan_tablet_id.is_valid() && (rowkey_cid_tablet_id_ != scan_tablet_id)); rowkey_cid_scan_param_.tablet_id_ = rowkey_cid_tablet_id_; - rowkey_cid_scan_param_.ls_id_ = ls_id_; if (OB_FAIL(rowkey_cid_iter_->rescan())) { LOG_WARN("fail to rescan cid vec table scan iterator.", K(ret)); @@ -766,7 +762,7 @@ int ObDASIvfBaseScanIter::try_write_centroid_cache( } } // end for int tmp_ret = (ret == OB_ITER_END) ? OB_SUCCESS : ret; - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, centroid_iter_, centroid_scan_param_, centroid_tablet_id_))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(centroid_iter_, centroid_scan_param_, centroid_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -804,10 +800,9 @@ int ObDASIvfBaseScanIter::get_centers_cache(bool is_vectorized, vec_aux_ctdef_->get_ivf_centroid_tbl_idx(), ObTSCIRScanType::OB_VEC_IVF_CENTROID_SCAN); // pq/flat both use centroid_tablet_id_ - if (OB_FAIL(vec_index_service->acquire_ivf_cache_mgr_guard( - ls_id_, centroid_tablet_id_, vec_index_param_, dim_, centroid_ctdef->ref_table_id_, cache_guard))) { + if (OB_FAIL(vec_index_service->acquire_ivf_cache_mgr_guard(centroid_tablet_id_, vec_index_param_, dim_, centroid_ctdef->ref_table_id_, cache_guard))) { LOG_WARN("failed to get ObPluginVectorIndexAdapter", - K(ret), K(ls_id_), K(centroid_tablet_id_), K(vec_index_param_)); + K(ret), K(centroid_tablet_id_), K(vec_index_param_)); } else if (OB_ISNULL(cache_mgr = cache_guard.get_ivf_cache_mgr())) { ret = OB_ERR_NULL_VALUE; LOG_WARN("invalid null cache mgr", K(ret)); @@ -1503,7 +1498,7 @@ int ObDASIvfScanIter::filter_rowkey_by_cid(bool is_vectorized, } int tmp_ret = ret; if (OB_FAIL( - ObDasVecScanUtils::reuse_iter(ls_id_, rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { + ObDasVecScanUtils::reuse_iter(rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -1730,7 +1725,7 @@ int ObDASIvfPQScanIter::inner_reuse() memset(near_cid_vec_ptrs_.get_data(), 0, near_cid_vec_ptrs_.count() * sizeof(float *)); } if (!pq_centroid_first_scan_ && OB_FAIL(ObDasVecScanUtils::reuse_iter( - ls_id_, pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { + pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { LOG_WARN("failed to reuse iter", K(ret)); } else if (OB_FAIL(ObDASIvfBaseScanIter::inner_reuse())) { LOG_WARN("fail to do inner reuse", K(ret)); @@ -1928,7 +1923,7 @@ int ObDASIvfPQScanIter::calc_distance_between_pq_ids_by_table( } } int tmp_ret = (ret == OB_ITER_END) ? OB_SUCCESS : ret; - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -2474,7 +2469,7 @@ int ObDASIvfBaseScanIter::get_rowkey_brute_post(bool is_vectorized, IvfRowkeyHea } } // end while int tmp_ret = (ret == OB_ITER_END) ? OB_SUCCESS : ret; - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, brute_iter_, brute_scan_param_, brute_tablet_id_))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(brute_iter_, brute_scan_param_, brute_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -2696,7 +2691,7 @@ int ObDASIvfPQScanIter::filter_rowkey_by_cid(bool is_vectorized, int64_t batch_r } // end for int tmp_ret = ret; if (OB_FAIL( - ObDasVecScanUtils::reuse_iter(ls_id_, rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { + ObDasVecScanUtils::reuse_iter(rowkey_cid_iter_, rowkey_cid_scan_param_, rowkey_cid_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -2919,7 +2914,7 @@ int ObDASIvfPQScanIter::try_write_pq_centroid_cache( } } int tmp_ret = (ret == OB_ITER_END) ? OB_SUCCESS : ret; - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(pq_centroid_iter_, pq_centroid_scan_param_, pq_centroid_tablet_id_))) { LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); } else { ret = tmp_ret; @@ -2957,10 +2952,9 @@ int ObDASIvfPQScanIter::get_pq_precomputetable_cache( const ObDASScanCtDef *centroid_ctdef = vec_aux_ctdef_->get_vec_aux_tbl_ctdef( vec_aux_ctdef_->get_ivf_centroid_tbl_idx(), ObTSCIRScanType::OB_VEC_IVF_CENTROID_SCAN); // pq/flat both use centroid_tablet_id_ - if (OB_FAIL(vec_index_service->acquire_ivf_cache_mgr_guard( - ls_id_, centroid_tablet_id_, vec_index_param_, dim_, centroid_ctdef->ref_table_id_, cache_guard))) { + if (OB_FAIL(vec_index_service->acquire_ivf_cache_mgr_guard(centroid_tablet_id_, vec_index_param_, dim_, centroid_ctdef->ref_table_id_, cache_guard))) { LOG_WARN("failed to get ObPluginVectorIndexAdapter", - K(ret), K(ls_id_), K(centroid_tablet_id_), K(vec_index_param_)); + K(ret), K(centroid_tablet_id_), K(vec_index_param_)); } else if (OB_ISNULL(cache_mgr = cache_guard.get_ivf_cache_mgr())) { ret = OB_ERR_NULL_VALUE; LOG_WARN("invalid null cache mgr", K(ret)); diff --git a/src/sql/das/iter/ob_das_ivf_scan_iter.h b/src/sql/das/iter/ob_das_ivf_scan_iter.h index 2ffbc6671..ebeeb7680 100644 --- a/src/sql/das/iter/ob_das_ivf_scan_iter.h +++ b/src/sql/das/iter/ob_das_ivf_scan_iter.h @@ -51,7 +51,7 @@ namespace sql } \ if (index_end) { \ int tmp_ret = (ret == OB_ITER_END) ? OB_SUCCESS : ret; \ - if (OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, iter, scan_param, tablet_id))) { \ + if (OB_FAIL(ObDasVecScanUtils::reuse_iter(iter, scan_param, tablet_id))) { \ LOG_WARN("failed to reuse rowkey cid iter.", K(ret)); \ } else { \ ret = tmp_ret; \ @@ -62,7 +62,6 @@ struct ObDASIvfScanIterParam : public ObDASIterParam { public: explicit ObDASIvfScanIterParam(const ObVectorIndexAlgorithmType index_type) : ObDASIterParam(ObDASIterType::DAS_ITER_IVF_SCAN), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), inv_idx_scan_iter_(nullptr), @@ -81,7 +80,7 @@ struct ObDASIvfScanIterParam : public ObDASIterParam { virtual bool is_valid() const override { - bool bret = ls_id_.is_valid() && nullptr != tx_desc_ && nullptr != snapshot_ && nullptr != inv_idx_scan_iter_ && + bool bret = nullptr != tx_desc_ && nullptr != snapshot_ && nullptr != inv_idx_scan_iter_ && nullptr != vec_aux_ctdef_ && nullptr != vec_aux_rtdef_; if (bret != true) { } else if (index_type == ObVectorIndexAlgorithmType::VIAT_IVF_FLAT) { @@ -99,7 +98,6 @@ struct ObDASIvfScanIterParam : public ObDASIterParam { return bret; } - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; @@ -163,7 +161,6 @@ class ObDASIvfBaseScanIter : public ObDASIter mem_context_(nullptr), vec_op_alloc_("IvfIdxLookupOp", OB_MALLOC_NORMAL_BLOCK_SIZE), persist_alloc_(ObMemAttr("IvfScan")), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), centroid_iter_(nullptr), @@ -217,11 +214,6 @@ class ObDASIvfBaseScanIter : public ObDASIter return inv_idx_scan_iter_; } - void set_ls_id(const share::ObLSID &ls_id) - { - ls_id_ = ls_id; - } - protected: virtual int inner_init(ObDASIterParam ¶m) override; virtual int inner_reuse() override; @@ -328,7 +320,6 @@ class ObDASIvfBaseScanIter : public ObDASIter // unlike vec_op_alloc_ do reset() in inner_resuse() // persist_alloc_ do reset() in inner_release() ObArenaAllocator persist_alloc_; - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; @@ -561,7 +552,7 @@ class ObDASIvfSQ8ScanIter : public ObDASIvfScanIter { int ret = OB_SUCCESS; if (!sq_meta_iter_first_scan_ && - OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, sq_meta_iter_, sq_meta_scan_param_, sq_meta_tablet_id_))) { + OB_FAIL(ObDasVecScanUtils::reuse_iter(sq_meta_iter_, sq_meta_scan_param_, sq_meta_tablet_id_))) { LOG_WARN("failed to reuse iter", K(ret)); } else { ret = ObDASIvfScanIter::inner_reuse(); diff --git a/src/sql/das/iter/ob_das_local_lookup_iter.cpp b/src/sql/das/iter/ob_das_local_lookup_iter.cpp index 18ef3cec1..85e359381 100644 --- a/src/sql/das/iter/ob_das_local_lookup_iter.cpp +++ b/src/sql/das/iter/ob_das_local_lookup_iter.cpp @@ -111,7 +111,6 @@ int ObDASLocalLookupIter::init_scan_param(ObTableScanParam ¶m, const ObDASSc LOG_WARN("unexpected nullptr ctdef or rtdef", K(ctdef), K(rtdef)); } else { param.tablet_id_ = lookup_tablet_id_; - param.ls_id_ = lookup_ls_id_; param.scan_allocator_ = &get_arena_allocator(); param.allocator_ = &rtdef->stmt_allocator_; param.tx_lock_timeout_ = rtdef->tx_lock_timeout_; @@ -260,11 +259,10 @@ int ObDASLocalLookupIter::do_index_lookup() } } else { // reuse -> real rescan - // reuse: store next tablet_id, ls_id and reuse storage iter; - // rescan: bind tablet_id, ls_id to scan_param and rescan; + // reuse: store the next tablet id and reuse the storage iterator; + // rescan: bind the tablet id to scan_param and rescan; if (DAS_ITER_FUNC_DATA != data_table_iter_->get_type()) { lookup_param_.tablet_id_ = lookup_tablet_id_; - lookup_param_.ls_id_ = lookup_ls_id_; } if (OB_FAIL(data_table_iter_->rescan())) { LOG_WARN("failed to rescan data table", K(ret)); @@ -324,7 +322,7 @@ int ObDASLocalLookupIter::check_index_lookup() "main table id", lookup_ctdef_->ref_table_id_, "main tablet id", lookup_tablet_id_, KPC_(trans_desc), KPC_(snapshot)); - concurrency_control::ObDataValidationService::set_delay_resource_recycle(lookup_ls_id_); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(); const ObTableScanParam &scan_param = scan_iter->get_scan_param(); if (trans_info_array_.count() == scan_param.key_ranges_.count()) { for (int64_t i = 0; i < trans_info_array_.count(); i++) { diff --git a/src/sql/das/iter/ob_das_local_lookup_iter.h b/src/sql/das/iter/ob_das_local_lookup_iter.h index e4fe36ff5..7e2031d7f 100644 --- a/src/sql/das/iter/ob_das_local_lookup_iter.h +++ b/src/sql/das/iter/ob_das_local_lookup_iter.h @@ -53,7 +53,6 @@ class ObDASLocalLookupIter : public ObDASLookupIter trans_info_array_(), lookup_param_(), lookup_tablet_id_(), - lookup_ls_id_(), trans_desc_(nullptr), snapshot_(nullptr), is_first_lookup_(true) @@ -62,7 +61,6 @@ class ObDASLocalLookupIter : public ObDASLookupIter storage::ObTableScanParam &get_lookup_param() { return lookup_param_; } void set_tablet_id(const ObTabletID &tablet_id) { lookup_tablet_id_ = tablet_id; } - void set_ls_id(const share::ObLSID &ls_id) { lookup_ls_id_ = ls_id; } int init_scan_param(storage::ObTableScanParam ¶m, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef); protected: @@ -82,11 +80,9 @@ class ObDASLocalLookupIter : public ObDASLookupIter protected: ObSEArray trans_info_array_; - // Local lookup das task could rescan multiple times during execution, lookup_tablet_id_ and - // lookup_ls_id_ store the lookup parameter for this time. + // Local lookup das task could rescan multiple times during execution. storage::ObTableScanParam lookup_param_; ObTabletID lookup_tablet_id_; - share::ObLSID lookup_ls_id_; transaction::ObTxDesc *trans_desc_; transaction::ObTxReadSnapshot *snapshot_; bool is_first_lookup_; diff --git a/src/sql/das/iter/ob_das_spiv_merge_iter.cpp b/src/sql/das/iter/ob_das_spiv_merge_iter.cpp index 41aac2dfa..baac71b1e 100644 --- a/src/sql/das/iter/ob_das_spiv_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_spiv_merge_iter.cpp @@ -260,7 +260,6 @@ int ObDASSPIVMergeIter::inner_init(ObDASIterParam ¶m) } else { ObDASSPIVMergeIterParam spiv_merge_param = static_cast(param); - ls_id_ = spiv_merge_param.ls_id_; tx_desc_ = spiv_merge_param.tx_desc_; snapshot_ = spiv_merge_param.snapshot_; @@ -405,8 +404,7 @@ int ObDASSPIVMergeIter::init_dim_iter_param(ObSPIVDimIterParam &dim_param, int64 } else if (OB_ISNULL(dim_param.inv_idx_scan_param_ = OB_NEWx(ObTableScanParam, &allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate memory for inv scan param", K(ret)); - } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, - dim_docid_value_tablet_id_, + } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(dim_docid_value_tablet_id_, spiv_scan_ctdef_, spiv_scan_rtdef_, tx_desc_, @@ -450,8 +448,7 @@ int ObDASSPIVMergeIter::create_dim_iters() if (OB_ISNULL(block_max_scan_params_[i] = OB_NEWx(ObTableScanParam, &allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate memory for block max scan param", K(ret)); - } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, - dim_docid_value_tablet_id_, + } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(dim_docid_value_tablet_id_, block_max_scan_ctdef_, block_max_scan_rtdef_, tx_desc_, @@ -588,7 +585,7 @@ int ObDASSPIVMergeIter::inner_reuse() } else { for (int i = 0; i < inv_dim_scan_iters_.count() && OB_SUCC(ret); ++i) { if (OB_FAIL(ObDasVecScanUtils::reuse_iter( - ls_id_, inv_dim_scan_iters_[i], *inv_scan_params_[i], dim_docid_value_tablet_id_))) { + inv_dim_scan_iters_[i], *inv_scan_params_[i], dim_docid_value_tablet_id_))) { LOG_WARN("failed to reuse inv dim scan iter", K(ret)); } } @@ -1216,8 +1213,7 @@ int ObDASSPIVMergeIter::do_aux_data_table_scan() int idx = get_aux_data_tbl_idx(); const ObDASScanCtDef *aux_data_ctdef = vec_aux_ctdef_->get_vec_aux_tbl_ctdef(idx, ObTSCIRScanType::OB_VEC_COM_AUX_SCAN); ObDASScanRtDef *aux_data_rtdef = vec_aux_rtdef_->get_vec_aux_tbl_rtdef(idx); - if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param(ls_id_, - aux_data_tablet_id_, + if (OB_FAIL(ObDasVecScanUtils::init_vec_aux_scan_param(aux_data_tablet_id_, aux_data_ctdef, aux_data_rtdef, tx_desc_, @@ -1248,8 +1244,7 @@ int ObDASSPIVMergeIter::do_rowkey_docid_table_scan() const ObDASScanCtDef *rowkey_docid_ctdef = vec_aux_ctdef_->get_vec_aux_tbl_ctdef( vec_aux_ctdef_->get_spiv_rowkey_docid_tbl_idx(), ObTSCIRScanType::OB_VEC_ROWKEY_VID_SCAN); ObDASScanRtDef *rowkey_docid_rtdef = vec_aux_rtdef_->get_vec_aux_tbl_rtdef(vec_aux_ctdef_->get_spiv_rowkey_docid_tbl_idx()); - if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, - rowkey_docid_tablet_id_, + if (OB_FAIL(ObDasVecScanUtils::init_scan_param(rowkey_docid_tablet_id_, rowkey_docid_ctdef, rowkey_docid_rtdef, tx_desc_, diff --git a/src/sql/das/iter/ob_das_spiv_merge_iter.h b/src/sql/das/iter/ob_das_spiv_merge_iter.h index 8a0d9ac2b..7106ec0f3 100644 --- a/src/sql/das/iter/ob_das_spiv_merge_iter.h +++ b/src/sql/das/iter/ob_das_spiv_merge_iter.h @@ -53,7 +53,6 @@ struct ObDASSPIVMergeIterParam : public ObDASIterParam public: ObDASSPIVMergeIterParam() : ObDASIterParam(ObDASIterType::DAS_ITER_SPIV_MERGE), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), inv_idx_scan_iter_(nullptr), @@ -70,14 +69,12 @@ struct ObDASSPIVMergeIterParam : public ObDASIterParam virtual bool is_valid() const override { - return ls_id_.is_valid() && - nullptr != tx_desc_ && + return nullptr != tx_desc_ && nullptr != snapshot_ && nullptr != vec_aux_ctdef_ && nullptr != vec_aux_rtdef_; } - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; @@ -102,7 +99,6 @@ class ObDASSPIVMergeIter : public ObDASIter : ObDASIter(ObDASIterType::DAS_ITER_SPIV_MERGE), mem_context_(nullptr), allocator_(lib::ObMemAttr("SPIVMergeIter"), OB_MALLOC_NORMAL_BLOCK_SIZE), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), inv_idx_scan_iter_(nullptr), @@ -153,7 +149,6 @@ class ObDASSPIVMergeIter : public ObDASIter dim_docid_value_tablet_id_ = related_tablet_ids.dim_docid_value_tablet_id_; } - void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } ObMapType* get_qvec() { return qvec_; }; int get_aux_data_tbl_idx() { int idx = vec_aux_ctdef_->get_spiv_aux_data_tbl_idx(); @@ -195,8 +190,8 @@ class ObDASSPIVMergeIter : public ObDASIter int do_aux_data_table_scan(); int do_rowkey_docid_table_scan(); - int reuse_aux_data_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, aux_data_iter_, aux_data_scan_param_, aux_data_tablet_id_); } - int reuse_rowkey_docid_iter() { return ObDasVecScanUtils::reuse_iter(ls_id_, rowkey_docid_iter_, rowkey_docid_scan_param_, rowkey_docid_tablet_id_); } + int reuse_aux_data_iter() { return ObDasVecScanUtils::reuse_iter(aux_data_iter_, aux_data_scan_param_, aux_data_tablet_id_); } + int reuse_rowkey_docid_iter() { return ObDasVecScanUtils::reuse_iter(rowkey_docid_iter_, rowkey_docid_scan_param_, rowkey_docid_tablet_id_); } int get_ctdef_with_rowkey_exprs(const ObDASScanCtDef *&ctdef, ObDASScanRtDef *&rtdef); int get_rowkey_pre_filter(ObIAllocator &allocator, bool is_vectorized, int64_t batch_count); @@ -247,7 +242,6 @@ class ObDASSPIVMergeIter : public ObDASIter lib::MemoryContext mem_context_; ObArenaAllocator allocator_; - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; diff --git a/src/sql/das/iter/ob_das_spiv_scan_iter.cpp b/src/sql/das/iter/ob_das_spiv_scan_iter.cpp index e2ac5ac07..8a5a1de1d 100644 --- a/src/sql/das/iter/ob_das_spiv_scan_iter.cpp +++ b/src/sql/das/iter/ob_das_spiv_scan_iter.cpp @@ -39,7 +39,6 @@ int ObDASSPIVScanIter::inner_init(ObDASIterParam ¶m) scan_iter_ = static_cast(spiv_scan_param.scan_iter_); scan_ctdef_ = spiv_scan_param.scan_ctdef_; scan_rtdef_ = spiv_scan_param.scan_rtdef_; - ls_id_ = spiv_scan_param.ls_id_; tx_desc_ = spiv_scan_param.tx_desc_; snapshot_ = spiv_scan_param.snapshot_; dim_ = spiv_scan_param.dim_; @@ -59,7 +58,7 @@ int ObDASSPIVScanIter::inner_init(ObDASIterParam ¶m) LOG_WARN("failed to build scan range", K(ret)); } else if (OB_FAIL(scan_iter_param_.key_ranges_.push_back(scan_range))) { LOG_WARN("failed to push lookup range", K(ret)); - } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(ls_id_, dim_docid_value_tablet_id_, scan_ctdef_, scan_rtdef_, + } else if (OB_FAIL(ObDasVecScanUtils::init_scan_param(dim_docid_value_tablet_id_, scan_ctdef_, scan_rtdef_, tx_desc_, snapshot_, scan_iter_param_, false))) { LOG_WARN("failed to init scan param", K(ret), K(dim_docid_value_tablet_id_)); } else { @@ -73,7 +72,7 @@ int ObDASSPIVScanIter::inner_init(ObDASIterParam ¶m) int ObDASSPIVScanIter::inner_reuse() { int ret = OB_SUCCESS; - if (OB_NOT_NULL(scan_iter_) && OB_FAIL(ObDasVecScanUtils::reuse_iter(ls_id_, scan_iter_, scan_iter_param_, dim_docid_value_tablet_id_))) { + if (OB_NOT_NULL(scan_iter_) && OB_FAIL(ObDasVecScanUtils::reuse_iter(scan_iter_, scan_iter_param_, dim_docid_value_tablet_id_))) { LOG_WARN("failed to reuse scan iter", K(ret)); } else if (is_first_scan_) { scan_iter_param_.need_switch_param_ = false; diff --git a/src/sql/das/iter/ob_das_spiv_scan_iter.h b/src/sql/das/iter/ob_das_spiv_scan_iter.h index 7d6b89c15..77d694238 100644 --- a/src/sql/das/iter/ob_das_spiv_scan_iter.h +++ b/src/sql/das/iter/ob_das_spiv_scan_iter.h @@ -33,7 +33,6 @@ struct ObDASSPIVScanIterParam : public ObDASIterParam public: ObDASSPIVScanIterParam() : ObDASIterParam(ObDASIterType::DAS_ITER_SPIV_SCAN), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), scan_iter_(nullptr), @@ -44,14 +43,12 @@ struct ObDASSPIVScanIterParam : public ObDASIterParam virtual bool is_valid() const override { - return ls_id_.is_valid() && - nullptr != tx_desc_ && + return nullptr != tx_desc_ && nullptr != snapshot_ && nullptr != scan_iter_ && dim_docid_value_tablet_id_.is_valid(); } - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; @@ -68,7 +65,6 @@ class ObDASSPIVScanIter : public ObDASIter ObDASSPIVScanIter() : ObDASIter(ObDASIterType::DAS_ITER_SPIV_SCAN), mem_context_(nullptr), - ls_id_(), tx_desc_(nullptr), snapshot_(nullptr), scan_iter_(nullptr), @@ -89,12 +85,6 @@ class ObDASSPIVScanIter : public ObDASIter int build_range(ObNewRange &range, uint64_t table_id); - void set_ls_tablet_id(const share::ObLSID &ls_id, const ObTabletID &related_tablet_id) - { - ls_id_ = ls_id; - dim_docid_value_tablet_id_ = related_tablet_id; - } - protected: virtual int inner_init(ObDASIterParam ¶m) override; virtual int inner_reuse() override; @@ -104,7 +94,6 @@ class ObDASSPIVScanIter : public ObDASIter private: lib::MemoryContext mem_context_; - share::ObLSID ls_id_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; diff --git a/src/sql/das/iter/ob_das_text_retrieval_iter.cpp b/src/sql/das/iter/ob_das_text_retrieval_iter.cpp index 41ae44120..8d03f1b74 100644 --- a/src/sql/das/iter/ob_das_text_retrieval_iter.cpp +++ b/src/sql/das/iter/ob_das_text_retrieval_iter.cpp @@ -33,7 +33,6 @@ ObDASTextRetrievalIter::ObDASTextRetrievalIter() ir_rtdef_(nullptr), tx_desc_(nullptr), snapshot_(nullptr), - ls_id_(), inv_idx_tablet_id_(), fwd_idx_tablet_id_(), inv_idx_scan_param_(), @@ -348,10 +347,8 @@ int ObDASTextRetrievalIter::rescan() int ret = OB_SUCCESS; inv_idx_scan_param_.tablet_id_ = inv_idx_tablet_id_; - inv_idx_scan_param_.ls_id_ = ls_id_; if (need_inv_idx_agg_) { inv_idx_agg_param_.tablet_id_ = inv_idx_tablet_id_; - inv_idx_agg_param_.ls_id_ = ls_id_; } if (OB_FAIL(inverted_idx_scan_iter_->rescan())) { LOG_WARN("failed to rescan inverted scan iter", K(ret)); @@ -461,7 +458,6 @@ int ObDASTextRetrievalIter::init_inv_idx_scan_param() { int ret = OB_SUCCESS; if (OB_FAIL(init_base_idx_scan_param( - ls_id_, inv_idx_tablet_id_, ir_ctdef_->get_inv_idx_scan_ctdef(), ir_rtdef_->get_inv_idx_scan_rtdef(), @@ -471,7 +467,6 @@ int ObDASTextRetrievalIter::init_inv_idx_scan_param() LOG_WARN("fail to init inverted index scan param", K(ret), KPC_(ir_ctdef)); } else if (need_inv_idx_agg_) { if (OB_FAIL(init_base_idx_scan_param( - ls_id_, inv_idx_tablet_id_, ir_ctdef_->get_inv_idx_agg_ctdef(), ir_rtdef_->get_inv_idx_agg_rtdef(), @@ -500,7 +495,6 @@ int ObDASTextRetrievalIter::init_fwd_idx_scan_param() if (!ir_ctdef_->need_calc_relevance()) { } else if (OB_FAIL(init_base_idx_scan_param( - ls_id_, fwd_idx_tablet_id_, ir_ctdef_->get_fwd_idx_agg_ctdef(), ir_rtdef_->get_fwd_idx_agg_rtdef(), @@ -513,7 +507,6 @@ int ObDASTextRetrievalIter::init_fwd_idx_scan_param() } int ObDASTextRetrievalIter::init_base_idx_scan_param( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, sql::ObDASScanRtDef *rtdef, @@ -524,7 +517,7 @@ int ObDASTextRetrievalIter::init_base_idx_scan_param( int ret = OB_SUCCESS; if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(tablet_id)); } else { @@ -554,7 +547,6 @@ int ObDASTextRetrievalIter::init_base_idx_scan_param( scan_param.need_scn_ = rtdef->need_scn_; scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; scan_param.fb_snapshot_ = rtdef->fb_snapshot_; - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) { scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; @@ -650,7 +642,6 @@ int ObDASTextRetrievalIter::do_token_cnt_agg(const ObDocIdExt &doc_id, int64_t & if (OB_SUCC(ret)) { if (not_first_fwd_agg_) { fwd_idx_scan_param_.tablet_id_ = fwd_idx_tablet_id_; - fwd_idx_scan_param_.ls_id_ = ls_id_; if (OB_FAIL(reuse_fwd_idx_iter())) { LOG_WARN("failed to reuse forward index iterator", K(ret)); } else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) { diff --git a/src/sql/das/iter/ob_das_text_retrieval_iter.h b/src/sql/das/iter/ob_das_text_retrieval_iter.h index e725e4c05..9d6839215 100644 --- a/src/sql/das/iter/ob_das_text_retrieval_iter.h +++ b/src/sql/das/iter/ob_das_text_retrieval_iter.h @@ -19,7 +19,6 @@ #include "ob_das_iter.h" #include "lib/container/ob_se_array.h" -#include "share/ob_ls_id.h" #include "src/storage/access/ob_dml_param.h" namespace oceanbase @@ -79,12 +78,9 @@ class ObDASTextRetrievalIter : public ObDASIter int set_query_token(const ObString &query_token); int set_query_token_and_rangekey(const ObString &query_token, const common::ObIArray &doc_id, const int64_t &batch_size); - void set_ls_tablet_ids( - const share::ObLSID &ls_id, - const ObTabletID &inv_tablet_id, - const ObTabletID &fwd_tablet_id) + void set_tablet_ids(const ObTabletID &inv_tablet_id, + const ObTabletID &fwd_tablet_id) { - ls_id_ = ls_id; inv_idx_tablet_id_ = inv_tablet_id; fwd_idx_tablet_id_ = fwd_tablet_id; } @@ -100,7 +96,6 @@ class ObDASTextRetrievalIter : public ObDASIter int init_inv_idx_scan_param(); int init_fwd_idx_scan_param(); static int init_base_idx_scan_param( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, @@ -175,7 +170,6 @@ class ObDASTextRetrievalIter : public ObDASIter ObDASIRScanRtDef *ir_rtdef_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; - share::ObLSID ls_id_; common::ObTabletID inv_idx_tablet_id_; common::ObTabletID fwd_idx_tablet_id_; storage::ObTableScanParam inv_idx_scan_param_; diff --git a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp index e78fe9572..fc7fdf62a 100644 --- a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp @@ -92,7 +92,6 @@ ObDASTextRetrievalMergeIter::ObDASTextRetrievalMergeIter() ir_rtdef_(nullptr), tx_desc_(nullptr), snapshot_(nullptr), - ls_id_(), domain_id_idx_tablet_id_(), query_tokens_(), cache_doc_ids_(), @@ -139,15 +138,12 @@ int ObDASTextRetrievalMergeIter::rescan() } int ObDASTextRetrievalMergeIter::set_related_tablet_ids( - const ObLSID &ls_id, const ObDASFTSTabletID &related_tablet_ids) { int ret = OB_SUCCESS; - ls_id_ = ls_id; domain_id_idx_tablet_id_ = related_tablet_ids.domain_id_idx_tablet_id_; for (int64_t i = 0; i < token_iters_.count(); ++i) { - token_iters_.at(i)->set_ls_tablet_ids( - ls_id, + token_iters_.at(i)->set_tablet_ids( related_tablet_ids.inv_idx_tablet_id_, related_tablet_ids.fwd_idx_tablet_id_); } @@ -450,7 +446,6 @@ int ObDASTextRetrievalMergeIter::inner_reuse() whole_doc_agg_param_.need_switch_param_ = whole_doc_agg_param_.need_switch_param_ || ((old_tablet_id.is_valid() && old_tablet_id != domain_id_idx_tablet_id_ ) ? true : false); whole_doc_agg_param_.tablet_id_ = domain_id_idx_tablet_id_; - whole_doc_agg_param_.ls_id_ = ls_id_; if (!force_return_docid_ || whole_doc_agg_param_.need_switch_param_) { doc_cnt_calculated_ = false; } @@ -723,7 +718,6 @@ int ObDASTextRetrievalMergeIter::init_total_doc_cnt_param( scan_param.need_scn_ = rtdef->need_scn_; scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; scan_param.fb_snapshot_ = rtdef->fb_snapshot_; - scan_param.ls_id_ = ls_id_; scan_param.tablet_id_ = domain_id_idx_tablet_id_; if (ctdef->pd_expr_spec_.pushdown_filters_.empty()) { scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; @@ -774,7 +768,6 @@ int ObDASTextRetrievalMergeIter::do_total_doc_cnt() whole_doc_agg_param_.need_switch_param_ = whole_doc_agg_param_.need_switch_param_ || ((old_tablet_id.is_valid() && old_tablet_id != domain_id_idx_tablet_id_ ) ? true : false); whole_doc_agg_param_.tablet_id_ = domain_id_idx_tablet_id_; - whole_doc_agg_param_.ls_id_ = ls_id_; if (!force_return_docid_ || whole_doc_agg_param_.need_switch_param_) { if (OB_FAIL(whole_doc_cnt_iter_->reuse())) { LOG_WARN("failed to reuse whole doc cnt iter", K(ret)); diff --git a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h index 49faba7d6..0ed1b85cc 100644 --- a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h +++ b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h @@ -115,9 +115,8 @@ class ObDASTextRetrievalMergeIter : public ObDASIter virtual int do_table_scan() override; virtual int rescan() override; void set_domain_id_idx_tablet_id(const ObTabletID &tablet_id) { domain_id_idx_tablet_id_ = tablet_id; } - void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } storage::ObTableScanParam &get_doc_agg_param() { return whole_doc_agg_param_; } - int set_related_tablet_ids(const share::ObLSID &ls_id, const ObDASFTSTabletID &related_tablet_ids); + int set_related_tablet_ids(const ObDASFTSTabletID &related_tablet_ids); virtual int set_merge_iters(const ObIArray &retrieval_iters); const ObIArray &get_query_tokens() { return query_tokens_; } bool is_taat_mode() { return RetrievalProcType::TAAT == processing_type_; } @@ -163,7 +162,6 @@ class ObDASTextRetrievalMergeIter : public ObDASIter ObDASIRScanRtDef *ir_rtdef_; transaction::ObTxDesc *tx_desc_; transaction::ObTxReadSnapshot *snapshot_; - share::ObLSID ls_id_; common::ObTabletID domain_id_idx_tablet_id_; ObArray query_tokens_; ObDASTokenRetrievalIterArray token_iters_; diff --git a/src/sql/das/iter/ob_das_vec_scan_utils.cpp b/src/sql/das/iter/ob_das_vec_scan_utils.cpp index 9dd7ed25a..c120456ff 100644 --- a/src/sql/das/iter/ob_das_vec_scan_utils.cpp +++ b/src/sql/das/iter/ob_das_vec_scan_utils.cpp @@ -333,8 +333,7 @@ int ObDasVecScanUtils::init_sort(const ObDASVecAuxScanCtDef *ir_ctdef, return ret; } -int ObDasVecScanUtils::reuse_iter(const share::ObLSID &ls_id, - ObDASScanIter *iter, +int ObDasVecScanUtils::reuse_iter(ObDASScanIter *iter, ObTableScanParam &scan_param, const ObTabletID tablet_id) { @@ -344,7 +343,6 @@ int ObDasVecScanUtils::reuse_iter(const share::ObLSID &ls_id, scan_param.need_switch_param_ = scan_param.need_switch_param_ || (scan_tablet_id.is_valid() && (tablet_id != scan_tablet_id)); scan_param.tablet_id_ = tablet_id; - scan_param.ls_id_ = ls_id; if (OB_NOT_NULL(iter) && OB_FAIL(iter->reuse())) { LOG_WARN("reuse iter failed", K(ret)); @@ -353,8 +351,7 @@ int ObDasVecScanUtils::reuse_iter(const share::ObLSID &ls_id, return ret; } -int ObDasVecScanUtils::init_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObDasVecScanUtils::init_scan_param(const common::ObTabletID &tablet_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, transaction::ObTxDesc *tx_desc, @@ -396,7 +393,6 @@ int ObDasVecScanUtils::init_scan_param(const share::ObLSID &ls_id, if (rtdef->is_for_foreign_check_) { scan_param.trans_desc_ = tx_desc; } - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; if (rtdef->sample_info_ != nullptr) { scan_param.sample_info_ = *rtdef->sample_info_; @@ -426,8 +422,7 @@ int ObDasVecScanUtils::init_scan_param(const share::ObLSID &ls_id, return ret; } -int ObDasVecScanUtils::init_vec_aux_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObDasVecScanUtils::init_vec_aux_scan_param(const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, sql::ObDASScanRtDef *rtdef, transaction::ObTxDesc *tx_desc, @@ -439,7 +434,7 @@ int ObDasVecScanUtils::init_vec_aux_scan_param(const share::ObLSID &ls_id, int ret = OB_SUCCESS; if (OB_FAIL( - ObDasVecScanUtils::init_scan_param(ls_id, tablet_id, ctdef, rtdef, tx_desc, snapshot, scan_param, is_get, scan_allocator))) { + ObDasVecScanUtils::init_scan_param(tablet_id, ctdef, rtdef, tx_desc, snapshot, scan_param, is_get, scan_allocator))) { LOG_WARN("failed to generate init vec aux scan param", K(ret)); } else { scan_param.is_for_foreign_check_ = false; diff --git a/src/sql/das/iter/ob_das_vec_scan_utils.h b/src/sql/das/iter/ob_das_vec_scan_utils.h index fc5a57505..03679e94f 100644 --- a/src/sql/das/iter/ob_das_vec_scan_utils.h +++ b/src/sql/das/iter/ob_das_vec_scan_utils.h @@ -58,12 +58,10 @@ class ObDasVecScanUtils ObDASSortRtDef *sort_rtdef, ObString &hybrid_search_vec, ObExpr *&distance_calc); - static int reuse_iter(const share::ObLSID &ls_id, - ObDASScanIter *iter, + static int reuse_iter(ObDASScanIter *iter, ObTableScanParam &scan_param, const ObTabletID tablet_id); - static int init_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + static int init_scan_param(const common::ObTabletID &tablet_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, transaction::ObTxDesc *tx_desc, @@ -71,8 +69,7 @@ class ObDasVecScanUtils ObTableScanParam &scan_param, bool is_get = true, ObIAllocator *allocator = nullptr); - static int init_vec_aux_scan_param(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + static int init_vec_aux_scan_param(const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, sql::ObDASScanRtDef *rtdef, transaction::ObTxDesc *tx_desc, diff --git a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp index 05f11094a..333c2e6d8 100644 --- a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp +++ b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp @@ -48,7 +48,6 @@ ObDASTRMergeIter::ObDASTRMergeIter() doc_length_est_param_(), doc_length_est_stat_cols_(), topk_limit_(0), - ls_id_(), total_doc_cnt_tablet_id_(), inv_idx_tablet_id_(), fwd_idx_tablet_id_(), @@ -126,7 +125,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() } if (OB_FAIL(ret)) { } else if (OB_FAIL(init_das_iter_scan_param( - ls_id_, total_doc_cnt_tablet_id_, + total_doc_cnt_tablet_id_, ir_ctdef_->get_doc_agg_ctdef(), ir_rtdef_->get_doc_agg_rtdef(), tx_desc_, snapshot_, mem_context_->get_arena_allocator(), @@ -175,7 +174,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() for (int64_t i = 0; OB_SUCC(ret) && i < dim_iter_cnt; ++i) { inv_scan_params_[i] = new (&scan_params[i]) ObTableScanParam(); if (OB_FAIL(init_das_iter_scan_param( - ls_id_, inv_idx_tablet_id_, + inv_idx_tablet_id_, ir_ctdef_->get_inv_idx_scan_ctdef(), ir_rtdef_->get_inv_idx_scan_rtdef(), tx_desc_, snapshot_, mem_context_->get_arena_allocator(), *inv_scan_params_[i]))) { @@ -196,7 +195,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() for (int64_t i = 0; OB_SUCC(ret) && i < dim_iter_cnt; ++i) { inv_agg_params_[i] = new (&agg_params[i]) ObTableScanParam(); if (OB_FAIL(init_das_iter_scan_param( - ls_id_, inv_idx_tablet_id_, + inv_idx_tablet_id_, ir_ctdef_->get_inv_idx_agg_ctdef(), ir_rtdef_->get_inv_idx_agg_rtdef(), tx_desc_, snapshot_, mem_context_->get_arena_allocator(), @@ -220,7 +219,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() for (int64_t i = 0; OB_SUCC(ret) && i < dim_iter_cnt; ++i) { fwd_scan_params_[i] = new (&fwd_params[i]) ObTableScanParam(); if (OB_FAIL(init_das_iter_scan_param( - ls_id_, fwd_idx_tablet_id_, + fwd_idx_tablet_id_, ir_ctdef_->get_fwd_idx_agg_ctdef(), ir_rtdef_->get_fwd_idx_agg_rtdef(), tx_desc_, snapshot_, mem_context_->get_arena_allocator(), @@ -259,7 +258,7 @@ int ObDASTRMergeIter::init_block_max_iter_param() for (int64_t i = 0; OB_SUCC(ret) && i < token_cnt; ++i) { block_max_scan_params_[i] = new (&scan_params[i]) ObTableScanParam(); if (OB_FAIL(init_das_iter_scan_param( - ls_id_, inv_idx_tablet_id_, + inv_idx_tablet_id_, ir_ctdef_->get_block_max_scan_ctdef(), ir_rtdef_->get_block_max_scan_rtdef(), tx_desc_, snapshot_, mem_context_->get_arena_allocator(), @@ -298,8 +297,7 @@ int ObDASTRMergeIter::init_doc_length_est_param() return ret; } -int ObDASTRMergeIter::init_das_iter_scan_param(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDASTRMergeIter::init_das_iter_scan_param(const ObTabletID &tablet_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, transaction::ObTxDesc *tx_desc, @@ -310,7 +308,7 @@ int ObDASTRMergeIter::init_das_iter_scan_param(const ObLSID &ls_id, int ret = OB_SUCCESS; if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(tablet_id)); } else { @@ -340,7 +338,6 @@ int ObDASTRMergeIter::init_das_iter_scan_param(const ObLSID &ls_id, scan_param.need_scn_ = rtdef->need_scn_; scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; scan_param.fb_snapshot_ = rtdef->fb_snapshot_; - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) { scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; @@ -895,8 +892,7 @@ int ObDASTRMergeIter::do_table_scan() return ret; } -int ObDASTRMergeIter::reuse_das_iter_scan_param(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDASTRMergeIter::reuse_das_iter_scan_param(const ObTabletID &tablet_id, ObTableScanParam &scan_param) { int ret = OB_SUCCESS; @@ -907,7 +903,6 @@ int ObDASTRMergeIter::reuse_das_iter_scan_param(const ObLSID &ls_id, scan_param.key_ranges_.reuse(); } scan_param.tablet_id_ = tablet_id; - scan_param.ls_id_ = ls_id; return ret; } @@ -922,27 +917,27 @@ int ObDASTRMergeIter::inner_reuse() LOG_WARN("sparse retrieval iter is null", K(ret)); } else if (query_tokens_.count() > 0) { if (OB_NOT_NULL(total_doc_cnt_scan_param_)) { - if (OB_FAIL(reuse_das_iter_scan_param(ls_id_, total_doc_cnt_tablet_id_, *total_doc_cnt_scan_param_))) { + if (OB_FAIL(reuse_das_iter_scan_param(total_doc_cnt_tablet_id_, *total_doc_cnt_scan_param_))) { LOG_WARN("failed to reuse total doc cnt scan param", K(ret)); } } for (int64_t i = 0; OB_SUCC(ret) && i < inv_scan_params_.count(); ++i) { - if (OB_FAIL(reuse_das_iter_scan_param(ls_id_, inv_idx_tablet_id_, *inv_scan_params_[i]))) { + if (OB_FAIL(reuse_das_iter_scan_param(inv_idx_tablet_id_, *inv_scan_params_[i]))) { LOG_WARN("failed to reuse inv scan param", K(ret), K(i)); } } for (int64_t i = 0; OB_SUCC(ret) && i < inv_agg_params_.count(); ++i) { - if (OB_FAIL(reuse_das_iter_scan_param(ls_id_, inv_idx_tablet_id_, *inv_agg_params_[i]))) { + if (OB_FAIL(reuse_das_iter_scan_param(inv_idx_tablet_id_, *inv_agg_params_[i]))) { LOG_WARN("failed to reuse inv agg param", K(ret), K(i)); } } for (int64_t i = 0; OB_SUCC(ret) && i < block_max_scan_params_.count(); ++i) { - if (OB_FAIL(reuse_das_iter_scan_param(ls_id_, inv_idx_tablet_id_, *block_max_scan_params_[i]))) { + if (OB_FAIL(reuse_das_iter_scan_param(inv_idx_tablet_id_, *block_max_scan_params_[i]))) { LOG_WARN("failed to reuse block max scan param", K(ret), K(i)); } } for (int64_t i = 0; i < OB_SUCC(ret) && fwd_scan_params_.count(); ++i) { - if (OB_FAIL(reuse_das_iter_scan_param(ls_id_, fwd_idx_tablet_id_, *fwd_scan_params_[i]))) { + if (OB_FAIL(reuse_das_iter_scan_param(fwd_idx_tablet_id_, *fwd_scan_params_[i]))) { LOG_WARN("failed to reuse fwd scan param", K(ret), K(i)); } } diff --git a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h index f819a96a7..7ec1ea580 100644 --- a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h +++ b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h @@ -85,7 +85,7 @@ class ObDASTRMergeIter : public ObDASIter INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(ir_ctdef), K_(ir_rtdef), K_(tx_desc), K_(snapshot), K_(query_tokens), K_(sr_iter_param), K_(sparse_retrieval_iter), K_(dim_iters), - K_(ls_id), K_(total_doc_cnt_tablet_id), K_(inv_idx_tablet_id), K_(fwd_idx_tablet_id), + K_(total_doc_cnt_tablet_id), K_(inv_idx_tablet_id), K_(fwd_idx_tablet_id), K_(flags), K_(check_rangekey_inited), K_(is_inited)); protected: virtual int inner_init(ObDASIterParam ¶m) override; @@ -95,11 +95,9 @@ class ObDASTRMergeIter : public ObDASIter virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; public: int set_related_tablet_ids( - const ObLSID &ls_id, const ObDASFTSTabletID &related_tablet_ids) { int ret = OB_SUCCESS; - ls_id_ = ls_id; total_doc_cnt_tablet_id_ = related_tablet_ids.domain_id_idx_tablet_id_; if (inv_idx_tablet_id_.is_valid() && inv_idx_tablet_id_ != related_tablet_ids.inv_idx_tablet_id_) { inv_idx_tablet_switched_ = true; @@ -124,7 +122,6 @@ class ObDASTRMergeIter : public ObDASIter private: int init_das_iter_scan_params(); static int init_das_iter_scan_param( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef, @@ -133,7 +130,6 @@ class ObDASTRMergeIter : public ObDASIter common::ObArenaAllocator &allocator, ObTableScanParam &scan_param); static int reuse_das_iter_scan_param( - const ObLSID &ls_id, const ObTabletID &tablet_id, ObTableScanParam &scan_param); int create_dim_iters(); @@ -176,7 +172,6 @@ class ObDASTRMergeIter : public ObDASIter ObBlockStatScanParam doc_length_est_param_; ObFixedArray doc_length_est_stat_cols_; int64_t topk_limit_; - ObLSID ls_id_; ObTabletID total_doc_cnt_tablet_id_; ObTabletID inv_idx_tablet_id_; ObTabletID fwd_idx_tablet_id_; diff --git a/src/sql/das/ob_das_context.cpp b/src/sql/das/ob_das_context.cpp index ced6287ff..39a676c6b 100644 --- a/src/sql/das/ob_das_context.cpp +++ b/src/sql/das/ob_das_context.cpp @@ -209,7 +209,6 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, tablet_loc = new(tablet_buf) ObDASTabletLoc(); tablet_loc->server_ = replica_loc.get_server(); tablet_loc->tablet_id_ = opt_tablet_loc.get_tablet_id(); - tablet_loc->ls_id_ = opt_tablet_loc.get_ls_id(); tablet_loc->partition_id_ = opt_tablet_loc.get_partition_id(); tablet_loc->first_level_part_id_ = opt_tablet_loc.get_first_level_part_id(); tablet_loc->loc_meta_ = table_loc.loc_meta_; @@ -266,7 +265,6 @@ OB_INLINE int ObDASCtx::build_related_tablet_loc(ObDASTabletLoc &tablet_loc) if (OB_SUCC(ret)) { related_tablet_loc = new(related_loc_buf) ObDASTabletLoc(); related_tablet_loc->tablet_id_ = rv->tablet_id_; - related_tablet_loc->ls_id_ = tablet_loc.ls_id_; related_tablet_loc->server_ = tablet_loc.server_; related_tablet_loc->loc_meta_ = related_table_loc->loc_meta_; related_tablet_loc->next_ = tablet_loc.next_; @@ -411,22 +409,6 @@ int ObDASCtx::build_table_loc_meta(const ObDASTableLocMeta &src, return ret; } -int ObDASCtx::get_all_lsid(share::ObLSArray &ls_ids) -{ - int ret = OB_SUCCESS; - FOREACH_X(table_node, table_locs_, OB_SUCC(ret)) { - ObDASTableLoc *table_loc = *table_node; - for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin(); - OB_SUCC(ret) && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) { - ObDASTabletLoc *tablet_loc = *tablet_node; - if (!is_contain(ls_ids, tablet_loc->ls_id_)) { - ret = ls_ids.push_back(tablet_loc->ls_id_); - } - } - } - return ret; -} - int64_t ObDASCtx::get_related_tablet_cnt() const { int64_t total_cnt = 0; diff --git a/src/sql/das/ob_das_context.h b/src/sql/das/ob_das_context.h index f09be0df3..3dd2cdab4 100644 --- a/src/sql/das/ob_das_context.h +++ b/src/sql/das/ob_das_context.h @@ -77,7 +77,7 @@ class ObDASCtx group_params_(nullptr), skip_scan_group_id_(-1), group_rescan_cnt_(-1), - use_gts_opt_(false), + use_snapshot_opt_(false), parent_table_set_(allocator), real_das_dop_(0), flags_(0) @@ -127,7 +127,6 @@ class ObDASCtx int get_das_tablet_mapper(const uint64_t ref_table_id, ObDASTabletMapper &tablet_mapper, const DASTableIDArrayWrap *related_table_ids = nullptr); - int get_all_lsid(share::ObLSArray &ls_ids); int64_t get_related_tablet_cnt() const; int set_snapshot(const transaction::ObTxReadSnapshot &snapshot) { return snapshot_.assign(snapshot); } transaction::ObTxReadSnapshot &get_snapshot() { return snapshot_; } @@ -161,8 +160,8 @@ class ObDASCtx bool &exist, uint64_t &array_idx); int64_t get_real_das_dop() { return real_das_dop_; } void set_real_das_dop(int64_t v) { real_das_dop_ = v; } - void set_use_gts_opt(bool v) { use_gts_opt_ = v; } - bool get_use_gts_opt() { return use_gts_opt_; } + void set_use_snapshot_opt(bool v) { use_snapshot_opt_ = v; } + bool get_use_snapshot_opt() { return use_snapshot_opt_; } TO_STRING_KV(K_(table_locs), K_(is_fk_cascading), @@ -191,7 +190,7 @@ class ObDASCtx const GroupParamArray *group_params_; //only allowed to be modified by GroupParamBackupGuard int64_t skip_scan_group_id_; //only allowed to be modified by GroupParamBackupGuard int64_t group_rescan_cnt_; //only allowed to be modified by GroupParamBackupGuard - bool use_gts_opt_; // without get gts + bool use_snapshot_opt_; DASTableIdList parent_table_set_; // The list of parent table, for inner sql. int64_t real_das_dop_; public: diff --git a/src/sql/das/ob_das_define.cpp b/src/sql/das/ob_das_define.cpp index cc2853dad..9d99c6599 100644 --- a/src/sql/das/ob_das_define.cpp +++ b/src/sql/das/ob_das_define.cpp @@ -68,7 +68,6 @@ OB_SERIALIZE_MEMBER(ObDASTableLocMeta, OB_SERIALIZE_MEMBER(ObDASTabletLoc, tablet_id_, - ls_id_, server_, flags_, partition_id_, diff --git a/src/sql/das/ob_das_define.h b/src/sql/das/ob_das_define.h index b99fbe878..89cabea08 100644 --- a/src/sql/das/ob_das_define.h +++ b/src/sql/das/ob_das_define.h @@ -17,7 +17,7 @@ #ifndef OBDEV_SRC_SQL_DAS_OB_DAS_DEFINE_H_ #define OBDEV_SRC_SQL_DAS_OB_DAS_DEFINE_H_ #include "share/ob_define.h" -#include "share/ob_ls_id.h" +#include "lib/list/ob_list.h" #include "share/location_cache/ob_location_struct.h" #include "common/ob_tablet_id.h" #include "sql/ob_phy_table_location.h" @@ -178,7 +178,6 @@ struct ObDASTabletLoc public: ObDASTabletLoc() : tablet_id_(), - ls_id_(), server_(), loc_meta_(nullptr), next_(this), @@ -189,7 +188,6 @@ struct ObDASTabletLoc ~ObDASTabletLoc() = default; TO_STRING_KV(K_(tablet_id), - K_(ls_id), K_(server), K_(loc_meta), K_(in_retry), @@ -203,9 +201,6 @@ struct ObDASTabletLoc * because ObDASTabletLoc will not be released */ common::ObTabletID tablet_id_; //the data_object_id corresponding to partition_id - //To reduce the conversion between tablet_id to ls_id, - //DAS caches ls_id_, SQL should not actually touch ls_id_ - share::ObLSID ls_id_; common::ObAddr server_; const ObDASTableLocMeta *loc_meta_; //reference the table location meta, not serialize it ObDASTabletLoc *next_; //to bind all data table and local index tablet location diff --git a/src/sql/das/ob_das_delete_op.cpp b/src/sql/das/ob_das_delete_op.cpp index 19dd16212..4d1ae0a2d 100644 --- a/src/sql/das/ob_das_delete_op.cpp +++ b/src/sql/das/ob_das_delete_op.cpp @@ -44,8 +44,7 @@ using namespace share; namespace sql { template <> -int ObDASIndexDMLAdaptor::write_rows(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDASIndexDMLAdaptor::write_rows(const ObTabletID &tablet_id, const CtDefType &ctdef, RtDefType &rtdef, ObDASDMLIterator &iter, @@ -54,48 +53,33 @@ int ObDASIndexDMLAdaptor::write_rows(cons int ret = OB_SUCCESS; ObAccessService *as = share::g_mp->access_service(); if (OB_UNLIKELY(ctdef.table_param_.get_data_table().is_vector_delta_buffer() && - !ctdef.is_access_mlog_as_master_table_)) { + !ctdef.is_access_main_table_)) { // for vector delta buffer, only do insert when DML with main table - if (OB_FAIL(as->insert_rows(ls_id, tablet_id, *tx_desc_, dml_param_, + if (OB_FAIL(as->insert_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("insert rows to access service failed", K(ret), K(tablet_id)); } } } else if (ctdef.table_param_.get_data_table().is_hybrid_vector_index() && - !ctdef.is_access_mlog_as_master_table_) { + !ctdef.is_access_main_table_) { // For hybrid vector index, check if it's embedded table if (share::schema::is_hybrid_vec_index_embedded_type(ctdef.table_param_.get_data_table().get_index_type())) { // For embedded table, perform actual delete operation - if (OB_FAIL(as->delete_rows(ls_id, tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { + if (OB_FAIL(as->delete_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("delete rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("delete rows to access service failed", K(ret), K(tablet_id)); } } } else if (share::schema::is_hybrid_vec_index_log_type(ctdef.table_param_.get_data_table().get_index_type())) { // For other hybrid vector index tables (like log table), perform insert to record delete mark - if (OB_FAIL(as->insert_rows(ls_id, tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { + if (OB_FAIL(as->insert_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("insert rows to access service failed", K(ret), K(tablet_id)); } } } - } else if (ctdef.table_param_.get_data_table().is_mlog_table() - && !ctdef.is_access_mlog_as_master_table_) { - ObDASMLogDMLIterator mlog_iter(ls_id, tablet_id, dml_param_, &iter, DAS_OP_TABLE_DELETE); - if (OB_FAIL(as->insert_rows(ls_id, - tablet_id, - *tx_desc_, - dml_param_, - ctdef.column_ids_, - &mlog_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret)); - } - } - } else if (OB_FAIL(as->delete_rows(ls_id, - tablet_id, + } else if (OB_FAIL(as->delete_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, @@ -138,14 +122,13 @@ int ObDASDeleteOp::open_op() del_adaptor.related_ctdefs_ = &related_ctdefs_; del_adaptor.related_rtdefs_ = &related_rtdefs_; del_adaptor.tablet_id_ = tablet_id_; - del_adaptor.ls_id_ = ls_id_; del_adaptor.related_tablet_ids_ = &related_tablet_ids_; - del_adaptor.is_do_gts_opt_ = das_gts_opt_info_.use_specify_snapshot_; + del_adaptor.use_snapshot_opt_ = das_snapshot_opt_info_.use_specify_snapshot_; del_adaptor.das_allocator_ = &op_alloc_; del_adaptor.ft_doc_word_infos_ = &doc_word_infos; - if (OB_FAIL(ObDASDomainUtils::build_ft_doc_word_infos(ls_id_, trans_desc_, snapshot_, related_ctdefs_, related_tablet_ids_, + if (OB_FAIL(ObDASDomainUtils::build_ft_doc_word_infos(trans_desc_, snapshot_, related_ctdefs_, related_tablet_ids_, del_ctdef_->is_main_table_in_fts_ddl_, doc_word_infos))) { - LOG_WARN("fail to build fulltext doc word infos", K(ret), K(ls_id_), KPC(snapshot_), K(related_ctdefs_), + LOG_WARN("fail to build fulltext doc word infos", K(ret), KPC(snapshot_), K(related_ctdefs_), K(related_tablet_ids_)); } else if (OB_FAIL(del_adaptor.write_tablet(dml_iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { diff --git a/src/sql/das/ob_das_domain_utils.cpp b/src/sql/das/ob_das_domain_utils.cpp index 7de1333f7..4ff74145f 100644 --- a/src/sql/das/ob_das_domain_utils.cpp +++ b/src/sql/das/ob_das_domain_utils.cpp @@ -236,7 +236,6 @@ int ObDASDomainUtils::generate_spatial_index_rows( } int ObDASDomainUtils::build_ft_doc_word_infos( - const share::ObLSID &ls_id, const transaction::ObTxDesc *trans_desc, const transaction::ObTxReadSnapshot *snapshot, const common::ObIArray &related_ctdefs, @@ -245,9 +244,9 @@ int ObDASDomainUtils::build_ft_doc_word_infos( common::ObIArray &doc_word_infos) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || OB_ISNULL(snapshot))) { + if (OB_UNLIKELY(OB_ISNULL(snapshot))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), KPC(snapshot)); + LOG_WARN("invalid arguments", K(ret), KPC(snapshot)); } for (int64_t i = 0; OB_SUCC(ret) && i < related_ctdefs.count(); ++i) { ObFTDocWordInfo doc_word_info; @@ -258,7 +257,6 @@ int ObDASDomainUtils::build_ft_doc_word_infos( } else if (related_ctdef->table_param_.get_data_table().is_fts_doc_word_aux()) { doc_word_info.table_id_ = related_ctdef->table_param_.get_data_table().get_table_id(); doc_word_info.doc_word_table_id_ = related_ctdef->table_param_.get_data_table().get_table_id(); - doc_word_info.doc_word_ls_id_ = ls_id; doc_word_info.doc_word_tablet_id_ = related_tablet_ids.at(i); doc_word_info.doc_word_schema_version_ = related_ctdef->table_param_.get_data_table().get_schema_version(); doc_word_info.doc_word_found_ = true; @@ -280,7 +278,6 @@ int ObDASDomainUtils::build_ft_doc_word_infos( } else if (doc_word_related_ctdef->table_param_.get_data_table().is_fts_doc_word_aux() && 0 == doc_word_related_ctdef->table_param_.get_data_table().get_index_name().case_compare(buf)) { doc_word_info.doc_word_table_id_ = doc_word_related_ctdef->table_param_.get_data_table().get_table_id(); - doc_word_info.doc_word_ls_id_ = ls_id; doc_word_info.doc_word_tablet_id_ = related_tablet_ids.at(j); doc_word_info.doc_word_schema_version_ = doc_word_related_ctdef->table_param_.get_data_table().get_schema_version(); doc_word_info.doc_word_found_ = true; @@ -300,7 +297,7 @@ int ObDASDomainUtils::build_ft_doc_word_infos( LOG_WARN("fail to refresh seq no", K(ret), K(i), KPC(related_ctdef), K(doc_word_info)); } } - LOG_TRACE("build_ft_doc_word_infos", K(ret), K(ls_id), K(snapshot), K(doc_word_infos), K(related_ctdefs), + LOG_TRACE("build_ft_doc_word_infos", K(ret), K(snapshot), K(doc_word_infos), K(related_ctdefs), K(related_tablet_ids)); return ret; } @@ -956,7 +953,6 @@ int ObFTDMLIterator::rewind() LOG_WARN("unexpected error, doc word info is nullptr", K(ret), KPC(doc_word_info_)); } else if (FALSE_IT(ft_doc_word_iter_.reset())) { } else if (OB_FAIL(ft_doc_word_iter_.init(doc_word_info_->doc_word_table_id_, - doc_word_info_->doc_word_ls_id_, doc_word_info_->doc_word_tablet_id_, &doc_word_info_->snapshot_, doc_word_info_->doc_word_schema_version_))) { @@ -991,7 +987,6 @@ int ObFTDMLIterator::init( ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, doc word info is nullptr", K(ret), KPC(doc_word_info_)); } else if (OB_FAIL(ft_doc_word_iter_.init(doc_word_info_->doc_word_table_id_, - doc_word_info_->doc_word_ls_id_, doc_word_info_->doc_word_tablet_id_, &doc_word_info_->snapshot_, doc_word_info_->doc_word_schema_version_))) { @@ -1054,7 +1049,6 @@ int ObFTDMLIterator::change_domain_dml_mode(const ObDomainDMLMode &mode) LOG_WARN("unexpected error, doc word info is nullptr", K(ret), KPC(doc_word_info_)); } else if (FALSE_IT(ft_doc_word_iter_.reset())) { } else if (OB_FAIL(ft_doc_word_iter_.init(doc_word_info_->doc_word_table_id_, - doc_word_info_->doc_word_ls_id_, doc_word_info_->doc_word_tablet_id_, &doc_word_info_->snapshot_, doc_word_info_->doc_word_schema_version_))) { diff --git a/src/sql/das/ob_das_domain_utils.h b/src/sql/das/ob_das_domain_utils.h index c1295c365..a1ba4014c 100644 --- a/src/sql/das/ob_das_domain_utils.h +++ b/src/sql/das/ob_das_domain_utils.h @@ -73,7 +73,6 @@ class ObFTDocWordInfo final ObFTDocWordInfo() : table_id_(OB_INVALID_ID), doc_word_table_id_(OB_INVALID_ID), - doc_word_ls_id_(), doc_word_tablet_id_(), snapshot_(), doc_word_schema_version_(), @@ -86,7 +85,6 @@ class ObFTDocWordInfo final int ret = OB_SUCCESS; table_id_ = src.table_id_; doc_word_table_id_ = src.doc_word_table_id_; - doc_word_ls_id_ = src.doc_word_ls_id_; doc_word_tablet_id_ = src.doc_word_tablet_id_; doc_word_schema_version_ = src.doc_word_schema_version_; doc_word_found_ = src.doc_word_found_; @@ -99,7 +97,6 @@ class ObFTDocWordInfo final TO_STRING_KV(K_(table_id), K_(doc_word_table_id), - K_(doc_word_ls_id), K_(doc_word_tablet_id), K_(snapshot), K_(doc_word_schema_version), @@ -107,7 +104,6 @@ class ObFTDocWordInfo final public: uint64_t table_id_; uint64_t doc_word_table_id_; - share::ObLSID doc_word_ls_id_; common::ObTabletID doc_word_tablet_id_; transaction::ObTxReadSnapshot snapshot_; int64_t doc_word_schema_version_; @@ -162,7 +158,6 @@ class ObDASDomainUtils final ~ObDASDomainUtils() = default; static int build_ft_doc_word_infos( - const share::ObLSID &ls_id, const transaction::ObTxDesc *trans_desc, const transaction::ObTxReadSnapshot *snapshot, const common::ObIArray &related_ctdef, diff --git a/src/sql/das/ob_das_id_cache.cpp b/src/sql/das/ob_das_id_cache.cpp deleted file mode 100644 index fc5715952..000000000 --- a/src/sql/das/ob_das_id_cache.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_DAS -#include "ob_das_id_rpc.h" -#include "ob_das_id_cache.h" -#include "ob_das_id_service.h" -#include "share/location_cache/ob_location_service.h" - -namespace oceanbase { -using namespace common; -using namespace share; -namespace sql -{ -int ObDASIDCache::init(const common::ObAddr &server) -{ - int ret = OB_SUCCESS; - void *request_buf = nullptr; - alloc_.set_attr(ObMemAttr("DASIDCache")); - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", KR(ret)); - } else if (OB_UNLIKELY(!server.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(server)); - } else if (OB_ISNULL(request_buf = alloc_.alloc(sizeof(ObDASIDRequestRpc)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc request rpc failed", KR(ret)); - } - if (OB_SUCC(ret)) { - id_request_rpc_ = new(request_buf) ObDASIDRequestRpc(); - if (OB_FAIL(id_request_rpc_->init(server, this))) { - LOG_WARN("init response rpc failed", KR(ret)); - } - } - if (OB_FAIL(ret)) { - if (NULL != id_request_rpc_) { - alloc_.free(id_request_rpc_); - id_request_rpc_ = NULL; - } - } else { - server_ = server; - is_inited_ = true; - } - return ret; -} - -void ObDASIDCache::destroy() -{ - if (NULL != id_request_rpc_) { - alloc_.free(id_request_rpc_); - id_request_rpc_ = NULL; - } - alloc_.reset(); - is_inited_ = false; -} - -void ObDASIDCache::reset() -{ - is_inited_ = false; - is_requesting_ = false; - for (int i=0; i= MAX_CACHE_NUM - 1) { - LOG_TRACE("drop das id", K(start_id), K(end_id)); - } else { - IdCache *id_cache = &(id_cache_[cache_idx % MAX_CACHE_NUM]); - inc_update(&(id_cache->start_id), start_id); - inc_update(&(id_cache->end_id), end_id); - (void)ATOMIC_FAA(&cache_idx_, 1); - retry_request_cnt_ = 0; - } - return ret; -} - -int ObDASIDCache::get_das_id(int64_t &das_id, const bool force_renew) -{ - int ret = OB_SUCCESS; - int save_ret = OB_SUCCESS; - - while (OB_SUCCESS == ret) { - const int64_t cur_idx = ATOMIC_LOAD(&cur_idx_); - IdCache *id_cache = &(id_cache_[cur_idx % MAX_CACHE_NUM]); - const int64_t tmp_end_id = ATOMIC_LOAD(&(id_cache->end_id)); - const int64_t tmp_start_id = ATOMIC_LOAD(&(id_cache->start_id)); - int64_t tmp_das_id = 0; - if (tmp_start_id < tmp_end_id && - (tmp_das_id = ATOMIC_FAA(&(id_cache->start_id), 1)) < tmp_end_id) { - das_id = tmp_das_id; - break; - } else { - if (OB_UNLIKELY(cur_idx >= ATOMIC_LOAD(&cache_idx_))) { - ret = OB_EAGAIN; - } else { - (void)ATOMIC_CAS(&cur_idx_, cur_idx, cur_idx + 1); - } - } - } - save_ret = ret; - const int64_t left_cache_count = cache_idx_ - cur_idx_; - if (left_cache_count < PRE_CACHE_NUM && !ATOMIC_LOAD(&is_requesting_)) { - const int64_t cur_ts = ObTimeUtility::current_time(); - const int64_t retry_timeout = min(OB_DAS_ID_RPC_TIMEOUT_MIN * max(retry_request_cnt_, 1), - OB_DAS_ID_RPC_TIMEOUT_MAX); - if (ATOMIC_BCAS(&is_requesting_, false, true)) { - ObDASIDRequest req; - obcall::ObDASIDRpcResult res; - retry_request_cnt_++; - if (OB_FAIL(req.init(get_preallocate_count_()))) { - LOG_WARN("ObDASIDRequest init fail", KR(ret)); - } else if (OB_FAIL(id_request_rpc_->fetch_new_range(req, res, retry_timeout, force_renew))) { - LOG_WARN("fetch new range failed", KR(ret)); - } else if (OB_UNLIKELY(!res.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("das id rpc result is invalid", KR(ret), K(res)); - } else if (OB_SUCCESS != res.get_status()) { - if (OB_NOT_MASTER == res.get_status() || OB_TENANT_NOT_IN_SERVER == res.get_status()) { - ret = OB_EAGAIN; - LOG_INFO("das id rpc need retry", KR(res.get_status()), KR(ret)); - } else { - ret = res.get_status(); - LOG_WARN("das id rpc failed", KR(ret), K(res)); - } - } else if (OB_FAIL(update_das_id(res.get_start_id(), res.get_end_id()))) { - LOG_WARN("update das id failed", KR(ret)); - } - ATOMIC_STORE(&is_requesting_, false); - if (OB_SUCC(ret)) { - LOG_TRACE("das id rpc succeeded", K(req), K(res)); - } - } - if (left_cache_count <= PRE_CACHE_NUM / 2) { - if (cache_idx_ < PRE_CACHE_NUM) { - // do not update preallocate count at boot time - } else { - update_preallocate_count_(); - } - } - } - return save_ret; -} - -void ObDASIDCache::update_preallocate_count_() -{ - const int64_t cur_ts = ObTimeUtility::current_time(); - if (cur_ts - last_update_ts_ > UPDATE_PREALLOCATE_COUNT_INTERVAL) { - const int64_t tmp_preallocate_count = ATOMIC_LOAD(&preallocate_count_); - int64_t new_preallocate_count = tmp_preallocate_count * UPDATE_FACTOR; - if (new_preallocate_count > MAX_PREALLOCATE_COUNT) { - new_preallocate_count = MAX_PREALLOCATE_COUNT; - } - if (ATOMIC_BCAS(&preallocate_count_, tmp_preallocate_count, new_preallocate_count)) { - last_update_ts_ = cur_ts; - } - } -} - -int64_t ObDASIDCache::get_preallocate_count_() -{ - const int64_t tmp_preallocate_count = ATOMIC_LOAD(&preallocate_count_); - int64_t new_preallocate_count = tmp_preallocate_count - MIN_PREALLOCATE_COUNT; - if (new_preallocate_count < MIN_PREALLOCATE_COUNT) { - new_preallocate_count = MIN_PREALLOCATE_COUNT; - } - (void)ATOMIC_BCAS(&preallocate_count_, tmp_preallocate_count, new_preallocate_count); - return tmp_preallocate_count; -} -} // namespace sql -} // namespace oceanbase diff --git a/src/sql/das/ob_das_id_cache.h b/src/sql/das/ob_das_id_cache.h deleted file mode 100644 index fd513581b..000000000 --- a/src/sql/das/ob_das_id_cache.h +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_OB_DAS_ID_CACHE_H -#define OCEANBASE_OB_DAS_ID_CACHE_H - -namespace oceanbase -{ -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} -namespace sql -{ -class ObDASIDRequestRpc; - -struct IdCache -{ - int64_t start_id CACHE_ALIGNED; - int64_t end_id; - TO_STRING_KV(K(start_id), K(end_id)); -}; -class ObDASIDCache -{ -public: - ObDASIDCache() { reset(); } - ~ObDASIDCache() { destroy(); } - int init(const common::ObAddr &server); - void destroy(); - void reset(); - int update_das_id(const int64_t start_id, const int64_t end_id); - int get_das_id(int64_t &das_id, const bool force_renew); -private: - void update_preallocate_count_(); - int64_t get_preallocate_count_(); -public: - TO_STRING_KV(K_(is_inited), K_(cur_idx), K_(cache_idx), K_(id_service_leader)); -public: - static const int64_t MIN_PREALLOCATE_COUNT = 1000000; // 1 million - static const int64_t MAX_PREALLOCATE_COUNT = MIN_PREALLOCATE_COUNT * 10; - static const int64_t UPDATE_FACTOR = 4; - static const int64_t MAX_CACHE_NUM = 16; - static const int64_t PRE_CACHE_NUM = MAX_CACHE_NUM / 4; - static const int64_t OB_DAS_ID_RPC_TIMEOUT_MIN = 100 * 1000L; // 100ms - static const int64_t OB_DAS_ID_RPC_TIMEOUT_MAX = 2 * 1000L * 1000L; // 2s - static const int64_t UPDATE_PREALLOCATE_COUNT_INTERVAL = OB_DAS_ID_RPC_TIMEOUT_MIN; -private: - bool is_inited_; - bool is_requesting_; - IdCache id_cache_[MAX_CACHE_NUM]; // Segment and scatter the Cache to avoid multiple threads competing concurrently, causing access hotspots - int64_t cur_idx_; - int64_t cache_idx_; - common::ObAddr server_; - ObDASIDRequestRpc *id_request_rpc_; - common::ObAddr id_service_leader_; - int64_t retry_request_cnt_; - common::ObLatch lock_; - int64_t preallocate_count_; - int64_t last_update_ts_; - common::ObArenaAllocator alloc_; -}; -} // namespace sql -} // namespace oceanbase -#endif // OCEANBASE_OB_DAS_ID_CACHE_H diff --git a/src/sql/das/ob_das_id_rpc.cpp b/src/sql/das/ob_das_id_rpc.cpp deleted file mode 100644 index dfe0e3419..000000000 --- a/src/sql/das/ob_das_id_rpc.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_DAS -#include "ob_das_id_rpc.h" -#include "ob_das_id_service.h" -#include "share/location_cache/ob_location_service.h" - -namespace oceanbase -{ -using namespace common; -using namespace share; -namespace sql -{ - -OB_SERIALIZE_MEMBER(ObDASIDRequest, range_); - -int ObDASIDRequest::init(const int64_t range) -{ - int ret = OB_SUCCESS; - if (!true || 0 >= range) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(range)); - } else { - range_ = range; - } - return ret; -} - -bool ObDASIDRequest::is_valid() const -{ - return true && range_ > 0; -} - -ObDASIDRequestRpc::ObDASIDRequestRpc() - : is_inited_(false), - is_running_(false), - self_(), - local_id_counter_(1) // Start from 1 to satisfy ObDASIDRpcResult validation (start_id > 0) -{ -} - -int ObDASIDRequestRpc::init(const common::ObAddr &self, - ObDASIDCache *id_cache) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - LOG_WARN("das id request rpc inited twice", KR(ret)); - } else if (!self.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(self)); - } else { - self_ = self; - is_inited_ = true; - } - return ret; -} - -void ObDASIDRequestRpc::destroy() -{ - if (is_inited_) { - self_.reset(); - local_id_counter_ = 1; // Reset to 1 for next init - is_inited_ = false; - } -} - -int ObDASIDRequestRpc::fetch_new_range(const ObDASIDRequest &msg, - obcall::ObDASIDRpcResult &res, - const int64_t timeout, - const bool force_renew) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("das id request rpc not inited", KR(ret)); - } else if (!msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid request", KR(ret), K(msg)); - } else { - // Use local auto-increment counter instead of RPC - // Counter resets to 1 on restart, no persistence needed - const int64_t range = msg.get_range(); - const int64_t start_id = ATOMIC_FAA(&local_id_counter_, range); - const int64_t end_id = start_id + range; - - - if (OB_FAIL(res.init(OB_SUCCESS, start_id, end_id))) { - LOG_WARN("init das id result failed", KR(ret), K(start_id), K(end_id)); - } else { - LOG_TRACE("fetch new DAS ID range from local counter", K(msg), K(res)); - } - } - return ret; -} -} // namespace sql - -namespace obcall -{ - -OB_SERIALIZE_MEMBER(ObDASIDRpcResult, status_, start_id_, end_id_); - -int ObDASIDRpcResult::init(const int status, - const int64_t start_id, - const int64_t end_id) -{ - int ret = OB_SUCCESS; - if (!true || - (OB_SUCCESS == status && (0 >= start_id || 0 >= end_id))) { - // when status is OB_SUCCESS, RPC should have succeeded with valid start id and end id - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(start_id), K(end_id)); - } else { - status_ = status; - start_id_ = start_id; - end_id_ = end_id; - } - return ret; -} - -bool ObDASIDRpcResult::is_valid() const -{ - // when status is not OB_SUCCESS, - // RPC has failed due to some error on the server side, e.g. NOT_MASTER - // otherwise, RPC should have succeeded with valid start id and end id - return true && - (OB_SUCCESS != status_ || (start_id_ > 0 && end_id_ > 0)); -} - -} // namespace obcall -} // namespace oceanbase diff --git a/src/sql/das/ob_das_id_rpc.h b/src/sql/das/ob_das_id_rpc.h deleted file mode 100644 index 10aba2a01..000000000 --- a/src/sql/das/ob_das_id_rpc.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OBDEV_SRC_SQL_DAS_OB_DAS_ID_RPC_H_ -#define OBDEV_SRC_SQL_DAS_OB_DAS_ID_RPC_H_ -#include "observer/ob_server_struct.h" -#include "ob_das_id_cache.h" -#include "lib/atomic/ob_atomic.h" -namespace oceanbase -{ -namespace sql -{ -class ObDASIDRequest -{ - OB_UNIS_VERSION(1); -public: - ObDASIDRequest() : range_(0) {} - ~ObDASIDRequest() {} - int init(const int64_t range); - bool is_valid() const; -public: - - int64_t get_range() const { return range_; } - TO_STRING_KV(K_(range)); -private: - int64_t range_; -}; -} // namespace sql - -namespace obcall -{ -class ObDASIDRpcResult -{ - OB_UNIS_VERSION(1); -public: - ObDASIDRpcResult() : status_(0), start_id_(0), end_id_(0) {} - virtual ~ObDASIDRpcResult() {} - int init(const int status, const int64_t start_id, const int64_t end_id); - - int get_status() const { return status_; } - int64_t get_start_id() const { return start_id_; } - int64_t get_end_id() const { return end_id_; } - bool is_valid() const; - TO_STRING_KV(K_(status), K_(start_id), K_(end_id)); -private: - int status_; - int64_t start_id_; - int64_t end_id_; -}; - -} // namespace obcall - -namespace sql -{ -class ObDASIDRequestRpc -{ -public: - ObDASIDRequestRpc(); - ~ObDASIDRequestRpc() { destroy(); } - int init(const common::ObAddr &self, - ObDASIDCache *id_cache); - void destroy(); - int fetch_new_range(const ObDASIDRequest &msg, - obcall::ObDASIDRpcResult &res, - const int64_t timeout, - const bool force_renew); -private: - bool is_inited_; - bool is_running_; - common::ObAddr self_; - int64_t local_id_counter_ CACHE_ALIGNED; // Local auto-increment counter for DAS ID, resets to 0 on restart -}; -} // namespace sql -} // namespace oceanbase -#endif // OBDEV_SRC_SQL_DAS_OB_DAS_ID_RPC_H_ diff --git a/src/sql/das/ob_das_id_service.cpp b/src/sql/das/ob_das_id_service.cpp deleted file mode 100644 index 5be5bce95..000000000 --- a/src/sql/das/ob_das_id_service.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_DAS -#include "ob_das_id_service.h" -namespace oceanbase -{ -namespace sql -{ -int ObDASIDService::mtl_init(ObDASIDService *&das_id_service) -{ - return das_id_service->init(); -} - -int ObDASIDService::init() -{ - self_ = GCTX.self_addr(); - service_type_ = ServiceType::DASIDService; - pre_allocated_range_ = DAS_ID_PREALLOCATED_RANGE; - return OB_SUCCESS; -} - -int ObDASIDService::handle_request(const ObDASIDRequest &request, obcall::ObDASIDRpcResult &result) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(request)); - } else { - - const int64_t range = request.get_range(); - int64_t start_id = 0; - int64_t end_id = 0; - if (OB_FAIL(get_number(range, 0, start_id, end_id))) { - LOG_WARN("get das id failed", KR(ret)); - } - // overwrite ret - if (OB_FAIL(result.init(ret, start_id, end_id))) { - LOG_WARN("das id result init failed", KR(ret), K(request)); - } - } - // overwrite ret - return OB_SUCCESS; -} -} // namespace sql -} // namespace oceanbase diff --git a/src/sql/das/ob_das_id_service.h b/src/sql/das/ob_das_id_service.h deleted file mode 100644 index bd0fdfeaa..000000000 --- a/src/sql/das/ob_das_id_service.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OBDEV_SRC_SQL_DAS_OB_DAS_ID_SERVICE_H_ -#define OBDEV_SRC_SQL_DAS_OB_DAS_ID_SERVICE_H_ -#include "storage/tx/ob_id_service.h" -#include "ob_das_id_rpc.h" -namespace oceanbase -{ -namespace sql -{ -class ObDASIDService : public transaction::ObIDService -{ -public: - ObDASIDService() {} - ~ObDASIDService() {} - int init(); - static int mtl_init(ObDASIDService *&das_id_service); - void destroy() { reset(); } - static const int64_t DAS_ID_PREALLOCATED_RANGE = 1000000; // 1 million - int handle_request(const ObDASIDRequest &request, obcall::ObDASIDRpcResult &result); -}; -} // namespace sql -} // namespace oceanbase -#endif // OBDEV_SRC_SQL_DAS_OB_DAS_ID_SERVICE_H_ diff --git a/src/sql/das/ob_das_insert_op.cpp b/src/sql/das/ob_das_insert_op.cpp index f47e3c318..a927c823a 100644 --- a/src/sql/das/ob_das_insert_op.cpp +++ b/src/sql/das/ob_das_insert_op.cpp @@ -45,8 +45,7 @@ using namespace share; namespace sql { template <> -int ObDASIndexDMLAdaptor::write_rows(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDASIndexDMLAdaptor::write_rows(const ObTabletID &tablet_id, const CtDefType &ctdef, RtDefType &rtdef, ObDASDMLIterator &iter, @@ -54,31 +53,15 @@ int ObDASIndexDMLAdaptor::write_rows(cons { int ret = OB_SUCCESS; ObAccessService *as = share::g_mp->access_service(); - if (ctdef.table_param_.get_data_table().is_mlog_table() - && !ctdef.is_access_mlog_as_master_table_) { - ObDASMLogDMLIterator mlog_iter(ls_id, tablet_id, dml_param_, &iter, DAS_OP_TABLE_INSERT); - if (OB_FAIL(as->insert_rows(ls_id, - tablet_id, - *tx_desc_, - dml_param_, - ctdef.column_ids_, - &mlog_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret)); - } - } - } else if (rtdef.use_put_) { - ret = as->put_rows(ls_id, - tablet_id, + if (rtdef.use_put_) { + ret = as->put_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows); } else { - ret = as->insert_rows(ls_id, - tablet_id, + ret = as->insert_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, @@ -111,7 +94,7 @@ int ObDASInsertOp::open_op() { int ret = OB_SUCCESS; if (ins_rtdef_->need_fetch_conflict_ && OB_FAIL(insert_row_with_fetch())) { - LOG_WARN("fail to do insert with conflict fetch", K(ret), K(das_gts_opt_info_)); + LOG_WARN("fail to do insert with conflict fetch", K(ret), K(das_snapshot_opt_info_)); } else if (!ins_rtdef_->need_fetch_conflict_ && OB_FAIL(insert_rows())) { LOG_WARN("fail to do insert", K(ret)); } @@ -133,9 +116,8 @@ int ObDASInsertOp::insert_rows() ins_adaptor.related_ctdefs_ = &related_ctdefs_; ins_adaptor.related_rtdefs_ = &related_rtdefs_; ins_adaptor.tablet_id_ = tablet_id_; - ins_adaptor.ls_id_ = ls_id_; ins_adaptor.related_tablet_ids_ = &related_tablet_ids_; - ins_adaptor.is_do_gts_opt_ = das_gts_opt_info_.use_specify_snapshot_; + ins_adaptor.use_snapshot_opt_ = das_snapshot_opt_info_.use_specify_snapshot_; ins_adaptor.das_allocator_ = &op_alloc_; if (OB_FAIL(ins_adaptor.write_tablet(dml_iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { @@ -167,10 +149,9 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, op_alloc_, store_ctx_guard, dml_param, - das_gts_opt_info_.use_specify_snapshot_))) { + das_snapshot_opt_info_.use_specify_snapshot_))) { LOG_WARN("init index dml param failed", K(ret), KPC(ins_ctdef), KPC(ins_rtdef)); - } else if (OB_FAIL(as->insert_rows_with_fetch_dup(ls_id_, - tablet_id, + } else if (OB_FAIL(as->insert_rows_with_fetch_dup(tablet_id, *trans_desc_, dml_param, ins_ctdef->column_ids_, @@ -217,24 +198,23 @@ int ObDASInsertOp::insert_row_with_fetch() transaction::ObTxReadSnapshot *snapshot = snapshot_; // write_flag should be inited before get store ctx, as it will be used in the call function - (void)ObDMLService::init_dml_write_flag(*ins_ctdef_, *ins_rtdef_, write_flag, das_gts_opt_info_.use_specify_snapshot_); - if (das_gts_opt_info_.get_specify_snapshot()) { + (void)ObDMLService::init_dml_write_flag(*ins_ctdef_, *ins_rtdef_, write_flag, das_snapshot_opt_info_.use_specify_snapshot_); + if (das_snapshot_opt_info_.get_specify_snapshot()) { transaction::ObTransService *txs = nullptr; - if (das_gts_opt_info_.isolation_level_ != transaction::ObTxIsolationLevel::RC) { + if (das_snapshot_opt_info_.isolation_level_ != transaction::ObTxIsolationLevel::RC) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected isolation_level", K(ret), K(das_gts_opt_info_)); + LOG_WARN("unexpected isolation_level", K(ret), K(das_snapshot_opt_info_)); } else if (OB_ISNULL(txs = MTL_WITH_CHECK(transaction::ObTransService*))) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("get_tx_service", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*trans_desc_, - das_gts_opt_info_.isolation_level_, - ls_id_, - THIS_WORKER.get_timeout_ts(), - *das_gts_opt_info_.get_response_snapshot()))) { - LOG_WARN("fail to get ls read_snapshot", K(ret), K(ls_id_), K(THIS_WORKER.get_timeout_ts())); + } else if (OB_FAIL(txs->get_read_snapshot(*trans_desc_, + das_snapshot_opt_info_.isolation_level_, + THIS_WORKER.get_timeout_ts(), + *das_snapshot_opt_info_.get_response_snapshot()))) { + LOG_WARN("fail to get read snapshot", K(ret), K(THIS_WORKER.get_timeout_ts())); } else { - snapshot = das_gts_opt_info_.get_response_snapshot(); - LOG_TRACE("succ get ls snaoshot", K(ls_id_), K(tablet_id_), KPC(snapshot)); + snapshot = das_snapshot_opt_info_.get_response_snapshot(); + LOG_TRACE("succ get read snapshot", K(tablet_id_), KPC(snapshot)); } } @@ -243,14 +223,13 @@ int ObDASInsertOp::insert_row_with_fetch() } else if (ins_ctdef_->table_rowkey_types_.empty()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_rowkey_types is invalid", K(ret)); - } else if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_, - ins_rtdef_->timeout_ts_, + } else if (OB_FAIL(as->get_write_store_ctx_guard(ins_rtdef_->timeout_ts_, *trans_desc_, *snapshot, write_branch_id_, write_flag, store_ctx_guard))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); } else if (OB_ISNULL(buf = op_alloc_.alloc(sizeof(ObDASConflictIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate ObDASConflictIterator", K(ret)); @@ -314,17 +293,9 @@ int ObDASInsertOp::insert_row_with_fetch() } else if (OB_FAIL(dml_iter.rewind(index_ins_ctdef, nullptr/*fts_doc_word_info*/))) { LOG_WARN("rewind dml iter failed", K(ret)); } else { - ObDASMLogDMLIterator mlog_iter(ls_id_, index_tablet_id, dml_param, &dml_iter, DAS_OP_TABLE_INSERT); - ObDatumRowIterator *new_iter = nullptr; - if (index_ins_ctdef->table_param_.get_data_table().is_mlog_table() - && !index_ins_ctdef->is_access_mlog_as_master_table_) { - new_iter = &mlog_iter; - } else { - new_iter = &dml_iter; - } if (OB_FAIL(insert_index_with_fetch(dml_param, as, - *new_iter, + dml_iter, result_iter, index_ins_ctdef, index_ins_rtdef, @@ -424,8 +395,8 @@ int ObDASInsertOp::decode_task_result(ObIDASTaskResult *task_result) result_ = insert_result; affected_rows_ = insert_result->get_affected_rows(); is_duplicated_ = insert_result->is_duplicated(); - if (das_gts_opt_info_.get_specify_snapshot()) { - if (OB_FAIL(das_gts_opt_info_.get_response_snapshot()->assign(insert_result->get_response_snapshot()))) { + if (das_snapshot_opt_info_.get_specify_snapshot()) { + if (OB_FAIL(das_snapshot_opt_info_.get_response_snapshot()->assign(insert_result->get_response_snapshot()))) { LOG_WARN("fail to assign snapshot", K(ret)); } } @@ -457,8 +428,8 @@ int ObDASInsertOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_mor ins_result.set_is_duplicated(is_duplicated_); has_more = false; memory_limit -= ins_result.get_result_buffer().get_mem_used(); - if (das_gts_opt_info_.get_specify_snapshot()) { - if (OB_FAIL(ins_result.get_response_snapshot().assign(*das_gts_opt_info_.get_response_snapshot()))) { + if (das_snapshot_opt_info_.get_specify_snapshot()) { + if (OB_FAIL(ins_result.get_response_snapshot().assign(*das_snapshot_opt_info_.get_response_snapshot()))) { LOG_WARN("fail to assign snapshot", K(ret)); } } diff --git a/src/sql/das/ob_das_location_router.cpp b/src/sql/das/ob_das_location_router.cpp index 5c40aa851..effafa95e 100755 --- a/src/sql/das/ob_das_location_router.cpp +++ b/src/sql/das/ob_das_location_router.cpp @@ -28,6 +28,21 @@ using namespace share::schema; using namespace transaction; namespace sql { +namespace +{ +int build_local_replica(ObLSReplicaLocation &replica) +{ + int ret = OB_SUCCESS; + ObReplicaProperty property; + ObLSRestoreStatus restore_status(ObLSRestoreStatus::NONE); + if (OB_FAIL(replica.init(GCTX.self_addr(), common::LEADER, GCONF.mysql_port, + REPLICA_TYPE_FULL, property, restore_status, 1))) { + LOG_WARN("init local replica failed", K(ret)); + } + return ret; +} +} + OB_SERIALIZE_MEMBER(DASRelatedTabletMap::MapEntry, key_.src_tablet_id_, key_.related_table_id_, @@ -791,117 +806,6 @@ ObDASLocationRouter::~ObDASLocationRouter() cur_errno_ = OB_SUCCESS; } -int ObDASLocationRouter::nonblock_get_readable_replica(const ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc, - const ObRoutePolicyType route_policy) -{ - int ret = OB_SUCCESS; - ObLSLocation ls_loc; - tablet_loc.tablet_id_ = tablet_id; - if (OB_FAIL(all_tablet_list_.push_back(tablet_id))) { - LOG_WARN("store access tablet id failed", K(ret)); - } else if (OB_FAIL(GCTX.location_service_->nonblock_get(tablet_id, - tablet_loc.ls_id_))) { - LOG_WARN("nonblock get ls id failed", K(ret), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->nonblock_get(GCONF.cluster_id, - tablet_loc.ls_id_, - ls_loc))) { - LOG_WARN("get ls replica location failed", K(ret), K(tablet_loc)); - } - if (is_partition_change_error(ret)) { - /*During the execution phase, if nonblock location interface is used to obtain the location - * and an exception occurs, retries are necessary. - * However, statement-level retries cannot rollback many execution states, - * so it is necessary to avoid retries in this scenario as much as possible. - * During the execution phase, when encountering a location exception for the first time, - * try to refresh the location once synchronously. - * If it fails, then proceed with statement-level retries.*/ - int tmp_ret = block_renew_tablet_location(tablet_id, ls_loc); - if (OB_UNLIKELY(OB_SUCCESS != tmp_ret)) { - LOG_WARN("block renew tablet location failed", KR(tmp_ret), K(tablet_id)); - } else { - tablet_loc.ls_id_ = ls_loc.get_ls_id(); - ret = OB_SUCCESS; - } - } - // Single-node single-replica: no replica blacklist. - ObSEArray remote_replicas; - const ObLSReplicaLocation *local_replica = nullptr; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_loc.get_replica_locations().count(); ++i) { - const ObLSReplicaLocation &tmp_replica_loc = ls_loc.get_replica_locations().at(i); - { - if (route_policy == FORCE_READONLY_ZONE && tmp_replica_loc.get_replica_type() != REPLICA_TYPE_READONLY) { - // skip the tmp_replica_loc - LOG_TRACE("skip the replica due to the replica policy.", K(ret), K(tmp_replica_loc.get_replica_type()), K(tmp_replica_loc)); - } else if (tmp_replica_loc.get_server() == GCTX.self_addr()) { - //prefer choose the local replica - local_replica = &tmp_replica_loc; - } else if (OB_FAIL(remote_replicas.push_back(&tmp_replica_loc))) { - LOG_WARN("store tmp replica failed", K(ret)); - } - } - } - if (OB_SUCC(ret)) { - if (local_replica != nullptr) { - tablet_loc.server_ = local_replica->get_server(); - } else if (remote_replicas.empty()) { - ret = OB_NO_READABLE_REPLICA; - LOG_WARN("there has no readable replica", K(ret), K(tablet_id), K(ls_loc), K(route_policy)); - } else { - //no local copy, randomly select a readable replica - int64_t select_idx = rand() % remote_replicas.count(); - const ObLSReplicaLocation *remote_loc = remote_replicas.at(select_idx); - tablet_loc.server_ = remote_loc->get_server(); - } - } - save_cur_exec_status(ret); - return ret; -} - -int ObDASLocationRouter::nonblock_get(const ObDASTableLocMeta &loc_meta, - const common::ObTabletID &tablet_id, - ObLSLocation &location) -{ - int ret = OB_SUCCESS; - - bool is_vt = is_virtual_table(loc_meta.ref_table_id_); - uint64_t ref_table_id = loc_meta.ref_table_id_; - if (OB_UNLIKELY(is_vt)) { - if (OB_FAIL(get_vt_ls_location(ref_table_id, tablet_id, location))) { - LOG_WARN("get virtual table ls location failed", K(ret), K(ref_table_id), K(tablet_id)); - } - } else { - ObLSID ls_id; - if (OB_FAIL(all_tablet_list_.push_back(tablet_id))) { - LOG_WARN("store all tablet list failed", K(ret), K(tablet_id)); - } else if (OB_FAIL(GCTX.location_service_->nonblock_get(tablet_id, ls_id))) { - LOG_WARN("nonblock get ls id failed", K(ret)); - } else if (OB_FAIL(GCTX.location_service_->nonblock_get(GCONF.cluster_id, - ls_id, - location))) { - LOG_WARN("fail to get tablet locations", K(ret), K(1UL), K(ls_id)); - } - if (is_partition_change_error(ret)) { - /*During the execution phase, if nonblock location interface is used to obtain the location - * and an exception occurs, retries are necessary. - * However, statement-level retries cannot rollback many execution states, - * so it is necessary to avoid retries in this scenario as much as possible. - * During the execution phase, when encountering a location exception for the first time, - * try to refresh the location once synchronously. - * If it fails, then proceed with statement-level retries.*/ - int tmp_ret = block_renew_tablet_location(tablet_id, location); - if (OB_UNLIKELY(OB_SUCCESS != tmp_ret)) { - LOG_WARN("block renew tablet location failed", KR(tmp_ret), K(tablet_id)); - } else { - ret = OB_SUCCESS; - } - } - } - save_cur_exec_status(ret); - - return ret; -} - int ObDASLocationRouter::nonblock_get_candi_tablet_locations(const ObDASTableLocMeta &loc_meta, const ObIArray &tablet_ids, const ObIArray &partition_ids, @@ -909,34 +813,28 @@ int ObDASLocationRouter::nonblock_get_candi_tablet_locations(const ObDASTableLoc ObIArray &candi_tablet_locs) { int ret = OB_SUCCESS; + UNUSED(loc_meta); NG_TRACE(get_location_cache_begin); candi_tablet_locs.reset(); int64_t N = tablet_ids.count(); if (OB_FAIL(candi_tablet_locs.prepare_allocate(N))) { LOG_WARN("Partition location list prepare error", K(ret)); } else { - ObLSLocation location; + ObLSReplicaLocation local_replica; int64_t i = 0; + if (OB_FAIL(build_local_replica(local_replica))) { + LOG_WARN("build local replica failed", K(ret)); + } for (; OB_SUCC(ret) && i < N; ++i) { - location.reset(); ObCandiTabletLoc &candi_tablet_loc = candi_tablet_locs.at(i); - //after 4.1, all modules that need to access location will use nonblock_get to fetch location - //if the location has expired, DAS location router will refresh all accessed tablets - if (OB_FAIL(nonblock_get(loc_meta, tablet_ids.at(i), location))) { - LOG_WARN("Get partition error, the location cache will be renewed later", - K(ret), "tablet_id", tablet_ids.at(i), K(candi_tablet_loc)); - } else { - ObObjectID first_level_part_id = first_level_part_ids.empty() ? OB_INVALID_ID : first_level_part_ids.at(i); - if (OB_FAIL(candi_tablet_loc.set_part_loc_with_only_readable_replica(partition_ids.at(i), - first_level_part_id, - tablet_ids.at(i), - location, - static_cast(loc_meta.route_policy_)))) { - LOG_WARN("fail to set partition location with only readable replica", - K(ret),K(i), K(location), K(candi_tablet_locs), K(tablet_ids), K(partition_ids)); - } - LOG_DEBUG("set partition location with only readable replica", - K(ret),K(i), K(location), K(candi_tablet_locs), K(tablet_ids), K(partition_ids)); + ObObjectID first_level_part_id = first_level_part_ids.empty() ? OB_INVALID_ID : first_level_part_ids.at(i); + if (OB_FAIL(all_tablet_list_.push_back(tablet_ids.at(i)))) { + LOG_WARN("store all tablet list failed", K(ret), K(i)); + } else if (OB_FAIL(candi_tablet_loc.set_local_tablet_loc(partition_ids.at(i), + first_level_part_id, + tablet_ids.at(i), + local_replica))) { + LOG_WARN("set local tablet location failed", K(ret), K(i)); } } // for end } @@ -948,23 +846,8 @@ int ObDASLocationRouter::get_tablet_loc(const ObDASTableLocMeta &loc_meta, const ObTabletID &tablet_id, ObDASTabletLoc &tablet_loc) { - int ret = OB_SUCCESS; - - bool is_vt = is_virtual_table(loc_meta.ref_table_id_); - if (OB_UNLIKELY(is_vt)) { - if (OB_FAIL(get_vt_tablet_loc(loc_meta.ref_table_id_, tablet_id, tablet_loc))) { - LOG_WARN("get virtual tablet loc failed", K(ret), K(loc_meta)); - } - } else { - if (OB_LIKELY(loc_meta.select_leader_) || OB_UNLIKELY(last_errno_ == OB_NOT_MASTER)) { - //if this statement is retried because of OB_NOT_MASTER, we will choose the leader directly - ret = nonblock_get_leader(tablet_id, tablet_loc); - } else { - ret = nonblock_get_readable_replica(tablet_id, tablet_loc, - static_cast(loc_meta.route_policy_)); - } - } - return ret; + UNUSED(loc_meta); + return nonblock_get_leader(tablet_id, tablet_loc); } int ObDASLocationRouter::nonblock_get_leader(const ObTabletID &tablet_id, @@ -975,60 +858,12 @@ int ObDASLocationRouter::nonblock_get_leader(const ObTabletID &tablet_id, if (OB_FAIL(all_tablet_list_.push_back(tablet_id))) { LOG_WARN("store access tablet id failed", K(ret), K(tablet_id)); } else { - tablet_loc.ls_id_ = share::ObLSID::SYS_LS_ID; tablet_loc.server_ = GCTX.self_addr(); } save_cur_exec_status(ret); return ret; } -int ObDASLocationRouter::get_leader(const ObTabletID &tablet_id, - ObAddr &leader_addr, - int64_t expire_renew_time) -{ - int ret = OB_SUCCESS; - bool is_cache_hit = false; - ObLSID ls_id; - if (OB_FAIL(GCTX.location_service_->get(tablet_id, - expire_renew_time, - is_cache_hit, - ls_id))) { - LOG_WARN("nonblock get ls id failed", K(ret)); - } else if (OB_FAIL(GCTX.location_service_->get_leader(GCONF.cluster_id, - ls_id, - false, - leader_addr))) { - LOG_WARN("nonblock get ls location failed", K(ret)); - } - return ret; -} - - -int ObDASLocationRouter::get_full_ls_replica_loc(const ObDASTabletLoc &tablet_loc, - ObLSReplicaLocation &replica_loc) -{ - int ret = OB_SUCCESS; - bool is_cache_hit = false; - ObLSLocation ls_loc; - if (OB_FAIL(GCTX.location_service_->nonblock_get(GCONF.cluster_id, - tablet_loc.ls_id_, - ls_loc))) { - LOG_WARN("get ls replica location failed", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_loc.get_replica_locations().count(); ++i) { - const ObLSReplicaLocation &tmp_replica_loc = ls_loc.get_replica_locations().at(i); - if (tmp_replica_loc.get_server() == tablet_loc.server_) { - replica_loc = tmp_replica_loc; - break; - } - } - if (OB_SUCC(ret) && !replica_loc.is_valid()) { - ret = OB_LOCATION_NOT_EXIST; - LOG_WARN("replica location not found", K(ret), K(tablet_loc)); - } - return ret; -} - int ObDASLocationRouter::get_vt_svr_pair(uint64_t vt_id, const VirtualSvrPair *&vt_pair) { int ret = OB_SUCCESS; @@ -1043,18 +878,11 @@ int ObDASLocationRouter::get_vt_svr_pair(uint64_t vt_id, const VirtualSvrPair *& } else if (nullptr == vt_pair) { VirtualSvrPair empty_pair; VirtualSvrPair *tmp_pair = nullptr; - bool is_cache_hit = false; ObSEArray part_locations; if (OB_FAIL(virtual_server_list_.push_back(empty_pair))) { LOG_WARN("extend virtual server list failed", K(ret)); - } else if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("location_service_ is null", KR(ret)); - } else if (OB_FAIL(GCTX.location_service_->vtable_get(vt_id, - 0,/*expire_renew_time*/ - is_cache_hit, - part_locations))) { - LOG_WARN("fail to get virtual table location", KR(ret), K(vt_id)); + } else if (OB_FAIL(part_locations.push_back(GCTX.self_addr()))) { + LOG_WARN("add local virtual table location failed", K(ret), K(vt_id)); } else { tmp_pair = &virtual_server_list_.get_last(); if (OB_FAIL(tmp_pair->init(allocator_, vt_id, part_locations))) { @@ -1067,68 +895,6 @@ int ObDASLocationRouter::get_vt_svr_pair(uint64_t vt_id, const VirtualSvrPair *& return ret; } -OB_NOINLINE int ObDASLocationRouter::get_vt_tablet_loc(uint64_t table_id, - const ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc) -{ - int ret = OB_SUCCESS; - VirtualSvrPair *final_pair = nullptr; - FOREACH(tmp_node, virtual_server_list_) { - if (tmp_node->get_table_id() == table_id) { - final_pair = &(*tmp_node); - break; - } - } - if (OB_ISNULL(final_pair)) { - ret = OB_LOCATION_NOT_EXIST; - LOG_WARN("virtual table location not exists", K(table_id), K(virtual_server_list_)); - } else if (OB_FAIL(final_pair->get_server_by_tablet_id(tablet_id, tablet_loc.server_))) { - LOG_WARN("get server by tablet id failed", K(ret), K(tablet_id)); - } else { - tablet_loc.tablet_id_ = tablet_id; - tablet_loc.ls_id_ = ObLSID::VT_LS_ID; - } - return ret; -} - -OB_NOINLINE int ObDASLocationRouter::get_vt_ls_location(uint64_t table_id, - const ObTabletID &tablet_id, - ObLSLocation &location) -{ - int ret = OB_SUCCESS; - bool is_cache_hit = false; - VirtualSvrPair *server_pair = nullptr; - FOREACH(tmp_node, virtual_server_list_) { - if (tmp_node->get_table_id() == table_id) { - server_pair = &(*tmp_node); - break; - } - } - if (OB_ISNULL(server_pair)) { - ret = OB_LOCATION_NOT_EXIST; - LOG_WARN("not found virtual location", K(ret), K(tablet_id), K(virtual_server_list_)); - } else { - // mock ls location - int64_t now = ObTimeUtility::current_time(); - ObReplicaProperty mock_prop; - ObLSReplicaLocation ls_replica; - ObAddr server; - ObLSRestoreStatus restore_status(ObLSRestoreStatus::NONE); - if (OB_FAIL(location.init(GCONF.cluster_id, ObLSID(ObLSID::VT_LS_ID), now))) { - LOG_WARN("init location failed", KR(ret)); - } else if (OB_FAIL(server_pair->get_server_by_tablet_id(tablet_id, server))) { - LOG_WARN("get server by tablet id failed", K(ret)); - } else if (OB_FAIL(ls_replica.init(server, common::LEADER, - GCONF.mysql_port, REPLICA_TYPE_FULL, mock_prop, - restore_status, 1 /*proposal_id*/))) { - LOG_WARN("init ls replica failed", K(ret)); - } else if (OB_FAIL(location.add_replica_location(ls_replica))) { - LOG_WARN("add replica location failed", K(ret)); - } - } - return ret; -} - bool ObDASLocationRouter::is_refresh_location_error(int err_no) const { return is_master_changed_error(err_no) || @@ -1143,7 +909,7 @@ void ObDASLocationRouter::refresh_location_cache_by_errno(bool is_nonblock, int { NG_TRACE_TIMES(1, get_location_cache_begin); if (is_refresh_location_error(err_no)) { - // Refresh tablet ls mapping and ls locations according to err_no. + // Refresh tablet location metadata according to err_no. // // The timeout has been set inner the interface when renewing location synchronously. // It will use the timeout of ObTimeoutCtx or THIS_WORKER if it has been set. @@ -1162,40 +928,6 @@ void ObDASLocationRouter::force_refresh_location_cache(bool is_nonblock, int err succ_tablet_list_.clear(); } -int ObDASLocationRouter::block_renew_tablet_location(const ObTabletID &tablet_id, ObLSLocation &ls_loc) -{ - int ret = OB_SUCCESS; - const int64_t expire_renew_time = INT64_MAX; // means must renew location - bool is_cache_hit = false; - ObLSID ls_id; - int64_t query_timeout_ts = THIS_WORKER.get_timeout_ts(); - ObTimeoutCtx timeout_ctx; - timeout_ctx.set_timeout(GCONF.location_cache_refresh_sql_timeout); - //The maximum timeout period is location_cache_refresh_sql_timeout - if (timeout_ctx.get_abs_timeout() > query_timeout_ts && query_timeout_ts > 0) { - timeout_ctx.set_abs_timeout(query_timeout_ts); - } - //the timeout limit for "refresh location" is within 1s - THIS_WORKER.set_timeout_ts(timeout_ctx.get_abs_timeout()); - if (OB_FAIL(GCTX.location_service_->get(tablet_id, - expire_renew_time, - is_cache_hit, - ls_id))) { - LOG_WARN("fail to get ls id", K(ret)); - } else if (OB_FAIL(GCTX.location_service_->get(GCONF.cluster_id, - ls_id, - expire_renew_time, - is_cache_hit, - ls_loc))) { - LOG_WARN("failed to get location", K(ls_id), K(ret)); - } else { - LOG_INFO("LOCATION: block refresh table cache succ", K(tablet_id), K(ls_loc)); - } - //recover query timeout ts - THIS_WORKER.set_timeout_ts(query_timeout_ts); - return ret; -} - void ObDASLocationRouter::set_retry_info(const ObQueryRetryInfo* retry_info) { last_errno_ = retry_info->get_last_query_retry_err(); diff --git a/src/sql/das/ob_das_location_router.h b/src/sql/das/ob_das_location_router.h index 03b7b659d..de838e19b 100644 --- a/src/sql/das/ob_das_location_router.h +++ b/src/sql/das/ob_das_location_router.h @@ -28,12 +28,6 @@ namespace common class ObTabletID; class ObNewRow; } // namespace common -namespace share -{ -class ObLSID; -class ObLSLocation; -class ObLSReplicaLocation; -} // namespace share namespace sql { struct ObDASTableLocMeta; @@ -362,10 +356,6 @@ class ObDASLocationRouter public: ObDASLocationRouter(common::ObIAllocator &allocator); ~ObDASLocationRouter(); - int nonblock_get(const ObDASTableLocMeta &loc_meta, - const common::ObTabletID &tablet_id, - share::ObLSLocation &location); - int nonblock_get_candi_tablet_locations(const ObDASTableLocMeta &loc_meta, const common::ObIArray &tablet_ids, const common::ObIArray &partition_ids, @@ -377,14 +367,8 @@ class ObDASLocationRouter ObDASTabletLoc &tablet_loc); int nonblock_get_leader(const ObTabletID &tablet_id, ObDASTabletLoc &tablet_loc); - int get_leader(const common::ObTabletID &tablet_id, - ObAddr &leader_addr, - int64_t expire_renew_time); - int get_full_ls_replica_loc(const ObDASTabletLoc &tablet_loc, - share::ObLSReplicaLocation &replica_loc); void refresh_location_cache_by_errno(bool is_nonblock, int err_no); void force_refresh_location_cache(bool is_nonblock, int err_no); - int block_renew_tablet_location(const common::ObTabletID &tablet_id, share::ObLSLocation &ls_loc); int save_touched_tablet_id(const common::ObTabletID &tablet_id) { return all_tablet_list_.push_back(tablet_id); } void set_last_errno(int err_no) { last_errno_ = err_no; } int get_last_errno() const { return last_errno_; } @@ -412,15 +396,6 @@ class ObDASLocationRouter TO_STRING_KV(K(all_tablet_list_)); private: int get_vt_svr_pair(uint64_t vt_id, const VirtualSvrPair *&vt_svr_pair); - int get_vt_tablet_loc(uint64_t table_id, - const common::ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc); - int get_vt_ls_location(uint64_t table_id, - const common::ObTabletID &tablet_id, - share::ObLSLocation &location); - int nonblock_get_readable_replica(const common::ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc, - const ObRoutePolicyType route_policy); private: int last_errno_; int cur_errno_; diff --git a/src/sql/das/ob_das_lock_op.cpp b/src/sql/das/ob_das_lock_op.cpp index a153b8b5e..7a771118a 100644 --- a/src/sql/das/ob_das_lock_op.cpp +++ b/src/sql/das/ob_das_lock_op.cpp @@ -59,14 +59,13 @@ int ObDASLockOp::open_op() ObAccessService *as = share::g_mp->access_service(); storage::ObStoreCtxGuard store_ctx_guard; - if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_, - lock_rtdef_->timeout_ts_, + if (OB_FAIL(as->get_write_store_ctx_guard(lock_rtdef_->timeout_ts_, *trans_desc_, *snapshot_, write_branch_id_, dml_param.write_flag_, store_ctx_guard))) { - LOG_WARN("fail to get_write_access_tx_ctx_guard", K(ret), K(ls_id_)); + LOG_WARN("fail to get_write_access_tx_ctx_guard", K(ret)); } else if (OB_FAIL(ObDMLService::init_dml_param( *lock_ctdef_, *lock_rtdef_, @@ -75,10 +74,9 @@ int ObDASLockOp::open_op() op_alloc_, store_ctx_guard, dml_param, - das_gts_opt_info_.use_specify_snapshot_))) { + das_snapshot_opt_info_.use_specify_snapshot_))) { LOG_WARN("init dml param failed", K(ret)); - } else if (OB_FAIL(as->lock_rows(ls_id_, - tablet_id_, + } else if (OB_FAIL(as->lock_rows(tablet_id_, *trans_desc_, dml_param, lock_rtdef_->for_upd_wait_time_, diff --git a/src/sql/das/ob_das_parallel_handler.h b/src/sql/das/ob_das_parallel_handler.h index 3dd129d9a..1153b07c8 100644 --- a/src/sql/das/ob_das_parallel_handler.h +++ b/src/sql/das/ob_das_parallel_handler.h @@ -15,6 +15,7 @@ */ #ifndef OBDEV_SRC_SQL_DAS_OB_DAS_PARALLEL_HANDLER_H_ #define OBDEV_SRC_SQL_DAS_OB_DAS_PARALLEL_HANDLER_H_ +#include "observer/ob_srv_task.h" #include "sql/engine/dml/ob_dml_ctx_define.h" namespace oceanbase { diff --git a/src/sql/das/ob_das_ref.cpp b/src/sql/das/ob_das_ref.cpp index a542f51bf..805c51de2 100644 --- a/src/sql/das/ob_das_ref.cpp +++ b/src/sql/das/ob_das_ref.cpp @@ -531,7 +531,7 @@ int ObDASRef::close_all_task() bool merge_trans_result_fail = (ret != OB_SUCCESS); // any fail during merge trans_result, // need set trans_result incomplete, in order to - // indicate some transaction participants info unknown + // indicate transaction write state info unknown if (merge_trans_result_fail && OB_NOT_NULL(session)) { LOG_WARN("close all task fail, set trans_result to incomplete", K(ret)); session->get_trans_result().set_incomplete(); @@ -573,10 +573,9 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc, task_op->set_task_id(task_id); task_op->in_stmt_retry_ = session->get_is_in_retry(); task_op->set_tablet_id(tablet_loc->tablet_id_); - task_op->set_ls_id(tablet_loc->ls_id_); task_op->set_tablet_loc(tablet_loc); - if (is_do_gts_opt() && OB_FAIL(task_op->init_das_gts_opt_info(session->get_tx_isolation()))) { - LOG_WARN("fail to init gts opt info", K(ret), K(session->get_tx_isolation())); + if (is_snapshot_opt_enabled() && OB_FAIL(task_op->init_das_snapshot_opt_info(session->get_tx_isolation()))) { + LOG_WARN("fail to init snapshot opt info", K(ret), K(session->get_tx_isolation())); } else if (OB_FAIL(add_aggregated_task(task_op, op_type))) { LOG_WARN("failed to add aggregated task", K(ret)); } @@ -661,35 +660,6 @@ int ObDASRef::add_aggregated_task(ObIDASTaskOp *das_task, ObDASOpType op_type) return ret; } -bool ObDASRef::check_tasks_same_ls_and_is_local(share::ObLSID &ls_id) -{ - ObIDASTaskOp *first_das_op = nullptr; - bool is_all_same = true; - DASTaskIter task_iter = begin_task_iter(); - const common::ObAddr &ctrl_addr = share::g_mp->data_access_service()->get_ctrl_addr(); - if (!has_task()) { - is_all_same = false; - } - while (is_all_same && !task_iter.is_end()) { - ObIDASTaskOp *das_op = *task_iter; - if (OB_ISNULL(first_das_op)) { - first_das_op = das_op; - if (first_das_op->get_tablet_loc()->server_ != ctrl_addr) { - is_all_same = false; - } - } else if (first_das_op->get_ls_id() != das_op->get_ls_id() || - first_das_op->get_tablet_loc()->server_ != das_op->get_tablet_loc()->server_) { - is_all_same = false; - } - ++task_iter; - } - - if (is_all_same) { - ls_id = first_das_op->get_ls_id(); - } - return is_all_same; -} - void ObDASRef::reset() { das_factory_.cleanup(); diff --git a/src/sql/das/ob_das_ref.h b/src/sql/das/ob_das_ref.h index 562712129..d56646049 100644 --- a/src/sql/das/ob_das_ref.h +++ b/src/sql/das/ob_das_ref.h @@ -232,7 +232,6 @@ class ObDASRef explicit ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx); ~ObDASRef() { reset(); } - bool check_tasks_same_ls_and_is_local(share::ObLSID &ls_id); DASOpResultIter begin_result_iter(); DASTaskIter begin_task_iter() { return batched_tasks_.begin(); } ObDASTaskFactory &get_das_factory() { return das_factory_; } @@ -266,8 +265,8 @@ class ObDASRef int retry_all_fail_tasks(common::ObIArray &failed_tasks); int close_all_task(); bool is_all_local_task() const; - bool is_do_gts_opt() { return do_gts_opt_; } - void set_do_gts_opt(bool v) { do_gts_opt_ = v; } + bool is_snapshot_opt_enabled() { return use_snapshot_opt_; } + void set_use_snapshot_opt(bool v) { use_snapshot_opt_ = v; } void set_execute_directly(bool v) { execute_directly_ = v; } bool is_execute_directly() const { return execute_directly_; } common::ObIAllocator &get_das_alloc() { return das_alloc_; } @@ -321,7 +320,7 @@ class ObDASRef struct { // FARM COMPAT WHITELIST uint64_t execute_directly_ : 1; uint64_t enable_rich_format_ : 1; - uint64_t do_gts_opt_ : 1; + uint64_t use_snapshot_opt_ : 1; uint64_t reserved_ : 62; }; }; @@ -355,4 +354,3 @@ OB_INLINE int ObDASRef::prepare_das_task(const ObDASTabletLoc *tablet_loc, DASOp } // namespace sql } // namespace oceanbase #endif /* OBDEV_SRC_SQL_DAS_OB_DAS_REF_H_ */ - diff --git a/src/sql/das/ob_das_retry_ctrl.cpp b/src/sql/das/ob_das_retry_ctrl.cpp index 8bf2ad9d5..c184b669a 100644 --- a/src/sql/das/ob_das_retry_ctrl.cpp +++ b/src/sql/das/ob_das_retry_ctrl.cpp @@ -29,7 +29,8 @@ namespace sql { * DAS cannot unconditionally retry for the error of tablet_location or ls_location, like -4725, -4721, * and needs to determine whether the real cause of the error is due to DDL operations or stale location cache. * 1. When the table, partition or tenant was dropped, which is caused by DDL, das task cannot be retried. - * 2. When tablet location cache is stale, tablet location cache should be updated and das task needs to be retried. + * 2. When tablet location cache is stale after metadata changes, tablet location cache should + * be updated and das task needs to be retried. * **/ void ObDASRetryCtrl::tablet_location_retry_proc(ObDASRef &das_ref, diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 8db23bebe..de5dfeb6c 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -319,7 +319,6 @@ int ObDASScanOp::init_scan_param() if (scan_rtdef_->is_for_foreign_check_) { scan_param_.trans_desc_ = trans_desc_; } - scan_param_.ls_id_ = ls_id_; scan_param_.tablet_id_ = tablet_id_; if (scan_rtdef_->sample_info_ != nullptr) { scan_param_.sample_info_ = *scan_rtdef_->sample_info_; @@ -745,10 +744,8 @@ int ObDASScanOp::rescan() int &ret = errcode_; reset_access_datums_ptr(); scan_param_.tablet_id_ = tablet_id_; - scan_param_.ls_id_ = ls_id_; LOG_DEBUG("begin to das table rescan", - "ls_id", scan_param_.ls_id_, "tablet_id", scan_param_.tablet_id_, "scan_range", scan_param_.key_ranges_, "range_pos", scan_param_.range_array_pos_); @@ -800,14 +797,13 @@ int ObDASScanOp::reuse_iter() case ITER_TREE_LOCAL_LOOKUP: { ObDASLocalLookupIter *lookup_iter = static_cast(result_); lookup_iter->set_tablet_id(tablet_ids_.lookup_tablet_id_); - lookup_iter->set_ls_id(ls_id_); break; } case ITER_TREE_MATCH: case ITER_TREE_TEXT_RETRIEVAL: { ObDASIter *result_iter = static_cast(result_); if (OB_FAIL(ObDASIterUtils::set_text_retrieval_related_ids( - attach_ctdef_, attach_rtdef_, tablet_ids_, ls_id_, result_iter))) { + attach_ctdef_, attach_rtdef_, tablet_ids_, result_iter))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } break; @@ -815,7 +811,7 @@ int ObDASScanOp::reuse_iter() case ITER_TREE_INDEX_MERGE: { ObDASIter *result_iter = static_cast(result_); if (OB_FAIL(ObDASIterUtils::set_index_merge_related_ids( - attach_ctdef_, attach_rtdef_, tablet_ids_, ls_id_, result_iter))) { + attach_ctdef_, attach_rtdef_, tablet_ids_, result_iter))) { LOG_WARN("failed to set index merge related ids", K(ret)); } break; @@ -823,7 +819,7 @@ int ObDASScanOp::reuse_iter() case ITER_TREE_FUNC_LOOKUP: { ObDASIter *result_iter = static_cast(result_); if (OB_FAIL(ObDASIterUtils::set_func_lookup_iter_related_ids( - attach_ctdef_, attach_rtdef_, tablet_ids_, ls_id_, result_iter))) { + attach_ctdef_, attach_rtdef_, tablet_ids_, result_iter))) { LOG_WARN("failed to set text retrieval related ids", K(ret)); } break; @@ -832,20 +828,18 @@ int ObDASScanOp::reuse_iter() if (OB_NOT_NULL(get_lookup_ctdef())) { ObDASLocalLookupIter *lookup_iter = static_cast(result_); lookup_iter->set_tablet_id(tablet_ids_.lookup_tablet_id_); - lookup_iter->set_ls_id(ls_id_); } break; } case ITER_TREE_GIS_LOOKUP: { ObDASLocalLookupIter *lookup_iter = static_cast(result_); lookup_iter->set_tablet_id(tablet_ids_.lookup_tablet_id_); - lookup_iter->set_ls_id(ls_id_); break; } case ITER_TREE_VEC_LOOKUP: { ObDASIter *result_iter = static_cast(result_); if (OB_FAIL(ObDASIterUtils::set_vec_lookup_related_ids( - attach_ctdef_, attach_rtdef_, tablet_ids_, ls_id_, result_iter))) { + attach_ctdef_, attach_rtdef_, tablet_ids_, result_iter))) { LOG_WARN("failed to set vector index related ids", K(ret)); } break; @@ -857,7 +851,7 @@ int ObDASScanOp::reuse_iter() } if (FAILEDx(result->get_domain_id_merge_iter(domain_id_merge_iter))) { LOG_WARN("fail to get domain id merge iter", K(ret)); - } else if (OB_NOT_NULL(domain_id_merge_iter) && OB_FAIL(domain_id_merge_iter->set_domain_id_merge_related_ids(tablet_ids_, ls_id_))) { + } else if (OB_NOT_NULL(domain_id_merge_iter) && OB_FAIL(domain_id_merge_iter->set_domain_id_merge_related_ids(tablet_ids_))) { LOG_WARN("fail to set domain id merge related ids", K(ret)); } else if (OB_FAIL(result->reuse())) { LOG_WARN("failed to reuse das iter tree", K(ret)); @@ -1794,7 +1788,6 @@ int ObLocalIndexLookupOp::do_index_lookup() const ObTabletID &storage_tablet_id = scan_param_.tablet_id_; scan_param_.need_switch_param_ = (storage_tablet_id.is_valid() && storage_tablet_id != tablet_id_ ? true : false); scan_param_.tablet_id_ = tablet_id_; - scan_param_.ls_id_ = ls_id_; if (OB_FAIL(reuse_iter())) { LOG_WARN("failed to reuse iter", K(ret)); } else if (OB_FAIL(tsc_service.table_rescan(scan_param_, storage_iter))) { @@ -1853,7 +1846,7 @@ int ObLocalIndexLookupOp::check_lookup_row_cnt() "data_table_tablet_id", tablet_id_ , KPC_(snapshot), KPC_(tx_desc)); - concurrency_control::ObDataValidationService::set_delay_resource_recycle(ls_id_); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(); (void)print_trans_info_and_key_range_(); } } @@ -1905,7 +1898,6 @@ OB_INLINE int ObLocalIndexLookupOp::init_scan_param() scan_param_.pd_storage_flag_ = lookup_ctdef_->pd_expr_spec_.pd_storage_flag_.pd_flag_; scan_param_.table_scan_opt_ = lookup_ctdef_->table_scan_opt_; scan_param_.fb_snapshot_ = lookup_rtdef_->fb_snapshot_; - scan_param_.ls_id_ = ls_id_; scan_param_.tablet_id_ = tablet_id_; scan_param_.main_table_scan_stat_.tsc_monitor_info_ = lookup_rtdef_->tsc_monitor_info_; if (lookup_rtdef_->is_for_foreign_check_) { diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index ab555d7f5..b6d9517ed 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -489,7 +489,6 @@ class ObLocalIndexLookupOp : public common::ObNewRowIterator, public ObIndexLook rowkey_iter_(nullptr), lookup_iter_(), tablet_id_(), - ls_id_(), scan_param_(), lookup_memctx_(), status_(0), @@ -522,7 +521,6 @@ class ObLocalIndexLookupOp : public common::ObNewRowIterator, public ObIndexLook bool is_group_scan() const { return is_group_scan_; } void set_tablet_id(const common::ObTabletID &tablet_id) { tablet_id_ = tablet_id; } void set_index_tablet_id(const common::ObTabletID &tablet_id) { index_tablet_id_ = tablet_id; } - void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } void set_rowkey_iter(common::ObNewRowIterator *rowkey_iter) {rowkey_iter_ = rowkey_iter;} common::ObNewRowIterator *get_rowkey_iter() { return rowkey_iter_; } int reuse_iter(); @@ -532,7 +530,6 @@ class ObLocalIndexLookupOp : public common::ObNewRowIterator, public ObIndexLook KPC_(tx_desc), KPC_(snapshot), K_(tablet_id), - K_(ls_id), K_(state), K_(index_end)); common::ObITabletScan &get_tsc_service(); @@ -552,7 +549,6 @@ class ObLocalIndexLookupOp : public common::ObNewRowIterator, public ObIndexLook common::ObNewRowIterator *rowkey_iter_; common::ObNewRowIterator *lookup_iter_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; storage::ObTableScanParam scan_param_; ObSEArray trans_info_array_; diff --git a/src/sql/das/ob_das_simple_op.cpp b/src/sql/das/ob_das_simple_op.cpp index 217a74a85..fdbccce89 100644 --- a/src/sql/das/ob_das_simple_op.cpp +++ b/src/sql/das/ob_das_simple_op.cpp @@ -60,14 +60,13 @@ int ObDASSplitRangesOp::open_op() { int ret = OB_SUCCESS; ObAccessService *access_service = share::g_mp->access_service(); - if (OB_FAIL(access_service->split_multi_ranges(ls_id_, - tablet_id_, + if (OB_FAIL(access_service->split_multi_ranges(tablet_id_, timeout_us_, ranges_, expected_task_count_, op_alloc_, multi_range_split_array_))) { - LOG_WARN("failed to split multi ranges", K(ret), K_(ls_id), K_(tablet_id)); + LOG_WARN("failed to split multi ranges", K(ret), K_(tablet_id)); } return ret; } @@ -210,12 +209,11 @@ int ObDASRangesCostOp::open_op() { int ret = OB_SUCCESS; ObAccessService *access_service = share::g_mp->access_service(); - if (OB_FAIL(access_service->get_multi_ranges_cost(ls_id_, - tablet_id_, + if (OB_FAIL(access_service->get_multi_ranges_cost(tablet_id_, timeout_us_, ranges_, total_size_))) { - LOG_WARN("failed to get multi ranges cost", K(ret), K_(ls_id), K_(tablet_id)); + LOG_WARN("failed to get multi ranges cost", K(ret), K_(tablet_id)); } return ret; } diff --git a/src/sql/das/ob_das_task.cpp b/src/sql/das/ob_das_task.cpp index 3ec0c3924..df89aa24f 100644 --- a/src/sql/das/ob_das_task.cpp +++ b/src/sql/das/ob_das_task.cpp @@ -109,12 +109,12 @@ int ObIDASTaskOp::swizzling_remote_task(ObDASRemoteInfo *remote_info) int ret = OB_SUCCESS; if (remote_info != nullptr) { snapshot_ = &remote_info->snapshot_; - if (das_gts_opt_info_.use_specify_snapshot_) { - if (OB_ISNULL(das_gts_opt_info_.get_specify_snapshot())) { + if (das_snapshot_opt_info_.use_specify_snapshot_) { + if (OB_ISNULL(das_snapshot_opt_info_.get_specify_snapshot())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr of specify_snapshot", K(ret)); } else { - snapshot_ = das_gts_opt_info_.get_specify_snapshot(); + snapshot_ = das_snapshot_opt_info_.get_specify_snapshot(); } } } @@ -172,13 +172,13 @@ int ObIDASTaskOp::end_das_task() return ret; } -int ObIDASTaskOp::init_das_gts_opt_info(transaction::ObTxIsolationLevel isolation_level) +int ObIDASTaskOp::init_das_snapshot_opt_info(transaction::ObTxIsolationLevel isolation_level) { int ret = OB_SUCCESS; - if (OB_FAIL(get_das_gts_opt_info().init(isolation_level))) { - LOG_WARN("fail to init das gts opt", K(ret), K(isolation_level)); + if (OB_FAIL(get_das_snapshot_opt_info().init(isolation_level))) { + LOG_WARN("fail to init das snapshot opt", K(ret), K(isolation_level)); } else { - snapshot_ = get_das_gts_opt_info().get_specify_snapshot(); + snapshot_ = get_das_snapshot_opt_info().get_specify_snapshot(); } return ret; } @@ -187,16 +187,15 @@ OB_SERIALIZE_MEMBER(ObIDASTaskOp, task_id_, task_flag_, tablet_id_, - ls_id_, related_ctdefs_, related_rtdefs_, related_tablet_ids_, attach_ctdef_, attach_rtdef_, - das_gts_opt_info_, + das_snapshot_opt_info_, plan_line_id_); -OB_DEF_SERIALIZE(ObDASGTSOptInfo) +OB_DEF_SERIALIZE(ObDASSnapshotOptInfo) { int ret = OB_SUCCESS; bool serialize_specify_snapshot = specify_snapshot_ == nullptr ? false : true; @@ -210,7 +209,7 @@ OB_DEF_SERIALIZE(ObDASGTSOptInfo) return ret; } -OB_DEF_DESERIALIZE(ObDASGTSOptInfo) +OB_DEF_DESERIALIZE(ObDASSnapshotOptInfo) { int ret = OB_SUCCESS; bool serialize_specify_snapshot = false; @@ -220,7 +219,7 @@ OB_DEF_DESERIALIZE(ObDASGTSOptInfo) serialize_specify_snapshot); if (serialize_specify_snapshot) { if (OB_FAIL(init(isolation_level_))) { - LOG_WARN("fail to init gts_opt_info", K(ret)); + LOG_WARN("fail to init snapshot opt info", K(ret)); } else { OB_UNIS_DECODE(*specify_snapshot_); } @@ -228,7 +227,7 @@ OB_DEF_DESERIALIZE(ObDASGTSOptInfo) return ret; } -OB_DEF_SERIALIZE_SIZE(ObDASGTSOptInfo) +OB_DEF_SERIALIZE_SIZE(ObDASSnapshotOptInfo) { int64_t len = 0; bool serialize_specify_snapshot = specify_snapshot_ == nullptr ? false : true; @@ -242,7 +241,7 @@ OB_DEF_SERIALIZE_SIZE(ObDASGTSOptInfo) return len; } -int ObDASGTSOptInfo::init(transaction::ObTxIsolationLevel isolation_level) +int ObDASSnapshotOptInfo::init(transaction::ObTxIsolationLevel isolation_level) { int ret = OB_SUCCESS; void *buf = nullptr; diff --git a/src/sql/das/ob_das_task.h b/src/sql/das/ob_das_task.h index 63d3ddacd..43fa6b723 100644 --- a/src/sql/das/ob_das_task.h +++ b/src/sql/das/ob_das_task.h @@ -45,11 +45,11 @@ struct ObDASTCBInterruptInfo; typedef ObDLinkNode DasTaskNode; typedef ObDList DasTaskLinkedList; -struct ObDASGTSOptInfo +struct ObDASSnapshotOptInfo { OB_UNIS_VERSION(1); public: - ObDASGTSOptInfo(common::ObIAllocator &alloc) + ObDASSnapshotOptInfo(common::ObIAllocator &alloc) : alloc_(alloc), use_specify_snapshot_(false), isolation_level_(), @@ -58,7 +58,7 @@ struct ObDASGTSOptInfo { } - ~ObDASGTSOptInfo() + ~ObDASSnapshotOptInfo() { if (specify_snapshot_ != nullptr) { specify_snapshot_->~ObTxReadSnapshot(); @@ -174,7 +174,7 @@ class ObIDASTaskOp op_result_(nullptr), attach_ctdef_(nullptr), attach_rtdef_(nullptr), - das_gts_opt_info_(op_alloc), + das_snapshot_opt_info_(op_alloc), plan_line_id_(0), das_task_start_timestamp_(0) { @@ -190,8 +190,6 @@ class ObIDASTaskOp const common::ObTabletID &get_tablet_id() const { return tablet_id_; } void set_task_id(const int64_t task_id) { task_id_ = task_id; } int64_t get_task_id() const { return task_id_; } - void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } - const share::ObLSID &get_ls_id() const { return ls_id_; } void set_tablet_loc(const ObDASTabletLoc *tablet_loc) { tablet_loc_ = tablet_loc; } // tablet_loc_ will not be serialized, therefore it cannot be accessed during the execution phase // of DASTaskOp. It can only be touched through das_ref and data_access_service layer. @@ -240,7 +238,6 @@ class ObIDASTaskOp KPC_(trans_desc), KPC_(snapshot), K_(tablet_id), - K_(ls_id), KPC_(tablet_loc), K_(related_ctdefs), K_(related_rtdefs), @@ -285,8 +282,8 @@ class ObIDASTaskOp void set_inner_rescan(bool flag) { inner_rescan_ = flag; } void set_write_buff_full(bool v) { write_buff_full_ = v; } bool is_write_buff_full() { return write_buff_full_; } - ObDASGTSOptInfo &get_das_gts_opt_info() { return das_gts_opt_info_; } - int init_das_gts_opt_info(transaction::ObTxIsolationLevel isolation_level); + ObDASSnapshotOptInfo &get_das_snapshot_opt_info() { return das_snapshot_opt_info_; } + int init_das_snapshot_opt_info(transaction::ObTxIsolationLevel isolation_level); protected: int start_das_task(); @@ -322,7 +319,6 @@ class ObIDASTaskOp }; int16_t write_branch_id_; // branch id for parallel write, required for partially rollback common::ObTabletID tablet_id_; - share::ObLSID ls_id_; // tablet_loc_ will not be serialized, therefore it cannot be accessed during the execution phase // of DASTaskOp. It can only be touched through das_ref and data_access_service layer. const ObDASTabletLoc *tablet_loc_; @@ -349,7 +345,7 @@ class ObIDASTaskOp //rowkey merging for index merge operations, and so on. const ObDASBaseCtDef *attach_ctdef_; ObDASBaseRtDef *attach_rtdef_; - ObDASGTSOptInfo das_gts_opt_info_; + ObDASSnapshotOptInfo das_snapshot_opt_info_; int64_t plan_line_id_; //plan operator id public: int64_t das_task_start_timestamp_; diff --git a/src/sql/das/ob_das_update_op.cpp b/src/sql/das/ob_das_update_op.cpp index 3f1e8c6f7..35594cb72 100644 --- a/src/sql/das/ob_das_update_op.cpp +++ b/src/sql/das/ob_das_update_op.cpp @@ -395,8 +395,7 @@ int ObDASUpdIterator::get_next_domain_index_rows(ObDatumRow *&rows, int64_t &row } template <> -int ObDASIndexDMLAdaptor::write_rows(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDASIndexDMLAdaptor::write_rows(const ObTabletID &tablet_id, const CtDefType &ctdef, RtDefType &rtdef, ObDASUpdIterator &iter, @@ -406,42 +405,27 @@ int ObDASIndexDMLAdaptor::write_rows(cons ObAccessService *as = share::g_mp->access_service(); if (OB_UNLIKELY((ctdef.table_param_.get_data_table().is_vector_delta_buffer() || ctdef.table_param_.get_data_table().is_hybrid_vector_index_log()) && - !ctdef.is_access_mlog_as_master_table_)) { + !ctdef.is_access_main_table_)) { // for vector delta buffer/hybrid log table, only do insert when DML with main table - if (OB_FAIL(as->insert_rows(ls_id, tablet_id, *tx_desc_, dml_param_, + if (OB_FAIL(as->insert_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("insert rows to access service failed", K(ret), K(tablet_id)); } } } else if (OB_UNLIKELY(ctdef.table_param_.get_data_table().is_domain_index())) { - if (OB_FAIL(as->delete_rows(ls_id, tablet_id, *tx_desc_, dml_param_, + if (OB_FAIL(as->delete_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("delete rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("delete rows to access service failed", K(ret), K(tablet_id)); } - } else if (OB_FAIL(as->insert_rows(ls_id, tablet_id, *tx_desc_, dml_param_, + } else if (OB_FAIL(as->insert_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, &iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("insert rows to access service failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("insert rows to access service failed", K(ret), K(tablet_id)); } } - } else if (ctdef.table_param_.get_data_table().is_mlog_table() - && !ctdef.is_access_mlog_as_master_table_) { - ObDASMLogDMLIterator mlog_iter(ls_id, tablet_id, dml_param_, &iter, DAS_OP_TABLE_UPDATE); - if (OB_FAIL(as->insert_rows(ls_id, - tablet_id, - *tx_desc_, - dml_param_, - ctdef.column_ids_, - &mlog_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("delete rows to access service failed", K(ret)); - } - } - } else if (OB_FAIL(as->update_rows(ls_id, - tablet_id, + } else if (OB_FAIL(as->update_rows(tablet_id, *tx_desc_, dml_param_, ctdef.column_ids_, @@ -485,14 +469,13 @@ int ObDASUpdateOp::open_op() upd_adaptor.related_ctdefs_ = &related_ctdefs_; upd_adaptor.related_rtdefs_ = &related_rtdefs_; upd_adaptor.tablet_id_ = tablet_id_; - upd_adaptor.ls_id_ = ls_id_; upd_adaptor.related_tablet_ids_ = &related_tablet_ids_; - upd_adaptor.is_do_gts_opt_ = das_gts_opt_info_.use_specify_snapshot_; + upd_adaptor.use_snapshot_opt_ = das_snapshot_opt_info_.use_specify_snapshot_; upd_adaptor.das_allocator_ = &op_alloc_; upd_adaptor.ft_doc_word_infos_ = &doc_word_infos; - if (OB_FAIL(ObDASDomainUtils::build_ft_doc_word_infos(ls_id_, trans_desc_, snapshot_, related_ctdefs_, related_tablet_ids_, + if (OB_FAIL(ObDASDomainUtils::build_ft_doc_word_infos(trans_desc_, snapshot_, related_ctdefs_, related_tablet_ids_, upd_ctdef_->is_main_table_in_fts_ddl_, doc_word_infos))) { - LOG_WARN("fail to build fulltext doc word infos", K(ret), K(ls_id_), KPC(snapshot_), K(related_ctdefs_), + LOG_WARN("fail to build fulltext doc word infos", K(ret), KPC(snapshot_), K(related_ctdefs_), K(related_tablet_ids_)); } else if (OB_FAIL(upd_adaptor.write_tablet(upd_iter, affected_rows))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { diff --git a/src/sql/das/ob_das_utils.h b/src/sql/das/ob_das_utils.h index ed18a9592..426e178ff 100644 --- a/src/sql/das/ob_das_utils.h +++ b/src/sql/das/ob_das_utils.h @@ -17,7 +17,6 @@ #ifndef DEV_SRC_SQL_DAS_OB_DAS_UTILS_H_ #define DEV_SRC_SQL_DAS_OB_DAS_UTILS_H_ #include "share/ob_define.h" -#include "share/ob_ls_id.h" #include "share/location_cache/ob_location_struct.h" #include "common/ob_tablet_id.h" #include "sql/dtl/ob_dtl_task.h" @@ -132,12 +131,6 @@ class ObDASUtils } return ret; } - static int generate_mlog_row(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const storage::ObDMLBaseParam &dml_param, - blocksstable::ObDatumRow &row, - ObDASOpType op_type, - bool is_old_row); }; } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_data_access_service.cpp b/src/sql/das/ob_data_access_service.cpp index 1f34e0f7e..141eff7ec 100644 --- a/src/sql/das/ob_data_access_service.cpp +++ b/src/sql/das/ob_data_access_service.cpp @@ -38,7 +38,7 @@ namespace sql ObDataAccessService::ObDataAccessService() : ctrl_addr_(), - id_cache_(), + next_das_id_(1), task_result_mgr_(), das_concurrency_limit_(INT32_MAX) { @@ -47,14 +47,8 @@ ObDataAccessService::ObDataAccessService() int ObDataAccessService::mtl_init(ObDataAccessService *&das) { - int ret = OB_SUCCESS; const ObAddr &self = GCTX.self_addr(); - if (OB_FAIL(das->id_cache_.init(self))) { - LOG_ERROR("init das id service failed", K(ret)); - } else if (OB_FAIL(das->init(self))) { - LOG_ERROR("init data access service failed", K(ret)); - } - return ret; + return das->init(self); } void ObDataAccessService::mtl_destroy(ObDataAccessService *&das) @@ -108,34 +102,9 @@ int ObDataAccessService::execute_das_task( int ObDataAccessService::get_das_task_id(int64_t &das_id) { - int ret = OB_SUCCESS; - const int MAX_RETRY_TIMES = 50; - int64_t tmp_das_id = 0; - bool force_renew = false; - int64_t total_sleep_time = 0; - int64_t cur_sleep_time = 1000; // 1ms - int64_t max_sleep_time = ObDASIDCache::OB_DAS_ID_RPC_TIMEOUT_MIN * 2; // 200ms - do { - if (OB_SUCC(id_cache_.get_das_id(tmp_das_id, force_renew))) { - } else if (OB_EAGAIN == ret) { - if (total_sleep_time >= max_sleep_time) { - // TODO chenxuan change error code - ret = OB_GTI_NOT_READY; - LOG_WARN("get das id not ready", K(ret), K(total_sleep_time), K(max_sleep_time)); - } else { - force_renew = true; - ob_usleep(cur_sleep_time); - total_sleep_time += cur_sleep_time; - cur_sleep_time = cur_sleep_time * 2; - } - } else { - LOG_WARN("get das id failed", K(ret)); - } - } while (OB_EAGAIN == ret); - if (OB_SUCC(ret)) { - das_id = tmp_das_id; - } - return ret; +; + das_id = ATOMIC_FAA(&next_das_id_, 1); + return OB_SUCCESS; } @@ -187,7 +156,6 @@ int ObDataAccessService::refresh_task_location_info(ObDASRef &das_ref, ObIDASTas *tablet_loc))) { LOG_WARN("get tablet location failed", K(ret), KPC(tablet_loc)); } else { - task_op.set_ls_id(tablet_loc->ls_id_); if (!task_op.is_local_task()) { int64_t task_id; if (OB_FAIL(share::g_mp->data_access_service()->get_das_task_id(task_id))) { diff --git a/src/sql/das/ob_data_access_service.h b/src/sql/das/ob_data_access_service.h index aa8980d7b..3b85d628c 100644 --- a/src/sql/das/ob_data_access_service.h +++ b/src/sql/das/ob_data_access_service.h @@ -16,8 +16,8 @@ #ifndef OBDEV_SRC_SQL_DAS_OB_DATA_ACCESS_SERVICE_H_ #define OBDEV_SRC_SQL_DAS_OB_DATA_ACCESS_SERVICE_H_ +#include "lib/atomic/ob_atomic.h" #include "share/ob_define.h" -#include "sql/das/ob_das_id_cache.h" #include "sql/das/ob_das_task_result.h" #include "sql/das/ob_das_ref.h" namespace oceanbase @@ -68,7 +68,7 @@ class ObDataAccessService ObDASBaseRtDef *attach_rtdef); private: common::ObAddr ctrl_addr_; - ObDASIDCache id_cache_; + int64_t next_das_id_ CACHE_ALIGNED; ObDASTaskResultMgr task_result_mgr_; int32_t das_concurrency_limit_; }; diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 69b92b166..312901110 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -56,13 +56,7 @@ int ObFreezeExecutor::execute(ObExecContext &ctx, ObFreezeStmt &stmt) } else { if (!stmt.is_major_freeze()) { ObRootMinorFreezeArg arg; - if (OB_FAIL(arg.server_list_.assign(stmt.get_server_list()))) { - LOG_WARN("failed to assign server_list", K(ret)); - } else { - arg.zone_ = stmt.get_zone(); - arg.tablet_id_ = stmt.get_tablet_id(); - arg.ls_id_ = stmt.get_ls_id(); - } + arg.tablet_id_ = stmt.get_tablet_id(); // access check: // not allow user_tenant to freeze other tenants if (OB_SUCC(ret)) { diff --git a/src/sql/engine/cmd/ob_load_data_impl.cpp b/src/sql/engine/cmd/ob_load_data_impl.cpp index 01e6d5448..6ad97d61b 100644 --- a/src/sql/engine/cmd/ob_load_data_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_impl.cpp @@ -234,41 +234,26 @@ int ObLoadDataBase::memory_wait_local(ObExecContext &ctx, } bool need_wait_freeze = true; - ObAddr leader_addr; - ObDASLocationRouter &loc_router = DAS_CTX(ctx).get_location_router(); while (OB_SUCC(ret) && need_wait_freeze) { ob_usleep(WAIT_INTERVAL_US); - leader_addr.reset(); res.reuse(); - char leader_ip_str[MAX_IP_ADDR_LENGTH]; - const int64_t retry_us = 200 * 1000; - //Try to use the results in the cache as much as possible, without forcing a cache refresh. - const int64_t expire_renew_time = 0; if (OB_FAIL(ObLoadDataUtils::check_session_status(*session))) { LOG_WARN("session is not valid during wait", K(ret)); - } else if (OB_FAIL(loc_router.get_leader(tablet_id, leader_addr, expire_renew_time))) { - LOG_WARN("failed to get location", K(ret)); - ob_usleep(retry_us); - } else { - LOG_DEBUG("get participants", K(tablet_id), K(leader_addr)); } if (OB_FAIL(ret)) { - } else if (!leader_addr.ip_to_string(leader_ip_str, sizeof(leader_ip_str))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("format leader ip failed", K(ret), K(leader_addr)); } else if (OB_FAIL(sql.assign_fmt(SERVER_TENANT_MEMORY_EXAMINE_SQL))) { - LOG_WARN("fail to append sql", K(ret), K(leader_addr)); + LOG_WARN("fail to append sql", K(ret)); } else if (OB_FAIL(sql_proxy_->read(res, sql.ptr()))) { LOG_WARN("fail to execute sql", K(ret), K(sql)); } else if (NULL == (result = res.get_result())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get sql result", K(ret)); } else if (OB_FAIL(result->next())) { - LOG_WARN("fail to get result, force renew location", K(ret), K(leader_addr)); + LOG_WARN("fail to get result", K(ret)); if (OB_ITER_END == ret) { ret = OB_SUCCESS; } @@ -277,24 +262,14 @@ int ObLoadDataBase::memory_wait_local(ObExecContext &ctx, //LOG_INFO("LOAD DATA is waiting for tenant memory available", //K(waited_seconds), K(total_wait_secs)); } - //if it is location exception, refresh location cache with block interface - //because load data can only local retry - loc_router.refresh_location_cache_by_errno(false, ret); } //print info if (OB_SUCC(ret)) { int64_t wait_secs = (ObTimeUtil::current_time() - start_wait_ts) / 1000000; total_wait_secs += wait_secs; - if (leader_addr != server_addr) { - LOG_INFO("LOAD DATA location change", - "old_addr", server_addr, - "new_addr", leader_addr); - server_addr = leader_addr; - is_leader_changed = true; - } else { - is_leader_changed = false; - } + server_addr = GCTX.self_addr(); + is_leader_changed = false; LOG_INFO("LOAD DATA is resumed", "waited_seconds", wait_secs, K(total_wait_secs)); @@ -2248,36 +2223,13 @@ void ObDataFragMgr::distory_datafrag(ObDataFrag *frag) { int ObPartDataFragMgr::update_part_location(ObExecContext &ctx) { int ret = OB_SUCCESS; - const int64_t retry_us = 200 * 1000; - const int64_t retry_timeout = - std::min(ObTimeUtil::current_time() + 30 * USECS_PER_SEC, // the RTO is 30s - ctx.get_my_session()->get_query_timeout_ts()); + UNUSED(ctx); if (OB_UNLIKELY(!tablet_id_.is_valid())) { ret = OB_NOT_INIT; LOG_WARN("invalid partition key", K(ret)); } else { - bool force_renew = false; - ObDASLocationRouter &loc_router = DAS_CTX(ctx).get_location_router(); - do { - const int64_t expire_renew_time = force_renew ? INT64_MAX : 0; - if (OB_FAIL(loc_router.get_leader(tablet_id_, leader_addr_, expire_renew_time))) { - if (is_location_service_renew_error(ret) && !force_renew) { - // retry one time - force_renew = true; - LOG_WARN("failed to get location and force renew", K(ret), K(tablet_id_)); - } else { - LOG_WARN("failed to get location", K(ret), K(tablet_id_)); - if (ObTimeUtil::current_time() + retry_us > retry_timeout) { - force_renew = false; - } else { - ob_usleep(retry_us); - } - } - } else { - LOG_DEBUG("get participants", K(tablet_id_), K(leader_addr_)); - } - } while (is_location_service_renew_error(ret) && force_renew); + leader_addr_ = GCTX.self_addr(); } return ret; diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index dbaa2ef6d..3ddf0e900 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -37,7 +37,6 @@ #include "sql/printer/ob_select_stmt_printer.h" #include "observer/ob_server_event_history_table_operator.h" -#include "storage/mview/cmd/ob_mview_executor_util.h" #include "storage/ob_partition_pre_split.h" namespace oceanbase @@ -535,7 +534,6 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) LOG_WARN("get first statement failed", K(ret)); } else if (table_schema.is_duplicate_table()) { - // TODO@jingyu_cr: make sure whether sys log stream have to be duplicated ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "create duplicate table under sys or meta tenant"); LOG_WARN("create dup table not supported", KR(ret), K(table_schema)); @@ -578,23 +576,6 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) LOG_WARN("fail to execute parallel ddl", KR(ret), K(create_table_arg), K(res)); } } - if (OB_SUCC(ret)) { - if (create_table_arg.schema_.is_materialized_view()) { - ObSQLSessionInfo *session_info = ctx.get_my_session(); - if (session_info == nullptr) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("session_info should not be nullptr", KR(ret)); - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(res.task_id_, false/*do not retry at executor*/, session_info, true))) { - if (storage::ObMViewExecutorUtil::is_mview_refresh_retry_ret_code(ret)) { - LOG_WARN("retry create mview", KR(ret), "task_id", res.task_id_); - ret = OB_EAGAIN; - } else { - LOG_WARN("fail to create mview", KR(ret), "task_id", res.task_id_); - } - } - } - } - } else { if (OB_FAIL(execute_ctas(ctx, stmt))){ // Processing of query-based table creation LOG_WARN("execute create table as select failed", KR(ret)); diff --git a/src/sql/engine/cmd/ob_variable_set_executor.cpp b/src/sql/engine/cmd/ob_variable_set_executor.cpp index dd2870eca..9defff2da 100644 --- a/src/sql/engine/cmd/ob_variable_set_executor.cpp +++ b/src/sql/engine/cmd/ob_variable_set_executor.cpp @@ -915,27 +915,14 @@ int ObVariableSetExecutor::process_session_autocommit_hook(ObExecContext &exec_c // in xa trans if (in_trans && my_session->associated_xa()) { const transaction::ObXATransID xid = my_session->get_xid(); - transaction::ObTxDesc *tx_desc = my_session->get_tx_desc(); - const transaction::ObGlobalTxType global_tx_type = tx_desc->get_global_tx_type(xid); // not allow to set autocommit to on if (false == orig_ac && 1 == autocommit) { ret = OB_TRANS_XA_ERR_COMMIT; LOG_WARN("not allow to set autocommit on in xa trans", K(ret), K(xid)); } else if (true == orig_ac && 1 == autocommit) { - if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { - // do nothing - } else { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("unexpected global trans type", K(ret), K(xid), K(global_tx_type)); - } + // do nothing } else { - // in xa trans - if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { - LOG_INFO("set autocommit off in xa trans", K(ret), K(xid)); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("unexpected global trans type", K(ret), K(xid), K(global_tx_type)); - } + LOG_INFO("set autocommit off in xa trans", K(ret), K(xid)); } } else if (false == orig_ac && true == in_trans && 1 == autocommit) { // set autocommit = 1 won't clear next scope transaction settings: diff --git a/src/sql/engine/dml/ob_conflict_checker.cpp b/src/sql/engine/dml/ob_conflict_checker.cpp index 039e05d78..78518500f 100644 --- a/src/sql/engine/dml/ob_conflict_checker.cpp +++ b/src/sql/engine/dml/ob_conflict_checker.cpp @@ -30,7 +30,6 @@ namespace sql int ObTabletSnapshotMaping::assign(const ObTabletSnapshotMaping& other) { int ret = OB_SUCCESS; - this->ls_id_ = other.ls_id_; this->tablet_id_ = other.tablet_id_; if (OB_FAIL(this->snapshot_.assign(other.snapshot_))) { LOG_WARN("assign snapshot fail", K(ret), K(other)); @@ -126,7 +125,7 @@ OB_DEF_SERIALIZE_SIZE(ObConflictCheckerCtdef) bool ObTabletSnapshotMaping::operator==(const ObTabletSnapshotMaping &other) const { - return ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_; + return tablet_id_ == other.tablet_id_; } @@ -243,7 +242,7 @@ int ObConflictChecker::create_conflict_map(int64_t replace_row_cnt) // Initialize map array, map creation of hash_bucket will be deferred int ObConflictChecker::init_conflict_checker(const ObExprFrameInfo *expr_frame_info, ObDASTableLoc *table_loc, - bool use_partition_gts_opt) + bool use_response_snapshot) { int ret = OB_SUCCESS; ObSQLSessionInfo *session = eval_ctx_.exec_ctx_.get_my_session(); @@ -258,7 +257,7 @@ int ObConflictChecker::init_conflict_checker(const ObExprFrameInfo *expr_frame_i // Here attention is needed das_ref_.set_execute_directly(!checker_ctdef_.use_dist_das_); das_ref_.set_mem_attr(mem_attr); - das_ref_.set_do_gts_opt(use_partition_gts_opt); + das_ref_.set_use_snapshot_opt(use_response_snapshot); } OZ(init_das_scan_rtdef()); return ret; @@ -1187,14 +1186,11 @@ int ObConflictChecker::collect_all_snapshot(transaction::ObTxReadSnapshot &snaps int ret = OB_SUCCESS; ObTabletSnapshotMaping tablet_snapshot_maping; tablet_snapshot_maping.tablet_id_ = tablet_loc->tablet_id_; - tablet_snapshot_maping.ls_id_ = tablet_loc->ls_id_; if (OB_ISNULL(tablet_loc)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); } else if (FALSE_IT(tablet_snapshot_maping.tablet_id_ = tablet_loc->tablet_id_)) { // do nothing - } else if (FALSE_IT(tablet_snapshot_maping.ls_id_ = tablet_loc->ls_id_)) { - // do nothing } else if (OB_FAIL(tablet_snapshot_maping.snapshot_.assign(snapshot))) { LOG_WARN("fail to assign snapshot", K(ret), K(snapshot)); } else if (OB_FAIL(has_exist_in_array(snapshot_maping_, tablet_snapshot_maping))) { @@ -1203,18 +1199,18 @@ int ObConflictChecker::collect_all_snapshot(transaction::ObTxReadSnapshot &snaps } else if (OB_FAIL(snapshot_maping_.push_back(tablet_snapshot_maping))) { LOG_WARN("fail to push back snapshot", K(ret), K(tablet_snapshot_maping)); } else { - LOG_TRACE("collect snaoshot after try_insert", K(tablet_snapshot_maping)); + LOG_TRACE("collect snapshot after try_insert", K(tablet_snapshot_maping)); } return ret; } -int ObConflictChecker::get_snapshot_by_ids(ObTabletID tablet_id, share::ObLSID ls_id, bool &founded, transaction::ObTxReadSnapshot &snapshot) +int ObConflictChecker::get_snapshot_by_tablet(ObTabletID tablet_id, bool &founded, transaction::ObTxReadSnapshot &snapshot) { int ret = OB_SUCCESS; founded = false; for (int64_t i = 0; OB_SUCC(ret) && !founded && i < snapshot_maping_.count(); i++) { ObTabletSnapshotMaping &snapshot_maping = snapshot_maping_.at(i); - if (snapshot_maping.tablet_id_ == tablet_id && snapshot_maping.ls_id_ == ls_id) { + if (snapshot_maping.tablet_id_ == tablet_id) { if (OB_FAIL(snapshot.assign(snapshot_maping.snapshot_))) { LOG_WARN("fail to assign snapshot", K(ret)); } else { @@ -1240,7 +1236,7 @@ int ObConflictChecker::post_all_das_scan_tasks() int ObConflictChecker::set_partition_snapshot_for_das_task(ObDASRef &das_ref) { int ret = OB_SUCCESS; - if (!das_ref.is_do_gts_opt()) { + if (!das_ref.is_snapshot_opt_enabled()) { // do nothing } else { DASTaskIter task_iter = das_ref.begin_task_iter(); @@ -1251,20 +1247,20 @@ int ObConflictChecker::set_partition_snapshot_for_das_task(ObDASRef &das_ref) if (OB_ISNULL(das_op)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); - } else if (OB_ISNULL(snapshot = das_op->get_das_gts_opt_info().get_specify_snapshot())) { + } else if (OB_ISNULL(snapshot = das_op->get_das_snapshot_opt_info().get_specify_snapshot())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), KPC(snapshot)); - } else if (OB_FAIL(get_snapshot_by_ids(das_op->get_tablet_id(), das_op->get_ls_id(), founded, *snapshot))) { - LOG_WARN("fail to get snapshot by ids", K(das_op->get_tablet_id()), K(das_op->get_ls_id()), K(snapshot_maping_)); + } else if (OB_FAIL(get_snapshot_by_tablet(das_op->get_tablet_id(), founded, *snapshot))) { + LOG_WARN("fail to get snapshot by tablet", K(das_op->get_tablet_id()), K(snapshot_maping_)); } else if (!founded) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to found snapshot", K(ret), K(das_op->get_tablet_id()), K(das_op->get_type()), K(das_op->get_ls_id())); + LOG_WARN("fail to found snapshot", K(ret), K(das_op->get_tablet_id()), K(das_op->get_type())); } else if (!snapshot->is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("snapshot is invalid", K(ret), K(das_op->get_tablet_id()), K(das_op->get_type())); } else { LOG_TRACE("set specify snapshot for current das_task", KPC(snapshot), - K(das_op->get_tablet_id()), K(das_op->get_ls_id()), K(das_op->get_type())); + K(das_op->get_tablet_id()), K(das_op->get_type())); } ++task_iter; } diff --git a/src/sql/engine/dml/ob_conflict_checker.h b/src/sql/engine/dml/ob_conflict_checker.h index 1b6c749ec..30cc858ed 100644 --- a/src/sql/engine/dml/ob_conflict_checker.h +++ b/src/sql/engine/dml/ob_conflict_checker.h @@ -34,16 +34,14 @@ class ObTabletSnapshotMaping public: ObTabletSnapshotMaping() : snapshot_(), - tablet_id_(), - ls_id_() + tablet_id_() {} ~ObTabletSnapshotMaping() = default; int assign(const ObTabletSnapshotMaping &other); bool operator==(const ObTabletSnapshotMaping &other) const; - TO_STRING_KV(K_(snapshot), K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(snapshot), K_(tablet_id)); transaction::ObTxReadSnapshot snapshot_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; private: DISALLOW_COPY_AND_ASSIGN(ObTabletSnapshotMaping); }; @@ -214,7 +212,7 @@ class ObConflictChecker // Initial conflict_checker int init_conflict_checker(const ObExprFrameInfo *expr_frame_info, ObDASTableLoc *table_loc, - bool use_partition_gts_opt); + bool use_response_snapshot); void set_local_tablet_loc(ObDASTabletLoc *tablet_loc) { local_tablet_loc_ = tablet_loc; } // Initial conflict_map int create_conflict_map(int64_t replace_row_cnt); @@ -264,7 +262,7 @@ class ObConflictChecker int reuse(); int collect_all_snapshot(transaction::ObTxReadSnapshot &snapshot, const ObDASTabletLoc *tablet_loc); - int get_snapshot_by_ids(ObTabletID tablet_id, share::ObLSID ls_id, bool &founded, transaction::ObTxReadSnapshot &snapshot); + int get_snapshot_by_tablet(ObTabletID tablet_id, bool &founded, transaction::ObTxReadSnapshot &snapshot); int set_partition_snapshot_for_das_task(ObDASRef &das_ref); diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index 76350b664..8aad76c0d 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -1599,7 +1599,7 @@ int ObDMLService::init_dml_param(const ObDASDMLBaseCtDef &base_ctdef, ObIAllocator &das_alloc, storage::ObStoreCtxGuard &store_ctx_gurad, storage::ObDMLBaseParam &dml_param, - bool is_insert_up_gts_opt) + bool use_snapshot_opt) { int ret = OB_SUCCESS; dml_param.timeout_ = base_rtdef.timeout_ts_; @@ -1627,14 +1627,14 @@ int ObDMLService::init_dml_param(const ObDASDMLBaseCtDef &base_ctdef, } dml_param.branch_id_ = write_branch_id; dml_param.store_ctx_guard_ = &store_ctx_gurad; - init_dml_write_flag(base_ctdef, base_rtdef, dml_param.write_flag_, is_insert_up_gts_opt); + init_dml_write_flag(base_ctdef, base_rtdef, dml_param.write_flag_, use_snapshot_opt); return ret; } void ObDMLService::init_dml_write_flag(const ObDASDMLBaseCtDef &base_ctdef, ObDASDMLBaseRtDef &base_rtdef, concurrent_control::ObWriteFlag &write_flag, - bool is_insert_up_gts_opt) + bool use_snapshot_opt) { if (base_ctdef.is_batch_stmt_) { write_flag.set_is_dml_batch_opt(); @@ -1661,8 +1661,8 @@ void ObDMLService::init_dml_write_flag(const ObDASDMLBaseCtDef &base_ctdef, if (base_rtdef.is_immediate_row_conflict_check_ && base_ctdef.is_update_pk_) { write_flag.set_immediate_row_check(); } - if (is_insert_up_gts_opt) { - write_flag.set_plain_insert_gts_opt(); + if (use_snapshot_opt) { + write_flag.set_snapshot_opt(); } } @@ -2536,7 +2536,7 @@ int ObDMLService::rollback_local_savepoint(ObTxDesc &tx_desc, const transaction: { int ret = OB_SUCCESS; ObTransService *tx = share::g_mp->trans_service(); - if (OB_FAIL(tx->rollback_to_implicit_savepoint(tx_desc, savepoint, expire_ts, nullptr))) { + if (OB_FAIL(tx->rollback_to_implicit_savepoint(tx_desc, savepoint, expire_ts, false))) { LOG_WARN("rollback to implicit local savepoint failed", K(ret)); } return ret; @@ -2552,8 +2552,7 @@ int ObDMLService::check_local_index_affected_rows(int64_t table_affected_rows, int ret = OB_SUCCESS; if (GCONF.enable_defensive_check()) { if (table_affected_rows != index_affected_rows - && !related_ctdef.table_param_.get_data_table().is_domain_index() - && !related_ctdef.table_param_.get_data_table().is_mlog_table()) { + && !related_ctdef.table_param_.get_data_table().is_domain_index()) { ret = OB_ERR_DEFENSIVE_CHECK; ObString func_name = ObString::make_string("check_local_index_affected_rows"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); diff --git a/src/sql/engine/dml/ob_dml_service.h b/src/sql/engine/dml/ob_dml_service.h index 147335c90..87928ca02 100644 --- a/src/sql/engine/dml/ob_dml_service.h +++ b/src/sql/engine/dml/ob_dml_service.h @@ -194,7 +194,7 @@ class ObDMLService static void init_dml_write_flag(const ObDASDMLBaseCtDef &base_ctdef, ObDASDMLBaseRtDef &base_rtdef, concurrent_control::ObWriteFlag &write_flag, - bool is_insert_up_gts_opt); + bool use_snapshot_opt); static int init_dml_param(const ObDASDMLBaseCtDef &base_ctdef, ObDASDMLBaseRtDef &base_rtdef, @@ -203,7 +203,7 @@ class ObDMLService common::ObIAllocator &das_alloc, storage::ObStoreCtxGuard &store_ctx_gurad, storage::ObDMLBaseParam &dml_param, - bool is_insert_up_gts_opt); + bool use_snapshot_opt); static int init_das_dml_rtdef(ObDMLRtCtx &dml_rtctx, const ObDASDMLBaseCtDef &das_ctdef, ObDASDMLBaseRtDef &das_rtdef, @@ -349,16 +349,14 @@ class ObDASIndexDMLAdaptor related_ctdefs_(nullptr), related_rtdefs_(nullptr), tablet_id_(), - ls_id_(), related_tablet_ids_(nullptr), das_allocator_(nullptr), ft_doc_word_infos_(nullptr), - is_do_gts_opt_(false), + use_snapshot_opt_(false), dml_param_() { } int write_tablet(DMLIterator &iter, int64_t &affected_rows); - int write_rows(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + int write_rows(const common::ObTabletID &tablet_id, const CtDefType &ctdef, RtDefType &rtdef, DMLIterator &iter, @@ -373,11 +371,10 @@ class ObDASIndexDMLAdaptor const DASCtDefFixedArray *related_ctdefs_; DASRtDefFixedArray *related_rtdefs_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; ObTabletIDFixedArray *related_tablet_ids_; common::ObIAllocator *das_allocator_; common::ObIArray *ft_doc_word_infos_; - bool is_do_gts_opt_; + bool use_snapshot_opt_; private: storage::ObDMLBaseParam dml_param_; }; @@ -388,7 +385,7 @@ int ObDASIndexDMLAdaptor::write_tablet(DMLIterator &iter, int64_ int ret = common::OB_SUCCESS; affected_rows = 0; SQL_DAS_LOG(DEBUG, "begin to write the main tablet", - K(ls_id_), K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_)); + K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_)); if (ctdef_->is_ignore_) { if (OB_FAIL(write_tablet_with_ignore(iter, affected_rows))) { LOG_WARN("write tablet with ignore failed", K(ret)); @@ -399,22 +396,21 @@ int ObDASIndexDMLAdaptor::write_tablet(DMLIterator &iter, int64_ concurrent_control::ObWriteFlag write_flag; // write_flag should be inited before get store ctx, as it will be used in the call function - (void)ObDMLService::init_dml_write_flag(*ctdef_, *rtdef_, write_flag, is_do_gts_opt_); - if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_, - rtdef_->timeout_ts_, + (void)ObDMLService::init_dml_write_flag(*ctdef_, *rtdef_, write_flag, use_snapshot_opt_); + if (OB_FAIL(as->get_write_store_ctx_guard(rtdef_->timeout_ts_, *tx_desc_, *snapshot_, write_branch_id_, write_flag, store_ctx_guard))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); } else if (OB_FAIL(ObDMLService::init_dml_param( - *ctdef_, *rtdef_, *snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_, is_do_gts_opt_))) { + *ctdef_, *rtdef_, *snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_, use_snapshot_opt_))) { SQL_DAS_LOG(WARN, "init dml param failed", K(ret), K(ctdef_->table_id_), K(ctdef_->index_tid_)); - } else if (OB_FAIL(write_rows(ls_id_, tablet_id_, *ctdef_, *rtdef_, iter, affected_rows))) { + } else if (OB_FAIL(write_rows(tablet_id_, *ctdef_, *rtdef_, iter, affected_rows))) { SQL_DAS_LOG(WARN, "write rows failed", K(ret), - K(ls_id_), K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_)); + K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_)); } else if (related_ctdefs_ != nullptr && !related_ctdefs_->empty()) { //write local index for (int64_t i = 0; OB_SUCC(ret) && i < related_ctdefs_->count(); ++i) { @@ -424,16 +420,15 @@ int ObDASIndexDMLAdaptor::write_tablet(DMLIterator &iter, int64_ ObFTDocWordInfo *doc_word_info = nullptr == ft_doc_word_infos_ ? nullptr : &(ft_doc_word_infos_->at(i)); int64_t index_affected_rows = 0; SQL_DAS_LOG(DEBUG, "rewind iterator and write local index tablet", - K(ls_id_), K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_), + K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_), KPC(doc_word_info)); if (OB_FAIL(iter.rewind(related_ctdef, doc_word_info))) { SQL_DAS_LOG(WARN, "rewind iterator failed", K(ret)); } else if (OB_FAIL(ObDMLService::init_dml_param(*related_ctdef, *related_rtdef, - *snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_, is_do_gts_opt_))) { + *snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_, use_snapshot_opt_))) { SQL_DAS_LOG(WARN, "init index dml param failed", K(ret), K(related_ctdef->table_id_), K(related_ctdef->index_tid_)); - } else if (OB_FAIL(write_rows(ls_id_, - related_tablet_id, + } else if (OB_FAIL(write_rows(related_tablet_id, *related_ctdef, *related_rtdef, iter, @@ -486,32 +481,30 @@ int ObDASIndexDMLAdaptor::write_tablet_with_ignore(DMLIterator & ret = OB_ERR_UNEXPECTED; SQL_DAS_LOG(WARN, "add row to single row buffer failed", K(ret)); } else { - SQL_DAS_LOG(TRACE, "write table dml row with ignore", KPC(dml_row), K(ls_id_), K(tablet_id_), + SQL_DAS_LOG(TRACE, "write table dml row with ignore", KPC(dml_row), K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_)); DMLIterator single_row_iter(ctdef_, single_row_buffer, *das_allocator_); storage::ObStoreCtxGuard store_ctx_guard; concurrent_control::ObWriteFlag write_flag; - (void)ObDMLService::init_dml_write_flag(*ctdef_, *rtdef_, write_flag, is_do_gts_opt_); - if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_, - rtdef_->timeout_ts_, + (void)ObDMLService::init_dml_write_flag(*ctdef_, *rtdef_, write_flag, use_snapshot_opt_); + if (OB_FAIL(as->get_write_store_ctx_guard(rtdef_->timeout_ts_, *tx_desc_, *snapshot_, write_branch_id_, write_flag, store_ctx_guard))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); } else if (OB_FAIL(ObDMLService::init_dml_param(*ctdef_, *rtdef_, *snapshot_, write_branch_id_, - *das_allocator_, store_ctx_guard, dml_param_, is_do_gts_opt_))) { + *das_allocator_, store_ctx_guard, dml_param_, use_snapshot_opt_))) { SQL_DAS_LOG(WARN, "init dml param failed", K(ret), KPC_(ctdef), KPC_(rtdef)); } else if (with_local_index && FALSE_IT(dml_param_.write_flag_.set_skip_flush_redo())) { - } else if (OB_FAIL(write_rows(ls_id_, - tablet_id_, + } else if (OB_FAIL(write_rows(tablet_id_, *ctdef_, *rtdef_, single_row_iter, table_affected_rows))) { - SQL_DAS_LOG(WARN, "write rows failed", K(ret), K(ls_id_), K(tablet_id_), KPC(ctdef_), KPC(rtdef_)); + SQL_DAS_LOG(WARN, "write rows failed", K(ret), K(tablet_id_), KPC(ctdef_), KPC(rtdef_)); } else if (with_local_index) { //write local index for (int64_t i = 0; OB_SUCC(ret) && i < related_ctdefs_->count(); ++i) { @@ -521,7 +514,7 @@ int ObDASIndexDMLAdaptor::write_tablet_with_ignore(DMLIterator & ObTabletID related_tablet_id = related_tablet_ids_->at(i); int64_t index_affected_rows = 0; SQL_DAS_LOG(TRACE, "rewind and write index dml row with ignore", KPC(dml_row), - K(ls_id_), K(related_tablet_id), + K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_)); if (OB_FAIL(single_row_iter.rewind(related_ctdef, doc_word_info))) { SQL_DAS_LOG(WARN, "rewind iterator failed", K(ret)); @@ -532,12 +525,11 @@ int ObDASIndexDMLAdaptor::write_tablet_with_ignore(DMLIterator & *das_allocator_, store_ctx_guard, dml_param_, - is_do_gts_opt_))) { + use_snapshot_opt_))) { SQL_DAS_LOG(WARN, "init index dml param failed", K(ret), KPC(related_ctdef), KPC(related_rtdef)); } else if (i == related_ctdefs_->count() - 1 && FALSE_IT(dml_param_.write_flag_.unset_skip_flush_redo())) { - } else if (OB_FAIL(write_rows(ls_id_, - related_tablet_id, + } else if (OB_FAIL(write_rows(related_tablet_id, *related_ctdef, *related_rtdef, single_row_iter, @@ -573,14 +565,13 @@ int ObDASIndexDMLAdaptor::write_tablet_with_ignore(DMLIterator & } template -int ObDASIndexDMLAdaptor::write_rows(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObDASIndexDMLAdaptor::write_rows(const common::ObTabletID &tablet_id, const CtDefType &ctdef, RtDefType &rtdef, DMLIterator &iter, int64_t &affected_rows) { - UNUSEDx(ls_id, tablet_id, ctdef, rtdef, iter, affected_rows); + UNUSEDx(tablet_id, ctdef, rtdef, iter, affected_rows); return common::OB_NOT_IMPLEMENT; } } // namespace sql diff --git a/src/sql/engine/dml/ob_table_insert_up_op.cpp b/src/sql/engine/dml/ob_table_insert_up_op.cpp index 86c0d65e3..d89e343c9 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_up_op.cpp @@ -169,17 +169,17 @@ int ObTableInsertUpOp::inner_open_with_das() int ret = OB_SUCCESS; const ObExprFrameInfo *expr_frame_info = NULL; ObDASTableLoc *table_loc = nullptr; - bool use_partition_gts_opt = false; + bool use_response_snapshot = false; expr_frame_info = nullptr != MY_SPEC.expr_frame_info_ ? MY_SPEC.expr_frame_info_ : &MY_SPEC.plan_->get_expr_frame_info(); - if (ctx_.get_das_ctx().get_use_gts_opt()) { - gts_state_ = MY_SPEC.has_global_unique_index_ == true ? - WITH_UNIQUE_GLOBAL_INDEX_STATE : USE_PARTITION_SNAPSHOT_STATE; - if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { - dml_rtctx_.das_ref_.set_do_gts_opt(true); - upd_rtctx_.das_ref_.set_do_gts_opt(true); - use_partition_gts_opt = true; + if (ctx_.get_das_ctx().get_use_snapshot_opt()) { + snapshot_state_ = MY_SPEC.has_global_unique_index_ == true ? + USE_STMT_SNAPSHOT_STATE : USE_RESPONSE_SNAPSHOT_STATE; + if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { + dml_rtctx_.das_ref_.set_use_snapshot_opt(true); + upd_rtctx_.das_ref_.set_use_snapshot_opt(true); + use_response_snapshot = true; } } if (OB_FAIL(init_insert_up_rtdef())) { @@ -189,7 +189,7 @@ int ObTableInsertUpOp::inner_open_with_das() LOG_WARN("table location is nullptr", K(ret)); } else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info, table_loc, - use_partition_gts_opt))) { + use_response_snapshot))) { LOG_WARN("init conflict_checker fail", K(ret)); } else { // init update das_ref @@ -1081,28 +1081,15 @@ int ObTableInsertUpOp::post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx) { int ret = OB_SUCCESS; if (dml_rtctx.das_ref_.has_task()) { - if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) { - share::ObLSID ls_id; + if (snapshot_state_ == USE_STMT_SNAPSHOT_STATE) { ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_); ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - if (dml_rtctx.das_ref_.check_tasks_same_ls_and_is_local(ls_id)) { - if (OB_FAIL(ObSqlTransControl::get_ls_read_snapshot(my_session, - plan_ctx, - ls_id, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get ls read snapshot", K(ret)); - } else { - LOG_TRACE("all task with same ls_id get ls snapshot", K(ls_id), K(ctx_.get_das_ctx().get_snapshot())); - } + if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, + plan_ctx, + ctx_.get_das_ctx().get_snapshot()))) { + LOG_WARN("fail to get read snapshot", K(ret)); } else { - if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, - plan_ctx, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get global read snapshot", K(ret)); - } else { - gts_state_ = GTE_GTS_STATE; - LOG_TRACE("tablets in different ls_id, we get_gts", K(ctx_.get_das_ctx().get_snapshot())); - } + LOG_TRACE("get read snapshot", K(ctx_.get_das_ctx().get_snapshot())); } } @@ -1121,7 +1108,7 @@ int ObTableInsertUpOp::post_all_dml_das_task(ObDMLRtCtx &dml_rtctx) int ret = OB_SUCCESS; NG_TRACE_TIMES(2, insertup_final_write); if (dml_rtctx.das_ref_.has_task()) { - if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { + if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { if (OB_FAIL(conflict_checker_.set_partition_snapshot_for_das_task(dml_rtctx.das_ref_))) { LOG_WARN("fail to set partition snapshot", K(ret)); } @@ -1223,12 +1210,12 @@ int ObTableInsertUpOp::fetch_conflict_rowkey(int64_t row_cnt) ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); if (OB_FAIL(ret)) { - } else if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { + } else if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); while (OB_SUCC(ret) && !task_iter.is_end()) { ObDASInsertOp *ins_op = static_cast(*task_iter); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); - transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_gts_opt_info().get_response_snapshot(); + transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_snapshot_opt_info().get_response_snapshot(); if (OB_ISNULL(snapshot)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); @@ -1240,31 +1227,6 @@ int ObTableInsertUpOp::fetch_conflict_rowkey(int64_t row_cnt) } ++task_iter; } - } else if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) { - bool refresh_snapshot = false; - share::ObLSID try_exec_ls_id; - share::ObLSID lookup_ls_id; - if (!dml_rtctx_.das_ref_.check_tasks_same_ls_and_is_local(try_exec_ls_id)) { - LOG_TRACE("tablets in different ls_id when try insert", K(ctx_.get_das_ctx().get_snapshot())); - } else if (!conflict_checker_.das_ref_.check_tasks_same_ls_and_is_local(lookup_ls_id)) { - refresh_snapshot = true; - LOG_TRACE("tablets in different ls_id when lookup get conflicted row", K(ctx_.get_das_ctx().get_snapshot())); - } else if ((OB_UNLIKELY(try_exec_ls_id != lookup_ls_id))) { - refresh_snapshot = true; - LOG_TRACE("tablets in ls_id of try execution are different with lookup", K(ctx_.get_das_ctx().get_snapshot())); - } - if (refresh_snapshot) { - ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_); - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, - plan_ctx, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get global read snapshot", K(ret)); - } else { - gts_state_ = GTE_GTS_STATE; - LOG_TRACE("get new snapshot", K(ctx_.get_das_ctx().get_snapshot())); - } - } } return ret; } diff --git a/src/sql/engine/dml/ob_table_insert_up_op.h b/src/sql/engine/dml/ob_table_insert_up_op.h index 3f475a3c1..6b11b817c 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.h +++ b/src/sql/engine/dml/ob_table_insert_up_op.h @@ -94,7 +94,7 @@ class ObTableInsertUpOp : public ObTableModifyOp MY_SPEC.conflict_checker_ctdef_), insert_up_row_store_("InsertUpRow"), is_ignore_(false), - gts_state_(WITHOUT_GTS_OPT_STATE), + snapshot_state_(WITHOUT_SNAPSHOT_OPT_STATE), has_guarantee_last_insert_id_(false) { } @@ -245,7 +245,7 @@ class ObTableInsertUpOp : public ObTableModifyOp common::ObArrayWrap insert_up_rtdefs_; ObChunkDatumStore insert_up_row_store_; // All the rows of insert_up collection bool is_ignore_; // temporarily record whether it is an ignore insert_up SQL statement - ObDmlGTSOptState gts_state_; + ObDmlSnapshotOptState snapshot_state_; bool has_guarantee_last_insert_id_; }; } // end namespace sql diff --git a/src/sql/engine/dml/ob_table_modify_op.h b/src/sql/engine/dml/ob_table_modify_op.h index 130373799..a5096d05a 100644 --- a/src/sql/engine/dml/ob_table_modify_op.h +++ b/src/sql/engine/dml/ob_table_modify_op.h @@ -28,12 +28,11 @@ namespace oceanbase namespace sql { -enum ObDmlGTSOptState +enum ObDmlSnapshotOptState { - WITHOUT_GTS_OPT_STATE = 0, - USE_PARTITION_SNAPSHOT_STATE = 1, - WITH_UNIQUE_GLOBAL_INDEX_STATE = 2, - GTE_GTS_STATE = 3, + WITHOUT_SNAPSHOT_OPT_STATE = 0, + USE_RESPONSE_SNAPSHOT_STATE = 1, + USE_STMT_SNAPSHOT_STATE = 2, }; class ForeignKeyHandle diff --git a/src/sql/engine/dml/ob_table_replace_op.cpp b/src/sql/engine/dml/ob_table_replace_op.cpp index 3173daf00..6a33050a9 100644 --- a/src/sql/engine/dml/ob_table_replace_op.cpp +++ b/src/sql/engine/dml/ob_table_replace_op.cpp @@ -19,7 +19,6 @@ #include "sql/engine/dml/ob_table_replace_op.h" #include "sql/engine/dml/ob_dml_service.h" #include "sql/das/ob_das_insert_op.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -169,17 +168,17 @@ OB_INLINE int ObTableReplaceOp::inner_open_with_das() int ret = OB_SUCCESS; const ObExprFrameInfo *expr_frame_info = NULL; ObDASTableLoc *table_loc = nullptr; - bool use_partition_gts_opt = false; + bool use_response_snapshot = false; expr_frame_info = nullptr != MY_SPEC.expr_frame_info_ ? MY_SPEC.expr_frame_info_ : &MY_SPEC.plan_->get_expr_frame_info(); - if (ctx_.get_das_ctx().get_use_gts_opt()) { - gts_state_ = MY_SPEC.has_global_unique_index_ == true ? - WITH_UNIQUE_GLOBAL_INDEX_STATE : USE_PARTITION_SNAPSHOT_STATE; - if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { - dml_rtctx_.das_ref_.set_do_gts_opt(true); - replace_rtctx_.das_ref_.set_do_gts_opt(true); - use_partition_gts_opt = true; + if (ctx_.get_das_ctx().get_use_snapshot_opt()) { + snapshot_state_ = MY_SPEC.has_global_unique_index_ == true ? + USE_STMT_SNAPSHOT_STATE : USE_RESPONSE_SNAPSHOT_STATE; + if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { + dml_rtctx_.das_ref_.set_use_snapshot_opt(true); + replace_rtctx_.das_ref_.set_use_snapshot_opt(true); + use_response_snapshot = true; } } if (OB_FAIL(init_replace_rtdef())) { @@ -189,8 +188,8 @@ OB_INLINE int ObTableReplaceOp::inner_open_with_das() LOG_WARN("table location is invalid", K(ret)); } else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info, table_loc, - use_partition_gts_opt))) { - LOG_WARN("init conflict_checker fail", K(ret), K(use_partition_gts_opt)); + use_response_snapshot))) { + LOG_WARN("init conflict_checker fail", K(ret), K(use_response_snapshot)); } else { // init update das_ref ObSQLSessionInfo *session = GET_MY_SESSION(ctx_); @@ -501,12 +500,12 @@ int ObTableReplaceOp::fetch_conflict_rowkey(int64_t replace_row_cnt) ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); if (OB_FAIL(ret)) { - } else if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { + } else if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); while (OB_SUCC(ret) && !task_iter.is_end()) { ObDASInsertOp *ins_op = static_cast(*task_iter); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); - transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_gts_opt_info().get_response_snapshot(); + transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_snapshot_opt_info().get_response_snapshot(); if (OB_ISNULL(snapshot)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); @@ -518,31 +517,6 @@ int ObTableReplaceOp::fetch_conflict_rowkey(int64_t replace_row_cnt) } ++task_iter; } - } else if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) { - bool refresh_snapshot = false; - share::ObLSID try_exec_ls_id; - share::ObLSID lookup_ls_id; - if (!dml_rtctx_.das_ref_.check_tasks_same_ls_and_is_local(try_exec_ls_id)) { - LOG_TRACE("tablets in different ls_id when try insert", K(ctx_.get_das_ctx().get_snapshot())); - } else if (!conflict_checker_.das_ref_.check_tasks_same_ls_and_is_local(lookup_ls_id)) { - refresh_snapshot = true; - LOG_TRACE("tablets in different ls_id when lookup get conflicted row", K(ctx_.get_das_ctx().get_snapshot())); - } else if ((OB_UNLIKELY(try_exec_ls_id != lookup_ls_id))) { - refresh_snapshot = true; - LOG_TRACE("tablets in ls_id of try execution are different with lookup", K(ctx_.get_das_ctx().get_snapshot())); - } - if (refresh_snapshot) { - ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_); - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, - plan_ctx, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get global read snapshot", K(ret)); - } else { - gts_state_ = GTE_GTS_STATE; - LOG_TRACE("get new snapshot", K(ctx_.get_das_ctx().get_snapshot())); - } - } } return ret; } @@ -595,28 +569,15 @@ int ObTableReplaceOp::post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx) int ret = OB_SUCCESS; NG_TRACE_TIMES(2, replace_try_insert); if (dml_rtctx.das_ref_.has_task()) { - if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) { - share::ObLSID ls_id; + if (snapshot_state_ == USE_STMT_SNAPSHOT_STATE) { ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_); ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - if (dml_rtctx.das_ref_.check_tasks_same_ls_and_is_local(ls_id)) { - if (OB_FAIL(ObSqlTransControl::get_ls_read_snapshot(my_session, - plan_ctx, - ls_id, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get ls read snapshot", K(ret)); - } else { - LOG_TRACE("all tasks with same ls_id, get ls snapshot", K(ls_id), K(ctx_.get_das_ctx().get_snapshot())); - } + if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, + plan_ctx, + ctx_.get_das_ctx().get_snapshot()))) { + LOG_WARN("fail to get read snapshot", K(ret)); } else { - if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session, - plan_ctx, - ctx_.get_das_ctx().get_snapshot()))) { - LOG_WARN("fail to get global read snapshot", K(ret)); - } else { - gts_state_ = GTE_GTS_STATE; - LOG_TRACE("task in different ls_id, get gts", K(ctx_.get_das_ctx().get_snapshot())); - } + LOG_TRACE("get read snapshot", K(ctx_.get_das_ctx().get_snapshot())); } } @@ -635,7 +596,7 @@ int ObTableReplaceOp::post_all_dml_das_task(ObDMLRtCtx &dml_rtctx) NG_TRACE_TIMES(2, replace_final_write); if (dml_rtctx.das_ref_.has_task()) { - if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) { + if (snapshot_state_ == USE_RESPONSE_SNAPSHOT_STATE) { if (OB_FAIL(conflict_checker_.set_partition_snapshot_for_das_task(dml_rtctx.das_ref_))) { LOG_WARN("fail to set partition snapshot", K(ret)); } diff --git a/src/sql/engine/dml/ob_table_replace_op.h b/src/sql/engine/dml/ob_table_replace_op.h index a7412781e..d3c0ba0e5 100644 --- a/src/sql/engine/dml/ob_table_replace_op.h +++ b/src/sql/engine/dml/ob_table_replace_op.h @@ -81,7 +81,7 @@ class ObTableReplaceOp : public ObTableModifyOp MY_SPEC.conflict_checker_ctdef_), replace_row_store_("ReplaceRow"), replace_rtctx_(eval_ctx_, ctx, *this), - gts_state_(WITHOUT_GTS_OPT_STATE) + snapshot_state_(WITHOUT_SNAPSHOT_OPT_STATE) {} virtual ~ObTableReplaceOp() {} @@ -173,7 +173,7 @@ class ObTableReplaceOp : public ObTableModifyOp common::ObArrayWrap replace_rtdefs_; ObChunkDatumStore replace_row_store_; // All the rows of replace operations ObDMLRtCtx replace_rtctx_; - ObDmlGTSOptState gts_state_; + ObDmlSnapshotOptState snapshot_state_; }; class ObTableReplaceOpInput : public ObTableModifyOpInput diff --git a/src/sql/engine/expr/ob_expr_vec_vector.cpp b/src/sql/engine/expr/ob_expr_vec_vector.cpp index 43fed42f3..ddc052903 100644 --- a/src/sql/engine/expr/ob_expr_vec_vector.cpp +++ b/src/sql/engine/expr/ob_expr_vec_vector.cpp @@ -20,6 +20,8 @@ #include "share/rc/ob_module_provider.h" #include "sql/engine/expr/ob_array_expr_utils.h" #include "sql/engine/ob_exec_context.h" +#include "storage/lob/ob_lob_access_param.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { @@ -154,4 +156,3 @@ int ObExprVecVector::cg_expr( } // namespace sql } // namespace oceanbase - diff --git a/src/sql/engine/ob_physical_plan.cpp b/src/sql/engine/ob_physical_plan.cpp index 98f62f8a2..c861d7f9c 100644 --- a/src/sql/engine/ob_physical_plan.cpp +++ b/src/sql/engine/ob_physical_plan.cpp @@ -118,7 +118,6 @@ ObPhysicalPlan::ObPhysicalPlan(MemoryContext &mem_context /* = CURRENT_CONTEXT * is_batch_params_execute_(false), all_local_session_vars_(&allocator_), udf_has_dml_stmt_(false), - mview_ids_(&allocator_), enable_inc_direct_load_(false), enable_replace_(false), insert_overwrite_(false), @@ -128,7 +127,7 @@ ObPhysicalPlan::ObPhysicalPlan(MemoryContext &mem_context /* = CURRENT_CONTEXT * data_complement_gen_doc_id_(false), dml_table_ids_(&allocator_), direct_load_need_sort_(false), - insertup_can_do_gts_opt_(false), + insertup_can_use_snapshot_opt_(false), px_node_policy_(ObPxNodePolicy::INVALID), px_node_addrs_(&allocator_), px_node_count_(ObPxNodeHint::UNSET_PX_NODE_COUNT), @@ -222,7 +221,6 @@ void ObPhysicalPlan::reset() udf_has_dml_stmt_ = false; is_inner_sql_ = false; is_batch_params_execute_ = false; - mview_ids_.reset(); enable_inc_direct_load_ = false; enable_replace_ = false; insert_overwrite_ = false; @@ -232,7 +230,7 @@ void ObPhysicalPlan::reset() data_complement_gen_doc_id_ = false; dml_table_ids_.reset(); direct_load_need_sort_ = false; - insertup_can_do_gts_opt_ = false; + insertup_can_use_snapshot_opt_ = false; px_node_policy_ = ObPxNodePolicy::INVALID; px_node_count_ = ObPxNodeHint::UNSET_PX_NODE_COUNT; px_node_addrs_.reset(); @@ -801,7 +799,6 @@ OB_SERIALIZE_MEMBER(ObPhysicalPlan, disable_auto_memory_mgr_, udf_has_dml_stmt_, stat_.format_sql_id_, - mview_ids_, enable_inc_direct_load_, enable_replace_, insert_overwrite_, diff --git a/src/sql/engine/ob_physical_plan.h b/src/sql/engine/ob_physical_plan.h index 74d241afd..2a18685b6 100644 --- a/src/sql/engine/ob_physical_plan.h +++ b/src/sql/engine/ob_physical_plan.h @@ -439,8 +439,6 @@ class ObPhysicalPlan : public ObPlanCacheObject const ObSubSchemaCtx &get_subschema_ctx() const { return subschema_ctx_; } int set_all_local_session_vars(ObIArray *all_local_session_vars); ObIArray & get_all_local_session_vars() { return all_local_session_vars_; } - inline const ObIArray &get_mview_ids() const { return mview_ids_; } - int set_mview_ids(const ObIArray &mview_ids) { return mview_ids_.assign(mview_ids); } ObFixedArray &get_dml_table_ids() { return dml_table_ids_; } const ObIArray &get_dml_table_ids() const { return dml_table_ids_; } void set_direct_load_need_sort(const bool direct_load_need_sort) @@ -448,8 +446,8 @@ class ObPhysicalPlan : public ObPlanCacheObject direct_load_need_sort_ = direct_load_need_sort; } bool get_direct_load_need_sort() const { return direct_load_need_sort_; } - inline bool get_insertup_can_do_gts_opt() const {return insertup_can_do_gts_opt_; } - inline void set_insertup_can_do_gts_opt(bool v) { insertup_can_do_gts_opt_ = v; } + inline bool get_insertup_can_use_snapshot_opt() const {return insertup_can_use_snapshot_opt_; } + inline void set_insertup_can_use_snapshot_opt(bool v) { insertup_can_use_snapshot_opt_ = v; } void set_is_use_auto_dop(bool use_auto_dop) { stat_.is_use_auto_dop_ = use_auto_dop; } bool get_is_use_auto_dop() const { return stat_.is_use_auto_dop_; } void set_px_node_policy(ObPxNodePolicy px_node_policy) @@ -646,7 +644,6 @@ class ObPhysicalPlan : public ObPlanCacheObject public: bool udf_has_dml_stmt_; private: - common::ObFixedArray mview_ids_; bool enable_inc_direct_load_; // for incremental direct load bool enable_replace_; // for incremental direct load bool insert_overwrite_; // for insert overwrite @@ -660,7 +657,7 @@ class ObPhysicalPlan : public ObPlanCacheObject // to decide whether it read uncommitted data common::ObFixedArray dml_table_ids_; bool direct_load_need_sort_; - bool insertup_can_do_gts_opt_; + bool insertup_can_use_snapshot_opt_; ObPxNodePolicy px_node_policy_; common::ObFixedArray px_node_addrs_; int64_t px_node_count_; diff --git a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp index edb1f1b6d..8fecbc099 100644 --- a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp @@ -1039,7 +1039,7 @@ int ObPxMultiPartSSTableInsertOp::sync_tablet_doc_id(ObISliceWriter *slice_write slice_writer->get_row_count()); if (OB_FAIL(get_data_tablet_id(tablet_id, data_tablet_id))) { LOG_WARN("fail to get data tablet id", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(tablet_context->ls_id_, data_tablet_id, last_autoinc_val))) { + } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(data_tablet_id, last_autoinc_val))) { LOG_WARN("set tablet autoinc seq failed", K(ret), KPC(slice_writer)); } } diff --git a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.h b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.h index e514ee22e..11075083d 100644 --- a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.h +++ b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.h @@ -24,7 +24,6 @@ namespace oceanbase { namespace storage { -typedef std::pair LSTabletIDPair; struct ObInsertMonitor; struct ObTabletSliceParam; class ObDirectLoadMgrAgent; diff --git a/src/sql/engine/px/ob_px_coord_op.cpp b/src/sql/engine/px/ob_px_coord_op.cpp index bab24c323..4af8575e4 100644 --- a/src/sql/engine/px/ob_px_coord_op.cpp +++ b/src/sql/engine/px/ob_px_coord_op.cpp @@ -829,16 +829,9 @@ int ObPxCoordOp::check_all_sqc(ObIArray &active_dfos, sqc.set_server_not_alive(false); sqc.set_interrupt_by_dm(false); const DASTabletLocIArray &access_locations = sqc.get_access_table_locations(); - for (int64_t i = 0; i < access_locations.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(ctx_.get_my_session()->get_trans_result().add_touched_ls(access_locations.at(i)->ls_id_))) { - LOG_WARN("add touched ls failed", K(ret)); - } - } const DASTabletLocIArray &extra_access_locations = sqc.get_extra_access_table_locations(); - for (int64_t i = 0; i < extra_access_locations.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(ctx_.get_my_session()->get_trans_result().add_touched_ls(extra_access_locations.at(i)->ls_id_))) { - LOG_WARN("add touched ls failed", K(ret)); - } + if (access_locations.count() > 0 || extra_access_locations.count() > 0) { + ctx_.get_my_session()->get_trans_result().mark_touched_storage(); } LOG_WARN("server not alive", K(sqc), K(access_locations), K(sqc.get_access_table_location_keys()), K(extra_access_locations)); diff --git a/src/sql/engine/px/ob_px_scheduler.cpp b/src/sql/engine/px/ob_px_scheduler.cpp index bfe9922b8..0d7946e4d 100644 --- a/src/sql/engine/px/ob_px_scheduler.cpp +++ b/src/sql/engine/px/ob_px_scheduler.cpp @@ -293,12 +293,10 @@ int ObPxMsgProc::process_sqc_finish_msg_once(ObExecContext &ctx, const ObPxFinis LOG_WARN("fail merge result", K(ret), "packet_trans_result", pkt.get_trans_result(), "tx_desc", *session->get_tx_desc()); - } else if (pkt.get_trans_result().get_touched_ls().count() > 0 - && OB_FAIL(session->get_trans_result() - .add_touched_ls(pkt.get_trans_result().get_touched_ls()))) { - LOG_WARN("fail add touched ls for tx", K(ret), - "touched_ls", pkt.get_trans_result().get_touched_ls()); } else { + if (pkt.get_trans_result().touches_storage()) { + session->get_trans_result().mark_touched_storage(); + } LOG_TRACE("on_sqc_finish_msg trans_result", "packet_trans_result", pkt.get_trans_result(), "tx_desc", *session->get_tx_desc(), @@ -617,12 +615,10 @@ int ObPxTerminateMsgProc::on_sqc_finish_msg(ObExecContext &ctx, const ObPxFinish LOG_WARN("fail report tx result", K(ret), "packet_trans_result", pkt.get_trans_result(), "tx_desc", *session->get_tx_desc()); - } else if (pkt.get_trans_result().get_touched_ls().count() > 0 - && OB_FAIL(session->get_trans_result() - .add_touched_ls(pkt.get_trans_result().get_touched_ls()))) { - LOG_WARN("fail add touched ls for tx", K(ret), - "touched_ls", pkt.get_trans_result().get_touched_ls()); } else { + if (pkt.get_trans_result().touches_storage()) { + session->get_trans_result().mark_touched_storage(); + } LOG_TRACE("on_sqc_finish_msg trans_result", "packet_trans_result", pkt.get_trans_result(), "tx_desc", *session->get_tx_desc()); diff --git a/src/sql/engine/px/ob_px_sub_coord.cpp b/src/sql/engine/px/ob_px_sub_coord.cpp index d0ec90d36..4f689ef12 100644 --- a/src/sql/engine/px/ob_px_sub_coord.cpp +++ b/src/sql/engine/px/ob_px_sub_coord.cpp @@ -919,7 +919,7 @@ int ObPxSubCoord::start_ddl() ddl_dag_param.ddl_task_param_.schema_version_ = schema_version; // for idempotence, the schema version must be fixed, so get it from task record ddl_dag_param.ddl_task_param_.is_no_logging_ = is_no_logging; ddl_dag_param.ddl_task_param_.is_offline_index_rebuild_ = is_offline_index_rebuild; - if (OB_FAIL(get_participants(sqc_arg_.sqc_, ddl_table_id, ddl_dag_param.ls_tablet_ids_))) { + if (OB_FAIL(get_participants(sqc_arg_.sqc_, ddl_table_id, ddl_dag_param.tablet_ids_))) { LOG_WARN("fail to get tablet ids", K(ret), K(ddl_task_id), K(ddl_table_id)); } else if (OB_FAIL(ObTenantDagScheduler::alloc_dag(exec_ctx->get_allocator(), false/*is_ha_dag*/, ddl_dag_))) { LOG_WARN("alloc ddl dag failed", K(ret), K(ddl_task_id), KP(ddl_dag_)); @@ -972,13 +972,16 @@ void ObPxSubCoord::ddl_rewrite_ret_code(int &ret_code) int ObPxSubCoord::get_participants(ObPxSqcMeta &sqc, const int64_t table_id, - ObIArray> &ls_tablet_ids) const + ObIArray &tablet_ids) const { int ret = OB_SUCCESS; const DASTabletLocIArray &locations = sqc.get_access_table_locations(); for (int64_t i = 0; OB_SUCC(ret) && i < locations.count(); ++i) { ObDASTabletLoc *tablet_loc = ObDASUtils::get_related_tablet_loc(*locations.at(i), table_id); - if (OB_FAIL(add_var_to_array_no_dup(ls_tablet_ids, std::make_pair(tablet_loc->ls_id_, tablet_loc->tablet_id_)))) { + if (OB_ISNULL(tablet_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet location is null", K(ret), K(table_id), K(i)); + } else if (OB_FAIL(add_var_to_array_no_dup(tablet_ids, tablet_loc->tablet_id_))) { LOG_WARN("add var to array no dup failed", K(ret)); } } diff --git a/src/sql/engine/px/ob_px_sub_coord.h b/src/sql/engine/px/ob_px_sub_coord.h index f7ef17aab..e98e8e1bd 100644 --- a/src/sql/engine/px/ob_px_sub_coord.h +++ b/src/sql/engine/px/ob_px_sub_coord.h @@ -92,7 +92,7 @@ class ObPxSubCoord void set_is_single_tsc_leaf_dfo(bool flag) { is_single_tsc_leaf_dfo_ = flag; } int get_participants(ObPxSqcMeta &sqc, const int64_t table_id, - ObIArray> &ls_tablet_ids) const; + ObIArray &tablet_ids) const; void destroy_shared_rf_msgs(); private: int setup_loop_proc(ObSqcCtx &sqc_ctx) const; diff --git a/src/sql/engine/sequence/ob_sequence_op.h b/src/sql/engine/sequence/ob_sequence_op.h index e06007822..06d619724 100644 --- a/src/sql/engine/sequence/ob_sequence_op.h +++ b/src/sql/engine/sequence/ob_sequence_op.h @@ -23,10 +23,6 @@ namespace oceanbase { -namespace obcall -{ -class ObGAISNextSequenceValRpcResult; -} namespace share { class ObGAISNextSequenceValReq; @@ -84,8 +80,6 @@ class ObLocalSequenceExecutor : public ObSequenceExecutor { virtual void reset() override; virtual void destroy() override; virtual int get_nextval(ObExecContext &ctx) override; - int handle_gais_request(const share::ObGAISNextSequenceValReq &request, - obcall::ObGAISNextSequenceValRpcResult &result); private: // sequence exposes to user layer is a cache // cache underlying responsible for sequence cache update and global coordination diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index c8247926c..d38c5c59d 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -2101,7 +2101,6 @@ int ObTableScanOp::local_iter_reuse() MY_INPUT.tablet_loc_ != nullptr); if (MY_INPUT.tablet_loc_ != nullptr) { scan_op->set_tablet_id(MY_INPUT.tablet_loc_->tablet_id_); - scan_op->set_ls_id(MY_INPUT.tablet_loc_->ls_id_); scan_op->set_tablet_loc(MY_INPUT.tablet_loc_); } if (MY_SPEC.gi_above_) { @@ -4278,7 +4277,6 @@ DEF_TO_STRING(ObVTableScanParam) int64_t pos = 0; J_OBJ_START(); J_KV(K_(tablet_id), - K_(ls_id), N_COLUMN_IDS, column_ids_, N_INDEX_ID, index_id_, N_KEY_RANGES, key_ranges_, diff --git a/src/sql/executor/ob_remote_scheduler.cpp b/src/sql/executor/ob_remote_scheduler.cpp index 6d284b1a0..3e868a6ad 100644 --- a/src/sql/executor/ob_remote_scheduler.cpp +++ b/src/sql/executor/ob_remote_scheduler.cpp @@ -148,7 +148,6 @@ int ObRemoteScheduler::build_remote_task(ObExecContext &ctx, ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx(); ObTaskExecutorCtx &task_exec_ctx = ctx.get_task_exec_ctx(); ObSQLSessionInfo *session = nullptr; - share::ObLSArray task_ls_list; if (OB_FAIL(remote_task.assign_dependency_tables(dependency_tables))) { LOG_WARN("fail to assign dependency_tables", K(ret)); } @@ -160,21 +159,16 @@ int ObRemoteScheduler::build_remote_task(ObExecContext &ctx, K(task_exec_ctx.get_query_sys_begin_schema_version())); remote_task.set_remote_sql_info(&plan_ctx->get_remote_sql_info()); ObDASTabletLoc *first_tablet_loc = nullptr; - if (OB_FAIL(ret)){ + if (OB_FAIL(ret)) { } else if (OB_ISNULL(session = ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("session is null", K(ret)); - } else if (OB_FAIL(DAS_CTX(ctx).get_all_lsid(task_ls_list))) { - LOG_WARN("get ls ids failed", K(ret)); - } else if(OB_FAIL(remote_task.assign_ls_list(task_ls_list))) { - LOG_WARN("fail to assign ls list", K(ret)); - } else if (OB_FAIL(session->get_trans_result().add_touched_ls(task_ls_list))) { - LOG_WARN("add touched ls failed", K(ret)); } else if (OB_ISNULL(DAS_CTX(ctx).get_table_loc_list().get_first()) || OB_ISNULL(first_tablet_loc = DAS_CTX(ctx).get_table_loc_list().get_first()->get_first_tablet_loc())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected empty table loc list", K(ret), K(DAS_CTX(ctx).get_table_loc_list())); } else { + session->get_trans_result().mark_touched_storage(); remote_task.set_runner_svr(first_tablet_loc->server_); ObTaskID task_id; task_id.set_execution_id(session->get_current_execution_id()); diff --git a/src/sql/executor/ob_remote_task_executor.cpp b/src/sql/executor/ob_remote_task_executor.cpp index 365de3409..1961a96b6 100644 --- a/src/sql/executor/ob_remote_task_executor.cpp +++ b/src/sql/executor/ob_remote_task_executor.cpp @@ -150,7 +150,6 @@ int ObRemoteTaskExecutor::build_task(ObExecContext &query_ctx, */ ObOpSpec *root_spec = NULL; const ObPhysicalPlan *phy_plan = NULL; - share::ObLSArray ls_list; if (OB_ISNULL(root_spec = job.get_root_spec())) { ret = OB_NOT_INIT; @@ -160,11 +159,8 @@ int ObRemoteTaskExecutor::build_task(ObExecContext &query_ctx, LOG_WARN("physical plan is NULL", K(ret)); } else if (OB_FAIL(build_task_op_input(query_ctx, task_info, *root_spec))) { LOG_WARN("fail build op inputs", K(ret)); - } else if (OB_FAIL(DAS_CTX(query_ctx).get_all_lsid(ls_list))) { - LOG_WARN("get ls ids failed.", K(ret)); - } else if (OB_FAIL(query_ctx.get_my_session()->get_trans_result().add_touched_ls(ls_list))) { - LOG_WARN("add touched ls failed.", K(ret)); } else { + query_ctx.get_my_session()->get_trans_result().mark_touched_storage(); const ObTaskInfo::ObRangeLocation &range_loc = task_info.get_range_location(); for (int64_t i = 0; OB_SUCC(ret) && i < range_loc.part_locs_.count(); ++i) { if (OB_FAIL(task.assign_ranges(range_loc.part_locs_.at(i).scan_ranges_))) { @@ -214,24 +210,11 @@ int ObRemoteTaskExecutor::handle_tx_after_rpc(ObScanner *scanner, if (OB_FAIL(ret)) { if (exec_ctx.use_remote_sql()) { // ignore ret - LOG_WARN("remote execute use sql failed, tx will rollback", K(ret)); + LOG_WARN("remote execute use sql fail with location_error, tx will rollback", K(ret)); session->get_trans_result().set_incomplete(); } else { - ObDASCtx &das_ctx = DAS_CTX(exec_ctx); - share::ObLSArray ls_ids; - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(das_ctx.get_all_lsid(ls_ids))) { - LOG_WARN("get all ls_ids failed", K(tmp_ret)); - } else if (OB_TMP_FAIL(session->get_trans_result().add_touched_ls(ls_ids))) { - LOG_WARN("add touched ls to txn failed", K(tmp_ret)); - } else { - LOG_INFO("add touched ls succ", K(ls_ids)); - } - if (OB_TMP_FAIL(tmp_ret)) { - LOG_WARN("remote execute use plan failed and try add touched ls failed, tx will rollback", K(tmp_ret)); - session->get_trans_result().set_incomplete(); - ret = COVER_SUCC(tmp_ret); - } + session->get_trans_result().mark_touched_storage(); + LOG_INFO("mark transaction as having touched storage"); } } } diff --git a/src/sql/executor/ob_task.cpp b/src/sql/executor/ob_task.cpp index 85622f044..3ea033953 100644 --- a/src/sql/executor/ob_task.cpp +++ b/src/sql/executor/ob_task.cpp @@ -336,7 +336,6 @@ OB_DEF_SERIALIZE(ObRemoteTask) } OB_UNIS_ENCODE(remote_sql_info_->is_original_ps_mode_); OB_UNIS_ENCODE(remote_sql_info_->sql_from_pl_); - OB_UNIS_ENCODE(ls_list_); return ret; } @@ -380,7 +379,6 @@ OB_DEF_SERIALIZE_SIZE(ObRemoteTask) } OB_UNIS_ADD_LEN(remote_sql_info_->is_original_ps_mode_); OB_UNIS_ADD_LEN(remote_sql_info_->sql_from_pl_); - OB_UNIS_ADD_LEN(ls_list_); } return len; } @@ -463,7 +461,6 @@ OB_DEF_DESERIALIZE(ObRemoteTask) } OB_UNIS_DECODE(remote_sql_info_->is_original_ps_mode_); OB_UNIS_DECODE(remote_sql_info_->sql_from_pl_); - OB_UNIS_DECODE(ls_list_); } } return ret; @@ -477,33 +474,6 @@ int ObRemoteTask::assign_dependency_tables(const DependenyTableStore &dependency return ret; } -int ObRemoteTask::assign_ls_list(const share::ObLSArray ls_ids) { - int ret = OB_SUCCESS; - if (OB_FAIL(ls_list_.assign(ls_ids))) { - LOG_WARN("failed to assign ls list", K(ret)); - } - return ret; -} -// Need to ensure that elements in the two ls arrays are not duplicated -bool ObRemoteTask::check_ls_list(share::ObLSArray &ls_ids) const { - bool is_valid = true; - - if (ls_ids.count() > ls_list_.count()) { - is_valid = false; - } else { - ARRAY_FOREACH_X(ls_ids, i, ls_ids_cnt, is_valid) { - is_valid = false; - ARRAY_FOREACH_X(ls_list_, j, ls_list_cnt, !is_valid) { - if (ls_ids.at(i) == ls_list_.at(j)) { - is_valid = true; - } - } - } - } - - return is_valid; -} - DEFINE_TO_YSON_KV(ObRemoteTask, OB_ID(task_id), task_id_, OB_ID(runner_svr), runner_svr_, OB_ID(ctrl_svr), ctrl_svr_); diff --git a/src/sql/executor/ob_task.h b/src/sql/executor/ob_task.h index 719c2f790..50099abfc 100644 --- a/src/sql/executor/ob_task.h +++ b/src/sql/executor/ob_task.h @@ -206,8 +206,7 @@ class ObRemoteTask : public obcall::ObIFill exec_ctx_(NULL), inner_alloc_("RemoteTask"), dependency_tables_(&inner_alloc_), - snapshot_(), - ls_list_() + snapshot_() { } ~ObRemoteTask() = default; @@ -219,9 +218,6 @@ class ObRemoteTask : public obcall::ObIFill } int64_t get_tenant_schema_version() const { return tenant_schema_version_; } int64_t get_sys_schema_version() const { return sys_schema_version_; } - int assign_ls_list(const share::ObLSArray ls_ids); - const share::ObLSArray &get_all_ls() const { return ls_list_; } - bool check_ls_list(share::ObLSArray &ls_ids) const; void set_session(ObSQLSessionInfo *session_info) { session_info_ = session_info; } void set_runner_svr(const common::ObAddr &runner_svr) { runner_svr_ = runner_svr; } const common::ObAddr &get_runner_svr() const { return runner_svr_; } @@ -250,8 +246,7 @@ class ObRemoteTask : public obcall::ObIFill K_(ctrl_svr), K_(task_id), KPC_(remote_sql_info), - K_(snapshot), - K_(ls_list)); + K_(snapshot)); DECLARE_TO_YSON_KV; private: int64_t tenant_schema_version_; @@ -270,8 +265,6 @@ class ObRemoteTask : public obcall::ObIFill common::ModulePageAllocator inner_alloc_; DependenyTableStore dependency_tables_; transaction::ObTxReadSnapshot snapshot_; - // remote execution pre-save ls - share::ObLSArray ls_list_; }; } } diff --git a/src/sql/ob_phy_table_location.cpp b/src/sql/ob_phy_table_location.cpp index 9a4c8c26f..2627ae5ee 100644 --- a/src/sql/ob_phy_table_location.cpp +++ b/src/sql/ob_phy_table_location.cpp @@ -27,18 +27,12 @@ namespace sql -//OB_SERIALIZE_MEMBER(ObPhyTableLocation, table_location_key_, -// ref_table_id_, part_loc_list_, partition_location_list_); - OB_DEF_SERIALIZE(ObPhyTableLocation) { int ret = OB_SUCCESS; - static ObPartitionLocationSEArray unused_part_loc_list; LST_DO_CODE(OB_UNIS_ENCODE, table_location_key_, ref_table_id_, - unused_part_loc_list, - partition_location_list_, duplicate_type_); return ret; } @@ -46,32 +40,19 @@ OB_DEF_SERIALIZE(ObPhyTableLocation) OB_DEF_DESERIALIZE(ObPhyTableLocation) { int ret = OB_SUCCESS; - ObPartitionLocationSEArray unused_part_loc_list; LST_DO_CODE(OB_UNIS_DECODE, table_location_key_, ref_table_id_, - unused_part_loc_list, - partition_location_list_, duplicate_type_); - - // build local index cache - if (OB_SUCC(ret)) { - if (OB_FAIL(try_build_location_idx_map())) { - LOG_WARN("fail build location idx map", K(ret)); - } - } return ret; } OB_DEF_SERIALIZE_SIZE(ObPhyTableLocation) { int64_t len = 0; - static ObPartitionLocationSEArray unused_part_loc_list; LST_DO_CODE(OB_UNIS_ADD_LEN, table_location_key_, ref_table_id_, - unused_part_loc_list, - partition_location_list_, duplicate_type_); return len; } @@ -79,7 +60,6 @@ OB_DEF_SERIALIZE_SIZE(ObPhyTableLocation) ObPhyTableLocation::ObPhyTableLocation() : table_location_key_(OB_INVALID_ID), ref_table_id_(OB_INVALID_ID), - partition_location_list_(), duplicate_type_(ObDuplicateType::NOT_DUPLICATE) { } @@ -89,7 +69,6 @@ void ObPhyTableLocation::reset() table_location_key_ = OB_INVALID_ID; ref_table_id_ = OB_INVALID_ID; duplicate_type_ = ObDuplicateType::NOT_DUPLICATE; - partition_location_list_.reset(); } int ObPhyTableLocation::assign(const ObPhyTableLocation &other) @@ -98,9 +77,6 @@ int ObPhyTableLocation::assign(const ObPhyTableLocation &other) table_location_key_ = other.table_location_key_; ref_table_id_ = other.ref_table_id_; duplicate_type_ = other.duplicate_type_; - if (OB_FAIL(partition_location_list_.assign(other.partition_location_list_))) { - LOG_WARN("Failed to assign partition location list", K(ret)); - } return ret; } @@ -109,171 +85,10 @@ int ObPhyTableLocation::assign(const ObPhyTableLocation &other) int ObPhyTableLocation::assign_from_phy_table_loc_info(const ObCandiTableLoc &other) { int ret = OB_SUCCESS; - ObSEArray phy_part_loc_info_sorted_list; - ObPartitionReplicaLocation part_replica_loc; - ObReplicaLocation replica_loc; table_location_key_ = other.get_table_location_key(); ref_table_id_ = other.get_ref_table_id(); - const ObCandiTabletLocIArray &phy_part_loc_info_list = other.get_phy_part_loc_info_list(); - // if (other.get_direction() == UNORDERED) { - // if (OB_FAIL(partition_location_list_.reserve(phy_part_loc_info_list.count()))) { - // LOG_WARN("fail reserve memory", K(ret), K(phy_part_loc_info_list.count())); - // } - // for (int64_t i = 0; OB_SUCC(ret) && i < phy_part_loc_info_list.count(); ++i) { - // part_replica_loc.reset(); - // replica_loc.reset(); - // const ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(i); - // const ObOptTabletLoc &part_loc = phy_part_loc_info.get_partition_location(); - // if (OB_FAIL(phy_part_loc_info.get_selected_replica(replica_loc))) { - // LOG_WARN("fail to get selected replica", K(ret), K(phy_part_loc_info)); - // } else if (OB_FAIL(part_replica_loc.assign(part_loc.get_table_id(), - // part_loc.get_partition_id(), - // part_loc.get_partition_cnt(), - // replica_loc, - // part_loc.get_renew_time(), - // part_loc.get_pg_key()))) { - // LOG_WARN("fail to assin part replica loc", K(ret), K(part_loc), K(replica_loc)); - // } else if (OB_FAIL(partition_location_list_.push_back(part_replica_loc))) { - // LOG_WARN("fail to push back part replica loc", K(ret), K(part_replica_loc)); - // } - // } - // } else { - // if (OB_FAIL(phy_part_loc_info_sorted_list.reserve(phy_part_loc_info_list.count()))) { - // LOG_WARN("fail reserve memory", K(ret), K(phy_part_loc_info_list.count())); - // } else if (OB_FAIL(partition_location_list_.reserve(phy_part_loc_info_list.count()))) { - // LOG_WARN("fail reserve memory", K(ret), K(phy_part_loc_info_list.count())); - // } - // for (int64_t i = 0; OB_SUCC(ret) && i < phy_part_loc_info_list.count(); ++i) { - // if (OB_FAIL(phy_part_loc_info_sorted_list.push_back(&phy_part_loc_info_list.at(i)))) { - // LOG_WARN("fail to push back phy part loc info", K(ret), K(phy_part_loc_info_list.at(i)), K(i)); - // } - // } - // if (OB_SUCC(ret) && phy_part_loc_info_sorted_list.count() > 0) { -// //output partition order according to the specified direction order. -// //Currently dependency on partition order represents range order. In the future, after operations like split, etc., we will need to rely on part_mgr(schema) for sorting - // if (is_ascending_direction(other.get_direction())) { - // lib::ob_sort(&phy_part_loc_info_sorted_list.at(0), - // &phy_part_loc_info_sorted_list.at(0) + phy_part_loc_info_sorted_list.count(), - // ObPhyTableLocation::compare_phy_part_loc_info_asc); - // } else if (is_descending_direction(other.get_direction())) { - // lib::ob_sort(&phy_part_loc_info_sorted_list.at(0), - // &phy_part_loc_info_sorted_list.at(0) + phy_part_loc_info_sorted_list.count(), - // ObPhyTableLocation::compare_phy_part_loc_info_desc); - // } else { - // ret = OB_ERR_UNEXPECTED; - // LOG_ERROR("unexpected order", K(ret), K(other.get_direction()), K(other)); - // } - // } - // for (int64_t i = 0; OB_SUCC(ret) && i < phy_part_loc_info_sorted_list.count(); ++i) { - // part_replica_loc.reset(); - // replica_loc.reset(); - // const ObCandiTabletLoc *phy_part_loc_info = phy_part_loc_info_sorted_list.at(i); - // if (OB_ISNULL(phy_part_loc_info)) { - // ret = OB_ERR_UNEXPECTED; - // LOG_ERROR("phy part loc info is NULL", K(ret), K(i)); - // } else { - // const ObOptTabletLoc &part_loc = phy_part_loc_info->get_partition_location(); - // if (OB_FAIL(phy_part_loc_info->get_selected_replica(replica_loc))) { - // LOG_WARN("fail to get selected replica", K(ret), K(*phy_part_loc_info)); - // } else if (OB_FAIL(part_replica_loc.assign(part_loc.get_table_id(), - // part_loc.get_partition_id(), - // part_loc.get_partition_cnt(), - // replica_loc, - // part_loc.get_renew_time(), - // part_loc.get_pg_key()))) { - // LOG_WARN("fail to assin part replica loc", K(ret), K(part_loc), K(replica_loc)); - // } else if (OB_FAIL(partition_location_list_.push_back(part_replica_loc))) { - // LOG_WARN("fail to push back part replica loc", K(ret), K(part_replica_loc)); - // } - // } - // } - // } - - if (OB_SUCC(ret)) { - set_duplicate_type(other.get_duplicate_type()); - } - return ret; -} - - -int ObPhyTableLocation::add_partition_location( - const ObPartitionReplicaLocation &partition_location) -{ - int ret = OB_SUCCESS; - bool duplicated = false; - for (int64_t i = 0; OB_SUCC(ret) && !duplicated - && i < partition_location_list_.count(); ++i) { - if (partition_location.get_table_id() - == partition_location_list_.at(i).get_table_id() - && partition_location.get_partition_id() - == partition_location_list_.at(i).get_partition_id()) { - duplicated = true; - } - } - if (!duplicated) { - if (OB_FAIL(partition_location_list_.push_back(partition_location))) { - LOG_WARN("Fail to add partition location to list", K(ret), K(partition_location)); - } - } - return ret; -} - -const ObPartitionReplicaLocation *ObPhyTableLocation::get_part_replic_by_part_id(int64_t part_id) const -{ - const ObPartitionReplicaLocation *ret = NULL; - int64_t i = 0; - if (OB_SUCCESS == get_location_idx_by_part_id(part_id, i)) { - ret = &partition_location_list_.at(i); - } - return ret; -} - - -int ObPhyTableLocation::try_build_location_idx_map() -{ - int ret = OB_SUCCESS; - // if more than FAST_LOOKUP_LOC_IDX_SIZE_THRES partitions in a 'Task/SQC/Worker', - // build a hash map to accelerate lookup speed - if (OB_UNLIKELY(partition_location_list_.count() > FAST_LOOKUP_LOC_IDX_SIZE_THRES)) { - if (OB_FAIL(location_idx_map_.create( - partition_location_list_.count(), "LocIdxPerfMap", "LocIdxPerfMap"))) { - LOG_WARN("fail init location idx map", K(ret)); - } else { - ARRAY_FOREACH(partition_location_list_, idx) { - // ObPartitionReplicaLocation - auto &item = partition_location_list_.at(idx); - if (OB_FAIL(location_idx_map_.set_refactored(item.get_partition_id(), idx))) { - LOG_WARN("fail set value to map", K(ret), K(item)); - } - } - } - } - return ret; -} - -int ObPhyTableLocation::get_location_idx_by_part_id(int64_t part_id, int64_t &loc_idx) const -{ - int ret = OB_ENTRY_NOT_EXIST; - if (OB_UNLIKELY(location_idx_map_.size() > 0)) { - ret = location_idx_map_.get_refactored(part_id, loc_idx); - LOG_DEBUG("lookup location index from map", K(partition_location_list_.count()), - K(part_id), K(loc_idx), K(table_location_key_), K(ref_table_id_)); - } else { - // single part table scan - for (int64_t idx = 0; idx < partition_location_list_.count(); ++idx) { - if (partition_location_list_.at(idx).get_partition_id() == part_id) { - loc_idx = idx; - ret = OB_SUCCESS; - break; - } - } - } + set_duplicate_type(other.get_duplicate_type()); return ret; } - -// The reason for not using the remove interface is that -// the remove interface will not free the memory of the removed node. -// If the interface is called multiple times for anti-corruption, -// the problem of memory bloat will occur. }/* ns sql*/ }/* ns oceanbase */ diff --git a/src/sql/ob_phy_table_location.h b/src/sql/ob_phy_table_location.h index 9edf2e222..1866bda27 100644 --- a/src/sql/ob_phy_table_location.h +++ b/src/sql/ob_phy_table_location.h @@ -17,8 +17,7 @@ #ifndef OCEANBASE_SQL_OB_PHY_TABLE_LOCATION_ #define OCEANBASE_SQL_OB_PHY_TABLE_LOCATION_ -#include "common/ob_range.h" -#include "share/partition_table/ob_partition_location.h" +#include "share/ob_define.h" namespace oceanbase { @@ -27,12 +26,6 @@ namespace sql class ObCandiTableLoc; class ObCandiTabletLoc; -typedef common::ObIArray ObPartitionLocationIArray; -typedef common::ObSEArray ObPartitionLocationSEArray; - -typedef common::ObIArray ObPartitionReplicaLocationIArray; -typedef common::ObSEArray ObPartitionReplicaLocationSEArray; - #ifdef _WIN32 // nb30.h defines DUPLICATE=0x06 (NetBIOS constant). Permanently undef to avoid conflicts. // OceanBase does not use NetBIOS APIs. @@ -69,63 +62,22 @@ class ObPhyTableLocation final inline uint64_t get_table_location_key() const { return table_location_key_; } inline uint64_t get_ref_table_id() const { return ref_table_id_; } - int add_partition_location(const share::ObPartitionReplicaLocation &partition_location); - inline const ObPartitionReplicaLocationIArray &get_partition_location_list() const - { - return partition_location_list_; - } - // Dangerous interface - inline ObPartitionReplicaLocationIArray &get_partition_location_list() - { - return const_cast - ((static_cast(*this)).get_partition_location_list()); - } - - int64_t get_partition_cnt() const { return partition_location_list_.count(); } TO_STRING_KV(K_(table_location_key), K_(ref_table_id), - K_(partition_location_list), K_(duplicate_type)); - const share::ObPartitionReplicaLocation *get_part_replic_by_part_id(int64_t part_id) const; - template - int find_not_include_part_ids(const SrcArray &all_part_ids, DstArray &expected_part_ids); - int get_location_idx_by_part_id(int64_t part_id, int64_t &loc_idx) const; bool is_duplicate_table() const { return ObDuplicateType::NOT_DUPLICATE != duplicate_type_; } bool is_duplicate_table_not_in_dml() const { return ObDuplicateType::DUPLICATE == duplicate_type_; } void set_duplicate_type(ObDuplicateType v) { duplicate_type_ = v; } ObDuplicateType get_duplicate_type() const { return duplicate_type_; } -private: - int try_build_location_idx_map(); private: /* Used for addressing location by table ID (possibly generated alias id) */ uint64_t table_location_key_; /* Used to get the actual physical table ID */ uint64_t ref_table_id_; - // The following two list has one element for each partition - ObPartitionReplicaLocationSEArray partition_location_list_; ObDuplicateType duplicate_type_; - - // for lookup performance - static const int FAST_LOOKUP_LOC_IDX_SIZE_THRES = 3; - common::hash::ObHashMap location_idx_map_; }; -template -int ObPhyTableLocation::find_not_include_part_ids(const SrcArray &all_part_ids, - DstArray &expected_part_ids) -{ - int ret = common::OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < all_part_ids.count(); ++i) { - if (OB_ISNULL(get_part_replic_by_part_id(all_part_ids.at(i)))) { - if (OB_FAIL(expected_part_ids.push_back(all_part_ids.at(i)))) { - SQL_LOG(WARN, "push back not included partition id failed", K(ret)); - } - } - } - return ret; -} - class ObPhyTableLocationGuard { public: diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index 32247c58f..09fe09c53 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -22,7 +22,6 @@ #include "sql/engine/px/ob_px_admission.h" #include "sql/engine/cmd/ob_table_direct_insert_service.h" #include "src/sql/plan_cache/ob_plan_cache.h" -#include "src/rootserver/mview/ob_mview_maintenance_service.h" #include "src/sql/ob_sql_ccl_rule_manager.h" #include "sql/resolver/dml/ob_select_stmt.h" #include "sql/monitor/show_trace/ob_show_trace.h" @@ -261,19 +260,9 @@ int ObResultSet::implicit_commit_before_cmd_execute(ObSQLSessionInfo &session_in { int ret = OB_SUCCESS; if (session_info.is_in_transaction() && session_info.associated_xa()) { - int tmp_ret = OB_SUCCESS; - transaction::ObTxDesc *tx_desc = session_info.get_tx_desc(); const transaction::ObXATransID xid = session_info.get_xid(); - const transaction::ObGlobalTxType global_tx_type = tx_desc->get_global_tx_type(xid); - if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { - // commit is not allowed in xa trans - ret = OB_TRANS_XA_ERR_COMMIT; - LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid), K(global_tx_type), - KPC(tx_desc)); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected global trans type", K(ret), K(xid), K(global_tx_type), KPC(tx_desc)); - } + ret = OB_TRANS_XA_ERR_COMMIT; + LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid)); exec_ctx.set_need_disconnect(false); } else { ret = ObSqlTransControl::end_trans_before_cmd_execute(session_info, @@ -482,8 +471,6 @@ OB_INLINE int ObResultSet::do_open_plan(ObExecContext &ctx) int ret = OB_SUCCESS; ctx.reset_op_env(); exec_result_ = &(ctx.get_task_exec_ctx().get_execute_result()); - rootserver::ObMViewMaintenanceService *mview_maintenance_service = - share::g_mp->m_view_maintenance_service(); if (stmt::T_PREPARE != stmt_type_) { if (OB_FAIL(ctx.init_phy_op(physical_plan_->get_phy_operator_size()))) { LOG_WARN("fail init exec phy op ctx", K(ret)); @@ -500,18 +487,8 @@ OB_INLINE int ObResultSet::do_open_plan(ObExecContext &ctx) if (OB_FAIL(ret)) { - } else if (OB_ISNULL(mview_maintenance_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mview_maintenance_service is null", K(ret), KP(mview_maintenance_service)); } else if (OB_FAIL(start_stmt())) { LOG_WARN("fail start stmt", K(ret)); - } else if (!physical_plan_->get_mview_ids().empty() && OB_PHY_PLAN_REMOTE != physical_plan_->get_plan_type() - && OB_FAIL((mview_maintenance_service->get_mview_refresh_info(physical_plan_->get_mview_ids(), - ctx.get_sql_proxy(), - ctx.get_das_ctx().get_snapshot().core_.version_, - ctx.get_physical_plan_ctx()->get_mview_ids(), - ctx.get_physical_plan_ctx()->get_last_refresh_scns())))) { - LOG_WARN("fail to set last_refresh_scns", K(ret), K(physical_plan_->get_mview_ids())); } else { ObPhysicalPlanCtx *plan_ctx = NULL; if (OB_ISNULL(plan_ctx = get_exec_context().get_physical_plan_ctx())) { diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 0a65847f2..a050b3316 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -7279,11 +7279,11 @@ int ObSPIService::resolve_into_params(const ParseResult &parse_result, ret = OB_ERR_UNEXPECTED; LOG_WARN("result tree is NULL or invalid result tree", K(ret)); } else if (T_CREATE_VIEW == parse_result.result_tree_->children_[0]->type_) { - if (parse_result.result_tree_->children_[0]->num_child_ <= 4 - || OB_ISNULL(parse_result.result_tree_->children_[0]->children_[4])) { + if (parse_result.result_tree_->children_[0]->num_child_ <= 3 + || OB_ISNULL(parse_result.result_tree_->children_[0]->children_[3])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("result tree is NULL or invalid result tree", K(ret)); - } else if (OB_FAIL(ObResolverUtils::get_select_into_node(*parse_result.result_tree_->children_[0]->children_[4], + } else if (OB_FAIL(ObResolverUtils::get_select_into_node(*parse_result.result_tree_->children_[0]->children_[3], into_node, true))) { LOG_WARN("wrong usage of into clause", K(ret)); @@ -7580,7 +7580,7 @@ int ObSPIService::setup_cursor_snapshot_verify_(ObPLCursorInfo *cursor, ObSPIRes LOG_ERROR("for update cursor opened but not trans id invalid", K(ret), KPC(tx), K(snapshot)); } need_register_snapshot = true; - } else if (cursor->is_streaming() && tx && tx->is_in_tx() && !tx->is_all_parts_clean()) { + } else if (cursor->is_streaming() && tx && tx->is_in_tx() && !tx->is_write_state_clean()) { if (exec_ctx.get_my_session()->enable_enhanced_cursor_validation()) { need_register_snapshot = false; LOG_TRACE("enable cursor open check read uncommitted"); diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index f7c9b3294..36062e87d 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -660,10 +660,7 @@ int ObSqlTransControl::stmt_setup_snapshot_(ObSQLSessionInfo *session, bool can_plain_insert = false; if (cl == ObConsistencyLevel::WEAK || cl == ObConsistencyLevel::FROZEN) { SCN snapshot_version = SCN::min_scn(); - const bool local_single_ls = plan->is_local_plan() && - OB_PHY_PLAN_LOCAL == plan->get_location_type(); if (OB_FAIL(txs->get_weak_read_snapshot_version(session->get_ob_max_read_stale_time(), - local_single_ls, snapshot_version))) { TRANS_LOG(WARN, "get weak read snapshot fail", KPC(txs)); int64_t stale_time = session->get_ob_max_read_stale_time(); @@ -682,48 +679,25 @@ int ObSqlTransControl::stmt_setup_snapshot_(ObSQLSessionInfo *session, TRANS_LOG(WARN, "check can do plain insert failed", KPC(txs)); } else if (can_plain_insert) { ObTxDesc &tx_desc = *session->get_tx_desc(); - das_ctx.set_use_gts_opt(true); + das_ctx.set_use_snapshot_opt(true); snapshot.init_none_read(); snapshot.core_.tx_id_ = tx_desc.get_tx_id(); snapshot.core_.scn_ = tx_desc.get_tx_seq(); } else { ObTxDesc &tx_desc = *session->get_tx_desc(); int64_t stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session); - share::ObLSID first_ls_id; - bool local_single_ls_plan = false; - bool is_single_tablet = false; - const bool local_single_ls_plan_maybe = plan->is_local_plan() && - OB_PHY_PLAN_LOCAL == plan->get_location_type(); - if (local_single_ls_plan_maybe) { - if (OB_FAIL(get_first_lsid(das_ctx, first_ls_id, is_single_tablet))) { - } else if (!first_ls_id.is_valid()) { - // do nothing - // get_ls_read_snapshot may degenerate into get_gts, so it can be used even if the ls is not local. - // This is mainly to solve the problem of strong reading performance in some single-tablet scenarios. - } else if (OB_FAIL(txs->get_ls_read_snapshot(tx_desc, - session->get_tx_isolation(), - first_ls_id, - stmt_expire_ts, - snapshot))) { - } else if (is_single_tablet && snapshot.snapshot_ls_role_ != ObRole::FOLLOWER) { - // performance for single tablet scenario - local_single_ls_plan = true; - } else { - local_single_ls_plan = has_same_lsid(das_ctx, snapshot, first_ls_id); - } - } - // per-opt: set read elr for DML stmt - if (OB_SUCC(ret) && local_single_ls_plan && !plan->is_plain_select() && txs->get_tx_elr_util().is_can_tenant_elr()) { - snapshot.try_set_read_elr(); - } - if (OB_SUCC(ret) && !local_single_ls_plan) { + if (OB_SUCC(ret)) { ret = txs->get_read_snapshot(tx_desc, session->get_tx_isolation(), stmt_expire_ts, snapshot); + // per-opt: set read elr for DML stmt + if (OB_SUCC(ret) && !plan->is_plain_select() && txs->get_tx_elr_util().is_can_tenant_elr()) { + snapshot.try_set_read_elr(); + } } if (OB_FAIL(ret)) { - LOG_WARN("fail to get snapshot", K(ret), K(local_single_ls_plan), K(first_ls_id), KPC(session)); + LOG_WARN("fail to get snapshot", K(ret), KPC(session)); } } return ret; @@ -760,27 +734,15 @@ int ObSqlTransControl::set_fk_check_snapshot(ObExecContext &exec_ctx) ObTxReadSnapshot &snapshot = das_ctx.get_snapshot(); ObTxDesc &tx_desc = *session->get_tx_desc(); int64_t stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session); - share::ObLSID local_ls_id; - bool local_single_ls_plan = plan->is_local_plan() - && OB_PHY_PLAN_LOCAL == plan->get_location_type() - && has_same_lsid(das_ctx, snapshot, local_ls_id); if (OB_FAIL(get_tx_service(session, txs))) { LOG_WARN("failed to get transaction service", K(ret)); } else { - if (local_single_ls_plan) { - ret = txs->get_ls_read_snapshot(tx_desc, - session->get_tx_isolation(), - local_ls_id, - stmt_expire_ts, - snapshot); - } else { - ret = txs->get_read_snapshot(tx_desc, - session->get_tx_isolation(), - stmt_expire_ts, - snapshot); - } + ret = txs->get_read_snapshot(tx_desc, + session->get_tx_isolation(), + stmt_expire_ts, + snapshot); if (OB_FAIL(ret)) { - LOG_WARN("fail to get snapshot", K(ret), K(local_ls_id), KPC(session)); + LOG_WARN("fail to get snapshot", K(ret), KPC(session)); } } } @@ -806,19 +768,16 @@ int ObSqlTransControl::can_do_plain_insert(ObSQLSessionInfo *session, K(ObSQLUtils::is_nested_sql(&exec_ctx)), K(session->get_tx_isolation())); } else if (plan->is_plain_insert()) { can_plain_insert = true; - } else if (session->enable_insertup_replace_gts_opt() && plan->get_insertup_can_do_gts_opt()) { - if (session->is_user_session()) { - can_plain_insert = true; - } + } else if (plan->get_insertup_can_use_snapshot_opt() && session->is_user_session()) { + can_plain_insert = true; } if (OB_SUCC(ret)) { - if (plan->get_insertup_can_do_gts_opt()) { + if (plan->get_insertup_can_use_snapshot_opt()) { LOG_TRACE("whether can do_batch_insert", K(can_plain_insert), K(plan->is_plain_insert()), - K(last_query_retry_err), K(plan->get_insertup_can_do_gts_opt()), + K(last_query_retry_err), K(plan->get_insertup_can_use_snapshot_opt()), K(plan->get_need_serial_exec()), K(session->is_user_session()), - K(session->get_tx_isolation()), K(ObSQLUtils::is_nested_sql(&exec_ctx)), - K(session->enable_insertup_replace_gts_opt())); + K(session->get_tx_isolation()), K(ObSQLUtils::is_nested_sql(&exec_ctx))); } } return ret; @@ -845,28 +804,6 @@ int ObSqlTransControl::get_read_snapshot(ObSQLSessionInfo *session, return ret; } -int ObSqlTransControl::get_ls_read_snapshot(ObSQLSessionInfo *session, - ObPhysicalPlanCtx *plan_ctx, - const share::ObLSID &local_ls_id, - transaction::ObTxReadSnapshot &snapshot) -{ - int ret = OB_SUCCESS; - ObTxIsolationLevel isolation = session->get_tx_isolation(); - const ObPhysicalPlan *plan = plan_ctx->get_phy_plan(); - int64_t expire_ts = get_stmt_expire_ts(plan_ctx, *session); - transaction::ObTransService *txs = NULL; - transaction::ObTxDesc &tx_desc = *session->get_tx_desc(); - if (OB_FAIL(get_tx_service(session, txs))) { - LOG_WARN("failed to get transaction service", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(tx_desc, isolation, local_ls_id, expire_ts, snapshot))) { - LOG_WARN("failed to set snapshot", K(ret)); - } else if (!snapshot.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected invalid snapshot", K(ret), K(tx_desc), K(isolation), K(local_ls_id)); - } - return ret; -} - int ObSqlTransControl::stmt_setup_savepoint_(ObSQLSessionInfo *session, ObDASCtx &das_ctx, ObPhysicalPlanCtx *plan_ctx, @@ -915,74 +852,6 @@ uint32_t ObSqlTransControl::get_real_session_id(ObSQLSessionInfo &session) return session.get_xid().empty() ? 0 : session.get_server_sid(); } -int ObSqlTransControl::get_first_lsid(const ObDASCtx &das_ctx, share::ObLSID &first_lsid, bool &is_single_tablet) -{ - int ret = OB_SUCCESS; - const DASTableLocList &table_locs = das_ctx.get_table_loc_list(); - if (!table_locs.empty()) { - const ObDASTableLoc *first_table_loc = table_locs.get_first(); - const DASTabletLocList &tablet_locs = first_table_loc->get_tablet_locs(); - if (!tablet_locs.empty()) { - const ObDASTabletLoc *tablet_loc = tablet_locs.get_first(); - first_lsid = tablet_loc->ls_id_; - } - is_single_tablet = (1 == table_locs.size() && 1 == tablet_locs.size()); - } - return ret; -} - -bool ObSqlTransControl::has_same_lsid(const ObDASCtx &das_ctx, - const ObTxReadSnapshot &snapshot, - share::ObLSID &first_lsid) -{ - int ret = OB_SUCCESS; - bool bret = true; - ObLSHandle ls_handle; - const share::SCN snapshot_version = snapshot.core_.version_; - const DASTableLocList &table_locs = das_ctx.get_table_loc_list(); - FOREACH_X(table_node, table_locs, bret) { - ObDASTableLoc *table_loc = *table_node; - for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin(); - bret && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) { - ObDASTabletLoc *tablet_loc = *tablet_node; - const ObTabletID tablet_id = tablet_loc->tablet_id_; - if (first_lsid != tablet_loc->ls_id_) { - bret = false; - } - if (bret && !ls_handle.is_valid()) { - ObLSService *ls_svr = NULL; - if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { - bret = false; - } else if (OB_FAIL(ls_svr->get_ls(first_lsid, ls_handle, ObLSGetMod::TRANS_MOD))) { - bret = false; - } else { - // do nothing - } - } - if (bret) { - ObLS *ls = NULL; - ObLSTabletService *ls_tablet_service = NULL; - if (OB_ISNULL(ls = ls_handle.get_ls())) { - bret = false; - } else if (OB_ISNULL(ls_tablet_service = ls->get_tablet_svr())) { - bret = false; - } else { - ObTablet *tablet = NULL; - ObTabletHandle tablet_handle; - if (OB_FAIL(ls_tablet_service->get_tablet(tablet_id, tablet_handle, 100 * 1000))) { - bret = false; - } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { - bret = false; - } else { - // do nothing - } - } - } - } - } - return bret; -} - int ObSqlTransControl::start_hook_if_need_(ObSQLSessionInfo &session, ObTransService *txs, bool &start_hook) @@ -1138,18 +1007,18 @@ int ObSqlTransControl::end_stmt(ObExecContext &exec_ctx, const bool rollback, co need_rollback = true; save_ret = ret; ret = OB_SUCCESS; - } - if (need_rollback) { - const int64_t stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session); - const share::ObLSArray &touched_ls = tx_result.get_touched_ls(); - const ObTxCleanPolicy policy = decide_stmt_rollback_tx_clean_policy_(exec_errcode, will_retry); - OZ (txs->rollback_to_implicit_savepoint(*tx_desc, - savepoint, - stmt_expire_ts, - &touched_ls, - policy), - savepoint, stmt_expire_ts, touched_ls, policy); - // prioritize returning session error code + } + if (need_rollback) { + const int64_t stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session); + const bool touched_storage = tx_result.touches_storage(); + const ObTxCleanPolicy policy = decide_stmt_rollback_tx_clean_policy_(exec_errcode, will_retry); + OZ (txs->rollback_to_implicit_savepoint(*tx_desc, + savepoint, + stmt_expire_ts, + touched_storage, + policy), + savepoint, stmt_expire_ts, touched_storage, policy); + // prioritize returning session error code if (session->is_terminate(ret)) { LOG_INFO("trans has terminated when end stmt", K(ret), K(tx_id_before_rollback)); } @@ -1272,7 +1141,7 @@ int ObSqlTransControl::rollback_savepoint(ObExecContext &exec_ctx, const ObTxSEQ OZ (get_tx_service(session, txs)); CK (OB_NOT_NULL(session->get_tx_desc())); OX (expire_ts = get_stmt_expire_ts(plan_ctx, *session)); - OZ (txs->rollback_to_implicit_savepoint(*session->get_tx_desc(), savepoint, expire_ts, nullptr)); + OZ (txs->rollback_to_implicit_savepoint(*session->get_tx_desc(), savepoint, expire_ts, false)); return ret; } /* @@ -1463,47 +1332,6 @@ void ObSqlTransControl::clear_xa_branch(const ObXATransID &xid, ObTxDesc *&tx_de } -int ObSqlTransControl::check_ls_readable(const share::ObLSID &ls_id, - const common::ObAddr &addr, - const int64_t max_stale_time_us, - bool &can_read) -{ - int ret = OB_SUCCESS; - can_read = false; - - if (!ls_id.is_valid() - || !addr.is_valid() - || max_stale_time_us == 0 - || max_stale_time_us <= -2) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ls_id), K(addr), K(max_stale_time_us)); - } else if (observer::ObServer::get_instance().get_self() == addr && max_stale_time_us > 0) { - storage::ObLSService *ls_svr = share::g_mp->ls_service(); - storage::ObLSHandle handle; - ObLS *ls = nullptr; - - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream service is NULL", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::TRANS_MOD))) { - LOG_WARN("get ls handle failed", K(ret)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls handle is null", K(ret)); - } else if (ObTimeUtility::current_time() - max_stale_time_us - < ls->get_ls_wrs_handler()->get_ls_weak_read_ts().convert_to_ts()) { - can_read = true; - } else if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - LOG_WARN("log stream weak read ts too stale", K(ls_id), K(addr), K(max_stale_time_us)); - } - } else { - // Single-node single-replica: the only replica is local and there is no - // replica blacklist, so the log stream is always readable here. - can_read = true; - } - return ret; -} - int ObSqlTransControl::alloc_branch_id(ObExecContext &exec_ctx, const int64_t count, int16_t &branch_id) { int ret = OB_SUCCESS; diff --git a/src/sql/ob_sql_trans_control.h b/src/sql/ob_sql_trans_control.h index 8bfbc7d61..c61a7512c 100644 --- a/src/sql/ob_sql_trans_control.h +++ b/src/sql/ob_sql_trans_control.h @@ -59,26 +59,18 @@ class TransState static const uint32_t END_TRANS_EXECUTED_BIT = (1 << 2); static const uint32_t START_STMT_EXECUTED_BIT = (1 << 4); static const uint32_t END_STMT_EXECUTED_BIT = (1 << 6); - static const uint32_t START_PART_EXECUTED_BIT = (1 << 8); - static const uint32_t END_PART_EXECUTED_BIT = (1 << 10); static const uint32_t START_TRANS_SUCC_BIT = (1 << 1); static const uint32_t END_TRANS_SUCC_BIT = (1 << 3); static const uint32_t START_STMT_SUCC_BIT = (1 << 5); static const uint32_t END_STMT_SUCC_BIT = (1 << 7); - static const uint32_t START_PART_SUCC_BIT = (1 << 9); - static const uint32_t END_PART_SUCC_BIT = (1 << 11); static const uint32_t START_TRANS_EXECUTED_SHIFT = 1; static const uint32_t END_TRANS_EXECUTED_SHIFT = 3; static const uint32_t START_STMT_EXECUTED_SHIFT = 5; static const uint32_t END_STMT_EXECUTED_SHIFT = 7; - static const uint32_t START_PART_EXECUTED_SHIFT = 9; - static const uint32_t END_PART_EXECUTED_SHIFT = 11; static const uint32_t START_TRANS_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 0)); static const uint32_t END_TRANS_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 2)); static const uint32_t START_STMT_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 4)); static const uint32_t END_STMT_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 6)); - static const uint32_t START_PART_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 8)); - static const uint32_t END_PART_EXECUTED_MASK = (0xFFFFFFFF ^ (0x3 << 10)); public: TransState() : state_(0) {} ~TransState() {} @@ -90,23 +82,15 @@ class TransState { state_ = ((state_ | START_STMT_EXECUTED_BIT) | (is_succ << START_STMT_EXECUTED_SHIFT)); } void set_end_stmt_executed(bool is_succ) { state_ = ((state_ | END_STMT_EXECUTED_BIT) | (is_succ << END_STMT_EXECUTED_SHIFT)); } - void set_start_participant_executed(bool is_succ) - { state_ = ((state_ | START_PART_EXECUTED_BIT) | (is_succ << START_PART_EXECUTED_SHIFT)); } - void set_end_participant_executed(bool is_succ) - { state_ = ((state_ | END_PART_EXECUTED_BIT) | (is_succ << END_PART_EXECUTED_SHIFT)); } void clear_start_trans_executed() { state_ = (state_ & START_TRANS_EXECUTED_MASK); } void clear_start_stmt_executed() { state_ = (state_ & START_STMT_EXECUTED_MASK); } - void clear_start_participant_executed() - { state_ = (state_ & START_PART_EXECUTED_MASK); } void clear_end_trans_executed() { state_ = (state_ & END_TRANS_EXECUTED_MASK); } void clear_end_stmt_executed() { state_ = (state_ & END_STMT_EXECUTED_MASK); } - void clear_end_participant_executed() - { state_ = (state_ & END_PART_EXECUTED_MASK); } bool is_start_trans_executed() const { return state_ & START_TRANS_EXECUTED_BIT; } @@ -116,10 +100,6 @@ class TransState { return state_ & START_STMT_EXECUTED_BIT; } bool is_end_stmt_executed() const { return state_ & END_STMT_EXECUTED_BIT; } - bool is_start_participant_executed() const - { return state_ & START_PART_EXECUTED_BIT; } - bool is_end_participant_executed() const - { return state_ & END_PART_EXECUTED_BIT; } bool is_start_trans_success() const { return state_ & START_TRANS_SUCC_BIT; } bool is_end_trans_success() const @@ -128,17 +108,13 @@ class TransState { return state_ & START_STMT_SUCC_BIT; } bool is_end_stmt_success() const { return state_ & END_STMT_SUCC_BIT; } - bool is_start_participant_success() const - { return state_ & START_PART_SUCC_BIT; } - bool is_end_participant_success() const - { return state_ & END_PART_SUCC_BIT; } void reset() { state_ = 0; } private: uint32_t state_; - // cached for start_stmt, start_participants, end_participants + // cached for transaction and statement lifecycle }; /* Inner class, used only within this file. Converts Consistency from the SQL layer to the transaction layer. @@ -208,10 +184,6 @@ class ObSqlTransControl const int64_t expire_ts, ObEndTransAsyncCallback *callback); static int start_stmt(ObExecContext &ctx); - static int get_ls_read_snapshot(ObSQLSessionInfo *session, - ObPhysicalPlanCtx *plan_ctx, - const share::ObLSID &local_ls_id, - transaction::ObTxReadSnapshot &snapshot); static int get_read_snapshot(ObSQLSessionInfo *session, ObPhysicalPlanCtx *plan_ctx, transaction::ObTxReadSnapshot &snapshot); @@ -270,10 +242,6 @@ class ObSqlTransControl const transaction::tablelock::ObTableLockMode lock_mode, const int64_t wait_lock_seconds); static void clear_xa_branch(const transaction::ObXATransID &xid, transaction::ObTxDesc *&tx_desc); - static int check_ls_readable(const share::ObLSID &ls_id, - const common::ObAddr &addr, - const int64_t max_stale_time_us, - bool &can_read); static uint32_t get_real_session_id(ObSQLSessionInfo &session); private: DISALLOW_COPY_AND_ASSIGN(ObSqlTransControl); @@ -286,10 +254,6 @@ class ObSqlTransControl static int start_hook_if_need_(ObSQLSessionInfo &session, transaction::ObTransService *txs, bool &start_hook); - static int get_first_lsid(const ObDASCtx &das_ctx, share::ObLSID &first_lsid, bool &is_single_tablet); - static bool has_same_lsid(const ObDASCtx &das_ctx, - const transaction::ObTxReadSnapshot &snapshot, - share::ObLSID &first_lsid); public: /* * create a savepoint without name @@ -304,11 +268,11 @@ class ObSqlTransControl * rollback to savepoint * * [convention]: - * transaction layer use trans_result (which maintained by SQL-engine) to decide rollback participants + * transaction layer use trans_result (which maintained by SQL-engine) to decide rollback write state * therefore if trans_result not been collected completed, trans_result.incomplete flag must be set * before do rollback. - * and if trans_result was incomplete, SQL-engine should pass the participants to transaction layer - * (the participants was calculated from table-locations inner this function) + * and if trans_result was incomplete, SQL-engine should pass the participant to transaction layer + * (the participant was calculated from table-locations inner this function) * * for example: the sql-task executed timeout and its result was unknown, and then do rollback_savepoint; * in this case, the trans_result was incomplete, the flag must been set. diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 0424584ed..c45a037b5 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -1097,8 +1097,6 @@ bool ObSQLUtils::cause_implicit_commit(ParseResult &result) || T_DROP_INDEX == type || T_CREATE_DATABASE == type || T_CREATE_INDEX == type - || T_CREATE_MLOG == type - || T_DROP_MLOG == type /* pl item type*/ || T_SP_CREATE_TYPE == type || T_SP_DROP_TYPE == type @@ -2452,38 +2450,10 @@ int ObSQLUtils::choose_best_replica_for_estimation( { int ret = OB_SUCCESS; best_partition.reset(); - const ObIArray &replica_loc_array = - phy_part_loc_info.get_partition_location().get_replica_locations(); - bool found = false; - // 2. check whether best partition can find in local - for (int64_t i = -1; !found && i < addrs_list.count(); ++i) { - const ObAddr &addr = (i == -1? local_addr : addrs_list.at(i)); - for (int64_t j = 0; !found && j < replica_loc_array.count(); ++j) { - if (addr == replica_loc_array.at(j).get_server() && - 0 != replica_loc_array.at(j).get_property().get_memstore_percent()) { - found = true; - best_partition.set(addr, - phy_part_loc_info.get_partition_location().get_tablet_id(), - phy_part_loc_info.get_partition_location().get_ls_id()); - } - } - } - if (!found && !no_use_remote) { - // best partition not find in local - ObAddr remote_addr; - if (OB_FAIL(choose_best_partition_replica_addr(local_addr, - phy_part_loc_info, - false, - remote_addr))) { - LOG_WARN("failed to get best partition replica addr", K(ret)); - // choose partition replica failed doesn't affect execution, we will decide whether use - // storage estimation interface by (!use_local && remote_addr.is_valid()). - ret = OB_SUCCESS; - } - best_partition.set(remote_addr, - phy_part_loc_info.get_partition_location().get_tablet_id(), - phy_part_loc_info.get_partition_location().get_ls_id()); - } + UNUSED(addrs_list); + UNUSED(no_use_remote); + best_partition.set(local_addr, + phy_part_loc_info.get_partition_location().get_tablet_id()); return ret; } diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index dc9021cc0..cef9c91ec 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -24,7 +24,6 @@ #include "lib/container/ob_2d_array.h" #include "common/mysqlclient/ob_mysql_connection.h" #include "share/geo/ob_s2adapter.h" -#include "share/partition_table/ob_partition_location.h" #include "share/ob_i_sql_expression.h" // ObISqlExpression,ObExprCtx #include "storage/access/ob_table_param.h" // ObColDesc #include "share/schema/ob_multi_version_schema_service.h" // ObMultiVersionSchemaService @@ -78,30 +77,26 @@ struct ObPlanCacheCtx; struct EstimatedPartition { common::ObAddr addr_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; - EstimatedPartition() : addr_(), tablet_id_(), ls_id_() + EstimatedPartition() : addr_(), tablet_id_() {} bool is_valid() const { - return addr_.is_valid() && tablet_id_.is_valid() && ls_id_.is_valid(); + return addr_.is_valid() && tablet_id_.is_valid(); } void reset() { addr_.reset(); tablet_id_.reset(); - ls_id_.reset(); } void set(const common::ObAddr &addr, - const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id) { + const common::ObTabletID &tablet_id) { addr_ = addr; tablet_id_ = tablet_id; - ls_id_ = ls_id; } - TO_STRING_KV(K_(addr), K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(addr), K_(tablet_id)); }; struct ObHiddenColumnItem diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index c2cc5ba00..4943f8031 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -1160,17 +1160,9 @@ int ObAccessPathEstimation::choose_leader_replica(const ObCandiTabletLoc &part_l EstimatedPartition &best_partition) { int ret = OB_SUCCESS; - const ObIArray &replica_loc_array = - part_loc_info.get_partition_location().get_replica_locations(); - for (int64_t i = 0; i < replica_loc_array.count(); ++i) { - if (replica_loc_array.at(i).is_strong_leader() && - (can_use_remote || local_addr == replica_loc_array.at(i).get_server())) { - best_partition.set(replica_loc_array.at(i).get_server(), - part_loc_info.get_partition_location().get_tablet_id(), - part_loc_info.get_partition_location().get_ls_id()); - break; - } - } + UNUSED(can_use_remote); + best_partition.set(local_addr, + part_loc_info.get_partition_location().get_tablet_id()); return ret; } @@ -1545,7 +1537,6 @@ int ObAccessPathEstimation::add_index_info(ObOptimizerContext &ctx, index_est_arg->scan_flag_.index_back_ = ap.est_cost_info_.index_meta_info_.is_index_back_; index_est_arg->range_columns_count_ = ap.est_cost_info_.range_columns_.count(); index_est_arg->tablet_id_ = part.tablet_id_; - index_est_arg->ls_id_ = part.ls_id_; index_est_arg->tx_id_ = ctx.get_session_info()->get_tx_id(); @@ -1907,18 +1898,14 @@ int ObAccessPathEstimation::estimate_full_table_rowcount(ObOptimizerContext &ctx //if the part loc infos more than 1, we see the dml info inner table and storage inner table. } else if (part_loc_info_array.count() > 1) { ObSEArray all_tablet_ids; - ObSEArray all_ls_ids; for (int64_t i = 0; OB_SUCC(ret) && i < part_loc_info_array.count(); ++i) { const ObOptTabletLoc &part_loc = part_loc_info_array.at(i).get_partition_location(); if (OB_FAIL(all_tablet_ids.push_back(part_loc.get_tablet_id()))) { LOG_WARN("failed to push back tablet id", K(ret)); - } else if (OB_FAIL(all_ls_ids.push_back(part_loc.get_ls_id()))) { - LOG_WARN("failed to push back tablet id", K(ret)); } } if (OB_SUCC(ret)) { - if (OB_FAIL(estimate_full_table_rowcount_by_meta_table(ctx, all_tablet_ids, - all_ls_ids, meta))) { + if (OB_FAIL(estimate_full_table_rowcount_by_meta_table(ctx, all_tablet_ids, meta))) { LOG_WARN("failed to estimate full table rowcount by meta table", K(ret)); } else { LOG_TRACE("succeed to estimate full table rowcount", K(meta)); @@ -1979,7 +1966,6 @@ int ObAccessPathEstimation::storage_estimate_full_table_rowcount(ObOptimizerCont path_arg.range_columns_count_ = meta.table_rowkey_count_; path_arg.batch_.type_ = ObSimpleBatch::T_SCAN; path_arg.tablet_id_ = best_index_part.tablet_id_; - path_arg.ls_id_ = best_index_part.ls_id_; path_arg.tx_id_ = ctx.get_session_info()->get_tx_id(); if (OB_FAIL(ObSQLUtils::make_whole_range(arena, @@ -2103,7 +2089,6 @@ int ObAccessPathEstimation::storage_estimate_range_rowcount(ObOptimizerContext & path_arg.range_columns_count_ = meta.table_rowkey_count_; path_arg.batch_.type_ = ObSimpleBatch::T_SCAN; path_arg.tablet_id_ = best_index_part.tablet_id_; - path_arg.ls_id_ = best_index_part.ls_id_; path_arg.tx_id_ = ctx.get_session_info()->get_tx_id(); if (OB_FAIL(construct_scan_range_batch(ctx.get_allocator(), chosen_scan_ranges, path_arg.batch_))) { @@ -2167,7 +2152,6 @@ int ObAccessPathEstimation::get_key_ranges(ObOptimizerContext &ctx, int ObAccessPathEstimation::estimate_full_table_rowcount_by_meta_table(ObOptimizerContext &ctx, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObTableMetaInfo &meta) { int ret = OB_SUCCESS; @@ -2178,7 +2162,6 @@ int ObAccessPathEstimation::estimate_full_table_rowcount_by_meta_table(ObOptimiz LOG_WARN("get unexpected null", K(ret), K(ctx.get_session_info()), K(ctx.get_opt_stat_manager())); } else if (OB_FAIL(ctx.get_opt_stat_manager()->get_table_rowcnt(meta.ref_table_id_, all_tablet_ids, - all_ls_ids, meta.table_row_count_))) { LOG_WARN("failed to get table rowcnt", K(ret)); } else { diff --git a/src/sql/optimizer/ob_access_path_estimation.h b/src/sql/optimizer/ob_access_path_estimation.h index cbca6eb4e..e4cf751c9 100644 --- a/src/sql/optimizer/ob_access_path_estimation.h +++ b/src/sql/optimizer/ob_access_path_estimation.h @@ -327,7 +327,6 @@ class ObAccessPathEstimation static int estimate_full_table_rowcount_by_meta_table(ObOptimizerContext &ctx, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObTableMetaInfo &meta); static int get_need_dynamic_sampling_columns(const ObLogPlan* log_plan, diff --git a/src/sql/optimizer/ob_intersect_route_policy.cpp b/src/sql/optimizer/ob_intersect_route_policy.cpp deleted file mode 100644 index 931add405..000000000 --- a/src/sql/optimizer/ob_intersect_route_policy.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_OPT -#include "sql/optimizer/ob_intersect_route_policy.h" -#include "sql/optimizer/ob_log_plan.h" -using namespace oceanbase::common; -using namespace oceanbase::share; -using namespace oceanbase::storage; -namespace oceanbase -{ -namespace sql -{ - -}//sql -}//oceanbase diff --git a/src/sql/optimizer/ob_intersect_route_policy.h b/src/sql/optimizer/ob_intersect_route_policy.h deleted file mode 100644 index 587d33605..000000000 --- a/src/sql/optimizer/ob_intersect_route_policy.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SQL_OB_INTERSET_ROUTE_POLICY_H -#define OCEANBASE_SQL_OB_INTERSET_ROUTE_POLICY_H -#include "share/ob_define.h" -#include "lib/net/ob_addr.h" -#include "lib/container/ob_iarray.h" -#include "lib/list/ob_list.h" -#include "lib/allocator/page_arena.h" -#include "sql/optimizer/ob_route_policy.h" -#include "common/ob_zone_status.h" -#include "share/partition_table/ob_partition_location.h" -#include "share/ob_server_locality_cache.h" -namespace oceanbase -{ -namespace sql -{ -class ObCandiTabletLoc; -class ObCandiTableLoc; -class ObIntersectRoutePolicy:public ObRoutePolicy -{ -public: - using ObRoutePolicy::ObRoutePolicy; -}; - - - - - -}//sql -}//oceanbase -#endif diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index a0f848ea0..df03ef520 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -488,8 +488,7 @@ int ObJoinOrder::compute_sharding_info_with_part_info(ObTableLocationType locati bool can_reselect_replica = (phy_tbl_info.is_duplicate_table_not_in_dml() && (1 == phy_tbl_info.get_phy_part_loc_info_list().count()) && !session_info->get_is_in_retry_for_dup_tbl() - && !is_modified - && !session_info->is_dup_ls_modified()); + && !is_modified); sharding_info = new(sharding_info) ObShardingInfo(); sharding_info->set_location_type(location_type); if (OB_FAIL(sharding_info->init_partition_info( @@ -4253,7 +4252,6 @@ int ObJoinOrder::get_valid_index_ids(const uint64_t table_id, } else if (OB_FAIL(schema_guard->get_can_read_index_array(ref_table_id, tids, index_count, - false, can_use_global_index, false /*domain index*/, false /*spatial index*/, @@ -4517,7 +4515,6 @@ int ObJoinOrder::fill_opt_info_index_name(const uint64_t table_id, } else if (OB_FAIL(schema_guard->get_can_read_index_array(base_table_id, index_ids, index_count, - false, true /*global index*/, true /*domain index*/))) { LOG_WARN("failed to get can read index", K(base_table_id), K(ret)); @@ -16748,7 +16745,6 @@ int ObJoinOrder::compute_fd_item_set_for_table_scan(const uint64_t table_id, } else if (OB_FAIL(schema_guard->get_can_read_index_array(table_ref_id, index_tids, index_count, - false, true /*global index*/, false /*domain index*/))) { LOG_WARN("failed to get can read index", K(ret), K(table_ref_id)); diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 6716e9eee..13e230751 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -996,8 +996,7 @@ int ObLogPlan::select_replicas(ObExecContext &exec_ctx, } } else { const bool sess_in_retry = session->get_is_in_retry_for_dup_tbl(); // Do not optimize replica selection for duplicate tables during retry state - const bool is_dup_ls_modified = session->is_dup_ls_modified(); - if (OB_FAIL(ObLogPlan::strong_select_replicas(local_server, phy_tbl_loc_info_list, is_hit_partition, sess_in_retry, is_dup_ls_modified))) { + if (OB_FAIL(ObLogPlan::strong_select_replicas(local_server, phy_tbl_loc_info_list, is_hit_partition, sess_in_retry))) { LOG_WARN("fail to strong select replicas", K(ret), K(local_server), K(phy_tbl_loc_info_list.count())); } else { session->partition_hit().try_set_bool(is_hit_partition); @@ -1009,8 +1008,7 @@ int ObLogPlan::select_replicas(ObExecContext &exec_ctx, int ObLogPlan::strong_select_replicas(const ObAddr &local_server, ObIArray &phy_tbl_loc_info_list, bool &is_hit_partition, - bool sess_in_retry, // current session is in retry - bool is_dup_ls_modified) + bool sess_in_retry) // current session is in retry { int ret = OB_SUCCESS; // Select all as leader @@ -1028,7 +1026,6 @@ int ObLogPlan::strong_select_replicas(const ObAddr &local_server, // The partition count of this table is 0, skip } else { if (!sess_in_retry - && !is_dup_ls_modified && phy_tbl_loc_info->is_duplicate_table_not_in_dml()) { if (OB_FAIL(phy_tbl_loc_info->all_select_local_replica_or_leader(is_on_same_server, cur_same_server, local_server))) { LOG_WARN("fail to all select leader", K(ret), K(*phy_tbl_loc_info)); @@ -1073,68 +1070,13 @@ int ObLogPlan::weak_select_replicas(const ObAddr &local_server, bool &is_hit_partition) { int ret = OB_SUCCESS; - is_hit_partition = true;//Currently there is no way to determine if it can be selected on one machine, so this value is set to true - ObCandiTableLoc * phy_tbl_loc_info = nullptr; - ObArenaAllocator allocator(ObModIds::OB_SQL_OPTIMIZER_SELECT_REPLICA); - ObList intersect_server_list(allocator); - SMART_VAR(ObRoutePolicy, route_policy, local_server) { - ObRoutePolicyCtx route_policy_ctx; - route_policy_ctx.policy_type_ = route_type; - route_policy_ctx.consistency_level_ = WEAK; - route_policy_ctx.max_read_stale_time_ = max_read_stale_time; - - - if (OB_FAIL(route_policy.init())) { - LOG_WARN("fail to init route policy", K(ret)); - } - for (int64_t i = 0 ; OB_SUCC(ret) && i < phy_tbl_loc_info_list.count(); ++i) { - if (OB_ISNULL(phy_tbl_loc_info = phy_tbl_loc_info_list.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("phy table loc info is NULL", K(phy_tbl_loc_info), K(i), K(ret)); - } else { - ObCandiTabletLocIArray &phy_part_loc_info_list = phy_tbl_loc_info->get_phy_part_loc_info_list_for_update(); - for (int64_t j = 0; OB_SUCC(ret) && j < phy_part_loc_info_list.count(); ++j) { - ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(j); - if (phy_part_loc_info.has_selected_replica()) {//do nothing - } else { - ObIArray &replica_array = phy_part_loc_info.get_partition_location().get_replica_locations(); - if (OB_FAIL(route_policy.init_candidate_replicas(replica_array))) { - LOG_WARN("fail to init candidate replicas", K(replica_array), K(ret)); - } else if (OB_FAIL(route_policy.calculate_replica_priority(local_server, - phy_part_loc_info.get_ls_id(), - replica_array, - route_policy_ctx, - is_inner_table(phy_tbl_loc_info->get_ref_table_id())))) { - LOG_WARN("fail to calculate replica priority", K(replica_array), K(route_policy_ctx), K(ret)); - } else if (OB_FAIL(route_policy.select_replica_with_priority(route_policy_ctx, replica_array, phy_part_loc_info))) { - LOG_WARN("fail to select replica", K(replica_array), K(ret)); - } - } - } - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(route_policy.select_intersect_replica(route_policy_ctx, - phy_tbl_loc_info_list, - intersect_server_list, - is_hit_partition))) { - LOG_WARN("fail to select intersect replica", K(route_policy_ctx), K(phy_tbl_loc_info_list), K(intersect_server_list), K(ret)); - } - - if (OB_SUCC(ret)) { - ObArenaAllocator allocator(ObModIds::OB_SQL_OPTIMIZER_SELECT_REPLICA); - ObAddrList intersect_servers(allocator); - if (OB_FAIL(calc_hit_partition_for_compat(phy_tbl_loc_info_list, local_server, is_hit_partition, intersect_servers))) { - LOG_WARN("fail to calc hit partition for compat", K(ret)); - } - } - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) {// Print once every 10 seconds} - LOG_INFO("selected replica ", "intersect_server_list", intersect_server_list, - "\n phy_tbl_loc_info_list", phy_tbl_loc_info_list, - "\n route_policy", route_policy, - "\n route_policy_ctx", route_policy_ctx, - "\n is_hit_partition", is_hit_partition); + UNUSED(route_type); + UNUSED(max_read_stale_time); + is_hit_partition = true; + for (int64_t i = 0; OB_SUCC(ret) && i < phy_tbl_loc_info_list.count(); ++i) { + if (OB_ISNULL(phy_tbl_loc_info_list.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("phy table location is null", K(ret), K(i)); } } return ret; @@ -1171,48 +1113,32 @@ int ObLogPlan::calc_intersect_servers(const ObIArray &phy_tbl_ ObAddrList &candidate_server_list) { int ret = OB_SUCCESS; - bool can_select_one_server = true; - ObRoutePolicy::CandidateReplica tmp_replica; candidate_server_list.reset(); - for (int64_t i = 0; OB_SUCC(ret) && can_select_one_server && i < phy_tbl_loc_info_list.count(); ++i) { + bool has_server = false; + ObAddr local_server; + for (int64_t i = 0; OB_SUCC(ret) && i < phy_tbl_loc_info_list.count(); ++i) { const ObCandiTableLoc *phy_tbl_loc_info = phy_tbl_loc_info_list.at(i); if (OB_ISNULL(phy_tbl_loc_info)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("phy_tbl_loc_info is NULL", K(ret), K(i), K(phy_tbl_loc_info_list.count())); } else { const ObCandiTabletLocIArray &phy_part_loc_info_list = phy_tbl_loc_info->get_phy_part_loc_info_list(); - for (int64_t j = 0; OB_SUCC(ret) && can_select_one_server && j < phy_part_loc_info_list.count(); ++j) { - const ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(j); - const ObIArray &replica_loc_list = phy_part_loc_info.get_partition_location().get_replica_locations(); - if (0 == i && 0 == j) { // first partition - for (int64_t k = 0; OB_SUCC(ret) && k < replica_loc_list.count(); ++k) { - if (OB_FAIL(candidate_server_list.push_back(replica_loc_list.at(k).get_server()))) { - LOG_WARN("fail to push back candidate server", K(ret), K(k), K(replica_loc_list.at(k))); - } - } - } else { // not the first partition - ObAddrList::iterator candidate_server_list_iter = candidate_server_list.begin(); - for (; OB_SUCC(ret) && candidate_server_list_iter != candidate_server_list.end(); candidate_server_list_iter++) { - const ObAddr &candidate_server = *candidate_server_list_iter; - bool has_replica = false; - for (int64_t k = 0; OB_SUCC(ret) && !has_replica && k < replica_loc_list.count(); ++k) { - if (replica_loc_list.at(k).get_server() == candidate_server) { - has_replica = true; - } - } - if (OB_SUCC(ret) && !has_replica) { - if (OB_FAIL(candidate_server_list.erase(candidate_server_list_iter))) { - LOG_WARN("fail to erase from list", K(ret), K(replica_loc_list), K(candidate_server)); - } - } - } - if (OB_SUCC(ret) && candidate_server_list.empty()) { - can_select_one_server = false; - } + for (int64_t j = 0; OB_SUCC(ret) && j < phy_part_loc_info_list.count(); ++j) { + const ObAddr &server = + phy_part_loc_info_list.at(j).get_partition_location().get_local_replica().get_server(); + if (!has_server) { + local_server = server; + has_server = true; + } else if (local_server != server) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("single-machine locations disagree", K(ret), K(local_server), K(server)); } } } } + if (OB_SUCC(ret) && has_server && OB_FAIL(candidate_server_list.push_back(local_server))) { + LOG_WARN("store local candidate server failed", K(ret), K(local_server)); + } return ret; } @@ -1229,26 +1155,11 @@ int ObLogPlan::select_one_server(const ObAddr &selected_server, ObCandiTabletLocIArray &phy_part_loc_info_list = phy_tbl_loc_info->get_phy_part_loc_info_list_for_update(); for (int64_t j = 0; OB_SUCC(ret) && j < phy_part_loc_info_list.count(); ++j) { - ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(j); - if (phy_part_loc_info.has_selected_replica()) { - // Already selected, skip - } else { - const ObIArray &replica_loc_list = - phy_part_loc_info.get_partition_location().get_replica_locations(); - bool replica_is_selected = false; - for (int64_t k = 0; OB_SUCC(ret) && !replica_is_selected && k < replica_loc_list.count(); ++k) { - if (selected_server == replica_loc_list.at(k).get_server()) { - if (OB_FAIL(phy_part_loc_info.set_selected_replica_idx(k))) { - LOG_WARN("fail to set selected replica idx", K(ret), K(k), K(phy_part_loc_info)); - } else { - replica_is_selected = true; - } - } - } - if (OB_SUCC(ret) && OB_UNLIKELY(!replica_is_selected)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("has no selected replica", K(ret), K(selected_server), K(replica_loc_list)); - } + const ObAddr &local_server = phy_part_loc_info_list.at(j) + .get_partition_location().get_local_replica().get_server(); + if (selected_server != local_server) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("selected server is not local", K(ret), K(selected_server), K(local_server)); } } } @@ -2974,7 +2885,6 @@ int ObLogPlan::allocate_access_path(AccessPath *ap, scan->set_skip_scan(OptSkipScanState::SS_DISABLE != ap->use_skip_scan_); scan->set_table_type(table_schema->get_table_type()); scan->set_index_prefix(ap->index_prefix_); - scan->set_mr_mv_scan(table_item->mr_mv_flags_); if (!ap->is_inner_path_ && OB_FAIL(scan->set_query_ranges(ap->get_cost_table_scan_info().ranges_, ap->get_cost_table_scan_info().ss_ranges_))) { @@ -11476,14 +11386,11 @@ int ObLogPlan::choose_duplicate_table_replica(ObLogicalOperator *op, ObCandiTableLoc &phy_loc = table_scan->get_table_partition_info()->get_phy_tbl_location_info_for_update(); for (int64_t i = 0; OB_SUCC(ret) && i < phy_loc.get_partition_cnt(); ++i) { - int64_t dup_table_pos = OB_INVALID_INDEX; ObCandiTabletLoc &phy_part_loc = phy_loc.get_phy_part_loc_info_list_for_update().at(i); - if (!phy_part_loc.is_server_in_replica(addr, dup_table_pos)) { + if (!phy_part_loc.is_local_server(addr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("no server in replica", K(addr), K(table_scan->get_table_id()), K(ret)); - } else { - phy_part_loc.set_selected_replica_idx(dup_table_pos); } } } else if (log_op_def::LOG_EXCHANGE == op->get_type()) { @@ -11603,10 +11510,6 @@ int ObLogPlan::gen_das_table_location_info(ObLogTableScan *table_scan, } else if (OB_FALSE_IT(das_location.set_use_das(true))) { } else if (OB_FALSE_IT(das_location.set_is_das_empty_part(table_partition_info->get_table_location().is_das_empty_part()))) { } else if (OB_FALSE_IT(das_location.set_broadcast_table(table_meta_info->is_broadcast_table_))) { - } else if (OB_FALSE_IT(das_location.set_dynamic_replica_select_table( - ObConsistencyLevel::WEAK == stmt->get_query_ctx()->get_global_hint().read_consistency_ - && table_meta_info->is_broadcast_table_ - && opt_ctx->get_session_info()->get_ddl_info().is_mview_complete_refresh()))) { } else if (das_location.is_all_partition() && !das_location.is_dynamic_replica_select_table()) { // do nothing @@ -14395,7 +14298,6 @@ int ObLogPlan::adjust_dup_table_replica_by_cons( const ObCandiTableLoc &phy_tbl_info = phy_tbl_info_list.at(i); if (phy_tbl_info.get_partition_cnt() == 1) { const ObCandiTabletLoc &dup_tbl_loc = phy_tbl_info.get_phy_part_loc_info_list().at(0); - int64_t replica_idx = 0; int64_t left_tbl_pos = -1; // find first constraint for (int64_t j = 0; OB_SUCC(ret) && j < dup_table_replica_cons.count(); ++j) { @@ -14412,16 +14314,9 @@ int ObLogPlan::adjust_dup_table_replica_by_cons( share::ObLSReplicaLocation replica_loc; if (OB_FAIL(left_tbl_part_loc_info.get_selected_replica(replica_loc))) { LOG_WARN("failed to set selected replica idx", K(ret), K(left_tbl_part_loc_info)); - } else if (dup_tbl_loc.is_server_in_replica(replica_loc.get_server(), replica_idx)) { - LOG_DEBUG("reselect replica index according to pwj constraints will happen", - K(dup_tbl_loc), K(replica_idx), K(replica_loc.get_server()), K(replica_loc)); - ObRoutePolicy::CandidateReplica replica; - if (OB_FAIL(dup_tbl_loc.get_priority_replica(replica_idx, replica))) { - LOG_WARN("failed to get priority replica", K(ret)); - } else if (OB_FAIL(const_cast(dup_tbl_loc) - .set_selected_replica_idx(replica_idx))) { - LOG_WARN("failed to set selected replica idx", K(ret), K(replica_idx)); - } + } else if (!dup_tbl_loc.is_local_server(replica_loc.get_server())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pwj locations are not on the local server", K(ret), K(replica_loc)); } } } @@ -14461,15 +14356,12 @@ int ObLogPlan::compute_duplicate_table_replicas(ObLogicalOperator *op) basic_addr = valid_addrs.at(0); } if (OB_SUCC(ret)) { - int64_t dup_table_pos = OB_INVALID_INDEX; ObCandiTableLoc *phy_loc =sharding->get_phy_table_location_info(); ObCandiTabletLoc &phy_part_loc = phy_loc->get_phy_part_loc_info_list_for_update().at(0); - if (!phy_part_loc.is_server_in_replica(basic_addr, dup_table_pos)) { + if (!phy_part_loc.is_local_server(basic_addr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("no server in replica", K(basic_addr), K(ret)); - } else { - phy_part_loc.set_selected_replica_idx(dup_table_pos); } } } diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index c673e2ad2..54d9a5539 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1796,8 +1796,7 @@ class ObLogPlan static int strong_select_replicas(const common::ObAddr &local_server, common::ObIArray &phy_tbl_loc_info_list, bool &is_hit_partition, - bool sess_in_retry, - bool is_dup_ls_modified); + bool sess_in_retry); static int weak_select_replicas(const common::ObAddr &local_server, ObRoutePolicyType route_type, int64_t max_read_stale_time, diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index 144cd4c73..d79de14e6 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -7351,12 +7351,9 @@ int ObOptimizerUtil::get_duplicate_table_replica(const ObCandiTableLoc &phy_tabl LOG_WARN("get unexpected partition count", K(ret), K(phy_table_loc.get_partition_cnt())); } else { const ObCandiTabletLoc &phy_part_loc = phy_table_loc.get_phy_part_loc_info_list().at(0); - const ObIArray &replicas = - phy_part_loc.get_partition_location().get_replica_locations(); - for (int64_t i = 0; OB_SUCC(ret) && i < replicas.count(); ++i) { - if (OB_FAIL(valid_addrs.push_back(replicas.at(i).get_server()))) { - LOG_WARN("failed to push back replica address", K(ret)); - } else { /*do nothing*/ } + if (OB_FAIL(valid_addrs.push_back( + phy_part_loc.get_partition_location().get_local_replica().get_server()))) { + LOG_WARN("failed to push back local address", K(ret)); } } return ret; @@ -7371,7 +7368,6 @@ int ObOptimizerUtil::compute_duplicate_table_sharding(const ObAddr &local_addr, { int ret = OB_SUCCESS; ObCandiTableLoc *phy_table_loc = NULL; - int64_t replica_index = OB_INVALID_INDEX; target_sharding = NULL; if (OB_ISNULL(target_sharding = reinterpret_cast( allocator.alloc(sizeof(ObShardingInfo))))) { @@ -7397,14 +7393,12 @@ int ObOptimizerUtil::compute_duplicate_table_sharding(const ObAddr &local_addr, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected partition count", K(ret)); } else { - int64_t dup_table_pos = OB_INVALID_INDEX; ObCandiTabletLoc &phy_part_loc = phy_table_loc->get_phy_part_loc_info_list_for_update().at(0); - if (!phy_part_loc.is_server_in_replica(selected_addr, dup_table_pos)) { + if (!phy_part_loc.is_local_server(selected_addr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("no server in replica", K(selected_addr), K(ret)); } else { - phy_part_loc.set_selected_replica_idx(dup_table_pos); if (local_addr == selected_addr) { target_sharding->set_local(); } else { @@ -7436,19 +7430,12 @@ int ObOptimizerUtil::generate_duplicate_table_replicas(ObIAllocator &allocator, // do nothing } else if (OB_FAIL(target_table_loc->assign(*source_table_loc))) { LOG_WARN("failed to assign table location", K(ret)); - } else { - ObCandiTabletLoc &phy_part_loc = - target_table_loc->get_phy_part_loc_info_list_for_update().at(0); - ObOptTabletLoc &opt_tablet_loc = phy_part_loc.get_partition_location(); - ObIArray &replica_loc_list = opt_tablet_loc.get_replica_locations(); - for (int64_t i = replica_loc_list.count() - 1; OB_SUCC(ret) && i >= 0; --i) { - if (ObOptimizerUtil::find_item(valid_addrs, - replica_loc_list.at(i).get_server())) { - // do nothing - } else if (OB_FAIL(replica_loc_list.remove(i))) { - LOG_WARN("failed to remove relica loc list", K(ret)); - } - } + } else if (!ObOptimizerUtil::find_item( + valid_addrs, + target_table_loc->get_phy_part_loc_info_list().at(0) + .get_partition_location().get_local_replica().get_server())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("local replica is not valid for duplicate table", K(ret), K(valid_addrs)); } return ret; } diff --git a/src/sql/optimizer/ob_phy_table_location_info.cpp b/src/sql/optimizer/ob_phy_table_location_info.cpp index e795717b3..fa4a392a0 100644 --- a/src/sql/optimizer/ob_phy_table_location_info.cpp +++ b/src/sql/optimizer/ob_phy_table_location_info.cpp @@ -29,7 +29,8 @@ namespace sql ObOptTabletLoc::ObOptTabletLoc() : partition_id_(OB_INVALID_INDEX), first_level_part_id_(OB_INVALID_INDEX), - replica_locations_("SqlOptimLocaCac", OB_MALLOC_NORMAL_BLOCK_SIZE) + tablet_id_(), + local_replica_() { } @@ -42,92 +43,45 @@ void ObOptTabletLoc::reset() partition_id_ = OB_INVALID_INDEX; first_level_part_id_ = OB_INVALID_INDEX; tablet_id_.reset(); - ls_id_.reset(); - replica_locations_.reset(); + local_replica_.reset(); } int ObOptTabletLoc::assign(const ObOptTabletLoc &other) { int ret = OB_SUCCESS; tablet_id_ = other.tablet_id_; - ls_id_ = other.ls_id_; partition_id_ = other.partition_id_; first_level_part_id_ = other.first_level_part_id_; - if (OB_FAIL(replica_locations_.assign(other.replica_locations_))) { - LOG_WARN("Failed to assign replica locations", K(ret)); - } + local_replica_ = other.local_replica_; return ret; } -int ObOptTabletLoc::assign_with_only_readable_replica(const ObObjectID &partition_id, - const ObObjectID &first_level_part_id, - const common::ObTabletID &tablet_id, - const ObLSLocation &ls_location, - const ObRoutePolicyType route_policy) +int ObOptTabletLoc::assign_local_replica(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, + const common::ObTabletID &tablet_id, + const ObLSReplicaLocation &replica) { int ret = OB_SUCCESS; reset(); partition_id_ = partition_id; first_level_part_id_ = first_level_part_id; tablet_id_ = tablet_id; - ls_id_ = ls_location.get_ls_id(); - int64_t leader_replica_idx = OB_INVALID_INDEX; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_location.get_replica_locations().count(); ++i) { - const ObLSReplicaLocation &replica_loc = ls_location.get_replica_locations().at(i); - if (replica_loc.is_strong_leader()) { - leader_replica_idx = i; - } - if (ObReplicaTypeCheck::is_readable_replica(replica_loc.get_replica_type())) { - // Single-node single-replica: no replica blacklist. - if (route_policy == FORCE_READONLY_ZONE && replica_loc.get_replica_type() != REPLICA_TYPE_READONLY) { - // skip the tmp_replica_loc - LOG_TRACE("skip the replica due to the replica policy.", K(ret), K(replica_loc)); - } else if (OB_FAIL(replica_locations_.push_back(replica_loc))) { - LOG_WARN("Failed to push back replica locations", - K(ret), K(i), K(replica_loc), K(replica_locations_)); - } - } - } - - // all replicas are in blacklist, add leader replica forcibly. - if (OB_SUCC(ret) && 0 == replica_locations_.count()) { - if (OB_INVALID_INDEX == leader_replica_idx) { - LOG_INFO("there is no leader replica"); - } else if (OB_FAIL(replica_locations_.push_back(ls_location.get_replica_locations().at(leader_replica_idx)))) { - LOG_WARN("failed to push back leader replica", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(0 == replica_locations_.count())) { - ret = OB_NO_READABLE_REPLICA; - LOG_WARN("there has no readable replica", K(ret), K(ls_location.get_replica_locations())); - } - } + local_replica_ = replica; return ret; } bool ObOptTabletLoc::is_valid() const { - // For compatibility considerations, after upgrading from 1.4.x and 2.1 to 2.2, pg_key may be invalid, therefore pg_key is not checked here - return OB_INVALID_INDEX != partition_id_; + return OB_INVALID_INDEX != partition_id_ + && tablet_id_.is_valid() + && local_replica_.is_valid(); } int ObOptTabletLoc::get_strong_leader(ObLSReplicaLocation &replica_location, int64_t &replica_idx) const { - int ret = OB_LS_LOCATION_LEADER_NOT_EXIST; - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), "self", *this); - } else { - ARRAY_FOREACH_X(replica_locations_, i, cnt, OB_LS_LOCATION_LEADER_NOT_EXIST == ret) { - if (replica_locations_.at(i).is_strong_leader()) { - replica_location = replica_locations_.at(i); - replica_idx = i; - ret = OB_SUCCESS; - } - } - } + int ret = OB_SUCCESS; + replica_location = local_replica_; + replica_idx = 0; return ret; } @@ -138,9 +92,7 @@ int ObOptTabletLoc::get_strong_leader(ObLSReplicaLocation &replica_location) con } ObCandiTabletLoc::ObCandiTabletLoc() - : opt_tablet_loc_(), - selected_replica_idx_(OB_INVALID_INDEX), - priority_replica_idxs_() + : opt_tablet_loc_() { } @@ -152,206 +104,32 @@ ObCandiTabletLoc::~ObCandiTabletLoc() int ObCandiTabletLoc::assign(const ObCandiTabletLoc &other) { int ret = OB_SUCCESS; - /* - if (OB_UNLIKELY(other.selected_replica_idx_ < 0 - || other.selected_replica_idx_ >= - other.opt_tablet_loc_.get_replica_locations().count())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("other select replica idx is invalid", K(ret), K(other.selected_replica_idx_), - K(other.opt_tablet_loc_.get_replica_locations().count()), - K(other.opt_tablet_loc_)); - } else - */ - if (OB_FAIL(opt_tablet_loc_.assign(other.opt_tablet_loc_))) { LOG_WARN("fail to assign other opt_tablet_loc_", K(ret), K(other.opt_tablet_loc_)); - } else if (OB_FAIL(priority_replica_idxs_.assign(other.priority_replica_idxs_))) { - LOG_WARN("fail to assign replica idxs", K(ret), K(other.priority_replica_idxs_)); - } else { - selected_replica_idx_ = other.selected_replica_idx_; - } - return ret; -} - -int ObCandiTabletLoc::set_selected_replica_idx(int64_t selected_replica_idx) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(selected_replica_idx < 0 - || selected_replica_idx >= - opt_tablet_loc_.get_replica_locations().count())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("select replica idx is invalid", K(ret), K(selected_replica_idx), - K(opt_tablet_loc_.get_replica_locations().count()), K(opt_tablet_loc_)); - } else { - selected_replica_idx_ = selected_replica_idx; - } - return ret; -} - -int ObCandiTabletLoc::set_selected_replica_idx_with_priority() -{ - int ret = OB_SUCCESS; - ObRoutePolicy::CandidateReplica selected_replica; - ObRoutePolicy::CandidateReplica cur_replica; - int64_t selected_idx = OB_INVALID_INDEX; - if (OB_UNLIKELY(priority_replica_idxs_.count() <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid priority replica count", K(priority_replica_idxs_), K(ret)); - } else if (priority_replica_idxs_.count() == 1) { - selected_idx = priority_replica_idxs_.at(0); - if (OB_FAIL(set_selected_replica_idx(selected_idx))) { - LOG_WARN("fail to set selected replica idx", K(priority_replica_idxs_), K(ret)); - } - } else if (priority_replica_idxs_.count() > 1) { - // Multiple priorities, local priority first; if priorities of multiple replicas are the same, select randomly - ObSEArray same_priority_ids; - bool is_first = true; - selected_replica.attr_.pos_type_ = ObRoutePolicy::POSITION_TYPE_MAX; - for (int64_t i = 0; OB_SUCC(ret) && i < priority_replica_idxs_.count(); ++i) { - int64_t cur_priority_idx = priority_replica_idxs_.at(i); - cur_replica.reset(); - if (OB_FAIL(get_priority_replica(cur_priority_idx, cur_replica))) { - LOG_WARN("fail to get priority replca", K(cur_priority_idx), K(ret)); - } else if (is_first) { - is_first = false; - selected_replica = cur_replica; - if (OB_FAIL(same_priority_ids.push_back(cur_priority_idx))) { - LOG_WARN("fail to push back id", K(same_priority_ids), K(cur_priority_idx), K(ret)); - } - } else if (cur_replica.attr_.pos_type_ == selected_replica.attr_.pos_type_) { - if (OB_FAIL(same_priority_ids.push_back(cur_priority_idx))) { - LOG_WARN("fail to push back id", K(same_priority_ids), K(cur_priority_idx), K(ret)); - } - } else if (cur_replica.attr_.pos_type_ < selected_replica.attr_.pos_type_) { - selected_replica = cur_replica; - same_priority_ids.reset(); - if (OB_FAIL(same_priority_ids.push_back(cur_priority_idx))) { - LOG_WARN("fail to push back id", K(same_priority_ids), K(cur_priority_idx), K(ret)); - } - } else { - /*cur_replica.attr_.pos_type_ < selected_replica.attr_.pos_type_ do nothing*/ - } - } - if (OB_SUCC(ret)) { - int64_t rand_idx = rand() % same_priority_ids.count(); - selected_idx = same_priority_ids.at(rand_idx); - if (OB_FAIL(set_selected_replica_idx(selected_idx))) { - LOG_WARN("fail to set selected replica idx", K(priority_replica_idxs_), K(ret)); - } - } - } else {/*do nothing*/} - - return ret; -} - -int ObCandiTabletLoc::add_priority_replica_idx(int64_t priority_replica_idx) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(priority_replica_idxs_.push_back(priority_replica_idx))) { - LOG_WARN("fail to push back priority replica idx", K(priority_replica_idx), K(ret)); - } - return ret; -} - -int ObCandiTabletLoc::get_priority_replica(int64_t selected_replica_idx, - share::ObLSReplicaLocation &replica_loc) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(get_priority_replica_base(selected_replica_idx, replica_loc))) { - LOG_WARN("fail to get priority replica", K(replica_loc), K(ret)); - } - return ret; -} - -int ObCandiTabletLoc::get_priority_replica(int64_t selected_replica_idx, - ObRoutePolicy::CandidateReplica &replica_loc) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(get_priority_replica_base(selected_replica_idx, replica_loc))) { - LOG_WARN("fail to get priority replica", K(replica_loc), K(ret)); - } - return ret; -} - -template -int ObCandiTabletLoc::get_priority_replica_base(int64_t selected_replica_idx, T &replica_loc) const -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_INDEX == selected_replica_idx)) { - ret = OB_NOT_INIT; - LOG_ERROR("ObCandiTabletLoc is not inited", K(ret), K(opt_tablet_loc_)); - } else { - const ObIArray &replica_loc_list = opt_tablet_loc_.get_replica_locations(); - if (OB_UNLIKELY(selected_replica_idx < 0 || selected_replica_idx >= replica_loc_list.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("select replica idx is invalid", K(ret), - K(selected_replica_idx), K(replica_loc_list.count())); - } else { - replica_loc = replica_loc_list.at(selected_replica_idx); - } } return ret; } -bool ObCandiTabletLoc::is_server_in_replica(const ObAddr &server, - int64_t &idx) const +bool ObCandiTabletLoc::is_local_server(const ObAddr &server) const { - int ret = OB_SUCCESS; - bool found_flag = false; - ObRoutePolicy::CandidateReplica tmp_replica; - idx = 0; - for (int64_t replica_idx = 0; - !found_flag && OB_SUCC(ret) && replica_idx < get_partition_location().get_replica_locations().count(); - ++replica_idx) { - tmp_replica.reset(); - if (OB_FAIL(get_priority_replica(replica_idx, tmp_replica))) { - LOG_WARN("fail to get priority replica", K(replica_idx), K(ret)); - } else if (tmp_replica.get_server() == server) { - idx = replica_idx; - found_flag = true; - } - } - return found_flag; + return opt_tablet_loc_.get_local_replica().get_server() == server; } int ObCandiTabletLoc::get_selected_replica(share::ObLSReplicaLocation &replica_loc) const { - int ret = OB_SUCCESS; - if (OB_FAIL(get_priority_replica(selected_replica_idx_, replica_loc))) { - LOG_WARN("fail to get priority replica", K(replica_loc), K(selected_replica_idx_), K(ret)); - } - return ret; -} - -int ObCandiTabletLoc::get_selected_replica(ObRoutePolicy::CandidateReplica &replica_loc) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(get_priority_replica(selected_replica_idx_, replica_loc))) { - LOG_WARN("fail to get priority replica", K(replica_loc), K(selected_replica_idx_), K(ret)); - } - return ret; + replica_loc = opt_tablet_loc_.get_local_replica(); + return OB_SUCCESS; } -int ObCandiTabletLoc::set_part_loc_with_only_readable_replica(const ObObjectID &partition_id, - const ObObjectID &first_level_part_id, - const common::ObTabletID &tablet_id, - const ObLSLocation &partition_location, - const ObRoutePolicyType route_policy) +int ObCandiTabletLoc::set_local_tablet_loc(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, + const common::ObTabletID &tablet_id, + const ObLSReplicaLocation &replica) { - int ret = OB_SUCCESS; - if (OB_UNLIKELY(has_selected_replica())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("partition location has not been set yet, but replica idx has been selected", - K(ret), K(*this), K(partition_location)); - } else if (OB_FAIL(opt_tablet_loc_.assign_with_only_readable_replica(partition_id, - first_level_part_id, - tablet_id, - partition_location, - route_policy))) { - LOG_WARN("fail to assign partition location with only readable replica", - K(ret), K(partition_location)); - } - return ret; + return opt_tablet_loc_.assign_local_replica(partition_id, + first_level_part_id, + tablet_id, + replica); } ObCandiTableLoc::ObCandiTableLoc() @@ -386,43 +164,13 @@ int ObCandiTableLoc::all_select_local_replica_or_leader(bool &is_on_same_server, int ret = OB_SUCCESS; is_on_same_server = true; ObAddr first_server; - ObLSReplicaLocation replica_location; - int64_t replica_idx = OB_INVALID_INDEX; - int64_t local_replica_idx = OB_INVALID_INDEX; - ObAddr replica_addr; for (int64_t i = 0; OB_SUCC(ret) && i < candi_tablet_locs_.count(); ++i) { - replica_location.reset(); - replica_idx = OB_INVALID_INDEX; - local_replica_idx = OB_INVALID_INDEX; - ObCandiTabletLoc &phy_part_loc_info = candi_tablet_locs_.at(i); - replica_addr.reset(); - if (OB_FAIL(phy_part_loc_info.get_partition_location().get_strong_leader(replica_location, replica_idx))) { - LOG_WARN("fail to get leader", K(ret), K(phy_part_loc_info.get_partition_location())); - } else { - replica_addr = replica_location.get_server(); - if (phy_part_loc_info.is_server_in_replica(local_server, local_replica_idx)) { - if (replica_idx != local_replica_idx) { - LOG_TRACE("about to choose local replica rather than leader replica for duplicate table", K(replica_idx), K(local_replica_idx)); - replica_idx = local_replica_idx; - replica_addr = local_server; - } - } - } - if (OB_FAIL(ret)) { - //do nothing ... - } else if (phy_part_loc_info.has_selected_replica() - && phy_part_loc_info.get_selected_replica_idx() != replica_idx) { - // FIXME qianfu The weak attribute of the subquery and the main query are different, which might cause this error, it will be fixed later - ret = OB_ERR_UNEXPECTED; - LOG_WARN("selected replica is not leader", K(ret), K(phy_part_loc_info)); - } else if (OB_FAIL(phy_part_loc_info.set_selected_replica_idx(replica_idx))) { - LOG_WARN("fail to set selected replica idx", K(ret), K(replica_idx), K(phy_part_loc_info)); - } else { - if (0 == i) { - first_server = replica_addr; - } else if (first_server != replica_addr) { - is_on_same_server = false; - } + const ObAddr &replica_addr = + candi_tablet_locs_.at(i).get_partition_location().get_local_replica().get_server(); + if (0 == i) { + first_server = replica_addr; + } else if (first_server != replica_addr) { + is_on_same_server = false; } } if (OB_SUCC(ret) && is_on_same_server) { @@ -437,27 +185,13 @@ int ObCandiTableLoc::all_select_leader(bool &is_on_same_server, int ret = OB_SUCCESS; is_on_same_server = true; ObAddr first_server; - ObLSReplicaLocation replica_location; - int64_t replica_idx = OB_INVALID_INDEX; for (int64_t i = 0; OB_SUCC(ret) && i < candi_tablet_locs_.count(); ++i) { - replica_location.reset(); - replica_idx = OB_INVALID_INDEX; - ObCandiTabletLoc &phy_part_loc_info = candi_tablet_locs_.at(i); - if (OB_FAIL(phy_part_loc_info.get_partition_location().get_strong_leader(replica_location, replica_idx))) { - LOG_WARN("fail to get leader", K(ret), K(phy_part_loc_info.get_partition_location())); - } else if (phy_part_loc_info.has_selected_replica() - && phy_part_loc_info.get_selected_replica_idx() != replica_idx) { - // FIXME qianfu The weak attribute of the subquery and the main query are different, which might cause this error, it will be fixed later - ret = OB_ERR_UNEXPECTED; - LOG_WARN("selected replica is not leader", K(ret), K(phy_part_loc_info)); - } else if (OB_FAIL(phy_part_loc_info.set_selected_replica_idx(replica_idx))) { - LOG_WARN("fail to set selected replica idx", K(ret), K(replica_idx), K(phy_part_loc_info)); - } else { - if (0 == i) { - first_server = replica_location.get_server(); - } else if (first_server != replica_location.get_server()) { - is_on_same_server = false; - } + const ObAddr &replica_addr = + candi_tablet_locs_.at(i).get_partition_location().get_local_replica().get_server(); + if (0 == i) { + first_server = replica_addr; + } else if (first_server != replica_addr) { + is_on_same_server = false; } } if (OB_SUCC(ret) && is_on_same_server) { @@ -465,9 +199,6 @@ int ObCandiTableLoc::all_select_leader(bool &is_on_same_server, } return ret; } -// Currently only used for mv, if to be used elsewhere, note the OB_NO_READABLE_REPLICA return code, -// Check if the usage is appropriate, as it involves retries. - int ObCandiTableLoc::get_all_servers(common::ObIArray &servers) const { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_phy_table_location_info.h b/src/sql/optimizer/ob_phy_table_location_info.h index 4d67a3b3c..75d55f6a8 100644 --- a/src/sql/optimizer/ob_phy_table_location_info.h +++ b/src/sql/optimizer/ob_phy_table_location_info.h @@ -19,7 +19,7 @@ #include "sql/ob_phy_table_location.h" #include "sql/ob_sql_define.h" -#include "sql/optimizer/ob_route_policy.h" +#include "share/location_cache/ob_location_struct.h" namespace oceanbase { namespace sql @@ -28,22 +28,16 @@ class DASRelatedTabletMap; class ObOptTabletLoc { OB_UNIS_VERSION(1); - //friend class ObPartitionReplicaLocation; public: - typedef common::ObSEArray ObSmartReplicaLocationArray; - ObOptTabletLoc(); virtual ~ObOptTabletLoc(); void reset(); int assign(const ObOptTabletLoc &partition_location); - int assign_with_only_readable_replica(const ObObjectID &partition_id, - const ObObjectID &first_level_part_id, - const common::ObTabletID &tablet_id, - const share::ObLSLocation &partition_location, - const ObRoutePolicyType route_policy); + int assign_local_replica(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, + const common::ObTabletID &tablet_id, + const share::ObLSReplicaLocation &replica); bool is_valid() const; bool operator==(const ObOptTabletLoc &other) const; @@ -66,24 +60,19 @@ class ObOptTabletLoc inline common::ObTabletID get_tablet_id() const { return tablet_id_; } - inline const share::ObLSID &get_ls_id() const { return ls_id_; } - - inline const common::ObIArray &get_replica_locations() const { return replica_locations_; } - - inline common::ObIArray &get_replica_locations() { return replica_locations_; } + inline const share::ObLSReplicaLocation &get_local_replica() const { return local_replica_; } + inline share::ObLSReplicaLocation &get_local_replica() { return local_replica_; } TO_STRING_KV(K_(partition_id), K_(tablet_id), - K_(ls_id), - K_(replica_locations)); + K_(local_replica)); private: int64_t partition_id_; // first level part id, only valid for subpartitioned table int64_t first_level_part_id_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; - ObSmartReplicaLocationArray replica_locations_; + share::ObLSReplicaLocation local_replica_; }; class ObCandiTabletLoc @@ -94,35 +83,18 @@ class ObCandiTabletLoc int assign(const ObCandiTabletLoc &other); - int set_selected_replica_idx(int64_t selected_replica_idx); - int set_selected_replica_idx_with_priority(); - int add_priority_replica_idx(int64_t priority_replica_idx); - int64_t get_selected_replica_idx() const { return selected_replica_idx_; } - bool has_selected_replica() const { return common::OB_INVALID_INDEX != selected_replica_idx_; } - const share::ObLSID &get_ls_id() const { return opt_tablet_loc_.get_ls_id(); } int get_selected_replica(share::ObLSReplicaLocation &replica_loc) const; - int get_selected_replica(ObRoutePolicy::CandidateReplica &replica_loc) const; - int get_priority_replica(int64_t idx, share::ObLSReplicaLocation &replica_loc) const; - int get_priority_replica(int64_t idx, ObRoutePolicy::CandidateReplica &replica_loc) const; - template - int get_priority_replica_base(int64_t selected_replica_idx, T &replica_loc) const; - int set_part_loc_with_only_readable_replica(const ObObjectID &partition_id, - const ObObjectID &first_level_part_id, - const common::ObTabletID &tablet_id, - const share::ObLSLocation &partition_location, - const ObRoutePolicyType route_policy); + int set_local_tablet_loc(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, + const common::ObTabletID &tablet_id, + const share::ObLSReplicaLocation &replica); const ObOptTabletLoc &get_partition_location() const { return opt_tablet_loc_; } ObOptTabletLoc &get_partition_location() { return opt_tablet_loc_; } - const common::ObIArray &get_priority_replica_idxs() const { return priority_replica_idxs_; } - bool is_server_in_replica(const common::ObAddr &server, int64_t &idx) const; - TO_STRING_KV(K_(opt_tablet_loc), K_(selected_replica_idx), K_(priority_replica_idxs)); + bool is_local_server(const common::ObAddr &server) const; + TO_STRING_KV(K_(opt_tablet_loc)); private: ObOptTabletLoc opt_tablet_loc_; - // The result after computing the intersection of all partitions is the index of the finally selected replica - int64_t selected_replica_idx_; - // After priority judgment of all replicas for the current partition, store the replica index with the highest priority here - common::ObSEArray priority_replica_idxs_; private: DISALLOW_COPY_AND_ASSIGN(ObCandiTabletLoc); }; diff --git a/src/sql/optimizer/ob_pwj_comparer.h b/src/sql/optimizer/ob_pwj_comparer.h index 2ffbfee30..8861767fb 100644 --- a/src/sql/optimizer/ob_pwj_comparer.h +++ b/src/sql/optimizer/ob_pwj_comparer.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_SQL_OPTIMIZER_OB_PWJ_COMPARER_H #define OCEANBASE_SQL_OPTIMIZER_OB_PWJ_COMPARER_H 1 #include "lib/container/ob_array.h" -#include "share/partition_table/ob_partition_location.h" #include "sql/optimizer/ob_table_partition_info.h" #include "sql/resolver/expr/ob_raw_expr.h" #include "sql/optimizer/ob_sharding_info.h" diff --git a/src/sql/optimizer/ob_replica_compare.cpp b/src/sql/optimizer/ob_replica_compare.cpp deleted file mode 100644 index d51b6c478..000000000 --- a/src/sql/optimizer/ob_replica_compare.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_OPT -#include "sql/optimizer/ob_replica_compare.h" -using namespace oceanbase::common; -namespace oceanbase -{ -namespace sql -{ - -ObReplicaCompare::ObReplicaCompare(ObRoutePolicyType policy_type) - :ret_(OB_SUCCESS), - policy_type_(policy_type), - readonly_zone_first_{ZONE_TYPE, MERGE_STATUS, POS_TYPE}, - only_readonly_zone_{ZONE_TYPE, MERGE_STATUS, POS_TYPE,}, - unmerge_zone_first_{MERGE_STATUS, ZONE_TYPE, POS_TYPE}, - force_readonly_zone_{ZONE_TYPE, MERGE_STATUS, POS_TYPE} - { - static_assert(sizeof(readonly_zone_first_) == sizeof(only_readonly_zone_), "invalid array size"); - static_assert(sizeof(readonly_zone_first_) == sizeof(unmerge_zone_first_), "invalid array size"); - static_assert(sizeof(readonly_zone_first_) == sizeof(force_readonly_zone_), "invalid array size"); - static_assert((sizeof(readonly_zone_first_)/sizeof(CompareType)) == (sizeof(cmp_func_array_)/sizeof(CmpFuncPtr)), "invalid array size"); - - cmp_func_array_[ZONE_TYPE] = &ObReplicaCompare::compare_zone_type; - cmp_func_array_[MERGE_STATUS] = &ObReplicaCompare::compare_merge_status; - cmp_func_array_[POS_TYPE] = &ObReplicaCompare::compare_pos_type; - } - - -bool ObReplicaCompare::operator()(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2) -{ - int ret = OB_SUCCESS; - bool bool_ret = false; - CompareType *cmp_type_array = NULL; - if (OB_FAIL(ret_)) {//do nothing if we already have an error - } else { - if (READONLY_ZONE_FIRST == policy_type_) { - cmp_type_array = readonly_zone_first_; - } else if (ONLY_READONLY_ZONE == policy_type_) { - cmp_type_array = only_readonly_zone_; - } else if (UNMERGE_ZONE_FIRST == policy_type_) { - cmp_type_array = unmerge_zone_first_; - } else if (FORCE_READONLY_ZONE == policy_type_) { - cmp_type_array = force_readonly_zone_; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected policy type", K(policy_type_), K(ret)); - } - CompareRes cmp_res = EQUAL; - for(int64_t i = 0 ; OB_SUCC(ret) && cmp_res == EQUAL && i < CMP_CNT; ++i) { - CompareType cmp_type = cmp_type_array[i]; - CmpFuncPtr cmp_fun = cmp_func_array_[cmp_type]; - if (OB_ISNULL(cmp_fun)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("cmp fun is NULL", K(cmp_type), K(ret)); - } else { - cmp_res = (this->*cmp_fun)(replica1, replica2); - } - } - bool_ret = (cmp_res == LESS); - ret_ = ret; - } - return bool_ret; -} - -}//sql -}//oceanbase diff --git a/src/sql/optimizer/ob_replica_compare.h b/src/sql/optimizer/ob_replica_compare.h deleted file mode 100644 index df2c781ee..000000000 --- a/src/sql/optimizer/ob_replica_compare.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_SQL_OB_REPLICA_COMPARE_H -#define OCEANBASE_SQL_OB_REPLICA_COMPARE_H -#include "share/ob_define.h" -#include "sql/optimizer/ob_route_policy.h" -namespace oceanbase -{ -namespace sql -{ -class ObReplicaCompare -{ -public: - enum CompareType - { - ZONE_TYPE, - MERGE_STATUS, - POS_TYPE, - CMP_CNT - }; - enum CompareRes - { - EQUAL, - LESS, - GREATER - }; -public: - explicit ObReplicaCompare(ObRoutePolicyType policy_type); - virtual ~ObReplicaCompare() {} - bool operator()(const ObRoutePolicy::CandidateReplica &replica1, const ObRoutePolicy::CandidateReplica &replica2); - int get_sort_ret() const { return ret_; } - - inline CompareRes compare_zone_type(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2); - inline CompareRes compare_merge_status(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2); - inline CompareRes compare_pos_type(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2); - - typedef CompareRes (ObReplicaCompare::*CmpFuncPtr)(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2); -private: - int ret_; - ObRoutePolicyType policy_type_; - CmpFuncPtr cmp_func_array_[CMP_CNT]; - CompareType readonly_zone_first_[CMP_CNT]; - CompareType only_readonly_zone_[CMP_CNT]; - CompareType unmerge_zone_first_[CMP_CNT]; - CompareType force_readonly_zone_[CMP_CNT]; -}; - - -ObReplicaCompare::CompareRes ObReplicaCompare::compare_zone_type(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2) -{ - CompareRes cmp_ret = EQUAL; - if (replica1.attr_.zone_type_ == common::ZONE_TYPE_READONLY - && replica2.attr_.zone_type_ != common::ZONE_TYPE_READONLY) { - cmp_ret = LESS; - } else if (replica1.attr_.zone_type_ != common::ZONE_TYPE_READONLY - && replica2.attr_.zone_type_ == common::ZONE_TYPE_READONLY) { - cmp_ret = GREATER; - } else {/*do nothing*/} - return cmp_ret; -} - -ObReplicaCompare::CompareRes ObReplicaCompare::compare_merge_status(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2) -{ - CompareRes cmp_ret = EQUAL; - if (replica1.attr_.merge_status_ != ObRoutePolicy::MERGING - && replica2.attr_.merge_status_ == ObRoutePolicy::MERGING) { - cmp_ret = LESS; - } else if (replica1.attr_.merge_status_ == ObRoutePolicy::MERGING - && replica2.attr_.merge_status_ != ObRoutePolicy::MERGING) { - cmp_ret = GREATER; - } else {/*do nothing*/} - return cmp_ret; -} - -ObReplicaCompare::CompareRes ObReplicaCompare::compare_pos_type(const ObRoutePolicy::CandidateReplica &replica1, - const ObRoutePolicy::CandidateReplica &replica2) -{ - CompareRes cmp_ret = EQUAL; - return cmp_ret; -} - -}//sql -}//oceanbase -#endif diff --git a/src/sql/optimizer/ob_route_policy.cpp b/src/sql/optimizer/ob_route_policy.cpp deleted file mode 100644 index d6afdccaa..000000000 --- a/src/sql/optimizer/ob_route_policy.cpp +++ /dev/null @@ -1,358 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SQL_OPT -#include "ob_route_policy.h" -#include "sql/optimizer/ob_replica_compare.h" -#include "sql/optimizer/ob_log_plan.h" -using namespace oceanbase::common; -using namespace oceanbase::share; -using namespace oceanbase::storage; -namespace oceanbase -{ -namespace sql -{ - - -int ObRoutePolicy::weak_sort_replicas(ObIArray& candi_replicas, ObRoutePolicyCtx &ctx) -{ - int ret = OB_SUCCESS; - if (candi_replicas.count() > 1) { - auto first = &candi_replicas.at(0); - ObRoutePolicyType policy_type = get_calc_route_policy_type(ctx); - ObReplicaCompare replica_cmp(policy_type); - lib::ob_sort(first, first + candi_replicas.count(), replica_cmp); - if (OB_FAIL(replica_cmp.get_sort_ret())) { - LOG_WARN("fail sort", K(candi_replicas), K(ret)); - } - } - return ret; -} - - -int ObRoutePolicy::filter_replica(const ObAddr &local_server, - const ObLSID &ls_id, - ObIArray& candi_replicas, - ObRoutePolicyCtx &ctx) -{ - int ret = OB_SUCCESS; - ObRoutePolicyType policy_type = get_calc_route_policy_type(ctx); - bool need_break = false; - for (int64_t i = 0; !need_break && OB_SUCC(ret) && i < candi_replicas.count(); ++i) { - CandidateReplica &cur_replica = candi_replicas.at(i); - bool can_read = true; - bool is_local = cur_replica.get_server() == local_server; - - if (is_local && OB_FAIL(ObSqlTransControl::check_ls_readable( - ls_id, - cur_replica.get_server(), - ctx.max_read_stale_time_, - can_read))) { - LOG_WARN("fail to check ls readable", K(ctx), K(cur_replica), K(ret)); - } else { - LOG_TRACE("check ls readable", K(ctx), K(ls_id), K(cur_replica.get_server()), K(can_read)); - if ((policy_type == ONLY_READONLY_ZONE && cur_replica.attr_.zone_type_ == ZONE_TYPE_READWRITE) - || (policy_type == FORCE_READONLY_ZONE && cur_replica.get_replica_type() != REPLICA_TYPE_READONLY) - || cur_replica.attr_.zone_status_ == ObZoneStatus::INACTIVE - || cur_replica.attr_.server_status_ != ObServerStatus::OB_SERVER_ACTIVE - || cur_replica.attr_.start_service_time_ == 0 - || cur_replica.attr_.server_stop_time_ != 0 - || (0 == cur_replica.get_property().get_memstore_percent() - && is_follower(cur_replica.get_role()))// As a Follower, the D role cannot be selected - || !can_read) { - cur_replica.is_filter_ = true; - } - - // if is local replica and can read, filter all replicas and only select this replica. - if (is_local && !cur_replica.is_filter_) { - for (int64_t j = 0; j < candi_replicas.count(); ++j) { - candi_replicas.at(i).is_filter_ = true; - } - cur_replica.is_filter_ = false; - need_break = true; - } - } - } - if (OB_SUCC(ret) && policy_type == FORCE_READONLY_ZONE) { - for (int64_t i = candi_replicas.count() - 1; OB_SUCC(ret) && i >= 0; --i) { - CandidateReplica &cur_replica = candi_replicas.at(i); - if (cur_replica.is_filter_ && OB_FAIL(candi_replicas.remove(i))) { - LOG_WARN("failed to remove filtered replica", K(ret)); - } - } - if (OB_SUCC(ret) && candi_replicas.count() == 0) { - ret = OB_NO_READABLE_REPLICA; - LOG_WARN("all replicas are filted", K(ret), K(policy_type)); - } - } - return ret; -} - -int ObRoutePolicy::calculate_replica_priority(const ObAddr &local_server, - const ObLSID &ls_id, - ObIArray& candi_replicas, - ObRoutePolicyCtx &ctx, - bool is_inner_table) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (candi_replicas.count() <= 1) { - } else if (WEAK == ctx.consistency_level_) { - if (OB_FAIL(filter_replica(local_server, ls_id, candi_replicas, ctx))) { - LOG_WARN("fail to filter replicas", K(candi_replicas), K(ctx), K(ret)); - } else if (OB_FAIL(weak_sort_replicas(candi_replicas, ctx))) { - LOG_WARN("fail to sort replicas", K(candi_replicas), K(ctx), K(ret)); - } - } else if (STRONG == ctx.consistency_level_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("strong consistency cant't be here", K(ctx), K(ret)); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected consistency level_", K(ctx)); - } - return ret; -} - -int ObRoutePolicy::init_candidate_replica(CandidateReplica &candi_replica) -{ - int ret = OB_SUCCESS; - ObServerLocality candi_locality; - candi_replica.attr_.zone_status_ = ObZoneStatus::ACTIVE; - candi_replica.attr_.merge_status_ = NOMERGING; - candi_replica.attr_.pos_type_ = SAME_SERVER; - candi_replica.attr_.zone_type_ = ObZoneType::ZONE_TYPE_READWRITE; - candi_replica.attr_.start_service_time_ = GCTX.start_service_time_; - candi_replica.attr_.server_stop_time_ = 0; - candi_replica.attr_.server_status_ = ObServerStatus::OB_SERVER_ACTIVE; - return ret; -} - -int ObRoutePolicy::init_candidate_replicas(common::ObIArray &candi_replicas) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } - for (int64_t i = 0 ; OB_SUCC(ret) && i < candi_replicas.count(); ++i) { - if (OB_FAIL(init_candidate_replica(candi_replicas.at(i)))) { - LOG_WARN("fail to candidate replica", K(i), K(candi_replicas), K(ret)); - } - } - return ret; -} - -int ObRoutePolicy::select_replica_with_priority(const ObRoutePolicyCtx &route_policy_ctx, - const ObIArray &replica_array, - ObCandiTabletLoc &phy_part_loc_info) -{ - int ret = OB_SUCCESS; - bool has_found = false; - bool same_priority = true; - ReplicaAttribute priority_attr; - for (int64_t i = 0; OB_SUCC(ret) && same_priority && i < replica_array.count(); ++i) { - if (replica_array.at(i).is_usable()/*+meet max_read_stale_time transaction delay*/) { - if (has_found) { - if (priority_attr == replica_array.at(i).attr_) { - if (OB_FAIL(phy_part_loc_info.add_priority_replica_idx(i))) { - LOG_WARN("fail to select replica", K(i), K(priority_attr), K(ret)); - } - } else { - same_priority = false; - } - } else { - if (OB_FAIL(phy_part_loc_info.add_priority_replica_idx(i))) { - LOG_WARN("fail to select replica", K(i), K(ret)); - } else { - has_found = true; - priority_attr = replica_array.at(i).attr_; - } - } - } - } - // In extreme cases, if all contents in replica_array become unreadable, then randomly select one - if (OB_UNLIKELY(false == has_found)) { - int64_t select_idx = rand() % replica_array.count(); - if (OB_FAIL(phy_part_loc_info.add_priority_replica_idx(select_idx))) { - LOG_WARN("fail to select replica", K(select_idx), K(ret)); - } - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) {// Print once every 10 seconds} - LOG_WARN("all replica is not usable currently", K(replica_array), K(route_policy_ctx), K(select_idx)); - } - } - return ret; -} - -int ObRoutePolicy::calc_intersect_repllica(const common::ObIArray &phy_tbl_loc_info_list, - ObList &intersect_server_list) -{ - int ret = OB_SUCCESS; - ObRoutePolicy::CandidateReplica tmp_replica; - bool can_select_one_server = true; - for (int64_t i = 0; OB_SUCC(ret) && can_select_one_server && i < phy_tbl_loc_info_list.count(); ++i) { - const ObCandiTableLoc *phy_tbl_loc_info = phy_tbl_loc_info_list.at(i); - if (OB_ISNULL(phy_tbl_loc_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("phy_tbl_loc_info is NULL", K(ret), K(i), K(phy_tbl_loc_info_list.count())); - } else { - const ObCandiTabletLocIArray &phy_part_loc_info_list = phy_tbl_loc_info->get_phy_part_loc_info_list(); - for (int64_t j = 0; OB_SUCC(ret) && can_select_one_server && j < phy_part_loc_info_list.count(); ++j) { - const ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(j); - const ObIArray &priority_replica_idxs = phy_part_loc_info.get_priority_replica_idxs(); - if (0 == i && 0 == j) { // first partition - if (phy_part_loc_info.has_selected_replica()) { // replica has already been selected - tmp_replica.reset(); - if (OB_FAIL(phy_part_loc_info.get_selected_replica(tmp_replica))) { - LOG_WARN("fail to get selected replica", K(ret), K(phy_part_loc_info)); - } else if (OB_FAIL(intersect_server_list.push_back(tmp_replica))) { - LOG_WARN("fail to push back candidate server", K(ret), K(tmp_replica)); - } - } else { // No replica has been selected yet - for (int64_t k = 0; OB_SUCC(ret) && k < priority_replica_idxs.count(); ++k) { - tmp_replica.reset(); - int64_t replica_idx = priority_replica_idxs.at(k); - if (OB_FAIL(phy_part_loc_info.get_priority_replica(replica_idx, tmp_replica))) { - LOG_WARN("fail to get priority replica", K(k), K(priority_replica_idxs), K(ret)); - } else if (OB_FAIL(intersect_server_list.push_back(tmp_replica))) { - LOG_WARN("fail to push back server ", K(k), K(priority_replica_idxs), K(ret)); - } - } - } - } else {// not the first partition - ObList::iterator intersect_server_list_iter = intersect_server_list.begin(); - for (; OB_SUCC(ret) && intersect_server_list_iter != intersect_server_list.end(); intersect_server_list_iter++) { - const ObAddr &candidate_server = intersect_server_list_iter->get_server(); - bool has_replica = false; - if (phy_part_loc_info.has_selected_replica()) { // replica has already been selected - tmp_replica.reset(); - if (OB_FAIL(phy_part_loc_info.get_selected_replica(tmp_replica))) { - LOG_WARN("fail to get selected replica", K(ret), K(phy_part_loc_info)); - } else if (tmp_replica.get_server() == candidate_server) { - has_replica = true; - } - } else { // Copy not yet selected - for (int64_t k = 0; OB_SUCC(ret) && !has_replica && k < priority_replica_idxs.count(); ++k) { - tmp_replica.reset(); - int64_t replica_idx = priority_replica_idxs.at(k); - if (OB_FAIL(phy_part_loc_info.get_priority_replica(replica_idx, tmp_replica))) { - LOG_WARN("fail to get priority replica", K(k), K(priority_replica_idxs), K(ret)); - } else if (candidate_server == tmp_replica.get_server()) { - has_replica = true; - } - } - } - if (OB_SUCC(ret) && !has_replica) { - if (OB_FAIL(intersect_server_list.erase(intersect_server_list_iter))) { - LOG_WARN("fail to erase from list", K(ret), K(candidate_server)); - } - } - } - if (OB_SUCC(ret) && intersect_server_list.empty()) { - can_select_one_server = false; - } - } - } - } - } - return ret; -} - -int ObRoutePolicy::select_intersect_replica(ObRoutePolicyCtx &route_policy_ctx, - common::ObIArray &phy_tbl_loc_info_list, - ObList &intersect_server_list, - bool &is_proxy_hit) -{ - UNUSED(route_policy_ctx); - int ret = OB_SUCCESS; - if (OB_FAIL(calc_intersect_repllica(phy_tbl_loc_info_list, intersect_server_list))) { - LOG_WARN("fail to calc intersect replica", K(phy_tbl_loc_info_list), K(ret)); - } else if (intersect_server_list.empty()) {//No intersection case, each partition selects replica separately - is_proxy_hit = true; - for (int64_t i = 0; OB_SUCC(ret) && i < phy_tbl_loc_info_list.count(); ++i) { - ObCandiTableLoc *phy_tbl_loc_info = phy_tbl_loc_info_list.at(i); - if (OB_ISNULL(phy_tbl_loc_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("phy_tbl_loc_info is NULL", K(ret), K(i), K(phy_tbl_loc_info_list.count())); - } else { - ObCandiTabletLocIArray &phy_part_loc_info_list = phy_tbl_loc_info->get_phy_part_loc_info_list_for_update(); - for (int64_t j = 0; OB_SUCC(ret) && j < phy_part_loc_info_list.count(); ++j) { - ObCandiTabletLoc &phy_part_loc_info = phy_part_loc_info_list.at(j); - if (phy_part_loc_info.has_selected_replica()) { - // do nothing - } else if (OB_FAIL(phy_part_loc_info.set_selected_replica_idx_with_priority())) { - LOG_WARN("fail to set selected replica idx", K(ret)); - } - } - } - } - } else { - CandidateReplica selected_replica; - ObList::iterator replica_iter = intersect_server_list.begin(); - ObSEArray same_priority_servers; - selected_replica.attr_.pos_type_ = POSITION_TYPE_MAX; - bool is_first = true; - for (; OB_SUCC(ret) && replica_iter != intersect_server_list.end(); replica_iter++) { - if (is_first) { - selected_replica = *replica_iter; - is_first = false; - if (OB_FAIL(same_priority_servers.push_back(replica_iter->get_server()))) { - LOG_WARN("fail to replica iterator", K(replica_iter), K(ret)); - } - } else if (replica_iter->attr_.pos_type_ == selected_replica.attr_.pos_type_) { - // Aggregate same priority servers for random selection later - if (OB_FAIL(same_priority_servers.push_back(replica_iter->get_server()))) { - LOG_WARN("fail to replica iterator", K(replica_iter), K(ret)); - } - } else if (replica_iter->attr_.pos_type_ < selected_replica.attr_.pos_type_) { - selected_replica = *replica_iter; - same_priority_servers.reset(); - if (OB_FAIL(same_priority_servers.push_back(replica_iter->get_server()))) { - LOG_WARN("fail to replica iterator", K(replica_iter), K(ret)); - } - } else { - /* replica_iter->attr_.pos_type_ > selected_replica.attr_.pos_type_ do nothing */ - } - } - - if (OB_SUCC(ret)) {//select server for all partitions of the query - int64_t selected_idx = rand() % same_priority_servers.count(); - const ObAddr &selected_server = same_priority_servers.at(selected_idx); - if (OB_FAIL(ObLogPlan::select_one_server(selected_server, phy_tbl_loc_info_list))) { - LOG_WARN("fail to select one server", K(selected_idx), K(selected_server), K(ret)); - } else { - is_proxy_hit = (local_addr_ == selected_server); - } - } - } - return ret; -} - -int ObRoutePolicy::init() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret)); - } else { - is_inited_ = true; - } - return ret; -} - -}//sql -}//oceanbase diff --git a/src/sql/optimizer/ob_route_policy.h b/src/sql/optimizer/ob_route_policy.h index b1a30cf00..3ef8b65ba 100644 --- a/src/sql/optimizer/ob_route_policy.h +++ b/src/sql/optimizer/ob_route_policy.h @@ -16,205 +16,24 @@ #ifndef OCEANBASE_SQL_OB_ROUTE_POLICY_H #define OCEANBASE_SQL_OB_ROUTE_POLICY_H -#include "share/ob_define.h" -#include "lib/net/ob_addr.h" -#include "lib/container/ob_iarray.h" -#include "lib/list/ob_list.h" -#include "common/ob_zone_status.h" -#include "common/ob_zone_type.h" -#include "share/location_cache/ob_location_struct.h" -#include "share/partition_table/ob_partition_location.h" -#include "share/ob_server_locality_cache.h" -#include "share/ob_server_status.h" + namespace oceanbase { namespace sql { -class ObCandiTabletLoc; -class ObCandiTableLoc; + enum ObRoutePolicyType { INVALID_POLICY = 0, READONLY_ZONE_FIRST = 1, ONLY_READONLY_ZONE = 2, UNMERGE_ZONE_FIRST = 3, - // Only valid in non-read-write separation scenarios, the routing rules on the server side are the same as regular weak reads in non-read-write separation, - // Even if the client routes the request to the partition leader, it is still executed locally, - // The difference lies in the feedback returned to OCJ && ObProxy; UNMERGE_FOLLOWER_FIRST = 4, FORCE_READONLY_ZONE = 5, POLICY_TYPE_MAX }; -struct ObRoutePolicyCtx -{ - ObRoutePolicyCtx() - :policy_type_(POLICY_TYPE_MAX), - consistency_level_(common::INVALID_CONSISTENCY), - max_read_stale_time_(0) - {} - - TO_STRING_KV(K(policy_type_), - K(consistency_level_), - K(max_read_stale_time_)); - - ObRoutePolicyType policy_type_; - common::ObConsistencyLevel consistency_level_; - int64_t max_read_stale_time_; -}; - -class ObRoutePolicy -{ -public: - enum PositionType - { - SAME_SERVER = 0, - POSITION_TYPE_MAX, - }; - enum MergeStatus - { - MERGING, - NOMERGING, - MERGE_STATUS_MAX, - }; - class ReplicaAttribute - { - public: - ReplicaAttribute() - :pos_type_(POSITION_TYPE_MAX), - merge_status_(MERGE_STATUS_MAX), - zone_status_(share::ObZoneStatus::UNKNOWN), - zone_type_(common::ZONE_TYPE_INVALID), - start_service_time_(0), - server_stop_time_(0), - server_status_(share::ObServerStatus::OB_DISPLAY_MAX) - {} - ~ReplicaAttribute(){} - bool operator==(const ReplicaAttribute &other_attr) const - { - bool bool_ret = false; - if (merge_status_ == other_attr.merge_status_ - && zone_type_ == other_attr.zone_type_) { - if (pos_type_ == SAME_SERVER) { - bool_ret = (other_attr.pos_type_ == SAME_SERVER); - } else { - bool_ret = (pos_type_ == other_attr.pos_type_); - } - } - return bool_ret; - } - void reset() - { - pos_type_ = POSITION_TYPE_MAX; - merge_status_ = MERGE_STATUS_MAX; - zone_status_ = share::ObZoneStatus::UNKNOWN; - zone_type_ = common::ZONE_TYPE_INVALID; - start_service_time_ = 0; - server_stop_time_ = 0; - server_status_ = share::ObServerStatus::OB_DISPLAY_MAX; - } - TO_STRING_KV(K(pos_type_), K(merge_status_), K(zone_type_), K(zone_status_), K(start_service_time_), K(server_stop_time_), K(server_status_)); - PositionType pos_type_; - MergeStatus merge_status_; - share::ObZoneStatus::Status zone_status_; - common::ObZoneType zone_type_; - int64_t start_service_time_; - int64_t server_stop_time_; - share::ObServerStatus::DisplayStatus server_status_; - }; - class CandidateReplica final : public share::ObLSReplicaLocation - { - public: - CandidateReplica() - : ObLSReplicaLocation(), - attr_(), - is_filter_(false), - replica_idx_(common::OB_INVALID_INDEX) - {} - CandidateReplica(const share::ObLSReplicaLocation &replica_location) - : ObLSReplicaLocation(replica_location), - attr_(), - is_filter_(false), - replica_idx_(common::OB_INVALID_INDEX) - {} - bool is_usable() const { return is_filter_ == false; } - void reset() - { - ObLSReplicaLocation::reset(); - attr_.reset(); - is_filter_ = false; - } - //to_string only outputs the key information that SQL execution is concerned with, - //and other information can be obtained through the corresponding virtual tables. - TO_STRING_KV(K_(server), - K_(role), - K_(sql_port), - K_(is_filter)); - public: - ReplicaAttribute attr_; - bool is_filter_; - int64_t replica_idx_;//invalid - }; -public: - ObRoutePolicy(const common::ObAddr &addr) - :local_addr_(addr), - has_refresh_locality_(false), - is_inited_(false) - {} - ~ObRoutePolicy() {} - int init(); - int calculate_replica_priority(const ObAddr &local_server, - const share::ObLSID &ls_id, - common::ObIArray& candi_replicas, - ObRoutePolicyCtx &ctx, - bool is_inner_table = false); - int init_candidate_replicas(common::ObIArray &candi_replicas); - int select_replica_with_priority(const ObRoutePolicyCtx &route_policy_ctx, - const common::ObIArray &replica_array, - ObCandiTabletLoc &phy_part_loc_info); - int select_intersect_replica(ObRoutePolicyCtx &route_policy_ctx, - common::ObIArray &phy_tbl_loc_info_list, - common::ObList &intersect_server_list, - bool &is_proxy_hit); - - TO_STRING_KV(K(local_addr_), - K(has_refresh_locality_), - K(is_inited_)); - - inline bool is_follower_first_route_policy_type(const ObRoutePolicyCtx &ctx) const - { - return UNMERGE_FOLLOWER_FIRST == ctx.policy_type_; - } - -protected: - int init_candidate_replica(CandidateReplica &candi_replica); - int calc_intersect_repllica(const common::ObIArray &phy_tbl_loc_info_list, - common::ObList &intersect_server_list); - - int filter_replica(const ObAddr &local_server, - const share::ObLSID &ls_id, - common::ObIArray& candi_replicas, - ObRoutePolicyCtx &ctx); - int weak_sort_replicas(common::ObIArray& candi_replicas, ObRoutePolicyCtx &ctx); - inline ObRoutePolicyType get_calc_route_policy_type(const ObRoutePolicyCtx &ctx) const - { - // When the cluster is a read-write zone, ignore the value of the ob_route_policy system variable, and handle as READONLY_ZONE_FIRST - // When the cluster is a read-write zone, and ob_route_policy is UNMERGE_FOLLOWER_FIRST, it is also handled as READONLY_ZONE_FIRST, but with additional feedback content - // When the cluster has a read-only zone, and ob_route_policy is UNMERGE_FOLLOWER_FIRST, it is also handled as READONLY_ZONE_FIRST, at which point no additional feedback content will be added - ObRoutePolicyType type = INVALID_POLICY; - if (FORCE_READONLY_ZONE == ctx.policy_type_) { - type = FORCE_READONLY_ZONE; - } else { - type = READONLY_ZONE_FIRST; - } - return type; - } -protected: - common::ObAddr local_addr_; - bool has_refresh_locality_; - bool is_inited_; -}; +} // namespace sql +} // namespace oceanbase -}//sql -}//oceanbase #endif diff --git a/src/sql/optimizer/ob_sel_estimator.cpp b/src/sql/optimizer/ob_sel_estimator.cpp index 1aa471bd4..f9feb7f2d 100644 --- a/src/sql/optimizer/ob_sel_estimator.cpp +++ b/src/sql/optimizer/ob_sel_estimator.cpp @@ -24,7 +24,7 @@ namespace oceanbase { namespace sql { -inline double revise_ndv(double ndv) { return ndv < 1.0 ? 1.0 : ndv; } +inline double revise_sel_estimator_ndv(double ndv) { return ndv < 1.0 ? 1.0 : ndv; } void SimpleRange::set_whole_range() { @@ -2664,12 +2664,12 @@ double ObInequalJoinSelEstimator::get_equal_sel(double min1, overlap = 0.0; } if (max1 - min1 > OB_DOUBLE_EPSINON) { - overlap_ndv1 = revise_ndv(ndv1 * overlap / (max1 - min1)) ; + overlap_ndv1 = revise_sel_estimator_ndv(ndv1 * overlap / (max1 - min1)) ; } else { overlap_ndv1 = 1; } if (max2 - min2 > OB_DOUBLE_EPSINON) { - overlap_ndv2 = revise_ndv(ndv2 * overlap / (max2 - min2)) ; + overlap_ndv2 = revise_sel_estimator_ndv(ndv2 * overlap / (max2 - min2)) ; } else { overlap_ndv2 = 1; } diff --git a/src/sql/optimizer/ob_sharding_info.h b/src/sql/optimizer/ob_sharding_info.h index 4ab43cf1e..a18dbd07c 100644 --- a/src/sql/optimizer/ob_sharding_info.h +++ b/src/sql/optimizer/ob_sharding_info.h @@ -21,7 +21,6 @@ #include "lib/container/ob_fixed_array.h" #include "lib/allocator/ob_allocator.h" #include "lib/allocator/page_arena.h" -#include "share/partition_table/ob_partition_location.h" #include "sql/optimizer/ob_table_partition_info.h" #include "sql/resolver/expr/ob_raw_expr.h" #include "sql/optimizer/ob_phy_table_location_info.h" @@ -288,7 +287,7 @@ class ObShardingInfo } else { ret = phy_table_location_info_->get_phy_part_loc_info_list().at(0) .get_partition_location() - .get_replica_locations().count() > 0; + .get_local_replica().is_valid(); } return ret; } diff --git a/src/sql/optimizer/ob_storage_estimator.cpp b/src/sql/optimizer/ob_storage_estimator.cpp index c31bec9cb..666a85e85 100644 --- a/src/sql/optimizer/ob_storage_estimator.cpp +++ b/src/sql/optimizer/ob_storage_estimator.cpp @@ -33,7 +33,7 @@ int ObStorageEstimator::estimate_row_count(const obcall::ObEstPartArg &arg, //est path rows ObTableScanParam param; share::SCN max_readable_scn; - if (OB_FAIL(OB_TS_MGR.get_gts(nullptr, max_readable_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts(max_readable_scn))) { LOG_WARN("failed to get gts", K(ret)); } else { param.frozen_version_ = static_cast(max_readable_scn.get_val_for_sql()); @@ -44,7 +44,6 @@ int ObStorageEstimator::estimate_row_count(const obcall::ObEstPartArg &arg, param.index_id_ = arg.index_params_.at(i).index_id_; param.scan_flag_ = arg.index_params_.at(i).scan_flag_; param.tablet_id_ = arg.index_params_.at(i).tablet_id_; - param.ls_id_ = arg.index_params_.at(i).ls_id_; param.tx_id_ = arg.index_params_.at(i).tx_id_; if (OB_FAIL(storage_estimate_rowcount(param, arg.index_params_.at(i).batch_, @@ -178,8 +177,7 @@ int ObStorageEstimator::storage_estimate_block_count_and_row_count( if (OB_ISNULL(access_service = share::g_mp->access_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(access_service)); - } else if (OB_FAIL(access_service->estimate_block_count_and_row_count(arg.ls_id_, - arg.tablet_id_, + } else if (OB_FAIL(access_service->estimate_block_count_and_row_count(arg.tablet_id_, timeout_us, macro_block_count, micro_block_count, diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 23d4872ee..1dbaf1fb9 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -34,10 +34,6 @@ namespace oceanbase { -namespace share -{ - class ObLSLocation; -} namespace sql { class ObRawExpr; diff --git a/src/sql/optimizer/ob_table_partition_info.cpp b/src/sql/optimizer/ob_table_partition_info.cpp index 22cf27d71..6ba9177a9 100644 --- a/src/sql/optimizer/ob_table_partition_info.cpp +++ b/src/sql/optimizer/ob_table_partition_info.cpp @@ -135,17 +135,6 @@ int ObTablePartitionInfo::calc_phy_table_loc_and_select_leader(ObExecContext &ex } else if (OB_SUCCESS != (tmp_ret = candi_table_loc.assign(candi_table_loc_))) { LOG_WARN("fail to assign", K(tmp_ret), K(candi_table_loc_)); } else { - ObCandiTabletLocIArray &info_array = candi_table_loc.get_phy_part_loc_info_list_for_update(); - - for (int64_t i = 0; i < info_array.count() && OB_SUCCESS == tmp_ret; i++) { - ObCandiTabletLoc &info = info_array.at(i); - if (info.get_partition_location().get_replica_locations().count() <= 0) { - //nothing todo - } else if (OB_SUCCESS != (tmp_ret = info.set_selected_replica_idx(0))) { - LOG_WARN("fail to set selected replica index", KR(ret)); - } - } - if (OB_SUCCESS != tmp_ret) { //nothing todo } else if (OB_SUCCESS != (tmp_ret = task_exec_ctx diff --git a/src/sql/optimizer/stat/ob_basic_stats_estimator.cpp b/src/sql/optimizer/stat/ob_basic_stats_estimator.cpp index 67b486738..c33dc92f7 100644 --- a/src/sql/optimizer/stat/ob_basic_stats_estimator.cpp +++ b/src/sql/optimizer/stat/ob_basic_stats_estimator.cpp @@ -376,7 +376,6 @@ int ObBasicStatsEstimator::do_estimate_block_count_and_row_count(ObExecContext & arg_element.tablet_id_ = candi_tablet_locs.at(j).get_partition_location().get_tablet_id(); - arg_element.ls_id_ = candi_tablet_locs.at(j).get_partition_location().get_ls_id(); if (OB_FAIL(arg.tablet_params_arg_.push_back(arg_element))) { LOG_WARN("failed to push back", K(ret)); } else if (OB_FAIL(skip_idx_set.add_member(j))) {//record diff --git a/src/sql/optimizer/stat/ob_opt_stat_manager.cpp b/src/sql/optimizer/stat/ob_opt_stat_manager.cpp index 57e24ad2e..e88ae8d83 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_manager.cpp +++ b/src/sql/optimizer/stat/ob_opt_stat_manager.cpp @@ -874,10 +874,9 @@ int ObOptStatManager::flush_evals(ObIAllocator *alloc, int ObOptStatManager::get_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt) { - return stat_service_.get_table_rowcnt(table_id, all_tablet_ids, all_ls_ids, table_rowcnt); + return stat_service_.get_table_rowcnt(table_id, all_tablet_ids, table_rowcnt); } //we need check the stat tables are valid, now we only check the stat table are exist. in some situation, diff --git a/src/sql/optimizer/stat/ob_opt_stat_manager.h b/src/sql/optimizer/stat/ob_opt_stat_manager.h index e76b7cfa2..b12337c47 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_manager.h +++ b/src/sql/optimizer/stat/ob_opt_stat_manager.h @@ -169,7 +169,6 @@ class ObOptStatManager int get_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt); static ObOptStatManager &get_instance() diff --git a/src/sql/optimizer/stat/ob_opt_stat_service.cpp b/src/sql/optimizer/stat/ob_opt_stat_service.cpp index f297ce487..91f5ef915 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_service.cpp +++ b/src/sql/optimizer/stat/ob_opt_stat_service.cpp @@ -379,17 +379,15 @@ int ObOptStatService::init_table_stats(ObIAllocator &allocator, int ObOptStatService::get_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt) { int ret = OB_SUCCESS; table_rowcnt = 0; - if (OB_UNLIKELY(all_tablet_ids.count() != all_ls_ids.count())) { + if (OB_UNLIKELY(all_tablet_ids.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error"); + LOG_WARN("tablet ids is empty", K(ret), K(all_tablet_ids)); } else { ObSEArray reload_tablet_ids; - ObSEArray reload_ls_ids; storage::ObTabletStat unused_tablet_stat; share::schema::ObTableModeFlag unused_mode; for (int64_t i = 0; OB_SUCC(ret) && i < all_tablet_ids.count(); ++i) { @@ -399,8 +397,7 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, // we need to fetch statistics from inner table if it is not yet available from cache if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("get table stat from cache failed", K(ret), K(key)); - } else if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i))) || - OB_FAIL(reload_ls_ids.push_back(all_ls_ids.at(i)))) { + } else if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i)))) { LOG_WARN("failed to push back", K(ret)); } } else if (OB_ISNULL(handle.stat_)) { @@ -408,8 +405,7 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, LOG_WARN("cache hit but value is NULL. BUG here.", K(ret), K(key)); //check is stale } else if (handle.stat_->is_arrived_expired_time()) { - if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i))) || - OB_FAIL(reload_ls_ids.push_back(all_ls_ids.at(i)))) { + if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i)))) { LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } else { @@ -417,9 +413,9 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, storage::ObTabletStat tablet_stat; //try check the latest tablet stat from stroage if (stat_mgr != NULL) { - if (OB_FAIL(stat_mgr->get_latest_tablet_stat(all_ls_ids.at(i), all_tablet_ids.at(i), tablet_stat, unused_tablet_stat, unused_mode))) { + if (OB_FAIL(stat_mgr->get_latest_tablet_stat(all_tablet_ids.at(i), tablet_stat, unused_tablet_stat, unused_mode))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get latest tablet stat", K(ret), K(all_ls_ids.at(i)), K(all_tablet_ids.at(i))); + LOG_WARN("failed to get latest tablet stat", K(ret), K(all_tablet_ids.at(i))); } else { ret = OB_SUCCESS; } @@ -427,8 +423,7 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, } LOG_TRACE("cache stat compare", KPC(handle.stat_), K(tablet_stat)); if (handle.stat_->get_row_count() < tablet_stat.insert_row_cnt_ - tablet_stat.delete_row_cnt_) { - if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i))) || - OB_FAIL(reload_ls_ids.push_back(all_ls_ids.at(i)))) { + if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i)))) { LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } else { @@ -438,8 +433,7 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, } if (OB_SUCC(ret) && !reload_tablet_ids.empty()) { int64_t reload_row_cnt = 0; - if (OB_FAIL(load_table_rowcnt_and_put_cache( table_id, reload_tablet_ids, - reload_ls_ids, reload_row_cnt))) { + if (OB_FAIL(load_table_rowcnt_and_put_cache( table_id, reload_tablet_ids, reload_row_cnt))) { LOG_WARN("load and put cache table stat failed.", K(ret)); } else { table_rowcnt += reload_row_cnt; @@ -453,7 +447,6 @@ int ObOptStatService::get_table_rowcnt(const uint64_t table_id, int ObOptStatService::load_table_rowcnt_and_put_cache(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt) { int ret = OB_SUCCESS; @@ -463,7 +456,7 @@ int ObOptStatService::load_table_rowcnt_and_put_cache(const uint64_t table_id, ret = OB_NOT_INIT; LOG_WARN("table statistics service is not initialized. ", K(ret)); } else if (OB_FAIL(sql_service_.fetch_table_rowcnt(table_id, - all_tablet_ids, all_ls_ids, + all_tablet_ids, tstats))) { if (!is_sys_table(table_id) && !is_virtual_table(table_id)) {//sys table and virtual table failed, use default LOG_WARN("failed to fetch table rowcnt ", K(ret)); diff --git a/src/sql/optimizer/stat/ob_opt_stat_service.h b/src/sql/optimizer/stat/ob_opt_stat_service.h index 402365ade..1bb0f0b65 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_service.h +++ b/src/sql/optimizer/stat/ob_opt_stat_service.h @@ -57,7 +57,6 @@ class ObOptStatService int get_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt); int get_system_stat(const ObOptSystemStat::Key &key, @@ -82,7 +81,6 @@ class ObOptStatService int load_table_rowcnt_and_put_cache(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, int64_t &table_rowcnt); int init_table_stats(ObIAllocator &allocator, diff --git a/src/sql/optimizer/stat/ob_opt_stat_sql_service.cpp b/src/sql/optimizer/stat/ob_opt_stat_sql_service.cpp index 4164f3f10..b4a7bff69 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_sql_service.cpp +++ b/src/sql/optimizer/stat/ob_opt_stat_sql_service.cpp @@ -1886,16 +1886,15 @@ int ObOptStatSqlService::batch_update_online_col_state(const uint64_t table_id, int ObOptStatSqlService::fetch_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObIArray &tstats) { int ret = OB_SUCCESS; ObSqlString raw_sql; ObSqlString tablet_list_str; - ObSqlString tablet_ls_list_str; + ObSqlString tablet_tuple_list_str; uint64_t real_table_id = share::is_real_table_mapping_virtual_table(table_id) ? ObSchemaUtils::get_real_table_mappings_tid(table_id) : table_id; - if (OB_FAIL(gen_tablet_list_str(all_tablet_ids, all_ls_ids, tablet_list_str, tablet_ls_list_str))) { + if (OB_FAIL(gen_tablet_list_str(all_tablet_ids, tablet_list_str, tablet_tuple_list_str))) { LOG_WARN("failed to gen tablet list str", K(ret)); } else if (OB_FAIL(raw_sql.append_fmt("select /*+opt_param('enable_in_range_optimization','true') opt_param('use_default_opt_stat','true')*/"\ "tablet_id, max(row_count) from "\ @@ -1909,7 +1908,7 @@ int ObOptStatSqlService::fetch_table_rowcnt(const uint64_t table_id, tablet_list_str.ptr(), share::OB_ALL_TABLET_CHECKSUM_TNAME, share::OB_ALL_FREEZE_INFO_TNAME, - tablet_ls_list_str.ptr()))) { + tablet_tuple_list_str.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); } else { SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { @@ -1961,14 +1960,13 @@ int ObOptStatSqlService::fetch_table_rowcnt(const uint64_t table_id, } int ObOptStatSqlService::gen_tablet_list_str(const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObSqlString &tablet_list_str, - ObSqlString &tablet_ls_list_str) + ObSqlString &tablet_tuple_list_str) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(all_tablet_ids.empty() || all_tablet_ids.count() != all_ls_ids.count())) { + if (OB_UNLIKELY(all_tablet_ids.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(all_tablet_ids), K(all_ls_ids)); + LOG_WARN("get unexpected error", K(ret), K(all_tablet_ids)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < all_tablet_ids.count(); ++i) { char prefix = i == 0 ? '(' : ' '; @@ -1978,10 +1976,10 @@ int ObOptStatSqlService::gen_tablet_list_str(const ObIArray &all_tab all_tablet_ids.at(i).id(), suffix))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_FAIL(tablet_ls_list_str.append_fmt("%c(%lu)%c", - prefix, - all_tablet_ids.at(i).id(), - suffix))) { + } else if (OB_FAIL(tablet_tuple_list_str.append_fmt("%c(%lu)%c", + prefix, + all_tablet_ids.at(i).id(), + suffix))) { LOG_WARN("failed to append fmt", K(ret)); } else {/*do nothing*/} } diff --git a/src/sql/optimizer/stat/ob_opt_stat_sql_service.h b/src/sql/optimizer/stat/ob_opt_stat_sql_service.h index 9bf6385cb..f541371cd 100644 --- a/src/sql/optimizer/stat/ob_opt_stat_sql_service.h +++ b/src/sql/optimizer/stat/ob_opt_stat_sql_service.h @@ -193,7 +193,6 @@ class ObOptStatSqlService int fetch_table_rowcnt(const uint64_t table_id, const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObIArray &tstats); int update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat); @@ -266,9 +265,8 @@ class ObOptStatSqlService ObObjMeta &endpoint_meta); int gen_tablet_list_str(const ObIArray &all_tablet_ids, - const ObIArray &all_ls_ids, ObSqlString &tablet_list_str, - ObSqlString &tablet_ls_list_str); + ObSqlString &tablet_tuple_list_str); int get_gather_stat_value(const ObOptStatGatherStat &gather_stat, ObSqlString &values_list); diff --git a/src/sql/parser/gen_type_name.sh b/src/sql/parser/gen_type_name.sh index f040314c5..0ac977441 100755 --- a/src/sql/parser/gen_type_name.sh +++ b/src/sql/parser/gen_type_name.sh @@ -2,5 +2,5 @@ echo -e '#include "sql/parser/ob_item_type.h"' echo -e "const char* get_type_name(int type)\n{" echo -e "\tswitch(type){" -sed -rn 's/\s*(T_[_A-Z1-9]+)[ =0-9]*,/\tcase \1 : return \"\1\";/p' $1 +sed -rn 's/\s*(T_[_A-Z0-9]+)[ =0-9]*,/\tcase \1 : return \"\1\";/p' $1 echo -e '\tdefault:return "Unknown";\n\t}\n}' diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 47b57aa26..90cf7347d 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -471,7 +471,6 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"lag", LAG}, {"language", LANGUAGE}, {"last", LAST}, - {"last_refresh_scn", LAST_REFRESH_SCN}, {"last_value", LAST_VALUE}, {"lateral", LATERAL}, {"lead", LEAD}, @@ -517,7 +516,6 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"longtext", LONGTEXT}, {"loop", LOOP}, {"low_priority", LOW_PRIORITY}, - {"ls", LS}, {"major", MAJOR}, {"manhattan", MANHATTAN}, {"manual", MANUAL}, @@ -1081,7 +1079,6 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"var_samp", VAR_SAMP}, {"verbose", VERBOSE}, {"verify", VERIFY}, - {"materialized", MATERIALIZED}, {"validate", VALIDATE}, {"values", VALUES}, {"varying", VARYING}, diff --git a/src/sql/parser/ob_item_type.h b/src/sql/parser/ob_item_type.h index 1c426d335..cad94f7d6 100644 --- a/src/sql/parser/ob_item_type.h +++ b/src/sql/parser/ob_item_type.h @@ -981,7 +981,6 @@ typedef enum ObItemType T_FUN_SYS_GTID_SUBTRACT = 2013, T_FUN_SYS_WAIT_FOR_EXECUTED_GTID_SET = 2014, T_FUN_SYS_WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS = 2015, - T_FUN_SYS_LAST_REFRESH_SCN = 2016, T_FUN_SUM_OPNSIZE = 2017, T_FUN_SYS_GET_LOCK = 2018, T_FUN_SYS_IS_FREE_LOCK = 2019, @@ -1233,7 +1232,7 @@ typedef enum ObItemType T_INDEX_TABLE_ID = 3390, T_TABLET_ID = 3391, T_TENANT_TABLET = 3392, - T_TENANT_LS_TABLET = 3393, + T_RESERVED_3393 = 3393, T_VIRTUAL_COLUMN_ID = 3394, T_DATABASE_ID = 3395, T_REPLICA_NUM = 3396, @@ -1620,13 +1619,13 @@ typedef enum ObItemType T_ERR_LOG_LIMIT = 3780, T_DEFAULT_INT = 3781, T_DEFAULT_NULL = 3782, - T_LS = 3783, + T_RESERVED_3783 = 3783, T_PARTITION_ID_SERVER = 3784, T_SERVER_TENANT = 3785, - T_LS_SERVER_TENANT = 3786, + T_RESERVED_3786 = 3786, T_ZONE_TENANT = 3787, T_CREATE_TIMESTAMP = 3790, - T_RECYCLE_REPLICA = 3791, + T_RESERVED_3791 = 3791, T_MERGE_CONTROL = 3792, T_TENANT_NAME = 3793, T_CACHE_NAME = 3794, @@ -1857,8 +1856,8 @@ typedef enum ObItemType T_EVENT_JOB_ALTER = 4023, T_EVENT_JOB_DROP = 4024, - T_CLEAR_BALANCE_TASK = 4025, - T_BALANCE_TASK_TYPE = 4026, + T_RESERVED_4025 = 4025, + T_RESERVED_4026 = 4026, T_CREATE_SYNONYM = 4027, T_DROP_SYNONYM = 4028, T_CREATE_FUNC = 4029, @@ -2159,7 +2158,6 @@ typedef enum ObItemType T_AUDIT_CLUSTER = 4303, T_AUDIT_CONTEXT = 4304, T_AUDIT_INDEX = 4306, - T_AUDIT_MATERIALIZED_VIEW = 4307, T_AUDIT_NOT_EXIST = 4308, T_AUDIT_OUTLINE = 4309, T_AUDIT_PROCEDURE = 4310, @@ -2457,34 +2455,6 @@ typedef enum ObItemType T_NO_DECORRELATE = 4580, T_COLUMN_ADD_WITH_LOB_PARAMS = 4582, - //for materialized view - T_MV_REFRESH_INFO = 4583, - T_MV_REFRESH_ON_CLAUSE = 4584, - T_MV_REFRESH_METHOD = 4585, - T_MV_REFRESH_INTERVAL = 4586, - T_MV_REFRESH_START_EXPR = 4587, - - // for materialized view log - T_CREATE_MLOG = 4588, - T_MLOG_WITH = 4589, - T_MLOG_WITH_VALUES = 4590, - T_MLOG_WITH_SPECIAL_COLUMN_LIST = 4591, - T_MLOG_WITH_SPECIAL_COLUMN = 4592, - T_MLOG_WITH_REFERENCE_COLUMN = 4593, - T_MLOG_WITH_PRIMARY_KEY = 4594, - T_MLOG_WITH_ROWID = 4595, - T_MLOG_WITH_SEQUENCE = 4596, - T_MLOG_NEW_VALUES = 4597, - T_MLOG_INCLUDING_NEW_VALUES = 4598, - T_MLOG_EXCLUDING_NEW_VALUES = 4599, - T_MLOG_PURGE = 4600, - T_MLOG_PURGE_IMMEDIATE = 4601, - T_MLOG_PURGE_IMMEDIATE_SYNC = 4602, - T_MLOG_PURGE_IMMEDIATE_ASYNC = 4603, - T_MLOG_PURGE_START_NEXT = 4604, - T_MLOG_PURGE_START_TIME_EXPR = 4605, - T_DROP_MLOG = 4606, - T_COL_SKIP_INDEX = 4607, T_COL_SKIP_INDEX_LIST = 4608, T_COL_SKIP_INDEX_MIN_MAX = 4609, @@ -2520,9 +2490,6 @@ typedef enum ObItemType T_PARALLEL_DAS_DML = 4651, T_DISABLE_PARALLEL_DAS_DML = 4652, T_ENABLE_LOB_PREFETCH = 4653, - T_MV_OPTIONS = 4654, - T_MV_REWRITE = 4655, - T_MV_NO_REWRITE = 4656, // select into outfile T_INTO_FILE_LIST = 4657, T_SINGLE_OPT = 4658, @@ -2561,9 +2528,6 @@ typedef enum ObItemType T_VEC_INDEX_COLUMN = 4691, T_VEC_INDEX_COLUMN_EXPR = 4692, // 4693: T_BACKUP_CLUSTER_PARAMETERS abandoned, id reserved - // create mv build deferred - T_MV_REFRESH_OPT = 4694, - T_MV_BUILD_OPT = 4695, T_PSEUDO_EXTERNAL_FILE_ROW = 4696, T_EXTERNAL_TABLE_AUTO_REFRESH = 4697, @@ -2624,8 +2588,6 @@ typedef enum ObItemType T_UNION_MERGE_HINT = 4740, T_UNION_MERGE_LIST = 4741, - T_PSEUDO_OLD_NEW_COL = 4742, - T_TRANSFORM_DISTINCT_AGG = 4743, T_NO_TRANSFORM_DISTINCT_AGG = 4744, @@ -2643,8 +2605,8 @@ typedef enum ObItemType T_TG_SYS_EVENT = 4754, T_TG_DB_EVENT_LIST = 4755, T_FILE_EXTENSION = 4756, - T_LS_ATTR_LIST = 4757, - T_ALTER_LS = 4758, + T_RESERVED_4757 = 4757, + T_RESERVED_4758 = 4758, T_UNIT_GROUP = 4759, T_TRANSPOSE_TABLE = 4760, T_FUN_UNPIVOT = 4761, @@ -2679,7 +2641,6 @@ typedef enum ObItemType T_PARSER_DICT_TABLE = 4783, T_PARSER_QUANTIFIER_TABLE = 4784, - T_ALTER_MLOG_OPTIONS = 4785, // macro block bloom filter T_ENABLE_MACRO_BLOCK_BLOOM_FILTER = 4786, T_DISTRIBUTE_HASH_LOCAL = 4787, @@ -2790,7 +2751,6 @@ typedef enum ObItemType T_UDF_PROPERTY_LIST = 4866, T_MICRO_BLOCK_FORMAT_VERSION = 4867, - T_MV_NESTED_REFRESH_CLAUSE = 4868, // placeholder for mview T_ALTER_SUBPARTITION_EXCHANGE = 4869, T_CREATE_SENSITIVE_RULE = 4870, T_DROP_SENSITIVE_RULE = 4871, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 886d10c23..be1a20a33 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -186,7 +186,7 @@ COALESCE_SQ NO_COALESCE_SQ COUNT_TO_EXISTS NO_COUNT_TO_EXISTS LEFT_TO_ANTI NO_LE ELIMINATE_JOIN NO_ELIMINATE_JOIN PUSH_LIMIT NO_PUSH_LIMIT PULLUP_EXPR NO_PULLUP_EXPR WIN_MAGIC NO_WIN_MAGIC AGGR_FIRST_UNNEST NO_AGGR_FIRST_UNNEST JOIN_FIRST_UNNEST NO_JOIN_FIRST_UNNEST COALESCE_AGGR NO_COALESCE_AGGR WITH_PULLUP WO_PULLUP -MV_REWRITE NO_MV_REWRITE TRANSFORM_DISTINCT_AGG NO_TRANSFORM_DISTINCT_AGG PUSH_SUBQ NO_PUSH_SUBQ +TRANSFORM_DISTINCT_AGG NO_TRANSFORM_DISTINCT_AGG PUSH_SUBQ NO_PUSH_SUBQ DECORRELATE NO_DECORRELATE // optimize hint INDEX_HINT INDEX_ASC_HINT INDEX_DESC_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT UNION_MERGE_HINT @@ -314,9 +314,9 @@ END_P SET_VAR DELIMITER KEYWORD KEY_BLOCK_SIZE KEY_VERSION KEYTAB KRB5CONF KVCACHE - LAG LANGUAGE LAST LAST_REFRESH_SCN LAST_VALUE LATERAL LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ + LAG LANGUAGE LAST LAST_VALUE LATERAL LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ LISTAGG LOB_INROW_THRESHOLD LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS - LEVEL LN LOG LS LINK LOG_RESTORE_SOURCE LINE_DELIMITER LOCATIONS + LEVEL LN LOG LINK LOG_RESTORE_SOURCE LINE_DELIMITER LOCATIONS MAJOR MAP MANHATTAN MANUAL MASTER MASTER_AUTO_POSITION MASTER_CONNECT_RETRY MASTER_DELAY MASTER_HEARTBEAT_PERIOD MASTER_HOST MASTER_LOG_FILE MASTER_LOG_POS MASTER_PASSWORD MASTER_PORT MASTER_RETRY_COUNT @@ -326,7 +326,7 @@ END_P SET_VAR DELIMITER MAX_UPDATES_PER_HOUR MAX_USER_CONNECTIONS MEDIUM MEMORY MEMTABLE MESSAGE_TEXT META MICROSECOND MIGRATE MIN MIN_CPU MIN_IOPS MIN_MAX MIN_NGRAM_SIZE MIN_TOKEN_SIZE MINOR MIN_ROWS MINUS MINUTE MISMATCH MODE MODIFY MODULE MONTH MOVE MODEL MULTILINESTRING MULTIPOINT MULTIPOLYGON MULTIVALUE MUTEX MYSQL_ERRNO MIGRATION MAX_USED_PART_ID MAXIMIZE - MATERIALIZED MEMBER MEMSTORE_PERCENT MINVALUE MY_NAME MERGE_ENGINE + MEMBER MEMSTORE_PERCENT MINVALUE MY_NAME MERGE_ENGINE NAME NAMES NAMESPACE NATIONAL NCHAR NDB NDBCLUSTER NESTED NEW NEXT NO NOAUDIT NODEGROUP NONE NORMAL NOW NOWAIT NEVER NGRAM_TOKEN_SIZE NOMINVALUE NOMAXVALUE NOORDER NOCYCLE NOCACHE NO_WAIT NULLS NUMBER NVARCHAR NTILE NTH_VALUE NOARCHIVELOG NETWORK NET_BANDWIDTH_WEIGHT NOPARALLEL @@ -423,7 +423,7 @@ END_P SET_VAR DELIMITER %type sort_list sort_key opt_asc_desc sort_list_for_group_by sort_key_for_group_by opt_asc_desc_for_group_by opt_column_id sort_list_opt_null sort_key_opt_null opt_asc_desc_null %type opt_query_expression_option_list query_expression_option_list query_expression_option opt_distinct opt_distinct_or_all opt_separator projection %type from_list table_references table_reference table_factor normal_relation_factor dot_relation_factor relation_factor -%type relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list relation_factor_in_mv_hint_list opt_relation_factor_in_mv_hint_list +%type relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list %type relation_factor_in_leading_hint_list joined_table tbl_name table_subquery table_subquery_alias %type relation_factor_with_star relation_with_star_list opt_with_star %type index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list opt_index_prefix union_merge_list @@ -461,7 +461,7 @@ END_P SET_VAR DELIMITER %type opt_with_consistent_snapshot opt_config_scope opt_index_keyname opt_full opt_mode_flag opt_extended opt_extended_or_full %type opt_priority opt_low_priority delete_option delete_option_list opt_delete_option_list %type opt_work begin_stmt commit_stmt rollback_stmt opt_ignore opt_ignore_or_replace ignore_or_replace xa_begin_stmt xa_end_stmt xa_prepare_stmt xa_commit_stmt xa_rollback_stmt xa_recover_stmt xa_xid opt_join_or_resume opt_suspend opt_one_phase opt_convert_xid -%type alter_table_stmt alter_table_actions alter_table_action_list alter_table_action alter_column_option alter_index_option alter_constraint_option standalone_alter_action alter_partition_option opt_to alter_tablegroup_option opt_table opt_tablegroup_option_list alter_tg_partition_option alter_mview_stmt alter_mview_actions alter_mview_action_list alter_mview_action alter_mlog_stmt alter_mlog_actions alter_mlog_action_list alter_mlog_action +%type alter_table_stmt alter_table_actions alter_table_action_list alter_table_action alter_column_option alter_index_option alter_constraint_option standalone_alter_action alter_partition_option opt_to alter_tablegroup_option opt_table opt_tablegroup_option_list alter_tg_partition_option %type tablegroup_option_list tablegroup_option alter_tablegroup_actions alter_tablegroup_action tablegroup_option_list_space_seperated %type opt_tg_partition_option tg_hash_partition_option tg_key_partition_option tg_range_partition_option tg_subpartition_option tg_list_partition_option %type alter_column_behavior opt_set opt_position_column @@ -471,8 +471,6 @@ END_P SET_VAR DELIMITER %type opt_hint_list hint_option select_with_opt_hint update_with_opt_hint delete_with_opt_hint hint_list_with_end global_hint transform_hint optimize_hint %type create_index_stmt index_name sort_column_list sort_column_key opt_index_option_list opt_fulltext_index_option_list index_option fulltext_index_option opt_sort_column_key_length opt_index_using_algorithm index_using_algorithm visibility_option opt_constraint_name constraint_name create_with_opt_hint index_expr alter_with_opt_hint %type opt_when check_state constraint_definition -%type create_mlog_stmt opt_mlog_option_list opt_mlog_options mlog_option opt_mlog_with mlog_with_values mlog_with_special_columns mlog_with_reference_columns mlog_with_special_column_list mlog_with_reference_column_list mlog_with_special_column mlog_with_reference_column opt_mlog_new_values mlog_including_or_excluding opt_mlog_purge mlog_purge_values mlog_purge_immediate_sync_or_async mlog_purge_start mlog_purge_next -%type drop_mlog_stmt %type unreserved_keyword unreserved_keyword_normal unreserved_keyword_special unreserved_keyword_extra unreserved_keyword_ambiguous_roles unreserved_keyword_for_role_name %type mysql_reserved_keyword %type consistency_level use_plan_cache_type @@ -480,12 +478,10 @@ END_P SET_VAR DELIMITER %type drop_index_stmt hint_options opt_expr_as_list expr_as_list expr_with_opt_alias substr_params opt_comma substr_or_substring %type /*frozen_type*/ opt_binary %type ip_port -%type create_view_stmt view_name opt_column_list opt_mv_column_list mv_column_list opt_table_id opt_tablet_id view_select_stmt opt_check_option opt_tablet_id_no_empty -%type create_mview_stmt create_mview_opts mview_refresh_opt mv_refresh_on_clause mv_refresh_mode mv_refresh_interval mv_start_clause mv_next_clause mv_nested_refresh_opt mview_nested_refresh_mode -%type mv_refresh_method mview_enable_disable +%type create_view_stmt view_name opt_column_list opt_table_id opt_tablet_id view_select_stmt opt_check_option opt_tablet_id_no_empty %type name_list %type zone_desc opt_zone_desc server_or_zone opt_server_or_zone opt_partitions opt_subpartitions -%type ls opt_tenant_list_or_ls_or_tablet_id +%type opt_tenant_list_or_tablet_id %type opt_tenant_list_v2 %type suspend_or_resume tenant_name opt_tenant_name cache_name opt_cache_name file_id opt_file_id cancel_task_type %type sql_id_or_schema_id_expr opt_sql_id_or_schema_id outline_type @@ -639,8 +635,6 @@ stmt: | variable_set_stmt { $$ = $1; question_mark_issue($$, result); } | execute_stmt { $$ = $1; check_question_mark($$, result); } | alter_table_stmt { $$ = $1; check_question_mark($$, result); } - | alter_mview_stmt { $$ = $1; check_question_mark($$, result); } - | alter_mlog_stmt { $$ = $1; check_question_mark($$, result); } | alter_system_stmt { $$ = $1; check_question_mark($$, result); } | deallocate_prepare_stmt { $$ = $1; check_question_mark($$, result); } | create_user_stmt { $$ = $1; check_question_mark($$, result); } @@ -659,8 +653,6 @@ stmt: | create_index_stmt { $$ = $1; check_question_mark($$, result); } | drop_index_stmt { $$ = $1; check_question_mark($$, result); } | kill_stmt { $$ = $1; question_mark_issue($$, result); } - | create_mlog_stmt { $$ = $1; check_question_mark($$, result); } - | drop_mlog_stmt { $$ = $1; check_question_mark($$, result); } | help_stmt { $$ = $1; check_question_mark($$, result); } | create_view_stmt { @@ -681,7 +673,6 @@ stmt: $$->value_ = result->question_mark_ctx_.count_; } } - | create_mview_stmt { $$ = $1; check_question_mark($$, result); } | create_tablegroup_stmt { $$ = $1; check_question_mark($$, result); } | drop_tablegroup_stmt { $$ = $1; check_question_mark($$, result); } | alter_tablegroup_stmt { $$ = $1; check_question_mark($$, result); } @@ -3243,10 +3234,6 @@ MOD '(' expr ',' expr ')' malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_ST_ASMVT, 5, $3, $5, $7, $9, $11); $$->reserved_ = 0; } -| LAST_REFRESH_SCN '(' INTNUM ')' -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_LAST_REFRESH_SCN, 1, $3); -} | SUM_OPNSIZE '(' expr ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SUM_OPNSIZE, 2, NULL, $3); @@ -8498,16 +8485,14 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW UNUSED($3); UNUSED($4); UNUSED($5); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, - NULL, /* opt_materialized, not support*/ + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 7, $7, /* view name */ $8, /* column list */ $9, /* table_id */ $11, /* select_stmt */ $2, $12, /* with option */ - NULL, /* force view opt */ - NULL, NULL, NULL, NULL, NULL + NULL /* force view opt */ ); dup_expr_string($11, result, @11.first_column, @11.last_column); $$->reserved_ = 0; /* is create view */ @@ -8519,366 +8504,20 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW UNUSED($2); UNUSED($3); UNUSED($4); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, - NULL, /* opt_materialized */ + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 7, $6, /* view name */ $7, /* column list */ $8, /* table_id */ $10, /* select_stmt */ NULL, $11, /* with option */ - NULL, /* force view opt */ - NULL, NULL, NULL, NULL, NULL + NULL /* force view opt */ ); dup_expr_string($10, result, @10.first_column, @10.last_column); $$->reserved_ = 1; /* is alter view */ } ; -/***************************************************************************** - * - * create materialized view - * - *****************************************************************************/ -create_mview_stmt: -create_with_opt_hint MATERIALIZED VIEW view_name opt_mv_column_list opt_table_option_list opt_partition_option create_mview_opts -AS view_select_stmt opt_check_option -{ - ParseNode *table_options = NULL; - merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); - - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, - NULL, /* opt_materialized, not support*/ - $4, /* view name */ - $5, /* column list */ - NULL, /* table_id */ - $10, /* select_stmt */ - NULL, - $11, /* with option */ - NULL, /* force view opt */ - $8, /* mview options */ - $7, /* partition option */ - table_options, /* table options */ - $1, /* hint */ - NULL /* reserved */ - ); - dup_expr_string($10, result, @10.first_column, @10.last_column); - $$->reserved_ = 2; /* create materialized view */ -} -; - -create_mview_opts: -mview_refresh_opt -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); - $$->value_ = 0; -} -| mview_refresh_opt mview_enable_disable ON QUERY COMPUTATION -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); - $$->value_ = $2[0]; -} -| mview_refresh_opt mview_enable_disable QUERY REWRITE -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); - $$->value_ = $2[0] << 1; -} -| mview_refresh_opt mview_enable_disable ON QUERY COMPUTATION mview_enable_disable QUERY REWRITE -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); - $$->value_ = $2[0] | ($6[0] << 1); -} -| mview_refresh_opt mview_enable_disable QUERY REWRITE mview_enable_disable ON QUERY COMPUTATION -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); - $$->value_ = $5[0] | ($2[0] << 1); -} -; - -mview_enable_disable: -DISABLE -{ - $$[0] = 0; -} -| ENABLE -{ - $$[0] = 1; -} -; - -mview_refresh_opt: -REFRESH mv_refresh_method mv_nested_refresh_opt mv_refresh_on_clause mv_refresh_interval -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_INFO, - 3, $3, $4, $5); - $$->int32_values_[0] = 0; - $$->int32_values_[1] = $2[0]; -} -| -NEVER REFRESH -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_INFO); - $$->int32_values_[0] = 1; -} -| -{ -$$ = NULL; -} -; - -mv_refresh_on_clause: -ON mv_refresh_mode -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_ON_CLAUSE, 1, - $2); - -} -| -{ -$$ = NULL; -} -; - -mv_nested_refresh_opt: -mview_nested_refresh_mode -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_NESTED_REFRESH_CLAUSE, 1, - $1); -} -| -{ -$$ = NULL; -} -; - -mview_nested_refresh_mode: -INDIVIDUAL -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_NESTED_REFRESH_CLAUSE); - $$->value_ = 0; -} -| -INCONSISTENT -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_NESTED_REFRESH_CLAUSE); - $$->value_ = 1; -} -| -CONSISTENT -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_NESTED_REFRESH_CLAUSE); - $$->value_ = 2; -}; - -mv_refresh_method: -FAST -{ -$$[0] = 0; -} -| -COMPLETE -{ -$$[0] = 1; -} -| -FORCE -{ -$$[0] = 2; -} -; - -mv_refresh_mode: -DEMAND -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_METHOD); - $$->value_ = 0; -} -| -COMMIT -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_METHOD); - $$->value_ = 1; -} -| -STATEMENT -{ - malloc_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_METHOD); - $$->value_ = 2; -} -; - -mv_refresh_interval: -mv_start_clause mv_next_clause -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_INTERVAL, 2, $1, $2); -} -; - -mv_start_clause: -START WITH bit_expr -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_START_EXPR, 1, $3); -} -| -{ -$$ = NULL; -}; - -mv_next_clause: -NEXT bit_expr -{ -dup_string($2, result, @2.first_column, @2.last_column); -$$ = $2; -} -| -{ -$$ = NULL; -}; - -alter_mview_stmt: -alter_with_opt_hint MATERIALIZED VIEW view_name alter_mview_actions -{ - ParseNode *table_actions = NULL; - merge_nodes(table_actions, result, T_ALTER_TABLE_ACTION_LIST, $5); - malloc_non_terminal_node($$, result->malloc_pool_, T_ALTER_TABLE, 4, $4, table_actions, NULL, $1); - $$->value_ = 0; -} -; - -alter_mview_actions: -alter_mview_action_list -{ - $$ = $1; -} -; - -alter_mview_action_list: -alter_mview_action -{ - $$ = $1; -} -| alter_mview_actions ',' alter_mview_action -{ - if ($3 != NULL) { - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); - } else { - $$ = $1; - } -} -; - -alter_mview_action: -mview_enable_disable ON QUERY COMPUTATION -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, NULL); - $$->value_ = 0; - $$->int32_values_[0] = 1; - $$->int32_values_[1] = $1[0]; -} -| mview_enable_disable QUERY REWRITE -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, NULL); - $$->value_ = 0; - $$->int32_values_[0] = 2; - $$->int32_values_[1] = $1[0]; -} -| opt_set parallel_option -{ - (void)$1; - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $2); - $$->value_ = 0; - $$->int32_values_[0] = 3; -} -| REFRESH mv_refresh_method -{ - ParseNode *refresh_info = NULL; - malloc_non_terminal_node(refresh_info, result->malloc_pool_, T_MV_REFRESH_INFO, 2, NULL, NULL); - refresh_info->int32_values_[0] = 0; - refresh_info->int32_values_[1] = $2[0]; - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, refresh_info); - $$->value_ = 0; -} -| NEVER REFRESH -{ - ParseNode *refresh_info = NULL; - malloc_non_terminal_node(refresh_info, result->malloc_pool_, T_MV_REFRESH_INFO, 2, NULL, NULL); - refresh_info->int32_values_[0] = 1; - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, refresh_info); - $$->value_ = 0; -} -| REFRESH mv_refresh_interval -{ - ParseNode *refresh_info = NULL; - malloc_non_terminal_node(refresh_info, result->malloc_pool_, T_MV_REFRESH_INFO, 2, NULL, $2); - refresh_info->int32_values_[0] = 2; - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, refresh_info); - $$->value_ = 0; -} -| REFRESH mview_nested_refresh_mode -{ - ParseNode *nested_refresh_node = NULL; - malloc_terminal_node(nested_refresh_node, result->malloc_pool_, T_MV_NESTED_REFRESH_CLAUSE); - nested_refresh_node->int32_values_[0] = $2->value_; - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, nested_refresh_node); - $$->value_ = 0; - $$->int32_values_[0] = 4; -} -; - -alter_mlog_stmt: -alter_with_opt_hint MATERIALIZED VIEW LOG ON relation_factor alter_mlog_actions -{ - ParseNode *table_actions = NULL; - merge_nodes(table_actions, result, T_ALTER_TABLE_ACTION_LIST, $7); - malloc_non_terminal_node($$, result->malloc_pool_, T_ALTER_TABLE, 4, $6, table_actions, NULL, $1); - $$->value_ = 0; -} -; - -alter_mlog_actions: -alter_mlog_action_list -{ - $$ = $1; -} -; - -alter_mlog_action_list: -alter_mlog_action -{ - $$ = $1; -} -| alter_mlog_actions ',' alter_mlog_action -{ - if ($3 != NULL) { - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); - } else { - $$ = $1; - } -} -; - -alter_mlog_action: -parallel_option -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_ALTER_MLOG_OPTIONS, 1, $1); - $$->value_ = 1; -} -| PURGE mv_refresh_interval -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_ALTER_MLOG_OPTIONS, 1, $2); - $$->value_ = 2; -} -| LOB_INROW_THRESHOLD opt_equal_mark INTNUM -{ - (void)$2; - ParseNode *lob_thres = NULL; - malloc_non_terminal_node(lob_thres, result->malloc_pool_, T_LOB_INROW_THRESHOLD, 1, $3); - malloc_non_terminal_node($$, result->malloc_pool_, T_ALTER_MLOG_OPTIONS, 1, lob_thres); - $$->value_ = 3; -} -; - - opt_algorithm: ALGORITHM COMP_EQ view_algorithm { @@ -8951,35 +8590,6 @@ relation_factor { $$ = $1; } ; -opt_mv_column_list: -'(' mv_column_list ')' -{ - merge_nodes($$, result, T_COLUMN_LIST, $2); -} -| /*EMPTY*/ { $$ = NULL; } -; - -mv_column_list: -column_name_list -{ - $$ = $1; -} -| column_name_list ',' PRIMARY KEY opt_index_using_algorithm '(' column_name_list ')' opt_index_using_algorithm opt_comment -{ - ParseNode *col_list= NULL; - ParseNode *pk_node = NULL; - merge_nodes(col_list, result, T_COLUMN_LIST, $7); - malloc_non_terminal_node(pk_node, result->malloc_pool_, T_PRIMARY_KEY, 3, col_list, NULL != $9 ? $9 : $5, $10); - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, pk_node); -} -| PRIMARY KEY opt_index_using_algorithm '(' column_name_list ')' opt_index_using_algorithm opt_comment -{ - ParseNode *col_list= NULL; - merge_nodes(col_list, result, T_COLUMN_LIST, $5); - malloc_non_terminal_node($$, result->malloc_pool_, T_PRIMARY_KEY, 3, col_list, NULL != $7 ? $7 : $3, $8); -} -; - opt_column_list: '(' column_name_list ')' { @@ -9432,226 +9042,6 @@ TEMPORARY { $$ = NULL; } ; -/***************************************************************************** - * - * create mlog grammar - * - *****************************************************************************/ -create_mlog_stmt: -create_with_opt_hint MATERIALIZED VIEW LOG ON relation_factor opt_mlog_option_list opt_mlog_with opt_mlog_new_values opt_mlog_purge -{ - (void)($1); - ParseNode *mlog_options = NULL; - merge_nodes(mlog_options, result, T_TABLE_OPTION_LIST, $7); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_MLOG, 5, - $6, /* 1. table name */ - mlog_options, /* 2. mlog options */ - $8, /* 3. with */ - $9, /* 4. new values */ - $10 /* 5. purge */ - ); - $$->reserved_ = 0; -} -| create_with_opt_hint OR REPLACE MATERIALIZED VIEW LOG ON relation_factor opt_mlog_option_list opt_mlog_with opt_mlog_new_values opt_mlog_purge -{ - (void)($1); - ParseNode *mlog_options = NULL; - merge_nodes(mlog_options, result, T_TABLE_OPTION_LIST, $9); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_MLOG, 5, - $8, /* 1. table name */ - mlog_options, /* 2. mlog options */ - $10, /* 3. with */ - $11, /* 4. new values */ - $12 /* 5. purge */ - ); - $$->reserved_ = 1; -} -; - -opt_mlog_option_list: -/*EMPTY*/ -{ - $$ = NULL; -} -| opt_mlog_options -{ - $$ = $1; -} -; - -opt_mlog_options: -mlog_option -{ - $$ = $1; -} -| opt_mlog_options mlog_option -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $2); -} -; - -mlog_option: -parallel_option -{ - $$ = $1; -} - -opt_mlog_with: -/* empty */ { $$ = NULL; } -| WITH mlog_with_values -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_WITH, 1, $2); -}; - -mlog_with_values: -mlog_with_special_columns mlog_with_reference_columns -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_VALUES, 2, $1, $2); -}; - -mlog_with_special_columns: -/* empty */ { $$ = NULL; } -| mlog_with_special_column_list -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_SPECIAL_COLUMN_LIST, 1, $1); -}; - -mlog_with_special_column_list: -mlog_with_special_column -{ - $$ = $1; -} -| mlog_with_special_column ',' mlog_with_special_column_list -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_SPECIAL_COLUMN, 2, $1, $3); -}; - -mlog_with_special_column: -PRIMARY KEY -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_PRIMARY_KEY); -} -| ROWID -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_ROWID); -} -| SEQUENCE -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_SEQUENCE); -} -; - -mlog_with_reference_columns: -/* empty */ { $$ = NULL; } -| '(' ')' { $$ = NULL; } -| '(' mlog_with_reference_column_list ')' -{ - merge_nodes($$, result, T_COLUMN_LIST, $2); -}; - -mlog_with_reference_column_list: -mlog_with_reference_column -{ - $$ = $1; -} -| mlog_with_reference_column ',' mlog_with_reference_column_list -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_WITH_REFERENCE_COLUMN, 2, $1, $3); -} -; - -mlog_with_reference_column: -column_name -{ - $$ = $1; -} -; - -opt_mlog_new_values: -/* empty */ { $$ = NULL; } -| mlog_including_or_excluding NEW VALUES -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_NEW_VALUES, 1, $1); -} -; - -mlog_including_or_excluding: -INCLUDING -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_INCLUDING_NEW_VALUES); -} -| EXCLUDING -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_EXCLUDING_NEW_VALUES); -} -; - -opt_mlog_purge: -/* empty */ { $$ = NULL; } -| PURGE mlog_purge_values -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE, 1, $2); -} -; - -mlog_purge_values: -IMMEDIATE mlog_purge_immediate_sync_or_async -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE_IMMEDIATE, 1, $2); -} -| mlog_purge_start mlog_purge_next -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE_START_NEXT, 2, $1, $2); -} -; - -mlog_purge_immediate_sync_or_async: -/* empty */ { $$ = NULL; } -| SYNCHRONOUS -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE_IMMEDIATE_SYNC); -} -| ASYNCHRONOUS -{ - malloc_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE_IMMEDIATE_ASYNC); -} -; - -mlog_purge_start: -/* empty */ { $$ = NULL; } -| START WITH bit_expr -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MLOG_PURGE_START_TIME_EXPR, 1, $3); -} -; - -mlog_purge_next: -/* empty */ { $$ = NULL; } -| NEXT bit_expr -{ - dup_string($2, result, @2.first_column, @2.last_column); - $$ = $2; -} -; - -/***************************************************************************** - * - * drop mlog grammar - * - *****************************************************************************/ -drop_mlog_stmt: -DROP MATERIALIZED VIEW LOG ON relation_factor -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_DROP_MLOG, 1, $6); -} -; - -/***************************************************************************** - * - * drop table grammar - * - *****************************************************************************/ - drop_table_stmt: DROP opt_temporary table_or_tables opt_if_exists table_list opt_drop_behavior { @@ -9680,14 +9070,6 @@ DROP VIEW opt_if_exists table_list opt_drop_behavior merge_nodes(views, result, T_VIEW_LIST, $4); malloc_non_terminal_node($$, result->malloc_pool_, T_DROP_VIEW, 3, NULL, $3, views); } -| DROP MATERIALIZED VIEW opt_if_exists table_list opt_drop_behavior -{ - ParseNode *views = NULL; - merge_nodes(views, result, T_VIEW_LIST, $5); - ParseNode *mv = NULL; - malloc_terminal_node(mv, result->malloc_pool_, T_MATERIALIZE); - malloc_non_terminal_node($$, result->malloc_pool_, T_DROP_VIEW, 3, mv, $4, views); -} ; opt_if_exists: @@ -11186,18 +10568,6 @@ NO_REWRITE opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_REWRITE, 1, $2); } -| MV_REWRITE -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REWRITE, 2, NULL, NULL); -} -| MV_REWRITE '(' qb_name_option opt_relation_factor_in_mv_hint_list ')' -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REWRITE, 2, $3, $4); -} -| NO_MV_REWRITE opt_qb_name -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_MV_NO_REWRITE, 1, $2); -} | MERGE_HINT opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_MERGE_HINT, 2, $2, NULL); @@ -13550,30 +12920,6 @@ relation_sep_option: {} ; -relation_factor_in_mv_hint_list: -normal_relation_factor -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_RELATION_FACTOR_IN_HINT, 2, $1, NULL); -} -| relation_factor_in_mv_hint_list relation_sep_option normal_relation_factor -{ - ParseNode *mock_rel_in_hint_node = NULL; - malloc_non_terminal_node(mock_rel_in_hint_node, result->malloc_pool_, T_RELATION_FACTOR_IN_HINT, 2, $3, NULL); - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, mock_rel_in_hint_node); -} -; - -opt_relation_factor_in_mv_hint_list: -/* EMPTY */ -{ - $$ = NULL; -} -| relation_factor_in_mv_hint_list -{ - merge_nodes($$, result, T_RELATION_FACTOR_IN_HINT_LIST, $1); -} -; - opt_relation_factor_in_hint_list: /* EMPTY */ { @@ -18064,7 +17410,7 @@ alter_with_opt_hint SYSTEM CANCEL cancel_task_type TASK STRING_VALUE malloc_non_terminal_node($$, result->malloc_pool_, T_CANCEL_TASK, 2, $4, $6); } | -alter_with_opt_hint SYSTEM MAJOR FREEZE opt_tenant_list_or_ls_or_tablet_id +alter_with_opt_hint SYSTEM MAJOR FREEZE opt_tenant_list_or_tablet_id { (void)($1); ParseNode *type = NULL; @@ -18082,7 +17428,7 @@ alter_with_opt_hint SYSTEM CHECKPOINT malloc_non_terminal_node($$, result->malloc_pool_, T_FREEZE, 3, type, NULL, NULL); } | -alter_with_opt_hint SYSTEM MINOR FREEZE opt_tenant_list_or_ls_or_tablet_id +alter_with_opt_hint SYSTEM MINOR FREEZE opt_tenant_list_or_tablet_id { (void)($1); ParseNode *type = NULL; @@ -18638,23 +17984,11 @@ server_or_zone } ; -ls: -LS opt_equal_mark INTNUM -{ - (void)($2); - malloc_non_terminal_node($$, result->malloc_pool_, T_LS, 1, $3); -} -; - -opt_tenant_list_or_ls_or_tablet_id: +opt_tenant_list_or_tablet_id: tenant_list_tuple opt_tablet_id { malloc_non_terminal_node($$, result->malloc_pool_, T_TENANT_TABLET, 2, $1, $2); } -| tenant_list_tuple ls opt_tablet_id -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_TENANT_LS_TABLET, 3, $1, $2, $3); -} | opt_tablet_id_no_empty { malloc_non_terminal_node($$, result->malloc_pool_, T_TABLET_ID, 1, $1); @@ -21970,7 +21304,6 @@ ACCESS_INFO | LATERAL %prec LOWER_PARENS | LANGUAGE | LAST -| LAST_REFRESH_SCN | LAST_VALUE | LEAD | LEADER @@ -22141,7 +21474,6 @@ ACCESS_INFO | PARTIAL_UPDATE | PARTITION_ID | PATH -| LS | PARTITIONING | PARTITIONS | PARTITION_TYPE @@ -22448,7 +21780,6 @@ ACCESS_INFO | VECTOR_DISTANCE | VECTOR_SIMILARITY | VIRTUAL_COLUMN_ID -| MATERIALIZED | VIEW | VERIFY | WAIT diff --git a/src/sql/plan_cache/ob_adaptive_auto_dop.cpp b/src/sql/plan_cache/ob_adaptive_auto_dop.cpp index 7240df6a6..9e6d3cf76 100644 --- a/src/sql/plan_cache/ob_adaptive_auto_dop.cpp +++ b/src/sql/plan_cache/ob_adaptive_auto_dop.cpp @@ -274,7 +274,6 @@ int ObAdaptiveAutoDop::add_estimation_tasks(const ObTableScanSpec &tsc_spec, index_est_arg->scan_flag_ = tsc_spec.get_query_flag(); index_est_arg->range_columns_count_ = cost_tsc_info.get_range_columns_count(); index_est_arg->tablet_id_ = tablet_loc->tablet_id_; - index_est_arg->ls_id_ = tablet_loc->ls_id_; index_est_arg->tx_id_ = sql_ctx->session_info_->get_tx_id(); diff --git a/src/sql/plan_cache/ob_plan_cache_util.cpp b/src/sql/plan_cache/ob_plan_cache_util.cpp index 6f1e65ad8..5afc66deb 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.cpp +++ b/src/sql/plan_cache/ob_plan_cache_util.cpp @@ -112,7 +112,6 @@ int ObPhyLocationGetter::reselect_duplicate_table_best_replica(const ObIArray candi_addrs; - ObSEArray new_replic_idxs; int64_t proj_cnt = phy_locations.count(); ObLSReplicaLocation replica_location; for (int64_t i = 0; OB_SUCC(ret) && is_same && i < proj_cnt; ++i) { @@ -137,12 +136,8 @@ int ObPhyLocationGetter::reselect_duplicate_table_best_replica(const ObIArray &replicas = - part_info.get_partition_location().get_replica_locations(); - for (int64_t j = 0; OB_SUCC(ret) && j < replicas.count(); ++j) { - if (OB_FAIL(candi_addrs.push_back(replicas.at(j).get_server()))) { - LOG_WARN("failed to push back servers", K(ret)); - } + if (OB_FAIL(candi_addrs.push_back(replica_location.get_server()))) { + LOG_WARN("failed to store local server", K(ret)); } duplicate_table_addr = replica_location.get_server(); has_duplicate_tbl = true; @@ -180,9 +175,6 @@ int ObPhyLocationGetter::reselect_duplicate_table_best_replica(const ObIArray(phy_locations.at(j)); - if (!ptli.is_duplicate_table_not_in_dml()) { - // do nothing - } else if (OB_FAIL(ptli.get_phy_part_loc_info_list_for_update().at(0). - set_selected_replica_idx(new_replic_idxs.at(j)))) { - SQL_PC_LOG(WARN, "failed to set selected replica idx", K(ret)); + is_valid = ptli.get_phy_part_loc_info_list().at(0).is_local_server(addr); } } if (OB_SUCC(ret) && is_valid) { @@ -243,8 +224,6 @@ int ObPhyLocationGetter::get_phy_locations(const ObIArray &tabl const ObTableLocation &table_location = table_locations.at(i); ObCandiTableLoc &candi_table_loc = candi_table_locs.at(i); NG_TRACE(calc_partition_location_begin); - // Here it is assumed that the materialized view's replica table has a copy on each server, - // Therefore here we do not check if materialized view can be generated, it is guaranteed to be generated if (OB_FAIL(table_location.calculate_candi_tablet_locations(exec_ctx, params, candi_table_loc.get_phy_part_loc_info_list_for_update(), @@ -277,17 +256,13 @@ int ObPhyLocationGetter::get_phy_locations(const ObIArray &tabl } //Only check the on_same_server when has table location in the phy_plan. - bool is_dup_ls_modified = false; - if (OB_NOT_NULL(pc_ctx.sql_ctx_.session_info_)) { - is_dup_ls_modified = pc_ctx.sql_ctx_.session_info_->is_dup_ls_modified(); - } if (OB_SUCC(ret) && N!=0 ) { if (OB_FAIL(ObLogPlan::select_replicas(exec_ctx, table_location_ptrs, exec_ctx.get_addr(), phy_location_info_ptrs))) { LOG_WARN("failed to select replicas", K(ret), K(table_locations), K(exec_ctx.get_addr()), K(phy_location_info_ptrs)); - } else if (is_dup_ls_modified || is_retrying) { + } else if (is_retrying) { // do nothing } else if (OB_FAIL(reselect_duplicate_table_best_replica(candi_table_locs, on_same_server))) { diff --git a/src/sql/plan_cache/ob_plan_match_helper.cpp b/src/sql/plan_cache/ob_plan_match_helper.cpp index 2e3e18969..40b519082 100644 --- a/src/sql/plan_cache/ob_plan_match_helper.cpp +++ b/src/sql/plan_cache/ob_plan_match_helper.cpp @@ -33,7 +33,6 @@ int ObPlanMatchHelper::match_plan(const ObPlanCacheCtx &pc_ctx, int ret = OB_SUCCESS; bool has_duplicate_table = false; bool is_retrying = false; - bool is_dup_ls_modified = false; is_matched = true; const ObAddr &server = pc_ctx.exec_ctx_.get_addr(); const ObIArray& base_cons = plan->get_base_constraints(); @@ -51,11 +50,10 @@ int ObPlanMatchHelper::match_plan(const ObPlanCacheCtx &pc_ctx, // match all is_matched = true; } else { - is_dup_ls_modified = GET_MY_SESSION(pc_ctx.exec_ctx_)->is_dup_ls_modified(); if (OB_NOT_NULL(plan_set_) && plan_set_->has_duplicate_table()) { if (OB_FAIL(pc_ctx.is_retry(is_retrying))) { LOG_WARN("failed to test if retrying", K(ret)); - } else if (is_retrying || is_dup_ls_modified) { + } else if (is_retrying) { has_duplicate_table = false; } else { has_duplicate_table = true; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 48c6e6f95..92543dda0 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -212,21 +212,6 @@ int ObAlterSystemResolverUtil::resolve_tenant(const ParseNode *parse_tree, return ret; } -int ObAlterSystemResolverUtil::resolve_ls_id(const ParseNode *parse_tree, int64_t &ls_id) -{ - int ret = OB_SUCCESS; - if (NULL == parse_tree) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("node should not be null"); - } else if (OB_FAIL(sanity_check(parse_tree, T_LS))) { - LOG_WARN("sanity check failed"); - } else { - ls_id = parse_tree->children_[0]->value_; - FLOG_INFO("resolve ls id", K(ls_id)); - } - return ret; -} - int ObAlterSystemResolverUtil::resolve_tablet_id(const ParseNode *opt_tablet_id, ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -405,9 +390,9 @@ int ObFreezeResolver::resolve(const ParseNode &parse_tree) ret = OB_ERR_UNEXPECTED; LOG_WARN("wrong freeze parse tree", K(parse_tree.num_child_)); } else { - ParseNode *opt_tenant_list_or_ls_or_tablet_id = parse_tree.children_[1]; - if (OB_FAIL(resolve_minor_freeze_(freeze_stmt, opt_tenant_list_or_ls_or_tablet_id))) { - LOG_WARN("resolve minor freeze failed", KR(ret), KP(opt_tenant_list_or_ls_or_tablet_id)); + ParseNode *opt_tenant_list_or_tablet_id = parse_tree.children_[1]; + if (OB_FAIL(resolve_minor_freeze_(freeze_stmt, opt_tenant_list_or_tablet_id))) { + LOG_WARN("resolve minor freeze failed", KR(ret), KP(opt_tenant_list_or_tablet_id)); } } } else { @@ -430,11 +415,8 @@ int ObFreezeResolver::resolve_major_freeze_(ObFreezeStmt *freeze_stmt, ParseNode } else if (OB_UNLIKELY(nullptr == opt_tenant_list_or_tablet_id->children_ || 0 == opt_tenant_list_or_tablet_id->num_child_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("children of tenant should not be null", KR(ret), KP(opt_tenant_list_or_tablet_id)); - } else if (OB_FAIL(resolve_tenant_ls_tablet_(freeze_stmt, opt_tenant_list_or_tablet_id))) { + } else if (OB_FAIL(resolve_tenant_tablet_(freeze_stmt, opt_tenant_list_or_tablet_id))) { LOG_WARN("fail to resolve tenant or tablet", KR(ret)); - } else if (OB_UNLIKELY(share::ObLSID::INVALID_LS_ID != freeze_stmt->get_ls_id())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not support to specify ls to major freeze", K(ret), "ls_id", freeze_stmt->get_ls_id()); } else if (freeze_stmt->get_tablet_id().is_valid()) { // tablet major freeze if (GCTX.is_shared_storage_mode()) { ret = OB_NOT_SUPPORTED; @@ -460,17 +442,16 @@ int ObFreezeResolver::resolve_major_freeze_(ObFreezeStmt *freeze_stmt, ParseNode } int ObFreezeResolver::resolve_minor_freeze_(ObFreezeStmt *freeze_stmt, - ParseNode *opt_tenant_list_or_ls_or_tablet_id) + ParseNode *opt_tenant_list_or_tablet_id) { int ret = OB_SUCCESS; - if (OB_NOT_NULL(opt_tenant_list_or_ls_or_tablet_id)) { - if (OB_FAIL(resolve_tenant_ls_tablet_(freeze_stmt, opt_tenant_list_or_ls_or_tablet_id))) { - LOG_WARN("resolve tenant ls table failed", KR(ret)); - } else if (T_TABLET_ID == opt_tenant_list_or_ls_or_tablet_id->type_) { + if (OB_NOT_NULL(opt_tenant_list_or_tablet_id)) { + if (OB_FAIL(resolve_tenant_tablet_(freeze_stmt, opt_tenant_list_or_tablet_id))) { + LOG_WARN("resolve tenant or tablet failed", KR(ret)); + } else if (T_TABLET_ID == opt_tenant_list_or_tablet_id->type_) { freeze_stmt->reset_tenant_count(); - freeze_stmt->get_ls_id() = share::ObLSID::INVALID_LS_ID; freeze_stmt->inc_tenant_count(); // if tenant is not explicitly specified, add owned tenant } } @@ -479,13 +460,13 @@ int ObFreezeResolver::resolve_minor_freeze_(ObFreezeStmt *freeze_stmt, return ret; } -int ObFreezeResolver::resolve_tenant_ls_tablet_(ObFreezeStmt *freeze_stmt, - ParseNode *opt_tenant_list_or_ls_or_tablet_id) +int ObFreezeResolver::resolve_tenant_tablet_(ObFreezeStmt *freeze_stmt, + ParseNode *opt_tenant_list_or_tablet_id) { int ret = OB_SUCCESS; - if (OB_ISNULL(opt_tenant_list_or_ls_or_tablet_id->children_)) { + if (OB_ISNULL(opt_tenant_list_or_tablet_id->children_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("children of tenant should not be null", KR(ret)); } else { @@ -494,45 +475,30 @@ int ObFreezeResolver::resolve_tenant_ls_tablet_(ObFreezeStmt *freeze_stmt, bool affect_all_meta = false; const ParseNode *tenant_list_tuple = nullptr; const ParseNode *opt_tablet_id = nullptr; - const ParseNode *ls_id = nullptr; - switch (opt_tenant_list_or_ls_or_tablet_id->type_) { + switch (opt_tenant_list_or_tablet_id->type_) { case T_TENANT_TABLET: - if (opt_tenant_list_or_ls_or_tablet_id->num_child_ != 2) { + if (opt_tenant_list_or_tablet_id->num_child_ != 2) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid child num", K(ret), K(opt_tenant_list_or_ls_or_tablet_id->num_child_)); + LOG_WARN("invalid child num", K(ret), K(opt_tenant_list_or_tablet_id->num_child_)); } else { - tenant_list_tuple = opt_tenant_list_or_ls_or_tablet_id->children_[0]; - opt_tablet_id = opt_tenant_list_or_ls_or_tablet_id->children_[1]; + tenant_list_tuple = opt_tenant_list_or_tablet_id->children_[0]; + opt_tablet_id = opt_tenant_list_or_tablet_id->children_[1]; if (OB_ISNULL(tenant_list_tuple)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant list is nullptr", KR(ret), KP(tenant_list_tuple), KP(ls_id), KP(opt_tablet_id)); - } - } - break; - case T_TENANT_LS_TABLET: - if (opt_tenant_list_or_ls_or_tablet_id->num_child_ != 3) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid child num", K(ret), K(opt_tenant_list_or_ls_or_tablet_id->num_child_)); - } else { - tenant_list_tuple = opt_tenant_list_or_ls_or_tablet_id->children_[0]; - ls_id = opt_tenant_list_or_ls_or_tablet_id->children_[1]; - opt_tablet_id = opt_tenant_list_or_ls_or_tablet_id->children_[2]; - if (OB_ISNULL(tenant_list_tuple) || OB_ISNULL(ls_id)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant_list or ls_id is nullptr", KR(ret), KP(tenant_list_tuple), KP(ls_id), KP(opt_tablet_id)); + LOG_WARN("tenant list is nullptr", KR(ret), KP(tenant_list_tuple), KP(opt_tablet_id)); } } break; case T_TABLET_ID: - if (opt_tenant_list_or_ls_or_tablet_id->num_child_ != 1) { + if (opt_tenant_list_or_tablet_id->num_child_ != 1) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid child num", K(ret), K(opt_tenant_list_or_ls_or_tablet_id->num_child_)); + LOG_WARN("invalid child num", K(ret), K(opt_tenant_list_or_tablet_id->num_child_)); } else { - opt_tablet_id = opt_tenant_list_or_ls_or_tablet_id->children_[0]; + opt_tablet_id = opt_tenant_list_or_tablet_id->children_[0]; if (OB_ISNULL(opt_tablet_id)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant_list or ls_id is nullptr", KR(ret), KP(opt_tablet_id)); + LOG_WARN("tablet id is nullptr", KR(ret), KP(opt_tablet_id)); } } break; @@ -540,8 +506,7 @@ int ObFreezeResolver::resolve_tenant_ls_tablet_(ObFreezeStmt *freeze_stmt, ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid parse node type", K(T_TENANT_TABLET), - K(T_TENANT_LS_TABLET), - K(opt_tenant_list_or_ls_or_tablet_id->type_)); + K(opt_tenant_list_or_tablet_id->type_)); break; } @@ -551,8 +516,6 @@ int ObFreezeResolver::resolve_tenant_ls_tablet_(ObFreezeStmt *freeze_stmt, freeze_stmt->tenant_count_ref(), affect_all, affect_all_user, affect_all_meta))) { LOG_WARN("fail to resolve tenant", KR(ret)); - } else if (OB_NOT_NULL(ls_id) && OB_FAIL(Util::resolve_ls_id(ls_id, freeze_stmt->get_ls_id()))) { - LOG_WARN("fail to resolve tablet id", KR(ret)); } else if (OB_NOT_NULL(opt_tablet_id) && OB_FAIL(Util::resolve_tablet_id(opt_tablet_id, freeze_stmt->get_tablet_id()))) { LOG_WARN("fail to resolve tablet id", KR(ret)); diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 8fc04cce6..d6c4c671e 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -49,8 +49,6 @@ class ObAlterSystemResolverUtil // resolve opt_tenant_name static int resolve_tenant(const ParseNode *parse_tree, common::ObFixedLengthString < common::OB_MAX_TENANT_NAME_LENGTH + 1 > &tenant_name); - static int resolve_ls_id(const ParseNode *parse_tree, int64_t &ls_id); - static int resolve_replica_type(const ParseNode *parse_tree, common::ObReplicaType &replica_type); static int resolve_string(const ParseNode *parse_tree, common::ObString &string); @@ -143,9 +141,9 @@ class ObFreezeResolver : public ObSystemCmdResolver { private: int resolve_major_freeze_(ObFreezeStmt *freeze_stmt, ParseNode *opt_tenant_list_or_tablet_id); int resolve_minor_freeze_(ObFreezeStmt *freeze_stmt, - ParseNode *opt_tenant_list_or_ls_or_tablet_id); + ParseNode *opt_tenant_list_or_tablet_id); - int resolve_tenant_ls_tablet_(ObFreezeStmt *freeze_stmt, ParseNode *opt_tenant_list_or_ls_or_tablet_id); + int resolve_tenant_tablet_(ObFreezeStmt *freeze_stmt, ParseNode *opt_tenant_list_or_tablet_id); }; diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index 5f96d18b7..2e7adfe7d 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -41,16 +41,14 @@ class ObFreezeStmt : public ObSystemCmdStmt freeze_all_flag_(0), opt_server_list_(), opt_tenant_count_(0), - opt_tablet_id_(), - opt_ls_id_(share::ObLSID::INVALID_LS_ID) {} + opt_tablet_id_() {} ObFreezeStmt(common::ObIAllocator *name_pool) : ObSystemCmdStmt(name_pool, stmt::T_FREEZE), major_freeze_(false), freeze_all_flag_(0), opt_server_list_(), opt_tenant_count_(0), - opt_tablet_id_(), - opt_ls_id_(share::ObLSID::INVALID_LS_ID) {} + opt_tablet_id_() {} virtual ~ObFreezeStmt() {} bool is_major_freeze() const { return major_freeze_; } @@ -69,13 +67,12 @@ class ObFreezeStmt : public ObSystemCmdStmt inline void reset_tenant_count() { opt_tenant_count_ = 0; } inline common::ObZone &get_zone() { return opt_zone_; } inline common::ObTabletID &get_tablet_id() { return opt_tablet_id_; } - inline int64_t &get_ls_id() { return opt_ls_id_; } inline int push_server(const common::ObAddr& server) { return opt_server_list_.push_back(server); } - TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(major_freeze), K(freeze_all_flag_), - K(opt_server_list_), K(opt_tenant_count_), K(opt_tablet_id_), K(opt_ls_id_)); + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(major_freeze), K(freeze_all_flag_), + K(opt_server_list_), K(opt_tenant_count_), K(opt_tablet_id_)); private: bool major_freeze_; // for major_freeze, it is ignore server list @@ -90,7 +87,6 @@ class ObFreezeStmt : public ObSystemCmdStmt // for minor_freeze only common::ObTabletID opt_tablet_id_; - int64_t opt_ls_id_; }; class ObFlushCacheStmt : public ObSystemCmdStmt diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 03b1fad5e..854874757 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -100,7 +100,7 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_REFRESH_SCHEMA, get_sys_tenant_alter_system_priv, OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CREATE_DATABASE, get_create_database_stmt_need_privs, 96) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_USE_DATABASE, no_priv_needed, 97) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ADMIN_SERVER, get_sys_tenant_alter_system_priv, 98) -OB_STMT_TYPE_DEF_UNKNOWN_AT(T_RECYCLE_REPLICA, get_sys_tenant_alter_system_priv, 103) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_RESERVED_3791, get_sys_tenant_alter_system_priv, 103) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ADMIN_MERGE, get_sys_tenant_alter_system_priv, 104) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ALTER_DATABASE, get_alter_database_stmt_need_privs, 105) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_DROP_DATABASE, get_drop_database_stmt_need_privs, 106) @@ -256,8 +256,6 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ACTIVATE_STANDBY, get_sys_tenant_alter_system_priv OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SWITCHOVER_TO_STANDBY, get_sys_tenant_alter_system_priv, 291) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SWITCHOVER_TO_PRIMARY, get_sys_tenant_alter_system_priv, 293) OB_STMT_TYPE_DEF(T_ALTER_SYSTEM_RESET_PARAMETER, get_sys_tenant_alter_system_priv, 292, ACTION_TYPE_ALTER_SYSTEM) -OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CREATE_MLOG, get_create_mlog_stmt_need_privs, 295) -OB_STMT_TYPE_DEF_UNKNOWN_AT(T_DROP_MLOG, get_drop_mlog_stmt_need_privs, 296) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_FLUSH_PRIVILEGES, no_priv_needed, 298) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ALTER_LS_REPLICA, get_sys_tenant_alter_system_priv, 299) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_PROCEDURE_CODE, err_stmt_type_priv, 300) diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 1e097b7b2..2baf92d43 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -45,7 +45,6 @@ #include "sql/engine/ob_physical_plan.h" #include "sql/ob_sql_context.h" #include "sql/ob_sql_trans_util.h" -#include "share/partition_table/ob_partition_location.h" #include "common/sql_mode/ob_sql_mode_utils.h" #include "sql/parser/ob_parser_utils.h" @@ -712,7 +711,6 @@ class ObBasicSessionInfo int get_regexp_time_limit(int64_t &v) const; int get_regexp_session_vars(ObExprRegexpSessionVariables &vars) const; int get_activate_all_role_on_login(bool &v) const; - int get_mview_refresh_dop(uint64_t &v) const; int update_timezone_info(); const common::ObTimeZoneInfo *get_timezone_info() const { return tz_info_wrap_.get_time_zone_info(); } const common::ObTimeZoneInfoWrap &get_tz_info_wrap() const { return tz_info_wrap_; } @@ -1064,8 +1062,6 @@ class ObBasicSessionInfo // The following helper function is for conveniently viewing the value of a system variable int if_aggr_pushdown_allowed(bool &aggr_pushdown_allowed) const; int is_transformation_enabled(bool &transformation_enabled) const; - int get_query_rewrite_enabled(int64_t &query_rewrite_enabled) const; - int get_query_rewrite_integrity(int64_t &query_rewrite_integrity) const; int is_serial_set_order_forced(bool &force_set_order) const; int is_old_charset_aggregation_enabled(bool &is_enable) const; int is_storage_estimation_enabled(bool &storage_estimation_enabled) const; @@ -1412,7 +1408,6 @@ class ObBasicSessionInfo bool has_explicit_start_trans() const { return tx_desc_ != NULL && tx_desc_->is_explicit(); } bool is_in_transaction() const { return tx_desc_ != NULL && tx_desc_->is_in_tx(); } bool has_active_autocommit_trans(transaction::ObTransID &trans_id); - bool is_dup_ls_modified() const { return tx_desc_ != NULL && tx_desc_->is_dup_ls_modified(); } bool get_in_transaction() const { return is_in_transaction(); } uint64_t get_trans_flags() const { return trans_flags_.get_flags(); } void set_has_exec_inner_dml(bool value) { trans_flags_.set_has_exec_inner_dml(value); } diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index ff71bf7a9..36031654b 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -20,6 +20,7 @@ #include "ob_sql_session_info.h" #include "share/rc/ob_module_provider.h" #include "storage/memtable/mvcc/ob_btree_iter_cache.h" +#include "storage/tx/ob_ts_mgr.h" #include "rootserver/ob_rs_serial_call.h" #include "pl/ob_pl_package.h" #include "observer/mysql/obmp_stmt_send_piece_data.h" @@ -2329,10 +2330,8 @@ void ObSQLSessionInfo::ObCachedTenantConfigInfo::refresh() } else { ATOMIC_STORE(&data_version_, data_version); } - // 1.Does it support external consistency - is_external_consistent_ = transaction::ObTsMgr::get_instance().is_external_consistent(); if (OB_LIKELY(true)) { - // 2.Is batch_multi_statement allowed + // 1.Is batch_multi_statement allowed enable_batched_multi_statement_ = GCONF.ob_enable_batched_multi_statement; // 3.Is bloom_filter allowed if (GCONF._bloom_filter_enabled) { @@ -2344,7 +2343,6 @@ void ObSQLSessionInfo::ObCachedTenantConfigInfo::refresh() ATOMIC_STORE(&sort_area_size_, GCONF._sort_area_size); ATOMIC_STORE(&hash_area_size_, GCONF._hash_area_size); ATOMIC_STORE(&enable_query_response_time_stats_, GCONF.query_response_time_stats); - ATOMIC_STORE(&enable_insertup_replace_gts_opt_, GCONF._enable_insertup_replace_gts_opt); ATOMIC_STORE(&enable_immediate_row_conflict_check_, GCONF._ob_immediate_row_conflict_check); ATOMIC_STORE(&range_optimizer_max_mem_size_, GCONF.range_optimizer_max_mem_size); ATOMIC_STORE(&_query_record_size_limit_, GCONF._query_record_size_limit); diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index f6642be0f..dc46af59b 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -638,7 +638,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI { public: ObCachedTenantConfigInfo(ObSQLSessionInfo *session) : - is_external_consistent_(false), enable_batched_multi_statement_(false), enable_sql_extension_(false), enable_bloom_filter_(true), @@ -648,7 +647,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI hash_area_size_(128*1024*1024), data_version_(0), enable_query_response_time_stats_(false), - enable_insertup_replace_gts_opt_(false), enable_immediate_row_conflict_check_(false), range_optimizer_max_mem_size_(128*1024*1024), _query_record_size_limit_(65536), @@ -669,7 +667,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI } ~ObCachedTenantConfigInfo() {} void refresh(); - bool get_is_external_consistent() const { return is_external_consistent_; } bool get_enable_batched_multi_statement() const { return enable_batched_multi_statement_; } bool get_enable_bloom_filter() const { return enable_bloom_filter_; } bool get_enable_sql_extension() const { return enable_sql_extension_; } @@ -677,7 +674,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI int64_t get_hash_area_size() const { return ATOMIC_LOAD(&hash_area_size_); } uint64_t get_data_version() const { return ATOMIC_LOAD(&data_version_); } bool enable_query_response_time_stats() const { return enable_query_response_time_stats_; } - bool enable_insertup_replace_gts_opt() const { return ATOMIC_LOAD(&enable_insertup_replace_gts_opt_); } int64_t get_print_sample_ppm() const { return ATOMIC_LOAD(&print_sample_ppm_); } bool get_px_join_skew_handling() const { return px_join_skew_handling_; } int64_t get_px_join_skew_minfreq() const { return px_join_skew_minfreq_; } @@ -712,7 +708,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI private: // Tenant-level configuration item cache session, avoid refreshing every time it is retrieved - bool is_external_consistent_; bool enable_batched_multi_statement_; bool enable_sql_extension_; bool enable_bloom_filter_; @@ -722,7 +717,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI int64_t hash_area_size_; uint64_t data_version_; bool enable_query_response_time_stats_; - bool enable_insertup_replace_gts_opt_; bool enable_immediate_row_conflict_check_; int64_t range_optimizer_max_mem_size_; int64_t _query_record_size_limit_; @@ -1197,11 +1191,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI void set_xa_last_result(const int result) { xa_last_result_ = result; } // For performance optimization, tenant-level configuration items do not need to be retrieved in real time, they are cached in the session, and refreshed every 5s void refresh_tenant_config() { cached_tenant_config_info_.refresh(); } - bool is_support_external_consistent() - { - cached_tenant_config_info_.refresh(); - return cached_tenant_config_info_.get_is_external_consistent(); - } bool is_enable_batched_multi_statement() { cached_tenant_config_info_.refresh(); @@ -1250,11 +1239,6 @@ class ObSQLSessionInfo: public common::ObVersionProvider, public ObBasicSessionI cached_tenant_config_info_.refresh(); return cached_tenant_config_info_.enable_query_response_time_stats(); } - bool enable_insertup_replace_gts_opt() - { - cached_tenant_config_info_.refresh(); - return cached_tenant_config_info_.enable_insertup_replace_gts_opt(); - } bool enable_immediate_row_conflict_check() { cached_tenant_config_info_.refresh(); diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index fe1287d9c..20387c908 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -214,7 +214,6 @@ ob_set_subtarget(ob_storage slog slog/ob_storage_log.cpp slog/ob_storage_log_write_buffer.cpp slog/ob_storage_log_writer.cpp - slog/ob_storage_logger_manager.cpp slog/ob_storage_logger.cpp ) @@ -336,24 +335,13 @@ ob_set_subtarget(ob_storage tablet ob_set_subtarget(ob_storage tx tx/ob_ctx_tx_data.cpp - tx/ob_gti_rpc.cpp tx/ob_gti_source.cpp - tx/ob_gts_local_cache.cpp - tx/ob_gts_msg.cpp - tx/ob_gts_rpc.cpp - tx/ob_gts_source.cpp - tx/ob_gts_task_queue.cpp - tx/ob_ts_response_handler.cpp - tx/ob_ts_worker.cpp tx/ob_id_service.cpp tx/ob_keep_alive_ls_handler.cpp tx/ob_tx_loop_worker.cpp - tx/ob_tx_ls_state_mgr.cpp tx/ob_tx_log_cb_define.cpp tx/ob_tx_log_cb_mgr.cpp tx/ob_tx_ctx_log_cb_helper.cpp - tx/ob_location_adapter.cpp - tx/ob_tablet_to_ls_cache.cpp tx/ob_timestamp_access.cpp tx/ob_timestamp_service.cpp tx/ob_weak_read_util.cpp @@ -370,10 +358,9 @@ ob_set_subtarget(ob_storage tx tx/ob_trans_factory.cpp tx/ob_trans_id_service.cpp tx/ob_trans_memory_stat.cpp - tx/ob_trans_part_ctx.cpp + tx/ob_tx_ctx.cpp tx/ob_defensive_check_mgr.cpp tx/ob_trans_result.cpp - tx/ob_trans_rpc.cpp tx/ob_tx_result_struct.cpp tx/ob_trans_service.cpp tx/ob_trans_service_v4.cpp @@ -389,15 +376,14 @@ ob_set_subtarget(ob_storage tx tx/ob_trans_timer.cpp tx/ob_trans_version_mgr.cpp tx/ob_ts_mgr.cpp + tx/ob_tx_timestamp_waiter.cpp tx/ob_tx_data_define.cpp tx/ob_tx_data_op.cpp tx/ob_tx_data_functor.cpp - tx/ob_tx_serialization.cpp + tx/ob_tx_print_time_guard.cpp tx/ob_tx_log.cpp tx/ob_tx_log_adapter.cpp tx/ob_tx_big_segment_buf.cpp - tx/ob_tx_ls_log_writer.cpp - tx/ob_tx_msg.cpp tx/ob_tx_on_demand_print.cpp tx/ob_tx_replay_executor.cpp tx/ob_xa_define.cpp @@ -427,11 +413,7 @@ ob_set_subtarget(ob_storage tx_storage tx_storage/ob_access_service.cpp tx_storage/ob_checkpoint_service.cpp tx_storage/ob_ls_freeze_thread.cpp - tx_storage/ob_ls_handle.cpp - tx_storage/ob_ls_map.cpp - tx_storage/ob_ls_safe_destroy_task.cpp tx_storage/ob_ls_service.cpp - tx_storage/ob_safe_destroy_handler.cpp tx_storage/ob_tenant_freezer.cpp tx_storage/ob_tenant_freezer_common.cpp tx_storage/ob_tenant_freezer_rpc.cpp @@ -464,8 +446,6 @@ ob_set_subtarget(ob_storage ls ls/ob_ls_ddl_log_handler.cpp ls/ob_ls_lock.cpp ls/ob_ls_meta.cpp - ls/ob_ls_meta_package.cpp - ls/ob_ls_role_handler.cpp ls/ob_ls_state.cpp ls/ob_ls_sync_tablet_seq_handler.cpp ls/ob_ls_switch_checker.cpp @@ -499,7 +479,6 @@ ob_set_subtarget(ob_storage access access/ob_multiple_scan_merge.cpp access/ob_multiple_skip_scan_merge.cpp access/ob_multiple_multi_skip_scan_merge.cpp - access/ob_multiple_mview_merge.cpp access/ob_index_skip_scanner.cpp access/ob_table_scan_iterator.cpp access/ob_store_row_iterator.cpp @@ -612,7 +591,7 @@ ob_set_subtarget(ob_storage common ob_storage_schema_recorder.cpp ob_storage_clog_recorder.cpp ob_storage_struct.cpp - ob_tablet_ha_status.cpp + ob_tablet_restore_state.cpp ob_storage_ha_struct.cpp ob_storage_table_guard.cpp ob_storage_util.cpp @@ -697,7 +676,6 @@ ob_set_subtarget(ob_storage compaction compaction/ob_sstable_merge_history.cpp compaction/ob_batch_freeze_tablets_dag.cpp compaction/ob_progressive_merge_helper.cpp - compaction/ob_mview_compaction_util.cpp compaction/ob_medium_loop.cpp compaction/ob_basic_schedule_tablet_func.cpp compaction/ob_schedule_tablet_func.cpp @@ -719,7 +697,6 @@ ob_set_subtarget(ob_storage memtable memtable/ob_memtable_block_row_scanner.cpp memtable/ob_memtable_compact_writer.cpp memtable/ob_memtable_context.cpp - memtable/ob_memtable_interface.cpp memtable/ob_memtable_iterator.cpp memtable/ob_memtable_mutator.cpp memtable/ob_memtable_read_row_util.cpp @@ -918,24 +895,6 @@ ob_set_subtarget(ob_storage lob lob/ob_lob_locator_struct.cpp ) -ob_set_subtarget(ob_storage mview - mview/cmd/ob_mview_executor_util.cpp - mview/cmd/ob_mview_purge_log_executor.cpp - mview/cmd/ob_mview_refresh_executor.cpp - mview/cmd/ob_mview_stats_purge_refresh_stats_executor.cpp - mview/cmd/ob_mview_stats_set_mvref_stats_params_executor.cpp - mview/cmd/ob_mview_stats_set_system_default_executor.cpp - mview/ob_mlog_purge.cpp - mview/ob_mview_refresh_helper.cpp - mview/ob_mview_refresh_stats_collect.cpp - mview/ob_mview_refresh_stats_purge.cpp - mview/ob_mview_refresh.cpp - mview/ob_mview_transaction.cpp - mview/ob_mview_sched_job_utils.cpp - mview/ob_major_mv_merge_info.cpp - mview/ob_mview_mds.cpp -) - ob_set_subtarget(ob_storage vector_index vector_index/cmd/ob_vector_refresh_index_executor.cpp vector_index/ob_vector_index_refresh.cpp diff --git a/src/storage/access/ob_dml_param.cpp b/src/storage/access/ob_dml_param.cpp index e7781bf75..869905aea 100644 --- a/src/storage/access/ob_dml_param.cpp +++ b/src/storage/access/ob_dml_param.cpp @@ -286,7 +286,6 @@ DEF_TO_STRING(ObTableScanParam) int64_t pos = 0; J_OBJ_START(); J_KV(K_(tablet_id), - K_(ls_id), N_COLUMN_IDS, column_ids_, N_INDEX_ID, index_id_, N_KEY_RANGES, key_ranges_, diff --git a/src/storage/access/ob_dml_param.h b/src/storage/access/ob_dml_param.h index f9e657a2c..eb0d9b3a4 100644 --- a/src/storage/access/ob_dml_param.h +++ b/src/storage/access/ob_dml_param.h @@ -209,9 +209,6 @@ class ObTableScanParam : public common::ObVTableScanParam OB_INLINE bool use_index_skip_scan() const { return (1 == ss_key_ranges_.count()) && (!ss_key_ranges_.at(0).is_whole_range()); } - OB_INLINE bool is_mview_query() const { - return nullptr != op_filters_ && scan_flag_.is_mr_mview_query(); - } void destroy() override { if (OB_UNLIKELY(ss_key_ranges_.get_capacity() > OB_DEFAULT_RANGE_COUNT)) { @@ -296,7 +293,7 @@ struct ObDMLBaseParam const blocksstable::ObDatumRow *data_row_for_lob_; // for tablet split bool is_main_table_in_fts_ddl_; // whether the main table is in fts ddl when dml is executed // Set by DAS layer when the table has async-mode indexes (e.g. sync_mode=async HNSW). - // Propagated to ObPartTransCtx::has_async_index_redo_ -> ObTxLogBlockHeader::HAS_ASYNC_INDEX + // Propagated to ObTxCtx::has_async_index_redo_ -> ObTxLogBlockHeader::HAS_ASYNC_INDEX // for Change Stream fast filtering in the Fetcher. bool has_async_index_; bool is_valid() const { return (timeout_ > 0 && schema_version_ >= 0) && nullptr != store_ctx_guard_; } diff --git a/src/storage/access/ob_multiple_merge.cpp b/src/storage/access/ob_multiple_merge.cpp index 7e3e6ac82..56fd35377 100644 --- a/src/storage/access/ob_multiple_merge.cpp +++ b/src/storage/access/ob_multiple_merge.cpp @@ -22,7 +22,8 @@ #include "ob_aggregated_store.h" #include "ob_aggregated_store_vec.h" #include "sql/engine/expr/ob_expr_lob_utils.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/concurrency_control/ob_data_validation_service.h" #include "storage/truncate_info/ob_truncate_partition_filter.h" @@ -167,9 +168,6 @@ int ObMultipleMerge::init( } unprojected_row_.count_ = 0; get_table_param_ = &get_table_param; - if (access_ctx_->is_mview_query()) { - access_param_->iter_param_.is_delete_insert_ = false; - } access_param_->iter_param_.set_tablet_handle(get_table_param.tablet_iter_.get_tablet_handle_ptr()); const ObITableReadInfo *read_info = access_param_->iter_param_.get_read_info(); const int64_t batch_size = access_param_->iter_param_.vectorized_enabled_ ? access_param_->get_op()->get_batch_size() : 1; @@ -259,7 +257,6 @@ int ObMultipleMerge::build_extra_access_ctx() if (OB_FAIL(fork_snapshot_scn.convert_for_tx(fork_info.get_fork_snapshot_version()))) { LOG_WARN("fail to convert snapshot version", KR(ret), K(fork_info)); } else if (OB_FAIL(fork_store_ctx->init_for_read( - access_ctx_->store_ctx_->ls_id_, fork_src_tablet_id, access_ctx_->store_ctx_->timeout_, 0, // tx_lock_timeout @@ -1139,7 +1136,6 @@ void ObMultipleMerge::report_tablet_stat() int tmp_ret = OB_SUCCESS; bool report_succ = false; /*placeholder*/ storage::ObTabletStat tablet_stat; - tablet_stat.ls_id_ = access_ctx_->ls_id_.id(); tablet_stat.tablet_id_ = access_ctx_->tablet_id_.id(); tablet_stat.query_cnt_ = 1; tablet_stat.scan_logical_row_cnt_ = access_ctx_->table_store_stat_.logical_read_cnt_; @@ -1993,7 +1989,7 @@ int ObMultipleMerge::refresh_table_on_demand() int ObMultipleMerge::refresh_tablet_iter() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; if (OB_UNLIKELY(!get_table_param_->tablet_iter_.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet iter is invalid", K(ret), K(get_table_param_->tablet_iter_)); @@ -2001,18 +1997,14 @@ int ObMultipleMerge::refresh_tablet_iter() // reset first, in case get_read_tables fail and rowkey_read_info_ become dangling access_param_->iter_param_.rowkey_read_info_ = nullptr; const int64_t remain_timeout = THIS_WORKER.get_timeout_remain(); - const share::ObLSID &ls_id = access_ctx_->ls_id_; const common::ObTabletID &tablet_id = get_table_param_->tablet_iter_.get_tablet()->get_tablet_meta().tablet_id_; const int64_t snapshot_version = generate_read_tables_version(); if (OB_UNLIKELY(remain_timeout <= 0)) { ret = OB_TIMEOUT; - LOG_WARN("timeout reached", K(ret), K(ls_id), K(tablet_id), K(remain_timeout)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_handle)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_read_tables( + LOG_WARN("timeout reached", K(ret), K(tablet_id), K(remain_timeout)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->get_read_tables( tablet_id, remain_timeout, snapshot_version, @@ -2021,7 +2013,7 @@ int ObMultipleMerge::refresh_tablet_iter() false/*allow_not_ready*/, true/*need_split_src_table*/, true/*need_split_dst_table*/))) { - LOG_WARN("failed to refresh tablet iterator", K(ret), K(ls_id), K_(get_table_param), KP_(access_param)); + LOG_WARN("failed to refresh tablet iterator", K(ret), K_(get_table_param), KP_(access_param)); } else { get_table_param_->refreshed_merge_ = this; access_param_->iter_param_.rowkey_read_info_ = @@ -2198,7 +2190,7 @@ int ObMultipleMerge::handle_4377(const char* func) OB_ERR_DEFENSIVE_CHECK, "msg", "Fatal Error!!! Catch a defensive error!", "index lookup: row not found in data-table"); - concurrency_control::ObDataValidationService::set_delay_resource_recycle(access_ctx_->ls_id_); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(); dump_table_statistic_for_4377(); dump_tx_statistic_for_4377(access_ctx_->store_ctx_); } @@ -2270,7 +2262,7 @@ int ObMultipleMerge::check_base_version(const bool is_di_merge_scan) const { void ObMultipleMerge::set_base_version() const { // When the major table is currently being processed, the snapshot version is taken and placed // in the current context for base version to filter unnecessary rows in the mini or minor sstable - if (!access_ctx_->is_mview_query() && is_scan()) { + if (is_scan()) { access_ctx_->trans_version_range_.base_version_ = major_table_version_; LOG_DEBUG("set base version", K_(access_ctx_->trans_version_range)); } diff --git a/src/storage/access/ob_pushdown_aggregate.cpp b/src/storage/access/ob_pushdown_aggregate.cpp index 8fc4f5945..9f1f36c9e 100644 --- a/src/storage/access/ob_pushdown_aggregate.cpp +++ b/src/storage/access/ob_pushdown_aggregate.cpp @@ -19,6 +19,7 @@ #include "sql/engine/aggregate/ob_aggregate_util.h" #include "sql/engine/expr/ob_datum_cast.h" #include "sql/engine/expr/ob_array_expr_utils.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { diff --git a/src/storage/access/ob_pushdown_aggregate_vec.cpp b/src/storage/access/ob_pushdown_aggregate_vec.cpp index 6ad14ecb7..57a5b83a3 100644 --- a/src/storage/access/ob_pushdown_aggregate_vec.cpp +++ b/src/storage/access/ob_pushdown_aggregate_vec.cpp @@ -18,6 +18,7 @@ #include "ob_pushdown_aggregate_vec.h" #include "sql/engine/expr/ob_datum_cast.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { diff --git a/src/storage/access/ob_rows_info.cpp b/src/storage/access/ob_rows_info.cpp index 69a589001..0239ec6dc 100644 --- a/src/storage/access/ob_rows_info.cpp +++ b/src/storage/access/ob_rows_info.cpp @@ -81,7 +81,6 @@ int ObRowsInfo::ExistHelper::init(const ObRelativeTable &table, table_iter_param_.table_id_ = table.get_table_id(); table_iter_param_.tablet_id_ = table.get_tablet_id(); if (nullptr != table.tablet_iter_.get_tablet()) { - table_iter_param_.ls_id_ = table.tablet_iter_.get_tablet()->get_tablet_meta().ls_id_; } table_iter_param_.out_cols_project_ = NULL; table_iter_param_.read_info_ = &rowkey_read_info; diff --git a/src/storage/access/ob_table_access_context.cpp b/src/storage/access/ob_table_access_context.cpp index 864961898..fbb3fc748 100644 --- a/src/storage/access/ob_table_access_context.cpp +++ b/src/storage/access/ob_table_access_context.cpp @@ -54,7 +54,6 @@ ObTableAccessContext::ObTableAccessContext() is_fork_ctx_(false), use_fuse_row_cache_(false), need_scn_(false), - need_release_mview_scan_info_(true), need_release_truncate_part_filter_(true), timeout_(0), query_flag_(), @@ -79,7 +78,6 @@ ObTableAccessContext::ObTableAccessContext() block_row_store_(nullptr), sample_filter_(nullptr), trans_state_mgr_(nullptr), - mview_scan_info_(nullptr), scan_resume_point_(nullptr), truncate_part_filter_(nullptr), mds_collector_(nullptr), @@ -93,14 +91,12 @@ ObTableAccessContext::~ObTableAccessContext() if (is_fork_ctx_) { // fork ctx shallow-copies many pointer members from the main ctx. It should NOT manage // their lifecycles during destruction; just detach them. - need_release_mview_scan_info_ = false; need_release_truncate_part_filter_ = false; reset_lob_locator_helper(); // fork-aware: won't destruct shared helper cached_iter_node_ = nullptr; stmt_iter_pool_ = nullptr; block_row_store_ = nullptr; sample_filter_ = nullptr; - mview_scan_info_ = nullptr; truncate_part_filter_ = nullptr; } else { reset_lob_locator_helper(); @@ -115,12 +111,6 @@ ObTableAccessContext::~ObTableAccessContext() if (OB_UNLIKELY(nullptr != sample_filter_)) { ObRowSampleFilterFactory::destroy_sample_filter(sample_filter_); } - if (OB_UNLIKELY(need_release_mview_scan_info_ && nullptr != mview_scan_info_)) { - release_mview_scan_info(stmt_allocator_, mview_scan_info_); - need_release_mview_scan_info_ = false; - } else { - mview_scan_info_ = nullptr; - } if (OB_UNLIKELY(need_release_truncate_part_filter_ && nullptr != truncate_part_filter_)) { ObTruncatePartitionFilterFactory::destroy_truncate_partition_filter(truncate_part_filter_); } else { @@ -153,10 +143,9 @@ int ObTableAccessContext::build_lob_locator_helper(ObTableScanParam &scan_param, } else if (FALSE_IT(lob_locator_helper_ = new (buf) ObLobLocatorHelper())) { } else if (OB_FAIL(lob_locator_helper_->init(scan_param, ctx, - scan_param.ls_id_, trans_version_range.snapshot_version_))) { STORAGE_LOG(WARN, "Failed to init lob locator helper", - K(ret), KPC(scan_param.table_param_), K(scan_param.ls_id_), K(trans_version_range)); + K(ret), KPC(scan_param.table_param_), K(trans_version_range)); reset_lob_locator_helper(); } else { STORAGE_LOG(DEBUG, "succ to init lob locator helper", KPC(lob_locator_helper_)); @@ -181,9 +170,8 @@ int ObTableAccessContext::build_lob_locator_helper(const ObStoreCtx &ctx, } else if (OB_FAIL(lob_locator_helper_->init(tablet_id_.id(), tablet_id_.id(), ctx, - ls_id_, trans_version_range.snapshot_version_))) { - STORAGE_LOG(WARN, "Failed to init lob locator helper limit", K(ret), K(ls_id_), K(trans_version_range)); + STORAGE_LOG(WARN, "Failed to init lob locator helper limit", K(ret), K(trans_version_range)); reset_lob_locator_helper(); } else { STORAGE_LOG(DEBUG, "succ to init lob locator helper", KPC(lob_locator_helper_)); @@ -208,7 +196,6 @@ int ObTableAccessContext::init(ObTableScanParam &scan_param, stmt_allocator_ = scan_param.allocator_; cached_iter_node_ = cached_iter_node; range_allocator_ = nullptr; - ls_id_ = scan_param.ls_id_; tablet_id_ = scan_param.tablet_id_; query_flag_ = scan_param.scan_flag_; sql_mode_ = scan_param.sql_mode_; @@ -273,7 +260,6 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, stmt_allocator_ = &stmt_allocator; range_allocator_ = nullptr; trans_version_range_ = trans_version_range; - ls_id_ = ctx.ls_id_; tablet_id_ = ctx.tablet_id_; // handle lob types without ObTableScanParam: // 1. use lob locator instead of full lob data @@ -324,7 +310,6 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, stmt_allocator_ = &allocator; range_allocator_ = nullptr; trans_version_range_ = trans_version_range; - ls_id_ = ctx.ls_id_; tablet_id_ = ctx.tablet_id_; lob_locator_helper_ = nullptr; cached_iter_node_ = cached_iter_node; @@ -347,47 +332,6 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, return ret; } -int ObTableAccessContext::init_for_mview(common::ObIAllocator *allocator, const ObTableAccessContext &access_ctx, ObStoreCtx &store_ctx) -{ - int ret = OB_SUCCESS; - if (is_inited_ && stmt_allocator_ != access_ctx.stmt_allocator_) { - ret = OB_INIT_TWICE; - LOG_WARN("cannot init twice", K(ret)); - } else { - stmt_allocator_ = access_ctx.stmt_allocator_; - allocator_ = allocator; - cached_iter_node_ = nullptr; - range_allocator_ = nullptr; - ls_id_ = access_ctx.ls_id_; - tablet_id_ = access_ctx.tablet_id_; - query_flag_ = access_ctx.query_flag_; - sql_mode_ = access_ctx.sql_mode_; - timeout_ = access_ctx.timeout_; - store_ctx_ = &store_ctx; - table_scan_stat_ = nullptr; - limit_param_ = nullptr; - trans_version_range_.base_version_ = 0; - trans_version_range_.snapshot_version_ = access_ctx.trans_version_range_.base_version_; - need_scn_ = false; - range_array_pos_ = nullptr; - use_fuse_row_cache_ = true; - lob_locator_helper_ = nullptr; - if (!micro_block_handle_mgr_.is_valid() && - OB_FAIL(micro_block_handle_mgr_.init( - false, - table_store_stat_, - table_scan_stat_, - query_flag_))) { - LOG_WARN("Failed to init micro block handle mgr", K(ret)); - } else { - mview_scan_info_ = access_ctx.mview_scan_info_; - need_release_mview_scan_info_ = false; - is_inited_ = true; - } - } - return ret; -} - int ObTableAccessContext::init_for_fork(ObTableAccessContext &other, ObStoreCtx *store_ctx, const ObForkTabletInfo &fork_info) { @@ -399,7 +343,6 @@ int ObTableAccessContext::init_for_fork(ObTableAccessContext &other, allocator_ = other.allocator_; cached_iter_node_ = other.cached_iter_node_; range_allocator_ = other.range_allocator_; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; query_flag_ = other.query_flag_; // disable row cache for fork @@ -423,11 +366,9 @@ int ObTableAccessContext::init_for_fork(ObTableAccessContext &other, block_row_store_ = other.block_row_store_; sample_filter_ = other.sample_filter_; trans_state_mgr_ = other.trans_state_mgr_; - mview_scan_info_ = other.mview_scan_info_; scan_resume_point_ = other.scan_resume_point_; truncate_part_filter_ = other.truncate_part_filter_; // fork ctx shallow-copies these shared pointers, should not release them - need_release_mview_scan_info_ = false; need_release_truncate_part_filter_ = false; mds_collector_ = other.mds_collector_; row_scan_cnt_ = other.row_scan_cnt_; @@ -439,7 +380,6 @@ int ObTableAccessContext::init_for_fork(ObTableAccessContext &other, query_flag_))) { LOG_WARN("Failed to init micro block handle mgr", K(ret)); } else { - need_release_mview_scan_info_ = false; is_inited_ = true; } return ret; @@ -474,28 +414,12 @@ int ObTableAccessContext::init_scan_allocator(ObTableScanParam &scan_param) return ret; } -int ObTableAccessContext::init_mview_scan_info(const int64_t multi_version_start, const sql::ObExprPtrIArray *op_filters, sql::ObEvalCtx &eval_ctx) -{ - int ret = OB_SUCCESS; - if (nullptr != mview_scan_info_) { - } else if (OB_FAIL(build_mview_scan_info_if_need(query_flag_, op_filters, eval_ctx, stmt_allocator_, mview_scan_info_))) { - LOG_WARN("Failed to build mview scan info", K(ret)); - } - if (OB_FAIL(ret) || nullptr == mview_scan_info_) { - } else if (OB_FAIL(mview_scan_info_->check_and_update_version_range(multi_version_start, trans_version_range_))) { - LOG_WARN("Failed to check and update version range", K(ret), K(multi_version_start), K(trans_version_range_), KPC_(mview_scan_info)); - } - return ret; -} - void ObTableAccessContext::reset() { if (is_fork_ctx_) { // fork ctx shallow-copies many pointer members from main ctx; don't release/mutate them. - need_release_mview_scan_info_ = false; need_release_truncate_part_filter_ = false; reset_lob_locator_helper(); // fork-aware - mview_scan_info_ = nullptr; truncate_part_filter_ = nullptr; cached_iter_node_ = nullptr; stmt_iter_pool_ = nullptr; @@ -503,12 +427,6 @@ void ObTableAccessContext::reset() sample_filter_ = nullptr; } else { reset_lob_locator_helper(); - if (OB_UNLIKELY(need_release_mview_scan_info_ && nullptr != mview_scan_info_)) { - release_mview_scan_info(stmt_allocator_, mview_scan_info_); - need_release_mview_scan_info_ = false; - } else { - mview_scan_info_ = nullptr; - } if (OB_UNLIKELY(need_release_truncate_part_filter_ && nullptr != truncate_part_filter_)) { ObTruncatePartitionFilterFactory::destroy_truncate_partition_filter(truncate_part_filter_); } else { @@ -525,7 +443,6 @@ void ObTableAccessContext::reset() } is_inited_ = false; timeout_ = 0; - ls_id_.reset(); tablet_id_.reset(); query_flag_.reset(); sql_mode_ = 0; @@ -578,7 +495,6 @@ int ObTableAccessContext::rescan_reuse(ObTableScanParam &scan_param) void ObTableAccessContext::reuse() { timeout_ = 0; - ls_id_.reset(); tablet_id_.reset(); query_flag_.reset(); sql_mode_ = 0; diff --git a/src/storage/access/ob_table_access_context.h b/src/storage/access/ob_table_access_context.h index d4a6ad46a..d195b3914 100644 --- a/src/storage/access/ob_table_access_context.h +++ b/src/storage/access/ob_table_access_context.h @@ -169,8 +169,8 @@ struct ObTableAccessContext inline bool enable_get_fuse_row_cache(const int64_t threshold) const { return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_get_fuse_row_cache(threshold) && !need_scn_ && !tablet_id_.is_ls_inner_tablet() && !has_truncate_filter(); } - inline bool enable_put_fuse_row_cache(const int64_t threshold, const bool is_mview_table_scan) const { - return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_put_fuse_row_cache(threshold) && (!need_scn_ || is_mview_table_scan) && !tablet_id_.is_ls_inner_tablet() && !has_truncate_filter(); + inline bool enable_put_fuse_row_cache(const int64_t threshold) const { + return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_put_fuse_row_cache(threshold) && !need_scn_ && !tablet_id_.is_ls_inner_tablet() && !has_truncate_filter(); } inline bool is_limit_end() const { return (nullptr != limit_param_ && limit_param_->limit_ >= 0 && (out_cnt_ - limit_param_->offset_ >= limit_param_->limit_)); @@ -221,26 +221,13 @@ struct ObTableAccessContext const common::ObVersionRange &trans_version_range, memtable::ObMvccMdsFilter *mds_filter = nullptr, CachedIteratorNode *cached_iter_node = nullptr); - // used for mview table scan - int init_for_mview(common::ObIAllocator *allocator, - const ObTableAccessContext &access_ctx, - ObStoreCtx &store_ctx); // used for fork table scan int init_for_fork(ObTableAccessContext &other, ObStoreCtx *store_ctx, const ObForkTabletInfo &fork_info); - OB_INLINE bool is_mview_query() const - { - return nullptr != mview_scan_info_; - } - OB_INLINE StorageScanType get_scan_type() const - { - return is_mview_query() ? mview_scan_info_->scan_type_ : StorageScanType::NORMAL; - } int alloc_iter_pool(); void inc_micro_access_cnt(); int init_scan_allocator(ObTableScanParam &scan_param); - int init_mview_scan_info(const int64_t multi_version_start, const sql::ObExprPtrIArray *op_filters, sql::ObEvalCtx &eval_ctx); int check_filtered_by_base_version(ObDatumRow &row); OB_INLINE bool has_truncate_filter() const { @@ -286,10 +273,8 @@ struct ObTableAccessContext K_(is_inited), K_(use_fuse_row_cache), K_(need_scn), - K_(need_release_mview_scan_info), K_(need_release_truncate_part_filter), K_(timeout), - K_(ls_id), K_(tablet_id), K_(query_flag), K_(sql_mode), @@ -308,7 +293,6 @@ struct ObTableAccessContext KP_(stmt_iter_pool), KP_(block_row_store), KP_(sample_filter), - KPC_(mview_scan_info), K_(table_store_stat), KP_(truncate_part_filter), KP_(mds_collector), @@ -346,10 +330,8 @@ struct ObTableAccessContext bool is_fork_ctx_; bool use_fuse_row_cache_; // temporary code bool need_scn_; - bool need_release_mview_scan_info_; bool need_release_truncate_part_filter_; int64_t timeout_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; common::ObQueryFlag query_flag_; ObSQLMode sql_mode_; @@ -377,7 +359,6 @@ struct ObTableAccessContext ObBlockRowStore *block_row_store_; ObRowSampleFilter *sample_filter_; compaction::ObCachedTransStateMgr *trans_state_mgr_; - ObMviewScanInfo *mview_scan_info_; ScanResumePoint *scan_resume_point_; // for scan pause ObTruncatePartitionFilter *truncate_part_filter_; ObMdsReadInfoCollector *mds_collector_; // used for collect mds info when query mds sstable diff --git a/src/storage/access/ob_table_access_param.cpp b/src/storage/access/ob_table_access_param.cpp index ffd425b72..9ea8f0fdc 100644 --- a/src/storage/access/ob_table_access_param.cpp +++ b/src/storage/access/ob_table_access_param.cpp @@ -30,7 +30,6 @@ namespace storage ObTableIterParam::ObTableIterParam() : table_id_(0), tablet_id_(), - ls_id_(), read_info_(nullptr), rowkey_read_info_(nullptr), tablet_handle_(nullptr), @@ -84,7 +83,6 @@ void ObTableIterParam::reset() { table_id_ = 0; tablet_id_.reset(); - ls_id_.reset(); read_info_ = nullptr; rowkey_read_info_ = nullptr; tablet_handle_ = nullptr; @@ -139,10 +137,10 @@ int ObTableIterParam::refresh_lob_column_out_status() return ret; } -bool ObTableIterParam::enable_fuse_row_cache(const ObQueryFlag &query_flag, const StorageScanType scan_type) const +bool ObTableIterParam::enable_fuse_row_cache(const ObQueryFlag &query_flag) const { bool bret = query_flag.is_use_fuse_row_cache() && !query_flag.is_read_latest() && - nullptr != rowkey_read_info_ && (!need_scn_ || is_mview_table_scan(scan_type)) && + nullptr != rowkey_read_info_ && !need_scn_ && is_same_schema_column_ && !has_virtual_columns_ && !has_lob_column_out_; return bret; } @@ -178,7 +176,6 @@ DEF_TO_STRING(ObTableIterParam) J_OBJ_START(); J_KV(K_(table_id), K_(tablet_id), - K_(ls_id), KPC_(read_info), KPC_(rowkey_read_info), KPC_(out_cols_project), @@ -270,7 +267,6 @@ int ObTableAccessParam::init( const share::schema::ObTableParam &table_param = *scan_param.table_param_; iter_param_.table_id_ = table_param.get_table_id(); iter_param_.tablet_id_ = scan_param.tablet_id_; - iter_param_.ls_id_ = scan_param.ls_id_; iter_param_.read_info_ = &table_param.get_read_info(); if (nullptr == tablet_handle) { @@ -317,7 +313,6 @@ int ObTableAccessParam::init( iter_param_.table_scan_opt_.storage_rowsets_size_ = 1; } iter_param_.pushdown_filter_ = scan_param.pd_storage_filters_; - iter_param_.ls_id_ = scan_param.ls_id_; // disable blockscan if scan order is KeepOrder // disable blockscan if use index skip scan as no large range to scan if (OB_UNLIKELY(ObQueryFlag::KeepOrder == scan_param.scan_flag_.scan_order_ || @@ -335,8 +330,7 @@ int ObTableAccessParam::init( iter_param_.limit_prefetch_ = (nullptr == op_filters_ || op_filters_->empty()); iter_param_.is_mds_query_ = scan_param.is_mds_query_; - if (scan_param.need_switch_param_ || - scan_param.is_mview_query()) { + if (scan_param.need_switch_param_) { iter_param_.set_use_stmt_iter_pool(); } @@ -437,7 +431,6 @@ int ObTableAccessParam::init_dml_access_param( iter_param_.table_id_ = table.get_table_id(); iter_param_.tablet_id_ = table.get_tablet_id(); if (nullptr != table.tablet_iter_.get_tablet()) { - iter_param_.ls_id_ = table.tablet_iter_.get_tablet()->get_tablet_meta().ls_id_; } iter_param_.read_info_ = &schema_param.get_read_info(); iter_param_.rowkey_read_info_ = &rowkey_read_info; diff --git a/src/storage/access/ob_table_access_param.h b/src/storage/access/ob_table_access_param.h index 0a0512ba2..34f547b02 100644 --- a/src/storage/access/ob_table_access_param.h +++ b/src/storage/access/ob_table_access_param.h @@ -60,7 +60,7 @@ struct ObTableIterParam && (nullptr == rowkey_read_info_ || rowkey_read_info_->is_valid()); } int refresh_lob_column_out_status(); - bool enable_fuse_row_cache(const ObQueryFlag &query_flag, const StorageScanType scan_type) const; + bool enable_fuse_row_cache(const ObQueryFlag &query_flag) const; int build_index_filter_for_row_store(common::ObIAllocator *allocator); const ObITableReadInfo *get_read_info(const bool is_get = false) const { @@ -111,10 +111,6 @@ struct ObTableIterParam { return (read_info_ != nullptr && read_info_ != rowkey_read_info_) ? read_info_->get_group_idx_col_index() : common::OB_INVALID_INDEX; } - OB_INLINE int64_t get_mview_old_new_col_index() const - { - return (read_info_ != nullptr && read_info_ != rowkey_read_info_) ? read_info_->get_mview_old_new_col_index() : common::OB_INVALID_INDEX; - } OB_INLINE bool need_truncate_filter() const { return (read_info_ != nullptr && read_info_ != rowkey_read_info_) ? read_info_->need_truncate_filter() : false; @@ -186,7 +182,6 @@ struct ObTableIterParam public: uint64_t table_id_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; const ObITableReadInfo *read_info_; const ObITableReadInfo *rowkey_read_info_; const ObTabletHandle *tablet_handle_; //for ddl merge_query diff --git a/src/storage/access/ob_table_scan_iterator.cpp b/src/storage/access/ob_table_scan_iterator.cpp index 1f1743fa7..c96eca4fd 100644 --- a/src/storage/access/ob_table_scan_iterator.cpp +++ b/src/storage/access/ob_table_scan_iterator.cpp @@ -43,7 +43,6 @@ ObTableScanIterator::ObTableScanIterator() row_sample_iterator_(NULL), block_sample_iterator_(NULL), // i_sample_iter_(NULL), - mview_merge_wrapper_(NULL), main_table_param_(), main_table_ctx_(), get_table_param_(), @@ -73,7 +72,6 @@ void ObTableScanIterator::reset() reset_scan_iter(skip_scan_merge_); reset_scan_iter(memtable_row_sample_iterator_); reset_scan_iter(block_sample_iterator_); - reset_scan_iter(mview_merge_wrapper_); reset_scan_iter(ddl_block_sample_iterator_); // reset_scan_iter(i_sample_iter_); if (nullptr != cached_iter_node_) { @@ -122,7 +120,6 @@ void ObTableScanIterator::reuse_row_iters() REUSE_SCAN_ITER(block_sample_iterator_); REUSE_SCAN_ITER(ddl_block_sample_iterator_); // REUSE_SCAN_ITER(i_sample_iter_); - REUSE_SCAN_ITER(mview_merge_wrapper_); #undef REUSE_SCAN_ITER } @@ -154,8 +151,7 @@ bool ObTableScanIterator::can_use_global_iter_pool(const ObQRIterType iter_type) !scan_param_->sample_info_.is_no_sample() || main_table_param_.iter_param_.enable_pd_aggregate() || main_table_param_.iter_param_.enable_pd_group_by() || - main_table_param_.iter_param_.has_lob_column_out_ || - scan_param_->is_mview_query()) { + main_table_param_.iter_param_.has_lob_column_out_) { } else { const int64_t table_cnt = get_table_param_.tablet_iter_.table_iter()->count(); int64_t col_cnt = MAX(scan_param_->table_param_->get_read_info().get_schema_column_count(), @@ -227,17 +223,6 @@ int ObTableScanIterator::prepare_table_context() STORAGE_LOG(WARN, "trans version range is not valid", K(ret), K(trans_version_range)); } else if (OB_FAIL(main_table_ctx_.init(*scan_param_, ctx_guard_.get_store_ctx(), trans_version_range, cached_iter_node_))) { STORAGE_LOG(WARN, "failed to init main table ctx", K(ret)); - } else if (scan_param_->is_mview_query()) { - const ObTabletMeta &tablet_meta = get_table_param_.tablet_iter_.get_tablet()->get_tablet_meta(); - if (OB_ISNULL(main_table_param_.op_filters_) || scan_param_->table_param_->use_lob_locator()) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected null scn filter or use lob locator in mview query", K(ret), KP(main_table_param_.op_filters_), - K(scan_param_->table_param_->use_lob_locator())); - } else if (OB_FAIL(main_table_ctx_.init_mview_scan_info(tablet_meta.multi_version_start_, - main_table_param_.op_filters_, - main_table_param_.get_op()->get_eval_ctx()))) { - STORAGE_LOG(WARN, "failed to init mview scan info", K(ret)); - } } } return ret; @@ -452,7 +437,6 @@ int ObTableScanIterator::rescan_for_iter() RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, skip_scan_merge_); RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, memtable_row_sample_iterator_); RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, block_sample_iterator_); - RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, mview_merge_wrapper_); RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, ddl_block_sample_iterator_); get_table_param_.refreshed_merge_ = nullptr; } @@ -478,11 +462,6 @@ int ObTableScanIterator::switch_param_for_iter() SWITCH_PARAM_FOR_ITER(multi_scan_merge_, ret); SWITCH_PARAM_FOR_ITER(skip_scan_merge_, ret); #undef SWITCH_PARAM_FOR_ITER - if (OB_SUCC(ret) && nullptr != mview_merge_wrapper_) { - if (OB_FAIL(mview_merge_wrapper_->switch_param(main_table_param_, main_table_ctx_, get_table_param_))) { - STORAGE_LOG(WARN, "Failed to switch param", K(ret)); - } - } return ret; } @@ -572,15 +551,7 @@ int ObTableScanIterator::open_iter() } else { get_table_param_.frozen_version_ = scan_param_->frozen_version_; get_table_param_.sample_info_ = scan_param_->sample_info_; - if (main_table_ctx_.is_mview_query()) { - ObMviewMerge *mview_merge = nullptr; - if (OB_FAIL(ObMviewMergeWrapper::alloc_mview_merge(main_table_param_, main_table_ctx_, get_table_param_, - table_scan_range_, mview_merge_wrapper_, mview_merge))) { - STORAGE_LOG(WARN, "Failed to alloc mview merge", K(ret)); - } else { - main_iter_ = mview_merge; - } - } else if (table_scan_range_.is_get()) { + if (table_scan_range_.is_get()) { if (OB_FAIL(init_and_open_get_merge_iter_())) { STORAGE_LOG(WARN, "init and open get merge iterator failed", KR(ret)); } @@ -782,19 +753,6 @@ int ObTableScanIterator::get_next_rows(int64_t &count, int64_t capacity) return ret; } -int ObTableScanIterator::check_ls_offline_after_read() -{ - int ret = OB_SUCCESS; - - memtable::ObMvccAccessCtx &acc_ctx = ctx_guard_.get_store_ctx().mvcc_acc_ctx_; - - if (acc_ctx.tx_table_guards_.check_ls_offline()) { - ret = OB_LS_OFFLINE; - STORAGE_LOG(WARN, "ls offline during the read operation", K(ret), K(acc_ctx.snapshot_)); - } - return ret; -} - int ObTableScanIterator::check_txn_status_if_read_uncommitted_() { int ret = OB_SUCCESS; @@ -822,16 +780,14 @@ int ObTableScanIterator::check_advance_scan_supported() if (OB_UNLIKELY(scan_param_->scan_flag_.scan_order_ != ObQueryFlag::Forward || !scan_param_->sample_info_.is_no_sample() || scan_param_->use_index_skip_scan() || - scan_param_->is_mview_query() || main_table_param_.iter_param_.enable_pd_aggregate() || main_table_param_.iter_param_.enable_pd_group_by())) { ret = OB_NOT_SUPPORTED; - STORAGE_LOG(WARN, "advance_scan not supported for not forward scan/sample/index skip scan/mview/group by pushdown", + STORAGE_LOG(WARN, "advance_scan not supported for not forward scan/sample/index skip scan/group by pushdown", K(ret), K(scan_param_->scan_flag_.scan_order_), K(scan_param_->sample_info_.is_no_sample()), K(scan_param_->use_index_skip_scan()), - K(scan_param_->is_mview_query()), K(main_table_param_.iter_param_.enable_pd_aggregate()), K(main_table_param_.iter_param_.enable_pd_group_by()), K(lbt())); diff --git a/src/storage/access/ob_table_scan_iterator.h b/src/storage/access/ob_table_scan_iterator.h index 049f7619a..2e28c2620 100644 --- a/src/storage/access/ob_table_scan_iterator.h +++ b/src/storage/access/ob_table_scan_iterator.h @@ -32,9 +32,7 @@ #include "ob_multiple_skip_scan_merge.h" #include "ob_multiple_multi_skip_scan_merge.h" #include "ob_single_merge.h" -#include "ob_multiple_mview_merge.h" #include "storage/tx_storage/ob_access_service.h" -#include "storage/tx_storage/ob_ls_map.h" #include "storage/tx/ob_trans_service.h" #include "ob_table_scan_range.h" #include "ob_global_iterator_pool.h" @@ -71,7 +69,6 @@ class ObTableScanIterator : public common::ObNewRowIterator // A offline ls will disable replay status and kill all part_ctx on the follower. // We can not read the uncommitted data which has not replay commit log yet. - int check_ls_offline_after_read(); public: static constexpr int64_t RP_MAX_FREE_LIST_NUM = 1024; static constexpr const char LABEL[] = "RPTableScanIter"; @@ -116,7 +113,6 @@ class ObTableScanIterator : public common::ObNewRowIterator ObMemtableRowSampleIterator *memtable_row_sample_iterator_; ObRowSampleIterator *row_sample_iterator_; ObBlockSampleIterator *block_sample_iterator_; // TODO: @yuanzhe refactor - ObMviewMergeWrapper *mview_merge_wrapper_; // we should consider the constructor cost ObTableAccessParam main_table_param_; ObTableAccessContext main_table_ctx_; diff --git a/src/storage/access/ob_table_scan_range.cpp b/src/storage/access/ob_table_scan_range.cpp index 5c6305a78..f313e9760 100644 --- a/src/storage/access/ob_table_scan_range.cpp +++ b/src/storage/access/ob_table_scan_range.cpp @@ -149,8 +149,7 @@ int ObTableScanRange::init( ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid argument to init table scan range", K(ret), K(simple_batch)); } else if (FALSE_IT(allocator_ = &allocator)) { - } else if (OB_FAIL(split_query.get_tablet_handle(scan_param.tablet_id_, - scan_param.ls_id_, tablet_handle))) { + } else if (OB_FAIL(split_query.get_tablet_handle(scan_param.tablet_id_, tablet_handle))) { STORAGE_LOG(WARN, "fail to get tablet handle", K(ret), K(scan_param)); } else if (OB_FAIL(ObTabletSplitMdsHelper::get_is_spliting(*tablet_handle.get_obj(), is_tablet_spliting))) { STORAGE_LOG(WARN, "fail to get tablet split status", K(ret)); @@ -308,7 +307,6 @@ int ObTableScanRange::init_ranges( ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid argument to init ranges", K(ret), K(allocator_)); } else { - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const int64_t range_cnt = ranges.count(); if (0 == range_cnt) { @@ -343,7 +341,7 @@ int ObTableScanRange::init_ranges( *allocator_, datum_range, is_false))) { - STORAGE_LOG(WARN, "Failed to get split datum range", K(ret), K(tablet_id), K(ls_id)); + STORAGE_LOG(WARN, "Failed to get split datum range", K(ret), K(tablet_id)); } else if (is_false) { STORAGE_LOG(INFO, "Range after split is empty", K(ret), K(range_cnt), K(i), K(tablet_id), K(range)); } else if (OB_FAIL(ranges_.push_back(datum_range))) { @@ -381,7 +379,6 @@ int ObTableScanRange::init_ranges_in_skip_scan(const ObTablet &tablet, ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid argument to init ranges", K(ret), K(allocator_), K(ranges.count()), K(skip_scan_ranges.count())); } else { - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; common::ObSEArray wrapped_ranges_; const int64_t range_cnt = ranges.count(); @@ -406,7 +403,7 @@ int ObTableScanRange::init_ranges_in_skip_scan(const ObTablet &tablet, *allocator_, wrapped_range.datum_range_, is_false))) { - STORAGE_LOG(WARN, "Failed to get split datum range", K(ret), K(tablet_id), K(ls_id)); + STORAGE_LOG(WARN, "Failed to get split datum range", K(ret), K(tablet_id)); } else if (is_false) { STORAGE_LOG(INFO, "Range after split is empty", K(ret), K(range_cnt), K(i), K(tablet_id), K(range), K(skip_scan_range)); } else if (OB_FAIL(wrapped_range.datum_skip_range_.from_range(skip_scan_range, *allocator_, enable_new_false_range_))) { diff --git a/src/storage/access/ob_vector_store.cpp b/src/storage/access/ob_vector_store.cpp index 113177499..3db3c1bcf 100644 --- a/src/storage/access/ob_vector_store.cpp +++ b/src/storage/access/ob_vector_store.cpp @@ -20,6 +20,7 @@ #include "ob_aggregated_store_vec.h" #include "storage/blocksstable/ob_micro_block_row_scanner.h" #include "storage/access/ob_pushdown_aggregate.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { diff --git a/src/storage/allocator/ob_mds_allocator.cpp b/src/storage/allocator/ob_mds_allocator.cpp index 84c161482..3b9baeeba 100644 --- a/src/storage/allocator/ob_mds_allocator.cpp +++ b/src/storage/allocator/ob_mds_allocator.cpp @@ -79,8 +79,8 @@ void ObTenantMdsAllocator::adaptive_update_limit(const int64_t holding_size, // moved definition to storage mds_tenant_service.cpp(MDS service-record real user) -ObMdsThrottleGuard::ObMdsThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time) - : ls_id_(ls_id), for_replay_(for_replay), abs_expire_time_(abs_expire_time) +ObMdsThrottleGuard::ObMdsThrottleGuard(const bool for_replay, const int64_t abs_expire_time) + : for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(share::g_mp->shared_mem_alloc_mgr()->share_resource_throttle_tool()); if (0 == abs_expire_time) { diff --git a/src/storage/allocator/ob_mds_allocator.h b/src/storage/allocator/ob_mds_allocator.h index 81d00fa1a..709e97926 100644 --- a/src/storage/allocator/ob_mds_allocator.h +++ b/src/storage/allocator/ob_mds_allocator.h @@ -19,7 +19,6 @@ #include "lib/allocator/ob_vslice_alloc.h" #include "storage/throttle/ob_share_throttle_define.h" -#include "share/ob_ls_id.h" namespace oceanbase { namespace share { @@ -68,11 +67,10 @@ struct ObTenantBufferCtxAllocator : public ObIAllocator// for now, it is just a class ObMdsThrottleGuard { public: - ObMdsThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time); + ObMdsThrottleGuard(const bool for_replay, const int64_t abs_expire_time); ~ObMdsThrottleGuard(); private: - const share::ObLSID ls_id_; bool for_replay_; int64_t abs_expire_time_; share::TxShareThrottleTool *throttle_tool_; diff --git a/src/storage/allocator/ob_tx_data_allocator.cpp b/src/storage/allocator/ob_tx_data_allocator.cpp index 17cb68e0f..9951c3575 100644 --- a/src/storage/allocator/ob_tx_data_allocator.cpp +++ b/src/storage/allocator/ob_tx_data_allocator.cpp @@ -64,10 +64,9 @@ void ObTenantTxDataAllocator::reset() // moved definition to storage ls_service.cpp(TX_DATA_SLICE_SIZE real user) -ObTxDataThrottleGuard::ObTxDataThrottleGuard(const ObLSID ls_id, - const bool for_replay, +ObTxDataThrottleGuard::ObTxDataThrottleGuard(const bool for_replay, const int64_t abs_expire_time) - : ls_id_(ls_id), for_replay_(for_replay), abs_expire_time_(abs_expire_time) + : for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(share::g_mp->shared_mem_alloc_mgr()->share_resource_throttle_tool()); if (0 == abs_expire_time) { diff --git a/src/storage/allocator/ob_tx_data_allocator.h b/src/storage/allocator/ob_tx_data_allocator.h index 5c1500bac..81fdf8ea5 100644 --- a/src/storage/allocator/ob_tx_data_allocator.h +++ b/src/storage/allocator/ob_tx_data_allocator.h @@ -20,12 +20,10 @@ #include "lib/allocator/ob_slice_alloc.h" #include "lib/allocator/ob_vslice_alloc.h" #include "share/ob_delegate.h" -#include "share/ob_ls_id.h" #include "storage/throttle/ob_share_throttle_define.h" namespace oceanbase { namespace share { -class ObLSID; OB_INLINE int64_t &tx_data_throttled_alloc() { @@ -73,11 +71,10 @@ class ObTenantTxDataAllocator { class ObTxDataThrottleGuard { public: - ObTxDataThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time); + ObTxDataThrottleGuard(const bool for_replay, const int64_t abs_expire_time); ~ObTxDataThrottleGuard(); private: - const share::ObLSID ls_id_; bool for_replay_; int64_t abs_expire_time_; share::TxShareThrottleTool *throttle_tool_; diff --git a/src/storage/blocksstable/encoding/ob_new_column_decoder.cpp b/src/storage/blocksstable/encoding/ob_new_column_decoder.cpp index baf8312b9..92982e84c 100644 --- a/src/storage/blocksstable/encoding/ob_new_column_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_new_column_decoder.cpp @@ -18,6 +18,7 @@ #include "ob_new_column_decoder.h" #include "storage/access/ob_aggregate_base.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp index f13a72803..fe9f39859 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp @@ -18,7 +18,7 @@ #include "ob_index_block_row_scanner.h" #include "storage/access/ob_rows_info.h" #include "storage/access/ob_index_skip_scanner.h" -#include "src/storage/tx_storage/ob_ls_map.h" +#include "storage/tablet/ob_tablet.h" namespace oceanbase { diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h index 56fa1730a..7b3ca6578 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h +++ b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h @@ -333,7 +333,6 @@ class ObIndexBlockRowScanner const common::ObQueryFlag &query_flag, const int64_t nested_offset, const ObITableReadInfo *table_read_info = nullptr); - // todo :qilu get ls_id from MTL() after ddl_kv_mgr split to tenant int open( const MacroBlockId ¯o_id, const ObMicroBlockData &idx_block_data, diff --git a/src/storage/blocksstable/ob_bloom_filter_cache.cpp b/src/storage/blocksstable/ob_bloom_filter_cache.cpp index b22bd82b4..b7e385465 100644 --- a/src/storage/blocksstable/ob_bloom_filter_cache.cpp +++ b/src/storage/blocksstable/ob_bloom_filter_cache.cpp @@ -780,7 +780,6 @@ int ObBloomFilterCache::may_contain( int ObBloomFilterCache::inc_empty_read( const uint64_t table_id, - const share::ObLSID &ls_id, const storage::ObITable::TableKey &sstable_key, const MacroBlockId ¯o_id, const int64_t empty_read_prefix, @@ -817,7 +816,7 @@ int ObBloomFilterCache::inc_empty_read( } else if (cell->check_timeout()) { // do nothing } else if (cur_cnt > bf_cache_miss_count_threshold_) { - if (ls_id.is_valid() && sstable_key.is_valid() && (nullptr != read_handle) && read_handle->has_macro_block_bf_) { + if (sstable_key.is_valid() && (nullptr != read_handle) && read_handle->has_macro_block_bf_) { bool need_load = false; const ObDatumRowkey *rowkey = nullptr; if (OB_UNLIKELY(!read_handle->is_valid())) { @@ -835,7 +834,7 @@ int ObBloomFilterCache::inc_empty_read( } else if (cell->check_waiting()) { // bf is on the way, do nothing } else if (OB_FAIL(share::g_mp->tenant_meta_mem_mgr() - ->schedule_load_bloomfilter(sstable_key, ls_id, macro_id, *rowkey))) { + ->schedule_load_bloomfilter(sstable_key, macro_id, *rowkey))) { LOG_WARN("fail to schedule load bf", K(ret), K(sstable_key), K(macro_id)); } else { cell->set_waiting(); diff --git a/src/storage/blocksstable/ob_bloom_filter_cache.h b/src/storage/blocksstable/ob_bloom_filter_cache.h index 230abe2ff..59396845f 100644 --- a/src/storage/blocksstable/ob_bloom_filter_cache.h +++ b/src/storage/blocksstable/ob_bloom_filter_cache.h @@ -191,7 +191,6 @@ class ObBloomFilterCache : public common::ObKVCache *&array); @@ -160,7 +156,6 @@ class ObMacroBlockBloomFilterLoadThread : public lib::ThreadPool void destroy(); void run1() override; int add_load_task(const storage::ObITable::TableKey &sstable_key, - const share::ObLSID &ls_id, const MacroBlockId ¯o_id, const ObDatumRowkey &rowkey); diff --git a/src/storage/blocksstable/ob_data_store_desc.cpp b/src/storage/blocksstable/ob_data_store_desc.cpp index aa101f5fa..dc9375dee 100644 --- a/src/storage/blocksstable/ob_data_store_desc.cpp +++ b/src/storage/blocksstable/ob_data_store_desc.cpp @@ -36,8 +36,7 @@ ObStaticDataStoreDesc::ObStaticDataStoreDesc() bool ObStaticDataStoreDesc::is_valid() const { bool is_valid = - ls_id_.is_valid() - && tablet_id_.is_valid() + tablet_id_.is_valid() && compressor_type_ > ObCompressorType::INVALID_COMPRESSOR && snapshot_version_ > 0 && schema_version_ >= 0; @@ -56,7 +55,6 @@ int ObStaticDataStoreDesc::assign(const ObStaticDataStoreDesc &desc) is_ddl_ = desc.is_ddl_; merge_type_ = desc.merge_type_; compressor_type_ = desc.compressor_type_; - ls_id_ = desc.ls_id_; tablet_id_ = desc.tablet_id_; macro_block_size_ = desc.macro_block_size_; macro_store_size_ = desc.macro_store_size_; @@ -118,7 +116,6 @@ void ObStaticDataStoreDesc::init_block_size(const ObMergeSchema &merge_schema) int ObStaticDataStoreDesc::init( const bool is_ddl, const ObMergeSchema &merge_schema, - const share::ObLSID &ls_id, const common::ObTabletID tablet_id, const compaction::ObMergeType merge_type, const int64_t snapshot_version, @@ -132,7 +129,7 @@ int ObStaticDataStoreDesc::init( { int ret = OB_SUCCESS; const bool is_major = compaction::is_major_or_meta_merge_type(merge_type); - if (OB_UNLIKELY(!merge_schema.is_valid() || !ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 + if (OB_UNLIKELY(!merge_schema.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || (!is_major && !end_scn.is_valid()) || !is_valid_exec_mode(exec_mode))) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "arguments is invalid", K(ret), K(merge_schema), K(snapshot_version), K(end_scn), @@ -141,7 +138,6 @@ int ObStaticDataStoreDesc::init( reset(); is_ddl_ = is_ddl; merge_type_ = merge_type; - ls_id_ = ls_id; tablet_id_ = tablet_id; exec_mode_ = exec_mode; encoding_granularity_ = encoding_granularity; @@ -482,44 +478,30 @@ int ObDataStoreDesc::get_emergency_row_store_type() char *endptr = nullptr; ObTabletID emergency_tablet_id(std::strtoull(partition_key, &endptr, 0)); - uint64_t emergency_ls_id = 0; if (OB_ISNULL(endptr)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid emergency tablet_id for skiping encoding", K(ret), K(partition_key)); } else { - // skip space and get ls id + // Skip spaces before tenant id. while ('\0' != *endptr && isspace(*endptr)) { endptr++; } - if ('#' != *endptr) { + if ('@' != *endptr) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid emergency partition key for skiping encoding", K(ret), K(partition_key)); } else { endptr++; - emergency_ls_id = std::strtoull(endptr, &endptr, 0); - - // skip space and get tenant id - while ('\0' != *endptr && isspace(*endptr)) { - endptr++; - } - if ('@' != *endptr) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid emergency partition key for skiping encoding", K(ret), K(partition_key)); - } else { - endptr++; - (void)(std::strtoull(endptr, &endptr, 0)); - } + (void)(std::strtoull(endptr, &endptr, 0)); if (OB_SUCC(ret)) { oceanbase::share::ObTaskController::get().allow_next_syslog(); - if (get_tablet_id() == emergency_tablet_id - && get_ls_id().id() == emergency_ls_id - && true) { + if (EMERGENCY_TABLET_ID_MAGIC == emergency_tablet_id + || get_tablet_id() == emergency_tablet_id) { STORAGE_LOG(INFO, "Succ to find specified emergency partition to skip encoding", - K(emergency_ls_id), K(emergency_tablet_id), K(*this)); + K(emergency_tablet_id), K(*this)); row_store_type_ = FLAT_ROW_STORE; } else { STORAGE_LOG(INFO, "this partition is not the emergency partition to skip encoding", - K(emergency_ls_id), K(emergency_tablet_id), K(*this)); + K(emergency_tablet_id), K(*this)); } } } @@ -741,7 +723,6 @@ int ObWholeDataStoreDesc::init( int ObWholeDataStoreDesc::init( const bool is_ddl, const ObMergeSchema &merge_schema, - const share::ObLSID &ls_id, const common::ObTabletID tablet_id, const compaction::ObMergeType merge_type, const int64_t snapshot_version, @@ -763,7 +744,7 @@ int ObWholeDataStoreDesc::init( } - if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, ls_id, tablet_id, merge_type, + if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, tablet_id, merge_type, snapshot_version, end_scn, cluster_version, exec_mode, micro_index_clustered, concurrent_cnt, need_submit_io, encoding_granularity))) { diff --git a/src/storage/blocksstable/ob_data_store_desc.h b/src/storage/blocksstable/ob_data_store_desc.h index 80df1b202..134f21341 100644 --- a/src/storage/blocksstable/ob_data_store_desc.h +++ b/src/storage/blocksstable/ob_data_store_desc.h @@ -23,7 +23,6 @@ #include "storage/compaction/ob_compaction_memory_context.h" #include "common/ob_tablet_id.h" #include "common/ob_store_format.h" -#include "share/ob_ls_id.h" #include "share/scn.h" namespace oceanbase @@ -57,7 +56,6 @@ struct ObStaticDataStoreDesc int init( const bool is_ddl, const share::schema::ObMergeSchema &merge_schema, - const share::ObLSID &ls_id, const common::ObTabletID tablet_id, const compaction::ObMergeType merge_type, const int64_t snapshot_version, @@ -72,7 +70,6 @@ struct ObStaticDataStoreDesc void reset(); int assign(const ObStaticDataStoreDesc &desc); TO_STRING_KV( - K_(ls_id), K_(tablet_id), K_(concurrent_cnt), "merge_type", merge_type_to_str(merge_type_), @@ -107,7 +104,6 @@ struct ObStaticDataStoreDesc bool is_ddl_; // only used to print ERROR or WARN log compaction::ObMergeType merge_type_; ObCompressorType compressor_type_; - share::ObLSID ls_id_; ObTabletID tablet_id_; int64_t concurrent_cnt_; int64_t macro_block_size_; @@ -262,7 +258,6 @@ struct ObDataStoreDesc STATIC_DESC_FUNC(int64_t, macro_store_size); STATIC_DESC_FUNC(int64_t, micro_block_size_limit); STATIC_DESC_FUNC(compaction::ObMergeType, merge_type); - STATIC_DESC_FUNC(const share::ObLSID&, ls_id); STATIC_DESC_FUNC(const ObTabletID&, tablet_id); STATIC_DESC_FUNC(int64_t, progressive_merge_round); STATIC_DESC_FUNC(int64_t, schema_version); @@ -355,7 +350,6 @@ struct ObWholeDataStoreDesc int init( const bool is_ddl, const share::schema::ObMergeSchema &merge_schema, - const share::ObLSID &ls_id, const common::ObTabletID tablet_id, const compaction::ObMergeType merge_type, const int64_t snapshot_version, diff --git a/src/storage/blocksstable/ob_macro_block_id.cpp b/src/storage/blocksstable/ob_macro_block_id.cpp index 47605280a..e698abf6d 100644 --- a/src/storage/blocksstable/ob_macro_block_id.cpp +++ b/src/storage/blocksstable/ob_macro_block_id.cpp @@ -231,8 +231,7 @@ bool MacroBlockId::is_private_data_or_meta() const return is_id_mode_share() && ( static_cast(ObStorageObjectType::PRIVATE_DATA_MACRO) == storage_object_type_ || - static_cast(ObStorageObjectType::PRIVATE_META_MACRO) == storage_object_type_ || - static_cast(ObStorageObjectType::PRIVATE_TABLET_META) == storage_object_type_ + static_cast(ObStorageObjectType::PRIVATE_META_MACRO) == storage_object_type_ ); } @@ -365,7 +364,7 @@ int MacroBlockId::memcpy_deserialize(const char* buf, const int64_t data_len, in } /* read through the following object types: - * SHARED_MAJOR_TABLET_META, COMPACTION_SERVER, SVR_COMPACTION_STATUS, COMPACTION_REPORT, + * SHARED_MAJOR_TABLET_META, COMPACTION_REPORT, * SHARED_MAJOR_GC_INFO, SHARED_MAJOR_META_LIST, MAJOR_PREWARM_DATA, MAJOR_PREWARM_DATA_INDEX, * MAJOR_PREWARM_META, MAJOR_PREWARM_META_INDEX */ @@ -398,8 +397,6 @@ bool is_object_type_only_store_remote(const ObStorageObjectType type) } /* files with the following object types are pin: - * LS_META, PRIVATE_TABLET_META, PRIVATE_TABLET_CURRENT_VERSION, - * LS_ACTIVE_TABLET_ARRAY, LS_PENDING_FREE_TABLET_ARRAY, LS_DUP_TABLE_META, * SERVER_META, TENANT_SUPER_BLOCK and TENANT_UNIT_META */ bool is_pin_storage_object_type(const ObStorageObjectType type) diff --git a/src/storage/blocksstable/ob_macro_block_id.h b/src/storage/blocksstable/ob_macro_block_id.h index 0520f491e..7fbff8905 100644 --- a/src/storage/blocksstable/ob_macro_block_id.h +++ b/src/storage/blocksstable/ob_macro_block_id.h @@ -177,80 +177,6 @@ namespace blocksstable (OB_DIR_MGR.get_local_tenant_dir(path, length, file_id.second_id(), file_id.third_id())), \ /*create_parent_dir*/ \ (OB_DIR_MGR.create_tenant_dir(file_id.second_id(), file_id.third_id()))) \ - STORAGE_OBJECT_TYPE_INFO(LS_META, "LS_META", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format: ls/ls_id_epoch_id/ls_meta */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s", \ - OB_DIR_MGR.get_local_cache_root_dir(), LS_DIR_STR, \ - file_id_.second_id(), ls_epoch_id_, get_storage_objet_type_str(object_type))), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id */ \ - (OB_DIR_MGR.get_ls_id_dir(path, length, file_id.second_id(), ls_epoch_id)), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_ls_id_dir(file_id.second_id(), ls_epoch_id))) \ - STORAGE_OBJECT_TYPE_INFO(LS_DUP_TABLE_META, "LS_DUP_TABLE_META", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format: ls/ls_id_epoch_id/ls_dup_table_meta */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s", \ - OB_DIR_MGR.get_local_cache_root_dir(), LS_DIR_STR, \ - file_id_.second_id(), ls_epoch_id_, get_storage_objet_type_str(object_type))), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id */ \ - (OB_DIR_MGR.get_ls_id_dir(path, length, file_id.second_id(), ls_epoch_id)), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_ls_id_dir(file_id.second_id(), ls_epoch_id))) \ - STORAGE_OBJECT_TYPE_INFO(LS_ACTIVE_TABLET_ARRAY, "LS_ACTIVE_TABLET_ARRAY", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format: ls/ls_id_epoch_id/tablet_id_array */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s", \ - OB_DIR_MGR.get_local_cache_root_dir(), LS_DIR_STR, \ - file_id_.second_id(), ls_epoch_id_, get_storage_objet_type_str(object_type))), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id */ \ - (OB_DIR_MGR.get_ls_id_dir(path, length, file_id.second_id(), ls_epoch_id)), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_ls_id_dir(file_id.second_id(), ls_epoch_id))) \ - STORAGE_OBJECT_TYPE_INFO(LS_PENDING_FREE_TABLET_ARRAY, "LS_PENDING_FREE_TABLET_ARRAY", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format: ls/ls_id_epoch_id/pending_free_tablet_array */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s", \ - OB_DIR_MGR.get_local_cache_root_dir(), LS_DIR_STR, \ - file_id_.second_id(), ls_epoch_id_, get_storage_objet_type_str(object_type))), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id */ \ - (OB_DIR_MGR.get_ls_id_dir(path, length, file_id.second_id(), ls_epoch_id)), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_ls_id_dir(file_id.second_id(), ls_epoch_id))) \ - STORAGE_OBJECT_TYPE_INFO(PRIVATE_TABLET_META, "PRIVATE_TABLET_META", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:tablet_id, fourth_id:meta_path_id+meta_version_id */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX) && (file_id_.third_id() > 0) && (file_id_.meta_path_id() >= 0) && (file_id_.meta_version_id() >= 0)), \ - /*to_local_path_format: ls/ls_id_epoch_id/tablet_meta/tablet_id/tablet_meta_version_path_id */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s/%ld/%ld_%ld", \ - OB_DIR_MGR.get_local_cache_root_dir(), \ - LS_DIR_STR, file_id_.second_id(), ls_epoch_id_, TABLET_META_DIR_STR, \ - file_id_.third_id(), file_id_.meta_version_id(), file_id_.meta_path_id())), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id/tablet_meta/tablet_id/ */ \ - (OB_DIR_MGR.get_tablet_meta_tablet_id_dir(path, length, file_id.second_id(), ls_epoch_id, file_id.third_id())), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_tablet_meta_tablet_id_dir(file_id.second_id(), ls_epoch_id, file_id.third_id()))) \ - STORAGE_OBJECT_TYPE_INFO(PRIVATE_TABLET_CURRENT_VERSION, "PRIVATE_TABLET_CURRENT_VERSION", true/*is_pin_local*/, false/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:tablet_id, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX) && (file_id_.third_id() > 0)), \ - /*to_local_path_format: ls/ls_id_epoch_id/tablet_meta/tablet_id/current_version */ \ - (databuff_printf(path_, length, pos, "%s/%s/%ld_%ld/%s/%ld/%s", \ - OB_DIR_MGR.get_local_cache_root_dir(), LS_DIR_STR, \ - file_id_.second_id(), ls_epoch_id_, TABLET_META_DIR_STR, file_id_.third_id(), \ - get_storage_objet_type_str(object_type))), \ - /*to_remote_path_format*/OB_NOT_SUPPORTED, \ - /*get_parent_dir: ls/ls_id_epoch_id/tablet_meta/tablet_id/ */ \ - (OB_DIR_MGR.get_tablet_meta_tablet_id_dir(path, length, file_id.second_id(), ls_epoch_id, file_id.third_id())), \ - /*create_parent_dir*/ \ - (OB_DIR_MGR.create_tablet_meta_tablet_id_dir(file_id.second_id(), ls_epoch_id, file_id.third_id()))) \ STORAGE_OBJECT_TYPE_INFO(SHARED_MAJOR_TABLET_META, "SHARED_MAJOR_TABLET_META", false/*is_pin_local*/, true/*is_read_through*/, \ /*is_valid second_id:tablet_id, third_id:meta_version_id, fourth_id:N/A */ \ ((file_id_.second_id() > 0) && (file_id_.second_id() < INT64_MAX) && (file_id_.third_id() >= 0)), \ @@ -262,28 +188,6 @@ namespace blocksstable SHARED_TABLET_META_DIR_STR, file_id_.third_id())), \ /*get_parent_dir*/OB_NOT_SUPPORTED, \ /*create_parent_dir*/OB_NOT_SUPPORTED) \ - STORAGE_OBJECT_TYPE_INFO(COMPACTION_SERVER, "COMPACTION_SERVER", false/*is_pin_local*/, true/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format*/OB_NOT_SUPPORTED, \ - /*to_remote_path_format: cluster_id/compaction/scheduler/ls_id_compaction_servers */ \ - (databuff_printf(path_, length, pos, "%s/%s_%ld/%s/%s/%ld_%s", \ - object_storage_root_dir, CLUSTER_DIR_STR, cluster_id, \ - COMPACTION_DIR_STR, SCHEDULER_DIR_STR, file_id_.second_id(), \ - get_storage_objet_type_str(object_type))), \ - /*get_parent_dir*/OB_NOT_SUPPORTED, \ - /*create_parent_dir*/OB_NOT_SUPPORTED) \ - STORAGE_OBJECT_TYPE_INFO(LS_SVR_COMPACTION_STATUS, "LS_SVR_COMPACTION_STATUS", false/*is_pin_local*/, true/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:server_id, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX) && (file_id_.third_id() > 0)), \ - /*to_local_path_format*/OB_NOT_SUPPORTED, \ - /*to_remote_path_format: cluster_id/compaction/compactor/ls_id_server_id_ls_svr_compaction_status */ \ - (databuff_printf(path_, length, pos, "%s/%s_%ld/%s/%s/%ld_%ld_%s", \ - object_storage_root_dir, CLUSTER_DIR_STR, cluster_id, \ - COMPACTION_DIR_STR, COMPACTOR_DIR_STR, file_id_.second_id(), file_id_.third_id(), \ - get_storage_objet_type_str(object_type))), \ - /*get_parent_dir*/OB_NOT_SUPPORTED, \ - /*create_parent_dir*/OB_NOT_SUPPORTED) \ STORAGE_OBJECT_TYPE_INFO(COMPACTION_REPORT, "COMPACTION_REPORT", false/*is_pin_local*/, true/*is_read_through*/, \ /*is_valid second_id:server_id, third_id:N/A, fourth_id:N/A */ \ ((file_id_.second_id() > 0) && (file_id_.second_id() < INT64_MAX)), \ @@ -317,17 +221,6 @@ namespace blocksstable SHARED_TABLET_META_DIR_STR, get_storage_objet_type_str(object_type))), \ /*get_parent_dir*/OB_NOT_SUPPORTED, \ /*create_parent_dir*/OB_NOT_SUPPORTED) \ - STORAGE_OBJECT_TYPE_INFO(LS_COMPACTION_STATUS, "LS_COMPACTION_STATUS", false/*is_pin_local*/, true/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format*/OB_NOT_SUPPORTED, \ - /*to_remote_path_format: cluster_id/compaction/scheduler/ls_id_ls_compaction_status */ \ - (databuff_printf(path_, length, pos, "%s/%s_%ld/%s/%s/%ld_%s", \ - object_storage_root_dir, CLUSTER_DIR_STR, cluster_id, \ - COMPACTION_DIR_STR, SCHEDULER_DIR_STR, file_id_.second_id(), \ - get_storage_objet_type_str(object_type))), \ - /*get_parent_dir*/OB_NOT_SUPPORTED, \ - /*create_parent_dir*/OB_NOT_SUPPORTED) \ STORAGE_OBJECT_TYPE_INFO(TABLET_COMPACTION_STATUS, "TABLET_COMPACTION_STATUS", false/*is_pin_local*/, true/*is_read_through*/, \ /*is_valid second_id:tablet_id, third_id:compaction_scn, fourth_id:N/A */ \ ((file_id_.second_id() > 0) && (file_id_.second_id() < INT64_MAX) && (file_id_.third_id() >= 0)), \ @@ -405,17 +298,6 @@ namespace blocksstable TABLET_IDS_DIR_STR, file_id_.second_id())), \ /*get_parent_dir*/OB_NOT_SUPPORTED, \ /*create_parent_dir*/OB_NOT_SUPPORTED) \ - STORAGE_OBJECT_TYPE_INFO(LS_COMPACTION_LIST, "LS_COMPACTION_LIST", false/*is_pin_local*/, true/*is_read_through*/, \ - /*is_valid second_id:ls_id, third_id:N/A, fourth_id:N/A */ \ - ((file_id_.second_id() >= 0) && (file_id_.second_id() < INT64_MAX)), \ - /*to_local_path_format*/OB_NOT_SUPPORTED, \ - /*to_remote_path_format: cluster_id/compaction/scheduler/ls_id_ls_compaction_list */ \ - (databuff_printf(path_, length, pos, "%s/%s_%ld/%s/%s/%ld_%s", \ - object_storage_root_dir, CLUSTER_DIR_STR, cluster_id, \ - COMPACTION_DIR_STR, SCHEDULER_DIR_STR, file_id_.second_id(), \ - get_storage_objet_type_str(object_type))), \ - /*get_parent_dir*/OB_NOT_SUPPORTED, \ - /*create_parent_dir*/OB_NOT_SUPPORTED) \ STORAGE_OBJECT_TYPE_INFO(IS_SHARED_TABLET_DELETED, "IS_SHARED_TABLET_DELETED", false/*is_pin_local*/, true/*is_read_through*/, \ /*is_valid second_id:tablet_id, third_id:N/A, fourth_id:N/A */ \ ((file_id_.second_id() > 0) && (file_id_.second_id() < INT64_MAX)), \ @@ -671,7 +553,6 @@ class MacroBlockId final int64_t macro_path_id_ : SF_BIT_PATH_ID; int64_t tenant_seq_ : SF_BIT_TENANT_SEQ; }; - // for PRIVATE_TABLET_META and PRIVATE_TABLET_CURRENT_VERSION struct { int64_t meta_path_id_ : SF_BIT_PATH_ID; uint64_t meta_version_id_ : SF_BIT_META_VERSION_ID; diff --git a/src/storage/blocksstable/ob_micro_block_row_getter.cpp b/src/storage/blocksstable/ob_micro_block_row_getter.cpp index badab535c..f1e5c691e 100644 --- a/src/storage/blocksstable/ob_micro_block_row_getter.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_getter.cpp @@ -271,7 +271,6 @@ int ObMicroBlockRowGetter::get_block_row(ObSSTableReadHandle &read_handle, if (!context_->query_flag_.is_index_back() && context_->query_flag_.is_use_bloomfilter_cache() && !sstable_->is_small_sstable()) { (void)OB_STORE_CACHE.get_bf_cache().inc_empty_read(param_->table_id_, - param_->ls_id_, sstable_->get_key(), read_handle.micro_handle_->macro_block_id_, read_handle.get_rowkey().get_datum_cnt(), diff --git a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp index 8a8bd298e..69e166016 100644 --- a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp @@ -43,7 +43,6 @@ void ObMicroBlockRowLockChecker::inc_empty_read(ObSSTableReadHandle &read_handle if (OB_NOT_NULL(context_) && OB_NOT_NULL(sstable_) && !context_->query_flag_.is_index_back() && context_->query_flag_.is_use_bloomfilter_cache() && !sstable_->is_small_sstable()) { (void)OB_STORE_CACHE.get_bf_cache().inc_empty_read(param_->table_id_, - param_->ls_id_, sstable_->get_key(), macro_id_, read_handle.get_rowkey_datum_cnt(), @@ -292,7 +291,6 @@ void ObMicroBlockRowLockMultiChecker::inc_empty_read(ObSSTableReadHandle &read_h && context_->query_flag_.is_use_bloomfilter_cache() && !sstable_->is_small_sstable() && empty_read_cnt_ > 0) { read_handle.current_rows_info_idx_ = rowkey_begin_idx_; (void)OB_STORE_CACHE.get_bf_cache().inc_empty_read(param_->table_id_, - param_->ls_id_, sstable_->get_key(), macro_id_, rows_info_->get_datum_cnt(), diff --git a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp index 57b640a09..d2696814e 100644 --- a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp @@ -1951,9 +1951,9 @@ int ObMultiVersionMicroBlockRowScanner::check_foreign_key( ObStoreRowLockState lock_state; const int64_t snapshot_version = context_->trans_version_range_.snapshot_version_; memtable::ObMvccAccessCtx &acc_ctx = context_->store_ctx_->mvcc_acc_ctx_; - bool is_plain_insert_gts_opt = context_->query_flag_.is_plain_insert_gts_opt(); + bool is_snapshot_opt = context_->query_flag_.is_snapshot_opt(); bool is_for_fk_check = context_->query_flag_.is_for_foreign_key_check(); - if ((is_plain_insert_gts_opt || is_for_fk_check) && + if ((is_snapshot_opt || is_for_fk_check) && OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_sstable( acc_ctx.get_tx_table_guards(), acc_ctx.get_tx_id(), @@ -1975,7 +1975,6 @@ int ObMultiVersionMicroBlockRowScanner::check_foreign_key( store_rowkey, lock_state, context_->tablet_id_, - context_->ls_id_, 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, sstable_->get_end_scn()); } diff --git a/src/storage/blocksstable/ob_object_manager.cpp b/src/storage/blocksstable/ob_object_manager.cpp index 4ca6c6445..feb3ed5ec 100644 --- a/src/storage/blocksstable/ob_object_manager.cpp +++ b/src/storage/blocksstable/ob_object_manager.cpp @@ -73,38 +73,6 @@ int64_t ObStorageObjectOpt::to_string(char *buf, const int64_t buf_len) const } break; } - case ObStorageObjectType::LS_META: - case ObStorageObjectType::LS_DUP_TABLE_META: - case ObStorageObjectType::LS_ACTIVE_TABLET_ARRAY: - case ObStorageObjectType::LS_PENDING_FREE_TABLET_ARRAY: { - if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu)", - get_storage_objet_type_str(object_type_), ss_ls_level_opt_.ls_id_))) { - LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)), - K(ss_ls_level_opt_.ls_id_)); - } - break; - } - case ObStorageObjectType::PRIVATE_TABLET_META: { - if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu,tablet_id=%lu,version=%lu)", - get_storage_objet_type_str(object_type_), - ss_private_tablet_opt_.ls_id_, ss_private_tablet_opt_.tablet_id_, - ss_private_tablet_opt_.version_))) { - LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)), - K(ss_private_tablet_opt_.ls_id_), K(ss_private_tablet_opt_.tablet_id_), - K(ss_private_tablet_opt_.version_)); - } - break; - } - case ObStorageObjectType::PRIVATE_TABLET_CURRENT_VERSION: { - if(OB_FAIL(databuff_printf(buf, buf_len, pos, - "object_type=%s (ls_id=%lu,tablet_id=%lu)", get_storage_objet_type_str(object_type_), - ss_private_tablet_current_version_opt_.ls_id_, ss_private_tablet_current_version_opt_.tablet_id_))) { - LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)), - K(ss_private_tablet_current_version_opt_.ls_id_), - K(ss_private_tablet_current_version_opt_.tablet_id_)); - } - break; - } case ObStorageObjectType::SHARED_MAJOR_TABLET_META: { if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (tablet_id=%lu,version=%lu)", get_storage_objet_type_str(object_type_), @@ -132,26 +100,6 @@ int64_t ObStorageObjectOpt::to_string(char *buf, const int64_t buf_len) const } break; } - case ObStorageObjectType::COMPACTION_SERVER: - case ObStorageObjectType::LS_COMPACTION_STATUS: - case ObStorageObjectType::LS_COMPACTION_LIST: { - if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu)", - get_storage_objet_type_str(object_type_), ss_compaction_scheduler_opt_.ls_id_))) { - LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)), - K(ss_compaction_scheduler_opt_.ls_id_)); - } - break; - } - case ObStorageObjectType::LS_SVR_COMPACTION_STATUS: { - if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu, server_id=%lu)", - get_storage_objet_type_str(object_type_), ss_ls_svr_compactor_opt_.ls_id_, - ss_ls_svr_compactor_opt_.server_id_))) { - LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)), - K(ss_ls_svr_compactor_opt_.ls_id_), - K(ss_ls_svr_compactor_opt_.server_id_)); - } - break; - } case ObStorageObjectType::COMPACTION_REPORT: { if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (server_id=%lu)", get_storage_objet_type_str(object_type_), ss_svr_compactor_opt_.server_id_))) { diff --git a/src/storage/blocksstable/ob_object_manager.h b/src/storage/blocksstable/ob_object_manager.h index 5903ce81f..89c81182a 100644 --- a/src/storage/blocksstable/ob_object_manager.h +++ b/src/storage/blocksstable/ob_object_manager.h @@ -77,29 +77,6 @@ class ObStorageObjectOpt final ss_tenant_level_opt_.tenant_epoch_id_ = tenant_epoch_id; }; - void set_ss_ls_level_meta_object_opt( - const ObStorageObjectType object_type, const int64_t ls_id) - { - object_type_ = object_type; - ss_ls_level_opt_.ls_id_ = ls_id; - }; - - void set_ss_private_tablet_meta_object_opt( - const int64_t ls_id, const uint64_t tablet_id, const uint64_t version) - { - object_type_ = ObStorageObjectType::PRIVATE_TABLET_META; - ss_private_tablet_opt_.ls_id_ = ls_id; - ss_private_tablet_opt_.tablet_id_ = tablet_id; - ss_private_tablet_opt_.version_ = version; - } - void set_ss_private_tablet_meta_current_verison_object_opt( - const int64_t ls_id, const uint64_t tablet_id) - { - object_type_ = ObStorageObjectType::PRIVATE_TABLET_CURRENT_VERSION; - ss_private_tablet_current_version_opt_.ls_id_ = ls_id; - ss_private_tablet_current_version_opt_.tablet_id_ = tablet_id; - } - void set_ss_share_tablet_meta_object_opt( const uint64_t tablet_id, const int64_t version) @@ -127,10 +104,9 @@ class ObStorageObjectOpt final } void set_ss_compaction_scheduler_object_opt( - const ObStorageObjectType object_type, const int64_t ls_id) + const ObStorageObjectType object_type) { object_type_ = object_type; - ss_compaction_scheduler_opt_.ls_id_ = ls_id; }; void set_ss_compactor_svr_object_opt( const ObStorageObjectType object_type, const int64_t server_id) @@ -138,13 +114,6 @@ class ObStorageObjectOpt final object_type_ = object_type; ss_svr_compactor_opt_.server_id_ = server_id; }; - void set_ss_compactor_ls_svr_object_opt( - const ObStorageObjectType object_type, const int64_t ls_id, const int64_t server_id) - { - object_type_ = object_type; - ss_ls_svr_compactor_opt_.ls_id_ = ls_id; - ss_ls_svr_compactor_opt_.server_id_ = server_id; - }; void set_ss_gc_info_object_opt( const uint64_t tablet_id) { @@ -209,25 +178,6 @@ class ObStorageObjectOpt final int64_t tenant_epoch_id_; }; - // ls level meta include: - // ls_meta/dup_table_meta/active_tablet_array/pending_free_tablet_array - struct SSLSLevelMetaObjectOpt - { - uint64_t ls_id_; - }; - - struct SSPrivateTabletMetaObjectOpt - { - uint64_t ls_id_; - uint64_t tablet_id_; - int64_t version_; - }; - struct SSPrivateTabletCurrentVersionObjectOpt - { - uint64_t ls_id_; - uint64_t tablet_id_; - }; - struct SSShareTabletMetaObjectOpt { uint64_t tablet_id_; @@ -236,17 +186,11 @@ class ObStorageObjectOpt final struct SSCompactionSchedulerObjectOpt { - uint64_t ls_id_; }; struct SSCompactorSvrObjectOpt { int64_t server_id_; }; - struct SSCompactorLSSvrObjectOpt - { - uint64_t ls_id_; - int64_t server_id_; - }; struct SSGCInfoObjectOpt { uint64_t tablet_id_; @@ -279,13 +223,9 @@ class ObStorageObjectOpt final SSTmpFileObjectOpt ss_tmp_file_opt_; SSServerLevelMetaObjectOpt ss_server_level_opt_; SSTenantLevelMetaObjectOpt ss_tenant_level_opt_; - SSLSLevelMetaObjectOpt ss_ls_level_opt_; - SSPrivateTabletMetaObjectOpt ss_private_tablet_opt_; - SSPrivateTabletCurrentVersionObjectOpt ss_private_tablet_current_version_opt_; SSShareTabletMetaObjectOpt ss_share_tablet_opt_; SSCompactionSchedulerObjectOpt ss_compaction_scheduler_opt_; SSCompactorSvrObjectOpt ss_svr_compactor_opt_; - SSCompactorLSSvrObjectOpt ss_ls_svr_compactor_opt_; SSGCInfoObjectOpt ss_gc_info_opt_; SSGCMetaListObjectOpt ss_meta_list_opt_; SSTabletCompactionStatusObjectOpt ss_tablet_compaction_status_opt_; diff --git a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp index fc26bc250..bed0cbc1a 100644 --- a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp +++ b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp @@ -529,10 +529,9 @@ int ObSharedMacroBlockMgr::update_tablet( ObTableStoreIterator table_store_iter; uint64_t data_version = 0; const ObTabletMeta &tablet_meta = tablet_handle.get_obj()->get_tablet_meta(); - const share::ObLSID &ls_id = tablet_meta.ls_id_; ObTabletHandle updated_tablet_handle; ObMetaDiskAddr cur_addr; - const ObTabletMapKey key(ls_id, tablet_meta.tablet_id_); + const ObTabletMapKey key(tablet_meta.tablet_id_); if (OB_FAIL(tablet_handle.get_obj()->get_all_sstables(table_store_iter))) { LOG_WARN("fail to get sstables of this tablet", K(ret)); @@ -620,21 +619,16 @@ int ObSharedMacroBlockMgr::update_tablet( if (OB_SUCC(ret) && !new_sstables.empty()) { ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; - - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls handle", K(ret), K(ls_id), KPC(tablet_handle.get_obj())); + ObLS *tenant_ls = nullptr; + + if (OB_FAIL(ls_svr->get_ls(tenant_ls))) { + LOG_WARN("fail to get ls", K(ret), KPC(tablet_handle.get_obj())); + } else if (OB_FAIL(tenant_ls->update_tablet_table_store( + updated_tablet_handle, new_sstables))) { + LOG_WARN("fail to replace small sstables in the tablet", + K(ret), K(updated_tablet_handle), K(new_sstables)); } else { - const int64_t rebuild_seq = ls_handle.get_ls()->get_rebuild_seq(); - if (OB_UNLIKELY(!ls_handle.is_valid())) { - LOG_WARN("la handle is invalid", K(ret), K(ls_handle)); - } else if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store( - rebuild_seq, updated_tablet_handle, new_sstables))) { - LOG_WARN("fail to replace small sstables in the tablet", - K(ret), K(rebuild_seq), K(updated_tablet_handle), K(new_sstables)); - } else { - rewrite_cnt += new_sstables.count(); - } + rewrite_cnt += new_sstables.count(); } } @@ -771,7 +765,6 @@ int ObSharedMacroBlockMgr::prepare_data_desc( } else if (OB_FAIL(data_desc.init( false/*is_ddl*/, *storage_schema, - tablet.get_tablet_meta().ls_id_, tablet.get_tablet_meta().tablet_id_, merge_type, snapshot_version, @@ -792,7 +785,6 @@ int ObSharedMacroBlockMgr::prepare_data_desc( if (FAILEDx(data_desc.init( false/*is_ddl*/, *storage_schema, - tablet.get_tablet_meta().ls_id_, tablet.get_tablet_meta().tablet_id_, merge_type, snapshot_version, diff --git a/src/storage/blocksstable/ob_shared_macro_block_manager.h b/src/storage/blocksstable/ob_shared_macro_block_manager.h index 3af41730e..cc4fb8d81 100644 --- a/src/storage/blocksstable/ob_shared_macro_block_manager.h +++ b/src/storage/blocksstable/ob_shared_macro_block_manager.h @@ -26,10 +26,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -} namespace blocksstable { struct ObMacroBlocksWriteCtx; diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index f166028d6..e16187b8c 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -935,14 +935,11 @@ bool ObSSTableMeta::is_shared_table() const || basic_meta_.table_backup_flag_.is_shared_sstable(); } -//================================== ObMigrationSSTableParam ================================== -ObMigrationSSTableParam::ObMigrationSSTableParam() +//================================== ObForkSSTableParam ================================== +ObForkSSTableParam::ObForkSSTableParam() : allocator_("SSTableParam", OB_MALLOC_NORMAL_BLOCK_SIZE), basic_meta_(), column_checksums_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator_)), - table_key_(), - column_default_checksums_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator_)), - is_small_sstable_(false), root_block_addr_(), root_block_buf_(nullptr), data_block_macro_meta_addr_(), @@ -951,18 +948,15 @@ ObMigrationSSTableParam::ObMigrationSSTableParam() { } -ObMigrationSSTableParam::~ObMigrationSSTableParam() +ObForkSSTableParam::~ObForkSSTableParam() { reset(); } -void ObMigrationSSTableParam::reset() +void ObForkSSTableParam::reset() { - table_key_.reset(); column_checksums_.reset(); - column_default_checksums_.reset(); basic_meta_.reset(); - is_small_sstable_ = false; root_block_addr_.reset(); root_block_buf_ = nullptr; data_block_macro_meta_addr_.reset(); @@ -971,293 +965,16 @@ void ObMigrationSSTableParam::reset() allocator_.reset(); } -bool ObMigrationSSTableParam::is_valid() const +bool ObForkSSTableParam::is_valid() const { - return basic_meta_.is_valid() && table_key_.is_valid() + return basic_meta_.is_valid() && (root_block_addr_.is_memory() ? (root_block_buf_ != nullptr) : (root_block_buf_ == nullptr)) && (data_block_macro_meta_addr_.is_memory() ? (data_block_macro_meta_buf_ != nullptr) : (data_block_macro_meta_buf_ == nullptr)); } -int ObMigrationSSTableParam::assign(const ObMigrationSSTableParam ¶m) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("assign migrate sstable param get invalid argument", K(ret), K(param)); - } else { - basic_meta_ = param.basic_meta_; - table_key_ = param.table_key_; - is_small_sstable_ = param.is_small_sstable_; - root_block_addr_ = param.root_block_addr_; - data_block_macro_meta_addr_ = param.data_block_macro_meta_addr_; - is_meta_root_ = param.is_meta_root_; - if (OB_FAIL(column_checksums_.assign(param.column_checksums_))) { - LOG_WARN("fail to assign column checksums", K(ret), K(param)); - } else if (OB_FAIL(column_default_checksums_.assign(param.column_default_checksums_))) { - LOG_WARN("fail to assign default column checksum", K(ret), K(param)); - } else if (root_block_addr_.is_memory() && (OB_ISNULL(root_block_buf_ = - static_cast(allocator_.alloc(root_block_addr_.size()))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc root block buf", K(ret)); - } else if (data_block_macro_meta_addr_.is_memory() && (OB_ISNULL(data_block_macro_meta_buf_ = - static_cast(allocator_.alloc(data_block_macro_meta_addr_.size()))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc data block macro meta buf", K(ret)); - } else { - if (root_block_addr_.is_memory()) { - MEMCPY(root_block_buf_, param.root_block_buf_, root_block_addr_.size()); - } - if (data_block_macro_meta_addr_.is_memory()) { - MEMCPY(data_block_macro_meta_buf_, param.data_block_macro_meta_buf_, data_block_macro_meta_addr_.size()); - } - } - if (OB_FAIL(ret)) { - reset(); - } - } - return ret; -} - -bool ObMigrationSSTableParam::is_empty_sstable() const -{ - return 0 == basic_meta_.data_macro_block_count_ && !is_only_shared_macro_blocks_sstable(); -} - -bool ObMigrationSSTableParam::is_shared_sstable() const -{ - return basic_meta_.table_shared_flag_.is_shared_sstable() - || basic_meta_.table_backup_flag_.is_shared_sstable(); -} - -//shared sstable contain shared macro blocks -bool ObMigrationSSTableParam::is_shared_macro_blocks_sstable() const -{ - return basic_meta_.table_shared_flag_.is_shared_macro_blocks() - || basic_meta_.table_backup_flag_.is_shared_sstable(); -} - -bool ObMigrationSSTableParam::is_only_shared_macro_blocks_sstable() const -{ - return basic_meta_.table_shared_flag_.is_only_shared_macro_blocks(); -} -// this is used for backup to get merge res from backup-ed migration sstable param -int ObMigrationSSTableParam::get_merge_res(blocksstable::ObSSTableMergeRes &res) const -{ - int ret = OB_SUCCESS; - res.root_desc_.addr_ = root_block_addr_; - res.root_desc_.buf_ = root_block_buf_; - res.root_desc_.height_ = basic_meta_.data_index_tree_height_; - res.data_root_desc_.addr_ = data_block_macro_meta_addr_; - res.data_root_desc_.buf_ = data_block_macro_meta_buf_; - res.data_root_desc_.is_meta_root_ = is_meta_root_; - res.index_blocks_cnt_ = basic_meta_.index_macro_block_count_; - res.data_blocks_cnt_ = basic_meta_.data_macro_block_count_; - res.micro_block_cnt_ = basic_meta_.data_micro_block_count_; - res.data_column_cnt_ = basic_meta_.column_cnt_; - res.row_count_ = basic_meta_.row_count_; - res.max_merged_trans_version_ = basic_meta_.max_merged_trans_version_; - res.contain_uncommitted_row_ = basic_meta_.contain_uncommitted_row_; - res.occupy_size_ = basic_meta_.occupy_size_; - res.original_size_ = basic_meta_.original_size_; - res.data_checksum_ = basic_meta_.data_checksum_; - res.use_old_macro_block_count_ = basic_meta_.use_old_macro_block_count_; - res.compressor_type_ = basic_meta_.compressor_type_; - res.encrypt_id_ = basic_meta_.encrypt_id_; - res.master_key_id_ = basic_meta_.master_key_id_; - res.root_row_store_type_ = basic_meta_.root_row_store_type_; - STATIC_ASSERT(ARRAYSIZEOF(basic_meta_.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, - "ObMigrationSSTableParam encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); - STATIC_ASSERT(ARRAYSIZEOF(res.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, - "ObSSTableMergeRes encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); - MEMCPY(res.encrypt_key_, basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - return ret; -} - -DEFINE_SERIALIZE(ObMigrationSSTableParam) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0 || pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("migration sstable param is invalid", K(ret), K(*this)); - } else { - int64_t tmp_pos = 0; - const int64_t len = get_serialize_size_(); - OB_UNIS_ENCODE(UNIS_VERSION); - OB_UNIS_ENCODE(len); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(serialize_(buf + pos, buf_len, tmp_pos))) { - LOG_WARN("fail to serialize_", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_UNLIKELY(len != tmp_pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, serialize may have bug", K(ret), K(len), K(tmp_pos), KPC(this)); - } else { - pos += tmp_pos; - } - } - return ret; -} -int ObMigrationSSTableParam::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(basic_meta_.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize basic meta", K(ret), KP(buf), K(buf_len), K(pos), K_(basic_meta)); - } else if (OB_FAIL(column_checksums_.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize column checksums", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(table_key_.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize table key", K(ret), KP(buf), K(buf_len), K(pos), K(table_key_)); - } else if (OB_FAIL(column_default_checksums_.serialize(buf, buf_len, pos))) { - LOG_WARN("fail to serialize default column checksum", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(serialization::encode_bool(buf, buf_len, pos, is_small_sstable_))) { - LOG_WARN("fail to serialize is_small_sstable_", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(addr_serialize(root_block_addr_, root_block_buf_, buf, buf_len, pos))) { - STORAGE_LOG(WARN, "fail to serialize address", K(ret), KP(buf), K(buf_len), K(pos), K(root_block_addr_)); - } else if (OB_FAIL(addr_serialize(data_block_macro_meta_addr_, data_block_macro_meta_buf_, buf, buf_len, pos))) { - STORAGE_LOG(WARN, "fail to serialize address", K(ret), KP(buf), K(buf_len), K(pos), K(data_block_macro_meta_addr_)); - } - LST_DO_CODE(OB_UNIS_ENCODE, is_meta_root_); - return ret; -} -DEFINE_DESERIALIZE(ObMigrationSSTableParam) -{ - int ret = OB_SUCCESS; - int64_t tmp_pos = 0; - int64_t len = 0; - int64_t version = 0; - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0 || pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(buf), K(data_len), K(pos)); - } else { - OB_UNIS_DECODE(version); - OB_UNIS_DECODE(len); - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(version != UNIS_VERSION)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("object version mismatch", K(ret), K(version)); - } else if (OB_UNLIKELY(data_len - pos < len)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("payload is out of the buf's boundary", K(ret), K(data_len), K(pos), K(len)); - } else if (OB_FAIL(deserialize_(buf + pos, len, tmp_pos))) { - LOG_WARN("fail to deserialize_", K(ret), KP(buf), K(data_len), K(pos)); - } else if (OB_UNLIKELY(len != tmp_pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, serialize may have bug", K(ret), K(len), K(tmp_pos), KPC(this)); - } else { - pos += tmp_pos; - } - } - return ret; -} - -int ObMigrationSSTableParam::deserialize_(const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - if (pos < data_len && OB_FAIL(basic_meta_.deserialize(buf, data_len, pos))) { - LOG_WARN("fail to deserialize basic meta", K(ret), KP(buf), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(column_checksums_.deserialize(buf, data_len, pos))) { - LOG_ERROR("fail to deserialize column checksums", K(ret), KP(buf), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(table_key_.deserialize(buf, data_len, pos))) { - LOG_WARN("fail to deserialize table key", K(ret), KP(buf), K(data_len), K(pos), K(table_key_)); - } else if (pos < data_len && OB_FAIL(column_default_checksums_.deserialize(buf, data_len, pos))) { - LOG_ERROR("fail to deserialize default column checksums", K(ret), KP(buf), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(serialization::decode_bool(buf, data_len, pos, &is_small_sstable_))) { - LOG_WARN("fail to deserialize is_small_sstable_", K(ret), KP(buf), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(addr_deserialize(buf, data_len, pos, root_block_addr_, root_block_buf_))) { - STORAGE_LOG(WARN, "fail to deserialize address and buf", K(ret), KP(buf), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(addr_deserialize(buf, data_len, pos, - data_block_macro_meta_addr_, data_block_macro_meta_buf_))) { - STORAGE_LOG(WARN, "fail to deserialize address and buf", K(ret), KP(buf), K(data_len), K(pos)); - } - - LST_DO_CODE(OB_UNIS_DECODE, is_meta_root_); - return ret; -} - - -DEFINE_GET_SERIALIZE_SIZE(ObMigrationSSTableParam) -{ - int64_t len = 0; - const int64_t payload_size = get_serialize_size_(); - OB_UNIS_ADD_LEN(UNIS_VERSION); - OB_UNIS_ADD_LEN(payload_size); - len += get_serialize_size_(); - return len; -} - -int64_t ObMigrationSSTableParam::get_serialize_size_() const -{ - int64_t len = 0; - len += basic_meta_.get_serialize_size(); - len += column_checksums_.get_serialize_size(); - len += table_key_.get_serialize_size(); - len += column_default_checksums_.get_serialize_size(); - len += serialization::encoded_length_bool(is_small_sstable_); - len += addr_get_serialize_size(root_block_addr_); - len += addr_get_serialize_size(data_block_macro_meta_addr_); - - LST_DO_CODE(OB_UNIS_ADD_LEN, is_meta_root_); - return len; -} - -int ObMigrationSSTableParam::addr_serialize(const ObMetaDiskAddr &addr, const char *addr_buf, - char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - if (OB_FAIL(addr.serialize(buf, buf_len, pos))) { - STORAGE_LOG(WARN, "fail to serialize address", K(ret), KP(buf), K(buf_len), K(pos), K(addr)); - } else if (addr.is_memory()) { - MEMCPY(buf + pos, addr_buf, addr.size()); - pos += addr.size(); - } - return ret; -} - -int ObMigrationSSTableParam::addr_deserialize(const char *buf, const int64_t data_len, int64_t &pos, - ObMetaDiskAddr &addr, char *&root_buf) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(addr.deserialize(buf, data_len, pos))) { - STORAGE_LOG(WARN, "fail to deserialize address", K(ret), KP(buf), K(data_len), K(pos)); - } else if (addr.is_memory()) { - char *data_buf = nullptr; - if (pos + addr.size() > data_len) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected data len is not enough", K(ret), K(pos), K(data_len), K(addr)); - } else if (OB_ISNULL(data_buf = static_cast(allocator_.alloc(addr.size())))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc data buffer", K(ret), K(addr)); - } else { - MEMCPY(data_buf, buf + pos, addr.size()); - root_buf = data_buf; - pos += addr.size(); - } - if (OB_FAIL(ret)) { - if (OB_NOT_NULL(data_buf)) { - allocator_.free(data_buf); - data_buf = nullptr; - } - } - } else { - root_buf = nullptr; - } - return ret; -} - -int64_t ObMigrationSSTableParam::addr_get_serialize_size(const ObMetaDiskAddr &addr) const -{ - int64_t len = 0; - len += addr.get_serialize_size(); - if (addr.is_memory()) { - len += addr.size(); - } - return len; -} int ObSSTableMetaChecker::check_sstable_meta_strict_equality( const ObSSTableMeta &old_sstable_meta, const ObSSTableMeta &new_sstable_meta) @@ -1302,19 +1019,6 @@ int ObSSTableMetaChecker::check_sstable_meta( return ret; } -int ObSSTableMetaChecker::check_sstable_meta( - const ObMigrationSSTableParam &migration_param, - const ObSSTableMeta &new_sstable_meta) -{ - int ret = OB_SUCCESS; - if (!migration_param.is_valid() || !new_sstable_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("migration param or new sstable meta is invalid", K(ret), K(migration_param), K(new_sstable_meta)); - } else if (OB_FAIL(check_sstable_basic_meta(migration_param.basic_meta_, new_sstable_meta.get_basic_meta()))) { - LOG_WARN("failed to check sstable basic meta", K(ret), K(migration_param), K(new_sstable_meta)); - } - return ret; -} int ObSSTableMetaChecker::check_sstable_basic_meta( const ObSSTableBasicMeta &old_sstable_basic_meta, diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 01d59dedf..6387a9f46 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -339,53 +339,35 @@ class ObSSTableMeta final DISALLOW_COPY_AND_ASSIGN(ObSSTableMeta); }; -class ObMigrationSSTableParam final +class ObForkSSTableParam final { public: - ObMigrationSSTableParam(); - ~ObMigrationSSTableParam(); + ObForkSSTableParam(); + ~ObForkSSTableParam(); bool is_valid() const; void reset(); - int assign(const ObMigrationSSTableParam ¶m); - bool is_empty_sstable() const; - bool is_shared_sstable() const; - bool is_shared_macro_blocks_sstable() const; - bool is_only_shared_macro_blocks_sstable() const; - int get_merge_res(blocksstable::ObSSTableMergeRes &res) const; - TO_STRING_KV(K_(basic_meta), - K(column_checksums_.count()), + TO_STRING_KV(K_(basic_meta), + K(column_checksums_.count()), K_(column_checksums), - K_(table_key), - K(column_default_checksums_.count()), - K_(column_default_checksums), - K_(is_small_sstable), K_(root_block_addr), KP_(root_block_buf), K_(data_block_macro_meta_addr), KP_(data_block_macro_meta_buf), K_(is_meta_root)); private: - int addr_serialize(const ObMetaDiskAddr &addr, const char *addr_buf, char *buf, const int64_t buf_len, int64_t &pos) const; - int addr_deserialize(const char *buf, const int64_t data_len, int64_t &pos, ObMetaDiskAddr &addr, char *&root__buf); - int64_t addr_get_serialize_size(const ObMetaDiskAddr &addr) const; - static const int64_t MIGRATION_SSTABLE_PARAM_VERSION = 1; typedef common::ObSEArray ColChecksumArray; public: common::ObArenaAllocator allocator_; ObSSTableBasicMeta basic_meta_; ColChecksumArray column_checksums_; - storage::ObITable::TableKey table_key_; - ColChecksumArray column_default_checksums_; - bool is_small_sstable_; // for shared data ObMetaDiskAddr root_block_addr_; char *root_block_buf_; ObMetaDiskAddr data_block_macro_meta_addr_; char *data_block_macro_meta_buf_; bool is_meta_root_; - OB_UNIS_VERSION(MIGRATION_SSTABLE_PARAM_VERSION); private: - DISALLOW_COPY_AND_ASSIGN(ObMigrationSSTableParam); + DISALLOW_COPY_AND_ASSIGN(ObForkSSTableParam); }; class ObSSTableMetaChecker @@ -398,9 +380,6 @@ class ObSSTableMetaChecker static int check_sstable_meta( const ObSSTableMeta &old_sstable_meta, const ObSSTableMeta &new_sstable_meta); - static int check_sstable_meta( - const ObMigrationSSTableParam &migration_param, - const ObSSTableMeta &new_sstable_meta); static int check_sstable_basic_meta( const ObSSTableBasicMeta &old_sstable_basic_meta, const ObSSTableBasicMeta &new_sstable_basic_meta); diff --git a/src/storage/checkpoint/ob_checkpoint_executor.cpp b/src/storage/checkpoint/ob_checkpoint_executor.cpp index d2fe0df25..569491b8d 100644 --- a/src/storage/checkpoint/ob_checkpoint_executor.cpp +++ b/src/storage/checkpoint/ob_checkpoint_executor.cpp @@ -18,6 +18,7 @@ #include "lib/stat/ob_diagnostic_info_guard.h" #include "ob_checkpoint_executor.h" +#include "storage/ls/ob_ls.h" #include "share/rc/ob_module_provider.h" #include "storage/tx_storage/ob_checkpoint_service.h" #include "logservice/ob_log_service.h" @@ -155,10 +156,8 @@ void ObCheckpointExecutor::add_server_event_history_for_update_clog_checkpoint( if (update_clog_checkpoint_times_ > 0) { int64_t cur_time = ObClockGenerator::getClock(); if (cur_time - last_add_server_history_time_ > ADD_SERVER_HISTORY_INTERVAL) { - - const int64_t ls_id = ls_->get_ls_id().id(); last_add_server_history_time_ = cur_time; - SERVER_EVENT_ADD("checkpoint", "update_clog_checkpoint", K(ls_id), K(checkpoint_scn), K(service_type), K_(update_clog_checkpoint_times)); + SERVER_EVENT_ADD("checkpoint", "update_clog_checkpoint", K(checkpoint_scn), K(service_type), K_(update_clog_checkpoint_times)); update_clog_checkpoint_times_ = 0; } } @@ -175,7 +174,7 @@ int ObCheckpointExecutor::update_clog_checkpoint() SCN checkpoint_scn; checkpoint_scn.set_max(); if (OB_FAIL(freezer->get_max_consequent_callbacked_scn(checkpoint_scn))) { - STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(freezer->get_ls_id())); + STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret)); } else { // used to record which handler provide the smallest rec_scn int min_rec_scn_service_type_index = 0; @@ -185,45 +184,44 @@ int ObCheckpointExecutor::update_clog_checkpoint() get_min_rec_scn_service_type_by_index_(min_rec_scn_service_type_index, service_type, buf_len); const SCN checkpoint_scn_in_ls_meta = ls_->get_clog_checkpoint_scn(); - const share::ObLSID ls_id = ls_->get_ls_id(); LSN clog_checkpoint_lsn; if (checkpoint_scn == checkpoint_scn_in_ls_meta) { STORAGE_LOG(INFO, "[CHECKPOINT] clog checkpoint no change", K(checkpoint_scn), - K(checkpoint_scn_in_ls_meta), K(ls_id), K(service_type)); + K(checkpoint_scn_in_ls_meta), K(service_type)); } else if (checkpoint_scn < checkpoint_scn_in_ls_meta) { if (min_rec_scn_service_type_index == 0) { STORAGE_LOG(INFO, "[CHECKPOINT] expexted when no log callbacked or replayed", - K(checkpoint_scn), K(checkpoint_scn_in_ls_meta), K(ls_id)); + K(checkpoint_scn), K(checkpoint_scn_in_ls_meta)); } else { STORAGE_LOG(ERROR, "[CHECKPOINT] can not advance clog checkpoint", K(checkpoint_scn), - K(checkpoint_scn_in_ls_meta), K(ls_id), K(service_type)); + K(checkpoint_scn_in_ls_meta), K(service_type)); } } else if (OB_FAIL(loghandler_->locate_by_scn_coarsely(checkpoint_scn, clog_checkpoint_lsn))) { if (OB_NOT_INIT == ret) { - STORAGE_LOG(WARN, "palf has been disabled", K(ret), K(checkpoint_scn), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "palf has been disabled", K(ret), K(checkpoint_scn)); ret = OB_SUCCESS; } else if (OB_NEED_RETRY == ret || OB_ENTRY_NOT_EXIST == ret) { - STORAGE_LOG(WARN, "locate_by_scn_coarsely need retry", K(checkpoint_scn), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "locate_by_scn_coarsely need retry", K(checkpoint_scn)); ret = OB_SUCCESS; } else { - STORAGE_LOG(ERROR, "locate lsn by logts failed", K(ret), K(ls_id), + STORAGE_LOG(ERROR, "locate lsn by logts failed", K(ret), K(checkpoint_scn), K(checkpoint_scn_in_ls_meta)); } } else if (OB_FAIL(ls_->set_clog_checkpoint(clog_checkpoint_lsn, checkpoint_scn, true/*write_slog*/))) { - STORAGE_LOG(WARN, "set clog checkpoint failed", K(ret), K(clog_checkpoint_lsn), K(checkpoint_scn), K(ls_id)); + STORAGE_LOG(WARN, "set clog checkpoint failed", K(ret), K(clog_checkpoint_lsn), K(checkpoint_scn)); } else { update_clog_checkpoint_times_++; FLOG_INFO("[CHECKPOINT] update clog checkpoint successfully", - K(clog_checkpoint_lsn), K(checkpoint_scn), K(ls_id), + K(clog_checkpoint_lsn), K(checkpoint_scn), K(service_type)); } add_server_event_history_for_update_clog_checkpoint(checkpoint_scn, service_type); } } else { - STORAGE_LOG(WARN, "freezer should not null", K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "freezer should not null"); } } else { - STORAGE_LOG(WARN, "update_checkpoint is not enabled", K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "update_checkpoint is not enabled"); } return ret; @@ -232,14 +230,13 @@ int ObCheckpointExecutor::update_clog_checkpoint() int ObCheckpointExecutor::advance_checkpoint_by_flush(const share::SCN input_recycle_scn) { int ret = OB_SUCCESS; - const ObLSID ls_id = ls_->get_ls_id(); SCN recycle_scn = input_recycle_scn; SCN max_decided_scn; RLockGuard guard(rwlock_); if (update_checkpoint_enabled_) { if (OB_FAIL(loghandler_->get_max_decided_scn(max_decided_scn))) { - STORAGE_LOG(WARN, "failed to get_max_decided_scn", K(ls_id)); + STORAGE_LOG(WARN, "failed to get_max_decided_scn"); } else if (!recycle_scn.is_valid() && OB_FAIL(calculate_recycle_scn_(max_decided_scn, recycle_scn))) { if (OB_EAGAIN != ret) { STORAGE_LOG(WARN, "calculate recycle scn failed", KR(ret)); @@ -247,16 +244,16 @@ int ObCheckpointExecutor::advance_checkpoint_by_flush(const share::SCN input_rec } else if (OB_FAIL(check_need_flush_(max_decided_scn, recycle_scn))) { STORAGE_LOG(WARN, "no need flush"); } else { - STORAGE_LOG(INFO, "start flush", K(recycle_scn), K(input_recycle_scn), K(ls_id)); + STORAGE_LOG(INFO, "start flush", K(recycle_scn), K(input_recycle_scn)); for (int i = 1; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { int tmp_ret = OB_SUCCESS; if (OB_NOT_NULL(handlers_[i]) && OB_TMP_FAIL(handlers_[i]->flush(recycle_scn))) { - STORAGE_LOG(WARN, "handler flush failed", KR(tmp_ret), K(recycle_scn), K(tmp_ret), K(i), K(ls_id)); + STORAGE_LOG(WARN, "handler flush failed", KR(tmp_ret), K(recycle_scn), K(tmp_ret), K(i)); } } } } else { - STORAGE_LOG(WARN, "update_checkpoint is not enabled", K(ls_id)); + STORAGE_LOG(WARN, "update_checkpoint is not enabled"); } return ret; @@ -327,7 +324,6 @@ int ObCheckpointExecutor::advance_checkpoint_by_flush(const share::SCN input_rec int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN &recycle_scn) { int ret = OB_SUCCESS; - const ObLSID ls_id = ls_->get_ls_id(); const LSN clog_checkpoint_lsn = ls_->get_clog_base_lsn(); const SCN clog_checkpoint_scn = ls_->get_clog_checkpoint_scn(); if (prev_clog_checkpoint_lsn_.is_valid() && (prev_clog_checkpoint_lsn_ == clog_checkpoint_lsn) && @@ -335,12 +331,11 @@ int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN recycle_scn = prev_recycle_scn_; reuse_recycle_scn_times_++; if (reuse_recycle_scn_times_ % 1000 == 0) { - STORAGE_LOG_RET(WARN, 0, "attention! clog checkpiont has not changed for a long time", K(ls_id)); + STORAGE_LOG_RET(WARN, 0, "attention! clog checkpiont has not changed for a long time"); recycle_scn.set_max(); } STORAGE_LOG(INFO, "clog checkpoint has not changed yet. use previous recycle_scn to advance checkpoint", - K(ls_id), K(reuse_recycle_scn_times_), K(clog_checkpoint_lsn), K(recycle_scn)); @@ -348,9 +343,9 @@ int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN SCN min_recycle_scn; SCN expected_recycle_scn; if (OB_FAIL(calculate_min_recycle_scn_(clog_checkpoint_lsn, min_recycle_scn))) { - STORAGE_LOG(WARN, "calculate min recycle scn failed", KR(ret), K(ls_id)); + STORAGE_LOG(WARN, "calculate min recycle scn failed", KR(ret)); } else if (OB_FAIL(calculate_expected_recycle_scn_(clog_checkpoint_lsn, expected_recycle_scn))) { - STORAGE_LOG(WARN, "calculate expected recycle scn failed", KR(ret), K(ls_id)); + STORAGE_LOG(WARN, "calculate expected recycle scn failed", KR(ret)); } else { recycle_scn = MIN(max_decided_scn, expected_recycle_scn); if (recycle_scn < min_recycle_scn) { @@ -358,7 +353,6 @@ int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN STORAGE_LOG(INFO, "recycle_scn too small, skip trigger flush once", KR(ret), - K(ls_id), K(min_recycle_scn), K(expected_recycle_scn), K(max_decided_scn)); @@ -368,7 +362,6 @@ int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN reuse_recycle_scn_times_ = 0; STORAGE_LOG(INFO, "advance checkpoint by flush to avoid clog disk full", - K(ls_id), K(recycle_scn), K(max_decided_scn), K(clog_checkpoint_lsn), @@ -381,44 +374,22 @@ int ObCheckpointExecutor::calculate_recycle_scn_(const SCN max_decided_scn, SCN return ret; } -/** - * @brief As calculate_recycle_scn_() comments show : There is a threshold of clog recycle and the default value is 5%. - *But if there are too many logstreams in a single tenant, for example, 20 logstreams. And each logstream used 4.9% clog - *disk, then all logstreams used 98% clog disk but the clog recycling cannot be triggered. So another parameter is added - *: MAX_TENANT_RECYCLE_CLOG_PERCENTAGE, which used to set the max recycle clog trigger. - * - *For example, in the previous logic, if there were a total of 10 logstreams and the minimum recycle threshold for each - *logstream was 5%, then the total threshold would be 50%. However, in the new logic, the minimum recycle threshold for - *each log stream cannot reach 5% but is calculated as MAX_TENANT_RECYCLE_CLOG_PERCENTAGE (30) divided by 10, which is - *3%. Therefore, regardless of the number of log streams, it ensures that the clog recycling can be triggered properly. - * - */ int ObCheckpointExecutor::calculate_min_recycle_scn_(const LSN clog_checkpoint_lsn, SCN &min_recycle_scn) { const int64_t DEFAULT_MIN_LS_RECYCLE_CLOG_PERCENTAGE = 5; - const int64_t MAX_TENANT_RECYCLE_CLOG_PERCENTAGE = ObCheckPointService::NEED_FLUSH_CLOG_DISK_PERCENT; int ret = OB_SUCCESS; int64_t used_size = 0; int64_t total_size = 0; ObLogService *log_service = nullptr; - ObLSService *ls_service = nullptr; - if (OB_ISNULL(log_service = share::g_mp->log_service()) || OB_ISNULL(ls_service = share::g_mp->ls_service())) { + if (OB_ISNULL(log_service = share::g_mp->log_service())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "get_log_service failed", K(ret)); } else if (OB_FAIL(log_service->get_palf_disk_usage(used_size, total_size))) { STORAGE_LOG(WARN, "get_disk_usage failed", K(ret), K(used_size), K(total_size)); } else { - int64_t ls_count = ls_service->get_ls_count(); - if (ls_count <= 0) { - ls_count = 1; - } - - int64_t ls_min_recycle_clog_percentage = - MIN(DEFAULT_MIN_LS_RECYCLE_CLOG_PERCENTAGE, MAX_TENANT_RECYCLE_CLOG_PERCENTAGE / ls_count); - - LSN min_recycle_lsn = clog_checkpoint_lsn + (total_size * ls_min_recycle_clog_percentage / 100); + LSN min_recycle_lsn = clog_checkpoint_lsn + (total_size * DEFAULT_MIN_LS_RECYCLE_CLOG_PERCENTAGE / 100); if (OB_FAIL(loghandler_->locate_by_lsn_coarsely(min_recycle_lsn, min_recycle_scn))) { STORAGE_LOG(WARN, "locate min_recycle_scn by lsn failed", KR(ret)); } @@ -445,7 +416,6 @@ int ObCheckpointExecutor::calculate_expected_recycle_scn_(const palf::LSN clog_c int ObCheckpointExecutor::check_need_flush_(const SCN max_decided_scn, const SCN recycle_scn) { int ret = OB_SUCCESS; - const ObLSID ls_id = ls_->get_ls_id(); const SCN clog_checkpoint_scn = ls_->get_clog_checkpoint_scn(); if (recycle_scn.is_max()) { // must do flush @@ -454,15 +424,13 @@ int ObCheckpointExecutor::check_need_flush_(const SCN max_decided_scn, const SCN STORAGE_LOG(WARN, "recycle_scn should not smaller than checkpoint_log_scn", K(recycle_scn), - K(clog_checkpoint_scn), - K(ls_id)); + K(clog_checkpoint_scn)); } else if (recycle_scn > max_decided_scn) { ret = OB_EAGAIN; STORAGE_LOG(WARN, "recycle_scn is larger than max_decided_scn", K(recycle_scn), K(clog_checkpoint_scn), - K(ls_id), K(max_decided_scn)); } return ret; @@ -502,12 +470,12 @@ int ObCheckpointExecutor::traversal_flush() const int ret = OB_SUCCESS; ObLSTxService *ls_tx_ser = nullptr; if (!update_checkpoint_enabled_) { - STORAGE_LOG(WARN, "update_checkpoint is not enabled", K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "update_checkpoint is not enabled"); } else if (OB_ISNULL(ls_tx_ser = ls_->get_tx_svr())) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls_tx_ser should not be null", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "ls_tx_ser should not be null", K(ret)); } else if (OB_FAIL(ls_tx_ser->traversal_flush())) { - STORAGE_LOG(WARN, "ls_tx_ser flush failed", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "ls_tx_ser flush failed", K(ret)); } return ret; } diff --git a/src/storage/checkpoint/ob_common_checkpoint.h b/src/storage/checkpoint/ob_common_checkpoint.h index 49186cfcf..057c9bdc7 100644 --- a/src/storage/checkpoint/ob_common_checkpoint.h +++ b/src/storage/checkpoint/ob_common_checkpoint.h @@ -18,7 +18,6 @@ #define OCEANBASE_STORAGE_OB_COMMON_CHECKPOINT_H_ #include -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "share/scn.h" diff --git a/src/storage/checkpoint/ob_data_checkpoint.cpp b/src/storage/checkpoint/ob_data_checkpoint.cpp index 3d47f4f80..1073c6441 100644 --- a/src/storage/checkpoint/ob_data_checkpoint.cpp +++ b/src/storage/checkpoint/ob_data_checkpoint.cpp @@ -193,36 +193,6 @@ int ObDataCheckpoint::init(ObLS *ls) return OB_SUCCESS; } -int ObDataCheckpoint::safe_to_destroy(bool &is_safe_destroy) -{ - int ret = OB_SUCCESS; - - is_safe_destroy = true; - // avoid start ls_freeze again after waiting ls_freeze finish - is_inited_ = false; - // wait until ls_freeze finish - while(!ls_freeze_finished_) { - ob_throttle_usleep(1000 * 1000, 0, ls_->get_ls_id().id()); - if (REACH_TIME_INTERVAL(10 * 1000L * 1000L)) { - STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "ls freeze cost too much time", K(ls_->get_ls_id())); - ret = OB_ERR_UNEXPECTED; - break; - } - } - - WLOCK(LS_FROZEN | NEW_CREATE | ACTIVE | PREPARE); - new_create_list_.reset(); - ls_frozen_list_.reset(); - active_list_.reset(); - prepare_list_.reset(); - - if (OB_FAIL(ret)) { - is_safe_destroy = false; - } - - return ret; -} - SCN ObDataCheckpoint::get_rec_scn() { RLOCK(LS_FROZEN | NEW_CREATE | ACTIVE | PREPARE); @@ -269,7 +239,7 @@ int ObDataCheckpoint::flush(SCN recycle_scn, bool need_freeze) if (OB_FAIL(ls_->logstream_freeze(is_sync, abs_timeout_ts, get_freeze_source()))) { - STORAGE_LOG(WARN, "minor freeze failed", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "minor freeze failed", K(ret)); } } else if (need_freeze) { SCN active_rec_scn = get_active_rec_scn(); @@ -280,10 +250,10 @@ int ObDataCheckpoint::flush(SCN recycle_scn, bool need_freeze) K(recycle_scn)); } else if (OB_FAIL(freeze_base_on_needs_(recycle_scn))) { STORAGE_LOG(WARN, "freeze_base_on_needs failed", - K(ret), K(ls_->get_ls_id()), K(recycle_scn)); + K(ret), K(recycle_scn)); } } else if (OB_FAIL(traversal_flush_())) { - STORAGE_LOG(WARN, "traversal_flush failed", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "traversal_flush failed", K(ret)); } return ret; } @@ -356,19 +326,17 @@ void ObDataCheckpoint::road_to_flush(SCN rec_scn) if (OB_UNLIKELY(!is_inited_)) { STORAGE_LOG_RET(WARN, OB_NOT_INIT, "ObDataCheckpoint not init", K(is_inited_)); } else { - STORAGE_LOG(INFO, "[Freezer] road_to_flush begin", K(ls_->get_ls_id())); + STORAGE_LOG(INFO, "[Freezer] road_to_flush begin"); // used to print log when stay at a cycle for a long time int64_t last_time = common::ObTimeUtility::fast_current_time(); // new_create_list -> ls_frozen_list pop_new_create_to_ls_frozen_(); last_time = common::ObTimeUtility::fast_current_time(); - STORAGE_LOG(INFO, "[Freezer] new_create_list to ls_frozen_list success", - K(ls_->get_ls_id())); + STORAGE_LOG(INFO, "[Freezer] new_create_list to ls_frozen_list success"); // ls_frozen_list -> active_list ls_frozen_to_active_(last_time); - STORAGE_LOG(INFO, "[Freezer] ls_frozen_list to active_list success", - K(ls_->get_ls_id())); + STORAGE_LOG(INFO, "[Freezer] ls_frozen_list to active_list success"); // active_list -> ls_frozen_list ObFreezeCheckpoint *last = nullptr; { @@ -376,15 +344,14 @@ void ObDataCheckpoint::road_to_flush(SCN rec_scn) last = active_list_.get_first_greater(rec_scn); } pop_active_list_to_ls_frozen_(last); - STORAGE_LOG(INFO, "[Freezer] active_list to ls_frozen_list success", - K(ls_->get_ls_id())); + STORAGE_LOG(INFO, "[Freezer] active_list to ls_frozen_list success"); // add diagnose info add_diagnose_info_for_ls_frozen_(); // ls_frozen_list -> prepare_list last_time = common::ObTimeUtility::fast_current_time(); ls_frozen_to_prepare_(last_time); - STORAGE_LOG(INFO, "[Freezer] road_to_flush end", K(ls_->get_ls_id())); + STORAGE_LOG(INFO, "[Freezer] road_to_flush end"); } set_ls_freeze_finished_(true); } @@ -471,9 +438,9 @@ void ObDataCheckpoint::ls_frozen_to_active_(int64_t &last_time) } if (!ls_frozen_list_is_empty) { - ob_throttle_usleep(LOOP_TRAVERSAL_INTERVAL_US, ret, ls_->get_ls_id().id()); + ob_throttle_usleep(LOOP_TRAVERSAL_INTERVAL_US, ret); if (task_reach_time_interval(3 * 1000 * 1000, last_time)) { - STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cost too much time in ls_frozen_list_", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cost too much time in ls_frozen_list_", K(ret)); RLOCK(LS_FROZEN); print_list_(ls_frozen_list_); } @@ -531,9 +498,9 @@ void ObDataCheckpoint::ls_frozen_to_prepare_(int64_t &last_time) } if (!ls_frozen_list_is_empty) { - ob_throttle_usleep(LOOP_TRAVERSAL_INTERVAL_US, ret, ls_->get_ls_id().id()); + ob_throttle_usleep(LOOP_TRAVERSAL_INTERVAL_US, ret); if (task_reach_time_interval(3 * 1000 * 1000, last_time)) { - STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cost too much time in ls_frozen_list_", K(ls_->get_ls_id())); + STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cost too much time in ls_frozen_list_"); RLOCK(LS_FROZEN); print_list_(ls_frozen_list_); } @@ -541,7 +508,7 @@ void ObDataCheckpoint::ls_frozen_to_prepare_(int64_t &last_time) break; } if (OB_EAGAIN == ret) { - ob_throttle_usleep(100000, ret, ls_->get_ls_id().id()); + ob_throttle_usleep(100000, ret); } } while (true); @@ -618,20 +585,16 @@ int ObDataCheckpoint::get_freezecheckpoint_info( RLOCK(LS_FROZEN | NEW_CREATE | ACTIVE | PREPARE); if (OB_FAIL(new_create_list_.get_freezecheckpoint_info( freeze_checkpoint_array))) { - STORAGE_LOG(ERROR, "iterator new_create_list fail", - K(ls_->get_ls_id())); + STORAGE_LOG(ERROR, "iterator new_create_list fail"); } else if (OB_FAIL(active_list_.get_freezecheckpoint_info( freeze_checkpoint_array))) { - STORAGE_LOG(ERROR, "iterator active_list_list fail", - K(ls_->get_ls_id())); + STORAGE_LOG(ERROR, "iterator active_list_list fail"); } else if (OB_FAIL(prepare_list_.get_freezecheckpoint_info( freeze_checkpoint_array))) { - STORAGE_LOG(ERROR, "iterator prepare_list_list fail", - K(ls_->get_ls_id())); + STORAGE_LOG(ERROR, "iterator prepare_list_list fail"); } else if (OB_FAIL(ls_frozen_list_.get_freezecheckpoint_info( freeze_checkpoint_array))) { - STORAGE_LOG(ERROR, "iterator ls_frozen_list_list fail", - K(ls_->get_ls_id())); + STORAGE_LOG(ERROR, "iterator ls_frozen_list_list fail"); } return ret; @@ -651,7 +614,7 @@ int ObDataCheckpoint::traversal_flush_() RLOCK(PREPARE); if (prepare_list_.is_empty()) { STORAGE_LOG(TRACE, "skip traversal_flush", K(ls_freeze_finished_), - K(prepare_list_.is_empty()), K(ls_->get_ls_id())); + K(prepare_list_.is_empty())); } else { ObCheckpointIterator iterator; prepare_list_.get_iterator(iterator); @@ -679,13 +642,13 @@ int ObDataCheckpoint::traversal_flush_() ObITable *table = flush_tasks[i].get_table(); ObITabletMemtable *tablet_memtable = static_cast(table); // Even if flush failed, we can continue to flush the next one except OB_SIZE_OVERFLOW - if (OB_TMP_FAIL(tablet_memtable->flush(ls_->get_ls_id())) + if (OB_TMP_FAIL(tablet_memtable->flush()) && tmp_ret != OB_NO_NEED_UPDATE) { - STORAGE_LOG(WARN, "memtable flush failed", K(tmp_ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "memtable flush failed", K(tmp_ret)); } } - STORAGE_LOG(INFO, "traversal_flush successfully", K(ls_->get_ls_id()), K(flush_tasks)); + STORAGE_LOG(INFO, "traversal_flush successfully", K(flush_tasks)); } return ret; @@ -924,14 +887,14 @@ int ObDataCheckpoint::freeze_base_on_needs_(share::SCN recycle_scn) if (OB_FAIL(ls_->logstream_freeze(is_sync, abs_timeout_ts, get_freeze_source()))) { - STORAGE_LOG(WARN, "minor freeze failed", K(ret), K(ls_->get_ls_id())); + STORAGE_LOG(WARN, "minor freeze failed", K(ret)); } } else if (OB_FAIL(ls_->tablet_freeze(need_flush_tablets, is_sync, abs_timeout_ts, false, /*need_rewrite_meta*/ get_freeze_source()))) { - STORAGE_LOG(WARN, "batch tablet freeze failed", K(ret), K(ls_->get_ls_id()), K(need_flush_tablets)); + STORAGE_LOG(WARN, "batch tablet freeze failed", K(ret), K(need_flush_tablets)); } } return ret; diff --git a/src/storage/checkpoint/ob_data_checkpoint.h b/src/storage/checkpoint/ob_data_checkpoint.h index 3ab9ea18f..b8fbddbac 100644 --- a/src/storage/checkpoint/ob_data_checkpoint.h +++ b/src/storage/checkpoint/ob_data_checkpoint.h @@ -95,7 +95,6 @@ class ObDataCheckpoint : public ObCommonCheckpoint // used for virtual table static const uint64_t LS_DATA_CHECKPOINT_TABLET_ID = 40000; int init(ObLS *ls); - int safe_to_destroy(bool &is_safe_destroy); ObCheckpointDList* get_checkpoint_list(const ObFreezeCheckpointLocation &location) { ObCheckpointDList *ret = NULL; diff --git a/src/storage/checkpoint/ob_freeze_checkpoint.h b/src/storage/checkpoint/ob_freeze_checkpoint.h index 11a5a56b4..e44766458 100644 --- a/src/storage/checkpoint/ob_freeze_checkpoint.h +++ b/src/storage/checkpoint/ob_freeze_checkpoint.h @@ -19,7 +19,6 @@ #include "lib/list/ob_dlink_node.h" #include "lib/utility/ob_print_utils.h" -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "share/scn.h" @@ -99,7 +98,7 @@ class ObFreezeCheckpoint : public common::ObDLinkBase void reset(); virtual void remove_from_data_checkpoint(); virtual share::SCN get_rec_scn() = 0; - virtual int flush(share::ObLSID ls_id) = 0; + virtual int flush() = 0; // judge rec_scn of the checkpoint unit won't get smaller // by comparing with max_consequent_callbacked_scn // a unit will only be moved once by rec_scn_stable_ diff --git a/src/storage/compaction/ob_basic_schedule_tablet_func.cpp b/src/storage/compaction/ob_basic_schedule_tablet_func.cpp index 7bf67aabd..6418824a6 100644 --- a/src/storage/compaction/ob_basic_schedule_tablet_func.cpp +++ b/src/storage/compaction/ob_basic_schedule_tablet_func.cpp @@ -43,22 +43,18 @@ void ObBasicScheduleTabletFunc::destroy() schedule_freeze_dag(true/*force*/); // schedule dag before destroy } -int ObBasicScheduleTabletFunc::switch_ls(ObLSHandle &ls_handle) +int ObBasicScheduleTabletFunc::init(ObLS *ls) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - schedule_freeze_dag(true/*force*/); // schedule dag before switch to next ls - - if (OB_FAIL(ls_status_.init_for_major(merge_version_, ls_handle))) { + if (OB_FAIL(ls_status_.init_for_major(merge_version_, ls))) { if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("failed to init ls status", KR(ret), K_(merge_version), K(ls_id)); + LOG_WARN("failed to init ls status", KR(ret), K_(merge_version)); } } else if (OB_UNLIKELY(merge_version_ > ObBasicMergeScheduler::INIT_COMPACTION_SCN && !ls_status_.can_merge())) { ret = OB_STATE_NOT_MATCH; - LOG_WARN("could not to merge now", K(ret), K(ls_id), K(ls_status_)); + LOG_WARN("could not to merge now", K(ret), K(ls_status_)); } else { - freeze_param_.ls_id_ = ls_id; freeze_param_.compaction_scn_ = merge_version_; } if (OB_SUCC(ret)) { @@ -73,8 +69,7 @@ void ObBasicScheduleTabletFunc::update_tenant_cached_status() if (OB_NOT_NULL(scheduler)) { is_skip_merge_tenant_ = scheduler->get_tenant_status().is_skip_merge_tenant(); ls_could_schedule_merge_ = scheduler->could_major_merge_start() && ls_status_.can_merge(); - // can only schedule new round on ls leader - ls_could_schedule_new_round_ = ls_could_schedule_merge_ && ls_status_.is_leader_; + ls_could_schedule_new_round_ = ls_could_schedule_merge_; if (!ls_status_.can_merge() && REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { LOG_INFO("should not schedule major merge for ls", K_(ls_status), @@ -107,13 +102,13 @@ void ObBasicScheduleTabletFunc::schedule_freeze_dag(const bool force) /* * diagnose section */ -int ObBasicScheduleTabletFunc::diagnose_switch_ls( - ObLSHandle &ls_handle) +int ObBasicScheduleTabletFunc::diagnose_init( + ObLS *ls) { int ret = OB_SUCCESS; - if (OB_FAIL(ls_status_.init_for_major(merge_version_, ls_handle))) { + if (OB_FAIL(ls_status_.init_for_major(merge_version_, ls))) { if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("failed to init ls status", KR(ret), K_(merge_version), K(ls_handle)); + LOG_WARN("failed to init ls status", KR(ret), K_(merge_version), KP(ls)); } } else { update_tenant_cached_status(); @@ -130,7 +125,6 @@ int ObBasicScheduleTabletFunc::check_with_schedule_scn( const ObTabletStatusCache &tablet_status, bool &can_merge) { - const ObLSID &ls_id = ls_status_.ls_id_; can_merge = false; int ret = OB_SUCCESS; bool need_force_freeze = false; @@ -167,7 +161,6 @@ int ObBasicScheduleTabletFunc::check_with_schedule_scn( K(need_merge), K(can_merge), K(schedule_scn), K(need_force_freeze), K(weak_read_ts_ready), K_(ls_status), K(tablet_status)); ADD_SUSPECT_INFO(MEDIUM_MERGE, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - ls_id, tablet_id, ObSuspectInfoType::SUSPECT_CANT_MAJOR_MERGE, schedule_scn, diff --git a/src/storage/compaction/ob_basic_schedule_tablet_func.h b/src/storage/compaction/ob_basic_schedule_tablet_func.h index 84bef3ca9..671e7c5c7 100644 --- a/src/storage/compaction/ob_basic_schedule_tablet_func.h +++ b/src/storage/compaction/ob_basic_schedule_tablet_func.h @@ -25,7 +25,6 @@ namespace storage class ObLS; class ObTablet; class ObTabletHandle; -class ObLSHandle; } namespace compaction { @@ -33,7 +32,7 @@ struct ObBasicScheduleTabletFunc { ObBasicScheduleTabletFunc(const int64_t merge_version, const int64_t loop_cnt = 0); virtual ~ObBasicScheduleTabletFunc() { destroy(); } - int switch_ls(storage::ObLSHandle &ls_handle); + int init(storage::ObLS *ls); void destroy(); const ObLSStatusCache &get_ls_status() const { return ls_status_; } ObScheduleTabletCnt &get_schedule_tablet_cnt() { return tablet_cnt_; } @@ -46,7 +45,7 @@ struct ObBasicScheduleTabletFunc /* * diagnose section */ - int diagnose_switch_ls(storage::ObLSHandle &ls_handle); + int diagnose_init(storage::ObLS *ls); protected: void update_tenant_cached_status(); virtual void schedule_freeze_dag(const bool force); diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp index e091c4ea3..9b726d510 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp @@ -106,16 +106,15 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param) is_tenant_major_merge_(false), is_backfill_(false), is_delete_insert_merge_(false), - is_ha_compeleted_(true), + is_restore_complete_(true), for_unittest_(false), merge_level_(MICRO_BLOCK_MERGE_LEVEL), merge_reason_(ObAdaptiveMergePolicy::AdaptiveMergeReason::NONE), sstable_logic_seq_(0), - ls_handle_(), + ls_(nullptr), tables_handle_(), concurrent_cnt_(0), data_version_(0), - ls_rebuild_seq_(-1), read_base_version_(0), create_snapshot_version_(0), start_time_(0), @@ -145,13 +144,13 @@ void ObStaticMergeParam::reset() } rowkey_read_info_ = nullptr; multi_version_column_descs_.reset(); - ls_handle_.reset(); // ls_handle could release before tablet_handle + ls_ = nullptr; tx_id_ = 0; tablet_schema_guard_.reset(); encoding_granularity_ = 0; for_unittest_ = false; is_delete_insert_merge_ = false; - is_ha_compeleted_ = true; + is_restore_complete_ = true; } bool ObStaticMergeParam::is_valid() const @@ -160,9 +159,9 @@ bool ObStaticMergeParam::is_valid() const if (OB_UNLIKELY(!dag_param_.is_valid())) { bret = false; LOG_WARN_RET(OB_ERR_UNEXPECTED, "dag param is invalid", K_(dag_param), K(dag_param_.is_valid())); - } else if (OB_UNLIKELY(!ls_handle_.is_valid() || tables_handle_.empty())) { + } else if (OB_UNLIKELY(OB_ISNULL(ls_) || tables_handle_.empty())) { bret = false; - LOG_WARN_RET(OB_ERR_UNEXPECTED, "ls_handle or tables_handle is invalid", K_(ls_handle), K_(tables_handle)); + LOG_WARN_RET(OB_ERR_UNEXPECTED, "ls or tables handle is invalid", KP_(ls), K_(tables_handle)); } else if (OB_UNLIKELY(is_multi_version_merge(get_merge_type()) && !scn_range_.is_valid())) { bret = false; LOG_WARN_RET(OB_ERR_UNEXPECTED, "scn range is invalid for multi_version merge", "merge_type", get_merge_type(), K_(scn_range)); @@ -187,12 +186,10 @@ int ObStaticMergeParam::init_static_info(ObTabletHandle &tablet_handle) } else { rowkey_read_info_ = static_cast(&(tablet_handle.get_obj()->get_rowkey_read_info())); } - const bool use_fixed_percentage = nullptr != schema_ && - schema_->mv_mode_.table_referenced_by_fast_lsm_mv_flag_ && - (is_mini_merge(get_merge_type()) || is_minor_merge(get_merge_type())); + const bool use_fixed_percentage = false; if (OB_FAIL(init_multi_version_column_descs())) { LOG_WARN("failed to init multi_version_column_descs", KR(ret)); - } else if (OB_FAIL(pre_warm_param_.init(get_ls_id(), get_tablet_id(), use_fixed_percentage))) { + } else if (OB_FAIL(pre_warm_param_.init(get_tablet_id(), use_fixed_percentage))) { LOG_WARN("failed to init pre warm param", KR(ret)); } return ret; @@ -444,7 +441,7 @@ int ObBasicTabletMergeCtx::build_ctx(bool &finish_flag) LOG_WARN(str, KR(ret), KPC(this)); CTX_SET_DIAGNOSE_LOCATION(*this); if (OB_FAIL(get_ls_and_tablet())) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_PRINT_WRAPPER("failed to get ls_handle/tablet_handle/rebuild_seq"); + LOG_PRINT_WRAPPER("failed to get ls, tablet handle, or rebuild sequence"); } } else if (OB_FAIL(get_merge_tables(get_merge_table_result))) { if (OB_NO_NEED_MERGE != ret) { @@ -586,23 +583,23 @@ bool ObBasicTabletMergeCtx::is_valid() const int ObBasicTabletMergeCtx::get_ls_and_tablet() { int ret = OB_SUCCESS; - ObLSHandle &ls_handle = static_param_.ls_handle_; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(static_param_.get_ls_id(), ls_handle, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get log stream", K(ret), K(static_param_.get_ls_id())); - } else if (ls_handle.get_ls()->is_offline()) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(static_param_.ls_))) { + LOG_WARN("failed to get single log stream", K(ret)); + } else if (static_param_.ls_->is_offline()) { ret = OB_CANCELED; LOG_INFO("ls offline, skip merge", K(ret), "param", get_dag_param()); - } else if (FALSE_IT(static_param_.ls_rebuild_seq_ = ls_handle.get_ls()->get_rebuild_seq())) { - } else if (get_dag_param().need_swap_tablet_flag_) { - if (OB_FAIL(swap_tablet())) { - LOG_WARN("failed to swap tablet", K(ret), K(static_param_)); + } else { + if (get_dag_param().need_swap_tablet_flag_) { + if (OB_FAIL(swap_tablet())) { + LOG_WARN("failed to swap tablet", K(ret), K(static_param_)); + } + } else if (OB_FAIL(static_param_.ls_->get_tablet_svr()->get_tablet( + static_param_.get_tablet_id(), + tablet_handle_, + 0/*timeout_us*/, + storage::ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("failed to get tablet", K(ret), K(static_param_)); } - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( - static_param_.get_tablet_id(), - tablet_handle_, - 0/*timeout_us*/, - storage::ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("failed to get tablet", K(ret), K(static_param_)); } return ret; } @@ -630,7 +627,7 @@ int ObBasicTabletMergeCtx::get_merge_tables(ObGetMergeTablesResult &get_merge_ta int ObBasicTabletMergeCtx::swap_tablet() { int ret = OB_SUCCESS; - const ObTabletMapKey key(get_ls_id(), get_tablet_id()); + const ObTabletMapKey key(get_tablet_id()); if (OB_FAIL(get_ls()->get_tablet_svr()->get_tablet_without_memtables( WashTabletPriority::WTP_LOW, key, mem_ctx_.get_allocator(), tablet_handle_))) { LOG_WARN("failed to get alloc tablet handle", K(ret), K(key)); @@ -666,16 +663,16 @@ int ObBasicTabletMergeCtx::get_storage_schema() { int ret = OB_SUCCESS; ObStorageSchema *schema_on_tablet = nullptr; - ObTabletHAStatus ha_status = get_tablet()->get_tablet_meta().ha_status_; + const ObTabletRestoreState &restore_state = get_tablet()->get_tablet_meta().restore_state_; if (OB_FAIL(get_tablet()->load_storage_schema(mem_ctx_.get_allocator(), schema_on_tablet))) { LOG_WARN("failed to load storage schema", K(ret), K_(tablet_handle)); - } else if (OB_UNLIKELY(!ha_status.is_valid())) { + } else if (OB_UNLIKELY(!restore_state.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ha status", K(ret), K(ha_status)); + LOG_WARN("invalid tablet restore state", K(ret), K(restore_state)); } else { static_param_.schema_ = schema_on_tablet; static_param_.is_delete_insert_merge_ = schema_on_tablet->is_delete_insert_merge_engine(); - static_param_.is_ha_compeleted_ = ha_status.is_none(); + static_param_.is_restore_complete_ = restore_state.is_restore_status_full(); } return ret; } @@ -838,10 +835,6 @@ void ObBasicTabletMergeCtx::add_sstable_merge_info( if (time_guard.need_print()) { ADD_COMMENT("time", time_guard); } - if (nullptr != static_param_.schema_ && static_param_.schema_->is_mv_major_refresh_table()) { - ADD_COMMENT("mv", 1); - } - #undef ADD_COMMENT ObInfoParamBuffer info_allocator; if (OB_SUCCESS == share::g_mp->dag_warning_history_manager()->get_with_param(hash, warning_info, info_allocator)) { @@ -880,7 +873,7 @@ int ObBasicTabletMergeCtx::init_static_desc() { int ret = OB_SUCCESS; static_param_.concurrent_cnt_ = get_concurrent_cnt(); - if (OB_FAIL(static_desc_.init(false/*is_ddl*/, *get_schema(), get_ls_id(), get_tablet_id(), + if (OB_FAIL(static_desc_.init(false/*is_ddl*/, *get_schema(), get_tablet_id(), get_merge_type(), get_snapshot(), static_param_.scn_range_.end_scn_, static_param_.data_version_, @@ -936,16 +929,14 @@ void ObBasicTabletMergeCtx::after_update_tablet_for_major() { int tmp_ret = OB_SUCCESS; if (is_major_merge_type(get_merge_type())) { - const ObLSID &ls_id = get_ls_id(); const ObTabletID &tablet_id = get_tablet_id(); - if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id, true/*need_diagnose*/))) { - LOG_ERROR_RET(tmp_ret, "failed to submit tablet update task to report", K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id, true/*need_diagnose*/))) { + LOG_ERROR_RET(tmp_ret, "failed to submit tablet update task to report", K(tablet_id)); } else if (OB_TMP_FAIL(get_ls()->get_tablet_svr()->update_tablet_report_status(tablet_id))) { - LOG_ERROR_RET(tmp_ret, "failed to update tablet report status", K(ls_id), K(tablet_id)); + LOG_ERROR_RET(tmp_ret, "failed to update tablet report status", K(tablet_id)); } - if (OB_TMP_FAIL(share::g_mp->tenant_medium_checker()->add_tablet_ls(tablet_id, ls_id, get_merge_version()))) { - LOG_WARN_RET(tmp_ret, "failed to add tablet ls for check", K(ls_id), - K(tablet_id), "merge_version", get_merge_version()); + if (OB_TMP_FAIL(share::g_mp->tenant_medium_checker()->add_tablet(tablet_id, get_merge_version()))) { + LOG_WARN_RET(tmp_ret, "failed to add tablet for check", K(tablet_id), "merge_version", get_merge_version()); } } } @@ -976,7 +967,6 @@ int ObBasicTabletMergeCtx::build_update_table_store_param( } param.storage_schema_ = static_param_.schema_; - param.rebuild_seq_ = get_ls_rebuild_seq(); param.ddl_info_.update_with_major_flag_ = false; param.sstable_ = sstable; @@ -1030,24 +1020,14 @@ int ObBasicTabletMergeCtx::try_set_upper_trans_version(blocksstable::ObSSTable & { int ret = OB_SUCCESS; const ObMergeType merge_type = get_inner_table_merge_type(); - const int64_t rebuild_seq = get_ls_rebuild_seq(); // update upper_trans_version for param.sstable_, and then update table store if (is_mini_merge(merge_type) || is_minor_merge(merge_type)) { - // upper_trans_version calculated from ls is invalid when ls is rebuilding, use rebuild_seq to prevent concurrency bug. int tmp_ret = OB_SUCCESS; ObLS *ls = get_ls(); int64_t new_upper_trans_version = INT64_MAX; - int64_t new_rebuild_seq = 0; - bool ls_is_migration = false; if (INT64_MAX != sstable.get_upper_trans_version()) { // all row committed, has set as max_merged_trans_version - } else if (OB_TMP_FAIL(ls->check_ls_migration_status(ls_is_migration, new_rebuild_seq))) { - LOG_WARN("failed to check ls migration status", K(tmp_ret), K(ls_is_migration), K(new_rebuild_seq)); - } else if (ls_is_migration) { - } else if (rebuild_seq != new_rebuild_seq) { - ret = OB_EAGAIN; - LOG_WARN("rebuild seq not same, need retry merge", K(ret), "ls_meta", ls->get_ls_meta(), K(new_rebuild_seq), K(rebuild_seq)); } else if (OB_TMP_FAIL(ObGCUpperTransHelper::try_get_sstable_upper_trans_version(*ls, sstable, new_upper_trans_version))) { LOG_WARN("failed to get new upper_trans_version for sstable", K(tmp_ret), K(sstable)); } else if (INT64_MAX != new_upper_trans_version @@ -1276,7 +1256,7 @@ int ObBasicTabletMergeCtx::cal_major_merge_param( LOG_WARN("found schema changed when compare sstable & schema but progressive merge round is not increasing", KR(ret), K(is_schema_changed), "param", get_dag_param(), KPC(get_schema())); #ifdef ERRSIM - SERVER_EVENT_SYNC_ADD("merge_errsim", "found_schema_changed", "ls_id", get_ls_id(), "tablet_id", get_tablet_id()); + SERVER_EVENT_SYNC_ADD("merge_errsim", "found_schema_changed", "tablet_id", get_tablet_id()); #endif } } @@ -1297,9 +1277,6 @@ int ObBasicTabletMergeCtx::swap_tablet(ObGetMergeTablesResult &get_merge_table_r if (OB_UNLIKELY(!is_major_or_meta_merge_type(get_merge_type()))) { ret = OB_NOT_SUPPORTED; LOG_WARN("other merge type not support swap tablet", KR(ret), "param", get_dag_param()); - } else if (OB_UNLIKELY(!get_tablet()->get_tablet_meta().ha_status_.is_data_status_complete())) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("ha status is not allowed major", KR(ret), KPC(this)); } else { ObTablesHandleArray &tables_handle = get_merge_table_result.handle_; ObProtectedMemtableMgrHandle *protected_handle = NULL; @@ -1402,7 +1379,6 @@ int ObBasicTabletMergeCtx::generate_macro_seq_info(const int64_t task_idx, int64 int ObBasicTabletMergeCtx::init_sstable_merge_history() { int ret = OB_SUCCESS; - static_history_.ls_id_ = get_ls_id(); static_history_.tablet_id_ = get_tablet_id(); static_history_.compaction_scn_ = static_param_.get_compaction_scn(); static_history_.merge_type_ = get_inner_table_merge_type(); diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.h b/src/storage/compaction/ob_basic_tablet_merge_ctx.h index 30c39ece2..7ad8dab9d 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.h @@ -28,6 +28,7 @@ class ObSSTable; } namespace storage { +class ObLS; class ObStorageSchema; class ObTabletHandle; } @@ -63,7 +64,6 @@ struct ObStaticMergeParam final scn_range_.end_scn_.get_val_for_tx() : version_range_.snapshot_version_; } OB_INLINE ObMergeType get_merge_type() const { return dag_param_.merge_type_; } - OB_INLINE const ObLSID &get_ls_id() const { return dag_param_.ls_id_; } OB_INLINE const ObTabletID &get_tablet_id() const { return dag_param_.tablet_id_; } OB_INLINE ObExecMode get_exec_mode() const { return dag_param_.exec_mode_; } int cal_minor_merge_param(const bool has_compaction_filter); @@ -84,8 +84,8 @@ struct ObStaticMergeParam final K_(sstable_logic_seq), K_(tables_handle), K_(is_schema_changed), K_(is_tenant_major_merge), K_(read_base_version), K_(merge_scn), K_(need_parallel_minor_merge), KP_(schema), "multi_version_column_descs_cnt", multi_version_column_descs_.count(), - K_(ls_handle), K_(snapshot_info), K_(is_backfill), K_(tablet_schema_guard), K_(for_unittest), - K_(is_delete_insert_merge), K_(is_ha_compeleted)); + KP_(ls), K_(snapshot_info), K_(is_backfill), K_(tablet_schema_guard), K_(for_unittest), + K_(is_delete_insert_merge), K_(is_restore_complete)); ObTabletMergeDagParam &dag_param_; bool is_full_merge_; // full merge or increment merge @@ -94,16 +94,15 @@ struct ObStaticMergeParam final bool is_tenant_major_merge_; bool is_backfill_; bool is_delete_insert_merge_; - bool is_ha_compeleted_; // only used for delete insert minor merge to control multi version row recycle logic, inited from tablet meta + bool is_restore_complete_; // controls multi-version row recycling while tablet restore is incomplete bool for_unittest_; ObMergeLevel merge_level_; ObAdaptiveMergePolicy::AdaptiveMergeReason merge_reason_; int16_t sstable_logic_seq_; - storage::ObLSHandle ls_handle_; + storage::ObLS *ls_; storage::ObTablesHandleArray tables_handle_; int64_t concurrent_cnt_; uint64_t data_version_; // for major, get from medium_info - int64_t ls_rebuild_seq_; int64_t read_base_version_; // use for major merge int64_t create_snapshot_version_; int64_t start_time_; @@ -252,7 +251,6 @@ struct ObBasicTabletMergeCtx #define PROGRESSIVE_FUNC(var_name) \ int64_t get_##var_name() const { return progressive_merge_mgr_.get_##var_name(); } DAG_PARAM_FUNC(ObMergeType, merge_type); - DAG_PARAM_FUNC(const ObLSID &, ls_id); DAG_PARAM_FUNC(const ObTabletID &, tablet_id); DAG_PARAM_FUNC(int64_t, merge_version); DAG_PARAM_FUNC(ObExecMode, exec_mode); @@ -260,7 +258,6 @@ struct ObBasicTabletMergeCtx STATIC_PARAM_FUNC(bool, is_full_merge); STATIC_PARAM_FUNC(bool, need_parallel_minor_merge); STATIC_PARAM_FUNC(int64_t, read_base_version); - STATIC_PARAM_FUNC(int64_t, ls_rebuild_seq); STATIC_PARAM_FUNC(const storage::ObTablesHandleArray &, tables_handle); STATIC_PARAM_FUNC(const ObTabletMergeDagParam &, dag_param); STATIC_PARAM_FUNC(const SCN &, merge_scn); @@ -274,7 +271,7 @@ struct ObBasicTabletMergeCtx OB_INLINE int64_t get_concurrent_cnt() const { return parallel_merge_ctx_.get_concurrent_cnt(); } OB_INLINE ObMergeType get_inner_table_merge_type() const { return get_is_tenant_major_merge() ? MAJOR_MERGE : get_merge_type(); } OB_INLINE const ObStorageSchema *get_schema() const { return static_param_.schema_; } - OB_INLINE ObLS *get_ls() const { return static_param_.ls_handle_.get_ls(); } + OB_INLINE storage::ObLS *get_ls() const { return static_param_.ls_; } ObTablet *get_tablet() const { return tablet_handle_.get_obj(); } OB_INLINE int64_t get_snapshot() const { return static_param_.version_range_.snapshot_version_; } OB_INLINE const SCN & get_end_scn() const { return static_param_.scn_range_.end_scn_; } diff --git a/src/storage/compaction/ob_batch_exec_dag.h b/src/storage/compaction/ob_batch_exec_dag.h index d200ae38c..94d17b5b1 100644 --- a/src/storage/compaction/ob_batch_exec_dag.h +++ b/src/storage/compaction/ob_batch_exec_dag.h @@ -37,11 +37,10 @@ struct ObBatchExecParam : public share::ObIDagInitParam ObBatchExecParam(const ObBatchExecParamType type); ObBatchExecParam( const ObBatchExecParamType type, - const share::ObLSID &ls_id, const int64_t merge_version, const int64_t batch_size = DEFAULT_BATCH_SIZE); virtual ~ObBatchExecParam() { tablet_info_array_.reset(); } - bool is_valid() const override { return ls_id_.is_valid() && compaction_scn_ > 0 && tablet_info_array_.count() > 0; } + bool is_valid() const override { return compaction_scn_ > 0 && tablet_info_array_.count() > 0; } int assign(const ObBatchExecParam &other); int64_t get_hash() const; int64_t get_task_cnt() const @@ -56,12 +55,11 @@ struct ObBatchExecParam : public share::ObIDagInitParam { tablet_info_array_.reset(); } - VIRTUAL_TO_STRING_KV(K_(param_type), K_(ls_id), K_(compaction_scn), "tablet_id_cnt", tablet_info_array_.count(), K_(tablet_info_array)); + VIRTUAL_TO_STRING_KV(K_(param_type), K_(compaction_scn), "tablet_id_cnt", tablet_info_array_.count(), K_(tablet_info_array)); public: static constexpr int64_t DEFAULT_BATCH_SIZE = 128; static constexpr int64_t DEFAULT_ARRAY_SIZE = 64; ObBatchExecParamType param_type_; - share::ObLSID ls_id_; int64_t compaction_scn_; int64_t batch_size_; common::ObSEArray tablet_info_array_; @@ -172,7 +170,6 @@ class ObBatchExecTask : public share::ObITask template ObBatchExecParam::ObBatchExecParam(const ObBatchExecParamType type) : param_type_(type), - ls_id_(), compaction_scn_(0), batch_size_(DEFAULT_BATCH_SIZE), tablet_info_array_() @@ -183,11 +180,9 @@ ObBatchExecParam::ObBatchExecParam(const ObBatchExecParamType type) template ObBatchExecParam::ObBatchExecParam( const ObBatchExecParamType type, - const share::ObLSID &ls_id, const int64_t merge_version, const int64_t batch_size) : param_type_(type), - ls_id_(ls_id), compaction_scn_(merge_version), batch_size_(batch_size), tablet_info_array_() @@ -205,7 +200,6 @@ int ObBatchExecParam::assign( } else if (OB_FAIL(tablet_info_array_.assign(other.tablet_info_array_))) { STORAGE_LOG(WARN, "failed to copy tablet ids", KR(ret)); } else { - ls_id_ = other.ls_id_; compaction_scn_ = other.compaction_scn_; batch_size_ = other.batch_size_; } @@ -217,7 +211,7 @@ int64_t ObBatchExecParam::get_hash() const { int64_t hash_val = 0; hash_val = common::murmurhash(¶m_type_, sizeof(param_type_), hash_val); - hash_val = common::murmurhash(&ls_id_, sizeof(ls_id_), hash_val); + hash_val = common::murmurhash(&compaction_scn_, sizeof(compaction_scn_), hash_val); return hash_val; } @@ -283,8 +277,7 @@ bool ObBatchExecDag::operator == (const ObIDag &other) const // same } else if (get_type() != other.get_type()) { is_same = false; - } else if ((param_.ls_id_ != static_cast(other).param_.ls_id_) - || (param_.compaction_scn_ != static_cast(other).param_.compaction_scn_)) { + } else if (param_.compaction_scn_ != static_cast(other).param_.compaction_scn_) { is_same = false; } return is_same; @@ -302,7 +295,6 @@ int ObBatchExecDag::fill_info_param( } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - param_.ls_id_.id(), param_.tablet_info_array_.count()))) { STORAGE_LOG(WARN, "failed to fill info param", KR(ret), K(param_)); } @@ -313,8 +305,8 @@ template int ObBatchExecDag::fill_dag_key(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; - if (OB_FAIL(databuff_printf(buf, buf_len, "ls_id=%ld tablet_cnt=%ld", - param_.ls_id_.id(), param_.tablet_info_array_.count()))) { + if (OB_FAIL(databuff_printf(buf, buf_len, "tablet_cnt=%ld", + param_.tablet_info_array_.count()))) { STORAGE_LOG(WARN, "failed to fill dag key", K(param_)); } return ret; @@ -325,7 +317,6 @@ int ObBatchExecDag::init_merge_history() { int ret = OB_SUCCESS; ObMergeStaticInfo &static_history = collector_.merge_history_.static_info_; - static_history.ls_id_ = param_.ls_id_; static_history.tablet_id_ = ObTabletID(ObTabletID::INVALID_TABLET_ID); // mock a special tablet id static_history.compaction_scn_ = param_.compaction_scn_; static_history.merge_type_ = BATCH_EXEC; diff --git a/src/storage/compaction/ob_batch_freeze_tablets_dag.cpp b/src/storage/compaction/ob_batch_freeze_tablets_dag.cpp index 04f660067..78a017ef6 100644 --- a/src/storage/compaction/ob_batch_freeze_tablets_dag.cpp +++ b/src/storage/compaction/ob_batch_freeze_tablets_dag.cpp @@ -45,7 +45,7 @@ int64_t ObBatchFreezeTabletsParam::to_string(char *buf, const int64_t buf_len) c { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(param_type), K_(ls_id), K_(compaction_scn), K_(batch_size), K_(loop_cnt)); + J_KV(K_(param_type), K_(compaction_scn), K_(batch_size), K_(loop_cnt)); if (tablet_info_array_.count() > 0) { J_COMMA(); J_NAME("tablet_info_array"); @@ -95,8 +95,7 @@ bool ObBatchFreezeTabletsDag::operator == (const ObIDag &other) const is_same = false; } else { const ObBatchFreezeTabletsDag &other_dag = static_cast(other); - if ((get_param().ls_id_ != other_dag.get_param().ls_id_) - || (get_param().compaction_scn_ != other_dag.get_param().compaction_scn_)) { + if (get_param().compaction_scn_ != other_dag.get_param().compaction_scn_) { is_same = false; } else { // to ensure dag with same loop_cnt is not same @@ -124,14 +123,10 @@ int ObBatchFreezeTabletsTask::inner_process() int64_t cost_ts = ObTimeUtility::fast_current_time(); const ObBatchFreezeTabletsParam ¶m = base_dag_->get_param(); - ObLSHandle ls_handle; ObLS *ls = nullptr; int64_t weak_read_ts = 0; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null ls", K(ret), K(param)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get single log stream", K(ret), K(param)); } else { weak_read_ts = ls->get_ls_wrs_handler()->get_ls_weak_read_ts().get_val_for_tx(); } @@ -152,8 +147,7 @@ int ObBatchFreezeTabletsTask::inner_process() LOG_WARN_RET(tmp_ret, "get invalid tablet pair", K(cur_pair)); } else if (cur_pair.schedule_merge_scn_ > weak_read_ts) { // no need to force freeze - } else if (OB_TMP_FAIL(share::g_mp->tenant_freezer()->tablet_freeze(param.ls_id_, - cur_pair.tablet_id_, + } else if (OB_TMP_FAIL(share::g_mp->tenant_freezer()->tablet_freeze(cur_pair.tablet_id_, true/*is_sync*/, max_retry_time_us, true,/*need_rewrite_meta*/ @@ -201,10 +195,10 @@ int ObBatchFreezeTabletsTask::schedule_tablet_major_after_freeze( } else if (!tablet->is_data_complete()) { // no need to schedule merge } else if (OB_FAIL(ObTenantTabletScheduler::schedule_merge_dag( - ls.get_ls_id(), *tablet, MEDIUM_MERGE, + *tablet, MEDIUM_MERGE, cur_pair.schedule_merge_scn_, EXEC_MODE_LOCAL))) { if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { - LOG_ERROR("failed to schedule medium merge dag", K(ret), "ls_id", ls.get_ls_id(), K(cur_pair)); + LOG_ERROR("failed to schedule medium merge dag", K(ret), K(cur_pair)); } } return ret; diff --git a/src/storage/compaction/ob_batch_freeze_tablets_dag.h b/src/storage/compaction/ob_batch_freeze_tablets_dag.h index 8cac1f6d7..52ddb043e 100644 --- a/src/storage/compaction/ob_batch_freeze_tablets_dag.h +++ b/src/storage/compaction/ob_batch_freeze_tablets_dag.h @@ -49,10 +49,9 @@ struct ObBatchFreezeTabletsParam : public ObBatchExecParam loop_cnt_(0) {} ObBatchFreezeTabletsParam( - const share::ObLSID &ls_id, const int64_t merge_version, const int64_t loop_cnt = 0) - : ObBatchExecParam(BATCH_FREEZE, ls_id, merge_version, DEFAULT_BATCH_SIZE), + : ObBatchExecParam(BATCH_FREEZE, merge_version, DEFAULT_BATCH_SIZE), loop_cnt_(loop_cnt) {} virtual ~ObBatchFreezeTabletsParam() = default; diff --git a/src/storage/compaction/ob_ckm_error_tablet_info.h b/src/storage/compaction/ob_ckm_error_tablet_info.h index e64bbd3e7..49b0e7e89 100644 --- a/src/storage/compaction/ob_ckm_error_tablet_info.h +++ b/src/storage/compaction/ob_ckm_error_tablet_info.h @@ -16,7 +16,6 @@ #ifndef OB_STORAGE_COMPACTION_CKM_ERROR_TABLET_INFO_H_ #define OB_STORAGE_COMPACTION_CKM_ERROR_TABLET_INFO_H_ #include "common/ob_tablet_id.h" -#include "share/tablet/ob_tablet_info.h" namespace oceanbase { namespace compaction @@ -44,7 +43,6 @@ struct ObCkmErrorStruct int64_t compaction_scn_; }; typedef ObCkmErrorStruct ObCkmErrorTabletInfo; -typedef ObCkmErrorStruct ObCkmErrorTabletLSInfo; } // namespace compaction } // namespace oceanbase diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index 0dae12c9f..20516465e 100644 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -45,7 +45,6 @@ bool ObScheduleSuspectInfo::is_valid() const { bool bret = true; if (OB_UNLIKELY(!is_valid_merge_type(merge_type_) - || !ls_id_.is_valid() || !tablet_id_.is_valid())) { bret = false; } @@ -57,7 +56,6 @@ void ObScheduleSuspectInfo::shallow_copy(ObIDiagnoseInfo *other) ObScheduleSuspectInfo *info = nullptr; if (OB_NOT_NULL(other) && OB_NOT_NULL(info = static_cast(other))) { merge_type_ = info->merge_type_; - ls_id_ = info->ls_id_; tablet_id_ = info->tablet_id_; priority_ = info->priority_; add_time_ = info->add_time_; @@ -506,14 +504,12 @@ int ObScheduleSuspectInfoMgr::add_suspect_info(const int64_t key, ObScheduleSusp * ObCompactionDiagnose implement * */ -#define ADD_DIAGNOSE_INFO(merge_type, ls_id, tablet_id, status, time, ...) \ -SET_DIAGNOSE_INFO((can_add_diagnose_info() ? &info_array_[idx_++] : NULL), normal_, merge_type, ls_id, tablet_id, status, time, __VA_ARGS__) +#define ADD_DIAGNOSE_INFO(merge_type, tablet_id, status, time, ...) \ +SET_DIAGNOSE_INFO((can_add_diagnose_info() ? &info_array_[idx_++] : NULL), normal_, merge_type, tablet_id, status, time, __VA_ARGS__) #define ADD_DIAGNOSE_INFO_FOR_TABLET(merge_type, status, time, ...) \ -ADD_DIAGNOSE_INFO(merge_type, ls_id, tablet_id, status, time, __VA_ARGS__) -#define ADD_DIAGNOSE_INFO_FOR_LS(merge_type, status, time, ...) \ -ADD_DIAGNOSE_INFO(merge_type, ls_id, UNKNOW_TABLET_ID, status, time, __VA_ARGS__) +ADD_DIAGNOSE_INFO(merge_type, tablet_id, status, time, __VA_ARGS__) #define ADD_COMMON_DIAGNOSE_INFO(merge_type, status, time, ...) \ -ADD_DIAGNOSE_INFO(merge_type, UNKNOW_LS_ID, UNKNOW_TABLET_ID, status, time, __VA_ARGS__) +ADD_DIAGNOSE_INFO(merge_type, UNKNOW_TABLET_ID, status, time, __VA_ARGS__) #define ADD_MAJOR_WAIT_SCHEDULE(time, info) \ if (ObTimeUtility::current_time_ns() > time) { \ @@ -527,7 +523,7 @@ ADD_DIAGNOSE_INFO(merge_type, UNKNOW_LS_ID, UNKNOW_TABLET_ID, status, time, __VA "compaction_scn", compaction_scn, \ "tablet_snapshot", tablet.get_snapshot_version(), \ "last_major_scn", last_major_snapshot_version))) { \ - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); \ + LOG_WARN("failed to add diagnose info", K(ret), K(tablet_id)); \ } \ ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE]; \ } @@ -543,7 +539,7 @@ ADD_DIAGNOSE_INFO(merge_type, UNKNOW_LS_ID, UNKNOW_TABLET_ID, status, time, __VA "max_receive_medium_scn", max_sync_medium_scn, \ "tablet_snapshot", tablet.get_snapshot_version(), \ "last_major_scn", last_major_snapshot_version))) { \ - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); \ + LOG_WARN("failed to add diagnose info", K(ret), K(tablet_id)); \ } \ ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MEDIUM_NOT_SCHEDULE]; \ } @@ -646,7 +642,6 @@ int ObCompactionDiagnoseMgr::init( int ObCompactionDiagnoseMgr::diagnose_dag( compaction::ObMergeType merge_type, - ObLSID ls_id, ObTabletID tablet_id, const int64_t merge_version, ObTabletMergeDag &dag, @@ -657,7 +652,6 @@ int ObCompactionDiagnoseMgr::diagnose_dag( ObTabletMergeDagParam param; param.merge_type_ = merge_type; param.merge_version_ = merge_version; - param.ls_id_ = ls_id; param.tablet_id_ = tablet_id; param.skip_get_tablet_ = true; param.is_reserve_mode_ = false; @@ -667,12 +661,12 @@ int ObCompactionDiagnoseMgr::diagnose_dag( } else if (is_minor_merge(merge_type)) { if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->diagnose_minor_exe_dag(&dag, progress))) { if (OB_HASH_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "failed to diagnose minor execute dag", K(ret), K(ls_id), K(tablet_id), K(progress)); + STORAGE_LOG(WARN, "failed to diagnose minor execute dag", K(ret), K(tablet_id), K(progress)); } } } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->diagnose_dag(&dag, progress))) { if (OB_HASH_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "failed to diagnose dag", K(ret), K(ls_id), K(tablet_id), K(progress)); + STORAGE_LOG(WARN, "failed to diagnose dag", K(ret), K(tablet_id), K(progress)); } } if (OB_HASH_NOT_EXIST == ret) { @@ -709,7 +703,6 @@ int ObCompactionDiagnoseMgr::diagnose_all_tablets() int ObCompactionDiagnoseMgr::get_and_set_suspect_info( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -717,9 +710,9 @@ int ObCompactionDiagnoseMgr::get_and_set_suspect_info( char tmp_str[common::OB_DIAGNOSE_INFO_LENGTH] = "\0"; share::ObSuspectInfoType suspect_info_type; - if (OB_FAIL(get_suspect_info(merge_type, ls_id, tablet_id, ret_info, suspect_info_type, tmp_str, sizeof(tmp_str)))) { + if (OB_FAIL(get_suspect_info(merge_type, tablet_id, ret_info, suspect_info_type, tmp_str, sizeof(tmp_str)))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed get suspect info", K(ret), K(ls_id)); + LOG_WARN("failed get suspect info", K(ret), K(tablet_id)); } } else if (OB_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( merge_type, @@ -733,7 +726,6 @@ int ObCompactionDiagnoseMgr::get_and_set_suspect_info( int ObCompactionDiagnoseMgr::get_suspect_info( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id, ObScheduleSuspectInfo &ret_info, share::ObSuspectInfoType &suspect_info_type, @@ -744,7 +736,6 @@ int ObCompactionDiagnoseMgr::get_suspect_info( suspect_info_type = share::ObSuspectInfoType::SUSPECT_INFO_TYPE_MAX; ObScheduleSuspectInfo input_info; input_info.merge_type_ = merge_type; - input_info.ls_id_ = ls_id; input_info.tablet_id_ = tablet_id; ObInfoParamBuffer allocator; // info_param_ will be invalid after return if (OB_FAIL(share::g_mp->schedule_suspect_info_mgr()->get_with_param(input_info.hash(), ret_info, allocator))) { @@ -773,7 +764,7 @@ int ObCompactionDiagnoseMgr::diagnose_tenant( //TODO(mingqiao): check tenant res diagnose_major_flag = true; // step 1: check common suspect info - (void) get_and_set_suspect_info(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID); + (void) get_and_set_suspect_info(MEDIUM_MERGE, UNKNOW_TABLET_ID); // step 2: check if major compaction is paused if (!MERGE_SCHEDULER_PTR->could_major_merge_start()) { @@ -808,18 +799,17 @@ int ObCompactionDiagnoseMgr::diagnose_tenant( //TODO(mingqiao): check tenant res return ret; } -void ObCompactionDiagnoseMgr::diagnose_tenant_ls( +void ObCompactionDiagnoseMgr::diagnose_tenant_merge_state( const bool diagnose_major_flag, const int64_t compaction_scn, const ObLSStatusCache &ls_status) { int tmp_ret = OB_SUCCESS; - // check weak read ts // ls level + // Check the tenant merge state derived from weak read progress. if (diagnose_major_flag && !ls_status.can_merge() && OB_TMP_FAIL(ADD_DIAGNOSE_INFO( MEDIUM_MERGE, - ls_status.ls_id_, UNKNOW_TABLET_ID, ObCompactionDiagnoseInfo::DIA_STATUS_FAILED, ObTimeUtility::fast_current_time(), @@ -829,26 +819,12 @@ void ObCompactionDiagnoseMgr::diagnose_tenant_ls( ls_status.weak_read_ts_.is_valid() ? ls_status.weak_read_ts_.get_val_for_tx() : -1))) { LOG_WARN_RET(tmp_ret, "failed to add dignose info about ls", K(tmp_ret), K(compaction_scn)); } - // check ls suspect info for memtable freezing // ls level - (void) get_and_set_suspect_info(MINI_MERGE, ls_status.ls_id_, UNKNOW_TABLET_ID); + // Check suspect information for memtable freezing. + (void) get_and_set_suspect_info(MINI_MERGE, UNKNOW_TABLET_ID); - // check ls suspect info for ls locality change - if (diagnose_major_flag && ls_status.is_leader_ && share::g_mp->tenant_medium_checker()->locality_cache_empty()) { - if (OB_TMP_FAIL(ADD_DIAGNOSE_INFO( - MEDIUM_MERGE, - ls_status.ls_id_, - UNKNOW_TABLET_ID, - ObCompactionDiagnoseInfo::DIA_STATUS_WARN, - ObTimeUtility::fast_current_time(), - "maybe bad case", - "ls leader is not in ls locality"))) { - LOG_WARN_RET(tmp_ret, "failed to add dignose info about ls leader not in locality cache", K(tmp_ret)); - } - } } void ObCompactionDiagnoseMgr::diagnose_failed_report_task( - const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t compaction_scn) { @@ -858,13 +834,13 @@ void ObCompactionDiagnoseMgr::diagnose_failed_report_task( ObScheduleSuspectInfo ret_info; char tmp_str[common::OB_DIAGNOSE_INFO_LENGTH] = "\0"; share::ObSuspectInfoType suspect_info_type; - if (OB_TMP_FAIL(get_suspect_info(MEDIUM_MERGE, ls_id, tablet_id, ret_info, suspect_info_type, tmp_str, sizeof(tmp_str)))) { - LOG_WARN_RET(tmp_ret, "failed to get suspect info", K(tmp_ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(get_suspect_info(MEDIUM_MERGE, tablet_id, ret_info, suspect_info_type, tmp_str, sizeof(tmp_str)))) { + LOG_WARN_RET(tmp_ret, "failed to get suspect info", K(tmp_ret), K(tablet_id)); } else if (is_compaction_report_info(suspect_info_type)) { - if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->check_exist(ls_id, tablet_id, exist))) { - LOG_WARN_RET(tmp_ret, "failed to check task exist", K(tmp_ret), K(ls_id), K(tablet_id)); - } else if (!exist && OB_TMP_FAIL(share::g_mp->tablet_table_updater()->check_processing_exist(ls_id, tablet_id, processing))) { - LOG_WARN_RET(tmp_ret, "failed to check processing task exist", K(tmp_ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->check_exist(tablet_id, exist))) { + LOG_WARN_RET(tmp_ret, "failed to check task exist", K(tmp_ret), K(tablet_id)); + } else if (!exist && OB_TMP_FAIL(share::g_mp->tablet_table_updater()->check_processing_exist(tablet_id, processing))) { + LOG_WARN_RET(tmp_ret, "failed to check processing task exist", K(tmp_ret), K(tablet_id)); } } @@ -894,7 +870,6 @@ void ObCompactionDiagnoseMgr::diagnose_existing_report_task() FOREACH(iter, waiting_tasks) { if (OB_TMP_FAIL(ADD_DIAGNOSE_INFO( MEDIUM_MERGE, - iter->get_ls_id(), iter->get_tablet_id(), ObCompactionDiagnoseInfo::DIA_STATUS_FAILED, ObTimeUtility::fast_current_time(), @@ -905,7 +880,6 @@ void ObCompactionDiagnoseMgr::diagnose_existing_report_task() FOREACH(iter, processing_tasks) { if (OB_TMP_FAIL(ADD_DIAGNOSE_INFO( MEDIUM_MERGE, - iter->get_ls_id(), iter->get_tablet_id(), ObCompactionDiagnoseInfo::DIA_STATUS_FAILED, ObTimeUtility::fast_current_time(), @@ -946,18 +920,6 @@ void ObCompactionDiagnoseMgr::diagnose_count_info() } } -struct ObDiagnoseTabletComparator final { -public: - ObDiagnoseTabletComparator(int &sort_ret) - : result_code_(sort_ret) - {} - bool operator()(const ObDiagnoseTablet &lhs, const ObDiagnoseTablet &rhs) - { - return lhs.ls_id_ < rhs.ls_id_; - } - int &result_code_; -}; - int ObCompactionDiagnoseMgr::diagnose_tenant_tablet() { int ret = OB_SUCCESS; @@ -983,61 +945,53 @@ int ObCompactionDiagnoseMgr::diagnose_tenant_tablet() DiagnoseTabletArray tablet_array; if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->get_diagnose_tablets(diagnose_tablets))) { LOG_WARN("failed to get all diagnose tablets", K(tmp_ret)); - } else { - ObDiagnoseTabletComparator cmp(ret); - ob_sort(diagnose_tablets.begin(), diagnose_tablets.end(), cmp); } - ObLSHandle ls_handle; ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObScheduleTabletFunc func(compaction_scn); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(func.diagnose_init(ls))) { + LOG_WARN("failed to initialize diagnose ls", K(ret)); + } else { + IGNORE_RETURN diagnose_tenant_merge_state(diagnose_major_flag, compaction_scn, func.get_ls_status()); + } ARRAY_FOREACH_NORET(diagnose_tablets, idx) { normal_ = true; bool need_merge = false; bool weak_read_ts_ready = false; const ObDiagnoseTablet &diagnose_tablet = diagnose_tablets.at(idx); - const ObLSID &ls_id = diagnose_tablet.ls_id_; const ObTabletID &tablet_id = diagnose_tablet.tablet_id_; - if (IS_UNKNOW_LS_ID(ls_id) || IS_UNKNOW_TABLET_ID(tablet_id)) { + if (OB_FAIL(ret) || IS_UNKNOW_TABLET_ID(tablet_id)) { continue; - } else if (func.get_ls_status().ls_id_ == ls_id) { - // do nothing - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, storage::ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (FALSE_IT(ls = ls_handle.get_ls())) { - } else if (OB_FAIL(func.diagnose_switch_ls(ls_handle))) { - LOG_WARN("failed to diagnose switch ls", K(ret), K(ls_id)); - } else { - IGNORE_RETURN diagnose_tenant_ls(diagnose_major_flag, compaction_scn, func.get_ls_status()); } if (OB_FAIL(ret)) { } else if (!func.get_ls_status().can_merge()) { // do nothing - } else if (OB_TMP_FAIL(ls->get_tablet(tablet_id, - tablet_handle, - ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT))) { - LOG_WARN("failed to get tablet", K(tmp_ret), K(ls_id), K(tablet_id)); + } else if (OB_TMP_FAIL(ls->get_tablet( + tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT))) { + LOG_WARN("failed to get tablet", K(tmp_ret), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { tmp_ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet handle", K(tmp_ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid tablet handle", K(tmp_ret), K(tablet_handle)); } else if (FALSE_IT(tablet = tablet_handle.get_obj())) { } else { if (diagnose_major_flag) { if (OB_TMP_FAIL(func.diagnose_switch_tablet(*ls, *tablet))) { - LOG_WARN("failed to get switch tablet", K(tmp_ret), K(compaction_scn), K(ls_id)); - } else if (OB_TMP_FAIL(diagnose_tablet_major_merge(compaction_scn, ls_id, func.get_tablet_status(), *tablet))) { - LOG_WARN("failed to get diagnose major merge", K(tmp_ret), K(compaction_scn), K(ls_id)); + LOG_WARN("failed to get switch tablet", K(tmp_ret), K(compaction_scn)); + } else if (OB_TMP_FAIL(diagnose_tablet_major_merge(compaction_scn, func.get_tablet_status(), *tablet))) { + LOG_WARN("failed to get diagnose major merge", K(tmp_ret), K(compaction_scn)); } } - if (OB_TMP_FAIL(diagnose_tablet_medium_merge(diagnose_major_flag, compaction_scn, ls_id, *tablet))) { + if (OB_TMP_FAIL(diagnose_tablet_medium_merge(diagnose_major_flag, compaction_scn, *tablet))) { LOG_WARN("failed to get diagnose medium merge", K(tmp_ret)); } - if (OB_TMP_FAIL(diagnose_tablet_mini_merge(ls_id, *tablet))) { + if (OB_TMP_FAIL(diagnose_tablet_mini_merge(*tablet))) { LOG_WARN("failed to get diagnose mini merge", K(tmp_ret)); } - if (OB_TMP_FAIL(diagnose_tablet_minor_merge(ls_id, *tablet))) { + if (OB_TMP_FAIL(diagnose_tablet_minor_merge(*tablet))) { LOG_WARN("failed to get diagnose minor merge", K(tmp_ret)); } } @@ -1118,9 +1072,9 @@ int ObCompactionDiagnoseMgr::do_tenant_major_merge_diagnose( LOG_WARN("invalid argument", KR(ret), KP(major_freeze_service)); } else if (major_freeze_service->is_paused()) { // major service in RS is paused, should not show suspect info - DEL_SUSPECT_INFO(MAJOR_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_RS_MAJOR_MERGE); + DEL_SUSPECT_INFO(MAJOR_MERGE, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_RS_MAJOR_MERGE); } else { - (void) get_and_set_suspect_info(MAJOR_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID); // get RS schedule suspect info + (void) get_and_set_suspect_info(MAJOR_MERGE, UNKNOW_TABLET_ID); // get RS schedule suspect info SMART_VARS_2((ObArray, uncompacted_tablets), (ObArray, uncompacted_table_ids)) { if (OB_FAIL(major_freeze_service->get_uncompacted_tablets(uncompacted_tablets, uncompacted_table_ids))) { @@ -1147,7 +1101,7 @@ int ObCompactionDiagnoseMgr::add_uncompacted_tablet_to_diagnose( ? "CHECKSUM_ERROR" : (compaction_scn_not_valid ? "compaction_scn_not_update" : "report_scn_not_update"); if (OB_FAIL(ADD_DIAGNOSE_INFO( - MAJOR_MERGE, uncompacted_tablets.at(i).get_ls_id(), + MAJOR_MERGE, uncompacted_tablets.at(i).get_tablet_id(), ObCompactionDiagnoseInfo::DIA_STATUS_RS_UNCOMPACTED, ObTimeUtility::fast_current_time(), "server", @@ -1176,9 +1130,7 @@ void ObCompactionDiagnoseMgr::add_uncompacted_table_ids_to_diagnose(const ObIArr } } -int ObCompactionDiagnoseMgr::diagnose_tablet_mini_merge( - const ObLSID &ls_id, - ObTablet &tablet) +int ObCompactionDiagnoseMgr::diagnose_tablet_mini_merge(ObTablet &tablet) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -1206,18 +1158,17 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_mini_merge( if (diagnose_flag) { if (OB_TMP_FAIL(diagnose_tablet_merge( MINI_MERGE, - ls_id, tablet))) { - LOG_WARN("diagnose failed", K(tmp_ret), K(ls_id), "tablet_id", tablet.get_tablet_meta().tablet_id_, KPC(latest_sstable)); + LOG_WARN("diagnose failed", K(tmp_ret), "tablet_id", tablet.get_tablet_meta().tablet_id_, KPC(latest_sstable)); } } else { - (void) get_and_set_suspect_info(MINI_MERGE, ls_id, tablet_id); + (void) get_and_set_suspect_info(MINI_MERGE, tablet_id); } } return ret; } -int ObCompactionDiagnoseMgr::diagnose_tablet_minor_merge(const ObLSID &ls_id, ObTablet &tablet) +int ObCompactionDiagnoseMgr::diagnose_tablet_minor_merge(ObTablet &tablet) { int ret = OB_SUCCESS; int64_t minor_compact_trigger = ObPartitionMergePolicy::DEFAULT_MINOR_COMPACT_TRIGGER; @@ -1229,9 +1180,8 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_minor_merge(const ObLSID &ls_id, Ob if (tablet.get_minor_table_count() >= minor_compact_trigger) { if (OB_FAIL(diagnose_tablet_merge( MINOR_MERGE, - ls_id, tablet))) { - LOG_WARN("diagnose failed", K(ret), K(ls_id), "tablet_id", tablet.get_tablet_meta().tablet_id_); + LOG_WARN("diagnose failed", K(ret), "tablet_id", tablet.get_tablet_meta().tablet_id_); } } return ret; @@ -1239,7 +1189,6 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_minor_merge(const ObLSID &ls_id, Ob int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( const int64_t compaction_scn, - const ObLSID &ls_id, const ObTabletStatusCache &tablet_status, ObTablet &tablet) { @@ -1248,9 +1197,6 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const int64_t last_major_snapshot_version = tablet.get_last_major_snapshot_version(); int64_t max_sync_medium_scn = 0; - ObArenaAllocator temp_allocator("GetSSchema", OB_MALLOC_NORMAL_BLOCK_SIZE); - ObStorageSchema *storage_schema = nullptr; - bool is_mv_major_refresh_tablet = false; if (tablet_id.is_ls_inner_tablet()) { // do nothing } else if (!tablet_status.can_merge()) { @@ -1261,20 +1207,17 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( // tablet status has null medium list, cannot check sycn medium scn } else if (OB_FAIL(ObMediumCompactionScheduleFunc::get_max_sync_medium_scn( tablet, *tablet_status.medium_list(), max_sync_medium_scn))) { - LOG_WARN("failed to get max sync medium scn", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(tablet.load_storage_schema(temp_allocator, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret), K(tablet)); - } else if (FALSE_IT(is_mv_major_refresh_tablet = storage_schema->is_mv_major_refresh())) { + LOG_WARN("failed to get max sync medium scn", K(ret), K(tablet_id)); } else if (tablet_status.tablet_merge_finish()) { - diagnose_failed_report_task(ls_id, tablet_id, compaction_scn); + diagnose_failed_report_task(tablet_id, compaction_scn); } else { - LOG_TRACE("diagnose tablet major merge", K(ls_id), K(tablet_id), K(compaction_scn), K(max_sync_medium_scn), K(last_major_snapshot_version)); + LOG_TRACE("diagnose tablet major merge", K(tablet_id), K(compaction_scn), K(max_sync_medium_scn), K(last_major_snapshot_version)); if (max_sync_medium_scn < compaction_scn) { // max_sync_medium_scn > last_major_snapshot_version means last compaction is not finished, // this will be diagnosed in diagnose_tablet_medium_merge if (max_sync_medium_scn == last_major_snapshot_version) { // now last compaction finish - if (OB_HASH_NOT_EXIST == get_and_set_suspect_info(MEDIUM_MERGE, ls_id, tablet_id)) { + if (OB_HASH_NOT_EXIST == get_and_set_suspect_info(MEDIUM_MERGE, tablet_id)) { if (ObTabletStatusCache::DIAGNOSE_NORMAL != tablet_status.get_new_round_state()) { ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL * 2, ObTabletStatusCache::new_round_state_to_str(tablet_status.get_new_round_state())); @@ -1287,20 +1230,12 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( } else if (tablet.get_snapshot_version() < compaction_scn) { // wait mini compaction or tablet freeze const char* info = "major wait for freeze"; ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); - } else if (is_mv_major_refresh_tablet && - ObBasicMergeScheduler::INIT_COMPACTION_SCN == last_major_snapshot_version) { - if (OB_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( - MEDIUM_MERGE, ObCompactionDiagnoseInfo::DIA_STATUS_NOT_SCHEDULE, - ObTimeUtility::fast_current_time(), "current_status", "wait for materialized view creation"))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); - } } if (OB_TMP_FAIL(diagnose_tablet_merge( MEDIUM_MERGE, - ls_id, tablet, compaction_scn))) { - LOG_WARN("diagnose failed", K(tmp_ret), K(ls_id), K(tablet_id), K(compaction_scn)); + LOG_WARN("diagnose failed", K(tmp_ret), K(tablet_id), K(compaction_scn)); } } return ret; @@ -1309,7 +1244,6 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( const bool diagnose_major_flag, const int64_t compaction_scn, - const ObLSID &ls_id, ObTablet &tablet) { int ret = OB_SUCCESS; @@ -1327,7 +1261,7 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( tablet, *medium_list, max_sync_medium_scn))) { LOG_WARN("failed to get max sync medium scn", K(ret), KPC(medium_list)); } else { - LOG_TRACE("diagnose tablet medium merge", K(ls_id), K(tablet_id), K(diagnose_major_flag), K(max_sync_medium_scn), K(compaction_scn), K(last_major_snapshot_version)); + LOG_TRACE("diagnose tablet medium merge", K(tablet_id), K(diagnose_major_flag), K(max_sync_medium_scn), K(compaction_scn), K(last_major_snapshot_version)); if (!diagnose_major_flag || (diagnose_major_flag && max_sync_medium_scn < compaction_scn)) { if (max_sync_medium_scn > last_major_snapshot_version) { if (tablet.get_snapshot_version() < max_sync_medium_scn) { // wait mini compaction or tablet freeze @@ -1338,10 +1272,9 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( ADD_MEDIUM_WAIT_SCHEDULE(max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); } else if (OB_TMP_FAIL(diagnose_tablet_merge( MEDIUM_MERGE, - ls_id, tablet, max_sync_medium_scn))) { - LOG_WARN("diagnose failed", K(tmp_ret), K(ls_id), K(tablet_id), K(max_sync_medium_scn), K(last_major_snapshot_version)); + LOG_WARN("diagnose failed", K(tmp_ret), K(tablet_id), K(max_sync_medium_scn), K(last_major_snapshot_version)); } } } @@ -1351,7 +1284,6 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( int ObCompactionDiagnoseMgr::diagnose_row_store_dag( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t compaction_scn) { @@ -1369,14 +1301,14 @@ int ObCompactionDiagnoseMgr::diagnose_row_store_dag( } if (OB_ISNULL(dag)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to diagnose dag", K(ret), K(ls_id), K(tablet_id), K(merge_type)); + LOG_WARN("failed to diagnose dag", K(ret), K(tablet_id), K(merge_type)); } else { ObDiagnoseTabletCompProgress progress; - if (OB_FAIL(diagnose_dag(merge_type, ls_id, tablet_id, compaction_scn, *dag, progress))) { + if (OB_FAIL(diagnose_dag(merge_type, tablet_id, compaction_scn, *dag, progress))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to diagnose dag", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(diagnose_no_dag(dag->hash(), merge_type, ls_id, tablet_id, compaction_scn))) { - LOG_WARN("failed to diagnose no dag", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to diagnose dag", K(ret), K(tablet_id)); + } else if (OB_FAIL(diagnose_no_dag(dag->hash(), merge_type, tablet_id, compaction_scn))) { + LOG_WARN("failed to diagnose no dag", K(ret), K(tablet_id)); } } else if (progress.is_valid()) { // dag exist, means compaction is running // check progress is normal @@ -1387,10 +1319,10 @@ int ObCompactionDiagnoseMgr::diagnose_row_store_dag( ObTimeUtility::fast_current_time(), "current_status", "dag may hang", "merge_progress", progress))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id), K(progress)); + LOG_WARN("failed to add diagnose info", K(ret), K(tablet_id), K(progress)); } - } else if (OB_FAIL(diagnose_no_dag(dag->hash(), merge_type, ls_id, tablet_id, compaction_scn))) { - LOG_WARN("failed to dagnose no dag", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(diagnose_no_dag(dag->hash(), merge_type, tablet_id, compaction_scn))) { + LOG_WARN("failed to dagnose no dag", K(ret), K(tablet_id)); } } return ret; @@ -1398,14 +1330,13 @@ int ObCompactionDiagnoseMgr::diagnose_row_store_dag( int ObCompactionDiagnoseMgr::diagnose_tablet_merge( const ObMergeType merge_type, - const ObLSID ls_id, ObTablet &tablet, const int64_t compaction_scn) { int ret = OB_SUCCESS; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; - if (OB_FAIL(diagnose_row_store_dag(merge_type, ls_id, tablet_id, compaction_scn))) { - LOG_WARN("failed to diagnose merge dag", K(ret), K(ls_id), K(tablet_id), K(merge_type), K(compaction_scn)); + if (OB_FAIL(diagnose_row_store_dag(merge_type, tablet_id, compaction_scn))) { + LOG_WARN("failed to diagnose merge dag", K(ret), K(tablet_id), K(merge_type), K(compaction_scn)); } return ret; } @@ -1413,7 +1344,6 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_merge( int ObCompactionDiagnoseMgr::get_suspect_and_warning_info( const int64_t dag_key, const ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, ObScheduleSuspectInfo &info, ObSuspectInfoType &suspect_type, @@ -1428,13 +1358,12 @@ int ObCompactionDiagnoseMgr::get_suspect_and_warning_info( ObInfoParamBuffer allocator; compaction::ObMergeDagHash dag_hash; dag_hash.merge_type_ = merge_type; - dag_hash.ls_id_ = ls_id; dag_hash.tablet_id_ = tablet_id; if (OB_FAIL(share::g_mp->schedule_suspect_info_mgr()->get_with_param(dag_hash.inner_hash(), info, allocator))) { if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("failed to get suspect info", K(ret), K(dag_hash)); } else { // no schedule suspect info - LOG_TRACE("no schedule suspect info", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("no schedule suspect info", K(ret), K(tablet_id)); info.info_param_ = nullptr; allocator.reuse(); char tmp_str[common::OB_DAG_WARNING_INFO_LENGTH] = "\0"; @@ -1484,7 +1413,6 @@ int ObCompactionDiagnoseMgr::get_suspect_and_warning_info( int ObCompactionDiagnoseMgr::diagnose_no_dag( const int64_t dag_key, const ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, const int64_t compaction_scn) { @@ -1492,10 +1420,10 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( ObScheduleSuspectInfo info; bool add_schedule_info = false; ObSuspectInfoType suspect_type = SUSPECT_INFO_TYPE_MAX; - LOG_TRACE("diagnose_no_dag", K(ret), K(merge_type), K(ls_id), K(tablet_id), K(compaction_scn)); + LOG_TRACE("diagnose_no_dag", K(ret), K(merge_type), K(tablet_id), K(compaction_scn)); char tmp_str[common::OB_DIAGNOSE_INFO_LENGTH] = "\0"; - if (OB_FAIL(get_suspect_and_warning_info(dag_key, merge_type, ls_id, tablet_id, info, suspect_type, tmp_str, sizeof(tmp_str)))) { - LOG_WARN("failed to get suspect and warning info", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(get_suspect_and_warning_info(dag_key, merge_type, tablet_id, info, suspect_type, tmp_str, sizeof(tmp_str)))) { + LOG_WARN("failed to get suspect and warning info", K(ret), K(tablet_id)); } else if (!info.is_valid()) { // do nothing } else if (is_medium_merge(merge_type)) { @@ -1503,12 +1431,12 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( ret = OB_ERR_UNEXPECTED; LOG_WARN("merge version or freeze ts is invalid", K(ret), K(compaction_scn)); } else { - LOG_INFO("diagnose major", K(ret), K(ls_id), K(tablet_id), "merge_type", merge_type_to_str(merge_type)); + LOG_INFO("diagnose major", K(ret), K(tablet_id), "merge_type", merge_type_to_str(merge_type)); ObDiagnoseTabletCompProgress progress; ObTabletMiniMergeDag mini_dag; - if (OB_FAIL(diagnose_dag(MINI_MERGE, ls_id, tablet_id, ObVersionRange::MIN_VERSION, mini_dag, progress))) { + if (OB_FAIL(diagnose_dag(MINI_MERGE, tablet_id, ObVersionRange::MIN_VERSION, mini_dag, progress))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to init dag", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to init dag", K(ret), K(tablet_id)); } else { add_schedule_info = true; ret = OB_SUCCESS; @@ -1521,7 +1449,7 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( ObTimeUtility::fast_current_time(), "current_status", "wait for mini merge", "mini_merge_progress", progress))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id), K(progress)); + LOG_WARN("failed to add diagnose info", K(ret), K(tablet_id), K(progress)); } } else { // no running mini dag add_schedule_info = true; @@ -1539,7 +1467,7 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( ObCompactionDiagnoseInfo::DIA_STATUS_NOT_SCHEDULE, info.add_time_, "schedule_suspect_info", tmp_str))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id), K(info)); + LOG_WARN("failed to add diagnose info", K(ret), K(tablet_id), K(info)); } } ++suspect_tablet_count_[suspect_type]; diff --git a/src/storage/compaction/ob_compaction_diagnose.h b/src/storage/compaction/ob_compaction_diagnose.h index b3548498d..aa8c4db6e 100644 --- a/src/storage/compaction/ob_compaction_diagnose.h +++ b/src/storage/compaction/ob_compaction_diagnose.h @@ -183,7 +183,7 @@ struct ObScheduleSuspectInfo : public ObIDiagnoseInfo, public ObMergeDagHash bool is_valid() const; virtual void shallow_copy(ObIDiagnoseInfo *other) override; virtual int64_t get_hash() const override; - TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(ls_id), K_(tablet_id), K_(add_time), K_(hash)); + TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(tablet_id), K_(add_time), K_(hash)); int64_t add_time_; int64_t hash_; @@ -383,7 +383,6 @@ struct ObCompactionDiagnoseInfo { ObCompactionDiagnoseInfo() : merge_type_(), - ls_id_(0), tablet_id_(0), timestamp_(0), status_(DIA_STATUS_MAX) @@ -400,11 +399,10 @@ struct ObCompactionDiagnoseInfo }; const static char *ObDiagnoseStatusStr[DIA_STATUS_MAX]; static const char * get_diagnose_status_str(ObDiagnoseStatus status); - TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(ls_id), K_(tablet_id), + TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(tablet_id), "status", get_diagnose_status_str(status_), K_(timestamp), K_(diagnose_info)); compaction::ObMergeType merge_type_; - int64_t ls_id_; int64_t tablet_id_; int64_t timestamp_; ObDiagnoseStatus status_; @@ -433,13 +431,12 @@ class ObCompactionDiagnoseMgr int diagnose_tenant( bool &diagnose_major_flag, int64_t &compaction_scn); - void diagnose_tenant_ls( + void diagnose_tenant_merge_state( const bool diagnose_major_flag, const int64_t compaction_scn, const ObLSStatusCache &ls_status); // diagnose failed report task void diagnose_failed_report_task( - const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t compaction_scn); // diagnose waiting or processing report task @@ -456,7 +453,6 @@ class ObCompactionDiagnoseMgr } static int diagnose_dag( const compaction::ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, const int64_t merge_version, ObTabletMergeDag &dag, @@ -468,38 +464,32 @@ class ObCompactionDiagnoseMgr } private: - int diagnose_tablet_mini_merge(const ObLSID &ls_id, ObTablet &tablet); - int diagnose_tablet_minor_merge(const ObLSID &ls_id, ObTablet &tablet); + int diagnose_tablet_mini_merge(ObTablet &tablet); + int diagnose_tablet_minor_merge(ObTablet &tablet); int diagnose_tablet_major_merge( const int64_t compaction_scn, - const ObLSID &ls_id, const ObTabletStatusCache &tablet_status, ObTablet &tablet); int diagnose_tablet_medium_merge( const bool diagnose_major_flag, const int64_t compaction_scn, - const ObLSID &ls_id, ObTablet &tablet); int diagnose_tablet_merge( const ObMergeType type, - const ObLSID ls_id, ObTablet &tablet, const int64_t merge_version = ObVersionRange::MIN_VERSION); int diagnose_row_store_dag( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t merge_version = ObVersionRange::MIN_VERSION); int diagnose_no_dag( const int64_t dag_key, const ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, const int64_t compaction_scn); int get_suspect_and_warning_info( const int64_t dag_key, const ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, ObScheduleSuspectInfo &info, ObSuspectInfoType &suspect_type, @@ -510,11 +500,9 @@ class ObCompactionDiagnoseMgr OB_INLINE bool can_add_diagnose_info() { return idx_ < max_cnt_; } int get_and_set_suspect_info( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id); int get_suspect_info( const ObMergeType merge_type, - const ObLSID &ls_id, const ObTabletID &tablet_id, ObScheduleSuspectInfo &ret_info, share::ObSuspectInfoType &suspect_info_type, @@ -569,17 +557,14 @@ class ObCompactionDiagnoseIterator bool is_opened_; }; -#define UNKNOW_LS_ID ObLSID(INT64_MAX) -#define IS_UNKNOW_LS_ID(ls_id) (ObLSID(INT64_MAX) == ls_id) #define UNKNOW_TABLET_ID ObTabletID(INT64_MAX) #define IS_UNKNOW_TABLET_ID(tablet_id) (ObTabletID(INT64_MAX) == tablet_id) -#define DEL_SUSPECT_INFO(type, ls_id, tablet_id, diagnose_type) \ +#define DEL_SUSPECT_INFO(type, tablet_id, diagnose_type) \ { \ int tmp_ret = OB_SUCCESS; \ compaction::ObMergeDagHash dag_hash; \ dag_hash.merge_type_ = type; \ - dag_hash.ls_id_ = ls_id; \ dag_hash.tablet_id_ = tablet_id; \ int64_t hash_value = dag_hash.inner_hash(); \ if (OB_TMP_FAIL(share::g_mp->schedule_suspect_info_mgr() \ @@ -589,11 +574,11 @@ class ObCompactionDiagnoseIterator K(tmp_ret), K(dag_hash)); \ } \ } else if (OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr() \ - ->delete_diagnose_tablet(ls_id, tablet_id, \ + ->delete_diagnose_tablet(tablet_id, \ diagnose_type))) { \ if (OB_HASH_NOT_EXIST != tmp_ret) { \ LOG_WARN_RET(tmp_ret, "failed to delete diagnose tablet", K(tmp_ret), \ - K(ls_id), K(tablet_id)); \ + K(tablet_id)); \ } \ } else { \ STORAGE_LOG(DEBUG, "success to delete suspect info", K(tmp_ret), \ @@ -605,7 +590,7 @@ class ObCompactionDiagnoseIterator template \ int SET_DIAGNOSE_INFO(ObCompactionDiagnoseInfo *diagnose_info, bool &is_normal, \ compaction::ObMergeType type, \ - const ObLSID ls_id, const ObTabletID tablet_id, \ + const ObTabletID tablet_id, \ ObCompactionDiagnoseInfo::ObDiagnoseStatus status, \ const int64_t timestamp, \ LOG_PARAMETER_KV##n) \ @@ -615,8 +600,6 @@ class ObCompactionDiagnoseIterator is_normal = false; \ if (OB_NOT_NULL(diagnose_info)) { \ diagnose_info->merge_type_ = type; \ - diagnose_info->ls_id_ = ls_id.id(); \ - \ diagnose_info->tablet_id_ = tablet_id.id(); \ diagnose_info->status_ = status; \ diagnose_info->timestamp_ = timestamp; \ @@ -695,17 +678,14 @@ DEFINE_DIAGNOSE_PRINT_KV(5) info_param->param_int_[6] = param_int7; #define ADD_COMMON_SUSPECT_INFO(merge_type, diagnose_type, info_type, ...) \ -ADD_SUSPECT_INFO(merge_type, diagnose_type, UNKNOW_LS_ID, UNKNOW_TABLET_ID, info_type, __VA_ARGS__) - -#define ADD_SUSPECT_LS_INFO(merge_type, diagnose_type, ls_id, info_type, ...) \ -ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, __VA_ARGS__) +ADD_SUSPECT_INFO(merge_type, diagnose_type, UNKNOW_TABLET_ID, info_type, __VA_ARGS__) #define INFO_PARAM_STR_LENGTH(n) (n * compaction::OB_DIAGNOSE_INFO_PARAM_STR_LENGTH) #define DEFINE_SUSPECT_INFO_ADD(n_int) \ template \ int ADD_SUSPECT_INFO(compaction::ObMergeType type, share::ObDiagnoseTabletType diagnose_type, \ - const ObLSID ls_id, const ObTabletID tablet_id, ObSuspectInfoType info_type, \ + const ObTabletID tablet_id, ObSuspectInfoType info_type, \ INFO_PARAM_INT##n_int) \ { \ int64_t __pos = 0; \ @@ -713,7 +693,6 @@ ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, compaction::ObScheduleSuspectInfo info; \ info.priority_ = static_cast(OB_SUSPECT_INFO_TYPES[info_type].priority); \ info.merge_type_ = type; \ - info.ls_id_ = ls_id; \ info.tablet_id_ = tablet_id; \ info.add_time_ = ObTimeUtility::fast_current_time(); \ info.hash_ = info.hash(); \ @@ -725,8 +704,8 @@ ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, info.info_param_ = info_param; \ if (OB_FAIL(share::g_mp->schedule_suspect_info_mgr()->add_suspect_info(info.hash(), info))) { \ STORAGE_LOG(WARN, "failed to add suspect info", K(ret), K(info)); \ - } else if (OB_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(ls_id, tablet_id, diagnose_type))) { \ - STORAGE_LOG(WARN, "failed to add diagnose tablet", K(ret), K(ls_id), K(tablet_id)); \ + } else if (OB_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(tablet_id, diagnose_type))) { \ + STORAGE_LOG(WARN, "failed to add diagnose tablet", K(ret), K(tablet_id)); \ } else { \ STORAGE_LOG(DEBUG, "success to add suspect info", K(ret), K(info), K(info_type), \ "info_type_str", OB_SUSPECT_INFO_TYPES[info_type].info_str, K(diagnose_type)); \ @@ -740,7 +719,7 @@ ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, #define DEFINE_SUSPECT_INFO_ADD_EXTRA(n, n_int) \ template \ int ADD_SUSPECT_INFO(compaction::ObMergeType type, share::ObDiagnoseTabletType diagnose_type, \ - const ObLSID ls_id, const ObTabletID tablet_id, ObSuspectInfoType info_type, \ + const ObTabletID tablet_id, ObSuspectInfoType info_type, \ INFO_PARAM_INT##n_int, LOG_PARAMETER_KV##n) \ { \ int64_t __pos = 0; \ @@ -748,7 +727,6 @@ ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, compaction::ObScheduleSuspectInfo info; \ info.priority_ = static_cast(OB_SUSPECT_INFO_TYPES[info_type].priority); \ info.merge_type_ = type; \ - info.ls_id_ = ls_id; \ info.tablet_id_ = tablet_id; \ info.add_time_ = ObTimeUtility::fast_current_time(); \ info.hash_ = info.hash(); \ @@ -765,8 +743,8 @@ ADD_SUSPECT_INFO(merge_type, diagnose_type, ls_id, UNKNOW_TABLET_ID, info_type, STORAGE_LOG(WARN, "fail to fill parameter kv into info param", K(ret)); \ } else if (OB_FAIL(share::g_mp->schedule_suspect_info_mgr()->add_suspect_info(info.hash(), info))) { \ STORAGE_LOG(WARN, "failed to add suspect info", K(ret), K(info)); \ - } else if (OB_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(ls_id, tablet_id, diagnose_type))) { \ - STORAGE_LOG(WARN, "failed to add diagnose tablet", K(ret), K(ls_id), K(tablet_id)); \ + } else if (OB_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(tablet_id, diagnose_type))) { \ + STORAGE_LOG(WARN, "failed to add diagnose tablet", K(ret), K(tablet_id)); \ } else { \ STORAGE_LOG(DEBUG, "success to add suspect info", K(ret), K(info), K(info_type), \ "info_type_str", OB_SUSPECT_INFO_TYPES[info_type].info_str, K(diagnose_type)); \ diff --git a/src/storage/compaction/ob_compaction_schedule_iterator.cpp b/src/storage/compaction/ob_compaction_schedule_iterator.cpp index 11f3b6732..dbf25bd22 100644 --- a/src/storage/compaction/ob_compaction_schedule_iterator.cpp +++ b/src/storage/compaction/ob_compaction_schedule_iterator.cpp @@ -54,77 +54,35 @@ int ObBasicMergeScheduleIterator::ObTabletArray::consume_tablet_id(ObTabletID &t ObBasicMergeScheduleIterator::ObBasicMergeScheduleIterator() : scan_finish_(false), merge_finish_(false), - ls_idx_(-1), schedule_tablet_cnt_(0), max_batch_tablet_cnt_(0), - cur_ls_handle_(), - ls_ids_(), + ls_(nullptr), tablet_ids_() { - ls_ids_.set_attr(ObMemAttr("CompIter")); } -int ObBasicMergeScheduleIterator::init(const int64_t schedule_batch_size) +int ObBasicMergeScheduleIterator::init( + const int64_t schedule_batch_size, + ObLS *ls) { int ret = OB_SUCCESS; if (OB_UNLIKELY(schedule_batch_size <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(schedule_batch_size)); } else if (!is_valid()) { - ls_ids_.reuse(); - if (OB_FAIL(share::g_mp->ls_service()->get_ls_ids(ls_ids_))) { - LOG_WARN("failed to get all ls id", K(ret)); - } else { - ls_idx_ = -1; - tablet_ids_.reset(); - scan_finish_ = false; - merge_finish_ = true; - cur_ls_handle_.reset(); - schedule_tablet_cnt_ = 0; - max_batch_tablet_cnt_ = schedule_batch_size; - LOG_TRACE("build iter", K(ret), KPC(this)); - } + tablet_ids_.reset(); + scan_finish_ = false; + merge_finish_ = true; + ls_ = ls; + schedule_tablet_cnt_ = 0; + max_batch_tablet_cnt_ = schedule_batch_size; + LOG_TRACE("build iter", K(ret), KPC(this)); } else { // iter is valid, no need to build, just set var to start cur batch (void) start_cur_batch(); } return ret; } -int ObBasicMergeScheduleIterator::get_next_ls(ObLSHandle &ls_handle) -{ - int ret = OB_SUCCESS; - if (-1 == ls_idx_ - || tablet_ids_.is_ls_iter_end()) { // tablet iter end, need get next ls - ++ls_idx_; - cur_ls_handle_.reset(); - tablet_ids_.reset(); - LOG_TRACE("tablet iter end", K(ret), K(ls_idx_), K(tablet_ids_)); - } - do { - if (ls_idx_ >= ls_ids_.count()) { - scan_finish_ = true; - LOG_DEBUG("schedule all ls finish", K(ret), K_(ls_idx), K_(ls_ids)); - ret = OB_ITER_END; - } else if (OB_FAIL(get_cur_ls_handle(ls_handle))) { - if (OB_LS_NOT_EXIST == ret) { - LOG_TRACE("ls not exist", K(ret), K(ls_idx_), K(ls_ids_[ls_idx_])); - skip_cur_ls(); - } else { - LOG_WARN("failed to get ls", K(ret), K(ls_idx_), K(ls_ids_[ls_idx_])); - } - } else { - cur_ls_handle_ = ls_handle; - } - if (OB_SUCC(ret) && schedule_tablet_cnt_ >= max_batch_tablet_cnt_) { - LOG_INFO("reach max batch tablet cnt, schedule next round", K(ret), K_(ls_idx), K_(ls_ids), - K_(schedule_tablet_cnt), K_(max_batch_tablet_cnt), "tablet_cnt", tablet_ids_.count(), - "tablet_idx", tablet_ids_.tablet_idx_); - ret = OB_ITER_END; - } - } while (OB_LS_NOT_EXIST == ret); - return ret; -} - #ifdef ERRSIM void errsim_set_batch_cnt( const ObBasicMergeScheduleIterator::ObTabletArray &tablet_ids, @@ -147,22 +105,28 @@ void errsim_set_batch_cnt( int ObBasicMergeScheduleIterator::get_next_tablet(ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!cur_ls_handle_.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected null", K(ret), KPC(this)); + if (scan_finish_) { + ret = OB_ITER_END; } else if (!tablet_ids_.is_inited_) { if (OB_FAIL(get_tablet_ids())) { LOG_WARN("failed to get tablet ids", K(ret)); } else { tablet_ids_.mark_inited(); - LOG_TRACE("build iter in get_next_tablet", K(ret), K_(ls_idx), "ls_id", ls_ids_[ls_idx_], K(tablet_ids_)); + LOG_TRACE("build iter in get_next_tablet", K(ret), K(tablet_ids_)); #ifdef ERRSIM (void) errsim_set_batch_cnt(tablet_ids_, max_batch_tablet_cnt_); #endif } } if (OB_FAIL(ret)) { + } else if (tablet_ids_.is_tablet_iter_end()) { + scan_finish_ = true; + ret = OB_ITER_END; + LOG_DEBUG("schedule tablet scan finish", K(ret)); } else if (schedule_tablet_cnt_ >= max_batch_tablet_cnt_) { + LOG_INFO("reach max batch tablet cnt, schedule next round", K(ret), + K_(schedule_tablet_cnt), K_(max_batch_tablet_cnt), "tablet_cnt", tablet_ids_.count(), + "tablet_idx", tablet_ids_.tablet_idx_); ret = OB_ITER_END; } else { ObTabletID tablet_id; @@ -170,10 +134,12 @@ int ObBasicMergeScheduleIterator::get_next_tablet(ObTabletHandle &tablet_handle) if (OB_FAIL(tablet_ids_.consume_tablet_id(tablet_id))) { if (OB_ITER_END != ret) { LOG_WARN("failed to get tablet id", KR(ret), K_(tablet_ids)); + } else { + scan_finish_ = true; } } else if (OB_FAIL(get_tablet_handle(tablet_id, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { - LOG_DEBUG("tablet not exist", K(ret), "ls_id", ls_ids_[ls_idx_], K(tablet_id), "tablet_cnt", tablet_ids_.count()); + LOG_DEBUG("tablet not exist", K(ret), K(tablet_id), "tablet_cnt", tablet_ids_.count()); } else { LOG_WARN("fail to get tablet", K(ret), K(tablet_ids_), K(tablet_id)); } @@ -190,29 +156,25 @@ void ObBasicMergeScheduleIterator::reset_basic_iter() { scan_finish_ = false; merge_finish_ = false; - ls_idx_ = 0; schedule_tablet_cnt_ = 0; - ls_ids_.reuse(); + max_batch_tablet_cnt_ = 0; tablet_ids_.reset(); - cur_ls_handle_.reset(); + ls_ = nullptr; } bool ObBasicMergeScheduleIterator::is_valid() const { - return ls_ids_.count() > 0 && ls_idx_ >= 0 && cur_ls_handle_.is_valid() - && (ls_idx_ < ls_ids_.count() - 1 - || (ls_idx_ == ls_ids_.count() - 1 && !tablet_ids_.is_ls_iter_end())); - // have remain ls or have remain tablet + return max_batch_tablet_cnt_ > 0 && !scan_finish_ && !tablet_ids_.is_tablet_iter_end(); } int64_t ObBasicMergeScheduleIterator::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(ls_idx), K_(ls_ids), K_(tablet_ids), K_(schedule_tablet_cnt), K_(max_batch_tablet_cnt)); - if (is_valid()) { + J_KV(K_(tablet_ids), K_(schedule_tablet_cnt), K_(max_batch_tablet_cnt)); + if (is_valid() && OB_NOT_NULL(ls_)) { J_COMMA(); - J_KV("cur_ls", ls_ids_.at(ls_idx_)); + J_KV("has_ls", OB_NOT_NULL(ls_)); } J_OBJ_END(); return pos; @@ -227,17 +189,27 @@ ObCompactionScheduleIterator::ObCompactionScheduleIterator( report_scn_flag_(false), tablet_get_mode_(storage::ObMDSGetTabletMode::READ_ALL_COMMITED) { - ls_ids_.set_attr(ObMemAttr("CompIter")); } int ObCompactionScheduleIterator::build_iter(const int64_t schedule_batch_size) { int ret = OB_SUCCESS; bool need_reset_report_scn = !is_valid(); + ObLS *ls = nullptr; - if (OB_FAIL(init(schedule_batch_size))) { + if (OB_UNLIKELY(schedule_batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(schedule_batch_size)); + } else if (!need_reset_report_scn) { + ls = ls_; + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } + + if (OB_SUCC(ret) && OB_NOT_NULL(ls) && OB_FAIL(init(schedule_batch_size, ls))) { LOG_WARN("failed to inner build iter", K(ret)); - } else if (need_reset_report_scn) { + } + if (OB_SUCC(ret) && need_reset_report_scn) { report_scn_flag_ = false; if (REACH_THREAD_TIME_INTERVAL(CHECK_REPORT_SCN_INTERVAL)) { report_scn_flag_ = true; @@ -255,22 +227,6 @@ void ObCompactionScheduleIterator::reset() report_scn_flag_ = false; } -int ObCompactionScheduleIterator::get_cur_ls_handle(ObLSHandle &ls_handle) -{ - int ret = share::g_mp->ls_service()->get_ls(ls_ids_[ls_idx_], ls_handle, ObLSGetMod::COMPACT_MODE); -#ifdef ERRSIM - if (OB_SUCC(ret) && ls_ids_[ls_idx_].id() > share::ObLSID::SYS_LS_ID) { - ret = OB_E(EventTable::EN_COMPACTION_ITER_LS_NOT_EXIST) ret; - if (OB_FAIL(ret)) { - FLOG_INFO("ERRSIM EN_COMPACTION_ITER_LS_NOT_EXIST", KR(ret)); - ret = OB_LS_NOT_EXIST; - ls_handle.reset(); - } - } -#endif - return ret; -} - #ifdef ERRSIM void errsim_iter_invalid_tablet_id(int &ret, ObBasicMergeScheduleIterator::ObTabletArray &tablet_ids) { if (OB_SUCC(ret)) { @@ -309,7 +265,7 @@ void errsim_iter_invalid_tablet_id(int &ret, ObBasicMergeScheduleIterator::ObTab int ObCompactionScheduleIterator::get_tablet_ids() { tablet_ids_.reset(); - int ret = cur_ls_handle_.get_ls()->get_tablet_svr()->get_all_tablet_ids(is_major_/*except_ls_inner_tablet*/, tablet_ids_.array_); + int ret = ls_->get_tablet_svr()->get_all_tablet_ids(is_major_/*except_ls_inner_tablet*/, tablet_ids_.array_); #ifdef ERRSIM (void) errsim_iter_invalid_tablet_id(ret, tablet_ids_); #endif @@ -319,7 +275,7 @@ int ObCompactionScheduleIterator::get_tablet_ids() int ObCompactionScheduleIterator::get_tablet_handle( const ObTabletID &tablet_id, ObTabletHandle &tablet_handle) { - int ret = cur_ls_handle_.get_ls()->get_tablet_svr()->get_tablet(tablet_id, tablet_handle, 0/*timeout*/, tablet_get_mode_); + int ret = ls_->get_tablet_svr()->get_tablet(tablet_id, tablet_handle, 0/*timeout*/, tablet_get_mode_); #ifdef ERRSIM if (OB_SUCC(ret) && tablet_id.id() > ObTabletID::MIN_USER_TABLET_ID) { ret = OB_E(EventTable::EN_COMPACTION_ITER_TABLET_NOT_EXIST) ret; diff --git a/src/storage/compaction/ob_compaction_schedule_iterator.h b/src/storage/compaction/ob_compaction_schedule_iterator.h index c89f7c8a6..4182c916b 100644 --- a/src/storage/compaction/ob_compaction_schedule_iterator.h +++ b/src/storage/compaction/ob_compaction_schedule_iterator.h @@ -15,20 +15,14 @@ */ #ifndef OB_STORAGE_COMPACTION_COMPACTION_SCHEDUL_ITERATOR_H_ #define OB_STORAGE_COMPACTION_COMPACTION_SCHEDUL_ITERATOR_H_ -#include "storage/ls/ob_ls_get_mod.h" #include "storage/tablet/ob_tablet_common.h" #include "lib/container/ob_se_array.h" #include "lib/literals/ob_literals.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "common/ob_tablet_id.h" #include "storage/ls/ob_ls.h" namespace oceanbase { -namespace share -{ -class ObLSID; -} namespace common { class ObTabletID; @@ -36,7 +30,6 @@ class ObTabletID; namespace storage { class ObLS; -class ObLSHandle; class ObTabletHandle; class ObLSTabletService; } @@ -50,10 +43,8 @@ class ObBasicMergeScheduleIterator { ObTabletArray(); ~ObTabletArray() { reset(); } - bool is_ls_iter_end() const + bool is_tablet_iter_end() const { - // not init means cur ls not iter before - // when init, need check array return is_inited_ && (array_.empty() || tablet_idx_ >= array_.count()); } void reset() @@ -72,15 +63,14 @@ class ObBasicMergeScheduleIterator TO_STRING_KV(K_(tablet_idx), "tablet_cnt", count(), K_(array), K_(is_inited)); static const int64_t TABLET_ID_ARRAY_CNT = 2000; int64_t tablet_idx_; - // array may be empty after inited on SS common::ObSEArray array_; bool is_inited_; }; public: ObBasicMergeScheduleIterator(); ~ObBasicMergeScheduleIterator() = default; - int init(const int64_t schedule_batch_size); - virtual int get_next_ls(storage::ObLSHandle &ls_handle); + int init(const int64_t schedule_batch_size, storage::ObLS *ls); + storage::ObLS *get_ls() const { return ls_; } int get_next_tablet(storage::ObTabletHandle &tablet_handle); bool is_scan_finish() const { return scan_finish_; } bool tenant_merge_finish() const { return merge_finish_ & scan_finish_; } @@ -89,10 +79,10 @@ class ObBasicMergeScheduleIterator } void reset_basic_iter(); bool is_valid() const; - void skip_cur_ls() + void finish_scan() { - ++ls_idx_; - cur_ls_handle_.reset(); + scan_finish_ = true; + ls_ = nullptr; tablet_ids_.reset(); } void start_cur_batch() @@ -101,18 +91,14 @@ class ObBasicMergeScheduleIterator } int64_t to_string(char *buf, const int64_t buf_len) const; protected: - virtual int get_cur_ls_handle(storage::ObLSHandle &ls_handle) = 0; virtual int get_tablet_ids() = 0; virtual int get_tablet_handle(const ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle) = 0; protected: - static const int64_t LS_ID_ARRAY_CNT = 10; bool scan_finish_; bool merge_finish_; - int64_t ls_idx_; int64_t schedule_tablet_cnt_; int64_t max_batch_tablet_cnt_; - storage::ObLSHandle cur_ls_handle_; - common::ObSEArray ls_ids_; + storage::ObLS *ls_; ObTabletArray tablet_ids_; }; @@ -128,7 +114,6 @@ class ObCompactionScheduleIterator : public ObBasicMergeScheduleIterator void set_tablet_get_mode(const storage::ObMDSGetTabletMode mode) { tablet_get_mode_ = mode; } void reset(); protected: - virtual int get_cur_ls_handle(storage::ObLSHandle &ls_handle) override; virtual int get_tablet_ids() override; virtual int get_tablet_handle(const ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle) override; protected: diff --git a/src/storage/compaction/ob_compaction_schedule_util.h b/src/storage/compaction/ob_compaction_schedule_util.h index 6700dd87d..95ea386ab 100644 --- a/src/storage/compaction/ob_compaction_schedule_util.h +++ b/src/storage/compaction/ob_compaction_schedule_util.h @@ -35,21 +35,21 @@ struct ObScheduleStatistics ~ObScheduleStatistics() {} OB_INLINE void reset() { - all_ls_weak_read_ts_ready_ = false; + weak_read_ts_ready_ = false; add_weak_read_ts_event_flag_ = false; check_weak_read_ts_cnt_ = 0; start_timestamp_ = 0; } OB_INLINE void start_merge() { - all_ls_weak_read_ts_ready_ = false; + weak_read_ts_ready_ = false; add_weak_read_ts_event_flag_ = true; check_weak_read_ts_cnt_ = 0; start_timestamp_ = ObTimeUtility::fast_current_time(); } - TO_STRING_KV(K_(all_ls_weak_read_ts_ready), K_(start_timestamp)); + TO_STRING_KV(K_(weak_read_ts_ready), K_(start_timestamp)); - bool all_ls_weak_read_ts_ready_; + bool weak_read_ts_ready_; bool add_weak_read_ts_event_flag_; int64_t check_weak_read_ts_cnt_; int64_t start_timestamp_; diff --git a/src/storage/compaction/ob_compaction_suggestion.cpp b/src/storage/compaction/ob_compaction_suggestion.cpp index f06d54695..5f3a83e04 100644 --- a/src/storage/compaction/ob_compaction_suggestion.cpp +++ b/src/storage/compaction/ob_compaction_suggestion.cpp @@ -16,7 +16,6 @@ #include "ob_compaction_suggestion.h" #include "share/rc/ob_module_provider.h" -#include "src/storage/tx_storage/ob_ls_map.h" namespace oceanbase { @@ -411,7 +410,6 @@ int ObCompactionSuggestionMgr::analyze_for_suggestion( suggestion.merge_type_ = MEDIUM_MERGE; } - suggestion.ls_id_ = UNKNOW_LS_ID.id(); suggestion.tablet_id_ = UNKNOW_TABLET_ID.id(); suggestion.merge_start_time_ = common::ObTimeUtility::fast_current_time(); suggestion.merge_finish_time_ = suggestion.merge_start_time_; @@ -555,7 +553,6 @@ int ObCompactionSuggestionMgr::analyze_merge_info( if (strlen(buf) > 0) { suggestion.merge_type_ = static_info.merge_type_; - suggestion.ls_id_ = static_info.ls_id_.id(); suggestion.tablet_id_ = static_info.tablet_id_.id(); suggestion.merge_start_time_ = running_info.merge_start_time_; suggestion.merge_finish_time_ = running_info.merge_finish_time_; diff --git a/src/storage/compaction/ob_compaction_suggestion.h b/src/storage/compaction/ob_compaction_suggestion.h index e1bd3cbb7..8c306bc8e 100644 --- a/src/storage/compaction/ob_compaction_suggestion.h +++ b/src/storage/compaction/ob_compaction_suggestion.h @@ -152,17 +152,15 @@ struct ObCompactionSuggestion { ObCompactionSuggestion() : merge_type_(compaction::INVALID_MERGE_TYPE), - ls_id_(0), tablet_id_(0), merge_start_time_(0), merge_finish_time_(0), suggestion_() {} - TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(ls_id), K_(tablet_id), K_(merge_start_time), + TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(tablet_id), K_(merge_start_time), K_(merge_finish_time), K_(suggestion)); compaction::ObMergeType merge_type_; - int64_t ls_id_; int64_t tablet_id_; int64_t merge_start_time_; int64_t merge_finish_time_; diff --git a/src/storage/compaction/ob_compaction_tablet_diagnose.cpp b/src/storage/compaction/ob_compaction_tablet_diagnose.cpp index 550d7ee9a..0ecc246a0 100644 --- a/src/storage/compaction/ob_compaction_tablet_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_tablet_diagnose.cpp @@ -63,7 +63,6 @@ void ObDiagnoseTabletMgr::destroy() // for diagnose int ObDiagnoseTabletMgr::add_diagnose_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::ObDiagnoseTabletType type) { @@ -72,13 +71,13 @@ int ObDiagnoseTabletMgr::add_diagnose_tablet( ret = OB_NOT_INIT; LOG_WARN("ObDiagnoseTabletMgr is not init", K(ret)); } else { - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() + if (OB_UNLIKELY(!tablet_id.is_valid() || !is_valid_diagnose_tablet_type(type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(type)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(type)); } else { lib::ObMutexGuard guard(diagnose_lock_); - ObDiagnoseTablet diagnose_tablet(ls_id, tablet_id); + ObDiagnoseTablet diagnose_tablet(tablet_id); int64_t flag = 0; if (OB_FAIL(diagnose_tablet_map_.get_refactored(diagnose_tablet, flag))) { if (OB_HASH_NOT_EXIST != ret) { @@ -121,7 +120,6 @@ int ObDiagnoseTabletMgr::get_diagnose_tablets(ObIArray &diagno } int ObDiagnoseTabletMgr::delete_diagnose_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::ObDiagnoseTabletType type) { @@ -130,13 +128,13 @@ int ObDiagnoseTabletMgr::delete_diagnose_tablet( ret = OB_NOT_INIT; LOG_WARN("ObDiagnoseTabletMgr is not init", K(ret)); } else { - if (!ls_id.is_valid() || !tablet_id.is_valid() + if (!tablet_id.is_valid() || !is_valid_diagnose_tablet_type(type)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(type)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(type)); } else { lib::ObMutexGuard guard(diagnose_lock_); - ObDiagnoseTablet diagnose_tablet(ls_id, tablet_id); + ObDiagnoseTablet diagnose_tablet(tablet_id); int64_t flag = 0; if (OB_FAIL(diagnose_tablet_map_.get_refactored(diagnose_tablet, flag))) { if (OB_HASH_NOT_EXIST != ret) { diff --git a/src/storage/compaction/ob_compaction_tablet_diagnose.h b/src/storage/compaction/ob_compaction_tablet_diagnose.h index 7545d51d8..191fa0ad9 100644 --- a/src/storage/compaction/ob_compaction_tablet_diagnose.h +++ b/src/storage/compaction/ob_compaction_tablet_diagnose.h @@ -25,28 +25,25 @@ namespace compaction { struct ObDiagnoseTablet { ObDiagnoseTablet() - : ls_id_(), - tablet_id_() + : tablet_id_() {} - ObDiagnoseTablet(const share::ObLSID &ls_id, const ObTabletID &tablet_id) - : ls_id_(ls_id), - tablet_id_(tablet_id) + explicit ObDiagnoseTablet(const ObTabletID &tablet_id) + : tablet_id_(tablet_id) {} ~ObDiagnoseTablet() {} inline bool operator == (const ObDiagnoseTablet &other) const { - return ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_; + return tablet_id_ == other.tablet_id_; } inline int hash(uint64_t &hash_value) const { int ret = common::OB_SUCCESS; - hash_value = murmurhash(&ls_id_, sizeof(ls_id_), 0); - hash_value = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_value); + hash_value = murmurhash(&tablet_id_, sizeof(tablet_id_), 0); return ret; } inline bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } static bool is_flagged(int64_t flag, const share::ObDiagnoseTabletType type) { @@ -69,9 +66,8 @@ struct ObDiagnoseTablet { input_flag &= ~flag; } } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); - share::ObLSID ls_id_; ObTabletID tablet_id_; }; @@ -86,12 +82,10 @@ class ObDiagnoseTabletMgr { // for diagnose int add_diagnose_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::ObDiagnoseTabletType type); int get_diagnose_tablets(ObIArray &diagnose_tablets); int delete_diagnose_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::ObDiagnoseTabletType type); void remove_diagnose_tablets( diff --git a/src/storage/compaction/ob_i_compaction_filter.cpp b/src/storage/compaction/ob_i_compaction_filter.cpp index b32a9331a..acf19339c 100644 --- a/src/storage/compaction/ob_i_compaction_filter.cpp +++ b/src/storage/compaction/ob_i_compaction_filter.cpp @@ -69,7 +69,6 @@ const char *ObICompactionFilter::ObFilterTypeStr[] = { "TX_DATA_MINOR", "MDS_MINOR_FILTER_DATA", - "MDS_MINOR_CROSS_LS", "MDS_IN_MEDIUM_INFO", "FILTER_TYPE_MAX" }; diff --git a/src/storage/compaction/ob_i_compaction_filter.h b/src/storage/compaction/ob_i_compaction_filter.h index 8659fad91..e913906a0 100644 --- a/src/storage/compaction/ob_i_compaction_filter.h +++ b/src/storage/compaction/ob_i_compaction_filter.h @@ -65,7 +65,6 @@ class ObICompactionFilter { TX_DATA_MINOR, MDS_MINOR_FILTER_DATA, - MDS_MINOR_CROSS_LS, MDS_IN_MEDIUM_INFO, FILTER_TYPE_MAX }; diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index d25f24d07..eee7b4640 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -19,6 +19,8 @@ #include "share/tablet/ob_tablet_table_operator.h" #include "share/schema/ob_tenant_schema_service.h" #include "logservice/ob_log_service.h" +#include "logservice/palf_handle_guard.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/ob_partition_range_spliter.h" #include "storage/truncate_info/ob_mds_info_distinct_mgr.h" @@ -37,7 +39,7 @@ int64_t ObMediumCompactionScheduleFunc::to_string(char *buf, const int64_t buf_l if (OB_ISNULL(buf) || buf_len <= 0) { } else { J_OBJ_START(); - J_KV("ls_id", ls_.get_ls_id(), "merge_reason", ObAdaptiveMergePolicy::merge_reason_to_str(merge_reason_), K_(least_medium_snapshot)); + J_KV("merge_reason", ObAdaptiveMergePolicy::merge_reason_to_str(merge_reason_), K_(least_medium_snapshot)); J_COMMA(); if (tablet_handle_.is_valid()) { J_KV("tablet_id", tablet_handle_.get_obj()->get_tablet_meta().tablet_id_); @@ -261,7 +263,6 @@ int ObMediumCompactionScheduleFunc::switch_to_choose_medium_snapshot( } int ObMediumCompactionScheduleFunc::get_status_from_inner_table( - const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletCompactionScnInfo &ret_info) { @@ -269,7 +270,6 @@ int ObMediumCompactionScheduleFunc::get_status_from_inner_table( ret_info.reset(); ObTabletCompactionScnInfo snapshot_info( - ls_id, tablet_id, ObTabletReplica::SCN_STATUS_IDLE); if (OB_FAIL(ObTabletMetaTableCompactionOperator::get_status(snapshot_info, ret_info))) { @@ -283,35 +283,26 @@ int ObMediumCompactionScheduleFunc::get_status_from_inner_table( return ret; } -// cal this func with PLAF LEADER ROLE && last_medium_scn_ = 0 -int ObMediumCompactionScheduleFunc::schedule_next_medium_for_leader( +int ObMediumCompactionScheduleFunc::schedule_next_medium( const int64_t major_snapshot, bool &medium_clog_submitted) { int ret = OB_SUCCESS; - ObRole role = INVALID_ROLE; - if (OB_FAIL(ls_.get_ls_role(role))) { - LOG_WARN("failed to get ls role", K(ret), KPC(this)); - } else if (LEADER == role) { - // only log_handler_leader can schedule #ifdef ERRSIM - ret = OB_E(EventTable::EN_SKIP_INDEX_MAJOR) ret; - // skip schedule major for user index table - ObTablet *tablet = nullptr; - if (OB_FAIL(ret) && tablet_handle_.is_valid() && OB_NOT_NULL(tablet = tablet_handle_.get_obj())) { - if (tablet->get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID - && tablet->get_tablet_meta().tablet_id_ != tablet->get_tablet_meta().data_tablet_id_) { - LOG_INFO("ERRSIM EN_SKIP_INDEX_MAJOR", K(ret), KPC(tablet), K(major_snapshot)); - return ret; - } else { - ret = OB_SUCCESS; - } + ret = OB_E(EventTable::EN_SKIP_INDEX_MAJOR) ret; + // skip schedule major for user index table + ObTablet *tablet = nullptr; + if (OB_FAIL(ret) && tablet_handle_.is_valid() && OB_NOT_NULL(tablet = tablet_handle_.get_obj())) { + if (tablet->get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID + && tablet->get_tablet_meta().tablet_id_ != tablet->get_tablet_meta().data_tablet_id_) { + LOG_INFO("ERRSIM EN_SKIP_INDEX_MAJOR", K(ret), KPC(tablet), K(major_snapshot)); + return ret; + } else { + ret = OB_SUCCESS; } -#endif - ret = schedule_next_medium_primary_cluster(major_snapshot, medium_clog_submitted); - } else { - LOG_TRACE("not leader", K(ret), K(role), K(ls_.get_ls_id())); } +#endif + ret = schedule_next_medium_primary_cluster(major_snapshot, medium_clog_submitted); return ret; } @@ -400,7 +391,7 @@ int ObMediumCompactionScheduleFunc::schedule_next_medium_primary_cluster( } else if (ObMediumCompactionInfo::MAJOR_COMPACTION == medium_info_list_->get_last_compaction_type()) { // for normal medium, checksum error happened, wait_check_medium_scn_ will never = 0 // for major, need select inner_table to check RS status - if (OB_FAIL(get_status_from_inner_table(ls_.get_ls_id(), tablet->get_tablet_meta().tablet_id_, ret_info))) { + if (OB_FAIL(get_status_from_inner_table(tablet->get_tablet_meta().tablet_id_, ret_info))) { LOG_WARN("failed to get status from inner tablet", K(ret), KPC(this)); } else if (ret_info.could_schedule_next_round(medium_info_list_->get_last_compaction_scn())) { LOG_INFO("success to check RS major checksum validation finished", K(ret), KPC(this), K(ret_info)); @@ -488,8 +479,8 @@ int ObMediumCompactionScheduleFunc::check_frequency( } else if (last_major_snapshot_version + time_interval > medium_snapshot) { // TODO(chengkong): for better performance, here should take meta major merge in the future. ObTableQueuingModeCfg queuing_cfg; - if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(ls_.get_ls_id(), tablet_id, queuing_cfg))) { - LOG_WARN_RET(tmp_ret, "failed to get table queuing mode, treat it as normal table", "ls_id", ls_.get_ls_id(), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(tablet_id, queuing_cfg))) { + LOG_WARN_RET(tmp_ret, "failed to get table queuing mode, treat it as normal table", K(tablet_id)); ret = OB_NO_NEED_MERGE; LOG_TRACE("schedule medium frequently", K(ret), K(last_major_snapshot_version), K(medium_snapshot), K(time_interval)); } else if (queuing_cfg.is_queuing_mode()) { @@ -657,7 +648,7 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot(bool &medium_clog_sub if (OB_FAIL(ret)) { // add schedule suspect info if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MEDIUM_MERGE, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - ls_.get_ls_id(), tablet_id, + tablet_id, ObSuspectInfoType::SUSPECT_SCHEDULE_MEDIUM_FAILED, medium_info.medium_snapshot_, medium_info.storage_schema_.store_column_cnt_, @@ -1099,9 +1090,6 @@ int ObMediumCompactionScheduleFunc::submit_medium_clog( if (OB_UNLIKELY(!tablet_handle_.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_)); - } else if (OB_UNLIKELY(medium_info.is_invalid_mview_compaction())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mview compaction", K(ret), K(tablet_handle_), K(medium_info)); } else if (FALSE_IT(tablet = tablet_handle_.get_obj())) { } else if (OB_FAIL(tablet->submit_medium_compaction_clog(medium_info, allocator_))) { LOG_WARN("failed to submit medium compaction clog", K(ret), K(medium_info)); @@ -1112,29 +1100,28 @@ int ObMediumCompactionScheduleFunc::submit_medium_clog( } int ObMediumCompactionScheduleFunc::batch_check_medium_meta_table( - const ObIArray &tablet_ls_infos, - ObIArray &finish_tablet_ls, + const ObIArray &tablet_check_infos, + ObIArray &finish_tablet_infos, ObCompactionTimeGuard &time_guard) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - if (tablet_ls_infos.empty()) { + if (tablet_check_infos.empty()) { } else { share::ObTabletReplicaFilterHolder filters; ObArrayWithMap tablet_infos; - if (OB_FAIL(tablet_infos.init(tablet_ls_infos.count()))) { - LOG_WARN("failed to reserve array", KR(ret), "array_cnt", tablet_ls_infos.count()); + if (OB_FAIL(tablet_infos.init(tablet_check_infos.count()))) { + LOG_WARN("failed to reserve array", KR(ret), "array_cnt", tablet_check_infos.count()); } else if (OB_FAIL(ObTabletTableOperator::batch_get_tablet_info(GCTX.sql_proxy_, - tablet_ls_infos, 0, tablet_infos))) { - LOG_WARN("failed to get tablet info", K(ret), K(tablet_ls_infos)); + tablet_check_infos, 0, tablet_infos))) { + LOG_WARN("failed to get tablet info", K(ret), K(tablet_check_infos)); } else { time_guard.click(ObCompactionScheduleTimeGuard::SEARCH_META_TABLE); - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_ls_infos.count(); ++idx) { + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_check_infos.count(); ++idx) { bool merge_finish = false; - const ObTabletCheckInfo &tablet_ls_info = tablet_ls_infos.at(idx); - const ObLSID &ls_id = tablet_ls_info.get_ls_id(); - const ObTabletID &tablet_id = tablet_ls_info.get_tablet_id(); - const int64_t check_medium_scn = tablet_ls_info.get_medium_scn(); + const ObTabletCheckInfo &tablet_check_info = tablet_check_infos.at(idx); + const ObTabletID &tablet_id = tablet_check_info.get_tablet_id(); + const int64_t check_medium_scn = tablet_check_info.get_medium_scn(); const ObTabletInfo *info = NULL; if (OB_TMP_FAIL(tablet_infos.get(tablet_id, info))) { if (OB_ENTRY_NOT_EXIST == tmp_ret) { @@ -1142,13 +1129,11 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_meta_table( } else { LOG_WARN("failed to get tablet info", KR(tmp_ret), K(tablet_id)); } - } else if (info->get_ls_id() != ls_id) { - LOG_INFO("tablet_ls_info has been deleted", K(tablet_ls_info), KPC(info)); } else if (OB_TMP_FAIL(check_medium_meta_table(check_medium_scn, *info, filters, merge_finish))) { LOG_WARN("failed to check medium meta table", K(tmp_ret), K(check_medium_scn), KPC(info)); } else if (merge_finish && - OB_TMP_FAIL(finish_tablet_ls.push_back(ObTabletCheckInfo(tablet_id, ls_id, check_medium_scn)))) { - LOG_WARN("fail to push back tablet_ls_infos", K(tmp_ret), KPC(info)); + OB_TMP_FAIL(finish_tablet_infos.push_back(ObTabletCheckInfo(tablet_id, check_medium_scn)))) { + LOG_WARN("fail to push back finish tablet infos", K(tmp_ret), KPC(info)); } } // for time_guard.click(ObCompactionScheduleTimeGuard::CHECK_META_TABLE); @@ -1166,8 +1151,6 @@ int ObMediumCompactionScheduleFunc::check_medium_meta_table( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; merge_finish = false; - const ObLSID &ls_id = tablet_info.get_ls_id(); - if (OB_UNLIKELY(check_medium_snapshot <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(check_medium_snapshot), K(tablet_info)); @@ -1175,30 +1158,26 @@ int ObMediumCompactionScheduleFunc::check_medium_meta_table( ret = OB_ERR_UNEXPECTED; LOG_WARN("tabled_id is invalid", K(ret), K(tablet_info)); } else { - const ObArray &replica_array = tablet_info.get_replicas(); + const ObTabletReplica &replica = tablet_info.get_replica(); int64_t unfinish_cnt = 0; int64_t filter_cnt = 0; bool pass = true; - for (int i = 0; OB_SUCC(ret) && i < replica_array.count(); ++i) { - const ObTabletReplica &replica = replica_array.at(i); - if (OB_UNLIKELY(!replica.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replica info is invalid", K(ret), K(tablet_info), K(replica)); - } else if (OB_FAIL(filters.check(replica, pass))) { - LOG_WARN("filter replica failed", K(ret), K(replica), K(filters)); - } else if (!pass) { - // do nothing - filter_cnt++; - } else if (replica.get_snapshot_version() >= check_medium_snapshot) { - // replica may have check_medium_snapshot = 2, but have received medium info of 3, - // when this replica is elected as leader, this will happened - } else { - LOG_TRACE("tablet unfinish", K(replica)); - unfinish_cnt++; - } - } // end of for + if (OB_UNLIKELY(!replica.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica info is invalid", K(ret), K(tablet_info), K(replica)); + } else if (OB_FAIL(filters.check(replica, pass))) { + LOG_WARN("filter replica failed", K(ret), K(replica), K(filters)); + } else if (!pass) { + filter_cnt++; + } else if (replica.get_snapshot_version() >= check_medium_snapshot) { + // replica may have check_medium_snapshot = 2, but have received medium info of 3, + // when this replica is elected as leader, this will happened + } else { + LOG_TRACE("tablet unfinish", K(replica)); + unfinish_cnt++; + } LOG_INFO("check_medium_compaction_finish", K(ret), K(tablet_info), K(check_medium_snapshot), - K(unfinish_cnt), K(filter_cnt), "total_cnt", replica_array.count()); + K(unfinish_cnt), K(filter_cnt)); if (0 == unfinish_cnt) { // merge finish merge_finish = true; @@ -1213,12 +1192,12 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( const int64_t end_idx, const bool is_medium_checker, ObTabletDataChecksumChecker &data_checksum_checker, - ObIArray &error_pairs, + ObIArray &error_pairs, int &check_ret) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObCkmErrorTabletLSInfo error_tablet_ls; + ObCkmErrorTabletInfo error_tablet; if (start_idx >= end_idx) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid idx range for check tablet checksums", K(ret), K(start_idx), K(end_idx)); @@ -1226,7 +1205,7 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( } else { const ObTabletReplicaChecksumItem *prev_item = nullptr; data_checksum_checker.reset(); - ObLSID prev_error_ls_id; + ObTabletID prev_error_tablet_id; for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx; ++idx) { const ObTabletReplicaChecksumItem &curr_item = checksum_items.at(idx); if (OB_ISNULL(prev_item)) { @@ -1243,13 +1222,13 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(tmp_ret), K(curr_item), KPC(prev_item), K(data_checksum_checker)); check_ret = OB_CHECKSUM_ERROR; - if (curr_item.ls_id_ != prev_error_ls_id) { - prev_error_ls_id = curr_item.ls_id_; - error_tablet_ls.reset(); - error_tablet_ls.tablet_info_ = ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_); - error_tablet_ls.compaction_scn_ = curr_item.compaction_scn_.get_val_for_tx(); - if (OB_TMP_FAIL(error_pairs.push_back(error_tablet_ls))) { - LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); + if (curr_item.tablet_id_ != prev_error_tablet_id) { + prev_error_tablet_id = curr_item.tablet_id_; + error_tablet.reset(); + error_tablet.tablet_info_ = curr_item.tablet_id_; + error_tablet.compaction_scn_ = curr_item.compaction_scn_.get_val_for_tx(); + if (OB_TMP_FAIL(error_pairs.push_back(error_tablet))) { + LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_); } } } else { @@ -1261,11 +1240,12 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( if (is_medium_checker && OB_SUCC(ret)) { ret = OB_E(EventTable::EN_MEDIUM_REPLICA_CHECKSUM_ERROR) OB_SUCCESS; if (OB_FAIL(ret)) { - STORAGE_LOG(INFO, "ERRSIM EN_MEDIUM_REPLICA_CHECKSUM_ERROR", K(ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); - error_tablet_ls.tablet_info_ = ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_); - error_tablet_ls.compaction_scn_ = curr_item.compaction_scn_.get_val_for_tx(); - if (OB_TMP_FAIL(error_pairs.push_back(error_tablet_ls))) { - LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); + STORAGE_LOG(INFO, "ERRSIM EN_MEDIUM_REPLICA_CHECKSUM_ERROR", K(ret), "tablet_id", curr_item.tablet_id_); + error_tablet.reset(); + error_tablet.tablet_info_ = curr_item.tablet_id_; + error_tablet.compaction_scn_ = curr_item.compaction_scn_.get_val_for_tx(); + if (OB_TMP_FAIL(error_pairs.push_back(error_tablet))) { + LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_); } check_ret = OB_CHECKSUM_ERROR; } @@ -1291,8 +1271,7 @@ int ObMediumCompactionScheduleFunc::check_replica_checksum_items( int64_t start_idx = 0; int64_t end_idx = 0; ObTabletID tablet_id = checksum_items.at(0).tablet_id_; - ObLSID ls_id = checksum_items.at(0).ls_id_; - ObSEArray error_pairs; + ObSEArray error_pairs; error_pairs.set_attr(ObMemAttr("MedCkmErrs")); ObTabletDataChecksumChecker data_checksum_checker; @@ -1307,18 +1286,18 @@ int ObMediumCompactionScheduleFunc::check_replica_checksum_items( } else { // update medium compaction info if (is_medium_checker && OB_SUCCESS == check_ret) { - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle unused_handle; - if (OB_TMP_FAIL((share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::COMPACT_MODE)))) { + if (OB_TMP_FAIL((share::g_mp->ls_service()->get_ls(ls)))) { if (OB_LS_NOT_EXIST == tmp_ret) { - LOG_TRACE("ls not exist", K(tmp_ret), K(ls_id)); + LOG_TRACE("ls not exist", K(tmp_ret)); } else { - LOG_WARN("failed to get ls", K(tmp_ret), K(ls_id)); + LOG_WARN("failed to get ls", K(tmp_ret)); } - } else if (OB_TMP_FAIL(ls_handle.get_ls()->update_medium_compaction_info(tablet_id, unused_handle))) { - LOG_WARN("failed to update medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + } else if (OB_TMP_FAIL(ls->update_medium_compaction_info(tablet_id, unused_handle))) { + LOG_WARN("failed to update medium compaction info", K(tmp_ret), K(tablet_id)); } else { - FLOG_INFO("finish check medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + FLOG_INFO("finish check medium compaction info", K(tmp_ret), K(tablet_id)); } } @@ -1329,7 +1308,6 @@ int ObMediumCompactionScheduleFunc::check_replica_checksum_items( if (OB_SUCC(ret) && end_idx < count) { start_idx = end_idx; tablet_id = checksum_items.at(end_idx).tablet_id_; - ls_id = checksum_items.at(end_idx).ls_id_; check_ret = OB_SUCCESS; } } @@ -1356,21 +1334,21 @@ int ObMediumCompactionScheduleFunc::check_replica_checksum_items( // for Leader, clean wait_check_medium_scn int ObMediumCompactionScheduleFunc::batch_check_medium_finish( - ObIArray &finish_tablet_ls_infos, - const ObIArray &tablet_ls_infos, + ObIArray &finish_tablet_infos, + const ObIArray &tablet_check_infos, ObCompactionTimeGuard &time_guard) { int ret = OB_SUCCESS; - if (tablet_ls_infos.empty()) { + if (tablet_check_infos.empty()) { } else { // different ObTabletCheckInfo have different medium_check_scn share::ObReplicaCkmArray checksum_items(false/*need_map*/); - if (OB_FAIL(batch_check_medium_meta_table(tablet_ls_infos, finish_tablet_ls_infos, time_guard))) { - LOG_WARN("failed to check inner table", K(ret), K(tablet_ls_infos)); - } else if (!finish_tablet_ls_infos.empty()) { - if (OB_FAIL(checksum_items.init(finish_tablet_ls_infos.count()))) { - LOG_WARN("failed to reserve ckm items", KR(ret), "array_cnt", finish_tablet_ls_infos.count()); - } else if (OB_FAIL(ObTabletReplicaChecksumOperator::get_tablets_replica_checksum(finish_tablet_ls_infos, checksum_items))) { + if (OB_FAIL(batch_check_medium_meta_table(tablet_check_infos, finish_tablet_infos, time_guard))) { + LOG_WARN("failed to check inner table", K(ret), K(tablet_check_infos)); + } else if (!finish_tablet_infos.empty()) { + if (OB_FAIL(checksum_items.init(finish_tablet_infos.count()))) { + LOG_WARN("failed to reserve ckm items", KR(ret), "array_cnt", finish_tablet_infos.count()); + } else if (OB_FAIL(ObTabletReplicaChecksumOperator::get_tablets_replica_checksum(finish_tablet_infos, checksum_items))) { LOG_WARN("failed to get tablet checksum", K(ret)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::SEARCH_CHECKSUM))) { } else if (OB_FAIL(check_replica_checksum_items(checksum_items, true /*is_medium_checker*/))) { @@ -1378,25 +1356,6 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_finish( } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::CHECK_CHECKSUM))) { } } - // TODO, sort tablet ls pair first - } - return ret; -} - -int ObMediumCompactionScheduleFunc::is_election_leader(const ObLSID &ls_id, bool &is_election_leader) -{ - int ret = OB_SUCCESS; - ObRole role = INVALID_ROLE; - int64_t unused_proposal_id = 0; - palf::PalfHandleGuard palf_handle_guard; - if (OB_FAIL(share::g_mp->log_service()->open_palf(ls_id, palf_handle_guard))) { - if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("failed to open palf", K(ret), K(ls_id)); - } - } else if (OB_FAIL(palf_handle_guard.get_role(role, unused_proposal_id))) { - LOG_WARN("failed to get palf handle role", K(ret), K(ls_id)); - } else { - is_election_leader = is_leader_by_election(role); } return ret; } diff --git a/src/storage/compaction/ob_medium_compaction_func.h b/src/storage/compaction/ob_medium_compaction_func.h index d19ad9b8b..eddaecdb2 100644 --- a/src/storage/compaction/ob_medium_compaction_func.h +++ b/src/storage/compaction/ob_medium_compaction_func.h @@ -56,7 +56,6 @@ class ObMediumCompactionScheduleFunc ~ObMediumCompactionScheduleFunc() {} - static int is_election_leader(const share::ObLSID &ls_id, bool &ls_election_leader); static int get_max_sync_medium_scn( const ObTablet &tablet, const ObMediumCompactionInfoList &medium_list, @@ -70,13 +69,13 @@ class ObMediumCompactionScheduleFunc storage::ObStorageSchema &storage_schema, bool &is_skip_merge_index); static int batch_check_medium_finish( - ObIArray &finish_tablet_ls_infos, - const ObIArray &tablet_ls_infos, + ObIArray &finish_tablet_infos, + const ObIArray &tablet_check_infos, ObCompactionTimeGuard &time_guard); static int check_replica_checksum_items( const ObReplicaCkmArray &checksum_items, const bool is_medium_checker); - int schedule_next_medium_for_leader( + int schedule_next_medium( const int64_t major_snapshot, bool &medium_clog_submitted); static int get_table_id( @@ -93,7 +92,6 @@ class ObMediumCompactionScheduleFunc protected: int decide_medium_snapshot(bool &medium_clog_submitted); static int get_status_from_inner_table( - const ObLSID &ls_id, const ObTabletID &tablet_id, share::ObTabletCompactionScnInfo &ret_info); int prepare_medium_info( @@ -110,8 +108,8 @@ class ObMediumCompactionScheduleFunc ObTableStoreIterator &table_iter); int submit_medium_clog(ObMediumCompactionInfo &medium_info); static int batch_check_medium_meta_table( - const ObIArray &tablet_ls_infos, - ObIArray &finish_tablet_ls, + const ObIArray &tablet_check_infos, + ObIArray &finish_tablet_infos, ObCompactionTimeGuard &time_guard); static int check_medium_meta_table( const int64_t medium_snapshot, @@ -124,7 +122,7 @@ class ObMediumCompactionScheduleFunc const int64_t end_idx, const bool is_medium_checker, ObTabletDataChecksumChecker &data_checksum_checker, - ObIArray &error_pairs, + ObIArray &error_pairs, int &check_ret); int choose_medium_snapshot( const int64_t max_sync_medium_scn, diff --git a/src/storage/compaction/ob_medium_compaction_mgr.cpp b/src/storage/compaction/ob_medium_compaction_mgr.cpp index 04a37e070..15c4b93aa 100644 --- a/src/storage/compaction/ob_medium_compaction_mgr.cpp +++ b/src/storage/compaction/ob_medium_compaction_mgr.cpp @@ -95,7 +95,6 @@ ObTabletMediumCompactionInfoRecorder::ObTabletMediumCompactionInfoRecorder() : ObIStorageClogRecorder(), is_inited_(false), ignore_medium_(false), - ls_id_(), tablet_id_(), tablet_handle_ptr_(nullptr), medium_info_(nullptr), @@ -118,13 +117,11 @@ void ObTabletMediumCompactionInfoRecorder::destroy() ignore_medium_ = false; ObIStorageClogRecorder::destroy(); free_allocated_info(); - ls_id_.reset(); tablet_id_.reset(); } int ObTabletMediumCompactionInfoRecorder::init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t max_saved_version, logservice::ObLogHandler *log_handler) @@ -141,10 +138,9 @@ int ObTabletMediumCompactionInfoRecorder::init( LOG_WARN("failed to init ObIStorageClogRecorder", K(ret), K(log_handler)); } else { ignore_medium_ = tablet_id.is_special_merge_tablet(); - ls_id_ = ls_id; tablet_id_ = tablet_id; is_inited_ = true; - LOG_INFO("success to init medium clog recorder", K(ret), K_(ls_id), K_(tablet_id), K(max_saved_version)); + LOG_INFO("success to init medium clog recorder", K(ret), K_(tablet_id), K(max_saved_version)); } return ret; } @@ -165,7 +161,7 @@ int ObTabletMediumCompactionInfoRecorder::submit_medium_compaction_info( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(medium_info)); } else if (FALSE_IT(medium_info_ = &medium_info)) { - } else if (OB_FAIL(try_update_for_leader(medium_info.medium_snapshot_, &allocator))) { + } else if (OB_FAIL(try_update(medium_info.medium_snapshot_, &allocator))) { LOG_WARN("failed to update for leader", K(ret), K(medium_info)); } medium_info_ = nullptr; @@ -210,9 +206,9 @@ int ObTabletMediumCompactionInfoRecorder::replay_medium_compaction_log( ret = OB_NOT_SUPPORTED; LOG_WARN("not support to replay medium compaction clog", K(ret), K_(tablet_id)); } else if (OB_FAIL(serialization::decode_i64(buf, size, pos, &update_version))) { - LOG_WARN("fail to deserialize table_version", K(ret), K_(ls_id), K_(tablet_id)); + LOG_WARN("fail to deserialize table_version", K(ret), K_(tablet_id)); } else if (OB_FAIL(ObIStorageClogRecorder::replay_clog(update_version, scn, buf, size, pos))) { - LOG_WARN("failed to replay clog", K(ret), K(scn), K_(ls_id), K_(tablet_id), K(update_version)); + LOG_WARN("failed to replay clog", K(ret), K(scn), K_(tablet_id), K(update_version)); } return ret; } @@ -237,7 +233,7 @@ int ObTabletMediumCompactionInfoRecorder::inner_replay_clog( ObTabletMediumClogReplayExecutor replay_executor(replay_medium_info); if (OB_FAIL(replay_executor.init(scn))) { LOG_WARN("failed to init replay executor", K(ret), K(scn)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_id_, tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, tablet_id_))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret) { ret = OB_SUCCESS; LOG_INFO("skip reply medium info", KR(ret), K(replay_medium_info)); @@ -245,14 +241,14 @@ int ObTabletMediumCompactionInfoRecorder::inner_replay_clog( LOG_ERROR("failed to replay medium info", K(ret), K(replay_medium_info)); } } else { - FLOG_INFO("success to save medium info", K(ret), K_(tablet_id), K_(ls_id), K(scn), K(replay_medium_info), K(max_saved_version_)); + FLOG_INFO("success to save medium info", K(ret), K_(tablet_id), K(scn), K(replay_medium_info), K(max_saved_version_)); } } return ret; } -int ObTabletMediumCompactionInfoRecorder::sync_clog_succ_for_leader(const int64_t update_version) +int ObTabletMediumCompactionInfoRecorder::on_sync_clog_success(const int64_t update_version) { int ret = OB_SUCCESS; if (OB_ISNULL(medium_info_)) { @@ -264,13 +260,13 @@ int ObTabletMediumCompactionInfoRecorder::sync_clog_succ_for_leader(const int64_ } else if (OB_FAIL(submit_trans_on_mds_table(true/*is_commit*/))) { LOG_WARN("failed to dec ref on memtable", K(ret), K_(tablet_id), KPC(medium_info_)); } else { - LOG_TRACE("success to save medium info for leader", K(ret), K_(ls_id), K_(tablet_id), KPC(medium_info_), + LOG_TRACE("success to save medium info for leader", K(ret), K_(tablet_id), KPC(medium_info_), K(max_saved_version_), K_(clog_scn)); } return ret; } -void ObTabletMediumCompactionInfoRecorder::sync_clog_failed_for_leader() +void ObTabletMediumCompactionInfoRecorder::on_sync_clog_failure() { submit_trans_on_mds_table(false/*is_commit*/); } @@ -283,7 +279,7 @@ int ObTabletMediumCompactionInfoRecorder::submit_trans_on_mds_table(const bool i || !tablet_handle_ptr_->is_valid() || nullptr == mds_ctx_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("medium info or tablet handle is unexpected null", K(ret), K_(ls_id), K_(tablet_id), + LOG_WARN("medium info or tablet handle is unexpected null", K(ret), K_(tablet_id), KP_(medium_info), K_(tablet_handle_ptr), KPC_(mds_ctx)); } else if (is_commit) { mds_ctx_->single_log_commit(clog_scn_, clog_scn_); @@ -358,8 +354,8 @@ int ObTabletMediumCompactionInfoRecorder::prepare_struct_in_lock( alloc_clog_buf = static_cast(buf) + alloc_buf_offset; } - if (FAILEDx(get_tablet_handle(ls_id_, tablet_id_, *tablet_handle_ptr_))) { - LOG_WARN("failed to get tablet handle", K(ret), K_(ls_id), K_(tablet_id)); + if (FAILEDx(get_tablet_handle(tablet_id_, *tablet_handle_ptr_))) { + LOG_WARN("failed to get tablet handle", K(ret), K_(tablet_id)); } else if (OB_FAIL(log_header.serialize(alloc_clog_buf, buf_len, pos))) { LOG_WARN("failed to serialize log header", K(ret)); } else if (OB_FAIL(tablet_id_.serialize(alloc_clog_buf, buf_len, pos))) { @@ -402,10 +398,10 @@ int ObTabletMediumCompactionInfoRecorder::submit_log( LOG_ERROR("fail to submit log", K(ret), K_(tablet_id), K(medium_info_)); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(submit_trans_on_mds_table(false))) { - LOG_ERROR("failed to dec ref on memtable", K(tmp_ret), K_(ls_id), K_(tablet_id)); + LOG_ERROR("failed to dec ref on memtable", K(tmp_ret), K_(tablet_id)); } } else { - LOG_INFO("success to submit medium log", K(ret), K_(ls_id), K_(tablet_id), K(medium_info_), + LOG_INFO("success to submit medium log", K(ret), K_(tablet_id), K(medium_info_), K_(clog_scn), "max_saved_version", get_max_saved_version()); } @@ -724,7 +720,6 @@ bool ObMediumCompactionInfoList::need_check_finish() const int ObMediumCompactionInfoList::get_next_schedule_info( const int64_t last_major_snapshot, const int64_t major_frozen_snapshot, - const bool is_mv_refresh_tablet, ObMediumCompactionInfo::ObCompactionType &compaction_type, int64_t &schedule_scn) const { @@ -734,8 +729,7 @@ int ObMediumCompactionInfoList::get_next_schedule_info( // finished, this medium info could recycle } else { if (info->is_medium_compaction() - || info->medium_snapshot_ <= major_frozen_snapshot - || is_mv_refresh_tablet) { + || info->medium_snapshot_ <= major_frozen_snapshot) { schedule_scn = info->medium_snapshot_; compaction_type = (ObMediumCompactionInfo::ObCompactionType)info->compaction_type_; } diff --git a/src/storage/compaction/ob_medium_compaction_mgr.h b/src/storage/compaction/ob_medium_compaction_mgr.h index 255db0355..dc6a11951 100644 --- a/src/storage/compaction/ob_medium_compaction_mgr.h +++ b/src/storage/compaction/ob_medium_compaction_mgr.h @@ -43,7 +43,6 @@ class ObTabletMediumCompactionInfoRecorder : public storage::ObIStorageClogRecor ObTabletMediumCompactionInfoRecorder(); ~ObTabletMediumCompactionInfoRecorder(); int init( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t max_saved_version, logservice::ObLogHandler *log_handler); @@ -57,15 +56,15 @@ class ObTabletMediumCompactionInfoRecorder : public storage::ObIStorageClogRecor const common::ObTabletID &tablet_id, const ObMediumCompactionInfo &medium_info, const logservice::ObLogBaseHeader *log_header); - INHERIT_TO_STRING_KV("ObIStorageClogRecorder", ObIStorageClogRecorder, K_(ignore_medium), K_(ls_id), K_(tablet_id)); + INHERIT_TO_STRING_KV("ObIStorageClogRecorder", ObIStorageClogRecorder, K_(ignore_medium), K_(tablet_id)); private: virtual int inner_replay_clog( const int64_t update_version, const share::SCN &scn, const char *buf, const int64_t size, - int64_t &pos) override; virtual int sync_clog_succ_for_leader(const int64_t update_version) override; - virtual void sync_clog_failed_for_leader() override; + int64_t &pos) override; virtual int on_sync_clog_success(const int64_t update_version) override; + virtual void on_sync_clog_failure() override; virtual int prepare_struct_in_lock( int64_t &update_version, @@ -86,7 +85,6 @@ class ObTabletMediumCompactionInfoRecorder : public storage::ObIStorageClogRecor private: bool is_inited_; bool ignore_medium_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; storage::ObTabletHandle *tablet_handle_ptr_; ObMediumCompactionInfo *medium_info_; @@ -138,7 +136,6 @@ class ObMediumCompactionInfoList final int get_next_schedule_info( const int64_t last_major_snapshot, const int64_t major_frozen_snapshot, - const bool is_mv_refresh_tablet, ObMediumCompactionInfo::ObCompactionType &compaction_type, int64_t &schedule_scn) const; OB_INLINE ObMediumCompactionInfo::ObCompactionType get_last_compaction_type() const diff --git a/src/storage/compaction/ob_medium_loop.cpp b/src/storage/compaction/ob_medium_loop.cpp index 6f558d0d0..31da038d3 100644 --- a/src/storage/compaction/ob_medium_loop.cpp +++ b/src/storage/compaction/ob_medium_loop.cpp @@ -54,8 +54,8 @@ int ObMediumLoop::init(const int64_t batch_size) if (OB_UNLIKELY(merge_version_ <= 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid merge_version", KR(ret), K_(merge_version)); - } else if (OB_FAIL(ls_tablet_iter_.build_iter(batch_size))) { - LOG_WARN("failed to init ls iterator", K(ret)); + } else if (OB_FAIL(tablet_iter_.build_iter(batch_size))) { + LOG_WARN("failed to init tablet iterator", K(ret)); } return ret; } @@ -66,87 +66,71 @@ int ObMediumLoop::loop() int tmp_ret = OB_SUCCESS; ObTenantTabletScheduler *scheduler = share::g_mp->tenant_tablet_scheduler(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; ObScheduleTabletFunc func(merge_version_, loop_cnt_); - ObLSID ls_id; - schedule_stats_.all_ls_weak_read_ts_ready_ = true; - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_tablet_iter_.get_next_ls(ls_handle))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get ls", K(ret), K_(ls_tablet_iter)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls)); - } else if (FALSE_IT(ls_id = ls->get_ls_id())) { - } else if (OB_TMP_FAIL(loop_in_ls(ls_handle, func))) { - LOG_TRACE("failed to loop ls", KR(ret), KPC(ls), K(func)); - ls_tablet_iter_.skip_cur_ls(); // for any errno, skip cur ls - ls_tablet_iter_.update_merge_finish(false); - if (OB_SIZE_OVERFLOW == tmp_ret) { - break; - } else if (!schedule_ignore_error(tmp_ret)) { - LOG_ERROR("failed to schedule ls merge", K(tmp_ret), K(ls_id)); + schedule_stats_.weak_read_ts_ready_ = true; + if (!tablet_iter_.is_scan_finish()) { + ObLS *ls = tablet_iter_.get_ls(); + if (OB_TMP_FAIL(loop_tablets(ls, func))) { + LOG_TRACE("failed to scan tablets", KR(ret), K(func)); + tablet_iter_.finish_scan(); + tablet_iter_.update_merge_finish(false); + if (OB_SIZE_OVERFLOW != tmp_ret && !schedule_ignore_error(tmp_ret)) { + LOG_ERROR("failed to schedule merge", K(tmp_ret)); } } - if (OB_SUCC(ret) && ls_tablet_iter_.need_report_scn()) { + if (OB_SUCC(ret) && tablet_iter_.need_report_scn()) { // loop tablet_meta table to update smaller report_scn because of migration tmp_ret = update_report_scn_as_ls_leader(*ls, func); #ifndef ERRSIM - LOG_INFO("try to update report scn as ls leader", K(tmp_ret), K(ls_id)); // low printing frequency + LOG_INFO("try to update report scn", K(tmp_ret)); // low printing frequency #endif } - } // while + } add_event_and_diagnose(func); - LOG_TRACE("finish schedule ls medium merge", K(tmp_ret), K(ret), K_(ls_tablet_iter), K(ls_id)); + LOG_TRACE("finish scheduling medium merge", K(tmp_ret), K(ret), K_(tablet_iter)); return ret; } -int ObMediumLoop::loop_in_ls( - ObLSHandle &ls_handle, +int ObMediumLoop::loop_tablets( + ObLS *ls, ObScheduleTabletFunc &func) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - if (OB_FAIL(func.switch_ls(ls_handle))) { + if (OB_FAIL(func.init(ls))) { if (OB_STATE_NOT_MATCH != ret) { - LOG_ERROR("failed to switch ls", KR(ret), K(ls_id), K(func)); + LOG_ERROR("failed to initialize compaction status", KR(ret), K(func)); } else { - ls_tablet_iter_.update_merge_finish(false); - schedule_stats_.all_ls_weak_read_ts_ready_ = false; + tablet_iter_.update_merge_finish(false); + schedule_stats_.weak_read_ts_ready_ = false; } } else { ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObTabletID tablet_id; bool tablet_merge_finish = false; - while (OB_SUCC(ret)) { // loop all tablet in ls - if (OB_FAIL(ls_tablet_iter_.get_next_tablet(tablet_handle))) { + while (OB_SUCC(ret)) { // process the remaining tablets + if (OB_FAIL(tablet_iter_.get_next_tablet(tablet_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_handle)); } } else if (OB_UNLIKELY(!tablet_handle.is_valid() || nullptr == (tablet = tablet_handle.get_obj()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet handle is invalid", KR(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("tablet handle is invalid", KR(ret), K(tablet_handle)); } else if (FALSE_IT(tablet_id = tablet->get_tablet_id())) { } else if (tablet_id.is_ls_inner_tablet()) { // do nothing } else if (OB_TMP_FAIL(func.schedule_tablet(tablet_handle, tablet_merge_finish))) { if (OB_STATE_NOT_MATCH != tmp_ret) { - LOG_ERROR("failed to schedule tablet", KR(tmp_ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to schedule tablet", KR(tmp_ret), K(tablet_id)); } - ls_tablet_iter_.update_merge_finish(false); + tablet_iter_.update_merge_finish(false); } else { - ls_tablet_iter_.update_merge_finish(tablet_merge_finish); + tablet_iter_.update_merge_finish(tablet_merge_finish); } } // while } @@ -157,10 +141,10 @@ void ObMediumLoop::add_event_and_diagnose(const ObScheduleTabletFunc &func) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - if (!ls_tablet_iter_.tenant_merge_finish() && merge_version_ > ObBasicMergeScheduler::INIT_COMPACTION_SCN) { + if (!tablet_iter_.tenant_merge_finish() && merge_version_ > ObBasicMergeScheduler::INIT_COMPACTION_SCN) { // not finish cur merge_version - if (schedule_stats_.all_ls_weak_read_ts_ready_) { // check schedule Timer Task - if (schedule_stats_.add_weak_read_ts_event_flag_ && ls_tablet_iter_.is_scan_finish()) { // all ls scan finish + if (schedule_stats_.weak_read_ts_ready_) { // check schedule Timer Task + if (schedule_stats_.add_weak_read_ts_event_flag_ && tablet_iter_.is_scan_finish()) { schedule_stats_.add_weak_read_ts_event_flag_ = false; ADD_COMPACTION_EVENT( merge_version_, @@ -172,7 +156,7 @@ void ObMediumLoop::add_event_and_diagnose(const ObScheduleTabletFunc &func) schedule_stats_.check_weak_read_ts_cnt_++; } - if (ls_tablet_iter_.is_scan_finish()) { + if (tablet_iter_.is_scan_finish()) { loop_cnt_++; if (REACH_THREAD_TIME_INTERVAL(ADD_LOOP_EVENT_INTERVAL)) { ADD_COMPACTION_EVENT( @@ -186,11 +170,11 @@ void ObMediumLoop::add_event_and_diagnose(const ObScheduleTabletFunc &func) } const int64_t merged_version = ObBasicMergeScheduler::get_merge_scheduler()->get_merged_version(); - if (ls_tablet_iter_.tenant_merge_finish() && merge_version_ > merged_version) { + if (tablet_iter_.tenant_merge_finish() && merge_version_ > merged_version) { ObBasicMergeScheduler::get_merge_scheduler()->update_merged_version(merge_version_); LOG_INFO("all tablet major merge finish", K(merged_version), K_(loop_cnt)); - DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); + DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_TABLET_ID, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); if (OB_TMP_FAIL(share::g_mp->tenant_compaction_progress_mgr()->finish_progress(merge_version_))) { LOG_WARN("failed to finish progress", K(tmp_ret), K_(merge_version)); } @@ -205,8 +189,8 @@ void ObMediumLoop::add_event_and_diagnose(const ObScheduleTabletFunc &func) } LOG_INFO("finish schedule all tablet merge", K(merge_version_), K(schedule_stats_), K_(loop_cnt), - "tenant_merge_finish", ls_tablet_iter_.tenant_merge_finish(), - "is_scan_all_tablet_finish", ls_tablet_iter_.is_scan_finish(), + "tenant_merge_finish", tablet_iter_.tenant_merge_finish(), + "is_scan_all_tablet_finish", tablet_iter_.is_scan_finish(), "schedule_tablet_cnt", func.get_schedule_tablet_cnt(), "time_guard", func.get_time_guard()); } @@ -215,16 +199,15 @@ void ObMediumLoop::add_event_and_diagnose(const ObScheduleTabletFunc &func) int ObMediumLoop::update_report_scn_as_ls_leader(ObLS &ls, const ObScheduleTabletFunc &func) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); const int64_t inner_table_merged_scn = ObBasicMergeScheduler::get_merge_scheduler()->get_inner_table_merged_scn(); const ObLSStatusCache &ls_status = func.get_ls_status(); - if (ls_status.can_merge() && ls_status.is_leader_) { + if (ls_status.can_merge()) { ObSEArray tablet_id_array; if (OB_FAIL(ls.get_tablet_svr()->get_all_tablet_ids(true/*except_ls_inner_tablet*/, tablet_id_array))) { - LOG_WARN("failed to get tablet id", K(ret), K(ls_id)); + LOG_WARN("failed to get tablet id", K(ret)); } else if (inner_table_merged_scn > ObBasicMergeScheduler::INIT_COMPACTION_SCN - && OB_FAIL(ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet(ls_id, inner_table_merged_scn, tablet_id_array))) { - LOG_WARN("failed to get unequal report scn", K(ret), K(ls_id), K(inner_table_merged_scn)); + && OB_FAIL(ObTabletMetaTableCompactionOperator::batch_update_unequal_report_scn_tablet(inner_table_merged_scn, tablet_id_array))) { + LOG_WARN("failed to get unequal report scn", K(ret), K(inner_table_merged_scn)); } } else { ret = OB_LS_LOCATION_LEADER_NOT_EXIST; @@ -236,37 +219,29 @@ int ObMediumLoop::update_report_scn_as_ls_leader(ObLS &ls, const ObScheduleTable int ObScheduleNewMediumLoop::loop() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; const int64_t frozen_version = ObBasicMergeScheduler::get_merge_scheduler()->get_frozen_version(); ObScheduleTabletFunc func(frozen_version); - // sort tablet ls info - if (OB_FAIL(sort_tablet_ls_info())) { + // sort tablet check info + if (OB_FAIL(sort_tablet_check_info())) { LOG_WARN("failed to sort", KR(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(func.init(ls))) { + if (OB_STATE_NOT_MATCH != ret) { + LOG_ERROR("failed to initialize compaction status", KR(ret)); + } else { + LOG_WARN("not support schedule medium", K(ret), K(func)); + } } - for (int64_t i = 0, idx = 0; i < tablet_ls_infos_.count(); ++i) { // ignore OB_FAIL - const ObLSID &ls_id = tablet_ls_infos_.at(i).get_ls_id(); - const ObTabletID &tablet_id = tablet_ls_infos_.at(i).get_tablet_id(); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_check_infos_.count(); ++i) { // ignore OB_FAIL + const ObTabletID &tablet_id = tablet_check_infos_.at(i).get_tablet_id(); ObTabletHandle tablet_handle; - if (func.get_ls_status().ls_id_ == ls_id) { - // do nothing, use old ls_handle - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(func.switch_ls(ls_handle))) { - if (OB_STATE_NOT_MATCH != ret) { - LOG_ERROR("failed to switch ls", KR(ret), K(ls_id), K(ls_id)); - } else { - LOG_WARN("not support schedule medium for ls", K(ret), K(ls_id), K(tablet_id), K(func)); - } - } - if (OB_FAIL(ret)) { - } else if (!func.get_ls_status().is_leader_) { - // not leader, can't schedule - LOG_TRACE("not ls leader, can't schedule medium", K(ret), K(ls_id), K(tablet_id), K(func)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( + if (OB_FAIL(ls->get_tablet_svr()->get_tablet( tablet_id, tablet_handle, 0 /*timeout_us*/))) { - LOG_WARN("get tablet failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } else if (OB_FAIL(func.request_schedule_new_round(tablet_handle, false/*user_request*/))) { - LOG_WARN("get tablet failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } } // end of for ret = OB_SUCCESS; @@ -274,24 +249,24 @@ int ObScheduleNewMediumLoop::loop() return ret; } -struct ObTabletLSInfoComparator final { +struct ObTabletCheckInfoComparator final { public: - ObTabletLSInfoComparator(int &sort_ret) + ObTabletCheckInfoComparator(int &sort_ret) : result_code_(sort_ret) {} bool operator()(const ObTabletCheckInfo &lhs, const ObTabletCheckInfo &rhs) { - return lhs.get_ls_id() < rhs.get_ls_id(); + return lhs.get_tablet_id().id() < rhs.get_tablet_id().id(); } int &result_code_; }; -int ObScheduleNewMediumLoop::sort_tablet_ls_info() +int ObScheduleNewMediumLoop::sort_tablet_check_info() { int ret = OB_SUCCESS; - if (tablet_ls_infos_.count() > 2) { - ObTabletLSInfoComparator cmp(ret); - ob_sort(tablet_ls_infos_.begin(), tablet_ls_infos_.end(), cmp); + if (tablet_check_infos_.count() > 2) { + ObTabletCheckInfoComparator cmp(ret); + ob_sort(tablet_check_infos_.begin(), tablet_check_infos_.end(), cmp); } return ret; } diff --git a/src/storage/compaction/ob_medium_loop.h b/src/storage/compaction/ob_medium_loop.h index 51f7a408d..6dd530615 100644 --- a/src/storage/compaction/ob_medium_loop.h +++ b/src/storage/compaction/ob_medium_loop.h @@ -29,7 +29,7 @@ struct ObMediumLoop ObMediumLoop() : merge_version_(ObBasicMergeScheduler::INIT_COMPACTION_SCN), loop_cnt_(0), - ls_tablet_iter_(true/*is_major*/) + tablet_iter_(true/*is_major*/) {} ~ObMediumLoop() {} int start_merge(const int64_t merge_version); @@ -42,8 +42,8 @@ struct ObMediumLoop || OB_LS_NOT_EXIST == ret; } private: - int loop_in_ls( - storage::ObLSHandle &ls_handle, + int loop_tablets( + storage::ObLS *ls, ObScheduleTabletFunc &tablet_schedule_func); int update_report_scn_as_ls_leader( storage::ObLS &ls, @@ -55,20 +55,20 @@ struct ObMediumLoop int64_t merge_version_; int64_t loop_cnt_; ObScheduleStatistics schedule_stats_; - ObCompactionScheduleIterator ls_tablet_iter_; + ObCompactionScheduleIterator tablet_iter_; }; struct ObScheduleNewMediumLoop { ObScheduleNewMediumLoop( - ObArray &tablet_ls_infos) - : tablet_ls_infos_(tablet_ls_infos) + ObArray &tablet_check_infos) + : tablet_check_infos_(tablet_check_infos) {} ~ObScheduleNewMediumLoop() {} int loop(); - int sort_tablet_ls_info(); + int sort_tablet_check_info(); private: - ObArray &tablet_ls_infos_; + ObArray &tablet_check_infos_; }; } // namespace compaction diff --git a/src/storage/compaction/ob_partition_merge_fuser.cpp b/src/storage/compaction/ob_partition_merge_fuser.cpp index b9e8b6175..da8dee668 100644 --- a/src/storage/compaction/ob_partition_merge_fuser.cpp +++ b/src/storage/compaction/ob_partition_merge_fuser.cpp @@ -18,6 +18,7 @@ #include "ob_partition_merge_fuser.h" #include "ob_tablet_merge_ctx.h" +#include "storage/lob/ob_lob_manager.h" namespace oceanbase { diff --git a/src/storage/compaction/ob_partition_merge_iter.cpp b/src/storage/compaction/ob_partition_merge_iter.cpp index 7958f48fb..0ffd3e8d9 100644 --- a/src/storage/compaction/ob_partition_merge_iter.cpp +++ b/src/storage/compaction/ob_partition_merge_iter.cpp @@ -16,10 +16,8 @@ #define USING_LOG_PREFIX STORAGE_COMPACTION #include "ob_partition_merge_iter.h" -#include "share/rc/ob_module_provider.h" #include "ob_tablet_merge_ctx.h" -#include "storage/compaction/ob_mview_compaction_util.h" -#include "observer/ob_inner_sql_result.h" +#include "storage/memtable/ob_memtable_iterator.h" namespace oceanbase { @@ -56,7 +54,7 @@ ObPartitionMergeIter::ObPartitionMergeIter(common::ObIAllocator &allocator) is_rowkey_first_row_already_output_(false), is_rowkey_shadow_row_reused_(false), is_delete_insert_merge_(false), - is_ha_compeleted_(true) + is_restore_complete_(true) { } @@ -91,7 +89,7 @@ void ObPartitionMergeIter::reset() is_rowkey_first_row_already_output_ = false; is_rowkey_shadow_row_reused_ = false; is_delete_insert_merge_ = false; - is_ha_compeleted_ = true; + is_restore_complete_ = true; ObMergeIter::reset(); } @@ -111,7 +109,7 @@ int ObPartitionMergeIter::init_query_base_params(const ObMergeParameter &merge_p LOG_WARN("Failed to init table access param", K(ret), KPC(this)); } else if (OB_FAIL(snapshot_version.convert_for_tx(merge_param.merge_version_range_.snapshot_version_))) { LOG_WARN("Failed to convert", K(ret), K_(merge_param.merge_version_range_.snapshot_version)); - } else if (OB_FAIL(store_ctx_.init_for_read(static_param.ls_handle_, + } else if (OB_FAIL(store_ctx_.init_for_read(static_param.ls_, INT64_MAX, // query_expire_ts -1, // lock_timeout_us snapshot_version))) { @@ -261,7 +259,7 @@ int64_t ObPartitionMergeIter::to_string(char *buf, const int64_t buf_len) const J_COMMA(); } J_KV(K_(iter_row_count), KPC(curr_row_), K_(iter_row_id), K_(last_macro_block_reused), - K_(is_rowkey_first_row_already_output), K_(is_base_iter), K_(is_delete_insert_merge), K_(is_ha_compeleted)); + K_(is_rowkey_first_row_already_output), K_(is_base_iter), K_(is_delete_insert_merge), K_(is_restore_complete)); } else { J_KV(K_(is_inited)); } @@ -1036,7 +1034,7 @@ int ObPartitionMinorRowMergeIter::common_minor_inner_init(const ObMergeParameter void *buf = nullptr; check_committing_trans_compacted_ = true; is_delete_insert_merge_ = merge_param.is_delete_insert_merge(); - is_ha_compeleted_ = merge_param.is_ha_compeleted(); + is_restore_complete_ = merge_param.is_restore_complete(); if (OB_FAIL(merge_param.get_schema()->get_stored_column_count_in_sstable(row_column_cnt))) { LOG_WARN("Failed to get full store column count", K(ret)); } else if (OB_FAIL(row_queue_.init(row_column_cnt))) { @@ -1939,179 +1937,5 @@ int ObPartitionMinorMacroMergeIter::get_curr_macro_block( return ret; } -ObPartitionMVRowMergeIter::ObMVSqlResource::ObMVSqlResource() - : read_result_(), - free_session_ctx_(), - session_(nullptr), - conn_(nullptr), - sql_result_(nullptr) -{ -} - -ObPartitionMVRowMergeIter::ObMVSqlResource::~ObMVSqlResource() -{ - read_result_.~ReadResult(); // need decons before session - ObMviewCompactionHelper::release_inner_connection(conn_); - ObMviewCompactionHelper::release_inner_session(free_session_ctx_, session_); - sql_result_ = nullptr; -} - -/* - * ObPartitionRowMergeIter used for mv major merge - */ -ObPartitionMVRowMergeIter::ObPartitionMVRowMergeIter(common::ObIAllocator &allocator) - : ObPartitionMergeIter(allocator), - is_delete_(false), - is_replace_(false), - sql_idx_(-1), - sql_read_col_cnt_(0), - store_col_cnt_(0), - result_row_(), - mv_sql_resource_() -{ -} - -ObPartitionMVRowMergeIter::~ObPartitionMVRowMergeIter() -{ -} - -int ObPartitionMVRowMergeIter::init(const ObMergeParameter &merge_param, - const int64_t refresh_sql_idx, - const ObITableReadInfo *read_info) -{ - int ret = OB_SUCCESS; - int64_t store_column_cnt = 0; - sql_idx_ = refresh_sql_idx; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObPartitionMVRowMergeIter init twice", K(ret)); - } else if (OB_UNLIKELY(!merge_param.is_valid() - || refresh_sql_idx < 0 || refresh_sql_idx >= merge_param.mview_merge_param_->refresh_sql_count_ - || nullptr == read_info)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("Invalid arguments to init ObPartitionMVRowMergeIter", K(ret), - K(merge_param), K(refresh_sql_idx), KP(read_info)); - } else if (OB_UNLIKELY(!inner_check(merge_param))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("Invalid argument to inner init", K(ret), K(*this), K(merge_param)); - } else if (OB_FAIL(inner_init(merge_param))) { - LOG_WARN("Failed to inner init", K(ret)); - } else if (OB_FAIL(merge_param.get_schema()->get_store_column_count(store_column_cnt, true /* full_col, unused */))) { - LOG_WARN("Failed to get store column cnt", K(ret)); - } else if (OB_FAIL(result_row_.init(allocator_, store_column_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()))) { - LOG_WARN("Failed to init result row", K(ret), K(store_column_cnt)); - } else { - is_delete_ = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].is_delete(); - is_replace_ = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].is_replace(); - schema_rowkey_column_cnt_ = read_info->get_schema_rowkey_count(); - result_row_.count_ = store_column_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - result_row_.storage_datums_[schema_rowkey_column_cnt_].set_int(-merge_param.merge_version_range_.snapshot_version_); - result_row_.storage_datums_[schema_rowkey_column_cnt_ + 1].set_int(0); - sql_read_col_cnt_ = is_replace_ ? store_column_cnt + 1 : // normal columns + old_new$$ - is_delete_ ? schema_rowkey_column_cnt_ : store_column_cnt; - store_col_cnt_ = is_replace_ ? sql_read_col_cnt_ - 1 : sql_read_col_cnt_; - iter_row_count_ = 0; - iter_row_id_ = -1; - curr_row_ = nullptr; - iter_end_ = false; - is_base_iter_ = false; - is_inited_ = true; - LOG_INFO("[MVIEW COMPACTION]: Succ to init partition mv merge iter", K(ret), K(*this)); - } - return ret; -} - -bool ObPartitionMVRowMergeIter::inner_check(const ObMergeParameter &merge_param) -{ - bool bret = true; - if (OB_UNLIKELY(is_multi_version_merge(merge_param.static_param_.get_merge_type()) || - !merge_param.is_mv_merge())) { - bret = false; - LOG_WARN_RET(OB_ERR_UNEXPECTED, "Unexpected merge type for mv merge", K(merge_param)); - } else if (OB_UNLIKELY(!merge_param.mview_merge_param_->is_valid())) { - bret = false; - LOG_WARN_RET(OB_ERR_UNEXPECTED, "Invalid mv merge param", K(merge_param)); - } - return bret; -} - -int ObPartitionMVRowMergeIter::inner_init(const ObMergeParameter &merge_param) -{ - int ret = OB_SUCCESS; - const ObSqlString &sql = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].sql_; - if (OB_FAIL(ObMviewCompactionHelper::create_inner_session(merge_param.mview_merge_param_->database_id_, - mv_sql_resource_.free_session_ctx_, - mv_sql_resource_.session_))) { - LOG_WARN("Failed to create inner session", K(ret), KPC(merge_param.mview_merge_param_)); - } else if (OB_FAIL(ObMviewCompactionHelper::create_inner_connection(mv_sql_resource_.session_, mv_sql_resource_.conn_))) { - LOG_WARN("Failed to create inner connection", K(ret), K_(sql_idx)); - } else if (OB_FAIL(mv_sql_resource_.conn_->execute_read(GCONF.cluster_id, sql.ptr(), mv_sql_resource_.read_result_))) { - LOG_WARN("Failed to execute", K(ret), K_(sql_idx), K(sql)); - } else if (OB_ISNULL(mv_sql_resource_.sql_result_ = static_cast(mv_sql_resource_.read_result_.get_result()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected null sql result", K(ret), K_(sql_idx), K(sql)); - } - return ret; -} - -int ObPartitionMVRowMergeIter::next() -{ - int ret = OB_SUCCESS; - bool is_delete_row = is_delete_; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObPartitionRowMergeIter is not inited", K(ret), K(*this)); - } else if (OB_UNLIKELY(iter_end_)) { - ret = OB_ITER_END; - } else if (FALSE_IT(curr_row_ = nullptr)) { - } else if (OB_FAIL(mv_sql_resource_.sql_result_->next())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("Failed to get next row", K(ret)); - } else { - iter_end_ = true; - } - } else { - const ObNewRow *new_row = mv_sql_resource_.sql_result_->get_row(); - if (OB_UNLIKELY(nullptr == new_row || sql_read_col_cnt_ != new_row->get_count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected result row", K(ret), K(schema_rowkey_column_cnt_), K(sql_read_col_cnt_), KPC(new_row)); - } - int64_t read_idx = 0; - for (; OB_SUCC(ret) && read_idx < schema_rowkey_column_cnt_; ++read_idx) { - if (OB_FAIL(result_row_.storage_datums_[read_idx].from_obj(new_row->get_cell(read_idx)))) { - LOG_WARN("Failed to convert obj to datum", K(ret)); - } - } - for (; OB_SUCC(ret) && read_idx < store_col_cnt_; ++read_idx) { - const int64_t datum_idx = read_idx + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - if (OB_FAIL(result_row_.storage_datums_[datum_idx].from_obj(new_row->get_cell(read_idx)))) { - LOG_WARN("Failed to convert obj to datum", K(ret)); - } - } - if (OB_SUCC(ret) && is_replace_) { - const ObObj &old_new_obj = new_row->get_cell(read_idx); - if (OB_UNLIKELY(old_new_obj.is_nop_value() || old_new_obj.is_null())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected null old_new$$ value", K(ret), K(old_new_obj)); - } else if (0 == ObString::make_string(ObMviewScanInfo::OLD_ROW).case_compare(old_new_obj.get_string())) { - is_delete_row = true; - } else if (0 == ObString::make_string(ObMviewScanInfo::NEW_ROW).case_compare(old_new_obj.get_string())) { - is_delete_row = false; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected old_new$$ value", K(ret), K(old_new_obj)); - } - } - } - if (OB_SUCC(ret)) { - is_delete_row ? result_row_.row_flag_.set_flag(ObDmlFlag::DF_DELETE) : result_row_.row_flag_.set_flag(ObDmlFlag::DF_INSERT); - curr_row_ = &result_row_; - iter_row_count_++; - iter_row_id_++; - LOG_DEBUG("[MVIEW COMPACTION]: iter next row", K(ret), K(*this), K_(result_row)); - } - return ret; -} - } //compaction } //oceanbase diff --git a/src/storage/compaction/ob_partition_merge_iter.h b/src/storage/compaction/ob_partition_merge_iter.h index 147f722ea..7ace337db 100644 --- a/src/storage/compaction/ob_partition_merge_iter.h +++ b/src/storage/compaction/ob_partition_merge_iter.h @@ -33,7 +33,6 @@ #include "storage/access/ob_table_access_param.h" #include "storage/access/ob_table_access_context.h" #include "storage/access/ob_micro_block_handle_mgr.h" -#include "sql/session/ob_sql_session_mgr.h" namespace oceanbase { @@ -43,11 +42,6 @@ namespace storage { struct ObTransNodeDMLStat; } -namespace observer -{ -class ObInnerSQLResult; -} - namespace compaction { struct ObMergeParameter; @@ -172,7 +166,7 @@ class ObPartitionMergeIter : public ObMergeIter bool is_rowkey_shadow_row_reused_; bool is_reserve_mode_; bool is_delete_insert_merge_; - bool is_ha_compeleted_; + bool is_restore_complete_; }; class ObPartitionRowMergeIter : public ObPartitionMergeIter @@ -298,7 +292,7 @@ class ObPartitionMinorRowMergeIter : public ObPartitionMergeIter const int64_t multi_version_start = access_context_.trans_version_range_.multi_version_start_; if (nullptr != curr_row_ && !curr_row_->is_uncommitted_row() && !curr_row_->is_last_multi_version_row()) { const int64_t commit_version = -curr_row_->storage_datums_[schema_rowkey_column_cnt_].get_int(); - if (is_delete_insert_merge_ && (!is_ha_compeleted_ || base_version <= 0)) { + if (is_delete_insert_merge_ && (!is_restore_complete_ || base_version <= 0)) { need_recycle = false; } else if (commit_version <= multi_version_start) { need_recycle = true; @@ -377,40 +371,6 @@ class ObPartitionMinorMacroMergeIter : public ObPartitionMinorRowMergeIter const bool reuse_uncommit_row_; }; -class ObPartitionMVRowMergeIter final : public ObPartitionMergeIter -{ -public: - struct ObMVSqlResource - { - ObMVSqlResource(); - ~ObMVSqlResource(); - TO_STRING_KV(K_(free_session_ctx), KP_(session), KP_(conn), KP_(sql_result)); - ObISQLClient::ReadResult read_result_; - sql::ObFreeSessionCtx free_session_ctx_; - sql::ObSQLSessionInfo *session_; - sqlclient::ObISQLConnection *conn_; - observer::ObInnerSQLResult *sql_result_; - }; - ObPartitionMVRowMergeIter(common::ObIAllocator &allocator); - virtual ~ObPartitionMVRowMergeIter(); - virtual int init(const ObMergeParameter &merge_param, - const int64_t refresh_sql_idx, - const ObITableReadInfo *read_info) override; - virtual int next() override; - TO_STRING_KV(K_(is_delete), K_(is_replace), K_(sql_idx), K_(sql_read_col_cnt), K_(store_col_cnt), K_(mv_sql_resource)); -protected: - virtual int inner_init(const ObMergeParameter &merge_param) override; - virtual bool inner_check(const ObMergeParameter &merge_param) override; -private: - bool is_delete_; - bool is_replace_; - int64_t sql_idx_; - int64_t sql_read_col_cnt_; - int64_t store_col_cnt_; - blocksstable::ObDatumRow result_row_; - ObMVSqlResource mv_sql_resource_; -}; - static const int64_t DEFAULT_ITER_COUNT = 16; static const int64_t DEFAULT_ITER_ARRAY_SIZE = DEFAULT_ITER_COUNT * sizeof(ObPartitionMergeIter *); typedef common::ObSEArray MERGE_ITER_ARRAY; diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index 4aab471f2..d2deac31f 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -640,9 +640,9 @@ int ObPartitionMergePolicy::find_minor_merge_tables( #endif if (OB_SUCC(ret)) { DEL_SUSPECT_INFO(MINOR_MERGE, - tablet.get_tablet_meta().ls_id_, tablet.get_tablet_meta().tablet_id_, + tablet.get_tablet_meta().tablet_id_, ObDiagnoseTabletType::TYPE_MINOR_MERGE); - if (OB_FAIL(refine_minor_merge_result(param.merge_type_, minor_compact_trigger, tablet.is_tablet_referenced_by_collect_mv(), result))) { + if (OB_FAIL(refine_minor_merge_result(param.merge_type_, minor_compact_trigger, result))) { if (OB_NO_NEED_MERGE != ret) { LOG_WARN("failed to refine minor_merge result", K(ret)); } @@ -657,7 +657,6 @@ int ObPartitionMergePolicy::find_minor_merge_tables( } } else if (OB_NO_NEED_MERGE == ret && tablet.get_minor_table_count() >= diagnose_table_cnt) { if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MINOR_MERGE, ObDiagnoseTabletType::TYPE_MINOR_MERGE, - tablet.get_tablet_meta().ls_id_, tablet.get_tablet_meta().tablet_id_, ObSuspectInfoType::SUSPECT_CANT_SCHEDULE_MINOR_MERGE, min_snapshot_version, max_snapshot_version, tablet.get_minor_table_count()))) { @@ -747,7 +746,7 @@ int ObPartitionMergePolicy::deal_hist_minor_merge( ObTabletMemberWrapper wrapper; ObSEArray freeze_infos; - if (FALSE_IT(hist_threshold = cal_hist_minor_merge_threshold(tablet.is_tablet_referenced_by_collect_mv()))) { + if (FALSE_IT(hist_threshold = cal_hist_minor_merge_threshold())) { } else if (OB_FAIL(tablet.fetch_table_store(wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_UNLIKELY(!wrapper.get_member()->is_valid())) { @@ -811,7 +810,6 @@ int ObPartitionMergePolicy::deal_hist_minor_merge( int ObPartitionMergePolicy::diagnose_minor_dag( ObMergeType merge_type, - const ObLSID ls_id, const ObTabletID tablet_id, char *buf, const int64_t buf_len) @@ -821,13 +819,12 @@ int ObPartitionMergePolicy::diagnose_minor_dag( ObDiagnoseTabletCompProgress progress; if (OB_FAIL(ObCompactionDiagnoseMgr::diagnose_dag( merge_type, - ls_id, tablet_id, ObVersion::MIN_VERSION, dag, progress))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to init dag", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to init dag", K(ret), K(tablet_id)); } else { // no minor merge dag ret = OB_SUCCESS; @@ -851,7 +848,6 @@ int ObPartitionMergePolicy::diagnose_table_count_unsafe( } // check min_reserved_snapshot - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; int64_t min_merged_snapshot = INT64_MAX; int64_t first_minor_start_scn = 0; @@ -881,15 +877,15 @@ int ObPartitionMergePolicy::diagnose_table_count_unsafe( // check have minor merge DAG const int64_t buf_len = compaction::OB_DIAGNOSE_INFO_PARAM_STR_LENGTH; char dag_str[buf_len] = "\0"; - if (OB_TMP_FAIL(diagnose_minor_dag(MINOR_MERGE, ls_id, tablet_id, dag_str, buf_len))) { - LOG_WARN("failed to diagnose minor dag", K(tmp_ret), K(ls_id), K(tablet_id), K(dag_str)); + if (OB_TMP_FAIL(diagnose_minor_dag(MINOR_MERGE, tablet_id, dag_str, buf_len))) { + LOG_WARN("failed to diagnose minor dag", K(tmp_ret), K(tablet_id), K(dag_str)); } - if (OB_TMP_FAIL(diagnose_minor_dag(HISTORY_MINOR_MERGE, ls_id, tablet_id, dag_str, buf_len))) { - LOG_WARN("failed to diagnose history minor dag", K(tmp_ret), K(ls_id), K(tablet_id), K(dag_str)); + if (OB_TMP_FAIL(diagnose_minor_dag(HISTORY_MINOR_MERGE, tablet_id, dag_str, buf_len))) { + LOG_WARN("failed to diagnose history minor dag", K(tmp_ret), K(tablet_id), K(dag_str)); } if (OB_TMP_FAIL(ADD_SUSPECT_INFO(merge_type, diagnose_type, - ls_id, tablet_id, ObSuspectInfoType::SUSPECT_SSTABLE_COUNT_NOT_SAFE, + tablet_id, ObSuspectInfoType::SUSPECT_SSTABLE_COUNT_NOT_SAFE, minor_compact_trigger, major_table_count, minor_table_count, first_minor_start_scn, "snapshot", snapshot_info, "dag", dag_str))) { @@ -938,7 +934,6 @@ int ObPartitionMergePolicy::refine_mini_merge_result( int ObPartitionMergePolicy::refine_minor_merge_result( const ObMergeType merge_type, const int64_t minor_compact_trigger, - const bool is_tablet_referenced_by_collect_mv, ObGetMergeTablesResult &result) { int ret = OB_SUCCESS; @@ -949,7 +944,7 @@ int ObPartitionMergePolicy::refine_minor_merge_result( } else if (OB_UNLIKELY(!is_minor_merge_type(merge_type))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected merge type to refine merge tables", K(result), K(ret)); - } else if (0 == minor_compact_trigger || result.handle_.get_count() >= OB_UNSAFE_TABLE_CNT || is_tablet_referenced_by_collect_mv) { + } else if (0 == minor_compact_trigger || result.handle_.get_count() >= OB_UNSAFE_TABLE_CNT) { // no refine } else { ObTablesHandleArray mini_tables; @@ -969,15 +964,6 @@ int ObPartitionMergePolicy::refine_minor_merge_result( } } -#ifdef ERRSIM -if (OB_SUCC(ret)) { - int tmp_ret = OB_E(EventTable::EN_MV_LARGE_SSTABLE_THRESHOLD) OB_SUCCESS; - if (tmp_ret) { - write_amplification_threshold = 10; - } -} -#endif - for (int64_t i = 0; OB_SUCC(ret) && i < result.handle_.get_count(); ++i) { ObTableHandleV2 tmp_table_handle; if (OB_FAIL(result.handle_.get_table(i, tmp_table_handle))) { @@ -1040,16 +1026,10 @@ if (OB_SUCC(ret)) { return ret; } -int64_t ObPartitionMergePolicy::cal_hist_minor_merge_threshold(const bool is_tablet_referenced_by_collect_mv) +int64_t ObPartitionMergePolicy::cal_hist_minor_merge_threshold() { - int64_t compact_trigger = DEFAULT_MINOR_COMPACT_TRIGGER; - - compact_trigger = GCONF.minor_compact_trigger; - - if (!is_tablet_referenced_by_collect_mv) { - compact_trigger = MIN((1 + compact_trigger) * OB_HIST_MINOR_FACTOR, MAX_TABLE_CNT_IN_STORAGE / 2); - } - return compact_trigger; + return MIN((1 + GCONF.minor_compact_trigger) * OB_HIST_MINOR_FACTOR, + MAX_TABLE_CNT_IN_STORAGE / 2); } int ObPartitionMergePolicy::get_multi_version_start( @@ -1064,12 +1044,8 @@ int ObPartitionMergePolicy::get_multi_version_start( if (tablet.is_ls_inner_tablet()) { result_version_range.multi_version_start_ = INT64_MAX; } else if (OB_FAIL(tablet.get_kept_snapshot_info(ls.get_min_reserved_snapshot(), snapshot_info))) { - // Minor Merge need read medium list to decide boundary snapshot and multi version start. - // Bug when ls is migrating, data is not complete and mds data can not be read. - // So if the sstable cnt is unsafe, a emergency minor should be scheduled. - const bool need_emergency_minor = OB_EAGAIN == ret && is_minor_merge_type(merge_type) - && !tablet.get_tablet_meta().ha_status_.is_data_status_complete() - && (tablet.get_major_table_count() + tablet.get_minor_table_count()) > OB_UNSAFE_TABLE_CNT; + const bool need_emergency_minor = OB_EAGAIN == ret && is_minor_merge_type(merge_type) + && (tablet.get_major_table_count() + tablet.get_minor_table_count()) > OB_UNSAFE_TABLE_CNT; if (is_mini_merge(merge_type) || OB_TENANT_NOT_EXIST == ret || need_emergency_minor) { snapshot_info.reset(); snapshot_info.snapshot_type_ = ObStorageSnapshotInfo::SNAPSHOT_MULTI_VERSION_START_ON_TABLET; @@ -1267,7 +1243,6 @@ bool compareScnRange(share::ObScnRange &a, share::ObScnRange &b) } int ObMinorExecuteRangeMgr::get_merge_ranges( - const ObLSID &ls_id, const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -1275,7 +1250,6 @@ int ObMinorExecuteRangeMgr::get_merge_ranges( ObTabletMergeDagParam param; param.merge_type_ = MINOR_MERGE; param.merge_version_ = ObVersion::MIN_VERSION; - param.ls_id_ = ls_id; param.tablet_id_ = tablet_id; param.skip_get_tablet_ = true; @@ -1560,7 +1534,7 @@ int ObAdaptiveMergePolicy::find_adaptive_merge_tables( } else if (scanty_tx_determ_table || scanty_inc_row_cnt) { int tmp_ret = OB_SUCCESS; ObTableQueuingModeCfg cfg; - if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(tablet.get_ls_id(), tablet.get_tablet_id(), cfg))) { + if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(tablet.get_tablet_id(), cfg))) { LOG_WARN_RET(tmp_ret, "failed to get table queuing mode, treat it as normal table"); } if (cfg.is_queuing_mode() && scanty_inc_row_cnt && !scanty_tx_determ_table) { @@ -1639,7 +1613,6 @@ int ObAdaptiveMergePolicy::get_adaptive_merge_reason( bool crazy_medium_flag = false; int64_t truncate_info_count = 0; int64_t truncate_newest_commit_version = 0; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; reason = AdaptiveMergeReason::NONE; @@ -1651,22 +1624,22 @@ int ObAdaptiveMergePolicy::get_adaptive_merge_reason( if (tablet_id.is_special_merge_tablet()) { // do nothing - } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(ls_id, tablet_id, tablet_analyzer))) { + } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(tablet_id, tablet_analyzer))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get tablet analyzer stat", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tablet analyzer stat", K(ret), K(tablet_id)); } else if (OB_TMP_FAIL(check_inc_sstable_row_cnt_percentage(tablet, reason))) { - LOG_WARN("failed to check sstable data situation", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check sstable data situation", K(tmp_ret), K(tablet_id)); } else { ret = OB_SUCCESS; } } else if (OB_TMP_FAIL(check_adaptive_merge_reason(tablet, tablet_analyzer, reason))) { - LOG_WARN("failed to check adaptive merge reason", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check adaptive merge reason", K(tmp_ret), K(tablet_id)); } if (OB_FAIL(ret) || AdaptiveMergeReason::NONE != reason || nullptr == read_info) { } else if (read_info->is_global_index_valid()) { read_truncate_info_flag = read_info->is_global_index_table(); } else if (OB_FAIL(tablet.load_storage_schema(tmp_allocator, schema_on_tablet))) { - LOG_WARN("failed to load storage schema", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to load storage schema", K(ret), K(tablet_id)); } else { read_truncate_info_flag = schema_on_tablet->is_global_index_table(); } @@ -1676,7 +1649,7 @@ int ObAdaptiveMergePolicy::get_adaptive_merge_reason( } else if (truncate_info_count > 0) { reason = AdaptiveMergeReason::RECYCLE_TRUNCATE_INFO; least_medium_snapshot = truncate_newest_commit_version; - LOG_INFO("[TRUNCATE INFO]success to get adaptive merge reason", KR(tmp_ret), K(ls_id), K(tablet_id), K(reason), K(least_medium_snapshot)); + LOG_INFO("[TRUNCATE INFO]success to get adaptive merge reason", KR(tmp_ret), K(tablet_id), K(reason), K(least_medium_snapshot)); } if (OB_NOT_NULL(schema_on_tablet)) { schema_on_tablet->~ObStorageSchema(); @@ -1696,7 +1669,7 @@ int ObAdaptiveMergePolicy::get_adaptive_merge_reason( #endif if (REACH_THREAD_TIME_INTERVAL(10 * 1000 * 1000 /*10s*/)) { - LOG_INFO("Check tablet adaptive merge reason", K(ret), K(ls_id), K(tablet_id), K(reason), K(tablet_analyzer)); + LOG_INFO("Check tablet adaptive merge reason", K(ret), K(tablet_id), K(reason), K(tablet_analyzer)); } return ret; } @@ -1707,20 +1680,19 @@ int ObAdaptiveMergePolicy::check_tombstone_reason( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; reason = AdaptiveMergeReason::NONE; ObTabletStatAnalyzer tablet_analyzer; if (tablet_id.is_special_merge_tablet()) { // do nothing - } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(ls_id, tablet_id, tablet_analyzer))) { + } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(tablet_id, tablet_analyzer))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get tablet analyzer stat", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tablet analyzer stat", K(ret), K(tablet_id)); } else { ret = OB_SUCCESS; } } else if (OB_TMP_FAIL(check_tombstone_situation(tablet_analyzer, reason))) { - LOG_WARN("failed to check tombstone scene", K(tmp_ret), K(ls_id), K(tablet_id), K(tablet_analyzer)); + LOG_WARN("failed to check tombstone scene", K(tmp_ret), K(tablet_id), K(tablet_analyzer)); } return ret; } @@ -1730,7 +1702,6 @@ int ObAdaptiveMergePolicy::check_inc_sstable_row_cnt_percentage( AdaptiveMergeReason &reason) { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; ObSSTable *last_major = nullptr; int64_t base_row_count = 0; @@ -1759,7 +1730,7 @@ int ObAdaptiveMergePolicy::check_inc_sstable_row_cnt_percentage( (inc_row_count * 100 / base_row_count) > LOAD_DATA_SCENE_THRESHOLD)) { reason = AdaptiveMergeReason::FREQUENT_WRITE; } - LOG_DEBUG("check_sstable_data_situation", K(ret), K(ls_id), K(tablet_id), K(reason), + LOG_DEBUG("check_sstable_data_situation", K(ret), K(tablet_id), K(reason), K(base_row_count), K(inc_row_count)); return ret; } @@ -1822,19 +1793,18 @@ int ObAdaptiveMergePolicy::check_adaptive_merge_reason( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = tablet.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_id(); if (OB_TMP_FAIL(check_tombstone_situation(tablet_analyzer, reason))) { - LOG_WARN("failed to check tombstone scene", K(tmp_ret), K(ls_id), K(tablet_id), K(tablet_analyzer)); + LOG_WARN("failed to check tombstone scene", K(tmp_ret), K(tablet_id), K(tablet_analyzer)); } if (AdaptiveMergeReason::NONE == reason && OB_TMP_FAIL(check_load_data_situation(tablet_analyzer, reason))) { - LOG_WARN("failed to check load data scene", K(tmp_ret), K(ls_id), K(tablet_id), K(tablet_analyzer)); + LOG_WARN("failed to check load data scene", K(tmp_ret), K(tablet_id), K(tablet_analyzer)); } if (AdaptiveMergeReason::NONE == reason && OB_TMP_FAIL(check_ineffecient_read(tablet_analyzer, reason))) { - LOG_WARN("failed to check ineffecient read", K(tmp_ret), K(ls_id), K(tablet_id), K(tablet_analyzer)); + LOG_WARN("failed to check ineffecient read", K(tmp_ret), K(tablet_id), K(tablet_analyzer)); } if (AdaptiveMergeReason::NONE == reason && OB_TMP_FAIL(check_inc_sstable_row_cnt_percentage(tablet, reason))) { - LOG_WARN("failed to check sstable data situation", K(tmp_ret), K(ls_id), K(tablet_id), K(tablet_analyzer)); + LOG_WARN("failed to check sstable data situation", K(tmp_ret), K(tablet_id), K(tablet_analyzer)); } return ret; @@ -1851,7 +1821,6 @@ int ObAdaptiveMergePolicy::check_adaptive_merge_reason_for_event( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; mode = ObTableModeFlag::TABLE_MODE_NORMAL; reason = AdaptiveMergeReason::NONE; @@ -1862,9 +1831,9 @@ int ObAdaptiveMergePolicy::check_adaptive_merge_reason_for_event( LOG_WARN("invalid compaction event", K(ret), K(event)); } else if (tablet_id.is_special_merge_tablet()) { // do nothing - } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(ls_id, tablet_id, tablet_analyzer))) { + } else if (OB_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_tablet_analyzer(tablet_id, tablet_analyzer))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get tablet analyzer stat", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tablet analyzer stat", K(ret), K(tablet_id)); } else { ret = OB_SUCCESS; } @@ -1878,7 +1847,7 @@ int ObAdaptiveMergePolicy::check_adaptive_merge_reason_for_event( reason = AdaptiveMergeReason::TOMBSTONE_SCENE; } else if (queuing_cfg.is_queuing_mode()) { if (OB_TMP_FAIL(check_adaptive_merge_reason(tablet, tablet_analyzer, reason))) { - LOG_WARN("failed to check adaptive merge reason", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check adaptive merge reason", K(tmp_ret), K(tablet_id)); } } } diff --git a/src/storage/compaction/ob_partition_merge_policy.h b/src/storage/compaction/ob_partition_merge_policy.h index 99ff9432d..176489c7c 100644 --- a/src/storage/compaction/ob_partition_merge_policy.h +++ b/src/storage/compaction/ob_partition_merge_policy.h @@ -148,14 +148,13 @@ class ObPartitionMergePolicy static int refine_minor_merge_result( const ObMergeType merge_type, const int64_t minor_compact_trigger, - const bool is_tablet_referenced_by_collect_mv, storage::ObGetMergeTablesResult &result); static int deal_with_minor_result( const compaction::ObMergeType &merge_type, storage::ObLS &ls, const storage::ObTablet &tablet, storage::ObGetMergeTablesResult &result); - static int64_t cal_hist_minor_merge_threshold(const bool is_tablet_referenced_by_collect_mv = false); + static int64_t cal_hist_minor_merge_threshold(); static int generate_input_result_array( const storage::ObGetMergeTablesResult &input_result, ObMinorExecuteRangeMgr &minor_range_mgr, @@ -174,7 +173,6 @@ class ObPartitionMergePolicy // diagnose part static int diagnose_minor_dag( compaction::ObMergeType merge_type, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, char *buf, const int64_t buf_len); @@ -211,7 +209,6 @@ struct ObMinorExecuteRangeMgr } int get_merge_ranges( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); bool in_execute_range(const storage::ObITable *table) const; int sort_ranges(); diff --git a/src/storage/compaction/ob_schedule_dag_func.cpp b/src/storage/compaction/ob_schedule_dag_func.cpp index 65c14c9c3..915203a98 100644 --- a/src/storage/compaction/ob_schedule_dag_func.cpp +++ b/src/storage/compaction/ob_schedule_dag_func.cpp @@ -155,7 +155,6 @@ int ObScheduleDagFunc::schedule_batch_freeze_dag( } int ObDagParamFunc::fill_param( - const share::ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, const int64_t &merge_snapshot_version, @@ -163,15 +162,12 @@ int ObDagParamFunc::fill_param( ObTabletMergeDagParam ¶m) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() - || !is_valid_merge_type(merge_type) + if (OB_UNLIKELY(!is_valid_merge_type(merge_type) || merge_snapshot_version < ObVersion::MIN_VERSION || !is_valid_exec_mode(exec_mode))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(merge_snapshot_version), - K(exec_mode)); + LOG_WARN("invalid argument", KR(ret), K(merge_snapshot_version), K(exec_mode)); } else { - param.ls_id_ = ls_id; param.tablet_id_ = tablet.get_tablet_meta().tablet_id_; param.merge_type_ = merge_type; param.merge_version_ = merge_snapshot_version; diff --git a/src/storage/compaction/ob_schedule_dag_func.h b/src/storage/compaction/ob_schedule_dag_func.h index a57ef5249..19c04acae 100644 --- a/src/storage/compaction/ob_schedule_dag_func.h +++ b/src/storage/compaction/ob_schedule_dag_func.h @@ -22,10 +22,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -} namespace storage { namespace mds @@ -92,7 +88,6 @@ class ObDagParamFunc final { public: static int fill_param( - const share::ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, const int64_t &merge_snapshot_version, diff --git a/src/storage/compaction/ob_schedule_status_cache.cpp b/src/storage/compaction/ob_schedule_status_cache.cpp index 6778785f6..c616626bd 100644 --- a/src/storage/compaction/ob_schedule_status_cache.cpp +++ b/src/storage/compaction/ob_schedule_status_cache.cpp @@ -28,7 +28,7 @@ namespace compaction const static char * ObLSStateStr[] = { "CAN_MERGE", "WEAK_READ_TS_NOT_READY", - "OFFLINE_OR_DELETED", + "OFFLINE", "STATE_MAX" }; @@ -45,44 +45,38 @@ const char * ObLSStatusCache::ls_state_to_str(const ObLSStatusCache::LSState &st } /* * check_list -* ls is deleted / offline +* ls is offline * weak_read_ts */ int ObLSStatusCache::init_for_major( const int64_t merge_version, - ObLSHandle &ls_handle) + ObLS *ls) { int ret = OB_SUCCESS; reset(); // reset before init - ObLS *ls = nullptr; - if (OB_UNLIKELY(merge_version < 0 || !ls_handle.is_valid() || nullptr == (ls = ls_handle.get_ls()))) { + if (OB_UNLIKELY(merge_version < 0 || OB_ISNULL(ls))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(merge_version), K(ls_handle)); - } else if (FALSE_IT(ls_id_ = ls->get_ls_id())) { - } else if (FALSE_IT(check_ls_state(*ls, state_))) { - } else if (!can_merge()) { - // do nothing - } else if (FALSE_IT(weak_read_ts_ = ls->get_ls_wrs_handler()->get_ls_weak_read_ts())) { - } else if (OB_UNLIKELY(!weak_read_ts_.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid weak read ts", KR(ret), K_(weak_read_ts)); - } else if (merge_version > 0 && weak_read_ts_.get_val_for_tx() < merge_version) { - state_ = WEAK_READ_TS_NOT_READY; - } else if (can_merge() && OB_FAIL(ObMediumCompactionScheduleFunc::is_election_leader(ls_id_, is_leader_))) { - is_leader_ = false; - if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("failed to get palf handle role", K(ret), K_(ls_id)); + LOG_WARN("invalid argument", KR(ret), K(merge_version), KP(ls)); + } else { + check_ls_state(*ls, state_); + if (can_merge()) { + weak_read_ts_ = ls->get_ls_wrs_handler()->get_ls_weak_read_ts(); + if (OB_UNLIKELY(!weak_read_ts_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid weak read ts", KR(ret), K_(weak_read_ts)); + } else if (merge_version > 0 && weak_read_ts_.get_val_for_tx() < merge_version) { + state_ = WEAK_READ_TS_NOT_READY; + } } } if (OB_SUCC(ret)) { - ls_handle_ = ls_handle; + ls_ = ls; if (!can_merge()) { - ADD_SUSPECT_LS_INFO(MAJOR_MERGE, - ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - ls_id_, - ObSuspectInfoType::SUSPECT_LS_CANT_MERGE, - weak_read_ts_.is_valid() ? weak_read_ts_.get_val_for_tx() : -1, - "ls_status", *this); + ADD_COMMON_SUSPECT_INFO(MAJOR_MERGE, + ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, + ObSuspectInfoType::SUSPECT_LS_CANT_MERGE, + weak_read_ts_.is_valid() ? weak_read_ts_.get_val_for_tx() : -1, + "ls_status", *this); if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { LOG_INFO("ls is not ready for compaction", KPC(this)); } @@ -93,19 +87,17 @@ int ObLSStatusCache::init_for_major( void ObLSStatusCache::reset() { - ls_id_.reset(); weak_read_ts_.reset(); - is_leader_ = false; state_ = STATE_MAX; - ls_handle_.reset(); + ls_ = nullptr; } void ObLSStatusCache::check_ls_state(ObLS &ls, LSState &state) { - if (ls.is_deleted() || ls.is_offline()) { - state = OFFLINE_OR_DELETED; + if (ls.is_offline()) { + state = OFFLINE; if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { - LOG_INFO("ls is deleted or offline", K(ls), K(ls.is_deleted()), K(ls.is_offline())); + LOG_INFO("ls is offline", K(ls), K(ls.is_offline())); } } else { state = CAN_MERGE; @@ -121,7 +113,7 @@ bool ObLSStatusCache::check_weak_read_ts_ready( if (weak_read_scn.get_val_for_tx() < merge_version) { FLOG_INFO("current slave_read_ts is smaller than freeze_ts, try later", - "ls_id", ls.get_ls_id(), K(merge_version), K(weak_read_scn)); + K(merge_version), K(weak_read_scn)); } else { is_ready_for_compaction = true; } @@ -152,7 +144,7 @@ const char * ObTabletStatusCache::tablet_execute_state_to_str(const ObTabletStat const static char * ObTabletScheduleNewRoundStateStr[] = { "CAN_SCHEDULE_NEW_ROUND", - "RESERVED_STATE", + "RESERVED_STATUS_BLOCKED", "DURING_SPLIT", "NEED_CHECK_LAST_MEDIUM_CKM", "EXIST_UNFINISH_MEDIUM", @@ -184,14 +176,13 @@ int ObTabletStatusCache::init_for_major( ret = OB_INIT_TWICE; LOG_WARN("init twice", KR(ret), KPC(this)); } else { - const ObLSID &ls_id = ls.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_id(); if (OB_FAIL(inner_init_state(merge_version, tablet, is_skip_merge_tenant))) { - LOG_WARN("failed to init state", KR(ret), K(merge_version), K(ls_id), K(tablet_id)); + LOG_WARN("failed to init state", KR(ret), K(merge_version), K(tablet_id)); } else if (OB_FAIL(update_tablet_report_status(ls, tablet))) { - LOG_WARN("failed to update tablet report status", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to update tablet report status", KR(ret), K(tablet_id)); } else { - inner_init_could_schedule_new_round(ls_id, tablet, + inner_init_could_schedule_new_round(tablet, ls_could_schedule_new_round, true /*normal_schedule*/); } @@ -222,7 +213,7 @@ int ObTabletStatusCache::init_for_diagnose( if (OB_FAIL(inner_init_state(merge_version, tablet, false/*is_skip_merge_tenant*/))) { LOG_WARN("failed to init state", KR(ret), K(merge_version), K(tablet_id)); } else { - inner_init_could_schedule_new_round(ls.get_ls_id(), tablet, + inner_init_could_schedule_new_round(tablet, true /*ls_could_schedule_new_round*/, false /*normal_schedule*/); } @@ -240,7 +231,6 @@ int ObTabletStatusCache::init_for_diagnose( * check_list * tablet is_data_complete * exist major sstable -* is_mv */ int ObTabletStatusCache::inner_init_state( const int64_t merge_version, @@ -261,18 +251,7 @@ int ObTabletStatusCache::inner_init_state( LOG_TRACE("no major", KR(ret), K(tablet_id), K(last_major_snapshot)); } else if (FALSE_IT(tablet_merge_finish_ = (last_major_snapshot >= merge_version))){ } else if (is_skip_merge_tenant) { - // temp solution(load storage schema to decide whether tablet is MV) - // TODO replace with new func on tablet later @lana - ObArenaAllocator temp_allocator("GetSSchema", OB_MALLOC_NORMAL_BLOCK_SIZE); - ObStorageSchema *storage_schema = nullptr; - if (OB_FAIL(tablet.load_storage_schema(temp_allocator, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret), K(tablet)); - } else if (!storage_schema->is_mv_major_refresh()) { - execute_state_ = TENANT_SKIP_MERGE; - } else { - // MV tablet should schedule merge in skip_merge_tenant - execute_state_ = CAN_MERGE; - } + execute_state_ = TENANT_SKIP_MERGE; } else { execute_state_ = CAN_MERGE; } @@ -283,7 +262,6 @@ int ObTabletStatusCache::inner_init_state( } void ObTabletStatusCache::inner_init_could_schedule_new_round( - const ObLSID &ls_id, const ObTablet &tablet, const bool ls_could_schedule_new_round, const bool normal_schedule) @@ -297,11 +275,11 @@ void ObTabletStatusCache::inner_init_could_schedule_new_round( new_round_state_ = NEW_ROUND_STATE_MAX; if (OB_FAIL(tablet.ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { LOG_WARN("failed to get tablet status", K(ret), K(tablet), K(user_data)); - } else if (ObTabletStatus::RESERVED_4 == user_data.tablet_status_ - || ObTabletStatus::RESERVED_6 == user_data.tablet_status_) { - new_round_state_ = RESERVED_STATE; + } else if (ObTabletStatus::RESERVED_STATUS_4 == user_data.tablet_status_ + || ObTabletStatus::RESERVED_STATUS_6 == user_data.tablet_status_) { + new_round_state_ = RESERVED_STATUS_BLOCKED; if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { - LOG_INFO("tablet status is reserved, merging is not allowed", K(user_data), K(tablet)); + LOG_INFO("reserved tablet status, merging is not allowed", K(user_data), K(tablet)); } } else if (ObTabletStatus::SPLIT_SRC == user_data.tablet_status_ || ObTabletStatus::SPLIT_SRC_DELETED == user_data.tablet_status_) { @@ -309,15 +287,15 @@ void ObTabletStatusCache::inner_init_could_schedule_new_round( if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { LOG_INFO("tablet status is split, merging is not allowed", K(user_data), K(tablet)); } - } else if (OB_FAIL(check_medium_list(ls_id, tablet, normal_schedule))) { + } else if (OB_FAIL(check_medium_list(tablet, normal_schedule))) { // call medium_list_->need_check_finish even if ls_could_schedule_new_round=false - LOG_WARN("failed to check medium list", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check medium list", K(ret), K(tablet_id)); } else if (!ls_could_schedule_new_round || NEW_ROUND_STATE_MAX != new_round_state_) { // do nothing } else if (normal_schedule) { if (OB_FAIL(register_map(tablet))) { // register_map must be the last step - LOG_WARN("failed to add tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to add tablet", K(ret), K(tablet_id)); } } else { // for diagnose new_round_state_ = DIAGNOSE_NORMAL; @@ -340,7 +318,6 @@ void ObTabletStatusCache::inner_destroy() } int ObTabletStatusCache::check_medium_list( - const ObLSID &ls_id, const ObTablet &tablet, const bool normal_schedule) { @@ -354,11 +331,11 @@ int ObTabletStatusCache::check_medium_list( new_round_state_ = NEED_CHECK_LAST_MEDIUM_CKM; if (normal_schedule) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(share::g_mp->tenant_medium_checker()->add_tablet_ls( - tablet_id, ls_id, medium_list_->get_wait_check_medium_scn()))) { - LOG_WARN("failed to add tablet", K(tmp_ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->tenant_medium_checker()->add_tablet( + tablet_id, medium_list_->get_wait_check_medium_scn()))) { + LOG_WARN("failed to add tablet", K(tmp_ret), K(tablet_id)); } else { - LOG_TRACE("success to add tablet into checker", KR(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("success to add tablet into checker", KR(ret), K(tablet_id)); } } } else if (!medium_list_->could_schedule_next_round(tablet.get_last_major_snapshot_version())) { @@ -392,11 +369,10 @@ int ObTabletStatusCache::update_tablet_report_status( int ret = OB_SUCCESS; if (tablet_merge_finish_) { int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_id(); if (tablet.get_tablet_meta().report_status_.need_report()) { - if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id, true/*need_diagnose*/))) { - LOG_WARN("failed to submit tablet update task to report", K(tmp_ret), K(tablet_id), K(ls_id)); + if (OB_TMP_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id, true/*need_diagnose*/))) { + LOG_WARN("failed to submit tablet update task to report", K(tmp_ret), K(tablet_id)); } else if (OB_TMP_FAIL(ls.get_tablet_svr()->update_tablet_report_status(tablet_id))) { LOG_WARN("failed to update tablet report status", K(tmp_ret), K(tablet_id)); } @@ -420,7 +396,6 @@ int ObTabletStatusCache::check_could_execute(const ObMergeType merge_type, const && !is_medium_merge(merge_type)) { need_merge = true; } else { - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; bool is_empty_shell = tablet.is_empty_shell(); if (is_minor_merge(merge_type) || is_mini_merge(merge_type)) { @@ -432,7 +407,7 @@ int ObTabletStatusCache::check_could_execute(const ObMergeType merge_type, const if (OB_FAIL(ret)) { } else if (!need_merge) { ret = OB_NO_NEED_MERGE; - LOG_INFO("tablet has no need to merge", K(ret), K(ls_id), K(tablet_id), + LOG_INFO("tablet has no need to merge", K(ret), K(tablet_id), "merge_type", merge_type_to_str(merge_type), K(is_empty_shell)); } } diff --git a/src/storage/compaction/ob_schedule_status_cache.h b/src/storage/compaction/ob_schedule_status_cache.h index 8a82e4efe..ea2054414 100644 --- a/src/storage/compaction/ob_schedule_status_cache.h +++ b/src/storage/compaction/ob_schedule_status_cache.h @@ -18,7 +18,6 @@ #include "share/scn.h" #include "storage/compaction/ob_compaction_schedule_util.h" #include "storage/compaction/ob_batch_freeze_tablets_dag.h" -#include "storage/tx_storage/ob_ls_handle.h" namespace oceanbase { namespace storage @@ -36,7 +35,7 @@ struct ObLSStatusCache final { CAN_MERGE = 0, WEAK_READ_TS_NOT_READY, - OFFLINE_OR_DELETED, + OFFLINE, STATE_MAX, }; static const char *ls_state_to_str(const LSState &state); @@ -45,14 +44,12 @@ struct ObLSStatusCache final return state >= CAN_MERGE && state < STATE_MAX; } ObLSStatusCache() - : ls_handle_(), - ls_id_(), + : ls_(nullptr), weak_read_ts_(), - is_leader_(false), state_(STATE_MAX) {} ~ObLSStatusCache() {} - int init_for_major(const int64_t merge_version, storage::ObLSHandle &ls_handle); + int init_for_major(const int64_t merge_version, storage::ObLS *ls); bool can_merge() const { return CAN_MERGE == state_; } void reset(); int check_restore_status(storage::ObLS &ls); @@ -60,18 +57,16 @@ struct ObLSStatusCache final static bool check_weak_read_ts_ready( const int64_t &merge_version, storage::ObLS &ls); - bool is_valid() const { return ls_id_.is_valid() && ls_handle_.is_valid(); } + bool is_valid() const { return OB_NOT_NULL(ls_); } storage::ObLS &get_ls() { OB_ASSERT_MSG(is_valid(), "ObLSStatusCache is not valid"); - return *ls_handle_.get_ls(); + return *ls_; } - TO_STRING_KV(K_(ls_id), K_(weak_read_ts), K_(is_leader), "state", ls_state_to_str(state_)); + TO_STRING_KV(K_(weak_read_ts), "state", ls_state_to_str(state_)); static const int64_t PRINT_LOG_INVERVAL = 2 * 60 * 1000 * 1000L; // 2m - storage::ObLSHandle ls_handle_; - share::ObLSID ls_id_; + storage::ObLS *ls_; share::SCN weak_read_ts_; - bool is_leader_; LSState state_; }; @@ -95,7 +90,7 @@ struct ObTabletStatusCache enum TabletScheduleNewRoundState : uint8_t { CAN_SCHEDULE_NEW_ROUND = 0, - RESERVED_STATE, + RESERVED_STATUS_BLOCKED, DURING_SPLIT, NEED_CHECK_LAST_MEDIUM_CKM, EXIST_UNFINISH_MEDIUM, @@ -155,12 +150,10 @@ struct ObTabletStatusCache const storage::ObTablet &tablet, const bool is_remote_tenant); int check_medium_list( - const share::ObLSID &ls_id, const storage::ObTablet &tablet, const bool normal_schedule); int register_map(const ObTablet &tablet); void inner_init_could_schedule_new_round( - const ObLSID &ls_id, const ObTablet &tablet, const bool ls_could_schedule_new_round, const bool normal_schedule); diff --git a/src/storage/compaction/ob_schedule_tablet_func.cpp b/src/storage/compaction/ob_schedule_tablet_func.cpp index 649011e30..da4a76efd 100644 --- a/src/storage/compaction/ob_schedule_tablet_func.cpp +++ b/src/storage/compaction/ob_schedule_tablet_func.cpp @@ -45,7 +45,6 @@ int ObScheduleTabletFunc::schedule_tablet( tablet_merge_finish = false; int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls_status_.ls_id_; ObTablet *tablet = nullptr; ObTabletID tablet_id; bool need_diagnose = false; @@ -87,7 +86,7 @@ int ObScheduleTabletFunc::schedule_tablet( } } if (need_diagnose - && OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(ls_id, tablet_id, + && OB_TMP_FAIL(share::g_mp->diagnose_tablet_mgr()->add_diagnose_tablet(tablet_id, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE))) { LOG_WARN("failed to add diagnose tablet", K(tmp_ret), K_(ls_status), K(tablet_id)); } @@ -102,7 +101,6 @@ int ObScheduleTabletFunc::schedule_tablet_new_round( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls_status_.ls_id_; const ObTabletID &tablet_id = tablet_handle.get_obj()->get_tablet_id(); bool medium_clog_submitted = false; @@ -116,15 +114,10 @@ int ObScheduleTabletFunc::schedule_tablet_new_round( ls_status_.get_ls(), tablet_handle, ls_status_.weak_read_ts_, *tablet_status_.medium_list(), &tablet_cnt_, merge_reason_); - if (OB_FAIL(func.schedule_next_medium_for_leader( + if (OB_FAIL(func.schedule_next_medium( tablet_status_.tablet_merge_finish() ? 0 : merge_version_, medium_clog_submitted))) { - if (OB_NOT_MASTER == ret) { - ls_status_.is_leader_ = false; - ls_could_schedule_new_round_ = false; - } else { - LOG_WARN("failed to schedule next medium", K(ret), K_(ls_status), K(tablet_id)); - } + LOG_WARN("failed to schedule next medium", K(ret), K_(ls_status), K(tablet_id)); } else if (medium_clog_submitted) { if (OB_TMP_FAIL(clear_stat_tablets_.push_back(tablet_id))) { LOG_WARN("failed to push back tablet_id for batch_freeze", KR(tmp_ret), K_(ls_status), K(tablet_id)); @@ -141,17 +134,12 @@ int ObScheduleTabletFunc::request_schedule_new_round( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObLSID &ls_id = ls_status_.ls_id_; ObTablet *tablet = nullptr; ObTabletID tablet_id; bool schedule_flag = false; if (OB_UNLIKELY(!ls_status_.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid ls status", KR(ret), K_(ls_status)); - } else if (!ls_status_.is_leader_) { - // not leader, can't schedule - ret = OB_LEADER_NOT_EXIST; - LOG_WARN("not ls leader, can't schedule medium", K(ret), K_(ls_status), K_(ls_status), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tablet handle", KR(ret), K(tablet_handle)); @@ -198,12 +186,11 @@ int ObScheduleTabletFunc::schedule_tablet_execute( return ret; } #endif - const ObLSID &ls_id = ls_status_.ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_id(); bool can_merge = false; int64_t schedule_scn = 0; - if (OB_FAIL(ObTenantTabletScheduler::check_ready_for_major_merge(ls_id, tablet, MEDIUM_MERGE))) { - LOG_WARN("failed to check ready for major merge", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(ObTenantTabletScheduler::check_ready_for_major_merge(tablet, MEDIUM_MERGE))) { + LOG_WARN("failed to check ready for major merge", K(ret), K(tablet_id)); } else if (OB_FAIL(get_schedule_execute_info(tablet, schedule_scn))) { if (OB_NO_NEED_MERGE == ret) { ret = OB_SUCCESS; @@ -213,7 +200,7 @@ int ObScheduleTabletFunc::schedule_tablet_execute( } else if (OB_FAIL(check_with_schedule_scn(tablet, schedule_scn, tablet_status_, can_merge))) { LOG_WARN("failed to check with schedule scn", KR(ret), K(schedule_scn)); } else if (can_merge) { - if (OB_FAIL(ObTenantTabletScheduler::schedule_merge_dag(ls_id, tablet, MEDIUM_MERGE, schedule_scn, EXEC_MODE_LOCAL))) { + if (OB_FAIL(ObTenantTabletScheduler::schedule_merge_dag(tablet, MEDIUM_MERGE, schedule_scn, EXEC_MODE_LOCAL))) { if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { LOG_ERROR("failed to schedule medium merge dag", K(ret), K_(ls_status), K(tablet_id)); } @@ -231,38 +218,15 @@ int ObScheduleTabletFunc::get_schedule_execute_info( int64_t &schedule_scn) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_status_.ls_id_; - const ObTabletID &tablet_id = tablet.get_tablet_id(); - bool schedule_flag = false; const int64_t last_major_snapshot = tablet.get_last_major_snapshot_version(); ObMediumCompactionInfo::ObCompactionType compaction_type = ObMediumCompactionInfo::COMPACTION_TYPE_MAX; schedule_scn = 0; // medium_snapshot in medium info - bool is_mv_major_refresh_tablet = false; - ObArenaAllocator temp_allocator("GetMediumInfo", OB_MALLOC_NORMAL_BLOCK_SIZE); // for load medium info - const ObMediumCompactionInfoList *medium_list = nullptr; - ObStorageSchema *storage_schema = nullptr; - if (OB_ISNULL(tablet_status_.medium_list())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("medium list in tablet status is null", KR(ret), K_(tablet_status)); - } else if (OB_FAIL(tablet.load_storage_schema(temp_allocator, storage_schema))) { - // temp solution(load storage schema to decide whether tablet is MV) - // TODO replace with new func on tablet later @lana - LOG_WARN("failed to load storage schema", K(ret), K(tablet)); - } else if (FALSE_IT(is_mv_major_refresh_tablet = storage_schema->is_mv_major_refresh())) { - } else if (is_mv_major_refresh_tablet && - ObBasicMergeScheduler::INIT_COMPACTION_SCN == last_major_snapshot) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - ls_id, tablet_id, ObSuspectInfoType::SUSPECT_MV_IN_CREATION, - last_major_snapshot))) { - LOG_WARN("failed to add suspect info", K(tmp_ret)); - } - LOG_INFO("mv creation has not finished, can not schedule mv tablet", K(ret), - K(last_major_snapshot)); } else if (OB_FAIL(tablet_status_.medium_list()->get_next_schedule_info( - last_major_snapshot, merge_version_, is_mv_major_refresh_tablet, compaction_type, schedule_scn))) { + last_major_snapshot, merge_version_, compaction_type, schedule_scn))) { if (OB_NO_NEED_MERGE != ret) { LOG_WARN("failed to get next schedule info", KR(ret), K(last_major_snapshot), K_(merge_version)); } @@ -281,8 +245,8 @@ void ObScheduleTabletFunc::schedule_freeze_dag(const bool force) int tmp_ret = OB_SUCCESS; IGNORE_RETURN ObBasicScheduleTabletFunc::schedule_freeze_dag(force); if (force || clear_stat_tablets_.count() > SCHEDULE_DAG_THREHOLD) { - if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->batch_clear_tablet_stat(ls_status_.ls_id_, clear_stat_tablets_))) { - LOG_WARN_RET(tmp_ret, "failed to batch clear tablet stats", K(ls_status_.ls_id_)); + if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->batch_clear_tablet_stat(clear_stat_tablets_))) { + LOG_WARN_RET(tmp_ret, "failed to batch clear tablet stats"); } clear_stat_tablets_.reset(); } diff --git a/src/storage/compaction/ob_sstable_merge_history.cpp b/src/storage/compaction/ob_sstable_merge_history.cpp index 3f1d1625f..2fd0e4b5a 100644 --- a/src/storage/compaction/ob_sstable_merge_history.cpp +++ b/src/storage/compaction/ob_sstable_merge_history.cpp @@ -115,8 +115,7 @@ void PartTableInfo::fill_info(char *buf, const int64_t buf_len) const * -------------------------------------------------------------------ObMergeStaticInfo------------------------------------------------------------------- */ ObMergeStaticInfo::ObMergeStaticInfo() - : ls_id_(), - tablet_id_(), + : tablet_id_(), merge_type_(INVALID_MERGE_TYPE), compaction_scn_(0), concurrent_cnt_(0), @@ -134,15 +133,13 @@ ObMergeStaticInfo::ObMergeStaticInfo() bool ObMergeStaticInfo::is_valid() const { - return ls_id_.is_valid() && - ((tablet_id_.is_valid() && is_valid_merge_type(merge_type_)) || + return ((tablet_id_.is_valid() && is_valid_merge_type(merge_type_)) || BATCH_EXEC == merge_type_) && compaction_scn_ > 0 && is_valid_exec_mode(exec_mode_); } void ObMergeStaticInfo::reset() { - ls_id_.reset(); tablet_id_.reset(); merge_type_ = INVALID_MERGE_TYPE; compaction_scn_ = 0; @@ -160,7 +157,6 @@ void ObMergeStaticInfo::reset() void ObMergeStaticInfo::shallow_copy(const ObMergeStaticInfo &other) { - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; merge_type_ = other.merge_type_; compaction_scn_ = other.compaction_scn_; diff --git a/src/storage/compaction/ob_sstable_merge_history.h b/src/storage/compaction/ob_sstable_merge_history.h index 165884016..8bdb8e4b4 100644 --- a/src/storage/compaction/ob_sstable_merge_history.h +++ b/src/storage/compaction/ob_sstable_merge_history.h @@ -16,7 +16,6 @@ #ifndef OB_STORAGE_COMPACTION_SSTABLE_MERGE_INFO_H_ #define OB_STORAGE_COMPACTION_SSTABLE_MERGE_INFO_H_ #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/compaction/ob_compaction_util.h" #include "storage/compaction/ob_tenant_freeze_info_mgr.h" #include "storage/compaction/ob_compaction_diagnose.h" @@ -114,7 +113,7 @@ struct ObMergeStaticInfo void reset(); bool is_valid() const; void shallow_copy(const ObMergeStaticInfo &other); - TO_STRING_KV(K_(ls_id), K_(tablet_id), "merge_type", merge_type_to_str(merge_type_), + TO_STRING_KV(K_(tablet_id), "merge_type", merge_type_to_str(merge_type_), K_(compaction_scn), K_(is_full_merge), K_(concurrent_cnt), "merge_level", merge_level_to_str(merge_level_), "exec_mode", exec_mode_to_str(exec_mode_), @@ -122,7 +121,6 @@ struct ObMergeStaticInfo K_(kept_snapshot_info), K_(participant_table_info), K_(mds_filter_info_str), K_(progressive_merge_round), K_(progressive_merge_num), K_(is_fake)); static const int64_t MDS_FILTER_INFO_LENGTH = 256; - share::ObLSID ls_id_; ObTabletID tablet_id_; ObMergeType merge_type_; int64_t compaction_scn_; // major_scn OR minor end_log_ts diff --git a/src/storage/compaction/ob_table_ckm_items.cpp b/src/storage/compaction/ob_table_ckm_items.cpp index 10bd605f9..01d0b9d59 100644 --- a/src/storage/compaction/ob_table_ckm_items.cpp +++ b/src/storage/compaction/ob_table_ckm_items.cpp @@ -158,7 +158,7 @@ ObTableCkmItems::ObTableCkmItems() table_id_(0), row_count_(0), table_schema_(nullptr), - tablet_pairs_(), + tablet_ids_(), ckm_items_(), sort_col_id_array_(), ckm_sum_array_() @@ -174,7 +174,7 @@ ObTableCkmItems::~ObTableCkmItems() int ObTableCkmItems::build( share::schema::ObSchemaGetterGuard &schema_guard, const schema::ObSimpleTableSchemaV2 &simple_schema, - const ObArray &input_tablet_pairs, + const ObIArray &input_tablet_ids, const ObReplicaCkmArray &input_ckm_items) { int ret = OB_SUCCESS; @@ -182,11 +182,11 @@ int ObTableCkmItems::build( if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("is inited before", KR(ret), KPC(this)); - } else if (OB_FAIL(tablet_pairs_.assign(input_tablet_pairs))) { - LOG_WARN("failed to assgin tablet ls pair array", KR(ret), K(input_tablet_pairs)); + } else if (OB_FAIL(tablet_ids_.assign(input_tablet_ids))) { + LOG_WARN("failed to assign tablet id array", KR(ret), K(input_tablet_ids)); } else if (OB_FAIL(ckm_items_.init(input_ckm_items))) { - LOG_WARN("failed to assgin tablet replica ckm array", KR(ret), K(input_ckm_items)); - } else if (OB_FAIL(schema_guard.get_table_schema( table_id, table_schema_))) { + LOG_WARN("failed to assign tablet replica ckm array", KR(ret), K(input_ckm_items)); + } else if (OB_FAIL(schema_guard.get_table_schema(table_id, table_schema_))) { LOG_WARN("fail to get table schema", KR(ret), K(table_id)); } else if ((!simple_schema.is_index_table() || simple_schema.is_fts_or_multivalue_index()) && OB_FAIL(sort_col_id_array_.build(*table_schema_))) { @@ -204,7 +204,6 @@ int ObTableCkmItems::build( int ObTableCkmItems::prepare_build( const uint64_t table_id, share::schema::ObSchemaGetterGuard &schema_guard, - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache, ObIArray &tablet_id_array) { int ret = OB_SUCCESS; @@ -221,8 +220,8 @@ int ObTableCkmItems::prepare_build( LOG_WARN("table schema is null", KR(ret), K(table_id), KP_(table_schema)); } else if (OB_FAIL(table_schema_->get_tablet_ids(tablet_id_array))) { LOG_WARN("fail to get tablet_ids from table schema", KR(ret), KPC(table_schema_)); - } else if (OB_FAIL(tablet_ls_pair_cache.get_tablet_ls_pairs(table_id, tablet_id_array, tablet_pairs_))) { - LOG_WARN("failed to get tablet ls pairs", KR(ret), K(table_id), K(tablet_id_array)); + } else if (OB_FAIL(tablet_ids_.assign(tablet_id_array))) { + LOG_WARN("failed to assign tablet ids", KR(ret), K(table_id), K(tablet_id_array)); } return ret; } @@ -231,18 +230,17 @@ int ObTableCkmItems::build( const uint64_t table_id, const share::SCN &compaction_scn, common::ObMySQLProxy &sql_proxy, - schema::ObSchemaGetterGuard &schema_guard, - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache) + schema::ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; ObSEArray tablet_id_array; - if (OB_FAIL(prepare_build(table_id, schema_guard, tablet_ls_pair_cache, tablet_id_array))) { + if (OB_FAIL(prepare_build(table_id, schema_guard, tablet_id_array))) { LOG_WARN("failed to prepare build ckm items", K(ret)); - } else if (OB_FAIL(ckm_items_.init(tablet_pairs_.count()))) { - STORAGE_LOG(WARN, "failed to init ckm array", K(ret), K(tablet_pairs_.count())); + } else if (OB_FAIL(ckm_items_.init(tablet_ids_.count()))) { + STORAGE_LOG(WARN, "failed to init ckm array", K(ret), K(tablet_ids_.count())); } else if (OB_FAIL(ObTabletReplicaChecksumOperator::get_tablet_replica_checksum_items(sql_proxy, - compaction_scn, tablet_pairs_, + compaction_scn, tablet_ids_, ckm_items_))) { LOG_WARN("failed to get table column checksum items", KR(ret)); } else if ((!table_schema_->is_index_table() || table_schema_->is_fts_or_multivalue_index()) @@ -305,9 +303,9 @@ int ObTableCkmItems::build_column_ckm_sum_array( { int ret = OB_SUCCESS; const int64_t items_cnt = ckm_items_.count(); - if (OB_UNLIKELY(ckm_items_.empty() || tablet_pairs_.empty())) { + if (OB_UNLIKELY(ckm_items_.empty() || tablet_ids_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("checksum items or tablet pairs are empty", KR(ret), K_(ckm_items), K_(tablet_pairs)); + LOG_WARN("checksum items or tablet ids are empty", KR(ret), K_(ckm_items), K_(tablet_ids)); } else if (!ckm_sum_array_.empty()) { LOG_INFO("use cached ckm array", KR(ret), K_(row_count), K_(ckm_sum_array), K(compaction_scn)); } else { @@ -320,11 +318,11 @@ int ObTableCkmItems::build_column_ckm_sum_array( LOG_WARN("failed to reserve tablet column checksum array", KR(ret)); } // items are order by tablet_id - for (int64_t pair_idx = 0; OB_SUCC(ret) && (pair_idx < tablet_pairs_.count()); ++pair_idx) { - const ObTabletID &tablet_id = tablet_pairs_.at(pair_idx).get_tablet_id(); + for (int64_t pair_idx = 0; OB_SUCC(ret) && (pair_idx < tablet_ids_.count()); ++pair_idx) { + const ObTabletID &tablet_id = tablet_ids_.at(pair_idx); const ObTabletReplicaChecksumItem *cur_item = nullptr; if (OB_FAIL(ckm_items_.get(tablet_id, cur_item))) { - LOG_WARN("failed to get ckm item", KR(ret), K(tablet_id), K(pair_idx), K(tablet_pairs_)); + LOG_WARN("failed to get ckm item", KR(ret), K(tablet_id), K(pair_idx), K(tablet_ids_)); } else if (OB_UNLIKELY(cur_item->column_meta_.column_checksums_.count() != column_checksums_cnt)) { // Why the first ckm_item can be selected as the based one? // Any partition alter operation will result in an increase in tablet_id, and ckm_items_ is generated order by the tablet_id. @@ -372,9 +370,9 @@ int ObTableCkmItems::build_column_ckm_sum_array( ckm_error_info.frozen_scn_ = compaction_scn; \ ckm_error_info.data_table_id_ = data_table_schema->get_table_id(); \ ckm_error_info.index_table_id_ = index_table_schema->get_table_id(); \ - if (tablet_array_idx >= 0 && tablet_array_idx < data_ckm.tablet_pairs_.count()) { \ - ckm_error_info.data_tablet_id_ = data_ckm.tablet_pairs_.at(tablet_array_idx).get_tablet_id(); \ - ckm_error_info.index_tablet_id_ = index_ckm.tablet_pairs_.at(tablet_array_idx).get_tablet_id(); \ + if (tablet_array_idx >= 0 && tablet_array_idx < data_ckm.tablet_ids_.count()) { \ + ckm_error_info.data_tablet_id_ = data_ckm.tablet_ids_.at(tablet_array_idx); \ + ckm_error_info.index_tablet_id_ = index_ckm.tablet_ids_.at(tablet_array_idx); \ } // for global index @@ -456,41 +454,41 @@ int ObTableCkmItems::validate_tablet_column_ckm( ret = OB_ERR_UNEXPECTED; LOG_WARN("data table and index table should not validate column checksum", KR(ret), KPC(data_table_schema), KPC(index_table_schema)); - } else if (OB_UNLIKELY(index_ckm.tablet_pairs_.count() != data_ckm.tablet_pairs_.count())) { + } else if (OB_UNLIKELY(index_ckm.tablet_ids_.count() != data_ckm.tablet_ids_.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet ls pairs count are not equal", KR(ret), K(data_ckm), K(index_ckm), - K(data_ckm.tablet_pairs_), K(index_ckm.tablet_pairs_)); + LOG_WARN("tablet id count are not equal", KR(ret), K(data_ckm), K(index_ckm), + K(data_ckm.tablet_ids_), K(index_ckm.tablet_ids_)); } else { const ObTabletReplicaChecksumItem *data_replica_ckm = NULL; const ObTabletReplicaChecksumItem *index_replica_ckm = NULL; - for (int64_t idx = 0; OB_SUCC(ret) && idx < index_ckm.tablet_pairs_.count(); ++idx) { + for (int64_t idx = 0; OB_SUCC(ret) && idx < index_ckm.tablet_ids_.count(); ++idx) { if (OB_FAIL(data_ckm.ckm_items_.get( - data_ckm.tablet_pairs_.at(idx).get_tablet_id(), data_replica_ckm))) { + data_ckm.tablet_ids_.at(idx), data_replica_ckm))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; LOG_INFO("failed to find replica_checksum idx of data tablet, skip verify", KR(ret), K(idx), - "tablet_id", data_ckm.tablet_pairs_.at(idx).get_tablet_id()); + "tablet_id", data_ckm.tablet_ids_.at(idx)); } else { LOG_ERROR("failed to find replica_checksum idx of data tablet", KR(ret), K(data_ckm), K(idx), - "tablet_id", data_ckm.tablet_pairs_.at(idx).get_tablet_id()); + "tablet_id", data_ckm.tablet_ids_.at(idx)); } } else if (OB_FAIL(index_ckm.ckm_items_.get( - index_ckm.tablet_pairs_.at(idx).get_tablet_id(), index_replica_ckm))) { + index_ckm.tablet_ids_.at(idx), index_replica_ckm))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; LOG_INFO("failed to find replica_checksum idx of index tablet, skip verify", KR(ret), K(idx), - "tablet_id", index_ckm.tablet_pairs_.at(idx).get_tablet_id()); + "tablet_id", index_ckm.tablet_ids_.at(idx)); } else { LOG_ERROR("failed to find replica_checksum idx of index tablet", KR(ret), KPC(data_replica_ckm), K(idx), - "tablet_id", index_ckm.tablet_pairs_.at(idx).get_tablet_id(), K(index_ckm.tablet_pairs_)); + "tablet_id", index_ckm.tablet_ids_.at(idx), K(index_ckm.tablet_ids_)); } } else { - if (OB_UNLIKELY(data_ckm.tablet_pairs_.at(idx).get_tablet_id() != data_replica_ckm->tablet_id_ - || index_ckm.tablet_pairs_.at(idx).get_tablet_id() != index_replica_ckm->tablet_id_)) { + if (OB_UNLIKELY(data_ckm.tablet_ids_.at(idx) != data_replica_ckm->tablet_id_ + || index_ckm.tablet_ids_.at(idx) != index_replica_ckm->tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id & replica checksum is not match", KR(ret), K(idx), - "data_tablet", data_ckm.tablet_pairs_.at(idx), K(data_replica_ckm), - "index_tablet", index_ckm.tablet_pairs_.at(idx), K(index_replica_ckm)); + "data_tablet", data_ckm.tablet_ids_.at(idx), K(data_replica_ckm), + "index_tablet", index_ckm.tablet_ids_.at(idx), K(index_replica_ckm)); } else if (OB_UNLIKELY(data_replica_ckm->compaction_scn_ != compaction_scn || index_replica_ckm->compaction_scn_ != compaction_scn)) { ret = OB_ITEM_NOT_MATCH; @@ -510,8 +508,8 @@ int ObTableCkmItems::validate_tablet_column_ckm( ckm_error_info))) { if (OB_CHECKSUM_ERROR != ret) { LOG_ERROR("failed to compare column checksum", KR(ret), - "data_tablet", data_ckm.tablet_pairs_.at(idx), KPC(data_replica_ckm), - "index_tablet", index_ckm.tablet_pairs_.at(idx), KPC(index_replica_ckm), + "data_tablet", data_ckm.tablet_ids_.at(idx), KPC(data_replica_ckm), + "index_tablet", index_ckm.tablet_ids_.at(idx), KPC(index_replica_ckm), K(data_ckm.ckm_items_), K(index_ckm.ckm_items_)); } } @@ -527,8 +525,8 @@ int ObTableCkmItems::validate_tablet_column_ckm( if (OB_CHECKSUM_ERROR == ret) { RECORD_CKM_ERROR_INFO(idx, false/*is_global_index*/); LOG_ERROR("failed to compare column checksum", KR(ret), K(ckm_error_info), - "data_tablet", data_ckm.tablet_pairs_.at(idx), "data_row_cnt", data_replica_ckm->row_count_, KPC(data_replica_ckm), - "index_tablet", index_ckm.tablet_pairs_.at(idx), "index_row_cnt", index_replica_ckm->row_count_, KPC(index_replica_ckm), + "data_tablet", data_ckm.tablet_ids_.at(idx), "data_row_cnt", data_replica_ckm->row_count_, KPC(data_replica_ckm), + "index_tablet", index_ckm.tablet_ids_.at(idx), "index_row_cnt", index_replica_ckm->row_count_, KPC(index_replica_ckm), K(data_ckm.ckm_items_), K(index_ckm.ckm_items_)); if (OB_TMP_FAIL(ObColumnChecksumErrorOperator::insert_column_checksum_err_info(sql_proxy, ckm_error_info))) { @@ -619,7 +617,7 @@ void ObTableCkmItems::reset() table_id_ = 0; row_count_ = 0; table_schema_ = NULL; - tablet_pairs_.reset(); + tablet_ids_.reset(); ckm_items_.reset(); sort_col_id_array_.reset(); ckm_sum_array_.reset(); @@ -648,10 +646,10 @@ int ObTableCkmItems::check_schema_change_after_major_freeze( } else { LOG_WARN("fail to get table schema", KR(ret), K(index_ckm)); } - } else if (old_table_schema->get_partition_num() != data_ckm.tablet_pairs_.count()) { + } else if (old_table_schema->get_partition_num() != data_ckm.tablet_ids_.count()) { FLOG_INFO("[IGNORE CHECKSUM_ERROR] partition num changed in data table", KR(ret), "old_partition_num", old_table_schema->get_partition_num(), - "new_partition_num", data_ckm.tablet_pairs_.count(), + "new_partition_num", data_ckm.tablet_ids_.count(), K(data_ckm)); #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("merge_errsim", "ignore_checksum_error", "reason", "partition_num_changed"); diff --git a/src/storage/compaction/ob_table_ckm_items.h b/src/storage/compaction/ob_table_ckm_items.h index 67ed26e2c..a31f991eb 100644 --- a/src/storage/compaction/ob_table_ckm_items.h +++ b/src/storage/compaction/ob_table_ckm_items.h @@ -17,7 +17,6 @@ #define OB_SHARE_COMPACTION_TABLE_CKM_ITEMS_H_ #include "common/ob_tablet_id.h" #include "share/ob_freeze_info_proxy.h" // ObFreezeInfo, previously hidden behind a transitive include, make the dependency explicit -#include "share/ob_ls_id.h" #include "share/schema/ob_table_schema.h" #include "share/ob_tablet_replica_checksum_operator.h" #include "share/compaction/ob_array_with_map.h" @@ -25,7 +24,6 @@ namespace oceanbase { namespace compaction { -struct ObTabletLSPairCache; struct ObIndexCkmValidatePair { public: @@ -118,18 +116,17 @@ struct ObTableCkmItems int64_t get_table_id() const { return table_id_; } const share::schema::ObTableSchema * get_table_schema() const { return table_schema_; } const common::ObIArray &get_ckm_items() const { return ckm_items_.get_array(); } - const common::ObIArray &get_tablet_ls_pairs() const { return tablet_pairs_; } + const common::ObIArray &get_tablet_ids() const { return tablet_ids_; } int build( share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObSimpleTableSchemaV2 &simple_schema, - const ObArray &input_tablet_pairs, + const common::ObIArray &input_tablet_ids, const share::ObReplicaCkmArray &input_ckm_items); int build( const uint64_t table_id, const share::SCN &compaction_scn, common::ObMySQLProxy &sql_proxy, - share::schema::ObSchemaGetterGuard &schema_guard, - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache); + share::schema::ObSchemaGetterGuard &schema_guard); int build_column_ckm_sum_array( const bool is_data_table, const share::SCN &compaction_scn, @@ -142,7 +139,7 @@ struct ObTableCkmItems ObTableCkmItems &index_ckm); static const int64_t FUNC_CNT = 2; static VALIDATE_CKM_FUNC validate_ckm_func[FUNC_CNT]; - TO_STRING_KV(K_(is_inited), K_(table_id), "tablet_cnt", tablet_pairs_.count(), + TO_STRING_KV(K_(is_inited), K_(table_id), "tablet_cnt", tablet_ids_.count(), "ckm_item_cnt", ckm_items_.count(), K_(sort_col_id_array), "col_ckm_sum_array_size", ckm_sum_array_.count()); @@ -168,8 +165,7 @@ struct ObTableCkmItems int prepare_build( const uint64_t table_id, share::schema::ObSchemaGetterGuard &schema_guard, - const compaction::ObTabletLSPairCache &tablet_ls_pair_cache, - common::ObIArray &tablet_id_array); + common::ObIArray &tablet_id_array); int check_tail_column_checksums_legal( const bool is_data_table, const ObIArray &base_column_checksums, @@ -185,7 +181,7 @@ struct ObTableCkmItems uint64_t table_id_; int64_t row_count_; const share::schema::ObTableSchema *table_schema_; - common::ObSEArray tablet_pairs_; + common::ObSEArray tablet_ids_; share::ObReplicaCkmArray ckm_items_; ObSortColumnIdArray sort_col_id_array_; // column_id -> array_idx common::ObSEArray ckm_sum_array_; // order by TableSchema::tablet_ids diff --git a/src/storage/compaction/ob_tablet_check_info.h b/src/storage/compaction/ob_tablet_check_info.h index 0420136a7..30f22e94c 100644 --- a/src/storage/compaction/ob_tablet_check_info.h +++ b/src/storage/compaction/ob_tablet_check_info.h @@ -19,7 +19,6 @@ #include "lib/utility/ob_print_utils.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -31,13 +30,11 @@ class ObTabletCheckInfo public: ObTabletCheckInfo() : tablet_id_(), - ls_id_(), check_medium_scn_(0) {} - ObTabletCheckInfo(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, int64_t medium_scn) + ObTabletCheckInfo(const common::ObTabletID &tablet_id, int64_t medium_scn) : tablet_id_(tablet_id), - ls_id_(ls_id), check_medium_scn_(medium_scn) {} @@ -50,14 +47,12 @@ class ObTabletCheckInfo return OB_SUCCESS; } const ObTabletID &get_tablet_id() const { return tablet_id_; } - const share::ObLSID &get_ls_id() const { return ls_id_; } int64_t get_medium_scn() const { return check_medium_scn_; } bool operator==(const ObTabletCheckInfo &other) const; - TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(check_medium_scn)); + TO_STRING_KV(K_(tablet_id), K_(check_medium_scn)); private: common::ObTabletID tablet_id_; - share::ObLSID ls_id_; int64_t check_medium_scn_; }; diff --git a/src/storage/compaction/ob_tablet_merge_ctx.cpp b/src/storage/compaction/ob_tablet_merge_ctx.cpp index 3ea54e42c..969905267 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_tablet_merge_ctx.cpp @@ -172,11 +172,9 @@ void ObTabletMiniMergeCtx::try_schedule_compaction_after_mini(ObTabletHandle &ta if (get_tablet_id().is_ls_inner_tablet() || 0 == get_merge_info().get_merge_history().get_macro_block_count()) { // do nothing - } else if (nullptr != static_param_.schema_ && static_param_.schema_->is_mv_major_refresh_table()) { - // do nothing } else if (FALSE_IT(try_report_tablet_stat_after_mini())) { // try report after mini every time for updating table mode for tablet. } else if (OB_TMP_FAIL(ObTenantTabletScheduler::try_schedule_adaptive_merge( - static_param_.ls_handle_, + static_param_.ls_, tablet_handle, ObAdaptiveMergePolicy::SCHEDULE_AFTER_MINI, info_collector_.tnode_stat_.update_row_count_, @@ -188,10 +186,10 @@ void ObTabletMiniMergeCtx::try_schedule_compaction_after_mini(ObTabletHandle &ta if (create_dag || 0 == get_merge_info().get_merge_history().get_macro_block_count()) { // no need to schedule minor merge } else if (OB_TMP_FAIL(ObTenantTabletScheduler::schedule_tablet_minor_merge( - static_param_.ls_handle_, tablet_handle))) { + static_param_.ls_, tablet_handle))) { if (OB_SIZE_OVERFLOW != tmp_ret) { LOG_ERROR_RET(tmp_ret, "failed to schedule special tablet minor merge", - "ls_id", get_ls_id(), "tablet_id", get_tablet_id()); + "tablet_id", get_tablet_id()); } } } @@ -201,7 +199,6 @@ void ObTabletMiniMergeCtx::try_schedule_compaction_after_mini(ObTabletHandle &ta int ObTabletMiniMergeCtx::try_report_tablet_stat_after_mini() { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = get_ls_id(); const ObTabletID &tablet_id = get_tablet_id(); const ObTransNodeDMLStat &tnode_stat = info_collector_.tnode_stat_; bool report_succ = false; @@ -212,7 +209,6 @@ int ObTabletMiniMergeCtx::try_report_tablet_stat_after_mini() // insufficient data, skip to report } else { ObTabletStat report_stat; - report_stat.ls_id_ = get_ls_id().id(); report_stat.tablet_id_ = get_tablet_id().id(); report_stat.merge_cnt_ = 1; report_stat.insert_row_cnt_ = tnode_stat.insert_row_count_; @@ -222,7 +218,7 @@ int ObTabletMiniMergeCtx::try_report_tablet_stat_after_mini() LOG_WARN("failed to report tablet stat", KR(ret)); } } - FLOG_INFO("try report tablet stat", KR(ret), K(ls_id), K(tablet_id), K(tnode_stat), K(report_succ)); + FLOG_INFO("try report tablet stat", KR(ret), K(tablet_id), K(tnode_stat), K(report_succ)); return ret; } diff --git a/src/storage/compaction/ob_tablet_merge_ctx.h b/src/storage/compaction/ob_tablet_merge_ctx.h index f93fb9cf2..7667169b9 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_tablet_merge_ctx.h @@ -19,8 +19,6 @@ #include "lib/utility/ob_print_utils.h" #include "storage/compaction/ob_partition_merge_progress.h" -#include "storage/tx_storage/ob_ls_map.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "share/scn.h" #include "storage/ob_tenant_tablet_stat_mgr.h" #include "storage/compaction/ob_tablet_merge_info.h" diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 549106d8e..04eb14a34 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -46,9 +46,7 @@ ObMergeParameter::ObMergeParameter( : static_param_(static_param), merge_range_(), trans_state_mgr_(nullptr), - error_location_(nullptr), - mview_merge_param_(nullptr), - allocator_(nullptr) + error_location_(nullptr) { } @@ -62,18 +60,11 @@ void ObMergeParameter::reset() merge_range_.reset(); trans_state_mgr_ = nullptr; error_location_ = nullptr; - if (nullptr != mview_merge_param_) { - mview_merge_param_->~ObMviewMergeParameter(); - allocator_->free(mview_merge_param_); - mview_merge_param_ = nullptr; - } - allocator_ = nullptr; } int ObMergeParameter::init( compaction::ObBasicTabletMergeCtx &merge_ctx, - const int64_t idx, - ObIAllocator *allocator) + const int64_t idx) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!merge_ctx.is_valid() || idx < 0 || idx >= static_param_.concurrent_cnt_)) { @@ -84,7 +75,6 @@ int ObMergeParameter::init( } else { const ObMergeType merge_type = static_param_.get_merge_type(); merge_version_range_ = static_param_.version_range_; - allocator_ = allocator; if (is_major_merge_type(merge_type)) { // major merge should only read data between two major freeze points // but there will be some minor sstables which across major freeze points @@ -108,14 +98,9 @@ int ObMergeParameter::init( } } } - if (OB_SUCC(ret)) { - const ObStorageSchema *schema = nullptr; - if (OB_ISNULL(schema = get_schema())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("schema is null", K(ret), K(*this));; - } else if (schema->is_mv_major_refresh_table() && is_major_merge_type(static_param_.get_merge_type()) && OB_FAIL(init_mview_merge_param(allocator))) { - STORAGE_LOG(WARN, "failed to init mview merge param", K(ret)); - } + if (OB_SUCC(ret) && OB_ISNULL(get_schema())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("schema is null", K(ret), K(*this)); } if (OB_SUCC(ret)) { FLOG_INFO("success to init ObMergeParameter", K(ret), K(idx), K_(static_param_.merge_scn), K_(merge_version_range)); @@ -123,18 +108,6 @@ int ObMergeParameter::init( return ret; } -int ObMergeParameter::init_mview_merge_param(ObIAllocator *allocator) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(mview_merge_param_ = OB_NEWx(ObMviewMergeParameter, allocator))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc mview merge param", K(ret)); - } else if (OB_FAIL(mview_merge_param_->init(*this))) { - STORAGE_LOG(WARN, "Failed to init mview merge param", K(ret)); - } - return ret; -} - const storage::ObTablesHandleArray & ObMergeParameter::get_tables_handle() const { return static_param_.tables_handle_; @@ -154,9 +127,9 @@ bool ObMergeParameter::is_delete_insert_merge() const return static_param_.is_delete_insert_merge_; } -bool ObMergeParameter::is_ha_compeleted() const +bool ObMergeParameter::is_restore_complete() const { - return static_param_.is_ha_compeleted_; + return static_param_.is_restore_complete_; } int64_t ObMergeParameter::to_string(char* buf, const int64_t buf_len) const @@ -218,14 +191,12 @@ ObTabletMergeDagParam::ObTabletMergeDagParam() exec_mode_(ObExecMode::EXEC_MODE_LOCAL), merge_type_(INVALID_MERGE_TYPE), merge_version_(0), - ls_id_(), tablet_id_() { } ObTabletMergeDagParam::ObTabletMergeDagParam( const compaction::ObMergeType merge_type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id) : skip_get_tablet_(false), need_swap_tablet_flag_(false), @@ -233,15 +204,13 @@ ObTabletMergeDagParam::ObTabletMergeDagParam( exec_mode_(ObExecMode::EXEC_MODE_LOCAL), merge_type_(merge_type), merge_version_(0), - ls_id_(ls_id), tablet_id_(tablet_id) { } bool ObTabletMergeDagParam::is_valid() const { - return ls_id_.is_valid() - && tablet_id_.is_valid() + return tablet_id_.is_valid() && is_valid_merge_type(merge_type_) && (!is_multi_version_merge(merge_type_) || merge_version_ >= 0) && is_valid_exec_mode(exec_mode_); @@ -279,13 +248,13 @@ int ObTabletMergeDag::get_tablet_and_compat_mode() int ret = OB_SUCCESS; // can't get tablet_handle now! because this func is called in create dag, // the last compaction dag is not finished yet, tablet is in old version - ObLSHandle tmp_ls_handle; + ObLS *ls = nullptr; ObTabletHandle tmp_tablet_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, tmp_ls_handle, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(tmp_ls_handle.get_ls()->get_tablet_svr()->get_tablet( + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get single log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet( tablet_id_, tmp_tablet_handle, 0/*timeout_us*/, storage::ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_id_)); } else if (OB_FAIL(ObTabletStatusCache::check_could_execute(merge_type_, *tmp_tablet_handle.get_obj()))) { if (OB_NO_NEED_MERGE != ret) { LOG_WARN("failed to check need merge", K(ret)); @@ -300,7 +269,7 @@ int ObTabletMergeDag::get_tablet_and_compat_mode() } else if (ObPartitionMergePolicy::is_sstable_count_not_safe(inc_sstable_cnt)) { ret = OB_TOO_MANY_SSTABLE; LOG_ERROR("Too many sstables in tablet, cannot schdule mini compaction, retry later", - K(ret), K_(ls_id), K_(tablet_id), K(inc_sstable_cnt), K(tmp_tablet_handle.get_obj())); + K(ret), K_(tablet_id), K(inc_sstable_cnt), K(tmp_tablet_handle.get_obj())); ObPartitionMergePolicy::diagnose_table_count_unsafe(MINI_MERGE, ObDiagnoseTabletType::TYPE_MINI_MERGE, *tmp_tablet_handle.get_obj()); } @@ -354,7 +323,6 @@ int ObTabletMergeDag::inner_init(const ObTabletMergeDagParam *param) } else { param_ = *param; merge_type_ = param->merge_type_; - ls_id_ = param->ls_id_; tablet_id_ = param->tablet_id_; if (param->skip_get_tablet_) { } else if (OB_FAIL(get_tablet_and_compat_mode())) { @@ -377,7 +345,6 @@ bool ObTabletMergeDag::operator == (const ObIDag &other) const } else { const ObTabletMergeDag &other_merge_dag = static_cast(other); if (merge_type_ != other_merge_dag.merge_type_ - || ls_id_ != other_merge_dag.ls_id_ || tablet_id_ != other_merge_dag.tablet_id_) { is_same = false; } @@ -390,7 +357,6 @@ uint64_t ObMergeDagHash::inner_hash() const uint64_t hash_value = 0; // make two merge type same hash_value = common::murmurhash(&merge_type_, sizeof(merge_type_), hash_value); - hash_value = common::murmurhash(&ls_id_, sizeof(ls_id_), hash_value); hash_value = common::murmurhash(&tablet_id_, sizeof(tablet_id_), hash_value); return hash_value; } @@ -399,8 +365,7 @@ bool ObMergeDagHash::belong_to_same_tablet(const ObMergeDagHash *other) const { bool bret = false; if (nullptr != other) { - bret = ls_id_ == other->ls_id_ - && tablet_id_ == other->tablet_id_; + bret = tablet_id_ == other->tablet_id_; } return bret; } @@ -433,7 +398,6 @@ int ObTabletMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, } else { const int64_t concurrent_cnt = OB_ISNULL(ctx_) ? 0 : ctx_->get_concurrent_cnt(); if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - ls_id_.id(), static_cast(tablet_id_.id()), param_.merge_version_, "exec_mode", exec_mode_to_str(param_.exec_mode_), @@ -447,8 +411,8 @@ int ObTabletMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, int ObTabletMergeDag::fill_dag_key(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; - if (OB_FAIL(databuff_printf(buf, buf_len, "ls_id=%ld tablet_id=%ld", ls_id_.id(), tablet_id_.id()))) { - LOG_WARN("failed to fill dag key", K(ret), K_(ls_id), K_(tablet_id)); + if (OB_FAIL(databuff_printf(buf, buf_len, "tablet_id=%ld", tablet_id_.id()))) { + LOG_WARN("failed to fill dag key", K(ret), K_(tablet_id)); } return ret; } @@ -497,7 +461,6 @@ void ObTabletMergeDag::fill_compaction_progress( progress.merge_type_ = ctx.get_inner_table_merge_type(); progress.merge_version_ = ctx.get_merge_version(); progress.status_ = get_dag_status(); - progress.ls_id_ = ctx.get_ls_id().id(); progress.tablet_id_ = ctx.get_tablet_id().id(); progress.dag_id_ = get_dag_id(); progress.create_time_ = add_time_; @@ -630,9 +593,10 @@ int ObTabletMergeExecuteDag::prepare_init( const ObTabletMergeDagParam ¶m, const lib::Worker::CompatMode compat_mode, const ObGetMergeTablesResult &result, - ObLSHandle &ls_handle) + ObLS *ls) { int ret = OB_SUCCESS; + UNUSED(ls); if (OB_UNLIKELY((!is_minor_merge_type(param.merge_type_) && !is_meta_major_merge(param.merge_type_) && !is_mds_minor_merge(param.merge_type_)) || !result.is_valid())) { @@ -641,7 +605,6 @@ int ObTabletMergeExecuteDag::prepare_init( } else { param_ = param; merge_type_ = param.merge_type_; - ls_id_ = param.ls_id_; tablet_id_ = param.tablet_id_; compat_mode_ = compat_mode; if (OB_FAIL(result_.assign(result))) { diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index e426aacf6..b45b3da95 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -21,7 +21,6 @@ #include "observer/scheduler/ob_tenant_dag_scheduler.h" #include "storage/ob_i_table.h" #include "storage/blocksstable/ob_datum_range.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/compaction/ob_i_compaction_filter.h" #include "storage/compaction/ob_compaction_util.h" #include "storage/ob_storage_struct.h" @@ -33,6 +32,7 @@ namespace oceanbase namespace storage { class ObITable; +class ObLS; class ObTablet; class ObTabletHandle; } @@ -45,7 +45,6 @@ struct ObStaticMergeParam; class ObPartitionMerger; struct ObCachedTransStateMgr; class ObPartitionMergeProgress; -class ObMviewMergeParameter; /* DAG : *PrepareTask -> ObTabletMergeTask* -> ObTabletMergeFinishTask @@ -61,16 +60,12 @@ struct ObMergeParameter { ~ObMergeParameter() { reset(); } bool is_valid() const; void reset(); - int init(ObBasicTabletMergeCtx &merge_ctx, const int64_t idx, ObIAllocator *allocator = nullptr); + int init(ObBasicTabletMergeCtx &merge_ctx, const int64_t idx); const storage::ObTablesHandleArray & get_tables_handle() const; const ObStorageSchema *get_schema() const; bool is_full_merge() const; - OB_INLINE bool is_mv_merge() const - { - return nullptr != mview_merge_param_; - } bool is_delete_insert_merge() const; - bool is_ha_compeleted() const; + bool is_restore_complete() const; const ObStaticMergeParam &static_param_; /* rest variables are different for MergeTask */ @@ -78,12 +73,9 @@ struct ObMergeParameter { blocksstable::ObDatumRange merge_range_; // rowkey_range compaction::ObCachedTransStateMgr *trans_state_mgr_; share::ObDiagnoseLocation *error_location_; - ObMviewMergeParameter *mview_merge_param_; - ObIAllocator *allocator_; int64_t to_string(char* buf, const int64_t buf_len) const; private: - int init_mview_merge_param(ObIAllocator *allocator); DISALLOW_COPY_AND_ASSIGN(ObMergeParameter); }; @@ -115,11 +107,10 @@ struct ObTabletMergeDagParam : public share::ObIDagInitParam ObTabletMergeDagParam(); ObTabletMergeDagParam( const compaction::ObMergeType merge_type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id); virtual bool is_valid() const override; VIRTUAL_TO_STRING_KV(K_(skip_get_tablet), "merge_type", merge_type_to_str(merge_type_), K_(merge_version), - K_(ls_id), K_(tablet_id), "exec_mode", exec_mode_to_str(exec_mode_), + K_(tablet_id), "exec_mode", exec_mode_to_str(exec_mode_), K_(need_swap_tablet_flag), K_(is_reserve_mode)); bool skip_get_tablet_; @@ -128,7 +119,6 @@ struct ObTabletMergeDagParam : public share::ObIDagInitParam ObExecMode exec_mode_; compaction::ObMergeType merge_type_; int64_t merge_version_; - share::ObLSID ls_id_; ObTabletID tablet_id_; ObCompactionParam compaction_param_; // used for adaptive compaction dag scheduling }; @@ -169,7 +159,6 @@ class ObMergeDagHash public: ObMergeDagHash() : merge_type_(compaction::ObMergeType::INVALID_MERGE_TYPE), - ls_id_(), tablet_id_() {} virtual ~ObMergeDagHash() {} @@ -177,16 +166,15 @@ class ObMergeDagHash bool is_valid() const { return merge_type_ > INVALID_MERGE_TYPE && merge_type_ < MERGE_TYPE_MAX - && ls_id_.is_valid() && tablet_id_.is_valid(); + && tablet_id_.is_valid(); } virtual uint64_t inner_hash() const; bool belong_to_same_tablet(const ObMergeDagHash *other) const; - TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(ls_id), K_(tablet_id)); + TO_STRING_KV("merge_type", merge_type_to_str(merge_type_), K_(tablet_id)); compaction::ObMergeType merge_type_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; @@ -200,7 +188,6 @@ class ObTabletMergeDag: public share::ObIDag, public ObMergeDagHash virtual ObBasicTabletMergeCtx *get_ctx() { return ctx_; } ObTabletMergeDagParam &get_param() { return param_; } const ObTabletMergeDagParam &get_param() const { return param_; } - virtual const share::ObLSID & get_ls_id() const { return param_.ls_id_; } bool is_reserve_mode() const { return param_.is_reserve_mode_; } void set_reserve_mode() { param_.is_reserve_mode_ = true; } virtual bool operator == (const ObIDag &other) const override; @@ -262,7 +249,7 @@ class ObTabletMergeExecuteDag: public ObTabletMergeDag const ObTabletMergeDagParam ¶m, const lib::Worker::CompatMode compat_mode, const ObGetMergeTablesResult &result, - storage::ObLSHandle &ls_handle); + storage::ObLS *ls); virtual bool operator == (const ObIDag &other) const override; const share::ObScnRange& get_merge_range() const { return result_.scn_range_; } const ObGetMergeTablesResult& get_result() const { return result_; } diff --git a/src/storage/compaction/ob_tenant_compaction_progress.cpp b/src/storage/compaction/ob_tenant_compaction_progress.cpp index b2a55a869..8b602139c 100644 --- a/src/storage/compaction/ob_tenant_compaction_progress.cpp +++ b/src/storage/compaction/ob_tenant_compaction_progress.cpp @@ -113,59 +113,41 @@ int ObTenantCompactionProgressMgr::loop_major_sstable_( { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("loop_major_sstable_to_calc_progress_size", 30 * 1000 * 1000); // 30s - ObSharedGuard ls_iter_guard; ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get ls iterator", K(ret)); - } + ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get single log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->build_tablet_iter(tablet_iter))) { + LOG_WARN("failed to build ls tablet iter", K(ret)); + } else { + ObTabletHandle tablet_handle; - while (OB_SUCC(ret)) { // loop all log_stream - ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); - if (OB_FAIL(ls_iter_guard.get_ptr()->get_next(ls))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get ls", K(ret), KP(ls_iter_guard.get_ptr())); - } - } else if (OB_ISNULL(ls)){ - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is nullptr", K(ret), KPC(ls)); - } else if (ls->is_deleted()) { - // do nothing - } else if (OB_FAIL(ls->get_tablet_svr()->build_tablet_iter(tablet_iter))) { - LOG_WARN("failed to build ls tablet iter", K(ret), K(ls)); - } else { - const ObLSID &ls_id = ls->get_ls_id(); - ObTabletHandle tablet_handle; - - while (OB_SUCC(ret)) { - if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_handle)); - } - } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet handle", K(ret), K(ls_id), K(tablet_handle)); - } else if (tablet_handle.get_obj()->get_tablet_meta().tablet_id_.is_special_merge_tablet()) { - // do nothing - } else if (0 == tablet_handle.get_obj()->get_last_major_snapshot_version() - || tablet_handle.get_obj()->get_last_major_snapshot_version() > merge_snapshot_version) { - // do nothing + while (OB_SUCC(ret)) { + if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; } else { - // ATTENTION: - // 1. it is hard to distinguish whether this major was generated by this compaction or whether it existed before the compaction, - // so maybe some more tablets will be calculated. - // 2. for the major sstable generated by this compaction, the size will be calculated larger than the old major sstable. - ++cnt; - size += tablet_handle.get_obj()->get_last_major_total_macro_block_count() * DEFAULT_MACRO_BLOCK_SIZE; + LOG_WARN("failed to get tablet", K(ret), K(tablet_handle)); } - } // end loop tablet - } - } // end loop ls + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle)); + } else if (tablet_handle.get_obj()->get_tablet_meta().tablet_id_.is_special_merge_tablet()) { + // do nothing + } else if (0 == tablet_handle.get_obj()->get_last_major_snapshot_version() + || tablet_handle.get_obj()->get_last_major_snapshot_version() > merge_snapshot_version) { + // do nothing + } else { + // ATTENTION: + // 1. it is hard to distinguish whether this major was generated by this compaction or whether it existed before the compaction, + // so maybe some more tablets will be calculated. + // 2. for the major sstable generated by this compaction, the size will be calculated larger than the old major sstable. + ++cnt; + size += tablet_handle.get_obj()->get_last_major_total_macro_block_count() * DEFAULT_MACRO_BLOCK_SIZE; + } + } // end loop tablet + } return ret; } diff --git a/src/storage/compaction/ob_tenant_compaction_progress.h b/src/storage/compaction/ob_tenant_compaction_progress.h index d862b75b6..228971ff3 100644 --- a/src/storage/compaction/ob_tenant_compaction_progress.h +++ b/src/storage/compaction/ob_tenant_compaction_progress.h @@ -154,17 +154,15 @@ struct ObTabletCompactionProgress : public ObCompactionProgress { ObTabletCompactionProgress() : ObCompactionProgress(), - ls_id_(0), tablet_id_(0), dag_id_(), progressive_merge_round_(0), create_time_(0) { } - INHERIT_TO_STRING_KV("ObCompactionProgress", ObCompactionProgress, K_(ls_id), - K_(tablet_id), K_(dag_id), K_(progressive_merge_round), K_(create_time)); + INHERIT_TO_STRING_KV("ObCompactionProgress", ObCompactionProgress, K_(tablet_id), + K_(dag_id), K_(progressive_merge_round), K_(create_time)); - int64_t ls_id_; int64_t tablet_id_; share::ObDagId dag_id_; int64_t progressive_merge_round_; diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp index 435795102..4b8260bc6 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp @@ -387,41 +387,6 @@ int64_t ObTenantFreezeInfoMgr::get_min_reserved_snapshot_for_tx() return snapshot_version; } -void ObTenantFreezeInfoMgr::check_tenant_in_restore_with_mv_( - bool &need_check_mview, - ObSchemaGetterGuard &schema_guard, - const ObSimpleTenantSchema *&tenant_schema) -{ - need_check_mview = false; - int ret = OB_SUCCESS; - if (MTL_TENANT_ROLE_CACHE_IS_INVALID()) { - need_check_mview = true; - } else if (MTL_TENANT_ROLE_CACHE_IS_PRIMARY()) { - need_check_mview = false; - } else { - // get tenant schema if pointer is nullptr - - if (OB_ISNULL(tenant_schema)) { - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_schema))) { - LOG_WARN("failed to get tenant info", K(ret)); - } - } - if (OB_FAIL(ret)) { - need_check_mview = true; - LOG_WARN("failed to get tenant schema, need check mview", K(ret), - K(need_check_mview), KP(tenant_schema)); - } else if (OB_ISNULL(tenant_schema)) { - need_check_mview = true; - LOG_WARN("tenant schema is null, need check mview", K(ret), - K(need_check_mview), KP(tenant_schema)); - } else if (tenant_schema->is_restore()) { - need_check_mview = true; - } - } -} - // get smallest kept snapshot int ObTenantFreezeInfoMgr::get_min_reserved_snapshot( const ObTabletID &tablet_id, @@ -433,9 +398,6 @@ int ObTenantFreezeInfoMgr::get_min_reserved_snapshot( int64_t duration = 0; bool unused = false; snapshot_info.reset(); - const ObSimpleTenantSchema *tenant_schema = nullptr; - ObSchemaGetterGuard schema_guard; - const int64_t abs_timeout_us = common::ObTimeUtility::current_time() + RLOCK_TIMEOUT_US; RLockGuardWithTimeout lock_guard(lock_, abs_timeout_us, ret); ObIArray &snapshots = snapshots_[cur_idx_]; @@ -464,27 +426,13 @@ int ObTenantFreezeInfoMgr::get_min_reserved_snapshot( snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_UNDO_RETENTION, snapshot_for_undo_retention); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_TX, snapshot_for_tx); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_MAJOR_FREEZE_TS, freeze_info.frozen_scn_.get_val_for_tx()); - bool exit_loop = false; - for (int64_t i = 0; i < snapshots.count() && OB_SUCC(ret) && !exit_loop; ++i) { + for (int64_t i = 0; i < snapshots.count() && OB_SUCC(ret); ++i) { bool related = false; const ObSnapshotInfo &snapshot = snapshots.at(i); if (OB_FAIL(is_snapshot_related_to_tablet(tablet_id, snapshot, related))) { STORAGE_LOG(WARN, "fail to check snapshot relation", K(ret), K(tablet_id), K(snapshot)); } else if (related) { snapshot_info.update_by_smaller_snapshot(snapshot.snapshot_type_, snapshot.snapshot_scn_.get_val_for_tx()); - if (ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV == snapshot.snapshot_type_) { - // if exist mview snapshot type and tenant in restore - // if tenant is invalid or restore, need check mview snapshot - // TODO siyu:: use tenant_status_cache - bool need_check_mview = false; - IGNORE_RETURN check_tenant_in_restore_with_mv_(need_check_mview, schema_guard, tenant_schema); - if (need_check_mview) { - exit_loop = true; - snapshot_info.update_by_smaller_snapshot(ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV, static_cast(0)); - LOG_INFO("exist new mv in restore", K(ret), K(snapshot_info), K(tablet_id), K(merged_version), K(need_check_mview)); - } - LOG_INFO("exist new mview when calc multi_version_start", K(ret), K(tablet_id), K(need_check_mview), K(snapshot_info), K(exit_loop)); - } } } LOG_TRACE("check_freeze_info_mgr", K(ret), K(snapshot_info), K(duration), K(snapshot_for_undo_retention), @@ -712,27 +660,19 @@ int ObTenantFreezeInfoMgr::try_update_reserved_snapshot() } } // end of lock - // loop all ls, try update reserved snapshot - ObSharedGuard ls_iter_guard; + // Try to update the reserved snapshot on the log stream. ObLS *ls = nullptr; if (OB_FAIL(ret) || reserved_snapshot <= 0) { - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get ls iterator", K(ret)); + } else if (OB_ISNULL(share::g_mp->ls_service())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "ls service is null", K(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get single log stream", K(ret)); } else { int tmp_ret = OB_SUCCESS; - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter_guard.get_ptr()->get_next(ls))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), KP(ls)); - } else if (OB_TMP_FAIL(ls->try_sync_reserved_snapshot(reserved_snapshot, true/*update_flag*/))) { - LOG_WARN("failed to update min reserved snapshot", K(tmp_ret), KPC(ls), K(reserved_snapshot)); - } - } // end of while + if (OB_TMP_FAIL(ls->try_sync_reserved_snapshot(reserved_snapshot, true/*update_flag*/))) { + LOG_WARN("failed to update min reserved snapshot", K(tmp_ret), KPC(ls), K(reserved_snapshot)); + } } cost_ts = ObTimeUtility::fast_current_time() - cost_ts; STORAGE_LOG(INFO, "update reserved snapshot finished", K(cost_ts), K(reserved_snapshot)); diff --git a/src/storage/compaction/ob_tenant_medium_checker.cpp b/src/storage/compaction/ob_tenant_medium_checker.cpp index dac190f3f..06c7f5c3e 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.cpp +++ b/src/storage/compaction/ob_tenant_medium_checker.cpp @@ -31,22 +31,19 @@ namespace compaction * */ bool ObTabletCheckInfo::is_valid() const { - return tablet_id_.is_valid() && ls_id_.is_valid() && - check_medium_scn_ != 0; + return tablet_id_.is_valid() && check_medium_scn_ != 0; } uint64_t ObTabletCheckInfo::hash() const { uint64_t hash_val = 0; hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); - hash_val = murmurhash(&ls_id_, sizeof(ls_id_), hash_val); return hash_val; } bool ObTabletCheckInfo::operator==(const ObTabletCheckInfo &other) const { - return tablet_id_ == other.tablet_id_ && - ls_id_ == other.ls_id_; + return tablet_id_ == other.tablet_id_; } /* @@ -76,11 +73,9 @@ int ObTenantMediumChecker::mtl_init(ObTenantMediumChecker *&tablet_medium_checke ObTenantMediumChecker::ObTenantMediumChecker() : is_inited_(false), - last_check_timestamp_(0), error_tablet_cnt_(0), - tablet_ls_set_(), - lock_(), - ls_locality_cache_empty_(true) + tablet_check_set_(), + lock_() {} ObTenantMediumChecker::~ObTenantMediumChecker() @@ -94,7 +89,7 @@ int ObTenantMediumChecker::init() if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTenantMediumChecker is inited before", KR(ret), KPC(this)); - } else if (OB_FAIL(tablet_ls_set_.create(DEFAULT_MAP_BUCKET, "MedCheckSet", "CheckSetNode"))) { + } else if (OB_FAIL(tablet_check_set_.create(DEFAULT_MAP_BUCKET, "MedCheckSet", "CheckSetNode"))) { LOG_WARN("failed to create set", K(ret)); } else { is_inited_ = true; @@ -108,41 +103,30 @@ int ObTenantMediumChecker::init() void ObTenantMediumChecker::destroy() { is_inited_ = false; - ls_locality_cache_empty_ = true; - if (tablet_ls_set_.created()) { - tablet_ls_set_.destroy(); + if (tablet_check_set_.created()) { + tablet_check_set_.destroy(); } } -int ObTenantMediumChecker::add_tablet_ls(const ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t medium_scn) +int ObTenantMediumChecker::add_tablet(const ObTabletID &tablet_id, const int64_t medium_scn) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); - } else if (!tablet_id.is_valid() || !ls_id.is_valid() || 0 == medium_scn) { + } else if (!tablet_id.is_valid() || 0 == medium_scn) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_id), K(ls_id), K(medium_scn)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(medium_scn)); } else { lib::ObMutexGuard guard(lock_); // just cover the old info - if (OB_FAIL(tablet_ls_set_.set_refactored(ObTabletCheckInfo(tablet_id, ls_id, medium_scn)))) { - LOG_WARN("failed to set tablet_ls_info", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(tablet_check_set_.set_refactored(ObTabletCheckInfo(tablet_id, medium_scn)))) { + LOG_WARN("failed to set tablet check info", K(ret), K(tablet_id)); } } return ret; } -bool ObTenantMediumChecker::locality_cache_empty() -{ - bool bret = true; - if (IS_INIT) { - bret = ls_locality_cache_empty_; - } - return bret; -} - int ObTenantMediumChecker::check_medium_finish_schedule() { int ret = OB_SUCCESS; @@ -151,45 +135,46 @@ int ObTenantMediumChecker::check_medium_finish_schedule() ret = OB_NOT_INIT; LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); } else { - DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); - TabletLSArray tablet_ls_infos; - tablet_ls_infos.set_attr(ObMemAttr("CheckInfos")); - TabletLSArray batch_tablet_ls_infos; - batch_tablet_ls_infos.set_attr(ObMemAttr("BCheckInfos")); - TabletLSArray finish_tablet_ls_infos; - finish_tablet_ls_infos.set_attr(ObMemAttr("FinishInfos")); - // copy the tablet_ls_infos from set // with lock + DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); + TabletCheckArray tablet_check_infos; + tablet_check_infos.set_attr(ObMemAttr("CheckInfos")); + TabletCheckArray batch_tablet_check_infos; + batch_tablet_check_infos.set_attr(ObMemAttr("BCheckInfos")); + TabletCheckArray finish_tablet_check_infos; + finish_tablet_check_infos.set_attr(ObMemAttr("FinishInfos")); + // copy the tablet check infos from set with lock { lib::ObMutexGuard guard(lock_); - if (OB_FAIL(tablet_ls_infos.reserve(tablet_ls_set_.size()))) { - LOG_WARN("fail to reserve array", K(ret), "size", tablet_ls_set_.size()); + if (OB_FAIL(tablet_check_infos.reserve(tablet_check_set_.size()))) { + LOG_WARN("fail to reserve array", K(ret), "size", tablet_check_set_.size()); } else { - for (TabletLSSet::const_iterator iter = tablet_ls_set_.begin(); - OB_SUCC(ret) && iter != tablet_ls_set_.end(); ++iter) { - if (OB_FAIL(tablet_ls_infos.push_back(iter->first))) { - LOG_WARN("failed to push back tablet_ls_info", K(ret), K(iter->first)); + for (TabletCheckSet::const_iterator iter = tablet_check_set_.begin(); + OB_SUCC(ret) && iter != tablet_check_set_.end(); ++iter) { + if (OB_FAIL(tablet_check_infos.push_back(iter->first))) { + LOG_WARN("failed to push back tablet check info", K(ret), K(iter->first)); } } } if (OB_SUCC(ret)) { - tablet_ls_set_.clear(); + tablet_check_set_.clear(); } } const int64_t batch_size = share::g_mp->tenant_tablet_scheduler()->get_checker_batch_size(); - if (OB_FAIL(ret) || tablet_ls_infos.empty()) { - } else if (OB_FAIL(batch_tablet_ls_infos.reserve(batch_size))) { + if (OB_FAIL(ret) || tablet_check_infos.empty()) { + } else if (OB_FAIL(batch_tablet_check_infos.reserve(batch_size))) { LOG_WARN("fail to reserve array", K(ret), "size", batch_size); - } else if (OB_FAIL(finish_tablet_ls_infos.reserve(batch_size))) { + } else if (OB_FAIL(finish_tablet_check_infos.reserve(batch_size))) { LOG_WARN("fail to reserve array", K(ret), "size", batch_size); } else { // batch check - int64_t info_count = tablet_ls_infos.count(); + int64_t info_count = tablet_check_infos.count(); int64_t start_idx = 0; int64_t end_idx = min(batch_size, info_count); int64_t cost_ts = ObTimeUtility::fast_current_time(); ObBatchFinishCheckStat stat; while (start_idx < end_idx) { - if (OB_TMP_FAIL(check_medium_finish(tablet_ls_infos, start_idx, end_idx, batch_tablet_ls_infos, finish_tablet_ls_infos, stat))) { + if (OB_TMP_FAIL(check_medium_finish(tablet_check_infos, start_idx, end_idx, + batch_tablet_check_infos, finish_tablet_check_infos, stat))) { LOG_WARN("failed to check medium finish", K(tmp_ret)); } start_idx = end_idx; @@ -222,54 +207,53 @@ int ObTenantMediumChecker::check_medium_finish_schedule() } int ObTenantMediumChecker::check_medium_finish( - const ObIArray &tablet_ls_infos, + const ObIArray &tablet_check_infos, int64_t start_idx, int64_t end_idx, - ObArray &check_tablet_ls_infos, - ObArray &finish_tablet_ls_infos, + ObArray &check_tablet_infos, + ObArray &finish_tablet_infos, ObBatchFinishCheckStat &stat) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - if (OB_UNLIKELY(tablet_ls_infos.empty() + if (OB_UNLIKELY(tablet_check_infos.empty() || start_idx < 0 || start_idx >= end_idx - || end_idx > tablet_ls_infos.count())) { + || end_idx > tablet_check_infos.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_ls_infos), K(start_idx), K(end_idx)); + LOG_WARN("invalid argument", KR(ret), K(tablet_check_infos), K(start_idx), K(end_idx)); } else { - check_tablet_ls_infos.reuse(); - finish_tablet_ls_infos.reuse(); + check_tablet_infos.reuse(); + finish_tablet_infos.reuse(); for (int64_t i = start_idx; OB_SUCC(ret) && i < end_idx; ++i) { - const ObLSID &ls_id = tablet_ls_infos.at(i).get_ls_id(); - if (OB_UNLIKELY(0 == tablet_ls_infos.at(i).get_medium_scn())) { + if (OB_UNLIKELY(0 == tablet_check_infos.at(i).get_medium_scn())) { tmp_ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected medium scn", K(tmp_ret), K(tablet_ls_infos.at(i))); + LOG_WARN("unexpected medium scn", K(tmp_ret), K(tablet_check_infos.at(i))); } else { - if (OB_FAIL(check_tablet_ls_infos.push_back(tablet_ls_infos.at(i)))) { - LOG_WARN("fail to push back check_tablet_ls_infos", K(ret), K(tablet_ls_infos.at(i))); + if (OB_FAIL(check_tablet_infos.push_back(tablet_check_infos.at(i)))) { + LOG_WARN("fail to push back check tablet infos", K(ret), K(tablet_check_infos.at(i))); } } } ObCompactionScheduleTimeGuard time_guard; - stat.filter_cnt_ += (end_idx - start_idx - check_tablet_ls_infos.count()); + stat.filter_cnt_ += (end_idx - start_idx - check_tablet_infos.count()); if (FAILEDx(ObMediumCompactionScheduleFunc::batch_check_medium_finish( - finish_tablet_ls_infos, check_tablet_ls_infos, time_guard))) { - LOG_WARN("failed to batch check medium finish", K(ret), K(tablet_ls_infos.count()), K(check_tablet_ls_infos.count()), - K(tablet_ls_infos), K(check_tablet_ls_infos)); - stat.fail_cnt_ += check_tablet_ls_infos.count(); + finish_tablet_infos, check_tablet_infos, time_guard))) { + LOG_WARN("failed to batch check medium finish", K(ret), K(tablet_check_infos.count()), K(check_tablet_infos.count()), + K(tablet_check_infos), K(check_tablet_infos)); + stat.fail_cnt_ += check_tablet_infos.count(); if (0 != stat.fail_cnt_) { - stat.failed_info_ = check_tablet_ls_infos.at(0); + stat.failed_info_ = check_tablet_infos.at(0); } - if (OB_TMP_FAIL(reput_check_info(check_tablet_ls_infos))) { + if (OB_TMP_FAIL(reput_check_info(check_tablet_infos))) { LOG_WARN("fail to reput failed check infos", K(tmp_ret)); } } else { - LOG_INFO("success to check medium finish", K(ret), K(tablet_ls_infos.count()), K(check_tablet_ls_infos.count()), - K(finish_tablet_ls_infos.count()), K(tablet_ls_infos), K(check_tablet_ls_infos), K(finish_tablet_ls_infos)); - stat.succ_cnt_ += check_tablet_ls_infos.count(); - stat.finish_cnt_ += finish_tablet_ls_infos.count(); - ObScheduleNewMediumLoop medium_loop(finish_tablet_ls_infos); + LOG_INFO("success to check medium finish", K(ret), K(tablet_check_infos.count()), K(check_tablet_infos.count()), + K(finish_tablet_infos.count()), K(tablet_check_infos), K(check_tablet_infos), K(finish_tablet_infos)); + stat.succ_cnt_ += check_tablet_infos.count(); + stat.finish_cnt_ += finish_tablet_infos.count(); + ObScheduleNewMediumLoop medium_loop(finish_tablet_infos); if (OB_FAIL(medium_loop.loop())) { LOG_WARN("failed to leader schedule", K(ret)); } else { @@ -277,13 +261,13 @@ int ObTenantMediumChecker::check_medium_finish( } } LOG_INFO("finish medium check", K(ret), K(start_idx), K(end_idx), - K(check_tablet_ls_infos.count()), K(finish_tablet_ls_infos.count()), K(time_guard)); + K(check_tablet_infos.count()), K(finish_tablet_infos.count()), K(time_guard)); } return ret; } -// just reput into set without ls status check -int ObTenantMediumChecker::reput_check_info(ObIArray &tablet_ls_infos) +// just reput into set +int ObTenantMediumChecker::reput_check_info(ObIArray &tablet_check_infos) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -292,15 +276,15 @@ int ObTenantMediumChecker::reput_check_info(ObIArray &tablet_ LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); } else { lib::ObMutexGuard guard(lock_); - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ls_infos.count(); ++i) { - const ObTabletCheckInfo &check_info = tablet_ls_infos.at(i); - if (OB_HASH_EXIST == (tmp_ret = tablet_ls_set_.exist_refactored(check_info))) { + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_check_infos.count(); ++i) { + const ObTabletCheckInfo &check_info = tablet_check_infos.at(i); + if (OB_HASH_EXIST == (tmp_ret = tablet_check_set_.exist_refactored(check_info))) { ret = OB_SUCCESS; // tablet exist } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to check exist in tablet set", K(ret), K(tmp_ret), K(check_info)); - } else if (OB_TMP_FAIL(tablet_ls_set_.set_refactored(check_info))) { - LOG_WARN("failed to set tablet_ls_info", K(tmp_ret), K(check_info)); + } else if (OB_TMP_FAIL(tablet_check_set_.set_refactored(check_info))) { + LOG_WARN("failed to set tablet check info", K(tmp_ret), K(check_info)); } } } diff --git a/src/storage/compaction/ob_tenant_medium_checker.h b/src/storage/compaction/ob_tenant_medium_checker.h index 106b1befe..f64fcf449 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.h +++ b/src/storage/compaction/ob_tenant_medium_checker.h @@ -60,14 +60,13 @@ class ObTenantMediumChecker void destroy(); int check_medium_finish_schedule(); int check_medium_finish( - const ObIArray &tablet_ls_infos, + const ObIArray &tablet_check_infos, int64_t start_idx, int64_t end_idx, - ObArray &check_tablet_ls_infos, - ObArray &finish_tablet_ls_infos, + ObArray &check_tablet_infos, + ObArray &finish_tablet_infos, ObBatchFinishCheckStat &stat); - int add_tablet_ls(const ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t medium_scn); - bool locality_cache_empty(); + int add_tablet(const ObTabletID &tablet_id, const int64_t medium_scn); int64_t get_error_tablet_cnt() { return ATOMIC_LOAD(&error_tablet_cnt_); } void clear_error_tablet_cnt() { ATOMIC_STORE(&error_tablet_cnt_, 0); } void update_error_tablet_cnt(const int64_t delta_cnt) @@ -75,29 +74,21 @@ class ObTenantMediumChecker // called when check tablet checksum error (void)ATOMIC_AAF(&error_tablet_cnt_, delta_cnt); } - TO_STRING_KV(K_(is_inited), K_(ls_locality_cache_empty)); + TO_STRING_KV(K_(is_inited)); private: - int reput_check_info(ObIArray &tablet_ls_infos); + int reput_check_info(ObIArray &tablet_check_infos); public: - static const int64_t LS_ID_ARRAY_CNT = 10; -#ifdef ERRSIM - static const int64_t CHECK_LS_LOCALITY_INTERVAL = 30 * 1000 * 1000L; // 30s -#else - static const int64_t CHECK_LS_LOCALITY_INTERVAL = 5 * 60 * 1000 * 1000L; // 5m -#endif static const int64_t DEFAULT_MAP_BUCKET = 1024; static const int64_t CLEAR_CKM_ERROR_INTERVAL = 2 * 60 * 1000 * 1000L; // 2m - typedef common::ObArray TabletLSArray; - typedef hash::ObHashSet TabletLSSet; + typedef common::ObArray TabletCheckArray; + typedef hash::ObHashSet TabletCheckSet; private: bool is_inited_; - int64_t last_check_timestamp_; int64_t error_tablet_cnt_; // for diagnose - TabletLSSet tablet_ls_set_; + TabletCheckSet tablet_check_set_; lib::ObMutex lock_; - bool ls_locality_cache_empty_; }; } diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index cd1a1e051..b995ab08d 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -78,11 +78,10 @@ int ObFastFreezeChecker::check_need_fast_freeze( need_fast_freeze = false; ObTableHandleV2 table_handle; ObITabletMemtable *memtable = nullptr; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; ObTableQueuingModeCfg queuing_cfg; - if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(ls_id, tablet_id, queuing_cfg))) { - LOG_WARN_RET(tmp_ret, "[FastFreeze] failed to get table queuing mode, treat it as normal table", K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_queuing_cfg(tablet_id, queuing_cfg))) { + LOG_WARN_RET(tmp_ret, "[FastFreeze] failed to get table queuing mode, treat it as normal table", K(tablet_id)); } const int64_t memtable_alive_threshold = queuing_cfg.get_memtable_alive_threshold(FAST_FREEZE_INTERVAL_US); if (OB_FAIL(tablet.get_active_memtable(table_handle))) { @@ -102,18 +101,18 @@ int ObFastFreezeChecker::check_need_fast_freeze( // do nothing } else if (ObTimeUtility::current_time() < memtable->get_timestamp() + memtable_alive_threshold) { if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - LOG_INFO("[FastFreeze] memtable is just created, no need to check", K(memtable_alive_threshold), K(ls_id), K(tablet_id), KPC(memtable)); + LOG_INFO("[FastFreeze] memtable is just created, no need to check", K(memtable_alive_threshold), K(tablet_id), KPC(memtable)); } } else { memtable::ObMemtable *mt = static_cast(memtable); check_hotspot_need_fast_freeze(*mt, need_fast_freeze); if (need_fast_freeze) { - FLOG_INFO("[FastFreeze] tablet detects hotspot row, need fast freeze", K(ls_id), K(tablet_id)); + FLOG_INFO("[FastFreeze] tablet detects hotspot row, need fast freeze", K(tablet_id)); } else { // Only queuing table need tombstone fast freeze in 4.2.x, but 4.3.0 has this before, so open it check_tombstone_need_fast_freeze(tablet, queuing_cfg, *mt, need_fast_freeze); if (need_fast_freeze) { - FLOG_INFO("[FastFreeze] tablet detects tombstone, need fast freeze", K(ls_id), K(tablet_id)); + FLOG_INFO("[FastFreeze] tablet detects tombstone, need fast freeze", K(tablet_id)); } } } @@ -138,13 +137,12 @@ void ObFastFreezeChecker::check_tombstone_need_fast_freeze( { need_fast_freeze = false; if (memtable.is_active_memtable()) { - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObMtStat &mt_stat = memtable.get_mt_stat(); // dirty read int64_t adaptive_threshold = queuing_cfg.get_tombstone_row_threshold(TOMBSTONE_DEFAULT_ROW_COUNT); if (!queuing_cfg.is_queuing_mode()) { // dynamically change adaptive_threshold by merge cnt in recent 10 mins - try_update_tablet_threshold(ObTabletStatKey(ls_id, tablet_id), mt_stat, memtable.get_timestamp(), queuing_cfg, adaptive_threshold); + try_update_tablet_threshold(ObTabletStatKey(tablet_id), mt_stat, memtable.get_timestamp(), queuing_cfg, adaptive_threshold); } need_fast_freeze = (mt_stat.update_row_count_ + mt_stat.delete_row_count_) >= adaptive_threshold || mt_stat.delete_row_count_ > queuing_cfg.total_delete_row_cnt_; @@ -197,7 +195,7 @@ void ObFastFreezeChecker::try_update_tablet_threshold( ObTabletStat tablet_stat; ObTabletStat total_stat; ObTableModeFlag mode = ObTableModeFlag::TABLE_MODE_NORMAL; - if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_latest_tablet_stat(key.ls_id_, key.tablet_id_, tablet_stat, total_stat, mode))) { + if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr()->get_latest_tablet_stat(key.tablet_id_, tablet_stat, total_stat, mode))) { if (OB_HASH_NOT_EXIST != tmp_ret) { LOG_WARN_RET(tmp_ret, "[FastFreeze] failed to get tablet stat", K(key)); } @@ -232,11 +230,10 @@ ObTenantTabletScheduler::ObTenantTabletScheduler() is_inited_(false), bf_queue_(), fast_freeze_checker_(), - minor_ls_tablet_iter_(false/*is_major*/), + minor_tablet_iter_(false/*is_major*/), gc_sst_tablet_iter_(false/*is_major*/), timer_task_mgr_(), - batch_size_mgr_(), - mview_validation_() + batch_size_mgr_() { STATIC_ASSERT(static_cast(NO_MAJOR_MERGE_TYPE_CNT) == ARRAYSIZEOF(MERGE_TYPES), "merge type array len is mismatch"); } @@ -260,7 +257,7 @@ void ObTenantTabletScheduler::reset() is_inited_ = false; ObBasicMergeScheduler::reset(); bf_queue_.destroy(); - minor_ls_tablet_iter_.reset(); + minor_tablet_iter_.reset(); gc_sst_tablet_iter_.reset(); LOG_INFO("The ObTenantTabletScheduler destroy"); } @@ -382,23 +379,14 @@ int ObTenantTabletScheduler::update_upper_trans_version_and_gc_sstable() gc_sst_tablet_iter_.set_tablet_get_mode(storage::ObMDSGetTabletMode::READ_WITHOUT_CHECK); } - ObLSHandle ls_handle; - ObLS *ls = nullptr; - while (OB_SUCC(ret)) { - if (OB_FAIL(gc_sst_tablet_iter_.get_next_ls(ls_handle))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get ls", K(ret), K(gc_sst_tablet_iter_)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls)); - } else if (ls->is_stopped()) { - } else if (OB_TMP_FAIL(try_update_upper_trans_version_and_gc_sstable(*ls, gc_sst_tablet_iter_))) { - gc_sst_tablet_iter_.skip_cur_ls(); - LOG_ERROR("failed to update upper trans version", K(tmp_ret), "ls_id", ls->get_ls_id()); + if (OB_SUCC(ret) && !gc_sst_tablet_iter_.is_scan_finish()) { + ObLS *ls = gc_sst_tablet_iter_.get_ls(); + if (ls->is_stopped()) { + gc_sst_tablet_iter_.finish_scan(); + } else if (OB_TMP_FAIL(try_update_upper_trans_version_and_gc_sstable( + *ls, gc_sst_tablet_iter_))) { + gc_sst_tablet_iter_.finish_scan(); + LOG_ERROR("failed to update upper trans version", K(tmp_ret)); } } return ret; @@ -409,18 +397,10 @@ int ObTenantTabletScheduler::try_update_upper_trans_version_and_gc_sstable( ObCompactionScheduleIterator &iter) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; common::ObTabletID tablet_id; - bool ls_is_migration = false; - int64_t rebuild_seq = 0; - if (OB_FAIL(ls.check_ls_migration_status(ls_is_migration, rebuild_seq))) { - LOG_WARN("failed to check ls migration status", K(ret), K(ls_id)); - } else if (ls_is_migration) { - // skip ls in migration, no need to iterate tablets - } else { - while (OB_SUCC(ret)) { + while (OB_SUCC(ret)) { if (OB_FAIL(iter.get_next_tablet(tablet_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -434,41 +414,38 @@ int ObTenantTabletScheduler::try_update_upper_trans_version_and_gc_sstable( } else if (FALSE_IT(tablet = tablet_handle.get_obj())) { } else if (FALSE_IT(tablet_id = tablet->get_tablet_meta().tablet_id_)) { } else if (tablet_id.is_special_merge_tablet()) { - } else if (!tablet->get_tablet_meta().ha_status_.check_allow_read()) { + } else if (!tablet->get_tablet_meta().restore_state_.check_allow_read()) { } else { int64_t multi_version_start = 0; int tmp_ret = OB_SUCCESS; bool need_update = false; // need update table store - /* - * 1. upper_trans_version calculated from ls is invalid when ls is rebuilding, use rebuild_seq to prevent concurrency bug. - * 2. new_upper_trans array comes from old table store, use end_scn of last minor to check if table store is updated by other thread. - */ + // new_upper_trans comes from the old table store; the last minor end_scn + // detects concurrent table-store updates. ObSEArray new_upper_trans; new_upper_trans.set_attr(ObMemAttr("NewUpTxnVer")); UpdateUpperTransParam upper_trans_param; upper_trans_param.new_upper_trans_ = &new_upper_trans; if (OB_TMP_FAIL(ObGCUpperTransHelper::check_need_gc_or_update_upper_trans_version(ls, *tablet, multi_version_start, upper_trans_param, need_update))) { - LOG_WARN("faild to check need gc or update", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("faild to check need gc or update", K(tmp_ret), K(tablet_id)); } else if (need_update) { ObArenaAllocator tmp_arena("RmOldTblTmp", OB_MALLOC_NORMAL_BLOCK_SIZE); ObStorageSchema *storage_schema = nullptr; if (OB_TMP_FAIL(tablet->load_storage_schema(tmp_arena, storage_schema))) { LOG_WARN("failed to load storage schema", K(tmp_ret), K(tablet)); } else { - ObUpdateTableStoreParam param(tablet->get_snapshot_version(), multi_version_start, storage_schema, rebuild_seq, upper_trans_param); + ObUpdateTableStoreParam param(tablet->get_snapshot_version(), multi_version_start, storage_schema, upper_trans_param); ObTabletHandle new_tablet_handle; // no use here if (OB_TMP_FAIL(ls.update_tablet_table_store(tablet_id, param, new_tablet_handle))) { - LOG_WARN("failed to update table store", K(tmp_ret), K(param), K(ls_id), K(tablet_id)); + LOG_WARN("failed to update table store", K(tmp_ret), K(param), K(tablet_id)); } else { - FLOG_INFO("success to remove old table in table store", K(tmp_ret), K(ls_id), + FLOG_INFO("success to remove old table in table store", K(tmp_ret), K(tablet_id), K(multi_version_start), KPC(tablet)); } } ObTabletObjLoadHelper::free(tmp_arena, storage_schema); } } - } // end while - } + } // end while return ret; } @@ -476,52 +453,33 @@ int ObTenantTabletScheduler::schedule_all_tablets_minor() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("The ObTenantTabletScheduler has not been inited", K(ret)); - } else if (OB_FAIL(minor_ls_tablet_iter_.build_iter(get_schedule_batch_size()))) { + } else if (OB_FAIL(minor_tablet_iter_.build_iter(get_schedule_batch_size()))) { LOG_WARN("failed to init iterator", K(ret)); } else { - LOG_INFO("start schedule all tablet minor merge", K(minor_ls_tablet_iter_)); + LOG_INFO("start schedule all tablet minor merge", K(minor_tablet_iter_)); } - while (OB_SUCC(ret)) { - if (OB_FAIL(minor_ls_tablet_iter_.get_next_ls(ls_handle))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get ls", K(ret), K(minor_ls_tablet_iter_)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls)); - } else { - const ObLSID &ls_id = ls->get_ls_id(); - if (OB_TMP_FAIL(schedule_ls_minor_merge(ls_handle))) { - LOG_TRACE("meet error when schedule", K(tmp_ret), K(minor_ls_tablet_iter_)); - minor_ls_tablet_iter_.skip_cur_ls(); - if (!schedule_ignore_error(tmp_ret)) { - LOG_ERROR("failed to schedule ls minor merge", K(tmp_ret), K(ls_id)); - } + if (OB_SUCC(ret) && !minor_tablet_iter_.is_scan_finish()) { + if (OB_TMP_FAIL(schedule_minor_merge(minor_tablet_iter_.get_ls()))) { + LOG_TRACE("meet error when schedule", K(tmp_ret), K(minor_tablet_iter_)); + minor_tablet_iter_.finish_scan(); + if (!schedule_ignore_error(tmp_ret)) { + LOG_ERROR("failed to schedule minor merge", K(tmp_ret)); } } - } // end of while + } return ret; } -int ObTenantTabletScheduler::check_ls_compaction_finish(const ObLSID &ls_id) +int ObTenantTabletScheduler::check_ls_compaction_finish() { int ret = OB_SUCCESS; bool exist = false; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->check_ls_compaction_dag_exist_with_cancel(ls_id, exist))) { - LOG_WARN("failed to check ls compaction dag", K(ret), K(ls_id)); + if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->check_compaction_dag_exist_with_cancel(exist))) { + LOG_WARN("failed to check compaction dag", K(ret)); } else if (exist) { // the compaction dag exists, need retry later. ret = OB_EAGAIN; @@ -629,7 +587,7 @@ bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const SCN &check_sc bool tx_table_ready = false; SCN max_decided_scn; if (OB_FAIL(ls.get_max_decided_scn(max_decided_scn))) { - LOG_WARN("failed to get max decided log_ts", K(ret), "ls_id", ls.get_ls_id()); + LOG_WARN("failed to get max decided log_ts", K(ret)); } else if (check_scn <= max_decided_scn) { tx_table_ready = true; LOG_INFO("tx table ready", "sstable_end_scn", check_scn, K(max_decided_scn)); @@ -639,18 +597,15 @@ bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const SCN &check_sc } int ObTenantTabletScheduler::check_ready_for_major_merge( - const ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type) { - UNUSED(ls_id); UNUSED(tablet); UNUSED(merge_type); return OB_SUCCESS; } int ObTenantTabletScheduler::schedule_merge_dag( - const ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, const int64_t &merge_snapshot_version, @@ -658,13 +613,13 @@ int ObTenantTabletScheduler::schedule_merge_dag( const ObDagId *dag_net_id /*= nullptr*/) { int ret = OB_SUCCESS; - if (OB_FAIL(check_ready_for_major_merge(ls_id, tablet, merge_type))) { - LOG_WARN("failed to check ready for major merge", K(ret), K(ls_id), K(tablet), K(merge_type)); + if (OB_FAIL(check_ready_for_major_merge(tablet, merge_type))) { + LOG_WARN("failed to check ready for major merge", K(ret), K(tablet), K(merge_type)); } else { UNUSED(dag_net_id); ObTabletMergeDagParam param; if (OB_FAIL(ObDagParamFunc::fill_param( - ls_id, tablet, merge_type, merge_snapshot_version, exec_mode, param))) { + tablet, merge_type, merge_snapshot_version, exec_mode, param))) { LOG_WARN("failed to fill param", KR(ret)); } else if (OB_FAIL(ObScheduleDagFunc::schedule_tablet_merge_dag(param))) { if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) { @@ -677,19 +632,18 @@ int ObTenantTabletScheduler::schedule_merge_dag( } int ObTenantTabletScheduler::schedule_tablet_meta_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, bool &has_created_dag) { int ret = OB_SUCCESS; has_created_dag = false; - if (OB_UNLIKELY(!ls_handle.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(OB_ISNULL(ls) || !tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_handle), K(tablet_handle)); + LOG_WARN("invalid argument", K(ret), KP(ls), K(tablet_handle)); } else { ObTablet *tablet = tablet_handle.get_obj(); - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; const int64_t last_major_snapshot_version = tablet->get_last_major_snapshot_version(); int64_t max_sync_medium_scn = 0; @@ -701,23 +655,23 @@ int ObTenantTabletScheduler::schedule_tablet_meta_merge( // check medium list if (OB_FAIL(tablet->read_medium_info_list(allocator, medium_list))) { - LOG_WARN("failed to read medium info list", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to read medium info list", K(ret), K(tablet_id)); } else if (OB_FAIL(ObMediumCompactionScheduleFunc::get_max_sync_medium_scn( *tablet, *medium_list, max_sync_medium_scn))) { - LOG_WARN("failed to get max sync medium snapshot", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get max sync medium snapshot", K(ret), K(tablet_id)); } else if ((nullptr != medium_list && medium_list->size() > 0) || max_sync_medium_scn > last_major_snapshot_version) { ret = OB_NO_NEED_MERGE; - LOG_WARN("tablet exists unfinished medium info, no need to do meta merge", K(ret), K(ls_id), K(tablet_id), + LOG_WARN("tablet exists unfinished medium info, no need to do meta merge", K(ret), K(tablet_id), K(last_major_snapshot_version), K(max_sync_medium_scn), KPC(medium_list)); } else { - LOG_INFO("start schedule meta merge", K(ls_id), K(tablet_id), KPC(tablet)); // tmp log, remove later + LOG_INFO("start schedule meta merge", K(tablet_id), KPC(tablet)); // tmp log, remove later ObGetMergeTablesParam param; ObGetMergeTablesResult result; param.merge_type_ = META_MAJOR_MERGE; if (OB_FAIL(ObAdaptiveMergePolicy::get_meta_merge_tables( param, - *ls_handle.get_ls(), + *ls, *tablet, result))) { if (OB_NO_NEED_MERGE != ret) { @@ -726,15 +680,15 @@ int ObTenantTabletScheduler::schedule_tablet_meta_merge( } else if (FALSE_IT(result.merge_version_ = result.version_range_.snapshot_version_)) { } else if (OB_UNLIKELY(tablet->get_multi_version_start() > result.merge_version_)) { ret = OB_SNAPSHOT_DISCARDED; - LOG_WARN("multi version data is discarded, should not compaction now", K(ret), K(ls_id), K(tablet_id), + LOG_WARN("multi version data is discarded, should not compaction now", K(ret), K(tablet_id), K(result.merge_version_)); } else { ObTabletMergeDagParam dag_param; if (OB_FAIL(ObDagParamFunc::fill_param( - ls_id, *tablet, META_MAJOR_MERGE, result.merge_version_, EXEC_MODE_LOCAL, dag_param))) { + *tablet, META_MAJOR_MERGE, result.merge_version_, EXEC_MODE_LOCAL, dag_param))) { LOG_WARN("failed to fill param", KR(ret)); } else if (OB_FAIL(schedule_merge_execute_dag( - dag_param, ls_handle, tablet_handle, result))) { + dag_param, ls, tablet_handle, result))) { if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { LOG_ERROR("failed to schedule tablet meta merge dag", K(ret), K(dag_param)); } @@ -744,7 +698,7 @@ int ObTenantTabletScheduler::schedule_tablet_meta_merge( } if (OB_SUCC(ret) && has_created_dag) { - share::g_mp->tenant_tablet_stat_mgr()->clear_tablet_stat(ls_id, tablet_id); + share::g_mp->tenant_tablet_stat_mgr()->clear_tablet_stat(tablet_id); LOG_INFO("success to schedule meta merge", K(ret), K(tablet_id)); } } @@ -798,16 +752,15 @@ int ObTenantTabletScheduler::fill_minor_compaction_param( template int ObTenantTabletScheduler::schedule_tablet_minor_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID &tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_; const int64_t schedule_type_cnt = tablet_id.is_special_merge_tablet() ? TX_TABLE_NO_MAJOR_MERGE_TYPE_CNT : NO_MAJOR_MERGE_TYPE_CNT; for (int i = 0; OB_SUCC(ret) && i < schedule_type_cnt; ++i) { - if (OB_FAIL(schedule_tablet_minor_merge(MERGE_TYPES[i], ls_handle, tablet_handle))) { - LOG_WARN("fail to schdule minor merge", K(ret), "merge_type", MERGE_TYPES[i], K(ls_id), K(tablet_id)); + if (OB_FAIL(schedule_tablet_minor_merge(MERGE_TYPES[i], ls, tablet_handle))) { + LOG_WARN("fail to schdule minor merge", K(ret), "merge_type", MERGE_TYPES[i], K(tablet_id)); } } return ret; @@ -816,18 +769,17 @@ int ObTenantTabletScheduler::schedule_tablet_minor_merge( template int ObTenantTabletScheduler::schedule_tablet_minor_merge( const ObMergeType &merge_type, - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID &tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_; ObGetMergeTablesParam param; ObGetMergeTablesResult result; param.merge_type_ = merge_type; if (OB_FAIL(ObPartitionMergePolicy::get_merge_tables[merge_type]( param, - *ls_handle.get_ls(), + *ls, *tablet_handle.get_obj(), result))) { if (OB_NO_NEED_MERGE == ret) { @@ -848,29 +800,29 @@ int ObTenantTabletScheduler::schedule_tablet_minor_merge( MinorParallelResultArray parallel_results; if (result.handle_.get_count() < minor_compact_trigger) { ret = OB_NO_NEED_MERGE; - } else if (OB_FAIL(minor_range_mgr.get_merge_ranges(ls_id, tablet_id))) { - LOG_WARN("failed to get merge range", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(minor_range_mgr.get_merge_ranges(tablet_id))) { + LOG_WARN("failed to get merge range", K(ret), K(tablet_id)); } else if (OB_FAIL(ObPartitionMergePolicy::generate_parallel_minor_interval(param.merge_type_, minor_compact_trigger, result, minor_range_mgr, parallel_results))) { if (OB_NO_NEED_MERGE != ret) { LOG_WARN("failed to generate parallel minor dag", K(ret), K(result)); } } else if (parallel_results.empty()) { - LOG_DEBUG("parallel results is empty, cannot schedule parallel minor merge", K(ls_id), K(tablet_id), + LOG_DEBUG("parallel results is empty, cannot schedule parallel minor merge", K(tablet_id), K(result), K(minor_range_mgr.exe_range_array_)); } else { const int64_t parallel_dag_cnt = minor_range_mgr.exe_range_array_.count() + parallel_results.count(); const int64_t total_sstable_cnt = result.handle_.get_count(); const int64_t create_time = common::ObTimeUtility::fast_current_time(); - ObTabletMergeDagParam dag_param(merge_type, ls_id, tablet_id); + ObTabletMergeDagParam dag_param(merge_type, tablet_id); for (int64_t k = 0; OB_SUCC(ret) && k < parallel_results.count(); ++k) { if (OB_UNLIKELY(parallel_results.at(k).handle_.get_count() <= 1)) { LOG_WARN("invalid parallel result", K(ret), K(k), K(parallel_results)); } else if (OB_FAIL(fill_minor_compaction_param(tablet_handle, parallel_results.at(k), total_sstable_cnt, parallel_dag_cnt, create_time, dag_param))) { LOG_WARN("failed to fill compaction param for ranking dags later", K(ret), K(k), K(parallel_results.at(k))); - } else if (OB_FAIL(schedule_merge_execute_dag(dag_param, ls_handle, tablet_handle, parallel_results.at(k)))) { + } else if (OB_FAIL(schedule_merge_execute_dag(dag_param, ls, tablet_handle, parallel_results.at(k)))) { LOG_WARN("failed to schedule minor execute dag", K(ret), K(k), K(parallel_results.at(k))); } else { - LOG_INFO("success to schedule tablet minor merge", K(ret), K(ls_id), K(tablet_id), + LOG_INFO("success to schedule tablet minor merge", K(ret), K(tablet_id), "table_cnt", parallel_results.at(k).handle_.get_count(), "merge_scn_range", parallel_results.at(k).scn_range_, K(merge_type)); } @@ -881,7 +833,7 @@ int ObTenantTabletScheduler::schedule_tablet_minor_merge( } int ObTenantTabletScheduler::schedule_tablet_ddl_major_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; @@ -892,15 +844,13 @@ int ObTenantTabletScheduler::schedule_tablet_ddl_major_merge( bool is_major_sstable_exist = false; bool has_freezed_ddl_kv = false; SCN ddl_commit_scn; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid argument", K(ret), K(tablet_handle)); } else if (OB_ISNULL(tenant_direct_load_mgr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_id, tablet_handle.get_obj()->get_tablet_meta().tablet_id_, true, /* is_full_direct_load */ direct_load_mgr_handle, @@ -913,7 +863,7 @@ int ObTenantTabletScheduler::schedule_tablet_ddl_major_merge( } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret)); } else if (FALSE_IT(ddl_commit_scn = direct_load_mgr_handle.get_full_obj()->get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()))) { - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->try_flush_ddl_commit_scn(ls_handle, tablet_handle, direct_load_mgr_handle, ddl_commit_scn))) { + } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->try_flush_ddl_commit_scn(ls, tablet_handle, direct_load_mgr_handle, ddl_commit_scn))) { LOG_WARN("try flush ddl commit scn failed", K(ret), "tablet_id", tablet_handle.get_obj()->get_tablet_meta().tablet_id_); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_freezed_ddl_kv(has_freezed_ddl_kv))) { LOG_WARN("check has freezed ddl kv failed", K(ret)); @@ -935,7 +885,7 @@ int ObTenantTabletScheduler::schedule_tablet_ddl_major_merge( template int ObTenantTabletScheduler::schedule_merge_execute_dag( const ObTabletMergeDagParam ¶m, - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, const ObGetMergeTablesResult &result) { @@ -944,7 +894,7 @@ int ObTenantTabletScheduler::schedule_merge_execute_dag( T *merge_exe_dag = nullptr; if (result.handle_.get_count() > 1 && - !ObTenantTabletScheduler::check_tx_table_ready(*ls_handle.get_ls(), result.scn_range_.end_scn_)) { + !ObTenantTabletScheduler::check_tx_table_ready(*ls, result.scn_range_.end_scn_)) { ret = OB_EAGAIN; LOG_INFO("tx table is not ready. waiting for max_decided_log_ts ...", KR(ret), "merge_scn", result.scn_range_.end_scn_); @@ -953,7 +903,7 @@ int ObTenantTabletScheduler::schedule_merge_execute_dag( } else if (OB_FAIL(merge_exe_dag->prepare_init(param, tablet_handle.get_obj()->get_tablet_meta().compat_mode_, result, - ls_handle))) { + ls))) { LOG_WARN("failed to init dag", K(ret), K(result)); } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->add_dag(merge_exe_dag, emergency))) { if (OB_EAGAIN != ret) { @@ -969,14 +919,12 @@ int ObTenantTabletScheduler::schedule_merge_execute_dag( return ret; } -int ObTenantTabletScheduler::schedule_ls_minor_merge( - ObLSHandle &ls_handle) +int ObTenantTabletScheduler::schedule_minor_merge( + ObLS *ls) { int ret = OB_SUCCESS; ObLSStatusCache::LSState state = ObLSStatusCache::STATE_MAX; - ObLS &ls = *ls_handle.get_ls(); - const ObLSID &ls_id = ls.get_ls_id(); - (void) ObLSStatusCache::check_ls_state(ls, state); + (void) ObLSStatusCache::check_ls_state(*ls, state); if (ObLSStatusCache::CAN_MERGE != state) { // no need to merge, do nothing ret = OB_STATE_NOT_MATCH; @@ -988,25 +936,25 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( ObSEArray need_fast_freeze_tablets; need_fast_freeze_tablets.set_attr(ObMemAttr("MinorBatch")); int64_t start_time_us = 0; - while (OB_SUCC(ret)) { // loop all tablet in ls + while (OB_SUCC(ret)) { // process the remaining tablets bool need_fast_freeze_flag = false; - if (OB_FAIL(minor_ls_tablet_iter_.get_next_tablet(tablet_handle))) { + if (OB_FAIL(minor_tablet_iter_.get_next_tablet(tablet_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; } else if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_handle)); } } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet handle", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle)); } else if (FALSE_IT(tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_)) { - } else if (OB_TMP_FAIL(schedule_tablet_minor(ls_handle, tablet_handle, schedule_minor_flag, need_fast_freeze_flag))) { - LOG_WARN("failed to schedule tablet minor", KR(tmp_ret), K(ls_id), K(tablet_id)); + } else if (OB_TMP_FAIL(schedule_tablet_minor(ls, tablet_handle, schedule_minor_flag, need_fast_freeze_flag))) { + LOG_WARN("failed to schedule tablet minor", KR(tmp_ret), K(tablet_id)); } if (need_fast_freeze_flag) { if (OB_TMP_FAIL(need_fast_freeze_tablets.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet_id for batch_freeze", KR(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to push back tablet_id for batch_freeze", KR(tmp_ret), K(tablet_id)); } } } // end of while @@ -1016,12 +964,12 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( start_time_us = ObClockGenerator::getClock(); if (need_fast_freeze_tablets.empty()) { // empty array. do not need freeze - } else if (OB_TMP_FAIL(ls.tablet_freeze(need_fast_freeze_tablets, + } else if (OB_TMP_FAIL(ls->tablet_freeze(need_fast_freeze_tablets, is_sync, 0, /*timeout, useless for async one*/ false, /*need_rewrite_meta*/ ObFreezeSourceFlag::FAST_FREEZE))) { - LOG_WARN("failt to batch freeze tablet", KR(tmp_ret), K(ls_id), K(need_fast_freeze_tablets)); + LOG_WARN("failt to batch freeze tablet", KR(tmp_ret), K(need_fast_freeze_tablets)); } else { LOG_INFO("fast freeze by batch_tablet_freeze finish", KR(tmp_ret), @@ -1032,8 +980,8 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( // Trigger TxData freeze after fast freeze ObTxTableGuard tx_table_guard; - LOG_INFO("Trigger tx data freeze by fast freeze", K(ls_id), K(tablet_id)); - if (OB_TMP_FAIL(ls.get_tx_table()->get_tx_table_guard(tx_table_guard))) { + LOG_INFO("Trigger tx data freeze by fast freeze", K(tablet_id)); + if (OB_TMP_FAIL(ls->get_tx_table()->get_tx_table_guard(tx_table_guard))) { LOG_WARN("get tx table guard failed", KR(tmp_ret), K(tx_table_guard)); } else { (void)tx_table_guard.self_freeze_task(); @@ -1046,7 +994,7 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( // schedule_minor_flag = false means minor dag array is full // but still need to loop tablet for ddl major & fast freeze int ObTenantTabletScheduler::schedule_tablet_minor( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle tablet_handle, bool &schedule_minor_flag, bool &need_fast_freeze_flag) @@ -1054,26 +1002,24 @@ int ObTenantTabletScheduler::schedule_tablet_minor( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; need_fast_freeze_flag = false; - - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTablet &tablet = *tablet_handle.get_obj(); const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; if (tablet.is_empty_shell()) { if (REACH_THREAD_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - LOG_INFO("can't schedule minor for empty shell tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("can't schedule minor for empty shell tablet", K(ret), K(tablet_id)); } } else if (schedule_minor_flag - && OB_TMP_FAIL(schedule_tablet_minor_merge(ls_handle, tablet_handle))) { + && OB_TMP_FAIL(schedule_tablet_minor_merge(ls, tablet_handle))) { if (OB_SIZE_OVERFLOW == tmp_ret) { schedule_minor_flag = false; } else if (OB_EAGAIN != tmp_ret) { - LOG_ERROR("failed to schedule tablet merge", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to schedule tablet merge", K(tmp_ret), K(tablet_id)); } } if (!tablet_id.is_ls_inner_tablet()) { // data tablet - if (OB_TMP_FAIL(schedule_ddl_tablet_merge(ls_handle, tablet_handle))) { + if (OB_TMP_FAIL(schedule_ddl_tablet_merge(ls, tablet_handle))) { if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) { - LOG_ERROR("failed to schedule tablet ddl merge", K(tmp_ret), K(ls_id), K(tablet_handle)); + LOG_ERROR("failed to schedule tablet ddl merge", K(tmp_ret), K(tablet_handle)); } } @@ -1084,9 +1030,9 @@ int ObTenantTabletScheduler::schedule_tablet_minor( if (share::g_mp->tenant_tablet_stat_mgr()->contain_extreme_tablet()) { bool unused_create_dag = false; // unused - if (OB_TMP_FAIL(ObTenantTabletScheduler::try_schedule_adaptive_merge(ls_handle, tablet_handle, + if (OB_TMP_FAIL(ObTenantTabletScheduler::try_schedule_adaptive_merge(ls, tablet_handle, ObAdaptiveMergePolicy::SCHEDULE_META, 0 /*update_cnt*/, 0 /*delete_cnt*/, unused_create_dag))) { - LOG_WARN("failed to schedule tablet meta merge", K(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to schedule tablet meta merge", K(tmp_ret), K(tablet_id)); } } } @@ -1094,33 +1040,32 @@ int ObTenantTabletScheduler::schedule_tablet_minor( } int ObTenantTabletScheduler::schedule_ddl_tablet_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; ObDDLKvMgrHandle ddl_kv_mgr_handle; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID tablet_id = tablet_handle.is_valid() ? tablet_handle.get_obj()->get_tablet_meta().tablet_id_ : ObTabletID(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid arg", K(ret), K(tablet_handle)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { - LOG_TRACE("kv mgr not exist", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("kv mgr not exist", K(ret), K(tablet_id)); ret = OB_SUCCESS; /* for empty table, ddl kv may not exist*/ } else { - LOG_WARN("get ddl kv mgr failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get ddl kv mgr failed", K(ret), K(tablet_id)); } } else { - if (OB_FAIL(schedule_tablet_ddl_major_merge(ls_handle, tablet_handle))) { + if (OB_FAIL(schedule_tablet_ddl_major_merge(ls, tablet_handle))) { if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { - LOG_ERROR("failed to schedule tablet ddl merge", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to schedule tablet ddl merge", K(ret), K(tablet_id)); } else { - LOG_TRACE("schedule ddl major merge failed", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("schedule ddl major merge failed", K(ret), K(tablet_id)); } } } - LOG_TRACE("schedule ddl tablet merge", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("schedule ddl tablet merge", K(ret), K(tablet_id)); return ret; } @@ -1138,7 +1083,6 @@ int ObTenantTabletScheduler::schedule_all_tablets_medium() const int64_t merge_version = get_frozen_version(); if (merge_version > merged_version_) { try_finish_merge_progress(merge_version); - mview_validation_.refresh(merge_version); } if (OB_FAIL(medium_loop_.init(get_schedule_batch_size()))) { LOG_WARN("failed to init medium loop", K(ret)); @@ -1162,43 +1106,42 @@ bool ObTenantTabletScheduler::need_fast_medium_loop() const } int ObTenantTabletScheduler::user_request_schedule_medium_merge( - const ObLSID &ls_id, const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - LOG_INFO("user_request_schedule_medium_merge", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("user_request_schedule_medium_merge", K(ret), K(tablet_id)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantTabletScheduler has not been inited", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(tablet_id.is_ls_inner_tablet())) { ret = OB_NOT_SUPPORTED; LOG_WARN("not supported to schedule medium for ls inner tablet", K(ret), K(tablet_id)); } else if (!could_major_merge_start()) { ret = OB_MAJOR_FREEZE_NOT_ALLOW; - LOG_WARN("major compaction is suspended", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); + LOG_WARN("major compaction is suspended", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { const int64_t merge_version = get_frozen_version(); const ObAdaptiveMergePolicy::AdaptiveMergeReason reason = ObAdaptiveMergePolicy::USER_REQUEST; ObScheduleTabletFunc func(merge_version, reason); - if (OB_FAIL(func.switch_ls(ls_handle))) { + if (OB_FAIL(func.init(ls))) { if (OB_STATE_NOT_MATCH != ret) { - LOG_ERROR("failed to switch ls", KR(ret), K(ls_id), K(func)); + LOG_ERROR("failed to initialize compaction status", KR(ret), K(func)); } else { - LOG_WARN("not support schedule medium for ls", K(ret), K(ls_id), K(tablet_id), K(func)); + LOG_WARN("not support schedule medium for ls", K(ret), K(tablet_id), K(func)); } - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( + } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet( tablet_id, tablet_handle, 0 /*timeout_us*/))) { - LOG_WARN("get tablet failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } else if (OB_FAIL(func.request_schedule_new_round(tablet_handle, true/*user_request*/))) { - LOG_WARN("failed to request schedule new round", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to request schedule new round", K(ret), K(tablet_id)); } } return ret; @@ -1223,7 +1166,6 @@ int ObTenantTabletScheduler::get_min_dependent_schema_version(int64_t &min_schem #ifdef ERRSIM void ObTenantTabletScheduler::errsim_after_mini_schedule_adaptive( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObAdaptiveMergePolicy::AdaptiveCompactionEvent &event, bool &medium_is_cooling_down, @@ -1250,7 +1192,7 @@ void ObTenantTabletScheduler::errsim_after_mini_schedule_adaptive( if (OB_FAIL(ret)) { } else if (OB_FAIL(EN_COMPACTION_DISABLE_META_MERGE_AFTER_MINI)) { reason = ObAdaptiveMergePolicy::NONE; - LOG_INFO("ERRSIM EN_COMPACTION_DISABLE_META_MERGE_AFTER_MINI: disable meta merge after mini", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("ERRSIM EN_COMPACTION_DISABLE_META_MERGE_AFTER_MINI: disable meta merge after mini", K(ret), K(tablet_id)); } bool is_tombstone_scene = ObAdaptiveMergePolicy::NONE != reason; @@ -1259,7 +1201,7 @@ void ObTenantTabletScheduler::errsim_after_mini_schedule_adaptive( #endif int ObTenantTabletScheduler::try_schedule_adaptive_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, const ObAdaptiveMergePolicy::AdaptiveCompactionEvent &event, const int64_t update_row_cnt, @@ -1269,53 +1211,52 @@ int ObTenantTabletScheduler::try_schedule_adaptive_merge( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; create_dag = false; - if (OB_UNLIKELY(!ls_handle.is_valid() || !tablet_handle.is_valid() || !ObAdaptiveMergePolicy::need_schedule_meta(event))) { + if (OB_UNLIKELY(OB_ISNULL(ls) || !tablet_handle.is_valid() || !ObAdaptiveMergePolicy::need_schedule_meta(event))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_handle), K(tablet_handle), K(event)); + LOG_WARN("invalid argument", K(ret), KP(ls), K(tablet_handle), K(event)); } else { ObTableModeFlag mode = ObTableModeFlag::TABLE_MODE_NORMAL; ObAdaptiveMergePolicy::AdaptiveMergeReason reason = ObAdaptiveMergePolicy::NONE; const ObTablet *tablet = tablet_handle.get_obj(); - const ObLSID &ls_id = tablet->get_ls_id(); const ObTabletID &tablet_id = tablet->get_tablet_id(); bool medium_is_cooling_down = tablet->get_last_major_snapshot_version() + ObAdaptiveMergePolicy::MEDIUM_COOLING_TIME_THRESHOLD_NS > ObTimeUtility::current_time_ns(); if (OB_FAIL(ObAdaptiveMergePolicy::check_adaptive_merge_reason_for_event( - *ls_handle.get_ls(), + *ls, *tablet, event, update_row_cnt, delete_row_cnt, mode, reason))) { - LOG_WARN("failed to check adaptive merge reason", K(ret), K(ls_handle), K(tablet_handle)); + LOG_WARN("failed to check adaptive merge reason", K(ret), KP(ls), K(tablet_handle)); #ifdef ERRSIM } else if (ObAdaptiveMergePolicy::AdaptiveCompactionEvent::SCHEDULE_AFTER_MINI ==event - && FALSE_IT(errsim_after_mini_schedule_adaptive(ls_id, tablet_id, event, medium_is_cooling_down, reason))) { + && FALSE_IT(errsim_after_mini_schedule_adaptive(tablet_id, event, medium_is_cooling_down, reason))) { #endif } else if (ObAdaptiveMergePolicy::NONE == reason) { } else if (ObAdaptiveMergePolicy::is_schedule_medium(mode) && ObAdaptiveMergePolicy::need_schedule_medium(event) && !medium_is_cooling_down) { ObScheduleTabletFunc func(0/*merge_version*/, reason); bool unused_tablet_merge_finish = false; - if (OB_TMP_FAIL(func.switch_ls(ls_handle))) { + if (OB_TMP_FAIL(func.init(ls))) { if (OB_STATE_NOT_MATCH != tmp_ret) { - LOG_ERROR("failed to switch ls", KR(tmp_ret), K(ls_id)); + LOG_ERROR("failed to initialize compaction status", KR(tmp_ret)); } } else if (OB_TMP_FAIL(func.schedule_tablet(tablet_handle, unused_tablet_merge_finish))) { - LOG_WARN("failed to schedule tablet", KR(tmp_ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to schedule tablet", KR(tmp_ret), K(tablet_id)); } } else if (ObAdaptiveMergePolicy::is_schedule_meta(mode)) { - if (OB_TMP_FAIL(ObTenantTabletScheduler::schedule_tablet_meta_merge(ls_handle, tablet_handle, create_dag))) { - LOG_ERROR_RET(tmp_ret, "failed to schedule meta merge for tablet", K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(ObTenantTabletScheduler::schedule_tablet_meta_merge(ls, tablet_handle, create_dag))) { + LOG_ERROR_RET(tmp_ret, "failed to schedule meta merge for tablet", K(tablet_id)); } else if (create_dag) { - LOG_INFO("[Buffer-Opt] Try to schedule tablet meta merge background", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("[Buffer-Opt] Try to schedule tablet meta merge background", K(ret), K(tablet_id)); } } if (ObAdaptiveMergePolicy::AdaptiveCompactionEvent::SCHEDULE_AFTER_MINI == event) { - LOG_INFO("[Buffer-Opt] Try to schedule tablet medium/meta after mini", K(ret), K(tmp_ret), K(ls_id), K(tablet_id), "is_tombstone_scene", ObAdaptiveMergePolicy::NONE != reason, + LOG_INFO("[Buffer-Opt] Try to schedule tablet medium/meta after mini", K(ret), K(tmp_ret), K(tablet_id), "is_tombstone_scene", ObAdaptiveMergePolicy::NONE != reason, "mode", table_mode_flag_to_str(mode), K(medium_is_cooling_down), K(event), K(update_row_cnt), K(delete_row_cnt), K(create_dag)); } else if (REACH_THREAD_TIME_INTERVAL(30 * 1000 * 1000 /*30s*/)) { - LOG_INFO("Try schedule tablet adaptive merge", K(ret), K(tmp_ret), K(ls_id), K(tablet_id), "is_tombstone_scene", ObAdaptiveMergePolicy::NONE != reason, K(event)); + LOG_INFO("Try schedule tablet adaptive merge", K(ret), K(tmp_ret), K(tablet_id), "is_tombstone_scene", ObAdaptiveMergePolicy::NONE != reason, K(event)); } } return ret; diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index d971b88a7..80c3b3439 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -18,7 +18,6 @@ #define STORAGE_OB_TENANT_TABLET_SCHEDULER_H_ #include "lib/thread/ob_dedup_queue.h" -#include "share/ob_ls_id.h" #include "share/tablet/ob_tablet_info.h" #include "storage/ob_i_store.h" #include "storage/compaction/ob_tablet_merge_task.h" @@ -31,7 +30,6 @@ #include "share/compaction/ob_schedule_batch_size_mgr.h" #include "storage/compaction/ob_compaction_schedule_util.h" #include "storage/compaction/ob_medium_loop.h" -#include "storage/compaction/ob_mview_compaction_util.h" namespace oceanbase { @@ -120,7 +118,7 @@ class ObTenantTabletScheduler : public ObBasicMergeScheduler virtual int schedule_merge(const int64_t broadcast_version) override; int update_upper_trans_version_and_gc_sstable(); int try_update_upper_trans_version_and_gc_sstable(ObLS &ls, ObCompactionScheduleIterator &iter); - int check_ls_compaction_finish(const share::ObLSID &ls_id); + int check_ls_compaction_finish(); int schedule_all_tablets_minor(); int gc_info(); @@ -142,48 +140,45 @@ class ObTenantTabletScheduler : public ObBasicMergeScheduler compaction::ObTabletMergeDagParam ¶m); template static int schedule_tablet_minor_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle); template static int schedule_tablet_minor_merge( const ObMergeType &merge_type, - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle); static int schedule_tablet_meta_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, bool &has_created_dag); template static int schedule_merge_execute_dag( const compaction::ObTabletMergeDagParam ¶m, - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, const ObGetMergeTablesResult &result); // Check whether major/medium compaction can be scheduled. static int check_ready_for_major_merge( - const ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type); static int schedule_merge_dag( - const share::ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, const int64_t &merge_snapshot_version, const ObExecMode exec_mode, const ObDagId *dag_net_id = nullptr); static int schedule_tablet_ddl_major_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle); #ifdef ERRSIM static void errsim_after_mini_schedule_adaptive( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObAdaptiveMergePolicy::AdaptiveCompactionEvent &event, bool &medium_is_cooling_down, ObAdaptiveMergePolicy::AdaptiveMergeReason &reason); #endif static int try_schedule_adaptive_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle, const ObAdaptiveMergePolicy::AdaptiveCompactionEvent &event, const int64_t update_row_cnt, @@ -191,23 +186,21 @@ class ObTenantTabletScheduler : public ObBasicMergeScheduler bool &create_dag); int get_min_dependent_schema_version(int64_t &min_schema_version); int user_request_schedule_medium_merge( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); OB_INLINE int64_t get_schedule_batch_size() const { return batch_size_mgr_.get_schedule_batch_size(); } OB_INLINE int64_t get_checker_batch_size() const { return batch_size_mgr_.get_checker_batch_size(); } - OB_INLINE ObMviewCompactionValidation &get_mview_validation() { return mview_validation_; } private: friend struct ObTenantTabletSchedulerTaskMgr; bool need_fast_medium_loop() const; int schedule_all_tablets_medium(); - int schedule_ls_minor_merge(ObLSHandle &ls_handle); + int schedule_minor_merge(ObLS *ls); OB_INLINE int schedule_tablet_minor( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle tablet_handle, bool &schedule_minor_flag, bool &need_fast_freeze_flag); int schedule_ddl_tablet_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle); public: typedef common::ObSEArray MinorParallelResultArray; @@ -228,12 +221,11 @@ class ObTenantTabletScheduler : public ObBasicMergeScheduler bool is_inited_; common::ObDedupQueue bf_queue_; ObFastFreezeChecker fast_freeze_checker_; - ObCompactionScheduleIterator minor_ls_tablet_iter_; + ObCompactionScheduleIterator minor_tablet_iter_; ObCompactionScheduleIterator gc_sst_tablet_iter_; ObTenantTabletSchedulerTaskMgr timer_task_mgr_; ObScheduleBatchSizeMgr batch_size_mgr_; ObMediumLoop medium_loop_; - ObMviewCompactionValidation mview_validation_; }; } // namespace compaction diff --git a/src/storage/concurrency_control/ob_data_validation_service.cpp b/src/storage/concurrency_control/ob_data_validation_service.cpp index 6efbc2b8f..05076d41c 100644 --- a/src/storage/concurrency_control/ob_data_validation_service.cpp +++ b/src/storage/concurrency_control/ob_data_validation_service.cpp @@ -16,6 +16,7 @@ #include "storage/concurrency_control/ob_data_validation_service.h" #include "share/rc/ob_module_provider.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -23,26 +24,18 @@ namespace concurrency_control { -void ObDataValidationService::set_delay_resource_recycle(const ObLSID ls_id) +void ObDataValidationService::set_delay_resource_recycle() { int ret = OB_SUCCESS; - ObLSHandle handle; - ObLSService *ls_service = share::g_mp->ls_service(); ObLS *ls = nullptr; const bool need_delay_opt = GCONF._delay_resource_recycle_after_correctness_issue; if (OB_LIKELY(!need_delay_opt)) { // do nothing - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { if (OB_LS_NOT_EXIST != ret) { - TRANS_LOG(DEBUG, "get log stream failed", K(ls_id), K(ret)); + TRANS_LOG(DEBUG, "get log stream failed", K(ret)); } - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "get log stream failed", K(ls_id), K(ret)); } else { ls->set_delay_resource_recycle(); } diff --git a/src/storage/concurrency_control/ob_data_validation_service.h b/src/storage/concurrency_control/ob_data_validation_service.h index 8e38bd942..6a9bae1d2 100644 --- a/src/storage/concurrency_control/ob_data_validation_service.h +++ b/src/storage/concurrency_control/ob_data_validation_service.h @@ -28,7 +28,7 @@ namespace concurrency_control class ObDataValidationService { public: - static void set_delay_resource_recycle(const ObLSID ls_id); + static void set_delay_resource_recycle(); }; } // namespace concurrency_control diff --git a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp index 97dcf5957..840445520 100644 --- a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp +++ b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp @@ -18,8 +18,8 @@ #include "share/rc/ob_module_provider.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx/ob_trans_service.h" +#include "storage/tx/ob_ts_mgr.h" #include "storage/tx/ob_weak_read_util.h" -#include "rootserver/mview/ob_mview_maintenance_service.h" #include "share/ob_server_struct.h" #include "lib/container/ob_array.h" #include "share/ob_io_device_helper.h" @@ -404,34 +404,17 @@ int ObMultiVersionGarbageCollector::study_min_unallocated_GTS(share::SCN &min_un { int ret = OB_SUCCESS; - const transaction::MonotonicTs stc_ahead = transaction::MonotonicTs::current_time() ; - transaction::MonotonicTs unused_receive_gts_ts(0); const int64_t timeout_us = 1 * 1000 * 1000; // 1s - const int64_t expire_time_us = common::ObTimeUtility::current_time() + timeout_us; share::SCN gts_scn; - do { - // We get the gts each 10ms in order not to report too much error during reboot - // and continue to fetch under the failure among 1s. - ret = OB_TS_MGR.get_gts(stc_ahead, - NULL, // gts task - gts_scn, - unused_receive_gts_ts); - if (ret == OB_EAGAIN) { - if (common::ObTimeUtility::current_time() > expire_time_us) { - ret = OB_TIMEOUT; - } else { - ob_usleep(10 * 1000/*10ms*/); - } - } else if (OB_FAIL(ret)) { - MVCC_LOG(WARN, "get gts fail", KR(ret)); - } else if (!gts_scn.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MVCC_LOG(ERROR, "get gts fail", K(gts_scn), K(ret)); - } else { - min_unallocated_GTS = gts_scn; - } - } while (ret == OB_EAGAIN); + if (OB_FAIL(OB_TS_MGR.get_gts_sync(timeout_us, gts_scn))) { + MVCC_LOG(WARN, "get gts fail", KR(ret)); + } else if (!gts_scn.is_valid()) { + ret = OB_ERR_UNEXPECTED; + MVCC_LOG(ERROR, "get gts fail", K(gts_scn), K(ret)); + } else { + min_unallocated_GTS = gts_scn; + } return ret; } @@ -449,7 +432,6 @@ int ObMultiVersionGarbageCollector::study_min_unallocated_WRS( if (OB_FAIL(share::g_mp->trans_service()->get_weak_read_snapshot_version( -1, // system variable : max read stale time for user - false, min_unallocated_WRS))) { MVCC_LOG(WARN, "fail to get weak read snapshot", K(ret)); if (OB_REPLICA_NOT_READABLE == ret) { @@ -465,8 +447,7 @@ int ObMultiVersionGarbageCollector::study_min_unallocated_WRS( return ret; } -// The read snapshot version may base on max committed txn version for the -// single ls txn, so we need study it on each machine. +// The read snapshot version may be based on the maximum committed transaction version. int ObMultiVersionGarbageCollector::study_max_committed_txn_version( share::SCN &max_committed_txn_version) { @@ -509,13 +490,6 @@ int ObMultiVersionGarbageCollector::study_min_active_txn_version( MVCC_LOG(WARN, "get min active snaphot version failed", K(ret)); } - share::SCN min_mview_mds_snapshot; - if (FAILEDx(share::g_mp->m_view_maintenance_service()->get_min_mview_mds_snapshot(min_mview_mds_snapshot))) { - MVCC_LOG(WARN, "get min mview active snaphot version failed", K(ret)); - } else if (min_mview_mds_snapshot.is_valid() && min_mview_mds_snapshot < min_active_txn_version) { - MVCC_LOG(INFO, "study_min_active_txn_version", K(min_active_txn_version), K(min_mview_mds_snapshot)); - min_active_txn_version = min_mview_mds_snapshot; - } return ret; } @@ -806,14 +780,14 @@ int ObMultiVersionGarbageCollector::update_status(const ObMultiVersionGCStatus s // value cannot be advanced and a large number of versions cannot be recycled. // So we need to handle snapshot processing in abnormal situations. // -// Firstly, we rely on the sys ls's ability to manage the ls. We hope that if -// the node cannot contact to the inner table due to abnormalities, then +// Firstly, we rely on the log stream lifecycle. If the node cannot contact +// the inner table due to abnormalities, then // eventually the entries of this node will be removed from the innner table, so // that we no longer rely on this Timestamps provided by the node as recycling // snapshots.(implemented in the reclaim) // // Secondly, we rely on the each node's ability to ignore faulty timestamps. We -// hope that if the sys ls cannot contact to the inner table and remove the +// hope that if the node cannot contact the inner table and remove the // entries in time, each node will ignore the timestamps that has not been // updated for a long time.(implemented in the ObMultiVersionGCSnapshotCalculator). // @@ -829,42 +803,34 @@ int ObMultiVersionGarbageCollector::update_status(const ObMultiVersionGCStatus s int ObMultiVersionGarbageCollector::reclaim() { int ret = OB_SUCCESS; - ObLS *ls = NULL; - storage::ObLSHandle ls_handle; - logservice::ObLogHandler *handler = nullptr; - int64_t old_proposal_id = 0; ObArray reclaimable_servers; ObArray snapshot_servers; bool is_this_server_disabled = false; - common::ObRole role; + bool has_ls = true; + storage::ObLS *ls = nullptr; ObTimeGuard timeguard(__func__, 1 * 1000 * 1000); - if (OB_FAIL(share::g_mp->ls_service()-> - get_ls(share::SYS_LS, ls_handle, ObLSGetMod::MULTI_VERSION_GARBAGE_COLLOECTOR_MOD))) { - MVCC_LOG(WARN, "get sys ls failed", K(ret)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + MVCC_LOG(WARN, "get ls failed", K(ret)); ret = OB_SUCCESS; - } else if (OB_ISNULL(ls = ls_handle.get_ls()) - || OB_ISNULL(handler = ls_handle.get_ls()->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - MVCC_LOG(ERROR, "log stream is NULL", K(ret), K(ls)); - } else if (OB_FAIL(handler->get_role(role, old_proposal_id))) { - MVCC_LOG(WARN, "fail to get role", KR(ret)); - } else if (common::is_leader_like(role)) { + has_ls = false; + } else { timeguard.click("get_leader"); + } + + // TODO(handora.qc): use nicer timer + const int64_t current_timestamp = ObClockGenerator::getRealClock(); - // TODO(handora.qc): use nicer timer - const int64_t current_timestamp = ObClockGenerator::getRealClock(); - - ObMultiVersionGCSnapshotOperator collector( - [current_timestamp, - &reclaimable_servers, - &snapshot_servers, - &is_this_server_disabled](const share::SCN snapshot_version, - const ObMultiVersionSnapshotType snapshot_type, - const ObMultiVersionGCStatus status, - const int64_t create_time, - const ObAddr addr) -> int { + ObMultiVersionGCSnapshotOperator collector( + [current_timestamp, + &reclaimable_servers, + &snapshot_servers, + &is_this_server_disabled](const share::SCN snapshot_version, + const ObMultiVersionSnapshotType snapshot_type, + const ObMultiVersionGCStatus status, + const int64_t create_time, + const ObAddr addr) -> int { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; bool need_reclaim = false; @@ -898,47 +864,48 @@ int ObMultiVersionGarbageCollector::reclaim() } return ret; - }); + }); - // collect all info for reclaimable servers and all reported servers - if (OB_FAIL(collect(collector))) { - MVCC_LOG(WARN, "collect snapshot info failed", K(ret)); - } else { - int tmp_ret = OB_SUCCESS; - timeguard.click("collect"); - - if (0 == reclaimable_servers.count()) { - // all snapshot info is not reclaimable - MVCC_LOG(INFO, "skip all alivavle snapshots info"); - // reclaim all uncessary servers - } else if (OB_TMP_FAIL((reclaim_(reclaimable_servers)))) { - MVCC_LOG(WARN, "reclaim snapshot info failed", K(tmp_ret), - K(reclaimable_servers)); - if (OB_SUCC(ret)) { - ret = tmp_ret; - } + // collect all info for reclaimable servers and all reported servers + if (!has_ls) { + // Skip this round until the tenant LS has been created. + } else if (OB_FAIL(collect(collector))) { + MVCC_LOG(WARN, "collect snapshot info failed", K(ret)); + } else { + int tmp_ret = OB_SUCCESS; + timeguard.click("collect"); + + if (0 == reclaimable_servers.count()) { + // all snapshot info is not reclaimable + MVCC_LOG(INFO, "skip all alivavle snapshots info"); + // reclaim all uncessary servers + } else if (OB_TMP_FAIL((reclaim_(reclaimable_servers)))) { + MVCC_LOG(WARN, "reclaim snapshot info failed", K(tmp_ret), + K(reclaimable_servers)); + if (OB_SUCC(ret)) { + ret = tmp_ret; } + } - timeguard.click("reclaim_"); - - if (0 == snapshot_servers.count()) { - MVCC_LOG(WARN, "no alive servers now, please check it clearly", - KPC(this), K(snapshot_servers), K(reclaimable_servers)); - // monitor all servers - } else if (OB_TMP_FAIL(monitor_(snapshot_servers))) { - MVCC_LOG(WARN, "snapshots servers are mintor failed", - KPC(this), K(snapshot_servers), K(tmp_ret)); - if (OB_SUCC(ret)) { - ret = tmp_ret; - } + timeguard.click("reclaim_"); + + if (0 == snapshot_servers.count()) { + MVCC_LOG(WARN, "no alive servers now, please check it clearly", + KPC(this), K(snapshot_servers), K(reclaimable_servers)); + // monitor all servers + } else if (OB_TMP_FAIL(monitor_(snapshot_servers))) { + MVCC_LOG(WARN, "snapshots servers are mintor failed", + KPC(this), K(snapshot_servers), K(tmp_ret)); + if (OB_SUCC(ret)) { + ret = tmp_ret; } - - timeguard.click("monitor_"); } + + timeguard.click("monitor_"); } MVCC_LOG(INFO, "reclaim multi version garabage collector end", KPC(this), - K(ret), K(role), K(reclaimable_servers), K(snapshot_servers), + K(ret), K(reclaimable_servers), K(snapshot_servers), K(is_this_server_disabled)); return ret; diff --git a/src/storage/ddl/ob_build_index_task.cpp b/src/storage/ddl/ob_build_index_task.cpp index 7b4efe8ff..f0a2c5d61 100644 --- a/src/storage/ddl/ob_build_index_task.cpp +++ b/src/storage/ddl/ob_build_index_task.cpp @@ -141,14 +141,11 @@ int ObUniqueIndexChecker::scan_table_with_column_checksum( query_flag.skip_read_lob_ = 1; bool allow_not_ready = false; ObArray need_reshape; - ObLSHandle ls_handle; + ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param_->ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get log stream", K(ret), K(param_->ls_id_)); - } else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, ls must not be nullptr", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_read_tables(param_->tablet_id_, + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet_svr()->get_read_tables(param_->tablet_id_, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, param.snapshot_version_, param.snapshot_version_, @@ -167,7 +164,7 @@ int ObUniqueIndexChecker::scan_table_with_column_checksum( } else if (param.task_id_ >= param_->ranges_.count() || 0 > param.task_id_ ) { ret = OB_ERR_UNEXPECTED; LOG_WARN("error unexpected, invalid task id", K(ret), K(param.task_id_), K(param_->ranges_.count())); - } else if (OB_FAIL(local_scan.table_scan(*param.data_table_schema_, param_->ls_id_, param_->tablet_id_, iterator, query_flag, param_->ranges_[param.task_id_]))) { + } else if (OB_FAIL(local_scan.table_scan(*param.data_table_schema_, param_->tablet_id_, iterator, query_flag, param_->ranges_[param.task_id_]))) { LOG_WARN("fail to table scan", K(ret)); } else { const ObColDescIArray &out_cols = *param.org_col_ids_; @@ -445,10 +442,10 @@ int ObUniqueIndexChecker::check_unique_index(ObIDag *dag, const int64_t task_id) LOG_WARN("invalid arguments", K(ret), KP(dag)); } else { MOD_SCOPE { - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param_->ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get log stream", K(ret), K(param_->ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, param_->tablet_id_, tablet_handle_))) { + ObLS *ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, param_->tablet_id_, tablet_handle_))) { LOG_WARN("fail to get tablet", K(ret), K(param_->tablet_id_), K(tablet_handle_)); } else if (param_->index_schema_->is_fts_index() || param_->index_schema_->is_vec_index()) { STORAGE_LOG(INFO, "do not need to check unique for domain index", "index_id", param_->index_schema_->get_table_id()); @@ -510,21 +507,21 @@ int ObUniqueIndexChecker::check_unique_index(ObIDag *dag, const int64_t task_id) int ObUniqueIndexChecker::wait_trans_end(ObIDag *dag) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObUniqueIndexChecker has not been inited", K(ret)); } else if (OB_FAIL(DDL_SIM(param_->task_id_, UNIQUE_INDEX_CHECKER_WAIT_TRANS_END_FAILED))) { LOG_WARN("ddl sim failure", K(ret), K(param_->task_id_)); - } else if (OB_FAIL(ls_service->get_ls(ObLSID(param_->ls_id_), ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(param_->ls_id_)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else { const int64_t now = ObTimeUtility::current_time(); const int64_t timeout_us = 1000L * 1000L * 60L; // 1 min while (OB_SUCC(ret) && !dag->has_set_stop()) { transaction::ObTransID pending_tx_id; - if (OB_FAIL(ls_handle.get_ls()->check_modify_time_elapsed(param_->tablet_id_, now, pending_tx_id))) { + if (OB_FAIL(ls->check_modify_time_elapsed(param_->tablet_id_, now, pending_tx_id))) { // when timeout with EAGAIN, ddl scheduler of root service will retry if (OB_EAGAIN == ret && ObTimeUtility::current_time() - now < timeout_us) { ret = OB_SUCCESS; @@ -552,7 +549,6 @@ ObUniqueCheckingDag::ObUniqueCheckingDag() } int ObUniqueCheckingDag::init( - const ObLSID &ls_id, const ObTabletID &tablet_id, const bool is_scan_index, const uint64_t index_table_id, @@ -566,7 +562,7 @@ int ObUniqueCheckingDag::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; STORAGE_LOG(WARN, "ObUniqueCheckingDag has already been inited", K(ret)); - } else if (OB_FAIL(param_.init(ls_id, tablet_id, is_scan_index, index_table_id, + } else if (OB_FAIL(param_.init(tablet_id, is_scan_index, index_table_id, schema_version, task_id, execution_id, snapshot_version, user_parallelism))) { STORAGE_LOG(WARN, "fail to init ObUniqueCheckingParam", KR(ret), K_(param)); } else if (OB_UNLIKELY(!param_.is_valid())) { @@ -849,8 +845,7 @@ int ObSimpleUniqueCheckingTask::process() "trace_id", *ObCurTraceId::get_trace_id(), "task_id", dag->get_task_id(), "snapshot_version", dag->get_snapshot_version(), - "tablet_id", param_->tablet_id_, - "info", dag->get_ls_id()); + "tablet_id", param_->tablet_id_); } LOG_INFO("simple unique check task process.", K(ret), "ddl_event_info", ObDDLEventInfo(), KPC(dag), K(task_id_)); return ret; @@ -999,7 +994,7 @@ int ObGlobalUniqueIndexCallback::operator()(const int ret_code) #ifdef ERRSIM if (OB_SUCC(ret)) { ret = OB_E(EventTable::EN_DDL_REPORT_REPLICA_BUILD_STATUS_FAIL) OB_SUCCESS; - LOG_INFO("report replica build status errsim", K(ret)); + LOG_INFO("report DDL build status errsim", K(ret)); } #endif if (OB_FAIL(ret)) { @@ -1024,7 +1019,6 @@ int ObLocalUniqueIndexCallback::operator()(const int ret_code) /* ObUniqueCheckingParam */ int ObUniqueCheckingParam::init( - const ObLSID &ls_id, const ObTabletID &tablet_id, const bool is_scan_index, const uint64_t index_table_id, @@ -1039,11 +1033,11 @@ int ObUniqueCheckingParam::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; STORAGE_LOG(WARN, "ObUniqueCheckingParam has already been inited", K(ret)); - } else if (OB_UNLIKELY(false || !ls_id.is_valid() || !tablet_id.is_valid() + } else if (OB_UNLIKELY(!tablet_id.is_valid() || OB_INVALID_ID == index_table_id || schema_version < 0 || task_id <= 0 || execution_id < 0 || snapshot_version < 0)) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(ls_id), K(tablet_id), + STORAGE_LOG(WARN, "invalid arguments", K(ret), K(tablet_id), K(index_table_id), K(schema_version), K(task_id), K(execution_id), K(snapshot_version)); } else { MOD_SCOPE { @@ -1067,7 +1061,6 @@ int ObUniqueCheckingParam::init( } else { compat_mode_ = lib::Worker::CompatMode::MYSQL; is_inited_ = true; - ls_id_ = ls_id; tablet_id_ = tablet_id; is_scan_index_ = is_scan_index; schema_service_ = schema_service; @@ -1089,7 +1082,7 @@ int ObUniqueCheckingParam::prepare_task_ranges() if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_task_ranges(task_id_, ls_id_, tablet_id_, is_scan_index_ ? index_schema_->get_tablet_size() + } else if (OB_FAIL(ObDDLUtil::get_task_ranges(task_id_, tablet_id_, is_scan_index_ ? index_schema_->get_tablet_size() : data_table_schema_->get_tablet_size(), user_parallelism_, allocator_, ranges_))) { LOG_WARN("get_task_ranges failed", K(ret), KPC(this)); } else { diff --git a/src/storage/ddl/ob_build_index_task.h b/src/storage/ddl/ob_build_index_task.h index 10a61f0b5..8dedd1d02 100644 --- a/src/storage/ddl/ob_build_index_task.h +++ b/src/storage/ddl/ob_build_index_task.h @@ -142,15 +142,14 @@ struct ObUniqueCheckingParam final { public: ObUniqueCheckingParam(): - is_inited_(false), ls_id_(share::ObLSID::INVALID_LS_ID), tablet_id_(), + is_inited_(false), tablet_id_(), is_scan_index_(false), schema_service_(nullptr), schema_guard_(share::schema::ObSchemaMgrItem::MOD_UNIQ_CHECK), index_schema_(nullptr), data_table_schema_(nullptr), callback_(nullptr), execution_id_(0), snapshot_version_(0), task_id_(0), compat_mode_(lib::Worker::CompatMode::INVALID), user_parallelism_(0), concurrent_cnt_(0), ranges_(), allocator_("UniqueChecking", OB_MALLOC_NORMAL_BLOCK_SIZE) {} ~ObUniqueCheckingParam() { destroy(); } - int init(const ObLSID &ls_id, - const ObTabletID &tablet_id, + int init(const ObTabletID &tablet_id, const bool is_scan_index, const uint64_t index_table_id, const int64_t schema_version, @@ -161,7 +160,7 @@ struct ObUniqueCheckingParam final int prepare_task_ranges(); bool is_valid() const { - return true && ls_id_.is_valid() && tablet_id_.is_valid() && snapshot_version_ > 0 + return tablet_id_.is_valid() && snapshot_version_ > 0 && schema_service_ != nullptr && compat_mode_ != lib::Worker::CompatMode::INVALID && execution_id_ >= 0 && task_id_ > 0 && user_parallelism_ > 0; } @@ -172,7 +171,6 @@ struct ObUniqueCheckingParam final void destroy() { is_inited_ = false; - ls_id_.reset(); tablet_id_.reset(); is_scan_index_ = false; schema_service_ = nullptr; @@ -191,12 +189,11 @@ struct ObUniqueCheckingParam final concurrent_cnt_ = 0; ranges_.reset(); } - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(is_scan_index), KP_(index_schema), + TO_STRING_KV(K_(is_inited), K_(tablet_id), K_(is_scan_index), KP_(index_schema), KP_(data_table_schema), K_(execution_id), K_(snapshot_version), K_(task_id), K_(compat_mode), K_(user_parallelism), K_(concurrent_cnt), K_(ranges)); public: bool is_inited_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; bool is_scan_index_; share::schema::ObMultiVersionSchemaService *schema_service_; @@ -250,7 +247,6 @@ class ObUniqueCheckingDag : public share::ObIDag ObUniqueCheckingDag(); virtual ~ObUniqueCheckingDag() = default; int init( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const bool is_scan_index, const uint64_t index_table_id, const int64_t schema_version, @@ -274,8 +270,6 @@ class ObUniqueCheckingDag : public share::ObIDag virtual uint64_t hash() const override; virtual bool operator ==(const share::ObIDag &other) const; common::ObTabletID get_tablet_id() const { return param_.tablet_id_; } - - share::ObLSID get_ls_id() const { return param_.ls_id_; } virtual int fill_info_param(compaction::ObIBasicInfoParam *&out_param, ObIAllocator &allocator) const override; virtual bool ignore_warning() override; virtual int fill_dag_key(char *buf, const int64_t buf_len) const override; diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index cbbf7f059..0c63e9d79 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -23,6 +23,7 @@ #include "ob_complement_data_task.h" #include "rootserver/ob_root_service.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" #include "share/ob_ddl_checksum.h" #include "share/ob_ddl_sim_point.h" #include "share/schema/ob_part_mgr_util.h" @@ -77,12 +78,12 @@ void add_ddl_event(const ObComplementDataParam *param, const ObString &stmt) int ObComplementDataParam::fill_tablet_param() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(dest_ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, dest_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -102,8 +103,8 @@ int ObComplementDataParam::fill_tablet_param() } else if (mds_data.lob_meta_tablet_id_.is_valid()) { dest_lob_meta_tablet_id_ = mds_data.lob_meta_tablet_id_; ObTabletHandle lob_meta_tablet_handle; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, mds_data.lob_meta_tablet_id_, lob_meta_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("ddl get tablet failed", K(ret), K(ls_handle)); + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, mds_data.lob_meta_tablet_id_, lob_meta_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("ddl get tablet failed", K(ret), K(ls)); } else if (OB_FAIL(lob_meta_tablet_handle.get_obj()->load_storage_schema(allocator_, lob_meta_tablet_param_.storage_schema_))) { LOG_WARN("load storage schema failed", K(ret)); } else { @@ -196,8 +197,6 @@ int ObComplementDataParam::init(const ObDDLBuildSingleReplicaRequestArg &arg) dest_schema_version_ = dest_schema_version; orig_tablet_id_ = arg.source_tablet_id_; dest_tablet_id_ = arg.dest_tablet_id_; - orig_ls_id_ = arg.ls_id_; - dest_ls_id_ = arg.dest_ls_id_; task_id_ = arg.task_id_; execution_id_ = arg.execution_id_; tablet_task_id_ = arg.tablet_task_id_; @@ -239,7 +238,6 @@ int ObComplementDataParam::prepare_task_ranges() LOG_INFO("succeed to to init task ranges", K(ret), K(user_parallelism_), K(concurrent_cnt_), K(ranges_)); } } else if (OB_FAIL(split_task_ranges(task_id_, - orig_ls_id_, orig_tablet_id_, orig_schema_tablet_size_, user_parallelism_))) { @@ -263,7 +261,6 @@ int ObComplementDataParam::prepare_task_ranges() // split task ranges to do table scan based on the whole range on the specified tablet. int ObComplementDataParam::split_task_ranges( const int64_t task_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t tablet_size, const int64_t hint_parallelism) @@ -272,21 +269,18 @@ int ObComplementDataParam::split_task_ranges( if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(ObDDLUtil::get_task_ranges(task_id, ls_id, tablet_id, tablet_size, hint_parallelism, allocator_, ranges_))) { + } else if (OB_FAIL(ObDDLUtil::get_task_ranges(task_id, tablet_id, tablet_size, hint_parallelism, allocator_, ranges_))) { LOG_WARN("get_task_ranges failed", K(ret), KPC(this)); } else { - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletTableIterator iterator; ObLSTabletService *tablet_service = nullptr; - if (OB_UNLIKELY(task_id <= 0 || !ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(task_id <= 0 || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(task_id), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get log stream", K(ret), K(arg)); - } else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_handle)); - } else if (OB_ISNULL(tablet_service = ls_handle.get_ls()->get_tablet_svr())) { + LOG_WARN("invalid arguments", K(ret), K(task_id), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get log stream", K(ret)); + } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet service is nullptr", K(ret)); } else { @@ -305,7 +299,7 @@ int ObComplementDataContext::init( { int ret = OB_SUCCESS; UNUSED(hidden_table_schema); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; const ObSSTable *first_major_sstable = nullptr; ObTabletMemberWrapper table_store_wrapper; @@ -315,9 +309,9 @@ int ObComplementDataContext::init( } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(param)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.dest_ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, param.dest_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -325,7 +319,7 @@ int ObComplementDataContext::init( } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; LOG_WARN("tablet handle is null", K(ret), K(param)); - } else if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(param.dest_ls_id_, param.dest_tablet_id_, first_major_sstable, table_store_wrapper))) { + } else if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(param.dest_tablet_id_, first_major_sstable, table_store_wrapper))) { LOG_WARN("check if major sstable exist failed", K(ret), K(param)); } else if (nullptr != first_major_sstable) { LOG_INFO("major exists, skip create tablet direct load mgr", K(ret), K(param)); @@ -344,7 +338,7 @@ int ObComplementDataContext::init( LOG_WARN("failed to alloc memory", K(ret)); } else { tablet_ctx_ = new (buf) ObDDLTabletContext(); - if (OB_FAIL(tablet_ctx_->init(param.dest_ls_id_, param.dest_tablet_id_, param.user_parallelism_, param.snapshot_version_, param.direct_load_type_, param.ddl_table_schema_))) { + if (OB_FAIL(tablet_ctx_->init(param.dest_tablet_id_, param.user_parallelism_, param.snapshot_version_, param.direct_load_type_, param.ddl_table_schema_))) { LOG_WARN("failed to init tablet ctx", K(ret)); } } @@ -476,7 +470,6 @@ int ObComplementDataDag::calc_total_row_count() // FIXME(YIREN), How to calc the row count of the source tablet for restore table. // RPC? } else if (OB_FAIL(ObDDLUtil::get_tablet_physical_row_cnt( - param_.orig_ls_id_, param_.orig_tablet_id_, true, // calc_sstable = true true, // calc_memtable = true @@ -528,8 +521,7 @@ int ObComplementDataDag::create_first_task() ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret)); } else if (FALSE_IT(context_.tablet_ctx_ = new (buf) ObDDLTabletContext())) { - } else if (OB_FAIL(context_.tablet_ctx_->init(param_.dest_ls_id_, - param_.dest_tablet_id_, + } else if (OB_FAIL(context_.tablet_ctx_->init(param_.dest_tablet_id_, param_.user_parallelism_, param_.snapshot_version_, param_.direct_load_type_, @@ -669,7 +661,6 @@ uint64_t ObComplementDataDag::hash() const } else { hash_val = 1UL + 1UL + param_.orig_table_id_ + param_.dest_table_id_ - + param_.orig_ls_id_.hash() + param_.dest_ls_id_.hash() + param_.orig_tablet_id_.hash() + param_.dest_tablet_id_.hash() + ObDagType::DAG_TYPE_DDL; } return hash_val; @@ -689,7 +680,6 @@ bool ObComplementDataDag::operator==(const ObIDag &other) const } else { is_equal = (1UL == 1UL) && (1UL == 1UL) && (param_.orig_table_id_ == dag.param_.orig_table_id_) && (param_.dest_table_id_ == dag.param_.dest_table_id_) && - (param_.orig_ls_id_ == dag.param_.orig_ls_id_) && (param_.dest_ls_id_ == dag.param_.dest_ls_id_) && (param_.orig_tablet_id_ == dag.param_.orig_tablet_id_) && (param_.dest_tablet_id_ == dag.param_.dest_tablet_id_); } } @@ -711,7 +701,7 @@ int ObComplementDataDag::report_replica_build_status() #ifdef ERRSIM if (OB_SUCC(ret)) { ret = OB_E(EventTable::EN_DDL_REPORT_REPLICA_BUILD_STATUS_FAIL) OB_SUCCESS; - LOG_INFO("report replica build status errsim", K(ret)); + LOG_INFO("report DDL build status errsim", K(ret)); } #endif obcall::ObDDLBuildSingleReplicaResponseArg arg; @@ -719,8 +709,6 @@ int ObComplementDataDag::report_replica_build_status() - arg.ls_id_ = param_.orig_ls_id_; - arg.dest_ls_id_ = param_.dest_ls_id_; arg.tablet_id_ = param_.orig_tablet_id_; arg.source_table_id_ = param_.orig_table_id_; arg.dest_schema_id_ = param_.dest_table_id_; @@ -733,7 +721,7 @@ int ObComplementDataDag::report_replica_build_status() arg.row_inserted_ = context_.row_inserted_; arg.physical_row_count_ = context_.physical_row_count_; arg.server_addr_ = GCTX.self_addr(); - FLOG_INFO("send replica build status response to RS", K(ret), K(context_), K(arg)); + FLOG_INFO("send DDL build status response to RS", K(ret), K(context_), K(arg)); if (OB_FAIL(ret)) { } else if (OB_FAIL(GCTX.root_service_->build_ddl_single_replica_response(arg))) { LOG_WARN("fail to send build ddl single replica response", K(ret), K(arg)); @@ -754,7 +742,6 @@ int ObComplementDataDag::fill_info_param(compaction::ObIBasicInfoParam *&out_par ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param", K(ret), K(param_)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - param_.orig_ls_id_.id(), static_cast(param_.orig_tablet_id_.id()), static_cast(param_.dest_tablet_id_.id()), static_cast(param_.orig_table_id_), @@ -775,8 +762,8 @@ int ObComplementDataDag::fill_dag_key(char *buf, const int64_t buf_len) const } else if (OB_UNLIKELY(!param_.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid params", K(ret), K(param_)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, "logstream_id=%ld source_tablet_id=%ld dest_tablet_id=%ld", - param_.orig_ls_id_.id(), param_.orig_tablet_id_.id(), param_.dest_tablet_id_.id()))) { + } else if (OB_FAIL(databuff_printf(buf, buf_len, "source_tablet_id=%ld dest_tablet_id=%ld", + param_.orig_tablet_id_.id(), param_.dest_tablet_id_.id()))) { LOG_WARN("fill dag key for ddl table merge dag failed", K(ret), K(param_)); } return ret; @@ -1011,7 +998,6 @@ int ObComplementWriteTask::get_next_chunk(ObChunk *&next_chunk) int ObComplementWriteTask::fill_writer_param(ObWriteMacroParam ¶m) { int ret = OB_SUCCESS; - param.ls_id_ = param_->dest_ls_id_; param.tablet_id_ = param_->dest_tablet_id_; param.tenant_data_version_ = param_->data_format_version_; param.is_no_logging_ = param_->is_no_logging_; @@ -1255,21 +1241,18 @@ int ObComplementWriteTask::do_local_scan() false, false); const bool allow_not_ready = false; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletTableIterator iterator; ObSSTable *sstable = nullptr; const int64_t schema_version = param_->dest_schema_version_; scan_ = scan; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param_->orig_ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("fail to get log stream", K(ret), KPC(param_)); - } else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_handle)); } else if (OB_FAIL(DDL_SIM(param_->task_id_, COMPLEMENT_DATA_TASK_LOCAL_SCAN_FAILED))) { LOG_WARN("ddl sim failure", K(ret), KPC(param_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_read_tables(param_->orig_tablet_id_, + } else if (OB_FAIL(ls->get_tablet_svr()->get_read_tables(param_->orig_tablet_id_, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, param_->snapshot_version_, param_->snapshot_version_, iterator, allow_not_ready, false/*need_split_src_table*/, false/*need_split_dst_table*/))) { if (OB_REPLICA_NOT_READABLE == ret) { @@ -1315,7 +1298,6 @@ int ObComplementWriteTask::do_local_scan() false/*unique_index_checking*/))) { LOG_WARN("fail to init local scan param", K(ret), K(*param_)); } else if (OB_FAIL(scan->table_scan(*data_table_schema, - param_->orig_ls_id_, param_->orig_tablet_id_, iterator, query_flag, @@ -1415,7 +1397,7 @@ int ObComplementMergeTask::process() ObTablet *tablet = nullptr; ObArray report_col_checksums; ObArray report_col_ids; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_ISNULL(tmp_dag) || ObDagType::DAG_TYPE_DDL != tmp_dag->get_type()) { ret = OB_ERR_UNEXPECTED; @@ -1429,19 +1411,16 @@ int ObComplementMergeTask::process() ObTabletMemberWrapper table_store_wrapper; const ObSSTable *first_major_sstable = nullptr; ObSSTableMetaHandle sst_meta_hdl; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param_->dest_ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_handle)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, param_->dest_tablet_id_, tablet_handle, + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, param_->dest_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), KPC_(param)); + LOG_WARN("get tablet handle failed", K(ret), KPC_(param)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(ls_id), KPC_(param)); + LOG_WARN("tablet handle is null", K(ret), KPC_(param)); } else if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable( - param_->dest_ls_id_, param_->dest_tablet_id_, first_major_sstable, table_store_wrapper))) { + param_->dest_tablet_id_, first_major_sstable, table_store_wrapper))) { LOG_WARN("check if major sstable exist failed", K(ret), K(*param_)); } else if (OB_ISNULL(first_major_sstable)) { ret = OB_ERR_UNEXPECTED; @@ -1451,8 +1430,7 @@ int ObComplementMergeTask::process() } else { const int64_t *column_checksums = sst_meta_hdl.get_sstable_meta().get_col_checksum(); const int64_t column_count = sst_meta_hdl.get_sstable_meta().get_col_checksum_cnt(); - if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum(param_->dest_ls_id_, - param_->dest_tablet_id_, + if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum(param_->dest_tablet_id_, param_->dest_table_id_, 1 /* execution_id */, param_->task_id_, @@ -1460,7 +1438,7 @@ int ObComplementMergeTask::process() column_count, param_->data_format_version_))) { LOG_WARN("report ddl column checksum failed", K(ret), K(*param_)); - } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(param_->dest_ls_id_, param_->dest_tablet_id_))) { + } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(param_->dest_tablet_id_))) { LOG_WARN("fail to submit tablet update task", K(ret), K(*param_)); } } @@ -1487,7 +1465,7 @@ int ObComplementMergeTask::process() OB_SUCCESS != (tmp_ret = dag->report_replica_build_status())) { // do not override ret if it has already failed. ret = OB_SUCCESS == ret ? tmp_ret : ret; - LOG_WARN("fail to report replica build status", K(ret), K(tmp_ret)); + LOG_WARN("fail to report DDL build status", K(ret), K(tmp_ret)); } add_ddl_event(param_, "complement merge task"); @@ -1660,7 +1638,6 @@ int ObLocalScan::check_generated_column_exist( int ObLocalScan::table_scan( const ObTableSchema &data_table_schema, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletTableIterator &table_iter, ObQueryFlag &query_flag, @@ -1671,7 +1648,7 @@ int ObLocalScan::table_scan( LOG_WARN("fail to construct column schema", K(ret), K(col_params_)); } else if (OB_FAIL(construct_access_param(data_table_schema, tablet_id))) { LOG_WARN("fail to construct access param", K(ret), K(col_params_)); - } else if (OB_FAIL(construct_range_ctx(query_flag, ls_id))) { + } else if (OB_FAIL(construct_range_ctx(query_flag))) { LOG_WARN("fail to construct range ctx", K(ret), K(query_flag)); } else if (OB_FAIL(construct_multiple_scan_merge(table_iter, range))) { LOG_WARN("fail to construct multiple scan merge", K(ret), K(table_iter), K(range)); @@ -1788,8 +1765,7 @@ int ObLocalScan::construct_access_param( } //construct version range and ctx -int ObLocalScan::construct_range_ctx(ObQueryFlag &query_flag, - const share::ObLSID &ls_id) +int ObLocalScan::construct_range_ctx(ObQueryFlag &query_flag) { int ret = OB_SUCCESS; common::ObVersionRange trans_version_range; @@ -1798,13 +1774,12 @@ int ObLocalScan::construct_range_ctx(ObQueryFlag &query_flag, trans_version_range.base_version_ = 0; SCN tmp_scn; if (OB_FAIL(tmp_scn.convert_for_tx(snapshot_version_))) { - LOG_WARN("convert fail", K(ret), K(ls_id), K_(snapshot_version)); - } else if (OB_FAIL(ctx_.init_for_read(ls_id, - access_param_.iter_param_.tablet_id_, + LOG_WARN("convert fail", K(ret), K_(snapshot_version)); + } else if (OB_FAIL(ctx_.init_for_read(access_param_.iter_param_.tablet_id_, INT64_MAX, -1, tmp_scn))) { - LOG_WARN("fail to init store ctx", K(ret), K(ls_id)); + LOG_WARN("fail to init store ctx", K(ret)); } else if (OB_FAIL(access_ctx_.init(query_flag, ctx_, allocator_, allocator_, trans_version_range))) { LOG_WARN("fail to init accesss ctx", K(ret)); } diff --git a/src/storage/ddl/ob_complement_data_task.h b/src/storage/ddl/ob_complement_data_task.h index e72a3c29d..8d4c4c90d 100644 --- a/src/storage/ddl/ob_complement_data_task.h +++ b/src/storage/ddl/ob_complement_data_task.h @@ -64,7 +64,7 @@ struct ObComplementDataParam final public: ObComplementDataParam(): is_inited_(false), - orig_ls_id_(share::ObLSID::INVALID_LS_ID), dest_ls_id_(share::ObLSID::INVALID_LS_ID), orig_table_id_(common::OB_INVALID_ID), + orig_table_id_(common::OB_INVALID_ID), dest_table_id_(common::OB_INVALID_ID), orig_tablet_id_(ObTabletID::INVALID_TABLET_ID), dest_tablet_id_(ObTabletID::INVALID_TABLET_ID), row_store_type_(common::ENCODING_ROW_STORE), orig_schema_version_(0), dest_schema_version_(0), snapshot_version_(0), task_id_(0), execution_id_(-1), tablet_task_id_(0), compat_mode_(lib::Worker::CompatMode::INVALID), data_format_version_(0), @@ -76,14 +76,12 @@ struct ObComplementDataParam final int prepare_task_ranges(); int split_task_ranges( const int64_t task_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t tablet_size, const int64_t hint_parallelism); bool is_valid() const { - return true && true - && orig_ls_id_.is_valid() && dest_ls_id_.is_valid() && common::OB_INVALID_ID != orig_table_id_ + return common::OB_INVALID_ID != orig_table_id_ && common::OB_INVALID_ID != dest_table_id_ && orig_tablet_id_.is_valid() && dest_tablet_id_.is_valid() && snapshot_version_ > 0 && compat_mode_ != lib::Worker::CompatMode::INVALID && execution_id_ >= 0 && tablet_task_id_ > 0 && data_format_version_ > 0 && orig_schema_tablet_size_ > 0 && user_parallelism_ > 0; @@ -96,10 +94,6 @@ struct ObComplementDataParam final void destroy() { is_inited_ = false; - - - orig_ls_id_.reset(); - dest_ls_id_.reset(); orig_table_id_ = common::OB_INVALID_ID; dest_table_id_ = common::OB_INVALID_ID; orig_tablet_id_.reset(); @@ -132,7 +126,7 @@ struct ObComplementDataParam final } allocator_.reset(); } - TO_STRING_KV(K_(is_inited), K_(orig_ls_id), K_(dest_ls_id), + TO_STRING_KV(K_(is_inited), K_(orig_table_id), K_(dest_table_id), K_(orig_tablet_id), K_(dest_tablet_id), K_(orig_schema_version), K_(tablet_task_id), K_(dest_schema_version), K_(snapshot_version), K_(task_id), K_(execution_id), K_(compat_mode), K_(data_format_version), K_(orig_schema_tablet_size),K_(user_parallelism), @@ -147,10 +141,6 @@ struct ObComplementDataParam final bool &is_allow_parallel); public: bool is_inited_; - - - share::ObLSID orig_ls_id_; - share::ObLSID dest_ls_id_; uint64_t orig_table_id_; uint64_t dest_table_id_; ObTabletID orig_tablet_id_; @@ -245,7 +235,7 @@ class ObComplementDataDag final: public share::ObIDag virtual int create_first_task() override; virtual bool ignore_warning() override; virtual bool is_ha_dag() const override { return false; } - // report replica build status to RS. + // report DDL build status to RS. int report_replica_build_status(); int calc_total_row_count(); private: @@ -401,7 +391,6 @@ class ObLocalScan : public ObScan const share::schema::ObTableSchema &hidden_table_schema, const bool unique_index_checking); int table_scan(const share::schema::ObTableSchema &data_table_schema, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletTableIterator &table_iter, common::ObQueryFlag &query_flag, @@ -425,7 +414,7 @@ class ObLocalScan : public ObScan int construct_access_param( const share::schema::ObTableSchema &data_table_schema, const ObTabletID &tablet_id); - int construct_range_ctx(common::ObQueryFlag &query_flag, const share::ObLSID &ls_id); + int construct_range_ctx(common::ObQueryFlag &query_flag); int construct_multiple_scan_merge(ObTablet &tablet, blocksstable::ObDatumRange &range); int construct_multiple_scan_merge( ObTabletTableIterator &table_iter, diff --git a/src/storage/ddl/ob_ddl_change_tablet_to_table_helper.h b/src/storage/ddl/ob_ddl_change_tablet_to_table_helper.h index 7f89d97d8..d0240cd6e 100644 --- a/src/storage/ddl/ob_ddl_change_tablet_to_table_helper.h +++ b/src/storage/ddl/ob_ddl_change_tablet_to_table_helper.h @@ -21,7 +21,6 @@ #include "lib/container/ob_array.h" #include "lib/container/ob_array_serialization.h" #include "lib/ob_define.h" -#include "share/ob_ls_id.h" namespace oceanbase { diff --git a/src/storage/ddl/ob_ddl_clog.cpp b/src/storage/ddl/ob_ddl_clog.cpp index 4256d0935..bdec11791 100644 --- a/src/storage/ddl/ob_ddl_clog.cpp +++ b/src/storage/ddl/ob_ddl_clog.cpp @@ -155,7 +155,7 @@ void ObDDLStartClogCb::try_release() } ObDDLMacroBlockClogCb::ObDDLMacroBlockClogCb() - : is_inited_(false), status_(), ls_id_(), macro_block_id_(), + : is_inited_(false), status_(), macro_block_id_(), data_buffer_lock_(), is_data_buffer_freed_(false), ddl_macro_block_(), snapshot_version_(0), data_format_version_(0), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), block_checksum_(0), is_macro_block_exist_(false) @@ -170,8 +170,7 @@ ObDDLMacroBlockClogCb::~ObDDLMacroBlockClogCb() macro_block_id_.reset(); } -int ObDDLMacroBlockClogCb::init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, +int ObDDLMacroBlockClogCb::init(const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, ObTabletHandle &tablet_handle, const ObDirectLoadType &direct_load_type) @@ -180,14 +179,13 @@ int ObDDLMacroBlockClogCb::init(const share::ObLSID &ls_id, if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !redo_info.is_valid() || !macro_block_id.is_valid() || + } else if (OB_UNLIKELY(!redo_info.is_valid() || !macro_block_id.is_valid() || (direct_load_type <= ObDirectLoadType::DIRECT_LOAD_INVALID|| ObDirectLoadType::DIRECT_LOAD_MAX <= direct_load_type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(redo_info), K(macro_block_id)); + LOG_WARN("invalid argument", K(ret), K(redo_info), K(macro_block_id)); } else if (OB_FAIL(OB_STORAGE_OBJECT_MGR.inc_ref(macro_block_id))) { LOG_WARN("inc reference count failed", K(ret), K(macro_block_id)); } else { - ls_id_ = ls_id; macro_block_id_ = macro_block_id; tablet_handle_ = tablet_handle; snapshot_version_ = redo_info.table_key_.get_snapshot_version(); @@ -272,7 +270,7 @@ int ObDDLMacroBlockClogCb::on_success() } else if (ObDDLUtil::use_idempotent_mode()) { /* Do not fetch direct load mgr. */ } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_id_, tablet->get_tablet_meta().tablet_id_, true/*is_full_direct_load*/, direct_load_mgr_handle, is_major_sstable_exist))) { + tablet->get_tablet_meta().tablet_id_, true/*is_full_direct_load*/, direct_load_mgr_handle, is_major_sstable_exist))) { if (OB_ENTRY_NOT_EXIST == ret && is_major_sstable_exist) { ret = OB_TASK_EXPIRED; LOG_INFO("major sstable already exist", K(ret), "tablet_id", tablet->get_tablet_meta().tablet_id_); @@ -331,13 +329,12 @@ int ObDDLMacroBlockClogCb::on_failure() } ObDDLCommitClogCb::ObDDLCommitClogCb() - : is_inited_(false), status_(), ls_id_(), tablet_id_(), start_scn_(SCN::min_scn()), lock_tid_(0), direct_load_mgr_handle_(), lob_direct_load_mgr_handle_() + : is_inited_(false), status_(), tablet_id_(), start_scn_(SCN::min_scn()), lock_tid_(0), direct_load_mgr_handle_(), lob_direct_load_mgr_handle_() { } -int ObDDLCommitClogCb::init(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObDDLCommitClogCb::init(const common::ObTabletID &tablet_id, const share::SCN &start_scn, const uint32_t lock_tid, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -347,16 +344,15 @@ int ObDDLCommitClogCb::init(const share::ObLSID &ls_id, if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !start_scn.is_valid_and_not_min() + } else if (OB_UNLIKELY(!tablet_id.is_valid() || !start_scn.is_valid_and_not_min() || 0 == lock_tid)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(start_scn), K(lock_tid)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(start_scn), K(lock_tid)); } else if (OB_FAIL(direct_load_mgr_handle_.assign(direct_load_mgr_handle))) { LOG_WARN("assign handle failed", K(ret)); } else if (OB_FAIL(lob_direct_load_mgr_handle_.assign(lob_direct_load_mgr_handle))) { LOG_WARN("assign handle failed", K(ret)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; start_scn_ = start_scn; lock_tid_ = lock_tid; @@ -550,7 +546,7 @@ int ObTabletSchemaVersionChangeLog::init(const ObTabletID &tablet_id, const int6 OB_SERIALIZE_MEMBER(ObTabletSchemaVersionChangeLog, tablet_id_, schema_version_); -OB_SERIALIZE_MEMBER(ObDDLBarrierLog, ls_id_, hidden_tablet_ids_); +OB_SERIALIZE_MEMBER(ObDDLBarrierLog, hidden_tablet_ids_); ObTabletSplitInfo::ObTabletSplitInfo() : rowkey_allocator_("SplitRangeClog"), diff --git a/src/storage/ddl/ob_ddl_clog.h b/src/storage/ddl/ob_ddl_clog.h index eeaa89ead..bfd862641 100644 --- a/src/storage/ddl/ob_ddl_clog.h +++ b/src/storage/ddl/ob_ddl_clog.h @@ -130,8 +130,7 @@ class ObDDLMacroBlockClogCb : public logservice::AppendCb public: ObDDLMacroBlockClogCb(); virtual ~ObDDLMacroBlockClogCb(); - int init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, + int init(const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, ObTabletHandle &tablet_handle, const ObDirectLoadType &direct_load_type); @@ -146,7 +145,6 @@ class ObDDLMacroBlockClogCb : public logservice::AppendCb private: bool is_inited_; ObDDLClogCbStatus status_; - share::ObLSID ls_id_; blocksstable::MacroBlockId macro_block_id_; ObSpinLock data_buffer_lock_; bool is_data_buffer_freed_; @@ -164,8 +162,7 @@ class ObDDLCommitClogCb : public logservice::AppendCb public: ObDDLCommitClogCb(); virtual ~ObDDLCommitClogCb() = default; - int init(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + int init(const common::ObTabletID &tablet_id, const share::SCN &start_scn, const uint32_t lock_tid, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -178,11 +175,10 @@ class ObDDLCommitClogCb : public logservice::AppendCb int get_ret_code() const { return status_.get_ret_code(); } void try_release(); const char *get_cb_name() const override { return "DDLCommitClogCb"; } - TO_STRING_KV(K(is_inited_), K(status_), K(ls_id_), K(tablet_id_), K(start_scn_), K_(lock_tid)); + TO_STRING_KV(K(is_inited_), K(status_), K(tablet_id_), K(start_scn_), K_(lock_tid)); private: bool is_inited_; ObDDLClogCbStatus status_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; share::SCN start_scn_; uint32_t lock_tid_; @@ -289,13 +285,12 @@ class ObTabletSchemaVersionChangeLog final class ObDDLBarrierLog final { public: - ObDDLBarrierLog() : ls_id_(), hidden_tablet_ids_() {} + ObDDLBarrierLog() : hidden_tablet_ids_() {} ~ObDDLBarrierLog() {} - bool is_valid() const { return ls_id_.is_valid() && hidden_tablet_ids_.count() > 0; } - TO_STRING_KV(K_(ls_id), K_(hidden_tablet_ids)); + bool is_valid() const { return hidden_tablet_ids_.count() > 0; } + TO_STRING_KV(K_(hidden_tablet_ids)); OB_UNIS_VERSION_V(1); public: - share::ObLSID ls_id_; common::ObSArray hidden_tablet_ids_; }; @@ -317,8 +312,8 @@ class ObTabletSplitInfo final common::ObArenaAllocator rowkey_allocator_; // alloc buf for datum rowkey. uint64_t table_id_; // scan rows needed, index table id or main table id. uint64_t lob_table_id_; // scan rows needed, valid when split lob tablet. - int64_t schema_version_; // report replica build status needed. - int64_t task_id_; // report replica build status needed. + int64_t schema_version_; // report DDL build status needed. + int64_t task_id_; // report DDL build status needed. common::ObTabletID source_tablet_id_; common::ObSArray dest_tablets_id_; int64_t compaction_scn_; diff --git a/src/storage/ddl/ob_ddl_common_storage_impl.cpp b/src/storage/ddl/ob_ddl_common_storage_impl.cpp index 99e76be8e..5bf61cdf1 100644 --- a/src/storage/ddl/ob_ddl_common_storage_impl.cpp +++ b/src/storage/ddl/ob_ddl_common_storage_impl.cpp @@ -42,7 +42,6 @@ #include "storage/direct_load/ob_direct_load_insert_data_table_ctx.h" #include "storage/tablet/ob_tablet_obj_load_helper.h" #include "storage/tablet/ob_tablet.h" -#include "share/location_cache/ob_location_service.h" #include "lib/worker.h" #include "storage/ddl/ob_ddl_write_stat_util.h" #include "share/ob_ddl_error_message_table_operator.h" // extract_index_key relocated-definition owner @@ -168,7 +167,6 @@ int check_skip_handle_lob_column( } int ObDDLUtil::report_ddl_checksum_from_major_sstable( - const ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t target_table_id, const int64_t execution_id, @@ -176,14 +174,14 @@ int ObDDLUtil::report_ddl_checksum_from_major_sstable( const int64_t tenant_data_version) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || OB_INVALID_ID == target_table_id || execution_id < 0 || ddl_task_id < 0 || tenant_data_version < 0)) { + if (OB_UNLIKELY(!tablet_id.is_valid() || OB_INVALID_ID == target_table_id || execution_id < 0 || ddl_task_id < 0 || tenant_data_version < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle))) { + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id)); } else { ObSSTable *first_major_sstable = nullptr; @@ -192,16 +190,15 @@ int ObDDLUtil::report_ddl_checksum_from_major_sstable( LOG_WARN("fetch table store failed", K(ret)); } else if (OB_ISNULL(first_major_sstable = static_cast(table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/)))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("no major after wait merge success", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(report_ddl_sstable_checksum(ls_id, tablet_id, target_table_id, execution_id, ddl_task_id, tenant_data_version, tablet_handle, first_major_sstable))) { - LOG_WARN("report ddl sstable checksum failed", K(ret), K(ls_id), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version)); + LOG_WARN("no major after wait merge success", K(ret), K(tablet_id)); + } else if (OB_FAIL(report_ddl_sstable_checksum(tablet_id, target_table_id, execution_id, ddl_task_id, tenant_data_version, tablet_handle, first_major_sstable))) { + LOG_WARN("report ddl sstable checksum failed", K(ret), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version)); } } return ret; } int ObDDLUtil::report_ddl_sstable_checksum( - const ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t target_table_id, const int64_t execution_id, @@ -212,26 +209,25 @@ int ObDDLUtil::report_ddl_sstable_checksum( { int ret = OB_SUCCESS; ObSSTableMetaHandle sst_meta_hdl; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || OB_INVALID_ID == target_table_id || + if (OB_UNLIKELY(!tablet_id.is_valid() || OB_INVALID_ID == target_table_id || execution_id < 0 || ddl_task_id < 0 || tenant_data_version < 0 || nullptr == first_major_sstable || !tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version), KPC(first_major_sstable), K(tablet_handle)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(target_table_id), K(execution_id), K(ddl_task_id), K(tenant_data_version), KPC(first_major_sstable), K(tablet_handle)); } else if (OB_FAIL(first_major_sstable->get_meta(sst_meta_hdl))) { LOG_WARN("fail to get sstable meta handle", K(ret)); } else { const int64_t *column_checksums = sst_meta_hdl.get_sstable_meta().get_col_checksum(); int64_t column_count = sst_meta_hdl.get_sstable_meta().get_col_checksum_cnt(); for (int64_t retry_cnt = 10; retry_cnt > 0; retry_cnt--) { // overwrite ret - if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum(ls_id, - tablet_id, + if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum(tablet_id, target_table_id, execution_id, ddl_task_id, column_checksums, column_count, tenant_data_version))) { - LOG_WARN("report ddl column checksum failed", K(ret), K(ls_id), K(tablet_id), K(ddl_task_id)); + LOG_WARN("report ddl column checksum failed", K(ret), K(tablet_id), K(ddl_task_id)); } else { break; } @@ -516,7 +512,6 @@ int ObDDLUtil::convert_to_storage_row( int ObDDLUtil::get_task_ranges( const int64_t task_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t tablet_size, const int64_t hint_parallelism, @@ -526,18 +521,15 @@ int ObDDLUtil::get_task_ranges( int ret = OB_SUCCESS; ObFreezeInfo frozen_status; const bool allow_not_ready = false; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletTableIterator iterator; ObLSTabletService *tablet_service = nullptr; - if (OB_UNLIKELY(task_id <= 0 || !ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(task_id <= 0 || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(task_id), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get log stream", K(ret), K(ls_id)); - } else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_handle)); - } else if (OB_ISNULL(tablet_service = ls_handle.get_ls()->get_tablet_svr())) { + LOG_WARN("invalid arguments", K(ret), K(task_id), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("fail to get log stream", K(ret)); + } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet service is nullptr", K(ret)); } else if (OB_FAIL(DDL_SIM(task_id, COMPLEMENT_DATA_TASK_SPLIT_RANGE_FAILED))) { @@ -604,7 +596,6 @@ int ObDDLUtil::get_task_ranges( } int ObDDLUtil::get_tablet_physical_row_cnt( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool calc_sstable, const bool calc_memtable, @@ -615,20 +606,20 @@ int ObDDLUtil::get_tablet_physical_row_cnt( // get total rows of the table; physical // src_tablet_id -> tablet -> sstables -> sstable_metas -> row_count // -> memtables -> physical_row_cnt - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObTableStoreIterator table_store_iter; physical_row_count = 0; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("fail to get tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid arg", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("fail to get tablet", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpecter error", K(ret), K(tablet_handle)); @@ -688,23 +679,23 @@ int ObDDLUtil::get_tablet_physical_row_cnt( return ret; } -int ObDDLUtil::is_major_exist(const ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &is_major_exist) +int ObDDLUtil::is_major_exist(const common::ObTabletID &tablet_id, bool &is_major_exist) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObLSService* ls_svr = share::g_mp->ls_service(); is_major_exist = false; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(ls_svr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ddl_get_tablet(ls_handle, tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet id", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ls_svr->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(ddl_get_tablet(ls, tablet_id, tablet_handle))) { + LOG_WARN("failed to get tablet id", K(ret), K(tablet_id)); } else { is_major_exist = tablet_handle.get_obj()->get_major_table_count() > 0 || tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable(); @@ -788,7 +779,6 @@ int ObDDLUtil::fill_writer_param( LOG_WARN("get ddl tablet context", K(ret), K(tablet_id), K(slice_idx)); } else { const ObDDLTaskParam &ddl_task_param = dag->get_ddl_task_param(); - param.ls_id_ = tablet_context->ls_id_; param.tablet_id_ = tablet_id; param.lob_meta_tablet_id_ = tablet_context->lob_meta_tablet_id_; param.tenant_data_version_ = ddl_task_param.tenant_data_version_; @@ -850,22 +840,21 @@ int ObDDLUtil::init_batch_rows( } int ObDDLUtil::ddl_get_tablet( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle, storage::ObMDSGetTabletMode mode) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; const int64_t DDL_GET_TABLET_RETRY_TIMEOUT = 30 * 1000 * 1000; // 30s const int64_t timeout_ts = ObTimeUtility::current_time() + DDL_GET_TABLET_RETRY_TIMEOUT; - if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); + if (OB_ISNULL(ls)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid ls", K(ret), KP(ls), K(tablet_id)); } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet_with_timeout(tablet_id, - tablet_handle, - timeout_ts, - mode))) { + tablet_handle, + timeout_ts, + mode))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id)); if (OB_ALLOCATE_MEMORY_FAILED == ret) { ret = OB_TIMEOUT; @@ -874,19 +863,6 @@ int ObDDLUtil::ddl_get_tablet( return ret; } -int ObDDLUtil::is_ls_leader(ObLS &ls, bool &is_leader) -{ - int ret = OB_SUCCESS; - ObRole role; - int64_t proposal_id = 0; - if (OB_FAIL(ls.get_log_handler()->get_role(role, proposal_id))) { - LOG_WARN("failed to get role", K(ret), K(ls.get_ls_id())); - } else { - is_leader = is_strong_leader(role); - } - return ret; -} - int ObDDLUtil::alloc_storage_macro_block_writer( const ObWriteMacroParam ¶m, ObIAllocator &allocator, @@ -987,17 +963,16 @@ int ObSplitUtil::deserializ_parallel_datum_rowkey( // ===== storage-clean static methods from ObDDLUtil demoted to storage::ObDDLStorageUtil members (A-set member-split cleanup)===== #include "storage/ddl/ob_ddl_storage_util.h" -int ObDDLUtil::set_tablet_autoinc_seq(const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t seq_value) +int ObDDLUtil::set_tablet_autoinc_seq(const ObTabletID &tablet_id, const int64_t seq_value) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || seq_value < 0)) { + if (OB_UNLIKELY(!tablet_id.is_valid() || seq_value < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(seq_value)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(seq_value)); } else { ObMigrateTabletAutoincSeqParam tablet_autoinc_param; obcall::ObBatchSetTabletAutoincSeqArg arg; obcall::ObBatchSetTabletAutoincSeqRes res; - arg.ls_id_ = ls_id; tablet_autoinc_param.src_tablet_id_ = tablet_id; tablet_autoinc_param.dest_tablet_id_ = tablet_id; tablet_autoinc_param.autoinc_seq_ = seq_value; diff --git a/src/storage/ddl/ob_ddl_inc_clog_callback.cpp b/src/storage/ddl/ob_ddl_inc_clog_callback.cpp index eaccadd1e..211ea6803 100644 --- a/src/storage/ddl/ob_ddl_inc_clog_callback.cpp +++ b/src/storage/ddl/ob_ddl_inc_clog_callback.cpp @@ -29,22 +29,21 @@ using namespace share; using namespace common; - ObDDLIncStartClogCb::ObDDLIncStartClogCb() - : is_inited_(false), ls_id_(), log_basic_() +ObDDLIncStartClogCb::ObDDLIncStartClogCb() + : is_inited_(false), log_basic_() { } - int ObDDLIncStartClogCb::init(const ObLSID &ls_id, const ObDDLIncLogBasic &log_basic) + int ObDDLIncStartClogCb::init(const ObDDLIncLogBasic &log_basic) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !log_basic.is_valid())) { + } else if (OB_UNLIKELY(!log_basic.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(log_basic)); + LOG_WARN("invalid argument", K(ret), K(log_basic)); } else { - ls_id_ = ls_id; log_basic_ = log_basic; is_inited_ = true; } @@ -59,7 +58,7 @@ status_.set_ret_code(ret); status_.set_state(STATE_SUCCESS); SCN scn = __get_scn(); - FLOG_INFO("write ddl inc start log success", K(ls_id_), K(scn), K(log_basic_)); + FLOG_INFO("write ddl inc start log success", K(scn), K(log_basic_)); try_release(); return OB_SUCCESS; } @@ -82,8 +81,8 @@ } } - ObDDLIncRedoClogCb::ObDDLIncRedoClogCb() - : is_inited_(false), ls_id_(), redo_info_(), macro_block_id_(), +ObDDLIncRedoClogCb::ObDDLIncRedoClogCb() + : is_inited_(false), redo_info_(), macro_block_id_(), data_buffer_lock_(), is_data_buffer_freed_(false) { @@ -98,8 +97,7 @@ macro_block_id_.reset(); } - int ObDDLIncRedoClogCb::init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, + int ObDDLIncRedoClogCb::init(const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, ObTabletHandle &tablet_handle) { @@ -107,14 +105,13 @@ if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !redo_info.is_valid() || !macro_block_id.is_valid())) { + } else if (OB_UNLIKELY(!redo_info.is_valid() || !macro_block_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(redo_info), K(macro_block_id)); + LOG_WARN("invalid argument", K(ret), K(redo_info), K(macro_block_id)); } else if (OB_FAIL(OB_STORAGE_OBJECT_MGR.inc_ref(macro_block_id))) { LOG_WARN("inc reference count failed", K(ret), K(macro_block_id)); } else { redo_info_ = redo_info; - ls_id_ = ls_id; macro_block_id_ = macro_block_id; tablet_handle_ = tablet_handle; } @@ -184,22 +181,21 @@ return OB_SUCCESS; } - ObDDLIncCommitClogCb::ObDDLIncCommitClogCb() - : is_inited_(false), ls_id_(), log_basic_() +ObDDLIncCommitClogCb::ObDDLIncCommitClogCb() + : is_inited_(false), log_basic_() { } - int ObDDLIncCommitClogCb::init(const ObLSID &ls_id, const ObDDLIncLogBasic &log_basic) + int ObDDLIncCommitClogCb::init(const ObDDLIncLogBasic &log_basic) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !log_basic.is_valid())) { + } else if (OB_UNLIKELY(!log_basic.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(log_basic)); + LOG_WARN("invalid argument", K(ret), K(log_basic)); } else { - ls_id_ = ls_id; log_basic_ = log_basic; is_inited_ = true; } @@ -212,12 +208,8 @@ int ret = OB_SUCCESS; common::ObTimeGuard timeguard("ObDDLIncCommitClogCb::on_success", 1 * 1000 * 1000); // 1s ObLS *ls = nullptr; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(log_basic_.get_tablet_id())); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else { const bool is_sync = false; (void)ls->tablet_freeze(log_basic_.get_tablet_id(), @@ -233,7 +225,7 @@ ObFreezeSourceFlag::DIRECT_INC_START); } SCN scn = __get_scn(); - FLOG_INFO("write ddl inc commit log success", K(ls_id_), K(scn), K(log_basic_)); + FLOG_INFO("write ddl inc commit log success", K(scn), K(log_basic_)); } status_.set_ret_code(ret); diff --git a/src/storage/ddl/ob_ddl_inc_clog_callback.h b/src/storage/ddl/ob_ddl_inc_clog_callback.h index 7fe0e5ab6..bb4402532 100644 --- a/src/storage/ddl/ob_ddl_inc_clog_callback.h +++ b/src/storage/ddl/ob_ddl_inc_clog_callback.h @@ -49,15 +49,14 @@ public: ObDDLIncStartClogCb(); virtual ~ObDDLIncStartClogCb() = default; - int init(const share::ObLSID &ls_id, const ObDDLIncLogBasic &log_basic); + int init(const ObDDLIncLogBasic &log_basic); virtual int on_success() override; virtual int on_failure() override; virtual void try_release() override; const char *get_cb_name() const override { return "DDLIncStartClogCb"; } - INHERIT_TO_STRING_KV("ObDDLIncClogCb", ObDDLIncClogCb, K(is_inited_), K(ls_id_), K(log_basic_)); + INHERIT_TO_STRING_KV("ObDDLIncClogCb", ObDDLIncClogCb, K(is_inited_), K(log_basic_)); private: bool is_inited_; - share::ObLSID ls_id_; ObDDLIncLogBasic log_basic_; }; @@ -66,8 +65,7 @@ public: ObDDLIncRedoClogCb(); virtual ~ObDDLIncRedoClogCb(); - int init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, + int init(const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, storage::ObTabletHandle &tablet_handle); virtual int on_success() override; @@ -76,7 +74,6 @@ const char *get_cb_name() const override { return "DDLIncRedoClogCb"; } private: bool is_inited_; - share::ObLSID ls_id_; storage::ObDDLMacroBlockRedoInfo redo_info_; blocksstable::MacroBlockId macro_block_id_; ObSpinLock data_buffer_lock_; @@ -89,15 +86,14 @@ public: ObDDLIncCommitClogCb(); virtual ~ObDDLIncCommitClogCb() = default; - int init(const share::ObLSID &ls_id, const ObDDLIncLogBasic &log_basic); + int init(const ObDDLIncLogBasic &log_basic); virtual int on_success() override; virtual int on_failure() override; virtual void try_release() override; const char *get_cb_name() const override { return "DDLIncCommitClogCb"; } - INHERIT_TO_STRING_KV("ObDDLIncClogCb", ObDDLIncClogCb, K(is_inited_), K(ls_id_), K(log_basic_)); + INHERIT_TO_STRING_KV("ObDDLIncClogCb", ObDDLIncClogCb, K(is_inited_), K(log_basic_)); private: bool is_inited_; - share::ObLSID ls_id_; ObDDLIncLogBasic log_basic_; }; diff --git a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp index 8ca03bb2b..969bfb7bd 100644 --- a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp @@ -1,4 +1,3 @@ -#include "share/ob_ex_rpc.h" #include "share/rc/ob_module_provider.h" /* * Copyright (c) 2025 OceanBase. @@ -19,9 +18,7 @@ #define USING_LOG_PREFIX STORAGE #include "storage/ddl/ob_ddl_inc_redo_log_writer.h" -#include "storage/ob_storage_rpc.h" -#include "storage/ob_storage_rpc_arg.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_trans_service.h" #include "storage/tx_storage/ob_ls_service.h" @@ -84,8 +81,7 @@ int ObDDLIncLogHandle::wait(const int64_t timeout) } ObDDLIncRedoLogWriter::ObDDLIncRedoLogWriter() - : is_inited_(false), remote_write_(false), - ls_id_(), tablet_id_(), ddl_inc_log_handle_(), leader_addr_(), leader_ls_id_(), buffer_(nullptr) + : is_inited_(false), tablet_id_(), ddl_inc_log_handle_(), buffer_(nullptr) { } @@ -98,17 +94,16 @@ ObDDLIncRedoLogWriter::~ObDDLIncRedoLogWriter() } } -int ObDDLIncRedoLogWriter::init(const ObLSID &ls_id, const ObTabletID &tablet_id) +int ObDDLIncRedoLogWriter::init(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("inited twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid arguments", K(ret), K(tablet_id)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; is_inited_ = true; } @@ -118,17 +113,13 @@ int ObDDLIncRedoLogWriter::init(const ObLSID &ls_id, const ObTabletID &tablet_id void ObDDLIncRedoLogWriter::reset() { is_inited_ = false; - remote_write_ = false; - ls_id_.reset(); tablet_id_.reset(); ddl_inc_log_handle_.reset(); - leader_addr_.reset(); - leader_ls_id_.reset(); } -bool ObDDLIncRedoLogWriter::need_retry(int ret_code, bool allow_remote_write) +bool ObDDLIncRedoLogWriter::need_retry(int ret_code) { - return OB_TX_NOLOGCB == ret_code || (allow_remote_write && OB_NOT_MASTER == ret_code); + return OB_TX_NOLOGCB == ret_code; } int ObDDLIncRedoLogWriter::write_inc_start_log( @@ -183,7 +174,6 @@ int ObDDLIncRedoLogWriter::write_inc_redo_log( } int ObDDLIncRedoLogWriter::write_inc_commit_log( - const bool allow_remote_write, const ObTabletID &lob_meta_tablet_id, transaction::ObTxDesc *tx_desc) { @@ -195,7 +185,7 @@ int ObDDLIncRedoLogWriter::write_inc_commit_log( } else if (OB_UNLIKELY(tx_desc == nullptr)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KP(tx_desc)); - } else if (!remote_write_) { + } else { ObDDLIncLogBasic log_basic; ObDDLIncCommitLog log; if (OB_FAIL(log_basic.init(tablet_id_, lob_meta_tablet_id))) { @@ -203,20 +193,9 @@ int ObDDLIncRedoLogWriter::write_inc_commit_log( } else if (OB_FAIL(log.init(log_basic))) { LOG_WARN("fail to init DDLIncCommitLog", K(ret), K(log_basic)); } else if (OB_FAIL(local_write_inc_commit_log(log, tx_desc))) { - if (ObDDLUtil::need_remote_write(ret) && allow_remote_write) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("fail to switch to remote write", K(ret), K(tablet_id_)); - } - } else { - LOG_WARN("local write inc commit log fail", K(ret), K(tablet_id_)); - } + LOG_WARN("local write inc commit log fail", K(ret), K(tablet_id_)); } } - if (OB_SUCC(ret) && remote_write_) { - if (OB_FAIL(retry_remote_write_inc_commit_log(lob_meta_tablet_id, tx_desc))) { - LOG_WARN("remote write inc commit log fail", K(ret), K(tablet_id_)); - } - } return ret; } @@ -227,8 +206,6 @@ int ObDDLIncRedoLogWriter::wait_inc_redo_log_finish() if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); - } else if (remote_write_) { - // remote write no need to wait local handle } else if (OB_UNLIKELY(!ddl_inc_log_handle_.is_valid())) { // no redo log has been written yet } else { @@ -261,10 +238,10 @@ int ObDDLIncRedoLogWriter::write_inc_start_log_with_retry( LOG_WARN("already timeout", K(ret), K(start_ts)); } else if (OB_FAIL(write_inc_start_log(lob_meta_tablet_id, tx_desc, start_scn))) { LOG_WARN("write inc ddl start log failed", K(ret)); - if (ObDDLIncRedoLogWriter::need_retry(ret, false/*allow_remote_write*/)) { + if (ObDDLIncRedoLogWriter::need_retry(ret)) { usleep(1000L * 1000L); // 1s ++retry_count; - LOG_WARN("retry write ddl inc start log", K(ret), K(ls_id_), K(tablet_id_), K(retry_count)); + LOG_WARN("retry write ddl inc start log", K(ret), K(tablet_id_), K(retry_count)); ret = OB_SUCCESS; } } else { @@ -288,10 +265,10 @@ int ObDDLIncRedoLogWriter::write_inc_redo_log_with_retry( } else if (OB_FAIL(write_inc_redo_log(redo_info, macro_block_id, task_id, tx_desc))) { LOG_WARN("write inc ddl redo log failed", K(ret)); } - if (ObDDLIncRedoLogWriter::need_retry(ret, false/*allow_remote_write*/)) { + if (ObDDLIncRedoLogWriter::need_retry(ret)) { usleep(1000L * 1000L); // 1s ++retry_count; - LOG_WARN("retry write ddl inc redo log", K(ret), K(ls_id_), K(tablet_id_), K(retry_count)); + LOG_WARN("retry write ddl inc redo log", K(ret), K(tablet_id_), K(retry_count)); ret = OB_SUCCESS; } else { break; @@ -302,7 +279,6 @@ int ObDDLIncRedoLogWriter::write_inc_redo_log_with_retry( } int ObDDLIncRedoLogWriter::write_inc_commit_log_with_retry( - const bool allow_remote_write, const ObTabletID &lob_meta_tablet_id, ObTxDesc *tx_desc) { @@ -316,12 +292,12 @@ int ObDDLIncRedoLogWriter::write_inc_commit_log_with_retry( } else if (ObTimeUtility::fast_current_time() - start_ts > timeout_us) { ret = OB_TIMEOUT; LOG_WARN("already timeout", K(ret), K(start_ts)); - } else if (OB_FAIL(write_inc_commit_log(allow_remote_write, lob_meta_tablet_id, tx_desc))) { + } else if (OB_FAIL(write_inc_commit_log(lob_meta_tablet_id, tx_desc))) { LOG_WARN("write inc ddl commit log failed", K(ret)); - if (ObDDLIncRedoLogWriter::need_retry(ret, false/*allow_remote_write*/)) { + if (ObDDLIncRedoLogWriter::need_retry(ret)) { usleep(1000L * 1000L); // 1s ++retry_count; - LOG_WARN("retry write ddl inc commit log", K(ret), K(ls_id_), K(tablet_id_), K(retry_count)); + LOG_WARN("retry write ddl inc commit log", K(ret), K(tablet_id_), K(retry_count)); ret = OB_SUCCESS; } } else { @@ -333,66 +309,31 @@ int ObDDLIncRedoLogWriter::write_inc_commit_log_with_retry( int ObDDLIncRedoLogWriter::get_write_store_ctx_guard( ObTxDesc *tx_desc, - ObStoreCtxGuard &ctx_guard, - storage::ObLS *&ls) + ObStoreCtxGuard &ctx_guard) { int ret = OB_SUCCESS; + ObLS *ls = nullptr; ObStoreCtx &ctx = ctx_guard.get_store_ctx(); - if (OB_NOT_NULL(ls)) { - ls->reset(); - ls = nullptr; - } - if (OB_FAIL(ctx_guard.init(ls_id_))) { - LOG_WARN("ctx_guard init fail", K(ret), K(ls_id_)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ctx_guard.get_ls_handle(), ObLSGetMod::DAS_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(ls_id_)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ctx_guard.init(ls))) { + LOG_WARN("ctx_guard init fail", K(ret)); } else { ctx.ls_ = ls; ctx.timeout_ = DEFAULT_RETRY_TIMEOUT_US; ObDMLBaseParam dml_param; dml_param.snapshot_.init_none_read(); dml_param.spec_seq_no_ = ObTxSEQ(1, 1); - if (OB_FAIL(ls->get_write_store_ctx(*tx_desc, - dml_param.snapshot_, - dml_param.write_flag_, - ctx_guard.get_store_ctx(), - dml_param.spec_seq_no_))) { - LOG_WARN("can not get write store ctx", K(ret), K(ls_id_), K(*tx_desc)); + if (OB_FAIL(ctx_guard.get_ls()->get_write_store_ctx( + *tx_desc, dml_param.snapshot_, dml_param.write_flag_, + ctx_guard.get_store_ctx(), dml_param.spec_seq_no_))) { + LOG_WARN("can not get write store ctx", K(ret), K(*tx_desc)); } } return ret; } -int ObDDLIncRedoLogWriter::switch_to_remote_write() -{ - int ret = OB_SUCCESS; - - share::ObLocationService *location_service = nullptr; - bool is_cache_hit = false; - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location service is null", K(ret), KP(location_service)); - } else if (OB_FAIL(location_service->get(tablet_id_, - INT64_MAX/*expire_renew_time*/, - is_cache_hit, - leader_ls_id_))) { - LOG_WARN("fail to get log stream id", K(ret), K_(tablet_id)); - } else if (OB_FAIL(location_service->get_leader(GCONF.cluster_id, - leader_ls_id_, - true, /*force_renew*/ - leader_addr_))) { - LOG_WARN("get leader failed", K(ret), K(leader_ls_id_)); - } else { - remote_write_ = true; - LOG_INFO("switch to remote write", K(ret), K_(tablet_id), K_(leader_ls_id), K_(leader_addr)); - } - return ret; -} - int ObDDLIncRedoLogWriter::local_write_inc_start_log( ObDDLIncStartLog &log, ObTxDesc *tx_desc, @@ -400,10 +341,9 @@ int ObDDLIncRedoLogWriter::local_write_inc_start_log( { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; ObTabletID lob_meta_tablet_id = log.get_log_basic().get_lob_meta_tablet_id(); ObDDLIncStartClogCb *cb = nullptr; - ObPartTransCtx *trans_ctx = nullptr; + ObTxCtx *trans_ctx = nullptr; ObDDLIncLogHandle handle; const bool is_sync = true; const int64_t abs_timeout_ts = ObClockGenerator::getClock() + DEFAULT_RETRY_TIMEOUT_US; @@ -412,19 +352,19 @@ int ObDDLIncRedoLogWriter::local_write_inc_start_log( if (OB_UNLIKELY(!log.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(log)); - } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard, ls))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls) || !tablet_id_.is_valid()) { + } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard))) { + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); + } else if (!tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KP(ls), K(tablet_id_)); - } else if (OB_FAIL(ls->tablet_freeze(tablet_id_, + LOG_WARN("invalid argument", K(tablet_id_)); + } else if (OB_FAIL(ctx_guard.get_ls()->tablet_freeze(tablet_id_, is_sync, abs_timeout_ts, false, /*need_rewrite_meta*/ ObFreezeSourceFlag::DIRECT_INC_START))) { LOG_WARN("sync tablet freeze failed", K(ret), K(tablet_id_)); } else if (lob_meta_tablet_id.is_valid() && - OB_FAIL(ls->tablet_freeze(lob_meta_tablet_id, + OB_FAIL(ctx_guard.get_ls()->tablet_freeze(lob_meta_tablet_id, is_sync, abs_timeout_ts, false, /*need_rewrite_meta*/ @@ -433,7 +373,7 @@ int ObDDLIncRedoLogWriter::local_write_inc_start_log( } else if (OB_ISNULL(cb = OB_NEW(ObDDLIncStartClogCb, ObMemAttr("DDL_IRLW")))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); - } else if (OB_FAIL(cb->init(ls_id_, log.get_log_basic()))) { + } else if (OB_FAIL(cb->init(log.get_log_basic()))) { LOG_WARN("failed to init cb", K(ret)); } else if (OB_ISNULL(trans_ctx = ctx_guard.get_store_ctx().mvcc_acc_ctx_.tx_ctx_)) { ret = OB_ERR_UNEXPECTED; @@ -468,7 +408,6 @@ int ObDDLIncRedoLogWriter::local_write_inc_redo_log( int ret = OB_SUCCESS; ObDDLRedoLog log; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObDDLIncRedoClogCb *cb = nullptr; @@ -481,9 +420,11 @@ int ObDDLIncRedoLogWriter::local_write_inc_redo_log( LOG_WARN("invalid arguments", K(ret), K(redo_info)); } else if (OB_FAIL(log.init(redo_info))) { LOG_WARN("fail to init DDLRedoLog", K(ret), K(redo_info)); - } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard, ls))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); - } else if (OB_FAIL(ls->get_tablet(redo_info.table_key_.tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard))) { + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); + } else if (OB_FAIL(ctx_guard.get_ls()->get_tablet( + redo_info.table_key_.tablet_id_, tablet_handle, + ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet_handle failed", K(ret), K(redo_info)); } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -493,8 +434,8 @@ int ObDDLIncRedoLogWriter::local_write_inc_redo_log( int tmp_ret = OB_SUCCESS; int64_t real_sleep_us = 0; buffer_size = log.get_serialize_size(); - if (OB_TMP_FAIL(ObDDLCtrlSpeedHandle::get_instance().limit_and_sleep(ls_id_, buffer_size, task_id, checker, real_sleep_us))) { - LOG_WARN("fail to limit and sleep", K(tmp_ret), K(task_id), K(ls_id_), K(buffer_size), K(real_sleep_us)); + if (OB_TMP_FAIL(ObDDLCtrlSpeedHandle::get_instance().limit_and_sleep(buffer_size, task_id, checker, real_sleep_us))) { + LOG_WARN("fail to limit and sleep", K(tmp_ret), K(task_id), K(buffer_size), K(real_sleep_us)); } } @@ -508,10 +449,10 @@ int ObDDLIncRedoLogWriter::local_write_inc_redo_log( } else if (FALSE_IT(pos = 0)) { } else if (OB_FAIL(tmp_redo_info.deserialize(buffer_, buffer_size, pos))) { LOG_WARN("fail to deserialize ddl redo log", K(ret)); - } else if (OB_FAIL(cb->init(ls_id_, tmp_redo_info, macro_block_id, tablet_handle))) { + } else if (OB_FAIL(cb->init(tmp_redo_info, macro_block_id, tablet_handle))) { LOG_WARN("init ddl clog callback failed", K(ret)); } else { - ObPartTransCtx *trans_ctx = nullptr; + ObTxCtx *trans_ctx = nullptr; ObRandom rand; int64_t replay_hint = rand.get(); SCN scn; @@ -541,26 +482,24 @@ int ObDDLIncRedoLogWriter::local_write_inc_commit_log( { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; ObTabletID lob_meta_tablet_id = log.get_log_basic().get_lob_meta_tablet_id(); ObDDLIncCommitClogCb *cb = nullptr; - ObPartTransCtx *trans_ctx = nullptr; + ObTxCtx *trans_ctx = nullptr; SCN base_scn = SCN::min_scn(); SCN scn = SCN::min_scn(); - bool is_external_consistent = false; ObDDLIncLogHandle handle; if (OB_UNLIKELY(!log.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(log)); - } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard, ls))) { - LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_)); + } else if (OB_FAIL(get_write_store_ctx_guard(tx_desc, ctx_guard))) { + LOG_WARN("fail to get_write_store_ctx_guard", K(ret)); } else if (OB_ISNULL(cb = OB_NEW(ObDDLIncCommitClogCb, ObMemAttr("DDL_IRLW")))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); - } else if (OB_FAIL(cb->init(ls_id_, log.get_log_basic()))) { + } else if (OB_FAIL(cb->init(log.get_log_basic()))) { LOG_WARN("failed to init cb", K(ret)); - } else if (OB_FAIL(OB_TS_MGR.get_ts_sync(ObDDLIncLogHandle::DDL_INC_LOG_TIMEOUT, base_scn, is_external_consistent))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts_sync(ObDDLIncLogHandle::DDL_INC_LOG_TIMEOUT, base_scn))) { LOG_WARN("fail to get gts sync", K(ret), K(log)); } else if (OB_ISNULL(trans_ctx = ctx_guard.get_store_ctx().mvcc_acc_ctx_.tx_ctx_)) { ret = OB_ERR_UNEXPECTED; @@ -568,17 +507,8 @@ int ObDDLIncRedoLogWriter::local_write_inc_commit_log( } else if (OB_FAIL(trans_ctx->submit_direct_load_inc_commit_log(log, cb, scn))) { LOG_WARN("fail to submit ddl inc commit log", K(ret), K(log)); } else { - bool need_retry = true; - while (need_retry) { - if (OB_FAIL(OB_TS_MGR.wait_gts_elapse(scn))) { - if (OB_EAGAIN != ret) { - LOG_WARN("fail to wait gts elapse", K(ret), K(log)); - } else { - ob_usleep(1000); - } - } else { - need_retry = false; - } + if (OB_FAIL(OB_TS_MGR.wait_gts_elapse(scn))) { + LOG_WARN("fail to wait gts elapse", K(ret), K(log)); } { common::ObTimeGuard timeguard("write_inc_commit_log wait", 5 * 1000 * 1000); // 5s @@ -600,78 +530,6 @@ int ObDDLIncRedoLogWriter::local_write_inc_commit_log( return ret; } -int ObDDLIncRedoLogWriter::retry_remote_write_inc_commit_log( - const ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc) -{ - int ret = OB_SUCCESS; - int retry_cnt = 0; - const int64_t MAX_REMOTE_WRITE_RETRY_CNT = 800; - while (OB_SUCC(ret)) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("flush ls leader location failed", K(ret)); - } else if (OB_FAIL(remote_write_inc_commit_log(lob_meta_tablet_id, tx_desc))) { - if (OB_NOT_MASTER == ret && retry_cnt++ < MAX_REMOTE_WRITE_RETRY_CNT) { - ob_usleep(10 * 1000); // 10 ms. - ret = OB_SUCCESS; - } else { - LOG_WARN("remote write ddl inc commit log failed", K(ret), K_(leader_ls_id), K_(leader_addr)); - } - } else { - break; - } - } - - return ret; -} - -int ObDDLIncRedoLogWriter::remote_write_inc_commit_log( - const ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc) -{ - int ret = OB_SUCCESS; - obcall::ObCallRemoteWriteDDLIncCommitLogArg arg; - obcall::ObCallRemoteWriteDDLIncCommitLogRes res; - if (OB_FAIL(arg.init(leader_ls_id_, tablet_id_, lob_meta_tablet_id, tx_desc))) { - LOG_WARN("fail to init ObCallRemoteWriteDDLIncCommitLogArg", K(ret)); - } else if (OB_FAIL(ex_rpc::sync_call([&]() -> int { - if (OB_UNLIKELY(!arg.is_valid())) return OB_INVALID_ARGUMENT; - int r = OB_SUCCESS; - MOD_SCOPE { - ObRole role = INVALID_ROLE; - ObDDLIncRedoLogWriter sstable_redo_writer; - ObLSService *ls_service = share::g_mp->ls_service(); - ObTransService *trans_service = share::g_mp->trans_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - r = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (ObRole::LEADER != role) { - r = OB_NOT_MASTER; - LOG_INFO("not leader", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(sstable_redo_writer.init(arg.ls_id_, arg.tablet_id_))) { - LOG_WARN("init sstable redo writer", K(ret), K(arg.tablet_id_)); - } else if (OB_FAIL(sstable_redo_writer.write_inc_commit_log_with_retry( - false, arg.lob_meta_tablet_id_, arg.tx_desc_))) { - LOG_ERROR("fail to write inc commit log", K(ret), K(arg)); - } else if (OB_FAIL(trans_service->get_tx_exec_result(*arg.tx_desc_, res.tx_result_))) { - LOG_WARN("fail to get_tx_exec_result", K(ret), K(arg)); - } - } - return r; - }))) { - LOG_WARN("write inc commit log failed", K(ret), K_(leader_ls_id), K_(leader_addr)); - } else if (OB_FAIL(share::g_mp->trans_service()->add_tx_exec_result(*arg.tx_desc_, res.tx_result_))) { - LOG_WARN("fail to get_tx_exec_result", K(ret), K(*arg.tx_desc_)); - } - return ret; -} - ObDDLIncRedoLogWriterCallback::ObDDLIncRedoLogWriterCallback() : is_inited_(false), redo_info_(), @@ -702,7 +560,7 @@ int ObDDLIncRedoLogWriterCallback::init(ObDDLRedoLogWriterCallbackInitParam &par } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(param)); - } else if (OB_FAIL(ddl_inc_writer_.init(param.ls_id_, param.tablet_id_))) { + } else if (OB_FAIL(ddl_inc_writer_.init(param.tablet_id_))) { LOG_WARN("fail to init ddl_inc_writer_", K(ret), K(param)); } else { block_type_ = param.block_type_; diff --git a/src/storage/ddl/ob_ddl_inc_redo_log_writer.h b/src/storage/ddl/ob_ddl_inc_redo_log_writer.h index 323c29493..8e58ad5d4 100644 --- a/src/storage/ddl/ob_ddl_inc_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_inc_redo_log_writer.h @@ -48,11 +48,9 @@ class ObDDLIncRedoLogWriter final static const int64_t DEFAULT_RETRY_TIMEOUT_US = 60L * 1000L * 1000L; // 1min ObDDLIncRedoLogWriter(); ~ObDDLIncRedoLogWriter(); - int init( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id); + int init(const ObTabletID &tablet_id); void reset(); - static bool need_retry(int ret_code, bool allow_remote_write); + static bool need_retry(int ret_code); int write_inc_start_log_with_retry( const ObTabletID &lob_meta_tablet_id, transaction::ObTxDesc *tx_desc, @@ -64,7 +62,6 @@ class ObDDLIncRedoLogWriter final transaction::ObTxDesc *tx_desc); int wait_inc_redo_log_finish(); int write_inc_commit_log_with_retry( - const bool allow_remote_write, const ObTabletID &lob_meta_tablet_id, transaction::ObTxDesc *tx_desc); private: @@ -78,14 +75,11 @@ class ObDDLIncRedoLogWriter final const int64_t task_id, transaction::ObTxDesc *tx_desc); int write_inc_commit_log( - const bool allow_remote_write, const ObTabletID &lob_meta_tablet_id, transaction::ObTxDesc *tx_desc); int get_write_store_ctx_guard( transaction::ObTxDesc *tx_desc, - ObStoreCtxGuard &ctx_guard, - storage::ObLS *&ls); - int switch_to_remote_write(); + ObStoreCtxGuard &ctx_guard); int local_write_inc_start_log( ObDDLIncStartLog &log, transaction::ObTxDesc *tx_desc, @@ -98,20 +92,10 @@ class ObDDLIncRedoLogWriter final int local_write_inc_commit_log( ObDDLIncCommitLog &log, transaction::ObTxDesc *tx_desc); - int retry_remote_write_inc_commit_log( - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc); - int remote_write_inc_commit_log( - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc); private: bool is_inited_; - bool remote_write_; - share::ObLSID ls_id_; ObTabletID tablet_id_; ObDDLIncLogHandle ddl_inc_log_handle_; - ObAddr leader_addr_; - share::ObLSID leader_ls_id_; char *buffer_; }; diff --git a/src/storage/ddl/ob_ddl_inc_task.cpp b/src/storage/ddl/ob_ddl_inc_task.cpp index 66c398282..dd5dfc0d7 100644 --- a/src/storage/ddl/ob_ddl_inc_task.cpp +++ b/src/storage/ddl/ob_ddl_inc_task.cpp @@ -51,7 +51,7 @@ int ObDDLIncStartTask::generate_next_task(ObITask *&next_task) next_task = nullptr; ObDDLIndependentDag *dag = static_cast(dag_); const int64_t next_tablet_idx = tablet_idx_ + 1; - if (next_tablet_idx >= dag->get_ls_tablet_ids().count()) { + if (next_tablet_idx >= dag->get_tablet_ids().count()) { ret = OB_ITER_END; } else { ObDDLIncStartTask *start_task = nullptr; @@ -68,16 +68,14 @@ int ObDDLIncStartTask::process() { int ret = OB_SUCCESS; ObDDLIndependentDag *dag = static_cast(dag_); - const std::pair &ls_tablet_id = - dag->get_ls_tablet_ids().at(tablet_idx_); + const ObTabletID &tablet_id = dag->get_tablet_ids().at(tablet_idx_); ObDDLTabletContext *tablet_ctx = nullptr; ObDDLIncRedoLogWriter redo_writer; SCN start_scn; - if (OB_FAIL(dag->get_tablet_context(ls_tablet_id.second, tablet_ctx))) { - LOG_WARN("fail to get tablet context", KR(ret), K(ls_tablet_id.second)); - } else if (OB_FAIL(redo_writer.init(ls_tablet_id.first, - ls_tablet_id.second))) { - LOG_WARN("fail to init inc redo writer", KR(ret), K(ls_tablet_id.first), K(ls_tablet_id.second), + if (OB_FAIL(dag->get_tablet_context(tablet_id, tablet_ctx))) { + LOG_WARN("fail to get tablet context", KR(ret), K(tablet_id)); + } else if (OB_FAIL(redo_writer.init(tablet_id))) { + LOG_WARN("fail to init inc redo writer", KR(ret), K(tablet_id), K(dag->get_direct_load_type()), K(dag->get_tx_info().trans_id_), K(dag->get_tx_info().seq_no_), K(dag->is_inc_major_log())); } else if (OB_FAIL(redo_writer.write_inc_start_log_with_retry(tablet_ctx->lob_meta_tablet_id_, dag->get_tx_info().tx_desc_, @@ -110,7 +108,7 @@ int ObDDLIncCommitTask::generate_next_task(ObITask *&next_task) const int64_t next_tablet_idx = tablet_idx_ + 1; if (tablet_idx_ == -1) { ret = OB_ITER_END; - } else if (next_tablet_idx >= dag->get_ls_tablet_ids().count()) { + } else if (next_tablet_idx >= dag->get_tablet_ids().count()) { ret = OB_ITER_END; } else { ObDDLIncCommitTask *commit_task = nullptr; @@ -128,20 +126,16 @@ int ObDDLIncCommitTask::process() int ret = OB_SUCCESS; ObDDLIndependentDag *dag = static_cast(dag_); if (tablet_idx_ != -1) { - const std::pair &ls_tablet_id = - dag->get_ls_tablet_ids().at(tablet_idx_); - tablet_id_ = ls_tablet_id.second; + tablet_id_ = dag->get_tablet_ids().at(tablet_idx_); } ObDDLTabletContext *tablet_ctx = nullptr; ObDDLIncRedoLogWriter redo_writer; if (OB_FAIL(dag->get_tablet_context(tablet_id_, tablet_ctx))) { LOG_WARN("fail to get tablet context", KR(ret), K(tablet_id_)); - } else if (OB_FAIL(redo_writer.init(tablet_ctx->ls_id_, - tablet_id_))) { - LOG_WARN("fail to init inc redo writer", KR(ret), K(tablet_ctx->ls_id_), K(tablet_id_), + } else if (OB_FAIL(redo_writer.init(tablet_id_))) { + LOG_WARN("fail to init inc redo writer", KR(ret), K(tablet_id_), K(dag->get_direct_load_type()), K(dag->get_tx_info().trans_id_), K(dag->get_tx_info().seq_no_), K(dag->is_inc_major_log())); - } else if (OB_FAIL(redo_writer.write_inc_commit_log_with_retry(true /*allow_remote_write*/, - tablet_ctx->lob_meta_tablet_id_, + } else if (OB_FAIL(redo_writer.write_inc_commit_log_with_retry(tablet_ctx->lob_meta_tablet_id_, dag->get_tx_info().tx_desc_))) { LOG_WARN("fail to write inc commit log", KR(ret), KPC(tablet_ctx), K(dag->get_tx_info())); } diff --git a/src/storage/ddl/ob_ddl_independent_dag.cpp b/src/storage/ddl/ob_ddl_independent_dag.cpp index 4c7e28eab..f6b9c7477 100644 --- a/src/storage/ddl/ob_ddl_independent_dag.cpp +++ b/src/storage/ddl/ob_ddl_independent_dag.cpp @@ -70,7 +70,7 @@ void ObDDLIndependentDag::reuse() ObTabletObjLoadHelper::free(arena_, ddl_table_schema_.lob_meta_storage_schema_); ddl_table_schema_.reset(); tx_info_.reset(); - ls_tablet_ids_.reset(); + tablet_ids_.reset(); FOREACH(tc_it, tablet_context_map_) { ObDDLTabletContext *tablet_context = tc_it->second; free_tablet_context(arena_, tablet_context); @@ -92,8 +92,8 @@ int ObDDLIndependentDag::init_by_param(const share::ObIDagInitParam *param) } else if (init_param->ddl_task_param_.tenant_data_version_ < DDL_IDEM_DATA_FORMAT_VERSION) { ret = OB_NOT_SUPPORTED; LOG_WARN("reject execute dag when request comes from old version", K(ret), KPC(init_param)); - } else if (OB_FAIL(ls_tablet_ids_.assign(init_param->ls_tablet_ids_))) { - LOG_WARN("assign ls tablet id array failed", K(ret), K(init_param->ls_tablet_ids_)); + } else if (OB_FAIL(tablet_ids_.assign(init_param->tablet_ids_))) { + LOG_WARN("assign tablet id array failed", K(ret), K(init_param->tablet_ids_)); } else { direct_load_type_ = init_param->direct_load_type_; ddl_thread_count_ = init_param->ddl_thread_count_; @@ -108,7 +108,7 @@ int ObDDLIndependentDag::init_by_param(const share::ObIDagInitParam *param) is_inited_ = true; } } - FLOG_INFO("ddl independent dag init", K(ret), KPC(this), K(ddl_table_schema_), K(tx_info_), K(ls_tablet_ids_), K(tablet_context_map_.size())); + FLOG_INFO("ddl independent dag init", K(ret), KPC(this), K(ddl_table_schema_), K(tx_info_), K(tablet_ids_), K(tablet_context_map_.size())); return ret; } @@ -124,18 +124,17 @@ int ObDDLIndependentDag::init_ddl_table_schema() int ObDDLIndependentDag::init_tablet_context_map() { int ret = OB_SUCCESS; - if (OB_FAIL(tablet_context_map_.create(ls_tablet_ids_.count(), ObMemAttr("ddl_dag_ctx_map")))) { - LOG_WARN("create tablet context map failed", K(ret), K(ls_tablet_ids_.count())); + if (OB_FAIL(tablet_context_map_.create(tablet_ids_.count(), ObMemAttr("ddl_dag_ctx_map")))) { + LOG_WARN("create tablet context map failed", K(ret), K(tablet_ids_.count())); } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablet_ids_.count(); ++i) { - const ObLSID &ls_id = ls_tablet_ids_.at(i).first; - const ObTabletID &tablet_id = ls_tablet_ids_.at(i).second; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids_.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids_.at(i); ObDDLTabletContext *tablet_context = nullptr; if (OB_ISNULL(tablet_context = OB_NEWx(ObDDLTabletContext, &arena_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory for tablet context failed", K(ret)); - } else if (OB_FAIL(tablet_context->init(ls_id, tablet_id, ddl_thread_count_, ddl_task_param_.snapshot_version_, direct_load_type_, ddl_table_schema_))) { - LOG_WARN("init ddl tablet context failed", K(ret), K(ls_id), K(tablet_id), K(ddl_thread_count_)); + } else if (OB_FAIL(tablet_context->init(tablet_id, ddl_thread_count_, ddl_task_param_.snapshot_version_, direct_load_type_, ddl_table_schema_))) { + LOG_WARN("init ddl tablet context failed", K(ret), K(tablet_id), K(ddl_thread_count_)); } else if (use_tablet_mode() && OB_FAIL(alloc_task(tablet_context->scan_task_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc tablet scan task failed", K(ret), K(tablet_id)); @@ -172,9 +171,9 @@ int ObDDLIndependentDag::schedule_tablet_merge_task() ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablet_ids_.count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids_.count(); ++i) { share::SCN mock_start_scn; - const ObTabletID &tablet_id = ls_tablet_ids_.at(i).second; + const ObTabletID &tablet_id = tablet_ids_.at(i); ObDDLTabletContext *tablet_context = nullptr; @@ -362,13 +361,13 @@ int ObDDLIndependentDag::add_vector_index_append_pipeline(const ObIndexType &ind int ObDDLIndependentDag::alloc_vector_index_write_and_build_pipeline( const ObIndexType &index_type, - const ObIArray> &ls_tablet_ids, + const ObIArray &tablet_ids, ObIArray &vector_index_task_array) { int ret = OB_SUCCESS; vector_index_task_array.reuse(); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablet_ids.count(); ++i) { - const ObTabletID &tablet_id = ls_tablet_ids.at(i).second; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); ObITask *vector_index_task = nullptr; if (schema::is_vec_index_snapshot_data_type(index_type)) { ObHNSWBuildAndWritePipeline *pipeline = nullptr; @@ -472,12 +471,11 @@ int ObDDLIndependentDag::check_is_first_ddl_kv(bool &is_first) { int ret = OB_SUCCESS; is_first = true; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablet_ids_.count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids_.count(); ++i) { ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletMapKey key; - key.ls_id_ = ls_tablet_ids_.at(i).first; - key.tablet_id_ = ls_tablet_ids_.at(i).second; + key.tablet_id_ = tablet_ids_.at(i); bool tmp_is_first = false; ObDDLKvMgrHandle ddl_kv_mgr_handle; @@ -620,7 +618,7 @@ int ObDDLIndependentDag::full_generate_write_macro_block_tasks(ObIArray &dat int ret = OB_SUCCESS; data_merge_tasks.reset(); lob_merge_tasks.reset(); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablet_ids_.count(); ++i) { - const ObTabletID &tablet_id = ls_tablet_ids_.at(i).second; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids_.count(); ++i) { + const ObTabletID &tablet_id = tablet_ids_.at(i); ObITask *data_merge_task = nullptr; ObITask *lob_merge_task = nullptr; if (OB_FAIL(init_tablet_merge_task(tablet_id, for_major, data_merge_task, lob_merge_task))) { diff --git a/src/storage/ddl/ob_ddl_independent_dag.h b/src/storage/ddl/ob_ddl_independent_dag.h index 8a54a97b0..11f437576 100644 --- a/src/storage/ddl/ob_ddl_independent_dag.h +++ b/src/storage/ddl/ob_ddl_independent_dag.h @@ -41,7 +41,7 @@ struct ObDDLIndependentDagInitParam : public share::ObIDagInitParam ddl_thread_count_(other.ddl_thread_count_), ddl_task_param_(other.ddl_task_param_), tx_info_(other.tx_info_), - ls_tablet_ids_(other.ls_tablet_ids_), + tablet_ids_(other.tablet_ids_), is_inc_major_log_(other.is_inc_major_log_) {} virtual bool is_valid() const override { @@ -49,16 +49,16 @@ struct ObDDLIndependentDagInitParam : public share::ObIDagInitParam ddl_thread_count_ > 0 && ddl_task_param_.is_valid() && (!is_incremental_direct_load(direct_load_type_) || tx_info_.is_valid()) && - ls_tablet_ids_.count() > 0; + tablet_ids_.count() > 0; } - VIRTUAL_TO_STRING_KV(K(direct_load_type_), K(ddl_thread_count_), K(ddl_task_param_), K(ls_tablet_ids_), K(is_inc_major_log_)); + VIRTUAL_TO_STRING_KV(K(direct_load_type_), K(ddl_thread_count_), K(ddl_task_param_), K(tablet_ids_), K(is_inc_major_log_)); public: ObDirectLoadType direct_load_type_; int64_t ddl_thread_count_; ObDDLTaskParam ddl_task_param_; ObDirectLoadTxInfo tx_info_; - ObArray> ls_tablet_ids_; + ObArray tablet_ids_; bool is_inc_major_log_; }; @@ -75,7 +75,7 @@ class ObDDLIndependentDag : public share::ObIndependentDag const ObDDLTaskParam &get_ddl_task_param() const { return ddl_task_param_; } const ObDDLTableSchema &get_ddl_table_schema() const { return ddl_table_schema_; } const ObDirectLoadTxInfo &get_tx_info() const { return tx_info_; } - const ObIArray> &get_ls_tablet_ids() { return ls_tablet_ids_; } + const ObIArray &get_tablet_ids() { return tablet_ids_; } int64_t get_pipeline_count() const { return ATOMIC_LOAD(&pipeline_count_); } void inc_pipeline_count() { ATOMIC_INC(&pipeline_count_); } void dec_pipeline_count() { ATOMIC_DEC(&pipeline_count_); } @@ -97,7 +97,7 @@ class ObDDLIndependentDag : public share::ObIndependentDag protected: int alloc_vector_index_write_and_build_pipeline( const ObIndexType &index_type, - const ObIArray> &ls_tablet_ids, + const ObIArray &tablet_ids, ObIArray &vector_index_task_array); private: int init_ddl_table_schema(); @@ -128,7 +128,7 @@ class ObDDLIndependentDag : public share::ObIndependentDag ObDDLTaskParam ddl_task_param_; ObDDLTableSchema ddl_table_schema_; ObDirectLoadTxInfo tx_info_; - ObArray> ls_tablet_ids_; + ObArray tablet_ids_; hash::ObHashMap tablet_context_map_; int64_t pipeline_count_; int ret_code_; diff --git a/src/storage/ddl/ob_ddl_insert_dag.cpp b/src/storage/ddl/ob_ddl_insert_dag.cpp index cca681d50..1a91511c8 100644 --- a/src/storage/ddl/ob_ddl_insert_dag.cpp +++ b/src/storage/ddl/ob_ddl_insert_dag.cpp @@ -115,11 +115,11 @@ int ObDDLInsertDag::update_tablet_range_count() LOG_WARN("no partition range", K(ret), K(ddl_slice_info)); } else if (1 == part_ranges.count() && 0 == part_ranges.at(0).tablet_id_) { // for unpartitioned table, there is only one tablet and its tablet id is 0 - if (ls_tablet_ids_.count() != 1) { + if (tablet_ids_.count() != 1) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet count not match", K(ret), K(part_ranges), K(ls_tablet_ids_)); + LOG_WARN("tablet count not match", K(ret), K(part_ranges), K(tablet_ids_)); } else { - const ObTabletID &tablet_id = ls_tablet_ids_.at(0).second; + const ObTabletID &tablet_id = tablet_ids_.at(0); total_slice_count_ = part_ranges.at(0).range_cut_.count() + 1; ObDDLTabletContext *tablet_context = nullptr; if (OB_FAIL(get_tablet_context(tablet_id, tablet_context))) { diff --git a/src/storage/ddl/ob_ddl_lock.cpp b/src/storage/ddl/ob_ddl_lock.cpp index 8764e26ec..3d29fc85e 100644 --- a/src/storage/ddl/ob_ddl_lock.cpp +++ b/src/storage/ddl/ob_ddl_lock.cpp @@ -16,7 +16,7 @@ #define USING_LOG_PREFIX STORAGE -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "storage/ddl/ob_ddl_lock.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" #include "storage/tablelock/ob_lock_utils.h" @@ -25,7 +25,6 @@ #include "common/mysqlclient/ob_mysql_result.h" using namespace oceanbase::transaction::tablelock; -using oceanbase::share::ObLSID; using oceanbase::share::schema::ObTableSchema; using oceanbase::observer::ObInnerSQLConnection; @@ -125,11 +124,7 @@ int ObDDLLock::lock_for_add_drop_index( LOG_WARN("failed to lock data table tablet", K(ret)); } else if (OB_FAIL(do_table_lock(data_table_id, data_tablet_ids, ROW_SHARE, lock_owner, timeout_us, true/*is_lock*/, trans))) { LOG_WARN("failed to lock data tablet", K(ret)); - } else if (index_schema.is_storage_local_index_table() || index_schema.is_mlog_table()) { - if (OB_FAIL(check_tablet_in_same_ls(data_table_schema, index_schema, trans))) { - LOG_WARN("failed to check tablet in same ls", K(ret)); - } - } else { + } else if (!index_schema.is_storage_local_index_table()) { if (OB_FAIL(ObOnlineDDLLock::lock_table(index_table_id, EXCLUSIVE, lock_owner, timeout_us, trans))) { LOG_WARN("failed to lock index table", K(ret)); } @@ -240,63 +235,65 @@ int ObDDLLock::unlock_for_rebuild_index( return ret; } -int ObDDLLock::lock_for_split_partition( +int ObDDLLock::lock_src_and_dst_for_split_partition( const ObTableSchema &table_schema, - const ObLSID *ls_id, - const ObIArray *src_tablet_ids, + const ObIArray &src_tablet_ids, const ObIArray &dst_tablet_ids, const ObTableLockOwnerID lock_owner, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - const uint64_t table_id = table_schema.get_table_id(); const int64_t timeout_us = DEFAULT_TIMEOUT; - const bool lock_src_and_dst_od = nullptr == ls_id && nullptr != src_tablet_ids; - const bool lock_dst_table_lock = nullptr != ls_id && nullptr == src_tablet_ids && ls_id->is_valid(); - if (OB_UNLIKELY(!(lock_src_and_dst_od ^ lock_dst_table_lock))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), KPC(ls_id), KPC(src_tablet_ids), K(dst_tablet_ids)); - } else if (table_schema.is_global_index_table()) { - if (lock_src_and_dst_od) { - if (OB_FAIL(ObOnlineDDLLock::lock_table(table_id, ROW_SHARE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock data table", K(ret)); - } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(*src_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock tablets", K(ret)); - } else if (OB_FAIL(do_table_lock(table_schema.get_data_table_id(), ROW_SHARE, lock_owner, timeout_us, true/*is_lock*/, trans))) { - LOG_WARN("failed to lock data table", K(ret)); - } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(dst_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock tablets", K(ret)); - } + if (table_schema.is_global_index_table()) { + if (OB_FAIL(ObOnlineDDLLock::lock_table(table_id, ROW_SHARE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock data table", K(ret)); + } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(src_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock tablets", K(ret)); + } else if (OB_FAIL(do_table_lock(table_schema.get_data_table_id(), ROW_SHARE, lock_owner, timeout_us, true/*is_lock*/, trans))) { + LOG_WARN("failed to lock data table", K(ret)); + } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(dst_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock tablets", K(ret)); } } else if (need_lock(table_schema)) { - if (lock_src_and_dst_od) { - if (OB_FAIL(ObOnlineDDLLock::lock_table(table_id, ROW_SHARE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock data table", K(ret)); - } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(*src_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock tablets", K(ret)); - } else if (OB_FAIL(do_table_lock(table_id, *src_tablet_ids, ROW_EXCLUSIVE, lock_owner, timeout_us, true/*is_lock*/, trans))) { - LOG_WARN("failed to lock data table", K(ret)); - } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(dst_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { - LOG_WARN("failed to lock tablets", K(ret)); - } - } else { - // performance crucial - ObLockAloneTabletRequest arg; - arg.lock_mode_ = ROW_EXCLUSIVE; - arg.op_type_ = ObTableLockOpType::OUT_TRANS_LOCK; - arg.owner_id_ = lock_owner; - arg.timeout_us_ = timeout_us; - arg.ls_id_ = *ls_id; - ObInnerSQLConnection *iconn = nullptr; - if (OB_ISNULL(iconn = static_cast(trans.get_connection()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid conn", K(ret)); - } else if (OB_FAIL(append(arg.tablet_ids_, dst_tablet_ids))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(ObInnerConnectionLockUtil::lock_tablet(arg, iconn))) { - LOG_WARN("failed to lock tablet", K(ret), K(arg)); - } + if (OB_FAIL(ObOnlineDDLLock::lock_table(table_id, ROW_SHARE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock data table", K(ret)); + } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(src_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock tablets", K(ret)); + } else if (OB_FAIL(do_table_lock(table_id, src_tablet_ids, ROW_EXCLUSIVE, lock_owner, timeout_us, true/*is_lock*/, trans))) { + LOG_WARN("failed to lock data table", K(ret)); + } else if (OB_FAIL(ObOnlineDDLLock::lock_tablets(dst_tablet_ids, EXCLUSIVE, lock_owner, timeout_us, trans))) { + LOG_WARN("failed to lock tablets", K(ret)); + } + } else { + LOG_INFO("skip ddl lock", K(ret), K(table_id)); + } + return ret; +} + +int ObDDLLock::lock_dst_for_split_partition( + const ObTableSchema &table_schema, + const ObIArray &dst_tablet_ids, + const ObTableLockOwnerID lock_owner, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const uint64_t table_id = table_schema.get_table_id(); + if (table_schema.is_global_index_table()) { + } else if (need_lock(table_schema)) { + ObLockAloneTabletRequest arg; + arg.lock_mode_ = ROW_EXCLUSIVE; + arg.op_type_ = ObTableLockOpType::OUT_TRANS_LOCK; + arg.owner_id_ = lock_owner; + arg.timeout_us_ = DEFAULT_TIMEOUT; + ObInnerSQLConnection *iconn = static_cast(trans.get_connection()); + if (OB_ISNULL(iconn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid conn", K(ret)); + } else if (OB_FAIL(append(arg.tablet_ids_, dst_tablet_ids))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(ObInnerConnectionLockUtil::lock_tablet(arg, iconn))) { + LOG_WARN("failed to lock tablet", K(ret), K(arg)); } } else { LOG_INFO("skip ddl lock", K(ret), K(table_id)); @@ -701,7 +698,6 @@ int ObDDLLock::lock_dst_table_and_global_indexes_for_fork( ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - const ObLSID ls_id(share::SYS_LS); const int64_t timeout_us = DEFAULT_TIMEOUT; ObSEArray data_tablet_ids; ObSEArray all_tablet_ids; @@ -756,7 +752,6 @@ int ObDDLLock::lock_dst_table_and_global_indexes_for_fork( arg.op_type_ = ObTableLockOpType::OUT_TRANS_LOCK; arg.owner_id_ = lock_owner; arg.timeout_us_ = timeout_us; - arg.ls_id_ = ls_id; ObInnerSQLConnection *iconn = nullptr; if (OB_ISNULL(iconn = static_cast(trans.get_connection()))) { ret = OB_ERR_UNEXPECTED; @@ -944,11 +939,8 @@ int ObDDLLock::lock_for_offline_ddl( LOG_INFO("skip ddl lock for non-user table", K(table_id)); } else if (OB_FAIL(do_table_lock(table_id, EXCLUSIVE, lock_owner, timeout_us, true/*is_lock*/, trans))) { LOG_WARN("failed to lock table lock", K(ret)); - } else if (nullptr != hidden_table_schema_to_check_bind) { - if (OB_FAIL(check_tablet_in_same_ls(table_schema, *hidden_table_schema_to_check_bind, trans))) { - LOG_WARN("failed to check tablet in same ls", K(ret)); - } } + UNUSED(hidden_table_schema_to_check_bind); return ret; } @@ -1087,96 +1079,29 @@ int ObDDLLock::do_table_lock(const uint64_t table_id, } if (OB_SUCC(ret)) { - ObArray ls_ids; - ObArray lock_args; - ObArray unlock_args; - int64_t arg_count = 0; - if (OB_FAIL(share::ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("failed to get tablet ls", K(ret)); - } else if (OB_UNLIKELY(ls_ids.count() != tablet_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls ids count", K(ret), K(ls_ids.count()), K(tablet_ids.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - const ObLSID &ls_id = ls_ids[i]; - int64_t j = 0; - for (; j < (is_lock ? lock_args.count() : unlock_args.count()); j++) { - const ObLSID &tmp_ls_id = is_lock ? lock_args[j].ls_id_ : unlock_args[j].ls_id_; - if (tmp_ls_id == ls_id) { - break; - } - } - arg_count = is_lock ? lock_args.count() : unlock_args.count(); - if (j == arg_count) { - ObLockAloneTabletRequest lock_arg; - ObUnLockAloneTabletRequest unlock_arg; - ObLockAloneTabletRequest &arg = is_lock ? lock_arg : unlock_arg; - arg.owner_id_ = lock_owner; - arg.lock_mode_ = lock_mode; - arg.op_type_ = op_type; - arg.timeout_us_ = timeout_us; - arg.ls_id_ = ls_id; - if (OB_FAIL(is_lock ? lock_args.push_back(lock_arg) : unlock_args.push_back(unlock_arg))) { - LOG_WARN("failed to push back modify arg", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObLockAloneTabletRequest &arg = is_lock ? lock_args.at(j) : unlock_args.at(j); - if (OB_FAIL(arg.tablet_ids_.push_back(tablet_ids.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } - } - } - arg_count = is_lock ? lock_args.count() : unlock_args.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < arg_count; i++) { - if (is_lock) { - if (OB_FAIL(ObInnerConnectionLockUtil::lock_tablet(lock_args[i], iconn))) { - LOG_WARN("failed to lock tablet", K(ret), K(lock_args[i])); - } + ObLockAloneTabletRequest lock_arg; + ObUnLockAloneTabletRequest unlock_arg; + ObLockAloneTabletRequest &arg = is_lock ? lock_arg : unlock_arg; + arg.owner_id_ = lock_owner; + arg.lock_mode_ = lock_mode; + arg.op_type_ = op_type; + arg.timeout_us_ = timeout_us; + if (OB_FAIL(append(arg.tablet_ids_, tablet_ids))) { + LOG_WARN("failed to append tablet ids", K(ret)); + } else if (is_lock && OB_FAIL(ObInnerConnectionLockUtil::lock_tablet(lock_arg, iconn))) { + LOG_WARN("failed to lock tablet", K(ret), K(lock_arg)); + } else if (!is_lock && OB_FAIL(ObInnerConnectionLockUtil::unlock_tablet(unlock_arg, iconn))) { + if (OB_OBJ_LOCK_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("table lock already unlocked", K(ret), K(unlock_arg)); } else { - if (OB_FAIL(ObInnerConnectionLockUtil::unlock_tablet(unlock_args[i], iconn))) { - if (OB_OBJ_LOCK_NOT_EXIST == ret) { - ret = OB_SUCCESS; - LOG_INFO("table lock already unlocked", K(ret), K(unlock_args[i])); - } else { - LOG_WARN("failed to unlock tablet", K(ret)); - } - } + LOG_WARN("failed to unlock tablet", K(ret)); } } } return ret; } -int ObDDLLock::check_tablet_in_same_ls( - const ObTableSchema &lhs_schema, - const ObTableSchema &rhs_schema, - ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - - ObSArray lhs_tablet_ids; - ObSArray rhs_tablet_ids; - ObSArray lhs_ls_ids; - ObSArray rhs_ls_ids; - if (OB_UNLIKELY(false)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant id not match", K(ret)); - } else if (OB_FAIL(lhs_schema.get_tablet_ids(lhs_tablet_ids))) { - LOG_WARN("failed to get orig tablet ids", K(ret)); - } else if (OB_FAIL(rhs_schema.get_tablet_ids(rhs_tablet_ids))) { - LOG_WARN("failed to get index tablet ids", K(ret)); - } else if (OB_FAIL(share::ObTabletToLSTableOperator::batch_get_ls(trans, lhs_tablet_ids, lhs_ls_ids))) { - LOG_WARN("failed to get data tablet ls", K(ret)); - } else if (OB_FAIL(share::ObTabletToLSTableOperator::batch_get_ls(trans, rhs_tablet_ids, rhs_ls_ids))) { - LOG_WARN("failed to get data tablet ls", K(ret)); - } else if (!is_array_equal(lhs_ls_ids, rhs_ls_ids)) { - ret = OB_EAGAIN; - LOG_WARN("tablet not in same ls", K(ret), K(lhs_ls_ids), K(rhs_ls_ids)); - } - return ret; -} - int ObDDLLock::replace_tablet_lock(const uint64_t table_id, const ObIArray &tablet_ids, @@ -1190,79 +1115,46 @@ int ObDDLLock::replace_tablet_lock(const uint64_t table_id, int ret = OB_SUCCESS; const ObTableLockOpType op_type = ObTableLockOpType::OUT_TRANS_UNLOCK; ObInnerSQLConnection *iconn = nullptr; - ObArray unlock_args; + ObUnLockAloneTabletRequest unlock_arg; if (OB_ISNULL(iconn = static_cast(trans.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid conn", K(ret)); } else if (OB_UNLIKELY(false || OB_INVALID_ID == table_id || tablet_ids.count() < 1)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(table_id), K(tablet_ids.count())); - } else if (OB_FAIL(get_unlock_alone_tablet_request_args(old_lock_mode, old_lock_owner, timeout_us, tablet_ids, unlock_args, trans))) { - LOG_WARN("fail to get unlock alone tablet request args", K(ret), K(old_lock_mode), K(old_lock_owner), K(timeout_us), K(tablet_ids)); + } else if (OB_FAIL(get_unlock_alone_tablet_request_arg(old_lock_mode, old_lock_owner, timeout_us, tablet_ids, unlock_arg))) { + LOG_WARN("fail to get unlock alone tablet request arg", K(ret), K(old_lock_mode), K(old_lock_owner), K(timeout_us), K(tablet_ids)); } else { - int64_t arg_count = unlock_args.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < arg_count; i++) { - ObReplaceLockRequest replace_req; - replace_req.new_lock_mode_ = new_lock_mode; - replace_req.new_lock_owner_ = new_lock_owner; - replace_req.unlock_req_ = &unlock_args[i]; - if (OB_FAIL(ObInnerConnectionLockUtil::replace_lock(replace_req, iconn))) { - LOG_WARN("failed to replace lock", K(ret), K(replace_req)); - } + ObReplaceLockRequest replace_req; + replace_req.new_lock_mode_ = new_lock_mode; + replace_req.new_lock_owner_ = new_lock_owner; + replace_req.unlock_req_ = &unlock_arg; + if (OB_FAIL(ObInnerConnectionLockUtil::replace_lock(replace_req, iconn))) { + LOG_WARN("failed to replace lock", K(ret), K(replace_req)); } } return ret; } -int ObDDLLock::get_unlock_alone_tablet_request_args(const transaction::tablelock::ObTableLockMode lock_mode, +int ObDDLLock::get_unlock_alone_tablet_request_arg(const transaction::tablelock::ObTableLockMode lock_mode, const transaction::tablelock::ObTableLockOwnerID lock_owner, const int64_t timeout_us, const ObIArray &tablet_ids, - ObArray &unlock_args, - ObMySQLTransaction &trans) + transaction::tablelock::ObUnLockAloneTabletRequest &unlock_arg) { int ret = OB_SUCCESS; - unlock_args.reset(); + unlock_arg.reset(); const ObTableLockOpType op_type = ObTableLockOpType::OUT_TRANS_UNLOCK; if (OB_UNLIKELY(false || tablet_ids.count() < 1)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_ids.count())); } else { - ObArray ls_ids; - int64_t arg_count = 0; - if (OB_FAIL(share::ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("failed to get tablet ls", K(ret)); - } else if (OB_UNLIKELY(ls_ids.count() != tablet_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls ids count", K(ret), K(ls_ids.count()), K(tablet_ids.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - const ObLSID &ls_id = ls_ids[i]; - int64_t j = 0; - for (; j < unlock_args.count(); j++) { - const ObLSID &tmp_ls_id = unlock_args[j].ls_id_; - if (tmp_ls_id == ls_id) { - break; - } - } - arg_count = unlock_args.count(); - if (j == arg_count) { - ObUnLockAloneTabletRequest unlock_arg; - unlock_arg.owner_id_ = lock_owner; - unlock_arg.lock_mode_ = lock_mode; - unlock_arg.op_type_ = op_type; - unlock_arg.timeout_us_ = timeout_us; - unlock_arg.ls_id_ = ls_id; - if (OB_FAIL(unlock_args.push_back(unlock_arg))) { - LOG_WARN("failed to push back modify arg", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObLockAloneTabletRequest &arg = unlock_args.at(j); - if (OB_FAIL(arg.tablet_ids_.push_back(tablet_ids.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } - } + unlock_arg.owner_id_ = lock_owner; + unlock_arg.lock_mode_ = lock_mode; + unlock_arg.op_type_ = op_type; + unlock_arg.timeout_us_ = timeout_us; + if (OB_FAIL(append(unlock_arg.tablet_ids_, tablet_ids))) { + LOG_WARN("failed to append tablet ids", K(ret)); } } return ret; diff --git a/src/storage/ddl/ob_ddl_lock.h b/src/storage/ddl/ob_ddl_lock.h index d071b074e..34530b720 100644 --- a/src/storage/ddl/ob_ddl_lock.h +++ b/src/storage/ddl/ob_ddl_lock.h @@ -65,10 +65,14 @@ class ObDDLLock { const transaction::tablelock::ObTableLockOwnerID lock_owner, ObMySQLTransaction &trans); - static int lock_for_split_partition( + static int lock_src_and_dst_for_split_partition( + const share::schema::ObTableSchema &table_schema, + const ObIArray &src_tablet_ids, + const ObIArray &dst_tablet_ids, + const transaction::tablelock::ObTableLockOwnerID lock_owner, + ObMySQLTransaction &trans); + static int lock_dst_for_split_partition( const share::schema::ObTableSchema &table_schema, - const share::ObLSID *ls_id, - const ObIArray *src_tablet_ids, const ObIArray &dst_tablet_ids, const transaction::tablelock::ObTableLockOwnerID lock_owner, ObMySQLTransaction &trans); @@ -181,10 +185,6 @@ class ObDDLLock { const int64_t timeout_us, const bool is_lock, ObMySQLTransaction &trans); - static int check_tablet_in_same_ls( - const share::schema::ObTableSchema &lhs_schema, - const share::schema::ObTableSchema &rhs_schema, - ObMySQLTransaction &trans); static int replace_tablet_lock(const uint64_t table_id, const ObIArray &tablet_ids, const transaction::tablelock::ObTableLockMode old_lock_mode, @@ -193,12 +193,11 @@ class ObDDLLock { const transaction::tablelock::ObTableLockOwnerID new_lock_owner, const int64_t timeout_us, ObMySQLTransaction &trans); - static int get_unlock_alone_tablet_request_args(const transaction::tablelock::ObTableLockMode lock_mode, + static int get_unlock_alone_tablet_request_arg(const transaction::tablelock::ObTableLockMode lock_mode, const transaction::tablelock::ObTableLockOwnerID lock_owner, const int64_t timeout_us, const ObIArray &tablet_ids, - ObArray &unlock_args, - ObMySQLTransaction &trans); + transaction::tablelock::ObUnLockAloneTabletRequest &unlock_arg); static int lock_table_and_global_indexes_for_fork( share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema &table_schema, diff --git a/src/storage/ddl/ob_ddl_macro_block_writer.cpp b/src/storage/ddl/ob_ddl_macro_block_writer.cpp index a7c67cc61..5dc240954 100644 --- a/src/storage/ddl/ob_ddl_macro_block_writer.cpp +++ b/src/storage/ddl/ob_ddl_macro_block_writer.cpp @@ -88,11 +88,10 @@ int ObDDLMacroBlockWriter::init( compaction::ObExecMode exec_mode = compaction::ObExecMode::EXEC_MODE_LOCAL; blocksstable::ObMacroMetaTempStore *macro_meta_store = nullptr; - if (OB_FAIL(pre_warm_param.init(param.ls_id_, table_key.tablet_id_))) { - LOG_WARN("fail to init pre warm param", KR(ret), K(param.ls_id_), K(table_key.tablet_id_)); + if (OB_FAIL(pre_warm_param.init(table_key.tablet_id_))) { + LOG_WARN("fail to init pre warm param", KR(ret), K(table_key.tablet_id_)); } else if (OB_FAIL(data_desc_.init(true/*is ddl*/, *tablet_param.storage_schema_, - param.ls_id_, table_key.get_tablet_id(), compaction::ObMergeType::MINOR_MERGE, 1L, @@ -120,7 +119,6 @@ int ObDDLMacroBlockWriter::init( LOG_WARN("fail to new ObDDLIncRedoLogWriterCallback", KR(ret)); } else { ObDDLRedoLogWriterCallbackInitParam init_param; - init_param.ls_id_ = param.ls_id_; init_param.tablet_id_ = table_key.tablet_id_; init_param.direct_load_type_ = param.direct_load_type_; init_param.block_type_ = DDL_MB_DATA_TYPE; @@ -158,7 +156,6 @@ int ObDDLMacroBlockWriter::init( } else { // full share::SCN mock_start_scn; IGNORE_RETURN mock_start_scn.convert_for_tx(SS_DDL_START_SCN_VAL); - const ObLSID ls_id = param.ls_id_; const uint64_t tenant_data_version = param.tenant_data_version_; const ObDDLMacroBlockType block_type = param.is_no_logging_ ? DDL_MB_SS_EMPTY_DATA_TYPE : DDL_MB_DATA_TYPE; const bool need_submit_io = true; @@ -175,11 +172,10 @@ int ObDDLMacroBlockWriter::init( macro_seq_param.seq_type_ = ObMacroSeqParam::SEQ_TYPE_INC; macro_seq_param.start_ = start_sequence.macro_data_seq_; - if (OB_FAIL(pre_warm_param.init(ls_id, table_key.tablet_id_))) { - LOG_WARN("fail to initialize pre warm param", K(ret), K(ls_id), K(table_key.tablet_id_)); + if (OB_FAIL(pre_warm_param.init(table_key.tablet_id_))) { + LOG_WARN("fail to initialize pre warm param", K(ret), K(table_key.tablet_id_)); } else if (OB_FAIL(data_desc_.init(true/*is ddl*/, *tablet_param.storage_schema_, - ls_id, table_key.get_tablet_id(), compaction::ObMergeType::MAJOR_MERGE, table_key.get_snapshot_version(), @@ -191,7 +187,7 @@ int ObDDLMacroBlockWriter::init( need_submit_io))) { LOG_WARN("fail to initialize data store desc", K(ret)); } else if (OB_FAIL(index_builder_.init(data_desc_.get_desc(), space_opt_mode/*small sstable op*/))) { - LOG_WARN("fail to initialize sstable index builder", K(ret), K(ls_id), K(table_key), K(data_desc_)); + LOG_WARN("fail to initialize sstable index builder", K(ret), K(table_key), K(data_desc_)); } else { // for build the tail index block in macro block data_desc_.get_desc().sstable_index_builder_ = &index_builder_; @@ -206,7 +202,6 @@ int ObDDLMacroBlockWriter::init( LOG_WARN("fail to new ObDDLRedoLogWriterCallback", KR(ret)); } else { ObDDLRedoLogWriterCallbackInitParam init_param; - init_param.ls_id_ = ls_id; init_param.tablet_id_ = table_key.tablet_id_; init_param.direct_load_type_ = param.direct_load_type_; init_param.block_type_ = block_type; @@ -235,7 +230,7 @@ int ObDDLMacroBlockWriter::init( *object_cleaner, ddl_redo_callback_))) { LOG_WARN("fail to open macro block writer", - K(ret), K(ls_id), K(table_key), K(data_desc_), K(start_sequence), KPC(object_cleaner)); + K(ret), K(table_key), K(data_desc_), K(start_sequence), KPC(object_cleaner)); } } } diff --git a/src/storage/ddl/ob_ddl_merge_helper.cpp b/src/storage/ddl/ob_ddl_merge_helper.cpp index 4ee1419f8..56764e578 100644 --- a/src/storage/ddl/ob_ddl_merge_helper.cpp +++ b/src/storage/ddl/ob_ddl_merge_helper.cpp @@ -76,20 +76,17 @@ int ObIDDLMergeHelper::freeze_ddl_kv(ObDDLTabletMergeDagParamV2 ¶m) { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; if (!param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(param)); - } else if (OB_FAIL(param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); - } else if (OB_FAIL(ObDDLMergeTaskUtils::freeze_ddl_kv(target_ls_id, - target_tablet_id, + } else if (OB_FAIL(ObDDLMergeTaskUtils::freeze_ddl_kv(target_tablet_id, param.direct_load_type_, param.start_scn_, param.ddl_task_param_.snapshot_version_, @@ -108,7 +105,6 @@ int ObSNDDLMergeHelperV2::set_ddl_complete(ObIDag *dag, ObTablet &tablet, ObDDLT ObStorageSchema *storage_schema = nullptr; ObWriteTabletParam *tablet_param = nullptr; ObDDLTabletContext *tablet_context = ddl_merge_param.get_tablet_ctx(); - ObLSID target_ls_id; ObTabletID target_tablet_id; /* ddl kv has already been freeze in prepare task */ if (OB_ISNULL(dag) || !ddl_merge_param.is_valid() || OB_ISNULL(tablet_context)) { @@ -121,7 +117,7 @@ int ObSNDDLMergeHelperV2::set_ddl_complete(ObIDag *dag, ObTablet &tablet, ObDDLT } else { LOG_WARN("get ddl kv mgr failed", K(ret), K(ddl_merge_param)); } - } else if (OB_FAIL(ddl_merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(ddl_merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } else if (OB_ISNULL(tablet_param)) { ret = OB_ERR_UNEXPECTED; @@ -132,7 +128,6 @@ int ObSNDDLMergeHelperV2::set_ddl_complete(ObIDag *dag, ObTablet &tablet, ObDDLT LOG_WARN("storage schema should not be null", K(ret), K(ddl_merge_param)); } else { complete_arg.has_complete_ = true; - complete_arg.ls_id_ = target_ls_id; complete_arg.tablet_id_ = target_tablet_id; complete_arg.direct_load_type_ = ddl_merge_param.direct_load_type_; complete_arg.start_scn_ = ddl_merge_param.start_scn_; @@ -151,62 +146,6 @@ int ObSNDDLMergeHelperV2::set_ddl_complete(ObIDag *dag, ObTablet &tablet, ObDDLT return ret; } -int ObSNDDLMergeHelperV2::set_ddl_complete_for_direct_load( - const ObLSID &ls_id, - const ObTabletID &tablet_id, - const ObDirectLoadType direct_load_type, - const int64_t snapshot_version, - const int64_t data_version) -{ - int ret = OB_SUCCESS; - ObLSService *ls_svr = nullptr; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObTabletHandle tablet_handle; - ObStorageSchema *storage_schema = nullptr; - ObArenaAllocator allocator(ObMemAttr("DLM_CLOSE")); - ObDDLKvMgrHandle ddl_kv_mgr_handle; - ObDDLWriteStat write_stats; // only used for empty direct load, so keep empty here - if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { - ret = OB_ERR_SYS; - LOG_WARN("MTL ObLSService is null", KR(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls", KR(ret), K(ls)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls is nullptr", KR(ret)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("fail to get tablet", KR(ret), K(tablet_id)); - } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(allocator, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, true /*try_create*/))) { - LOG_WARN("get ddl kv mgr failed", K(ret), K(ls_id), K(tablet_id)); - } else { - ObTabletDDLCompleteArg complete_arg; - complete_arg.has_complete_ = true; - complete_arg.ls_id_ = ls_id; - complete_arg.tablet_id_ = tablet_id; - complete_arg.direct_load_type_ = direct_load_type; - complete_arg.start_scn_ = SCN::base_scn(); - complete_arg.data_format_version_ = data_version; - complete_arg.snapshot_version_ = snapshot_version; - complete_arg.table_key_.reset(); - complete_arg.table_key_.tablet_id_ = tablet_id; - complete_arg.table_key_.table_type_ = ObITable::MAJOR_SSTABLE; - complete_arg.table_key_.version_range_.snapshot_version_ = snapshot_version; - - if (OB_FAIL(complete_arg.set_write_stat(write_stats))) { - LOG_WARN("failed to set write stat", K(ret), K(write_stats)); - } else if (OB_FAIL(complete_arg.set_storage_schema(*storage_schema))) { - LOG_WARN("failed to set storage_schema", K(ret), KPC(storage_schema)); - } else if (OB_FAIL(ObTabletDDLCompleteMdsHelper::record_ddl_complete_arg_to_mds(complete_arg, allocator))) { - LOG_WARN("failed to record ddl complete arg to mds", KR(ret), K(complete_arg)); - } - } - ObTabletObjLoadHelper::free(allocator, storage_schema); - return ret; -} - int ObSNDDLMergeHelperV2::process_prepare_task(ObIDag *dag, ObDDLTabletMergeDagParamV2 &ddl_merge_param, ObIArray &slice_ranges) @@ -226,7 +165,6 @@ int ObSNDDLMergeHelperV2::process_prepare_task(ObIDag *dag, const ObSSTable *first_major_sstable = nullptr; ObTabletMemberWrapper table_store_wrapper; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; ObDDLTabletContext::MergeCtx *merge_ctx = nullptr; @@ -234,7 +172,7 @@ int ObSNDDLMergeHelperV2::process_prepare_task(ObIDag *dag, if (!ddl_merge_param.is_valid() || nullptr == dag) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ddl_merge_param), KPC(dag)); - } else if (OB_FAIL(ddl_merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(ddl_merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } else if (FALSE_IT(for_major = ddl_merge_param.for_major_)) { } else if (OB_FAIL(slice_idxes.create(DDL_SLICE_BUCKET_NUM, ObMemAttr("slice_idx_set")))) { @@ -248,7 +186,7 @@ int ObSNDDLMergeHelperV2::process_prepare_task(ObIDag *dag, /* check major sstable exist */ if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_ls_id, target_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); @@ -402,7 +340,6 @@ int ObSNDDLMergeHelperV2::merge_slice(ObIDag *dag, ObArray tmp_metas; ObDDLWriteStat write_stat; - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; ObDDLTabletContext::MergeCtx *merge_ctx = nullptr; @@ -415,9 +352,9 @@ int ObSNDDLMergeHelperV2::merge_slice(ObIDag *dag, if (OB_ISNULL(dag) || start_slice_idx < 0 || end_slice_idx < start_slice_idx) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param", K(ret), K(dag), K(start_slice_idx), K(end_slice_idx)); - } else if (OB_FAIL(merge_param.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret), K(merge_param)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id, tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(merge_param)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -506,20 +443,18 @@ int ObIDDLMergeHelper::prepare_ddl_param(const ObDDLTabletMergeDagParamV2 &merge ObTabletDDLParam &ddl_param) { int ret = OB_SUCCESS; - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; if (!merge_param.is_valid() || !is_supported_direct_load_type(merge_param.direct_load_type_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(merge_param)); - } else if (OB_FAIL(merge_param.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get merge param", K(ret)); } else if (OB_ISNULL(tablet_param) || OB_ISNULL(tablet_param->storage_schema_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet param and storage schema should not be null", K(ret), KPC(tablet_param)); } else { ddl_param.direct_load_type_ = merge_param.direct_load_type_; - ddl_param.ls_id_ = ls_id; ddl_param.table_key_ = merge_param.table_key_; ddl_param.start_scn_ = merge_param.start_scn_; ddl_param.snapshot_version_ = merge_param.ddl_task_param_.snapshot_version_; @@ -563,15 +498,14 @@ int ObIDDLMergeHelper::get_rec_scn_from_ddl_kvs(ObDDLTabletMergeDagParamV2 &merg ObDDLTabletContext::MergeCtx *merge_ctx = nullptr; ObWriteTabletParam *tablet_param = nullptr; - ObLSID target_ls_id; ObTabletID target_tablet_id; if (!merge_param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(merge_param)); - } else if (OB_FAIL(merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_ls_id, target_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); } else if (OB_FAIL(merge_param.get_merge_ctx(merge_ctx))) { LOG_WARN("failed to get merge ctx", K(ret)); @@ -621,25 +555,24 @@ int ObIDDLMergeHelper::get_rec_scn_from_ddl_kvs(ObDDLTabletMergeDagParamV2 &merg return ret; } -int ObIDDLMergeHelper::remove_tablet_from_log_handler(const ObLSID &ls_id, const ObTabletID &tablet_id) +int ObIDDLMergeHelper::remove_tablet_from_log_handler(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; ObLS *ls = nullptr; ObSEArray tablet_ids; ObLSService *ls_service = share::g_mp->ls_service(); - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (nullptr == ls_service) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls service should not be null", K(ret)); } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { LOG_WARN("failed to push back tablet id", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->del_tablets(tablet_ids))) { - LOG_WARN("failed to del tablets", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(ls->get_ddl_log_handler()->del_tablets(tablet_ids))) { + LOG_WARN("failed to del tablets", K(ret), K(tablet_id)); } return ret; } @@ -653,7 +586,6 @@ int ObSNDDLMergeHelperV2::assemble_sstable(ObDDLTabletMergeDagParamV2 &merge_par { int ret = OB_SUCCESS; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; @@ -667,13 +599,13 @@ int ObSNDDLMergeHelperV2::assemble_sstable(ObDDLTabletMergeDagParamV2 &merge_par if (!merge_param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(merge_param)); - } else if (OB_FAIL(merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } /* check major sstable exist */ if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_ls_id, target_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); @@ -695,8 +627,7 @@ int ObSNDDLMergeHelperV2::assemble_sstable(ObDDLTabletMergeDagParamV2 &merge_par } else if (merge_param.for_major_ && !merge_param.for_replay_ && !merge_param.for_lob_ && - OB_FAIL(ObDDLUtil::report_ddl_checksum_from_major_sstable(target_ls_id, - target_tablet_id, + OB_FAIL(ObDDLUtil::report_ddl_checksum_from_major_sstable(target_tablet_id, merge_param.ddl_task_param_.target_table_id_, merge_param.ddl_task_param_.execution_id_, merge_param.ddl_task_param_.ddl_task_id_, @@ -706,7 +637,7 @@ int ObSNDDLMergeHelperV2::assemble_sstable(ObDDLTabletMergeDagParamV2 &merge_par /* release ddl kv when build major sstable */ if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_ls_id, target_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -721,8 +652,8 @@ int ObSNDDLMergeHelperV2::assemble_sstable(ObDDLTabletMergeDagParamV2 &merge_par /* remove tablet from log handler */ if (OB_FAIL(ret)) { - } else if (merge_param.for_major_ && OB_FAIL(ObIDDLMergeHelper::remove_tablet_from_log_handler(target_ls_id, target_tablet_id))) { - LOG_ERROR("failed to remove tablet from log handler", K(ret), K(target_ls_id), K(target_tablet_id)); + } else if (merge_param.for_major_ && OB_FAIL(ObIDDLMergeHelper::remove_tablet_from_log_handler(target_tablet_id))) { + LOG_ERROR("failed to remove tablet from log handler", K(ret), K(target_tablet_id)); } return ret; } diff --git a/src/storage/ddl/ob_ddl_merge_helper.h b/src/storage/ddl/ob_ddl_merge_helper.h index 2e268610f..d793508dd 100644 --- a/src/storage/ddl/ob_ddl_merge_helper.h +++ b/src/storage/ddl/ob_ddl_merge_helper.h @@ -52,7 +52,7 @@ class ObIDDLMergeHelper const ObDirectLoadType direct_load_type, ObIDDLMergeHelper *&helper); static int get_rec_scn_from_ddl_kvs(ObDDLTabletMergeDagParamV2 &merge_param); - static int remove_tablet_from_log_handler(const ObLSID &ls_id, const ObTabletID &tablet_id); + static int remove_tablet_from_log_handler(const ObTabletID &tablet_id); public: /* interface used for prpare_task*/ ObIDDLMergeHelper() {} @@ -114,13 +114,6 @@ class ObSNDDLMergeHelperV2: public ObIDDLMergeHelper int get_rec_scn(ObDDLTabletMergeDagParamV2 &merge_param) override; - // partition-level full bypass import fills an empty major for unspecified partitions - static int set_ddl_complete_for_direct_load(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, - const ObDirectLoadType direct_load_type, - const int64_t snapshot_version, - const int64_t data_version); - protected: bool is_supported_direct_load_type(const ObDirectLoadType direct_load_type) override ; int set_ddl_complete(ObIDag *dag, ObTablet &tablet, ObDDLTabletMergeDagParamV2 &ddl_merge_param); diff --git a/src/storage/ddl/ob_ddl_merge_schedule.cpp b/src/storage/ddl/ob_ddl_merge_schedule.cpp index a8748af63..79f315a47 100644 --- a/src/storage/ddl/ob_ddl_merge_schedule.cpp +++ b/src/storage/ddl/ob_ddl_merge_schedule.cpp @@ -115,7 +115,7 @@ int ObDDLMergeScheduler::check_need_merge_for_nidem_sn(ObTablet &tablet, ObArray ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, tenant direct load mgr is null", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - tablet.get_ls_id(), tablet.get_tablet_id(), true, direct_load_mgr_handle, is_major_sstable_exist))) { + tablet.get_tablet_id(), true, direct_load_mgr_handle, is_major_sstable_exist))) { if (OB_ENTRY_NOT_EXIST == ret && is_major_sstable_exist) { ret = OB_SUCCESS; } else { @@ -176,19 +176,18 @@ int ObDDLMergeScheduler::check_tablet_need_merge(ObTablet &tablet, ObDDLKvMgrHan return ret; } -int ObDDLMergeScheduler::schedule_ddl_merge(ObLSHandle &ls_handle, +int ObDDLMergeScheduler::schedule_ddl_merge(ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; ObDDLKvMgrHandle ddl_kv_mgr_handle; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID tablet_id = tablet_handle.is_valid() ? tablet_handle.get_obj()->get_tablet_meta().tablet_id_ : ObTabletID(); int tmp_ret = OB_SUCCESS; bool need_schedule_merge = false; ObDDLKVType ddl_kv_type = ObDDLKVType::DDL_KV_INVALID; /* used for decided using which direct load type*/ - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid arg", K(ret), K(tablet_handle)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { LOG_TRACE("kv mgr not exist", K(ret), K(tablet_handle.get_obj()->get_tablet_id())); @@ -198,9 +197,9 @@ int ObDDLMergeScheduler::schedule_ddl_merge(ObLSHandle &ls_handle, if (OB_FAIL(ret)) { } else if (OB_FAIL(ObDDLMergeScheduler::check_tablet_need_merge(*tablet_handle.get_obj(), ddl_kv_mgr_handle, need_schedule_merge, ddl_kv_type))) { - LOG_WARN("failed to check tablet need merge", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check tablet need merge", K(ret), K(tablet_id)); } else if (need_schedule_merge) { - LOG_INFO("need schedule merge", K(ret), K(ls_id), K(tablet_id), K(need_schedule_merge), K(ddl_kv_type)); + LOG_INFO("need schedule merge", K(ret), K(tablet_id), K(need_schedule_merge), K(ddl_kv_type)); } if (OB_FAIL(ret)) { @@ -208,11 +207,11 @@ int ObDDLMergeScheduler::schedule_ddl_merge(ObLSHandle &ls_handle, switch(ddl_kv_type) { case ObDDLKVType::DDL_KV_FULL: { - if (OB_FAIL(schedule_tablet_ddl_major_merge(ls_handle, tablet_handle))) { + if (OB_FAIL(schedule_tablet_ddl_major_merge(ls, tablet_handle))) { if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { - LOG_ERROR("failed to schedule tablet ddl merge", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to schedule tablet ddl merge", K(ret), K(tablet_id)); } else { - LOG_TRACE("schedule ddl major merge failed", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("schedule ddl major merge failed", K(ret), K(tablet_id)); } } } @@ -223,7 +222,7 @@ int ObDDLMergeScheduler::schedule_ddl_merge(ObLSHandle &ls_handle, break; } } - LOG_TRACE("schedule ddl tablet merge", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("schedule ddl tablet merge", K(ret), K(tablet_id)); return ret; } @@ -231,7 +230,7 @@ int ObDDLMergeScheduler::schedule_ddl_merge(ObLSHandle &ls_handle, * schedule to build ddl dump/major sstable in share nothing mode */ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( - ObLSHandle &ls_handle, + ObLS *ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; @@ -243,10 +242,9 @@ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( bool is_major_sstable_exist = false; bool has_freezed_ddl_kv = false; SCN ddl_commit_scn; - const ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_handle)); + LOG_WARN("invalid argument", K(ret), K(tablet_handle)); } else { need_schedule_merge = true; } @@ -258,7 +256,6 @@ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_id, tablet_handle.get_obj()->get_tablet_meta().tablet_id_, true, /* is_full_direct_load */ direct_load_mgr_handle, @@ -271,7 +268,7 @@ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret)); } else if (FALSE_IT(ddl_commit_scn = direct_load_mgr_handle.get_full_obj()->get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()))) { - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->try_flush_ddl_commit_scn(ls_handle, tablet_handle, direct_load_mgr_handle, ddl_commit_scn))) { + } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->try_flush_ddl_commit_scn(ls, tablet_handle, direct_load_mgr_handle, ddl_commit_scn))) { LOG_WARN("try flush ddl commit scn failed", K(ret), "tablet_id", tablet_handle.get_obj()->get_tablet_meta().tablet_id_); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_freezed_ddl_kv(has_freezed_ddl_kv))) { LOG_WARN("check has freezed ddl kv failed", K(ret)); @@ -296,10 +293,10 @@ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( ObDDLTableMergeDagParam param; ObArenaAllocator arena(ObMemAttr("DDL_Mrg_Par")); ObTabletDDLCompleteMdsUserData ddl_complete; - if (OB_FAIL(ObDDLUtil::is_major_exist(ls_id, tablet_handle.get_obj()->get_tablet_meta().tablet_id_, is_major_sstable_exist))) { - LOG_WARN("failed to check major sstable exist", K(ret), K(ls_id), K(tablet_handle.get_obj()->get_tablet_meta().tablet_id_)); + if (OB_FAIL(ObDDLUtil::is_major_exist(tablet_handle.get_obj()->get_tablet_meta().tablet_id_, is_major_sstable_exist))) { + LOG_WARN("failed to check major sstable exist", K(ret), K(tablet_handle.get_obj()->get_tablet_meta().tablet_id_)); } else if (is_major_sstable_exist) { - LOG_INFO("major sstable already exist, don't need to schdule ddl merge", K(ret), K(ls_id), K(tablet_handle.get_obj()->get_tablet_meta().tablet_id_)); + LOG_INFO("major sstable already exist, don't need to schdule ddl merge", K(ret), K(tablet_handle.get_obj()->get_tablet_meta().tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_freezed_ddl_kv(has_freezed_ddl_kv))) { @@ -319,7 +316,7 @@ int ObDDLMergeScheduler::schedule_tablet_ddl_major_merge( } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ", K(ret), K(param)); } else { - FLOG_INFO("schedule ddl merge task", K(ret), K(ls_handle.get_ls()->get_ls_id()), K(tablet_handle.get_obj()->get_tablet_id()), K(param)); + FLOG_INFO("schedule ddl merge task", K(ret), K(tablet_handle.get_obj()->get_tablet_id()), K(param)); } } } diff --git a/src/storage/ddl/ob_ddl_merge_schedule.h b/src/storage/ddl/ob_ddl_merge_schedule.h index 4cf5caba9..7ace1b16f 100644 --- a/src/storage/ddl/ob_ddl_merge_schedule.h +++ b/src/storage/ddl/ob_ddl_merge_schedule.h @@ -35,13 +35,13 @@ namespace oceanbase { namespace storage { -class ObLSHandle; +class ObLS; class ObTabletHandle; class ObDDLMergeScheduler { public: - static int schedule_ddl_merge(ObLSHandle &ls_handle, ObTabletHandle &tablet_handl); - static int schedule_tablet_ddl_major_merge(ObLSHandle &ls_handle, ObTabletHandle &tablet_handle); + static int schedule_ddl_merge(ObLS *ls, ObTabletHandle &tablet_handle); + static int schedule_tablet_ddl_major_merge(ObLS *ls, ObTabletHandle &tablet_handle); private: /* check need merge */ static int check_tablet_need_merge(ObTablet &tablet, ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool &need_schedule_merge, ObDDLKVType &ddl_kv_type); diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index f58c79b18..0a7f97bfe 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -87,7 +87,6 @@ ObDDLTableMergeDag::~ObDDLTableMergeDag() int ObDDLTableMergeDag::init_by_param(const share::ObIDagInitParam *param) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(ddl_param_)); @@ -115,13 +114,13 @@ int ObDDLTableMergeDag::check_allow_major_merge() if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ddl_param_.ls_id_, ddl_param_.tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ddl_param_.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(ddl_param_)); } else if (!tablet_handle.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet handle is invalid", K(ret), K(ddl_param_)); } else if (ddl_param_.is_commit_) { - if (!tablet_handle.get_obj()->get_tablet_meta().ha_status_.check_allow_read()) { + if (!tablet_handle.get_obj()->get_tablet_meta().restore_state_.check_allow_read()) { ddl_param_.is_commit_ = false; LOG_INFO("status not full change to dump task", K(ret)); } @@ -137,7 +136,7 @@ int ObDDLTableMergeDag::init_tablet_ctx() if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ddl_param_.ls_id_, ddl_param_.tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ddl_param_.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(ddl_param_)); } else if (tablet_ctx_ != nullptr) { LOG_INFO("tablet ctx already inited", K(ret)); @@ -146,7 +145,6 @@ int ObDDLTableMergeDag::init_tablet_ctx() LOG_WARN("failed to allocate memory", K(ret)); } else if (FALSE_IT(tablet_ctx_ = new (buf) ObDDLTabletContext())) { } else { - tablet_ctx_->ls_id_ = ddl_param_.ls_id_; tablet_ctx_->tablet_id_ = ddl_param_.tablet_id_; /* only sn major merge need to load storage schema from user data @@ -177,7 +175,7 @@ int ObDDLTableMergeDag::create_first_task() { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObArray ddl_kvs_handle; if (is_incremental_major_direct_load(ddl_param_.direct_load_type_)) { @@ -218,9 +216,9 @@ int ObDDLTableMergeDag::create_first_task() } } else { ObDDLTableMergeTask *merge_task = nullptr; - if (OB_FAIL(ls_service->get_ls(ddl_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(ddl_param_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, ddl_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -355,7 +353,6 @@ bool ObDDLTableMergeDag::operator == (const ObIDag &other) const const ObDDLTableMergeDag &other_dag = static_cast (other); // each tablet has max 1 dag in running, so that the compaction task is unique and no need to consider concurrency is_same = ddl_param_.tablet_id_ == other_dag.ddl_param_.tablet_id_ - && ddl_param_.ls_id_ == other_dag.ddl_param_.ls_id_ && ddl_param_.direct_load_type_ == other_dag.ddl_param_.direct_load_type_ && ddl_param_.trans_id_ == other_dag.ddl_param_.trans_id_ && ddl_param_.seq_no_ == other_dag.ddl_param_.seq_no_; @@ -375,7 +372,6 @@ int ObDDLTableMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_para ret = OB_NOT_INIT; LOG_WARN("ObDDLTableMergeDag has not been initialized", K(ret)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - ddl_param_.ls_id_.id(), static_cast(ddl_param_.tablet_id_.id()), static_cast(ddl_param_.rec_scn_.get_val_for_inner_table_field()), "is_commit", ddl_param_.is_commit_))) { @@ -387,8 +383,8 @@ int ObDDLTableMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_para int ObDDLTableMergeDag::fill_dag_key(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; - if (OB_FAIL(databuff_printf(buf, buf_len, "ddl table merge task: ls_id=%ld, tablet_id=%ld, rec_scn=%lu", - ddl_param_.ls_id_.id(), ddl_param_.tablet_id_.id(), ddl_param_.rec_scn_.get_val_for_inner_table_field()))) { + if (OB_FAIL(databuff_printf(buf, buf_len, "ddl table merge task: tablet_id=%ld, rec_scn=%lu", + ddl_param_.tablet_id_.id(), ddl_param_.rec_scn_.get_val_for_inner_table_field()))) { LOG_WARN("fill dag key for ddl table merge dag failed", K(ret), K(ddl_param_)); } return ret; @@ -433,7 +429,7 @@ int ObDDLTableMergeTask::init(const ObDDLTableMergeDagParam &ddl_dag_param, cons return ret; } -int wait_lob_tablet_major_exist(const ObDirectLoadType &direct_load_type, ObLSHandle &ls_handle, ObTablet &tablet) +int wait_lob_tablet_major_exist(const ObDirectLoadType &direct_load_type, ObLS *ls, ObTablet &tablet) { int ret = OB_SUCCESS; ObTabletBindingMdsUserData ddl_data; @@ -449,7 +445,7 @@ int wait_lob_tablet_major_exist(const ObDirectLoadType &direct_load_type, ObLSHa } else if (ddl_data.lob_meta_tablet_id_.is_valid()) { ObTabletHandle lob_tablet_handle; const ObTabletID lob_tablet_id = ddl_data.lob_meta_tablet_id_; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_tablet_id, lob_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, lob_tablet_id, lob_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get lob tablet handle failed", K(ret), K(lob_tablet_id)); } else if (is_idem_type(direct_load_type)) { if (OB_FAIL(lob_tablet_handle.get_obj()->get_ddl_complete(share::SCN::max_scn(), allocator, ddl_complete))) { @@ -467,7 +463,7 @@ int wait_lob_tablet_major_exist(const ObDirectLoadType &direct_load_type, ObLSHa if (!is_major_sstable_exist) { ret = OB_EAGAIN; int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ObDDLMergeScheduler::schedule_tablet_ddl_major_merge(ls_handle, lob_tablet_handle))) { + if (OB_TMP_FAIL(ObDDLMergeScheduler::schedule_tablet_ddl_major_merge(ls, lob_tablet_handle))) { LOG_WARN("schedule ddl major merge for lob tablet failed", K(tmp_ret), K(lob_tablet_id)); } } @@ -481,14 +477,14 @@ int ObDDLTableMergeTask::process() int ret = OB_SUCCESS; LOG_INFO("ddl merge task start process", K(*this), "ddl_event_info", ObDDLEventInfo()); ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(merge_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(merge_param_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, merge_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -502,23 +498,23 @@ int ObDDLTableMergeTask::process() FLOG_INFO("ERRSIM EN_DIRECT_LOAD_TASK_PROCESS", KR(ret)); #endif } else if (is_idem_type(merge_param_.direct_load_type_) && merge_param_.is_commit_ && - !tablet_handle.get_obj()->get_tablet_meta().ha_status_.check_allow_read()) { + !tablet_handle.get_obj()->get_tablet_meta().restore_state_.check_allow_read()) { LOG_INFO("skip since tablet not allow read", K(ret), K(merge_param_)); - } else if (OB_FAIL(merge_ddl_kvs(ls_handle, *(tablet_handle.get_obj())))) { + } else if (OB_FAIL(merge_ddl_kvs(ls, *(tablet_handle.get_obj())))) { LOG_WARN("fail to merge ddl kvs", K(ret)); } return ret; } -int ObDDLTableMergeTask::merge_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet) +int ObDDLTableMergeTask::merge_ddl_kvs(ObLS *ls, ObTablet &tablet) { int ret = OB_SUCCESS; if (is_full_direct_load(merge_param_.direct_load_type_)) { - if (OB_FAIL(merge_full_direct_load_ddl_kvs(ls_handle, tablet))) { + if (OB_FAIL(merge_full_direct_load_ddl_kvs(ls, tablet))) { LOG_WARN("fail to merge full direct load ddl kvs", K(ret)); } } else { - if (OB_FAIL(merge_incremental_direct_load_ddl_kvs(ls_handle, tablet))) { + if (OB_FAIL(merge_incremental_direct_load_ddl_kvs(ls, tablet))) { LOG_WARN("fail to merge incremental direct load ddl kvs", K(ret)); } } @@ -578,7 +574,6 @@ int prepare_ddl_param_for_idem_sn(const ObDDLTableMergeDagParam &merge_param, Ob LOG_WARN("only support diem direct load type", K(ret)); } else { ddl_param.direct_load_type_ = merge_param.direct_load_type_; - ddl_param.ls_id_ = merge_param.ls_id_; ddl_param.table_key_ = merge_param.table_key_; ddl_param.start_scn_ = merge_param.start_scn_; ddl_param.commit_scn_ = merge_param.rec_scn_; @@ -605,18 +600,18 @@ int prepare_full_direct_load_ddl_param(const ObDDLTableMergeDagParam &merge_para return ret; } -int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet) +int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs(ObLS *ls, ObTablet &tablet) { int ret = OB_SUCCESS; if (!is_full_direct_load(merge_param_.direct_load_type_)) { LOG_WARN("func can only be used for full direct load", K(ret), K(merge_param_)); - } else if (OB_FAIL(merge_full_direct_load_ddl_kvs_for_sn(ls_handle, tablet))) { + } else if (OB_FAIL(merge_full_direct_load_ddl_kvs_for_sn(ls, tablet))) { LOG_WARN("failed to merge full direct load", K(ret)); } return ret; } -int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_handle, ObTablet &tablet) +int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLS *ls, ObTablet &tablet) { int ret = OB_SUCCESS; ObTableStoreIterator ddl_table_iter; @@ -659,7 +654,7 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_ha const ObSSTable *first_major_sstable = nullptr; SCN compact_start_scn, compact_end_scn; if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable( - merge_param_.ls_id_, merge_param_.tablet_id_, first_major_sstable, table_store_wrapper))) { + merge_param_.tablet_id_, first_major_sstable, table_store_wrapper))) { LOG_WARN("check if major sstable exist failed", K(ret)); } else if (nullptr != first_major_sstable) { is_major_exist = true; @@ -667,7 +662,7 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_ha } else if (tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { ret = OB_TASK_EXPIRED; LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip"); - } else if (merge_param_.is_commit_ && OB_FAIL(wait_lob_tablet_major_exist(merge_param_.direct_load_type_, ls_handle, tablet))) { + } else if (merge_param_.is_commit_ && OB_FAIL(wait_lob_tablet_major_exist(merge_param_.direct_load_type_, ls, tablet))) { if (OB_EAGAIN != ret) { LOG_WARN("wait lob tablet major sstable exist faild", K(ret), K(merge_param_)); } else { @@ -700,7 +695,7 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_ha } else { // use the final table key of major, do nothing } - if (OB_FAIL(ObTabletDDLUtil::compact_ddl_kv(*ls_handle.get_ls(), + if (OB_FAIL(ObTabletDDLUtil::compact_ddl_kv(*ls, tablet, ddl_table_iter, frozen_ddl_kvs_, @@ -719,7 +714,7 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_ha if (OB_SUCC(ret) && merge_param_.is_commit_ && is_major_exist) { ObTenantDirectLoadMgr *tenant_direct_load_mgr = share::g_mp->tenant_direct_load_mgr(); - if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(merge_param_.ls_id_, merge_param_.tablet_id_))) { + if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(merge_param_.tablet_id_))) { LOG_WARN("fail to submit tablet update task", K(ret), K(merge_param_)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->release_ddl_kvs(ObDDLKVType::DDL_KV_FULL, compact_end_scn))) { LOG_WARN("release all ddl kv failed", K(ret), K(ddl_param)); @@ -774,7 +769,7 @@ int ObDDLMergeTaskUtils::refine_incremental_direct_load_merge_param(const ObTabl return ret; } -int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet) +int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLS *ls, ObTablet &tablet) { int ret = OB_SUCCESS; common::ObArenaAllocator allocator("DDLMergeTask", OB_MALLOC_NORMAL_BLOCK_SIZE); @@ -794,7 +789,6 @@ int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_ha // do nothing } else { ddl_param.direct_load_type_ = merge_param_.direct_load_type_; - ddl_param.ls_id_ = merge_param_.ls_id_; ddl_param.start_scn_ = ddl_kv->get_ddl_start_scn(); ddl_param.commit_scn_ = ddl_kv->get_rec_scn(); ddl_param.data_format_version_ = ddl_kv->get_data_format_version(); @@ -816,7 +810,7 @@ int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_ha ret = OB_EAGAIN; int tmp_ret = OB_SUCCESS; ObTabletHandle tmp_tablet_handle; - if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(merge_param_.tablet_id_, + if (OB_TMP_FAIL(ls->get_tablet(merge_param_.tablet_id_, tmp_tablet_handle, 0/*timeout_us*/, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -830,7 +824,7 @@ int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_ha LOG_ERROR("Unexpected uncontinuous scn_range in mini merge", K(ret), K(clog_checkpoint_scn), K(ddl_param), K(frozen_ddl_kvs_), K(tablet), KPC(tmp_tablet_handle.get_obj())); } - } else if (OB_FAIL(ObTabletDDLUtil::compact_ddl_kv(*ls_handle.get_ls(), + } else if (OB_FAIL(ObTabletDDLUtil::compact_ddl_kv(*ls, tablet, ddl_table_iter, frozen_ddl_kvs_, @@ -843,7 +837,7 @@ int ObDDLTableMergeTask::merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_ha if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; ObTabletHandle new_tablet_handle; - if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls, merge_param_.tablet_id_, new_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { @@ -970,17 +964,14 @@ int ObTabletDDLUtil::prepare_index_data_desc(const ObTablet &tablet, { int ret = OB_SUCCESS; data_desc.reset(); - ObLSService *ls_service = share::g_mp->ls_service(); const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const SCN end_scn = table_key.get_end_scn(); const bool micro_index_clustered = tablet.get_tablet_meta().micro_index_clustered_; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || data_format_version <= 0 || OB_ISNULL(storage_schema))) { + if (OB_UNLIKELY(!tablet_id.is_valid() || snapshot_version <= 0 || data_format_version <= 0 || OB_ISNULL(storage_schema))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(data_format_version), KP(storage_schema)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(snapshot_version), K(data_format_version), KP(storage_schema)); } else if (OB_FAIL(data_desc.init(true/*is_ddl*/, *storage_schema, - ls_id, tablet_id, table_key.is_minor_sstable() ? compaction::MINOR_MERGE : compaction::MAJOR_MERGE, snapshot_version, @@ -1128,7 +1119,6 @@ int ObTabletDDLUtil::update_ddl_table_store( LOG_WARN("invalid argument", K(ret), K(ddl_param), KP(storage_schema), KP(sstable)); } else { const bool is_major_sstable = ddl_param.table_key_.is_major_sstable(); - const int64_t rebuild_seq = ls.get_rebuild_seq(); int64_t snapshot_version = 0; int64_t multi_version_start = 0; if (is_full_direct_load(ddl_param.direct_load_type_)) { @@ -1144,7 +1134,6 @@ int ObTabletDDLUtil::update_ddl_table_store( ObUpdateTableStoreParam table_store_param(snapshot_version, multi_version_start, storage_schema, - rebuild_seq, sstable); if (OB_FAIL(table_store_param.init_with_compaction_info( ObCompactionTableStoreParam(is_major_sstable ? compaction::MEDIUM_MERGE : compaction::MINI_MERGE, @@ -1685,7 +1674,6 @@ int ObTabletDDLUtil::get_compact_scn( } int ObTabletDDLUtil::report_ddl_checksum( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t table_id, const int64_t execution_id, @@ -1772,28 +1760,27 @@ int ObTabletDDLUtil::report_ddl_checksum( return ret; } -int ObTabletDDLUtil::check_and_get_major_sstable(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObTabletDDLUtil::check_and_get_major_sstable(const ObTabletID &tablet_id, const blocksstable::ObSSTable *&first_major_sstable, ObTabletMemberWrapper &table_store_wrapper) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; first_major_sstable = nullptr; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("invalid argument", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet handle is null", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else { @@ -1805,7 +1792,7 @@ int ObTabletDDLUtil::check_and_get_major_sstable(const share::ObLSID &ls_id, int ObTabletDDLUtil::freeze_ddl_kv(const ObDDLTableMergeDagParam ¶m) { - return ObDDLMergeTaskUtils::freeze_ddl_kv(param.ls_id_, param.tablet_id_, param.direct_load_type_, + return ObDDLMergeTaskUtils::freeze_ddl_kv(param.tablet_id_, param.direct_load_type_, param.start_scn_, param.snapshot_version_, param.data_format_version_); } diff --git a/src/storage/ddl/ob_ddl_merge_task.h b/src/storage/ddl/ob_ddl_merge_task.h index 186428d5b..fbd411fce 100644 --- a/src/storage/ddl/ob_ddl_merge_task.h +++ b/src/storage/ddl/ob_ddl_merge_task.h @@ -95,9 +95,9 @@ class ObDDLTableMergeTask : public share::ObITask virtual int process() override; TO_STRING_KV(K_(is_inited), K_(merge_param)); private: - int merge_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet); - int merge_full_direct_load_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet); - int merge_full_direct_load_ddl_kvs_for_sn(ObLSHandle &ls_handle, ObTablet &tablet); + int merge_ddl_kvs(ObLS *ls, ObTablet &tablet); + int merge_full_direct_load_ddl_kvs(ObLS *ls, ObTablet &tablet); + int merge_full_direct_load_ddl_kvs_for_sn(ObLS *ls, ObTablet &tablet); int check_macro_intergrate_for_sn(const ObTabletDDLParam &ddl_param, ObTablet &tablet, SCN &compact_start_scn, @@ -106,7 +106,7 @@ class ObDDLTableMergeTask : public share::ObITask ObTablet &tablet, SCN &compact_start_scn, SCN &compact_end_scn); - int merge_incremental_direct_load_ddl_kvs(ObLSHandle &ls_handle, ObTablet &tablet); + int merge_incremental_direct_load_ddl_kvs(ObLS *ls, ObTablet &tablet); private: bool is_inited_; ObDDLTableMergeDagParam merge_param_; @@ -187,8 +187,7 @@ class ObTabletDDLUtil ObStorageSchema *&storage_schema, const ObIArray &frozen_ddl_kvs); - static int report_ddl_checksum(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, + static int report_ddl_checksum(const ObTabletID &tablet_id, const uint64_t table_id, const int64_t execution_id, const int64_t ddl_task_id, @@ -197,7 +196,6 @@ class ObTabletDDLUtil const uint64_t data_format_version); static int check_and_get_major_sstable( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const blocksstable::ObSSTable *&first_major_sstable, ObTabletMemberWrapper &table_store_wrapper); diff --git a/src/storage/ddl/ob_ddl_merge_task_utils.cpp b/src/storage/ddl/ob_ddl_merge_task_utils.cpp index abf7fdfa6..7cff512db 100644 --- a/src/storage/ddl/ob_ddl_merge_task_utils.cpp +++ b/src/storage/ddl/ob_ddl_merge_task_utils.cpp @@ -209,8 +209,7 @@ int ObDDLMergeTaskUtils::get_ddl_memtables(const ObIArray &frozen return ret; } -int ObDDLMergeTaskUtils::freeze_ddl_kv(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDDLMergeTaskUtils::freeze_ddl_kv(const ObTabletID &tablet_id, const ObDirectLoadType &direct_load_type, const share::SCN start_scn, const int64_t snapshot_version, @@ -219,25 +218,25 @@ int ObDDLMergeTaskUtils::freeze_ddl_kv(const ObLSID &ls_id, { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected err", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("unexpected err", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_TASK_EXPIRED; - LOG_INFO("ddl kv mgr not exist", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("ddl kv mgr not exist", K(ret), K(tablet_id)); } else { - LOG_WARN("get ddl kv mgr failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get ddl kv mgr failed", K(ret), K(tablet_id)); } } else if (is_full_direct_load(direct_load_type) && start_scn < tablet_handle.get_obj()->get_tablet_meta().ddl_start_scn_) { @@ -350,16 +349,14 @@ int ObDDLMergeTaskUtils::update_tablet_table_store(ObDDLTabletMergeDagParamV2 &d { int ret = OB_SUCCESS; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; bool for_major = dag_merge_param.for_major_; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); ObTabletHandle tablet_handle, new_tablet_handle; - int64_t rebuild_seq = -1; int64_t snapshot_version = 0; int64_t multi_version_start = 0; @@ -367,17 +364,13 @@ int ObDDLMergeTaskUtils::update_tablet_table_store(ObDDLTabletMergeDagParamV2 &d || (nullptr == major_sstable && slice_sstable_array.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(dag_merge_param), K(major_sstable), K(slice_sstable_array)); - } else if (OB_FAIL(dag_merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(dag_merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_ls_id, target_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(target_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(target_ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(target_ls_id)); - } else if (!ls_handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls handle should be valid", K(ret), K(ls_handle)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { - rebuild_seq = ls_handle.get_ls()->get_rebuild_seq(); snapshot_version = for_major ? max(dag_merge_param.ddl_task_param_.snapshot_version_, tablet_handle.get_obj()->get_snapshot_version()) : tablet_handle.get_obj()->get_snapshot_version(); multi_version_start = for_major ? max(dag_merge_param.ddl_task_param_.snapshot_version_, tablet_handle.get_obj()->get_multi_version_start()) : @@ -386,7 +379,7 @@ int ObDDLMergeTaskUtils::update_tablet_table_store(ObDDLTabletMergeDagParamV2 &d if (OB_FAIL(ret)) { } else { - ObUpdateTableStoreParam table_store_param(snapshot_version, multi_version_start, tablet_param->storage_schema_, rebuild_seq, major_sstable, true); + ObUpdateTableStoreParam table_store_param(snapshot_version, multi_version_start, tablet_param->storage_schema_, major_sstable, true); if (OB_FAIL(table_store_param.init_with_compaction_info(ObCompactionTableStoreParam(for_major ? compaction::MEDIUM_MERGE : compaction::MINI_MERGE, share::SCN::min_scn(), for_major /* need_report*/, @@ -409,9 +402,9 @@ int ObDDLMergeTaskUtils::update_tablet_table_store(ObDDLTabletMergeDagParamV2 &d } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(target_tablet_id, table_store_param, new_tablet_handle))) { + } else if (OB_FAIL(ls->update_tablet_table_store(target_tablet_id, table_store_param, new_tablet_handle))) { LOG_WARN("failed to update tablet table store", K(ret), K(target_tablet_id), K(table_store_param)); - } else if (for_major && OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(target_ls_id, target_tablet_id))) { + } else if (for_major && OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(target_tablet_id))) { LOG_ERROR("fail to submit tablet update task", K(ret), K(dag_merge_param)); } } @@ -429,7 +422,6 @@ int ObDDLMergeTaskUtils::build_sstable(ObDDLTabletMergeDagParamV2 &dag_merge_par slice_sstable_array.reset(); major_sstable = nullptr; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; hash::ObHashMap*> *slice_idx_sstables = nullptr; @@ -437,7 +429,7 @@ int ObDDLMergeTaskUtils::build_sstable(ObDDLTabletMergeDagParamV2 &dag_merge_par if (!dag_merge_param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(dag_merge_param)); - } else if (OB_FAIL(dag_merge_param.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(dag_merge_param.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } else if (OB_ISNULL(tablet_param->storage_schema_)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/ddl/ob_ddl_merge_task_utils.h b/src/storage/ddl/ob_ddl_merge_task_utils.h index 85da081fd..9d8e1cc33 100644 --- a/src/storage/ddl/ob_ddl_merge_task_utils.h +++ b/src/storage/ddl/ob_ddl_merge_task_utils.h @@ -53,8 +53,7 @@ class ObDDLMergeTaskUtils ObIAllocator &allocator, ObArray &sorted_metas); - static int freeze_ddl_kv(const ObLSID &ls_id, - const ObTabletID &tablet_id, + static int freeze_ddl_kv(const ObTabletID &tablet_id, const ObDirectLoadType &direct_load_type, const share::SCN start_scn, const int64_t snapshot_version, diff --git a/src/storage/ddl/ob_ddl_merge_task_v2.cpp b/src/storage/ddl/ob_ddl_merge_task_v2.cpp index 7b21eb7d4..d71078985 100644 --- a/src/storage/ddl/ob_ddl_merge_task_v2.cpp +++ b/src/storage/ddl/ob_ddl_merge_task_v2.cpp @@ -143,7 +143,6 @@ int ObDDLMergePrepareTask::inner_process() int64_t merge_slice_idx = 0; ObArray slice_ranges; - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; @@ -156,7 +155,7 @@ int ObDDLMergePrepareTask::inner_process() if (OB_ISNULL(dag)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("dag should not be null", K(ret)); - } else if (OB_FAIL(merge_param_.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param_.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } else if (OB_ISNULL(tablet_param)) { ret = OB_ERR_UNEXPECTED; @@ -267,7 +266,6 @@ int ObDDLMergePrepareTask::process() { int ret = OB_SUCCESS; bool has_lob = false; - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; if (!is_inited_) { @@ -275,7 +273,7 @@ int ObDDLMergePrepareTask::process() LOG_WARN("task is not inited", K(ret), KPC(this)); } else if (OB_FAIL(inner_process())) { LOG_WARN("failed to inner process prepare task", K(ret)); - } else if (OB_FAIL(merge_param_.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param_.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } @@ -285,12 +283,11 @@ int ObDDLMergePrepareTask::process() void ObDDLMergePrepareTask::task_debug_info_to_string(char *buf, const int64_t buf_len, int64_t &pos) const { - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; - if (OB_SUCCESS == merge_param_.get_tablet_param(ls_id, tablet_id, tablet_param)) { - BUF_PRINTF("DDL Merge Prepare Task: ls_id=%ld, tablet_id=%ld, is_inited=%s", - ls_id.id(), tablet_id.id(), is_inited_ ? "true" : "false"); + if (OB_SUCCESS == merge_param_.get_tablet_param(tablet_id, tablet_param)) { + BUF_PRINTF("DDL Merge Prepare Task: tablet_id=%ld, is_inited=%s", + tablet_id.id(), is_inited_ ? "true" : "false"); } else { BUF_PRINTF("DDL Merge Prepare Task: is_inited=%s", is_inited_ ? "true" : "false"); } @@ -305,8 +302,6 @@ int ObDDLMergeSliceTask::init(const ObDDLTabletMergeDagParamV2 &merge_param, const int64_t end_slice_idx) { int ret = OB_SUCCESS; - ObLSID target_ls_id; - ObTabletID target_tablet_id; if (!merge_param.is_valid() || start_slice_idx < 0 || end_slice_idx < start_slice_idx) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid merge param", K(ret), K(merge_param), K(start_slice_idx), K(end_slice_idx)); @@ -347,12 +342,11 @@ int ObDDLMergeSliceTask::process() void ObDDLMergeSliceTask::task_debug_info_to_string(char *buf, const int64_t buf_len, int64_t &pos) const { - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; - if (OB_SUCCESS == merge_param_.get_tablet_param(ls_id, tablet_id, tablet_param)) { - BUF_PRINTF("DDL Merge Slice Task: ls_id=%ld, tablet_id=%ld, start_slice=%ld, end_slice=%ld", - ls_id.id(), tablet_id.id(), start_slice_idx_, end_slice_idx_); + if (OB_SUCCESS == merge_param_.get_tablet_param(tablet_id, tablet_param)) { + BUF_PRINTF("DDL Merge Slice Task: tablet_id=%ld, start_slice=%ld, end_slice=%ld", + tablet_id.id(), start_slice_idx_, end_slice_idx_); } else { BUF_PRINTF("DDL Merge Slice Task: start_slice=%ld, end_slice=%ld", start_slice_idx_, end_slice_idx_); @@ -384,7 +378,6 @@ int ObDDLMergeAssembleTask::process() int ret = OB_SUCCESS; ObIDDLMergeHelper *merge_helper = nullptr; ObArenaAllocator allocator(ObMemAttr("Ddl_Assm_Task")); - ObLSID target_ls_id; ObTabletID target_tablet_id; ObWriteTabletParam *tablet_param = nullptr; ObTabletHandle tablet_handle; @@ -392,7 +385,7 @@ int ObDDLMergeAssembleTask::process() if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("assemble task has not been init", K(ret), KPC(this)); - } else if (OB_FAIL(merge_param_.get_tablet_param(target_ls_id, target_tablet_id, tablet_param))) { + } else if (OB_FAIL(merge_param_.get_tablet_param(target_tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret)); } else if (OB_FAIL(merge_param_.get_merge_helper(merge_helper))) { LOG_WARN("failed to get merge helper", K(ret), K(merge_param_)); @@ -410,12 +403,11 @@ int ObDDLMergeAssembleTask::process() void ObDDLMergeAssembleTask::task_debug_info_to_string(char *buf, const int64_t buf_len, int64_t &pos) const { - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; - if (OB_SUCCESS == merge_param_.get_tablet_param(ls_id, tablet_id, tablet_param)) { - BUF_PRINTF("DDL Merge Assemble Task: ls_id=%ld, tablet_id=%ld, is_inited=%s", - ls_id.id(), tablet_id.id(), is_inited_ ? "true" : "false"); + if (OB_SUCCESS == merge_param_.get_tablet_param(tablet_id, tablet_param)) { + BUF_PRINTF("DDL Merge Assemble Task: tablet_id=%ld, is_inited=%s", + tablet_id.id(), is_inited_ ? "true" : "false"); } else { BUF_PRINTF("DDL Merge Assemble Task: is_inited=%s", is_inited_ ? "true" : "false"); } diff --git a/src/storage/ddl/ob_ddl_pipeline.cpp b/src/storage/ddl/ob_ddl_pipeline.cpp index 0d61a9877..9e71042ce 100644 --- a/src/storage/ddl/ob_ddl_pipeline.cpp +++ b/src/storage/ddl/ob_ddl_pipeline.cpp @@ -88,7 +88,7 @@ int ObIDDLPipeline::process() } ObVectorIndexTabletContext::ObVectorIndexTabletContext() - : row_cnt_(0), vec_dim_(0), ls_id_(), tablet_id_(), vec_idx_param_(), ctx_(), + : row_cnt_(0), vec_dim_(0), tablet_id_(), vec_idx_param_(), ctx_(), vector_vid_col_idx_(-1), vector_col_idx_(-1), vector_key_col_idx_(-1), vector_data_col_idx_(-1), center_id_col_idx_(-1), center_vector_col_idx_(-1), meta_id_col_idx_(-1), meta_vector_col_idx_(-1), pq_center_id_col_idx_(-1), pq_center_vector_col_idx_(-1), extra_column_idx_types_(), lob_inrow_threshold_(0), rowkey_cnt_(0), column_cnt_(0), snapshot_version_(0), index_type_(share::VIAT_MAX), helper_(nullptr), @@ -100,23 +100,20 @@ ObVectorIndexTabletContext::ObVectorIndexTabletContext() } int ObVectorIndexTabletContext::init( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObIndexType &index_type, const int64_t snapshot_version, const ObDDLTableSchema &ddl_table_schema) { int ret = OB_SUCCESS; - if (!ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || !(ddl_table_schema.table_item_.vec_dim_ > 0)) { + if (!tablet_id.is_valid() || snapshot_version <= 0 || !(ddl_table_schema.table_item_.vec_dim_ > 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_id), K(ddl_table_schema), K(snapshot_version)); + LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(ddl_table_schema), K(snapshot_version)); } else { row_cnt_ = 0; vec_dim_ = ddl_table_schema.table_item_.vec_dim_; - ls_id_ = ls_id; tablet_id_ = tablet_id; vec_idx_param_ = ddl_table_schema.table_item_.vec_idx_param_; - ctx_.ls_id_ = ls_id_; lob_inrow_threshold_ = ddl_table_schema.table_item_.lob_inrow_threshold_; rowkey_cnt_ = ddl_table_schema.table_item_.rowkey_column_num_; column_cnt_ = ddl_table_schema.column_items_.count(); @@ -153,7 +150,7 @@ int ObVectorIndexTabletContext::init_hnsw_index(const ObDDLTableSchema &ddl_tabl { int ret = OB_SUCCESS; // get data tablet id and lob tablet id - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle five_tablet_handle; ObTabletHandle data_tablet_handle; ObTabletBindingMdsUserData ddl_data; @@ -166,15 +163,12 @@ int ObVectorIndexTabletContext::init_hnsw_index(const ObDDLTableSchema &ddl_tabl vector_data_col_idx_ = -1; int64_t pk_increment_col_idx = -1; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id_, five_tablet_handle))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet(tablet_id_, five_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else if (FALSE_IT(ctx_.data_tablet_id_ = five_tablet_handle.get_obj()->get_data_tablet_id())) { - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(ctx_.data_tablet_id_, data_tablet_handle))) { + } else if (OB_FAIL(ls->get_tablet(ctx_.data_tablet_id_, data_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(ctx_.data_tablet_id_)); } else if (OB_FAIL(data_tablet_handle.get_obj()->get_ddl_data(ddl_data))) { LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet_handle)); @@ -952,12 +946,10 @@ int ObHNSWIndexAppendBufferOperator::append_row( if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("error unexpected, vector index service is nullptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_adapter_guard( - tablet_context->vector_index_ctx_->ls_id_, tablet_id_, + } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(tablet_id_, ObIndexType::INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, adaptor_guard, &tablet_context->vector_index_ctx_->vec_idx_param_, tablet_context->vector_index_ctx_->vec_dim_))) { - LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(tablet_context->vector_index_ctx_->ls_id_), - K(tablet_id_)); + LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(tablet_id_)); } else if (OB_NOT_NULL(adaptor_guard.get_adatper()) && OB_FAIL(adaptor_guard.get_adatper()->get_extra_info_actual_size(extra_info_actual_size))) { LOG_WARN("failed to get extra info actual size.", K(ret)); @@ -1091,13 +1083,12 @@ int ObHNSWIndexBuildOperator::serialize_vector_index( if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(ctx.ls_id_, - tablet_id_, + } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(tablet_id_, ObIndexType::INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, adaptor_guard, &ctx.vec_idx_param_, ctx.vec_dim_))) { - LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(ctx.ls_id_), K(tablet_id_)); + LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(tablet_id_)); } else { ObHNSWSerializeCallback callback; ObOStreamBuf::Callback cb = callback; @@ -1114,7 +1105,7 @@ int ObHNSWIndexBuildOperator::serialize_vector_index( if (OB_ISNULL(tx_desc)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get tx desc, get nullptr", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ctx.ls_id_, timeout, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else { param.timeout_ = timeout; @@ -1173,7 +1164,7 @@ int ObHNSWIndexBuildOperator::execute( ObVectorIndexAlgorithmType index_type = VIAT_MAX; const uint64_t timeout_us = ObTimeUtility::current_time() + storage::ObInsertLobColumnHelper::LOB_TX_TIMEOUT; transaction::ObTxDesc *tx_desc = nullptr; - if (OB_FAIL(ObInsertLobColumnHelper::start_trans(tablet_context->ls_id_, false/*is_for_read*/, timeout_us, tx_desc))) { + if (OB_FAIL(ObInsertLobColumnHelper::start_trans(false/*is_for_read*/, timeout_us, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); } else if (OB_FAIL(serialize_vector_index(&op_allocator_, tx_desc, tablet_context->vector_index_ctx_->lob_inrow_threshold_, index_type, *tablet_context->vector_index_ctx_))) { LOG_WARN("serialize vector index failed", K(ret)); diff --git a/src/storage/ddl/ob_ddl_pipeline.h b/src/storage/ddl/ob_ddl_pipeline.h index 8ccfc9d06..23d1ca605 100644 --- a/src/storage/ddl/ob_ddl_pipeline.h +++ b/src/storage/ddl/ob_ddl_pipeline.h @@ -119,7 +119,6 @@ struct ObVectorIndexTabletContext destroy_ivf_build_helper(); } int init( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObIndexType &index_type, const int64_t snapshot_version, @@ -127,7 +126,7 @@ struct ObVectorIndexTabletContext int build_extra_column_idxs(const int32_t chunk_col_idx, common::ObSEArray &extra_column_idxs) const; -TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(snapshot_version), K_(index_type)); +TO_STRING_KV(K_(tablet_id), K_(snapshot_version), K_(index_type)); private: int init_hnsw_index(const ObDDLTableSchema &ddl_table_schema); @@ -142,8 +141,6 @@ TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(snapshot_version), K_(index_type)); public: int64_t row_cnt_; int64_t vec_dim_; - - share::ObLSID ls_id_; ObTabletID tablet_id_; common::ObString vec_idx_param_; share::ObVecIdxSnapshotDataWriteCtx ctx_; @@ -198,7 +195,7 @@ class ObHNSWIndexRowIterator : public ObVectorIndexRowIterator public: ObHNSWIndexRowIterator() : rowkey_cnt_(0), column_cnt_(0), snapshot_version_(0), index_type_(), - row_cnt_(0), ls_id_(),vec_idx_param_(), + row_cnt_(0), vec_idx_param_(), vector_vid_col_idx_(-1), vector_col_idx_(-1), vector_key_col_idx_(-1), vector_data_col_idx_(-1), ctx_(nullptr), extra_column_idx_types_() {} @@ -216,7 +213,6 @@ class ObHNSWIndexRowIterator : public ObVectorIndexRowIterator int64_t snapshot_version_; ObVectorIndexAlgorithmType index_type_; int64_t row_cnt_; - share::ObLSID ls_id_; common::ObString vec_idx_param_; int32_t vector_vid_col_idx_; int32_t vector_col_idx_; diff --git a/src/storage/ddl/ob_ddl_redo_log_replayer.cpp b/src/storage/ddl/ob_ddl_redo_log_replayer.cpp index 0436eca35..6ca18a4fb 100644 --- a/src/storage/ddl/ob_ddl_redo_log_replayer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_replayer.cpp @@ -63,7 +63,7 @@ int ObDDLRedoLogReplayer::replay_start(const ObDDLStartLog &log, const SCN &scn) LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("failed to init ddl start log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.get_table_key().tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.get_table_key().tablet_id_))) { if (OB_NO_NEED_UPDATE == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { @@ -86,7 +86,7 @@ int ObDDLRedoLogReplayer::replay_redo(const ObDDLRedoLog &log, const SCN &scn) LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("failed to init ddl redo log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.get_redo_info().table_key_.tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.get_redo_info().table_key_.tablet_id_))) { if (OB_NO_NEED_UPDATE == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { @@ -108,7 +108,7 @@ int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const SCN &sc LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("init replay executor failed", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.get_table_key().tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.get_table_key().tablet_id_))) { LOG_WARN("execute replay execute failed", K(ret)); } return ret; @@ -123,11 +123,11 @@ int ObDDLRedoLogReplayer::replay_split_start(const ObTabletSplitStartLog &log, c LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("failed to init ddl commit log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.basic_info_.source_tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.basic_info_.source_tablet_id_))) { if (OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay split start log", K(ret), K(scn), K(log), K(ls_->get_ls_id())); + LOG_ERROR("failed to replay split start log", K(ret), K(scn), K(log)); } } return ret; @@ -163,11 +163,11 @@ int ObDDLRedoLogReplayer::replay_split_finish(const ObTabletSplitFinishLog &log, LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("failed to init ddl commit log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.basic_info_.source_tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.basic_info_.source_tablet_id_))) { if (OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay split finish log", K(ret), K(scn), K(log), K(ls_->get_ls_id())); + LOG_ERROR("failed to replay split finish log", K(ret), K(scn), K(log)); } } return ret; @@ -183,11 +183,11 @@ int ObDDLRedoLogReplayer::replay_tablet_freeze(const ObTabletFreezeLog &log, con LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret)); } else if (OB_FAIL(replay_executor.init(ls_, log, scn))) { LOG_WARN("failed to init tablet freeze log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), log.tablet_id_))) { + } else if (OB_FAIL(replay_executor.execute(scn, log.tablet_id_))) { if (OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay tablet freeze log", K(ret), K(scn), K(log), K(ls_->get_ls_id())); + LOG_ERROR("failed to replay tablet freeze log", K(ret), K(scn), K(log)); } } return ret; @@ -207,11 +207,11 @@ int ObDDLRedoLogReplayer::replay_table_fork_freeze(const ObTableForkFreezeLog &l const ObSArray &tablet_ids = log.tablet_ids_; for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + if (OB_FAIL(replay_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay table fork freeze log", K(ret), K(scn), K(log), K(ls_->get_ls_id()), K(tablet_id)); + LOG_ERROR("failed to replay table fork freeze log", K(ret), K(scn), K(log), K(tablet_id)); } } } @@ -234,11 +234,11 @@ int ObDDLRedoLogReplayer::replay_table_fork_start(const ObTableForkStartLog &log const ObSEArray &source_tablet_ids = log.fork_info_.source_tablet_ids_; for (int64_t i = 0; OB_SUCC(ret) && i < source_tablet_ids.count(); ++i) { const ObTabletID &tablet_id = source_tablet_ids.at(i); - if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + if (OB_FAIL(replay_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay table fork start log", K(ret), K(scn), K(log), K(ls_->get_ls_id()), K(tablet_id)); + LOG_ERROR("failed to replay table fork start log", K(ret), K(scn), K(log), K(tablet_id)); } } } @@ -260,11 +260,11 @@ int ObDDLRedoLogReplayer::replay_table_fork_finish(const ObTableForkFinishLog &l const ObSEArray &source_tablet_ids = log.fork_info_.source_tablet_ids_; for (int64_t i = 0; OB_SUCC(ret) && i < source_tablet_ids.count(); ++i) { const ObTabletID &tablet_id = source_tablet_ids.at(i); - if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + if (OB_FAIL(replay_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret || OB_TASK_EXPIRED == ret) { ret = OB_SUCCESS; } else if (OB_EAGAIN != ret) { - LOG_ERROR("failed to replay fork table finish log", K(ret), K(scn), K(log), K(ls_->get_ls_id()), K(tablet_id)); + LOG_ERROR("failed to replay fork table finish log", K(ret), K(scn), K(log), K(tablet_id)); } } } @@ -308,7 +308,7 @@ int ObDDLRedoLogReplayer::do_replay_inc_start(const ObTabletID &tablet_id, const IncType replay_executor; if (OB_FAIL(replay_executor.init(ls_, tablet_id, scn, std::forward(args)...))) { STORAGE_LOG(WARN, "failed to init ddl inc major start log replay executor", KR(ret), K(tablet_id), K(scn)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + } else if (OB_FAIL(replay_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret) { FLOG_INFO("no need to replay ddl inc major start log", KR(ret)); ret = OB_SUCCESS; @@ -328,7 +328,7 @@ int ObDDLRedoLogReplayer::do_replay_inc_minor_commit( ObDDLIncMinorCommitReplayExecutor replay_executor; if (OB_FAIL(replay_executor.init(ls_, tablet_id, scn))) { STORAGE_LOG(WARN, "failed to init ddl inc minor commit log replay executor", KR(ret), K(tablet_id), K(scn)); - } else if (OB_FAIL(replay_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + } else if (OB_FAIL(replay_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret || OB_NO_NEED_UPDATE == ret) { FLOG_INFO("no need to replay ddl inc minor commit log", KR(ret)); ret = OB_SUCCESS; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index 12e9a4603..04a6c4f4b 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -1,5 +1,4 @@ #include "lib/stat/ob_diagnostic_info_guard.h" -#include "share/ob_ex_rpc.h" #include "share/rc/ob_module_provider.h" /* * Copyright (c) 2025 OceanBase. @@ -20,9 +19,8 @@ #define USING_LOG_PREFIX STORAGE #include "ob_ddl_redo_log_writer.h" -#include "storage/ob_storage_rpc.h" -#include "storage/ob_storage_rpc_arg.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/tx/ob_ts_mgr.h" #include "storage/ddl/ob_ddl_merge_task.h" #include "storage/ddl/ob_direct_insert_sstable_ctx_new.h" @@ -57,23 +55,19 @@ bool ObDDLIncNeedStopWriteChecker::check_need_stop_write() return ret_value; } -int ObDDLCtrlSpeedItem::init(const share::ObLSID &ls_id) +int ObDDLCtrlSpeedItem::init() { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("inited twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", K(ret), K(ls_id)); } else { - ls_id_ = ls_id; next_available_write_ts_ = ObTimeUtility::current_time(); if (OB_FAIL(refresh())) { LOG_WARN("fail to init write speed and clog disk used threshold", K(ret)); } else { is_inited_ = true; - LOG_INFO("succeed to init ObDDLCtrlSpeedItem", K(ret), K(is_inited_), K(ls_id_), + LOG_INFO("succeed to init ObDDLCtrlSpeedItem", K(ret), K(is_inited_), K(next_available_write_ts_), K(write_speed_), K(disk_used_stop_write_threshold_)); } } @@ -90,19 +84,11 @@ int ObDDLCtrlSpeedItem::refresh() bool force_wait = false; int64_t total_used_space = 0; // for current tenant, used bytes. int64_t total_disk_space = 0; // for current tenant, limit used bytes. - ObLSHandle ls_handle; palf::PalfOptions palf_opt; logservice::ObLogService *log_service = share::g_mp->log_service(); if (OB_ISNULL(log_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("error unexpected, nullptr found", K(ret), KP(log_service)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - if (OB_LS_NOT_EXIST == ret) { - // log stream may be removed during timer refresh task. - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get ls", K(ret), K(ls_id_)); - } } if (OB_FAIL(ret)) { @@ -123,7 +109,7 @@ int ObDDLCtrlSpeedItem::refresh() palf_opt.disk_options_.log_disk_utilization_limit_threshold_); need_stop_write_ = 100.0 * total_used_space / total_disk_space >= disk_used_stop_write_threshold_ ? true : false; } - LOG_DEBUG("current ddl clog write speed", K(ret), K(need_stop_write_), K(ls_id_), K(archive_speed), K(write_speed_), + LOG_DEBUG("current ddl clog write speed", K(ret), K(need_stop_write_), K(archive_speed), K(write_speed_), K(total_used_space), K(total_disk_space), K(disk_used_stop_write_threshold_), K(refresh_speed)); return ret; } @@ -154,30 +140,6 @@ int ObDDLCtrlSpeedItem::cal_limit(const int64_t bytes, int64_t &next_available_t return ret; } -int ObDDLCtrlSpeedItem::check_cur_node_is_leader(bool &is_leader) -{ - int ret = OB_SUCCESS; - is_leader = true; - ObRole role = INVALID_ROLE; - ObLS *ls = nullptr; - ObLSHandle handle; - ObLSService *ls_svr = share::g_mp->ls_service(); - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls_svr is nullptr", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id_, handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("fail to get ls handle", K(ret), K_(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is nullptr", K(ret)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get ls role failed", K(ret)); - } else if (role != ObRole::LEADER) { - is_leader = false; - } - return ret; -} - int ObDDLCtrlSpeedItem::do_sleep( const int64_t next_available_ts, const int64_t task_id, @@ -229,7 +191,7 @@ int ObDDLCtrlSpeedItem::do_sleep( } if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { ObTaskController::get().allow_next_syslog(); - FLOG_INFO("stop write ddl clog", K(ret), K(ls_id_), + FLOG_INFO("stop write ddl clog", K(ret), K(write_speed_), K(need_stop_write_), K(ref_cnt_), K(disk_used_stop_write_threshold_)); } @@ -256,16 +218,7 @@ int ObDDLCtrlSpeedItem::check_need_stop_write(ObDDLNeedStopWriteChecker &checker ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - bool is_leader = true; - if (OB_FAIL(check_cur_node_is_leader(is_leader))) { - LOG_WARN("check cur node is leader failed", K(ret)); - } else { - if (is_leader) { - is_need_stop_write = (checker.check_need_stop_write() || need_stop_write_); - } else { - is_need_stop_write = false; - } - } + is_need_stop_write = checker.check_need_stop_write() || need_stop_write_; } return ret; } @@ -334,8 +287,7 @@ int ObDDLCtrlSpeedHandle::init(common::ObTimer &timer) return ret; } -int ObDDLCtrlSpeedHandle::limit_and_sleep(const share::ObLSID &ls_id, - const int64_t bytes, +int ObDDLCtrlSpeedHandle::limit_and_sleep(const int64_t bytes, const int64_t task_id, ObDDLNeedStopWriteChecker &checker, int64_t &real_sleep_us) @@ -344,15 +296,15 @@ int ObDDLCtrlSpeedHandle::limit_and_sleep(const share::ObLSID &ls_id, if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if(OB_UNLIKELY(false || !ls_id.is_valid() || bytes < 0 || 0 == task_id)) { + } else if(OB_UNLIKELY(bytes < 0 || 0 == task_id)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_id), K(bytes)); + LOG_WARN("invalid argument", K(ret), K(task_id), K(bytes)); } else if (OB_FAIL(DDL_SIM(task_id, WRITE_DUPLICATED_DDL_REDO_LOG))) { LOG_WARN("ddl sim remote write", K(ret), K(task_id)); } else ; - if (OB_SUCC(ret) && OB_FAIL(speed_handle_item_.init(ls_id))) { + if (OB_SUCC(ret) && OB_FAIL(speed_handle_item_.init())) { if (OB_INIT_TWICE != ret) { - LOG_WARN("fail to init speed handle item", K(ret), K(ls_id)); + LOG_WARN("fail to init speed handle item", K(ret)); } else { ret = OB_SUCCESS; // already inited, treat as success } @@ -450,7 +402,6 @@ int ObDDLRedoLock::init() int ObDDLRedoLogWriter::local_write_ddl_macro_redo( const ObDDLMacroBlockRedoInfo &redo_info, - const share::ObLSID &ls_id, const int64_t task_id, logservice::ObLogHandler *log_handler, const blocksstable::MacroBlockId ¯o_block_id, @@ -478,7 +429,6 @@ int ObDDLRedoLogWriter::local_write_ddl_macro_redo( int64_t real_sleep_us = 0; int tmp_ret = OB_SUCCESS; - ObLSHandle ls_handle; ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; @@ -487,29 +437,25 @@ int ObDDLRedoLogWriter::local_write_ddl_macro_redo( || nullptr == log_handler || false || nullptr == buffer - || 0 == task_id - || !ls_id.is_valid())) { + || 0 == task_id)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(redo_info), KP(log_handler), KP(buffer), K(task_id), K(ls_id)); + LOG_WARN("invalid arguments", K(ret), K(redo_info), KP(log_handler), KP(buffer), K(task_id)); } else if (OB_FAIL(log.init(redo_info))) { LOG_WARN("fail to init DDLRedoLog", K(ret), K(redo_info)); } else if (FALSE_IT(buffer_size = base_header.get_serialize_size() + ddl_header.get_serialize_size() + log.get_serialize_size())) { - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (OB_FAIL(ls->get_tablet(log.get_redo_info().table_key_.tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(log.get_redo_info())); + LOG_WARN("get tablet handle failed", K(ret), K(log.get_redo_info())); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, ObDDLUtil::use_idempotent_mode()))) { LOG_WARN("create ddl kv mgr failed", K(ret)); } else { ObDDLFullNeedStopWriteChecker checker(ddl_kv_mgr_handle); - if (OB_TMP_FAIL(ObDDLCtrlSpeedHandle::get_instance().limit_and_sleep(ls_id, buffer_size, task_id, checker, real_sleep_us))) { - LOG_WARN("fail to limit and sleep", K(tmp_ret), K(task_id), K(ls_id), K(buffer_size), K(real_sleep_us)); + if (OB_TMP_FAIL(ObDDLCtrlSpeedHandle::get_instance().limit_and_sleep(buffer_size, task_id, checker, real_sleep_us))) { + LOG_WARN("fail to limit and sleep", K(tmp_ret), K(task_id), K(buffer_size), K(real_sleep_us)); } } if (OB_FAIL(ret)) { @@ -526,7 +472,7 @@ int ObDDLRedoLogWriter::local_write_ddl_macro_redo( } else if (OB_FAIL(tmp_log.deserialize(buffer, buffer_size, log_start_pos))) { LOG_WARN("fail to deserialize ddl redo log", K(ret)); /* use the ObString data_buffer_ in tmp_log.redo_info_, do not rely on the macro_block_buf in original log*/ - } else if (OB_FAIL(cb->init(ls_id, tmp_log.get_redo_info(), macro_block_id, tablet_handle, tmp_log.get_redo_info().type_))) { + } else if (OB_FAIL(cb->init(tmp_log.get_redo_info(), macro_block_id, tablet_handle, tmp_log.get_redo_info().type_))) { LOG_WARN("init ddl clog callback failed", K(ret), K(redo_info), K(tmp_log.get_redo_info()), K(macro_block_id)); } else if (OB_FAIL(DDL_SIM(task_id, DDL_REDO_WRITER_WRITE_MACRO_LOG_FAILED))) { LOG_WARN("ddl sim failure", K(ret), K(task_id)); @@ -555,7 +501,7 @@ int ObDDLRedoLogWriter::local_write_ddl_macro_redo( int ObDDLRedoLogWriter::local_write_ddl_start_log( const ObDDLStartLog &log, - ObLSHandle &ls_handle, + ObLS *ls, ObLogHandler *log_handler, ObDDLKvMgrHandle &ddl_kv_mgr_handle, ObDDLKvMgrHandle &lob_kv_mgr_handle, @@ -580,7 +526,6 @@ int ObDDLRedoLogWriter::local_write_ddl_start_log( const bool need_nonblock= false; const bool allow_compression = false; SCN scn = SCN::min_scn(); - bool is_external_consistent = false; ObDDLRedoLockGuard guard(log.get_table_key().get_tablet_id().hash()); if (OB_ISNULL(cb = op_alloc(ObDDLStartClogCb))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -594,9 +539,9 @@ int ObDDLRedoLogWriter::local_write_ddl_start_log( LOG_WARN("fail to seriaize ddl start log", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to seriaize ddl start log", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(log.get_table_key().get_tablet_id()))) { + } else if (OB_FAIL(ls->get_ddl_log_handler()->add_tablet(log.get_table_key().get_tablet_id()))) { LOG_WARN("add tablet failed", K(ret), "tablet_id", log.get_table_key().get_tablet_id()); - } else if (lob_kv_mgr_handle.is_valid() && OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(lob_kv_mgr_handle.get_obj()->get_tablet_id()))) { + } else if (lob_kv_mgr_handle.is_valid() && OB_FAIL(ls->get_ddl_log_handler()->add_tablet(lob_kv_mgr_handle.get_obj()->get_tablet_id()))) { LOG_WARN("add lob tablet failed", K(ret), "lob_tablet_id", lob_kv_mgr_handle.get_obj()->get_tablet_id()); } else if (OB_FAIL(log_handler->append(buffer, buffer_size, @@ -607,10 +552,6 @@ int ObDDLRedoLogWriter::local_write_ddl_start_log( lsn, scn))) { LOG_ERROR("fail to submit ddl start log", K(ret), K(buffer_size)); - if (ObDDLUtil::need_remote_write(ret)) { - ret = OB_NOT_MASTER; - LOG_INFO("overwrite return to OB_NOT_MASTER"); - } } else { ObDDLStartClogCb *tmp_cb = cb; cb = nullptr; @@ -653,7 +594,6 @@ int ObDDLRedoLogWriter::local_write_ddl_start_log( int ObDDLRedoLogWriter::local_write_ddl_commit_log( const ObDDLCommitLog &log, const ObDDLClogType clog_type, - const share::ObLSID &ls_id, ObLogHandler *log_handler, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, ObTabletDirectLoadMgrHandle &lob_direct_load_mgr_handle, @@ -678,22 +618,21 @@ int ObDDLRedoLogWriter::local_write_ddl_commit_log( const bool allow_compression = false; SCN base_scn = SCN::min_scn(); SCN scn = SCN::min_scn(); - bool is_external_consistent = false; if (OB_ISNULL(buffer = static_cast(ob_malloc(buffer_size, ObMemAttr("DDL_COMMIT_LOG"))))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); } else if (OB_ISNULL(cb = op_alloc(ObDDLCommitClogCb))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); - } else if (OB_FAIL(cb->init(ls_id, log.get_table_key().tablet_id_, log.get_start_scn(), lock_tid, direct_load_mgr_handle, lob_direct_load_mgr_handle))) { - LOG_WARN("init ddl commit log callback failed", K(ret), K(ls_id), K(log)); + } else if (OB_FAIL(cb->init(log.get_table_key().tablet_id_, log.get_start_scn(), lock_tid, direct_load_mgr_handle, lob_direct_load_mgr_handle))) { + LOG_WARN("init ddl commit log callback failed", K(ret), K(log)); } else if (OB_FAIL(base_header.serialize(buffer, buffer_size, pos))) { LOG_WARN("failed to serialize log base header", K(ret)); } else if (OB_FAIL(ddl_header.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to seriaize ddl commit log", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to seriaize ddl commit log", K(ret)); - } else if (OB_FAIL(OB_TS_MGR.get_ts_sync(ObDDLRedoLogHandle::DDL_REDO_LOG_TIMEOUT, base_scn, is_external_consistent))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts_sync(ObDDLRedoLogHandle::DDL_REDO_LOG_TIMEOUT, base_scn))) { LOG_WARN("fail to get gts sync", K(ret), K(log)); } else if (OB_FAIL(log_handler->append(buffer, buffer_size, @@ -708,17 +647,8 @@ if (OB_ISNULL(buffer = static_cast(ob_malloc(buffer_size, ObMemAttr("DDL ObDDLCommitClogCb *tmp_cb = cb; cb = nullptr; lock_tid = 0; - bool need_retry = true; - while (need_retry) { - if (OB_FAIL(OB_TS_MGR.wait_gts_elapse(scn))) { - if (OB_EAGAIN != ret) { - LOG_WARN("fail to wait gts elapse", K(ret), K(log)); - } else { - ob_usleep(1000); - } - } else { - need_retry = false; - } + if (OB_FAIL(OB_TS_MGR.wait_gts_elapse(scn))) { + LOG_WARN("fail to wait gts elapse", K(ret), K(log)); } if (OB_SUCC(ret)) { handle.cb_ = tmp_cb; @@ -742,7 +672,6 @@ if (OB_ISNULL(buffer = static_cast(ob_malloc(buffer_size, ObMemAttr("DDL template int ObDDLRedoLogWriter::write_auto_split_log( - const share::ObLSID &ls_id, const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const T &log, @@ -763,27 +692,18 @@ int ObDDLRedoLogWriter::write_auto_split_log( palf::LSN lsn; const bool need_nonblock= false; - ObLSHandle ls_handle; ObLS *ls = nullptr; - logservice::ObLogHandler *log_handler = nullptr; - if (OB_UNLIKELY(!ls_id.is_valid()) || - OB_UNLIKELY(ObDDLClogType::DDL_TABLET_SPLIT_START_LOG != clog_type && + if (OB_UNLIKELY(ObDDLClogType::DDL_TABLET_SPLIT_START_LOG != clog_type && ObDDLClogType::DDL_TABLET_SPLIT_FINISH_LOG != clog_type && ObDDLClogType::DDL_TABLET_FREEZE_LOG != clog_type) || OB_UNLIKELY(!log.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(clog_type), K(log)); + LOG_WARN("invalid arg", K(ret), K(clog_type), K(log)); } else if (OB_ISNULL(buffer = static_cast(tmp_arena.alloc(buffer_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc failed", K(ret), K(buffer_size)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(log)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get log handler failed", K(ret), K(log)); } else if (OB_ISNULL(cb = op_alloc(ObDDLClogCb))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); @@ -793,19 +713,15 @@ int ObDDLRedoLogWriter::write_auto_split_log( LOG_WARN("fail to seriaize ddl start log", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to seriaize ddl start log", K(ret)); - } else if (OB_FAIL(log_handler->append(buffer, - buffer_size, - SCN::min_scn(), - need_nonblock, - false/*allow_compression*/, - cb, - lsn, - scn))) { + } else if (OB_FAIL(ls->get_log_handler()->append(buffer, + buffer_size, + SCN::min_scn(), + need_nonblock, + false/*allow_compression*/, + cb, + lsn, + scn))) { LOG_ERROR("fail to submit ddl start log", K(ret), K(buffer_size)); - if (ObDDLUtil::need_remote_write(ret)) { - ret = OB_NOT_MASTER; - LOG_INFO("overwrite return to OB_NOT_MASTER"); - } } else { ObDDLClogCb *tmp_cb = cb; cb = nullptr; @@ -843,29 +759,25 @@ int ObDDLRedoLogWriter::write_auto_split_log( "clog_type", clog_type, "scn", scn, "trace_id", *ObCurTraceId::get_trace_id()); - LOG_INFO("write split log finished", K(ret), K(ls_id), K(clog_type), K(scn)); + LOG_INFO("write split log finished", K(ret), K(clog_type), K(scn)); return ret; } -template int ObDDLRedoLogWriter::write_auto_split_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_split_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTabletSplitStartLog &log, SCN &scn); -template int ObDDLRedoLogWriter::write_auto_split_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_split_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTabletSplitFinishLog &log, SCN &scn); -template int ObDDLRedoLogWriter::write_auto_split_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_split_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTabletFreezeLog &log, SCN &scn); template int ObDDLRedoLogWriter::write_auto_fork_log( - const share::ObLSID &ls_id, const ObDDLClogType &clog_type, const logservice::ObReplayBarrierType &replay_barrier_type, const T &log, @@ -886,27 +798,18 @@ int ObDDLRedoLogWriter::write_auto_fork_log( palf::LSN lsn; const bool need_nonblock = false; - ObLSHandle ls_handle; ObLS *ls = nullptr; - logservice::ObLogHandler *log_handler = nullptr; - if (OB_UNLIKELY(!ls_id.is_valid()) || - OB_UNLIKELY(ObDDLClogType::DDL_TABLE_FORK_FREEZE_LOG != clog_type && + if (OB_UNLIKELY(ObDDLClogType::DDL_TABLE_FORK_FREEZE_LOG != clog_type && ObDDLClogType::DDL_TABLE_FORK_START_LOG != clog_type && ObDDLClogType::DDL_TABLE_FORK_FINISH_LOG != clog_type) || OB_UNLIKELY(!log.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(clog_type), K(log)); + LOG_WARN("invalid arg", K(ret), K(clog_type), K(log)); } else if (OB_ISNULL(buffer = static_cast(tmp_arena.alloc(buffer_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc failed", K(ret), K(buffer_size)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), K(log)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get log handler failed", K(ret), K(log)); } else if (OB_ISNULL(cb = op_alloc(ObDDLClogCb))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); @@ -916,19 +819,15 @@ int ObDDLRedoLogWriter::write_auto_fork_log( LOG_WARN("fail to serialize ddl header", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to serialize fork log", K(ret)); - } else if (OB_FAIL(log_handler->append(buffer, - buffer_size, - SCN::min_scn(), - need_nonblock, - false/*allow_compression*/, - cb, - lsn, - scn))) { + } else if (OB_FAIL(ls->get_log_handler()->append(buffer, + buffer_size, + SCN::min_scn(), + need_nonblock, + false/*allow_compression*/, + cb, + lsn, + scn))) { LOG_ERROR("fail to submit ddl fork log", K(ret), K(buffer_size)); - if (ObDDLUtil::need_remote_write(ret)) { - ret = OB_NOT_MASTER; - LOG_INFO("overwrite return to OB_NOT_MASTER"); - } } else { ObDDLClogCb *tmp_cb = cb; cb = nullptr; @@ -967,31 +866,23 @@ int ObDDLRedoLogWriter::write_auto_fork_log( "replay_barrier", replay_barrier_type, "scn", scn, "trace_id", *ObCurTraceId::get_trace_id()); - LOG_INFO("write fork log finished", K(ret), K(ls_id), K(source_tablet_ids), K(clog_type), K(replay_barrier_type), K(scn)); + LOG_INFO("write fork log finished", K(ret), K(source_tablet_ids), K(clog_type), K(replay_barrier_type), K(scn)); return ret; } -template int ObDDLRedoLogWriter::write_auto_fork_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_fork_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTableForkFreezeLog &log, SCN &scn); -template int ObDDLRedoLogWriter::write_auto_fork_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_fork_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTableForkStartLog &log, SCN &scn); -template int ObDDLRedoLogWriter::write_auto_fork_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, +template int ObDDLRedoLogWriter::write_auto_fork_log(const ObDDLClogType &clog_type, const ObReplayBarrierType &replay_barrier_type, const ObTableForkFinishLog &log, SCN &scn); -bool ObDDLRedoLogWriter::need_retry(int ret_code) -{ - return OB_NOT_MASTER == ret_code; -} - ObDDLRedoLogHandle::ObDDLRedoLogHandle() : cb_(nullptr), scn_(SCN::min_scn()) { @@ -1097,80 +988,23 @@ void ObDDLCommitLogHandle::reset() } } -int ObDDLRedoLogWriter::remote_write_ddl_macro_redo( - const int64_t task_id, - const ObDDLMacroBlockRedoInfo &redo_info) -{ - int ret = OB_SUCCESS; - const int64_t wait_timeout_us = MAX(ObDDLRedoLogHandle::DDL_REDO_LOG_TIMEOUT, GCONF.rpc_timeout); - if (OB_UNLIKELY(!redo_info.is_valid() || 0 == task_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(redo_info)); - } else { - obcall::ObCallRemoteWriteDDLRedoLogArg arg; - if (OB_FAIL(arg.init(leader_ls_id_, redo_info, task_id))) { - LOG_WARN("fail to init arg", K(ret)); - } else if (OB_FAIL(ex_rpc::sync_call([&]() -> int { - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(arg)); - } else { - MOD_SCOPE { - ObRole role = INVALID_ROLE; - ObDDLRedoLogWriter sstable_redo_writer; - MacroBlockId macro_block_id; - ObLSService *ls_service = share::g_mp->ls_service(); - blocksstable::ObMacroBlockHandle macro_handle; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (ObRole::LEADER != role) { - ret = OB_NOT_MASTER; - LOG_INFO("not leader", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ObDDLRedoLogWriter::write_block_to_disk(arg.redo_info_, arg.ls_id_, macro_handle, macro_block_id))) { - LOG_WARN("failed to write block to disk", K(ret)); - } else if (OB_FAIL(sstable_redo_writer.init(arg.ls_id_, arg.redo_info_.table_key_.tablet_id_))) { - LOG_WARN("init sstable redo writer", K(ret), K(arg)); - } else if (OB_FAIL(sstable_redo_writer.write_macro_block_log(arg.redo_info_, macro_block_id, false, arg.task_id_))) { - LOG_WARN("fail to write macro redo", K(ret), K(arg), K(macro_block_id)); - } else if (OB_FAIL(sstable_redo_writer.wait_macro_block_log_finish(arg.redo_info_, macro_block_id))) { - LOG_WARN("fail to wait macro redo finish", K(ret), K(arg)); - } - } - } - return ret;}))) { - LOG_WARN("fail to write ddl redo log", K(ret), K_(leader_addr), K(arg)); - } - } - return ret; -} - ObDDLRedoLogWriter::ObDDLRedoLogWriter() - : is_inited_(false), remote_write_(false), - ls_id_(), tablet_id_(), ddl_redo_handle_array_(), leader_addr_(), leader_ls_id_(), buffer_(nullptr), allocator_(ObMemAttr("DldTabletMeta")), shared_tablet_() + : is_inited_(false), tablet_id_(), ddl_redo_handle_array_(), buffer_(nullptr), + allocator_(ObMemAttr("DldTabletMeta")), shared_tablet_() { ddl_redo_handle_array_.set_attr(lib::ObMemAttr("DdlWriteHdl")); } -int ObDDLRedoLogWriter::init(const ObLSID &ls_id, const ObTabletID &tablet_id) +int ObDDLRedoLogWriter::init(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ddl redo log writer has been inited twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid arguments", K(ret), K(tablet_id)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; is_inited_ = true; } @@ -1180,12 +1014,8 @@ int ObDDLRedoLogWriter::init(const ObLSID &ls_id, const ObTabletID &tablet_id) void ObDDLRedoLogWriter::reset() { is_inited_ = false; - remote_write_ = false; - ls_id_.reset(); tablet_id_.reset(); ddl_redo_handle_array_.reuse(); - leader_addr_.reset(); - leader_ls_id_.reset(); } int ObDDLRedoLogWriter::write_start_log( @@ -1202,7 +1032,6 @@ int ObDDLRedoLogWriter::write_start_log( int ret = OB_SUCCESS; ObDDLStartLog log; ObLS *ls = nullptr; - ObLSHandle ls_handle; ObTabletHandle tablet_handle; start_scn.set_min(); if (OB_UNLIKELY(!is_inited_)) { @@ -1214,14 +1043,11 @@ int ObDDLRedoLogWriter::write_start_log( } else if (OB_FAIL(log.init(table_key, data_format_version, execution_id, direct_load_type, lob_kv_mgr_handle.is_valid() ? lob_kv_mgr_handle.get_obj()->get_tablet_id() : ObTabletID()))) { LOG_WARN("fail to init DDLStartLog", K(ret), K(table_key), K(execution_id), K(data_format_version)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(table_key)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); /*} else if (OB_FAIL(DDL_SIM(ddl_task_id, DDL_REDO_WRITER_WRITE_START_LOG_FAILED))) { LOG_WARN("ddl sim failure", K(ret), K(ddl_task_id));*/ - } else if (OB_FAIL(local_write_ddl_start_log(log, ls_handle, ls->get_log_handler(), + } else if (OB_FAIL(local_write_ddl_start_log(log, ls, ls->get_log_handler(), ddl_kv_mgr_handle, lob_kv_mgr_handle, direct_load_mgr_handle, lock_tid, start_scn))) { LOG_WARN("fail to write ddl start log", K(ret), K(table_key)); } else { @@ -1239,11 +1065,9 @@ int ObDDLRedoLogWriter::write_start_log( int ObDDLRedoLogWriter::write_macro_block_log( const ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, - const bool allow_remote_write, const int64_t task_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; ObLS *ls = nullptr; const int64_t BUF_SIZE = 2 * 1024 * 1024 + 16 * 1024; if (OB_UNLIKELY(!is_inited_)) { @@ -1252,37 +1076,19 @@ int ObDDLRedoLogWriter::write_macro_block_log( } else if (OB_UNLIKELY(!redo_info.is_valid() || 0 == task_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(redo_info), K(task_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (nullptr == buffer_ && OB_ISNULL(buffer_ = static_cast(ob_malloc(BUF_SIZE, ObMemAttr("DDL_REDO_LOG"))))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret), K(BUF_SIZE)); - } else if (!remote_write_) { - if (OB_FAIL(ddl_redo_handle_array_.push_back(ObDDLRedoLogHandle()))) { - LOG_WARN("failed to push back new redo log handle", K(ret)); - } else if (OB_FAIL(local_write_ddl_macro_redo(redo_info, ls->get_ls_id(), task_id, ls->get_log_handler(), macro_block_id, buffer_, - ddl_redo_handle_array_.at(ddl_redo_handle_array_.count() - 1)))) { - if (ObDDLUtil::need_remote_write(ret) && allow_remote_write) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("fail to switch to remote write", K(ret)); - } - } else { - LOG_ERROR("fail to write ddl redo clog", K(ret), K(MTL_GET_TENANT_ROLE_CACHE())); - } - } else { - LOG_INFO("local write redo log of macro block", K(redo_info), K(macro_block_id)); - } - } - - if (OB_SUCC(ret) && remote_write_) { - if (OB_FAIL(retry_remote_write_macro_redo(task_id, redo_info))) { - LOG_WARN("remote write redo failed", K(ret), K(task_id)); - } else { - LOG_INFO("remote write redo log of macro block", K(redo_info), K(macro_block_id)); - } + } else if (OB_FAIL(ddl_redo_handle_array_.push_back(ObDDLRedoLogHandle()))) { + LOG_WARN("failed to push back new redo log handle", K(ret)); + } else if (OB_FAIL(local_write_ddl_macro_redo(redo_info, task_id, + ls->get_log_handler(), macro_block_id, buffer_, + ddl_redo_handle_array_.at(ddl_redo_handle_array_.count() - 1)))) { + LOG_ERROR("fail to write ddl redo clog", K(ret)); + } else { + LOG_INFO("local write redo log of macro block", K(redo_info), K(macro_block_id)); } return ret; } @@ -1298,8 +1104,7 @@ int ObDDLRedoLogWriter::wait_macro_block_log_finish( LOG_WARN("ddl redo log writer has not been inited", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ddl_redo_handle_array_.count(); i++) { - if (OB_ISNULL(ddl_redo_handle_array_.at(i).cb_)) { /* cb be null in remote write */ - } else if (!ddl_redo_handle_array_.at(i).is_valid()) { + if (!ddl_redo_handle_array_.at(i).is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid handle", K(ret), K(ddl_redo_handle_array_.at(i))); } else if (OB_FAIL(ddl_redo_handle_array_.at(i).wait())) { @@ -1316,45 +1121,12 @@ int ObDDLRedoLogWriter::wait_macro_block_log_finish( return ret; } -int ObDDLRedoLogWriter::write_commit_log_with_retry( - const bool allow_remote_write, - const ObITable::TableKey &table_key, - const share::SCN &start_scn, - ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, - ObTabletHandle &tablet_handle, - SCN &commit_scn, - bool &is_remote_write, - uint32_t &lock_tid) -{ - int ret = OB_SUCCESS; - int64_t start_ts = ObTimeUtility::fast_current_time(); - const int64_t timeout_us = ObDDLRedoLogWriter::DEFAULT_RETRY_TIMEOUT_US; - int64_t retry_count = 0; - do { - if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status failed", K(ret)); - } else if (OB_FAIL(write_commit_log(allow_remote_write, table_key, start_scn, direct_load_mgr_handle, tablet_handle, commit_scn, is_remote_write, lock_tid))) { - LOG_WARN("write ddl commit log failed", K(ret)); - } - if (ObDDLRedoLogWriter::need_retry(ret)) { - ob_usleep(1000L * 1000L); // 1s - ++retry_count; - LOG_INFO("retry write ddl commit log", K(ret), K(table_key), K(retry_count)); - } else { - break; - } - } while (ObTimeUtility::fast_current_time() - start_ts < timeout_us); - return ret; -} - int ObDDLRedoLogWriter::write_commit_log( - const bool allow_remote_write, const ObITable::TableKey &table_key, const share::SCN &start_scn, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, ObTabletHandle &tablet_handle, SCN &commit_scn, - bool &is_remote_write, uint32_t &lock_tid) { int ret = OB_SUCCESS; @@ -1364,8 +1136,6 @@ int ObDDLRedoLogWriter::write_commit_log( DEBUG_SYNC(BEFORE_DDL_WRITE_PREPARE_LOG); #endif commit_scn.set_min(); - is_remote_write = false; - ObLSHandle ls_handle; ObLS *ls = nullptr; ObDDLCommitLog log; ObDDLCommitLogHandle handle; @@ -1380,18 +1150,15 @@ int ObDDLRedoLogWriter::write_commit_log( LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_handle)); } else if (OB_FAIL(log.init(table_key, start_scn, ddl_data.lob_meta_tablet_id_))) { LOG_WARN("fail to init DDLCommitLog", K(ret), K(table_key), K(start_scn), K(ddl_data.lob_meta_tablet_id_)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(table_key)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (start_scn != direct_load_mgr_handle.get_obj()->get_start_scn()) { ret = OB_TASK_EXPIRED; LOG_WARN("current task is restarted", K(ret), K(start_scn), "current_start_scn", direct_load_mgr_handle.get_obj()->get_start_scn()); } else if (direct_load_mgr_handle.get_obj()->get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()).is_valid_and_not_min()) { commit_scn = direct_load_mgr_handle.get_obj()->get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()); LOG_WARN("already committed", K(ret), K(start_scn), K(commit_scn), K(direct_load_mgr_handle.get_obj()->get_start_scn()), K(log)); - } else if (!remote_write_) { + } else { // direct load mgr handle of lob meta tablet may not bind to data tablet handle, get it manually here ObTabletBindingMdsUserData ddl_data; ObTabletDirectLoadMgrHandle lob_direct_load_mgr_handle; @@ -1399,7 +1166,7 @@ int ObDDLRedoLogWriter::write_commit_log( LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_handle)); } else if (ddl_data.lob_meta_tablet_id_.is_valid()) { bool is_lob_major_sstable_exist = false; - if (OB_FAIL(share::g_mp->tenant_direct_load_mgr()->get_tablet_mgr_and_check_major(ls_id_, ddl_data.lob_meta_tablet_id_, + if (OB_FAIL(share::g_mp->tenant_direct_load_mgr()->get_tablet_mgr_and_check_major(ddl_data.lob_meta_tablet_id_, true/* is_full_direct_load */, lob_direct_load_mgr_handle, is_lob_major_sstable_exist))) { if (OB_ENTRY_NOT_EXIST == ret && is_lob_major_sstable_exist) { ret = OB_SUCCESS; @@ -1411,14 +1178,9 @@ int ObDDLRedoLogWriter::write_commit_log( } if (OB_FAIL(ret)) { } else if (OB_FAIL(local_write_ddl_commit_log( - log, ObDDLClogType::DDL_COMMIT_LOG, ls_id_, ls->get_log_handler(), direct_load_mgr_handle, lob_direct_load_mgr_handle, handle, lock_tid))) { - if (ObDDLUtil::need_remote_write(ret) && allow_remote_write) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("fail to switch to remote write", K(ret), K(table_key)); - } - } else { - LOG_ERROR("fail to write ddl commit log", K(ret), K(table_key)); - } + log, ObDDLClogType::DDL_COMMIT_LOG, ls->get_log_handler(), + direct_load_mgr_handle, lob_direct_load_mgr_handle, handle, lock_tid))) { + LOG_ERROR("fail to write ddl commit log", K(ret), K(table_key)); } else if (OB_FAIL(handle.wait())) { LOG_WARN("wait ddl commit log finish failed", K(ret), K(table_key)); } else { @@ -1426,200 +1188,16 @@ int ObDDLRedoLogWriter::write_commit_log( LOG_INFO("local write ddl commit log", K(ret), K(table_key), K(commit_scn)); } } - if (OB_SUCC(ret) && remote_write_) { - obcall::ObCallRemoteWriteDDLCommitLogArg arg; - if (OB_FAIL(arg.init(leader_ls_id_, table_key, start_scn))) { - LOG_WARN("fail to init ObCallRemoteWriteDDLCommitLogArg", K(ret)); - } else if (OB_FAIL(retry_remote_write_commit_clog(arg, commit_scn))) { - LOG_WARN("remote write ddl commit log failed", K(ret), K(arg)); - } else { - is_remote_write = !(leader_addr_ == GCTX.self_addr()); - LOG_INFO("remote write ddl commit log", K(ret), K(table_key), K(commit_scn), K(is_remote_write)); - } - } SERVER_EVENT_ADD("ddl", "ddl write commit log", "ret", ret, "trace_id", *ObCurTraceId::get_trace_id(), "start_scn", direct_load_mgr_handle.get_obj()->get_start_scn(), "tablet_id", tablet_id_, - "commit_scn", commit_scn, - "info", is_remote_write); + "commit_scn", commit_scn); LOG_INFO("ddl write commit log", K(ret), "ddl_event_info", ObDDLEventInfo()); return ret; } -int ObDDLRedoLogWriter::switch_to_remote_write() -{ - int ret = OB_SUCCESS; - - share::ObLocationService *location_service = nullptr; - bool is_cache_hit = false; - if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location service is null", K(ret), KP(location_service)); - } else if (OB_FAIL(location_service->get(tablet_id_, - INT64_MAX/*expire_renew_time*/, - is_cache_hit, - leader_ls_id_))) { - LOG_WARN("fail to get log stream id", K(ret), K_(tablet_id)); - } else if (OB_FAIL(location_service->get_leader(GCONF.cluster_id, - leader_ls_id_, - true, /*force_renew*/ - leader_addr_))) { - LOG_WARN("get leader failed", K(ret), K(leader_ls_id_)); - } else if (GCTX.self_addr() == leader_addr_) { - ret = OB_NOT_MASTER; // switch to local is unexpected, use retry ret code - remote_write_ = false; - LOG_WARN("leader is local", K(ret), K_(tablet_id), K_(leader_ls_id)); - } else { - remote_write_ = true; - LOG_INFO("switch to remote write", K(ret), K_(tablet_id), K_(leader_ls_id), K_(leader_addr)); - } - return ret; -} - -int ObDDLRedoLogWriter::retry_remote_write_macro_redo( - const int64_t task_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info) -{ - int ret = OB_SUCCESS; - int retry_cnt = 0; - const int64_t MAX_REMOTE_WRITE_RETRY_CNT = 800; - if (OB_UNLIKELY(!redo_info.is_valid() || 0 == task_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(redo_info)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("flush ls leader location failed", K(ret)); - } else if (OB_FAIL(remote_write_ddl_macro_redo(task_id, redo_info))) { - if (OB_NOT_MASTER == ret && retry_cnt++ < MAX_REMOTE_WRITE_RETRY_CNT) { - ob_usleep(10 * 1000); // 10 ms. - ret = OB_SUCCESS; - } else { - LOG_WARN("remote write macro redo failed", K(ret), K_(leader_ls_id), K_(leader_addr)); - } - } else { - break; // remote write ddl clog successfully. - } - } - } - return ret; -} - -int ObDDLRedoLogWriter::retry_remote_write_commit_clog( - const obcall::ObCallRemoteWriteDDLCommitLogArg &arg, - share::SCN &commit_scn) -{ - int ret = OB_SUCCESS; - int retry_cnt = 0; - const int64_t MAX_REMOTE_WRITE_RETRY_CNT = 800; - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(arg)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(switch_to_remote_write())) { - LOG_WARN("flush ls leader location failed", K(ret)); - } else if (OB_FAIL(remote_write_ddl_commit_redo(arg, commit_scn))) { - if (OB_NOT_MASTER == ret && retry_cnt++ < MAX_REMOTE_WRITE_RETRY_CNT) { - ob_usleep(10 * 1000); // 10 ms. - ret = OB_SUCCESS; - } else { - LOG_WARN("remote write macro redo failed", K(ret), K_(leader_ls_id), K_(leader_addr)); - } - } else { - break; // remote write ddl clog successfully. - } - } - } - return ret; -} - -int ObDDLRedoLogWriter::remote_write_ddl_commit_redo(const obcall::ObCallRemoteWriteDDLCommitLogArg &arg, SCN &commit_scn) -{ - int ret = OB_SUCCESS; - obcall::Int64 log_ns; - if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(arg)); - } else if (OB_FAIL(ex_rpc::sync_call([&]() -> int { - int ret = OB_SUCCESS; - MOD_SCOPE { - ObRole role = INVALID_ROLE; - const ObITable::TableKey &table_key = arg.table_key_; - ObDDLRedoLogWriter writer; - ObLSService *ls_svc = share::g_mp->ls_service(); - ObLSHandle ls_hdl; ObLS *ls = nullptr; - auto *tlm = share::g_mp->tenant_direct_load_mgr(); - ObTabletFullDirectLoadMgr *dtm = nullptr; ObTabletDirectLoadMgrHandle dmh; - dmh.reset(); bool major_exist = false; - if (OB_FAIL(ls_svc->get_ls(arg.ls_id_, ls_hdl, ObLSGetMod::OBSERVER_MOD))) { LOG_WARN("get ls failed", K(ret)); } - else if (OB_ISNULL(ls = ls_hdl.get_ls())) { ret = OB_ERR_UNEXPECTED; } - else if (OB_FAIL(ls->get_ls_role(role))) { LOG_WARN("get role failed", K(ret)); } - else if (ObRole::LEADER != role) { ret = OB_NOT_MASTER; } - else if (OB_ISNULL(tlm)) { ret = OB_ERR_UNEXPECTED; } - else if (OB_FAIL(tlm->get_tablet_mgr_and_check_major(arg.ls_id_, table_key.tablet_id_, true, dmh, major_exist))) { - if (OB_ENTRY_NOT_EXIST == ret && major_exist) { ret = OB_TASK_EXPIRED; } - } else if (OB_ISNULL(dtm = dmh.get_full_obj())) { - ret = OB_ERR_UNEXPECTED; - } - else if (OB_FAIL(writer.init(arg.ls_id_, table_key.tablet_id_))) { LOG_WARN("init failed", K(ret)); } - else { - uint32_t lock_tid = 0; SCN scn_val; bool remote = false; ObTabletHandle th; - if (OB_FAIL(dtm->wrlock(ObTabletDirectLoadMgr::TRY_LOCK_TIMEOUT, lock_tid))) { LOG_WARN("wrlock failed", K(ret)); } - else if (OB_FAIL(ls->get_tablet(table_key.tablet_id_, th, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("get tablet failed", K(ret)); } - else if (OB_FAIL(writer.write_commit_log(false, table_key, arg.start_scn_, dmh, th, scn_val, remote, lock_tid))) { LOG_WARN("write commit log failed", K(ret)); } - else if (!dtm->get_lob_mgr_handle().is_valid()) { - ObTabletBindingMdsUserData ddl_data; ObTabletDirectLoadMgrHandle lob_hdl; - if (OB_FAIL(th.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { LOG_WARN("get ddl data failed", K(ret)); } - else if (ddl_data.lob_meta_tablet_id_.is_valid()) { - bool lob_exist = false; - if (OB_FAIL(share::g_mp->tenant_direct_load_mgr()->get_tablet_mgr_and_check_major(arg.ls_id_, ddl_data.lob_meta_tablet_id_, true, lob_hdl, lob_exist))) { - if (OB_ENTRY_NOT_EXIST != ret || !lob_exist) { LOG_WARN("get lob mgr failed", K(ret)); } else { ret = OB_SUCCESS; } - } else if (OB_FAIL(lob_hdl.get_full_obj()->commit(*th.get_obj(), arg.start_scn_, scn_val, arg.table_id_, arg.ddl_task_id_, false))) { LOG_WARN("lob commit failed", K(ret)); } - } - } - if (OB_SUCC(ret) && OB_FAIL(dtm->commit(*th.get_obj(), arg.start_scn_, scn_val, arg.table_id_, arg.ddl_task_id_, false))) { LOG_WARN("kv commit failed", K(ret)); } - else if (OB_SUCC(ret)) { log_ns = scn_val.get_val_for_tx(); } - if (lock_tid != 0) { dtm->unlock(lock_tid); } - } - } - return ret; - }))) { - LOG_WARN("write ddl commit log failed", K(ret), K_(leader_ls_id), K_(leader_addr)); - } else if (OB_FAIL(commit_scn.convert_for_tx(log_ns))) { - LOG_WARN("convert for tx failed", K(ret)); - } - return ret; -} - -int ObDDLRedoLogWriter::write_block_to_disk(const ObDDLMacroBlockRedoInfo &redo_info, const ObLSID &ls_id, - blocksstable::ObMacroBlockHandle ¯o_handle, blocksstable::MacroBlockId ¯o_id) -{ - int ret = OB_SUCCESS; - macro_handle.reset(); - macro_id.reset(); - if (!redo_info.is_valid() || !ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(redo_info), K(ls_id)); - } else { - ObMacroBlockWriteInfo write_info; - write_info.buffer_ = redo_info.data_buffer_.ptr(); - write_info.size_= redo_info.data_buffer_.length(); - write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); - write_info.io_timeout_ms_ = max(DDL_FLUSH_MACRO_BLOCK_TIMEOUT / 1000L, GCONF._data_storage_io_timeout / 1000L); - if (OB_FAIL(ObBlockManager::async_write_block(write_info, macro_handle))) { - LOG_WARN("fail to async write block", K(ret), K(write_info), K(macro_handle)); - } else if (OB_FAIL(macro_handle.wait())) { - LOG_WARN("fail to wait macro block io finish", K(ret)); - } else { - macro_id = macro_handle.get_macro_id(); - } - } - return ret; -} - ObDDLRedoLogWriter::~ObDDLRedoLogWriter() { if (nullptr != buffer_) { @@ -1651,8 +1229,7 @@ ObDDLRedoLogWriterCallbackInitParam::~ObDDLRedoLogWriterCallbackInitParam() bool ObDDLRedoLogWriterCallbackInitParam::is_valid() const { - return ls_id_.is_valid() - && tablet_id_.is_valid() + return tablet_id_.is_valid() && table_key_.is_valid() && (DDL_MB_INVALID_TYPE != block_type_) && (0 != task_id_) @@ -1662,7 +1239,6 @@ bool ObDDLRedoLogWriterCallbackInitParam::is_valid() const void ObDDLRedoLogWriterCallbackInitParam::reset() { - ls_id_.reset(); tablet_id_.reset(); direct_load_type_ = DIRECT_LOAD_INVALID; block_type_ = ObDDLMacroBlockType::DDL_MB_INVALID_TYPE; @@ -1693,27 +1269,23 @@ ObDDLRedoLogWriterCallback::~ObDDLRedoLogWriterCallback() int ObDDLRedoLogWriterCallback::init(ObDDLRedoLogWriterCallbackInitParam &init_param) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObLSService *ls_service = nullptr; - bool is_cache_hit = false; - ObLSHandle ls_handle; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ddl redo log writer has been inited twice", K(ret)); } else if (OB_UNLIKELY(!init_param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid init param", KR(ret), K(init_param)); - } else if (OB_FAIL(ddl_writer_.init(init_param.ls_id_, init_param.tablet_id_))) { - LOG_WARN("fail to init ddl_writer_", K(ret), K(init_param.ls_id_), K(init_param.tablet_id_)); + } else if (OB_FAIL(ddl_writer_.init(init_param.tablet_id_))) { + LOG_WARN("fail to init ddl_writer_", K(ret), K(init_param.tablet_id_)); } else { // init kv mgr handle for idempotence check ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - if (OB_FAIL(ls_service->get_ls(init_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(init_param.ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, init_param.tablet_id_, tablet_handle))) { - LOG_WARN("get tablet failed", K(ret), K(init_param.ls_id_), K(init_param.tablet_id_)); + if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, init_param.tablet_id_, tablet_handle))) { + LOG_WARN("get tablet failed", K(ret), K(init_param.tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(kv_mgr_handle_, true /*try_create*/))) { LOG_WARN("get ddl kv mgr handle failed", K(ret)); } @@ -1818,16 +1390,8 @@ int ObDDLRedoLogWriterCallback::inner_write(const ObDDLMacroBlockRedoInfo &redo_ if (!redo_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(redo_info)); - } else if (OB_FAIL(ddl_writer_.write_macro_block_log(redo_info, redo_info.macro_block_id_, true/*allow remote write*/, param_.task_id_))) { + } else if (OB_FAIL(ddl_writer_.write_macro_block_log(redo_info, redo_info.macro_block_id_, param_.task_id_))) { LOG_ERROR("fail to write ddl redo log", K(ret), K(redo_info), K(param_.task_id_)); - if (ObDDLRedoLogWriter::need_retry(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(retry(ObDDLRedoLogWriter::DEFAULT_RETRY_TIMEOUT_US, redo_info, redo_info.macro_block_id_))) { - LOG_WARN("retry wirte ddl macro redo log failed", K(ret), K(tmp_ret), K(param_.task_id_), K(param_.table_key_)); - } else { - ret = OB_SUCCESS; // overwrite the return code - } - } } return ret; } @@ -1870,42 +1434,3 @@ int ObDDLRedoLogWriterCallback::wait() } return ret; } - -int ObDDLRedoLogWriterCallback::retry(const int64_t timeout_us, - const ObDDLMacroBlockRedoInfo &redo_info, - const blocksstable::MacroBlockId ¯o_block_id) -{ - int ret = OB_SUCCESS; - int64_t retry_count = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDDLRedoLogWriterCallback is not inited", K(ret)); - } else if (timeout_us <= 0) { - ret = OB_TIMEOUT; - LOG_WARN("timeout less than 0", K(ret), K(timeout_us)); - } else if (OB_UNLIKELY(!macro_block_id.is_valid() || !redo_info.is_valid())) { - ret = OB_ERR_SYS; - LOG_WARN("macro block id or redo info not valid", K(ret), K(macro_block_id), K(redo_info)); - } else { - int64_t start_ts = ObTimeUtility::fast_current_time(); - while (ObTimeUtility::fast_current_time() - start_ts < timeout_us) { // ignore ret - if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status failed", K(ret)); - } else if (OB_FAIL(ddl_writer_.write_macro_block_log(redo_info, macro_block_id, true/*allow remote write*/, param_.task_id_))) { - LOG_WARN("fail to write ddl redo log", K(ret)); - } else if (OB_FAIL(ddl_writer_.wait_macro_block_log_finish(redo_info, macro_block_id))) { - LOG_WARN("wait ddl redo log finish failed", K(ret)); - } else { - FLOG_INFO("retry write ddl macro redo success", K(ret), K(param_.table_key_), K(macro_block_id)); - } - if (ObDDLRedoLogWriter::need_retry(ret)) { - ob_usleep(1000L * 1000L); // 1s - ++retry_count; - LOG_INFO("retry write ddl macro redo log", K(ret), K(param_.table_key_), K(retry_count)); - } else { - break; - } - } - } - return ret; -} diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.h b/src/storage/ddl/ob_ddl_redo_log_writer.h index f587b67e3..39e004360 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_redo_log_writer.h @@ -18,13 +18,10 @@ #define OCEANBASE_STORAGE_OB_DDL_REDO_LOG_WRITER_H #include "lib/task/ob_timer.h" #include "common/ob_tablet_id.h" -#include "storage/ob_storage_rpc_arg.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "storage/ddl/ob_ddl_clog.h" #include "storage/ddl/ob_ddl_struct.h" #include "storage/blocksstable/ob_imacro_block_flush_callback.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/blocksstable/ob_logic_macro_id.h" namespace oceanbase @@ -49,7 +46,7 @@ class ObLogHandler; namespace storage { -class ObLSHandle; +class ObLS; class ObDDLNeedStopWriteChecker { public: @@ -83,12 +80,12 @@ class ObDDLIncNeedStopWriteChecker : public ObDDLNeedStopWriteChecker class ObDDLCtrlSpeedItem final { public: - ObDDLCtrlSpeedItem(): is_inited_(false), ls_id_(share::ObLSID::INVALID_LS_ID), + ObDDLCtrlSpeedItem(): is_inited_(false), next_available_write_ts_(-1), write_speed_(750), disk_used_stop_write_threshold_(-1), need_stop_write_(false), ref_cnt_(0) {} ~ObDDLCtrlSpeedItem() {}; void reset_need_stop_write() { need_stop_write_ = false; } - int init(const share::ObLSID &ls_id); + int init(); int refresh(); int limit_and_sleep(const int64_t bytes, const int64_t task_id, @@ -101,10 +98,9 @@ class ObDDLCtrlSpeedItem final int64_t dec_ref() { return ATOMIC_SAF(&ref_cnt_, 1); } int64_t get_ref() { return ATOMIC_LOAD(&ref_cnt_); } - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(next_available_write_ts), + TO_STRING_KV(K_(is_inited), K_(next_available_write_ts), K_(write_speed), K_(disk_used_stop_write_threshold), K_(need_stop_write), K_(ref_cnt)); private: - int check_cur_node_is_leader(bool &is_leader); int cal_limit(const int64_t bytes, int64_t &next_available_ts); int do_sleep(const int64_t next_available_ts, const int64_t task_id, @@ -114,7 +110,6 @@ class ObDDLCtrlSpeedItem final static const int64_t MIN_WRITE_SPEED = 50L; static const int64_t SLEEP_INTERVAL = 1 * 1000; // 1ms bool is_inited_; - share::ObLSID ls_id_; int64_t next_available_write_ts_; int64_t write_speed_; int64_t disk_used_stop_write_threshold_; // stop write threshold on tenant level. @@ -128,8 +123,7 @@ class ObDDLCtrlSpeedHandle final public: int init(common::ObTimer &timer); static ObDDLCtrlSpeedHandle &get_instance(); - int limit_and_sleep(const share::ObLSID &ls_id, - const int64_t bytes, + int limit_and_sleep(const int64_t bytes, const int64_t task_id, ObDDLNeedStopWriteChecker &checker, int64_t &real_sleep_us); @@ -220,9 +214,7 @@ class ObDDLRedoLogWriter final public: ObDDLRedoLogWriter(); ~ObDDLRedoLogWriter(); - int init( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id); + int init(const ObTabletID &tablet_id); void reset(); int write_start_log( const ObITable::TableKey &table_key, @@ -237,55 +229,33 @@ class ObDDLRedoLogWriter final int write_macro_block_log( const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, - const bool allow_remote_write, const int64_t task_id); int wait_macro_block_log_finish( const storage::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id); int write_commit_log( - const bool allow_remote_write, const ObITable::TableKey &table_key, const share::SCN &start_scn, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, ObTabletHandle &tablet_handle, share::SCN &commit_scn, - bool &is_remote_write, uint32_t &lock_tid); template - static int write_auto_split_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, + static int write_auto_split_log(const ObDDLClogType &clog_type, const logservice::ObReplayBarrierType &replay_barrier_type, const T &log, SCN &scn); template - static int write_auto_fork_log(const share::ObLSID &ls_id, - const ObDDLClogType &clog_type, + static int write_auto_fork_log(const ObDDLClogType &clog_type, const logservice::ObReplayBarrierType &replay_barrier_type, const T &log, SCN &scn); - int write_commit_log_with_retry( - const bool allow_remote_write, - const ObITable::TableKey &table_key, - const share::SCN &start_scn, - ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, - ObTabletHandle &tablet_handle, - share::SCN &commit_scn, - bool &is_remote_write, - uint32_t &lock_tid); - static const int64_t DEFAULT_RETRY_TIMEOUT_US = 60L * 1000L * 1000L; // 1min - static bool need_retry(int ret_code); - static int write_block_to_disk(const storage::ObDDLMacroBlockRedoInfo &redo_info, - const share::ObLSID &ls_id, - blocksstable::ObMacroBlockHandle ¯o_handle, - blocksstable::MacroBlockId &block_id); - const share::ObLSID &get_ls_id() const { return ls_id_; } const ObTabletID &get_tablet_id() const { return tablet_id_; } private: - int switch_to_remote_write(); int local_write_ddl_start_log( const ObDDLStartLog &log, - ObLSHandle &ls_handle, + ObLS *ls, logservice::ObLogHandler *log_handler, ObDDLKvMgrHandle &ddl_kv_mgr_handle, ObDDLKvMgrHandle &lob_kv_mgr_handle, @@ -295,40 +265,22 @@ class ObDDLRedoLogWriter final int local_write_ddl_commit_log( const ObDDLCommitLog &log, const ObDDLClogType clog_type, - const share::ObLSID &ls_id, logservice::ObLogHandler *log_handler, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, ObTabletDirectLoadMgrHandle &lob_direct_load_mgr_handle, ObDDLCommitLogHandle &handle, uint32_t &lock_tid); - int remote_write_ddl_commit_redo( - const obcall::ObCallRemoteWriteDDLCommitLogArg &arg, - share::SCN &commit_scn); - int retry_remote_write_macro_redo( - const int64_t task_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info); - int retry_remote_write_commit_clog( - const obcall::ObCallRemoteWriteDDLCommitLogArg &arg, - share::SCN &commit_scn); int local_write_ddl_macro_redo( const storage::ObDDLMacroBlockRedoInfo &redo_info, - const share::ObLSID &ls_id, const int64_t task_id, logservice::ObLogHandler *log_handler, const blocksstable::MacroBlockId ¯o_block_id, char *buffer, ObDDLRedoLogHandle &handle); - int remote_write_ddl_macro_redo( - const int64_t task_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info); private: bool is_inited_; - bool remote_write_; - share::ObLSID ls_id_; ObTabletID tablet_id_; ObSEArray ddl_redo_handle_array_; - ObAddr leader_addr_; - share::ObLSID leader_ls_id_; char *buffer_; ObArenaAllocator allocator_; ObTablet shared_tablet_; @@ -340,12 +292,11 @@ struct ObDDLRedoLogWriterCallbackInitParam ~ObDDLRedoLogWriterCallbackInitParam(); bool is_valid() const; void reset(); - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(direct_load_type), K_(block_type), + TO_STRING_KV(K_(tablet_id), K_(direct_load_type), K_(block_type), K_(table_key), K_(start_scn), K_(task_id), K_(data_format_version), K_(parallel_cnt), K_(need_delay), K_(need_submit_io), K_(merge_slice_idx), KP_(macro_meta_store), KP_(write_stat), KP_(tx_desc), K_(trans_id), K_(seq_no)); - share::ObLSID ls_id_; common::ObTabletID tablet_id_; storage::ObDirectLoadType direct_load_type_; storage::ObDDLMacroBlockType block_type_; @@ -383,9 +334,6 @@ class ObDDLRedoLogWriterCallback : public blocksstable::ObIMacroBlockFlushCallba private: int inner_write(const ObDDLMacroBlockRedoInfo &redo_info); int write_redo_info_array(); - int retry(const int64_t timeout_us, - const ObDDLMacroBlockRedoInfo &redo_info, - const blocksstable::MacroBlockId ¯o_block_id); private: bool is_inited_; ObDDLRedoLogWriterCallbackInitParam param_; diff --git a/src/storage/ddl/ob_ddl_replay_executor.cpp b/src/storage/ddl/ob_ddl_replay_executor.cpp index eca1dbf3a..a48c01ade 100644 --- a/src/storage/ddl/ob_ddl_replay_executor.cpp +++ b/src/storage/ddl/ob_ddl_replay_executor.cpp @@ -49,7 +49,6 @@ int ObDDLReplayExecutor::check_need_replay_ddl_log_( int ret = OB_SUCCESS; need_replay = true; ObTablet *tablet = nullptr; - ObMigrationStatus migration_status; if (OB_UNLIKELY(nullptr == ls || !tablet_handle.is_valid() || (!ObDDLUtil::use_idempotent_mode() && !ddl_start_scn.is_valid_and_not_min()) || !scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("not init", K(ret), KP(ls), K(tablet_handle), K(ddl_start_scn), K(scn)); @@ -87,7 +86,6 @@ int ObDDLReplayExecutor::check_need_replay_ddl_inc_log_( int ret = OB_SUCCESS; need_replay = true; ObTablet *tablet = nullptr; - ObMigrationStatus migration_status; #ifdef ERRSIM if (OB_SUCC(ret)) { ret = CHECK_NEED_REPLAY_DDL_INC_LOG ? : OB_SUCCESS; @@ -111,7 +109,7 @@ int ObDDLReplayExecutor::check_need_replay_ddl_inc_log_( LOG_WARN("fail to check need replay", K(ret), KP(ls), K(tablet_handle)); } else if (!need_replay) { // do nothing - } else if (tablet->get_tablet_meta().ha_status_.is_restore_status_empty()) { + } else if (tablet->get_tablet_meta().restore_state_.is_restore_status_empty()) { ret = OB_EAGAIN; need_replay = false; // TODO(xingzhi): add this fuction where supporting restore @@ -138,11 +136,10 @@ int ObDDLReplayExecutor::check_need_replay_ddl_inc_log_( } } #ifdef ERRSIM - if (tablet->get_tablet_meta().ha_status_.is_restore_status_remote()) { + if (tablet->get_tablet_meta().restore_state_.is_restore_status_remote()) { if (REACH_COUNT_INTERVAL(10000L)) { SERVER_EVENT_ADD("ddl_replay", "check_need_replay_ddl_inc_log", "result", ret, - "ls_id", ls->get_ls_id().id(), "tablet_id", tablet->get_tablet_meta().tablet_id_.id()); } } @@ -159,14 +156,7 @@ int ObDDLReplayExecutor::check_need_replay_( int ret = OB_SUCCESS; need_replay = true; ObTablet *tablet = nullptr; - ObMigrationStatus migration_status; - if (OB_FAIL(ls->get_ls_meta().get_migration_status(migration_status))) { - LOG_WARN("get migration status failed", K(ret), KPC(ls)); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_ADD_FAIL == migration_status - || ObMigrationStatus::OB_MIGRATION_STATUS_MIGRATE_FAIL == migration_status) { - need_replay = false; - LOG_INFO("ls migration failed, so ddl log skip replay", "ls_id", ls->get_ls_id(), K(tablet_handle), K(migration_status)); - } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is null", K(ret), K(tablet_handle)); } else if (tablet->is_empty_shell()) { @@ -175,12 +165,6 @@ int ObDDLReplayExecutor::check_need_replay_( LOG_INFO("no need to replay ddl log, because this tablet is empty shell", K(tablet_handle), "tablet_meta", tablet->get_tablet_meta()); } - } else if (tablet->get_tablet_meta().ha_status_.is_expected_status_deleted()) { - need_replay = false; - if (REACH_COUNT_INTERVAL(1000L)) { - LOG_INFO("no need to replay ddl log, because tablet will be deleted", - K(tablet_handle), "tablet_meta", tablet->get_tablet_meta()); - } } return ret; @@ -277,8 +261,6 @@ int ObDDLStartReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, const bool is_lob_meta_tablet) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; ObTabletDirectLoadMgrHandle direct_load_mgr_handle; ObTenantDirectLoadMgr *tenant_direct_load_mgr = share::g_mp->tenant_direct_load_mgr(); const int64_t unused_context_id = -1; @@ -308,7 +290,6 @@ int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, co direct_load_param.is_replay_ = true; bool is_major_sstable_exist = false; const int64_t snapshot_version = log_->get_table_key().get_snapshot_version(); - direct_load_param.common_param_.ls_id_ = tablet_handle.get_obj()->get_tablet_meta().ls_id_; direct_load_param.common_param_.tablet_id_ = tablet_id; direct_load_param.common_param_.data_format_version_ = log_->get_data_format_version(); direct_load_param.common_param_.direct_load_type_ = log_->get_direct_load_type(); @@ -326,7 +307,6 @@ int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, co if (OB_FAIL(tenant_direct_load_mgr->replay_create_tablet_direct_load(tablet_handle.get_obj(), log_->get_execution_id(), direct_load_param))) { LOG_WARN("create tablet manager failed", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_->get_ls_id(), tablet_id, true/* is_full_direct_load */, direct_load_mgr_handle, @@ -806,7 +786,6 @@ int ObDDLCommitReplayExecutor::replay_ddl_commit(ObTabletHandle &tablet_handle) ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret)); } else if (OB_FAIL(share::g_mp->tenant_direct_load_mgr()->get_tablet_mgr_and_check_major( - ls_->get_ls_id(), tablet_id, true/* is_full_direct_load */, direct_load_mgr_handle, @@ -864,19 +843,17 @@ int ObSplitStartReplayExecutor::init( } int ObSplitStartReplayExecutor::prepare_param_from_log( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObTabletSplitInfo &info, const share::SCN &scn, ObTabletSplitParam ¶m) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !handle.is_valid() || !info.is_valid() || !scn.is_valid_and_not_min())) { + if (OB_UNLIKELY(!handle.is_valid() || !info.is_valid() || !scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(handle), K(info), K(scn)); + LOG_WARN("invalid arg", K(ret), K(handle), K(info), K(scn)); } else { - - param.ls_id_ = ls_id; + param.table_id_ = info.table_id_; param.schema_version_ = info.schema_version_; param.task_id_ = info.task_id_; @@ -899,19 +876,17 @@ int ObSplitStartReplayExecutor::prepare_param_from_log( } int ObSplitStartReplayExecutor::prepare_param_from_log( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObTabletSplitInfo &info, const share::SCN &scn, ObLobSplitParam ¶m) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !handle.is_valid() || !info.is_valid() || !scn.is_valid_and_not_min())) { + if (OB_UNLIKELY(!handle.is_valid() || !info.is_valid() || !scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(handle), K(info), K(scn)); + LOG_WARN("invalid arg", K(ret), K(handle), K(info), K(scn)); } else { - - param.ls_id_ = ls_id; + param.source_table_id_ = info.table_id_; param.dest_schema_id_ = info.lob_table_id_; param.schema_version_ = info.schema_version_; @@ -936,7 +911,6 @@ int ObSplitStartReplayExecutor::prepare_param_from_log( } int ObSplitStartReplayExecutor::check_need_wait_split_finished( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObIArray &dest_tablets_id, bool &need_wait_split_finished) @@ -962,12 +936,12 @@ int ObSplitStartReplayExecutor::do_replay_(ObTabletHandle &handle) ret = OB_NULL_CHECK_ERROR; LOG_WARN("unexpected nullptr of dest_tablet_ids", K(ret), KPC(dest_tablet_ids)); } else if (OB_FAIL(check_need_wait_split_finished( - ls_->get_ls_id(), handle, *dest_tablet_ids, need_wait_split_finished))) { + handle, *dest_tablet_ids, need_wait_split_finished))) { LOG_WARN("check need wait split finished failed", K(ret)); } else if (log_->basic_info_.lob_col_idxs_.count() > 0) { // lob tablet. ObLobSplitParam param; - if (OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(ls_->get_ls_id(), handle, log_->basic_info_, scn_, param))) { + if (OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(handle, log_->basic_info_, scn_, param))) { LOG_WARN("prepare lob split param failed", K(ret), KPC(log_)); } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_lob_tablet_split_dag(param))) { if (!need_wait_split_finished) { @@ -983,7 +957,7 @@ int ObSplitStartReplayExecutor::do_replay_(ObTabletHandle &handle) } } else { ObTabletSplitParam param; - if (OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(ls_->get_ls_id(), handle, log_->basic_info_, scn_, param))) { + if (OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(handle, log_->basic_info_, scn_, param))) { LOG_WARN("prepare tablet split param failed", K(ret), KPC(log_)); } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_tablet_split_dag(param))) { if (!need_wait_split_finished) { @@ -999,8 +973,8 @@ int ObSplitStartReplayExecutor::do_replay_(ObTabletHandle &handle) } } if (OB_SUCC(ret) && need_wait_split_finished) { - if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(ls_->get_ls_id(), *dest_tablet_ids, is_data_split_finished))) { - LOG_WARN("check all tablets major exist failed", K(ret), "ls_id", ls_->get_ls_id(), KPC(dest_tablet_ids)); + if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(*dest_tablet_ids, is_data_split_finished))) { + LOG_WARN("check all tablets major exist failed", K(ret), KPC(dest_tablet_ids)); } else if (!is_data_split_finished) { ret = OB_EAGAIN; } @@ -1043,7 +1017,6 @@ int ObSplitFinishReplayExecutor::init( int ObSplitFinishReplayExecutor::do_replay_(ObTabletHandle &handle) { int ret = OB_SUCCESS; - ObLSID ls_id; ObLobSplitParam lob_split_param; ObTabletSplitParam data_split_param; const ObIArray *dest_tablet_ids = nullptr; @@ -1065,13 +1038,12 @@ int ObSplitFinishReplayExecutor::do_replay_(ObTabletHandle &handle) ret = OB_NULL_CHECK_ERROR; LOG_WARN("unexpected nullptr of dest_tablet_ids", K(ret), KPC(dest_tablet_ids)); } else if (OB_FALSE_IT(is_lob_tablet = log_->basic_info_.lob_col_idxs_.count() > 0)) { - } else if (OB_FALSE_IT(ls_id = ls_->get_ls_id())) { } else if (is_lob_tablet && - OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(ls_id, handle, log_->basic_info_, scn_, lob_split_param))) { - LOG_WARN("prepare lob split param failed", K(ret), K(ls_id), KPC(log_)); + OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(handle, log_->basic_info_, scn_, lob_split_param))) { + LOG_WARN("prepare lob split param failed", K(ret), KPC(log_)); } else if (!is_lob_tablet && - OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(ls_id, handle, log_->basic_info_, scn_, data_split_param))) { - LOG_WARN("prepare tablet split param failed", K(ret), K(ls_id), KPC(log_)); + OB_FAIL(ObSplitStartReplayExecutor::prepare_param_from_log(handle, log_->basic_info_, scn_, data_split_param))) { + LOG_WARN("prepare tablet split param failed", K(ret), KPC(log_)); } else { if (is_lob_tablet && OB_FAIL(compaction::ObScheduleDagFunc::schedule_lob_tablet_split_dag(lob_split_param))) { if (ObSplitStartReplayExecutor::is_split_log_retry_ret(ret)) { @@ -1091,8 +1063,8 @@ int ObSplitFinishReplayExecutor::do_replay_(ObTabletHandle &handle) } } if (OB_SUCC(ret)) { - if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(ls_id, *dest_tablet_ids, is_data_split_finished))) { - LOG_WARN("check all tablets major exist failed", K(ret), K(ls_id), KPC(dest_tablet_ids)); + if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(*dest_tablet_ids, is_data_split_finished))) { + LOG_WARN("check all tablets major exist failed", K(ret), KPC(dest_tablet_ids)); } else if (!is_data_split_finished) { ret = OB_EAGAIN; } @@ -1127,8 +1099,6 @@ int ObSplitFinishReplayExecutor::modify_tablet_restore_status_if_need( int ret = OB_SUCCESS; ObTablet *src_tablet; ObTabletRestoreStatus::STATUS src_restore_status = ObTabletRestoreStatus::STATUS::RESTORE_STATUS_MAX; - ObLSHandle ls_handle; - ObLSService *ls_service = nullptr; if (OB_UNLIKELY(dest_tablet_ids.count() <= 0 || !src_tablet_handle.is_valid() || OB_ISNULL(ls))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(dest_tablet_ids), K(src_tablet_handle), KP(ls)); @@ -1137,11 +1107,6 @@ int ObSplitFinishReplayExecutor::modify_tablet_restore_status_if_need( LOG_WARN("unexpected nullptr of src_tablet", K(ret), KP(src_tablet)); } else if (OB_FAIL(src_tablet->get_restore_status(src_restore_status))) { LOG_WARN("failed to get restore status of tablet", K(ret), K(src_tablet)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr of ls service", K(ret), KPC(ls), K(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(ls->get_ls_id(), ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls->get_ls_id())); } else if (ObTabletRestoreStatus::STATUS::UNDEFINED == src_restore_status) { /*skip the split, counting on restoring to fill the destination tablet data*/ ObTabletHandle des_handle; @@ -1150,7 +1115,7 @@ int ObSplitFinishReplayExecutor::modify_tablet_restore_status_if_need( for (int64_t i = 0; OB_SUCC(ret) && i < dest_tablet_ids.count(); ++i) { des_handle.reset(); const ObTabletID &t_id = dest_tablet_ids.at(i); - if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, t_id, des_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { + if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, t_id, des_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { LOG_WARN("get tablet failed", K(ret), K(t_id)); } else if (OB_ISNULL(tablet = des_handle.get_obj())) { ret = OB_NULL_CHECK_ERROR; @@ -1228,15 +1193,16 @@ int ObTabletFreezeReplayExecutor::do_replay_(ObTabletHandle &handle) const int64_t abs_timeout_ts = ObClockGenerator::getClock() + 10LL * 1000LL * 1000LL; const bool need_rewrite_tablet_meta = false; MOD_SCOPE { - const ObLSID ls_id = ls_->get_ls_id(); storage::ObTenantFreezer* freezer = nullptr; if (OB_ISNULL(freezer = share::g_mp->tenant_freezer())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObTenantFreezer shouldn't be null", K(ret)); - } else if (OB_FAIL(freezer->tablet_freeze(ls_id, tablet_id, true/*is_sync*/, abs_timeout_ts, need_rewrite_tablet_meta, ObFreezeSourceFlag::TABLET_SPLIT))) { - LOG_WARN("fail to freeze tablet", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(freezer->tablet_freeze(tablet_id, true/*is_sync*/, abs_timeout_ts, + need_rewrite_tablet_meta, + ObFreezeSourceFlag::TABLET_SPLIT))) { + LOG_WARN("fail to freeze tablet", K(ret), K(tablet_id)); } else { - LOG_INFO("succeed to freeze tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("succeed to freeze tablet", K(ret), K(tablet_id)); } } } @@ -1284,8 +1250,8 @@ int ObTabletForkFreezeReplayExecutor::do_replay_(ObTabletHandle &handle) LOG_WARN("invalid arguments", K(ret), KPC(log_), K(handle)); } else { const ObTabletID &tablet_id = handle.get_obj()->get_tablet_id(); - if (OB_FAIL(ObTabletForkUtil::freeze_tablet(ls_->get_ls_id(), tablet_id))) { - LOG_WARN("failed to freeze tablet from fork freeze log", K(ret), K(ls_->get_ls_id()), K(tablet_id), KPC(log_)); + if (OB_FAIL(ObTabletForkUtil::freeze_tablet(tablet_id))) { + LOG_WARN("failed to freeze tablet from fork freeze log", K(ret), K(tablet_id), KPC(log_)); } else { LOG_INFO("succeeded to replay table fork freeze log: tablet frozen", K(tablet_id)); } @@ -1475,7 +1441,7 @@ int ObDDLIncMinorStartReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) } } else if (!need_replay) { // do nothing - FLOG_INFO("no need to replay ddl inc minor start log", K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + FLOG_INFO("no need to replay ddl inc minor start log", K(scn_), K(tablet_id_)); } else { const bool is_sync = true; // try freeze for ten seconds @@ -1485,9 +1451,9 @@ int ObDDLIncMinorStartReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) abs_timeout_ts, false, /*need_rewrite_meta*/ ObFreezeSourceFlag::DIRECT_INC_START))) { - LOG_WARN("fail to sync tablet freeze", K(ret), K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + LOG_WARN("fail to sync tablet freeze", K(ret), K(scn_), K(tablet_id_)); } else { - FLOG_INFO("replay ddl inc minor start log success", K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + FLOG_INFO("replay ddl inc minor start log success", K(scn_), K(tablet_id_)); } } @@ -1540,7 +1506,7 @@ int ObDDLIncMinorCommitReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) } } else if (!need_replay) { // do nothing - FLOG_INFO("no need to replay ddl inc minor commit log", K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + FLOG_INFO("no need to replay ddl inc minor commit log", K(scn_), K(tablet_id_)); } else { const bool is_sync = true; // try freeze for ten seconds @@ -1550,9 +1516,9 @@ int ObDDLIncMinorCommitReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) abs_timeout_ts, false, /*need_rewrite_meta*/ ObFreezeSourceFlag::DIRECT_INC_END))) { - LOG_WARN("fail to sync tablet freeze", K(ret), K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + LOG_WARN("fail to sync tablet freeze", K(ret), K(scn_), K(tablet_id_)); } else { - FLOG_INFO("replay ddl inc minor commit log success", K(ls_->get_ls_id()), K(scn_), K(tablet_id_)); + FLOG_INFO("replay ddl inc minor commit log success", K(scn_), K(tablet_id_)); } } return ret; diff --git a/src/storage/ddl/ob_ddl_replay_executor.h b/src/storage/ddl/ob_ddl_replay_executor.h index 7b7e71010..87bd373fe 100644 --- a/src/storage/ddl/ob_ddl_replay_executor.h +++ b/src/storage/ddl/ob_ddl_replay_executor.h @@ -216,13 +216,11 @@ class ObSplitStartReplayExecutor final : public ObDDLReplayExecutor const ObTabletSplitStartLog &log, const share::SCN &scn); static int prepare_param_from_log( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObTabletSplitInfo &info, const share::SCN &scn, ObTabletSplitParam ¶m); static int prepare_param_from_log( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObTabletSplitInfo &info, const share::SCN &scn, @@ -241,7 +239,6 @@ class ObSplitStartReplayExecutor final : public ObDDLReplayExecutor private: int check_need_wait_split_finished( - const share::ObLSID &ls_id, const ObTabletHandle &handle, const ObIArray &dest_tablets_id, bool &need_wait_split_finished); diff --git a/src/storage/ddl/ob_ddl_server_client.cpp b/src/storage/ddl/ob_ddl_server_client.cpp index ba934ae63..db33e7d40 100644 --- a/src/storage/ddl/ob_ddl_server_client.cpp +++ b/src/storage/ddl/ob_ddl_server_client.cpp @@ -270,8 +270,7 @@ int ObDDLServerClient::finish_redef_table(const obcall::ObFinishRedefTableArg &f "trace_id", *ObCurTraceId::get_trace_id(), "task_id", finish_redef_arg.task_id_, "snapshot_version", build_single_arg.snapshot_version_, - "rpc_dst", rs_leader_addr, - "info", build_single_arg.ls_id_); + "rpc_dst", rs_leader_addr); LOG_INFO("finish redef table.", K(ret), "ddl_event_info", ObDDLEventInfo(), K(finish_redef_arg), K(build_single_arg), K(rs_leader_addr)); return ret; } diff --git a/src/storage/ddl/ob_ddl_struct.h b/src/storage/ddl/ob_ddl_struct.h index 06a01fad2..88fb2f572 100644 --- a/src/storage/ddl/ob_ddl_struct.h +++ b/src/storage/ddl/ob_ddl_struct.h @@ -532,7 +532,7 @@ struct ObWriteMacroParam final { public: ObWriteMacroParam() - : ls_id_(), tablet_id_(), tenant_data_version_(0), is_no_logging_(false), + : tablet_id_(), tenant_data_version_(0), is_no_logging_(false), schema_version_(0), slice_idx_(0), slice_count_(0), ddl_thread_count_(0), snapshot_version_(0), direct_load_type_(DIRECT_LOAD_INVALID), task_id_(0), is_index_table_(false), tx_info_(), ddl_table_schema_(), tablet_param_(), lob_meta_tablet_param_(), ddl_dag_(nullptr), tablet_context_(nullptr), max_batch_size_(0), start_sequence_(), row_offset_(0), @@ -541,19 +541,18 @@ struct ObWriteMacroParam final ~ObWriteMacroParam() = default; bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid() && tenant_data_version_ > 0 && schema_version_ > 0 + return tablet_id_.is_valid() && tenant_data_version_ > 0 && schema_version_ > 0 && slice_idx_ >= 0 && snapshot_version_ > 0 && direct_load_type_ > DIRECT_LOAD_INVALID && direct_load_type_ < DIRECT_LOAD_MAX && task_id_ > 0 && (!is_incremental_major_direct_load(direct_load_type_) || (is_incremental_major_direct_load(direct_load_type_) && tx_info_.is_valid())); } int64_t get_logic_parallel_count() const { return slice_count_ > 0 ? slice_count_ : ddl_thread_count_; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), K_(tenant_data_version), K_(is_no_logging), + TO_STRING_KV(K_(tablet_id), K_(lob_meta_tablet_id), K_(tenant_data_version), K_(is_no_logging), K_(schema_version), K_(slice_idx), K_(slice_count), K_(ddl_thread_count), K_(snapshot_version), K_(direct_load_type), K_(task_id), K_(is_index_table), K_(ddl_table_schema), K_(tablet_param), K_(lob_meta_tablet_param), KP_(tablet_context), K_(is_sorted_table_load)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; ObTabletID lob_meta_tablet_id_; int64_t tenant_data_version_; diff --git a/src/storage/ddl/ob_ddl_tablet_context.cpp b/src/storage/ddl/ob_ddl_tablet_context.cpp index dc0f7d6fa..75f9c2046 100644 --- a/src/storage/ddl/ob_ddl_tablet_context.cpp +++ b/src/storage/ddl/ob_ddl_tablet_context.cpp @@ -188,7 +188,6 @@ int init_tablet_param(ObTablet *tablet, ObStorageSchema *storage_schema, const O } int ObDDLTabletContext::init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t ddl_thread_count, const int64_t snapshot_version, @@ -199,13 +198,12 @@ int ObDDLTabletContext::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || ddl_thread_count <= 0 || + } else if (OB_UNLIKELY(!tablet_id.is_valid() || ddl_thread_count <= 0 || !is_valid_direct_load(direct_load_type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invlaid argument", K(ret), K(ls_id), K(tablet_id), K(ddl_thread_count), + LOG_WARN("invlaid argument", K(ret), K(tablet_id), K(ddl_thread_count), K(direct_load_type)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; bucket_count_ = ddl_thread_count * 2; if (OB_FAIL(slice_map_.create(bucket_count_, ObMemAttr("tblt_slice_map")))) { @@ -213,13 +211,13 @@ int ObDDLTabletContext::init( } else if (OB_FAIL(bucket_lock_.init(bucket_count_))) { LOG_WARN("init bucket lock failed", K(ret), K(bucket_count_)); } else { - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletBindingMdsUserData mds_data; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("ddl get tablet failed", K(ret), K(ls_handle), K(tablet_id)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("ddl get tablet failed", K(ret), K(ls), K(tablet_id)); } else if (OB_FAIL(init_tablet_param(tablet_handle.get_obj(), ddl_table_schema.storage_schema_, direct_load_type, arena_, tablet_param_))) { LOG_WARN("init tablet param failed", K(ret)); } else if (OB_FAIL(merge_ctx_.init(direct_load_type))) { @@ -232,8 +230,8 @@ int ObDDLTabletContext::init( } else if (mds_data.lob_meta_tablet_id_.is_valid()) { lob_meta_tablet_id_ = mds_data.lob_meta_tablet_id_; ObTabletHandle lob_meta_tablet_handle; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_meta_tablet_id_, lob_meta_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("ddl get tablet failed", K(ret), K(ls_handle), KPC(this)); + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, lob_meta_tablet_id_, lob_meta_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("ddl get tablet failed", K(ret), K(ls), KPC(this)); } else if (OB_FAIL(init_tablet_param(lob_meta_tablet_handle.get_obj(), ddl_table_schema.lob_meta_storage_schema_, direct_load_type, arena_, lob_meta_tablet_param_))) { LOG_WARN("init lob meta tablet param failed", K(ret)); } else if (OB_FAIL(lob_merge_ctx_.init(direct_load_type))) { @@ -263,7 +261,7 @@ int ObDDLTabletContext::init_vector_index_context(const int64_t snapshot_version LOG_WARN("allocate memory failed", K(ret)); } else { vector_index_ctx_ = new (buf) ObVectorIndexTabletContext(); - if (OB_FAIL(vector_index_ctx_->init(ls_id_, tablet_id_, tablet_param_.storage_schema_->get_index_type(), snapshot_version, ddl_table_schema))) { + if (OB_FAIL(vector_index_ctx_->init(tablet_id_, tablet_param_.storage_schema_->get_index_type(), snapshot_version, ddl_table_schema))) { LOG_WARN("init vector index ctx failed", K(ret)); } } @@ -274,7 +272,6 @@ int ObDDLTabletContext::init_vector_index_context(const int64_t snapshot_version void ObDDLTabletContext::reset() { is_inited_ = false; - ls_id_.reset(); tablet_id_.reset(); tablet_param_.reset(); lob_meta_tablet_id_.reset(); diff --git a/src/storage/ddl/ob_ddl_tablet_context.h b/src/storage/ddl/ob_ddl_tablet_context.h index 2e1bd5cc7..1183301d7 100644 --- a/src/storage/ddl/ob_ddl_tablet_context.h +++ b/src/storage/ddl/ob_ddl_tablet_context.h @@ -109,7 +109,6 @@ struct ObDDLTabletContext final ObDDLTabletContext(); ~ObDDLTabletContext(); int init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t ddl_thread_count, const int64_t snapshot_version, @@ -123,7 +122,7 @@ struct ObDDLTabletContext final int get_or_create_slice(const int64_t slice_idx, ObDDLSlice *&ddl_slice, bool &is_new_slice); int remove_slice(const int64_t slice_idx); int get_all_slices(ObIArray &ddl_slices); - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(tablet_param), K_(lob_meta_tablet_id), K_(lob_meta_tablet_param), + TO_STRING_KV(K_(is_inited), K_(tablet_id), K_(tablet_param), K_(lob_meta_tablet_id), K_(lob_meta_tablet_param), K_(slice_count), K_(table_slice_offset), K_(last_lob_id), K_(last_autoinc_val), K(bucket_count_), K(slice_map_.size())); private: int init_vector_index_context( @@ -138,7 +137,6 @@ struct ObDDLTabletContext final MergeCtx lob_merge_ctx_; // const param - share::ObLSID ls_id_; ObTabletID tablet_id_; ObWriteTabletParam tablet_param_; ObTabletID lob_meta_tablet_id_; diff --git a/src/storage/ddl/ob_delete_lob_meta_row_task.cpp b/src/storage/ddl/ob_delete_lob_meta_row_task.cpp index 3eaf74b6e..343521d2f 100644 --- a/src/storage/ddl/ob_delete_lob_meta_row_task.cpp +++ b/src/storage/ddl/ob_delete_lob_meta_row_task.cpp @@ -57,7 +57,6 @@ int ObDeleteLobMetaRowParam::init(const ObDDLBuildSingleReplicaRequestArg &arg) schema_version_ = arg.schema_version_; tablet_id_ = arg.source_tablet_id_; dest_tablet_id_ = arg.dest_tablet_id_; - ls_id_ = arg.ls_id_; task_id_ = arg.task_id_; execution_id_ = arg.execution_id_; tablet_task_id_ = arg.tablet_task_id_; @@ -134,7 +133,6 @@ uint64_t ObDeleteLobMetaRowDag::hash() const } else { hash_val = param_.table_id_ + param_.schema_id_ - + param_.ls_id_.hash() + param_.tablet_id_.hash() + param_.dest_tablet_id_.hash() + ObDagType::DAG_TYPE_DDL_DEL_LOB_META; @@ -156,7 +154,7 @@ bool ObDeleteLobMetaRowDag::operator==(const ObIDag &other) const } else { is_equal = (true) && (true) && (param_.table_id_ == dag.param_.table_id_) && (param_.schema_id_ == dag.param_.schema_id_) && - (param_.ls_id_ == dag.param_.ls_id_) && (param_.tablet_id_ == dag.param_.tablet_id_) && + (param_.tablet_id_ == dag.param_.tablet_id_) && (param_.dest_tablet_id_ == dag.param_.dest_tablet_id_) && (param_.delete_lob_meta_ret_ == dag.param_.delete_lob_meta_ret_); } @@ -174,7 +172,6 @@ int ObDeleteLobMetaRowDag::fill_info_param(compaction::ObIBasicInfoParam *&out_p ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param", K(ret), K(param_)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - param_.ls_id_.id(), static_cast(param_.table_id_), static_cast(param_.tablet_id_.id()), static_cast(param_.dest_tablet_id_.id()), @@ -194,8 +191,8 @@ int ObDeleteLobMetaRowDag::fill_dag_key(char *buf, const int64_t buf_len) const } else if (OB_UNLIKELY(!param_.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid params", K(ret), K(param_)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, "logstream_id=%ld tablet_id=%ld lob_meta_tablet_id=%ld", - param_.ls_id_.id(), param_.tablet_id_.id(), param_.dest_tablet_id_.id()))) { + } else if (OB_FAIL(databuff_printf(buf, buf_len, "tablet_id=%ld lob_meta_tablet_id=%ld", + param_.tablet_id_.id(), param_.dest_tablet_id_.id()))) { LOG_WARN("fill dag key for ddl table merge dag failed", K(ret), K(param_)); } return ret; @@ -214,15 +211,13 @@ int ObDeleteLobMetaRowDag::report_replica_build_status() #ifdef ERRSIM if (OB_SUCC(ret)) { ret = OB_E(EventTable::EN_DDL_REPORT_REPLICA_BUILD_STATUS_FAIL) OB_SUCCESS; - LOG_INFO("report replica build status errsim", K(ret)); + LOG_INFO("report DDL build status errsim", K(ret)); } #endif obcall::ObDDLBuildSingleReplicaResponseArg arg; ObAddr rs_addr = GCTX.self_addr(); - arg.ls_id_ = param_.ls_id_; - arg.dest_ls_id_ = param_.ls_id_; arg.tablet_id_ = param_.tablet_id_; arg.source_table_id_ = param_.table_id_; arg.dest_schema_id_ = param_.schema_id_; @@ -233,7 +228,7 @@ int ObDeleteLobMetaRowDag::report_replica_build_status() arg.task_id_ = param_.task_id_; arg.execution_id_ = param_.execution_id_; arg.server_addr_ = GCTX.self_addr(); - FLOG_INFO("send replica build status response to RS", K(ret), K(arg)); + FLOG_INFO("send DDL build status response to RS", K(ret), K(arg)); if (OB_FAIL(ret)) { } else if (OB_FAIL(GCTX.root_service_->build_ddl_single_replica_response(arg))) { LOG_WARN("fail to send build ddl single replica response", K(ret), K(arg)); @@ -292,7 +287,6 @@ int ObDeleteLobMetaRowTask::init_scan_param(ObTableScanParam& scan_param) scan_param.tablet_id_ = param_->tablet_id_; scan_param.schema_version_ = param_->schema_version_; scan_param.is_get_ = false; - scan_param.ls_id_ = param_->ls_id_; ObQueryFlag query_flag(ObQueryFlag::Forward, // scan_order false, // daily_merge false, // optimize @@ -396,9 +390,9 @@ int ObDeleteLobMetaRowTask::process() if (OB_ISNULL(txs) || OB_ISNULL(tsc_service) || OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("should not be null", K(ret), KP(txs), KP(tsc_service), KP(lob_mngr)); - } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(param_->ls_id_, true/*is_for_read*/, timeout_us, tx_desc))) { + } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(true/*is_for_read*/, timeout_us, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, param_->ls_id_, timeout_us, scan_param.snapshot_))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout_us, scan_param.snapshot_))) { LOG_WARN("fail to get snapshot", K(ret)); } else if (OB_FAIL(init_scan_param(scan_param))) { LOG_WARN("fail to init scan_param", K(ret)); @@ -422,7 +416,6 @@ int ObDeleteLobMetaRowTask::process() ret = OB_ERR_UNEXPECTED; LOG_WARN("get row column cnt invalid.", K(ret), K(datum_row->get_column_count())); } else if (OB_FAIL(ObInsertLobColumnHelper::delete_lob_column(param_->allocator_, - param_->ls_id_, param_->tablet_id_, collation_type_, datum_row->storage_datums_[0], @@ -459,7 +452,7 @@ int ObDeleteLobMetaRowTask::process() } else if (OB_SUCCESS != (tmp_ret = dag->report_replica_build_status())) { // do not override ret if it has already failed. ret = OB_SUCCESS == ret ? tmp_ret : ret; - LOG_WARN("fail to report replica build status", K(ret), K(tmp_ret)); + LOG_WARN("fail to report DDL build status", K(ret), K(tmp_ret)); } } diff --git a/src/storage/ddl/ob_delete_lob_meta_row_task.h b/src/storage/ddl/ob_delete_lob_meta_row_task.h index 2ce84ef04..28e1d3320 100644 --- a/src/storage/ddl/ob_delete_lob_meta_row_task.h +++ b/src/storage/ddl/ob_delete_lob_meta_row_task.h @@ -35,7 +35,7 @@ struct ObDeleteLobMetaRowParam final public: ObDeleteLobMetaRowParam(): is_inited_(false), - table_id_(common::OB_INVALID_ID), schema_id_(common::OB_INVALID_ID), ls_id_(share::ObLSID::INVALID_LS_ID), + table_id_(common::OB_INVALID_ID), schema_id_(common::OB_INVALID_ID), tablet_id_(ObTabletID::INVALID_TABLET_ID), dest_tablet_id_(ObTabletID::INVALID_TABLET_ID), row_store_type_(common::ENCODING_ROW_STORE), schema_version_(0), snapshot_version_(0), task_id_(0), execution_id_(-1), tablet_task_id_(0), delete_lob_meta_ret_(common::OB_SUCCESS), @@ -47,7 +47,7 @@ struct ObDeleteLobMetaRowParam final bool is_valid() const { - return true && ls_id_.is_valid() && common::OB_INVALID_ID != schema_id_ + return common::OB_INVALID_ID != schema_id_ && common::OB_INVALID_ID != table_id_ && tablet_id_.is_valid() && dest_tablet_id_.is_valid() && snapshot_version_ > 0 && compat_mode_ != lib::Worker::CompatMode::INVALID && execution_id_ >= 0 && tablet_task_id_ > 0 && data_format_version_ > 0; @@ -57,7 +57,6 @@ struct ObDeleteLobMetaRowParam final void destroy() { is_inited_ = false; - ls_id_.reset(); table_id_ = common::OB_INVALID_ID; schema_id_ = common::OB_INVALID_ID; tablet_id_.reset(); @@ -72,14 +71,13 @@ struct ObDeleteLobMetaRowParam final compat_mode_ = lib::Worker::CompatMode::INVALID; data_format_version_ = 0; } - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(table_id), K_(tablet_id), + TO_STRING_KV(K_(is_inited), K_(table_id), K_(tablet_id), K_(tablet_task_id), K_(schema_version), K_(snapshot_version), K_(task_id), K_(execution_id), K_(compat_mode), K_(data_format_version)); public: bool is_inited_; uint64_t table_id_; uint64_t schema_id_; - share::ObLSID ls_id_; ObTabletID tablet_id_; ObTabletID dest_tablet_id_; common::ObRowStoreType row_store_type_; diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index 598aebacf..005a3ef63 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -121,7 +121,6 @@ int ObTenantDirectLoadMgr::init() int ObTenantDirectLoadMgr::get_agent_exec_context( const int64_t context_id, - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObDirectLoadType &type, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -162,11 +161,11 @@ int ObTenantDirectLoadMgr::get_agent_exec_context( const bool is_full_direct_load_task = is_full_direct_load(type); if (OB_FAIL(get_tablet_mgr_no_lock(mgr_key, direct_load_mgr_handle))) { if (OB_ENTRY_NOT_EXIST == ret && is_full_direct_load_task) { - if (OB_FAIL(check_and_process_finished_tablet(ls_id, tablet_id))) { - LOG_WARN("check and report checksum if need failed", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(check_and_process_finished_tablet(tablet_id))) { + LOG_WARN("check and report checksum if need failed", K(ret), K(tablet_id)); } } else { - LOG_WARN("get table mgr failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get table mgr failed", K(ret), K(tablet_id)); } } if (OB_SUCC(ret)) { @@ -188,10 +187,9 @@ int ObTenantDirectLoadMgr::create_tablet_direct_load( const bool only_persisted_ddl_data) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = build_param.common_param_.ls_id_; const common::ObTabletID &tablet_id = build_param.common_param_.tablet_id_; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletBindingMdsUserData ddl_data; if (OB_UNLIKELY(!is_inited_)) { @@ -203,12 +201,12 @@ int ObTenantDirectLoadMgr::create_tablet_direct_load( } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(build_param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("get tablet handle failed", K(ret), K(build_param)); } else if (!only_persisted_ddl_data && OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_id)); } else if (only_persisted_ddl_data && OB_FAIL((tablet_handle.get_obj()->get_mds_data_from_tablet( mds::DummyKey(), share::SCN::max_scn(), @@ -218,7 +216,7 @@ int ObTenantDirectLoadMgr::create_tablet_direct_load( ddl_data.set_default_value(); ret = OB_SUCCESS; } else { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_id)); } } @@ -251,11 +249,11 @@ int ObTenantDirectLoadMgr::create_tablet_direct_load( LOG_WARN("lock multi buckets failed", K(ret)); } else if (!lob_meta_tablet_id.is_valid() || checkpoint_scn.is_valid_and_not_min()) { // has no lob, or recover from checkpoint. - LOG_DEBUG("do not create lob mgr handle when create data tablet mgr", K(ret), K(ls_id), K(lob_meta_tablet_id), + LOG_DEBUG("do not create lob mgr handle when create data tablet mgr", K(ret), K(lob_meta_tablet_id), K(checkpoint_scn), K(build_param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_meta_tablet_id, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, lob_meta_tablet_id, lob_tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(lob_meta_tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(lob_meta_tablet_id)); } else if (OB_FAIL(lob_tablet_handle.get_obj()->fetch_table_store(lob_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(try_create_tablet_direct_load_mgr_nolock( @@ -451,7 +449,6 @@ int ObTenantDirectLoadMgr::alloc_slice_id(int64_t &slice_id) int ObTenantDirectLoadMgr::open_tablet_direct_load( const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t context_id) { @@ -483,8 +480,8 @@ int ObTenantDirectLoadMgr::open_tablet_direct_load( // FIXME @SUZHI following key does no contain context id, incremental direct load always fail to get. if (OB_FAIL(get_tablet_mgr(mgr_key, handle))) { if (OB_ENTRY_NOT_EXIST == ret && is_full_direct_load_task) { - if (OB_FAIL(check_and_process_finished_tablet(ls_id, tablet_id))) { - LOG_WARN("check and report checksum if need failed", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(check_and_process_finished_tablet(tablet_id))) { + LOG_WARN("check and report checksum if need failed", K(ret), K(tablet_id)); } } else { LOG_WARN("get table mgr failed", K(ret), K(tablet_id), K(is_full_direct_load_task)); @@ -519,7 +516,6 @@ int ObTenantDirectLoadMgr::open_tablet_direct_load( int ObTenantDirectLoadMgr::close_tablet_direct_load( const int64_t context_id, const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit, const bool emergent_finish, @@ -534,22 +530,21 @@ int ObTenantDirectLoadMgr::close_tablet_direct_load( LOG_WARN("not init", K(ret)); } else if (OB_UNLIKELY(context_id <= 0 || !is_valid_direct_load(type) - || !ls_id.is_valid() || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(context_id), K(type), K(ls_id), K(tablet_id), + LOG_WARN("invalid argument", K(ret), K(context_id), K(type), K(tablet_id), K(task_id), K(table_id), K(execution_id)); } else if (is_idem_type(type)) { ret = OB_NOT_SUPPORTED; LOG_WARN("unsupported direct load type", K(ret), K(type)); } else if (is_shared_storage_dempotent_mode(type)) { - if (OB_FAIL(close_tablet_direct_load_for_ss(context_id, ls_id, tablet_id, need_commit))) { - LOG_WARN("close tablet direct load failed", K(ret), K(context_id), K(ls_id), K(tablet_id), K(need_commit)); + if (OB_FAIL(close_tablet_direct_load_for_ss(context_id, tablet_id, need_commit))) { + LOG_WARN("close tablet direct load failed", K(ret), K(context_id), K(tablet_id), K(need_commit)); } } else { - if (OB_FAIL(close_tablet_direct_load_for_sn(context_id, type, ls_id, tablet_id, need_commit, task_id, table_id, execution_id))) { - LOG_WARN("close tablet direct load failed", K(ret), K(context_id), - K(ls_id), K(tablet_id), K(need_commit), K(task_id), K(table_id), K(execution_id)); + if (OB_FAIL(close_tablet_direct_load_for_sn(context_id, type, tablet_id, need_commit, task_id, table_id, execution_id))) { + LOG_WARN("close tablet direct load failed", K(ret), K(context_id), + K(tablet_id), K(need_commit), K(task_id), K(table_id), K(execution_id)); } } return ret; @@ -558,7 +553,6 @@ int ObTenantDirectLoadMgr::close_tablet_direct_load( int ObTenantDirectLoadMgr::close_tablet_direct_load_for_sn( const int64_t context_id, const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit, const int64_t task_id, @@ -582,11 +576,11 @@ int ObTenantDirectLoadMgr::close_tablet_direct_load_for_sn( LOG_WARN("invalid argument", K(ret), K(tablet_id), K(context_id)); } else if (OB_FAIL(get_tablet_mgr(data_mgr_key, handle))) { if (OB_ENTRY_NOT_EXIST == ret && is_full_direct_load_task) { - if (OB_FAIL(check_and_process_finished_tablet(ls_id, tablet_id, nullptr/*row_iterator*/, task_id, table_id, execution_id))) { - LOG_WARN("check and report checksum if need failed", K(ret), K(ls_id), K(tablet_id), K(task_id), K(execution_id)); + if (OB_FAIL(check_and_process_finished_tablet(tablet_id, nullptr/*row_iterator*/, task_id, table_id, execution_id))) { + LOG_WARN("check and report checksum if need failed", K(ret), K(tablet_id), K(task_id), K(execution_id)); } } else { - LOG_WARN("get table mgr failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get table mgr failed", K(ret), K(tablet_id)); } } else if (need_commit) { ObTabletDirectLoadExecContext exec_context; @@ -638,13 +632,11 @@ int ObTenantDirectLoadMgr::close_tablet_direct_load_for_sn( int ObTenantDirectLoadMgr::close_tablet_direct_load_for_ss( const int64_t context_id, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - UNUSED(ls_id); ObTabletID lob_meta_tablet_id; ObTabletDirectLoadMgrHandle handle; ObTabletDirectLoadMgrKey data_mgr_key(tablet_id, context_id); @@ -655,7 +647,7 @@ int ObTenantDirectLoadMgr::close_tablet_direct_load_for_ss( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tablet_id), K(context_id)); } else if (OB_FAIL(get_tablet_mgr(data_mgr_key, handle))) { - LOG_WARN("get table mgr failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get table mgr failed", K(ret), K(tablet_id)); } else if (need_commit && OB_FAIL(handle.get_obj()->close(1/*placeholder*/, SCN::min_scn()/*placeholder*/))) { LOG_WARN("close to generate major sstable failed", K(ret)); @@ -737,7 +729,6 @@ int ObTenantDirectLoadMgr::get_tablet_cache_interval( } int ObTenantDirectLoadMgr::check_and_process_finished_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObIStoreRowIterator *row_iter, const int64_t task_id, @@ -745,7 +736,7 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( const int64_t execution_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObSSTableMetaHandle sst_meta_hdl; @@ -756,24 +747,24 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( ObTabletMemberWrapper table_store_wrapper; const int64_t max_wait_timeout_us = 30L * 1000L * 1000L; // 30s ObTimeGuard tg("ddl_retry_tablet", max_wait_timeout_us); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); + LOG_WARN("invalid arg", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); } while (OB_SUCC(ret)) { if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("check status failed", K(ret), K(tablet_id)); } else if (tg.get_diff() > max_wait_timeout_us) { ret = OB_NEED_RETRY; - LOG_WARN("process finished tablet timeout, need retry", K(ret), K(ls_id), K(tablet_id), K(tg)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("process finished tablet timeout, need retry", K(ret), K(tablet_id), K(tg)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet handle is null", K(ret), K(tablet_id)); } else if (task_id <= 0 || common::OB_INVALID_ID == table_id || execution_id < 0 || tablet_handle.get_obj()->get_tablet_meta().ddl_execution_id_ > execution_id) { // no need to report checkksum. @@ -785,11 +776,11 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( } else if (FALSE_IT(first_major_sstable = static_cast( table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/)))) { } else if (nullptr == first_major_sstable) { - LOG_INFO("major not exist, retry later", K(ret), K(ls_id), K(tablet_id), K(tg)); + LOG_INFO("major not exist, retry later", K(ret), K(tablet_id), K(tg)); usleep(100L * 1000L); // 100ms } else if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable( - ls_id, tablet_id, first_major_sstable, table_store_wrapper))) { - LOG_WARN("check if major sstable exist failed", K(ret), K(ls_id), K(tablet_id)); + tablet_id, first_major_sstable, table_store_wrapper))) { + LOG_WARN("check if major sstable exist failed", K(ret), K(tablet_id)); } else if (OB_FAIL(first_major_sstable->get_meta(sst_meta_hdl))) { LOG_WARN("fail to get sstable meta handle", K(ret)); } else if (OB_FAIL(ObDDLUtil::get_data_information(task_id, data_format_version, snapshot_version, unused_task_status))) { @@ -798,7 +789,6 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( const int64_t *column_checksums = sst_meta_hdl.get_sstable_meta().get_col_checksum(); int64_t column_count = sst_meta_hdl.get_sstable_meta().get_col_checksum_cnt(); if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum( - ls_id, tablet_id, table_id, execution_id, @@ -806,7 +796,7 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( column_checksums, column_count, data_format_version))) { - LOG_WARN("report ddl column checksum failed", K(ret), K(ls_id), K(tablet_id), K(execution_id)); + LOG_WARN("report ddl column checksum failed", K(ret), K(tablet_id), K(execution_id)); } else { break; } @@ -833,7 +823,7 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( ret = OB_SUCCESS; break; } else { - LOG_WARN("iter row failed", K(ret), K(task_id), K(execution_id), K(ls_id), K(tablet_id)); + LOG_WARN("iter row failed", K(ret), K(task_id), K(execution_id), K(tablet_id)); } } } @@ -842,7 +832,6 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( } int ObTenantDirectLoadMgr::get_tablet_mgr_and_check_major( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool is_full_direct_load, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -859,12 +848,12 @@ int ObTenantDirectLoadMgr::get_tablet_mgr_and_check_major( is_major_sstable_exist = false; if (OB_ENTRY_NOT_EXIST == ret) { int tmp_ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - if (OB_TMP_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(tmp_ret), K(ls_id)); - } else if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle))) { - LOG_WARN("get tablet handle failed", K(tmp_ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(tmp_ret)); + } else if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle))) { + LOG_WARN("get tablet handle failed", K(tmp_ret), K(tablet_id)); } else { is_major_sstable_exist = tablet_handle.get_obj()->get_major_table_count() > 0 || tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable(); @@ -954,7 +943,7 @@ int ObTenantDirectLoadMgr::GetGcCandidateOp::operator() (common::hash::HashMapPa // shared storage mgr shoule be freed by the front only. } else if (is_incremental_direct_load(key.direct_load_type_)) { // incremental direct load mgr shoule be freed by the front only. - } else if (OB_FAIL(candidate_mgrs_.push_back(std::make_pair(tablet_direct_load_mgr->get_ls_id(), key)))) { + } else if (OB_FAIL(candidate_mgrs_.push_back(key))) { LOG_WARN("failed to push back", K(ret)); } } @@ -965,7 +954,7 @@ int ObTenantDirectLoadMgr::gc_tablet_direct_load() { int ret = OB_SUCCESS; if (!tablet_mgr_map_.empty() && ObDDLUtil::reach_time_interval(10 * 1000 * 1000, last_gc_time_)) { - ObSEArray, 8> candidate_mgrs; + ObSEArray candidate_mgrs; { ObBucketTryRLockAllGuard guard(bucket_lock_); if (OB_SUCC(guard.get_ret())) { @@ -975,25 +964,20 @@ int ObTenantDirectLoadMgr::gc_tablet_direct_load() } for (int64_t i = 0; i < candidate_mgrs.count(); i++) { // overwrite ret - const share::ObLSID &ls_id = candidate_mgrs.at(i).first; - const ObTabletDirectLoadMgrKey &mgr_key = candidate_mgrs.at(i).second; + const ObTabletDirectLoadMgrKey &mgr_key = candidate_mgrs.at(i); ObLSService *ls_svr = share::g_mp->ls_service(); ObLS *ls = nullptr; - ObLSHandle ls_handle; ObTabletHandle tablet_handle; if (mgr_key.direct_load_type_ == DIRECT_LOAD_INCREMENTAL) { // skip } else if (OB_ISNULL(ls_svr)) { ret = OB_ERR_SYS; LOG_WARN("invalid mtl ObLSService", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get log stream failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream not exist", K(ret)); + } else if (OB_FAIL(ls_svr->get_ls(ls))) { + LOG_WARN("get single log stream failed", K(ret)); } else if (OB_FAIL(ls->get_tablet(mgr_key.tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(mgr_key)); + LOG_WARN("failed to get tablet", K(ret), K(mgr_key)); } else if (tablet_handle.get_obj()->get_major_table_count() > 0) { (void)remove_tablet_direct_load(mgr_key); } @@ -1153,7 +1137,7 @@ ObTabletDirectLoadMgr::~ObTabletDirectLoadMgr() bool ObTabletDirectLoadMgr::is_valid() { - return is_inited_ == true && ls_id_.is_valid() && tablet_id_.is_valid() + return is_inited_ == true && tablet_id_.is_valid() && is_valid_direct_load(direct_load_type_); } @@ -1165,7 +1149,7 @@ int ObTabletDirectLoadMgr::update( const int64_t bucket_num = 97L; // 97 const int64_t memory_limit = 1024LL * 1024LL * 1024LL * 10LL; // 10GB ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_UNLIKELY(!build_param.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -1173,9 +1157,9 @@ int ObTabletDirectLoadMgr::update( } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(build_param.common_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(build_param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, build_param.common_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -1248,7 +1232,6 @@ int ObTabletDirectLoadMgr::update( if (OB_FAIL(sqc_build_ctx_.build_param_.assign(build_param))) { LOG_WARN("assign build param failed", K(ret)); } else { - ls_id_ = build_param.common_param_.ls_id_; tablet_id_ = build_param.common_param_.tablet_id_; direct_load_type_ = build_param.common_param_.direct_load_type_; tenant_data_version_ = build_param.common_param_.data_format_version_; @@ -1565,7 +1548,7 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; - ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, + ObBatchSliceWriteInfo info(tablet_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_); @@ -1626,7 +1609,7 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; - ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, + ObBatchSliceWriteInfo info(tablet_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_); @@ -1831,7 +1814,7 @@ int ObTabletDirectLoadMgr::prepare_index_builder_if_need(const ObTableSchema &ta ObWholeDataStoreDesc index_block_desc; if (sqc_build_ctx_.index_builder_ != nullptr) { LOG_INFO("index builder is already prepared"); - } else if (OB_FAIL(index_block_desc.init(true/*is ddl*/, table_schema, ls_id_, tablet_id_, + } else if (OB_FAIL(index_block_desc.init(true/*is ddl*/, table_schema, tablet_id_, is_full_direct_load(direct_load_type_) ? compaction::ObMergeType::MAJOR_MERGE : compaction::ObMergeType::MINOR_MERGE, is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : 1L, tenant_data_version_, get_micro_index_clustered(), 0/*concurrent_cnt*/, @@ -1852,7 +1835,7 @@ int ObTabletDirectLoadMgr::prepare_index_builder_if_need(const ObTableSchema &ta index_block_desc.get_desc(), // index_block_desc is copied in index_builder ObSSTableIndexBuilder::DISABLE))) { LOG_WARN("failed to init index builder", K(ret), K(index_block_desc)); - } else if (OB_FAIL(sqc_build_ctx_.data_block_desc_.init(true/*is ddl*/, table_schema, ls_id_, tablet_id_, + } else if (OB_FAIL(sqc_build_ctx_.data_block_desc_.init(true/*is ddl*/, table_schema, tablet_id_, is_full_direct_load(direct_load_type_) ? compaction::ObMergeType::MAJOR_MERGE : compaction::ObMergeType::MINOR_MERGE, is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : 1L, tenant_data_version_, get_micro_index_clustered(), 0/*concurrent_cnt*/, @@ -1972,7 +1955,7 @@ int ObTabletFullDirectLoadMgr::open(const int64_t current_execution_id, share::S int ret = OB_SUCCESS; uint32_t lock_tid = 0; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletFullDirectLoadMgr *lob_tablet_mgr = nullptr; start_scn.reset(); @@ -1991,9 +1974,9 @@ int ObTabletFullDirectLoadMgr::open(const int64_t current_execution_id, share::S } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -2030,16 +2013,16 @@ int ObTabletFullDirectLoadMgr::open(const int64_t current_execution_id, share::S LOG_WARN("create ddl kv mgr failed", K(ret)); } else if (nullptr != lob_tablet_mgr) { ObTabletHandle lob_tablet_handle; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_tablet_mgr->get_tablet_id(), lob_tablet_handle))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), KPC(lob_tablet_mgr)); + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, lob_tablet_mgr->get_tablet_id(), lob_tablet_handle))) { + LOG_WARN("get tablet handle failed", K(ret), KPC(lob_tablet_mgr)); } else if (OB_FAIL(lob_tablet_handle.get_obj()->get_ddl_kv_mgr(lob_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create ddl kv mgr failed", K(ret)); } } if (OB_SUCC(ret)) { ObDDLRedoLogWriter redo_writer; - if (OB_FAIL(redo_writer.init(ls_id_, tablet_id_))) { - LOG_WARN("init redo writer failed", K(ret), K(ls_id_), K(tablet_id_)); + if (OB_FAIL(redo_writer.init(tablet_id_))) { + LOG_WARN("init redo writer failed", K(ret), K(tablet_id_)); } else if (OB_FAIL(redo_writer.write_start_log(table_key_, current_execution_id, sqc_build_ctx_.build_param_.common_param_.data_format_version_, direct_load_type_, ddl_kv_mgr_handle, lob_kv_mgr_handle, direct_load_mgr_handle, lock_tid, start_scn))) { @@ -2065,9 +2048,8 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star { int ret = OB_SUCCESS; SCN commit_scn; - bool is_remote_write = false; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletHandle new_tablet_handle; bool sstable_already_created = false; @@ -2081,9 +2063,9 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -2093,27 +2075,23 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star ObDDLRedoLogWriter redo_writer; #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("storage_ddl", "before_ddl_close", - "ls_id", ls_id_.id(), "tablet_id", tablet_id_.id(), "execution_id", execution_id, "start_scn", start_scn); #endif - DEBUG_SYNC(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG); if (OB_FAIL(wrlock(TRY_LOCK_TIMEOUT, lock_tid))) { LOG_WARN("failed to wrlock", K(ret), KPC(this)); } else if (FALSE_IT(sstable_already_created = sqc_build_ctx_.is_task_end_)) { } else if (sstable_already_created) { - // Why use is_task_end_ rather than commit_scn_. - // sqc may switch to follower, and the commit_scn will not be set. LOG_INFO("had already closed", K(ret)); - } else if (OB_FAIL(redo_writer.init(ls_id_, tablet_id_))) { - LOG_WARN("init redo writer failed", K(ret), K(ls_id_), K(tablet_id_)); + } else if (OB_FAIL(redo_writer.init(tablet_id_))) { + LOG_WARN("init redo writer failed", K(ret), K(tablet_id_)); } else { ObTabletDirectLoadMgrHandle direct_load_mgr_handle; if (OB_FAIL(direct_load_mgr_handle.set_obj(this))) { LOG_WARN("set direct load mgr handle failed", K(ret)); - } else if (OB_FAIL(redo_writer.write_commit_log_with_retry(true, table_key_, - start_scn, direct_load_mgr_handle, tablet_handle, commit_scn, is_remote_write, lock_tid))) { + } else if (OB_FAIL(redo_writer.write_commit_log(table_key_, start_scn, + direct_load_mgr_handle, tablet_handle, commit_scn, lock_tid))) { LOG_WARN("fail write ddl commit log", K(ret), K(table_key_), K(sqc_build_ctx_)); } } @@ -2128,8 +2106,6 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star sqc_build_ctx_.is_task_end_ = is_delay_build_major ? true : sqc_build_ctx_.is_task_end_; // skip report checksum #endif if (OB_FAIL(ret) || sstable_already_created) { - } else if (is_remote_write) { - LOG_INFO("ddl commit log is written in remote, need wait replay", K(sqc_build_ctx_), K(start_scn), K(commit_scn)); } else if (OB_UNLIKELY(!start_scn.is_valid_and_not_min()) || !commit_scn.is_valid_and_not_min()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret), KPC(this)); @@ -2144,12 +2120,12 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star K(sstable_already_created), K(is_delay_build_major)); } else if (OB_FAIL(schedule_merge_task(start_scn, commit_scn, true/*wait_major_generate*/, false/*is_replay*/))) { LOG_WARN("schedule merge task and wait real major generate", K(ret), - K(is_remote_write), K(sstable_already_created), K(start_scn), K(commit_scn)); + K(sstable_already_created), K(start_scn), K(commit_scn)); } else if (lob_mgr_handle_.is_valid() && OB_FAIL(lob_mgr_handle_.get_full_obj()->schedule_merge_task(start_scn, commit_scn, true/*wait_major_generate*/, false/*is_replay*/))) { LOG_WARN("schedule merge task and wait real major generate for lob failed", K(ret), - K(is_remote_write), K(sstable_already_created), K(start_scn), K(commit_scn)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id_, new_tablet_handle))) { + K(sstable_already_created), K(start_scn), K(commit_scn)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, new_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else { ObSSTableMetaHandle sst_meta_hdl; @@ -2160,7 +2136,7 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star } else if (OB_ISNULL(first_major_sstable = static_cast (table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/)))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("no major after wait merge success", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("no major after wait merge success", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(first_major_sstable->get_key() != table_key_)) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("ddl major sstable dropped, snapshot holding may have bug", @@ -2180,7 +2156,6 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star } else { for (int64_t retry_cnt = 10; retry_cnt > 0; retry_cnt--) { // overwrite ret if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum( - ls_id_, tablet_id_, sqc_build_ctx_.build_param_.runtime_only_param_.table_id_, execution_id, @@ -2188,7 +2163,7 @@ int ObTabletFullDirectLoadMgr::close(const int64_t execution_id, const SCN &star column_checksums, column_count, tenant_data_version_))) { - LOG_WARN("report ddl column checksum failed", K(ret), K(ls_id_), K(tablet_id_), K(execution_id), K(sqc_build_ctx_)); + LOG_WARN("report ddl column checksum failed", K(ret), K(tablet_id_), K(execution_id), K(sqc_build_ctx_)); } else { break; } @@ -2252,14 +2227,14 @@ int ObTabletFullDirectLoadMgr::start( } else if (OB_FAIL(tablet.get_ddl_kv_mgr(ddl_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create tablet ddl kv mgr handle failed", K(ret)); } else if (lob_mgr_handle_.is_valid()) { - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle lob_tablet_handle; if (OB_ISNULL(share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_mgr_handle_.get_obj()->get_tablet_id(), lob_tablet_handle))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, lob_mgr_handle_.get_obj()->get_tablet_id(), lob_tablet_handle))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_FAIL(lob_tablet_handle.get_obj()->get_ddl_kv_mgr(lob_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create tablet ddl kv mgr handle failed", K(ret)); @@ -2267,15 +2242,15 @@ int ObTabletFullDirectLoadMgr::start( } if (OB_SUCC(ret)) { - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls handle failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls()) || OB_ISNULL(ls_handle.get_ls()->get_ddl_log_handler())) { + ObLS *ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_ISNULL(ls->get_ddl_log_handler())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls or ddl log handler is null", K(ret), KPC(ls_handle.get_ls()), K(ls_id_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(tablet_id_))) { - LOG_WARN("add tablet id failed", K(ret), K(ls_id_), K(tablet_id_)); - } else if (lob_kv_mgr_handle.is_valid() && OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(lob_mgr_handle_.get_obj()->get_tablet_id()))) { + LOG_WARN("ls or ddl log handler is null", K(ret), KPC(ls)); + } else if (OB_FAIL(ls->get_ddl_log_handler()->add_tablet(tablet_id_))) { + LOG_WARN("add tablet id failed", K(ret), K(tablet_id_)); + } else if (lob_kv_mgr_handle.is_valid() && OB_FAIL(ls->get_ddl_log_handler()->add_tablet(lob_mgr_handle_.get_obj()->get_tablet_id()))) { LOG_WARN("add lob tablet id failed", K(ret), "lob_tablet_id", lob_mgr_handle_.get_obj()->get_tablet_id()); } } @@ -2340,13 +2315,13 @@ int ObTabletFullDirectLoadMgr::start_nolock( if (start_scn_.is_valid_and_not_min()) { if (execution_id >= execution_id_ && start_scn >= start_scn_) { is_brand_new = true; - LOG_INFO("execution id changed, need cleanup", K(ls_id_), K(tablet_id_), K(execution_id_), K(execution_id), K(start_scn_), K(start_scn)); + LOG_INFO("execution id changed, need cleanup", K(tablet_id_), K(execution_id_), K(execution_id), K(start_scn_), K(start_scn)); } else { if (!checkpoint_scn.is_valid_and_not_min()) { // only return error code when not start from checkpoint. ret = OB_TASK_EXPIRED; } - LOG_INFO("ddl start ignored", K(ls_id_), K(tablet_id_), K(execution_id_), K(execution_id), K(start_scn_), K(start_scn), K(checkpoint_scn)); + LOG_INFO("ddl start ignored", K(tablet_id_), K(execution_id_), K(execution_id), K(start_scn_), K(start_scn), K(checkpoint_scn)); } } else { is_brand_new = true; @@ -2429,21 +2404,17 @@ int ObTabletFullDirectLoadMgr::commit( } } if (OB_SUCC(ret) && lob_mgr_handle_.is_valid()) { - const share::ObLSID &ls_id = lob_mgr_handle_.get_full_obj()->get_ls_id(); const ObTabletID &lob_tablet_id = lob_mgr_handle_.get_full_obj()->get_tablet_id(); - ObLSHandle ls_handle; ObLS *ls = nullptr; ObTabletHandle lob_tablet_handle; if (OB_ISNULL(share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls->get_tablet(lob_tablet_id, lob_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(lob_tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ls->get_tablet(lob_tablet_id, lob_tablet_handle, + ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("get tablet handle failed", K(ret), K(lob_tablet_id)); } else if (OB_FAIL(lob_mgr_handle_.get_full_obj()->commit(*lob_tablet_handle.get_obj(), start_scn, commit_scn, table_id, ddl_task_id, is_replay))) { LOG_WARN("commit for lob failed", K(ret), K(start_scn), K(commit_scn)); } @@ -2472,7 +2443,6 @@ int ObTabletFullDirectLoadMgr::schedule_merge_task( } else { ObDDLTableMergeDagParam param; param.direct_load_type_ = direct_load_type_; - param.ls_id_ = ls_id_; param.tablet_id_ = tablet_id_; param.rec_scn_ = commit_scn; param.is_commit_ = true; @@ -2496,7 +2466,7 @@ int ObTabletFullDirectLoadMgr::schedule_merge_task( if (OB_SUCC(ret)) { const ObSSTable *first_major_sstable = nullptr; ObTabletMemberWrapper table_store_wrapper; - if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(ls_id_, tablet_id_, first_major_sstable, table_store_wrapper))) { + if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(tablet_id_, first_major_sstable, table_store_wrapper))) { LOG_WARN("check if major sstable exist failed", K(ret)); } else if (nullptr != first_major_sstable) { FLOG_INFO("major has already existed", KPC(this)); @@ -2504,7 +2474,7 @@ int ObTabletFullDirectLoadMgr::schedule_merge_task( } } if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) { - LOG_INFO("wait build ddl sstable", K(ret), K(ls_id_), K(tablet_id_), K(start_scn), K(commit_scn), + LOG_INFO("wait build ddl sstable", K(ret), K(tablet_id_), K(start_scn), K(commit_scn), "wait_elpased_s", (ObTimeUtility::fast_current_time() - wait_start_ts) / 1000000L); } } @@ -2523,7 +2493,7 @@ void ObTabletFullDirectLoadMgr::set_commit_scn_nolock(const share::SCN &scn) int ObTabletFullDirectLoadMgr::set_commit_scn(const share::SCN &commit_scn) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -2531,13 +2501,13 @@ int ObTabletFullDirectLoadMgr::set_commit_scn(const share::SCN &commit_scn) } else if (OB_UNLIKELY(!commit_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(commit_scn)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id_, + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet(tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id_)); } else { uint32_t lock_tid = 0; if (OB_FAIL(wrlock(TRY_LOCK_TIMEOUT, lock_tid))) { @@ -2616,7 +2586,6 @@ int ObTabletFullDirectLoadMgr::prepare_ddl_merge_param( LOG_WARN("check can schedule major compaction failed", K(ret)); } else if (can_schedule) { merge_param.direct_load_type_ = direct_load_type_; - merge_param.ls_id_ = ls_id_; merge_param.tablet_id_ = tablet_id_; merge_param.rec_scn_ = get_commit_scn(tablet.get_tablet_meta()); merge_param.is_commit_ = true; @@ -2625,7 +2594,6 @@ int ObTabletFullDirectLoadMgr::prepare_ddl_merge_param( merge_param.snapshot_version_ = table_key_.get_snapshot_version(); } else { merge_param.direct_load_type_ = direct_load_type_; - merge_param.ls_id_ = ls_id_; merge_param.tablet_id_ = tablet_id_; merge_param.start_scn_ = start_scn_; merge_param.data_format_version_ = tenant_data_version_; @@ -2652,7 +2620,6 @@ int ObTabletFullDirectLoadMgr::prepare_major_merge_param( LOG_WARN("ddl not started", K(ret)); } else { param.direct_load_type_ = direct_load_type_; - param.ls_id_ = ls_id_; param.table_key_ = table_key_; param.start_scn_ = start_scn_; param.commit_scn_ = commit_scn_; @@ -2676,15 +2643,14 @@ int ObTabletFullDirectLoadMgr::cleanup_unlock() LOG_INFO("cleanup expired sstables", K(*this)); ObLS *ls = nullptr; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -2710,7 +2676,7 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( { int ret = OB_SUCCESS; uint32_t lock_tid = 0; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObArenaAllocator tmp_arena("DDLUpdateTblTmp", OB_MALLOC_NORMAL_BLOCK_SIZE); ObStorageSchema *storage_schema = nullptr; @@ -2722,18 +2688,17 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( } else if (OB_UNLIKELY(!start_scn.is_valid_and_not_min() || snapshot_version <= 0 || !ddl_checkpoint_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(start_scn), K(snapshot_version), K(ddl_checkpoint_scn)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(tmp_arena, storage_schema))) { LOG_WARN("failed to load storage schema", K(ret), K(tablet_handle)); } else { ObTableHandleV2 table_handle; // empty - const int64_t rebuild_seq = ls_handle.get_ls()->get_rebuild_seq(); ObTableHandleV2 sstable_handle; ObTabletHandle new_tablet_handle; ObArray empty_meta_array; @@ -2741,7 +2706,6 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( ObTabletDDLParam ddl_param; ddl_param.direct_load_type_ = direct_load_type_; - ddl_param.ls_id_ = ls_id_; ddl_param.table_key_ = table_key_; ddl_param.start_scn_ = start_scn; ddl_param.commit_scn_ = commit_scn_; @@ -2753,8 +2717,7 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( ObUpdateTableStoreParam param(tablet_handle.get_obj()->get_snapshot_version(), ObVersionRange::MIN_VERSION, // multi_version_start - storage_schema, - rebuild_seq); + storage_schema); param.ddl_info_.keep_old_ddl_sstable_ = false; param.ddl_info_.ddl_start_scn_ = start_scn; param.ddl_info_.ddl_snapshot_version_ = snapshot_version; @@ -2767,10 +2730,10 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( } if (OB_FAIL(ret)) { } else if (FALSE_IT(param.sstable_ = static_cast(sstable_handle.get_table()))) { - } else if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) { - LOG_WARN("failed to update tablet table store", K(ret), K(ls_id_), K(tablet_id_), K(param)); + } else if (OB_FAIL(ls->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) { + LOG_WARN("failed to update tablet table store", K(ret), K(tablet_id_), K(param)); } else { - LOG_INFO("update tablet success", K(ls_id_), K(tablet_id_), + LOG_INFO("update tablet success", K(tablet_id_), K(ddl_param), "update_table_store_param", param, K(start_scn), K(snapshot_version), K(ddl_checkpoint_scn)); } } @@ -2784,32 +2747,31 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( int ObTabletFullDirectLoadMgr::update_major_sstable() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObArenaAllocator tmp_arena("DDLUpdateTblTmp", OB_MALLOC_NORMAL_BLOCK_SIZE); ObStorageSchema *storage_schema = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(tmp_arena, storage_schema))) { - LOG_WARN("load storage schema failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("load storage schema failed", K(ret), K(tablet_id_)); } else { ObTabletHandle new_tablet_handle; ObUpdateTableStoreParam param(tablet_handle.get_obj()->get_snapshot_version(), ObVersionRange::MIN_VERSION, // multi_version_start - storage_schema, - ls_handle.get_ls()->get_rebuild_seq()); + storage_schema); param.ddl_info_.keep_old_ddl_sstable_ = true; param.ddl_info_.ddl_commit_scn_ = get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()); // ddl commit scn may larger than ddl checkpoint scn - if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) { - LOG_WARN("failed to update tablet table store", K(ret), K(ls_id_), K(tablet_id_), K(param)); + if (OB_FAIL(ls->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) { + LOG_WARN("failed to update tablet table store", K(ret), K(tablet_id_), K(param)); } } ObTabletObjLoadHelper::free(tmp_arena, storage_schema); @@ -2838,7 +2800,7 @@ int ObTabletIncDirectLoadMgr::update( int ret = OB_SUCCESS; uint32_t lock_tid = 0; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObStorageSchema *storage_schema = nullptr; ObTabletHandle tablet_handle; ObArenaAllocator arena_allocator("TDL_UPDATE_SS", OB_MALLOC_NORMAL_BLOCK_SIZE); @@ -2853,9 +2815,9 @@ int ObTabletIncDirectLoadMgr::update( } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(build_param.common_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(build_param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, build_param.common_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -2909,8 +2871,8 @@ int ObTabletIncDirectLoadMgr::open( } if (OB_SUCC(ret)) { ObDDLIncRedoLogWriter redo_writer; - if (OB_FAIL(redo_writer.init(ls_id_, tablet_id_))) { - LOG_WARN("init redo writer failed", K(ret), K(ls_id_), K(tablet_id_)); + if (OB_FAIL(redo_writer.init(tablet_id_))) { + LOG_WARN("init redo writer failed", K(ret), K(tablet_id_)); } else if (OB_FAIL(redo_writer.write_inc_start_log_with_retry( nullptr != lob_tablet_mgr ? lob_tablet_mgr->get_tablet_id() : ObTabletID(), sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_, @@ -2965,10 +2927,9 @@ int ObTabletIncDirectLoadMgr::close(const int64_t current_execution_id, const sh LOG_WARN("unexpected err", K(ret), KPC(this)); } else { ObDDLIncRedoLogWriter redo_writer; - if (OB_FAIL(redo_writer.init(ls_id_, tablet_id_))) { - LOG_WARN("init redo writer failed", K(ret), K(ls_id_), K(tablet_id_)); + if (OB_FAIL(redo_writer.init(tablet_id_))) { + LOG_WARN("init redo writer failed", K(ret), K(tablet_id_)); } else if (OB_FAIL(redo_writer.write_inc_commit_log_with_retry( - true /*allow_remote_write*/, nullptr != lob_tablet_mgr ? lob_tablet_mgr->get_tablet_id() : ObTabletID(), sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_))) { LOG_WARN("fail write start log", K(ret), K(tenant_data_version_), K(sqc_build_ctx_)); diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h index 72b61b419..f2857481c 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h @@ -79,7 +79,6 @@ class ObTenantDirectLoadMgr final int alloc_slice_id(int64_t &slice_id); int get_agent_exec_context( const int64_t context_id, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const ObDirectLoadType &type, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -107,11 +106,9 @@ class ObTenantDirectLoadMgr final // to start the direct load, write start log in actually. // @param [in] type. - // @param [in] ls_id. // @param [in] tablet_id, the commit version for the full direct load, int open_tablet_direct_load( const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t context_id); @@ -125,7 +122,6 @@ class ObTenantDirectLoadMgr final int close_tablet_direct_load( const int64_t context_id, const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit, const bool emergent_finish = true, @@ -146,7 +142,6 @@ class ObTenantDirectLoadMgr final ObTabletDirectLoadMgrHandle &direct_load_mgr_handle); // TODO YIREN, adapt the interface to the shared-storage and shared-nothing mode. int get_tablet_mgr_and_check_major( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool is_full_direct_load, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, @@ -161,7 +156,6 @@ class ObTenantDirectLoadMgr final ObIAllocator &get_allocator() { return allocator_; } private: int check_and_process_finished_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObIStoreRowIterator *row_iter = nullptr, const int64_t task_id = 0, @@ -170,7 +164,6 @@ class ObTenantDirectLoadMgr final int close_tablet_direct_load_for_sn( const int64_t context_id, const ObDirectLoadType &type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit, const int64_t task_id = 0, @@ -178,19 +171,18 @@ class ObTenantDirectLoadMgr final const int64_t execution_id = -1); int close_tablet_direct_load_for_ss( const int64_t context_id, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const bool need_commit); private: struct GetGcCandidateOp final { public: - GetGcCandidateOp(ObIArray> &candidate_mgrs) + GetGcCandidateOp(ObIArray &candidate_mgrs) : candidate_mgrs_(candidate_mgrs) {} ~GetGcCandidateOp() {} int operator() (common::hash::HashMapPair &kv); private: DISALLOW_COPY_AND_ASSIGN(GetGcCandidateOp); - ObIArray> &candidate_mgrs_; + ObIArray &candidate_mgrs_; }; static int alloc_tablet_direct_load_mgr( @@ -354,7 +346,7 @@ class ObTabletDirectLoadMgr :public ObBaseTabletDirectLoadMgr int prepare_storage_schema(ObTabletHandle &tablet_handle); int64_t get_task_cnt() override { return task_cnt_; } bool get_is_no_logging() override { return is_no_logging_; } - VIRTUAL_TO_STRING_KV(K_(is_inited), K_(is_schema_item_ready), K_(ls_id), K_(tablet_id), K_(table_key), K_(ref_cnt), + VIRTUAL_TO_STRING_KV(K_(is_inited), K_(is_schema_item_ready), K_(tablet_id), K_(table_key), K_(ref_cnt), K_(direct_load_type), K_(sqc_build_ctx), KPC(lob_mgr_handle_.get_obj()), K_(schema_item), K_(column_items), K_(lob_column_idxs), K_(task_cnt), K_(micro_index_clustered), K_(is_no_logging)); diff --git a/src/storage/ddl/ob_direct_load_mgr_agent.cpp b/src/storage/ddl/ob_direct_load_mgr_agent.cpp index 62533ba8e..fc1cfc11f 100644 --- a/src/storage/ddl/ob_direct_load_mgr_agent.cpp +++ b/src/storage/ddl/ob_direct_load_mgr_agent.cpp @@ -78,7 +78,6 @@ int ObDirectLoadMgrAgent::init(ObBaseTabletDirectLoadMgr *direct_load_mgr, ObBas int ObDirectLoadMgrAgent::init( const int64_t context_id, - const share::ObLSID &ls_id, const ObTabletID &tablet_id/*always data tablet id.*/, const ObDirectLoadType &type) { @@ -88,22 +87,21 @@ int ObDirectLoadMgrAgent::init( ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); } else if (OB_UNLIKELY(context_id <= 0 - || !ls_id.is_valid() || !tablet_id.is_valid()) || !is_valid_direct_load(type)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(context_id), K(ls_id), K(tablet_id), K(type)); + LOG_WARN("invalid args", K(ret), K(context_id), K(tablet_id), K(type)); } else if (OB_ISNULL(tenant_direct_load_mgr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_agent_exec_context( - context_id, ls_id, tablet_id, type, mgr_handle_, start_scn_, execution_id_))) { + context_id, tablet_id, type, mgr_handle_, start_scn_, execution_id_))) { LOG_WARN("get agent exec context failed", K(ret), K(context_id), K(tablet_id)); } else if (is_shared_storage_dempotent_mode(type)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid direct load type which not used any more", K(ret)); - } else if (OB_FAIL(init_for_sn(ls_id, tablet_id))) { - LOG_WARN("init for sn failed", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(init_for_sn(tablet_id))) { + LOG_WARN("init for sn failed", K(ret), K(tablet_id)); } if (OB_SUCC(ret)) { direct_load_type_ = type; @@ -112,24 +110,22 @@ int ObDirectLoadMgrAgent::init( return ret; } -int ObDirectLoadMgrAgent::init_for_sn( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id) +int ObDirectLoadMgrAgent::init_for_sn(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("invalid args", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet handle is null", K(ret), K(tablet_id)); } else if (OB_LIKELY(mgr_handle_.is_valid())) { if (!start_scn_.is_valid_and_not_min() || execution_id_ < 0) { ret = OB_ERR_SYS; @@ -137,13 +133,13 @@ int ObDirectLoadMgrAgent::init_for_sn( } } else if (OB_UNLIKELY(is_incremental_direct_load(direct_load_type_))) { ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("mgr handle not exist", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("mgr handle not exist", K(ret), K(tablet_id)); } else { const blocksstable::ObSSTable *first_major_sstable = nullptr; ObTabletMemberWrapper table_store_wrapper; if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable( - ls_id, tablet_id, first_major_sstable, table_store_wrapper))) { - LOG_WARN("check if major sstable exist failed", K(ret), K(ls_id), K(tablet_id)); + tablet_id, first_major_sstable, table_store_wrapper))) { + LOG_WARN("check if major sstable exist failed", K(ret), K(tablet_id)); } else if (OB_NOT_NULL(first_major_sstable)) { LOG_INFO("skip, mgr handle is invalid due to major exist under shared-nothing mode", K(tablet_id)); } else { @@ -608,7 +604,7 @@ int ObDirectLoadMgrAgent::close(const int64_t context_id, const bool need_commit LOG_WARN("mgr handle is invalid", K(ret)); } else if (!is_idem_type(direct_load_type_)) { ObTenantDirectLoadMgr *tenant_direct_load_mgr = share::g_mp->tenant_direct_load_mgr(); - if (OB_FAIL(tenant_direct_load_mgr->close_tablet_direct_load(context_id, direct_load_type_, mgr_handle_.get_base_obj()->get_ls_id(), mgr_handle_.get_base_obj()->get_tablet_id(), need_commit, true/*emergent_finish*/, + if (OB_FAIL(tenant_direct_load_mgr->close_tablet_direct_load(context_id, direct_load_type_, mgr_handle_.get_base_obj()->get_tablet_id(), need_commit, true/*emergent_finish*/, mgr_handle_.get_base_obj()->get_ddl_task_id(), mgr_handle_.get_base_obj()->get_build_param().runtime_only_param_.table_id_, execution_id))) { LOG_WARN("close tablet direct load failed", K(ret), KPC(mgr_handle_.get_base_obj())); } diff --git a/src/storage/ddl/ob_direct_load_mgr_agent.h b/src/storage/ddl/ob_direct_load_mgr_agent.h index b9ca17937..8b10f180d 100644 --- a/src/storage/ddl/ob_direct_load_mgr_agent.h +++ b/src/storage/ddl/ob_direct_load_mgr_agent.h @@ -52,7 +52,6 @@ struct ObDirectLoadMgrAgent final public: int init( const int64_t context_id, - const share::ObLSID &ls_id, const ObTabletID &tablet_id/*always data tablet id.*/, const ObDirectLoadType &type); int open_sstable_slice( @@ -119,9 +118,7 @@ struct ObDirectLoadMgrAgent final TO_STRING_KV(K_(is_inited), K_(direct_load_type), K_(start_scn), K_(execution_id), KP(mgr_handle_.get_obj())); private: int close_for_idem(); - int init_for_sn( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id); + int init_for_sn(const ObTabletID &tablet_id); int init_for_ss(); int open_sstable_slice_for_ss( const blocksstable::ObMacroDataSeq &start_seq, diff --git a/src/storage/ddl/ob_direct_load_mgr_utils.cpp b/src/storage/ddl/ob_direct_load_mgr_utils.cpp index 29a78cc58..04e460463 100644 --- a/src/storage/ddl/ob_direct_load_mgr_utils.cpp +++ b/src/storage/ddl/ob_direct_load_mgr_utils.cpp @@ -25,26 +25,26 @@ namespace oceanbase namespace storage { -int ObDirectLoadMgrUtil::get_lob_tablet_id(const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletID &lob_tablet_id) +int ObDirectLoadMgrUtil::get_lob_tablet_id(const ObTabletID &tablet_id, ObTabletID &lob_tablet_id) { int ret = OB_SUCCESS; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletBindingMdsUserData ddl_data; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_id)); } else { lob_tablet_id = ddl_data.lob_piece_tablet_id_; } @@ -81,25 +81,25 @@ int ObDirectLoadMgrUtil::is_ddl_need_major_merge(const ObTablet &tablet, bool &d } -int ObDirectLoadMgrUtil::check_tablet_major_exist(const ObLSID &ls_id, const ObTabletID &tablet_id, bool &is_major_sstable_exist) +int ObDirectLoadMgrUtil::check_tablet_major_exist(const ObTabletID &tablet_id, bool &is_major_sstable_exist) { int ret = OB_SUCCESS; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletMemberWrapper table_store_wrapper; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->fetch_table_store(table_store_wrapper))) { - LOG_WARN("failed to fetch table store", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to fetch table store", K(ret), K(tablet_id)); } else { is_major_sstable_exist = nullptr != table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/); } @@ -158,7 +158,6 @@ int ObDirectLoadMgrUtil::create_idem_tablet_direct_load_mgr(const int64_t execut ObTabletDirectLoadMgrHandle &lob_direct_load_mgr_handle) { int ret = OB_SUCCESS; - const ObLSID &ls_id = build_param.common_param_.ls_id_; const ObTabletID &tablet_id = build_param.common_param_.tablet_id_; ObTabletID lob_tablet_id; /* reset param*/ @@ -169,10 +168,10 @@ int ObDirectLoadMgrUtil::create_idem_tablet_direct_load_mgr(const int64_t execut if (!build_param.is_valid() || 0 > execution_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(build_param), K(execution_id)); - } else if (OB_FAIL(get_lob_tablet_id(build_param.common_param_.ls_id_, build_param.common_param_.tablet_id_, lob_tablet_id))) { + } else if (OB_FAIL(get_lob_tablet_id(build_param.common_param_.tablet_id_, lob_tablet_id))) { LOG_WARN("failed to get lob_tablet_id", K(ret), K(build_param.common_param_)); - } else if (OB_FAIL(check_tablet_major_exist(ls_id, tablet_id, is_major_sstable_exist))) { - LOG_WARN("failed to check major sstable exist", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(check_tablet_major_exist(tablet_id, is_major_sstable_exist))) { + LOG_WARN("failed to check major sstable exist", K(ret), K(tablet_id)); } else { ObBaseTabletDirectLoadMgr* direct_load_mgr = nullptr; ObBaseTabletDirectLoadMgr* lob_direct_load_mgr = nullptr; @@ -255,7 +254,7 @@ int ObDirectLoadMgrUtil::create_tablet_direct_load_mgr(const int64_t execution_i } else if (!is_idem_type(build_param.common_param_.direct_load_type_)) { ObTenantDirectLoadMgr *tenant_direct_load_mgr = share::g_mp->tenant_direct_load_mgr(); ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; if (OB_ISNULL(tenant_direct_load_mgr)) { ret = OB_ERR_UNEXPECTED; @@ -268,9 +267,9 @@ int ObDirectLoadMgrUtil::create_tablet_direct_load_mgr(const int64_t execution_i } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(build_param.common_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(build_param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, build_param.common_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, build_param.common_param_.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("get tablet handle failed", K(ret), K(build_param)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -294,7 +293,6 @@ int ObDirectLoadMgrUtil::generate_merge_param(const ObTabletDDLCompleteArg &arg, LOG_WARN("invalid argument", K(ret), K(arg)); } else { merge_param.direct_load_type_ = arg.direct_load_type_; - merge_param.ls_id_ = arg.ls_id_; merge_param.tablet_id_ = arg.tablet_id_; merge_param.data_format_version_ = arg.data_format_version_; merge_param.snapshot_version_ = arg.snapshot_version_; @@ -316,7 +314,6 @@ int ObDirectLoadMgrUtil::generate_merge_param(const ObTabletDDLCompleteMdsUserDa LOG_WARN("invalid argument", K(ret), K(data)); } else if (data.has_complete_) { /* generate param for major merge */ merge_param.direct_load_type_ = data.direct_load_type_; - merge_param.ls_id_ = tablet.get_ls_id(); merge_param.tablet_id_ = tablet.get_tablet_id(); merge_param.data_format_version_ = data.data_format_version_; merge_param.snapshot_version_ = data.snapshot_version_; @@ -340,7 +337,6 @@ int ObDirectLoadMgrUtil::generate_merge_param(const ObTabletDDLCompleteMdsUserDa LOG_WARN("failed to get ddl kv_mgr_handle", K(ret)); } else { merge_param.direct_load_type_ = ObDirectLoadType::SN_IDEM_DIRECT_LOAD_DDL; // mock type - merge_param.ls_id_ = tablet.get_ls_id(); merge_param.tablet_id_ = tablet.get_tablet_id(); merge_param.data_format_version_ = ddl_kvs_handle.at(0).get_obj()->get_data_format_version(); merge_param.snapshot_version_ = ddl_kvs_handle.at(0).get_obj()->get_snapshot_version(); @@ -429,21 +425,21 @@ int ObDirectLoadMgrUtil::prepare_schema_item_for_vec_idx_data(ObSchemaGetterGuar return ret; } -int ObDirectLoadMgrUtil::get_tablet_handle(const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletHandle &tablet_handle) +int ObDirectLoadMgrUtil::get_tablet_handle(const ObTabletID &tablet_id, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObLSService * ls_service = nullptr; tablet_handle.reset(); - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/ddl/ob_direct_load_mgr_utils.h b/src/storage/ddl/ob_direct_load_mgr_utils.h index 93b5e7335..7836217af 100644 --- a/src/storage/ddl/ob_direct_load_mgr_utils.h +++ b/src/storage/ddl/ob_direct_load_mgr_utils.h @@ -35,7 +35,7 @@ static bool need_process_vec_index(const ObIndexType & index_type) || schema::is_vec_ivfpq_pq_centroid_index(index_type) || schema::is_vec_index_snapshot_data_type(index_type); } - static int get_tablet_handle(const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletHandle &tablet_handle); + static int get_tablet_handle(const ObTabletID &tablet_id, ObTabletHandle &tablet_handle); static int create_tablet_direct_load_mgr(const int64_t execution_id, const int64_t context_id, const ObTabletDirectLoadInsertParam &build_param, @@ -46,7 +46,6 @@ static bool need_process_vec_index(const ObIndexType & index_type) static ObDirectLoadType ddl_get_direct_load_type(const uint64_t data_format_version); static ObDirectLoadType load_data_get_direct_load_type(const bool is_incremental, const uint64_t data_format_version); - static int check_major_exist(const ObLSID &ls_id, const ObTabletID &talbet_id, bool is_major_eixst); static int generate_merge_param(const ObTabletDDLCompleteArg &arg, ObDDLTableMergeDagParam &merge_param); static int generate_merge_param(const ObTabletDDLCompleteMdsUserData &data, ObTablet &tablet, ObDDLTableMergeDagParam &merge_param); static int is_ddl_need_major_merge(const ObTablet &tablet, bool &ddl_need_merging); @@ -63,8 +62,8 @@ static bool need_process_vec_index(const ObIndexType & index_type) bool &is_major_sstable_exist, ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, ObTabletDirectLoadMgrHandle &lob_direct_load_mgr_handle); - static int check_tablet_major_exist(const ObLSID &ls_id, const ObTabletID &tablet_id, bool &is_major_sstable_exist); - static int get_lob_tablet_id(const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletID &lob_tablet_id); + static int check_tablet_major_exist(const ObTabletID &tablet_id, bool &is_major_sstable_exist); + static int get_lob_tablet_id(const ObTabletID &tablet_id, ObTabletID &lob_tablet_id); }; } // namespace storage } // namespace oceanbaes diff --git a/src/storage/ddl/ob_direct_load_mgr_v3.cpp b/src/storage/ddl/ob_direct_load_mgr_v3.cpp index 5d6142555..4d4a036f2 100644 --- a/src/storage/ddl/ob_direct_load_mgr_v3.cpp +++ b/src/storage/ddl/ob_direct_load_mgr_v3.cpp @@ -98,7 +98,6 @@ int ObTabletDirectLoadMgrV3::prepare_index_builder(const ObTabletDirectLoadInser { int ret = OB_SUCCESS; const uint64_t &data_format_version = build_param.common_param_.data_format_version_; - const ObLSID &ls_id = build_param.common_param_.ls_id_; const ObTabletID &tablet_id = build_param.common_param_.tablet_id_; const ObDirectLoadType &direct_load_type = build_param.common_param_.direct_load_type_; blocksstable::ObWholeDataStoreDesc index_block_desc; @@ -108,7 +107,7 @@ int ObTabletDirectLoadMgrV3::prepare_index_builder(const ObTabletDirectLoadInser if (!build_param.is_valid() || !table_key.is_valid() || !table_schema.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(build_param), K(table_schema), K(table_key)); - } else if (OB_FAIL(index_block_desc.init(true/*is ddl*/, table_schema, ls_id, tablet_id, + } else if (OB_FAIL(index_block_desc.init(true/*is ddl*/, table_schema, tablet_id, is_full_direct_load(direct_load_type) ? compaction::ObMergeType::MAJOR_MERGE : compaction::ObMergeType::MINOR_MERGE, is_full_direct_load(direct_load_type) ? table_key.get_snapshot_version() : 1L, data_format_version, table_schema.get_micro_index_clustered(), 0/*concurrent_cnt*/, @@ -127,7 +126,7 @@ int ObTabletDirectLoadMgrV3::prepare_index_builder(const ObTabletDirectLoadInser index_block_desc.get_desc(), // index_block_desc is copied in index_builder ObSSTableIndexBuilder::DISABLE))) { LOG_WARN("failed to init index builder", K(ret), K(index_block_desc)); - } else if (OB_FAIL(data_block_desc.init(true/*is ddl*/, table_schema, ls_id, tablet_id, + } else if (OB_FAIL(data_block_desc.init(true/*is ddl*/, table_schema, tablet_id, compaction::ObMergeType::MAJOR_MERGE, // TODO @zhuora.zzr to set virtual is_full_direct_load(direct_load_type) ? table_key.get_snapshot_version() : 1L, data_format_version, table_schema.get_micro_index_clustered(), 0/*concurrent_cnt*/, @@ -249,7 +248,7 @@ int ObTabletDirectLoadMgrV3::init_v2(const ObTabletDirectLoadInsertParam &build_ } else if (!build_param.is_valid() || !is_idem_type(build_param.common_param_.direct_load_type_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid build param", K(ret), K(build_param)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.ls_id_, build_param.common_param_.tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(build_param)); /* get tabel schema */ } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(schema_guard))) { @@ -289,7 +288,6 @@ int ObTabletDirectLoadMgrV3::init_v2(const ObTabletDirectLoadInsertParam &build_ } else { /* basic info */ build_param_.assign(build_param); - ls_id_ = build_param.common_param_.ls_id_; tablet_id_ = build_param.common_param_.tablet_id_; tenant_data_version_ = build_param.common_param_.data_format_version_; direct_load_type_ = build_param.common_param_.direct_load_type_; @@ -302,7 +300,7 @@ int ObTabletDirectLoadMgrV3::init_v2(const ObTabletDirectLoadInsertParam &build_ LOG_WARN("failed to create ddl kv mgr", K(ret)); } else { is_inited_ = true; - FLOG_INFO("[DIRECT_LOAD_MGR] success to create tablet direct load mgr", K(table_key_), K(ls_id_), K(tablet_id_), K(execution_id), + FLOG_INFO("[DIRECT_LOAD_MGR] success to create tablet direct load mgr", K(table_key_), K(tablet_id_), K(execution_id), K(direct_load_type_), KP(index_builder_), KPC(index_builder_), KP(storage_schema_)); } return ret; @@ -361,7 +359,7 @@ int ObTabletDirectLoadMgrV3::prepare_lob_param(const ObTabletDirectLoadInsertPar LOG_WARN("invalid argument", K(ret), K(build_param)); } else if (OB_FAIL(lob_param.assign(build_param))) { LOG_WARN("failed to assign lob param", K(ret), K(build_param)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.ls_id_, build_param.common_param_.tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet hanlde", K(ret), K(build_param)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { LOG_WARN("get ddl data failed", K(ret)); @@ -405,7 +403,7 @@ int ObTabletDirectLoadMgrV3::fill_sstable_slice_v2(const ObDirectLoadSliceInfo & } else { next_seq = blocksstable::ObMacroDataSeq(slice_writer.get_next_block_start_seq()); } - FLOG_INFO("[DIRECT_LOAD_MGR] fill sstable slice", K(ret), K(ls_id_), K(tablet_id_), K(execution_id_), K(slice_info), K(affected_rows), K(next_seq)); + FLOG_INFO("[DIRECT_LOAD_MGR] fill sstable slice", K(ret), K(tablet_id_), K(execution_id_), K(slice_info), K(affected_rows), K(next_seq)); return ret; } @@ -429,7 +427,7 @@ int ObTabletDirectLoadMgrV3::fill_sstable_slice_v2(const ObDirectLoadSliceInfo & direct_load_type_, column_items_, dir_id_, get_parallel(), slice_info.context_id_, insert_monitor))) { LOG_WARN("fill sstable slice failed", K(ret), KPC(this)); } - FLOG_INFO("[DIRECTL_LOAD_MGR] fill sstable slice", K(ret), K(ls_id_), K(tablet_id_), K(execution_id_), K(slice_info)); //TODO @zhuoran.zzr wait to remove to the sub class + FLOG_INFO("[DIRECTL_LOAD_MGR] fill sstable slice", K(ret), K(tablet_id_), K(execution_id_), K(slice_info)); //TODO @zhuoran.zzr wait to remove to the sub class return ret; } @@ -482,7 +480,7 @@ int ObTabletDirectLoadMgrV3::close_sstable_slice_v2(const ObDirectLoadSliceInfo ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret), KPC(this)); } else if (OB_FAIL(slice_writer.close())) { - LOG_WARN("failed to close_sstable_slice", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("failed to close_sstable_slice", K(ret), K(tablet_id_)); } else if (FALSE_IT(next_seq = slice_writer.get_next_block_start_seq())) { } else if (!slice_info.is_lob_slice_ && is_ddl_direct_load(direct_load_type_)) { int64_t task_finish_count = -1; @@ -510,7 +508,7 @@ int ObTabletDirectLoadMgrV3::close_sstable_slice_v2(const ObDirectLoadSliceInfo } } } - FLOG_INFO("[DIRECT_LOAD_MGR] close sstable slice", K(ret), K(ls_id_), K(tablet_id_), K(execution_id_), K(slice_info), K(is_all_task_finish)); + FLOG_INFO("[DIRECT_LOAD_MGR] close sstable slice", K(ret), K(tablet_id_), K(execution_id_), K(slice_info), K(is_all_task_finish)); return ret; } @@ -543,7 +541,6 @@ int ObTabletDirectLoadMgrV3::fill_lob_sstable_slice_row_v2(ObIAllocator &allocat /* prepare batch slice info with data table direct load mgr*/ ObBatchSliceWriteInfo info(data_direct_load_mgr->tablet_id_, - data_direct_load_mgr->ls_id_, trans_version, direct_load_type_, data_direct_load_mgr->build_param_.runtime_only_param_.trans_id_, @@ -582,7 +579,6 @@ int ObTabletDirectLoadMgrV3::fill_lob_sstable_slice_row_v2(ObIAllocator &allocat } else { const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; ObBatchSliceWriteInfo info(data_direct_load_mgr->tablet_id_, - data_direct_load_mgr->ls_id_, trans_version, direct_load_type_, data_direct_load_mgr->build_param_.runtime_only_param_.trans_id_, @@ -605,9 +601,8 @@ int ObTabletDirectLoadMgrV3::close() { int ret = OB_SUCCESS; SCN commit_scn; - bool is_remote_write = false; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle new_tablet_handle; bool sstable_already_created = false; @@ -617,11 +612,11 @@ int ObTabletDirectLoadMgrV3::close() } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (OB_FAIL(inner_close())) { LOG_WARN("failed to inner close", K(ret), KPC(this)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id_, new_tablet_handle))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id_, new_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else { ObSSTableMetaHandle sst_meta_hdl; @@ -632,7 +627,7 @@ int ObTabletDirectLoadMgrV3::close() } else if (OB_ISNULL(first_major_sstable = static_cast (table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/)))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("no major after wait merge success", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("no major after wait merge success", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(first_major_sstable->get_key() != table_key_)) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("ddl major sstable dropped, snapshot holding may have bug", @@ -644,7 +639,6 @@ int ObTabletDirectLoadMgrV3::close() int64_t column_count = sst_meta_hdl.get_sstable_meta().get_col_checksum_cnt(); for (int64_t retry_cnt = 10; retry_cnt > 0; retry_cnt--) { // overwrite ret if (OB_FAIL(ObTabletDDLUtil::report_ddl_checksum( - ls_id_, tablet_id_, build_param_.runtime_only_param_.table_id_, get_execution_id(), @@ -652,7 +646,7 @@ int ObTabletDirectLoadMgrV3::close() column_checksums, column_count, tenant_data_version_))) { - LOG_WARN("report ddl column checksum failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("report ddl column checksum failed", K(ret), K(tablet_id_)); } else { break; } @@ -722,7 +716,7 @@ int ObSNTabletDirectLoadMgr::schedule_merge_tablet_task(const ObTabletDDLComplet const int64_t wait_start_ts = ObTimeUtility::fast_current_time(); ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; - if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(arg.ls_id_, arg.tablet_id_, tablet_handle))) { + if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(arg.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet_handle", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, false /* not for repaly*/))) { LOG_WARN("failed to get ddl kv mgr", K(ret)); @@ -757,15 +751,15 @@ int ObSNTabletDirectLoadMgr::schedule_merge_tablet_task(const ObTabletDDLComplet if (OB_SUCC(ret)) { const ObSSTable *first_major_sstable = nullptr; ObTabletMemberWrapper table_store_wrapper; - if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(ls_id_, tablet_id_, first_major_sstable, table_store_wrapper))) { + if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(tablet_id_, first_major_sstable, table_store_wrapper))) { LOG_WARN("check if major sstable exist failed", K(ret)); } else if (nullptr != first_major_sstable) { - FLOG_INFO("major has already existed", K(ls_id_), K(tablet_id_)); + FLOG_INFO("major has already existed", K(tablet_id_)); break; } } if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) { - LOG_INFO("wait build ddl sstable", K(ret), K(ls_id_), K(tablet_id_), + LOG_INFO("wait build ddl sstable", K(ret), K(tablet_id_), "wait_elpased_s", (ObTimeUtility::fast_current_time() - wait_start_ts) / 1000000L); } } @@ -789,8 +783,8 @@ int ObSNTabletDirectLoadMgr::inner_close() if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("direct load mgr have not been inited", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id_, tablet_id_, tablet_handle))) { - LOG_WARN("failed to get tablet handle", K(ret), K(ls_id_), K(tablet_id_)); + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id_, tablet_handle))) { + LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id_)); } else if (!tablet_handle.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet handle", K(ret)); @@ -799,10 +793,9 @@ int ObSNTabletDirectLoadMgr::inner_close() } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->freeze_ddl_kv(start_scn_, table_key_.get_snapshot_version(), tenant_data_version_))) { LOG_WARN("failed to freeze ddl kv", K(ret), K(start_scn_), K(table_key_), K(tenant_data_version_)); } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(allocator, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("failed to load storage schema", K(ret), K(tablet_id_)); } else { complete_arg.has_complete_ = true; - complete_arg.ls_id_ = ls_id_; complete_arg.tablet_id_ = tablet_id_; complete_arg.direct_load_type_ = direct_load_type_; complete_arg.start_scn_ = start_scn_; @@ -812,7 +805,7 @@ int ObSNTabletDirectLoadMgr::inner_close() if (OB_FAIL(complete_arg.set_write_stat(write_stats))) { LOG_WARN("failed to set write stat", K(ret), K(write_stats)); } else if (OB_FAIL(complete_arg.set_storage_schema(*storage_schema))) { - LOG_WARN("failed to set storage_schema", K(ret), K(ls_id_), K(tablet_id_), KPC(storage_schema)); + LOG_WARN("failed to set storage_schema", K(ret), K(tablet_id_), KPC(storage_schema)); } else if (OB_FAIL(ObTabletDDLCompleteMdsHelper::record_ddl_complete_arg_to_mds(complete_arg, allocator))) { LOG_WARN("failed to record ddl complete arg to mds", KR(ret), K(complete_arg)); } else { @@ -822,7 +815,7 @@ int ObSNTabletDirectLoadMgr::inner_close() if (OB_FAIL(ret)) { } else if (OB_FAIL(schedule_merge_tablet_task(complete_arg, true /*wait major merge*/))) { - LOG_WARN("failed to wait merge task", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("failed to wait merge task", K(ret), K(tablet_id_)); } return ret; } @@ -867,7 +860,7 @@ int ObSSTabletDirectLoadMgr::init_v2(const ObTabletDirectLoadInsertParam &build_ LOG_WARN("invalid argument", K(ret), K(build_param), K(execution_id)); } else if (OB_FAIL(ObTabletDirectLoadMgrV3::init_v2(build_param, execution_id, role))) { LOG_WARN("failed to init direct load mgr", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.ls_id_, build_param.common_param_.tablet_id_, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(build_param.common_param_.tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(build_param)); } else if (!tablet_handle.is_valid()) { ret = OB_ERR_UNEXPECTED; @@ -942,7 +935,6 @@ int ObSSTabletDirectLoadMgr::create_ddl_ro_sstable(ObTablet &tablet, /* build index layer */ SMART_VARS_2((ObSSTableMergeRes, res), (ObTabletCreateSSTableParam, param)) { ObDDLRedoLogWriterCallbackInitParam init_param; - init_param.ls_id_ = ls_id_; init_param.tablet_id_ = tablet_id_; init_param.direct_load_type_ = direct_load_type_; init_param.block_type_ = DDL_MB_INDEX_TYPE; @@ -957,8 +949,8 @@ int ObSSTabletDirectLoadMgr::create_ddl_ro_sstable(ObTablet &tablet, ObMacroDataSeq tmp_seq(total_slice_cnt_ * compaction::MACRO_STEP_SIZE); tmp_seq.set_index_block(); - if (pre_warm_param.init(ls_id_, tablet_id_)) { - LOG_WARN("failed to init pre warm param", K(ret), K(ls_id_), K(tablet_id_)); + if (pre_warm_param.init(tablet_id_)) { + LOG_WARN("failed to init pre warm param", K(ret), K(tablet_id_)); } else if (!index_builder_->is_inited()) { ret = OB_NOT_INIT; LOG_WARN("index builder should be inited", K(ret), KPC(index_builder_)); @@ -1018,7 +1010,7 @@ int ObSSTabletDirectLoadMgr::inner_close() ObTabletHandle tablet_handle; ObTableHandleV2 sstable_handle; ObSSTable *ddl_major_sstable = nullptr; - if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id_, tablet_id_, tablet_handle))) { + if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret), K(tablet_handle)); } else if (OB_FAIL(create_ddl_ro_sstable(*tablet_handle.get_obj(), arena_allocator_, sstable_handle))) { LOG_WARN("failed to create ddl ro sstable ", K(ret)); @@ -1034,8 +1026,8 @@ int ObSSTabletDirectLoadMgr::inner_close() // sync max lob id if (OB_FAIL(ret) || ObDirectLoadMgrRole::LOB_TABLET_TYPE!= role_) { - } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(ls_id_,tablet_id_, last_lob_id_))) { - LOG_WARN("failed toset lob tablet autoinc seq", K(ret), K(ls_id_), K(tablet_id_), K(role_), K(last_lob_id_)); + } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(tablet_id_, last_lob_id_))) { + LOG_WARN("failed toset lob tablet autoinc seq", K(ret), K(tablet_id_), K(role_), K(last_lob_id_)); } return ret; } diff --git a/src/storage/ddl/ob_direct_load_mgr_v3.h b/src/storage/ddl/ob_direct_load_mgr_v3.h index ac28d5d67..02657e47c 100644 --- a/src/storage/ddl/ob_direct_load_mgr_v3.h +++ b/src/storage/ddl/ob_direct_load_mgr_v3.h @@ -95,7 +95,6 @@ class ObTabletDirectLoadMgrV3 : public ObBaseTabletDirectLoadMgr blocksstable::ObWholeDataStoreDesc &data_block_desc); virtual int inner_close() = 0; static int get_storage_schema(const ObTabletDirectLoadInsertParam &build_param, ObStorageSchema &storage_schema); - static int get_tablet_handle(const ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletHandle &tablet_handle); static int get_target_table_type(const ObStorageSchema &storage_schema, const ObDirectLoadType &direct_load_type, ObITable::TableType &table_type_); diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index c13a26613..bc1f6984d 100644 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -61,7 +61,6 @@ ObDDLInsertRowIterator::ObDDLInsertRowIterator() : is_inited_(false), ddl_agent_(nullptr), slice_row_iter_(nullptr), - ls_id_(), current_tablet_id_(), context_id_(-1), macro_seq_(), @@ -81,7 +80,6 @@ ObDDLInsertRowIterator::~ObDDLInsertRowIterator() int ObDDLInsertRowIterator::init( ObDirectLoadMgrAgent &agent, ObIStoreRowIterator *slice_row_iter, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t context_id, const ObTabletSliceParam &tablet_slice_param, @@ -94,14 +92,13 @@ int ObDDLInsertRowIterator::init( ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); } else if (OB_UNLIKELY(nullptr == slice_row_iter - || !ls_id.is_valid() || !tablet_id.is_valid() || context_id < 0 // no need check tablet slice param, invalid when slice empty || lob_cols_cnt < 0 || (is_idem_type(agent.get_direct_load_type()) && total_slice_cnt < 0))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(slice_row_iter), K(ls_id), K(tablet_id), K(context_id), K(tablet_slice_param), K(lob_cols_cnt), K(total_slice_cnt)); + LOG_WARN("invalid argument", K(ret), KP(slice_row_iter), K(tablet_id), K(context_id), K(tablet_slice_param), K(lob_cols_cnt), K(total_slice_cnt)); } else if (lob_cols_cnt > 0 && tablet_slice_param.is_valid() && OB_FAIL(lob_id_generator_.init(tablet_slice_param.slice_idx_ * ObTabletSliceParam::LOB_ID_SEQ_INTERVAL, // start ObTabletSliceParam::LOB_ID_SEQ_INTERVAL, // interval @@ -110,7 +107,6 @@ int ObDDLInsertRowIterator::init( ddl_agent_ = &agent; slice_row_iter_ = slice_row_iter; - ls_id_ = ls_id; current_tablet_id_ = tablet_id; context_id_ = context_id; lob_cols_cnt_ = lob_cols_cnt; @@ -148,7 +144,6 @@ int ObDDLInsertRowIterator::close_lob_sstable_slice() ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = true; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = current_tablet_id_; slice_info.slice_id_ = lob_slice_id_; slice_info.context_id_ = context_id_; @@ -189,7 +184,6 @@ int ObDDLInsertRowIterator::get_next_row( ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = true; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = current_tablet_id_; slice_info.slice_id_ = lob_slice_id_; slice_info.context_id_ = context_id_; @@ -225,7 +219,6 @@ int ObDDLInsertRowIterator::switch_to_new_lob_slice() uint64_t lob_id = 0; slice_info.is_full_direct_load_ = true; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = current_tablet_id_; slice_info.slice_id_ = lob_slice_id_; slice_info.context_id_ = context_id_; @@ -397,7 +390,6 @@ int64_t ObLobMetaRowIterator::get_seq_no() const ObTabletDDLParam::ObTabletDDLParam() : direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), - ls_id_(), start_scn_(SCN::min_scn()), commit_scn_(SCN::min_scn()), data_format_version_(0), @@ -758,14 +750,12 @@ int ObMacroBlockSliceStore::init( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(tablet_direct_load_mgr), K(data_seq)); } else { - const ObLSID &ls_id = tablet_direct_load_mgr->get_ls_id(); const ObITable::TableKey &table_key = tablet_direct_load_mgr->get_table_key(); // TODO(cangdi): fix it with right table key const int64_t ddl_task_id = tablet_direct_load_mgr->get_ddl_task_id(); const uint64_t data_format_version = tablet_direct_load_mgr->get_tenant_data_version(); const ObDirectLoadType direct_load_type = tablet_direct_load_mgr->get_direct_load_type(); const ObWholeDataStoreDesc &data_desc = tablet_direct_load_mgr->get_data_block_desc(); ObDDLRedoLogWriterCallbackInitParam init_param; - init_param.ls_id_ = ls_id; init_param.tablet_id_ = table_key.tablet_id_; init_param.direct_load_type_ = direct_load_type; init_param.table_key_ = table_key; @@ -800,8 +790,8 @@ int ObMacroBlockSliceStore::init( macro_seq_param.start_ = data_seq.macro_data_seq_; ObPreWarmerParam pre_warm_param; ObSSTablePrivateObjectCleaner *object_cleaner = nullptr; - if (OB_FAIL(pre_warm_param.init(ls_id, table_key.tablet_id_))) { - LOG_WARN("failed to init pre warm param", K(ret), K(ls_id), "tablet_id", table_key.tablet_id_); + if (OB_FAIL(pre_warm_param.init(table_key.tablet_id_))) { + LOG_WARN("failed to init pre warm param", K(ret), "tablet_id", table_key.tablet_id_); } else if (OB_FAIL(ObSSTablePrivateObjectCleaner::get_cleaner_from_data_store_desc( tablet_direct_load_mgr->get_data_block_desc().get_desc(), object_cleaner))) { @@ -858,7 +848,6 @@ int ObMacroBlockSliceStore::close() bool ObTabletDDLParam::is_valid() const { return is_valid_direct_load(direct_load_type_) - && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid_and_not_min() && commit_scn_.is_valid() && commit_scn_ != SCN::max_scn() @@ -1009,8 +998,8 @@ int ObDirectLoadSliceWriter::prepare_vector_slice_store( // is tablet_direct_load_mgr_ is null, no need to erase ivf_build_helper if (OB_NOT_NULL(tablet_direct_load_mgr_)) { ObIvfHelperKey key(tablet_direct_load_mgr_->get_tablet_id(), context_id); - if (OB_TMP_FAIL(ObPluginVectorIndexUtils::erase_ivf_build_helper(tablet_direct_load_mgr_->get_ls_id(), key))) { - LOG_WARN("failed to erase ivf build helper", K(tmp_ret), K(tablet_direct_load_mgr_->get_ls_id()), + if (OB_TMP_FAIL(ObPluginVectorIndexUtils::erase_ivf_build_helper(key))) { + LOG_WARN("failed to erase ivf build helper", K(tmp_ret), K(tablet_direct_load_mgr_->get_tablet_id())); } if (tmp_ret != OB_SUCCESS && tmp_ret != OB_HASH_NOT_EXIST) { @@ -1089,7 +1078,6 @@ int ObDirectLoadSliceWriter::prepare_iters( ObIAllocator &allocator, ObIAllocator &iter_allocator, blocksstable::ObStorageDatum &datum, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t trans_version, const ObObjType &obj_type, @@ -1143,11 +1131,11 @@ int ObDirectLoadSliceWriter::prepare_iters( if (is_incremental_direct_load(direct_load_type) && OB_ISNULL(tx_desc)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tx_desc should not be null if is incremental_direct_load", K(ret), K(direct_load_type), - K(ls_id), K(tablet_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); + K(tablet_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); } else if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, *lob_allocator_, tx_desc, pk_interval, ls_id, tablet_id/* tablet_id of main table */, tablet_direct_load_mgr_->get_tablet_id()/*tablet id of lob meta table*/, + allocator, *lob_allocator_, tx_desc, pk_interval, tablet_id/* tablet_id of main table */, tablet_direct_load_mgr_->get_tablet_id()/*tablet id of lob meta table*/, obj_type, cs_type, lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, *meta_write_iter_))) { - LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to insert_lob_col", K(ret), K(tablet_id)); } else if (OB_FAIL(row_iterator_->init(meta_write_iter_, trans_id, trans_version, seq_no, direct_load_type))) { LOG_WARN("fail to lob meta row iterator", K(ret), K(trans_id), K(trans_version), K(seq_no), K(direct_load_type)); @@ -1204,7 +1192,7 @@ int ObDirectLoadSliceWriter::fill_lob_into_memtable( int ret = OB_SUCCESS; const int64_t timeout_ts = ObTimeUtility::fast_current_time() + ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT; if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, info.ls_id_, info.data_tablet_id_, col_type.get_type(), col_type.get_collation_type(), + allocator, info.data_tablet_id_, col_type.get_type(), col_type.get_collation_type(), lob_storage_param, datum, timeout_ts, true/*has_lob_header*/))) { LOG_WARN("fail to insert_lob_col", K(ret), K(datum)); } @@ -1228,7 +1216,7 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( if (!datum.is_nop() && !datum.is_null()) { { ObLobMetaRowIterator *row_iter = nullptr; - if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, + if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.data_tablet_id_, info.trans_version_, col_type.get_type(), col_type.get_collation_type(), info.trans_id_, info.seq_no_, timeout_ts, lob_storage_param, info.direct_load_type_, info.tx_desc_, pk_interval, row_iter))) { @@ -2043,7 +2031,7 @@ int ObDirectLoadSliceWriter::inner_fill_hnsw_vector_index_data( if (OB_ISNULL(storage_schema) || snapshot_version < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(vec_idx_slice_store), KP(storage_schema), K(snapshot_version)); - } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(tablet_direct_load_mgr_->get_ls_id(), false/*is_for_read*/, timeout_us, tx_desc))) { + } else if (OB_FAIL(ObInsertLobColumnHelper::start_trans(false/*is_for_read*/, timeout_us, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); } else if (OB_FAIL(vec_idx_slice_store.serialize_vector_index(&allocator_, tx_desc, lob_inrow_threshold, index_type, snapshot_version))) { LOG_WARN("fail to do vector index snapshot data serialize", K(ret)); @@ -2140,8 +2128,8 @@ int ObDirectLoadSliceWriter::inner_fill_ivf_vector_index_data( { // what ever fail or success, we need to release ivf build helper int tmp_ret = OB_SUCCESS; ObIvfHelperKey key(vec_idx_slice_store.tablet_id_, vec_idx_slice_store.get_context_id()); - if (OB_TMP_FAIL(ObPluginVectorIndexUtils::erase_ivf_build_helper(tablet_direct_load_mgr_->get_ls_id(), key))) { - LOG_WARN("failed to erase ivf build helper", K(tmp_ret), K(tablet_direct_load_mgr_->get_ls_id()), K(vec_idx_slice_store.tablet_id_)); + if (OB_TMP_FAIL(ObPluginVectorIndexUtils::erase_ivf_build_helper(key))) { + LOG_WARN("failed to erase ivf build helper", K(tmp_ret), K(vec_idx_slice_store.tablet_id_)); } if (tmp_ret != OB_SUCCESS && tmp_ret != OB_HASH_NOT_EXIST) { ret = ret != OB_SUCCESS ? ret : tmp_ret; @@ -2228,24 +2216,20 @@ int ObVectorIndexSliceStore::init( } else { const ObIArray &col_desc_array = static_cast(tablet_direct_load_mgr)->get_sqc_build_ctx().data_block_desc_.get_desc().get_col_desc_array(); is_inited_ = true; - ctx_.ls_id_ = tablet_direct_load_mgr->get_ls_id(); tablet_id_ = tablet_direct_load_mgr->get_tablet_id(); vec_idx_param_ = vec_idx_param; vec_dim_ = vec_dim; // get data tablet id and lob tablet id - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle five_tablet_handle; ObTabletHandle data_tablet_handle; ObTabletBindingMdsUserData ddl_data; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ctx_.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ctx_.ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id_, five_tablet_handle))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet(tablet_id_, five_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(tablet_id_)); } else if (FALSE_IT(ctx_.data_tablet_id_ = five_tablet_handle.get_obj()->get_data_tablet_id())) { - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(ctx_.data_tablet_id_, data_tablet_handle))) { + } else if (OB_FAIL(ls->get_tablet(ctx_.data_tablet_id_, data_tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(ctx_.data_tablet_id_)); } else if (OB_FAIL(data_tablet_handle.get_obj()->get_ddl_data(ddl_data))) { LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet_handle)); @@ -2420,13 +2404,12 @@ int ObVectorIndexSliceStore::append_row(const blocksstable::ObDatumRow &datum_ro if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(ctx_.ls_id_, - tablet_id_, + } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(tablet_id_, ObIndexType::INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, adaptor_guard, &vec_idx_param_, vec_dim_))) { - LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(ctx_.ls_id_), K(tablet_id_)); + LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(tablet_id_)); } else { // get vid and vector ObString vec_str; @@ -2515,13 +2498,12 @@ int ObVectorIndexSliceStore::serialize_vector_index( if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(ctx_.ls_id_, - tablet_id_, + } else if (OB_FAIL(vec_index_service->acquire_adapter_guard(tablet_id_, ObIndexType::INDEX_TYPE_VEC_INDEX_SNAPSHOT_DATA_LOCAL, adaptor_guard, &vec_idx_param_, vec_dim_))) { - LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(ctx_.ls_id_), K(tablet_id_)); + LOG_WARN("fail to get ObMockPluginVectorIndexAdapter", K(ret), K(tablet_id_)); } else { ObHNSWSerializeCallback callback; ObOStreamBuf::Callback cb = callback; @@ -2538,7 +2520,7 @@ int ObVectorIndexSliceStore::serialize_vector_index( if (OB_ISNULL(tx_desc)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get tx desc, get nullptr", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ctx_.ls_id_, timeout, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else { param.timeout_ = timeout; @@ -2716,7 +2698,6 @@ int ObIvfCenterSliceStore::init( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(tablet_direct_load_mgr)); } else { - ObLSID ls_id = tablet_direct_load_mgr->get_ls_id(); tablet_id_ = tablet_direct_load_mgr->get_tablet_id(); vec_idx_param_ = vec_idx_param; vec_dim_ = vec_dim; @@ -2739,12 +2720,11 @@ int ObIvfCenterSliceStore::init( if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(ls_id, - key, + } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(key, ObIndexType::INDEX_TYPE_VEC_IVFFLAT_CENTROID_LOCAL, helper_guard_, vec_idx_param_))) { - LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(ls_id), K(tablet_id_)); + LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(tablet_id_)); } else if (OB_FAIL(get_spec_ivf_helper(helper))) { LOG_WARN("fail to get ivf flat helper", K(ret)); } else if (OB_FAIL(helper->init_ctx(vec_dim_))) { @@ -2954,7 +2934,6 @@ int ObIvfSq8MetaSliceStore::init( ret = OB_ERR_NULL_VALUE; LOG_WARN("invalid null tablet_direct_load_mgr", K(ret)); } else { - ObLSID ls_id = tablet_direct_load_mgr->get_ls_id(); tablet_id_ = tablet_direct_load_mgr->get_tablet_id(); vec_idx_param_ = vec_idx_param; vec_dim_ = vec_dim; @@ -2977,12 +2956,11 @@ int ObIvfSq8MetaSliceStore::init( if (OB_ISNULL(vec_index_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService ptr", K(ret)); - } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(ls_id, - key, + } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(key, ObIndexType::INDEX_TYPE_VEC_IVFSQ8_META_LOCAL, helper_guard_, vec_idx_param_))) { - LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(ls_id), K(tablet_id_)); + LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(tablet_id_)); } else if (OB_FAIL(get_spec_ivf_helper(helper))) { LOG_WARN("fail to get ivf flat helper", K(ret)); } else if (OB_FAIL(helper->init_ctx(vec_dim_))) { @@ -3152,7 +3130,6 @@ int ObIvfPqSliceStore::init( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(tablet_direct_load_mgr)); } else { - ObLSID ls_id = tablet_direct_load_mgr->get_ls_id(); tablet_id_ = tablet_direct_load_mgr->get_tablet_id(); table_id_ = static_cast(tablet_direct_load_mgr)->get_sqc_build_ctx().build_param_.runtime_only_param_.table_id_; vec_idx_param_ = vec_idx_param; @@ -3177,12 +3154,11 @@ int ObIvfPqSliceStore::init( if (OB_ISNULL(vec_index_service) || OB_ISNULL(GCTX.ddl_sql_proxy_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null ObPluginVectorIndexService or GCTX.ddl_sql_proxy_ ptr", K(ret), KP(vec_index_service)); - } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(ls_id, - key, + } else if (OB_FAIL(vec_index_service->acquire_ivf_build_helper_guard(key, ObIndexType::INDEX_TYPE_VEC_IVFPQ_PQ_CENTROID_LOCAL, helper_guard_, vec_idx_param_))) { - LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(ls_id), K(tablet_id_)); + LOG_WARN("failed to acquire ivf build helper guard", K(ret), K(tablet_id_)); } else if (OB_FAIL(get_spec_ivf_helper(helper))) { LOG_WARN("fail to get ivf flat helper", K(ret)); } else if (OB_FAIL(helper->init_ctx(vec_dim_))) { @@ -3365,7 +3341,6 @@ int ObDDLTableMergeDagParam::assign(const ObDDLTableMergeDagParam &merge_param) LOG_WARN("invalid argument", K(ret), K(merge_param)); } else { direct_load_type_ = merge_param.direct_load_type_; - ls_id_ = merge_param.ls_id_; tablet_id_ = merge_param.tablet_id_; rec_scn_ = merge_param.rec_scn_; is_commit_ = merge_param.is_commit_; @@ -3552,8 +3527,7 @@ int ObDDLTabletMergeDagParamV2::assign(const ObDDLTabletMergeDagParamV2 &merge_d return ret; } -int ObDDLTabletMergeDagParamV2::get_tablet_param(share::ObLSID &ls_id, - ObTabletID &tablet_id, +int ObDDLTabletMergeDagParamV2::get_tablet_param(ObTabletID &tablet_id, ObWriteTabletParam *&tablet_param) const { int ret = OB_SUCCESS; @@ -3569,12 +3543,10 @@ int ObDDLTabletMergeDagParamV2::get_tablet_param(share::ObLSID &ls_id, ret = OB_ERR_UNEXPECTED; LOG_WARN("lob tablet id not exist", K(ret), KPC(this)); } else { - ls_id = tablet_ctx_->ls_id_; tablet_id = tablet_ctx_->lob_meta_tablet_id_; tablet_param = &tablet_ctx_->lob_meta_tablet_param_; } } else { - ls_id = tablet_ctx_->ls_id_; tablet_id = tablet_ctx_->tablet_id_; tablet_param = &tablet_ctx_->tablet_param_; } @@ -3607,7 +3579,7 @@ int ObDDLTabletMergeDagParamV2::init_slice_sstable_array(hash::ObHashSet= 0 && context_id_ >= 0; } - TO_STRING_KV(K_(is_full_direct_load), K_(is_lob_slice), K_(ls_id), K_(data_tablet_id), K_(slice_id), K_(context_id), K_(is_task_finish), K_(total_slice_cnt), K_(slice_idx), K_(merge_slice_idx)); + bool is_valid() const { return data_tablet_id_.is_valid() && slice_id_ >= 0 && context_id_ >= 0; } + TO_STRING_KV(K_(is_full_direct_load), K_(is_lob_slice), K_(data_tablet_id), K_(slice_id), K_(context_id), K_(is_task_finish), K_(total_slice_cnt), K_(slice_idx), K_(merge_slice_idx)); public: bool is_full_direct_load_; bool is_lob_slice_; - share::ObLSID ls_id_; common::ObTabletID data_tablet_id_; int64_t slice_id_; int64_t context_id_; @@ -186,16 +183,15 @@ struct ObDirectInsertCommonParam final { public: ObDirectInsertCommonParam() - : ls_id_(), tablet_id_(), direct_load_type_(DIRECT_LOAD_INVALID), data_format_version_(0), read_snapshot_(0), is_no_logging_(false) + : tablet_id_(), direct_load_type_(DIRECT_LOAD_INVALID), data_format_version_(0), read_snapshot_(0), is_no_logging_(false) {} ~ObDirectInsertCommonParam() = default; - bool is_valid() const { return ls_id_.is_valid() && tablet_id_.is_valid() + bool is_valid() const { return tablet_id_.is_valid() && data_format_version_ >= 0 && read_snapshot_ >= 0 && DIRECT_LOAD_INVALID <= direct_load_type_ && direct_load_type_ <= DIRECT_LOAD_MAX; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(direct_load_type), K_(data_format_version), K_(read_snapshot), K_(is_no_logging)); + TO_STRING_KV(K_(tablet_id), K_(direct_load_type), K_(data_format_version), K_(read_snapshot), K_(is_no_logging)); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; ObDirectLoadType direct_load_type_; uint64_t data_format_version_; @@ -286,7 +282,6 @@ class ObDDLInsertRowIterator : public ObIDirectLoadRowIterator int init( ObDirectLoadMgrAgent &agent, ObIStoreRowIterator *slice_row_iter, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t context_id, const ObTabletSliceParam &tablet_slice_param, @@ -299,7 +294,7 @@ class ObDDLInsertRowIterator : public ObIDirectLoadRowIterator return get_next_row(is_skip_lob_, row); } int get_next_row(const bool skip_lob, const blocksstable::ObDatumRow *&row) override; - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(current_tablet_id), K_(context_id), K_(macro_seq), + TO_STRING_KV(K_(is_inited), K_(current_tablet_id), K_(context_id), K_(macro_seq), K_(lob_id_generator), K_(lob_id_cache), K_(lob_slice_id), K_(lob_cols_cnt), K_(is_skip_lob), K_(total_slice_cnt)); public: int switch_to_new_lob_slice(); @@ -316,7 +311,6 @@ class ObDDLInsertRowIterator : public ObIDirectLoadRowIterator // recover table ddl task needs it to scan rows. ObDirectLoadMgrAgent *ddl_agent_; ObIStoreRowIterator *slice_row_iter_; - share::ObLSID ls_id_; common::ObTabletID current_tablet_id_; // data_tablet_id rather than lob_meta_tablet_id. int64_t context_id_; blocksstable::ObMacroDataSeq macro_seq_; @@ -368,7 +362,6 @@ struct ObTabletDDLParam final ~ObTabletDDLParam(); bool is_valid() const; TO_STRING_KV(K_(direct_load_type), - K_(ls_id), K_(start_scn), K_(commit_scn), K_(data_format_version), @@ -378,7 +371,6 @@ struct ObTabletDDLParam final K_(seq_no)); public: ObDirectLoadType direct_load_type_; - share::ObLSID ls_id_; share::SCN start_scn_; share::SCN commit_scn_; uint64_t data_format_version_; @@ -393,7 +385,6 @@ struct ObDDLTableMergeDagParam : public share::ObIDagInitParam public: ObDDLTableMergeDagParam() : direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), - ls_id_(), tablet_id_(), rec_scn_(share::SCN::min_scn()), is_commit_(false), @@ -411,19 +402,18 @@ struct ObDDLTableMergeDagParam : public share::ObIDagInitParam { bool is_valid = data_format_version_ > 0 && snapshot_version_ > 0; if (is_full_direct_load(direct_load_type_)) { - is_valid = ls_id_.is_valid() && tablet_id_.is_valid() && start_scn_.is_valid_and_not_min(); + is_valid = tablet_id_.is_valid() && start_scn_.is_valid_and_not_min(); } else if (is_incremental_direct_load(direct_load_type_)) { - is_valid = ls_id_.is_valid() && tablet_id_.is_valid(); + is_valid = tablet_id_.is_valid(); } return is_valid; } int assign(const ObDDLTableMergeDagParam &merge_param); virtual ~ObDDLTableMergeDagParam() = default; - VIRTUAL_TO_STRING_KV(K_(direct_load_type), K_(ls_id), K_(tablet_id), K_(rec_scn), K_(is_commit), K_(start_scn), K_(data_format_version), + VIRTUAL_TO_STRING_KV(K_(direct_load_type), K_(tablet_id), K_(rec_scn), K_(is_commit), K_(start_scn), K_(data_format_version), K_(snapshot_version), K_(table_key), K_(user_data), K_(trans_id), K_(seq_no), K_(table_type)); public: ObDirectLoadType direct_load_type_; - share::ObLSID ls_id_; ObTabletID tablet_id_; share::SCN rec_scn_; bool is_commit_; @@ -461,7 +451,7 @@ struct ObDDLTabletMergeDagParamV2 int assign(const ObDDLTabletMergeDagParamV2 &merge_param); int init_slice_sstable_array(hash::ObHashSet &slice_idxes); int set_slice_sstable(const int64_t slice_idx, const ObTableHandleV2 &sstable_handle); - int get_tablet_param(share::ObLSID &ls_id, ObTabletID &tablet_id, ObWriteTabletParam *&tablet_param) const; + int get_tablet_param(ObTabletID &tablet_id, ObWriteTabletParam *&tablet_param) const; int get_merge_ctx(ObDDLTabletContext::MergeCtx *&merge_ctx); int get_storage_schema(ObStorageSchema *stroage_schema); void set_merge_all_slice() { merge_all_slice_ = true; } @@ -1042,7 +1032,6 @@ class ObDirectLoadSliceWriter final ObIAllocator &allocator, ObIAllocator &iter_allocator, blocksstable::ObStorageDatum &datum, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t trans_version, const ObObjType &obj_type, diff --git a/src/storage/ddl/ob_i_direct_load_mgr.cpp b/src/storage/ddl/ob_i_direct_load_mgr.cpp index 6eebd6b95..34af080eb 100644 --- a/src/storage/ddl/ob_i_direct_load_mgr.cpp +++ b/src/storage/ddl/ob_i_direct_load_mgr.cpp @@ -26,13 +26,12 @@ using namespace oceanbase::share::schema; using namespace oceanbase::sql; ObBaseTabletDirectLoadMgr::ObBaseTabletDirectLoadMgr() - : ls_id_(), tablet_id_(), table_key_(), tenant_data_version_(), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), ref_cnt_(0) + : tablet_id_(), table_key_(), tenant_data_version_(), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), ref_cnt_(0) { } ObBaseTabletDirectLoadMgr::~ObBaseTabletDirectLoadMgr() { - ls_id_.reset(); tablet_id_.reset(); table_key_.reset(); tenant_data_version_ = 0; diff --git a/src/storage/ddl/ob_i_direct_load_mgr.h b/src/storage/ddl/ob_i_direct_load_mgr.h index e2807b800..bcf1e186f 100644 --- a/src/storage/ddl/ob_i_direct_load_mgr.h +++ b/src/storage/ddl/ob_i_direct_load_mgr.h @@ -74,7 +74,7 @@ class ObBaseTabletDirectLoadMgr virtual ~ObBaseTabletDirectLoadMgr(); virtual bool is_valid() = 0; TO_STRING_KV(K_(table_key), K_(tenant_data_version), K_(direct_load_type), - K_(ls_id), K_(tablet_id)); + K_(tablet_id)); public: /* some baisc method */ void inc_ref() { ATOMIC_INC(&ref_cnt_); }; int64_t dec_ref() { return ATOMIC_SAF(&ref_cnt_, 1); } @@ -181,7 +181,6 @@ class ObBaseTabletDirectLoadMgr inline ObITable::TableKey get_table_key() const { return table_key_; } inline uint64_t get_tenant_data_version() const { return tenant_data_version_; } inline ObDirectLoadType get_direct_load_type() const { return direct_load_type_; } - inline share::ObLSID get_ls_id() const { return ls_id_;} inline ObTabletID get_tablet_id() const { return tablet_id_; } /* some getter method for compat * which should be remove @@ -198,7 +197,6 @@ class ObBaseTabletDirectLoadMgr protected: /* basic info */ - share::ObLSID ls_id_; ObTabletID tablet_id_; ObITable::TableKey table_key_; uint64_t tenant_data_version_; diff --git a/src/storage/ddl/ob_inc_ddl_merge_helper.cpp b/src/storage/ddl/ob_inc_ddl_merge_helper.cpp index 918228154..65b74dcdc 100644 --- a/src/storage/ddl/ob_inc_ddl_merge_helper.cpp +++ b/src/storage/ddl/ob_inc_ddl_merge_helper.cpp @@ -51,7 +51,6 @@ int ObIncMinDDLMergeHelper::process_prepare_task(ObIDag *dag, int ret = OB_SUCCESS; slice_ranges.reset(); - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; ObDDLTabletContext::MergeCtx *merge_ctx = nullptr; @@ -67,15 +66,15 @@ int ObIncMinDDLMergeHelper::process_prepare_task(ObIDag *dag, if (!dag_merge_param.is_valid() || nullptr == dag) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(dag_merge_param), KPC(dag)); - } else if (OB_FAIL(dag_merge_param.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(dag_merge_param.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret), K(dag_merge_param)); } else if (OB_FAIL(dag_merge_param.get_merge_ctx(merge_ctx))) { LOG_WARN("failed to get merge ctx", K(ret), K(dag_merge_param)); } else if (nullptr == tablet_param || nullptr == merge_ctx) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet param & merge ctx should not be null", K(ret)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id, tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet handle", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id, tablet_handle))) { + LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id)); } else if (!tablet_handle.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tablet handle should not be invalid", K(ret)); @@ -146,7 +145,6 @@ int ObIncMinDDLMergeHelper::merge_slice(ObIDag *dag, { int ret = OB_SUCCESS; - ObLSID ls_id; ObTabletID tablet_id; ObTabletHandle tablet_handle; ObWriteTabletParam *tablet_param = nullptr; @@ -168,14 +166,14 @@ int ObIncMinDDLMergeHelper::merge_slice(ObIDag *dag, if (nullptr == dag || !dag_merge_param.is_valid() || start_slice_idx < 0 || start_slice_idx > end_slice_idx) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(dag), K(dag_merge_param), K(start_slice_idx), K(end_slice_idx)); - } else if (OB_FAIL(dag_merge_param.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(dag_merge_param.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get tablet param", K(ret), K(dag_merge_param)); } else if (OB_FAIL(dag_merge_param.get_merge_ctx(merge_ctx))) { LOG_WARN("failed to get merge ctx", K(ret)); } else if (nullptr == merge_ctx || nullptr == tablet_param) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet param should not be null", K(ret), K(dag_merge_param)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id, tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet handle", K(ret)); } else if (OB_ISNULL(ddl_kv = merge_ctx->ddl_kv_handles_.at(0).get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -186,7 +184,6 @@ int ObIncMinDDLMergeHelper::merge_slice(ObIDag *dag, if (OB_FAIL(ret)) { } else { tablet_ddl_param.direct_load_type_ = dag_merge_param.direct_load_type_; - tablet_ddl_param.ls_id_ = ls_id; tablet_ddl_param.start_scn_ = ddl_kv->get_ddl_start_scn(); tablet_ddl_param.commit_scn_ = ddl_kv->get_ddl_start_scn(); tablet_ddl_param.data_format_version_ = dag_merge_param.ddl_task_param_.tenant_data_version_; @@ -250,13 +247,12 @@ int ObIncMinDDLMergeHelper::assemble_sstable(ObDDLTabletMergeDagParamV2 &dag_mer { int ret = OB_SUCCESS; - ObLSID ls_id; ObTabletID tablet_id; ObWriteTabletParam *tablet_param = nullptr; ObDDLTabletContext::MergeCtx *merge_ctx = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObDDLKV *ddl_kv = nullptr; ObTabletHandle tablet_handle; @@ -267,17 +263,17 @@ int ObIncMinDDLMergeHelper::assemble_sstable(ObDDLTabletMergeDagParamV2 &dag_mer if (!dag_merge_param.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(dag_merge_param)); - } else if (OB_FAIL(dag_merge_param.get_tablet_param(ls_id, tablet_id, tablet_param))) { + } else if (OB_FAIL(dag_merge_param.get_tablet_param(tablet_id, tablet_param))) { LOG_WARN("failed to get tablet ctx", K(ret), K(dag_merge_param)); - } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(ls_id, tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet handle", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ObDirectLoadMgrUtil::get_tablet_handle(tablet_id, tablet_handle))) { + LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tablet handle is invalid", K(ret)); } else if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(ls))) { LOG_WARN("failed to get ls", K(ret), K(dag_merge_param)); } else if (OB_FAIL(dag_merge_param.get_merge_ctx(merge_ctx))) { LOG_WARN("failed to get merge ctx", K(ret)); @@ -302,7 +298,6 @@ int ObIncMinDDLMergeHelper::assemble_sstable(ObDDLTabletMergeDagParamV2 &dag_mer ObUpdateTableStoreParam table_store_param(max(ddl_kv->get_snapshot_version(), tablet_handle.get_obj()->get_snapshot_version()), tablet_handle.get_obj()->get_multi_version_start(), tablet_param->storage_schema_, - ls_handle.get_ls()->get_rebuild_seq(), sstable); if (OB_FAIL(table_store_param.init_with_compaction_info(ObCompactionTableStoreParam(compaction::MINI_MERGE, share::SCN::min_scn(), @@ -312,7 +307,7 @@ int ObIncMinDDLMergeHelper::assemble_sstable(ObDDLTabletMergeDagParamV2 &dag_mer } else { table_store_param.compaction_info_.clog_checkpoint_scn_ = sstable->get_end_scn(); ObTabletHandle new_tablet_handle; - if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id, table_store_param, new_tablet_handle))) { + if (OB_FAIL(ls->update_tablet_table_store(tablet_id, table_store_param, new_tablet_handle))) { LOG_WARN("failed to update tablet table store", K(ret), K(dag_merge_param), K(table_store_param)); } else { FLOG_INFO("ddl update table store success", KPC(new_tablet_handle.get_obj()), K(table_store_param)); @@ -324,7 +319,7 @@ int ObIncMinDDLMergeHelper::assemble_sstable(ObDDLTabletMergeDagParamV2 &dag_mer if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; ObTabletHandle new_tablet_handle; - if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + if (OB_TMP_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, new_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { diff --git a/src/storage/ddl/ob_lob_macro_block_writer.cpp b/src/storage/ddl/ob_lob_macro_block_writer.cpp index 819ec98bc..29d293fc7 100644 --- a/src/storage/ddl/ob_lob_macro_block_writer.cpp +++ b/src/storage/ddl/ob_lob_macro_block_writer.cpp @@ -72,7 +72,6 @@ int ObLobMacroBlockWriter::init(const ObWriteMacroParam ¶m, lob_column_count_ = param.ddl_table_schema_.lob_column_idxs_.count(); lob_id_cache_.tablet_id_ = lob_meta_tablet_id_; lob_inrow_threshold_ = param.ddl_table_schema_.table_item_.lob_inrow_threshold_; - ls_id_ = param.ls_id_; tablet_id_ = param.tablet_id_; param_ = param; @@ -132,7 +131,6 @@ int ObLobMacroBlockWriter::write(const ObColumnSchemaItem &column_schema, ObIAll lob_arena_/*lob_allocator*/, param_.tx_info_.tx_desc_, lob_id_cache_, - ls_id_, tablet_id_, lob_meta_tablet_id_, column_schema.col_type_.get_type(), @@ -296,7 +294,7 @@ int ObLobMacroBlockWriter::close() uint64_t last_lob_id = 0; if (OB_FAIL(lob_id_cache_.get_value(last_lob_id))) { LOG_WARN("get last lob id failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(ls_id_, lob_meta_tablet_id_, last_lob_id))) { + } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(lob_meta_tablet_id_, last_lob_id))) { LOG_WARN("update max lob id failed", K(ret), K(last_lob_id)); } } diff --git a/src/storage/ddl/ob_lob_macro_block_writer.h b/src/storage/ddl/ob_lob_macro_block_writer.h index 389e8ba32..6a82b0826 100644 --- a/src/storage/ddl/ob_lob_macro_block_writer.h +++ b/src/storage/ddl/ob_lob_macro_block_writer.h @@ -60,7 +60,6 @@ class ObLobMacroBlockWriter private: bool is_inited_; - ObLSID ls_id_; ObTabletID tablet_id_; ObTabletID lob_meta_tablet_id_; diff --git a/src/storage/ddl/ob_table_fork_info.cpp b/src/storage/ddl/ob_table_fork_info.cpp index d73c80709..d0e444d84 100644 --- a/src/storage/ddl/ob_table_fork_info.cpp +++ b/src/storage/ddl/ob_table_fork_info.cpp @@ -29,8 +29,7 @@ namespace storage { ObTableForkInfo::ObTableForkInfo() - : ls_id_(), - table_id_(OB_INVALID_ID), + : table_id_(OB_INVALID_ID), schema_version_(0), task_id_(0), source_tablet_ids_(), dest_tablet_ids_(), fork_snapshot_version_(0), @@ -40,7 +39,6 @@ ObTableForkInfo::ObTableForkInfo() } ObTableForkInfo::ObTableForkInfo( - const share::ObLSID &ls_id, const uint64_t table_id, const int64_t schema_version, const int64_t task_id, @@ -49,8 +47,7 @@ ObTableForkInfo::ObTableForkInfo( const int64_t data_format_version, const common::ObIArray &source_tablet_ids, const common::ObIArray &dest_tablet_ids) - : ls_id_(ls_id), - table_id_(table_id), + : table_id_(table_id), schema_version_(schema_version), task_id_(task_id), source_tablet_ids_(), @@ -74,8 +71,6 @@ int ObTableForkInfo::assign(const ObTableForkInfo &info) } else if (OB_FAIL(dest_tablet_ids_.assign(info.dest_tablet_ids_))) { LOG_WARN("failed to assign dest tablet ids", K(ret)); } else { - - ls_id_ = info.ls_id_; table_id_ = info.table_id_; schema_version_ = info.schema_version_; task_id_ = info.task_id_; @@ -88,8 +83,7 @@ int ObTableForkInfo::assign(const ObTableForkInfo &info) bool ObTableForkInfo::is_valid() const { - return ls_id_.is_valid() - && OB_INVALID_ID != table_id_ + return OB_INVALID_ID != table_id_ && schema_version_ > 0 && task_id_ > 0 && source_tablet_ids_.count() > 0 @@ -113,7 +107,6 @@ int ObTableForkInfo::generate_fork_params(common::ObIArray &p for (int64_t i = 0; OB_SUCC(ret) && i < source_tablet_ids_.count(); ++i) { ObTabletForkParam fork_param; - fork_param.ls_id_ = ls_id_; fork_param.table_id_ = table_id_; fork_param.schema_version_ = schema_version_; fork_param.task_id_ = task_id_; @@ -152,11 +145,10 @@ int ObTableForkInfo::get_tablet_fork_param( } if (found_idx < 0) { ret = OB_ENTRY_NOT_EXIST; - LOG_INFO("tablet not in fork info", K(ret), K(tablet_id), K_(task_id), K_(table_id), K_(ls_id)); + LOG_INFO("tablet not in fork info", K(ret), K(tablet_id), K_(task_id), K_(table_id)); } else { tablet_fork_param.reset(); - tablet_fork_param.ls_id_ = ls_id_; tablet_fork_param.table_id_ = table_id_; tablet_fork_param.schema_version_ = schema_version_; tablet_fork_param.task_id_ = task_id_; @@ -174,7 +166,7 @@ int ObTableForkInfo::get_tablet_fork_param( OB_DEF_SERIALIZE(ObTableForkInfo) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, table_id_, schema_version_, task_id_, + LST_DO_CODE(OB_UNIS_ENCODE, table_id_, schema_version_, task_id_, source_tablet_ids_, dest_tablet_ids_, fork_snapshot_version_, compat_mode_, data_format_version_); return ret; @@ -183,7 +175,7 @@ OB_DEF_SERIALIZE(ObTableForkInfo) OB_DEF_DESERIALIZE(ObTableForkInfo) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, table_id_, schema_version_, task_id_, + LST_DO_CODE(OB_UNIS_DECODE, table_id_, schema_version_, task_id_, source_tablet_ids_, dest_tablet_ids_, fork_snapshot_version_, compat_mode_, data_format_version_); return ret; @@ -192,7 +184,7 @@ OB_DEF_DESERIALIZE(ObTableForkInfo) OB_DEF_SERIALIZE_SIZE(ObTableForkInfo) { int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, table_id_, schema_version_, task_id_, + LST_DO_CODE(OB_UNIS_ADD_LEN, table_id_, schema_version_, task_id_, source_tablet_ids_, dest_tablet_ids_, fork_snapshot_version_, compat_mode_, data_format_version_); return len; diff --git a/src/storage/ddl/ob_table_fork_info.h b/src/storage/ddl/ob_table_fork_info.h index 7e5c71976..d329b00a4 100644 --- a/src/storage/ddl/ob_table_fork_info.h +++ b/src/storage/ddl/ob_table_fork_info.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_STORAGE_DDL_OB_TABLE_FORK_INFO_H_ #define OCEANBASE_STORAGE_DDL_OB_TABLE_FORK_INFO_H_ -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "lib/container/ob_se_array.h" #include "lib/utility/ob_unify_serialize.h" @@ -36,7 +35,6 @@ class ObTableForkInfo final public: ObTableForkInfo(); ObTableForkInfo( - const share::ObLSID &ls_id, const uint64_t table_id, const int64_t schema_version, const int64_t task_id, @@ -50,13 +48,11 @@ class ObTableForkInfo final bool is_valid() const; int generate_fork_params(common::ObIArray ¶ms) const; int get_tablet_fork_param(const common::ObTabletID &tablet_id, ObTabletForkParam &tablet_fork_param) const; - TO_STRING_KV(K_(ls_id), K_(table_id), K_(schema_version), K_(task_id), + TO_STRING_KV(K_(table_id), K_(schema_version), K_(task_id), K_(source_tablet_ids), K_(dest_tablet_ids), K_(fork_snapshot_version), K_(compat_mode), K_(data_format_version)); public: - - share::ObLSID ls_id_; uint64_t table_id_; int64_t schema_version_; int64_t task_id_; diff --git a/src/storage/ddl/ob_tablet_ddl_kv.cpp b/src/storage/ddl/ob_tablet_ddl_kv.cpp index cfc258a71..7f0bf0b35 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv.cpp +++ b/src/storage/ddl/ob_tablet_ddl_kv.cpp @@ -779,8 +779,7 @@ ObDDLKV::~ObDDLKV() reset(); } -int ObDDLKV::init(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObDDLKV::init(const ObTabletID &tablet_id, const SCN &ddl_start_scn, const int64_t snapshot_version, const SCN &last_freezed_scn, @@ -795,18 +794,17 @@ int ObDDLKV::init(const ObLSID &ls_id, if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), KP(this)); - } else if (OB_UNLIKELY(!ls_id.is_valid() - || !tablet_id.is_valid() + } else if (OB_UNLIKELY(!tablet_id.is_valid() || !ddl_start_scn.is_valid_and_not_min() || snapshot_version <= 0 || !last_freezed_scn.is_valid_and_not_min() || data_format_version <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(ddl_start_scn), K(snapshot_version), K(last_freezed_scn), K(data_format_version)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(ddl_start_scn), K(snapshot_version), K(last_freezed_scn), K(data_format_version)); } else if ((max_end_scn_ != SCN::min_scn() && last_freezed_scn >= max_end_scn_) || last_freezed_scn >= rec_scn_) { ret = OB_SCN_OUT_OF_BOUND; - TRANS_LOG(ERROR, "cannot set start ts now", K(ret), K(ls_id), KPC(this)); + TRANS_LOG(ERROR, "cannot set start ts now", K(ret), KPC(this)); } else if (OB_UNLIKELY(!(storage::is_full_ddl_kv(ddl_kv_type)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("only support full and inc_major ddl kv", KR(ret), K(ddl_kv_type)); @@ -818,7 +816,6 @@ int ObDDLKV::init(const ObLSID &ls_id, } if (OB_SUCC(ret)) { - ls_id_ = ls_id; tablet_id_ = tablet_id; ddl_start_scn_ = ddl_start_scn; ddl_snapshot_version_ = snapshot_version; @@ -1035,7 +1032,6 @@ int ObDDLKV::set_macro_block( param.direct_load_type_ = data_format_version >= DDL_IDEM_DATA_FORMAT_VERSION ? ObDirectLoadType::SN_IDEM_DIRECT_LOAD_DDL : ObDirectLoadType::DIRECT_LOAD_DDL; } - param.ls_id_ = ls_id_; param.tablet_id_ = tablet_id_; param.start_scn_ = ddl_start_scn_; param.data_format_version_ = data_format_version; @@ -1070,15 +1066,15 @@ int ObDDLKV::set_macro_block( } else if (macro_block.ddl_start_scn_ > ddl_start_scn_) { ret = OB_EAGAIN; LOG_INFO("ddl start scn too large, retry", K(ret), - K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block)); + K(tablet_id_), K(ddl_start_scn_), K(macro_block)); } else { // filter out and do nothing LOG_INFO("ddl start scn too small, maybe from old build task, ignore", K(ret), - K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block)); + K(tablet_id_), K(ddl_start_scn_), K(macro_block)); } } else if (macro_block.scn_ > freeze_scn_) { ret = OB_EAGAIN; - LOG_INFO("this ddl kv is freezed, retry other ddl kv", K(ret), K(ls_id_), K(tablet_id_), K(macro_block), K(freeze_scn_)); + LOG_INFO("this ddl kv is freezed, retry other ddl kv", K(ret), K(tablet_id_), K(macro_block), K(freeze_scn_)); } else if (OB_UNLIKELY(snapshot_version != ddl_snapshot_version_ || data_format_version != data_format_version_ || macro_block.trans_id_ != trans_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret), K(macro_block), K(snapshot_version), K(data_format_version), KPC(this)); @@ -1194,14 +1190,14 @@ int ObDDLKV::full_load_freeze_(const SCN &freeze_scn) final_freeze_scn = max_scn_; } else { ret = OB_EAGAIN; - LOG_INFO("ddl kv not freezed, try again", K(ret), K(ls_id_), K(tablet_id_), K(get_macro_block_cnt())); + LOG_INFO("ddl kv not freezed, try again", K(ret), K(tablet_id_), K(get_macro_block_cnt())); } if (OB_SUCC(ret)) { if (OB_FAIL(set_end_scn(final_freeze_scn))) { LOG_WARN("fail to set end scn", K(ret), K(final_freeze_scn)); } else { ATOMIC_SET(&is_independent_freezed_, true); - LOG_INFO("ddl kv freezed", K(ret), K(ls_id_), K(tablet_id_), K(get_macro_block_cnt()), K(final_freeze_scn), K(freeze_scn)); + LOG_INFO("ddl kv freezed", K(ret), K(tablet_id_), K(get_macro_block_cnt()), K(final_freeze_scn), K(freeze_scn)); } } } @@ -1296,19 +1292,19 @@ int ObDDLKV::wait_pending() { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *ls = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else if (OB_UNLIKELY(!is_freezed())) { ret = OB_STATE_NOT_MATCH; LOG_WARN("ddl kv not freezed", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls handle failed", K(ret), K(ls_id_)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); } else { SCN max_decided_scn; - if (OB_FAIL(ls_handle.get_ls()->get_max_decided_scn(max_decided_scn))) { - LOG_WARN("get max decided log ts failed", K(ret), K(ls_id_)); + if (OB_FAIL(ls->get_max_decided_scn(max_decided_scn))) { + LOG_WARN("get max decided log ts failed", K(ret)); if (OB_STATE_NOT_MATCH == ret) { ret = OB_NEED_RETRY; } @@ -1318,7 +1314,7 @@ int ObDDLKV::wait_pending() const bool pending_finished = SCN::plus(max_decided_scn, 1) >= freeze_scn_ && !is_pending(); if (!pending_finished) { ret = OB_EAGAIN; - LOG_INFO("wait pending not finish", K(ret), K_(ls_id), K_(tablet_id), K_(freeze_scn), K_(min_scn), K_(max_scn), K(max_decided_scn)); + LOG_INFO("wait pending not finish", K(ret), K_(tablet_id), K_(freeze_scn), K_(min_scn), K_(max_scn), K(max_decided_scn)); } } } @@ -1339,7 +1335,7 @@ int64_t ObDDLKV::get_memory_used() const /**************** Implement ObITabletMemtable Function *****************/ int ObDDLKV::init(const ObITable::TableKey &table_key, - ObLSHandle &ls_handle, + ObLS *ls, ObFreezer *freezer, ObTabletMemtableMgr *memtable_mgr, const int64_t schema_version, @@ -1352,7 +1348,7 @@ int ObDDLKV::init(const ObITable::TableKey &table_key, TRANS_LOG(WARN, "init twice", K(*this)); ret = OB_INIT_TWICE; } else if (!table_key.is_valid() || OB_ISNULL(freezer) || OB_ISNULL(memtable_mgr) || schema_version < 0 || - OB_UNLIKELY(!ls_handle.is_valid())) { + OB_UNLIKELY(OB_ISNULL(ls))) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid param", @@ -1362,7 +1358,7 @@ int ObDDLKV::init(const ObITable::TableKey &table_key, KP(memtable_mgr), K(schema_version), K(freeze_clock), - K(ls_handle)); + K(ls)); } else if (OB_FAIL(set_memtable_mgr_(memtable_mgr))) { TRANS_LOG(WARN, "fail to set memtable mgr", K(ret), KP(memtable_mgr)); } else if (FALSE_IT(set_freeze_clock(freeze_clock))) { @@ -1374,7 +1370,6 @@ int ObDDLKV::init(const ObITable::TableKey &table_key, } else if (OB_FAIL(ObITable::init(table_key))) { TRANS_LOG(WARN, "failed to set_table_key", K(ret), K(table_key)); } else { - ls_id_ = freezer_->get_ls_id(); init_timestamp_ = ObClockGenerator::getClock(); (void)set_freeze_state(TabletMemtableFreezeState::ACTIVE); ddl_kv_type_ = ObDDLKVType::DDL_KV_INC_MINOR; @@ -1403,7 +1398,6 @@ bool ObDDLKV::ready_for_flush() { bool ObDDLKV::ready_for_flush_() { int ret = OB_SUCCESS; bool ready_for_flush = false; - const ObLSID ls_id = get_ls_id(); // STEP 1 : freeze ddl kv if needed if (ObITabletMemtable::get_end_scn().is_max()) { @@ -1428,7 +1422,6 @@ bool ObDDLKV::ready_for_flush_() { (void)set_last_print_time(cur_time); STORAGE_LOG(WARN, "direct load memtable not ready for flush for long time", - K(ls_id), K(get_frozen_time()), K(max_decided_scn), KPC(this)); @@ -1446,9 +1439,9 @@ bool ObDDLKV::data_has_completed_(SCN &max_decided_scn) if (OB_FAIL(ret)) { } else if (end_scn.is_max()) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("end_scn should not be max scn", K(ret), K(get_ls_id()), KPC(this)); + LOG_ERROR("end_scn should not be max scn", K(ret), KPC(this)); } else if (OB_FAIL(get_ls_current_right_boundary_(max_decided_scn))) { - LOG_WARN("get max decided scn failed", K(ret), K(get_ls_id())); + LOG_WARN("get max decided scn failed", K(ret)); } else if (max_decided_scn >= end_scn) { set_freeze_state(TabletMemtableFreezeState::READY_FOR_FLUSH); data_has_completed = true; @@ -1460,11 +1453,10 @@ bool ObDDLKV::data_has_completed_(SCN &max_decided_scn) int ObDDLKV::decide_right_boundary() { int ret = OB_SUCCESS; - const ObLSID ls_id = get_ls_id(); ObTabletMemtableMgr *mgr = get_memtable_mgr(); if (OB_ISNULL(mgr)) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet memtable mgr of ddl kv is unexpected null", KR(ret), K(ls_id), KPC(this)); + LOG_ERROR("tablet memtable mgr of ddl kv is unexpected null", KR(ret), KPC(this)); } else if (OB_FAIL(mgr->freeze_direct_load_memtable(this))) { LOG_WARN("fail to freeze direct load memtable", K(ret)); } else if (OB_ISNULL(freezer_)) { @@ -1474,7 +1466,7 @@ int ObDDLKV::decide_right_boundary() SCN snapshot_version = freezer_->get_freeze_snapshot_version(); if (snapshot_version.is_max() || snapshot_version.is_min() || !snapshot_version.is_valid()) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("invalid snapshot version", KR(ret), K(ls_id), K(snapshot_version), KP(freezer_)); + LOG_ERROR("invalid snapshot version", KR(ret), K(snapshot_version), KP(freezer_)); } else { set_snapshot_version(snapshot_version); } @@ -1485,7 +1477,6 @@ int ObDDLKV::decide_right_boundary() bool ObDDLKV::rec_scn_is_stable() { int ret = OB_SUCCESS; - ObLSID ls_id = get_ls_id(); bool rec_scn_is_stable = false; SCN max_decided_scn; const SCN rec_scn = ObITabletMemtable::get_rec_scn(); @@ -1493,7 +1484,7 @@ bool ObDDLKV::rec_scn_is_stable() // ddl kv do not have data yet rec_scn_is_stable = false; } else if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_decided_scn))) { - STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret)); } else if (max_decided_scn >= rec_scn) { rec_scn_is_stable = true; } @@ -1503,7 +1494,6 @@ bool ObDDLKV::rec_scn_is_stable() (get_frozen_time() != 0 && ObClockGenerator::getClock() - get_frozen_time() > WARN_LOG_INTERVAL)) { STORAGE_LOG(WARN, "direct load memtable rec_scn not stable for long time", - K(ls_id), KPC(this), K(mt_stat_.frozen_time_), K(max_decided_scn)); @@ -1533,7 +1523,7 @@ bool ObDDLKV::is_frozen_memtable() return bool_ret; } -int ObDDLKV::flush(share::ObLSID ls_id) +int ObDDLKV::flush() { int ret = OB_SUCCESS; @@ -1542,7 +1532,6 @@ int ObDDLKV::flush(share::ObLSID ls_id) ret = OB_NO_NEED_UPDATE; } else { ObDDLTableMergeDagParam param; - param.ls_id_ = ls_id; param.tablet_id_ = key_.tablet_id_; param.direct_load_type_ = DIRECT_LOAD_INCREMENTAL; param.start_scn_ = ddl_start_scn_; @@ -1562,7 +1551,6 @@ void ObDDLKV::print_ready_for_flush() { int ret = OB_SUCCESS; bool ready_for_flush = false; - const ObLSID ls_id = get_ls_id(); const common::ObTabletID tablet_id = key_.tablet_id_; bool frozen_memtable_flag = is_frozen_memtable(); int64_t write_ref = get_write_ref(); @@ -1571,13 +1559,13 @@ void ObDDLKV::print_ready_for_flush() SCN max_decided_scn = SCN::min_scn();; const SCN end_scn = ObITabletMemtable::get_end_scn(); if (OB_FAIL(get_ls_current_right_boundary_(max_decided_scn))) { - LOG_WARN("get max decided scn failed", K(ret), K(ls_id)); + LOG_WARN("get max decided scn failed", K(ret)); } else if (max_decided_scn >= end_scn) { ready_for_flush = true; } TRANS_LOG(INFO, "[ObFreezer] print_ready_for_flush", - KP(this), K(ls_id), K(tablet_id), + KP(this), K(tablet_id), K(ret), K(ready_for_flush), K(frozen_memtable_flag), K(write_ref), K(max_decided_scn), K(end_scn)); diff --git a/src/storage/ddl/ob_tablet_ddl_kv.h b/src/storage/ddl/ob_tablet_ddl_kv.h index 42b29ed9b..8bde8ab02 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv.h +++ b/src/storage/ddl/ob_tablet_ddl_kv.h @@ -46,6 +46,8 @@ class ObDataMacroBlockMeta; namespace storage { +class ObLS; + class ObBlockMetaTreeValue final { public: @@ -211,8 +213,7 @@ class ObDDLKV : public ObITabletMemtable ObDDLKV(); ~ObDDLKV(); // full direct load. - int init(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + int init(const common::ObTabletID &tablet_id, const share::SCN &ddl_start_scn, const int64_t snapshot_version, const share::SCN &last_freezed_scn, @@ -229,7 +230,7 @@ class ObDDLKV : public ObITabletMemtable public : // derived from ObITabletMemtable virtual bool is_inited() const override { return is_inited_; } virtual int init(const ObITable::TableKey &table_key, - ObLSHandle &ls_handle, + ObLS *ls, ObFreezer *freezer, ObTabletMemtableMgr *memtable_mgr, const int64_t schema_version, @@ -286,7 +287,7 @@ public : // derived from ObITabletMemtable // TODO : @jianyun.sjy ObDDLMemtable adapts check_rows_locked public: // derived from ObFreezeCheckpoint - virtual int flush(share::ObLSID ls_id) override; + virtual int flush() override; virtual bool ready_for_flush() override; virtual bool rec_scn_is_stable() override; virtual void set_allow_freeze(const bool allow_freeze) override; @@ -345,7 +346,6 @@ public : // derived from ObITabletMemtable K_(is_inited), K_(is_closed), K_(is_independent_freezed), - K_(ls_id), K_(tablet_id), K_(ddl_start_scn), K_(snapshot_version), diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp index 7ac249b7d..1da12b53d 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp @@ -43,7 +43,7 @@ bool ObDDLKVQueryParam::match_ddl_kv(const ObDDLKV &ddl_kv) const ObTabletDDLKvMgr::ObTabletDDLKvMgr() : is_inited_(false), - ls_id_(), tablet_id_(), + tablet_id_(), max_freeze_scn_(SCN::min_scn()), head_(0), tail_(0), lock_(), idem_checker_(), ref_cnt_(0) { @@ -67,31 +67,30 @@ void ObTabletDDLKvMgr::destroy() for (int64_t i = 0; i < MAX_DDL_KV_CNT_IN_STORAGE; ++i) { ddl_kv_handles_[i].reset(); } - ls_id_.reset(); tablet_id_.reset(); max_freeze_scn_.set_min(); idem_checker_.destroy(); is_inited_ = false; } -int ObTabletDDLKvMgr::init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) +int ObTabletDDLKvMgr::init(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ObTabletDDLKvMgr is already inited", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else if (OB_ISNULL(ls_service)) { ret = OB_ERR_SYS; LOG_WARN("ls service should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(tablet_id))) { - LOG_WARN("failed to add tablet", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(ls_service->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(ls->get_ddl_log_handler()->add_tablet(tablet_id))) { + LOG_WARN("failed to add tablet", K(ret), K(tablet_id)); } if (OB_FAIL(ret)) { @@ -100,7 +99,6 @@ int ObTabletDDLKvMgr::init(const share::ObLSID &ls_id, const common::ObTabletID if (OB_FAIL(add_idempotence_checker_nolock())) { LOG_WARN("failed to init idem checker", K(ret)); } else { - ls_id_ = ls_id; tablet_id_ = tablet_id; is_inited_ = true; } @@ -125,7 +123,7 @@ int ObTabletDDLKvMgr::set_max_freeze_scn(const share::SCN &checkpoint_scn) int ObTabletDDLKvMgr::get_rec_scn(SCN &rec_scn) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletFullDirectLoadMgr *tablet_mgr = nullptr; ObTabletDirectLoadMgrHandle direct_load_mgr_hdl; @@ -138,7 +136,6 @@ int ObTabletDDLKvMgr::get_rec_scn(SCN &rec_scn) ret = OB_ERR_SYS; LOG_WARN("error sys", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_id_, tablet_id_, true/* is_full_direct_load */, direct_load_mgr_hdl, @@ -152,16 +149,16 @@ int ObTabletDDLKvMgr::get_rec_scn(SCN &rec_scn) } } else if (OB_ISNULL(tablet_mgr = direct_load_mgr_hdl.get_full_obj())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected err", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("unexpected err", K(ret), K(tablet_id_)); } if (OB_SUCC(ret) && nullptr != tablet_mgr) { - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id_, + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ls->get_tablet(tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id_)); } // rec scn of ddl start log @@ -505,7 +502,7 @@ void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const SCN &scn, ObDDLKVHandle &kv_h ObDDLKV *tmp_kv = tmp_kv_handle.get_obj(); if (OB_ISNULL(tmp_kv)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(tmp_kv), K(i), K(head_), K(tail_)); + LOG_WARN("ddl kv is null", K(ret), K(tablet_id_), KP(tmp_kv), K(i), K(head_), K(tail_)); } else if (scn > tmp_kv->get_start_scn() && scn <= tmp_kv->get_freeze_scn()) { kv_handle = tmp_kv_handle; break; @@ -589,12 +586,12 @@ int ObTabletDDLKvMgr::release_ddl_kvs(const ObDDLKVType ddl_kv_type, const SCN & if (OB_FAIL(ret)) { } else if (OB_ISNULL(kv)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); + LOG_WARN("ddl kv is null", K(ret), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); } else if (kv->is_closed() && kv->get_freeze_scn() <= end_scn && kv->get_ddl_kv_type() == ddl_kv_type) { const SCN &freeze_scn = kv->get_freeze_scn(); free_ddl_kv(idx); ++head_; - LOG_INFO("succeed to release ddl kv", K(ls_id_), K(tablet_id_), K(freeze_scn)); + LOG_INFO("succeed to release ddl kv", K(tablet_id_), K(freeze_scn)); } } } @@ -615,7 +612,7 @@ int ObTabletDDLKvMgr::get_ddl_kv_min_scn(SCN &min_scn) ObDDLKV *kv = ddl_kv_handles_[idx].get_obj(); if (OB_ISNULL(kv)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); + LOG_WARN("ddl kv is null", K(ret), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); } else { min_scn = SCN::min(min_scn, kv->get_min_scn()); } @@ -641,7 +638,7 @@ int ObTabletDDLKvMgr::get_ddl_kvs_unlock( ObDDLKV *cur_kv = cur_kv_handle.get_obj(); if (OB_ISNULL(cur_kv)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(cur_kv), K(pos), K(head_), K(tail_)); + LOG_WARN("ddl kv is null", K(ret), K(tablet_id_), KP(cur_kv), K(pos), K(head_), K(tail_)); } else if ((!frozen_only || cur_kv->is_freezed()) && ddl_kv_query_param.match_ddl_kv(*cur_kv)) { if (OB_FAIL(kv_handle_array.push_back(cur_kv_handle))) { @@ -688,7 +685,7 @@ int ObTabletDDLKvMgr::get_ddl_kvs_for_query(ObTablet &tablet, ObIArrayget_tablet_meta().ddl_checkpoint_scn_ != commit_scn) {// only exist in memory SCN max_decided_scn; @@ -711,8 +708,8 @@ int ObTabletDDLKvMgr::try_flush_ddl_commit_scn( } if (already_freezed) { // do nothing - } else if (OB_FAIL(ls_handle.get_ls()->get_max_decided_scn(max_decided_scn))) { - LOG_WARN("get max decided log ts failed", K(ret), K(ls_handle.get_ls()->get_ls_id())); + } else if (OB_FAIL(ls->get_max_decided_scn(max_decided_scn))) { + LOG_WARN("get max decided log ts failed", K(ret)); } else if (SCN::plus(max_decided_scn, 1) >= commit_scn) { // commit_scn elapsed, means the prev clog already replayed or applied // max_decided_scn is the left border scn - 1 // the min deciding(replay or apply) scn (aka left border) is max_decided_scn + 1 @@ -755,7 +752,7 @@ int ObTabletDDLKvMgr::check_has_freezed_ddl_kv(bool &has_freezed_ddl_kv) ObDDLKV *cur_kv = cur_kv_handle.get_obj(); if (OB_ISNULL(cur_kv)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(cur_kv), K(pos), K(head_), K(tail_)); + LOG_WARN("ddl kv is null", K(ret), K(tablet_id_), KP(cur_kv), K(pos), K(head_), K(tail_)); } else if (cur_kv->is_freezed()) { has_freezed_ddl_kv = true; } @@ -801,8 +798,7 @@ int ObTabletDDLKvMgr::alloc_ddl_kv( } else if (OB_ISNULL(kv = tmp_kv_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ddl kv is null", K(ret)); - } else if (OB_FAIL(kv->init(ls_id_, - tablet_id_, + } else if (OB_FAIL(kv->init(tablet_id_, start_scn, snapshot_version, max_freeze_scn_, @@ -810,13 +806,13 @@ int ObTabletDDLKvMgr::alloc_ddl_kv( ddl_kv_type, trans_id, seq_no))) { - LOG_WARN("fail to init ddl kv", K(ret), K(ls_id_), K(tablet_id_), K(ddl_kv_type), K(trans_id), K(seq_no)); + LOG_WARN("fail to init ddl kv", K(ret), K(tablet_id_), K(ddl_kv_type), K(trans_id), K(seq_no)); } else { const int64_t idx = get_idx(tail_); tail_++; ddl_kv_handles_[idx] = tmp_kv_handle; kv_handle = tmp_kv_handle; - FLOG_INFO("succeed to add ddl kv", K(ls_id_), K(tablet_id_), K(head_), K(tail_), K(max_freeze_scn_), "ddl_kv_cnt", get_count_nolock(), KP(kv)); + FLOG_INFO("succeed to add ddl kv", K(tablet_id_), K(head_), K(tail_), K(max_freeze_scn_), "ddl_kv_cnt", get_count_nolock(), KP(kv)); } return ret; } @@ -838,7 +834,7 @@ void ObTabletDDLKvMgr::free_ddl_kv(const int64_t idx) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(idx)); } else { - FLOG_INFO("free ddl kv", K(ls_id_), K(tablet_id_), KPC(ddl_kv_handles_[idx].get_obj())); + FLOG_INFO("free ddl kv", K(tablet_id_), KPC(ddl_kv_handles_[idx].get_obj())); ddl_kv_handles_[idx].reset(); } } diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h index b7e6e7d2e..a6c4ea29c 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h @@ -22,7 +22,6 @@ #include "lib/lock/ob_tc_rwlock.h" #include "lib/ob_define.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "storage/ob_i_table.h" #include "storage/meta_mem/ob_tablet_pointer.h" @@ -30,6 +29,7 @@ namespace oceanbase { namespace storage { +class ObLS; class ObTablet; class ObDDLKV; @@ -117,7 +117,7 @@ class ObTabletDDLKvMgr final ObTabletDDLKvMgr(); ~ObTabletDDLKvMgr(); int register_to_tablet(ObDDLKvMgrHandle &kv_mgr_handle); - int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); // init before memtable mgr + int init(const common::ObTabletID &tablet_id); // init before memtable mgr int set_max_freeze_scn(const share::SCN &checkpoint_scn); int get_or_create_shared_nothing_ddl_kv( const share::SCN ¯o_redo_scn, @@ -146,7 +146,7 @@ class ObTabletDDLKvMgr final int release_ddl_kvs(const ObDDLKVType ddl_kv_type, const share::SCN &rec_scn); // release persistent ddl kv, used in ddl merge task for free ddl kv int check_has_effective_ddl_kv(bool &has_ddl_kv); // used in ddl log handler for checkpoint int try_flush_ddl_commit_scn( - ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &tablet_handle, const ObTabletDirectLoadMgrHandle &direct_load_mgr_handle, const share::SCN &commit_scn); @@ -181,7 +181,7 @@ class ObTabletDDLKvMgr final const ObITable::TableType table_type); int remove_idempotence_checker(); share::SCN get_max_freeze_scn() const { return max_freeze_scn_;} - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), + TO_STRING_KV(K_(is_inited), K_(tablet_id), K_(max_freeze_scn), K_(head), K_(tail), K_(ref_cnt)); @@ -217,7 +217,6 @@ class ObTabletDDLKvMgr final static const int64_t TRY_LOCK_TIMEOUT = 10 * 1000000; // 10s private: bool is_inited_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; share::SCN max_freeze_scn_; ObDDLKVHandle ddl_kv_handles_[MAX_DDL_KV_CNT_IN_STORAGE]; diff --git a/src/storage/ddl/ob_tablet_fork_task.cpp b/src/storage/ddl/ob_tablet_fork_task.cpp index 45f4cbb82..ff3b5c5e0 100644 --- a/src/storage/ddl/ob_tablet_fork_task.cpp +++ b/src/storage/ddl/ob_tablet_fork_task.cpp @@ -63,7 +63,6 @@ ObForkSnapshotRowScan::ObForkSnapshotRowScan() rowkey_read_info_(nullptr), access_param_(), allocator_("ForkSnapScan"), - ls_id_(), fork_snapshot_version_(0) {} @@ -142,7 +141,6 @@ int ObForkSnapshotRowScan::construct_access_param(const ObForkScanParam ¶m) } int ObForkSnapshotRowScan::construct_access_ctx( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t fork_snapshot_version) { @@ -165,19 +163,15 @@ int ObForkSnapshotRowScan::construct_access_ctx( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id)); } else { share::SCN snapshot_scn; if (OB_FAIL(snapshot_scn.convert_for_tx(fork_snapshot_version))) { LOG_WARN("failed to convert snapshot version", K(ret), K(fork_snapshot_version)); - } else if (OB_FAIL(ctx_.init_for_read(ls_id, - tablet_id, + } else if (OB_FAIL(ctx_.init_for_read(tablet_id, INT64_MAX, // query_expire_ts -1, // lock_timeout_us snapshot_scn))) { // Use fork_snapshot_version SCN instead of SCN::max_scn() - LOG_WARN("fail to init store ctx", K(ret), K(ls_id), K(snapshot_scn)); + LOG_WARN("fail to init store ctx", K(ret), K(snapshot_scn)); } else if (OB_FAIL(access_ctx_.init(query_flag, ctx_, allocator_, @@ -187,9 +181,9 @@ int ObForkSnapshotRowScan::construct_access_ctx( } } if (OB_FAIL(ret)) { - LOG_WARN("construct access ctx failed", KR(ret), K(ls_id), K(tablet_id), K(fork_snapshot_version)); + LOG_WARN("construct access ctx failed", KR(ret), K(tablet_id), K(fork_snapshot_version)); } else { - LOG_DEBUG("construct access ctx finished", K(ls_id), K(tablet_id), K(fork_snapshot_version)); + LOG_DEBUG("construct access ctx finished", K(tablet_id), K(fork_snapshot_version)); } return ret; } @@ -197,7 +191,6 @@ int ObForkSnapshotRowScan::construct_access_ctx( int ObForkSnapshotRowScan::init( const ObForkScanParam ¶m, blocksstable::ObSSTable &sstable, - const share::ObLSID &ls_id, const int64_t fork_snapshot_version) { int ret = OB_SUCCESS; @@ -205,15 +198,15 @@ int ObForkSnapshotRowScan::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!param.is_valid() || !sstable.is_valid() || !ls_id.is_valid() || fork_snapshot_version <= 0)) { + } else if (OB_UNLIKELY(!param.is_valid() || !sstable.is_valid() || fork_snapshot_version <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(param), K(sstable), K(ls_id), K(fork_snapshot_version)); + LOG_WARN("invalid args", K(ret), K(param), K(sstable), K(fork_snapshot_version)); } else if (OB_FAIL(construct_access_param(param))) { LOG_WARN("construct access param failed", K(ret), K(param)); } else { const ObTabletID &tablet_id = param.tablet_handle_.get_obj()->get_tablet_meta().tablet_id_; - if (OB_FAIL(construct_access_ctx(ls_id, tablet_id, fork_snapshot_version))) { - LOG_WARN("construct access ctx failed", K(ret), K(ls_id), K(tablet_id), K(fork_snapshot_version)); + if (OB_FAIL(construct_access_ctx(tablet_id, fork_snapshot_version))) { + LOG_WARN("construct access ctx failed", K(ret), K(tablet_id), K(fork_snapshot_version)); } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSSTableRowWholeScanner)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem failed", K(ret)); @@ -224,7 +217,6 @@ int ObForkSnapshotRowScan::init( param.query_range_))) { LOG_WARN("construct iterator failed", K(ret)); } else { - ls_id_ = ls_id; fork_snapshot_version_ = fork_snapshot_version; is_inited_ = true; } @@ -297,7 +289,6 @@ int ObForkSnapshotRowScan::get_next_row(const ObDatumRow *&tmp_row) ObTabletForkParam::ObTabletForkParam() : is_inited_(false), - ls_id_(), table_id_(OB_INVALID_ID), schema_version_(0), task_id_(0), @@ -311,7 +302,6 @@ ObTabletForkParam::ObTabletForkParam() ObTabletForkParam::ObTabletForkParam(const ObTabletForkParam &other) : is_inited_(other.is_inited_), - ls_id_(other.ls_id_), table_id_(other.table_id_), schema_version_(other.schema_version_), task_id_(other.task_id_), @@ -327,8 +317,6 @@ ObTabletForkParam &ObTabletForkParam::operator=(const ObTabletForkParam &other) { if (this != &other) { is_inited_ = other.is_inited_; - - ls_id_ = other.ls_id_; table_id_ = other.table_id_; schema_version_ = other.schema_version_; task_id_ = other.task_id_; @@ -349,8 +337,6 @@ ObTabletForkParam::~ObTabletForkParam() void ObTabletForkParam::reset() { is_inited_ = false; - - ls_id_.reset(); table_id_ = OB_INVALID_ID; schema_version_ = 0; task_id_ = 0; @@ -363,8 +349,7 @@ void ObTabletForkParam::reset() bool ObTabletForkParam::is_valid() const { - return ls_id_.is_valid() - && OB_INVALID_ID != table_id_ + return OB_INVALID_ID != table_id_ && schema_version_ > 0 && task_id_ > 0 && source_tablet_id_.is_valid() @@ -384,8 +369,6 @@ int ObTabletForkParam::init(const ObTabletForkParam ¶m) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(param)); } else { - - ls_id_ = param.ls_id_; table_id_ = param.table_id_; schema_version_ = param.schema_version_; task_id_ = param.task_id_; @@ -405,7 +388,7 @@ ObTabletForkCtx::ObTabletForkCtx() allocator_lock_(), is_inited_(false), complement_data_ret_(OB_SUCCESS), - ls_handle_(), + ls_(nullptr), src_tablet_handle_(), dst_tablet_handle_(), snapshot_table_store_(), @@ -414,17 +397,15 @@ ObTabletForkCtx::ObTabletForkCtx() clipped_schemas_map_(), created_sstable_handles_(), created_sstable_handles_lock_(), - row_inserted_(0), - ls_rebuild_seq_(-1) + row_inserted_(0) { } ObTabletForkCtx::~ObTabletForkCtx() { is_inited_ = false; - ls_rebuild_seq_ = -1; complement_data_ret_ = OB_SUCCESS; - ls_handle_.reset(); + ls_ = nullptr; src_tablet_handle_.reset(); dst_tablet_handle_.reset(); snapshot_table_store_.reset(); @@ -438,7 +419,7 @@ ObTabletForkCtx::~ObTabletForkCtx() bool ObTabletForkCtx::is_valid() const { - return is_inited_ && ls_handle_.is_valid() && src_tablet_handle_.is_valid() && dst_tablet_handle_.is_valid(); + return is_inited_ && OB_NOT_NULL(ls_) && src_tablet_handle_.is_valid() && dst_tablet_handle_.is_valid(); } int ObTabletForkCtx::add_created_sstable(const ObTableHandleV2 &handle) @@ -490,9 +471,9 @@ int ObTabletForkCtx::init(const ObTabletForkParam ¶m) } else { LOG_DEBUG("fork condition not satisfied yet, need retry", K(param)); } - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle_, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_))) { LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, param.source_tablet_id_, src_tablet_handle_, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get source tablet failed", K(ret), K(param.source_tablet_id_)); } else if (OB_FAIL(src_tablet_handle_.get_obj()->fetch_table_store(snapshot_table_store_))) { @@ -507,11 +488,10 @@ int ObTabletForkCtx::init(const ObTabletForkParam ¶m) table_store_iterator_, ObGetReadTablesMode::NORMAL))) { LOG_WARN("fail to fetch snapshot read tables", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, param.dest_tablet_id_, dst_tablet_handle_, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get destination tablet failed", K(ret), K(param.dest_tablet_id_)); } else { - ls_rebuild_seq_ = ls_handle_.get_ls()->get_rebuild_seq(); complement_data_ret_ = OB_SUCCESS; is_inited_ = true; } @@ -572,7 +552,7 @@ int ObTabletForkCtx::prepare_index_builder(const ObTabletForkParam ¶m) const int64_t snapshot_version = param.fork_snapshot_version_; if (OB_FAIL(data_desc.init( - true/*is_ddl*/, *storage_schema, param.ls_id_, + true/*is_ddl*/, *storage_schema, param.dest_tablet_id_, merge_type, snapshot_version, param.data_format_version_, dst_tablet_handle_.get_obj()->get_tablet_meta().micro_index_clustered_, 0/*concurrent_cnt*/, @@ -734,8 +714,7 @@ uint64_t ObTabletForkDag::hash() const ret = OB_ERR_SYS; LOG_ERROR("invalid argument", K(ret), K(is_inited_), K(param_)); } else { - hash_val = param_.ls_id_.hash() - + param_.table_id_ + param_.schema_version_ + hash_val = param_.table_id_ + param_.schema_version_ + param_.source_tablet_id_.hash() + param_.dest_tablet_id_.hash() + static_cast(param_.fork_snapshot_version_) + ObDagType::DAG_TYPE_FORK_TABLE; @@ -755,9 +734,7 @@ bool ObTabletForkDag::operator==(const ObIDag &other) const ret = OB_ERR_SYS; LOG_WARN("invalid argument", K(ret), K(param_), K(dag.param_)); } else { - is_equal = true - && param_.ls_id_ == dag.param_.ls_id_ - && param_.schema_version_ == dag.param_.schema_version_ + is_equal = param_.schema_version_ == dag.param_.schema_version_ && param_.source_tablet_id_ == dag.param_.source_tablet_id_ && param_.dest_tablet_id_ == dag.param_.dest_tablet_id_; } @@ -772,7 +749,7 @@ int ObTabletForkDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, ret = OB_NOT_INIT; LOG_WARN("ObTabletForkDag has not been initialized", K(ret)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - static_cast(param_.ls_id_.id()), static_cast(param_.source_tablet_id_.id())))) { + static_cast(param_.source_tablet_id_.id())))) { LOG_WARN("failed to fill info param", K(ret)); } return ret; @@ -788,9 +765,9 @@ int ObTabletForkDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param", K(ret), K(param_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "Fork table: src_tablet_id=%ld, dst_tablet_id=%ld, fork_snapshot_version=%ld, ls_id=%ld, schema_version=%ld", + "Fork table: src_tablet_id=%ld, dst_tablet_id=%ld, fork_snapshot_version=%ld, schema_version=%ld", param_.source_tablet_id_.id(), param_.dest_tablet_id_.id(), param_.fork_snapshot_version_, - param_.ls_id_.id(), param_.schema_version_))) { + param_.schema_version_))) { LOG_WARN("fail to fill dag key", K(ret), K(param_)); } return ret; @@ -939,7 +916,7 @@ int ObTabletForkReuseTask::process_reuse_sstable() int ret = OB_SUCCESS; ObTabletCreateSSTableParam param; ObTableHandleV2 table_handle; - blocksstable::ObMigrationSSTableParam mig_sstable_param; + blocksstable::ObForkSSTableParam fork_sstable_param; share::SCN fork_snapshot_scn; if (OB_ISNULL(sstable_)) { @@ -952,10 +929,10 @@ int ObTabletForkReuseTask::process_reuse_sstable() ObSSTableMetaHandle meta_handle; if (OB_FAIL(sstable_->get_meta(meta_handle))) { LOG_WARN("failed to get sstable meta", K(ret)); - } else if (OB_FAIL(context_->src_tablet_handle_.get_obj()->build_migration_sstable_param( - src_table_key, mig_sstable_param, true/*is_fork_table*/))) { - LOG_WARN("failed to build migration sstable param", K(ret), K(src_table_key)); - } else if (OB_FAIL(param.init_for_fork(mig_sstable_param, param_->dest_tablet_id_, src_table_key, meta_handle.get_sstable_meta(), fork_snapshot_scn))) { + } else if (OB_FAIL(context_->src_tablet_handle_.get_obj()->build_fork_sstable_param( + src_table_key, fork_sstable_param))) { + LOG_WARN("failed to build fork sstable param", K(ret), K(src_table_key)); + } else if (OB_FAIL(param.init_for_fork(fork_sstable_param, param_->dest_tablet_id_, src_table_key, meta_handle.get_sstable_meta(), fork_snapshot_scn))) { LOG_WARN("init for fork failed", K(ret), K(param_->dest_tablet_id_), K(src_table_key), K(fork_snapshot_scn)); } else if (OB_FAIL(context_->create_sstable(param, table_handle))) { LOG_ERROR("failed to create sstable with reused blocks", K(ret)); @@ -1142,7 +1119,6 @@ int ObTabletForkRewriteTask::prepare_macro_block_writer( if (OB_FAIL(data_desc.init( true/*is_ddl*/, clipped_storage_schema, - param_->ls_id_, param_->dest_tablet_id_, merge_type, snapshot_version, @@ -1163,7 +1139,7 @@ int ObTabletForkRewriteTask::prepare_macro_block_writer( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem failed", K(ret)); } else if (FALSE_IT(macro_block_writer = new (buf) ObMacroBlockWriter())) { - } else if (OB_FAIL(pre_warm_param.init(param_->ls_id_, param_->dest_tablet_id_))) { + } else if (OB_FAIL(pre_warm_param.init(param_->dest_tablet_id_))) { LOG_WARN("failed to init pre warm param", K(ret), K(param_->dest_tablet_id_), KPC(param_)); } else if (OB_FAIL(ObSSTablePrivateObjectCleaner::get_cleaner_from_data_store_desc( data_desc.get_desc(), @@ -1210,7 +1186,7 @@ int ObTabletForkRewriteTask::process_rewrite_sstable_task( whole_range, clipped_storage_schema); - if (OB_FAIL(row_scan.init(scan_param, *sstable_, param_->ls_id_, param_->fork_snapshot_version_))) { + if (OB_FAIL(row_scan.init(scan_param, *sstable_, param_->fork_snapshot_version_))) { LOG_WARN("init fork snapshot row scan failed", K(ret)); } else { const ObDatumRow *row = nullptr; @@ -1361,7 +1337,7 @@ int ObTabletForkMergeTask::create_sstables() bool need_fill_empty_sstable = false; share::SCN end_scn; if (OB_FAIL(ObTabletRebuildUtil::check_need_fill_empty_sstable( - context_->ls_handle_, + context_->ls_, max_end_scn_minor_sstable->is_minor_sstable(), max_end_scn_minor_sstable->get_key(), param_->dest_tablet_id_, @@ -1403,8 +1379,7 @@ int ObTabletForkMergeTask::create_sstables() // Batch update table store with all created sstables if (OB_SUCC(ret) && src_table_cnt > 0) { if (OB_FAIL(update_table_store_with_batch_tables( - context_->ls_rebuild_seq_, - context_->ls_handle_, + context_->ls_, context_->src_tablet_handle_, context_->dst_tablet_handle_, param_->dest_tablet_id_, @@ -1419,8 +1394,7 @@ int ObTabletForkMergeTask::create_sstables() } int ObTabletForkMergeTask::update_table_store_with_batch_tables( - const int64_t ls_rebuild_seq, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &src_tablet_handle, const ObTabletHandle &dst_tablet_handle, const ObTabletID &dst_tablet_id, @@ -1431,17 +1405,17 @@ int ObTabletForkMergeTask::update_table_store_with_batch_tables( ObBatchUpdateTableStoreParam param; param.reset(); ObSEArray batch_tables; - ObMigrationTabletParam src_tablet_param; + ObArenaAllocator schema_allocator(common::ObMemAttr("ForkSchema")); + ObStorageSchema *src_storage_schema = nullptr; if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(ls_rebuild_seq == -1 - || !ls_handle.is_valid() + } else if (OB_UNLIKELY(OB_ISNULL(ls) || !src_tablet_handle.is_valid() || !dst_tablet_handle.is_valid() || !dst_tablet_id.is_valid() || !is_valid_merge_type(merge_type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_rebuild_seq), K(ls_handle), K(src_tablet_handle), + LOG_WARN("invalid arg", K(ret), K(ls), K(src_tablet_handle), K(dst_tablet_handle), K(dst_tablet_id), K(merge_type)); } else if (OB_FAIL(tables_handle.get_tables(batch_tables))) { LOG_WARN("fork table: get batch sstables failed", KR(ret), KPC_(param), K(dst_tablet_id)); @@ -1450,16 +1424,16 @@ int ObTabletForkMergeTask::update_table_store_with_batch_tables( // TODO(fankun.fan): meta major sstable } else if (OB_UNLIKELY(src_tablet_handle.get_obj()->is_empty_shell())) { LOG_WARN("fork table: src tablet is empty shell, skip src storage schema", K(dst_tablet_id), K(src_tablet_handle)); - } else if (OB_FAIL(src_tablet_handle.get_obj()->build_migration_tablet_param(src_tablet_param))) { - LOG_WARN("fork table: build src tablet param failed", K(ret), K(dst_tablet_id), K(src_tablet_handle)); - } else if (OB_UNLIKELY(!src_tablet_param.storage_schema_.is_valid())) { - LOG_WARN("fork table: src storage schema is invalid, skip using it", K(dst_tablet_id), K(src_tablet_param.storage_schema_)); + } else if (OB_FAIL(src_tablet_handle.get_obj()->load_storage_schema(schema_allocator, src_storage_schema))) { + LOG_WARN("fork table: load src storage schema failed", K(ret), K(dst_tablet_id), K(src_tablet_handle)); + } else if (OB_ISNULL(src_storage_schema) || OB_UNLIKELY(!src_storage_schema->is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fork table: src storage schema is invalid", K(ret), K(dst_tablet_id), KP(src_storage_schema)); } else { - param.tablet_meta_ = &src_tablet_param; + param.storage_schema_ = src_storage_schema; } if (OB_SUCC(ret)) { - param.rebuild_seq_ = ls_rebuild_seq; param.release_mds_scn_.set_min(); param.tablet_fork_param_.snapshot_version_ = param_->fork_snapshot_version_; param.tablet_fork_param_.multi_version_start_ = param_->fork_snapshot_version_; @@ -1497,7 +1471,7 @@ int ObTabletForkMergeTask::update_table_store_with_batch_tables( param.tablet_fork_param_.clog_checkpoint_scn_ = clog_checkpoint_scn; param.tablet_fork_param_.mds_checkpoint_scn_ = mds_checkpoint_scn; - if (OB_FAIL(ls_handle.get_ls()->build_tablet_with_batch_tables(dst_tablet_id, param))) { + if (OB_FAIL(ls->build_tablet_with_batch_tables(dst_tablet_id, param))) { LOG_WARN("fork table: update tablet table store failed", KR(ret), KPC_(param), K(dst_tablet_id), "batch_cnt", batch_tables.count(), K(minor_cnt), K(major_cnt), "min_start_scn", min_start_scn_val, "max_end_scn", max_end_scn_val, @@ -1512,6 +1486,7 @@ int ObTabletForkMergeTask::update_table_store_with_batch_tables( } } + ObTabletObjLoadHelper::free(schema_allocator, src_storage_schema); return ret; } @@ -1522,16 +1497,16 @@ int ObTabletForkUtil::check_satisfy_fork_condition( int ret = OB_SUCCESS; is_satisfied = false; bool need_freeze = false; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObArray memtable_handles; if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(param)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(param.ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, param.source_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, param.source_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("failed to get tablet", K(ret), K(param.source_tablet_id_)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -1572,8 +1547,8 @@ int ObTabletForkUtil::check_satisfy_fork_condition( } if (OB_SUCC(ret) && need_freeze) { - if (OB_FAIL(ObTabletForkUtil::freeze_tablet(param.ls_id_, param.source_tablet_id_))) { - LOG_WARN("failed to freeze tablet", K(ret), K(param.ls_id_), K(param.source_tablet_id_)); + if (OB_FAIL(ObTabletForkUtil::freeze_tablet(param.source_tablet_id_))) { + LOG_WARN("failed to freeze tablet", K(ret), K(param.source_tablet_id_)); } ob_usleep(100 * 1000L); // 100ms } @@ -1588,23 +1563,23 @@ int ObTabletForkUtil::check_fork_data_complete( { int ret = OB_SUCCESS; is_complete = false; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; share::ObForkTabletInfo fork_info; if (OB_UNLIKELY(!dest_tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(SYS_LS), K(dest_tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(SYS_LS, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(SYS_LS)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, dest_tablet_id, + LOG_WARN("invalid arg", K(ret), K(dest_tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, dest_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet handle", K(ret), K(SYS_LS), K(dest_tablet_id)); + LOG_WARN("failed to get tablet handle", K(ret), K(dest_tablet_id)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(SYS_LS), K(dest_tablet_id)); + LOG_WARN("tablet handle is null", K(ret), K(dest_tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->get_fork_info(fork_info))) { - LOG_WARN("failed to get fork_info from tablet", K(ret), K(SYS_LS), K(dest_tablet_id)); + LOG_WARN("failed to get fork_info from tablet", K(ret), K(dest_tablet_id)); } else if (fork_info.get_fork_src_tablet_id().is_valid() && fork_info.is_complete()) { is_complete = true; LOG_DEBUG("tablet fork data complement complete", K(dest_tablet_id), K(fork_info)); @@ -1696,7 +1671,6 @@ int ObTabletForkUtil::try_schedule_fork_dags(const ObTableForkInfo &fork_info) int64_t fail_cnt = 0; LOG_INFO("fork table: schedule fork dags begin", "task_id", fork_info.task_id_, - K(fork_info.ls_id_), "table_id", fork_info.table_id_, "schema_version", fork_info.schema_version_, "fork_snapshot_version", fork_info.fork_snapshot_version_, @@ -1720,7 +1694,6 @@ int ObTabletForkUtil::try_schedule_fork_dags(const ObTableForkInfo &fork_info) } LOG_INFO("fork table: schedule fork dags finish", "task_id", fork_info.task_id_, - K(fork_info.ls_id_), "tablet_cnt", fork_params.count(), K(scheduled_cnt), K(exist_cnt), K(fail_cnt), KR(ret)); @@ -1729,7 +1702,6 @@ int ObTabletForkUtil::try_schedule_fork_dags(const ObTableForkInfo &fork_info) } int ObTabletForkUtil::freeze_tablet( - const share::ObLSID &ls_id, const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -1738,9 +1710,9 @@ int ObTabletForkUtil::freeze_tablet( if (OB_ISNULL(tenant_freezer)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObTenantFreezer is null", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid arguments", K(ret), K(tablet_id)); } else { // Freeze the tablet const bool is_sync = false; @@ -1748,8 +1720,7 @@ int ObTabletForkUtil::freeze_tablet( const bool need_rewrite_tablet_meta = false; const ObFreezeSourceFlag source = ObFreezeSourceFlag::FREEZE_TRIGGER; - if (OB_FAIL(tenant_freezer->tablet_freeze(ls_id, - tablet_id, + if (OB_FAIL(tenant_freezer->tablet_freeze(tablet_id, is_sync, max_retry_time_us, need_rewrite_tablet_meta, @@ -1768,17 +1739,16 @@ int ObTabletForkUtil::freeze_tablet( } int ObTabletForkUtil::freeze_tablets( - const share::ObLSID &ls_id, const ObIArray &tablet_ids) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(tablet_ids.empty())) { + if (OB_UNLIKELY(tablet_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_ids.count())); + LOG_WARN("invalid arguments", K(ret), K(tablet_ids.count())); } else { for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - if (OB_FAIL(ObTabletForkUtil::freeze_tablet(ls_id, tablet_id))) { + if (OB_FAIL(ObTabletForkUtil::freeze_tablet(tablet_id))) { LOG_WARN("failed to freeze tablet", K(ret), K(tablet_id)); } } diff --git a/src/storage/ddl/ob_tablet_fork_task.h b/src/storage/ddl/ob_tablet_fork_task.h index 327c9ac85..1cf23d3d4 100644 --- a/src/storage/ddl/ob_tablet_fork_task.h +++ b/src/storage/ddl/ob_tablet_fork_task.h @@ -30,7 +30,6 @@ #include "storage/tablet/ob_tablet_obj_load_helper.h" #include "storage/tablet/ob_tablet_member_wrapper.h" #include "lib/oblog/ob_log_module.h" -#include "storage/tx_storage/ob_ls_handle.h" // For ObLSHandle #include "storage/tablet/ob_tablet_create_delete_helper.h" #include "lib/lock/ob_mutex.h" #include "storage/ddl/ob_tablet_rebuild_util.h" @@ -41,8 +40,8 @@ namespace oceanbase { namespace storage { +class ObLS; class ObTableForkInfo; -struct ObMigrationTabletParam; struct ObForkScanParam final { @@ -77,7 +76,6 @@ class ObForkSnapshotRowScan : public ObIStoreRowIterator int init( const ObForkScanParam ¶m, ObSSTable &sstable, - const share::ObLSID &ls_id, const int64_t fork_snapshot_version); virtual int get_next_row(const blocksstable::ObDatumRow *&tmp_row) override; @@ -85,11 +83,10 @@ class ObForkSnapshotRowScan : public ObIStoreRowIterator const ObITableReadInfo *get_rowkey_read_info() const { return rowkey_read_info_; } storage::ObTxTableGuards &get_tx_table_guards() { return ctx_.mvcc_acc_ctx_.get_tx_table_guards(); } - TO_STRING_KV(K_(is_inited), K_(fork_snapshot_version), K_(ls_id), K_(ctx), K_(access_ctx), KPC_(rowkey_read_info), K_(access_param)); + TO_STRING_KV(K_(is_inited), K_(fork_snapshot_version), K_(ctx), K_(access_ctx), KPC_(rowkey_read_info), K_(access_param)); private: int construct_access_param(const ObForkScanParam ¶m); int construct_access_ctx( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t fork_snapshot_version); int build_rowkey_read_info(const ObForkScanParam ¶m); @@ -101,7 +98,6 @@ class ObForkSnapshotRowScan : public ObIStoreRowIterator ObRowkeyReadInfo *rowkey_read_info_; ObTableAccessParam access_param_; common::ObArenaAllocator allocator_; - share::ObLSID ls_id_; int64_t fork_snapshot_version_; }; @@ -118,13 +114,11 @@ struct ObTabletForkParam : public share::ObIDagInitParam void reset(); bool is_valid() const; int init(const ObTabletForkParam ¶m); - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(table_id), K_(schema_version), + TO_STRING_KV(K_(is_inited), K_(table_id), K_(schema_version), K_(task_id), K_(source_tablet_id), K_(dest_tablet_id), K_(fork_snapshot_version), K_(compat_mode), K_(data_format_version)); public: bool is_inited_; - - share::ObLSID ls_id_; uint64_t table_id_; int64_t schema_version_; int64_t task_id_; @@ -142,8 +136,7 @@ struct ObTabletForkCtx final ~ObTabletForkCtx(); int init(const ObTabletForkParam ¶m); bool is_valid() const; - TO_STRING_KV(K_(is_inited), K_(complement_data_ret), K_(row_inserted), - K_(ls_rebuild_seq)); + TO_STRING_KV(K_(is_inited), K_(complement_data_ret), K_(row_inserted)); public: int prepare_index_builder(const ObTabletForkParam ¶m); common::ObIAllocator &get_allocator() { return allocator_; } @@ -170,7 +163,7 @@ struct ObTabletForkCtx final typedef common::hash::ObHashMap INDEX_BUILDER_MAP; bool is_inited_; int complement_data_ret_; - ObLSHandle ls_handle_; + ObLS *ls_; ObTabletHandle src_tablet_handle_; ObTabletHandle dst_tablet_handle_; ObTabletMemberWrapper snapshot_table_store_; @@ -180,7 +173,6 @@ struct ObTabletForkCtx final ObTablesHandleArray created_sstable_handles_; lib::ObMutex created_sstable_handles_lock_; // Protect concurrent access to created_sstable_handles_ int64_t row_inserted_; - int64_t ls_rebuild_seq_; DISALLOW_COPY_AND_ASSIGN(ObTabletForkCtx); }; @@ -283,8 +275,7 @@ class ObTabletForkMergeTask final : public share::ObITask private: int create_sstables(); int update_table_store_with_batch_tables( - const int64_t ls_rebuild_seq, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &src_tablet_handle, const ObTabletHandle &dst_tablet_handle, const common::ObTabletID &dst_tablet_id, @@ -311,8 +302,8 @@ struct ObTabletForkUtil final const int64_t fork_snapshot_version, ObIArray &participants); static int try_schedule_fork_dags(const ObTableForkInfo &fork_info); - static int freeze_tablet(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); - static int freeze_tablets(const share::ObLSID &ls_id, const ObIArray &tablet_ids); + static int freeze_tablet(const common::ObTabletID &tablet_id); + static int freeze_tablets(const ObIArray &tablet_ids); }; } // end namespace storage diff --git a/src/storage/ddl/ob_tablet_lob_split_task.cpp b/src/storage/ddl/ob_tablet_lob_split_task.cpp index 2eab2c1ee..97434d805 100644 --- a/src/storage/ddl/ob_tablet_lob_split_task.cpp +++ b/src/storage/ddl/ob_tablet_lob_split_task.cpp @@ -94,7 +94,6 @@ int ObLobSplitParam::assign(const ObLobSplitParam &other) ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid param", K(ret), K(other)); } else { - ls_id_ = other.ls_id_; ori_lob_meta_tablet_id_ = other.ori_lob_meta_tablet_id_; schema_version_ = other.schema_version_; data_format_version_ = other.data_format_version_; @@ -140,7 +139,6 @@ int ObLobSplitParam::init(const obcall::ObDDLBuildSingleReplicaRequestArg &arg) ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid param", K(ret), K(arg)); } else { - ls_id_ = arg.ls_id_; ori_lob_meta_tablet_id_ = arg.source_tablet_id_; schema_version_ = arg.schema_version_; data_format_version_ = arg.data_format_version_; @@ -153,7 +151,7 @@ int ObLobSplitParam::init(const obcall::ObDDLBuildSingleReplicaRequestArg &arg) min_split_start_scn_ = arg.min_split_start_scn_; if (OB_FAIL(parallel_datum_rowkey_list_.assign(arg.parallel_datum_rowkey_list_))) { // shallow cpy. LOG_WARN("assign failed", K(ret), "parall_info", arg.parallel_datum_rowkey_list_); - } else if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ls_id_, ori_lob_meta_tablet_id_, new_lob_tablet_ids_, compat_mode_))) { + } else if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ori_lob_meta_tablet_id_, new_lob_tablet_ids_, compat_mode_))) { LOG_WARN("get split dest tablets failed", K(ret), K(arg)); } else if (OB_FAIL(lob_col_idxs_.assign(arg.lob_col_idxs_))) { LOG_WARN("failed to assign lob_col_idxs_", K(ret)); @@ -169,7 +167,6 @@ int ObLobSplitParam::init(const obcall::ObTabletSplitArg &arg) ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid param", K(ret), K(arg)); } else { - ls_id_ = arg.ls_id_; ori_lob_meta_tablet_id_ = arg.source_tablet_id_; schema_version_ = arg.schema_version_; data_format_version_ = arg.data_format_version_; @@ -181,7 +178,7 @@ int ObLobSplitParam::init(const obcall::ObTabletSplitArg &arg) compaction_scn_ = arg.compaction_scn_; min_split_start_scn_ = arg.min_split_start_scn_; ObArray unused_tablet_ids; - if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ls_id_, ori_lob_meta_tablet_id_, unused_tablet_ids, compat_mode_))) { + if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ori_lob_meta_tablet_id_, unused_tablet_ids, compat_mode_))) { LOG_WARN("get split dest tablets failed", K(ret), K(arg)); } else if (OB_FAIL(parallel_datum_rowkey_list_.assign(arg.parallel_datum_rowkey_list_))) { // shallow cpy. LOG_WARN("assign failed", K(ret), "parall_info", arg.parallel_datum_rowkey_list_); @@ -203,15 +200,15 @@ int ObLobSplitContext::init(const ObLobSplitParam& param) LOG_WARN("invalid param", K(ret), K(param)); } else if (OB_FAIL(init_maps(param))) { LOG_WARN("fail to init sort maps", K(ret), K(param)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle_, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_))) { LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, param.ori_lob_meta_tablet_id_, lob_meta_tablet_handle_, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("get tablet handle failed", K(ret), K(param)); } else if (OB_FAIL(ObTabletSplitUtil::check_satisfy_split_condition( - ls_handle_, lob_meta_tablet_handle_, param.new_lob_tablet_ids_, param.compaction_scn_, param.min_split_start_scn_))) { + ls_, lob_meta_tablet_handle_, param.new_lob_tablet_ids_, param.compaction_scn_, param.min_split_start_scn_))) { if (OB_NEED_RETRY == ret) { if (REACH_COUNT_INTERVAL(1000L)) { LOG_WARN("wait to satisfy the data split condition", K(ret), K(param)); @@ -224,7 +221,6 @@ int ObLobSplitContext::init(const ObLobSplitParam& param) } else if (OB_FAIL(ObTabletSplitUtil::convert_rowkey_to_range(range_allocator_, param.parallel_datum_rowkey_list_, main_table_ranges_))) { LOG_WARN("convert to range failed", K(ret), "parall_info", param.parallel_datum_rowkey_list_); } else { - ls_rebuild_seq_ = ls_handle_.get_ls()->get_rebuild_seq(); is_inited_ = true; LOG_INFO("show main tablet split info", K(param)); } @@ -247,7 +243,7 @@ int ObLobSplitContext::get_dst_lob_tablet_ids(const ObLobSplitParam& param) } else if (OB_FAIL(lob_meta_tablet_handle_.get_obj()->get_all_tables(lob_meta_table_store_iterator_))) { LOG_WARN("failed to get all sstables", K(ret), K(param)); } else if (FALSE_IT(main_tablet_id_ = lob_meta_tablet_handle_.get_obj()->get_tablet_meta().data_tablet_id_)) { - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, main_tablet_id_, main_tablet_handle_, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -270,13 +266,13 @@ int ObLobSplitContext::get_dst_lob_tablet_ids(const ObLobSplitParam& param) } else if (user_data.start_split_commit_version_ <= 0) { ret = OB_EAGAIN; LOG_WARN("failed to fetch the newest mds", K(ret)); - } else if (OB_FAIL(ObTabletSplitUtil::check_sstables_skip_data_split(ls_handle_, lob_meta_table_store_iterator_, + } else if (OB_FAIL(ObTabletSplitUtil::check_sstables_skip_data_split(ls_, lob_meta_table_store_iterator_, new_lob_tablet_ids_, user_data.start_split_commit_version_/*lob_major_snapshot*/, skipped_split_major_keys_))) { LOG_WARN("check sstables skip data split failed", K(ret)); } else { ObTabletHandle dst_lob_tablet_handle; for (int64_t i = 0; i < new_lob_tablet_ids_.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, new_lob_tablet_ids_.at(i), dst_lob_tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -329,8 +325,7 @@ int ObLobSplitContext::init_maps(const ObLobSplitParam& param) void ObLobSplitContext::destroy() { - ls_rebuild_seq_ = -1; - ls_handle_.reset(); + ls_ = nullptr; if (total_map_ != nullptr) { total_map_->clean_up(); allocator_.free(total_map_); @@ -421,8 +416,7 @@ bool ObTabletLobSplitDag::operator==(const ObIDag &other) const tmp_ret = OB_ERR_SYS; LOG_ERROR("invalid argument", K(tmp_ret), K(param_), K(dag.param_)); } else { - is_equal = (true) && (param_.ls_id_ == dag.param_.ls_id_) && - (param_.ori_lob_meta_tablet_id_ == dag.param_.ori_lob_meta_tablet_id_); + is_equal = param_.ori_lob_meta_tablet_id_ == dag.param_.ori_lob_meta_tablet_id_; } } return is_equal; @@ -440,7 +434,6 @@ int ObTabletLobSplitDag::calc_total_row_count() { // already calc, do nothing. LOG_TRACE("already calculated", K(context_.physical_row_count_)); } else if (OB_FAIL(ObDDLUtil::get_tablet_physical_row_cnt( - param_.ls_id_, param_.ori_lob_meta_tablet_id_, true, // calc_sstable = true; false, // calc_memtable = false; because memtable has been frozen. @@ -458,8 +451,7 @@ uint64_t ObTabletLobSplitDag::hash() const tmp_ret = OB_ERR_SYS; LOG_ERROR("table schema must not be NULL", K(tmp_ret), K(is_inited_), K(param_)); } else { - hash_val = param_.ls_id_.hash() - + param_.ori_lob_meta_tablet_id_.hash() + ObDagType::DAG_TYPE_LOB_SPLIT; + hash_val = param_.ori_lob_meta_tablet_id_.hash() + ObDagType::DAG_TYPE_LOB_SPLIT; } return hash_val; } @@ -471,7 +463,7 @@ int ObTabletLobSplitDag::fill_info_param(compaction::ObIBasicInfoParam *&out_par ret = OB_NOT_INIT; LOG_WARN("ObTabletLobSplitDag has not been initialized", K(ret)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - static_cast(param_.ls_id_.id()), static_cast(param_.ori_lob_meta_tablet_id_.id())))) { + static_cast(param_.ori_lob_meta_tablet_id_.id())))) { LOG_WARN("failed to fill info param", K(ret)); } return ret; @@ -487,9 +479,9 @@ int ObTabletLobSplitDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid params", K(ret), K(param_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "Regen macro block split: src_tablet_id=%ld, parallelism=%ld, ls_id=%ld, schema_version=%ld", + "Regen macro block split: src_tablet_id=%ld, parallelism=%ld, schema_version=%ld", param_.ori_lob_meta_tablet_id_.id(), param_.parallelism_, - param_.ls_id_.id(), param_.schema_version_))) { + param_.schema_version_))) { LOG_WARN("fail to fill comment", K(ret), K(param_)); } return ret; @@ -560,8 +552,6 @@ int ObTabletLobSplitDag::report_lob_split_status() ObAddr rs_addr = GCTX.self_addr(); - arg.ls_id_ = param_.ls_id_; - arg.dest_ls_id_ = param_.ls_id_; arg.tablet_id_ = param_.ori_lob_meta_tablet_id_; arg.source_table_id_ = param_.source_table_id_; arg.dest_schema_id_ = context_.main_tablet_id_.id(); // fill ori main tablet id @@ -635,7 +625,7 @@ int ObTabletLobBuildMapTask::process() LOG_WARN("build map task has already failed", "ret", ctx_->data_ret_); } else if (ctx_->is_lob_piece_) { // lob piece has no data, do nothing - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->new_lob_tablet_ids_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->new_lob_tablet_ids_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -734,7 +724,7 @@ int ObTabletLobBuildMapTask::build_sorted_map(ObIArray& iters) if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObTabletLobSplitTask is not inited", K(ret)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_, ctx_->main_tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { @@ -749,7 +739,7 @@ int ObTabletLobBuildMapTask::build_sorted_map(ObIArray& iters) ret = OB_ERR_UNEXPECTED; LOG_WARN("sub maps is null.", K(ret), K(ctx_->sub_maps_), K(task_id_)); } else if (OB_FAIL(ObTabletSplitMdsHelper::prepare_calc_split_dst( - *ctx_->ls_handle_.get_ls(), + *ctx_->ls_, *ctx_->main_tablet_handle_.get_obj(), ObTimeUtility::current_time() + (1 + ctx_->new_main_tablet_ids_.count()) * ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, src_split_data, @@ -883,7 +873,7 @@ int ObTabletLobMergeMapTask::process() LOG_WARN("dag is invalid", K(ret), KP(tmp_dag)); } else if (OB_SUCCESS != (ctx_->data_ret_)) { LOG_WARN("build map task has already failed", "ret", ctx_->data_ret_); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->new_lob_tablet_ids_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->new_lob_tablet_ids_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -1009,7 +999,7 @@ int ObTabletLobWriteDataTask::process() LOG_WARN("dag is invalid", K(ret), KP(tmp_dag)); } else if (OB_SUCCESS != (ctx_->data_ret_)) { LOG_WARN("write data task has already failed", "ret", ctx_->data_ret_); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->new_lob_tablet_ids_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->new_lob_tablet_ids_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -1217,7 +1207,7 @@ int ObTabletLobWriteDataTask::prepare_sstable_macro_writer(const ObTabletLobWrit if (OB_UNLIKELY(!new_tablet_id.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid new tablet id", K(ret), K(new_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_handle_, new_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_, new_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet failed", K(ret), K(new_tablet_id)); } else if (OB_FAIL(macro_start_seq.set_sstable_seq(write_sstable_ctx.data_seq_))) { LOG_WARN("set sstable logical seq failed", K(ret)); @@ -1232,8 +1222,8 @@ int ObTabletLobWriteDataTask::prepare_sstable_macro_writer(const ObTabletLobWrit if (OB_ISNULL(buf = ctx_->allocator_.alloc(sizeof(ObMacroBlockWriter)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem failed", K(ret)); - } else if (OB_FAIL(pre_warm_param.init(param_->ls_id_, new_tablet_id))) { - LOG_WARN("failed to init pre warm param", K(ret), "ls_id", param_->ls_id_, K(new_tablet_id)); + } else if (OB_FAIL(pre_warm_param.init(new_tablet_id))) { + LOG_WARN("failed to init pre warm param", K(ret), K(new_tablet_id)); } else if (OB_FAIL(ObSSTablePrivateObjectCleaner::get_cleaner_from_data_store_desc( data_desc.get_desc(), object_cleaner))) { @@ -1275,13 +1265,12 @@ int ObTabletLobWriteDataTask::prepare_sstable_index_builder(const ObTabletLobWri } else if (!new_tablet_id.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid new tablet id", K(ret), K(new_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_handle_, new_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ctx_->ls_, new_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_ISNULL(tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is null", K(ret), K(new_tablet_id), K(tablet_handle)); } else if (OB_FAIL(data_desc.init(true/*is_ddl*/, storage_schema, - param_->ls_id_, new_tablet_id, merge_type, snapshot_version, @@ -1505,7 +1494,7 @@ int ObTabletLobWriteDataTask::create_sstables( ObSEArray check_major_exist_tablets; if (OB_FAIL(check_major_exist_tablets.push_back(dst_tablet_id))) { LOG_WARN("push back failed", K(ret)); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, check_major_exist_tablets, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(check_major_exist_tablets, is_data_split_finished))) { LOG_WARN("check major exist failed", K(ret), K(check_major_exist_tablets), KPC(param_)); } else if (is_data_split_finished) { FLOG_INFO("skip to create sstable", K(ret), K(dst_tablet_id)); @@ -1528,7 +1517,7 @@ int ObTabletLobWriteDataTask::create_sstables( // fill empty minor sstable if scn not contiguous bool need_fill_empty_sstable = false; SCN end_scn; - if (OB_FAIL(ObTabletRebuildUtil::check_need_fill_empty_sstable(ctx_->ls_handle_, + if (OB_FAIL(ObTabletRebuildUtil::check_need_fill_empty_sstable(ctx_->ls_, is_minor_merge(write_sstable_ctx.merge_type_), write_sstable_ctx.table_key_, dst_tablet_id, @@ -1554,8 +1543,7 @@ int ObTabletLobWriteDataTask::create_sstables( if (batch_sstables_handle.at(i).empty() && !is_major_merge_type(merge_type)) { FLOG_INFO("already built, skip to update table store", K(ret), "tablet_id", ctx_->new_lob_tablet_ids_.at(i)); } else if (OB_FAIL(ObTabletSplitMergeTask::update_table_store_with_batch_tables( - ctx_->ls_rebuild_seq_, - ctx_->ls_handle_, + ctx_->ls_, ctx_->lob_meta_tablet_handle_, ctx_->new_lob_tablet_ids_.at(i), batch_sstables_handle.at(i), @@ -1570,7 +1558,7 @@ int ObTabletLobWriteDataTask::create_sstables( ObTablesHandleArray mds_sstables_handle; if (OB_FAIL(ObTabletSplitUtil::build_mds_sstable( build_mds_arena, - ctx_->ls_handle_, + ctx_->ls_, ctx_->lob_meta_tablet_handle_, ctx_->new_lob_tablet_ids_.at(i), mds_table_handle))) { @@ -1580,8 +1568,7 @@ int ObTabletLobWriteDataTask::create_sstables( } else if (OB_FAIL(mds_sstables_handle.add_table(mds_table_handle))) { LOG_WARN("add table failed", K(ret)); } else if (OB_FAIL(ObTabletSplitMergeTask::update_table_store_with_batch_tables( - ctx_->ls_rebuild_seq_, - ctx_->ls_handle_, + ctx_->ls_, ctx_->lob_meta_tablet_handle_, ctx_->new_lob_tablet_ids_.at(i), mds_sstables_handle, @@ -1971,7 +1958,6 @@ int ObTabletLobSplitUtil::process_write_split_start_log_request( int ret = OB_SUCCESS; ObLobSplitParam lob_split_param; ObTabletSplitParam data_split_param; - const share::ObLSID &ls_id = arg.ls_id_; const bool is_start_request = true; const bool is_lob_tablet = arg.lob_col_idxs_.count() > 0; if (OB_UNLIKELY(!arg.is_valid())) { @@ -1980,13 +1966,13 @@ int ObTabletLobSplitUtil::process_write_split_start_log_request( } else if (is_lob_tablet) { if (OB_FAIL(lob_split_param.init(arg))) { LOG_WARN("init param failed", K(ret)); - } else if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, ls_id, &lob_split_param, scn))) { + } else if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, &lob_split_param, scn))) { LOG_WARN("write split log failed", K(ret)); } } else { if (OB_FAIL(data_split_param.init(arg))) { LOG_WARN("init param failed", K(ret)); - } else if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, ls_id, &data_split_param, scn))) { + } else if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, &data_split_param, scn))) { LOG_WARN("write split log failed", K(ret)); } } @@ -2002,7 +1988,6 @@ int ObTabletLobSplitUtil::process_tablet_split_request( void *request_res) { int ret = OB_SUCCESS; - share::ObLSID ls_id; ObLobSplitParam lob_split_param; ObTabletSplitParam data_split_param; share::ObIDagInitParam *dag_param = nullptr; @@ -2021,7 +2006,6 @@ int ObTabletLobSplitUtil::process_tablet_split_request( } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_and_get_lob_tablet_split_dag(lob_split_param, lob_split_dag))) { LOG_WARN("failed to schedule dag", K(ret)); } - ls_id = lob_split_param.ls_id_; dag_param = &lob_split_param; stored_dag = lob_split_dag; } else { @@ -2033,14 +2017,13 @@ int ObTabletLobSplitUtil::process_tablet_split_request( } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_and_get_tablet_split_dag(data_split_param, data_split_dag))) { LOG_WARN("failed to schedule dag", K(ret)); } - ls_id = data_split_param.ls_id_; dag_param = &data_split_param; stored_dag = data_split_dag; } if (OB_SUCC(ret) && !is_start_request) { share::SCN unused_finish_scn = SCN::min_scn(); - if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, ls_id, dag_param, unused_finish_scn))) { + if (OB_FAIL(ObTabletLobSplitUtil::write_split_log(is_lob_tablet, is_start_request, dag_param, unused_finish_scn))) { LOG_WARN("write split log failed", K(ret)); } } @@ -2094,7 +2077,6 @@ int ObTabletLobSplitUtil::process_tablet_split_request( int ObTabletLobSplitUtil::write_split_log( const bool is_lob_tablet, const bool is_start_request, - const share::ObLSID &ls_id, const share::ObIDagInitParam *input_param, SCN &scn) { @@ -2102,9 +2084,9 @@ int ObTabletLobSplitUtil::write_split_log( ObTabletSplitStartLog split_start_log; ObTabletSplitFinishLog split_finish_log; ObTabletSplitInfo &split_info = is_start_request ? split_start_log.basic_info_ : split_finish_log.basic_info_; - if (OB_UNLIKELY(!ls_id.is_valid() || nullptr == input_param)) { + if (OB_UNLIKELY(nullptr == input_param)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), KP(input_param)); + LOG_WARN("invalid args", K(ret), KP(input_param)); } else if (is_lob_tablet) { const ObLobSplitParam *param = static_cast(input_param); if (OB_UNLIKELY(!param->is_valid())) { @@ -2155,13 +2137,13 @@ int ObTabletLobSplitUtil::write_split_log( if (OB_SUCC(ret)) { if (is_start_request) { if (OB_FAIL(ObDDLRedoLogWriter::write_auto_split_log( - ls_id, ObDDLClogType::DDL_TABLET_SPLIT_START_LOG, + ObDDLClogType::DDL_TABLET_SPLIT_START_LOG, logservice::ObReplayBarrierType::PRE_BARRIER, split_start_log, scn))) { LOG_WARN("write tablet split start log failed", K(ret), K(split_start_log)); } } else { if (OB_FAIL(ObDDLRedoLogWriter::write_auto_split_log( - ls_id, ObDDLClogType::DDL_TABLET_SPLIT_FINISH_LOG, + ObDDLClogType::DDL_TABLET_SPLIT_FINISH_LOG, logservice::ObReplayBarrierType::STRICT_BARRIER, split_finish_log, scn))) { LOG_WARN("write tablet split finish log failed", K(ret), K(split_finish_log)); } diff --git a/src/storage/ddl/ob_tablet_lob_split_task.h b/src/storage/ddl/ob_tablet_lob_split_task.h index ff8396fe2..2464d1db4 100644 --- a/src/storage/ddl/ob_tablet_lob_split_task.h +++ b/src/storage/ddl/ob_tablet_lob_split_task.h @@ -30,6 +30,8 @@ namespace oceanbase namespace storage { +class ObLS; + struct ObIStoreRowIteratorPtr { public: @@ -90,7 +92,7 @@ struct ObLobSplitParam : public share::ObIDagInitParam { public: ObLobSplitParam() : - rowkey_allocator_("LobSplitRowkey", OB_MALLOC_NORMAL_BLOCK_SIZE), ls_id_(share::ObLSID::INVALID_LS_ID), + rowkey_allocator_("LobSplitRowkey", OB_MALLOC_NORMAL_BLOCK_SIZE), ori_lob_meta_tablet_id_(ObTabletID::INVALID_TABLET_ID), new_lob_tablet_ids_(), schema_version_(0), data_format_version_(0), parallelism_(0), compaction_scn_(), @@ -104,14 +106,14 @@ struct ObLobSplitParam : public share::ObIDagInitParam int init(const obcall::ObDDLBuildSingleReplicaRequestArg &arg); int init(const obcall::ObTabletSplitArg &arg); bool is_valid() const { - return ls_id_.is_valid() && ori_lob_meta_tablet_id_.is_valid() + return ori_lob_meta_tablet_id_.is_valid() && new_lob_tablet_ids_.count() > 0 && schema_version_ > 0 && data_format_version_ > 0 && parallelism_ > 0 && compaction_scn_ > 0 && compat_mode_ != lib::Worker::CompatMode::INVALID && task_id_ > 0 && OB_INVALID_ID != source_table_id_ && OB_INVALID_ID != dest_schema_id_; } int assign(const ObLobSplitParam &other); - TO_STRING_KV(K_(ls_id), K_(ori_lob_meta_tablet_id), + TO_STRING_KV(K_(ori_lob_meta_tablet_id), K_(new_lob_tablet_ids), K_(schema_version), K_(data_format_version), K_(parallelism), K_(compaction_scn), K_(compat_mode), K_(task_id), K_(source_table_id), K_(dest_schema_id), K_(lob_col_idxs), @@ -120,7 +122,6 @@ struct ObLobSplitParam : public share::ObIDagInitParam private: common::ObArenaAllocator rowkey_allocator_; // for DatumRowkey. public: - share::ObLSID ls_id_; ObTabletID ori_lob_meta_tablet_id_; ObArray new_lob_tablet_ids_; int64_t schema_version_; @@ -143,14 +144,14 @@ struct ObLobSplitContext final ObLobSplitContext() : range_allocator_("LobSplitRange", OB_MALLOC_NORMAL_BLOCK_SIZE), is_inited_(false), data_ret_(0), is_lob_piece_(false), - ls_handle_(), main_tablet_id_(ObTabletID::INVALID_TABLET_ID), + ls_(nullptr), main_tablet_id_(ObTabletID::INVALID_TABLET_ID), allocator_(common::ObModIds::OB_LOB_ACCESS_BUFFER, OB_MALLOC_NORMAL_BLOCK_SIZE), main_tablet_handle_(), lob_meta_tablet_handle_(), m_allocator_(allocator_), new_main_tablet_ids_(OB_MALLOC_NORMAL_BLOCK_SIZE, m_allocator_), new_lob_tablet_ids_(OB_MALLOC_NORMAL_BLOCK_SIZE, m_allocator_), cmp_ret_(0), comparer_(cmp_ret_), total_map_(nullptr), sub_maps_(), skipped_split_major_keys_(), - row_inserted_(0), physical_row_count_(0), ls_rebuild_seq_(-1) + row_inserted_(0), physical_row_count_(0) {} ~ObLobSplitContext() { destroy(); } int init(const ObLobSplitParam& param); @@ -159,11 +160,10 @@ struct ObLobSplitContext final inline bool is_valid() const { return is_inited_; } void destroy(); TO_STRING_KV(K_(is_inited), K_(data_ret), K_(is_lob_piece), - K_(ls_handle), K_(main_tablet_id), K_(main_tablet_handle), + K_(ls), K_(main_tablet_id), K_(main_tablet_handle), K_(lob_meta_tablet_handle), K_(new_main_tablet_ids), K_(new_lob_tablet_ids), KPC_(total_map), K_(sub_maps), K_(main_table_ranges), - K_(skipped_split_major_keys), K_(row_inserted), K_(physical_row_count), - K_(ls_rebuild_seq)); + K_(skipped_split_major_keys), K_(row_inserted), K_(physical_row_count)); private: common::ObArenaAllocator range_allocator_; // for datum range. public: @@ -171,7 +171,7 @@ struct ObLobSplitContext final bool is_inited_; int data_ret_; bool is_lob_piece_; - ObLSHandle ls_handle_; + ObLS *ls_; ObTabletID main_tablet_id_; common::ObArenaAllocator allocator_; ObTabletHandle main_tablet_handle_; @@ -189,7 +189,6 @@ struct ObLobSplitContext final ObArray skipped_split_major_keys_; int64_t row_inserted_; int64_t physical_row_count_; - int64_t ls_rebuild_seq_; }; @@ -378,7 +377,6 @@ class ObTabletLobSplitUtil final static int write_split_log( const bool is_lob_tablet, const bool is_start_request, - const share::ObLSID &ls_id, const share::ObIDagInitParam *input_param, share::SCN &scn); }; diff --git a/src/storage/ddl/ob_tablet_rebuild_util.cpp b/src/storage/ddl/ob_tablet_rebuild_util.cpp index ee21eddd5..63684d48d 100644 --- a/src/storage/ddl/ob_tablet_rebuild_util.cpp +++ b/src/storage/ddl/ob_tablet_rebuild_util.cpp @@ -100,7 +100,7 @@ int ObTabletRebuildUtil::get_clipped_storage_schema_on_demand( } int ObTabletRebuildUtil::check_need_fill_empty_sstable( - ObLSHandle &ls_handle, + ObLS *ls, const bool is_minor_sstable, const ObITable::TableKey &table_key, const common::ObTabletID &dst_tablet_id, @@ -114,7 +114,7 @@ int ObTabletRebuildUtil::check_need_fill_empty_sstable( end_scn.reset(); if (is_minor_sstable) { - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, dst_tablet_id, dst_tablet_handle, + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, dst_tablet_id, dst_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet failed", K(ret), K(dst_tablet_id)); } else if (OB_FAIL(dst_tablet_handle.get_obj()->fetch_table_store(table_store_handle))) { diff --git a/src/storage/ddl/ob_tablet_rebuild_util.h b/src/storage/ddl/ob_tablet_rebuild_util.h index 82a040758..dbc2452e1 100644 --- a/src/storage/ddl/ob_tablet_rebuild_util.h +++ b/src/storage/ddl/ob_tablet_rebuild_util.h @@ -31,7 +31,6 @@ class ObIAllocator; namespace share { class SCN; -class ObLSID; } namespace blocksstable { @@ -40,7 +39,7 @@ class ObSSTable; } namespace storage { -class ObLSHandle; +class ObLS; class ObStorageSchema; class ObTabletCreateSSTableParam; @@ -119,7 +118,7 @@ class ObTabletRebuildUtil final const ObStorageSchema *&storage_schema); static int check_need_fill_empty_sstable( - ObLSHandle &ls_handle, + ObLS *ls, const bool is_minor_sstable, const ObITable::TableKey &table_key, const common::ObTabletID &dst_tablet_id, @@ -139,4 +138,3 @@ class ObTabletRebuildUtil final #endif // OCEANBASE_STORAGE_DDL_OB_TABLET_REBUILD_UTIL_H_ - diff --git a/src/storage/ddl/ob_tablet_slice_writer.cpp b/src/storage/ddl/ob_tablet_slice_writer.cpp index b9d39ff57..19a953521 100644 --- a/src/storage/ddl/ob_tablet_slice_writer.cpp +++ b/src/storage/ddl/ob_tablet_slice_writer.cpp @@ -492,7 +492,7 @@ int ObHeapRsSliceWriter::close() LOG_WARN("close row slice writer failed", K(ret)); } else if (OB_FAIL(heap_info_.get_last_autoinc_val(last_autoinc_val))) { LOG_WARN("get last autoinc val failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(writer_param_.ls_id_, tablet_id_, last_autoinc_val))) { + } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(tablet_id_, last_autoinc_val))) { LOG_WARN("set tablet autoinc seq failed", K(ret)); } return ret; @@ -1256,7 +1256,7 @@ int ObHeapBatchSliceWriter::close() LOG_WARN("close row slice writer failed", K(ret)); } else if (OB_FAIL(heap_info_.get_last_autoinc_val(last_autoinc_val))) { LOG_WARN("get last autoinc val failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(writer_param_.ls_id_, tablet_id_, last_autoinc_val))) { + } else if (OB_FAIL(ObDDLUtil::set_tablet_autoinc_seq(tablet_id_, last_autoinc_val))) { LOG_WARN("set tablet autoinc seq failed", K(ret)); } return ret; diff --git a/src/storage/ddl/ob_tablet_split_task.cpp b/src/storage/ddl/ob_tablet_split_task.cpp index a27bf97b5..cf67341ac 100644 --- a/src/storage/ddl/ob_tablet_split_task.cpp +++ b/src/storage/ddl/ob_tablet_split_task.cpp @@ -20,6 +20,7 @@ #include "share/rc/ob_module_provider.h" #include "rootserver/ob_root_service.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/ob_partition_range_spliter.h" #include "storage/tablet/ob_mds_scan_param_helper.h" #include "storage/tablet/ob_tablet_mds_table_mini_merger.h" @@ -44,7 +45,7 @@ namespace storage ObTabletSplitParam::ObTabletSplitParam() : rowkey_allocator_("SplitRangePar", OB_MALLOC_NORMAL_BLOCK_SIZE), - is_inited_(false), ls_id_(), table_id_(OB_INVALID_ID), + is_inited_(false), table_id_(OB_INVALID_ID), schema_version_(0), task_id_(0), source_tablet_id_(), dest_tablets_id_(), compaction_scn_(0), user_parallelism_(0), compat_mode_(lib::Worker::CompatMode::INVALID), data_format_version_(0), @@ -61,7 +62,7 @@ ObTabletSplitParam::~ObTabletSplitParam() bool ObTabletSplitParam::is_valid() const { - return ls_id_.is_valid() && OB_INVALID_ID != table_id_ + return OB_INVALID_ID != table_id_ && schema_version_ > 0 && task_id_ > 0 && source_tablet_id_.is_valid() && dest_tablets_id_.count() > 0 && compaction_scn_ > 0 && user_parallelism_ > 0 && compat_mode_ != lib::Worker::CompatMode::INVALID && data_format_version_ > 0 @@ -91,7 +92,6 @@ int ObTabletSplitParam::init( } } if (OB_SUCC(ret)) { - ls_id_ = param.ls_id_; table_id_ = param.table_id_; schema_version_ = param.schema_version_; task_id_ = param.task_id_; @@ -116,7 +116,6 @@ int ObTabletSplitParam::init(const obcall::ObDDLBuildSingleReplicaRequestArg &ar ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(arg)); } else { - ls_id_ = arg.ls_id_; table_id_ = arg.dest_schema_id_; schema_version_ = arg.schema_version_; task_id_ = arg.task_id_; @@ -129,7 +128,7 @@ int ObTabletSplitParam::init(const obcall::ObDDLBuildSingleReplicaRequestArg &ar min_split_start_scn_ = arg.min_split_start_scn_; if (OB_FAIL(parallel_datum_rowkey_list_.assign(arg.parallel_datum_rowkey_list_))) { // shallow cpy. LOG_WARN("convert to range failed", K(ret), "parall_info", arg.parallel_datum_rowkey_list_); - } else if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ls_id_, source_tablet_id_, dest_tablets_id_, compat_mode_))) { + } else if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(source_tablet_id_, dest_tablets_id_, compat_mode_))) { LOG_WARN("get split dest tablets failed", K(ret), K(arg)); } } @@ -143,7 +142,6 @@ int ObTabletSplitParam::init(const obcall::ObTabletSplitArg &arg) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(arg)); } else { - ls_id_ = arg.ls_id_; table_id_ = arg.table_id_; schema_version_ = arg.schema_version_; task_id_ = arg.task_id_; @@ -155,7 +153,7 @@ int ObTabletSplitParam::init(const obcall::ObTabletSplitArg &arg) can_reuse_macro_block_ = arg.can_reuse_macro_block_; min_split_start_scn_ = arg.min_split_start_scn_; ObArray unused_tablet_ids; - if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(ls_id_, source_tablet_id_, unused_tablet_ids, compat_mode_))) { + if (OB_FAIL(ObTabletSplitUtil::get_split_dest_tablets_info(source_tablet_id_, unused_tablet_ids, compat_mode_))) { LOG_WARN("get split dest tablets failed", K(ret), K(arg)); } else if (OB_FAIL(parallel_datum_rowkey_list_.assign(arg.parallel_datum_rowkey_list_))) { // shallow cpy. LOG_WARN("convert to range failed", K(ret), "parall_info", arg.parallel_datum_rowkey_list_); @@ -168,20 +166,18 @@ int ObTabletSplitParam::init(const obcall::ObTabletSplitArg &arg) ObTabletSplitCtx::ObTabletSplitCtx() : range_allocator_("SplitRangeCtx", OB_MALLOC_NORMAL_BLOCK_SIZE), - is_inited_(false), complement_data_ret_(OB_SUCCESS), ls_handle_(), tablet_handle_(), + is_inited_(false), complement_data_ret_(OB_SUCCESS), ls_(nullptr), tablet_handle_(), index_builder_map_(), clipped_schemas_map_(), allocator_("SplitCtx", OB_MALLOC_NORMAL_BLOCK_SIZE), - skipped_split_major_keys_(), row_inserted_(0), physical_row_count_(0), - ls_rebuild_seq_(-1) + skipped_split_major_keys_(), row_inserted_(0), physical_row_count_(0) { } ObTabletSplitCtx::~ObTabletSplitCtx() { is_inited_ = false; - ls_rebuild_seq_ = -1; complement_data_ret_ = OB_SUCCESS; - ls_handle_.reset(); + ls_ = nullptr; tablet_handle_.reset(); (void)ObTabletRebuildUtil::destroy_value_ptr_map(allocator_, index_builder_map_); (void)ObTabletRebuildUtil::destroy_value_ptr_map(allocator_, clipped_schemas_map_); @@ -194,7 +190,7 @@ ObTabletSplitCtx::~ObTabletSplitCtx() bool ObTabletSplitCtx::is_valid() const { - return is_inited_ && ls_handle_.is_valid() && tablet_handle_.is_valid(); + return is_inited_ && OB_NOT_NULL(ls_) && tablet_handle_.is_valid(); } int ObTabletSplitCtx::init(const ObTabletSplitParam ¶m) @@ -207,12 +203,12 @@ int ObTabletSplitCtx::init(const ObTabletSplitParam ¶m) } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(param)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle_, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_))) { LOG_WARN("failed to get log stream", K(ret), K(param)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, param.source_tablet_id_, tablet_handle_, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet failed", K(ret)); - } else if (OB_FAIL(ObTabletSplitUtil::check_satisfy_split_condition(ls_handle_, tablet_handle_, param.dest_tablets_id_, param.compaction_scn_, param.min_split_start_scn_))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_satisfy_split_condition(ls_, tablet_handle_, param.dest_tablets_id_, param.compaction_scn_, param.min_split_start_scn_))) { if (OB_NEED_RETRY == ret) { if (REACH_COUNT_INTERVAL(1000L)) { LOG_WARN("wait to satisfy the data split condition", K(ret), K(param)); @@ -223,14 +219,13 @@ int ObTabletSplitCtx::init(const ObTabletSplitParam ¶m) } else if (OB_FAIL(tablet_handle_.get_obj()->get_all_tables(table_store_iterator_))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(ObTabletSplitUtil::check_sstables_skip_data_split( - ls_handle_, table_store_iterator_, param.dest_tablets_id_, OB_INVALID_VERSION/*lob_major_snapshot*/, skipped_split_major_keys_))) { + ls_, table_store_iterator_, param.dest_tablets_id_, OB_INVALID_VERSION/*lob_major_snapshot*/, skipped_split_major_keys_))) { LOG_WARN("check sstables skip data split failed", K(ret)); } else if (OB_FAIL(ObTabletSplitUtil::convert_rowkey_to_range(range_allocator_, param.parallel_datum_rowkey_list_, data_split_ranges_))) { LOG_WARN("convert to range failed", K(ret), K(param)); } if (OB_SUCC(ret)) { - ls_rebuild_seq_ = ls_handle_.get_ls()->get_rebuild_seq(); complement_data_ret_ = OB_SUCCESS; is_inited_ = true; } @@ -286,10 +281,10 @@ int ObTabletSplitCtx::prepare_index_builder( param.source_tablet_id_, *sstable, *storage_schema, true/*try_create*/, clipped_storage_schema))) { LOG_WARN("get storage schema via sstable failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle_, param.dest_tablets_id_.at(j), tablet_handle))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_, param.dest_tablets_id_.at(j), tablet_handle))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_FAIL(data_desc.init( - true/*is_ddl*/, *clipped_storage_schema, param.ls_id_, + true/*is_ddl*/, *clipped_storage_schema, dst_tablet_id, merge_type, snapshot_version, param.data_format_version_, tablet_handle.get_obj()->get_tablet_meta().micro_index_clustered_, 0/*concurrent_cnt*/, @@ -345,7 +340,6 @@ int ObTabletSplitDag::calc_total_row_count() { ret = OB_INIT_TWICE; LOG_WARN("has calculated the row_count", K(ret), K(context_.physical_row_count_)); } else if (OB_FAIL(ObDDLUtil::get_tablet_physical_row_cnt( - param_.ls_id_, param_.source_tablet_id_, true, // calc_sstable = true; false, // calc_memtable = false; because memtable has been frozen. @@ -478,8 +472,7 @@ uint64_t ObTabletSplitDag::hash() const ret = OB_ERR_SYS; LOG_ERROR("invalid argument", K(ret), K(is_inited_), K(param_)); } else { - hash_val = param_.ls_id_.hash() - + param_.table_id_ + param_.schema_version_ + hash_val = param_.table_id_ + param_.schema_version_ + param_.source_tablet_id_.hash() + ObDagType::DAG_TYPE_TABLET_SPLIT; } return hash_val; @@ -498,7 +491,6 @@ bool ObTabletSplitDag::operator==(const ObIDag &other) const LOG_WARN("invalid argument", K(ret), K(param_), K(dag.param_)); } else { is_equal = true - && param_.ls_id_ == dag.param_.ls_id_ && param_.schema_version_ == dag.param_.schema_version_ && param_.source_tablet_id_ == dag.param_.source_tablet_id_; } @@ -513,7 +505,7 @@ int ObTabletSplitDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, ret = OB_NOT_INIT; LOG_WARN("ObComplementDataDag has not been initialized", K(ret)); } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), - static_cast(param_.ls_id_.id()), static_cast(param_.source_tablet_id_.id())))) { + static_cast(param_.source_tablet_id_.id())))) { LOG_WARN("failed to fill info param", K(ret)); } return ret; @@ -530,16 +522,16 @@ int ObTabletSplitDag::fill_dag_key(char *buf, const int64_t buf_len) const LOG_WARN("invalid params", K(ret), K(param_)); } else if (param_.can_reuse_macro_block_) { if (OB_FAIL(databuff_printf(buf, buf_len, - "Reuse macro block split: src_tablet_id=%ld, parallelism=%ld, ls_id=%ld, schema_version=%ld", + "Reuse macro block split: src_tablet_id=%ld, parallelism=%ld, schema_version=%ld", param_.source_tablet_id_.id(), param_.user_parallelism_, - param_.ls_id_.id(), param_.schema_version_))) { + param_.schema_version_))) { LOG_WARN("fail to fill comment", K(ret), K(param_)); } } else { if (OB_FAIL(databuff_printf(buf, buf_len, - "Regen macro block split: src_tablet_id=%ld, parallelism=%ld, ls_id=%ld, schema_version=%ld", + "Regen macro block split: src_tablet_id=%ld, parallelism=%ld, schema_version=%ld", param_.source_tablet_id_.id(), param_.user_parallelism_, - param_.ls_id_.id(), param_.schema_version_))) { + param_.schema_version_))) { LOG_WARN("fail to fill comment", K(ret), K(param_)); } } @@ -560,14 +552,12 @@ int ObTabletSplitDag::report_replica_build_status() #ifdef ERRSIM if (OB_SUCC(ret)) { ret = OB_E(EventTable::EN_DDL_REPORT_REPLICA_BUILD_STATUS_FAIL) OB_SUCCESS; - LOG_INFO("report replica build status errsim", K(ret)); + LOG_INFO("report DDL build status errsim", K(ret)); } #endif ObAddr rs_addr = GCTX.self_addr(); - arg.ls_id_ = param_.ls_id_; - arg.dest_ls_id_ = param_.ls_id_; arg.tablet_id_ = param_.source_tablet_id_; arg.source_table_id_ = param_.table_id_; arg.dest_schema_id_ = context_.tablet_handle_.get_obj()->get_tablet_meta().data_tablet_id_.id(); // to fetch DDL Task. @@ -644,7 +634,7 @@ int ObTabletSplitPrepareTask::process() LOG_WARN("unexpected err", K(ret), KP(tmp_dag), KP(dag)); } else if (OB_SUCCESS != (context_->complement_data_ret_)) { LOG_WARN("complement data has already failed", KPC(context_)); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->dest_tablets_id_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->dest_tablets_id_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -827,7 +817,7 @@ int ObTabletSplitWriteTask::process() LOG_WARN("not init", K(ret)); } else if (OB_SUCCESS != (context_->complement_data_ret_)) { LOG_WARN("complement data has already failed", KPC(context_)); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->dest_tablets_id_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->dest_tablets_id_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -884,14 +874,14 @@ int ObTabletSplitWriteTask::prepare_macro_block_writer( ObIArray ¯o_block_writer_arr) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObMacroDataSeq macro_start_seq(0); ObSSTableMetaHandle meta_handle; meta_handle.reset(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param_->ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("get ls failed", K(ret), KPC(param_)); } else if (OB_FAIL(sstable_->get_meta(meta_handle))) { LOG_WARN("get sstable meta failed", K(ret)); @@ -921,10 +911,9 @@ int ObTabletSplitWriteTask::prepare_macro_block_writer( sstable_->get_snapshot_version() : sstable_->get_end_scn().get_val_for_tx(); if (OB_FAIL(context_->index_builder_map_.get_refactored(task_key, sst_idx_builder))) { LOG_WARN("get refactored failed", K(ret)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, dst_tablet_id, tablet_handle))) { + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, dst_tablet_id, tablet_handle))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_FAIL(data_desc.init(true/*is_ddl*/, clipped_storage_schema, - param_->ls_id_, dst_tablet_id, merge_type, snapshot_version, @@ -942,7 +931,7 @@ int ObTabletSplitWriteTask::prepare_macro_block_writer( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem failed", K(ret)); } else if (FALSE_IT(macro_block_writer = new (buf) ObMacroBlockWriter())) { - } else if (OB_FAIL(pre_warm_param.init(param_->ls_id_, dst_tablet_id))) { + } else if (OB_FAIL(pre_warm_param.init(dst_tablet_id))) { LOG_WARN("failed to init pre warm param", K(ret), K(dst_tablet_id), KPC(param_)); } else if (OB_FAIL(ObSSTablePrivateObjectCleaner::get_cleaner_from_data_store_desc( data_desc.get_desc(), @@ -989,7 +978,7 @@ int ObTabletSplitWriteTask::prepare_sorted_high_bound_pair( ObTabletHandle tablet_handle; ObTabletSplitMdsUserData data; ObDatumRowkey data_tablet_end_partkey; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(context_->ls_handle_, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(context_->ls_, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_split_data(data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S))) { LOG_WARN("failed to get split data", K(ret)); @@ -1203,7 +1192,7 @@ int ObTabletSplitWriteTask::process_rows_for_rewrite_task( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(dest_tablets_id), K(macro_block_writer_arr), K(query_range)); } else if (OB_FAIL(ObTabletSplitMdsHelper::prepare_calc_split_dst( - *context_->ls_handle_.get_ls(), + *context_->ls_, *context_->tablet_handle_.get_obj(), ObTimeUtility::current_time() + (1 + dest_tablets_id.count()) * ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, src_split_data, @@ -1339,7 +1328,7 @@ int ObTabletSplitMergeTask::process() LOG_WARN("unexpected err", K(ret), KP(tmp_dag), KP(dag)); } else if (OB_SUCCESS != (context_->complement_data_ret_)) { LOG_WARN("complement data has already failed", KPC(context_)); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, param_->dest_tablets_id_, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->dest_tablets_id_, is_data_split_finished))) { LOG_WARN("check all major exist failed", K(ret)); } else if (is_data_split_finished) { LOG_INFO("split task has alreay finished", KPC(param_)); @@ -1361,7 +1350,7 @@ int ObTabletSplitMergeTask::process() DEBUG_SYNC(AFTER_TABLET_SPLIT_MERGE_TASK); if (OB_NOT_NULL(dag) && OB_FAIL(dag->report_replica_build_status())) { // do not worry about the error code overrided here, which is useless. - LOG_WARN("report replica build status failed", K(ret), KPC(context_)); + LOG_WARN("report DDL build status failed", K(ret), KPC(context_)); } return ret; } @@ -1392,7 +1381,7 @@ int ObTabletSplitMergeTask::create_sstable( ObSEArray check_major_exist_tablets; if (OB_FAIL(check_major_exist_tablets.push_back(dest_tablet_id))) { LOG_WARN("push back failed", K(ret)); - } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(param_->ls_id_, check_major_exist_tablets, is_data_split_finished))) { + } else if (OB_FAIL(ObTabletSplitUtil::check_data_split_finished(check_major_exist_tablets, is_data_split_finished))) { LOG_WARN("check major exist failed", K(ret), K(check_major_exist_tablets), KPC(param_)); } else if (is_data_split_finished) { FLOG_INFO("skip to create sstable", K(ret), K(dest_tablet_id)); @@ -1428,7 +1417,7 @@ int ObTabletSplitMergeTask::create_sstable( bool need_fill_empty_sstable = false; SCN end_scn; if (OB_FAIL(ObTabletRebuildUtil::check_need_fill_empty_sstable( - context_->ls_handle_, src_sstable->is_minor_sstable(), src_sstable->get_key(), + context_->ls_, src_sstable->is_minor_sstable(), src_sstable->get_key(), dest_tablet_id, need_fill_empty_sstable, end_scn))) { LOG_WARN("failed to check need fill", K(ret)); } else if (need_fill_empty_sstable) { @@ -1453,8 +1442,7 @@ int ObTabletSplitMergeTask::create_sstable( if (OB_SUCC(ret) && (src_table_cnt > 0 || is_major_merge_type(merge_type))) { // empty major result should also need to swap tablet, to update data_split_status and restore status. if (OB_FAIL(ObTabletSplitMergeTask::update_table_store_with_batch_tables( - context_->ls_rebuild_seq_, - context_->ls_handle_, + context_->ls_, context_->tablet_handle_, dest_tablet_id, batch_sstables_handle, @@ -1470,7 +1458,7 @@ int ObTabletSplitMergeTask::create_sstable( batch_sstables_handle.reset(); if (OB_FAIL(ObTabletSplitUtil::build_mds_sstable( build_mds_arena, - context_->ls_handle_, + context_->ls_, context_->tablet_handle_, dest_tablet_id, table_handle))) { @@ -1481,8 +1469,7 @@ int ObTabletSplitMergeTask::create_sstable( } else if (OB_FAIL(batch_sstables_handle.add_table(table_handle))) { LOG_WARN("add table failed", K(ret)); } else if (OB_FAIL(ObTabletSplitMergeTask::update_table_store_with_batch_tables( - context_->ls_rebuild_seq_, - context_->ls_handle_, + context_->ls_, context_->tablet_handle_, dest_tablet_id, batch_sstables_handle, @@ -1529,8 +1516,7 @@ int ObTabletSplitMergeTask::build_create_sstable_param( } int ObTabletSplitMergeTask::update_table_store_with_batch_tables( - const int64_t ls_rebuild_seq, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &src_tablet_handle, const ObTabletID &dst_tablet_id, const ObTablesHandleArray &tables_handle, @@ -1551,13 +1537,12 @@ int ObTabletSplitMergeTask::update_table_store_with_batch_tables( } #endif if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(ls_rebuild_seq == -1 - || !ls_handle.is_valid() + } else if (OB_UNLIKELY(OB_ISNULL(ls) || !src_tablet_handle.is_valid() || !dst_tablet_id.is_valid() || !is_valid_merge_type(merge_type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_rebuild_seq), K(ls_handle), K(src_tablet_handle), + LOG_WARN("invalid arg", K(ret), K(ls), K(src_tablet_handle), K(dst_tablet_id), K(merge_type)); } else if (OB_FAIL(tables_handle.get_tables(batch_tables))) { LOG_WARN("get batch sstables failed", K(ret)); @@ -1601,13 +1586,11 @@ int ObTabletSplitMergeTask::update_table_store_with_batch_tables( } if (OB_SUCC(ret)) { - const share::ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); param.tablet_split_param_.snapshot_version_ = src_tablet_handle.get_obj()->get_tablet_meta().snapshot_version_; param.tablet_split_param_.multi_version_start_ = src_tablet_handle.get_obj()->get_multi_version_start(); param.tablet_split_param_.merge_type_ = merge_type; - param.rebuild_seq_ = ls_rebuild_seq; // old rebuild seq. param.release_mds_scn_.set_min(); - if (OB_FAIL(ls_handle.get_ls()->build_tablet_with_batch_tables(dst_tablet_id, param))) { + if (OB_FAIL(ls->build_tablet_with_batch_tables(dst_tablet_id, param))) { LOG_WARN("failed to update tablet table store", K(ret), K(dst_tablet_id), K(param)); } FLOG_INFO("update batch sstables", K(ret), K(dst_tablet_id), K(batch_tables), K(param)); @@ -1641,7 +1624,6 @@ int ObRowScan::construct_access_param( int ret = OB_SUCCESS; ObTabletTableIterator table_iter; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); @@ -1660,7 +1642,6 @@ int ObRowScan::construct_access_param( // construct version range and ctx int ObRowScan::construct_access_ctx( - const share::ObLSID &ls_id, const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -1680,15 +1661,11 @@ int ObRowScan::construct_access_ctx( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id)); - } else if (OB_FAIL(ctx_.init_for_read(ls_id, - tablet_id, + } else if (OB_FAIL(ctx_.init_for_read(tablet_id, INT64_MAX, -1, SCN::max_scn()))) { - LOG_WARN("fail to init store ctx", K(ret), K(ls_id)); + LOG_WARN("fail to init store ctx", K(ret)); } else if (OB_FAIL(access_ctx_.init(query_flag, ctx_, allocator_, @@ -1744,7 +1721,7 @@ int ObRowScan::init( LOG_WARN("invalid args", K(ret), K(param), K(sstable)); } else if (OB_FAIL(construct_access_param(param))) { LOG_WARN("construct access param failed", K(ret), K(param)); - } else if (OB_FAIL(construct_access_ctx(param.src_tablet_.get_tablet_meta().ls_id_, param.src_tablet_.get_tablet_meta().tablet_id_))) { + } else if (OB_FAIL(construct_access_ctx(param.src_tablet_.get_tablet_meta().tablet_id_))) { LOG_WARN("construct access param failed", K(ret), K(param)); } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSSTableRowWholeScanner)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1786,7 +1763,7 @@ int ObRowScan::init( LOG_WARN("invalid args", K(ret), K(param), K(macro_desc), K(sstable)); } else if (OB_FAIL(construct_access_param(param))) { LOG_WARN("construct access param failed", K(ret), K(param)); - } else if (OB_FAIL(construct_access_ctx(param.src_tablet_.get_tablet_meta().ls_id_, param.src_tablet_.get_tablet_meta().tablet_id_))) { + } else if (OB_FAIL(construct_access_ctx(param.src_tablet_.get_tablet_meta().tablet_id_))) { LOG_WARN("construct access param failed", K(ret), K(param)); } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSSTableRowWholeScanner)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1877,7 +1854,6 @@ int ObSnapshotRowScan::init( LOG_WARN("init twice", K(ret)); } else { const ObTabletID &tablet_id = param.src_tablet_.get_tablet_meta().tablet_id_; - const ObLSID &ls_id = param.src_tablet_.get_tablet_meta().ls_id_; ObQueryFlag query_flag(ObQueryFlag::Forward, false, /* daily merge*/ true, /* use *optimize */ @@ -1909,7 +1885,7 @@ int ObSnapshotRowScan::init( } else if (OB_FALSE_IT(write_row_.row_flag_.set_flag(ObDmlFlag::DF_INSERT))) { } else if (OB_FAIL(construct_access_param(param.table_id_, tablet_id, read_info_))) { LOG_WARN("failed to init access param", K(ret), K(param)); - } else if (OB_FAIL(construct_range_ctx(query_flag, ls_id))) { + } else if (OB_FAIL(construct_range_ctx(query_flag))) { LOG_WARN("failed to init access ctx", K(ret)); } else if (OB_FAIL(construct_multiple_scan_merge(table_iter, range_))) { LOG_WARN("failed to init scan merge", K(ret), K(table_iter)); @@ -1952,8 +1928,7 @@ int ObSnapshotRowScan::construct_access_param( } int ObSnapshotRowScan::construct_range_ctx( - ObQueryFlag &query_flag, - const share::ObLSID &ls_id) + ObQueryFlag &query_flag) { int ret = OB_SUCCESS; common::ObVersionRange trans_version_range; @@ -1962,13 +1937,12 @@ int ObSnapshotRowScan::construct_range_ctx( trans_version_range.base_version_ = 0; SCN tmp_scn; if (OB_FAIL(tmp_scn.convert_for_tx(snapshot_version_))) { - LOG_WARN("convert fail", K(ret), K(ls_id), K_(snapshot_version)); - } else if (OB_FAIL(ctx_.init_for_read(ls_id, - access_param_.iter_param_.tablet_id_, + LOG_WARN("convert fail", K(ret), K_(snapshot_version)); + } else if (OB_FAIL(ctx_.init_for_read(access_param_.iter_param_.tablet_id_, INT64_MAX, -1, tmp_scn))) { - LOG_WARN("fail to init store ctx", K(ret), K(ls_id)); + LOG_WARN("fail to init store ctx", K(ret)); } else if (OB_FAIL(access_ctx_.init(query_flag, ctx_, allocator_, allocator_, trans_version_range))) { LOG_WARN("fail to init accesss ctx", K(ret)); } else if (OB_NOT_NULL(access_ctx_.lob_locator_helper_)) { @@ -2314,7 +2288,6 @@ int ObTabletSplitUtil::get_participants( int ObTabletSplitUtil::split_task_ranges( ObIAllocator &allocator, const share::ObDDLType ddl_type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t user_parallelism, const int64_t schema_tablet_size, @@ -2323,21 +2296,21 @@ int ObTabletSplitUtil::split_task_ranges( int ret = OB_SUCCESS; parallel_datum_rowkey_list.reset(); ObSEArray empty_skipped_keys; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTableStoreIterator table_store_iterator; ObSEArray store_ranges; ObSEArray tables; const bool is_table_restore = false; common::ObArenaAllocator tmp_arena("SplitRange", OB_MALLOC_NORMAL_BLOCK_SIZE); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("invalid args", K(ret), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->get_all_sstables(table_store_iterator))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(ObTabletSplitUtil::get_participants( @@ -2349,7 +2322,7 @@ int ObTabletSplitUtil::split_task_ranges( tmp_min_key.set_min_rowkey(); tmp_max_key.set_max_rowkey(); if (OB_FAIL(parallel_datum_rowkey_list.prepare_allocate(2))) { // min key and max key. - LOG_WARN("reserve failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("reserve failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tmp_min_key.deep_copy(parallel_datum_rowkey_list.at(0), allocator))) { LOG_WARN("failed to push min rowkey", K(ret)); } else if (OB_FAIL(tmp_max_key.deep_copy(parallel_datum_rowkey_list.at(1), allocator))) { @@ -2475,36 +2448,35 @@ int ObTabletSplitUtil::convert_datum_rowkey_to_range( // check whether the data tablet split finished. int ObTabletSplitUtil::check_data_split_finished( - const share::ObLSID &ls_id, const ObIArray &check_tablets_id, bool &is_finished) { int ret = OB_SUCCESS; is_finished = true; - ObLSHandle ls_handle; - if (OB_UNLIKELY(!ls_id.is_valid() || check_tablets_id.empty())) { + ObLS *ls = nullptr; + if (OB_UNLIKELY(check_tablets_id.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(check_tablets_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); + LOG_WARN("invalid arg", K(ret), K(check_tablets_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < check_tablets_id.count() && is_finished; i++) { const ObTabletID &tablet_id = check_tablets_id.at(i); ObTabletHandle tmp_tablet_handle; ObTabletMemberWrapper table_store_wrapper; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tmp_tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(nullptr == tmp_tablet_handle.get_obj())) { ret = OB_ERR_SYS; - LOG_WARN("tablet handle is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet handle is null", K(ret), K(tablet_id)); } else if (tmp_tablet_handle.get_obj()->get_tablet_meta().split_info_.is_data_incomplete()) { is_finished = false; } else if (OB_FAIL(tmp_tablet_handle.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_ISNULL(table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/))) { ret = OB_EAGAIN; - LOG_WARN("wait for migration to copy sstable finished", K(ret), K(ls_id), K(tablet_id), K(tmp_tablet_handle)); + LOG_WARN("wait for major sstable", K(ret), K(tablet_id), K(tmp_tablet_handle)); } } } @@ -2512,7 +2484,7 @@ int ObTabletSplitUtil::check_data_split_finished( } int ObTabletSplitUtil::check_satisfy_split_condition( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObArray &dest_tablets_id, const int64_t compaction_scn, @@ -2526,21 +2498,21 @@ int ObTabletSplitUtil::check_satisfy_split_condition( ObTabletMemberWrapper table_store_wrapper; bool is_tablet_status_need_to_split = false; share::SCN max_decided_scn; - if (OB_UNLIKELY(!ls_handle.is_valid() || !source_tablet_handle.is_valid() + if (OB_UNLIKELY(OB_ISNULL(ls) || !source_tablet_handle.is_valid() || dest_tablets_id.empty() || !min_split_start_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_handle), K(source_tablet_handle), K(dest_tablets_id), K(min_split_start_scn)); + LOG_WARN("invalid arg", K(ret), K(ls), K(source_tablet_handle), K(dest_tablets_id), K(min_split_start_scn)); } else if (OB_ISNULL(tablet = source_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret), K(source_tablet_handle)); - } else if (OB_FAIL(check_tablet_restore_status(dest_tablets_id, ls_handle, source_tablet_handle, is_tablet_status_need_to_split))) { + } else if (OB_FAIL(check_tablet_restore_status(dest_tablets_id, ls, source_tablet_handle, is_tablet_status_need_to_split))) { LOG_WARN("failed to check tablet status", K(ret), K(source_tablet_handle)); } else if (OB_UNLIKELY(!is_tablet_status_need_to_split)) { ret = OB_TABLET_STATUS_NO_NEED_TO_SPLIT; LOG_WARN("there is no need to split, because of the special restore status of src tablet or des tablets", K(ret), K(source_tablet_handle), K(dest_tablets_id)); - } else if (OB_FAIL(check_tablet_ha_status(ls_handle, source_tablet_handle, dest_tablets_id))) { + } else if (OB_FAIL(check_tablet_restore_state(ls, source_tablet_handle, dest_tablets_id))) { if (OB_NEED_RETRY != ret) { - LOG_WARN("check tablet ha status failed", K(ls_handle), K(source_tablet_handle), K(dest_tablets_id)); + LOG_WARN("check tablet ha status failed", K(ls), K(source_tablet_handle), K(dest_tablets_id)); } } else if (OB_FAIL(tablet->get_all_memtables_from_memtable_mgr(memtable_handles))) { LOG_WARN("failed to get_memtable_mgr for get all memtable", K(ret), KPC(tablet)); @@ -2549,17 +2521,15 @@ int ObTabletSplitUtil::check_satisfy_split_condition( if (REACH_COUNT_INTERVAL(1000L)) { LOG_INFO("should wait memtable dump", K(ret), "tablet_id", tablet->get_tablet_meta().tablet_id_, K(memtable_handles)); } - } else if (OB_FAIL(ls_handle.get_ls()->get_max_decided_scn(max_decided_scn))) { - LOG_WARN("get max decided log ts failed", K(ret), "ls_id", ls_handle.get_ls()->get_ls_id(), - "source_tablet_id", tablet->get_tablet_meta().tablet_id_); + } else if (OB_FAIL(ls->get_max_decided_scn(max_decided_scn))) { + LOG_WARN("get max decided log ts failed", K(ret), "source_tablet_id", tablet->get_tablet_meta().tablet_id_); if (OB_STATE_NOT_MATCH == ret) { ret = OB_NEED_RETRY; } } else if (SCN::plus(max_decided_scn, 1) < min_split_start_scn) { ret = OB_NEED_RETRY; if (REACH_COUNT_INTERVAL(1000L)) { - LOG_INFO("need wait max decided scn reach", K(ret), "ls_id", ls_handle.get_ls()->get_ls_id(), - "source_tablet_id", tablet->get_tablet_meta().tablet_id_, K(max_decided_scn), K(min_split_start_scn)); + LOG_INFO("need wait max decided scn reach", K(ret), "source_tablet_id", tablet->get_tablet_meta().tablet_id_, K(max_decided_scn), K(min_split_start_scn)); } } else { const ObMediumCompactionInfoList *medium_list = nullptr; @@ -2577,7 +2547,6 @@ int ObTabletSplitUtil::check_satisfy_split_condition( } int ObTabletSplitUtil::get_split_dest_tablets_info( - const share::ObLSID &ls_id, const ObTabletID &source_tablet_id, ObIArray &dest_tablets_id, lib::Worker::CompatMode &compat_mode) @@ -2586,25 +2555,25 @@ int ObTabletSplitUtil::get_split_dest_tablets_info( dest_tablets_id.reset(); compat_mode = lib::Worker::CompatMode::INVALID; ObTabletSplitMdsUserData data; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!ls_id.is_valid() || !source_tablet_id.is_valid())) { + if (OB_UNLIKELY(!source_tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_id), K(source_tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("invalid arg", K(ret), K(source_tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, source_tablet_id, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(source_tablet_id)); + LOG_WARN("get tablet handle failed", K(ret), K(source_tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_split_data(data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S))) { LOG_WARN("failed to get split data", K(ret)); } else if (OB_FAIL(data.get_split_dst_tablet_ids(dest_tablets_id))) { LOG_WARN("failed to get split dst tablet ids", K(ret)); } else if (OB_UNLIKELY(dest_tablets_id.count() < 2)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected mds data", K(ret), K(ls_id), K(source_tablet_id), K(dest_tablets_id)); + LOG_WARN("unexpected mds data", K(ret), K(source_tablet_id), K(dest_tablets_id)); } else { compat_mode = tablet_handle.get_obj()->get_tablet_meta().compat_mode_; } @@ -2616,15 +2585,15 @@ int ObTabletSplitUtil::check_medium_compaction_info_list_cnt( obcall::ObCheckMediumCompactionInfoListResult &result) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(arg)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("failed to get log stream", K(ret), K(arg)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, arg.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet handle failed", K(ret), K(arg)); } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { @@ -2672,23 +2641,18 @@ int ObTabletSplitUtil::check_medium_compaction_info_list_cnt( int ObTabletSplitUtil::check_tablet_restore_status( const ObIArray &dest_tablets_id, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, bool &is_tablet_status_need_to_split) { int ret = OB_SUCCESS; is_tablet_status_need_to_split = true; - ObTablet *source_tablet = nullptr; ObTabletRestoreStatus::STATUS source_restore_status = ObTabletRestoreStatus::STATUS::RESTORE_STATUS_MAX; - ObLS *ls = nullptr; - if (OB_UNLIKELY(dest_tablets_id.count() <= 0 || !ls_handle.is_valid() || !source_tablet_handle.is_valid())) { + if (OB_UNLIKELY(dest_tablets_id.count() <= 0 || OB_ISNULL(ls) || !source_tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(dest_tablets_id), K(source_tablet_handle), K(ls_handle)); - } else if (OB_ISNULL(source_tablet = source_tablet_handle.get_obj()) || OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_NULL_CHECK_ERROR; - LOG_WARN("unexpected nullptr", K(ret), KP(source_tablet), KP(ls)); - } else if (OB_FAIL(source_tablet->get_restore_status(source_restore_status))) { - LOG_WARN("failed to get restore status", K(ret), KP(source_tablet)); + LOG_WARN("invalid argument", K(ret), K(dest_tablets_id), K(source_tablet_handle), K(ls)); + } else if (OB_FAIL(source_tablet_handle.get_obj()->get_restore_status(source_restore_status))) { + LOG_WARN("failed to get restore status", K(ret), K(source_tablet_handle)); } else if (OB_UNLIKELY(ObTabletRestoreStatus::STATUS::UNDEFINED == source_restore_status || ObTabletRestoreStatus::STATUS::EMPTY == source_restore_status || ObTabletRestoreStatus::STATUS::REMOTE == source_restore_status)) { is_tablet_status_need_to_split = false; @@ -2700,7 +2664,7 @@ int ObTabletSplitUtil::check_tablet_restore_status( t_handle.reset(); des_restore_status = ObTabletRestoreStatus::STATUS::RESTORE_STATUS_MAX; const ObTabletID &t_id = dest_tablets_id.at(i); - if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, t_id, t_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { + if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, t_id, t_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { LOG_WARN("get tablet failed", K(ret), K(t_id)); } else if (OB_ISNULL(tablet = t_handle.get_obj())) { ret = OB_NULL_CHECK_ERROR; @@ -2733,7 +2697,7 @@ int ObTabletSplitUtil::check_tablet_restore_status( int ObTabletSplitUtil::build_mds_sstable( common::ObArenaAllocator &allocator, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObTabletID &dest_tablet_id, ObTableHandleV2 &mds_table_handle) @@ -2741,14 +2705,13 @@ int ObTabletSplitUtil::build_mds_sstable( int ret = OB_SUCCESS; mds_table_handle.reset(); ObTabletHandle dest_tablet_handle; - if (OB_UNLIKELY(!ls_handle.is_valid() || !source_tablet_handle.is_valid() || !dest_tablet_id.is_valid())) { + if (OB_UNLIKELY(OB_ISNULL(ls) || !source_tablet_handle.is_valid() || !dest_tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_handle), K(source_tablet_handle), K(dest_tablet_id)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + LOG_WARN("invalid arg", K(ret), K(ls), K(source_tablet_handle), K(dest_tablet_id)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, dest_tablet_id, dest_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet failed", K(ret), K(dest_tablet_id)); } else { - const share::ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); const ObTabletID &source_tablet_id = source_tablet_handle.get_obj()->get_tablet_id(); HEAP_VARS_4 ((compaction::ObTabletMergeDagParam, param), (compaction::ObTabletMergeCtx, tablet_merge_ctx, param, allocator), @@ -2759,30 +2722,28 @@ int ObTabletSplitUtil::build_mds_sstable( HEAP_VARS_2 ((ObTableScanParam, truncate_info_scan_param), (ObTabletTruncateInfoReader, truncate_info_reader)) { ObMdsReadInfoCollector unused_collector; - if (OB_FAIL(check_and_build_mds_sstable_merge_ctx(ls_handle, dest_tablet_handle, tablet_merge_ctx))) { - LOG_WARN("prepare medium mds merge ctx failed", K(ret), K(ls_handle), K(dest_tablet_id)); + if (OB_FAIL(check_and_build_mds_sstable_merge_ctx(ls, dest_tablet_handle, tablet_merge_ctx))) { + LOG_WARN("prepare medium mds merge ctx failed", K(ret), K(ls), K(dest_tablet_id)); } else if (tablet_merge_ctx.static_param_.scn_range_.end_scn_.is_base_scn()) { // = 1 - LOG_INFO("no need to build lost mds sstable again", K(ls_id), K(source_tablet_id), K(dest_tablet_id)); + LOG_INFO("no need to build lost mds sstable again", K(source_tablet_id), K(dest_tablet_id)); } else if (OB_FAIL(mds_mini_merger.init(tablet_merge_ctx, op))) { - LOG_WARN("fail to init mds mini merger", K(ret), K(tablet_merge_ctx), K(ls_id), K(dest_tablet_id)); + LOG_WARN("fail to init mds mini merger", K(ret), K(tablet_merge_ctx), K(dest_tablet_id)); } else if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, source_tablet_id, ObMdsScanParamHelper::get_whole_read_version_range(), unused_collector, medium_info_scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(source_tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(source_tablet_id)); } else if (OB_FAIL(medium_info_reader.init(*source_tablet_handle.get_obj(), medium_info_scan_param))) { LOG_WARN("failed to init medium info reader", K(ret)); } else if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, source_tablet_id, ObMdsScanParamHelper::get_whole_read_version_range(), unused_collector, truncate_info_scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(source_tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(source_tablet_id)); } else if (OB_FAIL(truncate_info_reader.init(*source_tablet_handle.get_obj(), truncate_info_scan_param))) { LOG_WARN("failed to init truncate info reader", K(ret)); } else { @@ -2797,7 +2758,7 @@ int ObTabletSplitUtil::build_mds_sstable( iter_arena.reuse(); if (OB_FAIL(truncate_info_reader.get_next_mds_kv(iter_arena, kv))) { if (OB_ITER_END != ret) { - LOG_WARN("iter medium mds failed", K(ret), K(ls_id), K(source_tablet_id)); + LOG_WARN("iter medium mds failed", K(ret), K(source_tablet_id)); } else { ret = OB_SUCCESS; break; @@ -2813,7 +2774,7 @@ int ObTabletSplitUtil::build_mds_sstable( } if (OB_SUCC(ret)) { if (!has_mds_row) { - LOG_INFO("no need to build mds sstable", K(ls_id), K(source_tablet_id), K(dest_tablet_id)); + LOG_INFO("no need to build mds sstable", K(source_tablet_id), K(dest_tablet_id)); } else if (OB_FAIL(op.finish())) { LOG_WARN("finish failed", K(ret)); } else if (OB_FAIL(mds_mini_merger.generate_mds_mini_sstable(allocator, mds_table_handle))) { @@ -2829,22 +2790,21 @@ int ObTabletSplitUtil::build_mds_sstable( } int ObTabletSplitUtil::check_and_build_mds_sstable_merge_ctx( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &dest_tablet_handle, compaction::ObTabletMergeCtx &tablet_merge_ctx) { int ret = OB_SUCCESS; ObLSService *ls_service = nullptr; share::SCN end_scn; - if (OB_UNLIKELY(!ls_handle.is_valid() || !dest_tablet_handle.is_valid())) { + if (OB_UNLIKELY(OB_ISNULL(ls) || !dest_tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_handle), K(dest_tablet_handle)); + LOG_WARN("invalid arg", K(ret), K(ls), K(dest_tablet_handle)); } else if (OB_FAIL(check_and_determine_mds_end_scn(dest_tablet_handle, end_scn))) { LOG_WARN("get mds sstable start scn failed", K(ret), K(dest_tablet_handle)); } else { compaction::ObStaticMergeParam &static_param = tablet_merge_ctx.static_param_; - static_param.ls_handle_ = ls_handle; - static_param.dag_param_.ls_id_ = ls_handle.get_ls()->get_ls_id(); + static_param.ls_ = ls; static_param.dag_param_.merge_type_ = compaction::ObMergeType::MDS_MINI_MERGE; static_param.dag_param_.tablet_id_ = dest_tablet_handle.get_obj()->get_tablet_id(); static_param.pre_warm_param_.type_ = ObPreWarmerType::MEM_PRE_WARM; @@ -2858,7 +2818,7 @@ int ObTabletSplitUtil::check_and_build_mds_sstable_merge_ctx( static_param.need_parallel_minor_merge_ = false; if (OB_FAIL(tablet_merge_ctx.init_tablet_merge_info())) { - LOG_WARN("failed to init tablet merge info", K(ret), K(ls_handle), K(dest_tablet_handle), K(tablet_merge_ctx)); + LOG_WARN("failed to init tablet merge info", K(ret), K(ls), K(dest_tablet_handle), K(tablet_merge_ctx)); } } return ret; @@ -2894,7 +2854,7 @@ int ObTabletSplitUtil::check_and_determine_mds_end_scn( } int ObTabletSplitUtil::check_sstables_skip_data_split( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTableStoreIterator &source_table_store_iter, const ObIArray &dest_tablets_id, const int64_t lob_major_snapshot/*OB_INVALID_VERSION for non lob tablets*/, @@ -2926,7 +2886,7 @@ int ObTabletSplitUtil::check_sstables_skip_data_split( const ObTabletID &this_tablet_id = dest_tablets_id.at(j); this_key.tablet_id_ = this_tablet_id; ObTabletMemberWrapper table_store_wrapper; - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, this_tablet_id, + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, this_tablet_id, tmp_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { LOG_WARN("get tablet handle failed", K(ret), K(this_tablet_id)); } else if (OB_UNLIKELY(nullptr == tmp_tablet_handle.get_obj())) { @@ -2953,18 +2913,17 @@ int ObTabletSplitUtil::check_sstables_skip_data_split( return ret; } -// For migration, wait data status COMPLETE. // For restore, wait restore status FULL/REMOTE. -int ObTabletSplitUtil::check_tablet_ha_status( - const ObLSHandle &ls_handle, +int ObTabletSplitUtil::check_tablet_restore_state( + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObIArray &dest_tablets_id) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_handle.is_valid() || !source_tablet_handle.is_valid() || dest_tablets_id.empty())) { + if (OB_UNLIKELY(OB_ISNULL(ls) || !source_tablet_handle.is_valid() || dest_tablets_id.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(source_tablet_handle), K(dest_tablets_id)); - } else if (!source_tablet_handle.get_obj()->get_tablet_meta().ha_status_.check_allow_read()) { + } else if (!source_tablet_handle.get_obj()->get_tablet_meta().restore_state_.check_allow_read()) { ret = OB_NEED_RETRY; if (REACH_COUNT_INTERVAL(1000L)) { LOG_INFO("should wait data complete", K(ret), @@ -2974,12 +2933,12 @@ int ObTabletSplitUtil::check_tablet_ha_status( for (int64_t i = 0; OB_SUCC(ret) && i < dest_tablets_id.count(); i++) { ObTabletHandle tmp_tablet_handle; const ObTabletID &tablet_id = dest_tablets_id.at(i); - if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tmp_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { + if ((OB_FAIL(ObDDLUtil::ddl_get_tablet(ls, tablet_id, tmp_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED)))) { LOG_WARN("get tablet failed", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(!tmp_tablet_handle.is_valid())) { ret = OB_ERR_SYS; LOG_WARN("invalid tablet", K(ret), K(tablet_id), K(tmp_tablet_handle)); - } else if (!tmp_tablet_handle.get_obj()->get_tablet_meta().ha_status_.check_allow_read()) { + } else if (!tmp_tablet_handle.get_obj()->get_tablet_meta().restore_state_.check_allow_read()) { ret = OB_NEED_RETRY; if (REACH_COUNT_INTERVAL(1000L)) { LOG_INFO("should wait data complete", K(ret), diff --git a/src/storage/ddl/ob_tablet_split_task.h b/src/storage/ddl/ob_tablet_split_task.h index e0c1d1ab0..e71e07f37 100644 --- a/src/storage/ddl/ob_tablet_split_task.h +++ b/src/storage/ddl/ob_tablet_split_task.h @@ -34,6 +34,7 @@ namespace oceanbase { namespace storage { +class ObLS; using common::ObTabletID; using ObSplitSSTableTaskKey = ObDdlSSTableTaskKey; @@ -48,7 +49,7 @@ struct ObTabletSplitParam : public share::ObIDagInitParam int init(const ObTabletSplitParam ¶m); int init(const obcall::ObDDLBuildSingleReplicaRequestArg &arg); int init(const obcall::ObTabletSplitArg &arg); - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(table_id), K_(schema_version), + TO_STRING_KV(K_(is_inited), K_(table_id), K_(schema_version), K_(task_id), K_(source_tablet_id), K_(dest_tablets_id), K_(compaction_scn), K_(user_parallelism), K_(compat_mode), K_(data_format_version), K_(can_reuse_macro_block), K_(split_sstable_type), K_(parallel_datum_rowkey_list), @@ -57,7 +58,6 @@ struct ObTabletSplitParam : public share::ObIDagInitParam common::ObArenaAllocator rowkey_allocator_; // for DatumRowkey. public: bool is_inited_; - share::ObLSID ls_id_; int64_t table_id_; int64_t schema_version_; // always the data table schema version int64_t task_id_; // ddl task id. @@ -82,8 +82,7 @@ struct ObTabletSplitCtx final int init(const ObTabletSplitParam ¶m); bool is_valid() const; TO_STRING_KV(K_(is_inited), K_(data_split_ranges), K_(complement_data_ret), - K_(row_inserted), K_(physical_row_count), K_(skipped_split_major_keys), - K(ls_rebuild_seq_)); + K_(row_inserted), K_(physical_row_count), K_(skipped_split_major_keys)); public: // generate index tree. int prepare_index_builder(const ObTabletSplitParam ¶m); @@ -95,7 +94,7 @@ struct ObTabletSplitCtx final bool is_inited_; ObArray data_split_ranges_; int complement_data_ret_; - ObLSHandle ls_handle_; + ObLS *ls_; ObTabletHandle tablet_handle_; // is important, rowkey_read_info, source_tables rely on it. ObTableStoreIterator table_store_iterator_; // for rewrite macro block task. @@ -105,7 +104,6 @@ struct ObTabletSplitCtx final ObArray skipped_split_major_keys_; int64_t row_inserted_; int64_t physical_row_count_; - int64_t ls_rebuild_seq_; DISALLOW_COPY_AND_ASSIGN(ObTabletSplitCtx); }; @@ -220,8 +218,7 @@ class ObTabletSplitMergeTask final : public share::ObITask int init(ObTabletSplitParam ¶m, ObTabletSplitCtx &ctx); virtual int process() override; static int update_table_store_with_batch_tables( - const int64_t ls_rebuild_seq, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &src_tablet_handle, const ObTabletID &dst_tablet_id, const ObTablesHandleArray &tables_handle, @@ -293,7 +290,6 @@ class ObRowScan : public ObIStoreRowIterator int construct_access_param( const ObSplitScanParam ¶m); int construct_access_ctx( - const share::ObLSID &ls_id, const ObTabletID &tablet_id); int build_rowkey_read_info( const ObSplitScanParam ¶m); @@ -324,7 +320,7 @@ class ObSnapshotRowScan final : public ObIStoreRowIterator const uint64_t table_id, const common::ObTabletID &tablet_id, const ObITableReadInfo &read_info); - int construct_range_ctx(ObQueryFlag &query_flag, const share::ObLSID &ls_id); + int construct_range_ctx(ObQueryFlag &query_flag); int construct_multiple_scan_merge(const ObTabletTableIterator &table_iter, const ObDatumRange &range); int add_extra_rowkey(const ObDatumRow &row); virtual int get_next_row(const blocksstable::ObDatumRow *&tmp_row) override; @@ -388,7 +384,6 @@ struct ObTabletSplitUtil final static int split_task_ranges( ObIAllocator &allocator, const share::ObDDLType ddl_type, - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t user_parallelism, const int64_t schema_tablet_size, @@ -404,17 +399,15 @@ struct ObTabletSplitUtil final const ObIArray & datum_rowkey_list, ObIArray &datum_ranges_array); static int check_data_split_finished( - const share::ObLSID &ls_id, const ObIArray &check_tablets_id, bool &is_finished); static int check_satisfy_split_condition( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObArray &dest_tablets_id, const int64_t compaction_scn, const share::SCN &min_split_start_scn); static int get_split_dest_tablets_info( - const share::ObLSID &ls_id, const ObTabletID &source_tablet_id, ObIArray &dest_tablets_id, lib::Worker::CompatMode &compat_mode); @@ -423,31 +416,31 @@ struct ObTabletSplitUtil final obcall::ObCheckMediumCompactionInfoListResult &result); static int check_tablet_restore_status( const ObIArray &dest_tablets_id, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, bool &is_tablet_status_need_to_split); static int build_mds_sstable( common::ObArenaAllocator &allocator, - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObTabletID &dest_tablet_id, ObTableHandleV2 &medium_mds_table_handle); static int check_sstables_skip_data_split( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTableStoreIterator &source_table_store_iter, const ObIArray &dest_tablets_id, const int64_t lob_major_snapshot/*OB_INVALID_VERSION for non lob tablets*/, ObIArray &skipped_split_major_keys); private: static int check_and_build_mds_sstable_merge_ctx( - const ObLSHandle &ls_handle, + ObLS *ls, const ObTabletHandle &dest_tablet_handle, compaction::ObTabletMergeCtx &tablet_merge_ctx); static int check_and_determine_mds_end_scn( const ObTabletHandle &dest_tablet_handle, share::SCN &end_scn); - static int check_tablet_ha_status( - const ObLSHandle &ls_handle, + static int check_tablet_restore_state( + ObLS *ls, const ObTabletHandle &source_tablet_handle, const ObIArray &dest_tablets_id); }; diff --git a/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.cpp index 330d12162..c5ccedce9 100755 --- a/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.cpp @@ -31,7 +31,7 @@ #include "storage/tx_storage/ob_ls_service.h" #include "storage/ddl/ob_ddl_independent_dag.h" #include "storage/ddl/ob_direct_load_mgr_utils.h" -#include "storage/tx/ob_ts_mgr.h" // transaction::get_tenant_gts +#include "storage/tx/ob_ts_mgr.h" namespace oceanbase { @@ -70,7 +70,6 @@ ObDirectLoadInsertDataTabletContext::~ObDirectLoadInsertDataTabletContext() } int ObDirectLoadInsertDataTabletContext::init(ObDirectLoadInsertDataTableContext *table_ctx, - const ObLSID &ls_id, const ObTabletID &origin_tablet_id, const ObTabletID &tablet_id) { @@ -78,16 +77,14 @@ int ObDirectLoadInsertDataTabletContext::init(ObDirectLoadInsertDataTableContext if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadInsertDataTabletContext init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == table_ctx || !ls_id.is_valid() || - !origin_tablet_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(nullptr == table_ctx || !origin_tablet_id.is_valid() || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(table_ctx), K(ls_id), K(origin_tablet_id), K(tablet_id)); + LOG_WARN("invalid args", KR(ret), KP(table_ctx), K(origin_tablet_id), K(tablet_id)); } else { table_ctx_ = table_ctx; param_ = &table_ctx->param_; context_id_ = table_ctx->ddl_ctrl_.context_id_; direct_load_type_ = table_ctx->ddl_ctrl_.direct_load_type_; - ls_id_ = ls_id; origin_tablet_id_ = origin_tablet_id; tablet_id_ = tablet_id; pk_tablet_id_ = origin_tablet_id_; // Get from original table, DDL will help sync to hidden table @@ -181,7 +178,6 @@ int ObDirectLoadInsertDataTabletContext::create_tablet_direct_load() direct_load_param.common_param_.direct_load_type_ = direct_load_type_; direct_load_param.common_param_.data_format_version_ = param_->data_version_; direct_load_param.common_param_.read_snapshot_ = param_->snapshot_version_; - direct_load_param.common_param_.ls_id_ = ls_id_; direct_load_param.common_param_.tablet_id_ = tablet_id_; direct_load_param.common_param_.is_no_logging_ = param_->is_no_logging_; direct_load_param.runtime_only_param_.exec_ctx_ = nullptr; @@ -216,10 +212,10 @@ int ObDirectLoadInsertDataTabletContext::open_tablet_direct_load() if (OB_ISNULL(sstable_insert_mgr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get tenant direct load mgr", K(ret)); - } else if (OB_FAIL(sstable_insert_mgr->open_tablet_direct_load(direct_load_type_, ls_id_, tablet_id_, + } else if (OB_FAIL(sstable_insert_mgr->open_tablet_direct_load(direct_load_type_, tablet_id_, context_id_))) { LOG_WARN("fail to open tablet direct load", KR(ret), K(tablet_id_)); - } else if (OB_FAIL(ddl_agent_.init(context_id_, ls_id_, tablet_id_, direct_load_type_))) { + } else if (OB_FAIL(ddl_agent_.init(context_id_, tablet_id_, direct_load_type_))) { LOG_WARN("init ddl agent failed", K(ret)); } } else { @@ -240,7 +236,7 @@ int ObDirectLoadInsertDataTabletContext::close_tablet_direct_load(bool commit) { int ret = OB_SUCCESS; if (OB_FAIL(ddl_agent_.close(context_id_, true /* need commit */, -1 /* execution id */))) { - LOG_WARN("fail to close tablet direct load", KR(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("fail to close tablet direct load", KR(ret), K(tablet_id_)); } else { is_closed_ = true; } @@ -272,7 +268,6 @@ int ObDirectLoadInsertDataTabletContext::open_sstable_slice(const ObMacroDataSeq ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = false; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -308,7 +303,6 @@ int ObDirectLoadInsertDataTabletContext::fill_sstable_slice(const int64_t &slice ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = false; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -335,7 +329,6 @@ int ObDirectLoadInsertDataTabletContext::fill_sstable_slice(const int64_t &slice ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = false; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -359,7 +352,6 @@ int ObDirectLoadInsertDataTabletContext::close_sstable_slice(const int64_t slice ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = false; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -427,7 +419,6 @@ int ObDirectLoadInsertDataTabletContext::open_lob_sstable_slice(const ObMacroDat ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -460,7 +451,6 @@ int ObDirectLoadInsertDataTabletContext::fill_lob_sstable_slice(ObIAllocator &al ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = lob_slice_id; slice_info.context_id_ = context_id_; @@ -488,7 +478,6 @@ int ObDirectLoadInsertDataTabletContext::fill_lob_sstable_slice(ObIAllocator &al ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = lob_slice_id; slice_info.context_id_ = context_id_; @@ -517,7 +506,6 @@ int ObDirectLoadInsertDataTabletContext::fill_lob_meta_sstable_slice(const int64 ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = lob_slice_id; slice_info.context_id_ = context_id_; @@ -540,7 +528,6 @@ int ObDirectLoadInsertDataTabletContext::close_lob_sstable_slice(const int64_t s ObDirectLoadSliceInfo slice_info; slice_info.is_full_direct_load_ = !param_->is_incremental_; slice_info.is_lob_slice_ = true; - slice_info.ls_id_ = ls_id_; slice_info.data_tablet_id_ = tablet_id_; slice_info.slice_id_ = slice_id; slice_info.context_id_ = context_id_; @@ -575,18 +562,18 @@ ObDirectLoadInsertDataTableContext::~ObDirectLoadInsertDataTableContext() int ObDirectLoadInsertDataTableContext::init( const ObDirectLoadInsertTableParam ¶m, - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadInsertDataTableContext init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid() || ls_partition_ids.empty() || - target_ls_partition_ids.empty() || - ls_partition_ids.count() != target_ls_partition_ids.count())) { + } else if (OB_UNLIKELY(!param.is_valid() || partition_ids.empty() || + target_partition_ids.empty() || + partition_ids.count() != target_partition_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), K(ls_partition_ids), K(target_ls_partition_ids)); + LOG_WARN("invalid args", KR(ret), K(param), K(partition_ids), K(target_partition_ids)); } else { param_ = param; if (OB_FAIL(inner_init())) { @@ -598,11 +585,11 @@ int ObDirectLoadInsertDataTableContext::init( if (OB_FAIL(ret)) { } else if (OB_FAIL(share::g_mp->tenant_direct_load_mgr()->alloc_execution_context_id(ddl_ctrl_.context_id_))) { LOG_WARN("alloc execution context id failed", K(ret)); - } else if (param_.enable_dag_ && OB_FAIL(init_dag(target_ls_partition_ids))) { + } else if (param_.enable_dag_ && OB_FAIL(init_dag(target_partition_ids))) { LOG_WARN("fail to init dag", KR(ret)); - } else if (OB_FAIL(create_all_tablet_contexts(ls_partition_ids, target_ls_partition_ids))) { - LOG_WARN("fail to create all tablet contexts", KR(ret), K(ls_partition_ids), - K(target_ls_partition_ids)); + } else if (OB_FAIL(create_all_tablet_contexts(partition_ids, target_partition_ids))) { + LOG_WARN("fail to create all tablet contexts", KR(ret), K(partition_ids), + K(target_partition_ids)); } else if (param_.online_opt_stat_gather_ && OB_FAIL(sql_stat_map_.create(1024, "TLD_SqlStatMap", "TLD_SqlStatMap"))) { LOG_WARN("fail to create sql stat map", KR(ret)); @@ -614,7 +601,7 @@ int ObDirectLoadInsertDataTableContext::init( } int ObDirectLoadInsertDataTableContext::init_dag( - const ObIArray &ls_partition_ids) + const ObIArray &partition_ids) { int ret = OB_SUCCESS; ObDDLIndependentDagInitParam init_param; @@ -632,7 +619,7 @@ int ObDirectLoadInsertDataTableContext::init_dag( // incremental direct load may generate multiple sstables, // the snapshot_version needs be updated each time to avoid duplicated MacroIds share::SCN current_scn; - if (OB_FAIL(transaction::get_tenant_gts(current_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts_sync(GCONF.rpc_timeout, current_scn))) { LOG_WARN("failed to get gts", KR(ret)); } else { init_param.ddl_task_param_.snapshot_version_ = current_scn.get_val_for_tx(); @@ -641,11 +628,10 @@ int ObDirectLoadInsertDataTableContext::init_dag( init_param.tx_info_.seq_no_ = param_.trans_param_.tx_seq_.cast_to_int(); } } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); i++) { if (OB_FAIL(add_var_to_array_no_dup( - init_param.ls_tablet_ids_, - std::make_pair(ls_partition_ids.at(i).ls_id_, - ls_partition_ids.at(i).part_tablet_id_.tablet_id_)))) { + init_param.tablet_ids_, + partition_ids.at(i).part_tablet_id_.tablet_id_))) { LOG_WARN("add var to array no dup failed", K(ret)); } } @@ -658,19 +644,18 @@ int ObDirectLoadInsertDataTableContext::init_dag( } int ObDirectLoadInsertDataTableContext::create_all_tablet_contexts( - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTabletID &origin_tablet_id = ls_partition_ids.at(i).part_tablet_id_.tablet_id_; - const ObLSID &ls_id = target_ls_partition_ids.at(i).ls_id_; - const ObTabletID &tablet_id = target_ls_partition_ids.at(i).part_tablet_id_.tablet_id_; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTabletID &origin_tablet_id = partition_ids.at(i).part_tablet_id_.tablet_id_; + const ObTabletID &tablet_id = target_partition_ids.at(i).part_tablet_id_.tablet_id_; ObDirectLoadInsertDataTabletContext *tablet_ctx = nullptr; if (OB_ISNULL(tablet_ctx = OB_NEWx(ObDirectLoadInsertDataTabletContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertDataTabletContext", KR(ret)); - } else if (OB_FAIL(tablet_ctx->init(this, ls_id, origin_tablet_id, tablet_id))) { + } else if (OB_FAIL(tablet_ctx->init(this, origin_tablet_id, tablet_id))) { LOG_WARN("fail to init tablet ctx", KR(ret)); } else if (OB_FAIL(tablet_ctx_map_.set_refactored(origin_tablet_id, tablet_ctx))) { LOG_WARN("fail to set tablet ctx map", KR(ret)); @@ -1185,7 +1170,7 @@ int ObDirectLoadInsertDataTabletContext::get_ddl_agent(ObDirectLoadMgrAgent &tmp } } else { /* build tmp agent for previous version, which used by shared storage */ /* TODO @zhuoran.zzr wait to remove it when ss mode ready */ - if (OB_FAIL(tmp_agent.init(context_id_, ls_id_, tablet_id_, direct_load_type_))) { + if (OB_FAIL(tmp_agent.init(context_id_, tablet_id_, direct_load_type_))) { LOG_WARN("fail to init tmp agent", K(ret)); } } diff --git a/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.h index 734582690..ad9329c0e 100755 --- a/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_data_table_ctx.h @@ -33,7 +33,7 @@ class ObDirectLoadInsertDataTabletContext : public ObDirectLoadInsertTabletConte public: ObDirectLoadInsertDataTabletContext(); virtual ~ObDirectLoadInsertDataTabletContext(); - int init(ObDirectLoadInsertDataTableContext *table_ctx, const share::ObLSID &ls_id, + int init(ObDirectLoadInsertDataTableContext *table_ctx, const common::ObTabletID &origin_tablet_id, const common::ObTabletID &tablet_id); int open() override; @@ -113,8 +113,8 @@ class ObDirectLoadInsertDataTableContext : public ObDirectLoadInsertTableContext ObDirectLoadInsertDataTableContext(); virtual ~ObDirectLoadInsertDataTableContext(); int init(const ObDirectLoadInsertTableParam ¶m, - const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids); + const common::ObIArray &partition_ids, + const common::ObIArray &target_partition_ids); int close() override; void set_lob_table_ctx(ObDirectLoadInsertTableContext *lob_table_ctx) { @@ -123,10 +123,10 @@ class ObDirectLoadInsertDataTableContext : public ObDirectLoadInsertTableContext ObDirectLoadInsertTableContext *get_lob_table_ctx() { return lob_table_ctx_; } private: - int init_dag(const common::ObIArray &ls_partition_ids); + int init_dag(const common::ObIArray &partition_ids); int create_all_tablet_contexts( - const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids); + const common::ObIArray &partition_ids, + const common::ObIArray &target_partition_ids); //////////////////////// sql stats interface //////////////////////// public: diff --git a/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.cpp index d62727ba5..f12b1fff2 100755 --- a/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.cpp @@ -41,7 +41,6 @@ ObDirectLoadInsertLobTabletContext::~ObDirectLoadInsertLobTabletContext() {} int ObDirectLoadInsertLobTabletContext::init(ObDirectLoadInsertLobTableContext *table_ctx, ObDirectLoadInsertDataTabletContext *data_tablet_ctx, - const ObLSID &ls_id, const ObTabletID &origin_tablet_id, const ObTabletID &tablet_id) { @@ -49,16 +48,15 @@ int ObDirectLoadInsertLobTabletContext::init(ObDirectLoadInsertLobTableContext * if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadInsertDataTabletContext init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == table_ctx || nullptr == data_tablet_ctx || !ls_id.is_valid() || + } else if (OB_UNLIKELY(nullptr == table_ctx || nullptr == data_tablet_ctx || !origin_tablet_id.is_valid() || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(table_ctx), KP(data_tablet_ctx), K(ls_id), + LOG_WARN("invalid args", KR(ret), KP(table_ctx), KP(data_tablet_ctx), K(origin_tablet_id), K(tablet_id)); } else { table_ctx_ = table_ctx; param_ = &table_ctx->param_; data_tablet_ctx_ = data_tablet_ctx; - ls_id_ = ls_id; origin_tablet_id_ = origin_tablet_id; tablet_id_ = tablet_id; pk_tablet_id_ = tablet_id_; // get from target table @@ -232,28 +230,28 @@ ObDirectLoadInsertLobTableContext::~ObDirectLoadInsertLobTableContext() {} int ObDirectLoadInsertLobTableContext::init( const ObDirectLoadInsertTableParam ¶m, ObDirectLoadInsertDataTableContext *data_table_ctx, - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids, - const ObIArray &data_ls_partition_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids, + const ObIArray &data_partition_ids) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadInsertDataTableContext init twice", KR(ret), KP(this)); } else if (OB_UNLIKELY(!param.is_valid() || nullptr == data_table_ctx || - ls_partition_ids.empty() || target_ls_partition_ids.empty() || - ls_partition_ids.count() != target_ls_partition_ids.count())) { + partition_ids.empty() || target_partition_ids.empty() || + partition_ids.count() != target_partition_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), KP(data_table_ctx), K(ls_partition_ids), - K(target_ls_partition_ids)); + LOG_WARN("invalid args", KR(ret), K(param), KP(data_table_ctx), K(partition_ids), + K(target_partition_ids)); } else { param_ = param; if (OB_FAIL(inner_init())) { LOG_WARN("fail to inner init", KR(ret)); } else if (OB_FAIL(create_all_tablet_contexts(data_table_ctx, - ls_partition_ids, - target_ls_partition_ids, - data_ls_partition_ids))) { + partition_ids, + target_partition_ids, + data_partition_ids))) { LOG_WARN("fail to create all tablet contexts", KR(ret)); } else { data_table_ctx->set_lob_table_ctx(this); @@ -266,16 +264,15 @@ int ObDirectLoadInsertLobTableContext::init( int ObDirectLoadInsertLobTableContext::create_all_tablet_contexts( ObDirectLoadInsertDataTableContext *data_table_ctx, - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids, - const ObIArray &data_ls_partition_ids) + const ObIArray &partition_ids, + const ObIArray &target_partition_ids, + const ObIArray &data_partition_ids) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTabletID &origin_tablet_id = ls_partition_ids.at(i).part_tablet_id_.tablet_id_; - const ObLSID &ls_id = target_ls_partition_ids.at(i).ls_id_; - const ObTabletID &tablet_id = target_ls_partition_ids.at(i).part_tablet_id_.tablet_id_; - const ObTabletID &data_tablet_id = data_ls_partition_ids.at(i).part_tablet_id_.tablet_id_; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTabletID &origin_tablet_id = partition_ids.at(i).part_tablet_id_.tablet_id_; + const ObTabletID &tablet_id = target_partition_ids.at(i).part_tablet_id_.tablet_id_; + const ObTabletID &data_tablet_id = data_partition_ids.at(i).part_tablet_id_.tablet_id_; ObDirectLoadInsertTabletContext *data_tablet_ctx = nullptr; ObDirectLoadInsertLobTabletContext *lob_tablet_ctx = nullptr; if (OB_FAIL(data_table_ctx->get_tablet_context(data_tablet_id, data_tablet_ctx))) { @@ -285,7 +282,7 @@ int ObDirectLoadInsertLobTableContext::create_all_tablet_contexts( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertLobTabletContext", KR(ret)); } else if (OB_FAIL(lob_tablet_ctx->init( - this, static_cast(data_tablet_ctx), ls_id, + this, static_cast(data_tablet_ctx), origin_tablet_id, tablet_id))) { LOG_WARN("fail to init tablet ctx", KR(ret)); } else if (OB_FAIL(tablet_ctx_map_.set_refactored(origin_tablet_id, lob_tablet_ctx))) { diff --git a/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.h index 94ca11f24..921140603 100755 --- a/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_lob_table_ctx.h @@ -32,7 +32,7 @@ class ObDirectLoadInsertLobTabletContext : public ObDirectLoadInsertTabletContex ObDirectLoadInsertLobTabletContext(); virtual ~ObDirectLoadInsertLobTabletContext(); int init(ObDirectLoadInsertLobTableContext *table_ctx, - ObDirectLoadInsertDataTabletContext *data_tablet_ctx, const share::ObLSID &ls_id, + ObDirectLoadInsertDataTabletContext *data_tablet_ctx, const common::ObTabletID &origin_tablet_id, const common::ObTabletID &tablet_id); int open() override; int close() override; @@ -88,9 +88,9 @@ class ObDirectLoadInsertLobTableContext : public ObDirectLoadInsertTableContext virtual ~ObDirectLoadInsertLobTableContext(); int init(const ObDirectLoadInsertTableParam ¶m, ObDirectLoadInsertDataTableContext *data_table_ctx, - const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids, - const common::ObIArray &data_ls_partition_ids); + const common::ObIArray &partition_ids, + const common::ObIArray &target_partition_ids, + const common::ObIArray &data_partition_ids); void set_data_table_ctx(ObDirectLoadInsertTableContext *data_table_ctx) { @@ -101,9 +101,9 @@ class ObDirectLoadInsertLobTableContext : public ObDirectLoadInsertTableContext private: int create_all_tablet_contexts( ObDirectLoadInsertDataTableContext *data_table_ctx, - const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids, - const common::ObIArray &data_ls_partition_ids); + const common::ObIArray &partition_ids, + const common::ObIArray &target_partition_ids, + const common::ObIArray &data_partition_ids); private: ObDirectLoadInsertTableContext *data_table_ctx_; diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 0767b5ad4..baa599e24 100755 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -94,7 +94,6 @@ bool ObDirectLoadInsertTableParam::is_valid() const ObDirectLoadInsertTabletContext::ObDirectLoadInsertTabletContext() : table_ctx_(nullptr), param_(nullptr), - ls_id_(), origin_tablet_id_(), tablet_id_(), pk_tablet_id_(), diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h index e04dcc117..1669cbe9b 100755 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h @@ -17,7 +17,6 @@ #include "lib/hash/ob_hashmap.h" #include "lib/lock/ob_mutex.h" -#include "share/ob_ls_id.h" #include "share/ob_tablet_autoincrement_param.h" #include "share/table/ob_table_load_define.h" #include "sql/engine/px/ob_sub_trans_ctrl.h" @@ -175,7 +174,6 @@ class ObDirectLoadInsertTabletContext public: OB_INLINE ObDirectLoadInsertTableContext *get_table_ctx() { return table_ctx_; } OB_INLINE const ObDirectLoadInsertTableParam *get_param() const { return param_; } - OB_INLINE const share::ObLSID &get_ls_id() const { return ls_id_; } OB_INLINE const common::ObTabletID &get_origin_tablet_id() const { return origin_tablet_id_; } OB_INLINE const common::ObTabletID &get_tablet_id() const { return tablet_id_; } @@ -232,13 +230,12 @@ class ObDirectLoadInsertTabletContext void inc_row_count(const int64_t row_count) { ATOMIC_AAF(&row_count_, row_count); } int64_t get_row_count() const { return ATOMIC_LOAD(&row_count_); } - VIRTUAL_TO_STRING_KV(KP_(table_ctx), KP_(param), K_(ls_id), K_(origin_tablet_id), K_(tablet_id), + VIRTUAL_TO_STRING_KV(KP_(table_ctx), KP_(param), K_(origin_tablet_id), K_(tablet_id), K_(pk_tablet_id), KP_(lob_tablet_ctx), K_(row_count), K_(is_inited)); protected: ObDirectLoadInsertTableContext *table_ctx_; const ObDirectLoadInsertTableParam *param_; - share::ObLSID ls_id_; common::ObTabletID origin_tablet_id_; common::ObTabletID tablet_id_; common::ObTabletID pk_tablet_id_; // get auto-increment pk from which tablet_id diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index 106238e0e..8e98c2f00 100755 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -100,18 +100,18 @@ void ObDirectLoadMergeCtx::reset() } int ObDirectLoadMergeCtx::init(const ObDirectLoadMergeParam ¶m, - const ObIArray &ls_partition_ids) + const ObIArray &partition_ids) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadMerger init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid() || ls_partition_ids.empty())) { + } else if (OB_UNLIKELY(!param.is_valid() || partition_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), K(ls_partition_ids)); + LOG_WARN("invalid args", KR(ret), K(param), K(partition_ids)); } else { param_ = param; - if (OB_FAIL(create_all_tablet_ctxs(ls_partition_ids))) { + if (OB_FAIL(create_all_tablet_ctxs(partition_ids))) { LOG_WARN("fail to create all tablet ctxs", KR(ret)); } else { struct @@ -131,17 +131,17 @@ int ObDirectLoadMergeCtx::init(const ObDirectLoadMergeParam ¶m, } int ObDirectLoadMergeCtx::create_all_tablet_ctxs( - const ObIArray &ls_partition_ids) + const ObIArray &partition_ids) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_partition_id = ls_partition_ids.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + const ObTableLoadTabletId &partition_id = partition_ids.at(i); ObDirectLoadTabletMergeCtx *tablet_ctx = nullptr; if (OB_ISNULL(tablet_ctx = OB_NEWx(ObDirectLoadTabletMergeCtx, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadTabletMergeCtx", KR(ret)); - } else if (OB_FAIL(tablet_ctx->init(this, ls_partition_id))) { - LOG_WARN("fail to init tablet ctx", KR(ret), K(ls_partition_id)); + } else if (OB_FAIL(tablet_ctx->init(this, partition_id))) { + LOG_WARN("fail to init tablet ctx", KR(ret), K(partition_id)); } else if (OB_FAIL(tablet_merge_ctx_array_.push_back(tablet_ctx))) { LOG_WARN("fail to push back", KR(ret)); } @@ -345,26 +345,25 @@ void ObDirectLoadTabletMergeCtx::reset() } int ObDirectLoadTabletMergeCtx::init(ObDirectLoadMergeCtx *merge_ctx, - const ObTableLoadLSIdAndPartitionId &ls_partition_id) + const ObTableLoadTabletId &partition_id) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadTabletMergeCtx init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == merge_ctx || !ls_partition_id.is_valid())) { + } else if (OB_UNLIKELY(nullptr == merge_ctx || !partition_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(merge_ctx), K(ls_partition_id)); + LOG_WARN("invalid args", KR(ret), KP(merge_ctx), K(partition_id)); } else { merge_ctx_ = merge_ctx; param_ = &merge_ctx->param_; - tablet_id_ = ls_partition_id.part_tablet_id_.tablet_id_; + tablet_id_ = partition_id.part_tablet_id_.tablet_id_; if (OB_FAIL(param_->insert_table_ctx_->get_tablet_context(tablet_id_, insert_tablet_ctx_))) { LOG_WARN("fail to get insert tablet ctx", KR(ret), K(tablet_id_)); } else if (ObDirectLoadMergeMode::merge_need_origin_table(param_->merge_mode_)) { ObDirectLoadOriginTableCreateParam origin_table_param; origin_table_param.table_id_ = param_->table_id_; - origin_table_param.ls_id_ = ls_partition_id.ls_id_; origin_table_param.tablet_id_ = tablet_id_; origin_table_param.tx_id_ = param_->trans_param_.tx_id_; origin_table_param.tx_seq_ = param_->trans_param_.tx_seq_; diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index b837228ad..ca77ec1bd 100755 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -136,7 +136,7 @@ class ObDirectLoadMergeCtx ~ObDirectLoadMergeCtx(); void reset(); int init(const ObDirectLoadMergeParam ¶m, - const common::ObIArray &ls_partition_ids); + const common::ObIArray &partition_ids); int build_merge_task(ObDirectLoadTableStore &table_store, int64_t thread_cnt); int build_del_lob_task(ObDirectLoadTableStore &table_store, int64_t thread_cnt, const bool for_dag); const common::ObIArray &get_tablet_merge_ctxs() const @@ -146,7 +146,7 @@ class ObDirectLoadMergeCtx private: int create_all_tablet_ctxs( - const common::ObIArray &ls_partition_ids); + const common::ObIArray &partition_ids); private: common::ObArenaAllocator allocator_; @@ -164,7 +164,7 @@ class ObDirectLoadTabletMergeCtx final ~ObDirectLoadTabletMergeCtx(); void reset(); int init(ObDirectLoadMergeCtx *merge_ctx, - const table::ObTableLoadLSIdAndPartitionId &ls_partition_id); + const table::ObTableLoadTabletId &partition_id); // No import data merge task int build_empty_data_merge_task(const ObDirectLoadTableDataDesc &table_data_desc, int64_t max_parallel_degree); diff --git a/src/storage/direct_load/ob_direct_load_origin_table.cpp b/src/storage/direct_load/ob_direct_load_origin_table.cpp index bfd7ceeb0..aaef7cb37 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.cpp +++ b/src/storage/direct_load/ob_direct_load_origin_table.cpp @@ -44,7 +44,7 @@ ObDirectLoadOriginTableCreateParam::~ObDirectLoadOriginTableCreateParam() bool ObDirectLoadOriginTableCreateParam::is_valid() const { - return OB_INVALID_ID != table_id_ && tablet_id_.is_valid() && ls_id_.is_valid(); + return OB_INVALID_ID != table_id_ && tablet_id_.is_valid(); } /** @@ -64,7 +64,6 @@ void ObDirectLoadOriginTableMeta::reset() { table_id_ = OB_INVALID_ID; tablet_id_.reset(); - ls_id_.reset(); tx_id_.reset(); tx_seq_.reset(); } @@ -104,24 +103,14 @@ int ObDirectLoadOriginTable::init(const ObDirectLoadOriginTableCreateParam ¶ LOG_WARN("invalid args", KR(ret), K(param)); } else { const ObTabletID &tablet_id = param.tablet_id_; - const ObLSID &ls_id = param.ls_id_; - ObLSService *ls_svr = nullptr; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { - ret = OB_ERR_SYS; - LOG_WARN("MTL ObLSService is null", KR(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls", KR(ret), K(ls)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls is nullptr", KR(ret)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle_))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("fail to get ls", KR(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, tablet_handle_))) { LOG_WARN("fail to get tablet", KR(ret), K(tablet_id)); } else if (OB_FAIL(prepare_tables())) { LOG_WARN("fail to prepare tables", KR(ret)); } else { - meta_.ls_id_ = param.ls_id_; meta_.table_id_ = param.table_id_; meta_.tablet_id_ = param.tablet_id_; meta_.tx_id_ = param.tx_id_; @@ -373,8 +362,7 @@ int ObDirectLoadOriginTableAccessor::init_table_access_ctx(bool skip_read_lob) share::SCN snapshot_scn; if (OB_FAIL(snapshot_scn.convert_for_tx(snapshot_version))) { LOG_WARN("fail to convert scn", KR(ret)); - } else if (OB_FAIL(store_ctx_.init_for_read(origin_table_->get_meta().ls_id_, - tablet_id, + } else if (OB_FAIL(store_ctx_.init_for_read(tablet_id, INT64_MAX, -1, snapshot_scn))) { diff --git a/src/storage/direct_load/ob_direct_load_origin_table.h b/src/storage/direct_load/ob_direct_load_origin_table.h index d082bcaf5..83893edfb 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.h +++ b/src/storage/direct_load/ob_direct_load_origin_table.h @@ -37,13 +37,11 @@ struct ObDirectLoadOriginTableCreateParam bool is_valid() const; TO_STRING_KV(K_(table_id), K_(tablet_id), - K_(ls_id), K_(tx_id), K_(tx_seq)); public: uint64_t table_id_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; transaction::ObTransID tx_id_; transaction::ObTxSEQ tx_seq_; }; @@ -56,13 +54,11 @@ struct ObDirectLoadOriginTableMeta void reset(); TO_STRING_KV(K_(table_id), K_(tablet_id), - K_(ls_id), K_(tx_id), K_(tx_seq)); public: uint64_t table_id_; common::ObTabletID tablet_id_; - share::ObLSID ls_id_; transaction::ObTransID tx_id_; transaction::ObTxSEQ tx_seq_; }; diff --git a/src/storage/fts/ob_fts_doc_word_iterator.cpp b/src/storage/fts/ob_fts_doc_word_iterator.cpp index aef060a3c..4664c95c5 100644 --- a/src/storage/fts/ob_fts_doc_word_iterator.cpp +++ b/src/storage/fts/ob_fts_doc_word_iterator.cpp @@ -45,7 +45,6 @@ ObFTDocWordScanIterator::~ObFTDocWordScanIterator() int ObFTDocWordScanIterator::init( const uint64_t table_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const transaction::ObTxReadSnapshot *snapshot, const int64_t schema_version) @@ -54,8 +53,8 @@ int ObFTDocWordScanIterator::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init fulltext doc word scan iterator twice", K(ret), K(is_inited_)); - } else if (OB_FAIL(init_scan_param(table_id, ls_id, tablet_id, snapshot, schema_version))) { - LOG_WARN("fail to init scan param", K(ret), K(table_id), K(ls_id), K(tablet_id), K(snapshot), K(schema_version)); + } else if (OB_FAIL(init_scan_param(table_id, tablet_id, snapshot, schema_version))) { + LOG_WARN("fail to init scan param", K(ret), K(table_id), K(tablet_id), K(snapshot), K(schema_version)); } else { is_inited_ = true; } @@ -155,7 +154,6 @@ int ObFTDocWordScanIterator::get_next_row(blocksstable::ObDatumRow *&datum_row) int ObFTDocWordScanIterator::init_scan_param( const uint64_t table_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const transaction::ObTxReadSnapshot *snapshot, const int64_t schema_version) @@ -172,16 +170,14 @@ int ObFTDocWordScanIterator::init_scan_param( false // read_latest ); if (OB_UNLIKELY(OB_INVALID_ID == table_id - || !ls_id.is_valid() || !tablet_id.is_valid() || nullptr == snapshot || schema_version < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(table_id), K(ls_id), K(tablet_id), KPC(snapshot), K(schema_version)); + LOG_WARN("invalid arguments", K(ret), K(table_id), K(tablet_id), KPC(snapshot), K(schema_version)); } else if (OB_FAIL(build_table_param(table_id, table_param_, scan_param_.column_ids_))) { LOG_WARN("fail to build table param", K(ret), K(table_id)); } else { - scan_param_.ls_id_ = ls_id; scan_param_.tablet_id_ = tablet_id; scan_param_.schema_version_ = schema_version; scan_param_.is_get_ = false; diff --git a/src/storage/fts/ob_fts_doc_word_iterator.h b/src/storage/fts/ob_fts_doc_word_iterator.h index bfa34a0c9..be7e21b7f 100644 --- a/src/storage/fts/ob_fts_doc_word_iterator.h +++ b/src/storage/fts/ob_fts_doc_word_iterator.h @@ -37,7 +37,6 @@ class ObFTDocWordScanIterator final int init( const uint64_t table_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const transaction::ObTxReadSnapshot *snapshot, const int64_t schema_version); @@ -50,7 +49,6 @@ class ObFTDocWordScanIterator final private: int init_scan_param( const uint64_t table_id, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const transaction::ObTxReadSnapshot *snapshot, const int64_t schema_version); diff --git a/src/storage/lob/ob_lob_access_param.cpp b/src/storage/lob/ob_lob_access_param.cpp index 5ffc1c6c3..60bd11250 100644 --- a/src/storage/lob/ob_lob_access_param.cpp +++ b/src/storage/lob/ob_lob_access_param.cpp @@ -48,7 +48,6 @@ int ObLobAccessParam::assign(const ObLobAccessParam& other) - this->ls_id_ = other.ls_id_; this->tablet_id_ = other.tablet_id_; this->lob_meta_tablet_id_ = other.lob_meta_tablet_id_; this->lob_piece_tablet_id_ = other.lob_piece_tablet_id_; @@ -485,7 +484,7 @@ int ObLobAccessParam::get_tx_read_snapshot(ObLobLocatorV2 &locator, transaction: } else if (OB_FAIL(locator.get_location_info(location_info))) { LOG_WARN("failed to get location info", K(ret), K(locator)); } else if (OB_FAIL(read_snapshot.build_snapshot_for_lob( - tx_info->snapshot_version_, tx_info->snapshot_tx_id_, tx_info->snapshot_seq_, share::ObLSID(location_info->ls_id_)))) { + tx_info->snapshot_version_, tx_info->snapshot_tx_id_, tx_info->snapshot_seq_))) { LOG_WARN("build_snapshot_for_lob fail", K(ret), KPC(tx_info), KPC(location_info), K(locator)); } } else { diff --git a/src/storage/lob/ob_lob_access_param.h b/src/storage/lob/ob_lob_access_param.h index 0ea931ef9..9b672d062 100644 --- a/src/storage/lob/ob_lob_access_param.h +++ b/src/storage/lob/ob_lob_access_param.h @@ -44,14 +44,14 @@ struct ObLobAccessParam { : tmp_allocator_(nullptr), allocator_(nullptr), tx_desc_(nullptr), snapshot_(), tx_id_(), sql_mode_(SMO_DEFAULT), dml_base_param_(nullptr), - ls_id_(), tablet_id_(), lob_meta_tablet_id_(), lob_piece_tablet_id_(), + tablet_id_(), lob_meta_tablet_id_(), lob_piece_tablet_id_(), coll_type_(), lob_locator_(nullptr), lob_common_(nullptr), lob_data_(nullptr), byte_size_(0), handle_size_(0), timeout_(0), fb_snapshot_(), offset_(0), len_(0), parent_seq_no_(), seq_no_st_(), used_seq_cnt_(0), total_seq_cnt_(0), checksum_(0), update_len_(0), op_type_(ObLobDataOutRowCtx::OpType::SQL), is_total_quantity_log_(true), read_latest_(false), scan_backward_(false), is_fill_zero_(false), from_rpc_(false), - inrow_read_nocopy_(false), is_store_char_len_(true), need_read_latest_(false), no_need_retry_(false), is_mlog_(false), try_flush_redo_(false), + inrow_read_nocopy_(false), is_store_char_len_(true), need_read_latest_(false), no_need_retry_(false), try_flush_redo_(false), main_table_rowkey_col_(false), is_index_table_(false), enable_remote_retry_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD), schema_chunk_size_(OB_DEFAULT_LOB_CHUNK_SIZE), access_ctx_(nullptr), addr_(), lob_id_geneator_(nullptr), data_row_(nullptr) @@ -118,11 +118,11 @@ struct ObLobAccessParam { bool skip_flush_redo() const { return !try_flush_redo_; } - TO_STRING_KV(KP(this), K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), K_(lob_piece_tablet_id), + TO_STRING_KV(KP(this), K_(tablet_id), K_(lob_meta_tablet_id), K_(lob_piece_tablet_id), KPC_(lob_locator), KPC_(lob_common), KPC_(lob_data), K_(byte_size), K_(handle_size), K_(timeout), KP_(allocator), KP_(tmp_allocator), K_(coll_type), K_(scan_backward), K_(offset), K_(len), K_(parent_seq_no), K_(seq_no_st), K_(used_seq_cnt), K_(total_seq_cnt), K_(checksum), K_(update_len), K_(op_type), K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(fb_snapshot), K_(tx_id), K_(read_latest), K_(is_total_quantity_log), - K_(inrow_read_nocopy), K_(schema_chunk_size), K_(inrow_threshold), K_(is_store_char_len), K_(need_read_latest), K(no_need_retry_), K_(is_mlog), K_(try_flush_redo), + K_(inrow_read_nocopy), K_(schema_chunk_size), K_(inrow_threshold), K_(is_store_char_len), K_(need_read_latest), K(no_need_retry_), K_(try_flush_redo), K_(main_table_rowkey_col), K_(is_index_table), K_(enable_remote_retry), KP_(access_ctx), K_(addr), KPC_(lob_id_geneator), KPC_(data_row)); private: @@ -140,7 +140,6 @@ struct ObLobAccessParam { // some lob manager func will access other lob for data // other lob can read from other tenant - share::ObLSID ls_id_; common::ObTabletID tablet_id_; common::ObTabletID lob_meta_tablet_id_; common::ObTabletID lob_piece_tablet_id_; @@ -197,7 +196,6 @@ struct ObLobAccessParam { bool need_read_latest_; // whether need retry when some error occur bool no_need_retry_; - bool is_mlog_; // before 4.3.4 lob meta tablet is writed before main tablet, // to avoid rollback cost too much there is a flag in ObWriteFlag to control skip flush redo. // but after 4.3.4, lob meta tablet is writed after main table, so no need skip flush redo @@ -268,4 +266,3 @@ struct ObLobStorageParam #endif // OCEANBASE_STORAGE_OB_LOB_ACCESS_PARAM_H_ - diff --git a/src/storage/lob/ob_lob_data_reader.cpp b/src/storage/lob/ob_lob_data_reader.cpp index 096f6d930..bc3b32210 100644 --- a/src/storage/lob/ob_lob_data_reader.cpp +++ b/src/storage/lob/ob_lob_data_reader.cpp @@ -94,9 +94,7 @@ int ObLobDataReader::read_lob_data_impl(blocksstable::ObStorageDatum &datum, ObC param.snapshot_.core_ = access_ctx_->store_ctx_->mvcc_acc_ctx_.snapshot_; param.snapshot_.valid_ = true; param.snapshot_.source_ = transaction::ObTxReadSnapshot::SRC::LS; - param.snapshot_.snapshot_lsid_ = access_ctx_->store_ctx_->ls_id_; param.sql_mode_ = access_ctx_->sql_mode_; - param.ls_id_ = access_ctx_->ls_id_; param.tablet_id_ = tablet_id_; param.allocator_ = &allocator_; param.lob_common_ = const_cast(&lob_common); diff --git a/src/storage/lob/ob_lob_location.cpp b/src/storage/lob/ob_lob_location.cpp index fc478ff17..274fca811 100644 --- a/src/storage/lob/ob_lob_location.cpp +++ b/src/storage/lob/ob_lob_location.cpp @@ -25,38 +25,6 @@ namespace oceanbase namespace storage { -int ObLobLocationUtil::get_ls_leader(ObLobAccessParam& param, - const share::ObLSID &ls_id, common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - const int64_t cluster_id = GCONF.cluster_id; - if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_NOT_INIT; - LOG_WARN("location cache is NULL", K(ret)); - } else if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid args", K(ret), K(ls_id)); - } else { - uint32_t renew_count = 0; - const uint32_t max_renew_count = 10; - const int64_t retry_us = 200 * 1000; - do { - if (OB_FAIL(GCTX.location_service_->nonblock_get_leader(cluster_id, ls_id, leader))) { - LOG_WARN("failed to get location", K(ret), K(ls_id), K(cluster_id)); - } else { - LOG_DEBUG("get ls leader", K(ls_id), K(leader), K(cluster_id)); - } - } while (OB_LS_LOCATION_NOT_EXIST == ret && renew_count < max_renew_count); - - if (OB_SUCC(ret) && !leader.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("leader addr is invalid", K(ret), K(ls_id), K(leader), K(cluster_id)); - } - } - return ret; -} - int ObLobLocationUtil::is_remote(ObLobAccessParam& param, bool& is_remote, common::ObAddr& dst_addr) { int ret = OB_SUCCESS; @@ -83,9 +51,7 @@ int ObLobLocationUtil::is_remote(ObLobAccessParam& param, bool& is_remote, commo dst_addr = retry_info->addr_; } } else { - if (OB_FAIL(get_ls_leader(param, param.ls_id_, dst_addr))) { - LOG_WARN("failed to get ls leader", K(ret), K(param.ls_id_)); - } + dst_addr = self_addr; } if (OB_SUCC(ret)) { // lob from other tenant also should read by rpc @@ -172,12 +138,6 @@ int ObLobLocationUtil::lob_refresh_location(ObLobAccessParam ¶m, int last_er LOG_WARN("tablet id is changed", K(ret), K(tablet_loc), K(param), KPC(location_info)); } else { param.addr_ = tablet_loc.server_; - if (tablet_loc.ls_id_ != param.ls_id_) { - LOG_INFO("[LOB RETRY] lob retry find tablet ls id is changed", - K(param.tablet_id_), K(param.ls_id_), K(tablet_loc.ls_id_), K(last_err), K(retry_cnt)); - param.ls_id_ = tablet_loc.ls_id_; - location_info->ls_id_ = tablet_loc.ls_id_.id(); - } } } LOG_TRACE("[LOB RETRY] after do fresh location", K(ret), K(last_err), K(retry_cnt), K(has_retry_info), K(param)); @@ -187,41 +147,8 @@ int ObLobLocationUtil::lob_refresh_location(ObLobAccessParam ¶m, int last_er int ObLobLocationUtil::get_ls_leader(ObLobAccessParam& param) { - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - const int64_t cluster_id = GCONF.cluster_id; - const share::ObLSID &ls_id = param.ls_id_; - ObAddr leader_addr; - - if (OB_ISNULL(GCTX.location_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("location service is null", K(ret), K(param)); - } else if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(param)); - } else { - uint32_t renew_count = 0; - const uint32_t max_renew_count = 10; - const int64_t retry_us = 200 * 1000; - do { - if (OB_FAIL(GCTX.location_service_->nonblock_get_leader(cluster_id, ls_id, leader_addr))) { - LOG_WARN("failed to get location", K(ret), K(ls_id), K(cluster_id), K(renew_count)); - } else { - LOG_TRACE("[LOB] get ls leader", K(ls_id), K(leader_addr), K(cluster_id), K(renew_count)); - } - } while (OB_LS_LOCATION_NOT_EXIST == ret && renew_count < max_renew_count); - - - if (OB_FAIL(ret)) { - } else if (!leader_addr.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("leader addr is invalid", K(ret), K(ls_id), K(leader_addr), K(cluster_id)); - } else { - param.addr_ = leader_addr; - } - } - return ret; + param.addr_ = MYADDR; + return OB_SUCCESS; } diff --git a/src/storage/lob/ob_lob_location.h b/src/storage/lob/ob_lob_location.h index c7870341a..d45471e5c 100644 --- a/src/storage/lob/ob_lob_location.h +++ b/src/storage/lob/ob_lob_location.h @@ -28,10 +28,6 @@ namespace storage class ObLobLocationUtil { public: - static int get_ls_leader( - ObLobAccessParam& param, - const share::ObLSID &ls_id, - common::ObAddr &leader); static int is_remote(ObLobAccessParam& param, bool& is_remote, common::ObAddr& dst_addr); static int lob_check_tablet_not_exist(ObLobAccessParam ¶m, uint64_t table_id); diff --git a/src/storage/lob/ob_lob_locator.cpp b/src/storage/lob/ob_lob_locator.cpp index bee156624..0502f2095 100644 --- a/src/storage/lob/ob_lob_locator.cpp +++ b/src/storage/lob/ob_lob_locator.cpp @@ -31,7 +31,6 @@ namespace storage ObLobLocatorHelper::ObLobLocatorHelper() : table_id_(OB_INVALID_ID), tablet_id_(OB_INVALID_ID), - ls_id_(OB_INVALID_ID), tx_read_snapshot_(), fb_snapshot_(), rowid_objs_(), @@ -52,7 +51,6 @@ ObLobLocatorHelper::~ObLobLocatorHelper() int ObLobLocatorHelper::init(const ObTableScanParam &scan_param, const ObStoreCtx &ctx, - const share::ObLSID &ls_id, const int64_t snapshot_version) { int ret = OB_SUCCESS; @@ -71,7 +69,6 @@ int ObLobLocatorHelper::init(const ObTableScanParam &scan_param, } else { table_id_ = table_param.get_table_id(); tablet_id_ = scan_param.tablet_id_.id(); - ls_id_ = ls_id.id(); enable_locator_v2_ = table_param.enable_lob_locator_v2(); scan_flag_ = scan_param.scan_flag_; tx_seq_base_ = scan_param.tx_seq_base_; @@ -95,7 +92,6 @@ int ObLobLocatorHelper::init(const ObTableScanParam &scan_param, int ObLobLocatorHelper::init(const uint64_t table_id, const uint64_t tablet_id, const ObStoreCtx &ctx, - const share::ObLSID &ls_id, const int64_t snapshot_version) { int ret = OB_SUCCESS; @@ -104,14 +100,13 @@ int ObLobLocatorHelper::init(const uint64_t table_id, STORAGE_LOG(WARN, "ObLobLocatorHelper init twice", K(ret), K(*this)); } else if (OB_UNLIKELY(snapshot_version <= 0)) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to init ObLobLocatorHelper", K(ret), K(ls_id), K(snapshot_version)); - } else if (OB_FAIL(tx_read_snapshot_.build_snapshot_for_lob(ctx.mvcc_acc_ctx_.snapshot_, ls_id))) { - STORAGE_LOG(WARN, "build_snapshot_for_lob fail", K(ret), K(ls_id), K(snapshot_version), K(ctx.mvcc_acc_ctx_.snapshot_)); + STORAGE_LOG(WARN, "Invalid argument to init ObLobLocatorHelper", K(ret), K(snapshot_version)); + } else if (OB_FAIL(tx_read_snapshot_.build_snapshot_for_lob(ctx.mvcc_acc_ctx_.snapshot_))) { + STORAGE_LOG(WARN, "build_snapshot_for_lob fail", K(ret), K(snapshot_version), K(ctx.mvcc_acc_ctx_.snapshot_)); } else { // table id is only used to determine if it is a systable, this interface created locator will not construct a real rowid table_id_ = table_id; tablet_id_ = tablet_id; - ls_id_ = ls_id.id(); enable_locator_v2_ = true; // must be called en locator v2 enabled is_inited_ = true; is_access_index_ = false; @@ -265,7 +260,7 @@ int ObLobLocatorHelper::fuse_mem_lob_header(ObObj &def_obj, uint64_t col_id, boo int64_t read_snapshot_size = 0; ObString read_snapshot_data; if (extern_flags.has_read_snapshot_) { - read_snapshot_size = tx_read_snapshot_.get_serialize_size_for_lob(share::ObLSID(ls_id_), fb_snapshot_); + read_snapshot_size = tx_read_snapshot_.get_serialize_size_for_lob(fb_snapshot_); } int64_t full_loc_size = ObLobLocatorV2::calc_locator_full_len(extern_flags, @@ -294,7 +289,7 @@ int ObLobLocatorHelper::fuse_mem_lob_header(ObObj &def_obj, uint64_t col_id, boo ObMemLobTxInfo tx_info(tx_read_snapshot_.version().get_val_for_tx(), tx_read_snapshot_.tx_id().get_id(), tx_read_snapshot_.tx_seq().cast_to_int()); - ObMemLobLocationInfo location_info(tablet_id_, ls_id_, def_obj.get_collation_type()); + ObMemLobLocationInfo location_info(tablet_id_, def_obj.get_collation_type()); if (OB_FAIL(locator.set_table_info(table_id_, col_id))) { // ToDo: @gehao should be column idx STORAGE_LOG(WARN, "Lob: set table info failed", K(ret), K(table_id_), K(col_id)); } else if (extern_flags.has_tx_info_ && OB_FAIL(locator.set_tx_info(tx_info))) { @@ -306,7 +301,7 @@ int ObLobLocatorHelper::fuse_mem_lob_header(ObObj &def_obj, uint64_t col_id, boo int64_t pos = 0; if (OB_FAIL(locator.get_read_snapshot_data(read_snapshot_data))) { STORAGE_LOG(WARN, "Lob: get_read_snapshot_data failed", K(ret), K(locator)); - } else if (OB_FAIL(tx_read_snapshot_.serialize_for_lob(share::ObLSID(ls_id_), fb_snapshot_, + } else if (OB_FAIL(tx_read_snapshot_.serialize_for_lob(fb_snapshot_, read_snapshot_data.ptr(), read_snapshot_data.length(), pos))) { STORAGE_LOG(WARN, "Lob: serialize_for_lob failed", K(ret), K(locator)); } @@ -389,7 +384,7 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator, } if (OB_SUCC(ret) && extern_flags.has_read_snapshot_) { - read_snapshot_size = tx_read_snapshot_.get_serialize_size_for_lob(share::ObLSID(ls_id_), fb_snapshot_); + read_snapshot_size = tx_read_snapshot_.get_serialize_size_for_lob(fb_snapshot_); } int64_t full_loc_size = 0; @@ -443,21 +438,17 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator, // if scan with index, get data tablet id common::ObTabletID target_tablet_id(tablet_id_); if (is_access_index_) { - share::ObLSID tmp_ls_id(ls_id_); - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle tablet_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(tmp_ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(target_tablet_id, tablet_handle))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(target_tablet_id, tablet_handle))) { LOG_WARN("fail to get tablet handle", K(ret), K(target_tablet_id)); } else { target_tablet_id = tablet_handle.get_obj()->get_data_tablet_id(); } } - ObMemLobLocationInfo location_info(target_tablet_id.id(), ls_id_, cs_type); + ObMemLobLocationInfo location_info(target_tablet_id.id(), cs_type); if (OB_FAIL(ret)) { } else if (has_extern && OB_FAIL(locator.set_table_info(table_id_, column_id))) { // should be column idx STORAGE_LOG(WARN, "Lob: set table info failed", K(ret), K(table_id_), K(column_id)); @@ -473,7 +464,7 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator, int64_t pos = 0; if (OB_FAIL(locator.get_read_snapshot_data(read_snapshot_data))) { STORAGE_LOG(WARN, "Lob: get_read_snapshot_data failed", K(ret), K(locator)); - } else if (OB_FAIL(tx_read_snapshot_.serialize_for_lob(share::ObLSID(ls_id_), fb_snapshot_, + } else if (OB_FAIL(tx_read_snapshot_.serialize_for_lob(fb_snapshot_, read_snapshot_data.ptr(), read_snapshot_data.length(), pos))) { STORAGE_LOG(WARN, "Lob: serialize_for_lob failed", K(ret), K(locator)); } @@ -521,7 +512,6 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator, ObArenaAllocator tmp_lob_allocator(ObModIds::OB_LOB_READER, OB_MALLOC_NORMAL_BLOCK_SIZE); ObLobAccessParam param; param.tx_desc_ = NULL; - param.ls_id_ = share::ObLSID(ls_id_); param.sql_mode_ = access_ctx.sql_mode_; param.tablet_id_ = ObTabletID(tablet_id_); diff --git a/src/storage/lob/ob_lob_locator.h b/src/storage/lob/ob_lob_locator.h index 492436b44..cdcd23b0b 100644 --- a/src/storage/lob/ob_lob_locator.h +++ b/src/storage/lob/ob_lob_locator.h @@ -37,12 +37,10 @@ class ObLobLocatorHelper } int init(const ObTableScanParam &scan_param, const ObStoreCtx &ctx, - const share::ObLSID &ls_id, const int64_t snapshot_version); int init(const uint64_t table_id, const uint64_t tablet_id, const ObStoreCtx &ctx, - const share::ObLSID &ls_id, const int64_t snapshot_version); int fill_lob_locator(blocksstable::ObDatumRow &row, bool is_projected_row, const ObTableAccessParam &access_param); @@ -61,7 +59,7 @@ class ObLobLocatorHelper } OB_INLINE bool is_valid() const { return is_inited_; } OB_INLINE bool enable_lob_locator_v2() const { return enable_locator_v2_; } - TO_STRING_KV(K_(table_id), K_(ls_id), K_(tx_id), K_(snapshot_version), K_(tx_read_snapshot), + TO_STRING_KV(K_(table_id), K_(tx_id), K_(snapshot_version), K_(tx_read_snapshot), K_(enable_locator_v2), K_(is_inited), K_(scan_flag), K_(tx_seq_base), K_(is_access_index)); private: static const int64_t DEFAULT_LOCATOR_OBJ_ARRAY_SIZE = 8; @@ -79,7 +77,6 @@ class ObLobLocatorHelper private: uint64_t table_id_; uint64_t tablet_id_; - int64_t ls_id_; int64_t tx_id_; int64_t snapshot_version_; transaction::ObTxReadSnapshot tx_read_snapshot_; diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index 2a4c7938f..de5c57079 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -1945,7 +1945,6 @@ int ObLobManager::build_lob_param(ObLobAccessParam& param, LOG_WARN("failed to get tx info", K(ret), K(lob)); } else if (OB_FAIL(lob.get_location_info(location_info))) { LOG_WARN("failed to get location info", K(ret), K(lob)); - } else if (OB_FALSE_IT(param.ls_id_ = share::ObLSID(location_info->ls_id_))) { } else if (OB_FALSE_IT(param.tablet_id_ = ObTabletID(location_info->tablet_id_))) { } else if (OB_FAIL(param.set_tx_read_snapshot(lob))) { LOG_WARN("set_tx_read_snapshot fail", K(ret), K(param), K(lob)); @@ -2094,12 +2093,6 @@ int ObLobManager::prepare_insert_task( LOG_WARN("fail to get lob byte len", K(ret), K(task)); } else if (new_byte_len <= lob_inrow_threshold) { // skip if inrow store - } else if (param.is_mlog_) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "columns more than lob inrow threshold in materialized view log is"); - LOG_WARN("columns more than lob inrow threshold in materialized view log is not supported ", - KR(ret), K(new_byte_len), K(lob_inrow_threshold), K(lbt())); } else if (OB_FAIL(prepare_outrow_locator(param, task))) { LOG_WARN("prepare_outrow_locator fail", K(ret), K(task), K(param)); } else { @@ -2326,7 +2319,7 @@ int ObLobManager::get_outrow_lob_full_data(common::ObLobTextIterCtx &ctx, if (OB_SUCC(init_lob_access_param(*this, param, &ctx, cs_type, tmp_alloc))) { param.len_ = (ctx.total_access_len_ == 0 ? param.byte_size_ : ctx.total_access_len_); - if (!param.ls_id_.is_valid() || !param.tablet_id_.is_valid()) { + if (!param.tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid param.", K(ret), K(param)); } else if (param.byte_size_ == 0) { @@ -2400,7 +2393,7 @@ int ObLobManager::get_delta_lob_full_data(common::ObLobTextIterCtx &ctx, ctx.locator_ = partial_data.locator_; if (OB_FAIL(init_lob_access_param(*this, param, &ctx, cs_type, allocator))) { COMMON_LOG(WARN, "init_lob_access_param fail", K(ret)); - } else if (!param.ls_id_.is_valid() || !param.tablet_id_.is_valid()) { + } else if (!param.tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid param.", K(ret), K(param)); } else if ((param.len_ = param.byte_size_) <= 0) { @@ -2461,7 +2454,7 @@ int ObLobManager::get_outrow_prefix_data(common::ObLobTextIterCtx &ctx, if (OB_SUCC(init_lob_access_param(*this, param, &ctx, cs_type, tmp_alloc))) { param.len_ = prefix_char_len; - if (!param.ls_id_.is_valid() || !param.tablet_id_.is_valid()) { + if (!param.tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid param.", K(ret), K(param)); } else if (param.byte_size_ < 0 || param.len_ == 0) { @@ -2524,7 +2517,7 @@ int ObLobManager::get_first_block(common::ObLobTextIterCtx &ctx, } } - if (!param.ls_id_.is_valid() || !param.tablet_id_.is_valid()) { + if (!param.tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid param.", K(ret), K(param)); } else if (param.byte_size_ == 0) { @@ -2639,7 +2632,7 @@ int ObLobManager::get_outrow_char_len(common::ObLobTextIterCtx &ctx, storage::ObLobAccessParam param; if (OB_SUCC(init_lob_access_param(*this, param, &ctx, cs_type, tmp_alloc))) { uint64_t length = 0; - if (!param.ls_id_.is_valid() || !param.tablet_id_.is_valid()) { + if (!param.tablet_id_.is_valid()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid param.", K(ret), K(param)); } else if (OB_FAIL(getlength(param, length))) { diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 3ebbf761b..8ba416954 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -54,7 +54,7 @@ int ObLobMetaScanIter::open_remote(ObLobAccessParam ¶m) cur_pos_ = 0; cur_byte_pos_ = 0; is_remote_ = true; - if (OB_FAIL(ObLobRemoteUtil::query(param, ObLobQueryArg::QueryType::READ, param.addr_, remote_ctx_))) { + if (OB_FAIL(ObLobRemoteUtil::query(param, obcall::ObLobQueryArg::QueryType::READ, param.addr_, remote_ctx_))) { LOG_WARN("fail to init remote query ctx", K(ret)); } return ret; diff --git a/src/storage/lob/ob_lob_persistent_adaptor.cpp b/src/storage/lob/ob_lob_persistent_adaptor.cpp index 8bf3017ac..645d6a383 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.cpp +++ b/src/storage/lob/ob_lob_persistent_adaptor.cpp @@ -206,25 +206,22 @@ int ObPersistentLobApator::fetch_lob_id_for_split_src(const ObLobAccessParam& pa { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle tablet_handle; ObTabletID dst_tablet_id; share::ObTabletAutoincrementService &auto_inc = share::ObTabletAutoincrementService::get_instance(); if (OB_ISNULL(param.data_row_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid data row", K(ret), K(lob_tablet_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(param.ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_with_timeout(lob_tablet_id, + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_with_timeout(lob_tablet_id, tablet_handle, param.timeout_, ObMDSGetTabletMode::READ_ALL_COMMITED, share::SCN::max_scn()))) { LOG_WARN("fail to get tablet handle", K(ret), K(lob_tablet_id), K(param)); - } else if (OB_FAIL(ObTabletSplitMdsHelper::calc_split_dst_lob(*ls_handle.get_ls(), *tablet_handle.get_obj(), *param.data_row_, param.timeout_, dst_tablet_id))) { + } else if (OB_FAIL(ObTabletSplitMdsHelper::calc_split_dst_lob(*tenant_ls, *tablet_handle.get_obj(), *param.data_row_, param.timeout_, dst_tablet_id))) { LOG_WARN("failed to calc split dst tablet", K(ret)); } else if (OB_FAIL(auto_inc.get_autoinc_seq(dst_tablet_id, lob_id, share::ObTabletAutoincrementService::LOB_CACHE_SIZE))) { LOG_WARN("get lob_id fail", K(ret), K(dst_tablet_id)); @@ -259,8 +256,7 @@ int ObPersistentLobApator::prepare_lob_meta_dml(ObLobAccessParam& param) if (OB_SUCC(ret)) { param.dml_base_param_->store_ctx_guard_->reset(); ObAccessService *oas = share::g_mp->access_service(); - if (OB_FAIL(oas->get_write_store_ctx_guard(param.ls_id_, - param.timeout_, + if (OB_FAIL(oas->get_write_store_ctx_guard(param.timeout_, *param.tx_desc_, param.snapshot_, 0,/*branch_id*/ @@ -350,7 +346,6 @@ int ObPersistentLobApator::build_common_scan_param( } if (OB_SUCC(ret)) { - scan_param.ls_id_ = param.ls_id_; scan_param.tablet_id_ = param.lob_meta_tablet_id_; scan_param.reserved_cell_count_ = scan_param.column_ids_.count(); @@ -410,14 +405,11 @@ int ObPersistentLobApator::inner_get_tablet( ObTabletHandle &handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(param.ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret)); - } else if (OB_FAIL(inner_get_tablet(param, tablet_id, ls_handle, handle))) { - LOG_WARN("failed to get tablet", K(ret), K(param.ls_id_), K(tablet_id)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(inner_get_tablet(param, tablet_id, tenant_ls, handle))) { + LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } return ret; } @@ -425,17 +417,17 @@ int ObPersistentLobApator::inner_get_tablet( int ObPersistentLobApator::inner_get_tablet( const ObLobAccessParam ¶m, const common::ObTabletID &tablet_id, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &handle) { int ret = OB_SUCCESS; - if (OB_FAIL(ls_handle.get_ls()->get_tablet_with_timeout(tablet_id, + if (OB_FAIL(tenant_ls->get_tablet_with_timeout(tablet_id, handle, param.timeout_, ObMDSGetTabletMode::READ_READABLE_COMMITED, param.snapshot_.core_.version_))) { if (OB_TABLET_IS_SPLIT_SRC == ret) { - if (OB_FAIL(ls_handle.get_ls()->get_tablet_with_timeout(tablet_id, + if (OB_FAIL(tenant_ls->get_tablet_with_timeout(tablet_id, handle, param.timeout_, ObMDSGetTabletMode::READ_ALL_COMMITED, @@ -507,7 +499,6 @@ int ObPersistentLobApator::scan_with_ctx( ObPersistLobReaderCacheKey key; ObLobMetaIterator *reader = nullptr; ObPersistLobReaderCache &cache = param.access_ctx_->reader_cache_; - key.ls_id_ = param.ls_id_; key.snapshot_ = param.snapshot_.core_.version_.get_val_for_tx(); key.tablet_id_ = param.tablet_id_; key.is_get_ = param.has_single_chunk(); @@ -560,24 +551,21 @@ int ObPersistentLobApator::prepare_scan_param_schema_version( int ObPersistentLobApator::prepare_lob_tablet_id(ObLobAccessParam& param) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle data_tablet; ObTabletBindingMdsUserData ddl_data; if (! param.tablet_id_.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet_id of main table is invalid", K(ret), K(param)); } else if (param.lob_meta_tablet_id_.is_valid() && param.lob_piece_tablet_id_.is_valid()) { - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(param.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(param.ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be null", K(ret)); - } else if (OB_FAIL(inner_get_tablet(param, param.tablet_id_, ls_handle, data_tablet))) { - LOG_WARN("failed to get tablet", K(ret), K(param.ls_id_), K(param.tablet_id_)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(inner_get_tablet(param, param.tablet_id_, tenant_ls, data_tablet))) { + LOG_WARN("failed to get tablet", K(ret), K(param.tablet_id_)); } else if (OB_FAIL(data_tablet.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); } else if (OB_UNLIKELY(check_lob_tablet_id(param.tablet_id_, ddl_data.lob_meta_tablet_id_, ddl_data.lob_piece_tablet_id_))) { - if (ls_handle.get_ls()->is_offline()) { + if (tenant_ls->is_offline()) { ret = OB_LS_OFFLINE; LOG_WARN("ls is offline, can not get lob tablet id", K(ret), K(param), K(ddl_data.lob_meta_tablet_id_), K(ddl_data.lob_piece_tablet_id_)); } else { @@ -633,9 +621,7 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObDatumRowIte } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(oas->delete_rows( - param.ls_id_, - param.lob_meta_tablet_id_, + } else if (OB_FAIL(oas->delete_rows(param.lob_meta_tablet_id_, *param.tx_desc_, *param.dml_base_param_, column_ids, @@ -670,9 +656,7 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObDatumRowIte } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(oas->insert_rows( - param.ls_id_, - param.lob_meta_tablet_id_, + } else if (OB_FAIL(oas->insert_rows(param.lob_meta_tablet_id_, *param.tx_desc_, *param.dml_base_param_, column_ids, @@ -713,9 +697,7 @@ int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObDatumRowIt } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(oas->update_rows( - param.ls_id_, - param.lob_meta_tablet_id_, + } else if (OB_FAIL(oas->update_rows(param.lob_meta_tablet_id_, *param.tx_desc_, *param.dml_base_param_, column_ids, @@ -782,4 +764,3 @@ int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInf } // storage } // oceanbase - diff --git a/src/storage/lob/ob_lob_persistent_adaptor.h b/src/storage/lob/ob_lob_persistent_adaptor.h index b4b74106e..877776ac1 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.h +++ b/src/storage/lob/ob_lob_persistent_adaptor.h @@ -101,7 +101,7 @@ class ObPersistentLobApator : public ObILobApator int inner_get_tablet( const ObLobAccessParam ¶m, const common::ObTabletID &tablet_id, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &handle); bool check_lob_tablet_id( @@ -154,4 +154,3 @@ class ObPersistentLobApator : public ObILobApator #endif - diff --git a/src/storage/lob/ob_lob_persistent_reader.h b/src/storage/lob/ob_lob_persistent_reader.h index c5029a8e7..7d0d04186 100644 --- a/src/storage/lob/ob_lob_persistent_reader.h +++ b/src/storage/lob/ob_lob_persistent_reader.h @@ -30,22 +30,20 @@ class ObLobMetaIterator; struct ObPersistLobReaderCacheKey { ObPersistLobReaderCacheKey(): - ls_id_(), tablet_id_(), snapshot_(0), is_get_(false) {} - share::ObLSID ls_id_; common::ObTabletID tablet_id_; int64_t snapshot_; bool is_get_; bool operator==(const ObPersistLobReaderCacheKey &other) const { - return snapshot_ == other.snapshot_ && tablet_id_ == other.tablet_id_ && ls_id_ == other.ls_id_ && is_get_ == other.is_get_; + return snapshot_ == other.snapshot_ && tablet_id_ == other.tablet_id_ && is_get_ == other.is_get_; } - TO_STRING_KV(K(ls_id_), K(tablet_id_), K(snapshot_)); + TO_STRING_KV(K(tablet_id_), K(snapshot_)); }; struct ObPersistLobReaderCacheNode : public ObDLinkBase diff --git a/src/storage/lob/ob_lob_tablet_dml.cpp b/src/storage/lob/ob_lob_tablet_dml.cpp index 971ff2ac8..26cd6789f 100644 --- a/src/storage/lob/ob_lob_tablet_dml.cpp +++ b/src/storage/lob/ob_lob_tablet_dml.cpp @@ -55,11 +55,8 @@ int ObLobTabletDmlHelper::build_common_lob_param_for_dml( lob_param.tx_desc_ = run_ctx.store_ctx_.mvcc_acc_ctx_.tx_desc_; lob_param.parent_seq_no_ = run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_; lob_param.tx_id_ = lob_param.tx_desc_->get_tx_id(); - lob_param.is_mlog_ = run_ctx.dml_param_.table_param_->get_data_table().is_mlog_table(); - lob_param.sql_mode_ = run_ctx.dml_param_.sql_mode_; lob_param.is_total_quantity_log_ = run_ctx.dml_param_.is_total_quantity_log_; - lob_param.ls_id_ = run_ctx.store_ctx_.ls_id_; lob_param.tablet_id_ = run_ctx.relative_table_.get_tablet_id(); lob_param.lob_meta_tablet_id_ = run_ctx.lob_dml_ctx_.lob_meta_tablet_id_; lob_param.lob_piece_tablet_id_ = run_ctx.lob_dml_ctx_.lob_piece_tablet_id_; @@ -80,8 +77,7 @@ int ObLobTabletDmlHelper::build_common_lob_param_for_dml( // NOTE: // lob_insert need table_scan, the snapshot already generated in // run_ctx.store_ctx, use it as an LS ReadSnapshot - lob_param.snapshot_.init_ls_read(run_ctx.store_ctx_.ls_id_, - run_ctx.store_ctx_.mvcc_acc_ctx_.snapshot_); + lob_param.snapshot_.init_ls_read(run_ctx.store_ctx_.mvcc_acc_ctx_.snapshot_); } return ret; } diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index f9d0ca385..b7b6c83bd 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -48,8 +48,7 @@ void ObLobCharsetUtil::transform_query_result_charset( byte_len = ObCharset::charpos(coll_type, data + byte_st, len - byte_st, byte_len); } -int ObInsertLobColumnHelper::start_trans(const share::ObLSID &ls_id, - const bool is_for_read, +int ObInsertLobColumnHelper::start_trans(const bool is_for_read, const int64_t timeout_ts, ObTxDesc *&tx_desc) { @@ -103,7 +102,6 @@ int ObInsertLobColumnHelper::end_trans(transaction::ObTxDesc *tx_desc, } int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObObjType &obj_type, const ObCollationType &cs_type, @@ -146,9 +144,9 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, } } } else { - if (OB_FAIL(start_trans(ls_id, false/*is_for_read*/, timeout_ts, tx_desc))) { + if (OB_FAIL(start_trans(false/*is_for_read*/, timeout_ts, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ls_id, timeout_ts, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout_ts, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else { // 4.0 text tc compatiable @@ -159,7 +157,6 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, LOG_WARN("assign snapshot fail", K(ret)); } else { lob_param.sql_mode_ = SMO_DEFAULT; - lob_param.ls_id_ = ls_id; lob_param.tablet_id_ = tablet_id; lob_param.coll_type_ = ObLobCharsetUtil::get_collation_type(obj_type, cs_type); lob_param.allocator_ = &allocator; @@ -196,7 +193,6 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, ObIAllocator &lob_allocator, transaction::ObTxDesc *tx_desc, share::ObTabletCacheInterval &lob_id_geneator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const common::ObTabletID lob_meta_tablet_id, const ObObjType &obj_type, @@ -255,7 +251,6 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, lob_param.tx_desc_ = tx_desc; // lob_param.snapshot_ = snapshot; lob_param.sql_mode_ = SMO_DEFAULT; - lob_param.ls_id_ = ls_id; lob_param.tablet_id_ = tablet_id; lob_param.coll_type_ = ObLobCharsetUtil::get_collation_type(obj_type, collation_type); lob_param.allocator_ = &allocator; @@ -284,7 +279,6 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, } int ObInsertLobColumnHelper::delete_lob_column(ObIAllocator &allocator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObCollationType& collation_type, blocksstable::ObStorageDatum &datum, @@ -310,9 +304,9 @@ int ObInsertLobColumnHelper::delete_lob_column(ObIAllocator &allocator, if (lob.has_inrow_data()) { // delete inrow lob no need to use the lob manager } else { - if (OB_FAIL(start_trans(ls_id, false/*is_for_read*/, timeout_ts, tx_desc))) { + if (OB_FAIL(start_trans(false/*is_for_read*/, timeout_ts, tx_desc))) { LOG_WARN("fail to get tx_desc", K(ret)); - } else if (OB_FAIL(txs->get_ls_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, ls_id, timeout_ts, snapshot))) { + } else if (OB_FAIL(txs->get_read_snapshot(*tx_desc, transaction::ObTxIsolationLevel::RC, timeout_ts, snapshot))) { LOG_WARN("fail to get snapshot", K(ret)); } else { // 4.0 text tc compatiable diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index be5ba1dc7..b1845579c 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -147,8 +147,7 @@ class ObInsertLobColumnHelper final static const uint64_t LOB_ACCESS_TX_TIMEOUT = 60000000; // 60s static const uint64_t LOB_ALLOCATOR_RESET_CYCLE = 128; public: - static int start_trans(const share::ObLSID &ls_id, - const bool is_for_read, + static int start_trans(const bool is_for_read, const int64_t timeout_ts, transaction::ObTxDesc *&tx_desc); static int end_trans(transaction::ObTxDesc *tx_desc, @@ -156,7 +155,6 @@ class ObInsertLobColumnHelper final const int64_t timeout_ts); static int insert_lob_column(ObIAllocator &allocator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObObjType &obj_type, const ObCollationType &cs_type, @@ -165,7 +163,6 @@ class ObInsertLobColumnHelper final const int64_t timeout_ts, const bool has_lob_header); static int delete_lob_column(ObIAllocator &allocator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObCollationType& collation_type, blocksstable::ObStorageDatum &datum, @@ -180,7 +177,6 @@ class ObInsertLobColumnHelper final ObIAllocator &lob_allocator, transaction::ObTxDesc *tx_desc, share::ObTabletCacheInterval &lob_id_geneator, - const share::ObLSID ls_id, const common::ObTabletID tablet_id, const common::ObTabletID lob_meta_tablet_id, const ObObjType &obj_type, @@ -276,5 +272,3 @@ struct ObLobPartialData } // oceanbase #endif - - diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index 3caa445df..29e16017b 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -452,12 +452,6 @@ int ObFreezer::init(ObLS *ls) return ret; } -/* ls info */ -share::ObLSID ObFreezer::get_ls_id() -{ - return OB_ISNULL(ls_) ? INVALID_LS : ls_->get_ls_id(); -} - checkpoint::ObDataCheckpoint* ObFreezer::get_ls_data_checkpoint() { return OB_ISNULL(ls_) ? nullptr : ls_->get_data_checkpoint(); @@ -489,31 +483,30 @@ int ObFreezer::logstream_freeze() int ret = OB_SUCCESS; SCN freeze_snapshot_version; SCN max_decided_scn; - share::ObLSID ls_id = get_ls_id(); - FLOG_INFO("[Freezer] logstream_freeze start", K(ret), K(ls_id)); + FLOG_INFO("[Freezer] logstream_freeze start", K(ret)); int64_t start_time = ObClockGenerator::getClock(); ObLSFreezeGuard guard(*this); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "[Freezer] not inited", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "[Freezer] not inited", K(ret)); } else if (OB_UNLIKELY(!enable_)) { ret = OB_NOT_RUNNING; - STORAGE_LOG(ERROR, "freezer is offline, can not freeze now", K(ret), K(ls_id)); + STORAGE_LOG(ERROR, "freezer is offline, can not freeze now", K(ret)); } else if (OB_FAIL(decide_max_decided_scn(max_decided_scn))) { - STORAGE_LOG(WARN, "[Freezer] decide max decided log ts failure", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "[Freezer] decide max decided log ts failure", K(ret)); } else if (OB_FAIL(get_ls_weak_read_scn(freeze_snapshot_version))) { - STORAGE_LOG(WARN, "[Freezer] get ls weak read ts failure", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "[Freezer] get ls weak read ts failure", K(ret)); } else if (freeze_snapshot_version.is_max() || ObScnRange::MIN_SCN >= freeze_snapshot_version) { ret = OB_MINOR_FREEZE_NOT_ALLOW; - STORAGE_LOG(WARN, "[Freezer] invalid weak read scn", K(ret), K(ls_id), K(freeze_snapshot_version)); + STORAGE_LOG(WARN, "[Freezer] invalid weak read scn", K(ret), K(freeze_snapshot_version)); #ifdef ERRSIM } else if (OB_FAIL(ret = ERRSIM_FREEZER_FREEZE_FAILURE)) { LOG_ERROR("[Freezer] errsim failure during freezer freeze", K(ret)); #endif } else if (OB_FAIL(set_freeze_flag())) { - FLOG_INFO("[Freezer] freeze is running", K(ret), K(ls_id)); + FLOG_INFO("[Freezer] freeze is running", K(ret)); } else { // ATTENTION! : once freeze_flag is set, ret code must be OB_SUCCESS max_decided_scn_ = max_decided_scn; @@ -552,7 +545,7 @@ void ObFreezer::submit_checkpoint_task() if (OB_FAIL(get_ls_data_checkpoint()->ls_freeze(SCN::max_scn()))) { TRANS_LOG(WARN, "[Freezer] data_checkpoint freeze failed", K(ret)); stat_.add_diagnose_info("data_checkpoint freeze failed"); - ob_throttle_usleep(100, 0, get_ls_id().id()); + ob_throttle_usleep(100, 0); } } while (OB_FAIL(ret)); } @@ -560,46 +553,29 @@ void ObFreezer::submit_checkpoint_task() int ObFreezer::wait_ls_freeze_finish() { int ret = OB_SUCCESS; - const share::ObLSID ls_id = get_ls_id(); const int64_t start_time = ObClockGenerator::getClock(); uint32_t freeze_clock = get_freeze_clock(); - { - PendTenantReplayHelper pend_replay_helper(*this, ls_); - (void)pend_replay_helper.set_skip_throttle_flag(); - - TRANS_LOG(INFO, - "[Freezer] wait freeze : Logstream ", - K(ls_id), - K(freeze_clock), - K(throttle_is_skipping_), - K(tenant_replay_is_pending_)); - - // wait till all memtables are moved from frozen_list to prepare_list - // this means that all memtables can be dumped - int64_t time_counter = 0; - while (!get_ls_data_checkpoint()->ls_freeze_finished()) { - if (TC_REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { - ++time_counter; - - // check pend condition every second - (void)pend_replay_helper.check_pend_condition_once(); - - // check resubmit log condition and report some debug info every 5 seconds - if (time_counter >= 5 && time_counter % 5 == 0) { - (void)resubmit_log_if_needed_(start_time, false /* is_tablet_freeze */, false /* is_try */); - } + TRANS_LOG(INFO, "[Freezer] wait freeze : Logstream ", K(freeze_clock)); + + // wait till all memtables are moved from frozen_list to prepare_list + // this means that all memtables can be dumped + int64_t time_counter = 0; + while (!get_ls_data_checkpoint()->ls_freeze_finished()) { + if (TC_REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { + ++time_counter; + if (time_counter >= 5 && time_counter % 5 == 0) { + (void)resubmit_log_if_needed_(start_time, false /* is_tablet_freeze */, false /* is_try */); } - ob_throttle_usleep(100, ret, get_ls_id().id()); } - - stat_.add_diagnose_info("logstream_freeze success"); - stat_.end_set_freeze_stat(ObFreezeState::FINISH, ObClockGenerator::getClock(), ret); - unset_freeze_(); - const int64_t wait_freeze_finish_spend_time = ObClockGenerator::getClock() - start_time; - FLOG_INFO("[Freezer] logstream_freeze success", K(ls_id), K(freeze_clock), K(wait_freeze_finish_spend_time)); + ob_throttle_usleep(100, ret); } - // freeze tx data out of PendTenantReplayGuard + stat_.add_diagnose_info("logstream_freeze success"); + stat_.end_set_freeze_stat(ObFreezeState::FINISH, ObClockGenerator::getClock(), ret); + unset_freeze_(); + const int64_t wait_freeze_finish_spend_time = ObClockGenerator::getClock() - start_time; + FLOG_INFO("[Freezer] logstream_freeze success", K(freeze_clock), K(wait_freeze_finish_spend_time)); + (void)try_freeze_tx_data_(); return ret; } @@ -609,7 +585,6 @@ void ObFreezer::resubmit_log_if_needed_(const int64_t start_time, const bool is_try, const ObITabletMemtable *memtable) { - const share::ObLSID ls_id = get_ls_id(); int64_t last_submit_log_time = start_time; if (need_resubmit_log() || // In order to prevent the txn has already passed the try_submit test @@ -626,7 +601,6 @@ void ObFreezer::resubmit_log_if_needed_(const int64_t start_time, TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "[Freezer] wait freeze finish costs too much time", - K(ls_id), K(is_tablet_freeze), K(is_try), K(cost_time), @@ -639,31 +613,22 @@ void ObFreezer::resubmit_log_if_needed_(const int64_t start_time, // Define a functor to avoid using lambda struct AsyncFreezeFunctor { const bool is_ls_freeze_; - const ObLSID ls_id_; ObFreezer *freezer_; - // hold ls handle to avoid logstream being destroyed - AsyncFreezeFunctor(const bool is_ls_freeze, ObFreezer *freezer, ObLSHandle &ls_handle) + AsyncFreezeFunctor(const bool is_ls_freeze, ObFreezer *freezer) : is_ls_freeze_(is_ls_freeze), - ls_id_(freezer->get_ls_id()), freezer_(freezer) {} int operator()() { int ret = OB_SUCCESS; common::ObDIActionGuard ag1("OccamThreadPool", "AsyncFreezer", "detect task"); - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_id_)); + // freezer_ belongs to the tenant LS, which outlives the tenant freezer task pool. + STORAGE_LOG(INFO, "[Freezer] An Async Freeze Task Start", K(is_ls_freeze_), KP(freezer_)); + if (is_ls_freeze_) { + common::ObDIActionGuard(common::ObDIActionGuard::NS_ACTION, "LSFreeze"); + (void)freezer_->async_ls_freeze_consumer(); } else { - // freezer_ cannot be nullptr because AsyncFreezeFunctor is constructed by ObFreezer::this pointer - STORAGE_LOG( - INFO, "[Freezer] An Async Freeze Task Start", K(ls_id_), K(is_ls_freeze_), KP(freezer_)); - if (is_ls_freeze_) { - common::ObDIActionGuard(common::ObDIActionGuard::NS_ACTION, "LSFreeze:%ld", ls_id_.id()); - (void)freezer_->async_ls_freeze_consumer(); - } else { - common::ObDIActionGuard(common::ObDIActionGuard::NS_ACTION, "TabletFreeze:%ld", ls_id_.id()); - (void)freezer_->async_tablet_freeze_consumer(); - } + common::ObDIActionGuard(common::ObDIActionGuard::NS_ACTION, "TabletFreeze"); + (void)freezer_->async_tablet_freeze_consumer(); } return ret; } @@ -684,41 +649,35 @@ void ObFreezer::submit_an_async_freeze_task(const bool is_ls_freeze) { int ret = OB_SUCCESS; ObTenantFreezer *tenant_freezer = nullptr; - share::ObLSID ls_id = get_ls_id(); - const int64_t start_time = ObClockGenerator::getClock(); bool submit_succ = false; if (OB_UNLIKELY(!enable_) || OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_RUNNING; - LOG_ERROR("freezer is offline, can not freeze now", K(ret), K(ls_id)); + LOG_ERROR("freezer is offline, can not freeze now", K(ret)); } else if (OB_ISNULL(tenant_freezer = share::g_mp->tenant_freezer())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ObTenantFreezer is null", K(ret), K(ls_id)); + TRANS_LOG(WARN, "ObTenantFreezer is null", K(ret)); } else if (async_freeze_task_already_exists_(is_ls_freeze)) { submit_succ = false; } else { ObSpinLockGuard freeze_thread_pool(tenant_freezer->freeze_thread_pool_lock_); - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_id)); - } else if (acquired_exec_async_task_permission_(is_ls_freeze)) { - AsyncFreezeFunctor async_freeze_functor(is_ls_freeze, this, ls_handle); + if (acquired_exec_async_task_permission_(is_ls_freeze)) { + AsyncFreezeFunctor async_freeze_functor(is_ls_freeze, this); do { ret = tenant_freezer->freeze_thread_pool_.commit_task_ignore_ret(async_freeze_functor); if (OB_FAIL(ret) && REACH_TIME_INTERVAL(5LL * 1000LL * 1000LL)) { - STORAGE_LOG(WARN, "commit task to freeze thread pool failed", KR(ret), K(ls_id)); + STORAGE_LOG(WARN, "commit task to freeze thread pool failed", KR(ret)); } } while (OB_FAIL(ret)); submit_succ = true; - STORAGE_LOG(INFO, "finish submit async freeze task", KR(ret), K(ls_id), K(is_ls_freeze), K(submit_succ)); + STORAGE_LOG(INFO, "finish submit async freeze task", KR(ret), K(is_ls_freeze), K(submit_succ)); } } if (!submit_succ && REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { TRANS_LOG(INFO, "async freeze task already exists, skip submitting one task", - K(ls_id), K(is_ls_freeze), K(is_async_tablet_freeze_task_existing_), K(is_async_ls_freeze_task_existing_)); @@ -751,7 +710,7 @@ void ObFreezer::async_ls_freeze_consumer() { int ret = OB_SUCCESS; if (OB_FAIL(ls_->logstream_freeze_task(INT64_MAX))) { - STORAGE_LOG(WARN, "async ls freeze failed", KR(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "async ls freeze failed", KR(ret)); } // reset task existing flag @@ -761,7 +720,7 @@ void ObFreezer::async_ls_freeze_consumer() void ObFreezer::async_tablet_freeze_consumer() { const int64_t start_time = ObClockGenerator::getClock(); - STORAGE_LOG(INFO, "Async Tablet Freeze Task Start", K(get_ls_id())); + STORAGE_LOG(INFO, "Async Tablet Freeze Task Start"); ObSEArray tablet_ids; tablet_ids.reuse(); @@ -785,7 +744,7 @@ void ObFreezer::async_tablet_freeze_consumer() // print some debug info const int64_t end_time = ObClockGenerator::getClock(); const int64_t spend_time_ms = (end_time - start_time) / 1000; - STORAGE_LOG(INFO, "Async Tablet Freeze Task finish", K(get_ls_id()), K(spend_time_ms)); + STORAGE_LOG(INFO, "Async Tablet Freeze Task finish", K(spend_time_ms)); // NOTE : reset task existing flag before submit another task ATOMIC_STORE(&is_async_tablet_freeze_task_existing_, false); @@ -806,9 +765,9 @@ void ObFreezer::try_freeze_tx_data_() if (OB_EAGAIN == ret) { // sleep 100ms and retry retry_times++; - ob_throttle_usleep(100LL * 1000LL, ret, ls_->get_ls_id().id()); + ob_throttle_usleep(100LL * 1000LL, ret); } else { - STORAGE_LOG(WARN, "freeze tx data table failed", KR(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "freeze tx data table failed", KR(ret)); } } } while (OB_EAGAIN == ret && ObClockGenerator::getClock() - start_freeze_ts < MAX_RETRY_DURATION); @@ -836,9 +795,9 @@ int ObFreezer::ls_inner_tablet_freeze(const ObTabletID &tablet_id) int ret = OB_SUCCESS; if (OB_ISNULL(get_ls_tx_svr())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "[Freezer] ls_tx_svr is unexpected nullptr", KR(ret), K(get_ls_id()), K(tablet_id)); + TRANS_LOG(WARN, "[Freezer] ls_tx_svr is unexpected nullptr", KR(ret), K(tablet_id)); } else if (OB_FAIL(get_ls_tx_svr()->flush_ls_inner_tablet(tablet_id))) { - TRANS_LOG(WARN, "[Freezer] freeze ls inner tablet failed", KR(ret), K(get_ls_id()), K(tablet_id)); + TRANS_LOG(WARN, "[Freezer] freeze ls inner tablet failed", KR(ret), K(tablet_id)); } return ret; } @@ -849,7 +808,6 @@ int ObFreezer::tablet_freeze(const ObIArray &tablet_ids, ObIArray &freeze_failed_tablets) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const int64_t start_time = ObClockGenerator::getClock(); int64_t max_loop_time = 0; ObTabletID record_tablet_id; @@ -859,27 +817,27 @@ int ObFreezer::tablet_freeze(const ObIArray &tablet_ids, frozen_memtable_handles.reuse(); freeze_failed_tablets.reuse(); - FLOG_INFO("[Freezer] tablet freeze start", K(ls_id), K(tablet_ids)); + FLOG_INFO("[Freezer] tablet freeze start", K(tablet_ids)); ObTabletFreezeGuard guard(*this, try_guard); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "[Freezer] not inited", K(ret), K(ls_id), K(tablet_ids)); + STORAGE_LOG(WARN, "[Freezer] not inited", K(ret), K(tablet_ids)); } else if (OB_UNLIKELY(!enable_)) { - STORAGE_LOG(ERROR, "[Freezer] freezer is offline, can not freeze now", K(ret), K(ls_id)); + STORAGE_LOG(ERROR, "[Freezer] freezer is offline, can not freeze now", K(ret)); } else if (OB_FAIL(frozen_memtable_handles.reserve(tablet_ids.count()))) { TRANS_LOG(WARN, "fail to reserve memtable handles", K(ret), K(tablet_ids)); } else if (OB_FAIL(freeze_failed_tablets.reserve(tablet_ids.count()))) { TRANS_LOG(WARN, "fail to reserve tablet ids", K(ret), K(tablet_ids)); } else if (OB_FAIL(get_ls_weak_read_scn(freeze_snapshot_version))) { - TRANS_LOG(WARN, "[Freezer] get ls weak read scn failure", K(ret), K(ls_id)); + TRANS_LOG(WARN, "[Freezer] get ls weak read scn failure", K(ret)); } else if (ObScnRange::MAX_SCN == freeze_snapshot_version || ObScnRange::MIN_SCN >= freeze_snapshot_version) { ret = OB_MINOR_FREEZE_NOT_ALLOW; - LOG_WARN("[Freezer] invalid weak read scn", K(ret), K(ls_id)); + LOG_WARN("[Freezer] invalid weak read scn", K(ret)); } else if (try_guard && OB_FAIL(guard.try_set_tablet_freeze_begin())) { - STORAGE_LOG(WARN, "[Freezer] ls freeze is running", KR(ret), K(ls_id), K(tablet_ids)); + STORAGE_LOG(WARN, "[Freezer] ls freeze is running", KR(ret), K(tablet_ids)); } else if (OB_FAIL(loop_set_freeze_flag(max_loop_time))) { - STORAGE_LOG(WARN, "[Freezer] another freeze function is running", KR(ret), K(ls_id), K(tablet_ids)); + STORAGE_LOG(WARN, "[Freezer] another freeze function is running", KR(ret), K(tablet_ids)); } else { // freeze flag has been set freeze_snapshot_version_ = freeze_snapshot_version; @@ -897,7 +855,7 @@ int ObFreezer::tablet_freeze(const ObIArray &tablet_ids, freeze_snapshot_version, frozen_memtable_handles, freeze_failed_tablets))) { - STORAGE_LOG(WARN, "[Freezer] batch_tablet_freeze failed", K(ls_id), K(tablet_ids)); + STORAGE_LOG(WARN, "[Freezer] batch_tablet_freeze failed", K(tablet_ids)); } if (frozen_memtable_handles.empty()) { @@ -985,7 +943,6 @@ int ObFreezer::tablet_freeze_(const ObIArray &tablet_ids, STORAGE_LOG(INFO, "[Freezer] TABLET FREEZE STEP 1 : set is_tablet_freeze flag done", KR(ret), - K(get_ls_id()), K(need_rewrite_meta), K(tablet_ids), K(freeze_failed_tablets)); @@ -1000,7 +957,6 @@ int ObFreezer::set_tablet_freeze_flag_(const ObTabletID tablet_id, int ret = OB_SUCCESS; ObTablet *tablet = nullptr; ObProtectedMemtableMgrHandle *protected_handle = nullptr; - ObLSID ls_id = get_ls_id(); ObTabletHandle tablet_handle; ObTableHandleV2 frozen_memtable_handle; @@ -1018,7 +974,7 @@ int ObFreezer::set_tablet_freeze_flag_(const ObTabletID tablet_id, LOG_ERROR("[Freezer] errsim failure during freezer freeze", K(ret)); #endif } else if (OB_FAIL(protected_handle->set_is_tablet_freeze_for_active_memtable(frozen_memtable_handle))) { - (void)handle_set_tablet_freeze_failed(need_rewrite_meta, tablet_id, ls_id, tablet, freeze_snapshot_version, ret); + (void)handle_set_tablet_freeze_failed(need_rewrite_meta, tablet_id, tablet, freeze_snapshot_version, ret); } else if (!frozen_memtable_handle.is_valid()) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "[Freezer] frozen_memtable_handle is invalid", K(ret), K(tablet_id)); @@ -1037,7 +993,6 @@ int ObFreezer::set_tablet_freeze_flag_(const ObTabletID tablet_id, void ObFreezer::handle_set_tablet_freeze_failed(const bool need_rewrite_meta, const ObTabletID &tablet_id, - const ObLSID &ls_id, const ObTablet *tablet, const share::SCN freeze_snapshot_version, int &ret) @@ -1047,11 +1002,11 @@ void ObFreezer::handle_set_tablet_freeze_failed(const bool need_rewrite_meta, ret = handle_no_active_memtable_(tablet_id, tablet, freeze_snapshot_version); } else { ret = OB_SUCCESS; - TRANS_LOG(INFO, "[Freezer] no need to freeze", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(INFO, "[Freezer] no need to freeze", K(ret), K(tablet_id)); stat_.add_diagnose_info("no need to freeze"); } } else { - TRANS_LOG(ERROR, "[Freezer] fail to set is_tablet_freeze", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(ERROR, "[Freezer] fail to set is_tablet_freeze", K(ret), K(tablet_id)); stat_.add_diagnose_info("fail to set is_tablet_freeze"); } } @@ -1063,8 +1018,6 @@ int ObFreezer::decide_real_snapshot_version_(const ObTabletID &tablet_id, { int ret = OB_SUCCESS; ObTabletCreateDeleteMdsUserData user_data; - bool is_committed = false; - share::SCN reserved_scn = share::SCN::max_scn(); mds::MdsWriter writer; mds::TwoPhaseCommitState trans_stat; share::SCN trans_version; @@ -1076,20 +1029,14 @@ int ObFreezer::decide_real_snapshot_version_(const ObTabletID &tablet_id, trans_stat, trans_version))) { LOG_WARN("fail to get latest tablet status", K(ret), KPC(tablet)); - } else if (ObTabletStatus::RESERVED_4 != user_data.tablet_status_ - && ObTabletStatus::RESERVED_6 != user_data.tablet_status_) { - //do nothing - } else if (user_data.reserved_scn_.is_valid() - && share::SCN::min_scn() != user_data.reserved_scn_) { - reserved_scn = user_data.reserved_scn_; + } else if (ObTabletStatus::RESERVED_STATUS_4 == user_data.tablet_status_ + || ObTabletStatus::RESERVED_STATUS_6 == user_data.tablet_status_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("reserved tablet status is not supported", K(ret), K(user_data)); } if (OB_SUCC(ret)) { - real_snapshot_version = MIN(freeze_snapshot_version, reserved_scn); - if (real_snapshot_version != freeze_snapshot_version) { - FLOG_INFO("update tablet snapshot version changed snapshot version", - K(real_snapshot_version), K(freeze_snapshot_version), K(reserved_scn), K(user_data)); - } + real_snapshot_version = freeze_snapshot_version; } return ret; @@ -1100,7 +1047,6 @@ int ObFreezer::handle_no_active_memtable_(const ObTabletID &tablet_id, SCN freeze_snapshot_version) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); ObProtectedMemtableMgrHandle *protected_handle = NULL; if (OB_FAIL(tablet->get_protected_memtable_mgr_handle(protected_handle))) { LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet)); @@ -1125,7 +1071,6 @@ int ObFreezer::handle_no_active_memtable_(const ObTabletID &tablet_id, ObTabletMergeDagParam param; param.merge_type_ = ObMergeType::MINI_MERGE; param.merge_version_ = ObVersionRange::MIN_VERSION; - param.ls_id_ = ls_id; param.tablet_id_ = tablet_id; param.skip_get_tablet_ = true; @@ -1135,34 +1080,33 @@ int ObFreezer::handle_no_active_memtable_(const ObTabletID &tablet_id, if (OB_FAIL(tmp_mini_dag.init_by_param(¶m))) { LOG_WARN("failed to init mini dag", K(ret), K(param)); } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->check_dag_exist(&tmp_mini_dag, is_exist))) { - LOG_WARN("failed to check dag exists", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to check dag exists", K(ret), K(tablet_id)); } else if (is_exist) { // we need to wait the current mini compaction dag to complete ret = OB_EAGAIN; - LOG_WARN("exist running mini compaction dag, try later", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("exist running mini compaction dag, try later", K(ret), K(tablet_id)); } else if (OB_FAIL(get_ls_tablet_svr()->update_tablet_snapshot_version(tablet_id, freeze_snapshot_version.get_val_for_tx()))) { - LOG_WARN("failed to update tablet snapshot version", K(ret), K(ls_id), K(tablet_id), K(freeze_snapshot_version)); + LOG_WARN("failed to update tablet snapshot version", K(ret), K(tablet_id), K(freeze_snapshot_version)); } else if (OB_FAIL(tablet->get_protected_memtable_mgr_handle(protected_handle))) { LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet)); } else if (protected_handle->get_max_saved_version_from_medium_info_recorder() >= freeze_snapshot_version.get_val_for_tx()) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(compaction::ObTenantTabletScheduler::schedule_merge_dag(ls_id, - *tablet, + if (OB_TMP_FAIL(compaction::ObTenantTabletScheduler::schedule_merge_dag(*tablet, MEDIUM_MERGE, freeze_snapshot_version.get_val_for_tx(), EXEC_MODE_LOCAL))) { if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) { ret = tmp_ret; - LOG_ERROR("failed to schedule medium merge dag", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to schedule medium merge dag", K(ret), K(tablet_id)); } } else { - TRANS_LOG(INFO, "[Freezer] memtable_mgr doesn't have memtable", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(INFO, "[Freezer] memtable_mgr doesn't have memtable", K(ret), K(tablet_id)); } } } else { - TRANS_LOG(INFO, "[Freezer] memtable_mgr has memtable", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(INFO, "[Freezer] memtable_mgr has memtable", K(ret), K(tablet_id)); } return ret; } @@ -1192,7 +1136,6 @@ int ObFreezer::wait_tablet_freeze_finish(ObIArray &memtable_han ObIArray &freeze_failed_tablets) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const int64_t start_time = ObClockGenerator::getClock(); for (int i = 0; i < memtable_handles.count(); ++i) { @@ -1216,7 +1159,6 @@ int ObFreezer::wait_tablet_freeze_finish(ObIArray &memtable_han STORAGE_LOG(INFO, "[Freezer] TABLET FREEZE STEP 2 : wait tablet freeze finish done", KR(ret), - K(get_ls_id()), K(wait_freeze_finish_spend_time), K(freeze_failed_tablets)); @@ -1233,7 +1175,7 @@ int ObFreezer::inner_wait_memtable_freeze_finish_(ObTableHandleV2 &memtable_hand if (!memtable_handle.is_valid()) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "memtable cannot be null", K(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "memtable cannot be null", K(ret)); } else if (OB_FAIL(memtable_handle.get_tablet_memtable(tablet_memtable))) { STORAGE_LOG(WARN, "fail to get memtable", K(ret), K(memtable_handle)); } else if (OB_ISNULL(tablet_memtable)) { @@ -1258,10 +1200,9 @@ int ObFreezer::inner_wait_memtable_freeze_finish_(ObTableHandleV2 &memtable_hand int ObFreezer::wait_data_memtable_freeze_finish_(ObITabletMemtable *tablet_memtable) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); ObMemtable *memtable = static_cast(tablet_memtable); if (OB_FAIL(wait_memtable_ready_for_flush_(memtable))) { - TRANS_LOG(WARN, "[Freezer] fail to wait memtable ready_for_flush", K(ret), K(ls_id)); + TRANS_LOG(WARN, "[Freezer] fail to wait memtable ready_for_flush", K(ret)); } else { int64_t read_lock = LSLOCKALL; int64_t write_lock = 0; @@ -1269,11 +1210,11 @@ int ObFreezer::wait_data_memtable_freeze_finish_(ObITabletMemtable *tablet_memta if (OB_FAIL(check_ls_state())) { TRANS_LOG(WARN, "ls state invalid", KR(ret)); } else if (OB_FAIL(memtable->finish_freeze())) { - TRANS_LOG(ERROR, "[Freezer] memtable cannot be flushed", K(ret), K(ls_id), KPC(memtable)); + TRANS_LOG(ERROR, "[Freezer] memtable cannot be flushed", K(ret), KPC(memtable)); stat_.add_diagnose_info("memtable cannot be flushed"); } else { stat_.add_diagnose_info("tablet_freeze success"); - FLOG_INFO("[Freezer] tablet_freeze_task success(DataMemtable)", K(ret), K(ls_id), KPC(memtable)); + FLOG_INFO("[Freezer] tablet_freeze_task success(DataMemtable)", K(ret), KPC(memtable)); } } return ret; @@ -1282,10 +1223,9 @@ int ObFreezer::wait_data_memtable_freeze_finish_(ObITabletMemtable *tablet_memta int ObFreezer::wait_direct_load_memtable_freeze_finish_(ObITabletMemtable *tablet_memtable) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); ObDDLKV *direct_load_memtable = static_cast(tablet_memtable); if (OB_FAIL(direct_load_memtable->decide_right_boundary())) { - STORAGE_LOG(WARN, "freeze direct load memtable failed", KR(ret), K(ls_id), KPC(tablet_memtable)); + STORAGE_LOG(WARN, "freeze direct load memtable failed", KR(ret), KPC(tablet_memtable)); } else { int64_t read_lock = LSLOCKALL; int64_t write_lock = 0; @@ -1293,11 +1233,11 @@ int ObFreezer::wait_direct_load_memtable_freeze_finish_(ObITabletMemtable *table if (OB_FAIL(check_ls_state())) { TRANS_LOG(WARN, "ls state invalid", KR(ret)); } else if (OB_FAIL(tablet_memtable->finish_freeze())) { - TRANS_LOG(ERROR, "[Freezer] direct load memtable cannot be flushed", K(ret), K(ls_id), KPC(tablet_memtable)); + TRANS_LOG(ERROR, "[Freezer] direct load memtable cannot be flushed", K(ret), KPC(tablet_memtable)); stat_.add_diagnose_info("direct load memtable cannot be flushed"); } else { stat_.add_diagnose_info("tablet_freeze success"); - FLOG_INFO("[Freezer] tablet_freeze_task success(DirectLoadMemtable)", K(ls_id), KP(this), KP(tablet_memtable)); + FLOG_INFO("[Freezer] tablet_freeze_task success(DirectLoadMemtable)", KP(this), KP(tablet_memtable)); } } return ret; @@ -1306,47 +1246,31 @@ int ObFreezer::wait_direct_load_memtable_freeze_finish_(ObITabletMemtable *table int ObFreezer::wait_memtable_ready_for_flush_(ObITabletMemtable *tablet_memtable) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const int64_t start_time = ObClockGenerator::getClock(); bool ready_for_flush = false; bool is_force_released = false; - { - PendTenantReplayHelper pend_replay_helper(*this, ls_); - (void)pend_replay_helper.set_skip_throttle_flag(); + TRANS_LOG(INFO, "[Freezer] wait freeze : Tablet", KP(tablet_memtable)); - TRANS_LOG(INFO, - "[Freezer] wait freeze : Tablet", - K(ls_id), - KP(tablet_memtable), - K(throttle_is_skipping_), - K(tenant_replay_is_pending_)); - - int64_t time_counter = 0; - do { - if (OB_FAIL(check_ls_state())) { - TRANS_LOG(WARN, "ls state invalid", KR(ret)); - } else if (OB_ISNULL(tablet_memtable)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "[Freezer] memtable cannot be null", K(ret)); - } else if (FALSE_IT(ready_for_flush = tablet_memtable->ready_for_flush())) { - } else if (FALSE_IT(is_force_released = tablet_memtable->is_force_released())) { - } else if (!ready_for_flush && !is_force_released) { - if (TC_REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { - ++time_counter; - - // check pend condition every second - (void)pend_replay_helper.check_pend_condition_once(); - - // check resubmit log condition and report some debug info every 5 seconds - if (time_counter >= 5 && time_counter % 5 == 0) { - (void)resubmit_log_if_needed_(start_time, true /* is_tablet_freeze */, false /* is_try */, tablet_memtable); - } + int64_t time_counter = 0; + do { + if (OB_FAIL(check_ls_state())) { + TRANS_LOG(WARN, "ls state invalid", KR(ret)); + } else if (OB_ISNULL(tablet_memtable)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "[Freezer] memtable cannot be null", K(ret)); + } else if (FALSE_IT(ready_for_flush = tablet_memtable->ready_for_flush())) { + } else if (FALSE_IT(is_force_released = tablet_memtable->is_force_released())) { + } else if (!ready_for_flush && !is_force_released) { + if (TC_REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { + ++time_counter; + if (time_counter >= 5 && time_counter % 5 == 0) { + (void)resubmit_log_if_needed_(start_time, true /* is_tablet_freeze */, false /* is_try */, tablet_memtable); } - ob_throttle_usleep(100, 0, get_ls_id().id()); } - } while (OB_SUCC(ret) && !ready_for_flush && !is_force_released); - } + ob_throttle_usleep(100, 0); + } + } while (OB_SUCC(ret) && !ready_for_flush && !is_force_released); return ret; } @@ -1370,7 +1294,6 @@ int ObFreezer::submit_log_for_freeze(const bool is_tablet_freeze, const bool is_ { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const int64_t start = ObClockGenerator::getClock(); ObTabletID tablet_id(INT64_MAX); // used for diagnose bool trace_id_need_reset = false; @@ -1388,7 +1311,7 @@ int ObFreezer::submit_log_for_freeze(const bool is_tablet_freeze, const bool is_ ObLSLockGuard lock_ls(ls_, ls_->lock_, read_lock, write_lock); if (OB_FAIL(check_ls_state())) { // ls has died, we need break the loop for submitting log - TRANS_LOG(WARN, "ls state has die", K(ls_id)); + TRANS_LOG(WARN, "ls state has die"); break; } } @@ -1400,12 +1323,12 @@ int ObFreezer::submit_log_for_freeze(const bool is_tablet_freeze, const bool is_ if (cost_time > 1000 * 1000) { if (TC_REACH_TIME_INTERVAL(5 * 1000 * 1000)) { TRANS_LOG(WARN, "[Freezer] traverse_trans_to_submit_redo_log failed", K(ret), - K(ls_id), K(cost_time), K(fail_tx_id)); + K(cost_time), K(fail_tx_id)); FreezeDiagnoseInfo diagnose("traverse_trans_to_submit_redo_log failed, traceId:%s, errorCode:%d", ret); ObCStringHelper helper; stat_.add_diagnose_info(helper.convert(diagnose)); if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MINI_MERGE, ObDiagnoseTabletType::TYPE_MINI_MERGE, - ls_id, tablet_id, ObSuspectInfoType::SUSPECT_SUBMIT_LOG_FOR_FREEZE, + tablet_id, ObSuspectInfoType::SUSPECT_SUBMIT_LOG_FOR_FREEZE, static_cast(ret), fail_tx_id.get_id()))) { TRANS_LOG(WARN, "failed to add suspect info", K(tmp_ret)); } @@ -1413,14 +1336,14 @@ int ObFreezer::submit_log_for_freeze(const bool is_tablet_freeze, const bool is_ } } if (OB_LOG_OUTOF_DISK_SPACE == ret) { - ob_throttle_usleep(100 * 1000, ret, ls_id.id()); + ob_throttle_usleep(100 * 1000, ret); } } while (!(is_try && (ObClockGenerator::getClock() - start > 10 * 1_s)) // we break the loop if we retry with a long time with the try semantic && OB_FAIL(ret)); if (OB_SUCC(ret)) { - DEL_SUSPECT_INFO(MINI_MERGE, ls_id, tablet_id, ObDiagnoseTabletType::TYPE_MINI_MERGE); + DEL_SUSPECT_INFO(MINI_MERGE, tablet_id, ObDiagnoseTabletType::TYPE_MINI_MERGE); if (OB_FAIL(get_ls_tx_svr()->traverse_trans_to_submit_next_log())) { TRANS_LOG(WARN, "traverse trans ctx to submit next log failed", K(ret)); @@ -1438,7 +1361,6 @@ int ObFreezer::submit_log_for_freeze(const bool is_tablet_freeze, const bool is_ int ObFreezer::loop_set_freeze_flag(const int64_t max_loop_time) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const int64_t start = ObClockGenerator::getClock(); int64_t cost_time = 0; do { @@ -1448,13 +1370,13 @@ int ObFreezer::loop_set_freeze_flag(const int64_t max_loop_time) if (cost_time > max_loop_time) { break; } - ob_throttle_usleep(100, ret, ls_id.id()); + ob_throttle_usleep(100, ret); } } while (OB_FAIL(ret)); const int64_t LOG_WARN_TIME = 10LL * 1000LL * 1000LL; // 10 seconds if (cost_time > LOG_WARN_TIME) { - TRANS_LOG(WARN, "[Freezer] wait the running freeze too long time", K(ls_id), K(cost_time)); + TRANS_LOG(WARN, "[Freezer] wait the running freeze too long time", K(cost_time)); } return ret; } @@ -1462,7 +1384,6 @@ int ObFreezer::loop_set_freeze_flag(const int64_t max_loop_time) int ObFreezer::set_freeze_flag_without_inc_freeze_clock() { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); uint32_t old_v = 0; uint32_t new_v = 0; @@ -1481,7 +1402,6 @@ int ObFreezer::set_freeze_flag_without_inc_freeze_clock() int ObFreezer::set_freeze_flag() { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); uint32_t old_v = 0; uint32_t new_v = 0; @@ -1492,7 +1412,7 @@ int ObFreezer::set_freeze_flag() old_v = ATOMIC_LOAD(&freeze_flag_); if (is_freeze(old_v)) { ret = OB_EAGAIN; - TRANS_LOG(WARN, "[Freezer] freeze is running!", K(ret), K(ls_id)); + TRANS_LOG(WARN, "[Freezer] freeze is running!", K(ret)); break; } new_v = (old_v + 1) | (1 << 31); @@ -1533,23 +1453,20 @@ bool ObFreezer::is_freeze(uint32_t freeze_flag) const int ObFreezer::decide_max_decided_scn(SCN &max_decided_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); - if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("[Freezer] not inited", K(ret), K(ls_id)); + LOG_WARN("[Freezer] not inited", K(ret)); } else if (OB_FAIL(get_ls_log_handler()->get_max_decided_scn(max_decided_scn))) { if (OB_STATE_NOT_MATCH == ret) { max_decided_scn.reset(); ret = OB_SUCCESS; } else { - TRANS_LOG(WARN, "[Freezer] fail to get max_decided_scn", K(ret), K(ls_id), - K(max_decided_scn)); + TRANS_LOG(WARN, "[Freezer] fail to get max_decided_scn", K(ret), K(max_decided_scn)); } } if (OB_SUCC(ret)) { - TRANS_LOG(TRACE, "[Freezer] decide max decided log ts", K(ret), K(ls_id), K(max_decided_scn)); + TRANS_LOG(TRACE, "[Freezer] decide max decided log ts", K(ret), K(max_decided_scn)); } return ret; @@ -1558,19 +1475,18 @@ int ObFreezer::decide_max_decided_scn(SCN &max_decided_scn) int ObFreezer::get_max_consequent_callbacked_scn(SCN &max_consequent_callbacked_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("[Freezer] not inited", K(ret), K(ls_id)); + LOG_WARN("[Freezer] not inited", K(ret)); } else if (OB_FAIL(get_ls_log_handler()->get_max_decided_scn(max_consequent_callbacked_scn))) { if (OB_STATE_NOT_MATCH == ret) { max_consequent_callbacked_scn.set_min(); ret = OB_SUCCESS; } else { - TRANS_LOG(ERROR, "[Freezer] fail to get min_unreplay_scn", K(ret), K(ls_id), K(max_consequent_callbacked_scn)); + TRANS_LOG(ERROR, "[Freezer] fail to get min_unreplay_scn", K(ret), K(max_consequent_callbacked_scn)); } } else { - TRANS_LOG(TRACE, "[Freezer] get_max_decided_scn", K(ret), K(ls_id), K(max_consequent_callbacked_scn)); + TRANS_LOG(TRACE, "[Freezer] get_max_decided_scn", K(ret), K(max_consequent_callbacked_scn)); } return ret; } @@ -1578,7 +1494,6 @@ int ObFreezer::get_max_consequent_callbacked_scn(SCN &max_consequent_callbacked_ int ObFreezer::get_ls_weak_read_scn(SCN &weak_read_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); weak_read_scn.reset(); if (IS_NOT_INIT) { @@ -1586,7 +1501,7 @@ int ObFreezer::get_ls_weak_read_scn(SCN &weak_read_scn) TRANS_LOG(WARN, "[Freezer] not inited", K(ret)); } else { weak_read_scn = get_ls_wrs_handler()->get_ls_weak_read_ts(); - TRANS_LOG(TRACE, "[Freezer] get_ls_weak_read_scn", K(ret), K(ls_id), K(weak_read_scn)); + TRANS_LOG(TRACE, "[Freezer] get_ls_weak_read_scn", K(ret), K(weak_read_scn)); } return ret; @@ -1596,7 +1511,6 @@ int ObFreezer::get_newest_clog_checkpoint_scn(const ObTabletID &tablet_id, SCN &clog_checkpoint_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); ObTabletHandle handle; clog_checkpoint_scn.reset(); @@ -1605,10 +1519,10 @@ int ObFreezer::get_newest_clog_checkpoint_scn(const ObTabletID &tablet_id, TRANS_LOG(WARN, "[Freezer] not inited", K(ret)); } else if (OB_FAIL(get_ls_tablet_svr()->get_tablet(tablet_id, handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - TRANS_LOG(WARN, "[Freezer] fail to get tablet", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(WARN, "[Freezer] fail to get tablet", K(ret), K(tablet_id)); } else { clog_checkpoint_scn = handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_; - TRANS_LOG(TRACE, "[Freezer] get_newest_clog_checkpoint_scn", K(ret), K(ls_id), K(tablet_id), + TRANS_LOG(TRACE, "[Freezer] get_newest_clog_checkpoint_scn", K(ret), K(tablet_id), K(clog_checkpoint_scn)); } @@ -1619,7 +1533,6 @@ int ObFreezer::get_newest_snapshot_version(const ObTabletID &tablet_id, SCN &snapshot_version) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); ObTabletHandle handle; snapshot_version.reset(); @@ -1628,11 +1541,11 @@ int ObFreezer::get_newest_snapshot_version(const ObTabletID &tablet_id, TRANS_LOG(WARN, "[Freezer] not inited", K(ret)); } else if (OB_FAIL(get_ls_tablet_svr()->get_tablet(tablet_id, handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - TRANS_LOG(WARN, "[Freezer] fail to get tablet", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(WARN, "[Freezer] fail to get tablet", K(ret), K(tablet_id)); } else if (OB_FAIL(snapshot_version.convert_for_tx(handle.get_obj()->get_snapshot_version()))) { - TRANS_LOG(WARN, "[Freezer] fail to convert from ts", K(ret), K(ls_id), K(tablet_id)); + TRANS_LOG(WARN, "[Freezer] fail to convert from ts", K(ret), K(tablet_id)); } else { - TRANS_LOG(TRACE, "[Freezer] get_snapshot_version", K(ret), K(ls_id), K(tablet_id), K(snapshot_version)); + TRANS_LOG(TRACE, "[Freezer] get_snapshot_version", K(ret), K(tablet_id), K(snapshot_version)); } return ret; @@ -1657,7 +1570,7 @@ void ObFreezer::print_freezer_statistics() { // print every 10s if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - TRANS_LOG(INFO, "[Freezer] empty table statistics: ", K(get_ls_id()), K(get_empty_memtable_cnt())); + TRANS_LOG(INFO, "[Freezer] empty table statistics: ", K(get_empty_memtable_cnt())); clear_empty_memtable_cnt(); } } @@ -1715,14 +1628,14 @@ int ObFreezer::pend_ls_replay() common::ObByteLockGuard guard(byte_lock_, ObWaitEventIds::LS_REPLAY_CTRL_LOCK); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this)); + TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), KP(this)); } else if (pend_replay_cnt_ < 0) { - TRANS_LOG(ERROR, "[Freezer] invalid pend_replay_cnt", KR(ret), K(get_ls_id()), KP(this), K(pend_replay_cnt_)); + TRANS_LOG(ERROR, "[Freezer] invalid pend_replay_cnt", KR(ret), KP(this), K(pend_replay_cnt_)); } else if (FALSE_IT(pend_replay_cnt_++)) { } else if (1 == pend_replay_cnt_) { if (OB_FAIL(get_ls_log_handler()->pend_submit_replay_log())) { pend_replay_cnt_--; - TRANS_LOG(WARN, "[Freezer] pend ls replay failed", KR(ret), K(get_ls_id()), KP(this)); + TRANS_LOG(WARN, "[Freezer] pend ls replay failed", KR(ret), KP(this)); } } return ret; @@ -1734,21 +1647,20 @@ int ObFreezer::restore_ls_replay() common::ObByteLockGuard guard(byte_lock_, ObWaitEventIds::LS_REPLAY_CTRL_LOCK); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this)); + TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), KP(this)); } else if (0 >= pend_replay_cnt_) { TRANS_LOG(INFO, "[Freezer] no need restore replay cause pend replay may failed", KR(ret), - K(get_ls_id()), KP(this), K(pend_replay_cnt_)); } else if (FALSE_IT(pend_replay_cnt_--)) { } else if (0 == pend_replay_cnt_) { if (OB_FAIL(get_ls_log_handler()->restore_submit_replay_log())) { if (get_ls_log_handler()->is_valid()) { - TRANS_LOG(ERROR, "restore replay failed. please check if logstream is removed", KR(ret), K(get_ls_id())); + TRANS_LOG(ERROR, "restore replay failed. please check if logstream is removed", KR(ret)); } else { - TRANS_LOG(WARN, "restore replay failed. ls log handler is invalid", KR(ret), K(get_ls_id())); + TRANS_LOG(WARN, "restore replay failed. ls log handler is invalid", KR(ret)); } } } @@ -1880,135 +1792,5 @@ int ObFreezer::ObTabletFreezeGuard::try_set_tablet_freeze_begin() } -void ObFreezer::PendTenantReplayHelper::set_skip_throttle_flag() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(current_freeze_ls_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "invalid ls pointer", KP(current_freeze_ls_)); - } else if (current_ls_is_leader_()){ - // leader do not need skip throttle - } else { - // pend replay before set_throttle_is_skipping to avoid skipping too much replay - if (remain_memory_is_exhausting_()) { - (void)pend_tenant_replay_(); - } - host_.set_throttle_is_skipping(); - } -} - -bool ObFreezer::PendTenantReplayHelper::current_ls_is_leader_() -{ - int ret = OB_SUCCESS; - // set default value as leader because leader do not skip throttle - bool is_leader = true; - ObRole role; - int64_t proposal_id = 0; - if (OB_FAIL(current_freeze_ls_->get_log_handler()->get_role(role, proposal_id))) { - LOG_WARN("get ls role failed", KR(ret), K(current_freeze_ls_->get_ls_id())); - } else if (common::is_strong_leader(role)) { - is_leader = true; - } else { - is_leader = false; - } - return is_leader; -} - -void ObFreezer::PendTenantReplayHelper::check_pend_condition_once() -{ - // only check pend condition when throttle is skipping - if (host_.throttle_is_skipping()) { - if (host_.tenant_replay_is_pending()) { - if (!remain_memory_is_exhausting_()) { - (void)restore_tenant_replay_(); - } else { - // keep pending replay - } - } else { - // tenant replay is not pending - if (remain_memory_is_exhausting_()) { - (void)pend_tenant_replay_(); - } else { - // keep not pending replay - } - } - } -} - -bool ObFreezer::PendTenantReplayHelper::remain_memory_is_exhausting_() { - TxShareThrottleTool &throttle_tool = (share::g_mp->shared_mem_alloc_mgr()->share_resource_throttle_tool()); - const bool has_triggered_throttle = throttle_tool.has_triggered_throttle(); - const bool remain_memory_is_exhausting = - has_triggered_throttle || share::g_mp->tenant_freezer()->memstore_remain_memory_is_exhausting(); - STORAGE_LOG(INFO, "finish check remain memory", K(has_triggered_throttle), K(remain_memory_is_exhausting)); - - return remain_memory_is_exhausting; -} - -void ObFreezer::PendTenantReplayHelper::pend_tenant_replay_() -{ - int ret = OB_SUCCESS; - ObLSService *ls_srv = share::g_mp->ls_service(); - common::ObSharedGuard iter; - if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "[ObFreezer] fail to get ls iterator", KR(ret)); - } else { - ObLS *ls = nullptr; - ls_handle_array_.reuse(); - int64_t iterate_ls_count = 0; - int64_t pend_ls_replay_count = 0; - - // Here we need to perform a pend_ls_replay on all ls, because if a switch_leader occurs for any ls during the - // freezing process, the absence of pend_ls_replay might result in an unthrottled memory usage - while (OB_SUCC(iter->get_next(ls))) { - ObLSHandle ls_handle; - iterate_ls_count++; - if (OB_FAIL(ls_srv->get_ls(ls->get_ls_id(), ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "[ObFreezer] get ls handle failed", KR(ret), KP(ls)); - } else if (OB_FAIL(ls_handle_array_.push_back(ls_handle))) { - STORAGE_LOG(WARN, "[ObFreezer] push back ls handle failed", KR(ret), KP(ls)); - } else if (OB_FAIL(ls->get_freezer()->pend_ls_replay())) { - STORAGE_LOG(WARN, "[ObFreezer] pend replay failed", KR(ret), KPC(ls)); - (void)ls_handle_array_.pop_back(); - } else { - pend_ls_replay_count++; - } - } - - // only skip throttle when pend all ls replay successfully, or reset this guard to restore replay - if (iterate_ls_count == pend_ls_replay_count) { - (void)host_.set_tenant_replay_is_pending(); - STORAGE_LOG(INFO, - "pend replay finish", - K(pend_ls_replay_count), - K(host_.throttle_is_skipping()), - K(host_.tenant_replay_is_pending())); - } else { - (void)restore_tenant_replay_(); - } - } -} - -void ObFreezer::PendTenantReplayHelper::restore_tenant_replay_() -{ - for (int64_t i = 0; i < ls_handle_array_.count(); i++) { - int ret = OB_SUCCESS; - ObLS *ls = nullptr; - if (OB_ISNULL(ls = ls_handle_array_.at(i).get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "[ObFreezer] invalid ls handle", KR(ret), KPC(ls)); - } else if (OB_FAIL(ls->get_freezer()->restore_ls_replay())) { - STORAGE_LOG(WARN, "[ObFreezer] restore replay failed", KR(ret), KPC(ls)); - } - } - host_.unset_tenant_replay_is_pending(); - STORAGE_LOG(INFO, - "restore tenant replay", - K(ls_handle_array_.count()), - K(host_.tenant_replay_is_pending()), - K(host_.throttle_is_skipping())); - ls_handle_array_.reuse(); -} - } // namespace storage } // namespace oceanbase diff --git a/src/storage/ls/ob_freezer.h b/src/storage/ls/ob_freezer.h index 3f6f157a3..7b8bb02dd 100644 --- a/src/storage/ls/ob_freezer.h +++ b/src/storage/ls/ob_freezer.h @@ -21,7 +21,6 @@ #include "lib/utility/ob_print_utils.h" #include "lib/string/ob_string_holder.h" #include "lib/container/ob_array.h" -#include "share/ob_ls_id.h" #include "storage/ls/ob_ls_wrs_handler.h" #include "storage/checkpoint/ob_freeze_checkpoint.h" #include "logservice/ob_log_handler.h" @@ -227,12 +226,12 @@ class ObFreezer uint32_t get_freeze_clock() { return ATOMIC_LOAD(&freeze_flag_) & (~(1 << 31)); } /* ls info */ - share::ObLSID get_ls_id(); checkpoint::ObDataCheckpoint *get_ls_data_checkpoint(); ObLSTxService *get_ls_tx_svr(); ObLSTabletService *get_ls_tablet_svr(); logservice::ObILogHandler *get_ls_log_handler(); ObLSWRSHandler *get_ls_wrs_handler(); + ObLS *get_ls() { return ls_; } /* freeze_snapshot_version */ share::SCN get_freeze_snapshot_version() { return freeze_snapshot_version_; } @@ -288,29 +287,6 @@ class ObFreezer bool need_release_; ObFreezer &parent_; }; - class PendTenantReplayHelper { - public: - PendTenantReplayHelper(ObFreezer &host, ObLS *current_freeze_ls) - : host_(host), current_freeze_ls_(current_freeze_ls) {} - ~PendTenantReplayHelper() { reset_pend_status_(); } - void set_skip_throttle_flag(); - void check_pend_condition_once(); - private: - bool current_ls_is_leader_(); - bool remain_memory_is_exhausting_(); - void pend_tenant_replay_(); - void restore_tenant_replay_(); - void reset_pend_status_() - { - (void)host_.unset_throttle_is_skipping(); - (void)restore_tenant_replay_(); - } - private: - ObFreezer &host_; - ObLS *current_freeze_ls_; - ObSEArray ls_handle_array_; - }; - private: /* freeze_flag */ int set_freeze_flag(); @@ -342,7 +318,6 @@ class ObFreezer SCN &real_snapshot_version); void handle_set_tablet_freeze_failed(const bool need_rewrite_meta, const ObTabletID &tablet_id, - const ObLSID &ls_id, const ObTablet *tablet, const share::SCN freeze_snapshot_version, int &ret); diff --git a/src/storage/ls/ob_freezer_define.h b/src/storage/ls/ob_freezer_define.h index f43da27ed..304490ff5 100644 --- a/src/storage/ls/ob_freezer_define.h +++ b/src/storage/ls/ob_freezer_define.h @@ -38,8 +38,8 @@ enum class ObFreezeSourceFlag : int64_t USER_MINOR_FREEZE = 2, FREEZE_TRIGGER = 3, BACKUP = 4, - RESERVED_5 = 5, - RESERVED_6 = 6, + RESERVED_SOURCE_5 = 5, + RESERVED_SOURCE_6 = 6, MAJOR_FREEZE = 7, FAST_FREEZE = 8, DIRECT_INC_START = 9, @@ -78,11 +78,11 @@ inline const char *obj_to_cstring(const ObFreezeSourceFlag type) case ObFreezeSourceFlag::BACKUP: ret = "BACKUP"; break; - case ObFreezeSourceFlag::RESERVED_5: - ret = "RESERVED_5"; + case ObFreezeSourceFlag::RESERVED_SOURCE_5: + ret = "RESERVED_SOURCE_5"; break; - case ObFreezeSourceFlag::RESERVED_6: - ret = "RESERVED_6"; + case ObFreezeSourceFlag::RESERVED_SOURCE_6: + ret = "RESERVED_SOURCE_6"; break; case ObFreezeSourceFlag::MAJOR_FREEZE: ret = "MAJOR_FREEZE"; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 702daef66..4abdadb72 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -27,7 +27,6 @@ #include "observer/ob_sys_tenant_load_sys_package_service.h" // for ObSysTenantLoadSysPackageService #include "share/ob_global_autoinc_service.h" #include "share/ob_internal_table_change_notifier.h" -#include "sql/das/ob_das_id_service.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/compaction/ob_tablet_merge_ctx.h" #include "storage/ls/ob_ls.h" @@ -35,8 +34,6 @@ #include "storage/tx/ob_timestamp_service.h" #include "storage/tx/ob_trans_id_service.h" #include "observer/vector_index/ob_plugin_vector_index_service.h" -#include "src/observer/table_load/resource/ob_table_load_resource_manager.h" -#include "rootserver/mview/ob_mview_maintenance_service.h" #include "storage/tx_storage/ob_tenant_freezer.h" namespace oceanbase @@ -80,44 +77,32 @@ ObLS::~ObLS() destroy(); } -int ObLS::init(const share::ObLSID &ls_id, - const ObMigrationStatus &migration_status, - const ObRestoreStatus &restore_status, - const SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info) +int ObLS::init(const ObRestoreStatus &restore_status, + const SCN &create_scn) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObLogService *logservice = share::g_mp->log_service(); ObTransService *txs_svr = share::g_mp->trans_service(); - ObLSService *ls_service = share::g_mp->ls_service(); - if (!ls_id.is_valid() || - !true || - !ObMigrationStatusHelper::is_valid(migration_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(migration_status)); - } else if (IS_INIT) { + if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ls is already initialized", K(ret), K_(ls_meta)); - } else if (OB_FAIL(ls_meta_.init(ls_id, - migration_status, - restore_status, - create_scn, - major_mv_merge_info))) { - LOG_WARN("failed to init ls meta", K(ret), K(ls_id), K(major_mv_merge_info)); + } else if (OB_FAIL(ls_meta_.init(restore_status, + create_scn))) { + LOG_WARN("failed to init ls meta", K(ret)); } else if (OB_FAIL(ls_freezer_.init(this))) { - LOG_WARN("init freezer failed", K(ret), K(ls_id)); + LOG_WARN("init freezer failed", K(ret)); } else { ObTxPalfParam tx_palf_param(get_log_handler()); common::ObInOutBandwidthThrottle *bandwidth_throttle = GCTX.bandwidth_throttle_; if (OB_ISNULL(bandwidth_throttle)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("bandwidth throttle should not be NULL", KR(ret)); - } else if (OB_FAIL(txs_svr->create_ls(ls_id, *this, &tx_palf_param, nullptr))) { - LOG_WARN("create trans service failed.", K(ret), K(ls_id)); + } else if (OB_FAIL(txs_svr->create_ls(*this, &tx_palf_param, nullptr))) { + LOG_WARN("create trans service failed.", K(ret)); } else if (OB_FAIL(ls_tablet_svr_.init(this))) { - LOG_WARN("ls tablet service init failed.", K(ret), K(ls_id)); + LOG_WARN("ls tablet service init failed.", K(ret)); } else if (OB_FAIL(tx_table_.init(this))) { LOG_WARN("init tx table failed",K(ret)); } else if (OB_FAIL(checkpoint_executor_.init(this, get_log_handler()))) { @@ -132,26 +117,26 @@ int ObLS::init(const share::ObLSID &ls_id, LOG_WARN("init ls sync tablet seq handler failed", K(ret)); } else if (OB_FAIL(ls_ddl_log_handler_.init(this))) { LOG_WARN("init ls ddl log handler failed", K(ret)); - } else if (OB_FAIL(keep_alive_ls_handler_.init(ls_meta_.ls_id_, get_log_handler()))) { + } else if (OB_FAIL(keep_alive_ls_handler_.init(get_log_handler()))) { LOG_WARN("init keep_alive_ls_handler failed", K(ret)); - } else if (OB_FAIL(ls_wrs_handler_.init(ls_meta_.ls_id_))) { + } else if (OB_FAIL(ls_wrs_handler_.init())) { LOG_WARN("ls loop worker init failed", K(ret)); } else if (OB_FAIL(tablet_gc_handler_.init(this))) { LOG_WARN("failed to init tablet gc handler", K(ret)); } else if (OB_FAIL(tablet_empty_shell_handler_.init(this))) { LOG_WARN("failed to init tablet_empty_shell_handler", K(ret)); } else if (OB_FAIL(reserved_snapshot_mgr_.init(this, &log_handler_))) { - LOG_WARN("failed to init reserved snapshot mgr", K(ret), K(ls_id)); + LOG_WARN("failed to init reserved snapshot mgr", K(ret)); } else if (OB_FAIL(reserved_snapshot_clog_handler_.init(this))) { - LOG_WARN("failed to init reserved snapshot clog handler", K(ret), K(ls_id)); + LOG_WARN("failed to init reserved snapshot clog handler", K(ret)); } else if (OB_FAIL(medium_compaction_clog_handler_.init(this))) { - LOG_WARN("failed to init medium compaction clog handler", K(ret), K(ls_id)); + LOG_WARN("failed to init medium compaction clog handler", K(ret)); } else if (OB_FAIL(register_to_service_())) { LOG_WARN("register to service failed", K(ret)); } else { need_delay_resource_recycle_ = false; is_inited_ = true; - LOG_INFO("ls init success", K(ls_id)); + LOG_INFO("ls init success"); } // do some rollback work if (OB_FAIL(ret)) { @@ -192,10 +177,7 @@ int ObLS::remove_ls_inner_tablet() return ret; } -int ObLS::create_ls(const share::ObTenantRole tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const ObReplicaType &replica_type, - const bool allow_log_sync) +int ObLS::create_ls(const palf::PalfBaseInfo &palf_base_info) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -207,17 +189,12 @@ int ObLS::create_ls(const share::ObTenantRole tenant_role, if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("ls do not init", K(ret)); - } else if (OB_FAIL(logservice->check_palf_exist(ls_meta_.ls_id_, is_palf_exist))) { + } else if (OB_FAIL(logservice->check_palf_exist(is_palf_exist))) { LOG_WARN("check_palf_exist failed", K(ret), K_(ls_meta)); } else if (is_palf_exist) { ret = OB_ERR_UNEXPECTED; LOG_WARN("palf should not exist now", K(ret), K_(ls_meta)); - } else if (OB_FAIL(logservice->create_ls(ls_meta_.ls_id_, - replica_type, - tenant_role, - palf_base_info, - allow_log_sync, - log_handler_))) { + } else if (OB_FAIL(logservice->create_ls(palf_base_info, log_handler_))) { LOG_WARN("create palf failed", K(ret), K_(ls_meta)); } else { if (OB_FAIL(ret)) { @@ -241,25 +218,22 @@ int ObLS::create_ls(const share::ObTenantRole tenant_role, return ret; } -int ObLS::load_ls(const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync) +int ObLS::load_ls() { int ret = OB_SUCCESS; ObLogService *logservice = share::g_mp->log_service(); bool is_palf_exist = false; - if (OB_FAIL(logservice->check_palf_exist(ls_meta_.ls_id_, is_palf_exist))) { + if (OB_FAIL(logservice->check_palf_exist(is_palf_exist))) { LOG_WARN("check_palf_exist failed", K(ret), K_(ls_meta)); } else if (!is_palf_exist) { LOG_WARN("there is no ls at disk, skip load", K_(ls_meta)); - } else if (OB_FAIL(logservice->add_ls(ls_meta_.ls_id_, - log_handler_))) { + } else if (OB_FAIL(logservice->add_ls(log_handler_))) { LOG_WARN("add ls failed", K(ret), K_(ls_meta)); } else { // TODO: add_ls has no interface to rollback now, something can not rollback. if (OB_FAIL(ret)) { - LOG_ERROR("load ls failed", K(ret), K(ls_meta_), K(tenant_role), K(palf_base_info)); + LOG_ERROR("load ls failed", K(ret), K(ls_meta_)); } } return ret; @@ -274,11 +248,8 @@ int ObLS::remove_ls() ret = OB_NOT_INIT; LOG_WARN("ls do not init", K(ret)); } else { - if (OB_TMP_FAIL(log_handler_.unregister_rebuild_cb())) { - LOG_WARN("unregister rebuild cb failed", K(ret), K(ls_meta_)); - } - if (OB_FAIL(logservice->remove_ls(ls_meta_.ls_id_, log_handler_))) { - LOG_ERROR("remove log stream from logservice failed", K(ret), K(ls_meta_.ls_id_)); + if (OB_FAIL(logservice->remove_ls(log_handler_))) { + LOG_ERROR("remove log stream from logservice failed", K(ret)); } } LOG_INFO("remove ls from disk", K(ret), K(ls_meta_)); @@ -301,17 +272,6 @@ int ObLS::set_start_work_state() return ret; } -int ObLS::set_start_ha_state() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ls_meta_.set_start_ha_state())) { - LOG_WARN("set start ha state failed", K(ret), K_(ls_meta)); - } else { - update_state_seq_(); - } - return ret; -} - int ObLS::set_remove_state() { int ret = OB_SUCCESS; @@ -331,7 +291,7 @@ ObLSPersistentState ObLS::get_persistent_state() const int ObLS::finish_create_ls() { int ret = OB_SUCCESS; - if (OB_FAIL(running_state_.create_finish(ls_meta_.ls_id_))) { + if (OB_FAIL(running_state_.create_finish())) { LOG_WARN("finish create ls failed", KR(ret), K(ls_meta_)); } else { update_state_seq_(); @@ -351,7 +311,7 @@ int ObLS::stop() LOG_WARN("ls is not inited", K(ret)); } else if (OB_FAIL(stop_())) { LOG_WARN("stop ls failed", K(ret), K(ls_meta_)); - } else if (OB_FAIL(running_state_.stop(ls_meta_.ls_id_))) { + } else if (OB_FAIL(running_state_.stop())) { LOG_WARN("set stop state failed", K(ret), K(ls_meta_)); } else { inc_update(&state_seq_, max(ObTimeUtil::current_time(), state_seq_ + 1)); @@ -438,54 +398,6 @@ int ObLS::prepare_for_safe_destroy_() return ret; } -bool ObLS::safe_to_destroy() -{ - int ret = OB_SUCCESS; - bool is_safe = false; - bool is_tablet_service_safe = false; - bool is_data_check_point_safe = false; - bool is_log_handler_safe = false; - bool is_vec_idx_scheduler_safe = false; - - if (OB_FAIL(ls_tablet_svr_.safe_to_destroy(is_tablet_service_safe))) { - LOG_WARN("ls tablet service check safe to destroy failed", K(ret), KPC(this)); - } else if (!is_tablet_service_safe) { - } else if (OB_FAIL(data_checkpoint_.safe_to_destroy(is_data_check_point_safe))) { - LOG_WARN("data_checkpoint check safe to destroy failed", K(ret), KPC(this)); - } else if (!is_data_check_point_safe) { - } else if (OB_FAIL(log_handler_.safe_to_destroy(is_log_handler_safe))) { - LOG_WARN("log_handler_ check safe to destroy failed", K(ret), KPC(this)); - } else if (!is_log_handler_safe) { - } else if (OB_FAIL(vector_idx_scheduler_safe_to_destroy_(is_vec_idx_scheduler_safe))) { - LOG_WARN("vector_idx_scheduler_ check safe to destroy failed", K(ret), KPC(this)); - } else if (!is_vec_idx_scheduler_safe) { - } else { - if (1 == ref_mgr_.get_total_ref_cnt()) { // only has one ref at the safe destroy task - is_safe = true; - } - } - - // check safe to destroy of all the sub module. - if (OB_SUCC(ret)) { - if (!is_safe) { - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - LOG_WARN("this ls is not safe to destroy", K(is_safe), - K(is_tablet_service_safe), K(is_data_check_point_safe), - K(is_log_handler_safe), - K(is_vec_idx_scheduler_safe), - "ls_ref", ref_mgr_.get_total_ref_cnt(), - K(ret), KP(this), KPC(this)); - ref_mgr_.print(); - PRINT_OBJ_LEAK(share::LEAK_CHECK_OBJ_LS_HANDLE); - READ_CHECKER_PRINT(ls_meta_.ls_id_); - } - } else { - LOG_INFO("this ls is safe to destroy", KP(this), KPC(this)); - } - } - return is_safe; -} - void ObLS::destroy() { // TODO: (yanyuan.cxf) destroy all the sub module. @@ -499,11 +411,11 @@ void ObLS::destroy() FLOG_INFO("ObLS destroy", K(this), K(*this), K(lbt())); if (running_state_.is_running()) { if (OB_TMP_FAIL(offline_(start_ts))) { - LOG_WARN("offline a running ls failed", K(tmp_ret), K(ls_meta_.ls_id_)); + LOG_WARN("offline a running ls failed", K(tmp_ret)); } } if (OB_TMP_FAIL(stop_())) { - LOG_WARN("ls stop failed.", K(tmp_ret), K(ls_meta_.ls_id_)); + LOG_WARN("ls stop failed.", K(tmp_ret)); } else { wait_(); if (OB_TMP_FAIL(prepare_for_safe_destroy_())) { @@ -520,10 +432,9 @@ void ObLS::destroy() if (OB_ISNULL(txs_svr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tx service is null, may be memory leak", KP(txs_svr)); - // Minority follower kills transaction forcefully during GC needs to follow non-gracefully process, majority GC writes offline log when it has already determined the transaction is finished - } else if (OB_FAIL(txs_svr->remove_ls(ls_meta_.ls_id_, false))) { + } else if (OB_FAIL(txs_svr->remove_ls(false))) { // we may has remove it before. - LOG_WARN("remove log stream from txs service failed", K(ret), K(ls_meta_.ls_id_)); + LOG_WARN("remove log stream from txs service failed", K(ret)); } checkpoint_executor_.reset(); log_handler_.destroy(); @@ -560,29 +471,95 @@ int ObLS::offline_compaction_() { int ret = OB_SUCCESS; if (FALSE_IT(ls_freezer_.offline())) { - } else if (OB_FAIL(share::g_mp->tenant_tablet_scheduler()-> - check_ls_compaction_finish(ls_meta_.ls_id_))) { + } else if (OB_FAIL(share::g_mp->tenant_tablet_scheduler()->check_ls_compaction_finish())) { LOG_WARN("check compaction finish failed", K(ret), K(ls_meta_)); } return ret; } +int ObLS::start_local_log_() +{ + int ret = OB_SUCCESS; + palf::LSN end_lsn; + bool is_done = false; + bool is_clear = false; + logservice::ObLogApplyService *apply_service = share::g_mp->log_service()->get_log_apply_service(); + logservice::ObLogReplayService *replay_service = share::g_mp->log_service()->get_log_replay_service(); + if (OB_FAIL(log_handler_.get_end_lsn(end_lsn))) { + LOG_WARN("get local log end failed", K(ret)); + } + while (OB_SUCC(ret) && !is_done) { + if (OB_FAIL(replay_service->is_replay_done(end_lsn, is_done))) { + LOG_WARN("check local replay failed", K(ret)); + } else if (!is_done) { + ob_usleep(50 * 1000); + } + } + if (OB_SUCC(ret) && OB_FAIL(apply_service->start_local_append())) { + LOG_WARN("start local apply failed", K(ret)); + } else if (OB_SUCC(ret) && OB_FAIL(replay_service->disable_local_replay())) { + LOG_WARN("stop local replay failed", K(ret)); + } + while (OB_SUCC(ret) && !is_clear) { + if (OB_FAIL(replay_service->is_submit_task_clear(is_clear))) { + LOG_WARN("check local replay tasks failed", K(ret)); + } else if (!is_clear) { + ob_usleep(1000); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(local_log_handler_set_.activate())) { + LOG_WARN("activate local log handlers failed", K(ret)); + } + } + return ret; +} + +int ObLS::stop_local_log_() +{ + int ret = OB_SUCCESS; + bool is_done = false; + palf::LSN end_lsn; + logservice::ObLogApplyService *apply_service = share::g_mp->log_service()->get_log_apply_service(); + logservice::ObLogReplayService *replay_service = share::g_mp->log_service()->get_log_replay_service(); + if (OB_FAIL(apply_service->wait_append_sync())) { + LOG_WARN("wait local append callbacks failed", K(ret)); + } else if (OB_FAIL(apply_service->stop_local_append())) { + LOG_WARN("stop local apply failed", K(ret)); + } + while (OB_SUCC(ret) && !is_done) { + if (OB_FAIL(apply_service->is_apply_done(is_done, end_lsn))) { + LOG_WARN("check local apply failed", K(ret)); + } else if (!is_done) { + ob_usleep(5 * 1000); + } + } + if (OB_SUCC(ret)) { + local_log_handler_set_.deactivate(); + if (OB_FAIL(replay_service->enable_local_replay(end_lsn))) { + LOG_WARN("stop local replay failed", K(ret)); + } + } + return ret; +} + int ObLS::offline_(const int64_t start_ts) { int ret = OB_SUCCESS; - // only follower can do this. if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); } else if (running_state_.is_stopped()) { LOG_INFO("ls is stopped state, do nothing", K(ret), K(ls_meta_)); - } else if (OB_FAIL(running_state_.pre_offline(ls_meta_.ls_id_))) { + } else if (OB_FAIL(running_state_.pre_offline())) { LOG_WARN("ls pre offline failed", K(ret), K(ls_meta_)); } else if (FALSE_IT(update_state_seq_())) { } else if (OB_FAIL(offline_advance_epoch_())) { } else if (FALSE_IT(checkpoint_executor_.offline())) { LOG_WARN("checkpoint executor offline failed", K(ret), K(ls_meta_)); + } else if (OB_FAIL(stop_local_log_())) { + LOG_WARN("failed to stop local log", K(ret)); } else if (OB_FAIL(log_handler_.offline())) { LOG_WARN("failed to offline log", K(ret)); // TODO: delete it if apply sequence @@ -608,7 +585,7 @@ int ObLS::offline_(const int64_t start_ts) LOG_WARN("tablet service offline failed", K(ret), K(ls_meta_)); } else if (OB_FAIL(tablet_empty_shell_handler_.offline())) { LOG_WARN("tablet_empty_shell_handler failed", K(ret), K(ls_meta_)); - } else if (OB_FAIL(running_state_.post_offline(ls_meta_.ls_id_))) { + } else if (OB_FAIL(running_state_.post_offline())) { LOG_WARN("ls post offline failed", KR(ret), K(ls_meta_)); } else { update_state_seq_(); @@ -629,7 +606,6 @@ int ObLS::offline() retry_times++; { ObLSLockGuard lock_myself(this, lock_, read_lock, write_lock); - // only follower can do this. if (OB_FAIL(offline_(start_ts))) { LOG_WARN("ls offline failed", K(ret), K(ls_meta_)); } @@ -641,7 +617,7 @@ int ObLS::offline() } } } while (OB_EAGAIN == ret); - FLOG_INFO("ls offline end", KR(ret), "ls_id", get_ls_id()); + FLOG_INFO("ls offline end", KR(ret)); return ret; } @@ -650,8 +626,8 @@ int ObLS::online_tx_() int ret = OB_SUCCESS; if (OB_FAIL(ls_tx_svr_.online())) { LOG_WARN("ls tx service online failed", K(ret), K(ls_meta_)); - } else if (OB_FAIL(ls_tx_svr_.set_max_replay_commit_version(ls_meta_.get_clog_checkpoint_scn()))) { - LOG_WARN("set max replay commit scn fail", K(ret), K(ls_meta_.get_clog_checkpoint_scn())); + } else if (OB_FAIL(ls_tx_svr_.advance_commit_version(ls_meta_.get_clog_checkpoint_scn()))) { + LOG_WARN("advance commit version failed", K(ret), K(ls_meta_.get_clog_checkpoint_scn())); } else if (OB_FAIL(tx_table_.online())) { LOG_WARN("tx table online failed", K(ret), K(ls_meta_)); } @@ -693,7 +669,6 @@ int ObLS::online_advance_epoch_() int ObLS::register_common_service() { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_meta_.ls_id_; REGISTER_TO_LOGSERVICE(TRANS_SERVICE_LOG_BASE_TYPE, &ls_tx_svr_); REGISTER_TO_LOGSERVICE(STORAGE_SCHEMA_LOG_BASE_TYPE, &ls_tablet_svr_); REGISTER_TO_LOGSERVICE(TABLET_SEQ_SYNC_LOG_BASE_TYPE, &ls_sync_tablet_seq_handler_); @@ -703,86 +678,37 @@ int ObLS::register_common_service() REGISTER_TO_LOGSERVICE(MEDIUM_COMPACTION_LOG_BASE_TYPE, &medium_compaction_clog_handler_); REGISTER_TO_LOGSERVICE(TABLE_LOCK_LOG_BASE_TYPE, &lock_table_); - if (ls_id == IDS_LS) { - REGISTER_TO_LOGSERVICE(TIMESTAMP_LOG_BASE_TYPE, share::g_mp->timestamp_service()); - REGISTER_TO_LOGSERVICE(TRANS_ID_LOG_BASE_TYPE, share::g_mp->trans_id_service()); - } - if (ls_id == MAJOR_FREEZE_LS) { - REGISTER_TO_LOGSERVICE(MAJOR_FREEZE_LOG_BASE_TYPE, share::g_mp->primary_major_freeze_service()); - } - if (ls_id == GAIS_LS) { - REGISTER_TO_LOGSERVICE(GAIS_LOG_BASE_TYPE, share::g_mp->global_auto_inc_service()); - share::g_mp->global_auto_inc_service()->set_cache_ls(this); - } + REGISTER_REPLAY_CHECKPOINT_HANDLER(TIMESTAMP_LOG_BASE_TYPE, share::g_mp->timestamp_service()); + REGISTER_REPLAY_CHECKPOINT_HANDLER(TRANS_ID_LOG_BASE_TYPE, share::g_mp->trans_id_service()); + REGISTER_TO_LOGSERVICE(MAJOR_FREEZE_LOG_BASE_TYPE, share::g_mp->primary_major_freeze_service()); return ret; } -int ObLS::register_sys_service() +int ObLS::register_local_services_() { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_meta_.ls_id_; - - if (ls_id == IDS_LS) { - REGISTER_TO_LOGSERVICE(DAS_ID_LOG_BASE_TYPE, share::g_mp->dasid_service()); - } - if (ls_id.is_sys_ls()) { - { - ObIngressBWAllocService *ingress_service = GCTX.net_frame_->get_ingress_service(); - REGISTER_TO_LOGSERVICE(NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, ingress_service); - REGISTER_TO_LOGSERVICE(MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE, share::g_mp->m_view_maintenance_service()); - REGISTER_TO_LOGSERVICE(TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE, share::g_mp->table_load_resource_service()); - REGISTER_TO_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); - REGISTER_TO_LOGSERVICE(SYS_DDL_SCHEDULER_LOG_BASE_TYPE, share::g_mp->ddl_scheduler()); - REGISTER_TO_LOGSERVICE(DDL_SERVICE_LAUNCHER_LOG_BASE_TYPE, share::g_mp->ddl_service_launcher()); - REGISTER_TO_LOGSERVICE(SYS_TENANT_LOAD_SYS_PACKAGE_SERVICE_LOG_BASE_TYPE, share::g_mp->sys_tenant_load_sys_package_service()); - // ObInternalTableChangeNotifier only needs role_change, not replay/checkpoint - if (OB_FAIL(ret)) { - } else if (OB_FAIL(role_change_handler_.register_handler( - INTERNAL_TABLE_NOTIFIER_LOG_BASE_TYPE, - &share::ObInternalTableChangeNotifier::get_instance()))) { - LOG_WARN("role_change_handler_ register notifier failed", K(ret), K(ls_id)); - } -#ifdef OB_BUILD_SYS_VEC_IDX - REGISTER_TO_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); - - if (OB_SUCC(ret)) { - // table_api removed from build: TTL_LOG_BASE_TYPE handler is gone, only the - // vector index scheduler (formerly hosted by the tablet ttl mgr) is kept. - if (OB_FAIL(init_vector_idx_scheduler_())) { - LOG_WARN("fail to init vector index scheduler", KR(ret)); - } else { - REGISTER_TO_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); - } - } -#endif - } - } - - return ret; -} - -int ObLS::register_user_service() -{ - int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_meta_.ls_id_; - - if (ls_id.is_sys_ls()) { - REGISTER_TO_LOGSERVICE(MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE, share::g_mp->m_view_maintenance_service()); - REGISTER_TO_LOGSERVICE(TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE, share::g_mp->table_load_resource_service()); - REGISTER_TO_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); - REGISTER_TO_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); + ObIngressBWAllocService *ingress_service = GCTX.net_frame_->get_ingress_service(); + REGISTER_TO_LOGSERVICE(NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, ingress_service); + REGISTER_TO_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); + REGISTER_TO_LOGSERVICE(SYS_DDL_SCHEDULER_LOG_BASE_TYPE, share::g_mp->ddl_scheduler()); + REGISTER_TO_LOGSERVICE(DDL_SERVICE_LAUNCHER_LOG_BASE_TYPE, share::g_mp->ddl_service_launcher()); + REGISTER_TO_LOGSERVICE(SYS_TENANT_LOAD_SYS_PACKAGE_SERVICE_LOG_BASE_TYPE, share::g_mp->sys_tenant_load_sys_package_service()); + // ObInternalTableChangeNotifier only needs local lifecycle callbacks. + if (OB_FAIL(ret)) { + } else if (OB_FAIL(local_log_handler_set_.register_handler( + INTERNAL_TABLE_NOTIFIER_LOG_BASE_TYPE, + &share::ObInternalTableChangeNotifier::get_instance()))) { + LOG_WARN("local_log_handler_set_ register notifier failed", K(ret)); } - - if (ls_id.is_user_ls()) { - if (OB_SUCC(ret)) { - // table_api removed from build: TTL_LOG_BASE_TYPE handler is gone, only the - // vector index scheduler (formerly hosted by the tablet ttl mgr) is kept. - if (OB_FAIL(init_vector_idx_scheduler_())) { - LOG_WARN("fail to init vector index scheduler", KR(ret)); - } else { - REGISTER_TO_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); - } + REGISTER_TO_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); + if (OB_SUCC(ret)) { + // table_api removed from build: TTL_LOG_BASE_TYPE handler is gone, only the + // vector index scheduler (formerly hosted by the tablet ttl mgr) is kept. + if (OB_FAIL(init_vector_idx_scheduler_())) { + LOG_WARN("fail to init vector index scheduler", KR(ret)); + } else { + REGISTER_TO_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); } } @@ -792,19 +718,11 @@ int ObLS::register_user_service() int ObLS::register_to_service_() { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_meta_.ls_id_; - // TODO-REVIEW: merge produced a register_to_service_ that only registered - // common + sys services with a master-flavored inline vector-index-scheduler - // init. HEAD moved the vector index - // scheduler lifecycle into register_sys_service()/register_user_service() - // via init_vector_idx_scheduler_(); restore that here. if (OB_FAIL(register_common_service())) { - LOG_WARN("common tenant register failed", K(ret), K(ls_id)); - } else if (OB_FAIL(register_sys_service())) { - LOG_WARN("register sys service failed", K(ret), K(ls_id)); - } else if (OB_FAIL(register_user_service())) { - LOG_WARN("register user service failed", K(ret), K(ls_id)); + LOG_WARN("common tenant register failed", K(ret)); + } else if (OB_FAIL(register_local_services_())) { + LOG_WARN("register local services failed", K(ret)); } return ret; @@ -844,17 +762,6 @@ void ObLS::destroy_vector_idx_scheduler_() } } -int ObLS::vector_idx_scheduler_safe_to_destroy_(bool &is_safe) -{ - int ret = OB_SUCCESS; - is_safe = true; - if (vector_idx_scheduler_timer_.inited() && - OB_FAIL(vector_idx_scheduler_.safe_to_destroy(is_safe))) { - LOG_WARN("fail to check vector index scheduler safe to destroy", KR(ret), K(is_safe)); - } - return ret; -} - void ObLS::unregister_common_service_() { UNREGISTER_FROM_LOGSERVICE(TRANS_SERVICE_LOG_BASE_TYPE, &ls_tx_svr_); @@ -865,69 +772,28 @@ void ObLS::unregister_common_service_() UNREGISTER_FROM_LOGSERVICE(RESERVED_SNAPSHOT_LOG_BASE_TYPE, &reserved_snapshot_clog_handler_); UNREGISTER_FROM_LOGSERVICE(MEDIUM_COMPACTION_LOG_BASE_TYPE, &medium_compaction_clog_handler_); UNREGISTER_FROM_LOGSERVICE(TABLE_LOCK_LOG_BASE_TYPE, &lock_table_); - if (ls_meta_.ls_id_ == IDS_LS) { - share::g_mp->trans_id_service()->reset_ls(); - share::g_mp->timestamp_service()->reset_ls(); - // temporary fix of - share::g_mp->dasid_service()->reset_ls(); - UNREGISTER_FROM_LOGSERVICE(TIMESTAMP_LOG_BASE_TYPE, share::g_mp->timestamp_service()); - UNREGISTER_FROM_LOGSERVICE(TRANS_ID_LOG_BASE_TYPE, share::g_mp->trans_id_service()); - } - if (ls_meta_.ls_id_ == MAJOR_FREEZE_LS) { - ObPrimaryMajorFreezeService *primary_major_freeze_service = share::g_mp->primary_major_freeze_service(); - UNREGISTER_FROM_LOGSERVICE(MAJOR_FREEZE_LOG_BASE_TYPE, primary_major_freeze_service); - } - if (ls_meta_.ls_id_ == GAIS_LS) { - UNREGISTER_FROM_LOGSERVICE(GAIS_LOG_BASE_TYPE, share::g_mp->global_auto_inc_service()); - share::g_mp->global_auto_inc_service()->set_cache_ls(nullptr); - } + UNREGISTER_REPLAY_CHECKPOINT_HANDLER(TIMESTAMP_LOG_BASE_TYPE); + UNREGISTER_REPLAY_CHECKPOINT_HANDLER(TRANS_ID_LOG_BASE_TYPE); + UNREGISTER_FROM_LOGSERVICE(MAJOR_FREEZE_LOG_BASE_TYPE, nullptr); } -void ObLS::unregister_sys_service_() +void ObLS::unregister_local_services_() { - if (ls_meta_.ls_id_ == IDS_LS) { - share::g_mp->dasid_service()->reset_ls(); - UNREGISTER_FROM_LOGSERVICE(DAS_ID_LOG_BASE_TYPE, share::g_mp->dasid_service()); - } - if (ls_meta_.ls_id_.is_sys_ls()) { - { - ObIngressBWAllocService *ingress_service = GCTX.net_frame_->get_ingress_service(); - UNREGISTER_FROM_LOGSERVICE(NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, ingress_service); - UNREGISTER_FROM_LOGSERVICE(MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE, share::g_mp->m_view_maintenance_service()); - UNREGISTER_FROM_LOGSERVICE(TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE, share::g_mp->table_load_resource_service()); - UNREGISTER_FROM_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); - UNREGISTER_FROM_LOGSERVICE(SYS_DDL_SCHEDULER_LOG_BASE_TYPE, share::g_mp->ddl_scheduler()); - UNREGISTER_FROM_LOGSERVICE(DDL_SERVICE_LAUNCHER_LOG_BASE_TYPE, share::g_mp->ddl_service_launcher()); - UNREGISTER_FROM_LOGSERVICE(SYS_TENANT_LOAD_SYS_PACKAGE_SERVICE_LOG_BASE_TYPE, share::g_mp->sys_tenant_load_sys_package_service()); - role_change_handler_.unregister_handler(INTERNAL_TABLE_NOTIFIER_LOG_BASE_TYPE); -#ifdef OB_BUILD_SYS_VEC_IDX - UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); - - UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); - destroy_vector_idx_scheduler_(); -#endif - } - } -} - -void ObLS::unregister_user_service_() -{ - if (ls_meta_.ls_id_.is_sys_ls()) { - UNREGISTER_FROM_LOGSERVICE(MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE, share::g_mp->m_view_maintenance_service()); - UNREGISTER_FROM_LOGSERVICE(TABLE_LOAD_RESOURCE_SERVICE_LOG_BASE_TYPE, share::g_mp->table_load_resource_service()); - UNREGISTER_FROM_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); - UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); - } - if (ls_meta_.ls_id_.is_user_ls()) { - UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); - destroy_vector_idx_scheduler_(); - } + UNREGISTER_FROM_LOGSERVICE(NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, nullptr); + UNREGISTER_FROM_LOGSERVICE(DBMS_SCHEDULER_LOG_BASE_TYPE, share::g_mp->dbms_sched_service()); + UNREGISTER_FROM_LOGSERVICE(SYS_DDL_SCHEDULER_LOG_BASE_TYPE, share::g_mp->ddl_scheduler()); + UNREGISTER_FROM_LOGSERVICE(DDL_SERVICE_LAUNCHER_LOG_BASE_TYPE, share::g_mp->ddl_service_launcher()); + UNREGISTER_FROM_LOGSERVICE(SYS_TENANT_LOAD_SYS_PACKAGE_SERVICE_LOG_BASE_TYPE, share::g_mp->sys_tenant_load_sys_package_service()); + local_log_handler_set_.unregister_handler(INTERNAL_TABLE_NOTIFIER_LOG_BASE_TYPE); + UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_SERVICE_LOG_BASE_TYPE, share::g_mp->plugin_vector_index_service()); + UNREGISTER_FROM_LOGSERVICE(VEC_INDEX_LOG_BASE_TYPE, &vector_idx_scheduler_); + destroy_vector_idx_scheduler_(); } void ObLS::unregister_from_service_() { unregister_common_service_(); - unregister_sys_service_(); + unregister_local_services_(); } int ObLS::online() @@ -958,6 +824,8 @@ int ObLS::online_without_lock() } else if (OB_FAIL(log_handler_.online(ls_meta_.get_clog_base_lsn(), ls_meta_.get_clog_checkpoint_scn()))) { LOG_WARN("failed to online log", K(ret)); + } else if (OB_FAIL(start_local_log_())) { + LOG_WARN("failed to start local log", K(ret)); } else if (OB_FAIL(ls_wrs_handler_.online())) { LOG_WARN("weak read handler online failed", K(ret), K(ls_meta_)); } else if (OB_FAIL(online_compaction_())) { @@ -966,46 +834,13 @@ int ObLS::online_without_lock() } else if (FALSE_IT(tablet_gc_handler_.online())) { } else if (FALSE_IT(tablet_empty_shell_handler_.online())) { } else if (OB_FAIL(online_advance_epoch_())) { - } else if (OB_FAIL(running_state_.online(ls_meta_.ls_id_))) { + } else if (OB_FAIL(running_state_.online())) { LOG_WARN("ls online failed", KR(ret), K(ls_meta_)); } else { update_state_seq_(); } - FLOG_INFO("ls online end", KR(ret), "ls_id", get_ls_id()); - return ret; -} - -int ObLS::get_ls_meta_package(const bool check_archive, ObLSMetaPackage &meta_package) -{ - int ret = OB_SUCCESS; - palf::LSN begin_lsn; - const int64_t cost_time = 10 * 1000 * 1000; // 10s - const ObLSID &id = get_ls_id(); - share::SCN tx_data_recycle_scn; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret)); - } else if (OB_FAIL(get_tx_data_sstable_recycle_scn(tx_data_recycle_scn))) { - LOG_WARN("failed to get tx data recycle scn", K(ret)); - } else { - meta_package.tx_data_recycle_scn_ = tx_data_recycle_scn; - meta_package.ls_meta_ = ls_meta_; - palf::LSN curr_lsn = meta_package.ls_meta_.get_clog_base_lsn(); - ObTimeGuard time_guard("get_ls_meta_package", cost_time); - if (! check_archive) { - LOG_TRACE("no need check archive", K(id), K(check_archive)); - } else { - LOG_WARN("archive is not implemanted"); - } - time_guard.click(); - if (OB_SUCC(ret) && OB_FAIL(log_handler_.get_palf_base_info(curr_lsn, - meta_package.palf_meta_))) { - LOG_WARN("get palf base info failed", K(ret), K(id), K(curr_lsn), K_(ls_meta)); - } - time_guard.click(); - - } + FLOG_INFO("ls online end", KR(ret)); return ret; } @@ -1017,13 +852,11 @@ int ObLS::set_ls_meta(const ObLSMeta &ls_meta) LOG_WARN("ls is not inited", K(ret)); } else { ls_meta_ = ls_meta; - if (IDS_LS == ls_meta_.ls_id_) { - ObAllIDMeta all_id_meta; - if (OB_FAIL(ls_meta_.get_all_id_meta(all_id_meta))) { - LOG_WARN("get all id meta failed", K(ret), K(ls_meta_)); - } else if (OB_FAIL(ObIDService::update_id_service(all_id_meta))) { - LOG_WARN("update id service fail", K(ret), K(all_id_meta), K(*this)); - } + ObAllIDMeta all_id_meta; + if (OB_FAIL(ls_meta_.get_all_id_meta(all_id_meta))) { + LOG_WARN("get all id meta failed", K(ret), K(ls_meta_)); + } else if (OB_FAIL(ObIDService::update_id_service(all_id_meta))) { + LOG_WARN("update id service fail", K(ret), K(all_id_meta), K(*this)); } } return ret; @@ -1052,20 +885,6 @@ int ObLS::get_ls_meta(ObLSMeta &ls_meta) const return ret; } -int ObLS::get_ls_role(ObRole &role) -{ - int ret = OB_SUCCESS; - role = INVALID_ROLE; - int64_t proposal_id = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret)); - } else if (OB_FAIL(log_handler_.get_role(role, proposal_id))) { - LOG_WARN("get ls role failed", K(ret), KPC(this)); - } - return ret; -} - int ObLS::try_sync_reserved_snapshot( const int64_t new_reserved_snapshot, const bool update_flag) @@ -1074,18 +893,8 @@ int ObLS::try_sync_reserved_snapshot( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); - } else { - ObRole role = INVALID_ROLE; - int64_t proposal_id = 0; - if (is_stopped()) { - // do nothing - } else if (OB_FAIL(log_handler_.get_role(role, proposal_id))) { - LOG_WARN("get ls role failed", K(ret), KPC(this)); - } else if (LEADER != role) { - // do nothing - } else { - ret = reserved_snapshot_mgr_.try_sync_reserved_snapshot(new_reserved_snapshot, update_flag); - } + } else if (!is_stopped()) { + ret = reserved_snapshot_mgr_.try_sync_reserved_snapshot(new_reserved_snapshot, update_flag); } return ret; } @@ -1093,51 +902,22 @@ int ObLS::try_sync_reserved_snapshot( int ObLS::get_ls_info(ObLSVTInfo &ls_info) { int ret = OB_SUCCESS; - ObRole role; - int64_t proposal_id = 0; - bool is_log_sync = false; - bool is_need_rebuild = false; - ObMigrationStatus migrate_status; bool tx_blocked = false; - int64_t required_data_disk_size = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); - } else if (OB_FAIL(log_handler_.get_role(role, proposal_id))) { - LOG_WARN("get ls role failed", K(ret), KPC(this)); - } else if (OB_FAIL(log_handler_.is_in_sync(is_log_sync, - is_need_rebuild))) { - LOG_WARN("get ls need rebuild info failed", K(ret), KPC(this)); - } else if (OB_FAIL(ls_meta_.get_migration_status(migrate_status))) { - LOG_WARN("get ls migrate status failed", K(ret), KPC(this)); } else if (OB_FAIL(ls_tx_svr_.check_tx_blocked(tx_blocked))) { LOG_WARN("check tx ls state error", K(ret),KPC(this)); - } else if (OB_ISNULL(get_tablet_svr())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret)); - } else if (OB_FAIL(get_tablet_svr()->get_ls_migration_required_size(required_data_disk_size))) { - LOG_WARN("fail to get required data disk size for migration", KR(ret)); } else { // The readable point of the primary tenant is weak read ts, // and the readable point of the standby tenant is readable scn ls_info.weak_read_scn_ = ls_wrs_handler_.get_ls_weak_read_ts(); if (OB_SUCC(ret)) { - ls_info.ls_id_ = ls_meta_.ls_id_; - ls_info.replica_type_ = ls_meta_.get_replica_type(); - ls_info.ls_state_ = role; - ls_info.migrate_status_ = migrate_status; ls_info.tablet_count_ = ls_tablet_svr_.get_tablet_count(); - ls_info.need_rebuild_ = is_need_rebuild; ls_info.checkpoint_scn_ = ls_meta_.get_clog_checkpoint_scn(); ls_info.checkpoint_lsn_ = ls_meta_.get_clog_base_lsn().val_; - ls_info.rebuild_seq_ = ls_meta_.get_rebuild_seq(); ls_info.tablet_change_checkpoint_scn_ = ls_meta_.get_tablet_change_checkpoint_scn(); - ls_info.reserved_scn_ = ls_meta_.get_reserved_scn(); ls_info.tx_blocked_ = tx_blocked; - ls_info.mv_major_merge_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_; - ls_info.mv_publish_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_publish_; - ls_info.mv_safe_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_safe_calc_; - ls_info.required_data_disk_size_ = required_data_disk_size; if (tx_blocked) { TRANS_LOG(INFO, "current ls is blocked", K(ls_info)); } @@ -1146,12 +926,6 @@ int ObLS::get_ls_info(ObLSVTInfo &ls_info) return ret; } -int ObLS::report_replica_info() -{ - int ret = OB_SUCCESS; - return ret; -} - int ObLS::ObLSInnerTabletIDIter::get_next(common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -1241,21 +1015,13 @@ int ObLS::update_tablet_table_store_without_lock_( } else if (OB_UNLIKELY(!tablet_id.is_valid() || !param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("update tablet table store get invalid argument", K(ret), K(tablet_id), K(param)); - } else { - const int64_t rebuild_seq = ls_meta_.get_rebuild_seq(); - if (param.rebuild_seq_ != rebuild_seq) { - ret = OB_EAGAIN; - LOG_WARN("update tablet table store rebuild seq not same, need retry", - K(ret), K(tablet_id), K(rebuild_seq), K(param)); - } else if (OB_FAIL(ls_tablet_svr_.update_tablet_table_store(tablet_id, param, handle))) { - LOG_WARN("failed to update tablet table store", K(ret), K(tablet_id), K(param)); - } + } else if (OB_FAIL(ls_tablet_svr_.update_tablet_table_store(tablet_id, param, handle))) { + LOG_WARN("failed to update tablet table store", K(ret), K(tablet_id), K(param)); } return ret; } int ObLS::update_tablet_table_store( - const int64_t ls_rebuild_seq, const ObTabletHandle &old_tablet_handle, const ObIArray &tables) { @@ -1268,14 +1034,9 @@ int ObLS::update_tablet_table_store( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(old_tablet_handle), K(tables)); } else { - const share::ObLSID &ls_id = ls_meta_.ls_id_; const common::ObTabletID &tablet_id = old_tablet_handle.get_obj()->get_tablet_meta().tablet_id_; - const int64_t rebuild_seq = ls_meta_.get_rebuild_seq(); - if (OB_UNLIKELY(ls_rebuild_seq != rebuild_seq)) { - ret = OB_EAGAIN; - LOG_WARN("rebuild seq has changed, retry", K(ret), K(ls_id), K(tablet_id), K(rebuild_seq), K(ls_rebuild_seq)); - } else if (OB_FAIL(ls_tablet_svr_.update_tablet_table_store(old_tablet_handle, tables))) { - LOG_WARN("fail to replace small sstables in the tablet", K(ret), K(ls_id), K(tablet_id), K(old_tablet_handle), K(tables)); + if (OB_FAIL(ls_tablet_svr_.update_tablet_table_store(old_tablet_handle, tables))) { + LOG_WARN("fail to replace small sstables in the tablet", K(ret), K(tablet_id), K(old_tablet_handle), K(tables)); } } return ret; @@ -1315,20 +1076,14 @@ int ObLS::inner_build_tablet_with_batch_tables_( { int ret = OB_SUCCESS; RDLockGuard guard(meta_rwlock_); - const share::ObLSID &ls_id = ls_meta_.ls_id_; - const int64_t rebuild_seq = ls_meta_.get_rebuild_seq(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); } else if (!tablet_id.is_valid() || !param.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("build ha tablet new table store get invalid argument", K(ret), K(ls_id), K(tablet_id), K(param)); - } else if (param.rebuild_seq_ != rebuild_seq) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("build ha tablet new table store rebuild seq not same, need retry", - K(ret), K(ls_id), K(tablet_id), K(rebuild_seq), K(param)); + LOG_WARN("build ha tablet new table store get invalid argument", K(ret), K(tablet_id), K(param)); } else if (OB_FAIL(ls_tablet_svr_.build_tablet_with_batch_tables(tablet_id, param))) { - LOG_WARN("failed to update tablet table store", K(ret), K(ls_id), K(tablet_id), K(param)); + LOG_WARN("failed to update tablet table store", K(ret), K(tablet_id), K(param)); } return ret; } @@ -1342,47 +1097,14 @@ int ObLS::build_new_tablet_from_mds_table( { int ret = OB_SUCCESS; RDLockGuard guard(meta_rwlock_); - const share::ObLSID &ls_id = ls_meta_.ls_id_; - const int64_t ls_rebuild_seq = ctx.get_ls_rebuild_seq(); - const int64_t rebuild_seq = ls_meta_.get_rebuild_seq(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); } else if (OB_UNLIKELY(!tablet_id.is_valid() || !flush_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(flush_scn)); - } else if (OB_UNLIKELY(ls_rebuild_seq != rebuild_seq)) { - ret = OB_EAGAIN; - LOG_WARN("rebuild seq from merge ctx is not the same with current ls, need retry", - K(ret), K(ls_id), K(tablet_id), K(ls_rebuild_seq), K(rebuild_seq), K(flush_scn)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(flush_scn)); } else if (OB_FAIL(ls_tablet_svr_.build_new_tablet_from_mds_table(ctx, tablet_id, mds_mini_sstable_handle, flush_scn, handle))) { - LOG_WARN("failed to build new tablet from mds table", K(ret), K(ls_id), K(tablet_id), K(flush_scn)); - } - return ret; -} - -int ObLS::check_ls_migration_status( - bool &ls_is_migration, - int64_t &rebuild_seq) -{ - int ret = OB_SUCCESS; - RDLockGuard guard(meta_rwlock_); - ls_is_migration = false; - rebuild_seq = 0; - ObMigrationStatus migration_status; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret)); - } else if (OB_UNLIKELY(is_stopped())) { - ret = OB_NOT_RUNNING; - LOG_WARN("ls stopped", K(ret), K_(ls_meta)); - } else if (OB_FAIL(ls_meta_.get_migration_status(migration_status))) { - LOG_WARN("failed to get migration status", K(ret), KPC(this)); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) { - //no need update upper trans version - ls_is_migration = true; - } else { - rebuild_seq = get_rebuild_seq(); + LOG_WARN("failed to build new tablet from mds table", K(ret), K(tablet_id), K(flush_scn)); } return ret; } @@ -1390,23 +1112,12 @@ int ObLS::check_ls_migration_status( int ObLS::finish_storage_meta_replay() { int ret = OB_SUCCESS; - ObMigrationStatus current_migration_status; - ObMigrationStatus new_migration_status; int64_t read_lock = 0; int64_t write_lock = LSLOCKALL; const int64_t start_ts = ObTimeUtility::current_time(); ObLSLockGuard lock_myself(this, lock_, read_lock, write_lock); - if (OB_FAIL(get_migration_status(current_migration_status))) { - LOG_WARN("failed to get migration status", K(ret), KPC(this)); - } else if (OB_FAIL(ObMigrationStatusHelper::trans_reboot_status(current_migration_status, - new_migration_status))) { - LOG_WARN("failed to trans fail status", K(ret), K(current_migration_status), - K(new_migration_status)); - } else if (ls_meta_.get_persistent_state().can_update_ls_meta() && - OB_FAIL(ls_meta_.set_migration_status(ls_epoch_, new_migration_status, false /*no need write slog*/))) { - LOG_WARN("failed to set migration status", K(ret), K(new_migration_status)); - } else if (OB_FAIL(running_state_.create_finish(ls_meta_.ls_id_))) { + if (OB_FAIL(running_state_.create_finish())) { LOG_WARN("create finish failed", KR(ret), K(ls_meta_)); } else { // after slog replayed, the ls must be offlined state. @@ -1422,7 +1133,7 @@ int ObLS::replay_get_tablet_no_check( ObTabletHandle &handle) const { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_meta_.ls_id_, tablet_id); + const ObTabletMapKey key(tablet_id); const SCN tablet_change_checkpoint_scn = ls_meta_.get_tablet_change_checkpoint_scn(); SCN max_scn; ObTabletHandle tablet_handle; @@ -1439,7 +1150,7 @@ int ObLS::replay_get_tablet_no_check( } else if (scn <= tablet_change_checkpoint_scn) { ret = OB_OBSOLETE_CLOG_NEED_SKIP; LOG_WARN("tablet already gc", K(ret), K(key), K(scn), K(tablet_change_checkpoint_scn)); - } else if (OB_FAIL(share::g_mp->log_service()->get_log_replay_service()->get_max_replayed_scn(ls_meta_.ls_id_, max_scn))) { + } else if (OB_FAIL(share::g_mp->log_service()->get_log_replay_service()->get_max_replayed_scn(max_scn))) { LOG_WARN("failed to get_max_replayed_scn", KR(ret), K_(ls_meta), K(scn), K(tablet_id)); } // double check for this scenario: @@ -1478,7 +1189,6 @@ int ObLS::replay_get_tablet( ObTabletHandle &handle) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls_meta_.ls_id_; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObTabletCreateDeleteMdsUserData data; @@ -1491,16 +1201,16 @@ int ObLS::replay_get_tablet( ret = OB_NOT_INIT; LOG_WARN("ls is not inited", KR(ret)); } else if (OB_FAIL(replay_get_tablet_no_check(tablet_id, scn, replay_allow_tablet_not_exist, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_id), K(scn)); } else if (tablet_id.is_ls_inner_tablet()) { // do nothing } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet should not be NULL", K(ret), KP(tablet), K(ls_id), K(tablet_id), K(scn)); + LOG_WARN("tablet should not be NULL", K(ret), KP(tablet), K(tablet_id), K(scn)); } else if (tablet->is_empty_shell()) { ObTabletStatus::Status tablet_status = ObTabletStatus::MAX; if (OB_FAIL(tablet->get_latest(data, writer, trans_stat, trans_version))) { - LOG_WARN("failed to get latest tablet status", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get latest tablet status", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(mds::TwoPhaseCommitState::ON_COMMIT != trans_stat)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is empty shell but user data is uncommitted, unexpected", K(ret), KPC(tablet)); @@ -1509,23 +1219,22 @@ int ObLS::replay_get_tablet( LOG_WARN("tablet is empty shell but user data is unexpected", K(ret), K(data), KPC(tablet)); } else { ret = OB_OBSOLETE_CLOG_NEED_SKIP; - LOG_INFO("tablet is already deleted, need skip", KR(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_INFO("tablet is already deleted, need skip", KR(ret), K(tablet_id), K(scn)); } } else if ((!is_update_mds_table && scn > tablet->get_clog_checkpoint_scn()) || (is_update_mds_table && scn > tablet->get_mds_checkpoint_scn())) { if (OB_FAIL(tablet->get_latest(data, writer, trans_stat, trans_version))) { if (OB_EMPTY_RESULT == ret) { ret = OB_EAGAIN; - LOG_INFO("read empty mds data, should retry", KR(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_INFO("read empty mds data, should retry", KR(ret), K(tablet_id), K(scn)); } else { LOG_WARN("failed to get latest tablet status", K(ret), KPC(tablet)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT != trans_stat) { if ((ObTabletStatus::NORMAL == data.tablet_status_ && data.create_commit_version_ == ObTransVersion::INVALID_TRANS_VERSION) - || ObTabletStatus::RESERVED_5 == data.tablet_status_ || ObTabletStatus::SPLIT_DST == data.tablet_status_) { ret = OB_EAGAIN; - LOG_INFO("latest transaction has not committed yet, should retry", KR(ret), K(ls_id), K(tablet_id), + LOG_INFO("latest transaction has not committed yet, should retry", KR(ret), K(tablet_id), K(scn), "clog_checkpoint_scn", tablet->get_clog_checkpoint_scn(), K(data)); } } @@ -1551,19 +1260,14 @@ int ObLS::logstream_freeze(const bool is_sync, const int64_t abs_timeout_ts = (0 == input_abs_timeout_ts) ? ObClockGenerator::getClock() + ObFreezer::SYNC_FREEZE_DEFAULT_RETRY_TIME : input_abs_timeout_ts; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_meta_.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_meta_.ls_id_)); - } else { - ret = logstream_freeze_task(abs_timeout_ts); - } + ret = logstream_freeze_task(abs_timeout_ts); } else { const bool is_ls_freeze = true; (void)ls_freezer_.submit_an_async_freeze_task(is_ls_freeze); } if (OB_SUCC(ret)) { - share::g_mp->tenant_freezer()->record_freezer_source_event(ls_meta_.ls_id_, source); + share::g_mp->tenant_freezer()->record_freezer_source_event(source); } return ret; @@ -1685,7 +1389,7 @@ int ObLS::tablet_freeze(const ObIArray &tablet_ids, } if (OB_SUCC(ret)) { - share::g_mp->tenant_freezer()->record_freezer_source_event(ls_meta_.ls_id_, source); + share::g_mp->tenant_freezer()->record_freezer_source_event(source); } return ret; @@ -1719,7 +1423,7 @@ int ObLS::tablet_freeze_task(const ObIArray &tablet_ids, } else if (OB_FAIL(ls_freezer_.tablet_freeze( tablet_ids, need_rewrite_meta, frozen_memtable_handles, freeze_failed_tablets))) { if (REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL)) { - STORAGE_LOG(WARN, "tablet freeze failed", KR(ret), K(ls_meta_.ls_id_), K(tablet_ids), K(freeze_failed_tablets)); + STORAGE_LOG(WARN, "tablet freeze failed", KR(ret), K(tablet_ids), K(freeze_failed_tablets)); } } } @@ -1771,7 +1475,7 @@ int ObLS::advance_checkpoint_by_flush(SCN recycle_scn, int ret = OB_SUCCESS; if (is_tenant_freeze) { ObDataCheckpoint::set_tenant_freeze(); - LOG_INFO("set tenant_freeze", K(ls_meta_.ls_id_)); + LOG_INFO("set tenant_freeze"); } ObDataCheckpoint::set_freeze_source(source); ret = checkpoint_executor_.advance_checkpoint_by_flush(recycle_scn); @@ -1795,45 +1499,12 @@ int ObLS::flush_to_recycle_clog() LOG_WARN("offline ls not allowed freeze", K(ret), K_(ls_meta)); } else if (FALSE_IT(ObDataCheckpoint::set_freeze_source(ObFreezeSourceFlag::CLOG_CHECKPOINT))) { } else if (OB_FAIL(checkpoint_executor_.advance_checkpoint_by_flush(SCN::invalid_scn() /*recycle_scn*/))) { - STORAGE_LOG(WARN, "advance_checkpoint_by_flush failed", KR(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "advance_checkpoint_by_flush failed", KR(ret)); } ObDataCheckpoint::reset_freeze_source(); return ret; } -int ObLS::get_ls_meta_package_and_tablet_metas( - const bool check_archive, - const HandleLSMetaFunc &handle_ls_meta_f, - const bool need_sorted_tablet_id, - const ObLSTabletService::HandleTabletMetaFunc &handle_tablet_meta_f) -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret)); - } else if (OB_UNLIKELY(is_stopped())) { - ret = OB_NOT_RUNNING; - LOG_WARN("ls stopped", K(ret), K_(ls_meta)); - } else if (OB_FAIL(tablet_gc_handler_.disable_gc())) { - LOG_WARN("failed to disable gc", K(ret), "ls_id", ls_meta_.ls_id_); - } else { - // TODO(wangxiaohui.wxh) 4.3, consider the ls is offline meanwhile. - // disable gc while get all tablet meta - ObLSMetaPackage meta_package; - if (OB_FAIL(get_ls_meta_package(check_archive, meta_package))) { - LOG_WARN("failed to get ls meta package", K(ret), K_(ls_meta)); - } else if (OB_FAIL(handle_ls_meta_f(meta_package))) { - LOG_WARN("failed to handle ls meta", K(ret), K_(ls_meta), K(meta_package)); - } else if (OB_FAIL(ls_tablet_svr_.ha_scan_all_tablets(handle_tablet_meta_f, need_sorted_tablet_id))) { - LOG_WARN("failed to scan all tablets", K(ret), K_(ls_meta)); - } - tablet_gc_handler_.enable_gc(); - } - - return ret; -} - int ObLS::check_ls_need_online(bool &need_online) { int ret = OB_SUCCESS; @@ -1861,76 +1532,25 @@ int ObLS::disable_replay() return ret; } -int ObLS::update_ls_meta(const bool update_restore_status, - const ObLSMeta &src_ls_meta) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret), K(ls_meta_)); - } else if (OB_UNLIKELY(is_stopped())) { - ret = OB_NOT_RUNNING; - LOG_WARN("ls stopped", K(ret), K_(ls_meta)); - } else if (OB_FAIL(ls_meta_.update_ls_meta(ls_epoch_, update_restore_status, src_ls_meta))) { - LOG_WARN("update ls meta fail", K(ret), K_(ls_meta), K(update_restore_status), K(src_ls_meta)); - } else if (IDS_LS == ls_meta_.ls_id_) { - ObAllIDMeta all_id_meta; - if (OB_FAIL(ls_meta_.get_all_id_meta(all_id_meta))) { - LOG_WARN("get all id meta failed", K(ret), K(ls_meta_)); - } else if (OB_FAIL(ObIDService::update_id_service(all_id_meta))) { - LOG_WARN("update id service fail", K(ret), K(all_id_meta), K(*this)); - } - } else { - // do nothing - } - - return ret; -} - int ObLS::diagnose(DiagnoseInfo &info) const { int ret = OB_SUCCESS; ObLogService *log_service = share::g_mp->log_service(); - share::ObLSID ls_id = get_ls_id(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "ls is not inited", K(ret)); - } else if (FALSE_IT(info.ls_id_ = ls_id.id()) || - FALSE_IT(info.rc_diagnose_info_.id_ = ls_id.id())) { } else if (OB_FAIL(checkpoint_executor_.get_clog_checkpoint_stat(info.ls_clog_checkpoint_stat_))) { - STORAGE_LOG(WARN, "get clog checkpoint stat failed", K(ret), K(ls_id)); - } else if (OB_FAIL(log_service->diagnose_apply(ls_id, info.apply_diagnose_info_))) { - STORAGE_LOG(WARN, "diagnose apply failed", K(ret), K(ls_id)); - } else if (OB_FAIL(log_service->diagnose_replay(ls_id, info.replay_diagnose_info_))) { - STORAGE_LOG(WARN, "diagnose replay failed", K(ret), K(ls_id)); - } else if (OB_FAIL(log_handler_.diagnose(info.log_handler_diagnose_info_))) { - STORAGE_LOG(WARN, "diagnose log handler failed", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get clog checkpoint stat failed", K(ret)); + } else if (OB_FAIL(log_service->diagnose_apply(info.apply_diagnose_info_))) { + STORAGE_LOG(WARN, "diagnose apply failed", K(ret)); + } else if (OB_FAIL(log_service->diagnose_replay(info.replay_diagnose_info_))) { + STORAGE_LOG(WARN, "diagnose replay failed", K(ret)); } else if (OB_FAIL(log_handler_.diagnose_palf(info.palf_diagnose_info_))) { - STORAGE_LOG(WARN, "diagnose palf failed", K(ret), K(ls_id)); - } else if (info.is_role_sync()) { - // Role synchronization does not require diagnosis role change service - info.rc_diagnose_info_.state_ = TakeOverState::TAKE_OVER_FINISH; - info.rc_diagnose_info_.log_type_ = ObLogBaseType::INVALID_LOG_BASE_TYPE; - } else if (OB_FAIL(log_service->diagnose_role_change(info.rc_diagnose_info_))) { - // election, palf, log handler roles are not unified when a leaderless situation may occur - STORAGE_LOG(WARN, "diagnose rc service failed", K(ret), K(ls_id)); - } - DiagnoseFunctor fn(ls_id, info.read_only_tx_info_, 0, sizeof(info.read_only_tx_info_)); - READ_CHECKER_FOR_EACH(fn); - STORAGE_LOG(INFO, "diagnose finish", K(ret), K(info), K(ls_id)); - return ret; -} - -int ObLS::inc_update_reserved_scn(const share::SCN &reserved_scn) -{ - int ret = OB_SUCCESS; - WRLockGuard guard(meta_rwlock_); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ls_meta_.inc_update_reserved_scn(ls_epoch_, reserved_scn))) { - LOG_WARN("fail to set reserved scn", K(ret), K(reserved_scn), K_(ls_meta)); - } else { - // do nothing + STORAGE_LOG(WARN, "diagnose palf failed", K(ret)); } + ObReadOnlyTxDiagnoseFunctor fn(info.read_only_tx_info_, 0, sizeof(info.read_only_tx_info_)); + READ_CHECKER_FOR_EACH(fn); + STORAGE_LOG(INFO, "diagnose finish", K(ret), K(info)); return ret; } @@ -1938,7 +1558,6 @@ int ObLS::set_restore_status(const ObRestoreStatus &restore_status) { int ret = OB_SUCCESS; WRLockGuard guard(meta_rwlock_); - bool allow_read = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -1953,69 +1572,6 @@ int ObLS::set_restore_status(const ObRestoreStatus &restore_status) STORAGE_LOG(WARN, "state not match, cannot update ls meta", K(ret), K(ls_meta_)); } else if (OB_FAIL(ls_meta_.set_restore_status(ls_epoch_, restore_status))) { LOG_WARN("failed to set restore status", K(ret), K(restore_status)); - } else if (OB_FAIL(inner_check_allow_read_(restore_status, allow_read))) { - LOG_WARN("failed to check allow to read", K(ret), K(restore_status)); - } else if (allow_read) { - ls_tablet_svr_.enable_to_read(); - } else { - ls_tablet_svr_.disable_to_read(); - } - return ret; -} - -int ObLS::set_ls_rebuild() -{ - int ret = OB_SUCCESS; - WRLockGuard guard(meta_rwlock_); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ls is not inited", K(ret), K(ls_meta_)); - } else if (OB_UNLIKELY(is_stopped())) { - ret = OB_NOT_RUNNING; - STORAGE_LOG(WARN, "ls stopped", K(ret), K_(ls_meta)); - } else if (!ls_meta_.get_persistent_state().can_update_ls_meta()) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "state not match, cannot update ls meta", K(ret), K(ls_meta_)); - } else if (OB_FAIL(ls_meta_.set_ls_rebuild(ls_epoch_))) { - LOG_WARN("failed to set ls rebuild", K(ret), K(ls_meta_)); - } else { - ls_tablet_svr_.disable_to_read(); - } - return ret; -} - -int ObLS::set_ls_migration_gc( - bool &allow_gc) -{ - int ret = OB_SUCCESS; - allow_gc = false; - const bool write_slog = true; - const ObMigrationStatus change_status = ObMigrationStatus::OB_MIGRATION_STATUS_GC; - ObMigrationStatus curr_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; - ObLSRebuildInfo rebuild_info; - WRLockGuard guard(meta_rwlock_); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ls is not inited", K(ret), K(ls_meta_)); - } else if (!ls_meta_.get_persistent_state().can_update_ls_meta()) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "state not match, cannot update ls meta", K(ret), K(ls_meta_)); - } else if (OB_FAIL(ls_meta_.get_rebuild_info(rebuild_info))) { - LOG_WARN("failed to get rebuild info", K(ret), K(ls_meta_)); - } else if (rebuild_info.is_in_rebuild()) { - allow_gc = false; - } else if (OB_FAIL(ls_meta_.get_migration_status(curr_status))) { - LOG_WARN("failed to get migration status", K(ret), K(ls_meta_)); - } else if (ObMigrationStatusHelper::check_allow_gc_abandoned_ls(curr_status)) { - allow_gc = true; - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != curr_status) { - allow_gc = false; - } else if (OB_FAIL(ls_meta_.set_migration_status(ls_epoch_, change_status, write_slog))) { - LOG_WARN("failed to set migration status", K(ret), K(change_status)); - } else { - allow_gc = true; } return ret; } @@ -2038,55 +1594,5 @@ void ObLS::clear_delay_resource_recycle() LOG_INFO("clear delay resource recycle", KPC(this)); } -int ObLS::check_allow_read(bool &allow_to_read) -{ - int ret = OB_SUCCESS; - ObRestoreStatus restore_status; - allow_to_read = false; - //allow ls is not init because create ls will schedule this interface - if (OB_FAIL(ls_meta_.get_restore_status(restore_status))) { - LOG_WARN("failed to get migration and restore status"); - } else if (OB_FAIL(inner_check_allow_read_(restore_status, allow_to_read))) { - LOG_WARN("failed to do inner check allow read", K(ret), K(restore_status)); - } - return ret; -} - -int ObLS::inner_check_allow_read_( - const ObRestoreStatus &restore_status, - bool &allow_read) -{ - int ret = OB_SUCCESS; - allow_read = false; - if (!restore_status.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("inner check allow read get invalid argument", K(ret), K(restore_status)); - } else if (restore_status.check_allow_read()) { - allow_read = true; - } else { - allow_read = false; - } - return ret; -} - -int ObLS::set_ls_allow_to_read() -{ - int ret = OB_SUCCESS; - bool allow_to_read = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls is not inited", K(ret)); - } else if (OB_FAIL(check_allow_read(allow_to_read))) { - LOG_WARN("failed to check ls allow read", K(ret)); - } else if (allow_to_read) { - ls_tablet_svr_.enable_to_read(); - LOG_INFO("set ls allow to read", "ls_id", get_ls_id()); - } else { - ls_tablet_svr_.disable_to_read(); - LOG_INFO("set ls not allow to read", "ls_id", get_ls_id()); - } - return ret; -} - } } diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 719d6bca7..8ace15a9c 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -18,16 +18,12 @@ #define OCEABASE_STORAGE_OB_LS_ #include "lib/utility/ob_print_utils.h" -#include "lib/task/ob_timer.h" -#include "common/ob_member_list.h" #include "share/ob_delegate.h" #include "lib/worker.h" #include "storage/ls/ob_ls_lock.h" #include "storage/ls/ob_ls_tablet_service.h" #include "storage/ls/ob_ls_tx_service.h" -#include "storage/ls/ob_ls_role_handler.h" #include "storage/ls/ob_ls_fo_handler.h" -#include "storage/ls/ob_ls_rebuild_handler.h" #include "storage/ls/ob_ls_meta.h" #include "storage/ls/ob_freezer.h" #include "storage/ls/ob_ls_sync_tablet_seq_handler.h" @@ -43,15 +39,11 @@ #include "logservice/applyservice/ob_log_apply_service.h" #include "logservice/replayservice/ob_replay_handler.h" #include "logservice/replayservice/ob_replay_status.h" -#include "logservice/rcservice/ob_role_change_handler.h" +#include "logservice/localservice/ob_local_log_handler_set.h" #include "logservice/ob_log_handler.h" -#include "storage/ls/ob_ls_meta_package.h" -#include "storage/ls/ob_ls_get_mod.h" #include "storage/tablelock/ob_lock_table.h" -#include "lib/hash/ob_multi_mod_ref_mgr.h" #include "storage/tx_storage/ob_tablet_gc_service.h" #include "storage/tx_storage/ob_empty_shell_task.h" -#include "storage/mview/ob_major_mv_merge_info.h" #include "observer/vector_index/ob_plugin_vector_index_scheduler.h" #include "storage/ls/ob_freezer_define.h" @@ -70,72 +62,37 @@ namespace storage struct ObLSVTInfo { - share::ObLSID ls_id_; - ObReplicaType replica_type_; - common::ObRole ls_state_; - ObMigrationStatus migrate_status_; int64_t tablet_count_; share::SCN weak_read_scn_; - bool need_rebuild_; share::SCN checkpoint_scn_; //TODO SCN int64_t checkpoint_lsn_; - int64_t rebuild_seq_; share::SCN tablet_change_checkpoint_scn_; - share::SCN reserved_scn_; bool tx_blocked_; - share::SCN mv_major_merge_scn_; - share::SCN mv_publish_scn_; - share::SCN mv_safe_scn_; - int64_t required_data_disk_size_; - TO_STRING_KV(K_(ls_id), - K_(replica_type), - K_(ls_state), - K_(migrate_status), - K_(tablet_count), + TO_STRING_KV(K_(tablet_count), K_(weak_read_scn), K_(checkpoint_scn), K_(checkpoint_lsn), - K_(rebuild_seq), K_(tablet_change_checkpoint_scn), - K_(reserved_scn), - K_(tx_blocked), - K_(mv_major_merge_scn), - K_(mv_publish_scn), - K_(mv_safe_scn), - K_(required_data_disk_size)); + K_(tx_blocked)); }; // Diagnose vtable statistics information struct DiagnoseInfo { DiagnoseInfo() { reset(); } ~DiagnoseInfo() { reset(); } - bool is_role_sync() { - return ((palf_diagnose_info_.election_role_ == palf_diagnose_info_.palf_role_) && - ((palf_diagnose_info_.palf_role_ == log_handler_diagnose_info_.log_handler_role_ && - palf_diagnose_info_.palf_proposal_id_ == log_handler_diagnose_info_.log_handler_proposal_id_))); - } - int64_t ls_id_; - logservice::LogHandlerDiagnoseInfo log_handler_diagnose_info_; palf::PalfDiagnoseInfo palf_diagnose_info_; - logservice::RCDiagnoseInfo rc_diagnose_info_; logservice::ApplyDiagnoseInfo apply_diagnose_info_; logservice::ReplayDiagnoseInfo replay_diagnose_info_; checkpoint::ObLsClogCheckpointStat ls_clog_checkpoint_stat_; char read_only_tx_info_[1024]; - TO_STRING_KV(K(ls_id_), - K(log_handler_diagnose_info_), - K(palf_diagnose_info_), - K(rc_diagnose_info_), + TO_STRING_KV(K(palf_diagnose_info_), K(apply_diagnose_info_), K(replay_diagnose_info_), K(ls_clog_checkpoint_stat_), K(read_only_tx_info_)); void reset() { - ls_id_ = -1; - log_handler_diagnose_info_.reset(); palf_diagnose_info_.reset(); - rc_diagnose_info_.reset(); apply_diagnose_info_.reset(); replay_diagnose_info_.reset(); ls_clog_checkpoint_stat_.reset(); @@ -145,7 +102,7 @@ struct DiagnoseInfo class ObIComponentFactory; -class ObLS : public common::ObLink +class ObLS { public: typedef common::ObLatch RWLock; @@ -201,16 +158,12 @@ class ObLS : public common::ObLink public: ObLS(); virtual ~ObLS(); - int init(const share::ObLSID &ls_id, - const ObMigrationStatus &migration_status, - const ObRestoreStatus &restore_status, - const share::SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info); + int init(const ObRestoreStatus &restore_status, + const share::SCN &create_scn); // I am ready to work now. int stop(); void wait(); int prepare_for_safe_destroy(); - bool safe_to_destroy(); void destroy(); int offline(); int online(); @@ -227,11 +180,7 @@ class ObLS : public common::ObLink ObTxTable *get_tx_table() { return &tx_table_; } ObLSWRSHandler *get_ls_wrs_handler() { return &ls_wrs_handler_; } ObLSTabletService *get_tablet_svr() { return &ls_tablet_svr_; } - share::ObLSID get_ls_id() const { return ls_meta_.ls_id_; } - bool is_sys_ls() const { return ls_meta_.ls_id_.is_sys_ls(); } - ObFreezer *get_freezer() { return &ls_freezer_; } - common::ObMultiModRefMgr &get_ref_mgr() { return ref_mgr_; } checkpoint::ObCheckpointExecutor *get_checkpoint_executor() { return &checkpoint_executor_; } checkpoint::ObDataCheckpoint *get_data_checkpoint() { return &data_checkpoint_; } transaction::ObKeepAliveLSHandler *get_keep_alive_ls_handler() { return &keep_alive_ls_handler_; } @@ -239,7 +188,6 @@ class ObLS : public common::ObLink // ObObLogHandler interface: // get the log_service pointer logservice::ObLogHandler *get_log_handler() { return &log_handler_; } - logservice::ObRoleChangeHandler *get_role_change_handler() { return &role_change_handler_;} //remove member handler @@ -248,40 +196,17 @@ class ObLS : public common::ObLink // get ls info int get_ls_info(ObLSVTInfo &ls_info); - int get_ls_role(ObRole &role); - // report the ls replica info to RS. - int report_replica_info(); - // set disk state of ls. int set_start_work_state(); - int set_start_ha_state(); int set_remove_state(); ObLSPersistentState get_persistent_state() const; int finish_create_ls(); - // for rebuild - // remove inner tablet, the memtable and minor sstable of data tablet, disable replay - // int prepare_rebuild(); - // int delete_tablet(); - // int disable_replay(); - // create myself at disk - // @param[in] tenant_role, role of tenant, which determains palf access mode // @param[in] palf_base_info, all the info that palf needed - // @param[in] allow_log_sync, if palf is allowed to sync log, will be removed - // after migrating as learner - int create_ls(const share::ObTenantRole tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const common::ObReplicaType &replica_type, - const bool allow_log_sync = false); + int create_ls(const palf::PalfBaseInfo &palf_base_info); // load ls info from disk - // @param[in] tenant_role, role of tenant, which determains palf access mode - // @param[in] palf_base_info, all the info that palf needed - // @param[in] allow_log_sync, if palf is allowed to sync log, will be removed - // after migrating as learner - int load_ls(const share::ObTenantRole &tenant_role, - const palf::PalfBaseInfo &palf_base_info, - const bool allow_log_sync); + int load_ls(); // remove the durable info of myself from disk. int remove_ls(); // create all the inner tablet. @@ -289,11 +214,6 @@ class ObLS : public common::ObLink const share::SCN &create_scn); int remove_ls_inner_tablet(); - // get the meta package of ls: ObLSMeta, PalfBaseInfo - // @param[in] check_archive, if need check archive, - // for backup task is false, migration/rebuild is true - // @param[out] meta_package, the meta package. - int get_ls_meta_package(const bool check_archive, ObLSMetaPackage &meta_package); // get only the meta of ls. const ObLSMeta &get_ls_meta() const { return ls_meta_; } // get current ls meta. @@ -305,9 +225,6 @@ class ObLS : public common::ObLink int64_t get_ls_epoch() const { return ls_epoch_; } int set_ls_epoch(const int64_t ls_epoch); - // do the work after storage meta replay - // 1) rewrite the migration status if it is failed. - // 2) load inner tablet and start to work if it is a normal ls. int finish_storage_meta_replay(); // get tablet while replaying clog @@ -326,7 +243,6 @@ class ObLS : public common::ObLink int try_sync_reserved_snapshot(const int64_t new_reserved_snapshot, const bool update_flag); int check_can_replay_clog(bool &can_replay); int check_ls_need_online(bool &need_online); - int check_allow_read(bool &allow_read); // for delaying the resource recycle after correctness issue bool need_delay_resource_recycle() const; @@ -347,6 +263,8 @@ class ObLS : public common::ObLink int online_compaction_(); int offline_tx_(const int64_t start_ts); int online_tx_(); + int start_local_log_(); + int stop_local_log_(); int update_tablet_table_store_without_lock_( const ObTabletID &tablet_id, const ObUpdateTableStoreParam ¶m, @@ -355,18 +273,13 @@ class ObLS : public common::ObLink int online_advance_epoch_(); int register_to_service_(); int register_common_service(); - int register_sys_service(); - int register_user_service(); + int register_local_services_(); void unregister_from_service_(); void unregister_common_service_(); - void unregister_sys_service_(); - void unregister_user_service_(); + void unregister_local_services_(); public: // ObLSMeta interface: - int update_ls_meta(const bool update_restore_status, - const ObLSMeta &src_ls_meta); - int update_id_meta(const int64_t service_type, const int64_t limited_id, const share::SCN &latest_scn, @@ -374,7 +287,6 @@ class ObLS : public common::ObLink { return ls_meta_.update_id_meta(ls_epoch_, service_type, limited_id, latest_scn, write_slog); } - int set_ls_rebuild(); // protect in ls lock int set_clog_checkpoint(const palf::LSN &clog_checkpoint_lsn, const share::SCN &clog_checkpoint_scn, @@ -393,25 +305,17 @@ class ObLS : public common::ObLink { return ls_meta_.clear_saved_info(ls_epoch_); } - CONST_DELEGATE_WITH_RET(ls_meta_, get_rebuild_seq, int64_t); CONST_DELEGATE_WITH_RET(ls_meta_, get_tablet_change_checkpoint_scn, share::SCN); DELEGATE_WITH_RET(ls_meta_, set_tablet_change_checkpoint_scn, int); int set_tablet_change_checkpoint_scn(const share::SCN &tablet_change_checkpoint_scn) { return ls_meta_.set_tablet_change_checkpoint_scn(ls_epoch_, tablet_change_checkpoint_scn); } - int set_major_mv_merge_scn(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn(ls_epoch_, scn); } - int set_major_mv_merge_scn_safe_calc(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn_safe_calc(ls_epoch_, scn); } - int set_major_mv_merge_scn_publish(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn_publish(ls_epoch_, scn); } int set_restore_status(const ObRestoreStatus &restore_status); // get restore status // @param [out] restore status. // int get_restore_status(share::ObRestoreStatus &status); DELEGATE_WITH_RET(ls_meta_, get_restore_status, int); - // get migration status - // @param [out] migration status. - // int get_migration_status(ObMigrationstatus &status); - DELEGATE_WITH_RET(ls_meta_, get_migration_status, int); // get offline ts // @param [in] offline ts. // int get_offline_scn(const share::SCN &offline_scn); @@ -426,31 +330,7 @@ class ObLS : public common::ObLink // @param [in] replayable point // int get_ls_replayable_point(int64_t &replayable_point); DELEGATE_WITH_RET(ls_meta_, get_ls_replayable_point, int); - int inc_update_reserved_scn(const share::SCN &reserved_scn); - int set_reserved_scn(const share::SCN &reserved_scn); - // get ls_meta_package and unsorted tablet_ids, add read lock of LSLOCKLSMETA. - // @param [in] check_archive if need check archive, for backup task is false, migration/rebuild is true - // @param [out] meta_package - // @param [out] tablet_ids - DELEGATE_WITH_RET(ls_meta_, get_migration_and_restore_status, int); - int set_rebuild_info(const ObLSRebuildInfo &rebuild_info) - { - return ls_meta_.set_rebuild_info(ls_epoch_, rebuild_info); - } - DELEGATE_WITH_RET(ls_meta_, get_rebuild_info, int); - DELEGATE_WITH_RET(ls_meta_, get_create_type, int); - // get ls_meta_package and sorted tablet_metas for backup. tablet gc is forbidden meanwhile. - // @param [in] check_archive if need check archive, migration/rebuild is true - // @param [in] handle_ls_meta_f, ls meta callback, will be first called. - // @param [in] handle_tablet_meta_f, tablet meta callback - typedef common::ObFunction HandleLSMetaFunc; - int get_ls_meta_package_and_tablet_metas( - const bool check_archive, - const HandleLSMetaFunc &handle_ls_meta_f, - const bool need_sorted_tablet_id, - const ObLSTabletService::HandleTabletMetaFunc &handle_tablet_meta_f); - - // ObLSTabletService interface: + // ObLSTabletService interface: // update tablet by checkpoint // @param [in] key, key of tablet that will be updated // @param [in] new_addr, new addr of the tablet @@ -472,17 +352,9 @@ class ObLS : public common::ObLink // get ls tablet iterator // @param [out] iterator, ls tablet iterator to iterate all tablets in ls // int build_tablet_iter(ObLSTabletIterator &iter); - // int build_tablet_iter(ObLSTabletIDIterator &iter); DELEGATE_WITH_RET(ls_tablet_svr_, build_tablet_iter, int); // update medium compaction info for tablet DELEGATE_WITH_RET(ls_tablet_svr_, update_medium_compaction_info, int); - // trim rebuild tablet - // @param [in] tablet_id ObTabletID, is_rollback bool - // @param [out] null - // int trim_rebuild_tablet( - // const ObTabletID &tablet_id, - // const bool is_rollback = false); - DELEGATE_WITH_RET(ls_tablet_svr_, trim_rebuild_tablet, int); // remove tablets // @param [in] tbalet_ids ObIArray // @param [out] null @@ -490,38 +362,27 @@ class ObLS : public common::ObLink // const common::ObIArray &tablet_id_array); DELEGATE_WITH_RET(ls_tablet_svr_, remove_tablets, int); // create_ls_inner_tablet - // @param [in] ls_id // @param [in] tablet_id // @param [in] frozen_timestamp // @param [in] create_tablet_schema // @param [in] create_scn // int create_ls_inner_tablet( - // const share::ObLSID &ls_id, // const common::ObTabletID &tablet_id, // const share::SCN &frozen_timestamp, // const ObCreateTabletSchema &create_tablet_schema, // const share::SCN &create_scn); DELEGATE_WITH_RET(ls_tablet_svr_, create_ls_inner_tablet, int); // remove_ls_inner_tablet - // @param [in] ls_id // @param [in] tablet_id // int remove_ls_inner_tablet( - // const share::ObLSID &ls_id, // const common::ObTabletID &tablet_id); DELEGATE_WITH_RET(ls_tablet_svr_, remove_ls_inner_tablet, int); - DELEGATE_WITH_RET(ls_tablet_svr_, rebuild_create_tablet, int); - DELEGATE_WITH_RET(ls_tablet_svr_, update_tablet_ha_data_status, int); - DELEGATE_WITH_RET(ls_tablet_svr_, ha_get_tablet, int); - DELEGATE_WITH_RET(ls_tablet_svr_, get_tablet_without_memtables, int); - DELEGATE_WITH_RET(ls_tablet_svr_, ha_get_tablet_without_memtables, int); + DELEGATE_WITH_RET(ls_tablet_svr_, update_tablet_column_store_schema, int); DELEGATE_WITH_RET(ls_tablet_svr_, update_tablet_restore_status, int); - DELEGATE_WITH_RET(ls_tablet_svr_, create_or_update_migration_tablet, int); DELEGATE_WITH_RET(ls_tablet_svr_, flush_mds_table, int); - DELEGATE_WITH_RET(ls_tablet_svr_, enable_to_read, void); - DELEGATE_WITH_RET(ls_tablet_svr_, disable_to_read, void); DELEGATE_WITH_RET(ls_tablet_svr_, get_tablet_with_timeout, int); DELEGATE_WITH_RET(ls_tablet_svr_, get_mds_table_mgr, int); - // check tablet no active memtable + // check that tablets have no active memtable DELEGATE_WITH_RET(ls_tablet_svr_, check_tablet_no_active_memtable, int); // ObLockTable interface: @@ -588,26 +449,7 @@ class ObLS : public common::ObLink DELEGATE_WITH_RET(lock_table_, check_and_clear_obj_lock, int); DELEGATE_WITH_RET(lock_table_, add_lock_into_queue, int); - // set the member_list of log_service - // @param [in] member_list, the member list to be set. - // @param [in] replica_num, the number of replica - // int set_initial_member_list(const common::ObMemberList &member_list, - // const int64_t paxos_replica_num); - DELEGATE_WITH_RET(log_handler_, set_initial_member_list, int); - // get the member list of log_service - // @param [out] member_list, the member_list of current log_service - // @param [out] quorum, the quorum of member_list - // int get_paxos_member_list(common::ObMemberList &member_list, int64_t &quorum) const; - CONST_DELEGATE_WITH_RET(log_handler_, get_paxos_member_list, int); - // get paxos member list and learner list of log_service - // @param [out] member_list, the member_list of current log_service - // @param [out] quorum, the quorum of member_list - // @param [out] learner_list, the learner list of log_service - CONST_DELEGATE_WITH_RET(log_handler_, get_paxos_member_list_and_learner_list, int); - // advance the base_lsn of log_handler. - // @param[in] palf_base_info, the palf meta used to advance base lsn. - // int advance_base_info(const palf::PalfBaseInfo &palf_base_info); - DELEGATE_WITH_RET(log_handler_, advance_base_info, int); + DELEGATE_WITH_RET(log_handler_, bootstrap, int); // disable clog sync. // with ls read lock and log write lock. @@ -619,11 +461,6 @@ class ObLS : public common::ObLink // @brief, get max decided log scn considering both apply and replay. // @param[out] share::SCN&, max decided log scn. DELEGATE_WITH_RET(log_handler_, get_max_decided_scn, int); - // @breif,get member stat: whether in paxos member list or learner list and whether is migrating - // @param[in] const common::ObAddr, request server. - // @param[out] bool &(is_valid_member), - // @param[out] LogMemberGCStat&, - DELEGATE_WITH_RET(log_handler_, get_member_gc_stat, int); // @brief append count bytes from the buffer starting at buf to the palf handle, return the LSN and timestamp // @param[in] const void *, the data buffer. // @param[in] const uint64_t, the length of data buffer. @@ -635,29 +472,9 @@ class ObLS : public common::ObLink // @param[out] int64_t&, the append timestamp. // @retval // OB_SUCCESS - // OB_NOT_MASTER, the prospoal_id of ObLogHandler is not same with PalfHandle. DELEGATE_WITH_RET(log_handler_, append, int); - // @breif, palf enable vote - // @param[in] null. - // @param[out] null. - DELEGATE_WITH_RET(log_handler_, enable_vote, int); - // @breif, palf disable vote - // @param[in] need_check. - // @param[out] null. - DELEGATE_WITH_RET(log_handler_, disable_vote, int); - DELEGATE_WITH_RET(log_handler_, remove_member, int); - DELEGATE_WITH_RET(log_handler_, remove_learner, int); - DELEGATE_WITH_RET(log_handler_, is_in_sync, int); DELEGATE_WITH_RET(log_handler_, get_end_scn, int); - DELEGATE_WITH_RET(log_handler_, disable_sync, int); - DELEGATE_WITH_RET(log_handler_, change_replica_num, int); DELEGATE_WITH_RET(log_handler_, get_end_lsn, int); - DELEGATE_WITH_RET(log_handler_, try_lock_config_change, int); - DELEGATE_WITH_RET(log_handler_, unlock_config_change, int); - DELEGATE_WITH_RET(log_handler_, get_config_change_lock_stat, int); - DELEGATE_WITH_RET(log_handler_, switch_acceptor_to_learner, int); - DELEGATE_WITH_RET(log_handler_, add_learner, int); - DELEGATE_WITH_RET(log_handler_, replace_learners, int); // Create a TxCtx whose tx_id is specified // @param [in] tx_id: transaction ID @@ -667,7 +484,6 @@ class ObLS : public common::ObLink // @param [out] tx_ctx: newly allocated or already exsited transaction context // Return Values That Need Attention: // @return OB_SUCCESS, if the tx_ctx newly allocated or already existed - // @return OB_NOT_MASTER, if this ls is a follower replica CONST_DELEGATE_WITH_RET(ls_tx_svr_, create_tx_ctx, int); // Find specified TxCtx from the ObLSTxCtxMgr; @@ -675,7 +491,6 @@ class ObLS : public common::ObLink // @param [in] for_replay: Identifies whether the TxCtx is used by replay processing; // @param [out] tx_ctx: context found through ObLSTxCtxMgr's hash table // Return Values That Need Attention: - // @return OB_NOT_MASTER, if the LogStream is follower replica // @return OB_TRANS_CTX_NOT_EXIST, if the specified TxCtx is not found; CONST_DELEGATE_WITH_RET(ls_tx_svr_, get_tx_ctx, int); CONST_DELEGATE_WITH_RET(ls_tx_svr_, get_tx_ctx_with_timeout, int); @@ -732,11 +547,6 @@ class ObLS : public common::ObLink // const int64_t timestamp, // ObTransID &block_tx_id); DELEGATE_WITH_RET(ls_tx_svr_, check_modify_time_elapsed, int); - // get tx scheduler in ls tx service - // @param [in] tx_id: wish to get this tx_id scheduler - // @param [out] scheduler: scheduler of this tx_id - // int get_tx_scheduler(const transaction::ObTransID &tx_id, ObAddr &scheduler) const; - CONST_DELEGATE_WITH_RET(ls_tx_svr_, get_tx_scheduler, int); // get tx start session_id in ls tx service // @param [in] tx_id: wish to get this tx_id start session_id // @param [out] session_id: session_id of this tx_id @@ -750,10 +560,6 @@ class ObLS : public common::ObLink DELEGATE_WITH_RET(ls_tx_svr_, get_tx_ctx_count, int); DELEGATE_WITH_RET(ls_tx_svr_, get_active_tx_count, int); DELEGATE_WITH_RET(ls_tx_svr_, print_all_tx_ctx, int); - DELEGATE_WITH_RET(ls_tx_svr_, retry_apply_start_working_log, int); - - DELEGATE_WITH_RET(ls_tx_svr_, wait_tx_write_end, int); - // ObReplayHandler interface: DELEGATE_WITH_RET(replay_handler_, replay, int); @@ -833,7 +639,6 @@ class ObLS : public common::ObLink const ObUpdateTableStoreParam ¶m, ObTabletHandle &handle); int update_tablet_table_store( - const int64_t ls_rebuild_seq, const ObTabletHandle &old_tablet_handle, const ObIArray &tables); int build_tablet_with_batch_tables( @@ -845,9 +650,6 @@ class ObLS : public common::ObLink const ObTableHandleV2 &mds_mini_sstable_handle, const share::SCN &flush_scn, ObTabletHandle &handle); - int check_ls_migration_status( - bool &ls_is_migration, - int64_t &rebuild_seq); int diagnose(DiagnoseInfo &info) const; DELEGATE_WITH_RET(reserved_snapshot_mgr_, replay_reserved_snapshot_log, int); @@ -855,10 +657,6 @@ class ObLS : public common::ObLink DELEGATE_WITH_RET(reserved_snapshot_mgr_, add_dependent_medium_tablet, int); DELEGATE_WITH_RET(reserved_snapshot_mgr_, del_dependent_medium_tablet, int); - int set_ls_migration_gc(bool &allow_gc); - int inner_check_allow_read_(const ObRestoreStatus &restore_status, bool &allow_read); - int set_ls_allow_to_read(); - private: void record_async_freeze_tablets_(const ObIArray &tablet_ids, const int64_t epoch); int inner_build_tablet_with_batch_tables_( @@ -871,7 +669,7 @@ class ObLS : public common::ObLink // log service for ls // log_service manager: create, remove and get logservice::ObLogHandler log_handler_; - logservice::ObRoleChangeHandler role_change_handler_; + logservice::ObLocalLogHandlerSet local_log_handler_set_; // trans service for ls ObLSTxService ls_tx_svr_; @@ -884,8 +682,6 @@ class ObLS : public common::ObLink ObFreezer ls_freezer_; // for FO // ObLSFailoverHandler ls_failover_handler_; - // for rebuild - // ObLSRebuildHandler ls_rebuild_handler_; // for restore ObTxTable tx_table_; checkpoint::ObDataCheckpoint data_checkpoint_; @@ -910,7 +706,6 @@ class ObLS : public common::ObLink int init_vector_idx_scheduler_(); void stop_vector_idx_scheduler_(); void destroy_vector_idx_scheduler_(); - int vector_idx_scheduler_safe_to_destroy_(bool &is_safe); // table_api removed from build: the vector index scheduler used to be hosted by // table::ObTenantTabletTTLMgr (together with tablet TTL); only the vector index // scheduler part is preserved here, driven by its own timer. @@ -928,7 +723,6 @@ class ObLS : public common::ObLink ObLSMeta ls_meta_; int64_t ls_epoch_; ObLSLock lock_; - common::ObMultiModRefMgr ref_mgr_; // this is used for the meta lock, and will be removed later RWLock meta_rwlock_; diff --git a/src/storage/ls/ob_ls_ddl_log_handler.cpp b/src/storage/ls/ob_ls_ddl_log_handler.cpp index 00dac7535..af5de50de 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.cpp +++ b/src/storage/ls/ob_ls_ddl_log_handler.cpp @@ -140,7 +140,7 @@ int ObActiveDDLKVIterator::get_next_ddl_kv_mgr(ObDDLKvMgrHandle &handle) LOG_WARN("push back to delete tablet id failed", K(ret)); } } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_->get_ls_id()), K(tablet_id)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } } else if (tablet_handle.get_obj()->get_tablet_meta().ddl_commit_scn_.is_valid_and_not_min() && tablet_handle.get_obj()->get_tablet_meta().ddl_checkpoint_scn_ >= tablet_handle.get_obj()->get_tablet_meta().ddl_commit_scn_) { @@ -346,30 +346,12 @@ int ObLSDDLLogHandler::replay(const void *buffer, return ret; } -void ObLSDDLLogHandler::switch_to_follower_forcedly() +void ObLSDDLLogHandler::deactivate() { // TODO } -int ObLSDDLLogHandler::switch_to_leader() -{ - int ret = OB_SUCCESS; - - //TODO - - return ret; -} - -int ObLSDDLLogHandler::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - - //TODO - - return ret; -} - -int ObLSDDLLogHandler::resume_leader() +int ObLSDDLLogHandler::activate() { int ret = OB_SUCCESS; @@ -423,7 +405,6 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) ObTabletDirectLoadMgrHandle direct_load_mgr_hdl; // TODO @zhuoran.zzr wait to remove it if (OB_TMP_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( - ls_->get_ls_id(), ddl_kv_mgr_handle.get_obj()->get_tablet_id(), true/* is_full_direct_load */, direct_load_mgr_hdl, @@ -435,7 +416,6 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) } } else { DEBUG_SYNC(BEFORE_DDL_CHECKPOINT); - param.ls_id_ = ls_->get_ls_id(); param.tablet_id_ = ddl_kv_mgr_handle.get_obj()->get_tablet_id(); param.start_scn_ = direct_load_mgr_hdl.get_full_obj()->get_start_scn(); param.rec_scn_ = rec_scn; @@ -457,11 +437,11 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) if (OB_FAIL(ls_->get_tablet(ddl_kv_mgr_handle.get_obj()->get_tablet_id(), tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet handle", K(ret), K(ls_->get_ls_id()), K(ddl_kv_mgr_handle.get_obj()->get_tablet_id())); + LOG_WARN("failed to get tablet handle", K(ret), K(ddl_kv_mgr_handle.get_obj()->get_tablet_id())); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_complete(share::SCN::max_scn(), arena, ddl_complete))) { if (OB_EMPTY_RESULT == ret) { ret = OB_SUCCESS; - LOG_INFO("no ddl complete", K(ret), K(ls_->get_ls_id()), K(ddl_kv_mgr_handle.get_obj()->get_tablet_id())); + LOG_INFO("no ddl complete", K(ret), K(ddl_kv_mgr_handle.get_obj()->get_tablet_id())); } else { LOG_WARN("failed to get ddl complete", K(ret)); } @@ -476,7 +456,7 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) } else if (OB_TMP_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ", K(tmp_ret), K(param)); } - FLOG_INFO("schedule ddl dump merge task", K(ret), K(ls_->get_ls_id()), K(tablet_handle.get_obj()->get_tablet_id())); + FLOG_INFO("schedule ddl dump merge task", K(ret), K(tablet_handle.get_obj()->get_tablet_id())); } } } @@ -528,7 +508,6 @@ SCN ObLSDDLLogHandler::get_rec_scn() } LOG_INFO("[CHECKPOINT] ObLSDDLLogHandler::get_rec_scn", K(ret), - "ls_id", OB_ISNULL(ls_) ? ObLSID() : ls_->get_ls_id(), K(barrier_tablet_id), K(rec_scn), K_(last_rec_scn)); return rec_scn; } @@ -582,7 +561,7 @@ int ObLSDDLLogHandler::replay_ddl_tablet_schema_version_change_log_(const char * LOG_WARN("fail to deserialize source barrier log", K(ret)); } else if (OB_FAIL(replay_executor.init(log, log_scn))) { LOG_WARN("failed to init tablet schema version change log replay executor", K(ret)); - } else if (OB_FAIL(replay_executor.execute(log_scn, ls_->get_ls_id(), log.get_tablet_id()))) { + } else if (OB_FAIL(replay_executor.execute(log_scn, log.get_tablet_id()))) { if (OB_NO_NEED_UPDATE == ret) { LOG_WARN("no need replay tablet schema version change log", K(ret), K(log), K(log_scn)); ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls_ddl_log_handler.h b/src/storage/ls/ob_ls_ddl_log_handler.h index 9ba686a71..7c0be069c 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.h +++ b/src/storage/ls/ob_ls_ddl_log_handler.h @@ -60,7 +60,7 @@ class ObActiveDDLKVIterator final }; class ObLSDDLLogHandler : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -71,7 +71,6 @@ class ObLSDDLLogHandler : public logservice::ObIReplaySubHandler, int init(ObLS *ls); void reset(); - // for migrate and rebuild int offline(); int online(); @@ -81,11 +80,9 @@ class ObLSDDLLogHandler : public logservice::ObIReplaySubHandler, const palf::LSN &lsn, const share::SCN &log_ts) override final; - // for role change - void switch_to_follower_forcedly() override final; - int switch_to_leader() override final; - int switch_to_follower_gracefully() override final; - int resume_leader() override final; + // local lifecycle + void deactivate() override final; + int activate() override final; // for checkpoint int flush(share::SCN &rec_scn) override final; diff --git a/src/storage/ls/ob_ls_get_mod.h b/src/storage/ls/ob_ls_get_mod.h deleted file mode 100644 index dedea0387..000000000 --- a/src/storage/ls/ob_ls_get_mod.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_STORAGE_OB_LS_GET_MOD -#define OCEANBASE_STORAGE_OB_LS_GET_MOD - -namespace oceanbase -{ -namespace storage -{ -enum class ObLSGetMod : int -{ - INVALID_MOD = -1, - TRANS_MOD = 0, - STORAGE_MOD = 1, - RS_MOD = 2, - LOG_MOD = 3, - OBSERVER_MOD = 4, - ARCHIVE_MOD = 5, - DAS_MOD = 6, - SHARE_MOD = 7, - APPLY_MOD = 8, - ADAPTER_MOD = 9, - DEADLOCK_MOD = 10, - TABLELOCK_MOD = 11, - HA_MOD = 12, - TABLET_MOD = 13, - DDL_MOD = 14, - TXSTORAGE_MOD = 15, - LEADER_COORDINATOR_MOD = 16, - DATA_DICT_MOD = 17, - DATA_MEMTABLE_MOD = 18, - MULTI_VERSION_GARBAGE_COLLOECTOR_MOD = 19, - MDS_TABLE_MOD = 20, - COMPACT_MODE = 21, - SS_PREWARM_MOD = 22, - SHARED_META_SERVICE = 23, - STORAGE_CACHE_POLICY_MOD = 24, - TOTAL_MAX_MOD = 25, -}; - -} -} - -#endif // OCEANBASE_STORAGE_OB_LS_GET_MOD diff --git a/src/storage/ls/ob_ls_lock.cpp b/src/storage/ls/ob_ls_lock.cpp index b09c85a08..b4ef8b85a 100644 --- a/src/storage/ls/ob_ls_lock.cpp +++ b/src/storage/ls/ob_ls_lock.cpp @@ -70,7 +70,7 @@ int64_t ObLSLock::lock(const ObLS *ls, int64_t hold, int64_t change, const int64 pos++; } if (tg.get_diff() >= LOCK_CONFLICT_WARN_TIME) { - LOG_WARN("get lock cost too much time", KP(ls), "ls_id", OB_ISNULL(ls) ? ObLSID(0) : ls->get_ls_id()); + LOG_WARN("get lock cost too much time", KP(ls)); } return res; } @@ -106,7 +106,7 @@ int64_t ObLSLock::try_lock(const ObLS *ls, int64_t hold, int64_t change) pos++; } if (tg.get_diff() >= LOCK_CONFLICT_WARN_TIME) { - LOG_WARN("get lock cost too much time", KP(ls), "ls_id", OB_ISNULL(ls) ? ObLSID(0) : ls->get_ls_id()); + LOG_WARN("get lock cost too much time", KP(ls)); } return res; @@ -204,39 +204,13 @@ ObLSLockGuard::~ObLSLockGuard() const int64_t end_ts = ObTimeUtility::current_time(); if (end_ts - start_ts_ > 5 * 1000 * 1000) { FLOG_INFO("ls lock cost too much time", K_(start_ts), - "cost_us", end_ts - start_ts_, KP(ls_), - "ls_id", OB_ISNULL(ls_) ? ObLSID(0) : ls_->get_ls_id(), K(lbt())); + "cost_us", end_ts - start_ts_, KP(ls_), K(lbt())); } start_ts_ = INT64_MAX; ls_ = nullptr; } -ObLSLockWithPendingReplayGuard::ObLSLockWithPendingReplayGuard( - ObLSLock &lock, - const share::ObLSID &ls_id, - int64_t hold, - int64_t change) - : lock_(lock), - ls_id_(ls_id), - mark_(0), - start_ts_(INT64_MAX) -{ - UNUSEDx(lock_, mark_, hold, change); - // TODO: cxf262476 wait the replay engine to be empty -} - -ObLSLockWithPendingReplayGuard::~ObLSLockWithPendingReplayGuard() -{ - // TODO: cxf262476 unlock - const int64_t end_ts = ObTimeUtility::current_time(); - if (end_ts - start_ts_ > 5 * 1000 * 1000) { - LOG_INFO("ls lock cost too much time", K_(start_ts), - "cost_us", end_ts - start_ts_, K(ls_id_), K(lbt())); - } - start_ts_ = INT64_MAX; -} - // ================== ls state guard ===================== ObLSStateGuard::ObLSStateGuard(ObLS *ls) : ls_(ls), diff --git a/src/storage/ls/ob_ls_lock.h b/src/storage/ls/ob_ls_lock.h index bc4fbe25e..6f9c1e4c2 100644 --- a/src/storage/ls/ob_ls_lock.h +++ b/src/storage/ls/ob_ls_lock.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_STORAGE_OB_LS_LOCK_ #define OCEANBASE_STORAGE_OB_LS_LOCK_ -#include "share/ob_ls_id.h" #include "lib/lock/ob_latch.h" namespace oceanbase @@ -36,29 +35,12 @@ static const int64_t LSLOCKALL = LSLOCKMASK; class ObLSLockGuard; class ObLSTryLockGuard; -class ObLSLockWithPendingReplayGuard; class ObLS; -// ATTENTION: -// TODO(handora.qc): the solution now is not graceful, better solution is -// needed. -// the LS lock should be used carefully with replay engine lock for -// the deadlock issue. -// -// the deadlock will happen as follows: -// 1. the lock will be added from replay engine to LS (replay -// ADD_TABLET_TO_LS clog) -// 2. the lock will be added from LS to replay engine (stop -// LS during rebuild) -// -// so the replay engine lock needs to guarantee the graceful exit until the -// LS lock and the action under the lock needs to provide the -// reentrant execution. class ObLSLock { friend ObLSLockGuard; friend ObLSTryLockGuard; - friend ObLSLockWithPendingReplayGuard; static const int64_t LOCK_CONFLICT_WARN_TIME = 100 * 1000; // 100 ms public: typedef common::ObLatch RWLock; @@ -105,34 +87,6 @@ class ObLSLockGuard const ObLS *ls_; }; -// ATTENTION: -// The lock is designed to resolve the deadlock of code desgin between replay -// engine and partition group. You need use the guard when adding replay engine -// lock inside LS -class ObLSLockWithPendingReplayGuard -{ -public: - ObLSLockWithPendingReplayGuard(ObLSLock &lock, - const share::ObLSID &ls_id, - int64_t hold, - int64_t change); - ~ObLSLockWithPendingReplayGuard(); - - ObLSLockWithPendingReplayGuard(const ObLSLockWithPendingReplayGuard&) = delete; - ObLSLockWithPendingReplayGuard& operator=(const ObLSLockWithPendingReplayGuard&) = delete; -public: - static const int64_t SLEEP_FOR_PENDING_REPLAY = 1000L * 1000L; // 1s - static const int64_t SLEEP_FOR_WAIT_EMPTY = 10L * 1000L; // 10ms - static const int64_t SLEEP_FOR_PENDING_REPLAY_CNT = 5000L; - static const int64_t TIMEOUT_FOR_PENDING_REPLAY = 100L * 1000L; // 100ms -private: - ObLSLock &lock_; - //ObReplayStatus &replay_status_; - share::ObLSID ls_id_; - int64_t mark_; - int64_t start_ts_; -}; - class ObLSStateGuard { public: diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index 4046363a2..8aebb0139 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -39,43 +39,29 @@ WriteSlog ObLSMeta::write_slog_ = [](const int64_t ls_epoch, const ObLSMeta &ls_ ObLSMeta::ObLSMeta() : rw_lock_(), update_lock_(), - ls_id_(), - unused_replica_type_(REPLICA_TYPE_FULL), ls_persistent_state_(), clog_checkpoint_scn_(ObScnRange::MIN_SCN), clog_base_lsn_(PALF_INITIAL_LSN_VAL), - rebuild_seq_(-1), - migration_status_(ObMigrationStatus::OB_MIGRATION_STATUS_MAX), offline_scn_(), restore_status_(ObRestoreStatus::Status::RESTORE_STATUS_MAX), replayable_point_(), tablet_change_checkpoint_scn_(SCN::min_scn()), all_id_meta_(), - saved_info_(), - reserved_scn_(SCN::min_scn()), - rebuild_info_(), - major_mv_merge_info_() + saved_info_() { } ObLSMeta::ObLSMeta(const ObLSMeta &ls_meta) : rw_lock_(), update_lock_(), - ls_id_(ls_meta.ls_id_), - unused_replica_type_(ls_meta.unused_replica_type_), ls_persistent_state_(ls_meta.ls_persistent_state_), clog_checkpoint_scn_(ls_meta.clog_checkpoint_scn_), clog_base_lsn_(ls_meta.clog_base_lsn_), - rebuild_seq_(ls_meta.rebuild_seq_), - migration_status_(ls_meta.migration_status_), offline_scn_(ls_meta.offline_scn_), restore_status_(ls_meta.restore_status_), replayable_point_(ls_meta.replayable_point_), tablet_change_checkpoint_scn_(ls_meta.tablet_change_checkpoint_scn_), - saved_info_(ls_meta.saved_info_), - reserved_scn_(ls_meta.reserved_scn_), - rebuild_info_(ls_meta.rebuild_info_), - major_mv_merge_info_(ls_meta.major_mv_merge_info_) + saved_info_(ls_meta.saved_info_) { int ret = OB_SUCCESS; all_id_meta_.update_all_id_meta(ls_meta.all_id_meta_); @@ -85,28 +71,14 @@ int ObLSMeta::set_start_work_state() { ObReentrantWLockGuard update_guard(update_lock_); ObReentrantWLockGuard guard(rw_lock_); - return ls_persistent_state_.start_work(ls_id_); -} - -int ObLSMeta::set_start_ha_state() -{ - ObReentrantWLockGuard update_guard(update_lock_); - ObReentrantWLockGuard guard(rw_lock_); - return ls_persistent_state_.start_ha(ls_id_); -} - -int ObLSMeta::set_finish_ha_state() -{ - ObReentrantWLockGuard update_guard(update_lock_); - ObReentrantWLockGuard guard(rw_lock_); - return ls_persistent_state_.finish_ha(ls_id_); + return ls_persistent_state_.start_work(); } int ObLSMeta::set_remove_state() { ObReentrantWLockGuard update_guard(update_lock_); ObReentrantWLockGuard guard(rw_lock_); - return ls_persistent_state_.remove(ls_id_); + return ls_persistent_state_.remove(); } const ObLSPersistentState &ObLSMeta::get_persistent_state() const @@ -121,11 +93,7 @@ ObLSMeta &ObLSMeta::operator=(const ObLSMeta &other) ObReentrantWLockGuard guard_myself(rw_lock_); if (this != &other) { - ls_id_ = other.ls_id_; - unused_replica_type_ = other.unused_replica_type_; ls_persistent_state_ = other.ls_persistent_state_; - rebuild_seq_ = other.rebuild_seq_; - migration_status_ = other.migration_status_; clog_base_lsn_ = other.clog_base_lsn_; clog_checkpoint_scn_ = other.clog_checkpoint_scn_; offline_scn_ = other.offline_scn_; @@ -134,9 +102,6 @@ ObLSMeta &ObLSMeta::operator=(const ObLSMeta &other) tablet_change_checkpoint_scn_ = other.tablet_change_checkpoint_scn_; all_id_meta_.update_all_id_meta(other.all_id_meta_); saved_info_ = other.saved_info_; - reserved_scn_ = other.reserved_scn_; - rebuild_info_ = other.rebuild_info_; - major_mv_merge_info_ = other.major_mv_merge_info_; } return *this; } @@ -146,20 +111,13 @@ void ObLSMeta::reset() ObReentrantWLockGuard update_guard(update_lock_); ObReentrantWLockGuard guard(rw_lock_); - ls_id_.reset(); - unused_replica_type_ = REPLICA_TYPE_FULL; clog_base_lsn_.reset(); clog_checkpoint_scn_ = ObScnRange::MIN_SCN; - rebuild_seq_ = -1; - migration_status_ = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; offline_scn_.reset(); restore_status_ = ObRestoreStatus::Status::RESTORE_STATUS_MAX; replayable_point_.reset(); tablet_change_checkpoint_scn_ = SCN::min_scn(); saved_info_.reset(); - reserved_scn_ = SCN::min_scn(); - rebuild_info_.reset(); - major_mv_merge_info_.reset(); } LSN ObLSMeta::get_clog_base_lsn() const @@ -226,7 +184,7 @@ int ObLSMeta::set_tablet_change_checkpoint_scn( LOG_WARN("clog_checkpoint write slog failed", K(ret)); } else { ObReentrantWLockGuard guard(rw_lock_); - LOG_INFO("update tablet change checkpoint scn", K(ls_id_), + LOG_INFO("update tablet change checkpoint scn", "old_scn", tablet_change_checkpoint_scn_, "new_scn", tablet_change_checkpoint_scn); tablet_change_checkpoint_scn_ = tablet_change_checkpoint_scn; } @@ -235,194 +193,9 @@ int ObLSMeta::set_tablet_change_checkpoint_scn( return ret; } -ObMajorMVMergeInfo ObLSMeta::get_major_mv_merge_info() const -{ - ObReentrantRLockGuard guard(rw_lock_); - return major_mv_merge_info_; -} - -int ObLSMeta::set_major_mv_merge_scn(const int64_t ls_epoch, const SCN &major_mv_merge_scn) -{ - int ret = OB_SUCCESS; - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (major_mv_merge_info_.major_mv_merge_scn_ >= major_mv_merge_scn) { - LOG_INFO("old_scn is less than new_scn, skip", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_, "new_scn", major_mv_merge_scn); - } else { - ObLSMeta tmp(*this); - tmp.major_mv_merge_info_.major_mv_merge_scn_ = major_mv_merge_scn; - - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - LOG_INFO("update major_mv_merge_scn", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_, "new_scn", major_mv_merge_scn); - major_mv_merge_info_.major_mv_merge_scn_ = major_mv_merge_scn; - } - } - - return ret; -} - -int ObLSMeta::set_major_mv_merge_scn_safe_calc(const int64_t ls_epoch, const SCN &major_mv_merge_scn_safe_calc) -{ - int ret = OB_SUCCESS; - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (major_mv_merge_info_.major_mv_merge_scn_safe_calc_ >= major_mv_merge_scn_safe_calc) { - LOG_INFO("old_scn is not less than new_scn, skip", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_safe_calc_, "new_scn", major_mv_merge_scn_safe_calc); - } else { - ObLSMeta tmp(*this); - tmp.major_mv_merge_info_.major_mv_merge_scn_safe_calc_ = major_mv_merge_scn_safe_calc; - - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - LOG_INFO("update major_mv_merge_scn_safe_calc", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_safe_calc_, "new_scn", major_mv_merge_scn_safe_calc); - major_mv_merge_info_.major_mv_merge_scn_safe_calc_ = major_mv_merge_scn_safe_calc; - } - } - - return ret; -} - -int ObLSMeta::set_major_mv_merge_scn_publish(const int64_t ls_epoch, const SCN &major_mv_merge_scn_publish) -{ - int ret = OB_SUCCESS; - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (major_mv_merge_info_.major_mv_merge_scn_publish_ >= major_mv_merge_scn_publish) { - LOG_INFO("old_scn is not less than new_scn, skip", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_publish_, "new_scn", major_mv_merge_scn_publish); - } else { - ObLSMeta tmp(*this); - tmp.major_mv_merge_info_.major_mv_merge_scn_publish_ = major_mv_merge_scn_publish; - - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - LOG_INFO("update major_mv_merge_scn_publish", K(ls_id_), - "old_scn", major_mv_merge_info_.major_mv_merge_scn_publish_, "new_scn", major_mv_merge_scn_publish); - major_mv_merge_info_.major_mv_merge_scn_publish_ = major_mv_merge_scn_publish; - } - } - - return ret; -} - -share::SCN ObLSMeta::get_reserved_scn() const -{ - ObReentrantRLockGuard guard(rw_lock_); - return reserved_scn_; -} - -int ObLSMeta::inc_update_reserved_scn(const int64_t ls_epoch, const share::SCN &new_reserved_scn) -{ - ObReentrantWLockGuard update_guard(update_lock_); - int ret = OB_SUCCESS; - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (reserved_scn_ > new_reserved_scn) { - LOG_INFO("reserved scn is small", K_(ls_id), K(new_reserved_scn), K_(reserved_scn)); - } else { - ObLSMeta tmp(*this); - tmp.reserved_scn_ = new_reserved_scn; - - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("clog_checkpoint write slog failed", K(ret), K(*this)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - reserved_scn_ = new_reserved_scn; - } - } - return ret; -} - bool ObLSMeta::is_valid() const { - return true - && ls_id_.is_valid() - && OB_MIGRATION_STATUS_MAX != migration_status_ - && restore_status_.is_valid() - && rebuild_seq_ >= 0; -} - -int64_t ObLSMeta::get_rebuild_seq() const -{ - ObReentrantRLockGuard guard(rw_lock_); - return rebuild_seq_; -} - -int ObLSMeta::set_migration_status( - const int64_t ls_epoch, - const ObMigrationStatus &migration_status, - const bool write_slog) -{ - int ret = OB_SUCCESS; - bool can_change = false; - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (!ObMigrationStatusHelper::is_valid(migration_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(migration_status), KPC(this)); - } else if (migration_status_ == migration_status) { - //do nothing - } else if (OB_FAIL(ObMigrationStatusHelper::check_can_change_status(migration_status_, - migration_status, - can_change))) { - LOG_WARN("failed to check can change stauts", K(ret), K(migration_status_), - K(migration_status)); - } else if (!can_change) { - ret = OB_OP_NOT_ALLOW; - LOG_ERROR("ls can not change to migrate status", K(ret), K(migration_status_), - K(migration_status)); - } else { - ObLSMeta tmp(*this); - tmp.migration_status_ = migration_status; - tmp.ls_persistent_state_ = (OB_MIGRATION_STATUS_NONE == migration_status && - ObRestoreStatus::Status::NONE == restore_status_.get_status() ? - ObLSPersistentState::State::LS_NORMAL : ls_persistent_state_); - - if (write_slog && OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("migration_status write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - if ((OB_MIGRATION_STATUS_NONE == migration_status - && ObRestoreStatus::Status::NONE == restore_status_.get_status()) - && OB_FAIL(set_finish_ha_state())) { - LOG_WARN("set finish ha state failed", K(ret), K(ls_id_)); - } else { - ObMigrationStatus original_status = migration_status_; - migration_status_ = migration_status; - FLOG_INFO("succeed to set ls migration status", K(ls_id_), "original status", - original_status, "current status", migration_status); - } - } - } - return ret; -} - -int ObLSMeta::get_migration_status(ObMigrationStatus &migration_status) const -{ - int ret = OB_SUCCESS; - ObReentrantRLockGuard guard(rw_lock_); - if (!is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream meta is not valid, cannot get migration status", K(ret), K(*this)); - } else { - migration_status = migration_status_; - } - return ret; + return restore_status_.is_valid() ; } int ObLSMeta::get_offline_scn(SCN &offline_scn) @@ -452,22 +225,14 @@ int ObLSMeta::set_restore_status(const int64_t ls_epoch, const ObRestoreStatus & } else { ObLSMeta tmp(*this); tmp.restore_status_ = restore_status; - tmp.ls_persistent_state_ = (restore_status.is_none() && - OB_MIGRATION_STATUS_NONE == migration_status_ ? - ObLSPersistentState::State::LS_NORMAL : ls_persistent_state_); if (OB_FAIL(write_slog_(ls_epoch, tmp))) { LOG_WARN("restore_status write slog failed", K(ret)); } else { ObReentrantWLockGuard guard(rw_lock_); - if ((restore_status.is_none() && OB_MIGRATION_STATUS_NONE == migration_status_) - && OB_FAIL(set_finish_ha_state())) { - LOG_WARN("set finish ha state failed", KR(ret), K(ls_id_)); - } else { - ObRestoreStatus original_status = restore_status_; - restore_status_ = restore_status; - FLOG_INFO("succeed to set ls restore status", K(ls_id_), "original status", - original_status, "current status", restore_status); - } + ObRestoreStatus original_status = restore_status_; + restore_status_ = restore_status; + FLOG_INFO("succeed to set ls restore status", "original status", + original_status, "current status", restore_status); } } return ret; @@ -524,125 +289,6 @@ int ObLSMeta::get_ls_replayable_point(SCN &replayable_point) return ret; } -//This interface for ha. Add parameters should check meta value need to update from src -int ObLSMeta::update_ls_meta( - const int64_t ls_epoch, - const bool update_restore_status, - const ObLSMeta &src_ls_meta) -{ - int ret = OB_SUCCESS; - ObRestoreStatus ls_restore_status; - - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (!src_ls_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("update ls meta get invalid argument", K(ret), K(src_ls_meta)); - } else if (update_restore_status - && OB_FAIL(src_ls_meta.get_restore_status(ls_restore_status))) { - LOG_WARN("failed to get restore status", K(ret), K(src_ls_meta)); - } else { - ObLSMeta tmp(*this); - tmp.clog_base_lsn_ = src_ls_meta.clog_base_lsn_; - tmp.clog_checkpoint_scn_ = src_ls_meta.clog_checkpoint_scn_; - tmp.replayable_point_ = src_ls_meta.replayable_point_; - tmp.tablet_change_checkpoint_scn_ = src_ls_meta.tablet_change_checkpoint_scn_; - tmp.reserved_scn_ = src_ls_meta.reserved_scn_; - tmp.rebuild_seq_++; - if (update_restore_status) { - tmp.restore_status_ = ls_restore_status; - } - tmp.offline_scn_ = src_ls_meta.offline_scn_; - update_guard.click(); - tmp.all_id_meta_.update_all_id_meta(src_ls_meta.all_id_meta_); - if (tmp.clog_checkpoint_scn_ < clog_checkpoint_scn_) { - // TODO(muwei.ym): now do not allow clog checkpoint ts rollback, may support it in 4.3 - ret = OB_ERR_UNEXPECTED; - LOG_WARN("do not allow clog checkpoint ts rollback", K(ret), K(src_ls_meta), KPC(this)); - } else if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("clog_checkpoint write slog failed", K(ret)); - } else { - update_guard.click(); - ObReentrantWLockGuard guard(rw_lock_); - clog_base_lsn_ = src_ls_meta.clog_base_lsn_; - clog_checkpoint_scn_ = src_ls_meta.clog_checkpoint_scn_; - replayable_point_ = src_ls_meta.replayable_point_; - tablet_change_checkpoint_scn_ = src_ls_meta.tablet_change_checkpoint_scn_; - all_id_meta_.update_all_id_meta(src_ls_meta.all_id_meta_); - rebuild_seq_ = tmp.rebuild_seq_; - offline_scn_ = src_ls_meta.offline_scn_; - reserved_scn_ = src_ls_meta.reserved_scn_; - if (update_restore_status) { - restore_status_ = ls_restore_status; - } - major_mv_merge_info_ = src_ls_meta.major_mv_merge_info_; - // store format doesn't change - } - LOG_INFO("update ls meta", K(ret), K(tmp), K(src_ls_meta), K(*this)); - } - return ret; -} - -int ObLSMeta::set_ls_rebuild(const int64_t ls_epoch) -{ - int ret = OB_SUCCESS; - const ObMigrationStatus change_status = ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD; - const ObLSPersistentState persistent_state = ObLSPersistentState::State::LS_HA; - bool can_change = false; - - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (change_status == migration_status_) { - //do nothing - } else { - ObLSMeta tmp(*this); - if (OB_FAIL(ObMigrationStatusHelper::check_can_change_status(tmp.migration_status_, - change_status, - can_change))) { - LOG_WARN("failed to check can change status", K(ret), K(migration_status_), K(change_status)); - } else if (!can_change) { - ret = OB_OP_NOT_ALLOW; - LOG_ERROR("ls can not change to rebuild status", K(ret), K(tmp), K(change_status)); - } else { - tmp.migration_status_ = change_status; - tmp.rebuild_seq_++; - tmp.ls_persistent_state_ = persistent_state; - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("clog_checkpoint write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - if (OB_FAIL(set_start_ha_state())) { - LOG_WARN("set start ha state failed", KR(ret), K(ls_id_)); - } else { - migration_status_ = change_status; - rebuild_seq_ = tmp.rebuild_seq_; - FLOG_INFO("succeed to set ls rebuild", "ls_id", ls_id_, KPC(this)); - } - } - } - } - return ret; -} - -int ObLSMeta::check_valid_for_backup() const -{ - int ret = OB_SUCCESS; - ObReentrantRLockGuard guard(rw_lock_); - if (!is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream meta is not valid", K(ret), KPC(this)); - } else if (!restore_status_.is_none()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("restore status is not none", K(ret), KPC(this)); - } else if (OB_MIGRATION_STATUS_NONE != migration_status_) { - ret = OB_REPLICA_CANNOT_BACKUP; - LOG_WARN("ls replica not valid for backup", K(ret), KPC(this)); - } - return ret; -} - int ObLSMeta::get_saved_info(ObLSSavedInfo &saved_info) { int ret = OB_SUCCESS; @@ -707,31 +353,18 @@ int ObLSMeta::clear_saved_info(const int64_t ls_epoch) } int ObLSMeta::init( - const share::ObLSID &ls_id, - const ObMigrationStatus &migration_status, const ObRestoreStatus &restore_status, - const SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info) + const SCN &create_scn) { int ret = OB_SUCCESS; - if (!ls_id.is_valid() - || !ObMigrationStatusHelper::is_valid(migration_status) - || !restore_status.is_valid() - || !major_mv_merge_info.is_valid()) { + if (!restore_status.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("init ls meta get invalid argument", K(ret), K(ls_id), - K(migration_status), K(restore_status)); + LOG_WARN("init ls meta get invalid argument", K(ret), K(restore_status)); } else { - - ls_id_ = ls_id; ls_persistent_state_ = ObLSPersistentState::State::LS_INIT; clog_checkpoint_scn_ = create_scn; clog_base_lsn_.val_ = PALF_INITIAL_LSN_VAL; - rebuild_seq_ = 0; - migration_status_ = migration_status; restore_status_ = restore_status; - reserved_scn_ = SCN::min_scn(); - major_mv_merge_info_ = major_mv_merge_info; } return ret; } @@ -790,89 +423,6 @@ int ObLSMeta::check_can_update_() return ret; } -int ObLSMeta::get_migration_and_restore_status( - ObMigrationStatus &migration_status, - ObRestoreStatus &ls_restore_status) -{ - int ret = OB_SUCCESS; - migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; - ls_restore_status = ObRestoreStatus::Status::RESTORE_STATUS_MAX; - - ObReentrantRLockGuard guard(rw_lock_); - if (!is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls meta is not valid, cannot get", K(ret), K(*this)); - } else { - migration_status = migration_status_; - ls_restore_status = restore_status_; - } - return ret; -} - -int ObLSMeta::set_rebuild_info(const int64_t ls_epoch, const ObLSRebuildInfo &rebuild_info) -{ - int ret = OB_SUCCESS; - ObReentrantWLockGuard update_guard(update_lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (!rebuild_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid rebuild info", K(ret), K(rebuild_info_), K(rebuild_info)); - } else if (rebuild_info_ == rebuild_info) { - //do nothing - } else if (ObLSRebuildStatus::CLEANUP == rebuild_info.status_ - && ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status_ - && ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD_FAIL != migration_status_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("migration status in unexpected, can not set rebuild info to cleanup", K(ret), - K(rebuild_info), K(migration_status_), KPC(this)); - } else { - ObLSMeta tmp(*this); - tmp.rebuild_info_ = rebuild_info; - if (OB_FAIL(write_slog_(ls_epoch, tmp))) { - LOG_WARN("rebuild_info write slog failed", K(ret)); - } else { - ObReentrantWLockGuard guard(rw_lock_); - rebuild_info_ = rebuild_info; - FLOG_INFO("succeed to set rebuild info", K(ls_id_), K(rebuild_info)); - } - } - return ret; -} - -int ObLSMeta::get_rebuild_info(ObLSRebuildInfo &rebuild_info) const -{ - int ret = OB_SUCCESS; - ObReentrantRLockGuard guard(rw_lock_); - if (!is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls meta is not valid, cannot get rebuild info", K(ret), K(*this)); - } else { - rebuild_info = rebuild_info_; - } - return ret; -} - -// need protect by lock -int ObLSMeta::get_create_type(int64_t &create_type) const -{ - int ret = OB_SUCCESS; - bool is_restore = false; - ObReentrantRLockGuard guard(rw_lock_); - create_type = ObLSCreateType::NORMAL; - if (!is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("log stream meta is not valid, cannot get restore status", K(ret), K(*this)); - } else if (!ObMigrationStatusHelper::need_online(migration_status_)) { - create_type = ObLSCreateType::MIGRATE; - } else if (restore_status_.is_restore_doing()) { - create_type = ObLSCreateType::RESTORE; - } else if (ls_persistent_state_.is_ha_state()) { - create_type = ObLSCreateType::MIGRATE; - } - return ret; -} - int ObLSMeta::check_ls_need_online(bool &need_online) const { int ret = OB_SUCCESS; @@ -880,10 +430,7 @@ int ObLSMeta::check_ls_need_online(bool &need_online) const if (!is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log stream meta is not valid", K(ret), K(*this)); - } else if (!ObMigrationStatusHelper::need_online(migration_status_)) { - need_online = false; - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE == migration_status_ && - !restore_status_.need_online()) { + } else if (!restore_status_.need_online()) { need_online = false; } return ret; @@ -954,13 +501,6 @@ ObLSMeta::ObReentrantRLockGuard::ObReentrantRLockGuard(ObLatch &lock, time_guard_.click("after lock"); } -void ObLSMeta::update_clog_checkpoint_in_ls_meta_package_(const share::SCN& clog_checkpoint_scn, - const palf::LSN& clog_base_lsn) -{ - clog_checkpoint_scn_ = clog_checkpoint_scn; - clog_base_lsn_ = clog_base_lsn; -} - ObLSMeta::ObReentrantRLockGuard::~ObReentrantRLockGuard() { if (OB_LIKELY(OB_SUCCESS == ret_) && first_locked_) { @@ -970,24 +510,16 @@ ObLSMeta::ObReentrantRLockGuard::~ObReentrantRLockGuard() } } -// add field should also consider ObLSMeta::update_ls_meta function OB_SERIALIZE_MEMBER(ObLSMeta, - ls_id_, - unused_replica_type_, ls_persistent_state_, // FARM COMPAT WHITELIST clog_checkpoint_scn_, clog_base_lsn_, - rebuild_seq_, - migration_status_, offline_scn_, restore_status_, replayable_point_, tablet_change_checkpoint_scn_, all_id_meta_, - saved_info_, - reserved_scn_, - rebuild_info_, - major_mv_merge_info_); + saved_info_); } } diff --git a/src/storage/ls/ob_ls_meta.h b/src/storage/ls/ob_ls_meta.h index 36d6de971..2e7557cf0 100644 --- a/src/storage/ls/ob_ls_meta.h +++ b/src/storage/ls/ob_ls_meta.h @@ -20,37 +20,25 @@ #include "lib/utility/utility.h" // ObTimeGuard #include "lib/lock/ob_spin_lock.h" #include "logservice/palf/lsn.h" -#include "share/ob_ls_id.h" #include "lib/ob_define.h" #include "lib/function/ob_function.h" #include "share/ob_unit_getter.h" #include "storage/ls/ob_ls_state.h" -#include "storage/ob_storage_ha_struct.h" #include "logservice/ob_log_handler.h" #include "share/ls/ob_ls_restore_status.h" #include "share/ls/ob_restore_status.h" #include "storage/tx/ob_id_service.h" #include "storage/ls/ob_ls_saved_info.h" #include "share/scn.h" -#include "storage/mview/ob_major_mv_merge_info.h" namespace oceanbase { namespace storage { -class ObLSCreateType -{ -public: - static const int64_t NORMAL = 0; - static const int64_t RESTORE = 1; - static const int64_t MIGRATE = 2; -}; - class ObLSMeta { - friend class ObLSMetaPackage; - OB_UNIS_VERSION_V(1); + OB_UNIS_VERSION_V(3); public: ObLSMeta(); ObLSMeta(const ObLSMeta &ls_meta); @@ -58,8 +46,6 @@ class ObLSMeta void reset(); bool is_valid() const; int set_start_work_state(); - int set_start_ha_state(); - int set_finish_ha_state(); int set_remove_state(); const ObLSPersistentState &get_persistent_state() const; ObLSMeta &operator=(const ObLSMeta &other); @@ -69,11 +55,6 @@ class ObLSMeta const palf::LSN &clog_checkpoint_lsn, const share::SCN &clog_checkpoint_scn, const bool write_slog); - int64_t get_rebuild_seq() const; - int set_migration_status(const int64_t ls_epoch, - const ObMigrationStatus &migration_status, - const bool write_slog = true); - int get_migration_status (ObMigrationStatus &migration_status) const; int get_offline_scn(share::SCN &offline_scn); int set_restore_status(const int64_t ls_epoch, const ObRestoreStatus &restore_status); @@ -81,18 +62,8 @@ class ObLSMeta int update_ls_replayable_point(const int64_t ls_epoch, const share::SCN &replayable_point); int get_ls_replayable_point(share::SCN &replayable_point); - //for ha batch update ls meta element - int update_ls_meta( - const int64_t ls_epoch, - const bool update_restore_status, - const ObLSMeta &src_ls_meta); - //for ha rebuild update ls meta - int set_ls_rebuild(const int64_t ls_epoch); - int check_valid_for_backup() const; share::SCN get_tablet_change_checkpoint_scn() const; int set_tablet_change_checkpoint_scn(const int64_t ls_epoch, const share::SCN &tablet_change_checkpoint_scn); - share::SCN get_reserved_scn() const; - int inc_update_reserved_scn(const int64_t ls_epoch, const share::SCN &new_reserved_scn); int update_id_meta(const int64_t ls_epoch, const int64_t service_type, const int64_t limited_id, @@ -102,26 +73,11 @@ class ObLSMeta int get_saved_info(ObLSSavedInfo &saved_info); int build_saved_info(const int64_t ls_epoch); int clear_saved_info(const int64_t ls_epoch); - int get_migration_and_restore_status( - ObMigrationStatus &migration_status, - ObRestoreStatus &ls_restore_status); - int set_rebuild_info(const int64_t ls_epoch, const ObLSRebuildInfo &rebuild_info); - int get_rebuild_info(ObLSRebuildInfo &rebuild_info) const; - int get_create_type(int64_t &create_type) const; int check_ls_need_online(bool &need_online) const; - ObMajorMVMergeInfo get_major_mv_merge_info() const; - int set_major_mv_merge_scn(const int64_t ls_epoch, const SCN &major_mv_merge_scn); - int set_major_mv_merge_scn_safe_calc(const int64_t ls_epoch, const SCN &major_mv_merge_scn_safe_calc); - int set_major_mv_merge_scn_publish(const int64_t ls_epoch, const SCN &major_mv_merge_scn_publish); int init( - const share::ObLSID &ls_id, - const ObMigrationStatus &migration_status, const ObRestoreStatus &restore_status, - const share::SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info); + const share::SCN &create_scn); - ObReplicaType get_replica_type() const - { return unused_replica_type_; } // IF I have locked with W: // lock with R/W will be succeed do nothing. // ELSE: @@ -158,24 +114,18 @@ class ObLSMeta common::ObLatch &lock_; int ret_; }; - TO_STRING_KV(K_(ls_id), K_(ls_persistent_state), + TO_STRING_KV(K_(ls_persistent_state), K_(clog_checkpoint_scn), K_(clog_base_lsn), - K_(rebuild_seq), K_(migration_status), K(offline_scn_), + K(offline_scn_), K_(restore_status), K_(replayable_point), K_(tablet_change_checkpoint_scn), - K_(all_id_meta), K_(reserved_scn), K_(rebuild_info)); + K_(all_id_meta)); private: int check_can_update_(); public: mutable common::ObLatch rw_lock_; // only for atomic read/write in memory. mutable common::ObLatch update_lock_; // only one process can update ls meta. both for write slog and memory - share::ObLSID ls_id_; - -private: - void update_clog_checkpoint_in_ls_meta_package_(const share::SCN& clog_checkpoint_scn, - const palf::LSN& clog_base_lsn); private: - ObReplicaType unused_replica_type_; ObLSPersistentState ls_persistent_state_; typedef common::ObFunction WriteSlog; // for test @@ -190,8 +140,6 @@ class ObLSMeta // 2. log_scn of log entry that clog_base_lsn_ points to is smaller than/equal to clog_checkpoint_scn_ // 3. clog starts to replay log entries from clog_base_lsn_ on crash recovery palf::LSN clog_base_lsn_; - int64_t rebuild_seq_; - ObMigrationStatus migration_status_; share::SCN offline_scn_; ObRestoreStatus restore_status_; share::SCN replayable_point_; @@ -199,9 +147,6 @@ class ObLSMeta share::SCN tablet_change_checkpoint_scn_; transaction::ObAllIDMeta all_id_meta_; ObLSSavedInfo saved_info_; - share::SCN reserved_scn_; - ObLSRebuildInfo rebuild_info_; - ObMajorMVMergeInfo major_mv_merge_info_; }; } // namespace storage diff --git a/src/storage/ls/ob_ls_meta_package.cpp b/src/storage/ls/ob_ls_meta_package.cpp deleted file mode 100644 index 857305570..000000000 --- a/src/storage/ls/ob_ls_meta_package.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "storage/ls/ob_ls_meta_package.h" - -namespace oceanbase -{ -namespace storage -{ -OB_SERIALIZE_MEMBER(ObLSMetaPackage, - ls_meta_, - palf_meta_, - tx_data_recycle_scn_); - -ObLSMetaPackage::ObLSMetaPackage() - : ls_meta_(), - palf_meta_(), - tx_data_recycle_scn_() -{ -} - -ObLSMetaPackage::ObLSMetaPackage(const ObLSMetaPackage &other) - : ls_meta_(other.ls_meta_), palf_meta_(other.palf_meta_), tx_data_recycle_scn_(other.tx_data_recycle_scn_) -{ -} - -ObLSMetaPackage &ObLSMetaPackage::operator=(const ObLSMetaPackage &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - ls_meta_ = other.ls_meta_; - palf_meta_ = other.palf_meta_; - tx_data_recycle_scn_ = other.tx_data_recycle_scn_; - } - return *this; -} - -void ObLSMetaPackage::reset() -{ - ls_meta_.reset(); - palf_meta_.reset(); - tx_data_recycle_scn_.reset(); -} - -bool ObLSMetaPackage::is_valid() const -{ - return (ls_meta_.is_valid() && - palf_meta_.is_valid()); -} - -void ObLSMetaPackage::update_clog_checkpoint_in_ls_meta(const share::SCN& clog_checkpoint_scn, - const palf::LSN& clog_base_lsn) -{ - ls_meta_.update_clog_checkpoint_in_ls_meta_package_(clog_checkpoint_scn, clog_base_lsn); -} - -} -} diff --git a/src/storage/ls/ob_ls_meta_package.h b/src/storage/ls/ob_ls_meta_package.h deleted file mode 100644 index ffe2e5a4b..000000000 --- a/src/storage/ls/ob_ls_meta_package.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_STORAGE_OB_LS_META_PACKAGE_ -#define OCEANBASE_STORAGE_OB_LS_META_PACKAGE_ -#include "storage/ls/ob_ls_meta.h" // ObLSMeta -#include "logservice/palf/palf_base_info.h" // PalfBaseInfo - -namespace oceanbase -{ -namespace storage -{ - -// this is a package of meta. -// it is a combination of ls meta and all its member's. -// we can rebuild the ls with this package. -class ObLSMetaPackage final -{ - OB_UNIS_VERSION_V(1); -public: - ObLSMetaPackage(); - ObLSMetaPackage(const ObLSMetaPackage &ls_meta); - ~ObLSMetaPackage() { reset(); } - ObLSMetaPackage &operator=(const ObLSMetaPackage &other); - void reset(); - bool is_valid() const; - void update_clog_checkpoint_in_ls_meta(const share::SCN& clog_checkpoint_scn, - const palf::LSN& clog_base_lsn); - - TO_STRING_KV(K_(ls_meta), K_(palf_meta)); -public: - ObLSMeta ls_meta_; // the meta of ls - palf::PalfBaseInfo palf_meta_; // the meta of palf - share::SCN tx_data_recycle_scn_; -}; - -} // storage -} // oceanbase -#endif diff --git a/src/storage/ls/ob_ls_rebuild_handler.h b/src/storage/ls/ob_ls_rebuild_handler.h deleted file mode 100644 index 8a8485dce..000000000 --- a/src/storage/ls/ob_ls_rebuild_handler.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEABASE_STORAGE_LS_REBUILD_HANDLER_ -#define OCEABASE_STORAGE_LS_REBUILD_HANDLER_ - -namespace oceanbase -{ -namespace storage -{ - -class ObLSRebuildHandler -{}; - - -} -} -#endif diff --git a/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp b/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp index a1ec68f33..338d340df 100644 --- a/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp +++ b/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp @@ -15,8 +15,6 @@ */ #define USING_LOG_PREFIX STORAGE #include "ob_ls_reserved_snapshot_mgr.h" -#include "share/rc/ob_module_provider.h" -#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -31,10 +29,9 @@ ObLSReservedSnapshotMgr::ObLSReservedSnapshotMgr() min_reserved_snapshot_(0), next_reserved_snapshot_(0), snapshot_lock_(ObLatchIds::LS_RESERVED_SNAPSHOT_LOCK), - sync_clog_lock_(), - ls_(nullptr), - ls_handle_(), - dependent_tablet_set_(), + sync_clog_lock_(), + ls_(nullptr), + dependent_tablet_set_(), clog_cb_(*this), last_print_log_ts_(ObTimeUtility::fast_current_time()), clog_buf_() @@ -63,7 +60,7 @@ int ObLSReservedSnapshotMgr::init(ObLS *ls, ObLogHandler *log_handler) } else { ls_ = ls; is_inited_ = true; - LOG_INFO("success to init snapshot mgr", K(ret), KP(ls), "ls_id", ls_->get_ls_id(), KP(this)); + LOG_INFO("success to init snapshot mgr", K(ret), KP(ls), KP(this)); } return ret; } @@ -76,7 +73,6 @@ void ObLSReservedSnapshotMgr::destroy() min_reserved_snapshot_ = 0; next_reserved_snapshot_ = 0; ls_ = nullptr; - ls_handle_.reset(); last_print_log_ts_ = 0; if (dependent_tablet_set_.created()) { dependent_tablet_set_.destroy(); @@ -97,9 +93,9 @@ int ObLSReservedSnapshotMgr::add_dependent_medium_tablet(const ObTabletID tablet } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != hash_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to check exist in tablet set", K(ret), K(hash_ret), - "ls_id", ls_->get_ls_id(), K(tablet_id)); + K(tablet_id)); } else if (OB_FAIL(dependent_tablet_set_.set_refactored(tablet_id.id()))) { - LOG_WARN("failed to set tablet_id", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id)); + LOG_WARN("failed to set tablet_id", K(ret), K(tablet_id)); } } return ret; @@ -115,8 +111,7 @@ int ObLSReservedSnapshotMgr::del_dependent_medium_tablet(const ObTabletID tablet } else { common::TCWLockGuard lock_guard(snapshot_lock_); if (OB_FAIL(dependent_tablet_set_.erase_refactored(tablet_id.id()))) { - LOG_WARN("failed to erase tablet id", K(ret), "ls_id", ls_->get_ls_id(), - K(tablet_id), K(dependent_tablet_set_.size()), KP(this)); + LOG_WARN("failed to erase tablet id", K(ret), K(tablet_id), K(dependent_tablet_set_.size()), KP(this)); } else if (0 == dependent_tablet_set_.size() && next_reserved_snapshot_ > 0 && next_reserved_snapshot_ > min_reserved_snapshot_) { @@ -152,16 +147,16 @@ int ObLSReservedSnapshotMgr::submit_log( ret = OB_ERR_UNEXPECTED; LOG_WARN("clog_buf or clog_len is invalid", K(ret), KP(clog_buf), K(clog_len)); } else if (OB_FAIL(write_clog(clog_buf, clog_len))) { - LOG_WARN("fail to submit log", K(ret), "ls_id", ls_->get_ls_id()); + LOG_WARN("fail to submit log", K(ret)); } else { - LOG_DEBUG("submit reserved snapshot log success", "ls_id", ls_->get_ls_id(), K(reserved_snapshot)); + LOG_DEBUG("submit reserved snapshot log success", K(reserved_snapshot)); } return ret; } // called by ObTenantFreezeInfoMgr, sync clog in Timer -int ObLSReservedSnapshotMgr::update_min_reserved_snapshot_for_leader(const int64_t new_snapshot_version) +int ObLSReservedSnapshotMgr::update_min_reserved_snapshot(const int64_t new_snapshot_version) { int ret = OB_SUCCESS; bool send_log_flag = false; @@ -172,8 +167,7 @@ int ObLSReservedSnapshotMgr::update_min_reserved_snapshot_for_leader(const int64 common::TCWLockGuard lock_guard(snapshot_lock_); if (new_snapshot_version < min_reserved_snapshot_) { ret = OB_SNAPSHOT_DISCARDED; - LOG_WARN("failed to update min reserved snapshot", K(ret), "ls_id", ls_->get_ls_id(), - K(new_snapshot_version), K(min_reserved_snapshot_)); + LOG_WARN("failed to update min reserved snapshot", K(ret), K(new_snapshot_version), K(min_reserved_snapshot_)); } else if (0 == dependent_tablet_set_.size()) { // no dependent tablet, can push snapshot forward if (new_snapshot_version > min_reserved_snapshot_) { // update min_reserved_snapshot and send clog @@ -191,8 +185,7 @@ int ObLSReservedSnapshotMgr::update_min_reserved_snapshot_for_leader(const int64 if (OB_FAIL(sync_clog(new_snapshot_version))) { LOG_WARN("failed to send update reserved snapshot log", K(ret), K(new_snapshot_version)); } else if (need_print_log()) { - LOG_INFO("submit reserved snapshot log success", "ls_id", ls_->get_ls_id(), - K(new_snapshot_version)); + LOG_INFO("submit reserved snapshot log success", K(new_snapshot_version)); } } return ret; @@ -210,16 +203,15 @@ int ObLSReservedSnapshotMgr::try_sync_reserved_snapshot( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(new_reserved_snapshot)); } else if (update_flag) { - if (OB_FAIL(update_min_reserved_snapshot_for_leader(new_reserved_snapshot))) { + if (OB_FAIL(update_min_reserved_snapshot(new_reserved_snapshot))) { if (OB_SNAPSHOT_DISCARDED != ret) { - LOG_WARN("failed to update min_reserved_snapshot", K(ret), "ls_id", ls_->get_ls_id(), K(new_reserved_snapshot)); + LOG_WARN("failed to update min_reserved_snapshot", K(ret), K(new_reserved_snapshot)); } } } else if (OB_FAIL(sync_clog(new_reserved_snapshot))) { LOG_WARN("failed to send update reserved snapshot log", K(ret), K(new_reserved_snapshot)); } else if (need_print_log()) { - LOG_INFO("submit reserved snapshot log success", "ls_id", ls_->get_ls_id(), - K(new_reserved_snapshot)); + LOG_INFO("submit reserved snapshot log success", K(new_reserved_snapshot)); } return ret; } @@ -228,8 +220,8 @@ int ObLSReservedSnapshotMgr::sync_clog(const int64_t new_reserved_snapshot) { int ret = OB_SUCCESS; ObMutexGuard guard(sync_clog_lock_); - if (OB_FAIL(try_update_for_leader(new_reserved_snapshot, nullptr/*allocator*/))) { - LOG_WARN("failed to send update reserved snapshot log", K(ret), "ls_id", ls_->get_ls_id(), K(new_reserved_snapshot)); + if (OB_FAIL(try_update(new_reserved_snapshot, nullptr/*allocator*/))) { + LOG_WARN("failed to send update reserved snapshot log", K(ret), K(new_reserved_snapshot)); } return ret; } @@ -243,10 +235,9 @@ int ObLSReservedSnapshotMgr::replay_reserved_snapshot_log( ret = OB_NOT_INIT; LOG_WARN("ObLSReservedSnapshotMgr not inited", K(ret), KP(ls_)); } else if (OB_FAIL(serialization::decode_i64(buf, size, pos, &reserved_snapshot))) { - LOG_WARN("fail to deserialize reserved_snapshot", K(ret), "ls_id", ls_->get_ls_id()); + LOG_WARN("fail to deserialize reserved_snapshot", K(ret)); } else if (OB_FAIL(ObIStorageClogRecorder::replay_clog(reserved_snapshot, scn, buf, size, pos))) { - LOG_WARN("failed to update reserved snapshot by log", K(ret), "ls_id", ls_->get_ls_id(), - K(min_reserved_snapshot_), K(reserved_snapshot)); + LOG_WARN("failed to update reserved snapshot by log", K(ret), K(min_reserved_snapshot_), K(reserved_snapshot)); } return ret; } @@ -267,7 +258,7 @@ int ObLSReservedSnapshotMgr::inner_replay_clog( return ret; } -int ObLSReservedSnapshotMgr::sync_clog_succ_for_leader(const int64_t update_version) +int ObLSReservedSnapshotMgr::on_sync_clog_success(const int64_t update_version) { int ret = OB_SUCCESS; if (OB_FAIL(inner_update_reserved_snapshot(update_version))) { @@ -286,8 +277,7 @@ int ObLSReservedSnapshotMgr::inner_update_reserved_snapshot(const int64_t reserv common::TCWLockGuard lock_guard(snapshot_lock_); if (reserved_snapshot > min_reserved_snapshot_) { min_reserved_snapshot_ = reserved_snapshot; - LOG_INFO("success to update reserved snapshot", K(ret), "ls_id", ls_->get_ls_id(), - K(min_reserved_snapshot_)); + LOG_INFO("success to update reserved snapshot", K(ret), K(min_reserved_snapshot_)); } } return ret; @@ -309,12 +299,10 @@ int ObLSReservedSnapshotMgr::prepare_struct_in_lock( ObLogBaseType::RESERVED_SNAPSHOT_LOG_BASE_TYPE, ObReplayBarrierType::PRE_BARRIER/*need_replay_pre_barrier*/); - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_->get_ls_id(), ls_handle_, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), "ls_id", ls_->get_ls_id()); - } else if (OB_FAIL(log_header.serialize(clog_buf_, CLOG_BUF_LEN, pos))) { + if (OB_FAIL(log_header.serialize(clog_buf_, CLOG_BUF_LEN, pos))) { LOG_WARN("failed to serialize log header", K(ret)); } else if (OB_FAIL(serialization::encode_i64(clog_buf_, CLOG_BUF_LEN, pos, update_version))) { - LOG_WARN("generate reserved snapshot log", K(ret), "ls_id", ls_->get_ls_id(), K(pos), K(CLOG_BUF_LEN)); + LOG_WARN("generate reserved snapshot log", K(ret), K(pos), K(CLOG_BUF_LEN)); } else { logcb_ptr_ = &clog_cb_; clog_buf = clog_buf_; @@ -325,7 +313,6 @@ int ObLSReservedSnapshotMgr::prepare_struct_in_lock( void ObLSReservedSnapshotMgr::free_struct_in_lock() { - ls_handle_.reset(); clog_cb_.reset(); } diff --git a/src/storage/ls/ob_ls_reserved_snapshot_mgr.h b/src/storage/ls/ob_ls_reserved_snapshot_mgr.h index d4cedccac..9101b3447 100644 --- a/src/storage/ls/ob_ls_reserved_snapshot_mgr.h +++ b/src/storage/ls/ob_ls_reserved_snapshot_mgr.h @@ -20,12 +20,10 @@ #include "logservice/ob_append_callback.h" #include "lib/utility/ob_print_utils.h" #include "lib/hash/ob_hashset.h" -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "lib/lock/ob_tc_rwlock.h" #include "storage/ob_storage_clog_recorder.h" #include "logservice/ob_log_base_header.h" -#include "storage/tx_storage/ob_ls_handle.h" namespace oceanbase { @@ -47,9 +45,7 @@ class ObLSReservedSnapshotMgr : public ObIStorageClogRecorder int init(storage::ObLS *ls, logservice::ObLogHandler *log_handler); virtual void destroy() override; - // for leader int try_sync_reserved_snapshot(const int64_t new_reserved_snapshot, const bool update_flag); - // follower int replay_reserved_snapshot_log(const share::SCN &scn, const char *buf, const int64_t size, int64_t &pos); // operate with write_lock int add_dependent_medium_tablet(const ObTabletID tablet_id); @@ -58,7 +54,7 @@ class ObLSReservedSnapshotMgr : public ObIStorageClogRecorder int64_t get_min_reserved_snapshot(); private: - int update_min_reserved_snapshot_for_leader(const int64_t new_reserved_snapshot); + int update_min_reserved_snapshot(const int64_t new_reserved_snapshot); int inner_update_reserved_snapshot(const int64_t reserved_snapshot); virtual int inner_replay_clog( @@ -67,8 +63,8 @@ class ObLSReservedSnapshotMgr : public ObIStorageClogRecorder const char *buf, const int64_t size, int64_t &pos) override; - virtual int sync_clog_succ_for_leader(const int64_t update_version) override; - virtual void sync_clog_failed_for_leader() override + virtual int on_sync_clog_success(const int64_t update_version) override; + virtual void on_sync_clog_failure() override { // do nothing } @@ -105,7 +101,6 @@ class ObLSReservedSnapshotMgr : public ObIStorageClogRecorder mutable common::TCRWLock snapshot_lock_; lib::ObMutex sync_clog_lock_; storage::ObLS *ls_; - ObLSHandle ls_handle_; common::hash::ObHashSet dependent_tablet_set_; // tablet_id ObStorageCLogCb clog_cb_; int64_t last_print_log_ts_; diff --git a/src/storage/ls/ob_ls_role_handler.cpp b/src/storage/ls/ob_ls_role_handler.cpp deleted file mode 100644 index bb5fc13d1..000000000 --- a/src/storage/ls/ob_ls_role_handler.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_ls_role_handler.h" -namespace oceanbase -{ -namespace storage -{ - -} -} diff --git a/src/storage/ls/ob_ls_role_handler.h b/src/storage/ls/ob_ls_role_handler.h deleted file mode 100644 index c27822f55..000000000 --- a/src/storage/ls/ob_ls_role_handler.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEABASE_STORAGE_LS_ROLE_HANDLER_ -#define OCEABASE_STORAGE_LS_ROLE_HANDLER_ - -#include "lib/net/ob_addr.h" // ObAddr -namespace oceanbase -{ -namespace storage -{ -class ObLS; - -class ObLSRoleHandler -{ -public: - ObLSRoleHandler() : is_inited_(false), ls_(nullptr) {} - ~ObLSRoleHandler() {} - int init(ObLS *ls); -public: - // coordinate log stream leader change - // @param [in] leader, new leader - int change_leader(const common::ObAddr &leader); - - // for change leader callback. - int leader_revoke(); - int leader_takeover(); - int leader_active(); - -private: - bool is_inited_; - ObLS *ls_; -}; - -} // storage -} // oceanbase -#endif diff --git a/src/storage/ls/ob_ls_state.cpp b/src/storage/ls/ob_ls_state.cpp index 736e34478..356d07351 100644 --- a/src/storage/ls/ob_ls_state.cpp +++ b/src/storage/ls/ob_ls_state.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX STORAGE #include "storage/ls/ob_ls_state.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -68,61 +67,61 @@ int ObLSRunningState::StateHelper::switch_state(const int64_t op) } } if (OB_SUCC(ret)) { - _LOG_INFO("ObLSRunningState switch state success(ls_id=%jd, %s ~> %s, op=%s)", - ls_id_.id(), State::state_str(last_state_), State::state_str(state_), Ops::op_str(op)); + _LOG_INFO("ObLSRunningState switch state success(%s ~> %s, op=%s)", + State::state_str(last_state_), State::state_str(state_), Ops::op_str(op)); } else { - _LOG_ERROR("ObLSRunningState switch state error(ret=%d, ls_id=%jd, state=%s, op=%s)", - ret, ls_id_.id(), State::state_str(state_), Ops::op_str(op)); + _LOG_ERROR("ObLSRunningState switch state error(ret=%d, state=%s, op=%s)", + ret, State::state_str(state_), Ops::op_str(op)); } return ret; } -int ObLSRunningState::create_finish(const share::ObLSID &ls_id) +int ObLSRunningState::create_finish() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::CREATE_FINISH))) { - LOG_WARN("create finish failed", K(ret), K(ls_id)); + LOG_WARN("create finish failed", K(ret)); } return ret; } -int ObLSRunningState::online(const share::ObLSID &ls_id) +int ObLSRunningState::online() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::ONLINE))) { - LOG_WARN("online failed", K(ret), K(ls_id)); + LOG_WARN("online failed", K(ret)); } return ret; } -int ObLSRunningState::pre_offline(const share::ObLSID &ls_id) +int ObLSRunningState::pre_offline() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::PRE_OFFLINE))) { - LOG_WARN("pre offline failed", K(ret), K(ls_id)); + LOG_WARN("pre offline failed", K(ret)); } return ret; } -int ObLSRunningState::post_offline(const share::ObLSID &ls_id) +int ObLSRunningState::post_offline() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::POST_OFFLINE))) { - LOG_WARN("post offline failed", K(ret), K(ls_id)); + LOG_WARN("post offline failed", K(ret)); } return ret; } -int ObLSRunningState::stop(const share::ObLSID &ls_id) +int ObLSRunningState::stop() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::STOP))) { - LOG_WARN("stop failed", K(ret), K(ls_id)); + LOG_WARN("stop failed", K(ret)); } return ret; } @@ -142,13 +141,12 @@ ObLSPersistentState &ObLSPersistentState::operator=(const int64_t state) } // the state machine of ObLSPersistentState -//STATE \ ACTION START_WORK START_HA FINISH_HA REMOVE -//------------------------------------------------------------------- -//INIT NORMAL HA N ZOMBIE -//NORMAL NORMAL HA N ZOMBIE -//CREATE_ABORTED N N N N -//ZOMBIE N N N ZOMBIE -//HA N HA NORMAL ZOMBIE +//STATE \ ACTION START_WORK REMOVE +//-------------------------------------- +//INIT NORMAL ZOMBIE +//NORMAL NORMAL ZOMBIE +//CREATE_ABORTED N N +//ZOMBIE N ZOMBIE int ObLSPersistentState::StateHelper::switch_state(const int64_t op) { int ret = OB_SUCCESS; @@ -157,15 +155,13 @@ int ObLSPersistentState::StateHelper::switch_state(const int64_t op) static const int64_t LS_NORMAL = State::LS_NORMAL; static const int64_t LS_CREATE_ABORTED = State::LS_CREATE_ABORTED; static const int64_t LS_ZOMBIE = State::LS_ZOMBIE; - static const int64_t LS_HA = State::LS_HA; static const int64_t STATE_MAP[State::MAX][Ops::MAX] = { - // START_WORK START_HA FINISH_HA REMOVE -/* INIT */ {LS_NORMAL, LS_HA, N, LS_ZOMBIE}, -/* NORMAL */ {LS_NORMAL, LS_HA, N, LS_ZOMBIE}, -/* CREATE_ABORTED */ {N, N, N, N}, -/* ZOMBIE */ {N, N, N, LS_ZOMBIE}, -/* HA */ {N, LS_HA, LS_NORMAL, LS_ZOMBIE}, + // START_WORK REMOVE +/* INIT */ {LS_NORMAL, LS_ZOMBIE}, +/* NORMAL */ {LS_NORMAL, LS_ZOMBIE}, +/* CREATE_ABORTED */ {N, N}, +/* ZOMBIE */ {N, LS_ZOMBIE}, }; if (OB_UNLIKELY(!Ops::is_valid(op))) { @@ -184,51 +180,31 @@ int ObLSPersistentState::StateHelper::switch_state(const int64_t op) } } if (OB_SUCC(ret)) { - _LOG_INFO("ObLSPersistentState switch state success(ls_id=%jd, %s ~> %s, op=%s)", - ls_id_.id(), State::state_str(last_state_), State::state_str(state_), Ops::op_str(op)); + _LOG_INFO("ObLSPersistentState switch state success(%s ~> %s, op=%s)", + State::state_str(last_state_), State::state_str(state_), Ops::op_str(op)); } else { - _LOG_ERROR("ObLSPersistentState switch state error(ret=%d, ls_id=%jd, state=%s, op=%s)", - ret, ls_id_.id(), State::state_str(state_), Ops::op_str(op)); + _LOG_ERROR("ObLSPersistentState switch state error(ret=%d, state=%s, op=%s)", + ret, State::state_str(state_), Ops::op_str(op)); } return ret; } -int ObLSPersistentState::start_work(const share::ObLSID &ls_id) +int ObLSPersistentState::start_work() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::START_WORK))) { - LOG_WARN("start work failed", K(ret), K(ls_id)); + LOG_WARN("start work failed", K(ret)); } return ret; } -int ObLSPersistentState::start_ha(const share::ObLSID &ls_id) +int ObLSPersistentState::remove() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); - if (OB_FAIL(state_helper.switch_state(Ops::START_HA))) { - LOG_WARN("start ha failed", K(ret), K(ls_id)); - } - return ret; -} - -int ObLSPersistentState::finish_ha(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); - if (OB_FAIL(state_helper.switch_state(Ops::FINISH_HA))) { - LOG_WARN("finish ha failed", K(ret), K(ls_id)); - } - return ret; -} - -int ObLSPersistentState::remove(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - StateHelper state_helper(ls_id, state_); + StateHelper state_helper(state_); if (OB_FAIL(state_helper.switch_state(Ops::REMOVE))) { - LOG_WARN("remove failed", K(ret), K(ls_id)); + LOG_WARN("remove failed", K(ret)); } return ret; } diff --git a/src/storage/ls/ob_ls_state.h b/src/storage/ls/ob_ls_state.h index 5875778da..6b88d7e76 100644 --- a/src/storage/ls/ob_ls_state.h +++ b/src/storage/ls/ob_ls_state.h @@ -22,11 +22,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -} - namespace storage { class ObLSRunningState @@ -50,11 +45,11 @@ class ObLSRunningState return (State::LS_INIT == state_ || State::LS_STOPPED == state_); } - int create_finish(const share::ObLSID &ls_id); - int online(const share::ObLSID &ls_id); - int pre_offline(const share::ObLSID &ls_id); - int post_offline(const share::ObLSID &ls_id); - int stop(const share::ObLSID &ls_id); + int create_finish(); + int online(); + int pre_offline(); + int post_offline(); + int stop(); private: class State { @@ -130,12 +125,11 @@ class ObLSRunningState class StateHelper { public: - explicit StateHelper(const share::ObLSID &ls_id, int64_t &state) - : ls_id_(ls_id), state_(state), last_state_(state) {} + explicit StateHelper(int64_t &state) + : state_(state), last_state_(state) {} ~StateHelper() {} int switch_state(const int64_t op); private: - const share::ObLSID &ls_id_; int64_t &state_; int64_t last_state_; }; @@ -157,14 +151,11 @@ class ObLSPersistentState int deserialize(const char* buf, const int64_t data_len, int64_t& pos); int64_t get_serialize_size() const; - int start_work(const share::ObLSID &ls_id); - int start_ha(const share::ObLSID &ls_id); - int finish_ha(const share::ObLSID &ls_id); - int remove(const share::ObLSID &ls_id); + int start_work(); + int remove(); inline bool can_update_ls_meta() const { - return (State::LS_NORMAL == state_ || - State::LS_HA == state_); + return State::LS_NORMAL == state_; } bool is_need_gc() const { @@ -185,10 +176,6 @@ class ObLSPersistentState return (State::LS_ZOMBIE == state_ || State::LS_CREATE_ABORTED == state_); } - bool is_ha_state() const - { - return (State::LS_HA == state_); - } public: class State { @@ -198,8 +185,7 @@ class ObLSPersistentState static const int64_t LS_NORMAL = 1; static const int64_t LS_CREATE_ABORTED = 2; static const int64_t LS_ZOMBIE = 3; - static const int64_t LS_HA = 4; - static const int64_t MAX = 5; + static const int64_t MAX = 4; public: static bool is_valid(const int64_t state) { return state > INVALID && state < MAX; } @@ -216,7 +202,6 @@ class ObLSPersistentState TCM_STATE_CASE_TO_STR(LS_NORMAL); TCM_STATE_CASE_TO_STR(LS_CREATE_ABORTED); TCM_STATE_CASE_TO_STR(LS_ZOMBIE); - TCM_STATE_CASE_TO_STR(LS_HA); default: break; } @@ -230,10 +215,8 @@ class ObLSPersistentState public: static const int64_t INVALID = -1; static const int64_t START_WORK = 0; - static const int64_t START_HA = 1; - static const int64_t FINISH_HA = 2; - static const int64_t REMOVE = 3; - static const int64_t MAX = 4; + static const int64_t REMOVE = 1; + static const int64_t MAX = 2; public: static bool is_valid(const int64_t op) { return op > INVALID && op < MAX; } @@ -248,8 +231,6 @@ class ObLSPersistentState const char* str = "INVALID"; switch (op) { TCM_OP_CASE_TO_STR(START_WORK); - TCM_OP_CASE_TO_STR(START_HA); - TCM_OP_CASE_TO_STR(FINISH_HA); TCM_OP_CASE_TO_STR(REMOVE); default: break; @@ -262,12 +243,11 @@ class ObLSPersistentState class StateHelper { public: - explicit StateHelper(const share::ObLSID &ls_id, int64_t &state) - : ls_id_(ls_id), state_(state), last_state_(state) {} + explicit StateHelper(int64_t &state) + : state_(state), last_state_(state) {} ~StateHelper() {} int switch_state(const int64_t op); private: - const share::ObLSID &ls_id_; int64_t &state_; int64_t last_state_; }; diff --git a/src/storage/ls/ob_ls_storage_clog_handler.h b/src/storage/ls/ob_ls_storage_clog_handler.h index fd5bc6f63..094baec50 100644 --- a/src/storage/ls/ob_ls_storage_clog_handler.h +++ b/src/storage/ls/ob_ls_storage_clog_handler.h @@ -26,7 +26,7 @@ namespace storage class ObLS; class ObLSStorageClogHandler : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -44,19 +44,11 @@ class ObLSStorageClogHandler : public logservice::ObIReplaySubHandler, const palf::LSN &lsn, const share::SCN &scn) override final; - // for role change - void switch_to_follower_forcedly() override final + // local lifecycle + void deactivate() override final { } - int switch_to_leader() override final - { - return OB_SUCCESS; - } - int switch_to_follower_gracefully() override final - { - return OB_SUCCESS; - } - int resume_leader() override final + int activate() override final { return OB_SUCCESS; } diff --git a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp index adbf465ec..88503db49 100644 --- a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp +++ b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp @@ -79,30 +79,12 @@ int ObLSSyncTabletSeqHandler::replay(const void *buffer, return ret; } -void ObLSSyncTabletSeqHandler::switch_to_follower_forcedly() +void ObLSSyncTabletSeqHandler::deactivate() { // TODO } -int ObLSSyncTabletSeqHandler::switch_to_leader() -{ - int ret = OB_SUCCESS; - - //TODO - - return ret; -} - -int ObLSSyncTabletSeqHandler::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - - //TODO - - return ret; -} - -int ObLSSyncTabletSeqHandler::resume_leader() +int ObLSSyncTabletSeqHandler::activate() { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls_sync_tablet_seq_handler.h b/src/storage/ls/ob_ls_sync_tablet_seq_handler.h index 6ddc8b2b0..63128bedd 100644 --- a/src/storage/ls/ob_ls_sync_tablet_seq_handler.h +++ b/src/storage/ls/ob_ls_sync_tablet_seq_handler.h @@ -29,7 +29,7 @@ namespace storage class ObLS; class ObLSSyncTabletSeqHandler : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -45,11 +45,9 @@ class ObLSSyncTabletSeqHandler : public logservice::ObIReplaySubHandler, const palf::LSN &lsn, const share::SCN &scn) override final; - // for role change - void switch_to_follower_forcedly() override final; - int switch_to_leader() override final; - int switch_to_follower_gracefully() override final; - int resume_leader() override final; + // local lifecycle + void deactivate() override final; + int activate() override final; // for checkpoint int flush(share::SCN &scn) override final; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index b4bbeacc5..70fa6c9ae 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -78,7 +78,6 @@ ObLSTabletService::ObLSTabletService() mds_table_mgr_(), tablet_id_set_(), bucket_lock_(), - allow_to_read_mgr_(), is_inited_(false), is_stopped_(false) { @@ -103,8 +102,6 @@ int ObLSTabletService::init( } else if (OB_FAIL(bucket_lock_.init(ObTabletCommon::BUCKET_LOCK_BUCKET_CNT, ObLatchIds::TABLET_BUCKET_LOCK, "TabletSvrBucket"))) { LOG_WARN("failed to init bucket lock", K(ret)); - } else if (OB_FAIL(set_allow_to_read_(ls))) { - LOG_WARN("failed to set allow to read", K(ret)); } else if (OB_FAIL(mds_table_mgr_.init(ls))) { LOG_WARN("fail to init mds table mgr", KR(ret)); } else { @@ -154,8 +151,8 @@ int ObLSTabletService::offline() LOG_WARN("not inited", K(ret), K_(is_inited)); } else if (OB_FAIL(offline_build_tablet_without_memtable_())) { LOG_WARN("failed to offline build tablet without memtable", K(ret)); - } else if (OB_FAIL(offline_gc_tablet_for_aborted_create_())) { - LOG_WARN("failed to offline_gc_tablet_for_aborted_create", K(ret)); + } else if (OB_FAIL(offline_gc_uncommitted_tablets_())) { + LOG_WARN("failed to offline gc uncommitted tablets", K(ret)); } else if (OB_FAIL(offline_destroy_memtable_and_mds_table_())) { LOG_WARN("failed to offline destroy memtable and mds table", K(ret)); } else { @@ -193,7 +190,7 @@ int ObLSTabletService::replay( LOG_WARN("fail to deserialize tablet id", K(ret), KP(log_buf), K(nbytes), K(pos)); } else if (OB_FAIL(replayer_executor.init(log_buf, nbytes, pos, scn))) { LOG_WARN("failed to init replayer", K(ret), KP(log_buf), K(nbytes), K(pos), K(lsn), K(scn)); - } else if (OB_FAIL(replayer_executor.execute(scn, ls_->get_ls_id(), tablet_id))) { + } else if (OB_FAIL(replayer_executor.execute(scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret) { ret = OB_SUCCESS; // TODO (gaishun.gs): unify multi data replay logic LOG_INFO("tablet does not exist, skip", K(ret), K(replayer_executor)); @@ -208,26 +205,12 @@ int ObLSTabletService::replay( return ret; } -void ObLSTabletService::switch_to_follower_forcedly() +void ObLSTabletService::deactivate() { // TODO } -int ObLSTabletService::switch_to_leader() -{ - int ret = OB_SUCCESS; - //TODO - return ret; -} - -int ObLSTabletService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - //TODO - return ret; -} - -int ObLSTabletService::resume_leader() +int ObLSTabletService::activate() { int ret = OB_SUCCESS; //TODO @@ -251,53 +234,6 @@ int ObLSTabletService::prepare_for_safe_destroy() if (OB_FAIL(delete_all_tablets())) { LOG_WARN("fail to delete all tablets", K(ret)); } -#ifdef ERRSIM - if (OB_NOT_NULL(ls_) && !ls_->get_ls_id().is_sys_ls()) { - SERVER_EVENT_SYNC_ADD("ls_tablet_service", "after_delete_all_tablets", - "ls_id", ls_->get_ls_id().id()); - DEBUG_SYNC(AFTER_LS_GC_DELETE_ALL_TABLETS); - } -#endif - return ret; -} - -int ObLSTabletService::safe_to_destroy(bool &is_safe) -{ - int ret = OB_SUCCESS; - ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - is_safe = true; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else { - int64_t tx_data_memtable_mgr_ref = tx_data_memtable_mgr_.get_ref(); - int64_t tx_ctx_memtable_mgr_ref = tx_ctx_memtable_mgr_.get_ref(); - int64_t lock_memtable_mgr_ref = lock_memtable_mgr_.get_ref(); - int64_t mds_table_mgr_ref = mds_table_mgr_.get_ref(); - if (0 != tx_data_memtable_mgr_ref || 0 != tx_ctx_memtable_mgr_ref - || 0 != lock_memtable_mgr_ref || 0 != mds_table_mgr_ref) { - if (REACH_TIME_INTERVAL(60L * 1000000)) { // 60s - LOG_WARN("inner tablet memtable mgr can't destroy", K(tx_data_memtable_mgr_ref), - K(tx_ctx_memtable_mgr_ref), K(lock_memtable_mgr_ref), K(mds_table_mgr_ref)); - } - is_safe = false; - } else { - tx_data_memtable_mgr_.destroy(); - tx_ctx_memtable_mgr_.destroy(); - lock_memtable_mgr_.destroy(); - mds_table_mgr_.destroy(); - } - if (is_safe) { - bool is_wait_gc = false; - if (OB_FAIL(t3m->has_meta_wait_gc(is_wait_gc))) { - LOG_WARN("failed to check has_meta_wait_gc", K(ret)); - is_safe = false; - } else { - is_safe = !is_wait_gc; - } - } - } - return ret; } @@ -305,7 +241,6 @@ int ObLSTabletService::delete_all_tablets() { int ret = OB_SUCCESS; if (OB_NOT_NULL(ls_)) { - const ObLSID &ls_id = ls_->get_ls_id(); ObSArray tablet_id_array; GetAllTabletIDOperator op(tablet_id_array); @@ -313,14 +248,14 @@ int ObLSTabletService::delete_all_tablets() common::ObBucketWLockAllGuard lock_guard(bucket_lock_); time_guard.click("Lock"); if (OB_FAIL(tablet_id_set_.foreach(op))) { - LOG_WARN("failed to traverse tablet id set", K(ret), K(ls_id)); + LOG_WARN("failed to traverse tablet id set", K(ret)); } else if (tablet_id_array.empty()) { // tablet id array is empty, do nothing } else { for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) { const ObTabletID &tablet_id = tablet_id_array.at(i); - if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to do remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(inner_remove_tablet(tablet_id))) { + LOG_ERROR("failed to do remove tablet", K(ret), K(tablet_id)); ob_usleep(1_s); ob_abort(); } @@ -347,10 +282,9 @@ int ObLSTabletService::remove_tablet(const ObTabletHandle& tablet_handle) LOG_WARN("invalid tablet", K(ret), K(tablet_handle)); } else { const ObTablet &target_tablet = *(tablet_handle.get_obj()); - const ObLSID ls_id = ls_->get_ls_id(); const int64_t ls_epoch = ls_->get_ls_epoch(); const ObTabletID tablet_id = target_tablet.get_tablet_meta().tablet_id_; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle cur_tablet_handle; ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); @@ -366,12 +300,12 @@ int ObLSTabletService::remove_tablet(const ObTabletHandle& tablet_handle) LOG_INFO("tablet object has been changed, need retry", K(ret), K(key), K(target_tablet), KPC(cur_tablet_handle.get_obj())); } else { const ObMetaDiskAddr &tablet_addr = tablet_handle.get_obj()->get_tablet_addr(); - if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.remove_tablet(ls_id, ls_epoch, tablet_handle))) { - LOG_WARN("failed to write remove tablet slog", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.remove_tablet(tablet_handle))) { + LOG_WARN("failed to write remove tablet slog", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->wait_release_memtables())) { LOG_ERROR("failed to release memtables", K(ret), K(tablet_id)); - } else if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to do remove tablet", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(inner_remove_tablet(tablet_id))) { + LOG_ERROR("failed to do remove tablet", K(ret), K(tablet_id)); ob_usleep(1_s); ob_abort(); } else { @@ -388,7 +322,6 @@ int ObLSTabletService::remove_tablets(const common::ObIArray const int64_t tablet_cnt = tablet_id_array.count(); ObSArray all_tablet_id_hash_array; ObSArray tablet_ids; - ObSArray tablet_addrs; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -417,11 +350,8 @@ int ObLSTabletService::remove_tablets(const common::ObIArray LOG_WARN("failed to lock multi buckets", K(ret)); } else { time_guard.click("Lock"); - - const share::ObLSID &ls_id = ls_->get_ls_id(); ObTabletHandle tablet_handle; ObTabletMapKey key; - key.ls_id_ = ls_id; // check tablet existence for (int64_t i = 0; OB_SUCC(ret) && i < tablet_cnt; ++i) { @@ -440,30 +370,28 @@ int ObLSTabletService::remove_tablets(const common::ObIArray } else if (OB_FAIL(tablet_handle.get_obj()->get_meta_disk_addr(tablet_addr))) { LOG_WARN("failed to get tablet addr", K(ret), K(key)); } else if (!tablet_addr.is_disked()) { - if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { + if (OB_FAIL(inner_remove_tablet(tablet_id))) { LOG_WARN("failed to remove non disked tablet from memory", K(ret), K(key)); } else { FLOG_INFO("succeeded to remove non disked tablet from memory", K(ret), K(key)); } } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); - } else if (OB_FAIL(tablet_addrs.push_back(tablet_addr))) { - LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); } } // write slog and do remove tablet if (OB_FAIL(ret)) { } else if (tablet_ids.empty()) { - LOG_INFO("all tablets already deleted, do nothing", K(ret), K(ls_id), K(tablet_id_array)); - } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.remove_tablets(ls_id, ls_->get_ls_epoch(), tablet_ids, tablet_addrs))) { - LOG_WARN("failed to remove tablets", K(ret), K(ls_id), K(tablet_ids)); + LOG_INFO("all tablets already deleted, do nothing", K(ret), K(tablet_id_array)); + } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.remove_tablets(tablet_ids))) { + LOG_WARN("failed to remove tablets", K(ret), K(tablet_ids)); } else { time_guard.click("WrSlog"); for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to do remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(inner_remove_tablet(tablet_id))) { + LOG_ERROR("failed to do remove tablet", K(ret), K(tablet_id)); ob_usleep(1_s); ob_abort(); } @@ -480,47 +408,40 @@ int ObLSTabletService::remove_tablets(const common::ObIArray return ret; } -int ObLSTabletService::do_remove_tablet( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +int ObLSTabletService::do_remove_tablet(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; ObTimeGuard time_guard("RmTabletLock", 1_s); ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); time_guard.click("Lock"); - if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_WARN("fail to remove tablet with lock", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(inner_remove_tablet(tablet_id))) { + LOG_WARN("fail to remove tablet with lock", K(ret), K(tablet_id)); } return ret; } // TODO(yunshan.tys) cope with failure of deleting tablet (tablet hasn't been loaded from disk) -int ObLSTabletService::inner_remove_tablet( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id) +int ObLSTabletService::inner_remove_tablet(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTenantDirectLoadMgr *tenant_direct_load_mgr = share::g_mp->tenant_direct_load_mgr(); - ObTransService *tx_svr = share::g_mp->trans_service(); if (OB_FAIL(tablet_id_set_.erase(tablet_id))) { if (OB_HASH_NOT_EXIST == ret) { // tablet id is already erased ret = OB_SUCCESS; } else { - LOG_WARN("fail to erase tablet id from set", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to erase tablet id from set", K(ret), K(tablet_id)); } - } else if (OB_FAIL(tx_svr->remove_tablet(tablet_id, ls_id))) { - LOG_ERROR("fail to remove tablet to ls cache", K(ret), K(tablet_id), K(ls_id)); } if (OB_SUCC(ret)) { // loop retry to delete tablet from t3m while (OB_FAIL(t3m->del_tablet(key))) { if (REACH_TIME_INTERVAL(10_s)) { - LOG_ERROR("failed to delete tablet from t3m", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to delete tablet from t3m", K(ret), K(tablet_id)); } } } @@ -531,13 +452,13 @@ int ObLSTabletService::inner_remove_tablet( if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; } else { - LOG_ERROR("remove tablet direct load failed", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("remove tablet direct load failed", K(ret), K(tablet_id)); } } } if (OB_SUCC(ret)) { - FLOG_INFO("succeeded to remove tablet", K(ret), K(ls_id), K(tablet_id)); + FLOG_INFO("succeeded to remove tablet", K(ret), K(tablet_id)); } return ret; @@ -550,7 +471,7 @@ int ObLSTabletService::get_tablet( const ObMDSGetTabletMode mode) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -589,12 +510,11 @@ int ObLSTabletService::get_tablet_addr(const ObTabletMapKey &key, ObMetaDiskAddr void ObLSTabletService::report_tablet_to_rs(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls_->get_ls_id(); if (tablet_id.is_ls_inner_tablet()) { // no need to report for ls inner tablet - } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id))) { - LOG_WARN("failed to report tablet info", KR(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id))) { + LOG_WARN("failed to report tablet info", KR(ret), K(tablet_id)); } } @@ -602,8 +522,6 @@ void ObLSTabletService::report_tablet_to_rs( const common::ObIArray &tablet_id_array) { int ret = OB_SUCCESS; - - const share::ObLSID &ls_id = ls_->get_ls_id(); // ignore ret on purpose for (int64_t i = 0; i < tablet_id_array.count(); ++i) { @@ -611,8 +529,8 @@ void ObLSTabletService::report_tablet_to_rs( if (tablet_id.is_ls_inner_tablet()) { // no need to report for ls inner tablet continue; - } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(ls_id, tablet_id))) { - LOG_WARN("failed to report tablet info", KR(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->tablet_table_updater()->submit_tablet_update_task(tablet_id))) { + LOG_WARN("failed to report tablet info", KR(ret), K(tablet_id)); } } } @@ -621,15 +539,10 @@ int ObLSTabletService::table_scan(ObTabletHandle &tablet_handle, ObTableScanIter { int ret = OB_SUCCESS; NG_TRACE(S_table_scan_begin); - bool allow_to_read = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (FALSE_IT(allow_to_read_mgr_.load_allow_to_read_info(allow_to_read))) { - } else if (!allow_to_read) { - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls is not allow to read", K(ret), KPC(ls_)); } else if (OB_FAIL(prepare_scan_table_param(param, *(share::g_mp->tenant_schema_service()->get_schema_service())))) { LOG_WARN("failed to prepare scan table param", K(ret), K(param)); } else if (OB_FAIL(inner_table_scan(tablet_handle, iter, param))) { @@ -644,7 +557,6 @@ int ObLSTabletService::table_rescan(ObTabletHandle &tablet_handle, ObTableScanPa { int ret = OB_SUCCESS; NG_TRACE(S_table_rescan_begin); - bool allow_to_read = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -652,10 +564,6 @@ int ObLSTabletService::table_rescan(ObTabletHandle &tablet_handle, ObTableScanPa } else if (OB_ISNULL(result)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret)); - } else if (FALSE_IT(allow_to_read_mgr_.load_allow_to_read_info(allow_to_read))) { - } else if (!allow_to_read) { - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls is not allow to read", K(ret), KPC(ls_)); } else if (OB_FAIL(prepare_scan_table_param(param, *(share::g_mp->tenant_schema_service()->get_schema_service())))) { LOG_WARN("failed to prepare scan table param", K(ret), K(result), K(param)); } else { @@ -669,13 +577,12 @@ int ObLSTabletService::table_rescan(ObTabletHandle &tablet_handle, ObTableScanPa } int ObLSTabletService::refresh_tablet_addr( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObUpdateTabletPointerParam ¶m, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); while (OB_SUCC(ret)) { @@ -701,220 +608,6 @@ int ObLSTabletService::refresh_tablet_addr( return ret; } -int ObLSTabletService::trim_old_tablets(const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle_head; - ObTimeGuard time_guard("ObLSTabletService::trim_old_tablets", 1_s); - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - time_guard.click("Lock"); - - if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle_head))) { - LOG_WARN("failed to check and get tablet", K(ret), K(tablet_id)); - } else if (tablet_handle_head.get_obj()->is_empty_shell()) { - LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(ret), "old_tablet", tablet_handle_head.get_obj()); - } else if (OB_UNLIKELY(!tablet_handle_head.get_obj()->get_tablet_meta().has_next_tablet_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("doesn't have old tablet", K(ret), "tablet_meta", tablet_handle_head.get_obj()->get_tablet_meta()); - } else { - time_guard.click("GetTablet"); - ObTablet *tablet_head = tablet_handle_head.get_obj(); - ObTabletHandle new_tablet_handle; - - tablet_head->trim_tablet_list(); - ObMetaDiskAddr disk_addr; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet_head, new_tablet_handle))) { - LOG_WARN("fail to persist and transfor tablet", K(ret), KPC(tablet_head), K(new_tablet_handle)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->get_tablet_addr())) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, tablet_handle_head, new_tablet_handle, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else { - FLOG_INFO("succeeded to trim tablets list", K(key), K(disk_addr)); - } - } - - return ret; -} - -int ObLSTabletService::rollback_rebuild_tablet(const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle_head; - ObTimeGuard time_guard("ObLSTabletService::rollback_rebuild_tablet", 1_s); - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - time_guard.click("Lock"); - - if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle_head))) { - LOG_WARN("failed to check and get tablet", K(ret), K(tablet_id)); - } else if (tablet_handle_head.get_obj()->is_empty_shell()) { - LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(ret), "old_tablet", tablet_handle_head.get_obj()); - } else { - time_guard.click("GetTablet"); - - ObTablet *tablet_head = tablet_handle_head.get_obj(); - ObTabletHandle next_tablet_handle; - ObMetaObj meta_obj; - tablet_head->get_next_tablet_guard().get_obj(meta_obj); - next_tablet_handle.set_obj(meta_obj); - next_tablet_handle.set_wash_priority(WashTabletPriority::WTP_LOW); - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); - ObMetaDiskAddr disk_addr; - ObTabletHandle new_tablet_handle; - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *next_tablet_handle.get_obj(), new_tablet_handle))) { - LOG_WARN("fail to persist and transform tablet", K(ret), K(next_tablet_handle), K(new_tablet_handle)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->get_tablet_addr())) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, tablet_handle_head, new_tablet_handle, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else { - FLOG_INFO("succeeded to rollback rebuild", K(key), K(disk_addr)); - } - } - - return ret; -} - -int ObLSTabletService::rebuild_tablet_with_old( - const ObMigrationTabletParam &mig_tablet_param, - ObTabletHandle &tablet_guard) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("ObLSTabletService::rebuild_tablet_with_old", 1_s); - common::ObArenaAllocator allocator(common::ObMemAttr("RebuildTablet")); - ObTabletHandle old_tablet_hdl; - ObTabletHandle tmp_tablet_hdl; - ObTabletHandle new_tablet_hdl; - ObTablet *tmp_tablet = nullptr; - ObFreezer *freezer = ls_->get_freezer(); - ObMetaDiskAddr disk_addr; - - const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_; - const share::ObLSID &ls_id = mig_tablet_param.ls_id_; - const ObTabletMapKey key(ls_id, tablet_id); - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) { - LOG_WARN("failed to get tablet", K(ret), K(key)); - } else if (FALSE_IT(time_guard.click("GetTablet"))) { - } else if (old_tablet_hdl.get_obj()->is_empty_shell()) { - LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(ret), "old_tablet", old_tablet_hdl.get_obj()); - } else if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) { - LOG_WARN("fail to acquire temporary tablet", K(ret), K(key)); - } else if (FALSE_IT(tmp_tablet = tmp_tablet_hdl.get_obj())) { - } else if (OB_FAIL(tmp_tablet->init_with_migrate_param(allocator, mig_tablet_param, true/*is_update*/, freezer))) { - LOG_WARN("failed to init tablet", K(ret), K(mig_tablet_param)); - } else if (FALSE_IT(time_guard.click("InitTablet"))) { - } else if (FALSE_IT(tmp_tablet->set_next_tablet_guard(tablet_guard))) { - } else if (FALSE_IT(time_guard.click("InitTablet"))) { - } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tmp_tablet, new_tablet_hdl))) { - LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tmp_tablet), K(new_tablet_hdl)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = new_tablet_hdl.get_obj()->tablet_addr_)) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, old_tablet_hdl, new_tablet_hdl, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else if (OB_FAIL(new_tablet_hdl.get_obj()->start_direct_load_task_if_need())) { - LOG_WARN("start ddl if need failed", K(ret), K(key)); - } else { - LOG_INFO("rebuild tablet with old succeed", K(ret), K(key), K(disk_addr)); - } - return ret; -} - -int ObLSTabletService::migrate_update_tablet( - const ObMigrationTabletParam &mig_tablet_param) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("ObLSTabletService::migrate_update_tablet", 1_s); - common::ObArenaAllocator allocator(common::ObMemAttr("MigUpTab")); - const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_; - const share::ObLSID &ls_id = mig_tablet_param.ls_id_; - const ObTabletMapKey key(mig_tablet_param.ls_id_, mig_tablet_param.tablet_id_); - ObTabletHandle old_tablet_hdl; - ObTabletHandle tmp_tablet_hdl; - ObTabletHandle new_tablet_hdl; - ObTablet *new_tablet = nullptr; - ObMetaDiskAddr disk_addr; - ObFreezer *freezer = ls_->get_freezer(); - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) { - LOG_WARN("failed to get tablet", K(ret), K(key)); - } else if (FALSE_IT(time_guard.click("GetTablet"))) { - } else if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) { - LOG_WARN("fail to acquire temporary tablet", K(ret), K(key)); - } else if (FALSE_IT(new_tablet = tmp_tablet_hdl.get_obj())) { - } else if (OB_FAIL(new_tablet->init_with_migrate_param(allocator, mig_tablet_param, true/*is_update*/, freezer))) { - LOG_WARN("failed to init tablet", K(ret), K(mig_tablet_param)); - } else if (FALSE_IT(time_guard.click("InitTablet"))) { - } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *new_tablet, new_tablet_hdl))) { - LOG_WARN("fail to persist and transform tablet", K(ret), KPC(new_tablet), K(new_tablet_hdl)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = new_tablet_hdl.get_obj()->tablet_addr_)) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, old_tablet_hdl, new_tablet_hdl, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else if (OB_FAIL(new_tablet_hdl.get_obj()->start_direct_load_task_if_need())) { - LOG_WARN("start ddl if need failed", K(ret)); - } else { - LOG_INFO("migrate update tablet succeed", K(ret), K(key), K(disk_addr)); - } - - return ret; -} - -int ObLSTabletService::migrate_create_tablet( - const ObMigrationTabletParam &mig_tablet_param, - ObTabletHandle &handle) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("ObLSTabletService::migrate_create_tablet", 1_s); - common::ObArenaAllocator allocator(common::ObMemAttr("MigCreateTab")); - const share::ObLSID &ls_id = mig_tablet_param.ls_id_; - const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_; - const ObTabletMapKey key(ls_id, tablet_id); - ObFreezer *freezer = ls_->get_freezer(); - ObTabletHandle tmp_tablet_hdl; - ObTabletHandle tablet_handle; - ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); - ObTransService *tx_svr = share::g_mp->trans_service(); - - if (OB_FAIL(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, tmp_tablet_hdl))) { - LOG_WARN("fail to create temporary tablet", K(ret), K(key)); - } else if (OB_FAIL(tmp_tablet_hdl.get_obj()->init_with_migrate_param(allocator, mig_tablet_param, false/*is_update*/, freezer))) { - LOG_WARN("fail to init tablet", K(ret), K(mig_tablet_param)); - } else if (FALSE_IT(time_guard.click("InitTablet"))) { - } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tmp_tablet_hdl.get_obj(), tablet_handle))) { - LOG_WARN("fail to persist and transform tablet", K(ret), K(tmp_tablet_hdl), K(tablet_handle)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = tablet_handle.get_obj()->tablet_addr_)) { - } else if (OB_FAIL(safe_create_cas_tablet(ls_id, tablet_id, disk_addr, tablet_handle, time_guard))) { - LOG_WARN("fail to create tablet and cas", K(ret), K(ls_id), K(tablet_id), K(disk_addr)); - } else if (OB_FAIL(tablet_handle.get_obj()->start_direct_load_task_if_need())) { - LOG_WARN("start ddl if need failed", K(ret)); - } else if (OB_FAIL(tx_svr->create_tablet(key.tablet_id_, key.ls_id_))) { - LOG_WARN("fail to create tablet cache", K(ret), K(key), K(tablet_handle)); - } else { - LOG_INFO("migrate create tablet succeed", K(ret), K(key), K(disk_addr)); - } - - if (OB_SUCC(ret)) { - handle = tablet_handle; - } else { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); - } - } - - return ret; -} - int ObLSTabletService::refresh_memtable_for_ckpt( const ObMetaDiskAddr &old_addr, const ObMetaDiskAddr &cur_addr, @@ -1025,10 +718,9 @@ int ObLSTabletService::update_tablet_table_store( time_guard.click("GetTablet"); ObTabletHandle tmp_tablet_hdl; ObTablet *tmp_tablet = nullptr; - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) { LOG_WARN("fail to acquire temporary tablet", K(ret), K(key)); @@ -1050,7 +742,7 @@ int ObLSTabletService::update_tablet_table_store( if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; ObServerAutoSplitScheduler &auto_split_scheduler = ObServerAutoSplitScheduler::get_instance(); - if (OB_TMP_FAIL(auto_split_scheduler.push_task(new_tablet_hdl, *ls_))) { + if (OB_TMP_FAIL(auto_split_scheduler.push_task(new_tablet_hdl))) { LOG_WARN("fail to push auto split task", K(tmp_ret)); } } @@ -1068,9 +760,7 @@ int ObLSTabletService::update_tablet_table_store( // TODO(@DanLing) use LocalArena later allocator.set_ctx_id(ObCtxIds::MERGE_RESERVE_CTX_ID); } - - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle old_tablet_hdl; ObTabletHandle tmp_tablet_hdl; ObTabletHandle new_tablet_hdl; @@ -1102,7 +792,7 @@ int ObLSTabletService::update_tablet_table_store( time_guard.click("GetTablet"); ObTablet *old_tablet = old_tablet_hdl.get_obj(); ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam persist_param(ls_->get_ls_epoch(), tablet_id); share::SCN not_used_scn; if (!is_mds_merge(param.compaction_info_.merge_type_) && OB_FAIL(tmp_tablet->init_for_merge(allocator, param, *old_tablet))) { LOG_WARN("failed to init tablet", K(ret), K(param), KPC(old_tablet)); @@ -1123,7 +813,7 @@ int ObLSTabletService::update_tablet_table_store( if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; ObServerAutoSplitScheduler &auto_split_scheduler= ObServerAutoSplitScheduler::get_instance(); - if (OB_TMP_FAIL(auto_split_scheduler.push_task(new_tablet_hdl, *ls_))) { + if (OB_TMP_FAIL(auto_split_scheduler.push_task(new_tablet_hdl))) { LOG_WARN("fail to push auto split task", K(tmp_ret)); } @@ -1134,8 +824,7 @@ int ObLSTabletService::update_tablet_table_store( int ObLSTabletService::update_tablet_to_empty_shell(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); common::ObArenaAllocator allocator(common::ObMemAttr("UpdEmptySh")); ObTabletHandle new_tablet_handle; ObTabletHandle tmp_tablet_handle; @@ -1164,7 +853,7 @@ int ObLSTabletService::update_tablet_to_empty_shell(const common::ObTabletID &ta ObTablet *tmp_tablet = tmp_tablet_handle.get_obj(); ObTablet *new_tablet = nullptr; ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(tmp_tablet->init_empty_shell(*tmp_tablet_handle.get_allocator(), *old_tablet))) { LOG_WARN("failed to init tablet", K(ret), KPC(old_tablet)); } else if (FALSE_IT(time_guard.click("InitNew"))) { @@ -1212,10 +901,9 @@ int ObLSTabletService::update_medium_compaction_info( ObTabletHandle new_tablet_hdl; ObTablet *tmp_tablet = nullptr; ObTablet *old_tablet = old_tablet_handle.get_obj(); - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -1248,8 +936,7 @@ int ObLSTabletService::build_new_tablet_from_mds_table( { int ret = OB_SUCCESS; common::ObArenaAllocator allocator(common::ObMemAttr("BuildMSD")); - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle old_tablet_hdl; ObTabletHandle tablet_for_mds_dump_handle; ObTabletHandle tmp_tablet_hdl; @@ -1299,11 +986,10 @@ int ObLSTabletService::build_new_tablet_from_mds_table( ObTablet *old_tablet = old_tablet_hdl.get_obj(); ObTablet *tmp_tablet = tmp_tablet_hdl.get_obj(); ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); ObUpdateTableStoreParam mds_param(ctx.static_param_.version_range_.snapshot_version_, 1/*multi_version_start*/, ObMdsSchemaHelper::get_instance().get_storage_schema(), - ctx.get_ls_rebuild_seq(), mds_sstable, false/*allow_duplicate_sstable*/); if (OB_FAIL(mds_param.init_with_compaction_info( @@ -1336,8 +1022,7 @@ int ObLSTabletService::update_tablet_release_memtable_for_offline( const SCN scn) { int ret = OB_SUCCESS; - const ObLSID ls_id(ls_->get_ls_id()); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObTimeGuard time_guard("ObLSTabletService::update_tablet_release_memtable", 1_s); @@ -1365,7 +1050,7 @@ int ObLSTabletService::update_tablet_release_memtable_for_offline( ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletHandle new_tablet_handle; ObUpdateTabletPointerParam param; - const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam persist_param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletPersister::copy_from_old_tablet(persist_param, *tablet, new_tablet_handle))) { LOG_WARN("fail to copy from old tablet", K(ret), KPC(tablet)); } else if (FALSE_IT(time_guard.click("CpTablet"))) { @@ -1395,8 +1080,8 @@ int ObLSTabletService::update_tablet_release_memtable_for_offline( K(ret), K(iter), KPC(table)); } else if (OB_FAIL(tablet_handle.get_obj()->wait_release_memtables())) { LOG_ERROR("failed to release memtables", K(ret), K(tablet_id)); - } else if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to do remove tablet", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(inner_remove_tablet(tablet_id))) { + LOG_ERROR("failed to do remove tablet", K(ret), K(tablet_id)); } else { time_guard.click("RmTablet"); } @@ -1424,7 +1109,7 @@ int ObLSTabletService::update_tablet_ddl_commit_scn( const SCN ddl_commit_scn) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle old_handle; ObTimeGuard time_guard("ObLSTabletService::update_tablet_ddl_commit_scn", 1_s); ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); @@ -1445,7 +1130,7 @@ int ObLSTabletService::update_tablet_ddl_commit_scn( ObUpdateTabletPointerParam param; ObTabletHandle new_handle; const ObTablet &old_tablet = *old_handle.get_obj(); - const ObTabletPersisterParam persist_param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam persist_param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletPersister::persist_and_transform_only_tablet_meta(persist_param, old_tablet, modifier, new_handle))) { LOG_WARN("fail to persist and transform only tablet meta", K(ret), K(old_tablet), K(ddl_commit_scn)); @@ -1482,7 +1167,7 @@ int ObLSTabletService::update_tablet_report_status(const common::ObTabletID &tab } else { time_guard.click("GetTablet"); ObMetaDiskAddr disk_addr; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); ObTablet *tablet = tablet_handle.get_obj(); ObTabletHandle new_tablet_handle; bool need_report = true; @@ -1495,7 +1180,7 @@ int ObLSTabletService::update_tablet_report_status(const common::ObTabletID &tab } if (need_report) { - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle))) { LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle)); } else if (FALSE_IT(time_guard.click("Persist"))) { @@ -1536,12 +1221,9 @@ int ObLSTabletService::update_tablet_snapshot_version( ObTabletHandle new_tablet_hdl; ObTablet *tmp_tablet = nullptr; ObTablet *old_tablet = old_tablet_handle.get_obj(); - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); - ObTabletDataStatus::STATUS current_status = ObTabletDataStatus::DATA_STATUS_MAX; - + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_TABLET_NOT_EXIST; @@ -1549,9 +1231,7 @@ int ObLSTabletService::update_tablet_snapshot_version( LOG_WARN("failed to acquire tablet", K(ret), K(key)); } } else if (FALSE_IT(tmp_tablet = tmp_tablet_hdl.get_obj())) { - } else if (OB_FAIL(old_tablet->tablet_meta_.ha_status_.get_data_status(current_status))) { - LOG_WARN("failed to get data status", K(ret), KPC(old_tablet)); - } else if (OB_FAIL(tmp_tablet->init_with_replace_members(allocator, *old_tablet, snapshot_version, current_status))) { + } else if (OB_FAIL(tmp_tablet->init_with_updated_members(allocator, *old_tablet, snapshot_version))) { LOG_WARN("failed to init tablet", K(ret), KPC(old_tablet)); } else if (FALSE_IT(time_guard.click("InitNew"))) { } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tmp_tablet, new_tablet_hdl))) { @@ -1591,24 +1271,24 @@ int ObLSTabletService::update_tablet_restore_status( } else { time_guard.click("GetTablet"); ObMetaDiskAddr disk_addr; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); ObTablet *tablet = tablet_handle.get_obj(); ObTabletHandle new_tablet_handle; const bool current_split_data_incomplete_status = tablet->tablet_meta_.split_info_.is_data_incomplete(); - if (OB_FAIL(tablet->tablet_meta_.ha_status_.get_restore_status(current_status))) { + if (OB_FAIL(tablet->tablet_meta_.restore_state_.get_restore_status(current_status))) { LOG_WARN("failed to get restore status", K(ret), KPC(tablet)); } else if (OB_FAIL(ObTabletRestoreStatus::check_can_change_status(current_status, restore_status, can_change))) { LOG_WARN("failed to check can change status", K(ret), K(current_status), K(restore_status), KPC(tablet)); } else if (!can_change) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("can not change restore status", K(ret), K(current_status), K(restore_status), KPC(tablet)); - } else if (OB_FAIL(tablet->tablet_meta_.ha_status_.set_restore_status(restore_status))) { + } else if (OB_FAIL(tablet->tablet_meta_.restore_state_.set_restore_status(restore_status))) { LOG_WARN("failed to set restore status", K(ret), K(restore_status), KPC(tablet)); } else if (need_to_set_split_data_complete && OB_FALSE_IT((void)tablet->tablet_meta_.split_info_.set_data_incomplete(false))) { } else { // TODO(jiahua.cjh) move check valid to tablet init after generate new version tablet. - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(tablet->check_valid())) { LOG_WARN("failed to check tablet valid", K(ret), K(restore_status), KPC(tablet)); } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle))) { @@ -1629,7 +1309,7 @@ int ObLSTabletService::update_tablet_restore_status( if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = tablet->tablet_meta_.ha_status_.set_restore_status(current_status))) { + if (OB_SUCCESS != (tmp_ret = tablet->tablet_meta_.restore_state_.set_restore_status(current_status))) { LOG_ERROR("failed to set restore status", K(tmp_ret), K(current_status), KPC(tablet)); ob_abort(); } @@ -1640,146 +1320,7 @@ int ObLSTabletService::update_tablet_restore_status( return ret; } -int ObLSTabletService::update_tablet_ha_data_status( - const common::ObTabletID &tablet_id, - const ObTabletDataStatus::STATUS &data_status) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle; - ObTabletDataStatus::STATUS current_status = ObTabletDataStatus::DATA_STATUS_MAX; - bool can_change = false; - - ObTimeGuard time_guard("ObLSTabletService::update_tablet_ha_data_status", 1_s); - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - time_guard.click("Lock"); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(is_stopped_)) { - ret = OB_NOT_RUNNING; - LOG_WARN("tablet service stopped", K(ret)); - } else if (OB_UNLIKELY(!tablet_id.is_valid()) - || OB_UNLIKELY(!ObTabletDataStatus::is_valid(data_status))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(tablet_id), K(data_status)); - } else if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); - } else if (tablet_handle.get_obj()->is_empty_shell()) { - LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(ret), "old_tablet", tablet_handle.get_obj()); - } else { - time_guard.click("GetTablet"); - ObMetaDiskAddr disk_addr; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); - ObTablet *old_tablet = tablet_handle.get_obj(); - ObTablet *tmp_tablet = nullptr; - common::ObArenaAllocator allocator("UpdateSchema", OB_MALLOC_NORMAL_BLOCK_SIZE, ObCtxIds::DEFAULT_CTX_ID); - ObTabletHandle new_tablet_handle; - ObTabletHandle tmp_tablet_handle; - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(old_tablet->tablet_meta_.ha_status_.get_data_status(current_status))) { - LOG_WARN("failed to get data status", K(ret), KPC(old_tablet)); - } else if (OB_FAIL(ObTabletDataStatus::check_can_change_status(current_status, data_status, can_change))) { - LOG_WARN("failed to check can change status", K(ret), K(current_status), K(data_status), KPC(old_tablet)); - } else if (!can_change) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can not change data status", K(ret), K(current_status), K(data_status), KPC(old_tablet)); - } else if (current_status == data_status) { - LOG_INFO("data status is same, skip update", K(tablet_id), K(current_status), K(data_status)); - } else if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_handle))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_TABLET_NOT_EXIST; - } else { - LOG_WARN("failed to acquire tablet", K(ret), K(key)); - } - } else if (FALSE_IT(tmp_tablet = tmp_tablet_handle.get_obj())) { - } else if (OB_FAIL(tmp_tablet->init_with_replace_members(allocator, *old_tablet, old_tablet->tablet_meta_.snapshot_version_, data_status))) { - LOG_WARN("failed to init tablet", K(ret), KPC(old_tablet)); - } else if (FALSE_IT(time_guard.click("InitNew"))) { - } else if (OB_FAIL(tmp_tablet->check_valid())) { - LOG_WARN("failed to check tablet valid", K(ret), K(data_status), KPC(tmp_tablet)); - } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tmp_tablet, new_tablet_handle))) { - LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tmp_tablet), K(new_tablet_handle)); - } else if (FALSE_IT(time_guard.click("Persist"))) { - } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->tablet_addr_)) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, tablet_handle, new_tablet_handle, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else { - LOG_INFO("succeeded to update tablet ha data status", K(ret), K(key), K(disk_addr), K(data_status), K(tablet_handle), K(tmp_tablet_handle), K(time_guard)); - } - } - return ret; -} -int ObLSTabletService::update_tablet_ha_expected_status( - const common::ObTabletID &tablet_id, - const ObTabletExpectedStatus::STATUS &expected_status) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle; - ObTabletExpectedStatus::STATUS current_status = ObTabletExpectedStatus::EXPECTED_STATUS_MAX; - bool can_change = false; - - ObTimeGuard time_guard("ObLSTabletService::update_tablet_ha_data_status", 1_s); - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - time_guard.click("Lock"); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(is_stopped_)) { - ret = OB_NOT_RUNNING; - LOG_WARN("tablet service stopped", K(ret)); - } else if (OB_UNLIKELY(!tablet_id.is_valid()) - || OB_UNLIKELY(!ObTabletExpectedStatus::is_valid(expected_status))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(tablet_id), K(expected_status)); - } else if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); - } else if (tablet_handle.get_obj()->is_empty_shell()) { - LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(ret), "old_tablet", tablet_handle.get_obj()); - } else { - time_guard.click("GetTablet"); - ObMetaDiskAddr disk_addr; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); - ObTablet *tablet = tablet_handle.get_obj(); - ObTabletHandle new_tablet_handle; - const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id); - - if (OB_FAIL(tablet->tablet_meta_.ha_status_.get_expected_status(current_status))) { - LOG_WARN("failed to get data status", K(ret), KPC(tablet)); - } else if (expected_status == current_status) { - LOG_INFO("tablet ha expected status is same, no need update", K(tablet_id), - K(current_status), K(expected_status)); - } else if (OB_FAIL(ObTabletExpectedStatus::check_can_change_status(current_status, expected_status, can_change))) { - LOG_WARN("failed to check can change status", K(ret), K(current_status), K(expected_status), KPC(tablet)); - } else if (!can_change) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can not change meta status", K(ret), K(current_status), K(expected_status), KPC(tablet)); - } else { - if (OB_FAIL(tablet->tablet_meta_.ha_status_.set_expected_status(expected_status))) { - LOG_WARN("failed to set ha meta status", K(ret), KPC(tablet), K(expected_status)); - } else { - if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle))) { - LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle)); - } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->tablet_addr_)) { - } else if (OB_FAIL(safe_update_cas_tablet(key, disk_addr, tablet_handle, new_tablet_handle, time_guard))) { - LOG_WARN("fail to update tablet", K(ret), K(key), K(disk_addr)); - } else { - LOG_INFO("succeeded to update tablet meta status", K(ret), K(key), K(disk_addr), K(expected_status), KPC(tablet)); - } - - if (OB_FAIL(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = tablet->tablet_meta_.ha_status_.set_expected_status(current_status))) { - LOG_WARN("failed to set expected status", K(tmp_ret), K(current_status), KPC(tablet)); - ob_abort(); - } - } - } - } - } - return ret; -} int ObLSTabletService::replay_create_inner_tablet( common::ObArenaAllocator &allocator, @@ -1798,7 +1339,7 @@ int ObLSTabletService::replay_create_inner_tablet( LOG_WARN("fail to read tablet buf from disk", K(ret), K(disk_addr)); } else if (OB_FAIL(tablet->deserialize_for_replay(allocator, buf, buf_len, pos))) { LOG_WARN("fail to deserialize tablet", K(ret), KP(buf), K(buf_len)); - } else if (OB_FAIL(tablet->init_shared_params(key.ls_id_, key.tablet_id_, tablet->get_tablet_meta().compat_mode_))) { + } else if (OB_FAIL(tablet->init_shared_params(key.tablet_id_, tablet->get_tablet_meta().compat_mode_))) { LOG_WARN("fail to init shared params", K(ret), K(key)); } return ret; @@ -1814,22 +1355,20 @@ int ObLSTabletService::replay_create_tablet( int ret = OB_SUCCESS; bool b_exist = false; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObTransService *tx_svr = share::g_mp->trans_service(); ObFreezer *freezer = ls_->get_freezer(); - const ObLSID &ls_id = ls_->get_ls_id(); common::ObArenaAllocator allocator(common::ObMemAttr("ReplayCreate")); ObTabletHandle tablet_hdl; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(has_tablet(ls_id, tablet_id, b_exist))) { - LOG_WARN("fail to check tablet existence", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(has_tablet(tablet_id, b_exist))) { + LOG_WARN("fail to check tablet existence", K(ret), K(tablet_id)); } else if (b_exist) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("restart replay tablet should not exist", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("restart replay tablet should not exist", K(ret), K(tablet_id)); } else { ObTimeGuard time_guard("ObLSTabletService::replay_create_tablet", 1_s); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTablet *tablet = nullptr; int64_t pos = 0; ObMetaDiskAddr old_addr; @@ -1837,7 +1376,7 @@ int ObLSTabletService::replay_create_tablet( int64_t try_cache_size = 0; ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); time_guard.click("Lock"); - if (OB_FAIL(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, tablet_hdl))) { + if (OB_FAIL(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, *ls_, tablet_hdl))) { LOG_WARN("fail to create temporary tablet", K(ret), K(key)); } else if (FALSE_IT(tablet = tablet_hdl.get_obj())) { } else if (FALSE_IT(tablet->tablet_addr_ = disk_addr)) { @@ -1846,10 +1385,9 @@ int ObLSTabletService::replay_create_tablet( } else if (OB_FAIL(tablet->deserialize_for_replay(allocator, buf, buf_len, pos))) { LOG_WARN("fail to deserialize tablet", K(ret), KP(buf), K(buf_len), K(pos)); } else if (FALSE_IT(time_guard.click("Deserialize"))) { - } else if (OB_FAIL(tablet->init_shared_params(ls_id, - tablet_id, + } else if (OB_FAIL(tablet->init_shared_params(tablet_id, tablet->get_tablet_meta().compat_mode_))) { - LOG_WARN("failed to init shared params", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to init shared params", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_id_set_.set(tablet_id))) { LOG_WARN("fail to set tablet id set", K(ret), K(tablet_id)); } else { @@ -1880,22 +1418,20 @@ int ObLSTabletService::replay_create_tablet( LOG_WARN("start ddl if need failed", K(ret)); } else if (OB_FAIL(tablet->inc_macro_ref_cnt())) { LOG_WARN("fail to increase macro blocks' ref cnt for meta and data", K(ret)); - } else if (OB_FAIL(tx_svr->create_tablet(key.tablet_id_, key.ls_id_))) { - LOG_WARN("fail to create tablet cache", K(ret), K(key), K(tablet_hdl)); } if (OB_SUCC(ret)) { } else { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(tablet_id)); } } } if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; ObServerAutoSplitScheduler &auto_split_scheduler= ObServerAutoSplitScheduler::get_instance(); - if (OB_TMP_FAIL(auto_split_scheduler.push_task(tablet_hdl, *ls_))) { + if (OB_TMP_FAIL(auto_split_scheduler.push_task(tablet_hdl))) { LOG_WARN("fail to push auto split task", K(tmp_ret)); } } @@ -1910,7 +1446,7 @@ int ObLSTabletService::get_tablet_with_timeout( const share::SCN &snapshot) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); const int64_t timeout_step_us = 10_s; const int64_t snapshot_version = snapshot.get_val_for_tx(); @@ -1933,61 +1469,13 @@ int ObLSTabletService::get_tablet_with_timeout( return ret; } -int ObLSTabletService::get_ls_migration_required_size(int64_t &required_size) -{ - int ret = OB_SUCCESS; - int64_t tmp_size = 0; - required_size = 0; - const ObLSID &ls_id = ls_->get_ls_id(); - ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObTabletMapKey key; - ObSArray tablet_ids; - GetAllTabletIDOperator op(tablet_ids); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_ISNULL(t3m)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("t3m should not be nullptr", K(ret), KP(t3m)); - } else if (OB_FAIL(tablet_id_set_.foreach(op))) { - LOG_WARN("fail to get all tablet ids from set", K(ret)); - } else { - key.ls_id_ = ls_id; - ObTabletResidentInfo info; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - key.tablet_id_ = tablet_ids.at(i); - if (OB_FAIL(t3m->get_tablet_resident_info(key, info))) { - if (OB_ENTRY_NOT_EXIST == ret) { - // do nothing (expected: add no lock when fetching tablets from tablet_id_set_) - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get tablet required_size", K(ret), K(key), K(tmp_size)); - } - } else if (!info.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("in_valid resident_info", K(ret), K(key), K(info)); - } else { - required_size += info.get_tablet_meta_size() + // meta_size - info.get_required_size() - // data_size - info.get_ss_public_sstable_occupy_size(); // shared_data size - } - } // end for - } - - if (OB_FAIL(ret)) { - required_size = 0; - } - return ret; -} - int ObLSTabletService::direct_get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &handle) { #ifdef ENABLE_DEBUG_LOG ObTimeGuard tg("direct_get_tablet", 10_ms); #endif int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, handle))) { if (OB_TABLET_NOT_EXIST != ret) { @@ -2053,13 +1541,12 @@ int ObLSTabletService::inner_table_scan( } int ObLSTabletService::has_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &b_exist) { int ret = OB_SUCCESS; b_exist = false; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); if (OB_FAIL(t3m->has_tablet(key, b_exist))) { @@ -2070,7 +1557,6 @@ int ObLSTabletService::has_tablet( } int ObLSTabletService::create_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -2092,8 +1578,7 @@ int ObLSTabletService::create_tablet( common::ObArenaAllocator tmp_allocator(common::ObMemAttr("CreateTab")); common::ObArenaAllocator *allocator = nullptr; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObTransService *tx_svr = share::g_mp->trans_service(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTablet *tablet = nullptr; ObFreezer *freezer = ls_->get_freezer(); tablet_handle.reset(); @@ -2110,29 +1595,27 @@ int ObLSTabletService::create_tablet( || OB_ISNULL(allocator = tablet_handle.get_allocator())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("new tablet is null", K(ret), KP(tablet), KP(allocator), K(tablet_handle)); - } else if (OB_FAIL(tablet->init_for_first_time_creation(*allocator, ls_id, tablet_id, data_tablet_id, + } else if (OB_FAIL(tablet->init_for_first_time_creation(*allocator, tablet_id, data_tablet_id, create_scn, snapshot_version, create_tablet_schema, need_create_empty_major_sstable, clog_checkpoint_scn, mds_checkpoint_scn, is_split_dest_tablet, split_src_tablet_id, micro_index_clustered, freezer, fork_info))) { - LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), + LOG_WARN("failed to init tablet", K(ret), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema)); } else if (OB_FAIL(tablet->get_updating_tablet_pointer_param(param))) { LOG_WARN("fail to get updating tablet pointer parameters", K(ret), KPC(tablet)); } else if (OB_FAIL(t3m->compare_and_swap_tablet(key, tablet_handle, tablet_handle, param))) { LOG_WARN("failed to compare and swap tablet", K(ret), K(key), K(tablet_handle), K(param)); } else if (OB_FAIL(tablet_id_set_.set(tablet_id))) { - LOG_WARN("fail to insert tablet id", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(tx_svr->create_tablet(key.tablet_id_, key.ls_id_))) { - LOG_WARN("fail to create tablet cache", K(ret), K(key), K(tablet_handle)); + LOG_WARN("fail to insert tablet id", K(ret), K(tablet_id)); } else { report_tablet_to_rs(tablet_id); } if (OB_SUCC(ret)) { - LOG_INFO("succeed to create tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("succeed to create tablet", K(ret), K(tablet_id)); } else { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(tablet_id)); } } } @@ -2141,7 +1624,6 @@ int ObLSTabletService::create_tablet( } int ObLSTabletService::create_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -2155,164 +1637,79 @@ int ObLSTabletService::create_inner_tablet( bool need_create_empty_major_old_version = true; common::ObArenaAllocator allocator(common::ObMemAttr("LSCreateTab")); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObTransService *tx_svr = share::g_mp->trans_service(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTablet *tmp_tablet = nullptr; ObFreezer *freezer = ls_->get_freezer(); ObTabletHandle tmp_tablet_hdl; ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); ObTimeGuard time_guard("ObLSTabletService::create_inner_tablet", 10_ms); const share::SCN clog_checkpoint_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; const share::SCN mds_checkpoint_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - if (OB_FAIL(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, tmp_tablet_hdl))) { + if (OB_FAIL(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, *ls_, tmp_tablet_hdl))) { LOG_WARN("failed to create temporary tablet", K(ret), K(key)); } else if (OB_ISNULL(tmp_tablet = tmp_tablet_hdl.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("new tablet is null", K(ret), KPC(tmp_tablet), K(tmp_tablet_hdl)); } else if (FALSE_IT(time_guard.click("CreateTablet"))) { - } else if (OB_FAIL(tmp_tablet->init_for_first_time_creation(allocator, ls_id, tablet_id, data_tablet_id, + } else if (OB_FAIL(tmp_tablet->init_for_first_time_creation(allocator, tablet_id, data_tablet_id, create_scn, snapshot_version, create_tablet_schema, true/*need_create_empty_major_sstable*/, clog_checkpoint_scn, mds_checkpoint_scn, false/*is_split_dest_tablet*/, ObTabletID()/*split_src_tablet_id*/, false/*micro_index_clustered*/, freezer))) { - LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), + LOG_WARN("failed to init tablet", K(ret), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema)); } else if (FALSE_IT(time_guard.click("InitTablet"))) { } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tmp_tablet, tablet_handle))) { LOG_WARN("fail to persist and transform tablet", K(ret), K(tmp_tablet_hdl), K(tablet_handle)); } else if (FALSE_IT(time_guard.click("Persist"))) { } else if (FALSE_IT(disk_addr = tablet_handle.get_obj()->get_tablet_addr())) { - } else if (OB_FAIL(safe_create_cas_tablet(ls_id, tablet_id, disk_addr, tablet_handle, time_guard))) { - LOG_WARN("fail to refresh tablet", K(ret), K(ls_id), K(tablet_id), K(disk_addr), K(tablet_handle)); - } else if (OB_FAIL(tx_svr->create_tablet(key.tablet_id_, key.ls_id_))) { - LOG_WARN("fail to create tablet cache", K(ret), K(key), K(tablet_handle)); + } else if (OB_FAIL(safe_create_cas_tablet(tablet_id, disk_addr, tablet_handle, time_guard))) { + LOG_WARN("fail to refresh tablet", K(ret), K(tablet_id), K(disk_addr), K(tablet_handle)); } if (OB_SUCC(ret)) { LOG_INFO("create ls inner tablet success", K(ret), K(key), K(disk_addr)); } else { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); - } - } - return ret; -} - -int ObLSTabletService::create_empty_shell_tablet( - const ObMigrationTabletParam ¶m, - ObTabletHandle &tablet_handle) -{ - int ret = OB_SUCCESS; - ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - const share::ObLSID &ls_id = param.ls_id_; - const common::ObTabletID &tablet_id = param.tablet_id_; - const ObTabletMapKey key(ls_id, tablet_id); - ObTabletHandle old_tablet_hdl; - ObTimeGuard time_guard("ObLSTabletService::create_empty_shell_tablet", 10_ms); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ls tablet svr hasn't been inited", K(ret)); - } else if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(key)); - } else if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) { - if (OB_TABLET_NOT_EXIST == ret) { - ret = OB_SUCCESS; - LOG_DEBUG("tablet not exist", K(ret), K(tablet_id)); - } else { - LOG_WARN("fail to get tablet", K(ret), K(tablet_id)); - } - } else if (OB_FAIL(remove_tablet(old_tablet_hdl))) { - LOG_WARN("failed to remove tablet", K(ret), K(key)); - } else { - time_guard.click("RemoveOld"); - } - - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); // must lock after prepare - common::ObArenaAllocator allocator(common::ObMemAttr("MigEmptyT")); - ObTabletHandle tmp_tablet_hdl; - if (FAILEDx(ObTabletCreateDeleteHelper::create_tmp_tablet(key, allocator, tmp_tablet_hdl))) { - LOG_WARN("fail to create temporary tablet", K(ret), K(key)); - } else { - time_guard.click("CreateTablet"); - ObFreezer *freezer = ls_->get_freezer(); - ObTablet *tmp_tablet = tmp_tablet_hdl.get_obj(); - ObTabletHandle tablet_handle; - ObTablet *new_tablet = nullptr; - ObMetaDiskAddr disk_addr; - const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id); - if (OB_FAIL(tmp_tablet->init_with_migrate_param(allocator, param, false/*is_update*/, freezer))) { - LOG_WARN("failed to init tablet", K(ret), K(param)); - } else if (FALSE_IT(time_guard.click("InitTablet"))) { - #ifdef ERRSIM - } else if (OB_SUCCESS != EN_CREATE_EMPTY_SHELL_TABLET_ERROR) { - ret = EN_CREATE_EMPTY_SHELL_TABLET_ERROR; - LOG_WARN("[ERRSIM] fake create empty shell tablet error", K(ret), K(param)); - #endif - } else if (OB_FAIL(ObTabletPersister::transform_empty_shell(persist_param, *tmp_tablet, tablet_handle))) { - LOG_WARN("failed to transform empty shell", K(ret), KPC(tmp_tablet)); - } else if (FALSE_IT(time_guard.click("Transform"))) { - } else { - if (OB_FAIL(safe_create_cas_empty_shell(ls_id, tablet_id, tablet_handle, time_guard))) { - LOG_WARN("fail to refresh empty shell", K(ret), K(ls_id), K(tablet_id), K(tablet_handle)); - } - } - if (OB_SUCC(ret)) { - ls_->get_tablet_gc_handler()->set_tablet_gc_trigger(); - LOG_INFO("succeeded to create empty shell tablet", K(ret), K(key), K(param)); - } else { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); - } + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(tablet_id)); } } - return ret; } -int ObLSTabletService::rollback_remove_tablet( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +int ObLSTabletService::rollback_remove_tablet(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - bool is_same = true; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ls tablet service do not init", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid args", K(ret), K(tablet_id)); } else { ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - if (OB_FAIL(check_rollback_tablet_is_same_(ls_id, tablet_id, is_same))) { - LOG_WARN("failed to check rollback tablet is same", K(ret), K(ls_id), K(tablet_id)); - } else if (!is_same) { - //do nothing - } else if (OB_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { - LOG_WARN("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(rollback_remove_tablet_without_lock(tablet_id))) { + LOG_WARN("fail to rollback remove tablet", K(ret), K(tablet_id)); } } return ret; } -int ObLSTabletService::rollback_remove_tablet_without_lock( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +int ObLSTabletService::rollback_remove_tablet_without_lock(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); if (OB_FAIL(tablet_id_set_.erase(tablet_id))) { if (OB_HASH_NOT_EXIST == ret) { // tablet id is already erased ret = OB_SUCCESS; - LOG_DEBUG("tablet id does not exist, maybe has not been inserted yet", K(ret), K(ls_id), K(tablet_id)); + LOG_DEBUG("tablet id does not exist, maybe has not been inserted yet", K(ret), K(tablet_id)); } else { - LOG_WARN("fail to erase tablet id from set", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to erase tablet id from set", K(ret), K(tablet_id)); } } @@ -2320,7 +1717,7 @@ int ObLSTabletService::rollback_remove_tablet_without_lock( // loop retry to delete tablet from t3m while (OB_FAIL(t3m->del_tablet(key))) { if (REACH_TIME_INTERVAL(10_s)) { - LOG_ERROR("failed to delete tablet from t3m", K(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("failed to delete tablet from t3m", K(ret), K(tablet_id)); } } } @@ -2333,7 +1730,7 @@ int ObLSTabletService::create_memtable(const common::ObTabletID &tablet_id, Crea int ret = OB_SUCCESS; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletHandle old_tablet_handle; - const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle new_tablet_handle; ObTimeGuard time_guard("ObLSTabletService::create_memtable", 10_ms); @@ -2361,7 +1758,7 @@ int ObLSTabletService::create_memtable(const common::ObTabletID &tablet_id, Crea share::SCN trans_version; ObTablet &old_tablet = *(old_tablet_handle.get_obj()); bool is_committed = false; - // forbid creating a new memtable while rollback is pending + // Do not create a new memtable while tablet status is being changed. if (arg.for_replay_) { } else if (OB_FAIL(old_tablet.ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { LOG_WARN("fail to get latest tablet status", K(ret)); @@ -2391,21 +1788,6 @@ int ObLSTabletService::create_memtable(const common::ObTabletID &tablet_id, Crea return ret; } -int ObLSTabletService::check_allow_to_read() -{ - int ret = OB_SUCCESS; - bool allow_to_read = false; - allow_to_read_mgr_.load_allow_to_read_info(allow_to_read); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (!allow_to_read) { - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls is not allow to read", K(ret), KPC(ls_)); - } - return ret; -} - // ATTENTION! // here we pass VALUE rather than REF for tablet id, // because tablet id may be from iter, which will be reset in function, @@ -2445,7 +1827,6 @@ int ObLSTabletService::inner_get_read_tables( int ret = OB_SUCCESS; ObTabletHandle &handle = iter.tablet_handle_; iter.reset(); - bool allow_to_read = false; ObTabletMapKey key; key.tablet_id_ = tablet_id; @@ -2458,11 +1839,6 @@ int ObLSTabletService::inner_get_read_tables( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(snapshot_version_for_tablet), K(snapshot_version_for_tables)); - } else if (FALSE_IT(allow_to_read_mgr_.load_allow_to_read_info(allow_to_read))) { - } else if (!allow_to_read) { - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls is not allow to read", K(ret), KPC(ls_), K(lbt())); - } else if (FALSE_IT(key.ls_id_ = ls_->get_ls_id())) { } else if (OB_FAIL(ObTabletCreateDeleteHelper::check_and_get_tablet(key, handle, timeout_us, mode, @@ -2503,7 +1879,6 @@ int ObLSTabletService::inner_get_read_tables_for_split_src( } else if (OB_UNLIKELY(!tablet_id.is_valid() || snapshot_version < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(snapshot_version)); - } else if (FALSE_IT(key.ls_id_ = ls_->get_ls_id())) { } else if (OB_FAIL(ObTabletCreateDeleteHelper::check_and_get_tablet(key, handle, timeout_us, ObMDSGetTabletMode::READ_ALL_COMMITED, @@ -2549,9 +1924,9 @@ int ObLSTabletService::set_tablet_status( LOG_WARN("fail to get tablet", K(ret)); } } else if (OB_FAIL(tablet_handle.get_obj()->set_tablet_status(tablet_status, ctx))) { - LOG_WARN("fail to set tablet status", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(tablet_status)); + LOG_WARN("fail to set tablet status", K(ret), K(tablet_id), K(tablet_status)); } else { - LOG_INFO("succeeded to set tablet status", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(tablet_status)); + LOG_INFO("succeeded to set tablet status", K(ret), K(tablet_id), K(tablet_status)); } } return ret; @@ -2583,7 +1958,7 @@ int ObLSTabletService::replay_set_tablet_status( } else if (OB_FAIL(tablet_handle.get_obj()->replay_set_tablet_status(scn, tablet_status, ctx))) { LOG_WARN("fail to replay set tablet status", K(ret), K(tablet_id), K(scn), K(tablet_status)); } else { - LOG_INFO("succeeded to replay set tablet status", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(scn), K(tablet_status)); + LOG_INFO("succeeded to replay set tablet status", K(ret), K(tablet_id), K(scn), K(tablet_status)); } } return ret; @@ -2613,7 +1988,7 @@ int ObLSTabletService::set_ddl_complete( } else if (OB_FAIL(tablet_handle.get_obj()->set_ddl_complete(key, ddl_complete, ctx, timeout_us))) { LOG_WARN("fail to set ddl info", K(ret), K(key), K(tablet_id), K(ddl_complete), K(timeout_us)); } else { - LOG_INFO("succeeded to set ddl info", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(key), K(ddl_complete), K(timeout_us)); + LOG_INFO("succeeded to set ddl info", K(ret), K(tablet_id), K(key), K(ddl_complete), K(timeout_us)); } } return ret; @@ -2645,7 +2020,7 @@ int ObLSTabletService::set_ddl_info( } else if (OB_FAIL(tablet_handle.get_obj()->set_ddl_info(ddl_data, ctx, timeout_us))) { LOG_WARN("fail to set ddl info", K(ret), K(tablet_id), K(ddl_data), K(timeout_us)); } else { - LOG_INFO("succeeded to set ddl info", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(ddl_data), K(timeout_us)); + LOG_INFO("succeeded to set ddl info", K(ret), K(tablet_id), K(ddl_data), K(timeout_us)); } } return ret; @@ -2677,7 +2052,7 @@ int ObLSTabletService::replay_set_ddl_info( } else if (OB_FAIL(tablet_handle.get_obj()->replay_set_ddl_info(scn, ddl_data, ctx))) { LOG_WARN("fail to set ddl info", K(ret), K(tablet_id), K(ddl_data), K(scn)); } else { - LOG_INFO("succeeded to set ddl info", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(ddl_data), K(scn)); + LOG_INFO("succeeded to set ddl info", K(ret), K(tablet_id), K(ddl_data), K(scn)); } } return ret; @@ -2710,7 +2085,7 @@ int ObLSTabletService::replay_set_ddl_complete( } else if (OB_FAIL(tablet_handle.get_obj()->replay_set_ddl_complete(scn, key, ddl_data, ctx))) { LOG_WARN("fail to replay set ddl info", K(ret), K(tablet_id), K(key), K(ddl_data), K(scn)); } else { - LOG_INFO("succeeded to replay set ddl info", K(ret), "ls_id", ls_->get_ls_id(), K(tablet_id), K(key), K(ddl_data), K(scn)); + LOG_INFO("succeeded to replay set ddl info", K(ret), K(tablet_id), K(key), K(ddl_data), K(scn)); } } return ret; @@ -3099,9 +2474,7 @@ int ObLSTabletService::update_rows( const bool delay_new = false; const ObColDescIArray &col_descs = *(run_ctx.col_descs_); - // for major mv base table need update full column - if (dml_param.table_param_->get_data_table().get_mv_mode().table_referenced_by_fast_lsm_mv_flag_ || - run_ctx.is_delete_insert_table_) { + if (run_ctx.is_delete_insert_table_) { ctx.update_full_column_ = true; } @@ -3615,186 +2988,63 @@ int ObLSTabletService::lock_rows( ++afct_num; } timeguard.click("Lock"); - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - affected_rows = afct_num; - } - } - } -} - NG_TRACE(S_lock_rows_end); - return ret; -} - -int ObLSTabletService::lock_row( - ObTabletHandle &tablet_handle, - ObStoreCtx &ctx, - const ObDMLBaseParam &dml_param, - blocksstable::ObDatumRow &row, - const ObLockFlag lock_flag, - const bool is_sfu) -{ - UNUSEDx(lock_flag, is_sfu); - int ret = OB_SUCCESS; - const ObTabletID &data_tablet_id = ctx.tablet_id_; - ObTimeGuard timeguard(__func__, 3_s); - int64_t afct_num = 0; - ObColDescArray col_desc; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("tablet service is not initialized", K(ret)); - } else if (OB_UNLIKELY(!ctx.is_valid() || !dml_param.is_valid() || !row.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ctx), K(dml_param), K(row)); - } else { - HEAP_VAR(ObDMLRunningCtx, run_ctx, ctx, - dml_param, - ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), - ObDmlFlag::DF_LOCK) { - if (OB_FAIL(prepare_dml_running_ctx(nullptr, nullptr, tablet_handle, run_ctx))) { - LOG_WARN("failed to prepare dml running ctx", K(ret)); - } else if (OB_FAIL(run_ctx.relative_table_.get_rowkey_column_ids(col_desc))) { - LOG_WARN("Fail to get column desc", K(ret)); - } else { - if (ObTimeUtility::current_time() > dml_param.timeout_) { - ret = OB_TIMEOUT; - int64_t cur_time = ObClockGenerator::getClock(); - LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); - } else if (OB_FAIL(lock_row_wrap(tablet_handle, dml_param.data_row_for_lob_, run_ctx.relative_table_, ctx, col_desc, row))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to lock row", K(row), K(ret)); - } - } else { - ++afct_num; - } - } - } -} - - return ret; -} - -int ObLSTabletService::trim_rebuild_tablet( - const ObTabletID &tablet_id, - const bool is_rollback) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(is_stopped_)) { - ret = OB_NOT_RUNNING; - LOG_WARN("tablet service stopped", K(ret)); - } else if (OB_UNLIKELY(!tablet_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tablet_id)); - } else if (is_rollback && OB_FAIL(rollback_rebuild_tablet(tablet_id))) { - LOG_ERROR("failed to rollback tablet rebuilt", K(ret), K(is_rollback), K(tablet_id)); - } else if (!is_rollback && OB_FAIL(trim_old_tablets(tablet_id))) { - LOG_WARN("failed to trim old tablets", K(ret), K(is_rollback), K(tablet_id)); - } - - return ret; -} - -int ObLSTabletService::create_or_update_migration_tablet( - const ObMigrationTabletParam &mig_tablet_param) -{ - int ret = OB_SUCCESS; - const share::ObLSID &ls_id = mig_tablet_param.ls_id_; - const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_; - ObTabletHandle tablet_handle; - bool b_exist = false; - const bool need_create_msd_tablet = mig_tablet_param.is_empty_shell(); - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(is_stopped_)) { - ret = OB_NOT_RUNNING; - LOG_WARN("tablet service stopped", K(ret)); - } else if (!mig_tablet_param.is_empty_shell() - && (OB_UNLIKELY(!mig_tablet_param.is_valid()) - || OB_UNLIKELY(ls_id != ls_->get_ls_id()))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(mig_tablet_param), K_(ls)); - } else if (need_create_msd_tablet) { - if (OB_FAIL(create_empty_shell_tablet(mig_tablet_param, tablet_handle))) { - LOG_WARN("failed to create empty shell tablet", K(ret), K(tablet_id), K(mig_tablet_param)); - } - } else { - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - if (OB_FAIL(has_tablet(ls_id, tablet_id, b_exist))) { - LOG_WARN("failed to check tablet existence", K(ls_id), K(tablet_id)); - } else if (b_exist - && OB_FAIL(migrate_update_tablet(mig_tablet_param))) { - LOG_ERROR("failed to update tablet meta", K(ret), K(tablet_id), K(mig_tablet_param)); - } else if (!b_exist - && OB_FAIL(migrate_create_tablet(mig_tablet_param, tablet_handle))) { - LOG_ERROR("failed to migrate create tablet", K(ret), K(mig_tablet_param)); + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + affected_rows = afct_num; + } } - } - + } +} + NG_TRACE(S_lock_rows_end); return ret; } -int ObLSTabletService::rebuild_create_tablet( - const ObMigrationTabletParam &mig_tablet_param, - const bool keep_old) +int ObLSTabletService::lock_row( + ObTabletHandle &tablet_handle, + ObStoreCtx &ctx, + const ObDMLBaseParam &dml_param, + blocksstable::ObDatumRow &row, + const ObLockFlag lock_flag, + const bool is_sfu) { + UNUSEDx(lock_flag, is_sfu); int ret = OB_SUCCESS; - const share::ObLSID &ls_id = mig_tablet_param.ls_id_; - const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_; - const ObTabletMapKey key(ls_id, tablet_id); - bool b_exist = false; - const bool need_create_msd_tablet = mig_tablet_param.is_empty_shell() && !keep_old; - ObTabletHandle new_tablet_handle; - ObTabletHandle old_tablet_handle; + const ObTabletID &data_tablet_id = ctx.tablet_id_; + ObTimeGuard timeguard(__func__, 3_s); + int64_t afct_num = 0; + ObColDescArray col_desc; - if (OB_UNLIKELY(!is_inited_)) { + if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(is_stopped_)) { - ret = OB_NOT_RUNNING; - LOG_WARN("tablet service stopped", K(ret)); - } else if (OB_UNLIKELY(!mig_tablet_param.is_valid()) - || OB_UNLIKELY(ls_id != ls_->get_ls_id())) { + LOG_WARN("tablet service is not initialized", K(ret)); + } else if (OB_UNLIKELY(!ctx.is_valid() || !dml_param.is_valid() || !row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(mig_tablet_param), K_(ls)); - } else if (OB_UNLIKELY(mig_tablet_param.is_empty_shell() && keep_old)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, rebuild create an empty shell tablet, but still need the old tablet", K(ret), - K(mig_tablet_param), K(keep_old)); - } else if (need_create_msd_tablet) { - if (OB_FAIL(create_empty_shell_tablet(mig_tablet_param, new_tablet_handle))) { - LOG_WARN("failed to create empty shell tablet", K(ret), K(tablet_id), K(mig_tablet_param)); - } + LOG_WARN("invalid argument", K(ret), K(ctx), K(dml_param), K(row)); } else { - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - if (OB_FAIL(has_tablet(ls_id, tablet_id, b_exist))) { - LOG_WARN("fail to check tablet existence", K(ls_id), K(tablet_id)); - } else if (!b_exist && - OB_FAIL(migrate_create_tablet(mig_tablet_param, new_tablet_handle))) { - LOG_ERROR("failed to rebuild create tablet", K(ret), K(tablet_id), K(mig_tablet_param)); - } else if (b_exist && !keep_old && - OB_FAIL(migrate_update_tablet(mig_tablet_param))) { - LOG_ERROR("failed to rebuild create tablet", K(ret), K(tablet_id), K(mig_tablet_param)); - } else if (b_exist && keep_old) { - if (OB_FAIL(ObTabletCreateDeleteHelper::check_and_get_tablet(key, old_tablet_handle, - ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, - ObMDSGetTabletMode::READ_WITHOUT_CHECK, - ObTransVersion::MAX_TRANS_VERSION))) { - LOG_WARN("failed to check and get tablet", K(ret), K(key)); - } else if (OB_UNLIKELY(old_tablet_handle.get_obj()->get_tablet_meta().has_next_tablet_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("number of nodes on list exceeds 2", K(ret)); - } else if (OB_FAIL(rebuild_tablet_with_old(mig_tablet_param, old_tablet_handle))) { - LOG_WARN("failed to rebuild tablet and maintain linked list", K(ret), K(tablet_id)); + HEAP_VAR(ObDMLRunningCtx, run_ctx, ctx, + dml_param, + ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), + ObDmlFlag::DF_LOCK) { + if (OB_FAIL(prepare_dml_running_ctx(nullptr, nullptr, tablet_handle, run_ctx))) { + LOG_WARN("failed to prepare dml running ctx", K(ret)); + } else if (OB_FAIL(run_ctx.relative_table_.get_rowkey_column_ids(col_desc))) { + LOG_WARN("Fail to get column desc", K(ret)); + } else { + if (ObTimeUtility::current_time() > dml_param.timeout_) { + ret = OB_TIMEOUT; + int64_t cur_time = ObClockGenerator::getClock(); + LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); + } else if (OB_FAIL(lock_row_wrap(tablet_handle, dml_param.data_row_for_lob_, run_ctx.relative_table_, ctx, col_desc, row))) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret) { + LOG_WARN("failed to lock row", K(row), K(ret)); + } + } else { + ++afct_num; } } - } + } +} return ret; } @@ -3856,9 +3106,8 @@ int ObLSTabletService::build_tablet_with_batch_tables( ObTablet *old_tablet = old_tablet_handle.get_obj(); ObTablet *tmp_tablet = nullptr; - const share::ObLSID &ls_id = ls_->get_ls_id(); - const ObTabletMapKey key(ls_id, tablet_id); - const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id); + const ObTabletMapKey key(tablet_id); + const ObTabletPersisterParam persist_param(ls_->get_ls_epoch(), tablet_id); if (OB_FAIL(ret)) { } else if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_handle))) { @@ -3891,15 +3140,12 @@ int ObLSTabletService::safe_update_cas_tablet( { int ret = OB_SUCCESS; ObUpdateTabletPointerParam param; - int64_t ls_epoch = 0; ObTenantCheckpointSlogHandler::ObCkptSlogROptLockGuard guard(share::g_mp->tenant_storage_meta_service()->get_ckpt_slog_hdl()); if (OB_FAIL(guard.get_ret())) { LOG_WARN("fail to get ckpt_slog_lock", K(ret)); - } else if (OB_FAIL(new_handle.get_obj()->get_ls_epoch(ls_epoch))) { - LOG_WARN("fail to get ls epoch", K(ret), K(key), K(new_handle)); } else if (OB_FAIL(new_handle.get_obj()->get_updating_tablet_pointer_param(param))) { LOG_WARN("fail to get updating tablet pointer parameters", K(ret), K(new_handle)); - } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.update_tablet(key.ls_id_, ls_epoch, key.tablet_id_, addr))) { + } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.update_tablet(key.tablet_id_, addr))) { LOG_WARN("fail to write update tablet slog", K(ret), K(key), K(addr)); } else if (FALSE_IT(time_guard.click("WrSlog"))) { } else if (OB_FAIL(share::g_mp->tenant_meta_mem_mgr()->compare_and_swap_tablet(key, old_handle, new_handle, param))) { @@ -3943,7 +3189,6 @@ int ObLSTabletService::safe_update_cas_empty_shell( } int ObLSTabletService::safe_create_cas_tablet( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObMetaDiskAddr &addr, ObTabletHandle &tablet_handle, @@ -3951,19 +3196,16 @@ int ObLSTabletService::safe_create_cas_tablet( { int ret = OB_SUCCESS; ObUpdateTabletPointerParam param; - int64_t ls_epoch = 0; ObTenantCheckpointSlogHandler::ObCkptSlogROptLockGuard guard(share::g_mp->tenant_storage_meta_service()->get_ckpt_slog_hdl()); if (OB_FAIL(guard.get_ret())) { LOG_WARN("fail to get ckpt_slog_lock", K(ret)); - } else if (OB_FAIL(tablet_handle.get_obj()->get_ls_epoch(ls_epoch))) { - LOG_WARN("fail to get ls epoch", K(ret), K(ls_id), K(tablet_id), K(addr)); } else if (OB_FAIL(tablet_handle.get_obj()->get_updating_tablet_pointer_param(param))) { LOG_WARN("fail to get updating tablet pointer parameters", K(ret), K(tablet_handle)); - } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.update_tablet(ls_id, ls_epoch, tablet_id, addr))) { - LOG_WARN("fail to write update tablet slog", K(ret), K(ls_id), K(tablet_id), K(addr)); + } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.update_tablet(tablet_id, addr))) { + LOG_WARN("fail to write update tablet slog", K(ret), K(tablet_id), K(addr)); } else if (FALSE_IT(time_guard.click("WrSlog"))) { - } else if (OB_FAIL(refresh_tablet_addr(ls_id, tablet_id, param, tablet_handle))) { - LOG_ERROR("failed to refresh tablet addr", K(ret), K(ls_id), K(tablet_id), K(param), K(lbt())); + } else if (OB_FAIL(refresh_tablet_addr(tablet_id, param, tablet_handle))) { + LOG_ERROR("failed to refresh tablet addr", K(ret), K(tablet_id), K(param), K(lbt())); ob_usleep(1_s); ob_abort(); } else { @@ -3972,36 +3214,6 @@ int ObLSTabletService::safe_create_cas_tablet( return ret; } -int ObLSTabletService::safe_create_cas_empty_shell( - const ObLSID &ls_id, - const ObTabletID &tablet_id, - ObTabletHandle &tablet_handle, - ObTimeGuard &time_guard) -{ - int ret = OB_SUCCESS; - ObTablet *tablet = tablet_handle.get_obj(); - ObUpdateTabletPointerParam param; - ObMetaDiskAddr addr; - ObTenantCheckpointSlogHandler::ObCkptSlogROptLockGuard guard(share::g_mp->tenant_storage_meta_service()->get_ckpt_slog_hdl()); - if (OB_FAIL(guard.get_ret())) { - LOG_WARN("fail to get ckpt_slog_lock", K(ret)); - } else if (OB_FAIL(tablet_handle.get_obj()->get_updating_tablet_pointer_param(param))) { - LOG_WARN("fail to get updating tablet pointer parameters", K(ret), K(tablet_handle)); - } else if (OB_FAIL(TENANT_STORAGE_META_PERSISTER.write_empty_shell_tablet(tablet, addr))) { - LOG_WARN("fail to write emtpy shell tablet", K(ret), K(tablet_id), K(addr)); - } else if (FALSE_IT(tablet->tablet_addr_ = addr)) { - } else if (FALSE_IT(param.tablet_addr_ = addr)) { - } else if (FALSE_IT(time_guard.click("WrSlog"))) { - } else if (OB_FAIL(refresh_tablet_addr(ls_id, tablet_id, param, tablet_handle))) { - LOG_ERROR("failed to refresh tablet addr", K(ret), K(ls_id), K(tablet_id), K(param), K(lbt())); - ob_usleep(1_s); - ob_abort(); - } else { - time_guard.click("Refresh"); - } - return ret; -} - int ObLSTabletService::check_old_row_legitimacy( const ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, @@ -4094,18 +3306,7 @@ int ObLSTabletService::check_old_row_legitimacy( if (OB_ERR_DEFENSIVE_CHECK == ret) { int tmp_ret = OB_SUCCESS; bool is_virtual_gen_col = false; - if (OB_TMP_FAIL(check_real_leader_for_4377_(store_ctx.ls_id_))) { - ret = tmp_ret; - LOG_WARN("check real leader for 4377 found exception", K(ret), K(old_row), K(data_table)); - } else if (nullptr != store_ctx.mvcc_acc_ctx_.tx_desc_ - && OB_TMP_FAIL(check_need_rollback_for_4377_(store_ctx.mvcc_acc_ctx_.tx_desc_, - data_tablet_handle))) { - ret = tmp_ret; - LOG_WARN("check need rollback for 4377 found exception", K(ret), K(old_row), K(data_table)); - } else if (OB_TMP_FAIL(check_parts_tx_state_for_4377_(store_ctx.mvcc_acc_ctx_.tx_desc_))) { - ret = tmp_ret; - LOG_WARN("check parts tx state for 4377 found exception", K(ret), K(old_row), K(data_table)); - } else if (is_udf) { + if (is_udf) { ret = OB_ERR_INDEX_KEY_NOT_FOUND; LOG_WARN("index key not found on udf column", K(ret), K(old_row)); } else if (data_table.is_index_table() && OB_TMP_FAIL(check_is_gencol_check_failed(data_table, err_col_id, is_virtual_gen_col))) { @@ -4131,7 +3332,7 @@ int ObLSTabletService::check_old_row_legitimacy( LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!"); - concurrency_control::ObDataValidationService::set_delay_resource_recycle(store_ctx.ls_id_); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(); LOG_ERROR("Dump data table info", K(ret), K(data_table)); store_ctx.force_print_trace_log(); } @@ -4521,8 +3722,7 @@ int ObLSTabletService::insert_vector_index_rows( // delta_buffer table columns def is: ObPluginVectorIndexService *vec_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexAdapterGuard adaptor_guard; - if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.store_ctx_.ls_id_, - run_ctx.relative_table_.get_tablet_id(), + if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.relative_table_.get_tablet_id(), ObIndexType::INDEX_TYPE_VEC_DELTA_BUFFER_LOCAL, adaptor_guard, &vec_idx_param, @@ -4574,8 +3774,7 @@ int ObLSTabletService::insert_vector_index_rows( int64_t vec_dim = run_ctx.dml_param_.table_param_->get_data_table().get_vec_dim(); ObPluginVectorIndexAdapterGuard adaptor_guard; ObPluginVectorIndexService *vec_index_service = share::g_mp->plugin_vector_index_service(); - if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.store_ctx_.ls_id_, - run_ctx.relative_table_.get_tablet_id(), + if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.relative_table_.get_tablet_id(), ObIndexType::INDEX_TYPE_HYBRID_INDEX_LOG_LOCAL, adaptor_guard, &vec_idx_param, @@ -4634,8 +3833,7 @@ int ObLSTabletService::insert_vector_index_rows( // hybrid vec embedded table columns def is: ObPluginVectorIndexService *vec_index_service = share::g_mp->plugin_vector_index_service(); ObPluginVectorIndexAdapterGuard adaptor_guard; - if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.store_ctx_.ls_id_, - run_ctx.relative_table_.get_tablet_id(), + if (OB_FAIL(vec_index_service->acquire_adapter_guard(run_ctx.relative_table_.get_tablet_id(), ObIndexType::INDEX_TYPE_HYBRID_INDEX_EMBEDDED_LOCAL, adaptor_guard, &vec_idx_param, @@ -4698,8 +3896,7 @@ int ObLSTabletService::insert_vector_index_rows( ObPluginVectorIndexAdapterGuard adaptor_guard; ObString vec_idx_param = run_ctx.dml_param_.table_param_->get_data_table().get_vec_index_param(); const int64_t vec_dim = run_ctx.dml_param_.table_param_->get_data_table().get_vec_dim(); - int tmp_ret = vec_index_service->acquire_adapter_guard(run_ctx.store_ctx_.ls_id_, - run_ctx.relative_table_.get_tablet_id(), + int tmp_ret = vec_index_service->acquire_adapter_guard(run_ctx.relative_table_.get_tablet_id(), ObIndexType::INDEX_TYPE_VEC_INDEX_ID_LOCAL, adaptor_guard, &vec_idx_param, @@ -5973,7 +5170,6 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( access_param.iter_param_.table_id_ = data_table.get_table_id(); access_param.iter_param_.tablet_id_ = data_table.tablet_iter_.get_tablet()->get_tablet_meta().tablet_id_; if (nullptr != data_table.tablet_iter_.get_tablet()) { - access_param.iter_param_.ls_id_ = data_table.tablet_iter_.get_tablet()->get_tablet_meta().ls_id_; } access_param.iter_param_.read_info_ = read_info; access_param.iter_param_.out_cols_project_ = &out_col_pros; @@ -6107,7 +5303,6 @@ int ObLSTabletService::get_ls_min_end_scn( SCN &min_end_scn_from_latest_tablets, SCN &min_end_scn_from_old_tablets) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_->get_ls_id(); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObSArray tablet_ids; GetAllTabletIDOperator op(tablet_ids, true/*except_ls_inner_tablet*/); @@ -6121,7 +5316,7 @@ int ObLSTabletService::get_ls_min_end_scn( } else { SCN ls_checkpoint = ls_->get_clog_checkpoint_scn(); for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - ObTabletMapKey key(ls_id, tablet_ids.at(i)); + ObTabletMapKey key(tablet_ids.at(i)); SCN min_end_scn_from_latest = SCN::max_scn(); SCN min_end_scn_from_old = SCN::max_scn(); if (OB_FAIL(t3m->get_min_end_scn_for_ls(key, @@ -6456,7 +5651,7 @@ int ObLSTabletService::estimate_block_count_and_row_count( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(OB_TS_MGR.get_gts(nullptr, max_readable_scn))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts(max_readable_scn))) { LOG_WARN("failed to get gts", K(ret)); } else if (FALSE_IT(snapshot_version_for_tablet = static_cast(max_readable_scn.get_val_for_sql()))) { } else if (FALSE_IT(snapshot_version_for_tables = static_cast(max_readable_scn.get_val_for_sql()))) { @@ -6585,7 +5780,6 @@ int ObLSTabletService::get_mds_table_mgr(mds::MdsTableMgrHandle &mgr_handle) } int ObLSTabletService::create_ls_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const SCN &major_frozen_scn, const ObCreateTabletSchema &create_tablet_schema, @@ -6601,29 +5795,25 @@ int ObLSTabletService::create_ls_inner_tablet( if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!major_frozen_scn.is_valid()) || OB_UNLIKELY(!create_tablet_schema.is_valid()) || OB_UNLIKELY(lib::Worker::CompatMode::INVALID == create_tablet_schema.get_compat_mode()) || OB_UNLIKELY(!create_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(major_frozen_scn), + LOG_WARN("invalid args", K(ret), K(tablet_id), K(major_frozen_scn), K(create_tablet_schema), K(create_scn)); - } else if (OB_UNLIKELY(ls_id != ls_->get_ls_id())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls id is unexpected", K(ret), "arg_ls_id", ls_id, "ls_id", ls_->get_ls_id()); } else if (OB_UNLIKELY(!tablet_id.is_ls_inner_tablet())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id is not ls inner tablet", K(ret), K(tablet_id)); - } else if (OB_FAIL(has_tablet(ls_id, tablet_id, b_exist))) { - LOG_WARN("failed to check tablet existence", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(has_tablet(tablet_id, b_exist))) { + LOG_WARN("failed to check tablet existence", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(b_exist)) { ret = OB_TABLET_EXIST; - LOG_WARN("tablet already exists", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(create_inner_tablet(ls_id, tablet_id, tablet_id/*data_tablet_id*/, + LOG_WARN("tablet already exists", K(ret), K(tablet_id)); + } else if (OB_FAIL(create_inner_tablet(tablet_id, tablet_id/*data_tablet_id*/, create_scn, snapshot_version, create_tablet_schema, tablet_handle))) { - LOG_WARN("failed to do create tablet", K(ret), K(ls_id), K(tablet_id), + LOG_WARN("failed to do create tablet", K(ret), K(tablet_id), K(create_scn), K(major_frozen_scn), K(create_tablet_schema)); } @@ -6631,7 +5821,6 @@ int ObLSTabletService::create_ls_inner_tablet( } int ObLSTabletService::remove_ls_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -6640,15 +5829,11 @@ int ObLSTabletService::remove_ls_inner_tablet( if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_UNLIKELY(ls_id != ls_->get_ls_id())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls id is unexpected", K(ret), "arg_ls_id", ls_id, "ls_id", ls_->get_ls_id()); - } else if (OB_FAIL(do_remove_tablet(ls_id, tablet_id))) { - LOG_WARN("failed to remove tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid args", K(ret), K(tablet_id)); + } else if (OB_FAIL(do_remove_tablet(tablet_id))) { + LOG_WARN("failed to remove tablet", K(ret), K(tablet_id)); } return ret; @@ -6694,44 +5879,7 @@ int ObLSTabletService::build_tablet_iter(ObLSTabletIterator &iter, const bool ex return ret; } -int ObLSTabletService::build_tablet_iter(ObHALSTabletIDIterator &iter) -{ - int ret = common::OB_SUCCESS; - GetAllTabletIDOperator op(iter.tablet_ids_); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(tablet_id_set_.foreach(op))) { - STORAGE_LOG(WARN, "fail to get all tablet ids from set", K(ret)); - } else if (OB_UNLIKELY(!iter.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("iter is invalid", K(ret), K(iter)); - } else if (OB_FAIL(iter.sort_tablet_ids_if_need())) { - LOG_WARN("failed to sort tablet ids if need", K(ret), K(iter)); - } - - if (OB_FAIL(ret)) { - iter.reset(); - } - return ret; -} - -int ObLSTabletService::build_tablet_iter(ObHALSTabletIterator &iter) -{ - int ret = common::OB_SUCCESS; - iter.ls_tablet_service_ = this; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(build_tablet_iter(iter.tablet_id_iter_))) { - STORAGE_LOG(WARN, "fail to build ha tablet id iterator", K(ret)); - } - if (OB_FAIL(ret)) { - iter.reset(); - } - return ret; -} int ObLSTabletService::is_tablet_exist(const common::ObTabletID &tablet_id, bool &is_exist) { @@ -6753,38 +5901,6 @@ int ObLSTabletService::is_tablet_exist(const common::ObTabletID &tablet_id, bool return ret; } -int ObLSTabletService::set_allow_to_read_(ObLS *ls) -{ - int ret = OB_SUCCESS; - bool allow_read = false; - - if (OB_ISNULL(ls)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("set allow to read get invalid argument", K(ret), KP(ls)); - } else { - if (OB_FAIL(ls->check_allow_read(allow_read))) { - LOG_WARN("failed to check allow read", K(ret), KPC(ls)); - } else if (!allow_read) { - allow_to_read_mgr_.disable_to_read(); - FLOG_INFO("set ls do not allow to read", KPC(ls)); - } else { - allow_to_read_mgr_.enable_to_read(); - FLOG_INFO("set ls allow to read", KPC(ls)); - } - } - return ret; -} - -void ObLSTabletService::enable_to_read() -{ - allow_to_read_mgr_.enable_to_read(); -} - -void ObLSTabletService::disable_to_read() -{ - allow_to_read_mgr_.disable_to_read(); -} - int ObLSTabletService::GetAllTabletIDOperator::operator()(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -6811,7 +5927,7 @@ int ObLSTabletService::DestroyMemtableAndMemberAndMdsTableOperator::operator()(c ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is null", K(ret)); } else { - const ObTabletMapKey key(tablet_svr_->ls_->get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); if (OB_FAIL(t3m->release_memtable_and_mds_table_for_ls_offline(key))) { LOG_WARN("failed to release memtables and mds table", K(ret), K(key)); } @@ -6843,36 +5959,6 @@ int ObLSTabletService::SetMemtableFrozenOperator::operator()(const common::ObTab return ret; } -void ObLSTabletService::AllowToReadMgr::disable_to_read() -{ - bool old_v = false; - bool new_v = false; - do { - old_v = ATOMIC_LOAD(&allow_to_read_); - if (old_v == new_v) { - break; - } - } while (ATOMIC_CAS(&allow_to_read_, old_v, new_v) != old_v); -} - -void ObLSTabletService::AllowToReadMgr::enable_to_read() -{ - bool old_v = false; - bool new_v = true; - do { - old_v = ATOMIC_LOAD(&allow_to_read_); - if (old_v == new_v) { - break; - } - } while (ATOMIC_CAS(&allow_to_read_, old_v, new_v) != old_v); -} - -void ObLSTabletService::AllowToReadMgr::load_allow_to_read_info( - bool &allow_to_read) -{ - allow_to_read = ATOMIC_LOAD(&allow_to_read_); -} - int ObLSTabletService::get_all_tablet_ids( const bool except_ls_inner_tablet, common::ObIArray &tablet_id_array) @@ -6913,101 +5999,6 @@ int ObLSTabletService::set_frozen_for_all_memtables() return ret; } -int ObLSTabletService::ha_scan_all_tablets( - const HandleTabletMetaFunc &handle_tablet_meta_f, - const bool need_sorted_tablet_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else { - ObHALSTabletIterator iterator(ls_->get_ls_id(), false/* need_initial_state */, need_sorted_tablet_id); - if (OB_FAIL(build_tablet_iter(iterator))) { - LOG_WARN("fail to build tablet iterator", K(ret), KPC(this)); - } else { - ObTabletHandle tablet_handle; - ObTablet *tablet = nullptr; - obcall::ObCopyTabletInfo tablet_info; - ObTabletCreateDeleteMdsUserData user_data; - mds::MdsWriter writer;// will be removed later - mds::TwoPhaseCommitState trans_stat;// will be removed later - share::SCN trans_version;// will be removed later - - while (OB_SUCC(ret)) { - tablet_info.reset(); - user_data.reset(); - trans_stat = mds::TwoPhaseCommitState::STATE_END; - if (OB_FAIL(iterator.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get tablet", K(ret)); - } - } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet is nullptr", K(ret), K(tablet_handle)); - } else if (OB_FAIL(tablet->get_latest(user_data, writer, trans_stat, trans_version))) { - if (OB_EMPTY_RESULT == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get latest tablet status", K(ret), KPC(tablet)); - } - } else if (trans_stat != mds::TwoPhaseCommitState::ON_COMMIT - && ObTabletStatus::RESERVED_5 == user_data.tablet_status_) { - //TODO(muwei.ym) CAN NOT USE this condition when MDS supports uncommitted transaction - - // why we should skip uncommitted reserved tablet status, because if we backup the uncommitted reserved status - // but the final result of the uncommitted reserved status is aborted, which means the tablet has no MDS Table, - // and the restore process will create this tablet nontheless, however this aborted tablet should be GC-ed at the end, - // because the aborted tablet has no MDS Table, the tablet can not be GC-ed correctly, resulting in the tablet dangling - // for details, see issue-51990749 - - // we can skip backup uncommitted reserved tablets because we backup ls meta first - // if the reserved transaction is not committed, the clog_checkpoint_scn recorded - // in ls meta will not be advanced, so that even though we skip backup uncommited - // reserved tablet status, we can still restore it by replaying clog - LOG_WARN("tablet is in reserved status but not committed, skip", K(tablet_handle), K(user_data)); - continue; - } else if (OB_FAIL(tablet->build_migration_tablet_param(tablet_info.param_))) { - LOG_WARN("failed to build migration tablet param", K(ret)); - } else if (OB_FAIL(tablet->get_ha_sstable_size(tablet_info.data_size_))) { - LOG_WARN("failed to get sstable size", K(ret), KPC(tablet)); - } else if (OB_FAIL(FALSE_IT(tablet_info.version_ = GET_MIN_CLUSTER_VERSION()))) { - LOG_WARN("failed to get server version", K(ret), K(tablet_info)); - } else { - tablet_info.tablet_id_ = tablet->get_tablet_meta().tablet_id_; - tablet_info.status_ = ObCopyTabletStatus::TABLET_EXIST; - if (OB_FAIL(handle_tablet_meta_f(tablet_info, tablet_handle))) { - LOG_WARN("fail to handle tablet meta", K(ret), K(tablet_info)); - } - } - } - } - } - - return ret; -} - -int ObLSTabletService::ha_get_tablet( - const common::ObTabletID &tablet_id, - ObTabletHandle &handle) -{ - int ret = OB_SUCCESS; - bool is_empty_shell = false; - ObTablet *tablet = nullptr; - if (OB_FAIL(direct_get_tablet(tablet_id, handle))) { - LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); - } else if (OB_ISNULL(tablet = handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet should not be NULL", K(ret), K(tablet_id)); - } else if (tablet->is_empty_shell()) { - // treat empty shell as tablet not exist. - ret = OB_TABLET_NOT_EXIST; - } - return ret; -} int ObLSTabletService::get_tablet_without_memtables( const WashTabletPriority &priority, @@ -7039,126 +6030,6 @@ int ObLSTabletService::get_tablet_without_memtables( return ret; } -int ObLSTabletService::ha_get_tablet_without_memtables( - const WashTabletPriority &priority, - const ObTabletMapKey &key, - common::ObArenaAllocator &allocator, - ObTabletHandle &handle) -{ - int ret = OB_SUCCESS; - ObTablet *tablet = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(get_tablet_without_memtables(priority, key, allocator, handle))) { - LOG_WARN("failed to get tablet without memtables", K(ret), K(priority), K(key)); - } else if (OB_ISNULL(tablet = handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet should not be NULL", K(ret), K(key)); - } else if (tablet->is_empty_shell()) { - // treat empty shell as tablet not exist. - ret = OB_TABLET_NOT_EXIST; - } - return ret; -} - -int ObLSTabletService::check_real_leader_for_4377_(const ObLSID ls_id) -{ - int ret = OB_SUCCESS; - ObLSService* ls_srv = nullptr; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - int64_t epoch = 0; - bool is_real_leader = false; - - if (OB_ISNULL(ls_srv = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("share::g_mp->ls_service() fail, MTL not init?", K(ret)); - } else if (OB_FAIL(ls_srv->get_ls(ls_id, - ls_handle, - ObLSGetMod::TRANS_MOD))) { - LOG_ERROR("ls_srv->get_ls() fail", KR(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("invalid ls", KR(ret)); - } else if (OB_FAIL(ls->get_tx_svr()->get_tx_ls_log_adapter()->get_role(is_real_leader, epoch))) { - LOG_WARN("get replica status fail", K(ret), KPC(ls)); - } else if (!is_real_leader) { - ret = OB_NOT_MASTER; - LOG_WARN("get follower status during 4377", K(ret), KPC(ls)); - } - - return ret; -} - -int ObLSTabletService::check_need_rollback_for_4377_(const transaction::ObTxDesc *tx_desc, - ObTabletHandle &data_tablet_handle) -{ - int ret = OB_SUCCESS; - ObTabletCreateDeleteMdsUserData user_data; - mds::MdsWriter unused_writer;// will be removed later - mds::TwoPhaseCommitState unused_trans_stat;// will be removed later - share::SCN unused_trans_version;// will be removed later - - if (OB_ISNULL(tx_desc)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tx_desc is null when check 4377", K(ret)); - } else if (OB_FAIL(data_tablet_handle.get_obj()->get_latest( - user_data, unused_writer, unused_trans_stat, unused_trans_version))) { - LOG_WARN("can not get the tablet status from ObITabletMdsInterface", K(ret)); - } else { - // If a reserved status SCN exists and is newer than this transaction's active timestamp, - // rollback the transaction to avoid returning an incorrect 4377 error. - if (user_data.reserved_scn_.is_valid()) { - if (user_data.reserved_scn_.convert_to_ts() > tx_desc->get_active_ts()) { - // TODO(yangyifei.yyf): active_ts is set by ObClockGenerator::get_clock, it's not a GTS. - // And reserved_scn is set by redo_scn, it's incorrect to compare it with active_ts. - // We should adjust this judgment later. - ret = OB_TRANS_NEED_ROLLBACK; - LOG_WARN("maybe meet reserved status during kill tx, which can cause 4377 error, so we will rollback it", - K(tx_desc), - K(user_data)); - } - } else { - // There's no reserved status after trans begin, still throw 4377 error. - } - } - return ret; -} - -int ObLSTabletService::check_parts_tx_state_for_4377_(transaction::ObTxDesc *tx_desc) -{ - int ret = OB_SUCCESS; - transaction::ObTransService *txs = share::g_mp->trans_service(); - transaction::ObTxPartList copy_parts; - bool is_alive = false; - - if (OB_ISNULL(tx_desc)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tx_desc is null", K(ret)); - } else if (OB_FAIL(tx_desc->get_parts_copy(copy_parts))) { - TRANS_LOG(WARN, "get participants copy error", K(ret)); - } else { - // Traverse all participant lists through RPC to confirm whether the txn - // context has been terminated. In this scenario, we consider the 4377 as - // misreported, and convert it into OB_TRANS_NEED_ROLLBACK. - for (int64_t i = 0; OB_SUCC(ret) && i < copy_parts.count(); i++) { - ObLSID ls_id = copy_parts[i].id_; - is_alive = false; - if (OB_FAIL(txs->ask_tx_state_for_4377(ls_id, - tx_desc->get_tx_id(), - is_alive))) { - TRANS_LOG(WARN, "fail to ask tx state for 4377", K(tx_desc)); - } else if (!is_alive) { - ret = OB_TRANS_NEED_ROLLBACK; - LOG_WARN("maybe meet terminated tx during kill tx, which can cause 4377 error, so we will rollback it", - K(tx_desc)); - } - } - } - - return ret; -} int ObLSTabletService::offline_build_tablet_without_memtable_() { @@ -7224,11 +6095,10 @@ int ObLSTabletService::check_tablet_no_active_memtable(const ObIArrayget_ls_id(); - LOG_INFO("start offline_gc_tablet_for_aborted_create", K(ret), K(ls_id)); + LOG_INFO("start offline gc uncommitted tablets", K(ret)); ObTabletIDArray deleted_tablets; ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); bool tablet_status_is_written = false; @@ -7282,10 +6152,10 @@ int ObLSTabletService::offline_gc_tablet_for_aborted_create_() if (OB_SUCC(ret)) { for (int64_t i = 0; OB_SUCC(ret) && i < deleted_tablets.count(); ++i) { const common::ObTabletID &tablet_id = deleted_tablets.at(i); - if (OB_FAIL(do_remove_tablet(ls_id, tablet_id))) { - LOG_WARN("failed to remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(do_remove_tablet(tablet_id))) { + LOG_WARN("failed to remove tablet", K(ret), K(tablet_id)); } else { - LOG_INFO("gc tablet finish", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("gc tablet finish", K(ret), K(tablet_id)); } } } @@ -8071,30 +6941,6 @@ int ObLSTabletService::check_old_row_legitimacy_wrap( return ret; } -int ObLSTabletService::check_rollback_tablet_is_same_( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - bool &is_same) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle; - ObTablet *tablet = nullptr; - is_same = true; - - if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle))) { - if (OB_TABLET_NOT_EXIST == ret) { - is_same = true; - ret = OB_SUCCESS; - } - } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet should not be NULL", K(ret), K(ls_id), K(tablet_id)); - } else { - is_same = true; - } - return ret; -} - int ObLSTabletService::process_lob_after_insert( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, @@ -8154,17 +7000,12 @@ int ObLSTabletService::scan_block_stat( ObBlockStatIterator &iter) { int ret = OB_SUCCESS; - bool allow_to_read = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); } else if (OB_UNLIKELY(!tablet_handle.is_valid() || !scan_param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_handle), K(scan_param)); - } else if (FALSE_IT(allow_to_read_mgr_.load_allow_to_read_info(allow_to_read))) { - } else if (OB_UNLIKELY(!allow_to_read)) { - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls not allow to read", K(ret), KPC_(ls)); } else if (OB_FAIL(prepare_scan_table_param(*scan_param.get_scan_param(), *(share::g_mp->tenant_schema_service()->get_schema_service())))) { LOG_WARN("fail to prepare scan table param", K(ret), K(scan_param), K(tablet_handle)); } else if (OB_UNLIKELY(scan_param.get_scan_param()->fb_snapshot_.is_min())) { diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index 2b923c6b4..e7965f83c 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -27,7 +27,7 @@ #include "storage/ob_dml_common.h" #include "storage/ob_relative_table.h" #include "storage/slog/ob_storage_log.h" -#include "storage/ob_tablet_ha_status.h" +#include "storage/ob_tablet_restore_state.h" #include "storage/tablelock/ob_lock_memtable_mgr.h" #include "storage/tx_table/ob_tx_ctx_memtable_mgr.h" #include "storage/tx_table/ob_tx_data_memtable_mgr.h" @@ -57,7 +57,6 @@ struct ColumnMap; namespace blocksstable { -class ObMigrationSSTableParam; struct ObDatumRowkey; enum ObDmlFlag; class ObDatumRowStore; @@ -85,21 +84,18 @@ class ObTableScanIterator; class ObRowGetter; class ObLSTabletIterator; class ObLSTabletAddrIterator; -class ObHALSTabletIDIterator; -class ObHALSTabletIterator; class ObLSTabletFastIter; class ObTabletMapKey; struct ObStorageLogParam; struct ObTabletCreateSSTableParam; struct ObUpdateTableStoreParam; -struct ObMigrationTabletParam; class ObTableScanRange; class ObTabletCreateDeleteMdsUserData; class ObBlockStatScanParam; class ObBlockStatIterator; class ObLSTabletService : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -116,18 +112,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, // TODO: delete it if apply sequence // set allocators frozen to reduce active tenant_memory in ObLS::offline_() int set_frozen_for_all_memtables(); -public: - class AllowToReadMgr final - { - public: - AllowToReadMgr() : allow_to_read_(false) {} - ~AllowToReadMgr() = default; - void disable_to_read(); - void enable_to_read(); - void load_allow_to_read_info(bool &allow_to_read); - private: - bool allow_to_read_; - }; private: // for replay virtual int replay( @@ -136,11 +120,9 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, const palf::LSN &lsn, const share::SCN &scn) override; - // for role change - virtual void switch_to_follower_forcedly() override; - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override; - virtual int resume_leader() override; + // local lifecycle + void deactivate() override; + int activate() override; // for checkpoint virtual int flush(share::SCN &recycle_scn) override; @@ -148,21 +130,17 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, public: int prepare_for_safe_destroy(); - int safe_to_destroy(bool &is_safe); // tablet operation int create_ls_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const share::SCN &frozen_timestamp, const ObCreateTabletSchema &create_tablet_schema, const share::SCN &create_scn); int remove_ls_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); int create_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -179,7 +157,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, ObTabletHandle &tablet_handle, const share::ObForkTabletInfo &fork_info = share::ObForkTabletInfo()); int rollback_remove_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); int get_tablet( @@ -193,7 +170,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, const int64_t retry_timeout_us, const ObMDSGetTabletMode mode = ObMDSGetTabletMode::READ_READABLE_COMMITED, const share::SCN &snapshot = share::SCN::max_scn()); - int get_ls_migration_required_size(int64_t &required_size); int remove_tablets(const common::ObIArray &tablet_id_array); // Exactly deletion compared with input tablets int remove_tablet(const ObTabletHandle& tablet_handle); @@ -242,26 +218,12 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, const common::ObTabletID &tablet_id, const ObTabletRestoreStatus::STATUS &restore_status, const bool need_to_set_split_data_complete); - int update_tablet_ha_data_status( - const common::ObTabletID &tablet_id, - const ObTabletDataStatus::STATUS &data_status); - int update_tablet_ha_expected_status( - const common::ObTabletID &tablet_id, - const ObTabletExpectedStatus::STATUS &expected_status); - // Get tablet handle but ignore empty shell. Return OB_TABLET_NOT_EXIST if it is empty shell. - int ha_get_tablet( - const common::ObTabletID &tablet_id, - ObTabletHandle &handle); + int update_tablet_column_store_schema(const common::ObTabletID &tablet_id); int get_tablet_without_memtables( const WashTabletPriority &priority, const ObTabletMapKey &key, common::ObArenaAllocator &allocator, ObTabletHandle &handle); - int ha_get_tablet_without_memtables( - const WashTabletPriority &priority, - const ObTabletMapKey &key, - common::ObArenaAllocator &allocator, - ObTabletHandle &handle); int update_tablet_to_empty_shell(const common::ObTabletID &tablet_id); int replay_create_tablet( const ObMetaDiskAddr &disk_addr, @@ -288,7 +250,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, const bool allow_no_ready_read, const bool need_split_src_table, const bool need_split_dst_table); - int check_allow_to_read(); int set_tablet_status( const common::ObTabletID &tablet_id, const ObTabletCreateDeleteMdsUserData &tablet_status, @@ -424,34 +385,16 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, // iterator int build_tablet_iter(ObLSTabletIterator &iter, const bool except_ls_inner_tablet = false); int build_tablet_iter(ObLSTabletAddrIterator &iter); - int build_tablet_iter(ObHALSTabletIDIterator &iter); - int build_tablet_iter(ObHALSTabletIterator &iter); int is_tablet_exist(const common::ObTabletID &tablet_id, bool &is_exist); - // migration section - typedef common::ObFunction HandleTabletMetaFunc; - int ha_scan_all_tablets( - const HandleTabletMetaFunc &handle_tablet_meta_f, - const bool need_sorted_tablet_id); - int trim_rebuild_tablet( - const ObTabletID &tablet_id, - const bool is_rollback = false); - int rebuild_create_tablet( - const ObMigrationTabletParam &mig_tablet_param, - const bool keep_old); - int create_or_update_migration_tablet( - const ObMigrationTabletParam &mig_tablet_param); int build_tablet_with_batch_tables( const ObTabletID &tablet_id, const ObBatchUpdateTableStoreParam ¶m); - void enable_to_read(); - void disable_to_read(); int get_all_tablet_ids(const bool except_ls_inner_tablet, common::ObIArray &tablet_id_array); int flush_mds_table(int64_t recycle_scn); - // check tablet write stop int check_tablet_no_active_memtable(const ObIArray &tablet_list, bool &has); protected: @@ -549,16 +492,10 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, ObTabletHandle &new_handle, ObTimeGuard &time_guard); int safe_create_cas_tablet( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObMetaDiskAddr &addr, ObTabletHandle &tablet_handle, ObTimeGuard &time_guard); - int safe_create_cas_empty_shell( - const ObLSID &ls_id, - const ObTabletID &tablet_id, - ObTabletHandle &tablet_handle, - ObTimeGuard &time_guard); void report_tablet_to_rs(const common::ObTabletID &tablet_id); void report_tablet_to_rs(const common::ObIArray &tablet_id_array); @@ -571,11 +508,9 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, ObTableScanParam ¶m); int get_tablet_addr(const ObTabletMapKey &key, ObMetaDiskAddr &addr); int has_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &b_exist); int create_inner_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -593,31 +528,18 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, int64_t &memtable_row_count); int refresh_tablet_addr( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObUpdateTabletPointerParam ¶m, ObTabletHandle &tablet_handle); int do_remove_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); int inner_remove_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); int rollback_remove_tablet_without_lock( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); - int rollback_rebuild_tablet(const ObTabletID &tablet_id); - int trim_old_tablets(const ObTabletID &tablet_id); - int rebuild_tablet_with_old( - const ObMigrationTabletParam &mig_tablet_param, - ObTabletHandle &tablet_guard); - int migrate_update_tablet(const ObMigrationTabletParam &mig_tablet_param); - int migrate_create_tablet( - const ObMigrationTabletParam &mig_tablet_param, - ObTabletHandle &handle); int delete_all_tablets(); int offline_build_tablet_without_memtable_(); - int offline_gc_tablet_for_aborted_create_(); + int offline_gc_uncommitted_tablets_(); int offline_destroy_memtable_and_mds_table_(); int inner_get_read_tables( @@ -645,10 +567,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, const ObTabletMapKey &key, const int64_t ls_epoch, ObTabletHandle &tablet_handle); - static int check_real_leader_for_4377_(const ObLSID ls_id); - static int check_need_rollback_for_4377_(const transaction::ObTxDesc *tx_desc, - ObTabletHandle &tablet_handle); - static int check_parts_tx_state_for_4377_(transaction::ObTxDesc *tx_desc); static int check_old_row_legitimacy( const blocksstable::ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, @@ -881,16 +799,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, static void dump_diag_info_for_old_row_loss( ObDMLRunningCtx &run_ctx, const blocksstable::ObDatumRow &datum_row); - int set_allow_to_read_(ObLS *ls); - // TODO(chenqingxiang.cqx): remove this - int create_empty_shell_tablet( - const ObMigrationTabletParam ¶m, - ObTabletHandle &tablet_handle); - int check_rollback_tablet_is_same_( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - bool &is_same); - // for lob tablet dml static int check_rowkey_length( const ObDMLRunningCtx &run_ctx, @@ -1028,7 +936,6 @@ class ObLSTabletService : public logservice::ObIReplaySubHandler, mds::ObMdsTableMgr mds_table_mgr_; ObTabletIDSet tablet_id_set_; common::ObBucketLock bucket_lock_; // for tablet update, not for dml - AllowToReadMgr allow_to_read_mgr_; bool is_inited_; bool is_stopped_; }; diff --git a/src/storage/ls/ob_ls_tx_service.cpp b/src/storage/ls/ob_ls_tx_service.cpp index 326f612ab..97af294e4 100644 --- a/src/storage/ls/ob_ls_tx_service.cpp +++ b/src/storage/ls/ob_ls_tx_service.cpp @@ -22,7 +22,7 @@ #include "storage/throttle/ob_throttle_unit.h" #include "storage/tx/ob_trans_service.h" #include "storage/tx/ob_tx_replay_executor.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx_storage/ob_tenant_freezer.h" @@ -37,16 +37,14 @@ namespace storage { using namespace checkpoint; -int ObLSTxService::init(const ObLSID &ls_id, - ObLSTxCtxMgr *mgr, +int ObLSTxService::init(ObLSTxCtxMgr *mgr, ObTransService *trans_service) { int ret = OB_SUCCESS; - if (!ls_id.is_valid() || OB_ISNULL(mgr) || OB_ISNULL(trans_service)) { + if (OB_ISNULL(mgr) || OB_ISNULL(trans_service)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id), KP(mgr), KP(trans_service)); + TRANS_LOG(WARN, "invalid argument", K(ret), KP(mgr), KP(trans_service)); } else { - ls_id_ = ls_id; mgr_ = mgr; trans_service_ = trans_service; } @@ -55,7 +53,7 @@ int ObLSTxService::init(const ObLSID &ls_id, int ObLSTxService::create_tx_ctx(ObTxCreateArg arg, bool &existed, - ObPartTransCtx *&ctx) const + ObTxCtx *&ctx) const { int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { @@ -69,7 +67,7 @@ int ObLSTxService::create_tx_ctx(ObTxCreateArg arg, int ObLSTxService::get_tx_ctx(const transaction::ObTransID &tx_id, const bool for_replay, - ObPartTransCtx *&ctx) const + ObTxCtx *&ctx) const { int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { @@ -83,7 +81,7 @@ int ObLSTxService::get_tx_ctx(const transaction::ObTransID &tx_id, int ObLSTxService::get_tx_ctx_with_timeout(const transaction::ObTransID &tx_id, const bool for_replay, - transaction::ObPartTransCtx *&tx_ctx, + transaction::ObTxCtx *&tx_ctx, const int64_t lock_timeout) const { int ret = OB_SUCCESS; @@ -97,34 +95,6 @@ int ObLSTxService::get_tx_ctx_with_timeout(const transaction::ObTransID &tx_id, return ret; } -int ObLSTxService::get_tx_scheduler(const transaction::ObTransID &tx_id, - ObAddr &scheduler) const -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", K(ret)); - } else { - ObPartTransCtx *ctx; - if (OB_FAIL(mgr_->get_tx_ctx_directly_from_hash_map(tx_id, ctx))) { - // TRANS_LOG(WARN, "get ctx failed", K(ret), K(tx_id)); - } else if (OB_ISNULL(ctx)) { - ret = OB_BAD_NULL_ERROR; - TRANS_LOG(WARN, "get ctx is null", K(ret), K(tx_id)); - } else { - scheduler = ctx->get_scheduler(); - if (OB_SUCCESS != (tmp_ret = mgr_->revert_tx_ctx(ctx))) { - TRANS_LOG(ERROR, "fail to revert tx", K(ret), K(tmp_ret), K(tx_id), KPC(ctx)); - } - if (!scheduler.is_valid()) {// follower ctx, not replay commit info yet, this may happed - ret = OB_TRANS_CTX_NOT_EXIST; - } - } - } - return ret; -} - int ObLSTxService::get_tx_start_session_id(const transaction::ObTransID &tx_id, uint32_t &session_id) const { int ret = OB_SUCCESS; @@ -133,17 +103,17 @@ int ObLSTxService::get_tx_start_session_id(const transaction::ObTransID &tx_id, ret = OB_NOT_INIT; TRANS_LOG(WARN, "not init", K(ret)); } else { - ObPartTransCtx *ctx; + ObTxCtx *ctx; if (OB_FAIL(mgr_->get_tx_ctx_directly_from_hash_map(tx_id, ctx))) { if (OB_TRANS_CTX_NOT_EXIST == ret) { ret = OB_SUCCESS; - TRANS_LOG(INFO, "ctx not existed on this LS", K(tx_id), K(ls_id_)); + TRANS_LOG(INFO, "ctx not existed", K(tx_id)); } else { - TRANS_LOG(WARN, "get ctx failed", K(ret), K(tx_id), K(ls_id_)); + TRANS_LOG(WARN, "get ctx failed", K(ret), K(tx_id)); } } else if (OB_ISNULL(ctx)) { ret = OB_BAD_NULL_ERROR; - TRANS_LOG(WARN, "get ctx is null", K(ret), K(tx_id), K(ls_id_)); + TRANS_LOG(WARN, "get ctx is null", K(ret), K(tx_id)); } else { session_id = (sql::ObSQLSessionInfo::INVALID_SESSID == ctx->get_client_sid()) @@ -182,7 +152,6 @@ int ObLSTxService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, } else if (OB_FAIL(mgr_->start_readonly_request())) { TRANS_LOG(WARN, "start readonly request failed", K(ret)); } else { - store_ctx.ls_id_ = ls_id_; store_ctx.is_read_store_ctx_ = true; ret = trans_service_->get_read_store_ctx(snapshot, read_latest, lock_timeout, store_ctx, tx_desc); if (OB_FAIL(ret)) { @@ -205,7 +174,6 @@ int ObLSTxService::get_read_store_ctx(const SCN &snapshot, } else if (OB_FAIL(mgr_->start_readonly_request())) { TRANS_LOG(WARN, "start readonly request failed", K(ret)); } else { - store_ctx.ls_id_ = ls_id_; store_ctx.is_read_store_ctx_ = true; ret = trans_service_->get_read_store_ctx(snapshot, lock_timeout, store_ctx); @@ -234,7 +202,7 @@ int ObLSTxService::get_write_store_ctx(ObTxDesc &tx, abs_expire_ts = ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; } - ObTxDataThrottleGuard tx_data_throttle_guard(ls_id_, false /* for_replay */, abs_expire_ts); + ObTxDataThrottleGuard tx_data_throttle_guard(false /* for_replay */, abs_expire_ts); ret = trans_service_->get_write_store_ctx(tx, snapshot, write_flag, store_ctx, spec_seq_no, false); } return ret; @@ -265,15 +233,15 @@ int ObLSTxService::revert_store_ctx(storage::ObStoreCtx &store_ctx) const return ret; } -int ObLSTxService::check_scheduler_status(SCN &min_start_scn, - transaction::MinStartScnStatus &status) +int ObLSTxService::check_tx_status(SCN &min_start_scn, + transaction::MinStartScnStatus &status) { int ret = OB_SUCCESS; if (OB_ISNULL(trans_service_)) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not init", K(ret)); } else { - ret = mgr_->check_scheduler_status(min_start_scn, status); + ret = mgr_->check_tx_status(min_start_scn, status); } return ret; } @@ -283,12 +251,12 @@ int ObLSTxService::check_all_tx_clean_up() const int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (mgr_->get_active_tx_count() > 0) { // there is some tx not finished, retry. ret = OB_EAGAIN; } else { - TRANS_LOG(INFO, "wait_all_tx_cleaned_up cleaned up success", K_(ls_id)); + TRANS_LOG(INFO, "wait_all_tx_cleaned_up cleaned up success"); } return ret; } @@ -300,15 +268,15 @@ int ObLSTxService::check_all_readonly_tx_clean_up() const int64_t active_readonly_request_count = 0; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if ((active_readonly_request_count = mgr_->get_total_active_readonly_request_count()) > 0) { if (REACH_TIME_INTERVAL(5000000)) { TRANS_LOG(INFO, "readonly requests are active", K(active_readonly_request_count)); - READ_CHECKER_PRINT(ls_id_); + READ_CHECKER_PRINT(); } ret = OB_EAGAIN; } else { - TRANS_LOG(INFO, "wait_all_readonly_tx_cleaned_up cleaned up success", K_(ls_id)); + TRANS_LOG(INFO, "wait_all_readonly_tx_cleaned_up cleaned up success"); } #ifdef ERRSIM @@ -328,11 +296,11 @@ int ObLSTxService::block_tx() bool unused_is_all_tx_clean_up = false; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->block_tx(unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "block rw tx failed", K_(ls_id)); + TRANS_LOG(WARN, "block rw tx failed"); } else { - TRANS_LOG(INFO, "block rw tx success", K_(ls_id)); + TRANS_LOG(INFO, "block rw tx success"); } return ret; } @@ -343,11 +311,11 @@ int ObLSTxService::block_all() bool unused_is_all_tx_clean_up = false; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->block_all(unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "block all failed", K_(ls_id)); + TRANS_LOG(WARN, "block all failed"); } else { - TRANS_LOG(INFO, "block all success", K_(ls_id)); + TRANS_LOG(INFO, "block all success"); } return ret; } @@ -358,11 +326,11 @@ int ObLSTxService::kill_all_tx(const bool graceful) bool unused_is_all_tx_clean_up = false; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->kill_all_tx(graceful, unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "kill_all_tx failed", K_(ls_id)); + TRANS_LOG(WARN, "kill_all_tx failed"); } else { - TRANS_LOG(INFO, "kill_all_tx success", K_(ls_id)); + TRANS_LOG(INFO, "kill_all_tx success"); } return ret; } @@ -374,7 +342,7 @@ int ObLSTxService::check_modify_schema_elapsed(const ObTabletID &tablet_id, int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(schema_version < 0)) { ret = OB_INVALID_ARGUMENT; @@ -399,7 +367,7 @@ int ObLSTxService::check_modify_time_elapsed(const ObTabletID &tablet_id, int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(timestamp < 0)) { ret = OB_INVALID_ARGUMENT; @@ -422,7 +390,7 @@ int ObLSTxService::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->iterate_tx_obj_lock_op(iter))) { TRANS_LOG(WARN, "get tx obj lock op iter failed", K(ret)); } else if (OB_FAIL(iter.set_ready())) { @@ -438,11 +406,11 @@ int ObLSTxService::iterate_tx_ctx(ObLSTxCtxIterator &iter) const int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(iter.set_ready(mgr_))) { - TRANS_LOG(WARN, "get tx obj lock op iter failed", K(ret), K_(ls_id)); + TRANS_LOG(WARN, "get tx obj lock op iter failed", K(ret)); } else { - TRANS_LOG(INFO, "iter set ready success", K(ret), K_(ls_id)); + TRANS_LOG(INFO, "iter set ready success", K(ret)); } return ret; } @@ -463,8 +431,7 @@ int ObLSTxService::replay(const void *buffer, LOG_WARN("log base header deserialize error", K(ret)); } else if (OB_FAIL(ObTxReplayExecutor::execute(parent_, this, log_buf, nbytes, tmp_pos, lsn, scn, - base_header, - ls_id_))) { + base_header))) { LOG_WARN("replay tx log error", K(ret), K(lsn), K(scn)); } return ret; @@ -479,68 +446,13 @@ int ObLSTxService::traverse_trans_to_submit_next_log() { return mgr_->traverse_t ObITxLogAdapter *ObLSTxService::get_tx_ls_log_adapter() { return mgr_->get_ls_log_adapter(); } -int ObLSTxService::replay_start_working_log(const ObTxStartWorkingLog &log, SCN &log_ts_ns) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(mgr_->replay_start_working_log(log, log_ts_ns))) { - TRANS_LOG(WARN, "replay start working log failed", KR(ret), K(log), K(ls_id_)); - } - return ret; -} - -void ObLSTxService::switch_to_follower_forcedly() +void ObLSTxService::deactivate() { - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(mgr_->switch_to_follower_forcedly())) { - TRANS_LOG(ERROR, "switch to follower forcedly failed", KR(ret), K(ls_id_)); - } - // TRANS_LOG(INFO, "[ObLSTxService] switch_to_follower_forcedly", KR(ret), K(ls_id_)); - return; } -int ObLSTxService::switch_to_leader() +int ObLSTxService::activate() { - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(mgr_->switch_to_leader())) { - TRANS_LOG(WARN, "switch to leader failed", KR(ret), K(ls_id_)); - } - // TRANS_LOG(INFO, "[ObLSTxService] switch_to_leader", KR(ret), K(ls_id_)); - return ret; -} - -int ObLSTxService::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(mgr_->switch_to_follower_gracefully())) { - TRANS_LOG(WARN, "switch to follower gracefully failed", KR(ret), K(ls_id_)); - } - // TRANS_LOG(INFO, "[ObLSTxService] switch_to_follower_gracefully", KR(ret), K(ls_id_)); - return ret; -} - -int ObLSTxService::resume_leader() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(mgr_->resume_leader())) { - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(ls_id_)); - } - // TRANS_LOG(INFO, "[ObLSTxService] resume_leader", KR(ret), K(ls_id_)); - return ret; + return OB_SUCCESS; } inline @@ -581,7 +493,7 @@ SCN ObLSTxService::get_rec_scn() TRANS_LOG(INFO, "[CHECKPOINT] ObLSTxService::get_rec_scn", K(common_checkpoint_type), KPC(common_checkpoints_[min_rec_scn_common_checkpoint_type_index]), - K(min_rec_scn), K(ls_id_)); + K(min_rec_scn)); return min_rec_scn; } @@ -599,7 +511,6 @@ int ObLSTxService::flush(SCN &recycle_scn) TRANS_LOG(INFO, "common_checkpoints flush", K(i), - K(ls_id_), K(common_checkpoints_[i])); if (OB_SUCCESS != (tmp_ret = common_checkpoints_[i]->flush(recycle_scn))) { TRANS_LOG(WARN, "obCommonCheckpoint flush failed", K(tmp_ret), K(common_checkpoints_[i])); @@ -695,19 +606,6 @@ int ObLSTxService::unregister_common_checkpoint(const ObCommonCheckpointType &ty return ret; } -int ObLSTxService::check_in_leader_serving_state(bool& bool_ret) -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObLSTxService::mgr_ is nullptr", K(ls_id_)); - } else { - bool_ret = mgr_->in_leader_serving_state(); - } - return ret; -} - int ObLSTxService::traversal_flush() { int ret = OB_SUCCESS; @@ -767,9 +665,9 @@ int ObLSTxService::prepare_offline(const int64_t start_ts) bool unused_is_all_tx_clean_up = false; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->block_all(unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "block all failed", K_(ls_id)); + TRANS_LOG(WARN, "block all failed"); } else if (ObTimeUtility::current_time() > start_ts + WAIT_READONLY_REQUEST_US) { // dont care readonly request } else { @@ -777,11 +675,11 @@ int ObLSTxService::prepare_offline(const int64_t start_ts) if (readonly_request_cnt > 0) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - TRANS_LOG(WARN, "readonly requests are active", K(ret), KP(mgr_), K_(ls_id), K(readonly_request_cnt)); + TRANS_LOG(WARN, "readonly requests are active", K(ret), KP(mgr_), K(readonly_request_cnt)); } } } - TRANS_LOG(INFO, "prepare offline ls", K(ret), K(start_ts), KP(mgr_), K_(ls_id)); + TRANS_LOG(INFO, "prepare offline ls", K(ret), K(start_ts), KP(mgr_)); return ret; } @@ -793,18 +691,15 @@ int ObLSTxService::offline() bool unused_is_all_tx_clean_up = false; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->block_all(unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "block all failed", K_(ls_id)); + TRANS_LOG(WARN, "block all failed"); } else if (OB_FAIL(mgr_->kill_all_tx(graceful, unused_is_all_tx_clean_up))) { - TRANS_LOG(WARN, "kill_all_tx failed", K_(ls_id)); - } else if (OB_FAIL(share::g_mp->trans_service()->get_ts_mgr()->interrupt_gts_callback_for_ls_offline( - ls_id_))) { - TRANS_LOG(WARN, "interrupt gts callback failed", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "kill_all_tx failed"); } else if (mgr_->get_tx_ctx_count() > 0) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - TRANS_LOG(WARN, "transaction not empty, try again", K(ret), KP(mgr_), K_(ls_id), K(mgr_->get_tx_ctx_count())); + TRANS_LOG(WARN, "transaction not empty, try again", K(ret), KP(mgr_), K(mgr_->get_tx_ctx_count())); } } return ret; @@ -816,9 +711,9 @@ int ObLSTxService::online() // need reset block. if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else if (OB_FAIL(mgr_->online())) { - TRANS_LOG(WARN, "ls tx service online failed", K(ret), K_(ls_id)); + TRANS_LOG(WARN, "ls tx service online failed", K(ret)); } else { // do nothing } @@ -833,7 +728,7 @@ int ObLSTxService::get_tx_ctx_count(int64_t &tx_ctx_count) tx_ctx_count = -1; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else { tx_ctx_count = mgr_->get_tx_ctx_count(); } @@ -846,7 +741,7 @@ int ObLSTxService::get_active_tx_count(int64_t &active_tx_count) active_tx_count = -1; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else { active_tx_count = mgr_->get_active_tx_count(); } @@ -858,7 +753,7 @@ int ObLSTxService::print_all_tx_ctx(const int64_t print_num) int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else { const bool verbose = true; mgr_->print_all_tx_ctx(print_num, verbose); @@ -866,28 +761,15 @@ int ObLSTxService::print_all_tx_ctx(const int64_t print_num) return ret; } -int ObLSTxService::retry_apply_start_working_log() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(mgr_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); - } else { - ret = mgr_->retry_apply_start_working_log(); - } - return ret; -} - -int ObLSTxService::set_max_replay_commit_version(share::SCN commit_version) +int ObLSTxService::advance_commit_version(share::SCN commit_version) { int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else { - mgr_->update_max_replay_commit_version(commit_version); share::g_mp->trans_service()->get_tx_version_mgr().update_max_commit_ts(commit_version, false /*elr*/); - TRANS_LOG(INFO, "succ set max_replay_commit_version", K(commit_version)); + TRANS_LOG(INFO, "advance commit version", K(commit_version)); } return ret; } @@ -897,23 +779,11 @@ int ObLSTxService::check_tx_blocked(bool &tx_blocked) const int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + TRANS_LOG(WARN, "not init", KR(ret)); } else { tx_blocked = mgr_->is_tx_blocked(); } return ret; } -int ObLSTxService::wait_tx_write_end(ObTimeoutCtx &timeout_ctx) -{ - int ret = OB_SUCCESS; - int64_t start_time = ObTimeUtility::current_time(); - if (OB_FAIL(mgr_->wait_tx_write_end(timeout_ctx))) { - TRANS_LOG(WARN, "for each tx ctx error", KR(ret)); - } - int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("wait_tx_write_end", KR(ret), "cost", end_time - start_time); - return ret; -} - } // transaction } // oceanbase diff --git a/src/storage/ls/ob_ls_tx_service.h b/src/storage/ls/ob_ls_tx_service.h index 536cf0c57..da8ba859e 100644 --- a/src/storage/ls/ob_ls_tx_service.h +++ b/src/storage/ls/ob_ls_tx_service.h @@ -19,12 +19,11 @@ #include "lib/ob_errno.h" #include "lib/lock/ob_spin_rwlock.h" // SpinRWLock -#include "share/ob_ls_id.h" #include "storage/checkpoint/ob_common_checkpoint.h" #include "storage/ob_i_store.h" #include "storage/tablelock/ob_table_lock_common.h" #include "logservice/ob_log_base_type.h" -#include "logservice/rcservice/ob_role_change_handler.h" +#include "logservice/localservice/ob_local_log_handler_set.h" #include "storage/tx/ob_keep_alive_ls_handler.h" namespace oceanbase @@ -46,8 +45,6 @@ class ObTxDesc; class ObLSTxCtxMgr; class ObTxRetainCtxMgr; class ObTransService; -class ObTxLSLogWriter; -class ObTxStartWorkingLog; class ObITxLogAdapter; class ObTxCreateArg; class ObLSTxCtxIterator; @@ -58,13 +55,12 @@ namespace storage { class ObLSTxService : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: ObLSTxService(ObLS *parent) : parent_(parent), - ls_id_(), mgr_(NULL), trans_service_(NULL), rwlock_(common::ObLatchIds::CLOG_CKPT_RWLOCK) { @@ -81,21 +77,18 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, // NB: block_normal and unblcok should pair used !!! // when you finish block_noraml, you should unblock_normal then push to other state public: - int init(const share::ObLSID &ls_id, - transaction::ObLSTxCtxMgr *mgr, + int init(transaction::ObLSTxCtxMgr *mgr, transaction::ObTransService *trans_service); int create_tx_ctx(transaction::ObTxCreateArg arg, bool &existed, - transaction::ObPartTransCtx *&ctx) const; + transaction::ObTxCtx *&ctx) const; int get_tx_ctx(const transaction::ObTransID &tx_id, const bool for_replay, - transaction::ObPartTransCtx *&ctx) const; + transaction::ObTxCtx *&ctx) const; int get_tx_ctx_with_timeout(const transaction::ObTransID &tx_id, const bool for_replay, - transaction::ObPartTransCtx *&tx_ctx, + transaction::ObTxCtx *&tx_ctx, const int64_t lock_timeout) const; - int get_tx_scheduler(const transaction::ObTransID &tx_id, - ObAddr &scheduler) const; int get_tx_start_session_id(const transaction::ObTransID &tx_id, uint32_t &session_id) const; int revert_tx_ctx(transaction::ObTransCtx *ctx) const; int get_read_store_ctx(const transaction::ObTxReadSnapshot &snapshot, @@ -118,7 +111,7 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, // submit next log when all trx in frozen memtable have submitted log int traverse_trans_to_submit_next_log(); // check schduler status for gc - int check_scheduler_status(share::SCN &min_start_scn, transaction::MinStartScnStatus &status); + int check_tx_status(share::SCN &min_start_scn, transaction::MinStartScnStatus &status); // for ls gc // @return OB_SUCCESS, all the tx of this ls cleaned up @@ -160,16 +153,10 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, int get_active_tx_count(int64_t &active_tx_count); int print_all_tx_ctx(const int64_t print_num); - int retry_apply_start_working_log(); - public: int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn); - - int replay_start_working_log(const transaction::ObTxStartWorkingLog &log, share::SCN &log_ts_ns); - void switch_to_follower_forcedly(); - int switch_to_leader(); - int switch_to_follower_gracefully(); - int resume_leader(); + void deactivate() override; + int activate() override; share::SCN get_rec_scn() override; int flush(share::SCN &recycle_scn) override; @@ -178,7 +165,6 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, int get_common_checkpoint_info( ObIArray &common_checkpoint_array); - int wait_tx_write_end(ObTimeoutCtx &timeout_ctx); public: transaction::ObTransService *get_trans_service() { return trans_service_; } @@ -191,8 +177,7 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, // undertake dump int traversal_flush(); virtual share::SCN get_ls_weak_read_ts(); - int check_in_leader_serving_state(bool& bool_ret); - int set_max_replay_commit_version(share::SCN commit_version); + int advance_commit_version(share::SCN commit_version); transaction::ObTxRetainCtxMgr *get_retain_ctx_mgr(); transaction::ObTxLogCbPoolMgr *get_log_cb_pool_mgr(); @@ -202,8 +187,7 @@ class ObLSTxService : public logservice::ObIReplaySubHandler, void reset_(); storage::ObLS *parent_; - - share::ObLSID ls_id_; + transaction::ObLSTxCtxMgr *mgr_; transaction::ObTransService *trans_service_; diff --git a/src/storage/ls/ob_ls_wrs_handler.cpp b/src/storage/ls/ob_ls_wrs_handler.cpp index 176e039c2..00b903c2b 100644 --- a/src/storage/ls/ob_ls_wrs_handler.cpp +++ b/src/storage/ls/ob_ls_wrs_handler.cpp @@ -17,6 +17,7 @@ #define USING_LOG_PREFIX STORAGE #include "ob_ls_wrs_handler.h" #include "storage/tx/ob_trans_service.h" +#include "storage/tx/ob_ts_mgr.h" namespace oceanbase { @@ -27,7 +28,7 @@ using namespace share; namespace storage { -int ObLSWRSHandler::init(const share::ObLSID &ls_id) +int ObLSWRSHandler::init() { int ret = OB_SUCCESS; @@ -38,7 +39,6 @@ int ObLSWRSHandler::init(const share::ObLSID &ls_id) is_enabled_ = false; ls_weak_read_ts_.set_min(); is_inited_ = true; - ls_id_ = ls_id; STORAGE_LOG(INFO, "ObLSWRSHandler init success", K(*this)); } @@ -51,7 +51,6 @@ void ObLSWRSHandler::reset() // set weak read ts to 0 ls_weak_read_ts_.set_min(); is_enabled_ = false; - ls_id_.reset(); } int ObLSWRSHandler::offline() @@ -82,7 +81,6 @@ int ObLSWRSHandler::online() int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, bool &need_skip, - bool &is_user_ls, SCN &wrs_version, const int64_t max_stale_time) { @@ -106,7 +104,7 @@ int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, STORAGE_LOG(INFO, "fail to generate weak read timestamp", KR(ret), K(max_stale_time)); } ret = OB_SUCCESS; - } else if (OB_FAIL(OB_TS_MGR.get_gts(NULL, gts_scn))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts(gts_scn))) { TRANS_LOG(WARN, "get gts scn error", K(ret), K(max_stale_time), K(*this)); } else { int64_t snapshot_version_barrier = gts_scn.convert_to_ts() - max_stale_time; @@ -124,15 +122,6 @@ int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, } } - // check replica type - if (OB_SUCC(ret) && false == need_skip) { - if (ls_id_.is_sys_ls()) { - is_user_ls = false; - } else { - is_user_ls = true; - } - } - // update weak read timestamp if (OB_SUCC(ret)) { if (!need_skip) { @@ -152,36 +141,32 @@ int ObLSWRSHandler::generate_weak_read_timestamp_(ObLS &ls, const int64_t max_st { int ret = OB_SUCCESS; SCN min_log_service_scn, min_tx_service_ts; - const ObLSID &ls_id = ls.get_ls_id(); - // the order of apply service, trx should not be changed if (OB_FAIL(ls.get_max_decided_scn(min_log_service_scn))) { if (OB_STATE_NOT_MATCH == ret) { // print one log per minute if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - STORAGE_LOG(WARN, "get_max_decided_log_ts_ns error", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get_max_decided_log_ts_ns error", K(ret)); } } else { - STORAGE_LOG(WARN, "get_max_decided_log_ts_ns error", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get_max_decided_log_ts_ns error", K(ret)); } } else if (OB_FAIL(ls.get_tx_svr() ->get_trans_service() - ->get_ls_min_uncommit_prepare_version(ls_id, - min_tx_service_ts))) { + ->get_min_uncommit_prepare_version(min_tx_service_ts))) { if (OB_PARTITION_NOT_EXIST == ret) { if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - STORAGE_LOG(WARN, "get_min_uncommit_prepare_version error", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get_min_uncommit_prepare_version error", K(ret)); } } else { - STORAGE_LOG(WARN, "get_min_uncommit_prepare_version error", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "get_min_uncommit_prepare_version error", K(ret)); } } else { timestamp = SCN::min(min_log_service_scn, min_tx_service_ts); const int64_t current_us = ObClockGenerator::getClock(); if (current_us - timestamp.convert_to_ts() > 3000 * 1000L /*3s*/ || REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - TRANS_LOG(INFO, "get wrs ts", K(ls_id), - "delta", current_us - timestamp.convert_to_ts(), + TRANS_LOG(INFO, "get wrs ts", "delta", current_us - timestamp.convert_to_ts(), "log_service_ts", min_log_service_scn.convert_to_ts(), "min_tx_service_ts", min_tx_service_ts.convert_to_ts(), K(timestamp)); diff --git a/src/storage/ls/ob_ls_wrs_handler.h b/src/storage/ls/ob_ls_wrs_handler.h index 42529185c..81fa98f53 100644 --- a/src/storage/ls/ob_ls_wrs_handler.h +++ b/src/storage/ls/ob_ls_wrs_handler.h @@ -22,7 +22,6 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/utility/ob_macro_utils.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "storage/tx/ob_trans_define.h" namespace oceanbase @@ -43,19 +42,18 @@ class ObLSWRSHandler public: ObLSWRSHandler() { reset(); } ~ObLSWRSHandler() { reset(); } - int init(const share::ObLSID &ls_id); + int init(); void reset(); int offline(); int online(); int generate_ls_weak_read_snapshot_version(oceanbase::storage::ObLS &ls, bool &need_skip, - bool &is_user_ls, share::SCN &wrs_version, const int64_t max_stale_time); share::SCN get_ls_weak_read_ts() const { return ls_weak_read_ts_; } bool can_skip_ls() const { return !is_enabled_; } - TO_STRING_KV(K_(is_inited), K_(is_enabled), K_(ls_id), K_(ls_weak_read_ts)); + TO_STRING_KV(K_(is_inited), K_(is_enabled), K_(ls_weak_read_ts)); private: int generate_weak_read_timestamp_(oceanbase::storage::ObLS &ls, const int64_t max_stale_time, share::SCN ×tamp); @@ -68,7 +66,6 @@ class ObLSWRSHandler common::ObSpinLock lock_; bool is_inited_; bool is_enabled_; - share::ObLSID ls_id_; share::SCN ls_weak_read_ts_; }; diff --git a/src/storage/memtable/mvcc/ob_mvcc.h b/src/storage/memtable/mvcc/ob_mvcc.h index 5156d3102..f6bda6670 100644 --- a/src/storage/memtable/mvcc/ob_mvcc.h +++ b/src/storage/memtable/mvcc/ob_mvcc.h @@ -129,7 +129,7 @@ class ObITransCallback // elr_trans_preparing is used for early lock release, if you want to release // the lock after proposing the commit log and even before the commit log - // successfully synced for single ls txn. + // successfully synced for the transaction. virtual int elr_trans_preparing() { return OB_SUCCESS; } // elr_trans_revoke is used to revoke elr after commit failed virtual void elr_trans_revoke() { } @@ -186,4 +186,3 @@ class ObITransCallback }; // namespace oceanbase #endif //OCEANBASE_MEMTABLE_MVCC_OB_MVCC_ - diff --git a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp index 2aa48f87c..2eb62a2a2 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp @@ -39,7 +39,7 @@ int ObMvccAccessCtx::get_write_seq(ObTxSEQ &seq) const { int ret = OB_SUCCESS; // for update uk or pk, set branch part to 0, in orer to let tx-callback fall into single list - if (tx_scn_.support_branch() && write_flag_.is_update_uk()) { + if (write_flag_.is_update_uk()) { const int branch = tx_scn_.get_branch(); if (branch == 0) { seq = tx_scn_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h index 1aa424957..64827e9f3 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h @@ -28,7 +28,7 @@ namespace oceanbase { namespace transaction { -class ObPartTransCtx; +class ObTxCtx; } namespace storage { @@ -156,7 +156,7 @@ class ObMvccAccessCtx && tx_table_guards_.is_valid() && (!tx_ctx_ || mem_ctx_); } - int init_read(transaction::ObPartTransCtx *tx_ctx, /* nullable */ + int init_read(transaction::ObTxCtx *tx_ctx, /* nullable */ ObMemtableCtx *mem_ctx, /* nullable */ storage::ObTxTable *tx_table, const transaction::ObTxSnapshot &snapshot, @@ -199,7 +199,7 @@ class ObMvccAccessCtx snapshot.version_ = snapshot_version; return init_read(NULL, NULL, tx_table, snapshot, timeout, tx_lock_timeout, false, false, NULL); } - int init_write(transaction::ObPartTransCtx &tx_ctx, + int init_write(transaction::ObTxCtx &tx_ctx, ObMemtableCtx &mem_ctx, const transaction::ObTransID &tx_id, const transaction::ObTxSEQ tx_scn, @@ -243,7 +243,7 @@ class ObMvccAccessCtx { abs_lock_timeout_ts_ = abs_lock_timeout; } - int init_replay(transaction::ObPartTransCtx &tx_ctx, + int init_replay(transaction::ObTxCtx &tx_ctx, ObMemtableCtx &mem_ctx, const transaction::ObTransID &tx_id) { @@ -350,7 +350,7 @@ class ObMvccAccessCtx // specials for MvccWrite transaction::ObTransID tx_id_; transaction::ObTxDesc *tx_desc_; // the txn descriptor - transaction::ObPartTransCtx *tx_ctx_; // the txn context + transaction::ObTxCtx *tx_ctx_; // the txn context ObMemtableCtx *mem_ctx_; // memtable-ctx transaction::ObTxSEQ tx_scn_; // the change's number of this modify concurrent_control::ObWriteFlag write_flag_; // the write flag of the write process diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp index bc9c24bd9..81f8ccf8a 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp @@ -15,7 +15,8 @@ */ #include "ob_mvcc_ctx.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" +#include "storage/ls/ob_freezer.h" namespace oceanbase { using namespace common; @@ -359,14 +360,14 @@ int ObIMvccCtx::register_ext_info_commit_cb( } #include "storage/tx/ob_trans_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { namespace memtable { ObMvccWriteGuard::~ObMvccWriteGuard() { if (NULL != ctx_) { int ret = OB_SUCCESS; - transaction::ObPartTransCtx *tx_ctx = ctx_->get_trans_ctx(); + transaction::ObTxCtx *tx_ctx = ctx_->get_trans_ctx(); ctx_->write_done(); if (OB_NOT_NULL(memtable_) diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_ctx.h index 90f0bc2b6..7f0a06674 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.h @@ -35,7 +35,7 @@ class ObTxTableGuard; } namespace transaction { -class ObPartTransCtx; +class ObTxCtx; namespace tablelock { class ObMemCtxLockOpLinkNode; @@ -113,7 +113,7 @@ class ObIMvccCtx virtual void add_trans_mem_total_size(const int64_t size) = 0; virtual void inc_pending_log_size(const int64_t size) = 0; virtual transaction::ObTransID get_tx_id() const = 0; - virtual transaction::ObPartTransCtx *get_trans_ctx() const = 0; + virtual transaction::ObTxCtx *get_trans_ctx() const = 0; // statics maintainness for txn logging virtual void inc_unsubmitted_cnt() = 0; virtual void dec_unsubmitted_cnt() = 0; diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp index 998ce4bfb..3805af706 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp @@ -106,7 +106,6 @@ int ObMvccEngine::try_compact_row_when_mvcc_read_(const SCN &snapshot_version, int ObMvccEngine::get(ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMemtableKey *parameter_key, - const share::ObLSID memtable_ls_id, ObMemtableKey *returned_key, ObMvccValueIterator &value_iter, ObStoreRowLockState &lock_state) @@ -131,7 +130,7 @@ int ObMvccEngine::get(ObMvccAccessCtx &ctx, if (OB_SUCCESS != (tmp_ret = try_compact_row_when_mvcc_read_(ctx.get_snapshot_version(), *value))) { TRANS_LOG(WARN, "fail to try to compact row", K(tmp_ret)); } - } else if (query_flag.is_for_foreign_key_check() || query_flag.is_plain_insert_gts_opt()) { + } else if (query_flag.is_for_foreign_key_check() || query_flag.is_snapshot_opt()) { ret = ObRowConflictHandler::check_foreign_key_constraint_for_memtable(ctx, value, lock_state); } else { // do nothing @@ -140,7 +139,6 @@ int ObMvccEngine::get(ObMvccAccessCtx &ctx, if (OB_FAIL(value_iter.init(ctx, returned_key, value, - memtable_ls_id, query_flag))) { TRANS_LOG(WARN, "ObMvccValueIterator init fail", KR(ret)); } @@ -155,7 +153,6 @@ int ObMvccEngine::scan( ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMvccScanRange &range, - const share::ObLSID memtable_ls_id, ObMvccRowIterator &row_iter) { int ret = OB_SUCCESS; @@ -168,7 +165,6 @@ int ObMvccEngine::scan( } else if (OB_FAIL(row_iter.init(*query_engine_, ctx, range, - memtable_ls_id, query_flag))) { TRANS_LOG(WARN, "row_iter init fail", K(ret)); } else { diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.h b/src/storage/memtable/mvcc/ob_mvcc_engine.h index c5281187a..9f5e373f8 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.h +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.h @@ -111,14 +111,12 @@ class ObMvccEngine int get(ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMemtableKey *parameter_key, - const share::ObLSID memtable_ls_id, ObMemtableKey *internal_key, ObMvccValueIterator &value_iter, storage::ObStoreRowLockState &lock_state); int scan(ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMvccScanRange &range, - const share::ObLSID memtable_ls_id, ObMvccRowIterator &row_iter); int scan(ObMvccAccessCtx &ctx, const ObMvccScanRange &range, diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp index ee4af5f39..3cde49ec6 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp @@ -32,7 +32,6 @@ namespace memtable int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, const ObMemtableKey *key, ObMvccRow *value, - const share::ObLSID memtable_ls_id, const ObQueryFlag &query_flag) { int ret = OB_SUCCESS; @@ -45,7 +44,6 @@ int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, is_inited_ = true; } else { value_ = value; - memtable_ls_id_ = memtable_ls_id; if (OB_FAIL(lock_for_read_(query_flag))) { TRANS_LOG(WARN, "fail to find start pos for iterator", K(ret)); } else { @@ -62,7 +60,6 @@ int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, K(query_flag.is_read_latest()), KPC(key), K(ctx), - K(memtable_ls_id), K(lbt())); return ret; } @@ -345,7 +342,6 @@ void ObMvccValueIterator::move_to_next_node_() ObMvccRowIterator::ObMvccRowIterator() : is_inited_(false), ctx_(NULL), - memtable_ls_id_(), query_flag_(), value_iter_(), query_engine_(NULL), @@ -362,7 +358,6 @@ int ObMvccRowIterator::init( ObQueryEngine &query_engine, ObMvccAccessCtx &ctx, const ObMvccScanRange &range, - const share::ObLSID memtable_ls_id, const ObQueryFlag &query_flag) { int ret = OB_SUCCESS; @@ -378,7 +373,6 @@ int ObMvccRowIterator::init( ctx_ = &ctx; query_flag_ = query_flag; query_engine_ = &query_engine; - memtable_ls_id_ = memtable_ls_id; is_inited_ = true; } return ret; @@ -408,7 +402,7 @@ int ObMvccRowIterator::get_next_row( } else if (NULL == (value = query_engine_iter_->get_value())) { TRANS_LOG(ERROR, "unexpected value null pointer", "ctx", *ctx_); ret = OB_ERR_UNEXPECTED; - } else if (query_flag_.is_for_foreign_key_check() || query_flag_.is_plain_insert_gts_opt()) { + } else if (query_flag_.is_for_foreign_key_check() || query_flag_.is_snapshot_opt()) { if (OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_memtable(*ctx_, value, lock_state))) { // we will throw error code if it's failed here, but we need to // post lock with key outside, so we have to set it here. @@ -417,7 +411,7 @@ int ObMvccRowIterator::get_next_row( } if (OB_FAIL(ret)) { - } else if (OB_FAIL(value_iter_.init(*ctx_, tmp_key, value, memtable_ls_id_, query_flag_))) { + } else if (OB_FAIL(value_iter_.init(*ctx_, tmp_key, value, query_flag_))) { TRANS_LOG(WARN, "value iter init fail", K(ret), "ctx", *ctx_, KP(value), K(*value)); } else if (!value_iter_.is_exist()) { // mvcc row is empty(no tnode), so we continue @@ -434,7 +428,6 @@ void ObMvccRowIterator::reset() { is_inited_ = false; ctx_ = NULL; - memtable_ls_id_.reset(); query_flag_.reset(); value_iter_.reset(); if (NULL != query_engine_iter_) { diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.h b/src/storage/memtable/mvcc/ob_mvcc_iterator.h index 15e4ac6f2..c68f00fc9 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.h +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.h @@ -102,7 +102,6 @@ class ObMvccValueIterator : is_inited_(false), ctx_(NULL), value_(NULL), - memtable_ls_id_(), version_iter_(NULL) { } @@ -111,7 +110,6 @@ class ObMvccValueIterator int init(ObMvccAccessCtx &ctx, const ObMemtableKey *key, ObMvccRow *value, - const share::ObLSID memtable_ls_id, const ObQueryFlag &query_flag); OB_INLINE bool is_exist() { @@ -123,7 +121,6 @@ class ObMvccValueIterator is_inited_ = false; ctx_ = NULL; value_ = NULL; - memtable_ls_id_.reset(); version_iter_ = NULL; } const transaction::ObTransID get_trans_id() const { return ctx_->get_tx_id(); } @@ -145,7 +142,7 @@ class ObMvccValueIterator transaction::ObTransID get_snapshot_tx_id() const { return ctx_->snapshot_.tx_id_; } int64_t get_major_snapshot() const { return ctx_->major_snapshot_; } - TO_STRING_KV(KPC_(value), KPC_(version_iter), KPC_(ctx), K_(memtable_ls_id), K(get_major_snapshot())); + TO_STRING_KV(KPC_(value), KPC_(version_iter), KPC_(ctx), K(get_major_snapshot())); private: int lock_for_read_(const ObQueryFlag &flag); @@ -160,7 +157,6 @@ class ObMvccValueIterator bool is_inited_; ObMvccAccessCtx *ctx_; ObMvccRow *value_; - share::ObLSID memtable_ls_id_; ObMvccTransNode *version_iter_; }; @@ -175,7 +171,6 @@ class ObMvccRowIterator int init(ObQueryEngine &query_engine, ObMvccAccessCtx &ctx, const ObMvccScanRange &range, - const share::ObLSID memtable_ls_id, const ObQueryFlag &query_flag); int get_next_row(const ObMemtableKey *&key, ObMvccValueIterator *&value_iter, @@ -188,7 +183,6 @@ class ObMvccRowIterator private: bool is_inited_; ObMvccAccessCtx *ctx_; - share::ObLSID memtable_ls_id_; ObQueryFlag query_flag_; ObMvccValueIterator value_iter_; ObQueryEngine *query_engine_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index e42a02bb2..e4e126875 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -18,7 +18,7 @@ #include "share/rc/ob_module_provider.h" #include "storage/memtable/ob_row_compactor.h" #include "storage/memtable/ob_lock_wait_mgr.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/access/ob_rows_info.h" #include "storage/truncate_info/ob_truncate_partition_filter.h" @@ -605,7 +605,6 @@ int ObMvccRow::insert_trans_node(ObIMvccCtx &ctx, TRANS_LOG(ERROR, "meet unexpected index_node", KR(ret), K(*prev), K(node), K(*index_node), K(*this)); abort_unless(0); } else if (prev->tx_id_ == node.tx_id_ - && prev->seq_no_.support_branch() && OB_UNLIKELY(prev->seq_no_ > node.seq_no_) // exclude the concurrently update uk case, which always in branch 0 && !(prev->seq_no_.get_branch() == 0 && node.seq_no_.get_branch() == 0)) { @@ -650,8 +649,7 @@ int ObMvccRow::insert_trans_node(ObIMvccCtx &ctx, } } if (OB_SUCC(ret) && OB_NOT_NULL(tmp) && tmp->tx_id_ == node.tx_id_) { - if (tmp->seq_no_.support_branch() - && OB_UNLIKELY(tmp->seq_no_ > node.seq_no_) + if (OB_UNLIKELY(tmp->seq_no_ > node.seq_no_) // exclude the concurrently update uk case, which always in branch 0 && !(tmp->seq_no_.get_branch() == 0 && node.seq_no_.get_branch() == 0)) { ret = OB_ERR_UNEXPECTED; @@ -816,14 +814,12 @@ int ObMvccRow::remove_callback(ObMvccRowCallback &cb) TRANS_LOG(WARN, "MTL(LockWaitMgr) is null", K(ret), KPC(this)); } else { auto tx_ctx = cb.get_trans_ctx(); - ObAddr tx_scheduler; if (OB_ISNULL(tx_ctx)) { int tmp_ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "trans ctx is null", KR(tmp_ret), K(cb)); - } else { - tx_scheduler = static_cast(tx_ctx)->get_scheduler(); } - share::g_mp->lock_wait_mgr()->transform_row_lock_to_tx_lock(cb.get_tablet_id(), *cb.get_key(), ObTransID(node->tx_id_), tx_scheduler); + share::g_mp->lock_wait_mgr()->transform_row_lock_to_tx_lock( + cb.get_tablet_id(), *cb.get_key(), ObTransID(node->tx_id_)); if (cb.is_non_unique_local_index_cb()) { // row lock holder is no need to set for non-unique local index, so the reset can be skipped } else { diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index c62c2ad1a..2e748ee28 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -17,7 +17,7 @@ #include "ob_mvcc_trans_ctx.h" #include "share/rc/ob_module_provider.h" #include "storage/memtable/ob_lock_wait_mgr.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -353,7 +353,7 @@ int ObTransCallbackMgr::append(ObITransCallback *node) node->set_epoch(write_epoch_); } const transaction::ObTxSEQ seq_no = node->get_seq_no(); - if (seq_no.support_branch()) { + { int slot = seq_no.get_branch() % MAX_CALLBACK_LIST_COUNT; if (slot > 0 && for_replay_ @@ -407,16 +407,6 @@ int ObTransCallbackMgr::append(ObITransCallback *node) } else { ret = callback_lists_[slot - 1].append_callback(node, for_replay_, parallel_replay_, is_serial_final_()); } - } else if (!for_replay_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "write by older version", K(ret), K(seq_no), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif - } else { - // for replay, before version 4.2.4 - ret = callback_list_.append_callback(node, for_replay_, parallel_replay_, true); - add_main_list_append_cnt(); } #ifdef ENABLE_DEBUG_LOG memtable_set_injection_sleep(); @@ -448,8 +438,7 @@ int ObTransCallbackMgr::append(ObITransCallback *head, // Step2: find the slot for register or replay const transaction::ObTxSEQ seq_no = head->get_seq_no(); - if (OB_LIKELY(seq_no.support_branch())) { - // NEW since version 4.2.4, select by branch + { int slot = seq_no.get_branch() % MAX_CALLBACK_LIST_COUNT; if (OB_UNLIKELY(slot > 0 && for_replay_ @@ -514,21 +503,6 @@ int ObTransCallbackMgr::append(ObITransCallback *head, parallel_replay_, is_serial_final_()); } - } else if (!for_replay_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "write by older version", K(ret), K(seq_no), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif - } else { - // for replay, before version 4.2.4 - ret = callback_list_.append_callback(head, - tail, - length, - for_replay_, - parallel_replay_, - true /*is_serial_final*/); - add_main_list_append_cnt(); } // Step3: revert the side effect if the append failed @@ -574,7 +548,7 @@ int ObTransCallbackMgr::rollback_to(const ObTxSEQ to_seq_no, int ret = OB_SUCCESS; int slot = -1; remove_cnt = callback_remove_for_rollback_to_count_; - if (OB_LIKELY(to_seq_no.support_branch())) { // since 4.2.4 + { // it is a global savepoint, rollback on all list if (to_seq_no.get_branch() == 0) { CALLBACK_LISTS_FOREACH(idx, list) { @@ -603,14 +577,6 @@ int ObTransCallbackMgr::rollback_to(const ObTxSEQ to_seq_no, KPC(this), KPC(get_trans_ctx()), K(replay_scn), K(to_seq_no), K(from_seq_no)); } } - } else if (!for_replay_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "leader rollback to with old version", K(ret), K(to_seq_no), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif - } else { // for replay, before 4.2.4 - ret = callback_list_.remove_callbacks_for_rollback_to(to_seq_no, from_seq_no, replay_scn); } if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback to fail", K(ret), K(slot), K(from_seq_no), K(to_seq_no)); @@ -619,7 +585,7 @@ int ObTransCallbackMgr::rollback_to(const ObTxSEQ to_seq_no, return ret; } -transaction::ObPartTransCtx *ObTransCallbackMgr::get_trans_ctx() const +transaction::ObTxCtx *ObTransCallbackMgr::get_trans_ctx() const { return host_.get_trans_ctx(); } @@ -2342,4 +2308,3 @@ int64_t ObTransCallbackMgr::get_flushed_log_size() const }; // end namespace mvcc }; // end namespace oceanbase - diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h index d99c61a0a..0fdd8caaa 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h @@ -329,7 +329,7 @@ class ObTransCallbackMgr return (idx == 0) || !for_replay_ || is_serial_final_(); } void print_statistics(char *buf, const int64_t buf_len, int64_t &pos) const; - transaction::ObPartTransCtx *get_trans_ctx() const; + transaction::ObTxCtx *get_trans_ctx() const; TO_STRING_KV(KP(this), K_(serial_final_scn), K_(serial_final_seq_no), diff --git a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp index b447bb456..2183380b4 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp +++ b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ #include "ob_tx_callback_list.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -531,7 +531,7 @@ int ObTxCallbackList::remove_callbacks_for_rollback_to(const transaction::ObTxSE TRANS_LOG(ERROR, "remove callback by rollback wont report error", K(ret), K(functor)); } else { int64_t removed = functor.get_remove_cnt(); - if (to_seq.support_branch() && to_seq.get_branch()) { + if (to_seq.get_branch()) { branch_removed_ += removed; } callback_mgr_.add_rollback_to_callback_remove_cnt(removed); @@ -887,7 +887,7 @@ void ObTxCallbackList::ensure_checksum_(const SCN scn) } } -transaction::ObPartTransCtx *ObTxCallbackList::get_trans_ctx() const +transaction::ObTxCtx *ObTxCallbackList::get_trans_ctx() const { return callback_mgr_.get_trans_ctx(); } @@ -895,14 +895,12 @@ transaction::ObPartTransCtx *ObTxCallbackList::get_trans_ctx() const DEF_TO_STRING(ObTxCallbackList) { int64_t pos = 0; - transaction::ObPartTransCtx *tx_ctx = get_trans_ctx(); + transaction::ObTxCtx *tx_ctx = get_trans_ctx(); const transaction::ObTransID tx_id = tx_ctx ? tx_ctx->get_trans_id() : transaction::ObTransID(); - const share::ObLSID ls_id = tx_ctx ? tx_ctx->get_ls_id() : ObLSID(); J_OBJ_START(); J_KV(K_(id), KP(tx_ctx), K(tx_id), - K(ls_id), K_(appended), K_(length), K_(logged), diff --git a/src/storage/memtable/mvcc/ob_tx_callback_list.h b/src/storage/memtable/mvcc/ob_tx_callback_list.h index 57275bfcc..60813bcac 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_list.h +++ b/src/storage/memtable/mvcc/ob_tx_callback_list.h @@ -111,7 +111,7 @@ class ObTxCallbackList // tx_elr_preparing will elr prepare all callbacks. And it will release the // lock after proposing the commit log and even before the commit log - // successfully synced for single ls txn. + // successfully synced for the transaction. int tx_elr_preparing(); // tx_elr_revoke will clear elr flag on TransNode @@ -206,7 +206,7 @@ class ObTxCallbackList checksum_scn_.inc_update(checksum_scn); } void inc_update_sync_scn(const share::SCN scn); - transaction::ObPartTransCtx *get_trans_ctx() const; + transaction::ObTxCtx *get_trans_ctx() const; bool pending_log_too_large(const int64_t limit) const { return ATOMIC_LOAD(&data_size_) - ATOMIC_LOAD(&logged_data_size_) > limit; diff --git a/src/storage/memtable/ob_concurrent_control.h b/src/storage/memtable/ob_concurrent_control.h index d3b2029c5..8a4ebccc6 100644 --- a/src/storage/memtable/ob_concurrent_control.h +++ b/src/storage/memtable/ob_concurrent_control.h @@ -41,7 +41,7 @@ struct ObWriteFlag #define OBWF_BIT_UPDATE_PK_DOP 1 #define OBWF_BIT_IMMEDIATE_CHECK 1 #define OBWF_BIT_DELETE_INSERT 1 - #define OBWF_BIT_PLAIN_INSERT_GTS_OPT 1 + #define OBWF_BIT_SNAPSHOT_OPT 1 #define OBWF_BIT_RESERVED 50 static const uint64_t OBWF_MASK_RESERVED_COMPAT_0 = (0x1UL << OBWF_BIT_RESERVED_COMPAT_0) - 1; @@ -70,7 +70,7 @@ struct ObWriteFlag uint64_t is_update_pk_dop_ : OBWF_BIT_UPDATE_PK_DOP; // 0: false(default), 1: true uint64_t immediate_row_check_ : OBWF_BIT_IMMEDIATE_CHECK; // 0: false(default), 1: true uint64_t is_delete_insert_ : OBWF_BIT_DELETE_INSERT; // 0: false(default), 1: true - uint64_t is_plain_ins_gts_opt_ : OBWF_BIT_PLAIN_INSERT_GTS_OPT; + uint64_t use_snapshot_opt_ : OBWF_BIT_SNAPSHOT_OPT; uint64_t reserved_ : OBWF_BIT_RESERVED; }; }; @@ -100,8 +100,8 @@ struct ObWriteFlag inline bool is_update_pk_dop() const { return is_update_pk_dop_; } inline void set_immediate_row_check() { immediate_row_check_ = true; } inline bool is_immediate_row_check() const { return immediate_row_check_; } - inline void set_plain_insert_gts_opt() { is_plain_ins_gts_opt_ = true; } - inline bool is_plain_insert_gts_opt() const { return is_plain_ins_gts_opt_; } + inline void set_snapshot_opt() { use_snapshot_opt_ = true; } + inline bool is_snapshot_opt() const { return use_snapshot_opt_; } inline void set_is_delete_insert() { is_delete_insert_ = true; } inline bool is_delete_insert() const { return is_delete_insert_; } @@ -117,7 +117,7 @@ struct ObWriteFlag "is_update_pk_dop", is_update_pk_dop_, "immediate_row_check", immediate_row_check_, "is_delete_insert", is_delete_insert_, - "is_plain_ins_gts_opt", is_plain_ins_gts_opt_); + "use_snapshot_opt", use_snapshot_opt_); OB_UNIS_VERSION(1); }; diff --git a/src/storage/memtable/ob_lock_wait_mgr.cpp b/src/storage/memtable/ob_lock_wait_mgr.cpp index 918ad1f06..b360880c9 100644 --- a/src/storage/memtable/ob_lock_wait_mgr.cpp +++ b/src/storage/memtable/ob_lock_wait_mgr.cpp @@ -561,7 +561,6 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, const uint32_t sess_id, const ObTransID &tx_id, const ObTransID &holder_tx_id, - const ObLSID &ls_id, ObFunction &rechecker) { int ret = OB_SUCCESS; @@ -590,11 +589,11 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, uint32_t client_sid = sql::ObSQLSessionInfo::INVALID_SESSID; if (OB_ISNULL(tx_service = share::g_mp->trans_service())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ObTransService is null", K(sess_id), K(tx_id), K(holder_tx_id), K(ls_id)); + TRANS_LOG(ERROR, "ObTransService is null", K(sess_id), K(tx_id), K(holder_tx_id)); } else if (OB_FAIL(sql::ObBasicSessionInfo::get_client_sid(sess_id, client_sid))) { TRANS_LOG(ERROR, "get client_sid failed", K(ret)); - } else if (OB_FAIL(tx_service->get_trans_start_session_id(ls_id, holder_tx_id, holder_session_id))) { - TRANS_LOG(WARN, "get transaction start session_id failed", K(sess_id), K(tx_id), K(holder_tx_id), K(ls_id)); + } else if (OB_FAIL(tx_service->get_trans_start_session_id(holder_tx_id, holder_session_id))) { + TRANS_LOG(WARN, "get transaction start session_id failed", K(sess_id), K(tx_id), K(holder_tx_id)); } else { ObCStringHelper helper; const char *row_key_str = helper.convert(row_key); @@ -637,7 +636,6 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, const transaction::ObTransID &tx_id, const transaction::ObTransID &holder_tx_id, const ObTableLockMode &lock_mode, - const ObLSID &ls_id, ObFunction &check_need_wait) { int ret = OB_SUCCESS; @@ -670,11 +668,11 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, uint32_t client_sid = sql::ObSQLSessionInfo::INVALID_SESSID; if (OB_ISNULL(tx_service = share::g_mp->trans_service())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ObTransService is null", K(sess_id), K(tx_id), K(holder_tx_id), K(ls_id)); + TRANS_LOG(ERROR, "ObTransService is null", K(sess_id), K(tx_id), K(holder_tx_id)); } else if (OB_FAIL(sql::ObBasicSessionInfo::get_client_sid(sess_id, client_sid))) { TRANS_LOG(ERROR, "get client_sid failed", K(ret)); - } else if (OB_FAIL(tx_service->get_trans_start_session_id(ls_id, holder_tx_id, holder_session_id))) { - TRANS_LOG(WARN, "get transaction start session_id failed", K(sess_id), K(tx_id), K(holder_tx_id), K(ls_id)); + } else if (OB_FAIL(tx_service->get_trans_start_session_id(holder_tx_id, holder_session_id))) { + TRANS_LOG(WARN, "get transaction start session_id failed", K(sess_id), K(tx_id), K(holder_tx_id)); } else { node->set((void*)node, hash, @@ -724,8 +722,7 @@ int ObLockWaitMgr::repost(Node* node) // wait lock on trans_id int ObLockWaitMgr::transform_row_lock_to_tx_lock(const ObTabletID &tablet_id, const Key &row_key, - const ObTransID &tx_id, - const ObAddr &tx_scheduler) + const ObTransID &tx_id) { int ret = OB_SUCCESS; @@ -737,11 +734,7 @@ int ObLockWaitMgr::transform_row_lock_to_tx_lock(const ObTabletID &tablet_id, while (NULL != (node = fetch_waiter(hash_row_key))) { int tmp_ret = OB_SUCCESS; ObTransID self_tx_id(node->tx_id_); - if (tx_scheduler.is_valid()) { - ObTransDeadlockDetectorAdapter::change_detector_waiting_obj_from_row_to_trans(self_tx_id, tx_id, tx_scheduler); - } else { - TRANS_LOG(WARN, "tx scheduler is invalid", K(tx_scheduler), K(tx_id), K(row_key)); - } + ObTransDeadlockDetectorAdapter::change_detector_waiting_obj_from_row_to_trans(self_tx_id, tx_id); node->change_hash(hash_tx_id, lock_seq); if (!wait(node)) { diff --git a/src/storage/memtable/ob_lock_wait_mgr.h b/src/storage/memtable/ob_lock_wait_mgr.h index 842b8dd3f..f4693ca13 100644 --- a/src/storage/memtable/ob_lock_wait_mgr.h +++ b/src/storage/memtable/ob_lock_wait_mgr.h @@ -80,16 +80,13 @@ class DeadLockBlockCallBack { int ret = OB_SUCCESS; UserBinaryKey user_key; ObTransID trans_id; - ObAddr trans_scheduler; ObDependencyResource resource; - #define PRINT_WRAPPER KR(ret), K_(hash), K(trans_id), K(trans_scheduler) + #define PRINT_WRAPPER KR(ret), K_(hash), K(trans_id) if (OB_FAIL(mapper_.get_hash_holder(hash_, trans_id))) { DETECT_LOG(WARN, "get hash holder failed", PRINT_WRAPPER); } else if (OB_FAIL(user_key.set_user_key(trans_id))) { DETECT_LOG(WARN, "set user key failed", PRINT_WRAPPER); - } else if (OB_FAIL(ObTransDeadlockDetectorAdapter::get_conflict_trans_scheduler(trans_id, trans_scheduler))) { - DETECT_LOG(WARN, "get trans scheduler failed", PRINT_WRAPPER); - } else if (OB_FAIL(resource.set_args(trans_scheduler, user_key))) { + } else if (OB_FAIL(resource.set_args(GCTX.self_addr(), user_key))) { DETECT_LOG(WARN, "resource set args failed", PRINT_WRAPPER); } else if (OB_FAIL(resource_array.push_back(resource))) { DETECT_LOG(WARN, "fail to push resource to array", PRINT_WRAPPER); @@ -243,7 +240,6 @@ class ObLockWaitMgr: public share::ObThreadPool const uint32_t sess_id, const transaction::ObTransID &tx_id, const transaction::ObTransID &holder_tx_id, - const ObLSID &ls_id, ObFunction &rechecker); int post_lock(const int tmp_ret, const ObTabletID &tablet_id, @@ -256,14 +252,12 @@ class ObLockWaitMgr: public share::ObThreadPool const transaction::ObTransID &tx_id, const transaction::ObTransID &holder_tx_id, const transaction::tablelock::ObTableLockMode &lock_mode, - const ObLSID &ls_id, ObFunction &check_need_wait); // when removing the callbacks of uncommitted transaction, we need move // the conflict dependency from rows to transactions int transform_row_lock_to_tx_lock(const ObTabletID &tablet_id, const Key &key, - const transaction::ObTransID &tx_id, - const ObAddr &tx_scheduler); + const transaction::ObTransID &tx_id); // wakeup the request waiting on the row void wakeup(const ObTabletID &tablet_id, const Key& key); // wakeup the request waiting on the transaction diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index cbebc87a2..61e938aa5 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -26,7 +26,7 @@ #include "storage/memtable/ob_row_compactor.h" #include "storage/compaction/ob_schedule_dag_func.h" #include "storage/access/ob_sstable_row_getter.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx_storage/ob_tenant_freezer.h" #include "storage/tx_storage/ob_tenant_freezer.h" #include "storage/access/ob_row_sample_iterator.h" @@ -36,6 +36,7 @@ #include "lib/hash/ob_hashmap.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -130,10 +131,10 @@ class ObDirectLoadMemtableRowsLockedChecker ObMemtable::ObMemtable() : is_inited_(false), - recommend_freeze_flag_(false), + recommend_snapshot_freeze_flag_(false), contain_hotspot_row_(false), is_delete_insert_table_(false), - ls_handle_(), + ls_(nullptr), local_allocator_(*this), query_engine_(local_allocator_), mvcc_engine_(), @@ -151,7 +152,7 @@ ObMemtable::~ObMemtable() } int ObMemtable::init(const ObITable::TableKey &table_key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, storage::ObFreezer *freezer, storage::ObTabletMemtableMgr *memtable_mgr, const int64_t schema_version, @@ -166,10 +167,10 @@ int ObMemtable::init(const ObITable::TableKey &table_key, OB_ISNULL(freezer) || OB_ISNULL(memtable_mgr) || schema_version < 0 || - OB_UNLIKELY(!ls_handle.is_valid())) { + OB_UNLIKELY(OB_ISNULL(tenant_ls))) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid param", K(ret), K(table_key), KP(freezer), KP(memtable_mgr), - K(schema_version), K(freeze_clock), K(ls_handle)); + K(schema_version), K(freeze_clock), K(tenant_ls)); } else if (OB_FAIL(set_memtable_mgr_(memtable_mgr))) { TRANS_LOG(WARN, "fail to set memtable mgr", K(ret), KP(memtable_mgr)); } else if (FALSE_IT(set_freeze_clock(freeze_clock))) { @@ -188,8 +189,7 @@ int ObMemtable::init(const ObITable::TableKey &table_key, } else if (OB_FAIL(ObITable::init(table_key))) { TRANS_LOG(WARN, "failed to set_table_key", K(ret), K(table_key)); } else { - ls_handle_ = ls_handle; - ls_id_ = ls_handle_.get_ls()->get_ls_id(); + ls_ = tenant_ls; if (table_key.get_tablet_id().is_sys_tablet()) { mode_ = lib::Worker::CompatMode::MYSQL; } else { @@ -217,7 +217,7 @@ void ObMemtable::pre_batch_destroy_keybtree() } int ObMemtable::batch_remove_unused_callback_for_uncommited_txn( - const ObLSID ls_id, const memtable::ObMemtableSet *memtable_set) + const memtable::ObMemtableSet *memtable_set) { int ret = OB_SUCCESS; // NB: Do not use cache here, because the trans_service may be destroyed under @@ -226,7 +226,7 @@ int ObMemtable::batch_remove_unused_callback_for_uncommited_txn( ::oceanbase::share::g_mp->trans_service(); if (NULL != txs_svr - && OB_FAIL(txs_svr->remove_callback_for_uncommited_txn(ls_id, memtable_set))) { + && OB_FAIL(txs_svr->remove_callback_for_uncommited_txn(memtable_set))) { TRANS_LOG(WARN, "remove callback for uncommited txn failed", K(ret), KPC(memtable_set)); } @@ -255,12 +255,12 @@ void ObMemtable::destroy() time_guard.click(); local_allocator_.destroy(); time_guard.click(); - ls_handle_.reset(); + ls_ = nullptr; max_data_schema_version_ = 0; max_column_cnt_ = 0; state_ = ObMemtableState::INVALID; reported_dml_stat_.reset(); - recommend_freeze_flag_ = false; + recommend_snapshot_freeze_flag_ = false; recommend_snapshot_version_.reset(); contain_hotspot_row_ = false; ObITabletMemtable::reset(); @@ -303,7 +303,7 @@ int ObMemtable::safe_to_destroy(bool &is_safe) if (!is_inited_) { is_safe = true; TRANS_LOG(INFO, "memtable is not inited and safe to destroy", KPC(this)); - } else if (OB_FAIL(ls_handle_.get_ls()->get_max_decided_scn(max_decided_scn))) { + } else if (OB_FAIL(ls_->get_max_decided_scn(max_decided_scn))) { TRANS_LOG(WARN, "fail to get max decided scn", K(ret), K(max_decided_scn)); is_safe = false; } else { @@ -317,8 +317,7 @@ int ObMemtable::safe_to_destroy(bool &is_safe) bool is_done = false; palf::LSN end_lsn; if (OB_FAIL(share::g_mp->log_service()->get_log_apply_service()-> - is_apply_done(ls_handle_.get_ls()->get_ls_id(), - is_done, + is_apply_done(is_done, end_lsn))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; @@ -703,18 +702,17 @@ int ObMemtable::get( } else { const ObColDescIArray &out_cols = read_info->get_columns_desc(); ObStoreRowLockState lock_state; - bool is_plain_insert_gts_opt = context.query_flag_.is_plain_insert_gts_opt(); + bool is_snapshot_opt = context.query_flag_.is_snapshot_opt(); if (OB_FAIL(parameter_mtk.encode(out_cols, &rowkey.get_store_rowkey()))) { TRANS_LOG(WARN, "mtk encode fail", "ret", ret); } else if (OB_FAIL(mvcc_engine_.get(context.store_ctx_->mvcc_acc_ctx_, context.query_flag_, ¶meter_mtk, - ls_id_, &returned_mtk, value_iter, lock_state))) { if (OB_TRY_LOCK_ROW_CONFLICT == ret || OB_TRANSACTION_SET_VIOLATION == ret) { - if (!context.query_flag_.is_for_foreign_key_check() && !is_plain_insert_gts_opt) { + if (!context.query_flag_.is_for_foreign_key_check() && !is_snapshot_opt) { ret = OB_ERR_UNEXPECTED; // to prevent retrying casued by throwing 6005 TRANS_LOG(WARN, "should not meet lock conflict if it's not for foreign key check", K(ret), K(context.query_flag_)); @@ -724,7 +722,6 @@ int ObMemtable::get( *parameter_mtk.get_rowkey(), lock_state, key_.tablet_id_, - get_ls_id(), 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, lock_state.trans_scn_); } @@ -1011,7 +1008,7 @@ int ObMemtable::replay_row(ObStoreCtx &ctx, ObRowData old_row; blocksstable::ObDmlFlag dml_flag = blocksstable::ObDmlFlag::DF_NOT_EXIST; ObMemtableCtx *mt_ctx = ctx.mvcc_acc_ctx_.mem_ctx_; - ObPartTransCtx *part_ctx = static_cast(mt_ctx->get_trans_ctx()); + ObTxCtx *part_ctx = static_cast(mt_ctx->get_trans_ctx()); common::ObTimeGuard timeguard("ObMemtable::replay_row", 5 * 1000); if (OB_FAIL(mmi->get_mutator_row().copy(table_id, rowkey, table_version, row, @@ -1609,14 +1606,13 @@ bool ObMemtable::ready_for_flush_() int ret = OB_SUCCESS; SCN current_right_boundary = ObScnRange::MIN_SCN; - share::ObLSID ls_id = get_ls_id(); if (bool_ret) { if (OB_FAIL(resolve_snapshot_version_())) { - TRANS_LOG(WARN, "fail to resolve snapshot version", K(ret), KPC(this), K(ls_id)); + TRANS_LOG(WARN, "fail to resolve snapshot version", K(ret), KPC(this)); } else if (OB_FAIL(resolve_max_end_scn_())) { - TRANS_LOG(WARN, "fail to resolve max_end_scn", K(ret), KPC(this), K(ls_id)); + TRANS_LOG(WARN, "fail to resolve max_end_scn", K(ret), KPC(this)); } else { - TRANS_LOG(INFO, "[resolve_right_boundary] ready_for_flush_", K(ls_id), KPC(this)); + TRANS_LOG(INFO, "[resolve_right_boundary] ready_for_flush_", KPC(this)); if (OB_FAIL(get_ls_current_right_boundary_(current_right_boundary))) { TRANS_LOG(WARN, "fail to get current right boundary", K(ret)); } @@ -1639,7 +1635,7 @@ bool ObMemtable::ready_for_flush_() } } - TRANS_LOG(INFO, "ready for flush", K(bool_ret), K(ret), K(current_right_boundary), K(ls_id), KPC(this)); + TRANS_LOG(INFO, "ready for flush", K(bool_ret), K(ret), K(current_right_boundary), KPC(this)); } } else if (is_frozen && get_logging_blocked()) { // ensure unset all frozen memtables'logging_block @@ -1662,7 +1658,7 @@ bool ObMemtable::ready_for_flush_() ObTimeUtility::current_time() - mt_stat_.frozen_time_ > 10 * 1000 * 1000L)) { if (ObTimeUtility::current_time() - mt_stat_.last_print_time_ > 10 * 1000) { STORAGE_LOG(WARN, "memtable not ready for flush for long time", - K(get_ls_id()), K(*this), K(mt_stat_.frozen_time_), + K(*this), K(mt_stat_.frozen_time_), K(current_right_boundary)); mt_stat_.last_print_time_ = ObTimeUtility::current_time(); } @@ -1675,7 +1671,7 @@ bool ObMemtable::ready_for_flush_() int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MINI_MERGE, ObDiagnoseTabletType::TYPE_MINI_MERGE, - ls_id, get_tablet_id(), + get_tablet_id(), ObSuspectInfoType::SUSPECT_NOT_READY_FOR_FLUSH, static_cast(is_frozen_memtable()), get_write_ref(), get_unsubmitted_cnt(), current_right_boundary.get_val_for_tx(), get_end_scn().get_val_for_tx()))) { @@ -1689,7 +1685,6 @@ bool ObMemtable::ready_for_flush_() void ObMemtable::print_ready_for_flush() { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); const common::ObTabletID tablet_id = key_.tablet_id_; bool frozen_memtable_flag = is_frozen_memtable(); int64_t write_ref = get_write_ref(); @@ -1705,7 +1700,7 @@ void ObMemtable::print_ready_for_flush() current_right_boundary >= end_scn; TRANS_LOG(INFO, "[ObFreezer] print_ready_for_flush", - KP(this), K(ls_id), K(tablet_id), + KP(this), K(tablet_id), K(ret), K(bool_ret), K(frozen_memtable_flag), K(write_ref), K(current_right_boundary), K(end_scn), @@ -1758,16 +1753,16 @@ int ObMemtable::resolve_snapshot_version_() } else if (SCN::invalid_scn() == freeze_snapshot_version) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "fail to get freeze_snapshot_version", K(ret), KPC(this)); - } else if (is_recommend_freeze()) { + } else if (has_recommend_snapshot_freeze()) { // freeze_snapshot_version is used for read tables decision which guarantees // that all version smaller than the freeze_snapshot_version belongs to - // table before the memtable. Some callers need it to be smaller than the - // reserved_scn, which requires ignoring the input snapshot version. + // table before the memtable. Some replay paths require a smaller boundary, + // so they set a recommended snapshot version explicitly. // // NOTICE: While the recommend snapshot may be unsafe, so user must ensure // its correctness. // - // So use the recommended snapshot version for this freeze. + // Use the recommended snapshot version when it is set explicitly. if (!recommend_snapshot_version_.is_valid() || ObScnRange::MAX_SCN == recommend_snapshot_version_ || ObScnRange::MIN_SCN == recommend_snapshot_version_) { @@ -1810,11 +1805,13 @@ int ObMemtable::resolve_max_end_scn_() TRANS_LOG(ERROR, "fail to get freeze_snapshot_version", K(ret)); } else if (SCN::invalid_scn() == max_decided_scn) { // Pass if not necessary - } else if (is_recommend_freeze()) { + } else if (has_recommend_snapshot_freeze()) { // max_decided_scn is critial for sstable read performance using larger // right boundary of memtable(You can learn from the comments that follow - // the class member). Some callers need the right boundary to stay smaller - // than the recommended SCN, so ignore max_decided_scn. + // the class member). Some replay paths require a smaller right boundary, + // so max_decided_scn is ignored when a recommended snapshot is set. + // + // Pass when recommended snapshot freeze is set. } else if (OB_TMP_FAIL(set_max_end_scn(max_decided_scn))) { TRANS_LOG(WARN, "fail to set max_end_scn", K(ret)); } @@ -1822,7 +1819,7 @@ int ObMemtable::resolve_max_end_scn_() return ret; } -int ObMemtable::flush(share::ObLSID ls_id) +int ObMemtable::flush() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -1832,7 +1829,6 @@ int ObMemtable::flush(share::ObLSID ls_id) ret = OB_NO_NEED_UPDATE; } else { ObTabletMergeDagParam param; - param.ls_id_ = ls_id; param.tablet_id_ = key_.tablet_id_; param.merge_type_ = MINI_MERGE; param.merge_version_ = ObVersion::MIN_VERSION; @@ -1851,10 +1847,10 @@ int ObMemtable::flush(share::ObLSID ls_id) mt_stat_.ready_for_flush_time_ != 0 && cur_time - mt_stat_.ready_for_flush_time_ > 30 * 1000 * 1000) { STORAGE_LOG(WARN, "memtable can not create dag successfully for long time", - K(ls_id), K(*this), K(mt_stat_.ready_for_flush_time_)); + K(*this), K(mt_stat_.ready_for_flush_time_)); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MINI_MERGE, ObDiagnoseTabletType::TYPE_MINI_MERGE, - ls_id, get_tablet_id(), + get_tablet_id(), ObSuspectInfoType::SUSPECT_MEMTABLE_CANT_CREATE_DAG, static_cast(ret), cur_time - mt_stat_.ready_for_flush_time_, mt_stat_.ready_for_flush_time_))) { @@ -1872,7 +1868,7 @@ void ObMemtable::fill_compaction_param_( { ObCompactionParam &compaction_param = param.compaction_param_; compaction_param.occupy_size_ = get_occupied_size(); - compaction_param.replay_interval_ = get_start_scn().get_val_for_tx() - ls_handle_.get_ls()->get_ls_meta().get_clog_checkpoint_scn().get_val_for_tx(); + compaction_param.replay_interval_ = get_start_scn().get_val_for_tx() - ls_->get_ls_meta().get_clog_checkpoint_scn().get_val_for_tx(); compaction_param.last_end_scn_ = get_end_scn(); compaction_param.add_time_ = current_time; compaction_param.estimate_phy_size_ = mt_stat_.row_size_; @@ -2923,9 +2919,7 @@ int ObMemtable::post_row_write_conflict_(ObMvccAccessCtx &acc_ctx, mem_ctx->add_conflict_trans_id(conflict_tx_id); mem_ctx->on_wlock_retry(row_key, conflict_tx_id); int tmp_ret = OB_SUCCESS; - transaction::ObPartTransCtx *tx_ctx = acc_ctx.tx_ctx_; transaction::ObTransID tx_id = acc_ctx.get_tx_id(); - bool remote_tx = tx_ctx->get_scheduler() != tx_ctx->get_addr(); ObFunction recheck_func([&](bool &locked, bool &wait_on_row) -> int { int ret = OB_SUCCESS; lock_state.is_locked_ = false; @@ -2951,13 +2945,12 @@ int ObMemtable::post_row_write_conflict_(ObMvccAccessCtx &acc_ctx, key_.get_tablet_id(), *row_key.get_rowkey(), lock_wait_expire_ts, - remote_tx, + false, last_compact_cnt, total_trans_node_cnt, acc_ctx.tx_desc_->get_assoc_session_id(), tx_id, conflict_tx_id, - get_ls_id(), recheck_func); if (OB_SUCCESS != tmp_ret) { TRANS_LOG(WARN, "post_lock after tx conflict failed", @@ -2973,10 +2966,10 @@ int ObMemtable::get_tx_table_guard(ObTxTableGuard &tx_table_guard) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_handle_.is_valid())) { + if (OB_UNLIKELY(OB_ISNULL(ls_))) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ls_handle is invalid", K(ret)); - } else if (OB_FAIL(ls_handle_.get_ls()->get_tx_table_guard(tx_table_guard))) { + TRANS_LOG(ERROR, "tenant_ls is invalid", K(ret)); + } else if (OB_FAIL(ls_->get_tx_table_guard(tx_table_guard))) { TRANS_LOG(WARN, "Get tx table guard from ls failed.", KR(ret)); } @@ -2992,7 +2985,7 @@ bool ObMemtable::rec_scn_is_stable() } else { SCN max_consequent_callbacked_scn; if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { - STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret)); } else { rec_scn_is_stable = (max_consequent_callbacked_scn >= get_rec_scn()); } @@ -3001,13 +2994,12 @@ bool ObMemtable::rec_scn_is_stable() (mt_stat_.frozen_time_ != 0 && ObTimeUtility::current_time() - mt_stat_.frozen_time_ > 10 * 1000 * 1000L)) { STORAGE_LOG(WARN, "memtable rec_scn not stable for long time", - K(get_ls_id()), K(*this), K(mt_stat_.frozen_time_), + K(*this), K(mt_stat_.frozen_time_), K(max_consequent_callbacked_scn)); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(ADD_SUSPECT_INFO(compaction::ObMergeType::MINI_MERGE, ObDiagnoseTabletType::TYPE_MINI_MERGE, - get_ls_id(), get_tablet_id(), ObSuspectInfoType::SUSPECT_REC_SCN_NOT_STABLE, get_rec_scn().get_val_for_tx(), @@ -3022,7 +3014,6 @@ bool ObMemtable::rec_scn_is_stable() int64_t ObMemtable::dec_write_ref() { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); // fix issue 47021079 // To avoid the following case where logging_block cannot be unset: @@ -3049,7 +3040,7 @@ int64_t ObMemtable::dec_write_ref() (void)unset_logging_blocked_for_active_memtable_(); share::SCN right_boundary; if (OB_FAIL(get_ls_current_right_boundary_(right_boundary))) { - TRANS_LOG(WARN, "get ls right bound fail", K(ret), K(ls_id), KPC(this)); + TRANS_LOG(WARN, "get ls right bound fail", K(ret), KPC(this)); } else if (right_boundary >= get_max_end_scn()) { resolve_right_boundary(); if (OB_LIKELY(!get_resolved_active_memtable_left_boundary())) { diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index 0b6415a5c..38001181e 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -195,7 +195,7 @@ class ObMemtable : public ObITabletMemtable public: // derived from ObITabletMemtable virtual int init(const ObITable::TableKey &table_key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, storage::ObFreezer *freezer, storage::ObTabletMemtableMgr *memtable_mgr, const int64_t schema_version, @@ -349,11 +349,10 @@ class ObMemtable : public ObITabletMemtable const ObMvccEngine &get_mvcc_engine() const { return mvcc_engine_; } void pre_batch_destroy_keybtree(); static int batch_remove_unused_callback_for_uncommited_txn( - const share::ObLSID ls_id, const memtable::ObMemtableSet *memtable_set); /* freeze */ - virtual int flush(share::ObLSID ls_id) override; + virtual int flush() override; bool is_empty() const override { @@ -369,12 +368,12 @@ class ObMemtable : public ObITabletMemtable // recommend scn should belong to the tables before the memtable and the // memtable. And under exception case, user need guarantee all new data is // bigger than the recommend_scn. - inline void set_recommend_freeze(const share::SCN recommend_scn) + inline void set_recommend_snapshot_freeze(const share::SCN recommend_scn) { recommend_snapshot_version_.atomic_set(recommend_scn); - ATOMIC_STORE(&recommend_freeze_flag_, true); + ATOMIC_STORE(&recommend_snapshot_freeze_flag_, true); } - inline bool is_recommend_freeze() const { return ATOMIC_LOAD(&recommend_freeze_flag_); } + inline bool has_recommend_snapshot_freeze() const { return ATOMIC_LOAD(&recommend_snapshot_freeze_flag_); } virtual void set_delete_insert_flag(const bool rhs) override { is_delete_insert_table_ = rhs; } inline bool is_delete_insert_table() const { return is_delete_insert_table_; } virtual uint32_t get_freeze_flag() override; @@ -402,8 +401,7 @@ class ObMemtable : public ObITabletMemtable K_(contain_hotspot_row), K_(snapshot_version), K_(contain_hotspot_row), - K_(ls_id), - K_(recommend_freeze_flag), + K_(recommend_snapshot_freeze_flag), K_(recommend_snapshot_version), K_(is_delete_insert_table)); private: @@ -501,11 +499,11 @@ class ObMemtable : public ObITabletMemtable private: DISALLOW_COPY_AND_ASSIGN(ObMemtable); bool is_inited_; - bool recommend_freeze_flag_; + bool recommend_snapshot_freeze_flag_; bool contain_hotspot_row_; bool is_delete_insert_table_; - storage::ObLSHandle ls_handle_; + storage::ObLS *ls_; ObSingleMemstoreAllocator local_allocator_; ObMTKVBuilder kv_builder_; ObQueryEngine query_engine_; diff --git a/src/storage/memtable/ob_memtable_context.cpp b/src/storage/memtable/ob_memtable_context.cpp index aa24a5cc7..59b507d4f 100644 --- a/src/storage/memtable/ob_memtable_context.cpp +++ b/src/storage/memtable/ob_memtable_context.cpp @@ -15,7 +15,7 @@ */ #include "ob_memtable_context.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tablelock/ob_lock_memtable.h" #include "storage/tx/ob_trans_deadlock_adapter.h" @@ -181,12 +181,6 @@ int64_t ObMemtableCtx::to_string(char *buf, const int64_t buf_len) const } else { common::databuff_printf(buf, buf_len, pos, ctx_->get_trans_id()); } - common::databuff_printf(buf, buf_len, pos, " ls_id="); - if (OB_ISNULL(ctx_)) { - common::databuff_printf(buf, buf_len, pos, "-1"); - } else { - common::databuff_printf(buf, buf_len, pos, ctx_->get_ls_id().id()); - } common::databuff_printf(buf, buf_len, pos, " row_callback[alloc:%ld, free:%ld, unsubmit:%ld] " "redo[fill:%ld,sync_succ:%ld, sync_fail:%ld] " "main_list_len=%ld pending_log_size=%ld ", @@ -206,7 +200,7 @@ void ObMemtableCtx::set_read_only() ATOMIC_STORE(&is_read_only_, true); } -void ObMemtableCtx::set_trans_ctx(ObPartTransCtx *ctx) +void ObMemtableCtx::set_trans_ctx(ObTxCtx *ctx) { ATOMIC_STORE(&ctx_, ctx); } @@ -249,21 +243,21 @@ int ObMemtableCtx::write_auth(const bool exclusive) if (ATOMIC_LOAD(&is_read_only_)) { ret = OB_ERR_READ_ONLY_TRANSACTION; TRANS_LOG(ERROR, "WriteAuth: readonly trans not support update operation", - "trans_id", ctx_->get_trans_id(), "ls_id", ctx_->get_ls_id(), K(ret)); + "trans_id", ctx_->get_trans_id(), K(ret)); } else if (!ATOMIC_LOAD(&is_master_)) { ret = OB_NOT_MASTER; TRANS_LOG(WARN, "WriteAuth: trans is already not master", - "trans_id", ctx_->get_trans_id(), "ls_id", ctx_->get_ls_id(), K(ret)); + "trans_id", ctx_->get_trans_id(), K(ret)); } else if (OB_SUCCESS != ATOMIC_LOAD(&end_code_)) { ret = ATOMIC_LOAD(&end_code_); TRANS_LOG(WARN, "WriteAuth: trans is already end", K(ret), - "trans_id", ctx_->get_trans_id(), "ls_id", ctx_->get_ls_id(), K_(end_code)); + "trans_id", ctx_->get_trans_id(), K_(end_code)); } else if (is_tx_rollbacked()) { // The txn has been killed during normal processing. So we return // OB_TRANS_KILLED to prompt this abnormal state. ret = OB_TRANS_KILLED; TRANS_LOG(WARN, "WriteAuth: trans is already end", K(ret), - "trans_id", ctx_->get_trans_id(), "ls_id", ctx_->get_ls_id(), K_(end_code)); + "trans_id", ctx_->get_trans_id(), K_(end_code)); } else if (lock_succ) { // all check passed after lock succ break; @@ -709,38 +703,15 @@ int ObMemtableCtx::get_callback_list_stat(ObIArray &stats) //////////////////////////////////////////////////////////////////////////////////////////////////// -int ObMemtableCtx::get_conflict_trans_ids(common::ObIArray &array) +int ObMemtableCtx::get_conflict_trans_ids(common::ObIArray &array) { int ret = OB_SUCCESS; - common::ObArray conflict_ids; { ObByteLockGuard guard(lock_, ObWaitEventIds::MEMTABLE_CTX_ACCESS_LOCK); - ret = conflict_ids.assign(conflict_trans_ids_); + ret = array.assign(conflict_trans_ids_); } if (OB_FAIL(ret)) { DETECT_LOG(ERROR, "fail to copy conflict_trans_ids_", KR(ret), KPC(this)); - } else if (OB_ISNULL(ctx_)) { - ret = OB_BAD_NULL_ERROR; - DETECT_LOG(ERROR, "trans part ctx on ObMemtableCtx is NULL", KR(ret), KPC(this)); - } else { - for (int64_t idx = 0; idx < conflict_ids.count() && OB_SUCC(ret); ++idx) { - ObAddr scheduler_addr; - if (OB_FAIL(ObTransDeadlockDetectorAdapter::get_trans_scheduler_info_on_participant(conflict_ids.at(idx), - ctx_->get_ls_id(), - scheduler_addr))) { - if (ret == OB_TRANS_CTX_NOT_EXIST) { - DETECT_LOG(INFO, "tx ctx not exist anymore, give up blocking on this tx_id", - KR(ret), KPC(this), K(conflict_ids), K(array), K(idx)); - ret = OB_SUCCESS; - } else { - DETECT_LOG(WARN, "fail to get conflict trans scheduler addr", - KR(ret), KPC(this), K(conflict_ids), K(array), K(idx)); - } - } else if (OB_FAIL(array.push_back({conflict_ids[idx], scheduler_addr}))) { - DETECT_LOG(ERROR, "copy block trans id failed", - KR(ret), KPC(this), K(conflict_ids), K(array), K(idx)); - } - } } return ret; } @@ -968,15 +939,6 @@ int ObMemtableCtx::get_table_lock_store_info(ObTableLockInfo &table_lock_info) return ret; } -int ObMemtableCtx::get_table_lock_for_reserved(ObTableLockInfo &table_lock_info, const ObIArray &tablet_list) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(lock_mem_ctx_.get_table_lock_for_reserved(table_lock_info, tablet_list))) { - TRANS_LOG(WARN, "get tablet lock for reserved status failed", K(ret)); - } - return ret; -} - int ObMemtableCtx::recover_from_table_lock_durable_info(const ObTableLockInfo &table_lock_info) { int ret = OB_SUCCESS; @@ -1203,7 +1165,7 @@ int ObMemtableCtx::register_multi_source_data_if_need_( int64_t serialize_size = lock_op.get_serialize_size(); int64_t pos = 0; ObTxDataSourceType type = ObTxDataSourceType::TABLE_LOCK; - ObPartTransCtx *part_ctx = static_cast(ctx_); + ObTxCtx *part_ctx = static_cast(ctx_); if (serialize_size > tablelock::ObTableLockOp::MAX_SERIALIZE_SIZE) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index 8dc3237fd..88d22693f 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -349,7 +349,7 @@ class ObMemtableCtx : public ObIMemtableCtx // transaction context, and carries it back after execution, for dead lock detect use virtual int add_conflict_trans_id(const transaction::ObTransID conflict_trans_id); void reset_conflict_trans_ids(); - int get_conflict_trans_ids(common::ObIArray &array); + int get_conflict_trans_ids(common::ObIArray &array); virtual int read_lock_yield() { return ATOMIC_LOAD(&end_code_); @@ -407,8 +407,8 @@ class ObMemtableCtx : public ObIMemtableCtx int log_submitted(const ObRedoLogSubmitHelper &helper); int sync_log_succ(const share::SCN scn, const ObCallbackScopeArray &callbacks); void sync_log_fail(const ObCallbackScopeArray &callbacks, const share::SCN &max_applied_scn); - virtual void set_trans_ctx(transaction::ObPartTransCtx *ctx); - virtual transaction::ObPartTransCtx *get_trans_ctx() const { return ctx_; } + virtual void set_trans_ctx(transaction::ObTxCtx *ctx); + virtual transaction::ObTxCtx *get_trans_ctx() const { return ctx_; } virtual void inc_truncate_cnt() override { truncate_cnt_++; } int get_memtable_key_arr(transaction::ObMemtableKeyArray &memtable_key_arr); uint64_t get_lock_for_read_retry_count() const { return lock_for_read_retry_count_; } @@ -533,7 +533,6 @@ class ObMemtableCtx : public ObIMemtableCtx const share::SCN &scn); int recover_from_table_lock_durable_info(const ObTableLockInfo &table_lock_info); int get_table_lock_store_info(ObTableLockInfo &table_lock_info); - int get_table_lock_for_reserved(ObTableLockInfo &table_lock_info, const ObIArray &tablet_list); // for deadlock detect. void set_table_lock_killed() { lock_mem_ctx_.set_killed(); } bool is_table_lock_killed() const { return lock_mem_ctx_.is_killed(); } @@ -592,7 +591,7 @@ class ObMemtableCtx : public ObIMemtableCtx ObMemtableCtxCbAllocator ctx_cb_allocator_; ObRedoLogGenerator log_gen_; RetryInfo retry_info_; - transaction::ObPartTransCtx *ctx_; + transaction::ObTxCtx *ctx_; int64_t truncate_cnt_; // the retry count of lock for read uint64_t lock_for_read_retry_count_; diff --git a/src/storage/memtable/ob_memtable_interface.cpp b/src/storage/memtable/ob_memtable_interface.cpp deleted file mode 100644 index 2029f0d6c..000000000 --- a/src/storage/memtable/ob_memtable_interface.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_memtable_interface.h" - -namespace oceanbase -{ -using namespace share; -namespace storage -{ - -int ObIMemtable::get_ls_id(ObLSID &ls_id) { - int ret = OB_SUCCESS; - ls_id = get_ls_id(); - return ret; -} -ObLSID ObIMemtable::get_ls_id() const { return ls_id_; } - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/memtable/ob_memtable_interface.h b/src/storage/memtable/ob_memtable_interface.h index 7a32be465..a81f397e8 100644 --- a/src/storage/memtable/ob_memtable_interface.h +++ b/src/storage/memtable/ob_memtable_interface.h @@ -68,10 +68,10 @@ class ObIMemtableCtx : public ObIMvccCtx virtual int replay_to_commit(const bool is_resume) = 0; //method called when leader revoke virtual int commit_to_replay() = 0; - virtual void set_trans_ctx(transaction::ObPartTransCtx *ctx) = 0; + virtual void set_trans_ctx(transaction::ObTxCtx *ctx) = 0; virtual void inc_truncate_cnt() = 0; - virtual int get_conflict_trans_ids(common::ObIArray &array) = 0; + virtual int get_conflict_trans_ids(common::ObIArray &array) = 0; VIRTUAL_TO_STRING_KV("", ""); public: // return OB_AGAIN/OB_SUCCESS @@ -112,18 +112,14 @@ struct CreateMemtableArg { class ObIMemtable : public storage::ObITable { public: ObIMemtable() - : ls_id_(), - snapshot_version_(share::SCN::max_scn()) + : snapshot_version_(share::SCN::max_scn()) {} virtual ~ObIMemtable() {} void reset() { ObITable::reset(); - ls_id_.reset(); snapshot_version_.set_max(); } - int get_ls_id(share::ObLSID &ls_id); - share::ObLSID get_ls_id() const; virtual ObTabletID get_tablet_id() const = 0; virtual int get(const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, @@ -191,7 +187,6 @@ class ObIMemtable : public storage::ObITable { virtual int64_t dec_ref() { return ObITable::dec_ref(); } protected: - share::ObLSID ls_id_; share::SCN snapshot_version_; }; } // namespace storage diff --git a/src/storage/memtable/ob_memtable_single_row_reader.cpp b/src/storage/memtable/ob_memtable_single_row_reader.cpp index 7d4794e6a..153e674fd 100644 --- a/src/storage/memtable/ob_memtable_single_row_reader.cpp +++ b/src/storage/memtable/ob_memtable_single_row_reader.cpp @@ -124,7 +124,6 @@ int ObMemtableSingleRowReader::init_a_new_range(const ObDatumRange &new_range_to if (OB_FAIL(mvcc_engine.scan(context_->store_ctx_->mvcc_acc_ctx_, context_->query_flag_, mvcc_scan_range, - memtable_->get_ls_id(), row_iter_))) { TRANS_LOG(WARN, "mvcc engine scan fail", K(ret), K(mvcc_scan_range)); } else if (OB_FAIL(bitmap_.init(read_info_->get_request_count(), read_info_->get_schema_rowkey_count()))) { @@ -313,7 +312,6 @@ int ObMemtableSingleRowReader::get_next_value_iter_(const ObMemtableKey *&key, O *tmp_rowkey, lock_state, context_->tablet_id_, - context_->ls_id_, 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, lock_state.trans_scn_); diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index 5d9da2ba3..ff66bc667 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -15,7 +15,7 @@ */ #include "ob_redo_log_generator.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { diff --git a/src/storage/memtable/ob_row_conflict_handler.cpp b/src/storage/memtable/ob_row_conflict_handler.cpp index 0de70ee22..143b791bf 100644 --- a/src/storage/memtable/ob_row_conflict_handler.cpp +++ b/src/storage/memtable/ob_row_conflict_handler.cpp @@ -50,7 +50,6 @@ int ObRowConflictHandler::check_row_locked(const storage::ObTableIterParam ¶ rowkey.get_store_rowkey(), lock_state, context.tablet_id_, - context.ls_id_, 0, 0, /* these two params get from mvcc_row, and for statistics, so we ignore them */ lock_state.trans_scn_); @@ -258,7 +257,6 @@ int ObRowConflictHandler::post_row_read_conflict(ObMvccAccessCtx &acc_ctx, const ObStoreRowkey &row_key, ObStoreRowLockState &lock_state, const ObTabletID tablet_id, - const share::ObLSID ls_id, const int64_t last_compact_cnt, const int64_t total_trans_node_cnt, const share::SCN &trans_scn) @@ -282,19 +280,7 @@ int ObRowConflictHandler::post_row_read_conflict(ObMvccAccessCtx &acc_ctx, } else { int tmp_ret = OB_SUCCESS; ObTransID tx_id = acc_ctx.get_tx_id(); - ObAddr conflict_scheduler_addr; - // register to deadlock detector - if (OB_FAIL(ObTransDeadlockDetectorAdapter:: - get_trans_scheduler_info_on_participant(conflict_tx_id, ls_id, conflict_scheduler_addr))) { - TRANS_LOG(WARN, "get transaction scheduler info fail", K(ret), K(conflict_tx_id), K(tx_id), K(ls_id)); - } else { - ObTransIDAndAddr conflict_tx(conflict_tx_id, conflict_scheduler_addr); - tx_desc->add_conflict_tx(conflict_tx); - } - // The addr in tx_desc is the scheduler_addr of current trans, - // and GCTX.self_addr() will return the addr where the row is stored - // (i.e. where the trans is executing) - bool remote_tx = tx_desc->get_addr() != GCTX.self_addr(); + tx_desc->add_conflict_tx(conflict_tx_id); ObFunction recheck_func([&](bool &locked, bool &wait_on_row) -> int { int ret = OB_SUCCESS; lock_state.is_locked_ = false; @@ -320,13 +306,12 @@ int ObRowConflictHandler::post_row_read_conflict(ObMvccAccessCtx &acc_ctx, tablet_id, row_key, lock_wait_expire_ts, - remote_tx, + false, last_compact_cnt, total_trans_node_cnt, tx_desc->get_assoc_session_id(), tx_id, conflict_tx_id, - ls_id, recheck_func); if (OB_SUCCESS != tmp_ret) { TRANS_LOG(WARN, "post_lock after tx conflict failed", diff --git a/src/storage/memtable/ob_row_conflict_handler.h b/src/storage/memtable/ob_row_conflict_handler.h index bb3768c45..8a9c23875 100644 --- a/src/storage/memtable/ob_row_conflict_handler.h +++ b/src/storage/memtable/ob_row_conflict_handler.h @@ -35,7 +35,6 @@ class ObStoreRowkey; class ObQueryFlag; } namespace share { -class ObLSID; class SCN; } namespace blocksstable { @@ -91,7 +90,6 @@ class ObRowConflictHandler { const common::ObStoreRowkey &row_key, storage::ObStoreRowLockState &lock_state, const common::ObTabletID tablet_id, - const share::ObLSID ls_id, const int64_t last_compact_cnt, const int64_t total_trans_node_cnt, const share::SCN &trans_scn); diff --git a/src/storage/memtable/ob_row_conflict_info.h b/src/storage/memtable/ob_row_conflict_info.h index c9ca5732f..9d22877f5 100644 --- a/src/storage/memtable/ob_row_conflict_info.h +++ b/src/storage/memtable/ob_row_conflict_info.h @@ -18,7 +18,6 @@ #include "lib/net/ob_addr.h" #include "meta_programming/ob_mover.h" #include "ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/tx/ob_trans_define.h" #include "lib/string/ob_string_holder.h" #include "lib/utility/ob_unify_serialize.h" @@ -34,7 +33,6 @@ struct ObRowConflictInfo public: ObRowConflictInfo() : conflict_happened_addr_(), - conflict_ls_(), conflict_tablet_(), conflict_row_key_str_(), conflict_sess_id_pair_(), @@ -56,7 +54,6 @@ struct ObRowConflictInfo ObRowConflictInfo &operator=(meta::ObMover rhs_mover) {// support move assignment ObRowConflictInfo &rhs = rhs_mover.get_object(); init(rhs.conflict_happened_addr_, - rhs.conflict_ls_, rhs.conflict_tablet_, meta::ObMover(rhs.conflict_row_key_str_), rhs.conflict_sess_id_pair_, @@ -65,8 +62,8 @@ struct ObRowConflictInfo rhs.conflict_tx_hold_seq_); return *this; } - int init(const common::ObAddr &, const share::ObLSID &, - const common::ObTabletID &, meta::ObMover, + int init(const common::ObAddr &, const common::ObTabletID &, + meta::ObMover, const transaction::SessionIDPair, const common::ObAddr &, const transaction::ObTransID &, const transaction::ObTxSEQ &) { int ret = OB_SUCCESS; @@ -79,7 +76,6 @@ struct ObRowConflictInfo int assign(meta::ObMover rhs_rvalue) {// support move assign ObRowConflictInfo &rhs = rhs_rvalue.get_object(); return init(rhs.conflict_happened_addr_, - rhs.conflict_ls_, rhs.conflict_tablet_, meta::ObMover(rhs.conflict_row_key_str_), rhs.conflict_sess_id_pair_, @@ -90,13 +86,12 @@ struct ObRowConflictInfo bool is_valid() const { return conflict_tx_scheduler_.is_valid() && conflict_tx_id_.is_valid(); } - TO_STRING_KV(K_(conflict_happened_addr), K_(conflict_ls), K_(conflict_tablet), + TO_STRING_KV(K_(conflict_happened_addr), K_(conflict_tablet), K_(conflict_row_key_str), K_(conflict_sess_id_pair), K_(conflict_tx_scheduler), K_(conflict_tx_id), K_(conflict_tx_hold_seq), K_(conflict_hash), K_(lock_seq), K_(abs_timeout), K_(self_tx_id), K_(lock_mode), K_(last_compact_cnt), K_(total_update_cnt), K_(assoc_sess_id), K_(holder_sess_id), K_(holder_tx_start_time)); common::ObAddr conflict_happened_addr_;// distributed info - share::ObLSID conflict_ls_;// resource related common::ObTabletID conflict_tablet_;// resource related common::ObStringHolder conflict_row_key_str_;// resource related transaction::SessionIDPair conflict_sess_id_pair_;// holder related @@ -114,8 +109,8 @@ struct ObRowConflictInfo uint32_t holder_sess_id_; // lock holder session id int64_t holder_tx_start_time_; // lock holder trans start time }; -OB_SERIALIZE_MEMBER_TEMP(inline, ObRowConflictInfo, conflict_happened_addr_, conflict_ls_, - conflict_tablet_, conflict_row_key_str_, conflict_sess_id_pair_, +OB_SERIALIZE_MEMBER_TEMP(inline, ObRowConflictInfo, conflict_happened_addr_, conflict_tablet_, + conflict_row_key_str_, conflict_sess_id_pair_, conflict_tx_scheduler_, conflict_tx_id_, conflict_tx_hold_seq_, conflict_hash_, lock_seq_, abs_timeout_, self_tx_id_, lock_mode_, last_compact_cnt_, total_update_cnt_, assoc_sess_id_, holder_sess_id_, holder_tx_start_time_); diff --git a/src/storage/meta_mem/ob_flying_tablet_pointer_map.cpp b/src/storage/meta_mem/ob_flying_tablet_pointer_map.cpp index c8085b177..18b0a925f 100644 --- a/src/storage/meta_mem/ob_flying_tablet_pointer_map.cpp +++ b/src/storage/meta_mem/ob_flying_tablet_pointer_map.cpp @@ -53,14 +53,13 @@ int ObFlyingTabletPointerMap::init() int ObFlyingTabletPointerMap::set(const ObDieingTabletMapKey &key, ObTabletPointerHandle &handle) { int ret = OB_SUCCESS; - int64_t ls_id = handle.get_resource_ptr()->get_ls()->get_ls_id().id(); if (OB_UNLIKELY(!is_inited_)) { ret = common::OB_NOT_INIT; LOG_WARN("ObResourceMap has not been inited", K(ret)); } else if (OB_FAIL(map_.set_refactored(key, handle))) { LOG_WARN("fail to set into ResourceMap", K(ret), K(key)); } else { - FLOG_INFO("success to push tablet_pointer to flying_map", K(ret), K(ls_id), K(key), KP(handle.get_resource_ptr()), KPC(handle.get_resource_ptr()), K(count())); + FLOG_INFO("success to push tablet_pointer to flying_map", K(ret), K(key), KP(handle.get_resource_ptr()), KPC(handle.get_resource_ptr()), K(count())); } return ret; } diff --git a/src/storage/meta_mem/ob_tablet_map_key.cpp b/src/storage/meta_mem/ob_tablet_map_key.cpp index d51d52e10..0104a95f1 100644 --- a/src/storage/meta_mem/ob_tablet_map_key.cpp +++ b/src/storage/meta_mem/ob_tablet_map_key.cpp @@ -21,16 +21,13 @@ namespace oceanbase namespace storage { ObTabletMapKey::ObTabletMapKey() - : ls_id_(), - tablet_id_() + : tablet_id_() { } ObTabletMapKey::ObTabletMapKey( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) - : ls_id_(ls_id), - tablet_id_(tablet_id) + : tablet_id_(tablet_id) { } @@ -41,7 +38,6 @@ ObTabletMapKey::~ObTabletMapKey() void ObTabletMapKey::reset() { - ls_id_.reset(); tablet_id_.reset(); } @@ -54,7 +50,6 @@ int ObTabletMapKey::hash(uint64_t &hash_val) const uint64_t ObTabletMapKey::hash() const { uint64_t hash_val = 0; - hash_val = common::murmurhash(&ls_id_, sizeof(ls_id_), hash_val); hash_val = common::murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); return hash_val; } diff --git a/src/storage/meta_mem/ob_tablet_map_key.h b/src/storage/meta_mem/ob_tablet_map_key.h index 4693dc8b9..5a244eec8 100644 --- a/src/storage/meta_mem/ob_tablet_map_key.h +++ b/src/storage/meta_mem/ob_tablet_map_key.h @@ -19,7 +19,6 @@ #include #include "lib/utility/ob_print_utils.h" -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" namespace oceanbase @@ -30,7 +29,7 @@ class ObTabletMapKey final { public: ObTabletMapKey(); - ObTabletMapKey(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); + explicit ObTabletMapKey(const common::ObTabletID &tablet_id); ~ObTabletMapKey(); void reset(); @@ -42,20 +41,19 @@ class ObTabletMapKey final int hash(uint64_t &hash_val) const; uint64_t hash() const; - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; inline bool ObTabletMapKey::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } inline bool ObTabletMapKey::operator ==(const ObTabletMapKey &other) const { - return ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_; + return tablet_id_ == other.tablet_id_; } inline bool ObTabletMapKey::operator !=(const ObTabletMapKey &other) const @@ -65,7 +63,7 @@ inline bool ObTabletMapKey::operator !=(const ObTabletMapKey &other) const inline bool ObTabletMapKey::operator <(const ObTabletMapKey &other) const { - return ls_id_ < other.ls_id_ && tablet_id_ < other.tablet_id_; + return tablet_id_ < other.tablet_id_; } class ObDieingTabletMapKey final diff --git a/src/storage/meta_mem/ob_tablet_pointer.cpp b/src/storage/meta_mem/ob_tablet_pointer.cpp index d458f9430..e96fccf40 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.cpp +++ b/src/storage/meta_mem/ob_tablet_pointer.cpp @@ -16,7 +16,7 @@ #include "ob_tablet_pointer.h" #include "share/rc/ob_module_provider.h" -#include "src/storage/tx_storage/ob_ls_map.h" +#include "storage/ls/ob_ls.h" #define USING_LOG_PREFIX STORAGE @@ -34,7 +34,7 @@ ERRSIM_POINT_DEF(EN_RELEASE_MDS_NODE_FAILED); ObTabletPointer::ObTabletPointer() : phy_addr_(), obj_(), - ls_handle_(), + ls_(nullptr), ddl_kv_mgr_handle_(), protected_memtable_mgr_handle_(), ddl_info_(), @@ -48,16 +48,16 @@ ObTabletPointer::ObTabletPointer() auto_part_size_(OB_INVALID_SIZE) { #if defined(__x86_64__) && !defined(ENABLE_OBJ_LEAK_CHECK) - static_assert(sizeof(ObTabletPointer) == 360, "The size of ObTabletPointer will affect the meta memory manager, and the necessity of adding new fields needs to be considered."); + static_assert(sizeof(ObTabletPointer) == 344, "The size of ObTabletPointer will affect the meta memory manager, and the necessity of adding new fields needs to be considered."); #endif } ObTabletPointer::ObTabletPointer( - const ObLSHandle &ls_handle, + ObLS *tenant_ls, const ObMemtableMgrHandle &memtable_mgr_handle) : phy_addr_(), obj_(), - ls_handle_(ls_handle), + ls_(tenant_ls), protected_memtable_mgr_handle_(memtable_mgr_handle), ddl_info_(), initial_state_(true), @@ -88,7 +88,7 @@ void ObTabletPointer::reset() old_version_chain_ = nullptr; reset_obj(); phy_addr_.reset(); - ls_handle_.reset(); + ls_ = nullptr; auto_part_size_ = OB_INVALID_SIZE; flying_ = false; } @@ -271,10 +271,10 @@ int ObTabletPointer::get_attr_for_obj(ObTablet *tablet) if (OB_ISNULL(tablet)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tablet is null", K(ret), KP(tablet)); - } else if (OB_UNLIKELY(!ls_handle_.is_valid())) { + } else if (OB_UNLIKELY(OB_ISNULL(ls_))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls handle is invalid", K(ret), K_(ls_handle)); - } else if (OB_ISNULL(log_handler = ls_handle_.get_ls()->get_log_handler())) { + LOG_WARN("ls is invalid", K(ret), KP_(ls)); + } else if (OB_ISNULL(log_handler = ls_->get_log_handler())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log handler is null", K(ret), KP(log_handler)); } else { @@ -304,11 +304,10 @@ int ObTabletPointer::dump_meta_obj(ObMetaObjGuard &guard, void *&free_ ret = OB_ERR_UNEXPECTED; LOG_ERROR("obj is still being used", K(ret), K(*this)); } else { - const ObLSID ls_id = obj_.ptr_->tablet_meta_.ls_id_; const ObTabletID tablet_id = obj_.ptr_->tablet_meta_.tablet_id_; const int64_t wash_score = obj_.ptr_->get_wash_score(); guard.get_obj(meta_obj); - const ObTabletPersisterParam param(ls_id, ls_handle_.get_ls()->get_ls_epoch(), tablet_id); + const ObTabletPersisterParam param(ls_->get_ls_epoch(), tablet_id); ObTablet *tmp_obj = obj_.ptr_; if (OB_NOT_NULL(meta_obj.ptr_) && obj_.ptr_->get_try_cache_size() <= ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE) { char *buf = reinterpret_cast(meta_obj.ptr_); @@ -334,7 +333,7 @@ int ObTabletPointer::dump_meta_obj(ObMetaObjGuard &guard, void *&free_ } else { ObMetaObjBufferHelper::del_meta_obj(tmp_obj); free_obj = ObMetaObjBufferHelper::get_meta_obj_buffer_ptr(reinterpret_cast(tmp_obj)); - FLOG_INFO("wash one tablet",K(ret), KP(free_obj), K(ls_id), K(tablet_id), K(wash_score), K(obj_), K(meta_obj), K(phy_addr_)); + FLOG_INFO("wash one tablet", K(ret), KP(free_obj), K(tablet_id), K(wash_score), K(obj_), K(meta_obj), K(phy_addr_)); } } return ret; @@ -369,7 +368,7 @@ int ObTabletPointer::deep_copy(char *buf, const int64_t buf_len, ObTabletPointer } if (OB_SUCC(ret)) { - pvalue->ls_handle_ = ls_handle_; + pvalue->ls_ = ls_; pvalue->ddl_kv_mgr_handle_ = ddl_kv_mgr_handle_; pvalue->protected_memtable_mgr_handle_ = protected_memtable_mgr_handle_; pvalue->ddl_info_ = ddl_info_; @@ -387,7 +386,7 @@ int ObTabletPointer::deep_copy(char *buf, const int64_t buf_len, ObTabletPointer ObTabletResidentInfo ObTabletPointer::get_tablet_resident_info(const ObTabletMapKey &key) const { - return ObTabletResidentInfo(attr_, phy_addr_, key.ls_id_, key.tablet_id_); + return ObTabletResidentInfo(attr_, phy_addr_, key.tablet_id_); } bool ObTabletPointer::get_initial_state() const @@ -401,16 +400,15 @@ void ObTabletPointer::set_initial_state(const bool initial_state) } int ObTabletPointer::create_ddl_kv_mgr( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool &is_created) { int ret = OB_SUCCESS; ddl_kv_mgr_handle.reset(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id)); } else { ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObByteLockGuard guard(ddl_kv_mgr_lock_); @@ -421,8 +419,8 @@ int ObTabletPointer::create_ddl_kv_mgr( ObDDLKvMgrHandle tmp_handle; if (OB_FAIL(t3m->acquire_tablet_ddl_kv_mgr(tmp_handle))) { LOG_WARN("fail to acquire tablet ddl kv mgr", K(ret)); - } else if (OB_FAIL(tmp_handle.get_obj()->init(ls_id, tablet_id))) { - LOG_WARN("init ddl kv mgr failed", K(ret), K(ls_id),K(tablet_id)); + } else if (OB_FAIL(tmp_handle.get_obj()->init(tablet_id))) { + LOG_WARN("init ddl kv mgr failed", K(ret), K(tablet_id)); } else { ddl_kv_mgr_handle_ = tmp_handle; is_created = true; @@ -478,25 +476,23 @@ int ObTabletPointer::get_mds_table(const ObTabletID &tablet_id, bool not_exist_create) { int ret = OB_SUCCESS; - ObLSID ls_id = ls_handle_.get_ls()->get_ls_id(); share::SCN mds_ckpt_scn = share::SCN::min_scn(); if (not_exist_create) { ScanAllVersionTabletsOp::GetMaxMdsCkptScnOp op(mds_ckpt_scn); if (OB_UNLIKELY(phy_addr_.is_none())) {// first time create, without phy addr, use min scn to init mds table - } else if (OB_FAIL(share::g_mp->tenant_meta_mem_mgr()->scan_all_version_tablets({ls_id, tablet_id}, op))) { + } else if (OB_FAIL(share::g_mp->tenant_meta_mem_mgr()->scan_all_version_tablets(ObTabletMapKey(tablet_id), op))) { LOG_WARN("failed to get mds_ckpt_scn", K(ret)); } } if (OB_FAIL(ret)) { - } else if (!ls_handle_.is_valid()) { + } else if (OB_ISNULL(ls_)) { ret = OB_NOT_INIT; - LOG_ERROR("invalid ls_handle_, maybe not init yet", K(ret)); + LOG_ERROR("invalid ls_, maybe not init yet", K(ret)); } else if (!tablet_id.is_valid()) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "tablet_id is invalid", K(ret)); } else if (OB_FAIL(mds_table_handler_.get_mds_table_handle(handle, tablet_id, - ls_handle_.get_ls()->get_ls_id(), mds_ckpt_scn, not_exist_create, this))) { @@ -665,7 +661,7 @@ int ObTabletPointer::scan_all_tablets_on_chain(const ObFunction ObLS *ObTabletPointer::get_ls() const { - return ls_handle_.get_ls(); + return ls_; } int ObTabletPointer::acquire_obj(ObTablet *&t) @@ -780,4 +776,3 @@ int ScanAllVersionTabletsOp::GetMaxMdsCkptScnOp::operator()(ObTablet &tablet) } // namespace storage } // namespace oceanbase - diff --git a/src/storage/meta_mem/ob_tablet_pointer.h b/src/storage/meta_mem/ob_tablet_pointer.h index ffde0d051..5245bc8a1 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.h +++ b/src/storage/meta_mem/ob_tablet_pointer.h @@ -21,7 +21,6 @@ #include "storage/meta_mem/ob_meta_obj_struct.h" #include "storage/multi_data_source/runtime_utility/mds_lock.h" #include "storage/tablet/ob_tablet_ddl_info.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/multi_data_source/mds_table_handler.h" #include "storage/ob_i_memtable_mgr.h" #include "storage/ob_protected_memtable_mgr_handle.h" @@ -31,6 +30,7 @@ namespace oceanbase { namespace storage { +class ObLS; class ObTablet; class ObTabletDDLKvMgr; struct ObTabletResidentInfo; @@ -41,7 +41,7 @@ struct ObTabletAttr final public: ObTabletAttr() :v_(0), - ha_status_(0), + restore_state_(0), all_sstable_data_occupy_size_(0), all_sstable_data_required_size_(0), tablet_meta_size_(0), @@ -52,7 +52,7 @@ struct ObTabletAttr final void reset() { v_ = 0; - ha_status_ = 0; + restore_state_ = 0; all_sstable_data_occupy_size_ = 0; all_sstable_data_required_size_ = 0; tablet_meta_size_ = 0; @@ -61,7 +61,7 @@ struct ObTabletAttr final } bool is_valid() const { return valid_; } TO_STRING_KV(K_(valid), K_(is_empty_shell), - K_(has_next_tablet), K_(has_nested_table), K_(ha_status), + K_(has_next_tablet), K_(has_nested_table), K_(restore_state), K_(all_sstable_data_occupy_size), K_(all_sstable_data_required_size), K_(tablet_meta_size), K_(ss_public_sstable_occupy_size), K_(backup_bytes) ); @@ -76,7 +76,7 @@ struct ObTabletAttr final }; }; - int64_t ha_status_; + int64_t restore_state_; // all sstable data occupy_size, include major sstable // + int64_t all_sstable_data_occupy_size_; @@ -101,7 +101,7 @@ class ObTabletPointer final friend class ObFlyingTabletPointerMap; public: ObTabletPointer(); - ObTabletPointer(const ObLSHandle &ls_handle, + ObTabletPointer(ObLS *tenant_ls, const ObMemtableMgrHandle &memtable_mgr_handle); ~ObTabletPointer(); int get_in_memory_obj(ObMetaObjGuard &guard); @@ -136,14 +136,14 @@ class ObTabletPointer final int dump_meta_obj(ObMetaObjGuard &guard, void *&free_obj); // do not KPC memtable_mgr, may dead lock - TO_STRING_KV(K_(phy_addr), K_(obj), K_(ls_handle), K_(ddl_kv_mgr_handle), K_(attr), + TO_STRING_KV(K_(phy_addr), K_(obj), KP_(ls), K_(ddl_kv_mgr_handle), K_(attr), K_(protected_memtable_mgr_handle), K_(ddl_info), K_(initial_state), KP_(old_version_chain), K_(flying)); public: bool get_initial_state() const; ObTabletResidentInfo get_tablet_resident_info(const ObTabletMapKey &key) const; ObTabletMDSTruncateLock &get_mds_truncate_lock() const { return mds_lock_; } void set_initial_state(const bool initial_state); - int create_ddl_kv_mgr(const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool &is_created); + int create_ddl_kv_mgr(const ObTabletID &tablet_id, ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool &is_created); void get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle); int set_ddl_kv_mgr(const ObDDLKvMgrHandle &ddl_kv_mgr_handle); int remove_ddl_kv_mgr(const ObDDLKvMgrHandle &ddl_kv_mgr_handle); @@ -181,7 +181,7 @@ class ObTabletPointer final private: ObMetaDiskAddr phy_addr_; // 48B ObMetaObj obj_; // 40B - ObLSHandle ls_handle_; // 24B + ObLS *ls_; // 8B ObDDLKvMgrHandle ddl_kv_mgr_handle_; // 48B ObProtectedMemtableMgrHandle protected_memtable_mgr_handle_; // 32B ObTabletDDLInfo ddl_info_; // 32B @@ -193,7 +193,7 @@ class ObTabletPointer final ObTablet *old_version_chain_; // 8B ObTabletAttr attr_; // 32B // protected by rw lock of tablet_map_ int64_t auto_part_size_; // 8B - DISALLOW_COPY_AND_ASSIGN(ObTabletPointer); // 376B + DISALLOW_COPY_AND_ASSIGN(ObTabletPointer); // 344B, 360B with ENABLE_OBJ_LEAK_CHECK }; struct ObTabletResidentInfo final @@ -203,9 +203,8 @@ struct ObTabletResidentInfo final ObTabletResidentInfo( const ObTabletAttr &attr, const ObMetaDiskAddr &tablet_addr, - const share::ObLSID &ls_id, const ObTabletID &tablet_id) - : attr_(attr), tablet_addr_(tablet_addr), ls_id_(ls_id), tablet_id_(tablet_id) + : attr_(attr), tablet_addr_(tablet_addr), tablet_id_(tablet_id) {} ~ObTabletResidentInfo() { reset(); }; bool is_valid() const { return attr_.valid_ && tablet_id_.is_valid() && tablet_addr_.is_valid(); } @@ -222,13 +221,11 @@ struct ObTabletResidentInfo final attr_.reset(); tablet_addr_.set_none_addr(); tablet_id_ = ObTabletID::INVALID_TABLET_ID; - ls_id_ = ObLSID::INVALID_LS_ID; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(tablet_addr), K_(attr)); + TO_STRING_KV(K_(tablet_id), K_(tablet_addr), K_(attr)); public: ObTabletAttr attr_; ObMetaDiskAddr tablet_addr_; // used to identify one tablet - share::ObLSID ls_id_; ObTabletID tablet_id_; }; diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index 2a6afcfe0..9fef6314d 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -42,7 +42,6 @@ ObTenantMetaMemStatus::ObTenantMetaMemStatus() void ObTabletBufferInfo::reset() { - ls_id_.reset(); tablet_id_.reset(); tablet_buffer_ptr_ = nullptr; tablet_ = nullptr; @@ -62,7 +61,6 @@ int ObTabletBufferInfo::fill_info(const ObTabletPoolType &pool_type, ObMetaObjBu tablet_ = reinterpret_cast(obj_buffer); const ObTabletMeta &tablet_meta = tablet_->get_tablet_meta(); tablet_id_ = tablet_meta.tablet_id_; - ls_id_ = tablet_meta.ls_id_; tablet_buffer_ptr_ = reinterpret_cast(node); in_map_ = ObMetaObjBufferHelper::is_in_map(obj_buffer); pool_type_ = pool_type; @@ -172,6 +170,8 @@ ObTenantMetaMemMgr::ObTenantMetaMemMgr() tablet_gc_queue_(), free_tables_queue_(), gc_queue_lock_(common::ObLatchIds::TENANT_META_MEM_MGR_LOCK), + gc_memtable_set_(nullptr), + gc_uninit_memtable_set_(nullptr), memtable_pool_(get_default_memtable_pool_count(), "MemTblObj", ObCtxIds::DEFAULT_CTX_ID), tablet_buffer_pool_(get_default_normal_tablet_pool_count(), "N_TabletPool", ObCtxIds::META_OBJ_CTX_ID, &wash_func_), large_tablet_buffer_pool_(get_default_large_tablet_pool_count(), "L_TabletPool", ObCtxIds::META_OBJ_CTX_ID, &wash_func_, false/*allow_over_max_free_num*/), @@ -230,8 +230,6 @@ int ObTenantMetaMemMgr::init() LOG_WARN("fail to initialize external tablet cnt map", K(ret), K(bucket_num)); } else if (OB_FAIL(flying_tablet_map_.init())) { LOG_WARN("fail to initialize tablet map", K(ret), K(bucket_num)); - } else if (OB_FAIL(gc_memtable_map_.create(10, "GCMemtableMap", "GCMemtableMap"))) { - LOG_WARN("fail to initialize gc memtable map", K(ret)); } else if (OB_FAIL(meta_cache_io_allocator_.init(OB_MALLOC_MIDDLE_BLOCK_SIZE, "StorMetaCacheIO", mem_limit))) { LOG_WARN("fail to init storage meta cache io allocator", K(ret), K(mem_limit)); } else if (OB_FAIL(fetch_tenant_config())) { @@ -360,26 +358,13 @@ void ObTenantMetaMemMgr::wait() void ObTenantMetaMemMgr::destroy() { - int ret = OB_SUCCESS; gc_timer_.destroy(); full_tablet_creator_.reset(); // must reset after gc_tablets flying_tablet_map_.destroy(); external_tablet_cnt_map_.destroy(); tablet_map_.destroy(); - for (common::hash::ObHashMap::iterator iter = gc_memtable_map_.begin(); - OB_SUCC(ret) && iter != gc_memtable_map_.end(); ++iter) { - memtable::ObMemtableSet *memtable_set = iter->second; - if (OB_NOT_NULL(memtable_set)) { - if (0 != memtable_set->size()) { - LOG_ERROR("leaked memtable", KPC(memtable_set)); - } - if (OB_FAIL(memtable_set->destroy())) { - LOG_ERROR("memtable set destroy failed", K(ret)); - } - ob_free(memtable_set); - } - } - gc_memtable_map_.destroy(); + destroy_gc_memtable_set_(gc_memtable_set_); + destroy_gc_memtable_set_(gc_uninit_memtable_set_); bucket_lock_.destroy(); for (int64_t i = 0; i <= ObITable::TableType::REMOTE_LOGICAL_MINOR_SSTABLE; i++) { pool_arr_[i] = nullptr; @@ -517,8 +502,8 @@ int ObTenantMetaMemMgr::gc_tables_in_queue(bool &all_table_cleaned) LOG_ERROR("the table type is invalid", K(ret), KP(item), KPC(item), KPC(table), K(table->get_key())); } else { if (ObITable::TableType::DATA_MEMTABLE == table_type) { - if (OB_FAIL(push_memtable_into_gc_map_(static_cast(table)))) { - LOG_WARN("push memtable into gc map failed", K(ret)); + if (OB_FAIL(push_memtable_into_gc_set_(static_cast(table)))) { + LOG_WARN("push memtable into gc set failed", K(ret)); } } else { pool_arr_[index]->free_obj(static_cast(table)); @@ -609,144 +594,135 @@ void ObTenantMetaMemMgr::batch_destroy_memtable_(memtable::ObMemtableSet *memtab memtable::ObMemtableKeyBtree::batch_destroy(); } -void ObTenantMetaMemMgr::batch_gc_memtable_() +void ObTenantMetaMemMgr::destroy_gc_memtable_set_(memtable::ObMemtableSet *&memtable_set) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; + if (OB_NOT_NULL(memtable_set)) { + if (0 != memtable_set->size()) { + LOG_ERROR("leaked memtable", KPC(memtable_set)); + } + if (OB_FAIL(memtable_set->destroy())) { + LOG_ERROR("memtable set destroy failed", K(ret)); + } + ob_free(memtable_set); + memtable_set = nullptr; + } +} - for (common::hash::ObHashMap::iterator iter = gc_memtable_map_.begin(); - iter != gc_memtable_map_.end(); ++iter) { - const ObLSID &ls_id = iter->first; - memtable::ObMemtableSet *memtable_set = iter->second; - if (OB_NOT_NULL(memtable_set) - && 0 != memtable_set->size()) { - if (ls_id.id() != 0) { - if (OB_TMP_FAIL(ObMemtable::batch_remove_unused_callback_for_uncommited_txn(ls_id, - memtable_set))) { - if (OB_NOT_RUNNING != tmp_ret) { - LOG_ERROR("batch remove memtable set failed", K(tmp_ret), KPC(memtable_set)); - } else { - LOG_WARN("batch remove memtable set failed", K(tmp_ret), KPC(memtable_set)); - } - for (common::hash::ObHashSet::iterator set_iter = memtable_set->begin(); - set_iter != memtable_set->end(); - ++set_iter) { - if (OB_TMP_FAIL(push_table_into_gc_queue((ObITable *)(set_iter->first), ObITable::TableType::DATA_MEMTABLE))) { - LOG_ERROR("push table into gc queue failed, maybe there will be leak", - K(tmp_ret), KPC(memtable_set)); - } - } +int ObTenantMetaMemMgr::prepare_gc_memtable_set_(memtable::ObMemtableSet *&memtable_set) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(memtable_set)) { + const ObMemAttr attr("memtable_set"); + void *buf = nullptr; + memtable::ObMemtableSet *tmp_memtable_set = nullptr; + if (OB_ISNULL(buf = ob_malloc(sizeof(memtable::ObMemtableSet), attr))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for memtable set", K(ret)); + } else if (OB_ISNULL(tmp_memtable_set = new (buf) ObMemtableSet())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for memtable set", K(ret)); + } else if (OB_FAIL(tmp_memtable_set->create(1024, + "MemtableSetBkt", + "MemtableSetNode"))) { + LOG_WARN("fail to create", K(ret)); + } else { + memtable_set = tmp_memtable_set; + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(tmp_memtable_set)) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(tmp_memtable_set->destroy())) { + LOG_ERROR("memtable set destroy failed", K(tmp_ret)); + } + ob_free(tmp_memtable_set); + } else if (OB_NOT_NULL(buf)) { + ob_free(buf); + } + } + } + return ret; +} - LOG_INFO("batch gc memtable failed and push into gc queue again", K(memtable_set->size())); - while (OB_TMP_FAIL(memtable_set->clear())) { - LOG_ERROR("clear memtable set failed", K(tmp_ret), KPC(memtable_set)); - } +void ObTenantMetaMemMgr::batch_gc_memtable_set_( + memtable::ObMemtableSet *memtable_set, + const bool remove_tx_callbacks) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + if (OB_NOT_NULL(memtable_set) + && 0 != memtable_set->size()) { + if (remove_tx_callbacks) { + if (OB_TMP_FAIL(ObMemtable::batch_remove_unused_callback_for_uncommited_txn(memtable_set))) { + if (OB_NOT_RUNNING != tmp_ret) { + LOG_ERROR("batch remove memtable set failed", K(tmp_ret), KPC(memtable_set)); } else { - (void)batch_destroy_memtable_(memtable_set); - - int64_t batch_destroyed_occupy_size = 0; - for (common::hash::ObHashSet::iterator set_iter = memtable_set->begin(); - set_iter != memtable_set->end(); - ++set_iter) { - batch_destroyed_occupy_size += ((ObMemtable *)(set_iter->first))->get_occupied_size(); - pool_arr_[static_cast(ObITable::TableType::DATA_MEMTABLE)]->free_obj((void *)(set_iter->first)); + LOG_WARN("batch remove memtable set failed", K(tmp_ret), KPC(memtable_set)); + } + for (common::hash::ObHashSet::iterator set_iter = memtable_set->begin(); + set_iter != memtable_set->end(); + ++set_iter) { + if (OB_TMP_FAIL(push_table_into_gc_queue((ObITable *)(set_iter->first), ObITable::TableType::DATA_MEMTABLE))) { + LOG_ERROR("push table into gc queue failed, maybe there will be leak", + K(tmp_ret), KPC(memtable_set)); } + } - FLOG_INFO("batch gc memtable successfully", K(memtable_set->size()), K(batch_destroyed_occupy_size)); - while (OB_TMP_FAIL(memtable_set->clear())) { - LOG_ERROR("clear memtable set failed", K(tmp_ret), KPC(memtable_set)); - } + LOG_INFO("batch gc memtable failed and push into gc queue again", K(memtable_set->size())); + while (OB_TMP_FAIL(memtable_set->clear())) { + LOG_ERROR("clear memtable set failed", K(tmp_ret), KPC(memtable_set)); } } else { + (void)batch_destroy_memtable_(memtable_set); + + int64_t batch_destroyed_occupy_size = 0; for (common::hash::ObHashSet::iterator set_iter = memtable_set->begin(); set_iter != memtable_set->end(); ++set_iter) { + batch_destroyed_occupy_size += ((ObMemtable *)(set_iter->first))->get_occupied_size(); pool_arr_[static_cast(ObITable::TableType::DATA_MEMTABLE)]->free_obj((void *)(set_iter->first)); } - LOG_INFO("batch gc memtable successfully", K(memtable_set->size())); + FLOG_INFO("batch gc memtable successfully", K(memtable_set->size()), K(batch_destroyed_occupy_size)); while (OB_TMP_FAIL(memtable_set->clear())) { LOG_ERROR("clear memtable set failed", K(tmp_ret), KPC(memtable_set)); } } - } - } - - if (REACH_THREAD_TIME_INTERVAL(1_hour)) { - for (common::hash::ObHashMap::iterator iter = gc_memtable_map_.begin(); - iter != gc_memtable_map_.end(); ++iter) { - memtable::ObMemtableSet *memtable_set = iter->second; - if (OB_NOT_NULL(memtable_set)) { - if (0 != memtable_set->size()) { - LOG_ERROR("leaked memtable", KPC(memtable_set)); - } - if (OB_FAIL(memtable_set->destroy())) { - LOG_ERROR("memtable set destroy failed", K(ret)); - } - ob_free(memtable_set); + } else { + for (common::hash::ObHashSet::iterator set_iter = memtable_set->begin(); + set_iter != memtable_set->end(); + ++set_iter) { + pool_arr_[static_cast(ObITable::TableType::DATA_MEMTABLE)]->free_obj((void *)(set_iter->first)); } - } - if (OB_TMP_FAIL(gc_memtable_map_.clear())) { - LOG_ERROR("clear gc memtable map failed", K(tmp_ret)); + LOG_INFO("batch gc memtable successfully", K(memtable_set->size())); + while (OB_TMP_FAIL(memtable_set->clear())) { + LOG_ERROR("clear memtable set failed", K(tmp_ret), KPC(memtable_set)); + } } } } -int ObTenantMetaMemMgr::push_memtable_into_gc_map_(memtable::ObMemtable *memtable) +void ObTenantMetaMemMgr::batch_gc_memtable_() { - int ret = OB_SUCCESS; - share::ObLSID ls_id; - const ObMemAttr attr("memtable_set"); - memtable::ObMemtableSet *memtable_set = nullptr; - - if (OB_FAIL(memtable->get_ls_id(ls_id))) { - LOG_WARN("get memtable ls id failed", K(ret), KPC(memtable)); - if (OB_NOT_INIT == ret) { - // special ls id for memtable which fail to init - ls_id = 0; - ret = OB_SUCCESS; - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(gc_memtable_map_.get_refactored(ls_id, memtable_set))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - memtable::ObMemtableSet *tmp_memtable_set; - void *buf = NULL; - if (OB_ISNULL(buf = ob_malloc(sizeof(memtable::ObMemtableSet), attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory for hash set", K(ret)); - } else if (OB_ISNULL(tmp_memtable_set = new (buf) ObMemtableSet())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory for hash set", K(ret)); - } else if (OB_FAIL(tmp_memtable_set->create(1024, - "MemtableSetBkt", - "MemtableSetNode"))) { - LOG_WARN("fail to create", K(ret)); - } else if (OB_FAIL(gc_memtable_map_.set_refactored(ls_id, tmp_memtable_set))) { - LOG_WARN("fail to set hash set", K(ret)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(tmp_memtable_set->destroy())) { - LOG_ERROR("memtable set destroy failed", K(tmp_ret)); - } - } else { - memtable_set = tmp_memtable_set; - } - - if (NULL != buf && OB_FAIL(ret)) { - ob_free(tmp_memtable_set); - } - } else { - LOG_WARN("map get failed", K(ret), KPC(memtable)); - } + batch_gc_memtable_set_(gc_memtable_set_, true); + batch_gc_memtable_set_(gc_uninit_memtable_set_, false); + if (REACH_THREAD_TIME_INTERVAL(1_hour)) { + destroy_gc_memtable_set_(gc_memtable_set_); + destroy_gc_memtable_set_(gc_uninit_memtable_set_); } +} - if (OB_SUCC(ret)) { - if (OB_FAIL(memtable_set->set_refactored((uint64_t)(memtable), 0/*flag, not overwrite*/))) { - LOG_WARN("map set failed", K(ret), KPC(memtable)); - } +int ObTenantMetaMemMgr::push_memtable_into_gc_set_(memtable::ObMemtable *memtable) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_gc_memtable_set_(gc_memtable_set_))) { + LOG_WARN("failed to prepare gc memtable set", K(ret), KPC(memtable)); + } else if (OB_ISNULL(gc_memtable_set_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("memtable set is null", K(ret), KPC(memtable)); + } else if (OB_FAIL(gc_memtable_set_->set_refactored((uint64_t)(memtable), 0/*flag, not overwrite*/))) { + LOG_WARN("memtable set insert failed", K(ret), KPC(memtable)); } return ret; @@ -784,7 +760,7 @@ int ObTenantMetaMemMgr::push_tablet_into_gc_queue(ObTablet *tablet) release_tablet(tablet, false/*return tablet buffer ptr after release*/); } else { const ObTabletMeta &tablet_meta = tablet->get_tablet_meta(); - const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_); + const ObTabletMapKey key(tablet_meta.tablet_id_); const ObDieingTabletMapKey dieing_tablet_key(key); ObBucketHashWLockGuard lock_guard(bucket_lock_, key.hash()); @@ -959,21 +935,14 @@ int ObTenantMetaMemMgr::get_min_end_scn_from_single_tablet(ObTablet *tablet, LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(tablet->ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { if (OB_EMPTY_RESULT == ret) { - // When OB_EMPTY_RESULT is returned, the tablet may have just been created and - // committed its tablet status. The old version tablet is not readable in this - // case, so it is safe to return max scn here. min_end_scn.set_max(); ret = OB_SUCCESS; } else { LOG_WARN("get tablet status failed", KR(ret), KP(tablet)); } - } else if (ObTabletStatus::RESERVED_5 == user_data.tablet_status_) { - if (!user_data.reserved_scn_.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("reserved_scn is invalid", K(ret), K(user_data)); - } else { - min_end_scn = SCN::scn_dec(user_data.reserved_scn_); - } + } else if (ObTabletStatus::RESERVED_STATUS_5 == user_data.tablet_status_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("reserved tablet status is not supported", K(ret), K(user_data)); } else { SCN cur_recycle_end_scn = SCN::max_scn(); bool contain_uncommitted_row = false; @@ -1329,7 +1298,7 @@ int ObTenantMetaMemMgr::create_tmp_tablet( const WashTabletPriority &priority, const ObTabletMapKey &key, common::ObArenaAllocator &allocator, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; @@ -1356,7 +1325,7 @@ int ObTenantMetaMemMgr::create_tmp_tablet( } else if (is_exist) { ret = OB_ENTRY_EXIST; LOG_WARN("This tablet pointer has exist, and don't create again", K(ret), K(key), K(is_exist)); - } else if (OB_FAIL(create_tablet(key, ls_handle, tablet_handle))) { + } else if (OB_FAIL(create_tablet(key, tenant_ls, tablet_handle))) { LOG_WARN("fail to create tablet in map", K(ret), K(key), K(tablet_handle)); } } @@ -1535,7 +1504,7 @@ int ObTenantMetaMemMgr::acquire_tmp_tablet( int ObTenantMetaMemMgr::create_msd_tablet( const WashTabletPriority &priority, const ObTabletMapKey &key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; @@ -1560,7 +1529,7 @@ int ObTenantMetaMemMgr::create_msd_tablet( } else if (OB_UNLIKELY(is_exist)) { ret = OB_ENTRY_EXIST; LOG_WARN("This tablet pointer has exist, and don't create again", K(ret), K(key), K(is_exist)); - } else if (OB_FAIL(create_tablet(key, ls_handle, tablet_handle))) { + } else if (OB_FAIL(create_tablet(key, tenant_ls, tablet_handle))) { LOG_WARN("fail to create tablet in map", K(ret), K(key), K(tablet_handle)); } } @@ -1572,34 +1541,33 @@ int ObTenantMetaMemMgr::create_msd_tablet( int ObTenantMetaMemMgr::create_tablet( const ObTabletMapKey &key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; ObMemtableMgrHandle memtable_mgr_hdl; - ObLS *ls = ls_handle.get_ls(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMetaMemMgr hasn't been initialized", K(ret)); - } else if (OB_UNLIKELY(!key.is_valid() || !ls_handle.is_valid() || !tablet_handle.is_valid())) { + } else if (OB_UNLIKELY(!key.is_valid() || OB_ISNULL(tenant_ls) || !tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(key), K(ls_handle), K(tablet_handle)); + LOG_WARN("invalid argument", K(ret), K(key), KP(tenant_ls), K(tablet_handle)); } else if (key.tablet_id_.is_ls_tx_data_tablet()) { - if (OB_FAIL(ls->get_tablet_svr()->get_tx_data_memtable_mgr(memtable_mgr_hdl))) { + if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tx_data_memtable_mgr(memtable_mgr_hdl))) { LOG_WARN("fail to get tx data memtable mgr", K(ret)); } } else if (key.tablet_id_.is_ls_tx_ctx_tablet()) { - if (OB_FAIL(ls->get_tablet_svr()->get_tx_ctx_memtable_mgr(memtable_mgr_hdl))) { + if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tx_ctx_memtable_mgr(memtable_mgr_hdl))) { LOG_WARN("fail to get tx data memtable mgr", K(ret)); } } else if (key.tablet_id_.is_ls_lock_tablet()) { - if (OB_FAIL(ls->get_tablet_svr()->get_lock_memtable_mgr(memtable_mgr_hdl))) { + if (OB_FAIL(tenant_ls->get_tablet_svr()->get_lock_memtable_mgr(memtable_mgr_hdl))) { LOG_WARN("fail to get lock memtable mgr", K(ret)); } } if (OB_SUCC(ret)) { ObTabletPointerHandle ptr_handle(tablet_map_); - ObTabletPointer tablet_ptr(ls_handle, memtable_mgr_hdl); + ObTabletPointer tablet_ptr(tenant_ls, memtable_mgr_hdl); ObMetaDiskAddr addr; addr.set_none_addr(); tablet_ptr.set_addr_with_reset_obj(addr); @@ -1715,50 +1683,6 @@ int ObTenantMetaMemMgr::get_tablet_with_allocator( return ret; } -int ObTenantMetaMemMgr::get_current_version_for_tablet( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id, - int64_t &tablet_version, - bool &allow_tablet_version_gc) -{ - int ret = OB_SUCCESS; - const ObTabletMapKey key(ls_id, tablet_id); - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObTenantMetaMemMgr hasn't been initialized", K(ret)); - } else if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(key)); - } else { - ObBucketHashRLockGuard lock_guard(bucket_lock_, key.hash()); - ObTabletPointer *tablet_ptr = nullptr; - ObTabletPointerHandle ptr_handle(tablet_map_); - ObTabletHandle tablet_handle; - if (OB_FAIL(tablet_map_.get(key, ptr_handle))) { - LOG_WARN("failed to get ptr handle", K(ret), K(key)); - } else if (OB_ISNULL(tablet_ptr = static_cast(ptr_handle.get_resource_ptr()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet ptr is NULL", K(ret), K(ptr_handle)); - } else if (OB_FAIL(get_tablet(WashTabletPriority::WTP_HIGH, key, tablet_handle))) { - LOG_WARN("failed to get tablet_handle"); - } else if (OB_ISNULL(tablet_handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet is NULL", K(ret), K(tablet_handle)); - } else { - const ObDieingTabletMapKey dieing_key(key); - bool exist_in_external = false; - if (OB_FAIL(external_tablet_cnt_map_.check_exist(dieing_key, exist_in_external))) { - LOG_WARN("fail to check ex_tablet exist or not", K(ret), K(dieing_key), K(exist_in_external)); - } else { - tablet_version = tablet_ptr->get_addr().block_id().meta_version_id(); - allow_tablet_version_gc = tablet_ptr->is_old_version_chain_empty() && !exist_in_external; - FLOG_INFO("PRINT TABLET ADDRESS", K(ret), K(tablet_ptr->get_addr()), K(tablet_handle.get_obj()->get_tablet_addr())); - } - } - } - return ret; -} - int ObTenantMetaMemMgr::build_tablet_handle_for_mds_scan( ObTablet *tablet, ObTabletHandle &handle) @@ -2034,12 +1958,11 @@ int ObTenantMetaMemMgr::dec_external_tablet_cnt(const uint64_t tablet_id) } int ObTenantMetaMemMgr::schedule_load_bloomfilter(const storage::ObITable::TableKey &sstable_key, - const share::ObLSID &ls_id, const MacroBlockId ¯o_id, const ObDatumRowkey &rowkey) { int ret = OB_SUCCESS; - if (OB_FAIL(bf_load_thread_.add_load_task(sstable_key, ls_id, macro_id, rowkey))) { + if (OB_FAIL(bf_load_thread_.add_load_task(sstable_key, macro_id, rowkey))) { LOG_WARN("fail to add bloom filter load task", K(ret), K(sstable_key), K(macro_id), K(rowkey)); } return ret; @@ -2505,13 +2428,12 @@ int ObTenantMetaMemMgr::get_wash_tablet_candidate(const std::type_info &type_inf if (curr->get_data().has_new_) { ObTablet *tablet = reinterpret_cast(ObMetaObjBufferHelper::get_obj_buffer(curr)); const ObTabletMeta &tablet_meta = tablet->get_tablet_meta(); - const ObTabletMapKey tablet_key(tablet_meta.ls_id_, tablet_meta.tablet_id_); + const ObTabletMapKey tablet_key(tablet_meta.tablet_id_); if (1 == tablet->get_ref() && ObMetaObjBufferHelper::is_in_map(reinterpret_cast(tablet))) { // While hope to wash normal buffer pool, we need to skip rebuild next tablet, which may be oversub macro ref cnt. const bool need_skip = type_info == typeid(ObNormalTabletBuffer) && tablet->get_next_tablet_guard().is_valid(); // this tablet is only hold by tablet map. CandidateTabletInfo candidate; - candidate.ls_id_ = tablet_key.ls_id_.id(); candidate.tablet_id_ = tablet_key.tablet_id_.id(); candidate.wash_score_ = tablet->get_wash_score(); LOG_DEBUG("get wash candidate", K(candidate)); @@ -2547,9 +2469,8 @@ int ObTenantMetaMemMgr::do_wash_candidate_tablet( void *&free_obj) { int ret = OB_SUCCESS; - const ObLSID ls_id(candidate.ls_id_); const ObTabletID tablet_id(candidate.tablet_id_); - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletPointerHandle ptr_handle(tablet_map_); if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -2680,23 +2601,6 @@ int ObTenantMetaMemMgr::try_wash_tablet(const std::type_info &type_info, void *& return ret; } -ObTenantMetaMemMgr::MinMinorSSTableInfo::MinMinorSSTableInfo( - const share::ObLSID &ls_id, - const ObITable::TableKey &table_key, - const ObTableHandleV2 &sstable_handle) - : ls_id_(ls_id), - table_key_(table_key), - sstable_handle_(sstable_handle) -{ -} - -ObTenantMetaMemMgr::MinMinorSSTableInfo::~MinMinorSSTableInfo() -{ - ls_id_.reset(); - table_key_.reset(); - sstable_handle_.reset(); -} - ObT3mTabletMapIterator::ObT3mTabletMapIterator(ObTenantMetaMemMgr &t3m) : tablet_map_(t3m.tablet_map_), tablet_items_(), diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h index d0265ab0e..187d49dc8 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h @@ -25,8 +25,6 @@ #include "lib/lock/ob_tc_rwlock.h" #include "lib/lock/ob_thread_cond.h" #include "lib/queue/ob_link_queue.h" -#include "lib/task/ob_timer.h" -#include "share/ob_ls_id.h" #include "share/resource_limit_calculator/ob_resource_limit_calculator.h" #include "storage/blocksstable/ob_macro_block_id.h" #include "storage/blocksstable/ob_bloom_filter_load_task.h" @@ -70,6 +68,7 @@ class ObLockMemtable; namespace storage { +class ObLS; class ObTenantMetaMemMgr; class ObTxDataMemtable; class ObTxCtxMemtable; @@ -102,7 +101,7 @@ struct ObTabletBufferInfo final { public: ObTabletBufferInfo() - : ls_id_(), tablet_id_(), + : tablet_id_(), tablet_buffer_ptr_(nullptr), tablet_(nullptr), pool_type_(), in_map_(false), last_access_time_(-1) { @@ -114,10 +113,9 @@ struct ObTabletBufferInfo final void reset(); int fill_info(const ObTabletPoolType &pool_type, ObMetaObjBufferNode *node); - TO_STRING_KV(K_(ls_id), K_(tablet_id), KP_(tablet_buffer_ptr), KP_(tablet), K_(pool_type), K_(in_map), K_(last_access_time)); + TO_STRING_KV(K_(tablet_id), KP_(tablet_buffer_ptr), KP_(tablet), K_(pool_type), K_(in_map), K_(last_access_time)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; char *tablet_buffer_ptr_; ObTablet *tablet_; @@ -138,9 +136,9 @@ class ObTenantMetaMemMgr final static const int64_t MAX_TABLET_CNT_IN_OBJ_POOL = 50000; static const int64_t MAX_MEMTABLE_CNT_IN_OBJ_POOL = 2 * MAX_TABLET_CNT_IN_OBJ_POOL; - static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; - static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; - static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; + static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT; + static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = 1; + static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = 1; static const int64_t MAX_DDL_KV_IN_OBJ_POOL = 5000; static const int64_t TABLET_TRANSFORM_INTERVAL_US = 2 * 1000 * 1000L; // 2s @@ -230,13 +228,13 @@ class ObTenantMetaMemMgr final int create_msd_tablet( const WashTabletPriority &priority, const ObTabletMapKey &key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &tablet_handle); int create_tmp_tablet( const WashTabletPriority &priority, const ObTabletMapKey &key, common::ObArenaAllocator &allocator, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObTabletHandle &tablet_handle); int acquire_tmp_tablet( const WashTabletPriority &priority, @@ -269,11 +267,6 @@ class ObTenantMetaMemMgr final ObTabletHandle &handle, const bool force_alloc_new = false); - int get_current_version_for_tablet( - const share::ObLSID &ls_id, - const ObTabletID &tablet_id, - int64_t &tablet_version, - bool &allow_tablet_version_gc); int check_allow_tablet_gc(const ObTabletID &tablet_id, bool &allow); int get_tablet_buffer_infos(ObIArray &buffer_infos); int get_tablet_addr(const ObTabletMapKey &key, ObMetaDiskAddr &addr); @@ -317,7 +310,6 @@ class ObTenantMetaMemMgr final int inc_external_tablet_cnt(const uint64_t tablet_id); int dec_external_tablet_cnt(const uint64_t tablet_id); int schedule_load_bloomfilter(const storage::ObITable::TableKey &sstable_key, - const share::ObLSID &ls_id, const MacroBlockId ¯o_id, const ObDatumRowkey &common_rowkey); @@ -355,13 +347,12 @@ class ObTenantMetaMemMgr final struct CandidateTabletInfo final { public: - CandidateTabletInfo() :ls_id_(0), tablet_id_(0), wash_score_(INT64_MIN) {} + CandidateTabletInfo() : tablet_id_(0), wash_score_(INT64_MIN) {} ~CandidateTabletInfo() = default; - bool is_valid() const { return ls_id_ > 0 && tablet_id_ > 0; } + bool is_valid() const { return tablet_id_ > 0; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(wash_score)); + TO_STRING_KV(K_(tablet_id), K_(wash_score)); - int64_t ls_id_; // to use ObBinaryHeap, the type here must is_trivially_copyable uint64_t tablet_id_; // to use ObBinaryHeap, the type here must is_trivially_copyable int64_t wash_score_; }; @@ -415,43 +406,6 @@ class ObTenantMetaMemMgr final virtual ~RefreshConfigTask() = default; virtual void runTimerTask() override; }; - class MinMinorSSTableInfo final - { - public: - MinMinorSSTableInfo() : ls_id_(), table_key_(), sstable_handle_() {} - MinMinorSSTableInfo( - const share::ObLSID &ls_id, - const ObITable::TableKey &table_key, - const ObTableHandleV2 &sstable_handle); - ~MinMinorSSTableInfo(); - OB_INLINE bool is_valid() const - { - return ls_id_.is_valid() - && table_key_.is_valid() - && sstable_handle_.is_valid() - && sstable_handle_.get_table()->is_minor_sstable(); - } - OB_INLINE bool operator ==(const MinMinorSSTableInfo &other) const - { - return sstable_handle_.get_table() == other.sstable_handle_.get_table(); - } - OB_INLINE bool operator !=(const MinMinorSSTableInfo &other) const { return !(*this == other); } - OB_INLINE uint64_t hash() const - { - const ObITable *table = sstable_handle_.get_table(); - return common::murmurhash(&table, sizeof(table), 0); - } - OB_INLINE int hash(uint64_t &hash_val) const - { - hash_val = hash(); - return OB_SUCCESS; - } - TO_STRING_KV(K_(ls_id), K_(table_key), K_(sstable_handle)); - public: - share::ObLSID ls_id_; - ObITable::TableKey table_key_; - ObTableHandleV2 sstable_handle_; - }; class TabletMapDumpOperator { public: @@ -497,14 +451,13 @@ class ObTenantMetaMemMgr final static const int64_t FLYING_TABLET_THRESHOLD = 100000; typedef common::ObBinaryHeap Heap; typedef common::ObDList TabletBufferList; - typedef common::hash::ObHashSet SSTableSet; typedef common::hash::ObHashSet PinnedTabletSet; private: int acquire_tablet(const ObTabletPoolType type, ObTabletHandle &tablet_handle); int acquire_tablet(ObITenantMetaObjPool *pool, ObTablet *&tablet); int acquire_tablet_ddl_kv_mgr(ObDDLKvMgrHandle &handle); - int create_tablet(const ObTabletMapKey &key, ObLSHandle &ls_handle, ObTabletHandle &tablet_handle); + int create_tablet(const ObTabletMapKey &key, ObLS *tenant_ls, ObTabletHandle &tablet_handle); int do_wash_candidate_tablet( const CandidateTabletInfo &info, ObTabletHandle &tablet_handle, @@ -533,8 +486,11 @@ class ObTenantMetaMemMgr final const char *name, common::ObIArray &info) const; int get_wash_tablet_candidate(const std::type_info &type_info, CandidateTabletInfo &info); - int push_memtable_into_gc_map_(memtable::ObMemtable *memtable); + int push_memtable_into_gc_set_(memtable::ObMemtable *memtable); + int prepare_gc_memtable_set_(memtable::ObMemtableSet *&memtable_set); + void destroy_gc_memtable_set_(memtable::ObMemtableSet *&memtable_set); void batch_gc_memtable_(); + void batch_gc_memtable_set_(memtable::ObMemtableSet *memtable_set, const bool remove_tx_callbacks); void batch_destroy_memtable_(memtable::ObMemtableSet *memtable_set); bool is_tablet_handle_leak_checker_enabled(); int push_tablet_pointer_to_fly_map_if_need_(const ObTabletMapKey &key); @@ -554,7 +510,8 @@ class ObTenantMetaMemMgr final TabletGCQueue tablet_gc_queue_; common::ObLinkQueue free_tables_queue_; common::ObSpinLock gc_queue_lock_; - common::hash::ObHashMap gc_memtable_map_; + memtable::ObMemtableSet *gc_memtable_set_; + memtable::ObMemtableSet *gc_uninit_memtable_set_; ObTabletLeakChecker leak_checker_; ObTenantMetaObjPool memtable_pool_; diff --git a/src/storage/meta_store/ob_server_storage_meta_replayer.cpp b/src/storage/meta_store/ob_server_storage_meta_replayer.cpp index 2f46b2abf..c7c6c4341 100644 --- a/src/storage/meta_store/ob_server_storage_meta_replayer.cpp +++ b/src/storage/meta_store/ob_server_storage_meta_replayer.cpp @@ -177,29 +177,14 @@ int ObServerStorageMetaReplayer::finish_storage_meta_replay_() if (OB_SUCC(ret)) { MOD_SCOPE { - common::ObSharedGuard ls_iter; - ObLS *ls = nullptr; - ObLSTabletService *ls_tablet_svr = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next ls", K(ret)); - } - } else if (nullptr == ls) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret)); - } else if (OB_FAIL(ls->finish_storage_meta_replay())) { - LOG_WARN("finish replay failed", K(ret), KPC(ls)); - } - } - if (OB_ITER_END == ret) { - if (OB_FAIL(share::g_mp->ls_service()->gc_ls_after_replay_slog())) { - LOG_WARN("fail to gc ls after replay slog", K(ret)); - } - } + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->finish_storage_meta_replay())) { + LOG_WARN("finish replay failed", K(ret)); + } + if (OB_SUCC(ret) && OB_FAIL(share::g_mp->ls_service()->gc_ls_after_replay_slog())) { + LOG_WARN("fail to gc ls after replay slog", K(ret)); } } } diff --git a/src/storage/meta_store/ob_tenant_storage_meta_persister.cpp b/src/storage/meta_store/ob_tenant_storage_meta_persister.cpp index d0484dbe1..54d9efc0a 100644 --- a/src/storage/meta_store/ob_tenant_storage_meta_persister.cpp +++ b/src/storage/meta_store/ob_tenant_storage_meta_persister.cpp @@ -37,18 +37,9 @@ int ObTenantStorageMetaPersister::init( { int ret = OB_SUCCESS; - const int64_t MEM_LIMIT = 512UL << 20; - lib::ObMemAttr attr("TntMetaPersist"); - const int64_t MAP_BUCKET_CNT = 256; - lib::ObMemAttr map_attr("PendingFreeMap"); - if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ObTenantStorageMetaPersister has inited", K(ret)); - } else if (OB_FAIL(allocator_.init(common::OB_MALLOC_NORMAL_BLOCK_SIZE, attr, MEM_LIMIT))) { - LOG_WARN("fail to init fifo allocator", K(ret)); - } else if (OB_FAIL(pending_free_tablet_arr_map_.create(MAP_BUCKET_CNT, map_attr))) { - LOG_WARN("fail to create pending_free_tablet_arr_map", K(ret)); } else { slogger_ = &slogger; @@ -60,21 +51,9 @@ int ObTenantStorageMetaPersister::init( void ObTenantStorageMetaPersister::destroy() { - int ret = OB_SUCCESS; if (IS_INIT) { - for (PendingFreeTabletArrayMap::iterator iter = pending_free_tablet_arr_map_.begin(); - iter != pending_free_tablet_arr_map_.end(); iter++) { - if (OB_ISNULL(iter->second)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("PendingFreeTabletArrayInfo is null", K(ret), K(iter->first)); - } else { - ob_delete(iter->second); - } - } - pending_free_tablet_arr_map_.destroy(); slogger_ = nullptr; ckpt_slog_handler_ = nullptr; - allocator_.reset(); is_inited_ = false; } } @@ -94,44 +73,43 @@ int ObTenantStorageMetaPersister::prepare_create_ls(const ObLSMeta &meta, int64_ return ret; } -int ObTenantStorageMetaPersister::commit_create_ls( - const share::ObLSID &ls_id, const int64_t ls_epoch) +int ObTenantStorageMetaPersister::commit_create_ls() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - if (OB_FAIL(write_commit_create_ls_slog_(ls_id))) { - LOG_WARN("fail to write commit create ls slog", K(ret), K(ls_id)); + if (OB_FAIL(write_commit_create_ls_slog_())) { + LOG_WARN("fail to write commit create ls slog", K(ret)); } } return ret; } -int ObTenantStorageMetaPersister::abort_create_ls(const share::ObLSID &ls_id, const int64_t ls_epoch) +int ObTenantStorageMetaPersister::abort_create_ls() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - if (OB_FAIL(write_abort_create_ls_slog_(ls_id))) { - LOG_WARN("fail to write abort create ls slog", K(ret), K(ls_id)); + if (OB_FAIL(write_abort_create_ls_slog_())) { + LOG_WARN("fail to write abort create ls slog", K(ret)); } } return ret; } -int ObTenantStorageMetaPersister::delete_ls(const share::ObLSID &ls_id, const int64_t ls_epoch) +int ObTenantStorageMetaPersister::delete_ls() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - if (OB_FAIL(write_delete_ls_slog_(ls_id))) { - LOG_WARN("fail to write delete ls slog", K(ret), K(ls_id)); + if (OB_FAIL(write_delete_ls_slog_())) { + LOG_WARN("fail to write delete ls slog", K(ret)); } } return ret; @@ -196,7 +174,7 @@ int ObTenantStorageMetaPersister::batch_update_tablet(const ObIArray(log_param.data_); - const ObTabletMapKey tablet_key(slog->ls_id_, slog->tablet_id_); + const ObTabletMapKey tablet_key(slog->tablet_id_); do { if (OB_FAIL(ckpt_slog_handler_->report_slog(tablet_key, log_param.disk_addr_))) { if (OB_ALLOCATE_MEMORY_FAILED != ret) { @@ -214,8 +192,6 @@ int ObTenantStorageMetaPersister::batch_update_tablet(const ObIArrayget_tablet_meta().ls_id_, tablet->get_tablet_meta().tablet_id_); - ObEmptyShellTabletLog slog_entry(tablet->get_tablet_meta().ls_id_, - tablet->get_tablet_meta().tablet_id_, + const ObTabletMapKey tablet_key(tablet->get_tablet_meta().tablet_id_); + ObEmptyShellTabletLog slog_entry(tablet->get_tablet_meta().tablet_id_, tablet); ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, @@ -278,9 +253,7 @@ int ObTenantStorageMetaPersister::write_empty_shell_tablet(ObTablet *tablet, ObM return ret; } -int ObTenantStorageMetaPersister::remove_tablet( - const share::ObLSID &ls_id, const int64_t ls_epoch, - const ObTabletHandle &tablet_handle) +int ObTenantStorageMetaPersister::remove_tablet(const ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -293,32 +266,27 @@ int ObTenantStorageMetaPersister::remove_tablet( const common::ObTabletID &tablet_id = tablet_handle.get_obj()->get_tablet_meta().tablet_id_; const ObMetaDiskAddr &tablet_addr = tablet_handle.get_obj()->get_tablet_addr(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !tablet_addr.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || !tablet_addr.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tablet_id), K(tablet_addr)); + LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(tablet_addr)); } else { - if (OB_FAIL(write_remove_tablet_slog_(ls_id, tablet_id))) { - LOG_WARN("fail to write remove tablet slog", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(write_remove_tablet_slog_(tablet_id))) { + LOG_WARN("fail to write remove tablet slog", K(ret), K(tablet_id)); } } } return ret; } -int ObTenantStorageMetaPersister::remove_tablets( - const share::ObLSID &ls_id, const int64_t ls_epoch, - const ObIArray &tablet_ids, const ObIArray &tablet_addrs) +int ObTenantStorageMetaPersister::remove_tablets(const ObIArray &tablet_ids) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(ls_id)); } else { - if (OB_FAIL(write_remove_tablets_slog_(ls_id, tablet_ids))) { - LOG_WARN("fail to write remove tablets slog", K(ret), K(ls_id)); + if (OB_FAIL(write_remove_tablets_slog_(tablet_ids))) { + LOG_WARN("fail to write remove tablets slog", K(ret)); } } return ret; @@ -340,11 +308,10 @@ int ObTenantStorageMetaPersister::write_prepare_create_ls_slog_(const ObLSMeta & return ret; } -int ObTenantStorageMetaPersister::write_commit_create_ls_slog_(const share::ObLSID &ls_id) +int ObTenantStorageMetaPersister::write_commit_create_ls_slog_() { int ret = OB_SUCCESS; - share::ObLSID tmp_ls_id = ls_id; - ObCreateLSCommitSLog slog_entry(tmp_ls_id); + ObCreateLSCommitSLog slog_entry; ObStorageLogParam log_param; log_param.data_ = &slog_entry; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, @@ -355,11 +322,10 @@ int ObTenantStorageMetaPersister::write_commit_create_ls_slog_(const share::ObLS return ret; } -int ObTenantStorageMetaPersister::write_abort_create_ls_slog_(const share::ObLSID &ls_id) +int ObTenantStorageMetaPersister::write_abort_create_ls_slog_() { int ret = OB_SUCCESS; - share::ObLSID tmp_ls_id = ls_id; - ObCreateLSAbortSLog slog_entry(tmp_ls_id); + ObCreateLSAbortSLog slog_entry; ObStorageLogParam log_param; log_param.data_ = &slog_entry; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, @@ -370,11 +336,10 @@ int ObTenantStorageMetaPersister::write_abort_create_ls_slog_(const share::ObLSI return ret; } -int ObTenantStorageMetaPersister::write_delete_ls_slog_(const share::ObLSID &ls_id) +int ObTenantStorageMetaPersister::write_delete_ls_slog_() { int ret = OB_SUCCESS; - share::ObLSID tmp_ls_id = ls_id; - ObDeleteLSLog slog_entry(tmp_ls_id); + ObDeleteLSLog slog_entry; ObStorageLogParam log_param; log_param.data_ = &slog_entry; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, @@ -400,16 +365,15 @@ int ObTenantStorageMetaPersister::write_update_ls_meta_slog_(const ObLSMeta &ls_ } int ObTenantStorageMetaPersister::write_update_tablet_slog_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObMetaDiskAddr &disk_addr) { int ret = OB_SUCCESS; - const ObTabletMapKey tablet_key(ls_id, tablet_id); + const ObTabletMapKey tablet_key(tablet_id); if (OB_FAIL(LOCAL_DEVICE_INSTANCE.fsync_block())) { // make sure that all data or meta written on the macro block is flushed LOG_WARN("fail to fsync_block", K(ret)); } else { - ObUpdateTabletLog slog_entry(ls_id, tablet_id, disk_addr); + ObUpdateTabletLog slog_entry(tablet_id, disk_addr); ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET); @@ -431,12 +395,10 @@ int ObTenantStorageMetaPersister::write_update_tablet_slog_( return ret; } -int ObTenantStorageMetaPersister::write_remove_tablet_slog_( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) +int ObTenantStorageMetaPersister::write_remove_tablet_slog_(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObDeleteTabletLog slog_entry(ls_id, tablet_id); + ObDeleteTabletLog slog_entry(tablet_id); ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, ObRedoLogSubType::OB_REDO_LOG_DELETE_TABLET); @@ -448,7 +410,7 @@ int ObTenantStorageMetaPersister::write_remove_tablet_slog_( } int ObTenantStorageMetaPersister::write_remove_tablets_slog_( - const ObLSID &ls_id, const common::ObIArray &tablet_ids) + const common::ObIArray &tablet_ids) { int ret = OB_SUCCESS; // We can split the tablet_ids array due to following reasons: @@ -473,7 +435,7 @@ int ObTenantStorageMetaPersister::write_remove_tablets_slog_( } } if (OB_FAIL(ret)){ - } else if (OB_FAIL(safe_batch_write_remove_tablets_slog_(ls_id, current_tablet_arr))){ + } else if (OB_FAIL(safe_batch_write_remove_tablets_slog_(current_tablet_arr))){ STORAGE_REDO_LOG(WARN, "inner write log fail", K(ret), K(cur_cnt), K(total_cnt), K(finish_cnt)); } else { finish_cnt += cur_cnt; @@ -484,7 +446,7 @@ int ObTenantStorageMetaPersister::write_remove_tablets_slog_( } int ObTenantStorageMetaPersister::safe_batch_write_remove_tablets_slog_( - const ObLSID &ls_id, const common::ObIArray &tablet_ids) + const common::ObIArray &tablet_ids) { int ret = OB_SUCCESS; const int64_t tablet_count = tablet_ids.count(); @@ -502,10 +464,10 @@ int ObTenantStorageMetaPersister::safe_batch_write_remove_tablets_slog_( } else { for (int64_t i = 0; OB_SUCC(ret) && i < tablet_count; ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); - ObDeleteTabletLog slog_entry(ls_id, tablet_id); + ObDeleteTabletLog slog_entry(tablet_id); if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet id is invalid", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet id is invalid", K(ret), K(tablet_id)); } else if (OB_FAIL(slog_array.push_back(slog_entry))) { LOG_WARN("fail to push slog entry into slog array", K(ret), K(slog_entry), K(i)); } @@ -528,93 +490,6 @@ int ObTenantStorageMetaPersister::safe_batch_write_remove_tablets_slog_( return ret; } -int ObTenantStorageMetaPersister::get_items_from_pending_free_tablet_array( - const ObLSID &ls_id, - const int64_t ls_epoch, - ObIArray &items) -{ - int ret = OB_SUCCESS; - items.reuse(); - PendingFreeTabletArrayKey key(ls_id, ls_epoch); - PendingFreeTabletArrayInfo *array_info = nullptr; - { - lib::ObMutexGuard guard(peding_free_map_lock_); - if (OB_FAIL(pending_free_tablet_arr_map_.get_refactored(key, array_info))) { - LOG_WARN("fail to get pending free tablet array info", K(ret), K(key)); - } - } - if (OB_FAIL(ret)) { - if (OB_HASH_NOT_EXIST == ret) { - array_info = nullptr; - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get pending free tablet array info from map", K(ret), K(key)); - } - } else if (OB_ISNULL(array_info)) { - ret = OB_ERR_UNEXPECTED; // get_refactored successfully, but array_info = nullptr - LOG_WARN("array info is nullptr", K(ret), K(key)); - } else { - lib::ObMutexGuard guard(array_info->lock_); - if (OB_FAIL(items.assign(array_info->pending_free_tablet_arr_.items_))) { - LOG_WARN("fail to assign array", K(ret), K(array_info->pending_free_tablet_arr_.items_)); - } - } - return ret; -} -int ObTenantStorageMetaPersister::delete_items_from_pending_free_tablet_array( - const ObLSID &ls_id, - const int64_t ls_epoch, - const ObIArray &items) -{ - int ret = OB_SUCCESS; - - PendingFreeTabletArrayKey key(ls_id, ls_epoch); - PendingFreeTabletArrayInfo *array_info = nullptr; - { - lib::ObMutexGuard guard(peding_free_map_lock_); - if (OB_FAIL(pending_free_tablet_arr_map_.get_refactored(key, array_info))) { - LOG_WARN("fail to get pending free tablet array info", K(ret), K(key)); - } - } // guard peding_free_map_lock_ - - if (OB_FAIL(ret)) { - if (OB_HASH_NOT_EXIST == ret) { - array_info = nullptr; - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get pending free tablet array info", K(ret), K(key)); - } - } else if (OB_ISNULL(array_info)) { - ret = OB_ERR_UNEXPECTED; // get_refactored successfully, but array_info = nullptr - LOG_WARN("array info is nullptr", K(ret), K(key)); - } else { - lib::ObMutexGuard guard(array_info->lock_); - const ObIArray &arr = array_info->pending_free_tablet_arr_.items_; - common::ObSArray tmp; - tmp.reserve(arr.count()); - int64_t delete_cnt = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < arr.count(); i++) { - if (has_exist_in_array(items, arr.at(i))) { - delete_cnt++; - } else if (OB_FAIL(tmp.push_back(arr.at(i)))) { - LOG_WARN("failed to push_back", K(ret), K(tmp), K(arr), K(items)); - } - } - if (OB_FAIL(ret)) { - // error occurred - } else if (items.count() == delete_cnt) { - // all deleted - if (OB_FAIL(array_info->pending_free_tablet_arr_.items_.assign(tmp))) { - LOG_WARN("failed to sync delete op to pending_free_items_arr", K(ret), K(tmp), K(arr), K(items)); - } - } else if (items.count() != delete_cnt) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("deleting item(s) do not all exist in pending_free_arr", K(ret), K(items), K(arr), K(tmp)); - } - } - return ret; -} - //=================================== Shared-Storage =============================================// diff --git a/src/storage/meta_store/ob_tenant_storage_meta_persister.h b/src/storage/meta_store/ob_tenant_storage_meta_persister.h index 7e90cb7b1..b9c5459fa 100644 --- a/src/storage/meta_store/ob_tenant_storage_meta_persister.h +++ b/src/storage/meta_store/ob_tenant_storage_meta_persister.h @@ -43,111 +43,38 @@ class ObTenantStorageMetaPersister void destroy(); int prepare_create_ls(const ObLSMeta &meta, int64_t &ls_epoch); - int commit_create_ls(const share::ObLSID &ls_id, const int64_t ls_epoch); - int abort_create_ls(const share::ObLSID &ls_id, const int64_t ls_epoch); - int delete_ls(const share::ObLSID &ls_id, const int64_t ls_epoch); + int commit_create_ls(); + int abort_create_ls(); + int delete_ls(); int update_ls_meta(const int64_t ls_epoch, const ObLSMeta &ls_meta); int update_tenant_preallocated_seqs(const ObTenantMonotonicIncSeqs &preallocated_seqs); int batch_update_tablet(const ObIArray &slog_arr); int batch_update_tablet(const ObIArray &slog_arr, ObIArray ¶m_arr); int update_tablet( - const share::ObLSID &ls_id, const int64_t ls_epoch, const common::ObTabletID &tablet_id, const ObMetaDiskAddr &disk_addr); int write_active_tablet_array(ObLS *ls); int write_empty_shell_tablet(ObTablet *tablet, ObMetaDiskAddr &tablet_addr); - int remove_tablet( - const share::ObLSID &ls_id, const int64_t ls_epoch, - const ObTabletHandle &tablet_handle); - int remove_tablets( - const share::ObLSID &ls_id, const int64_t ls_epoch, - const ObIArray &tablet_id_arr, const ObIArray &tablet_addr_arr); + int remove_tablet(const ObTabletHandle &tablet_handle); + int remove_tablets(const ObIArray &tablet_id_arr); - int get_items_from_pending_free_tablet_array( - const ObLSID &ls_id, - const int64_t ls_epoch, - ObIArray &items); - int delete_items_from_pending_free_tablet_array( - const ObLSID &ls_id, - const int64_t ls_epoch, - const ObIArray &items); private: int write_prepare_create_ls_slog_(const ObLSMeta &ls_meta); - int write_commit_create_ls_slog_(const share::ObLSID &ls_id); - int write_abort_create_ls_slog_(const share::ObLSID &ls_id); - int write_delete_ls_slog_(const share::ObLSID &ls_id); + int write_commit_create_ls_slog_(); + int write_abort_create_ls_slog_(); + int write_delete_ls_slog_(); int write_update_ls_meta_slog_(const ObLSMeta &ls_meta); int write_update_tablet_slog_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObMetaDiskAddr &tablet_addr); - int write_remove_tablet_slog_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); - int write_remove_tablets_slog_( - const ObLSID &ls_id, const common::ObIArray &tablet_ids); - int safe_batch_write_remove_tablets_slog_( - const ObLSID &ls_id, const common::ObIArray &tablet_ids); - -private: - struct PendingFreeTabletArrayKey - { - PendingFreeTabletArrayKey() : ls_id_(), ls_epoch_(0) {} - PendingFreeTabletArrayKey(const share::ObLSID &ls_id, const int64_t ls_epoch) - : ls_id_(ls_id), ls_epoch_(ls_epoch) {} - uint64_t hash() const - { - return ls_id_.hash() ^ ls_epoch_; - } - - int hash(uint64_t &hash_val) const - { - hash_val = hash(); - return OB_SUCCESS; - } - bool operator ==(const PendingFreeTabletArrayKey &other) const - { - return other.ls_id_ == ls_id_ && other.ls_epoch_ == ls_epoch_; - } - bool operator !=(const PendingFreeTabletArrayKey &other) const { return !(other == *this); } - bool operator <(const PendingFreeTabletArrayKey &other) const - { - bool bool_ret = false; - if (ls_id_ < other.ls_id_) { - bool_ret = true; - } else if (ls_id_ == other.ls_id_) { - bool_ret = (ls_epoch_ < other.ls_epoch_); - } - return bool_ret; - } - - TO_STRING_KV(K_(ls_id), K_(ls_epoch)); - - share::ObLSID ls_id_; - int64_t ls_epoch_; - }; - - struct PendingFreeTabletArrayInfo - { - PendingFreeTabletArrayInfo() - : lock_(), pending_free_tablet_arr_() {} - - lib::ObMutex lock_; - ObLSPendingFreeTabletArray pending_free_tablet_arr_; - }; - - typedef common::hash::ObHashMap< - PendingFreeTabletArrayKey, - PendingFreeTabletArrayInfo*, - common::hash::NoPthreadDefendMode> PendingFreeTabletArrayMap; + const common::ObTabletID &tablet_id, const ObMetaDiskAddr &tablet_addr); + int write_remove_tablet_slog_(const common::ObTabletID &tablet_id); + int write_remove_tablets_slog_(const common::ObIArray &tablet_ids); + int safe_batch_write_remove_tablets_slog_(const common::ObIArray &tablet_ids); private: bool is_inited_; storage::ObStorageLogger *slogger_; - common::ObConcurrentFIFOAllocator allocator_; - lib::ObMutex super_block_lock_; // protect tenant super block ObTenantCheckpointSlogHandler *ckpt_slog_handler_; - lib::ObMutex peding_free_map_lock_; // pending_free_tablet_arr_map_ - - PendingFreeTabletArrayMap pending_free_tablet_arr_map_; - }; } // namespace storage diff --git a/src/storage/meta_store/ob_tenant_storage_meta_replayer.h b/src/storage/meta_store/ob_tenant_storage_meta_replayer.h index c40310a44..d15e2ba5b 100644 --- a/src/storage/meta_store/ob_tenant_storage_meta_replayer.h +++ b/src/storage/meta_store/ob_tenant_storage_meta_replayer.h @@ -28,7 +28,6 @@ class ObStorageObjectOpt; namespace storage { class ObTenantSuperBlock; -class ObLSItem; class ObTenantStorageMetaPersister; class ObTenantCheckpointSlogHandler; class ObTenantStorageMetaReplayer diff --git a/src/storage/meta_store/ob_tenant_storage_meta_service.cpp b/src/storage/meta_store/ob_tenant_storage_meta_service.cpp index e83c932ac..6fa76d68a 100644 --- a/src/storage/meta_store/ob_tenant_storage_meta_service.cpp +++ b/src/storage/meta_store/ob_tenant_storage_meta_service.cpp @@ -19,6 +19,7 @@ #include "share/rc/ob_module_provider.h" #include "storage/meta_store/ob_storage_meta_io_util.h" #include "storage/meta_store/ob_server_storage_meta_service.h" +#include "storage/ob_file_system_router.h" #include "storage/tablet/ob_tablet_macro_info_iterator.h" #include "observer/omt/ob_tenant.h" #include "storage/ls/ob_ls.h" @@ -59,7 +60,10 @@ int ObTenantStorageMetaService::init() ret = OB_INIT_TWICE; LOG_WARN("has inited", K(ret)); } else if (!is_shared_storage && - OB_FAIL(slogger_.init(SERVER_STORAGE_META_SERVICE.get_slogger_manager()))) { + OB_FAIL(slogger_.init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec()))) { LOG_WARN("failed to init slogger", K(ret)); } else if (OB_FAIL(ckpt_slog_handler_.init(slogger_))) { LOG_WARN("fail to init tenant checkpoint slog hander", K(ret)); @@ -277,46 +281,5 @@ int ObTenantStorageMetaService::read_from_share_blk( return ret; } -int ObTenantStorageMetaService::ObLSItemIterator::get_next_ls_item( - storage::ObLSItem &item) -{ - int ret = OB_SUCCESS; - if (idx_ == tenant_super_block_.ls_cnt_) { - ret = OB_ITER_END; - } else { - item = tenant_super_block_.ls_item_arr_[idx_++]; - } - return ret; -} - -int ObTenantStorageMetaService::get_ls_items_by_status( - const storage::ObLSItemStatus status, - ObIArray &ls_items) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - ls_items.reuse(); - omt::ObTenant *tenant = static_cast(MTL_CTX()); - HEAP_VAR(ObLSItemIterator, ls_item_iter, tenant->get_super_block()) { - ObLSItem ls_item; - while (OB_SUCC(ls_item_iter.get_next_ls_item(ls_item))) { - if (status == ls_item.status_ && - OB_FAIL(ls_items.push_back(ls_item))) { - LOG_WARN("failed to push back tenant_item", K(ret), K(ls_item), K(ls_items), K(ls_item_iter)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get tenant items by status", K(ret), K(ls_item), K(ls_items), K(ls_item_iter)); - } - } - } - return ret; -} - } // namespace storage } // namespace oceanbase diff --git a/src/storage/meta_store/ob_tenant_storage_meta_service.h b/src/storage/meta_store/ob_tenant_storage_meta_service.h index b12606745..0878e37cd 100644 --- a/src/storage/meta_store/ob_tenant_storage_meta_service.h +++ b/src/storage/meta_store/ob_tenant_storage_meta_service.h @@ -75,24 +75,6 @@ class ObTenantStorageMetaService ObSharedObjectReaderWriter &get_shared_object_raw_reader_writer() { return shared_object_raw_rwriter_; } storage::ObStorageLogger &get_slogger() { return slogger_; } - class ObLSItemIterator final - { - public: - explicit ObLSItemIterator(const storage::ObTenantSuperBlock &super_block): - idx_(0), - tenant_super_block_(super_block) - {} - ~ObLSItemIterator() = default; - int get_next_ls_item(storage::ObLSItem &item); - TO_STRING_KV(K_(idx), K_(tenant_super_block)); - private: - int64_t idx_; - const storage::ObTenantSuperBlock tenant_super_block_; - DISALLOW_COPY_AND_ASSIGN(ObLSItemIterator); - }; - int get_ls_items_by_status( - const storage::ObLSItemStatus status, - ObIArray &ls_items); private: private: bool is_inited_; diff --git a/src/storage/multi_data_source/adapter_define/mds_dump_node.cpp b/src/storage/multi_data_source/adapter_define/mds_dump_node.cpp index 91e87d11d..4c3e9857b 100644 --- a/src/storage/multi_data_source/adapter_define/mds_dump_node.cpp +++ b/src/storage/multi_data_source/adapter_define/mds_dump_node.cpp @@ -480,20 +480,8 @@ int MdsDumpNode::deserialize(common::ObIAllocator &allocator, const char *buf, c version, mds_table_id_, mds_unit_id_, - writer_id_); - - - if (OB_FAIL(ret)) { - } else if (UNIS_VERSION_V1 == version) { - int64_t seq_no = 0; - LST_DO_CODE(OB_UNIS_DECODE, seq_no); - if (OB_SUCC(ret)) { - // compat logic, seq_no can be 0 or -1 in 431 - seq_no_ = ((0 == seq_no || -1 == seq_no) ? ObTxSEQ::MIN_VAL() : ObTxSEQ::mk_v0(seq_no)); - } - } else { - LST_DO_CODE(OB_UNIS_DECODE, seq_no_); - } + writer_id_, + seq_no_); LST_DO_CODE(OB_UNIS_DECODE, redo_scn_, @@ -517,13 +505,6 @@ int MdsDumpNode::deserialize(common::ObIAllocator &allocator, const char *buf, c } } - if (OB_FAIL(ret)) { - } else if (UNIS_VERSION_V1 == version) { - if (seq_no_.is_min()) { - crc_check_number_ = generate_hash(); - } - } - return ret; } diff --git a/src/storage/multi_data_source/adapter_define/mds_dump_node.h b/src/storage/multi_data_source/adapter_define/mds_dump_node.h index 263cc9936..7a8571865 100644 --- a/src/storage/multi_data_source/adapter_define/mds_dump_node.h +++ b/src/storage/multi_data_source/adapter_define/mds_dump_node.h @@ -22,7 +22,6 @@ #include "lib/hash_func/murmur_hash.h" #include "lib/utility/ob_unify_serialize.h" #include "meta_programming/ob_type_traits.h" -#include "share/ob_ls_id.h" #include "src/storage/multi_data_source/compile_utility/compile_mapper.h" #include "src/storage/multi_data_source/mds_node.h" #include "src/storage/multi_data_source/compile_utility/map_type_index_in_tuple.h" @@ -105,7 +104,7 @@ struct MdsDumpNode// RAII int64_t to_string(char *buf, const int64_t buf_len) const; int64_t simple_to_string(char *buf, const int64_t buf_len, int64_t &pos) const; template - int convert_to_user_mds_node(UserMdsNode &user_mds_node, const share::ObLSID &ls_id, const ObTabletID &tablet_id) const; + int convert_to_user_mds_node(UserMdsNode &user_mds_node) const; int assign(const MdsDumpNode &rhs, ObIAllocator &alloc); @@ -113,7 +112,6 @@ struct MdsDumpNode// RAII int deserialize(common::ObIAllocator &allocator, const char *buf, const int64_t data_len, int64_t &pos); int64_t get_serialize_size() const; - static constexpr int64_t UNIS_VERSION_V1 = 1; static constexpr int64_t UNIS_VERSION = 2; // member state @@ -285,7 +283,7 @@ inline int MdsDumpKey::convert_to_user_key(DummyKey &user_key) const } template -int MdsDumpNode::convert_to_user_mds_node(UserMdsNode &user_mds_node, const share::ObLSID &ls_id, const ObTabletID &tablet_id) const +int MdsDumpNode::convert_to_user_mds_node(UserMdsNode &user_mds_node) const { #define PRINT_WRAPPER KR(ret), K(*this), K(generated_hash), K(typeid(V).name()) int ret = OB_SUCCESS; diff --git a/src/storage/multi_data_source/compile_utility/mds_register.h b/src/storage/multi_data_source/compile_utility/mds_register.h index e6081db59..258027e53 100644 --- a/src/storage/multi_data_source/compile_utility/mds_register.h +++ b/src/storage/multi_data_source/compile_utility/mds_register.h @@ -70,13 +70,8 @@ #include "src/share/ob_tablet_autoincrement_param.h" #include "src/storage/compaction/ob_medium_compaction_info.h" #include "src/storage/multi_data_source/ob_tablet_create_mds_ctx.h" - #include "src/share/ob_standby_upgrade.h" - #include "src/storage/multi_data_source/ob_tablet_create_mds_ctx.h" - #include "src/share/ob_standby_upgrade.h" - #include "src/storage/mview/ob_major_mv_merge_info.h" #include "src/storage/truncate_info/ob_truncate_info.h" #include "src/storage/truncate_info/ob_truncate_info_mds_helper.h" - #include "src/storage/mview/ob_mview_mds.h" #include "src/storage/tablet/ob_tablet_ddl_complete_mds_helper.h" #include "src/storage/tablet/ob_tablet_ddl_complete_mds_data.h" #endif @@ -85,7 +80,7 @@ /**********************generate mds frame code with multi source transaction***********************/ // GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(HELPER_CLASS, BUFFER_CTX_TYPE, ID, ENUM_NAME) is an // interface MACRO to transfer necessary information to MDS FRAME to generate codes in transaction -// layer, mostly in ob_multi_data_source.cpp and ob_trans_part_ctx.cpp. +// layer, mostly in ob_multi_data_source.cpp and ob_tx_ctx.cpp. // // @param HELPER_CLASS the class must has two static method signatures(or COMPILE ERROR): // 1. static int on_register(const char* buf, @@ -139,38 +134,14 @@ _GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION_(HELPER_CLASS, BUFFER_CTX_TYPE, ID, ENU ::oceanbase::storage::mds::MdsCtx,\ 31,\ TABLET_SPLIT) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::share::ObUpgradeDataVersionMDSHelper, \ - ::oceanbase::storage::mds::MdsCtx, \ - 33,\ - STANDBY_UPGRADE_DATA_VERSION) GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObTabletBindingMdsHelper,\ ::oceanbase::storage::mds::MdsCtx,\ 34,\ TABLET_BINDING) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVPublishSCNHelper,\ - ::oceanbase::storage::ObUnUseCtx, \ - 35,\ - MV_PUBLISH_SCN) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVNoticeSafeHelper,\ - ::oceanbase::storage::ObUnUseCtx, \ - 36,\ - MV_NOTICE_SAFE) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVUpdateSCNHelper,\ - ::oceanbase::storage::ObUnUseCtx, \ - 37,\ - MV_UPDATE_SCN) GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObTruncateInfoMdsHelper,\ ::oceanbase::storage::mds::MdsCtx, \ 38,\ SYNC_TRUNCATE_INFO) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVMergeSCNHelper,\ - ::oceanbase::storage::ObUnUseCtx, \ - 39,\ - MV_MERGE_SCN) - GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMViewMdsOpHelper,\ - ::oceanbase::storage::ObMViewMdsOpCtx, \ - 40,\ - MVIEW_MDS_OP) GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObTabletDDLCompleteMdsHelper,\ ::oceanbase::storage::mds::MdsCtx,\ 41,\ diff --git a/src/storage/multi_data_source/mds_ctx.cpp b/src/storage/multi_data_source/mds_ctx.cpp index b30225e8e..b7337895d 100644 --- a/src/storage/multi_data_source/mds_ctx.cpp +++ b/src/storage/multi_data_source/mds_ctx.cpp @@ -86,7 +86,7 @@ int MdsCtx::inc_seq_no() if (OB_FAIL(ObSequence::get_and_inc_max_seq_no(0, seq))) { MDS_LOG(WARN, "fail to get and inc max seq no", K(ret), KPC(this)); } else { - seq_no_ = transaction::ObTxSEQ::mk_v0(seq); + seq_no_ = transaction::ObTxSEQ(seq, 0); } return ret; } diff --git a/src/storage/multi_data_source/mds_node.cpp b/src/storage/multi_data_source/mds_node.cpp index 9eb41e2ed..5b861242b 100644 --- a/src/storage/multi_data_source/mds_node.cpp +++ b/src/storage/multi_data_source/mds_node.cpp @@ -190,7 +190,6 @@ int MdsNodeInfoForVirtualTable::assign(const MdsNodeInfoForVirtualTable &rhs) } else if (OB_FAIL(user_data_.assign(rhs.user_data_))) { MDS_LOG(WARN, "fail to copy user data", KR(ret)); } else { - ls_id_ = rhs.ls_id_; tablet_id_ = rhs.tablet_id_; unit_id_ = rhs.unit_id_; version_idx_ = rhs.version_idx_; diff --git a/src/storage/multi_data_source/mds_node.h b/src/storage/multi_data_source/mds_node.h index 14bc053bf..22c69bf8a 100644 --- a/src/storage/multi_data_source/mds_node.h +++ b/src/storage/multi_data_source/mds_node.h @@ -23,7 +23,6 @@ #include "lib/queue/ob_link.h" #include "lib/time/ob_clock_generator.h" #include "ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "share/scn.h" #include "runtime_utility/common_define.h" #include "runtime_utility/list_helper.h" @@ -60,10 +59,9 @@ class MdsCtx; struct MdsNodeInfoForVirtualTable { MdsNodeInfoForVirtualTable() - : ls_id_(), tablet_id_(), unit_id_(UINT8_MAX), user_key_(), version_idx_(-1), writer_(), seq_no_(), redo_scn_(), + : tablet_id_(), unit_id_(UINT8_MAX), user_key_(), version_idx_(-1), writer_(), seq_no_(), redo_scn_(), end_scn_(), trans_version_(), node_type_(MdsNodeType::TYPE_END), state_(TwoPhaseCommitState::STATE_END), position_(NodePosition::POSITION_END), user_data_() {} - share::ObLSID ls_id_; common::ObTabletID tablet_id_; int64_t unit_id_; common::ObStringHolder user_key_; diff --git a/src/storage/multi_data_source/mds_node.ipp b/src/storage/multi_data_source/mds_node.ipp index 8572dbef2..4f4f39583 100644 --- a/src/storage/multi_data_source/mds_node.ipp +++ b/src/storage/multi_data_source/mds_node.ipp @@ -99,11 +99,6 @@ int64_t UserMdsNode::to_string(char * buf, const int64_t buf_len) const #else databuff_printf(buf, buf_len, pos, "this:0x%lx, ", (unsigned long)this); #endif - if (nullptr != p_mds_table) { - databuff_print_multi_objs(buf, buf_len, pos, "ls_id:", p_mds_table->ls_id_, ", "); - } else { - databuff_printf(buf, buf_len, pos, "ls_id:%s, ", "NULL"); - } if (nullptr != p_mds_table) { databuff_print_multi_objs(buf, buf_len, pos, "tablet_id:", p_mds_table->tablet_id_, ", "); } else { @@ -362,7 +357,6 @@ int UserMdsNode::fill_virtual_info(MdsNodeInfoForVirtualTable &mds_node_in MDS_LOG(WARN, "fail construct ObStringHolder", K(*this)); } else { if (OB_LIKELY(has_valid_link_back_ptr_())) { - mds_node_info.ls_id_ = p_mds_row_->p_mds_unit_->p_mds_table_->get_ls_id(); mds_node_info.tablet_id_ = p_mds_row_->p_mds_unit_->p_mds_table_->get_tablet_id(); } mds_node_info.writer_ = MdsWriter(status_.union_.field_.writer_type_, writer_id_); @@ -393,8 +387,7 @@ void UserMdsNode::report_event_(const char (&event_str)[N], } else if (OB_FAIL(fill_event_(event, event_str, stack_buffer, buffer_size))) { MDS_LOG(WARN, "fail fill mds event", K(*this)); } else { - observer::MdsEventKey key(p_mds_row_->p_mds_unit_->p_mds_table_->ls_id_, - p_mds_row_->p_mds_unit_->p_mds_table_->tablet_id_); + observer::MdsEventKey key(p_mds_row_->p_mds_unit_->p_mds_table_->tablet_id_); observer::ObMdsEventBuffer::append(key, event, p_mds_row_->p_mds_unit_->p_mds_table_, file, line, function_name); } } diff --git a/src/storage/multi_data_source/mds_row.ipp b/src/storage/multi_data_source/mds_row.ipp index 135f55f1f..da210326f 100644 --- a/src/storage/multi_data_source/mds_row.ipp +++ b/src/storage/multi_data_source/mds_row.ipp @@ -653,17 +653,17 @@ struct DumpNodeOP { MdsAllocator::get_instance()))) { MDS_LOG_SCAN(WARN, "failt to convert user mds node to dump node", K(node)); /**********************this is different logic from normal logic***************************************************/ - } else if (specified_node.user_data_.data_type_ == ObTabletMdsUserDataType::RESERVED_7) { + } else if (specified_node.user_data_.data_type_ == ObTabletMdsUserDataType::RESERVED_MDS_TYPE_7) { if (specified_node.seq_no_.is_min()) { - dump_kv_.v_.seq_no_ = transaction::ObTxSEQ::mk_v0(47); + dump_kv_.v_.seq_no_ = transaction::ObTxSEQ(47, 0); dump_kv_.v_.crc_check_number_ = dump_kv_.v_.generate_hash(); MDS_LOG(INFO, "convert reserved tablet status mds node's seq no from min to 47 for compat issue", K(node), K(dump_kv_)); } else { MDS_LOG(TRACE, "no need convert reserved tablet status mds node's seq no cause it is valid", K(node), K(dump_kv_)); } - } else if (specified_node.user_data_.data_type_ == ObTabletMdsUserDataType::RESERVED_3) { + } else if (specified_node.user_data_.data_type_ == ObTabletMdsUserDataType::RESERVED_MDS_TYPE_3) { if (specified_node.seq_no_.is_min()) { - dump_kv_.v_.seq_no_ = transaction::ObTxSEQ::mk_v0(100); + dump_kv_.v_.seq_no_ = transaction::ObTxSEQ(100, 0); dump_kv_.v_.crc_check_number_ = dump_kv_.v_.generate_hash(); MDS_LOG(INFO, "convert reserved tablet status mds node's seq no from min to 100 for compat issue", K(node), K(dump_kv_)); } else { @@ -881,8 +881,7 @@ void MdsRow::report_event_(const char (&event_str)[N], } else if (OB_FAIL(node.fill_event_(event, event_str, stack_buffer, buffer_size))) { MDS_LOG(WARN, "fail fill mds event", K(*this)); } else { - observer::MdsEventKey key(MdsRowBase::p_mds_unit_->p_mds_table_->ls_id_, - MdsRowBase::p_mds_unit_->p_mds_table_->tablet_id_); + observer::MdsEventKey key(MdsRowBase::p_mds_unit_->p_mds_table_->tablet_id_); observer::ObMdsEventBuffer::append(key, event, MdsRowBase::p_mds_unit_->p_mds_table_, file, line, function_name); } } diff --git a/src/storage/multi_data_source/mds_table_base.cpp b/src/storage/multi_data_source/mds_table_base.cpp index d5d95b01f..1bd165b94 100644 --- a/src/storage/multi_data_source/mds_table_base.cpp +++ b/src/storage/multi_data_source/mds_table_base.cpp @@ -49,21 +49,19 @@ int MdsTableBase::advance_state_to(State new_state) const } int MdsTableBase::init(const ObTabletID tablet_id, - const share::ObLSID ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action ObTabletPointer *pointer, ObMdsTableMgr *p_mgr) { int ret = OB_SUCCESS; MDS_TG(1_ms); - if (!ls_id.is_valid() || !tablet_id.is_valid()) { + if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - MDS_LOG(ERROR, "invalid argument", KR(ret), K(*this), K(ls_id), K(tablet_id)); + MDS_LOG(ERROR, "invalid argument", KR(ret), K(*this), K(tablet_id)); } else if (MDS_FAIL(advance_state_to(State::INIT))) { - MDS_LOG(ERROR, "mds table maybe init twice", KR(ret), K(*this), K(ls_id), K(tablet_id)); + MDS_LOG(ERROR, "mds table maybe init twice", KR(ret), K(*this), K(tablet_id)); } else { tablet_id_ = tablet_id; - ls_id_ = ls_id; last_inner_recycled_scn_ = mds_ckpt_scn_from_tablet; if (OB_NOT_NULL(p_mgr)) { mgr_handle_.set_mds_table_mgr(p_mgr); @@ -71,7 +69,7 @@ int MdsTableBase::init(const ObTabletID tablet_id, debug_info_.init_trace_id_ = *ObCurTraceId::get_trace_id(); debug_info_.init_ts_ = ObClockGenerator::getClock(); if (MDS_FAIL(register_to_mds_table_mgr())) { - MDS_LOG(ERROR, "fail to register mds table", KR(ret), K(*this), K(ls_id), K(tablet_id)); + MDS_LOG(ERROR, "fail to register mds table", KR(ret), K(*this), K(tablet_id)); } } } @@ -138,7 +136,7 @@ int MdsTableBase::unregister_from_mds_table_mgr() MDS_TG(1_ms); if (!mgr_handle_.is_valid()) { MDS_LOG(INFO, "no need unregister from mds_table_mgr cause invalid mds_table_mgr", KR(ret), K(*this)); - } else if (!ls_id_.is_valid() || !tablet_id_.is_valid()) { + } else if (!tablet_id_.is_valid()) { MDS_LOG(INFO, "no need unregister from mds_table_mgr cause invalid id", KR(ret), K(*this)); } else if (MDS_FAIL(mgr_handle_.get_mds_table_mgr()->unregister_from_mds_table_mgr(this))) { MDS_LOG(ERROR, "fail to unregister mds table", K(*this)); @@ -154,7 +152,7 @@ int MdsTableBase::unregister_from_removed_recorder() MDS_TG(1_ms); if (!mgr_handle_.is_valid()) { MDS_LOG(INFO, "no need unregister from mds_table_mgr cause invalid mds_table_mgr", KR(ret), K(*this)); - } else if (!ls_id_.is_valid() || !tablet_id_.is_valid()) { + } else if (!tablet_id_.is_valid()) { MDS_LOG(INFO, "no need unregister from mds_table_mgr cause invalid id", KR(ret), K(*this)); } else { mgr_handle_.get_mds_table_mgr()->unregister_from_removed_mds_table_recorder(this); @@ -167,17 +165,14 @@ int MdsTableBase::get_ls_max_consequent_callbacked_scn_(share::SCN &max_conseque { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; MDS_TG(1_ms); - if (!ls_id_.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG(WARN, "ls id not valid", KR(ret), K(*this)); - } else if (OB_ISNULL(ls_service)) { + if (OB_ISNULL(ls_service)) { ret = OB_BAD_NULL_ERROR; MDS_LOG(WARN, "ls tx service is null", KR(ret), K(*this)); - } else if (MDS_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - MDS_LOG(WARN, "fail to get ls handle", KR(ret), K(*this)); - } else if (MDS_FAIL(ls_handle.get_ls()->get_freezer()->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { + } else if (MDS_FAIL(ls_service->get_ls(tenant_ls))) { + MDS_LOG(WARN, "fail to get ls", KR(ret), K(*this)); + } else if (MDS_FAIL(tenant_ls->get_freezer()->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { MDS_LOG(WARN, "fail to get max_consequent_callbacked_scn", KR(ret), K(*this)); } return ret; @@ -187,7 +182,6 @@ int MdsTableBase::merge(const int64_t construct_sequence, const share::SCN &flus { int ret = OB_SUCCESS; ObMdsTableMergeDagParam param; - param.ls_id_ = ls_id_; param.tablet_id_ = tablet_id_; param.flush_scn_ = flushing_scn; param.mds_construct_sequence_ = construct_sequence; @@ -280,11 +274,6 @@ common::ObTabletID MdsTableBase::get_tablet_id() const return tablet_id_; } -share::ObLSID MdsTableBase::get_ls_id() const -{ - return ls_id_; -} - share::SCN MdsTableBase::get_rec_scn() { return rec_scn_.atomic_get(); diff --git a/src/storage/multi_data_source/mds_table_base.h b/src/storage/multi_data_source/mds_table_base.h index 828bbe319..d97446b83 100644 --- a/src/storage/multi_data_source/mds_table_base.h +++ b/src/storage/multi_data_source/mds_table_base.h @@ -102,7 +102,6 @@ class MdsTableBase : public ListNode public: MdsTableBase() : state_(State::UNKNOWN), - ls_id_(), tablet_id_(), flushing_scn_(), last_inner_recycled_scn_(share::SCN::min_scn()), @@ -113,7 +112,6 @@ class MdsTableBase : public ListNode lock_() { construct_sequence_ = ObMdsGlobalSequencer::generate_senquence(); } virtual ~MdsTableBase() {} int init(const ObTabletID tablet_id, - const share::ObLSID ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action ObTabletPointer *pointer, ObMdsTableMgr *p_mgr); @@ -172,7 +170,6 @@ class MdsTableBase : public ListNode const ScanNodeOrder scan_node_order) const = 0; virtual void on_flush(const share::SCN &flushed_scn, const int flush_ret) = 0; virtual int try_recycle(const share::SCN recycle_scn) = 0; - share::ObLSID get_ls_id() const; int64_t get_node_cnt() const; virtual share::SCN get_rec_scn(); virtual int operate(const ObFunction &operation) = 0; @@ -207,8 +204,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.reset(); event.event_ = "CONSTRUCTED"; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); } void report_destruct_event_(const char *file = __builtin_FILE(), @@ -219,8 +215,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.reset(); event.event_ = "DESTRUCTED"; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); } template @@ -241,8 +236,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.assign(stack_buffer, pos); event.event_ = event_str; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); } template @@ -261,8 +255,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.assign(stack_buffer, pos); event.event_ = event_str; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); } @@ -282,8 +275,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.assign(stack_buffer, pos); event.event_ = event_str; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); @@ -302,8 +294,7 @@ class MdsTableBase : public ListNode event.record_thread_info_(); event.info_str_.assign(stack_buffer, pos); event.event_ = "RECYCLE"; - observer::MdsEventKey key(ls_id_, - tablet_id_); + observer::MdsEventKey key(tablet_id_); observer::ObMdsEventBuffer::append(key, event, this, file, line, function_name); } public: @@ -331,7 +322,6 @@ class MdsTableBase : public ListNode ObCurTraceId::TraceId remove_trace_id_; } debug_info_;// 120B mutable State state_; - share::ObLSID ls_id_; ObTabletID tablet_id_; share::SCN flushing_scn_;// To tell if this mds table is flushing share::SCN last_inner_recycled_scn_;// To filter repeated release operation, and filter replay operation diff --git a/src/storage/multi_data_source/mds_table_handle.h b/src/storage/multi_data_source/mds_table_handle.h index 634d14c7c..8c2c4ef66 100644 --- a/src/storage/multi_data_source/mds_table_handle.h +++ b/src/storage/multi_data_source/mds_table_handle.h @@ -40,11 +40,9 @@ class MdsTableHandle bool operator==(const MdsTableHandle &rhs) const; ~MdsTableHandle(); int get_tablet_id(common::ObTabletID &tablet_id) const; - int get_ls_id(share::ObLSID &ls_id) const; template int init(ObIAllocator &allocator, const ObTabletID tablet_id, - const share::ObLSID ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action ObTabletPointer *pointer, ObMdsTableMgr *mgr_handle = nullptr); diff --git a/src/storage/multi_data_source/mds_table_handle.ipp b/src/storage/multi_data_source/mds_table_handle.ipp index 51d30a5b9..bd7e7d093 100644 --- a/src/storage/multi_data_source/mds_table_handle.ipp +++ b/src/storage/multi_data_source/mds_table_handle.ipp @@ -111,23 +111,9 @@ inline int MdsTableHandle::get_tablet_id(common::ObTabletID &tablet_id) const return ret; } -inline int MdsTableHandle::get_ls_id(share::ObLSID &ls_id) const -{ - int ret = OB_SUCCESS; - CHECK_MDS_TABLE_INIT(); - if (!p_mds_table_base_.is_valid()) { - ret = OB_BAD_NULL_ERROR; - MDS_LOG(WARN, "p_mds_table_base_ is invalid", K(*this)); - } else { - ls_id = p_mds_table_base_->get_ls_id(); - } - return ret; -} - template int MdsTableHandle::init(ObIAllocator &allocator, const ObTabletID tablet_id, - const share::ObLSID ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action ObTabletPointer *pointer, ObMdsTableMgr *p_mgr) @@ -146,7 +132,7 @@ int MdsTableHandle::init(ObIAllocator &allocator, if (OB_SUCC(ret)) { if (OB_FAIL(p_mds_table.construct(allocator))) { MDS_LOG(WARN, "construct mds table impl failed", KP(this), K(lbt())); - } else if (OB_FAIL(p_mds_table->init(tablet_id, ls_id, mds_ckpt_scn_from_tablet, pointer, p_mgr))) { + } else if (OB_FAIL(p_mds_table->init(tablet_id, mds_ckpt_scn_from_tablet, pointer, p_mgr))) { MDS_LOG(WARN, "init mds table failed", KR(ret), K(mds_table_id_), K(typeid(MdsTableType).name())); } else { diff --git a/src/storage/multi_data_source/mds_table_handler.cpp b/src/storage/multi_data_source/mds_table_handler.cpp index 95dc70d0f..3af1cd548 100644 --- a/src/storage/multi_data_source/mds_table_handler.cpp +++ b/src/storage/multi_data_source/mds_table_handler.cpp @@ -31,16 +31,13 @@ ObMdsTableHandler::~ObMdsTableHandler() { ATOMIC_STORE(&is_written_, false); } int ObMdsTableHandler::get_mds_table_handle(mds::MdsTableHandle &handle, const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action const bool not_exist_create, ObTabletPointer *pointer) { - #define PRINT_WRAPPER KR(ret), K(tablet_id), K(ls_id), K(not_exist_create), K(*this) + #define PRINT_WRAPPER KR(ret), K(tablet_id), K(not_exist_create), K(*this) int ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; mds::ObMdsTableMgr *mds_table_mgr; auto try_get_handle_directly = [this](mds::MdsTableHandle &handle) -> void { @@ -68,17 +65,10 @@ int ObMdsTableHandler::get_mds_table_handle(mds::MdsTableHandle &handle, // do nothing } else { if (OB_ISNULL(mds_table_mgr_handle_.get_mds_table_mgr())) { - if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_INIT(WARN, "ls service should not be NULL"); - } else if (MDS_FAIL(ls_service->get_ls(ls_id, - ls_handle, - ObLSGetMod::TABLET_MOD))) { + if (MDS_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { MDS_LOG_INIT(WARN, "failed to get ls"); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_INIT(WARN, "ls should not be NULL"); - } else if (MDS_FAIL(ls->get_tablet_svr()->get_mds_table_mgr(mds_table_mgr_handle_))) { + } else if (MDS_FAIL(tenant_ls->get_tablet_svr()->get_mds_table_mgr( + mds_table_mgr_handle_))) { MDS_LOG_INIT(WARN, "get mds table mgr failed"); } else if (OB_ISNULL(mds_table_mgr = mds_table_mgr_handle_.get_mds_table_mgr())) { ret = OB_ERR_UNEXPECTED; @@ -88,7 +78,6 @@ int ObMdsTableHandler::get_mds_table_handle(mds::MdsTableHandle &handle, if (OB_SUCC(ret)) { if (MDS_FAIL(mds_table_handle_.init(mds::MdsAllocator::get_instance(), tablet_id, - ls_id, mds_ckpt_scn_from_tablet, pointer, mds_table_mgr_handle_.get_mds_table_mgr()))) { diff --git a/src/storage/multi_data_source/mds_table_handler.h b/src/storage/multi_data_source/mds_table_handler.h index 42ebfb31b..1311dcee7 100644 --- a/src/storage/multi_data_source/mds_table_handler.h +++ b/src/storage/multi_data_source/mds_table_handler.h @@ -39,7 +39,6 @@ class ObMdsTableHandler ObMdsTableHandler &operator=(const ObMdsTableHandler &);// value sematic for tablet ponter deep copy int get_mds_table_handle(mds::MdsTableHandle &handle, const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const share::SCN mds_ckpt_scn_from_tablet,// this is used to filter replayed nodes after removed action const bool not_exist_create, ObTabletPointer *pointer); diff --git a/src/storage/multi_data_source/mds_table_impl.h b/src/storage/multi_data_source/mds_table_impl.h index cd54a5241..6fadb14d3 100644 --- a/src/storage/multi_data_source/mds_table_impl.h +++ b/src/storage/multi_data_source/mds_table_impl.h @@ -36,7 +36,6 @@ #include "adapter_define/mds_dump_node.h" #include "mds_writer.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "mds_table_base.h" #include "compile_utility/map_type_index_in_tuple.h" #include "storage/multi_data_source/compile_utility/compile_mapper.h" @@ -168,7 +167,7 @@ class MdsTableImpl final : public MdsTableBase const bool for_flush, const ScanRowOrder scan_row_order, const ScanNodeOrder scan_node_order); - TO_STRING_KV(KP(this), K_(ls_id), K_(tablet_id), K_(flushing_scn), K_(rec_scn), K_(max_aborted_scn), + TO_STRING_KV(KP(this), K_(tablet_id), K_(flushing_scn), K_(rec_scn), K_(max_aborted_scn), K_(last_inner_recycled_scn), K_(total_node_cnt), K_(construct_sequence), K_(debug_info)); template int for_each_scan_row(FowEachRowAction action_type, SCAN_OP &&op); diff --git a/src/storage/multi_data_source/mds_table_impl.ipp b/src/storage/multi_data_source/mds_table_impl.ipp index 21b9d2db9..f9206df0a 100644 --- a/src/storage/multi_data_source/mds_table_impl.ipp +++ b/src/storage/multi_data_source/mds_table_impl.ipp @@ -585,7 +585,7 @@ int MdsTableImpl::get_latest(int64_t unit_id, return ret; } -// Only normal mds table supports this compatibility method. +// Only normal mds table supports tablet status node inspection. template int MdsTableImpl::get_tablet_status_node(ObFunction &op, const int64_t read_seq) const { @@ -1260,7 +1260,7 @@ template ::type> int MdsTableImpl::scan_all_nodes_to_dump(DUMP_OP &&for_each_op, const int64_t mds_construct_sequence, - /* false scans active nodes in addition to flush nodes */ + /*false is used by non-flush dump paths to copy mds data*/ const bool for_flush, const ScanRowOrder scan_row_order, const ScanNodeOrder scan_node_order) { diff --git a/src/storage/multi_data_source/mds_table_iterator.ipp b/src/storage/multi_data_source/mds_table_iterator.ipp index e71b7729b..f8d592f16 100644 --- a/src/storage/multi_data_source/mds_table_iterator.ipp +++ b/src/storage/multi_data_source/mds_table_iterator.ipp @@ -166,7 +166,6 @@ int ObMdsUnitRowNodeScanIterator::init(mds::MdsTableHandle & const int64_t timeout_ts) { #define PRINT_WRAPPER KR(ret), K(*this), K(mds_table_handle) int ret = OB_SUCCESS; - share::ObLSID ls_id; if (is_inited_) { ret = OB_INIT_TWICE; MDS_LOG_NONE(WARN, "ObMdsUnitRowNodeScanIterator init twice"); @@ -175,15 +174,13 @@ int ObMdsUnitRowNodeScanIterator::init(mds::MdsTableHandle & MDS_LOG_NONE(WARN, "mds_table_handle invalid"); } else if (OB_FAIL(filter_function_.assign(filter))) { MDS_LOG_NONE(WARN, "failed to init filter function"); - } else if (OB_FAIL(mds_table_handle.get_ls_id(ls_id))) { - MDS_LOG_NONE(WARN, "failed to get ls_id from mds_table"); } else { int64_t current_ts = ObClockGenerator::getClock(); int64_t timeout_us = timeout_ts - current_ts > 0 ? timeout_ts - current_ts : 0; mds_table_handle_ = mds_table_handle; is_inited_ = true; is_first_scan_ = true; - new (&retry_param_) RetryParam(ls_id, timeout_us); + new (&retry_param_) RetryParam(timeout_us); } return ret; #undef PRINT_WRAPPER @@ -263,11 +260,6 @@ int ObMdsUnitRowNodeScanIterator::cache_all_nodes_in_row_(Kv ret = OB_SUCCESS; } else if (retry_param_.check_timeout()) { ret = OB_TIMEOUT; - } else if ((retry_param_.retry_cnt_ % 50) == 0) {// every 500ms - if (retry_param_.check_ls_in_gc_state()) { - ret = OB_REPLICA_NOT_READABLE; - MDS_LOG_NONE(INFO, "iter scan mds node meet ls gc"); - } } } } while (OB_EAGAIN == ret); diff --git a/src/storage/multi_data_source/mds_table_mgr.cpp b/src/storage/multi_data_source/mds_table_mgr.cpp index 1be0ad1ce..abfdc6014 100644 --- a/src/storage/multi_data_source/mds_table_mgr.cpp +++ b/src/storage/multi_data_source/mds_table_mgr.cpp @@ -143,7 +143,7 @@ struct OrderOp { }; int ObMdsTableMgr::flush(SCN recycle_scn, bool need_freeze) { - #define PRINT_WRAPPER KR(ret), K(ls_->get_ls_id()), K(recycle_scn), K(need_freeze), K(order_flusher_for_some),\ + #define PRINT_WRAPPER KR(ret), K(recycle_scn), K(need_freeze), K(order_flusher_for_some),\ K(max_consequent_callbacked_scn), K(*this) MDS_TG(10_s); int ret = OB_SUCCESS; @@ -196,7 +196,7 @@ void ObMdsTableMgr::order_flush_(FlusherForSome &order_flusher_for_some, share::SCN freezing_scn, share::SCN max_consequent_callbacked_scn) { - #define PRINT_WRAPPER KR(ret), K(ls_->get_ls_id()), K(freezing_scn), K(order_flusher_for_some),\ + #define PRINT_WRAPPER KR(ret), K(freezing_scn), K(order_flusher_for_some),\ K(third_sacn_mds_table_cnt), K(max_consequent_callbacked_scn), K(order_flusher_for_all.count()),\ K(*this) MDS_TG(10_s); diff --git a/src/storage/multi_data_source/mds_unit.ipp b/src/storage/multi_data_source/mds_unit.ipp index c923b6744..68f8ae052 100644 --- a/src/storage/multi_data_source/mds_unit.ipp +++ b/src/storage/multi_data_source/mds_unit.ipp @@ -328,7 +328,7 @@ int MdsUnit::set(MdsTableBase *p_mds_table, int ret = OB_SUCCESS; MDS_TG(10_ms); bool is_lvalue = std::is_lvalue_reference::value; - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); SetOP op(this, is_lvalue, p_mds_table, key, value, ctx, is_for_remove, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { if (OB_TIMEOUT == ret) { @@ -408,7 +408,7 @@ int MdsUnit::get_snapshot(const K &key, { int ret = OB_SUCCESS; MDS_TG(10_ms); - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); GetSnapShotOp::type> op(this, key, read_op, snapshot, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { if (OB_TIMEOUT == ret) { @@ -455,7 +455,7 @@ int MdsUnit::get_by_writer(const K &key, { int ret = OB_SUCCESS; MDS_TG(10_ms); - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); GetByWriterOp::type> op(this, key, read_op, writer, snapshot, read_seq, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { if (OB_TIMEOUT == ret) { @@ -630,8 +630,7 @@ void MdsUnit::report_event_(const char (&event_str)[N], } else { event.key_str_.assign(stack_buffer, pos); event.event_ = event_str; - observer::MdsEventKey key(MdsUnitBase::p_mds_table_->ls_id_, - MdsUnitBase::p_mds_table_->tablet_id_); + observer::MdsEventKey key(MdsUnitBase::p_mds_table_->tablet_id_); observer::ObMdsEventBuffer::append(key, event, MdsUnitBase::p_mds_table_, file, line, function_name); } } @@ -771,7 +770,7 @@ int MdsUnit::set(MdsTableBase *p_mds_table, { int ret = OB_SUCCESS; MDS_TG(10_ms); - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); bool is_lvalue = std::is_lvalue_reference::value; SetOP op(this, is_lvalue, value, ctx, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { @@ -829,7 +828,7 @@ int MdsUnit::get_snapshot(OP &&read_op, { int ret = OB_SUCCESS; MDS_TG(10_ms); - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); GetSnapShotOp::type> op(this, read_op, snapshot, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { if (OB_TIMEOUT == ret) { @@ -869,7 +868,7 @@ int MdsUnit::get_by_writer(OP &&read_op, { int ret = OB_SUCCESS; MDS_TG(10_ms); - RetryParam retry_param(MdsUnitBase::p_mds_table_->ls_id_, lock_timeout_us); + RetryParam retry_param(lock_timeout_us); GetByWriterOp::type> op(this, read_op, writer, snapshot, read_seq, retry_param); if (MDS_FAIL(retry_release_lock_with_op_until_timeout(lock_, retry_param, op))) { if (OB_TIMEOUT == ret) { diff --git a/src/storage/multi_data_source/ob_mds_minor_compaction_filter.cpp b/src/storage/multi_data_source/ob_mds_minor_compaction_filter.cpp index 9393e5010..4ff8f75b5 100644 --- a/src/storage/multi_data_source/ob_mds_minor_compaction_filter.cpp +++ b/src/storage/multi_data_source/ob_mds_minor_compaction_filter.cpp @@ -133,36 +133,5 @@ int ObMdsMinorFilter::filter_truncate_info( return ret; } -ObCrossLSMdsMinorFilter::ObCrossLSMdsMinorFilter() - : ObICompactionFilter() -{ -} - -int ObCrossLSMdsMinorFilter::filter( - const blocksstable::ObDatumRow &row, - ObFilterRet &filter_ret) -{ - int ret = OB_SUCCESS; - filter_ret = FILTER_RET_MAX; - mds::MdsDumpKVStorageAdapter kv_adapter; - constexpr uint8_t tablet_status_mds_unit_id = mds::TupleTypeIdx>::value; - - if (OB_FAIL(kv_adapter.convert_from_mds_multi_version_row(row))) { - LOG_WARN("fail to convert from mds multi version row", K(ret), K(row)); - } else if (tablet_status_mds_unit_id == kv_adapter.get_type()) { - if (OB_UNLIKELY(row.is_uncommitted_row() - || !row.is_compacted_multi_version_row())) { // not filter uncommitted row - ret = OB_INVALID_ARGUMENT; - LOG_WARN("uncommitted row or uncompacted row in mds table", K(ret), K(row)); - } else { - filter_ret = FILTER_RET_REMOVE; - LOG_DEBUG("filter tablet status for cross ls mds minor merge", K(ret)); - } - } else { - filter_ret = FILTER_RET_NOT_CHANGE; - } - - return ret; -} } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/ob_mds_minor_compaction_filter.h b/src/storage/multi_data_source/ob_mds_minor_compaction_filter.h index 4c340e0a6..79010e7cd 100644 --- a/src/storage/multi_data_source/ob_mds_minor_compaction_filter.h +++ b/src/storage/multi_data_source/ob_mds_minor_compaction_filter.h @@ -61,15 +61,6 @@ class ObMdsMinorFilter : public compaction::ObICompactionFilter ObArenaAllocator allocator_; }; -class ObCrossLSMdsMinorFilter : public compaction::ObICompactionFilter -{ -public: - ObCrossLSMdsMinorFilter(); - virtual ~ObCrossLSMdsMinorFilter() = default; -public: - virtual int filter(const blocksstable::ObDatumRow &row, ObFilterRet &filter_ret) override; - virtual CompactionFilterType get_filter_type() const override { return MDS_MINOR_CROSS_LS; } -}; } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/ob_mds_table_merge_dag.cpp b/src/storage/multi_data_source/ob_mds_table_merge_dag.cpp index f7ec2a822..2aa0bda5b 100644 --- a/src/storage/multi_data_source/ob_mds_table_merge_dag.cpp +++ b/src/storage/multi_data_source/ob_mds_table_merge_dag.cpp @@ -84,13 +84,13 @@ int ObMdsTableMergeDag::fill_compat_mode_() // between mds table lock on ObTabletPointer and other mds component // inner locks. So here use no_lock to get tablet. - ObLSHandle tmp_ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle tmp_tablet_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, tmp_ls_handle, ObLSGetMod::COMPACT_MODE))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); - } else if (OB_FAIL(tmp_ls_handle.get_ls()->get_tablet_svr()->get_tablet( + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tablet( tablet_id_, tmp_tablet_handle, 0/*timeout_us*/, storage::ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id_), K(tablet_id_)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_id_)); } else { compat_mode_ = tmp_tablet_handle.get_obj()->get_tablet_meta().compat_mode_; } @@ -127,7 +127,6 @@ int ObMdsTableMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_para LOG_WARN("ls basic tablet merge dag do not init", K(ret)); } else { if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, ObIDag::get_type(), - ls_id_.id(), static_cast(tablet_id_.id()), static_cast(flush_scn_.get_val_for_inner_table_field())))) { LOG_WARN("failed to fill info param", K(ret)); @@ -140,9 +139,9 @@ int ObMdsTableMergeDag::fill_dag_key(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; - if (OB_FAIL(databuff_printf(buf, buf_len, "mds table merge task, ls_id=%ld, tablet_id=%ld, flush_scn=%ld", - ls_id_.id(), tablet_id_.id(), flush_scn_.get_val_for_inner_table_field()))) { - LOG_WARN("failed to fill dag key", K(ret), K_(ls_id), K_(tablet_id), K_(flush_scn)); + if (OB_FAIL(databuff_printf(buf, buf_len, "mds table merge task, tablet_id=%ld, flush_scn=%ld", + tablet_id_.id(), flush_scn_.get_val_for_inner_table_field()))) { + LOG_WARN("failed to fill dag key", K(ret), K_(tablet_id), K_(flush_scn)); } return ret; diff --git a/src/storage/multi_data_source/ob_mds_table_merge_task.cpp b/src/storage/multi_data_source/ob_mds_table_merge_task.cpp index e36061a5f..cc215c51f 100644 --- a/src/storage/multi_data_source/ob_mds_table_merge_task.cpp +++ b/src/storage/multi_data_source/ob_mds_table_merge_task.cpp @@ -98,7 +98,6 @@ int ObMdsTableMergeTask::process() ObTablet *tablet = nullptr; ObTabletMergeCtx &ctx = *ctx_ptr; ctx.static_param_.start_time_ = common::ObTimeUtility::fast_current_time(); - const share::ObLSID &ls_id = ctx.get_ls_id(); const common::ObTabletID &tablet_id = ctx.get_tablet_id(); #ifdef ERRSIM if (GCONF.errsim_test_tablet_id.get_value() > 0 && tablet_id.id() == GCONF.errsim_test_tablet_id.get_value()) { @@ -119,28 +118,28 @@ int ObMdsTableMergeTask::process() LOG_WARN("failed to get ls and tablet", KR(ret), K(ctx), KPC(mds_merge_dag_)); } else if (OB_ISNULL(ls = ctx.get_ls())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), "ls_handle", ctx.static_param_.ls_handle_, KPC(mds_merge_dag_)); + LOG_WARN("ls is null", K(ret), "tenant_ls", ctx.static_param_.ls_, KPC(mds_merge_dag_)); } else if (ls->is_offline()) { ret = OB_CANCELED; LOG_INFO("ls offline, skip merge", K(ret), K(ctx), KPC(mds_merge_dag_)); } else if (OB_FAIL(ctx.init_tablet_merge_info())) { - LOG_WARN("failed to init tablet merge info", K(ret), K(ls_id), K(tablet_id), KPC(mds_merge_dag_)); + LOG_WARN("failed to init tablet merge info", K(ret), K(tablet_id), KPC(mds_merge_dag_)); } else if (OB_ISNULL(tablet = ctx.get_tablet())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet is null", K(ret), K(tablet_id)); } else if (CLICK_FAIL(tablet->get_mds_table_for_dump(mds_table))) { if (OB_EMPTY_RESULT != ret) { - LOG_WARN("fail to get mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get mds table", K(ret), K(tablet_id)); } else { ret = OB_NO_NEED_MERGE; } } else if (OB_UNLIKELY(!mds_table.get_mds_table_ptr()->is_construct_sequence_matched(mds_construct_sequence))) { ret = OB_NO_NEED_MERGE; - LOG_WARN("construct sequence does not match current mds table, no need to merge", K(ret), K(ls_id), K(tablet_id), K(mds_construct_sequence)); + LOG_WARN("construct sequence does not match current mds table, no need to merge", K(ret), K(tablet_id), K(mds_construct_sequence)); } else if (tablet->get_mds_checkpoint_scn() >= flush_scn) { need_schedule_mds_minor = false; FLOG_INFO("flush scn smaller than mds ckpt scn, only flush nodes of mds table and do not generate mds mini", - K(ret), K(ls_id), K(tablet_id), K(flush_scn), + K(ret), K(tablet_id), K(flush_scn), "mds_checkpoint_scn", tablet->get_mds_checkpoint_scn(), KPC(mds_merge_dag_)); ctx.time_guard_click(ObStorageCompactionTimeGuard::EXECUTE); @@ -148,23 +147,23 @@ int ObMdsTableMergeTask::process() } else if (FALSE_IT(ctx.static_param_.scn_range_.start_scn_ = tablet->get_mds_checkpoint_scn())) { } else if (MDS_FAIL(build_mds_sstable(ctx, mds_construct_sequence, table_handle))) { if (OB_EMPTY_RESULT != ret) { - LOG_ERROR("fail to build mds sstable", K(ret), K(ls_id), K(tablet_id), KPC(mds_merge_dag_)); + LOG_ERROR("fail to build mds sstable", K(ret), K(tablet_id), KPC(mds_merge_dag_)); } else { // abort transaction may trigger an empty mds table dump to relase mem and clog scn - // however, an uncommitted tablet for example create/reserved/split dest, need to + // however, an uncommitted tablet, for example create/split dest, needs to // transform to empty shell accoring to the mds table rec scn (See // ObTabletEmptyShellHandler::check_tablet_from_aborted_tx_). // So here double check tablet status that only flush those committed tablet status. ret = OB_SUCCESS; if (OB_FAIL(check_tablet_status_for_empty_mds_table_(*tablet))) { if (OB_NO_NEED_MERGE != ret) { - LOG_WARN("fail to check tablet status", K(ret), K(ls_id), K(tablet_id), KPC(mds_merge_dag_)); + LOG_WARN("fail to check tablet status", K(ret), K(tablet_id), KPC(mds_merge_dag_)); } else { - FLOG_INFO("skip uncommitted creation tablet", K(ls_id), K(tablet_id)); + FLOG_INFO("skip uncommitted creation tablet", K(tablet_id)); } } else { need_schedule_mds_minor = false; - FLOG_INFO("trigger empty mds table flush", K(ls_id), K(tablet_id)); + FLOG_INFO("trigger empty mds table flush", K(tablet_id)); } } } else if (CLICK_FAIL(ls->build_new_tablet_from_mds_table( @@ -173,7 +172,7 @@ int ObMdsTableMergeTask::process() table_handle, flush_scn, new_tablet_handle))) { - LOG_ERROR("failed to build new tablet from mds table", K(ret), K(ctx), K(ls_id), K(tablet_id), + LOG_ERROR("failed to build new tablet from mds table", K(ret), K(ctx), K(tablet_id), K(flush_scn), KPC(mds_merge_dag_)); } @@ -203,25 +202,24 @@ int ObMdsTableMergeTask::process() void ObMdsTableMergeTask::try_schedule_compaction_after_mds_mini(compaction::ObTabletMergeCtx &ctx, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ctx.get_ls_id(); const common::ObTabletID &tablet_id = ctx.get_tablet_id(); bool during_restore = false; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !tablet_handle.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || !tablet_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(tablet_handle), KPC(mds_merge_dag_)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(tablet_handle), KPC(mds_merge_dag_)); // when restoring, some log stream may be not ready, // thus the inner sql in ObTenantFreezeInfoMgr::try_update_info may timeout } else if (OB_SUCCESS == ObBasicMergeScheduler::get_merge_scheduler()->during_restore(during_restore) && !during_restore) { if (0 == ctx.get_merge_info().get_merge_history().block_info_.macro_block_count_) { // no need to schedule mds minor merge } else if (OB_FAIL(ObTenantTabletScheduler::schedule_tablet_minor_merge( - compaction::MDS_MINOR_MERGE, ctx.static_param_.ls_handle_, tablet_handle))) { + compaction::MDS_MINOR_MERGE, ctx.static_param_.ls_, tablet_handle))) { if (OB_SIZE_OVERFLOW != ret) { LOG_ERROR("failed to schedule special tablet minor merge which triggle mds", - K(ret), K(ls_id), K(tablet_id), KPC(mds_merge_dag_)); + K(ret), K(tablet_id), KPC(mds_merge_dag_)); } } else { - LOG_TRACE("succeed to try schedule mds minor after mds", K(ls_id), K(tablet_id), KPC(mds_merge_dag_)); + LOG_TRACE("succeed to try schedule mds minor after mds", K(tablet_id), KPC(mds_merge_dag_)); } } } @@ -247,14 +245,13 @@ int ObMdsTableMergeTask::build_mds_sstable( ObTableHandleV2 &table_handle) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ctx.get_ls_id(); const common::ObTabletID &tablet_id = ctx.get_tablet_id(); SMART_VARS_2((ObMdsTableMiniMerger, mds_mini_merger), (ObTabletDumpMds2MiniOperator, op)) { if (OB_FAIL(mds_mini_merger.init(ctx, op))) { - LOG_WARN("fail to init mds mini merger", K(ret), K(ctx), K(ls_id), K(tablet_id)); + LOG_WARN("fail to init mds mini merger", K(ret), K(ctx), K(tablet_id)); } else if (OB_FAIL(ctx.get_tablet()->scan_mds_table_with_op(mds_construct_sequence, op))) { - LOG_WARN("fail to scan mds table with op", K(ret), K(ctx), K(ls_id), K(tablet_id)); + LOG_WARN("fail to scan mds table with op", K(ret), K(ctx), K(tablet_id)); } else if (OB_FAIL(mds_mini_merger.generate_mds_mini_sstable( ctx.mem_ctx_.get_allocator(), table_handle))) { LOG_WARN("fail to generate mds mini sstable with mini merger", K(ret), K(mds_mini_merger)); diff --git a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp index 273d44001..851aed148 100644 --- a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp +++ b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp @@ -28,18 +28,12 @@ namespace storage namespace mds { ObTabletCreateMdsCtx::ObTabletCreateMdsCtx() - : MdsCtx(), - magic_(MAGIC), - version_(VERSION), - ls_id_() + : MdsCtx() { } ObTabletCreateMdsCtx::ObTabletCreateMdsCtx(const MdsWriter &writer) - : MdsCtx(writer), - magic_(MAGIC), - version_(VERSION), - ls_id_() + : MdsCtx(writer) { } @@ -48,129 +42,18 @@ void ObTabletCreateMdsCtx::on_abort(const share::SCN &abort_scn) mds::MdsCtx::on_abort(abort_scn); int ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; - if (OB_UNLIKELY(!ls_id_.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", K(ret), K_(ls_id)); - } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - LOG_WARN("fail to get ls", K(ret), K_(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K_(ls_id), KP(ls)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("fail to get ls", K(ret)); } else { - checkpoint::ObTabletEmptyShellHandler *handler = ls->get_tablet_empty_shell_handler(); + checkpoint::ObTabletEmptyShellHandler *handler = + tenant_ls->get_tablet_empty_shell_handler(); handler->set_empty_shell_trigger(true/*is_trigger*/); - LOG_INFO("tablet create tx aborted", K(ret), K_(ls_id), K(abort_scn)); + LOG_INFO("tablet create tx aborted", K(ret), K(abort_scn)); } } - -int ObTabletCreateMdsCtx::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - const int64_t serialize_size = get_serialize_size(); - int64_t tmp_pos = pos; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(buf_len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(buf_len), K(pos)); - } else if (OB_UNLIKELY(buf_len - pos < serialize_size)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer len is not enough to serialize", K(ret), K(buf_len), K(pos), K(serialize_size)); - } else if (VERSION == version_) { - if (OB_FAIL(MdsCtx::serialize(buf, buf_len, tmp_pos))) { - LOG_WARN("failed to serialize mds ctx", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_FAIL(serialization::encode(buf, buf_len, tmp_pos, magic_))) { - LOG_WARN("fail to serialize magic", K(ret), K(buf_len), K(tmp_pos), K_(magic)); - } else if (OB_FAIL(serialization::encode(buf, buf_len, tmp_pos, version_))) { - LOG_WARN("fail to serialize version", K(ret), K(buf_len), K(tmp_pos), K_(version)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, tmp_pos, serialize_size))) { - LOG_WARN("fail to serialize length", K(ret), K(buf_len), K(tmp_pos), K(serialize_size)); - } else if (OB_FAIL(ls_id_.serialize(buf, buf_len, tmp_pos))) { - LOG_WARN("fail to serialize ls id", K(ret), K(buf_len), K(tmp_pos), K_(ls_id)); - } else { - pos = tmp_pos; - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected version", K(ret), K_(version)); - } - - return ret; -} - -int ObTabletCreateMdsCtx::deserialize(const char *buf, const int64_t buf_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t origin_pos = pos; - int64_t tmp_pos = pos; - int32_t magic = -1; - int32_t version = -1; - int64_t serialize_size = 0; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(buf_len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(MdsCtx::deserialize(buf, buf_len, tmp_pos))) { - LOG_WARN("fail to deserialize mds ctx", K(ret), K(buf_len), K(tmp_pos)); - } else { - int tmp_ret = OB_SUCCESS; - pos = tmp_pos; - bool is_old_data = false; - if (tmp_pos == buf_len) { - LOG_WARN("buffer is not enough for magic deserialize", K(ret), K(buf_len), K(tmp_pos)); - is_old_data = true; - } else if (OB_TMP_FAIL(serialization::decode(buf, buf_len, tmp_pos, magic))) { - LOG_WARN("decode magic from buffer failed", K(tmp_ret), K(ret), K(buf_len), K(tmp_pos)); - is_old_data = true; - } else if (magic != MAGIC) { - LOG_WARN("magic not match", K(tmp_ret), K(ret), K(buf_len), K(tmp_pos)); - is_old_data = true; - } else if (OB_FAIL(serialization::decode(buf, buf_len, tmp_pos, version))) { - LOG_WARN("failed to deserialize version", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_UNLIKELY(VERSION != version)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("version does not match", K(ret), K(version)); - } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, tmp_pos, &serialize_size))) { - LOG_WARN("failed to deserialize serialize size", K(ret), K(buf_len), K(tmp_pos)); - } else if (tmp_pos - origin_pos < serialize_size && OB_FAIL(ls_id_.deserialize(buf, buf_len, tmp_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_UNLIKELY(tmp_pos - origin_pos != serialize_size)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("deserialize length does not match", K(ret), K(buf_len), K(pos), K(tmp_pos), K(serialize_size)); - } else { - version_ = version; - pos = tmp_pos; - } - - if (is_old_data) { - FLOG_INFO("maybe meet old version data", K(ret), K(magic), LITERAL_K(MAGIC)); - version_ = VERSION; - ls_id_ = ObLSID::INVALID_LS_ID; - } - } - - return ret; -} - -int64_t ObTabletCreateMdsCtx::get_serialize_size() const -{ - int64_t size = 0; - int64_t serialize_size = 0; // dummy - size += MdsCtx::get_serialize_size(); - size += serialization::encoded_length(magic_); - size += serialization::encoded_length(version_); - size += serialization::encoded_length_i64(serialize_size); - size += ls_id_.get_serialize_size(); - return size; -} } // namespace mds } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.h b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.h index aa0e02762..c6514e23c 100644 --- a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.h +++ b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.h @@ -18,7 +18,6 @@ #define OCEANBASE_STORAGE_OB_CREATE_TABLET_MDS_CTX #include "storage/multi_data_source/mds_ctx.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -34,24 +33,7 @@ class ObTabletCreateMdsCtx : public MdsCtx virtual ~ObTabletCreateMdsCtx() = default; public: virtual void on_abort(const share::SCN &abort_scn) override; - virtual int serialize(char *buf, const int64_t buf_len, int64_t &pos) const override; - virtual int deserialize(const char *buf, const int64_t buf_len, int64_t &pos) override; - virtual int64_t get_serialize_size() const override; -public: - void set_ls_id(const share::ObLSID &ls_id); -private: - static constexpr int32_t MAGIC = 0xdead; - static constexpr int32_t VERSION = 1; - - const int32_t magic_; - int32_t version_; - share::ObLSID ls_id_; }; - -inline void ObTabletCreateMdsCtx::set_ls_id(const share::ObLSID &ls_id) -{ - ls_id_ = ls_id; -} } // namespace mds } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.cpp b/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.cpp index e28aeb9f5..d8c03f7e8 100644 --- a/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.cpp +++ b/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.cpp @@ -96,7 +96,6 @@ int ObTabletMdsMinorMergeCtx::update_tablet(ObTabletHandle &new_tablet_handle) ObUpdateTableStoreParam mds_param(static_param_.version_range_.snapshot_version_, 1/*multi_version_start*/, ObMdsSchemaHelper::get_instance().get_storage_schema(), - get_ls_rebuild_seq(), sstable, false/*allow_duplicate_sstable*/); if (OB_FAIL(mds_param.init_with_compaction_info( @@ -113,83 +112,5 @@ int ObTabletMdsMinorMergeCtx::update_tablet(ObTabletHandle &new_tablet_handle) return ret; } -ObTabletCrossLSMdsMinorMergeCtx::ObTabletCrossLSMdsMinorMergeCtx(ObTabletMergeDagParam ¶m, ObArenaAllocator &allocator) - : ObTabletMergeCtx(param, allocator) -{ -} - -int ObTabletCrossLSMdsMinorMergeCtx::prepare_schema() -{ - int ret = OB_SUCCESS; - static_param_.schema_ = ObMdsSchemaHelper::get_instance().get_storage_schema(); - return ret; -} - -int ObTabletCrossLSMdsMinorMergeCtx::prepare_index_tree() -{ - return ObBasicTabletMergeCtx::build_index_tree( - merge_info_, - ObMdsSchemaHelper::get_instance().get_rowkey_read_info()); -} - -void ObTabletCrossLSMdsMinorMergeCtx::free_schema() -{ - static_param_.schema_ = nullptr; -} - -int ObTabletCrossLSMdsMinorMergeCtx::get_merge_tables(ObGetMergeTablesResult &get_merge_table_result) -{ - int ret = OB_SUCCESS; - get_merge_table_result.reset(); - const ObMergeType merge_type = static_param_.get_merge_type(); - ObTablet *tablet = nullptr; - - if (!tablet_handle_.is_valid() || static_param_.tables_handle_.empty()) { - ret = OB_NOT_INIT; - LOG_WARN("tablet cross ls mds minor merge ctx do not init", K(ret), K(static_param_)); - } else if (OB_UNLIKELY(!is_mds_minor_merge(merge_type))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid arguments", K(ret), "merge_type", merge_type_to_str(merge_type)); - } else if (OB_FAIL(prepare_compaction_filter())) { - LOG_WARN("fail to prepare compaction filter", K(ret)); - } else if (OB_FAIL(get_merge_table_result.handle_.assign(static_param_.tables_handle_))) { - LOG_WARN("failed to assign table handle array", K(ret), K(static_param_)); - } else if (OB_ISNULL(tablet = tablet_handle_.get_obj())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet should not be NULL", K(ret), K(tablet_handle_)); - } else { - get_merge_table_result.scn_range_.start_scn_ = static_param_.tables_handle_.get_table(0)->get_start_scn(); - get_merge_table_result.scn_range_.end_scn_ = static_param_.tables_handle_.get_table(static_param_.tables_handle_.get_count() - 1)->get_end_scn(); - get_merge_table_result.version_range_.snapshot_version_ = tablet->get_snapshot_version(); - } - return ret; -} - -int ObTabletCrossLSMdsMinorMergeCtx::update_tablet(ObTabletHandle &new_tablet_handle) -{ - int ret = OB_NOT_SUPPORTED; - // only called in finish merge task, such as ObTabletMergeFinishTask/ObCOMergeFinishTask - LOG_WARN("cross ls minor merge should not update tablet", K(ret), KPC(this)); - return ret; -} - -int ObTabletCrossLSMdsMinorMergeCtx::prepare_compaction_filter() -{ - int ret = OB_SUCCESS; - void *buf = nullptr; - - if (OB_ISNULL(buf = mem_ctx_.alloc(sizeof(ObCrossLSMdsMinorFilter)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret), "size", sizeof(ObCrossLSMdsMinorFilter)); - } else { - ObCrossLSMdsMinorFilter *filter = new (buf) ObCrossLSMdsMinorFilter(); - filter_ctx_.compaction_filter_ = filter; - FLOG_INFO("success to init mds compaction filter", K(ret)); - } - - return ret; -} - - } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.h b/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.h index ed1a9cd14..001515d61 100644 --- a/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.h +++ b/src/storage/multi_data_source/ob_tablet_mds_merge_ctx.h @@ -46,22 +46,6 @@ class ObTabletMdsMinorMergeCtx : public compaction::ObTabletExeMergeCtx virtual int update_tablet(ObTabletHandle &new_tablet_handle) override; }; -class ObTabletCrossLSMdsMinorMergeCtx : public compaction::ObTabletMergeCtx -{ -public: - ObTabletCrossLSMdsMinorMergeCtx(compaction::ObTabletMergeDagParam ¶m, common::ObArenaAllocator &allocator); - virtual ~ObTabletCrossLSMdsMinorMergeCtx() { free_schema(); } -public: - virtual int prepare_schema() override; - virtual int prepare_index_tree() override; - virtual void free_schema() override; - virtual int get_merge_tables(ObGetMergeTablesResult &get_merge_table_result) override; - virtual int update_tablet(ObTabletHandle &new_tablet_handle) override; - -private: - int prepare_compaction_filter(); - -}; } // namespace storage } // namespace oceanbase diff --git a/src/storage/multi_data_source/runtime_utility/mds_factory.h b/src/storage/multi_data_source/runtime_utility/mds_factory.h index 93f02e1ea..c03b3f49b 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_factory.h +++ b/src/storage/multi_data_source/runtime_utility/mds_factory.h @@ -22,7 +22,6 @@ #include #include "lib/atomic/ob_atomic.h" #include "ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "src/share/ob_errno.h" #include "common/meta_programming/ob_type_traits.h" #include "mds_tenant_service.h" diff --git a/src/storage/multi_data_source/runtime_utility/mds_retry_control.cpp b/src/storage/multi_data_source/runtime_utility/mds_retry_control.cpp index 328289331..5ca5efd85 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_retry_control.cpp +++ b/src/storage/multi_data_source/runtime_utility/mds_retry_control.cpp @@ -24,10 +24,6 @@ namespace storage namespace mds { -bool RetryParam::check_ls_in_gc_state() const { - return false; -} - } } } diff --git a/src/storage/multi_data_source/runtime_utility/mds_retry_control.h b/src/storage/multi_data_source/runtime_utility/mds_retry_control.h index 7da421145..1147fedef 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_retry_control.h +++ b/src/storage/multi_data_source/runtime_utility/mds_retry_control.h @@ -31,14 +31,12 @@ namespace mds struct RetryParam { RetryParam(): - ls_id_(0), start_ts_(0), last_print_ts_(0), timeout_ts_(0), retry_cnt_(0), print_interval_(0) {} - RetryParam(share::ObLSID ls_id, int64_t lock_timeout_us, int64_t print_interval = 500_ms) : - ls_id_(ls_id), + RetryParam(int64_t lock_timeout_us, int64_t print_interval = 500_ms) : start_ts_(ObClockGenerator::getClock()), last_print_ts_(0), // to avoid over MAX limit, signed number overlimit bahavior is not defined by standard @@ -55,9 +53,7 @@ struct RetryParam { return ret; } bool check_timeout() const { return ObClockGenerator::getClock() > timeout_ts_; } - bool check_ls_in_gc_state() const; - TO_STRING_KV(K_(ls_id), KTIME_(start_ts), KTIME_(last_print_ts), KTIME_(timeout_ts), K_(retry_cnt), K_(print_interval)); - share::ObLSID ls_id_; + TO_STRING_KV(KTIME_(start_ts), KTIME_(last_print_ts), KTIME_(timeout_ts), K_(retry_cnt), K_(print_interval)); int64_t start_ts_; mutable int64_t last_print_ts_; int64_t timeout_ts_; @@ -97,15 +93,6 @@ int retry_release_lock_with_op_until_timeout(const MdsLock &lock, } } } // release lock - if (OB_EAGAIN == ret && MODE == LockMode::READ) { - if ((retry_param.retry_cnt_ % 50) == 0) {// every 500ms check ls status -#ifndef UNITTEST_DEBUG - if (retry_param.check_ls_in_gc_state()) { - ret = OB_REPLICA_NOT_READABLE; - } -#endif - } - } } while (OB_EAGAIN == ret && ({ ob_usleep(10_ms); ++retry_param; true; })); return ret; #undef PRINT_WRAPPER diff --git a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp index cc20aa866..9297a031a 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp +++ b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp @@ -17,7 +17,6 @@ #include "mds_tenant_service.h" #include "share/rc/ob_module_provider.h" #include "storage/tx_storage/ob_ls_service.h" -#include "storage/tablet/ob_tablet_iterator.h" #include "storage/allocator/ob_mds_allocator.h" // relocated-definition owner #include "storage/allocator/ob_tenant_vector_allocator.h" @@ -91,16 +90,23 @@ void ObTenantMdsService::run_recyle_timer_task() } void ObTenantMdsService::try_recycle_mds_table_task() { - #define PRINT_WRAPPER KR(ret), KPC(this), K(tablet_oldest_scn), K(mds_table_handle) + #define PRINT_WRAPPER KR(ret) int ret = OB_SUCCESS; + MDS_TG(1_s); ObCurTraceId::init(GCONF.self_addr_); - ObTenantMdsService::for_each_ls_in_tenant([](ObLS &ls) -> int { - ObTenantMdsService::for_each_mds_table_in_ls(ls, [](ObTablet &tablet) -> int {// FIXME: there is no need scan all tablets + ObLS *tenant_ls = nullptr; + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_NONE(WARN, "ls service is null"); + } else if (MDS_FAIL(ls_service->get_ls(tenant_ls))) { + MDS_LOG_NONE(WARN, "fail to get log stream"); + } else if (MDS_FAIL(for_each_mds_table_(*tenant_ls, [](ObTablet &tablet) -> int { (void) process_with_tablet_(tablet); return OB_SUCCESS;// keep doing ignore error - }); - return OB_SUCCESS;// keep doing ignore error - }); + }))) { + MDS_LOG_NONE(WARN, "fail to scan mds tables"); + } #undef PRINT_WRAPPER } @@ -113,10 +119,18 @@ void ObTenantMdsService::run_dump_status_timer_task() void ObTenantMdsService::dump_special_mds_table_status_task() { #define PRINT_WRAPPER KR(ret) + int ret = OB_SUCCESS; MDS_TG(1_s); ObCurTraceId::init(GCONF.self_addr_); - ObTenantMdsService::for_each_ls_in_tenant([](ObLS &ls) -> int { - int ret = OB_SUCCESS; + ObLS *tenant_ls = nullptr; + ObLSService *ls_service = share::g_mp->ls_service(); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_NONE(WARN, "ls service is null"); + } else if (MDS_FAIL(ls_service->get_ls(tenant_ls))) { + MDS_LOG_NONE(WARN, "fail to get log stream"); + } else { + ObLS &ls = *tenant_ls; MDS_TG(1_s); MdsTableMgrHandle mds_table_mge_handle; share::SCN ls_mds_freezing_scn; @@ -146,103 +160,26 @@ void ObTenantMdsService::dump_special_mds_table_status_task() }); return OB_SUCCESS;// keep iterating }); - }; - return OB_SUCCESS;// keep doing ignore error - }); - #undef PRINT_WRAPPER -} - -int ObTenantMdsService::for_each_ls_in_tenant(const ObFunction &op) -{ - #define PRINT_WRAPPER KR(ret) - int ret = OB_SUCCESS; - MDS_TG(3_s); - ObSharedGuard iter; - ObLS *ls = nullptr; - int64_t succ_num = 0; - if (!op.is_valid()) { - ret = OB_INVALID_ARGUMENT; - MDS_LOG_NONE(WARN, "invalid op"); - } else if (OB_ISNULL(share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_NONE(WARN, "ls service is null", K(ret)); - } else if (MDS_FAIL(share::g_mp->ls_service()->get_ls_iter(iter, ObLSGetMod::MDS_TABLE_MOD))) { - MDS_LOG_NONE(WARN, "fail to get ls iterator"); - } else { - do { - if (MDS_FAIL(iter->get_next(ls))) { - if (OB_ITER_END != ret) { - MDS_LOG_NONE(WARN, "get next iter failed"); - } else { - ret = OB_SUCCESS; - break; - } - } else if (MDS_FAIL(op(*ls))) { - MDS_LOG_NONE(WARN, "fail to for each ls", K(succ_num)); - } else { - MDS_LOG_NONE(DEBUG, "succeed to operate one ls", K(ret), "ls_id", ls->get_ls_id()); - } - } while (++succ_num && OB_SUCC(ret)); - } - MDS_LOG_NONE(INFO, "for each ls", K(succ_num)); - return ret; - #undef PRINT_WRAPPER -} - -int ObTenantMdsService::for_each_tablet_in_ls(ObLS &ls, const ObFunction &op) -{ - #define PRINT_WRAPPER KR(ret), K(ls) - int ret = OB_SUCCESS; - int64_t succ_num = 0; - ObLSTabletIterator tablet_iter(storage::ObMDSGetTabletMode::READ_WITHOUT_CHECK); - MDS_TG(500_ms); - if (!op.is_valid()) { - ret = OB_INVALID_ARGUMENT; - MDS_LOG_NONE(WARN, "invalid op"); - } else if (MDS_FAIL(ls.build_tablet_iter(tablet_iter))) { - MDS_LOG_NONE(WARN, "failed to build ls tablet iter"); - } else { - ObTabletHandle tablet_handle; - ObTablet *tablet = nullptr; - do { - tablet_handle.reset(); - tablet = nullptr; - if (MDS_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { - if (OB_ITER_END != ret && OB_EMPTY_RESULT != ret) { - MDS_LOG_NONE(WARN, "failed to get tablet"); - } else { - ret = OB_SUCCESS; - break; - } - } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_NONE(WARN, "tablet should not be NULL", KPC(tablet)); - } else if (tablet->get_tablet_meta().tablet_id_.is_ls_inner_tablet()) { - // FIXME: there is no mds table on ls inner tablet yet, but there will be - } else { - op(*tablet); - } - } while (++succ_num && OB_SUCC(ret)); - MDS_LOG_NONE(INFO, "for each tablet", K(succ_num)); + } } - return ret; #undef PRINT_WRAPPER } -int ObTenantMdsService::for_each_mds_table_in_ls(ObLS &ls, const ObFunction &op) +int ObTenantMdsService::for_each_mds_table_(ObLS &ls, const ObFunction &op) { - #define PRINT_WRAPPER KR(ret), K(ls), K(mds_table_total_num), K(ids_in_t3m_array.count()) + #define PRINT_WRAPPER KR(ret), K(ids_in_t3m_array.count()) int ret = OB_SUCCESS; - int64_t succ_num = 0; MDS_TG(10_s); - int64_t mds_table_total_num = 0; MdsTableMgrHandle mgr_handle; ObArray ids_in_t3m_array; - if (MDS_FAIL(ls.get_mds_table_mgr(mgr_handle))) { + if (!op.is_valid()) { + ret = OB_INVALID_ARGUMENT; + MDS_LOG_NONE(WARN, "invalid tablet operation"); + } else if (MDS_FAIL(ls.get_mds_table_mgr(mgr_handle))) { MDS_LOG_NONE(WARN, "fail to get mds table mgr"); } else if (MDS_FAIL(mgr_handle.get_mds_table_mgr()->for_each_in_t3m_mds_table( - [&mds_table_total_num, &ids_in_t3m_array, &ls](MdsTableBase &mds_table) -> int {// with map's bucket lock protected + [&ids_in_t3m_array](MdsTableBase &mds_table) -> int {// with map's bucket lock protected MDS_TG(1_s); int ret = OB_SUCCESS; if (MDS_FAIL(ids_in_t3m_array.push_back(mds_table.get_tablet_id()))) { @@ -257,6 +194,9 @@ int ObTenantMdsService::for_each_mds_table_in_ls(ObLS &ls, const ObFunctiontenant_meta_mem_mgr())) { ret = OB_BAD_NULL_ERROR; MDS_LOG_GC(ERROR, "MTL ObTenantMetaMemMgr is NULL"); - } else if (MDS_FAIL(share::g_mp->tenant_meta_mem_mgr()->scan_all_version_tablets(ObTabletMapKey(ls_id, tablet_id), op))) { + } else if (MDS_FAIL(share::g_mp->tenant_meta_mem_mgr()->scan_all_version_tablets(ObTabletMapKey(tablet_id), op))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; MDS_LOG_GC(WARN, "get_min_mds_ckpt_scn meet OB_ENTRY_NOT_EXIST"); diff --git a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h index 071d2067c..ac8f68b36 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h +++ b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h @@ -25,10 +25,8 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/utility/ob_print_utils.h" #include "lib/allocator/ob_vslice_alloc.h" -#include "share/ob_ls_id.h" #include "share/ob_occam_timer.h" #include "storage/allocator/ob_mds_allocator.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "lib/hash/ob_linear_hash_map.h" namespace oceanbase @@ -40,7 +38,6 @@ class SCN; namespace storage { class ObLS; -class ObLSHandle; class ObTablet; class ObTabletHandle; namespace mds @@ -67,9 +64,6 @@ class ObTenantMdsService static void mtl_stop(ObTenantMdsService* &); static void mtl_wait(ObTenantMdsService* &); void destroy() { this->~ObTenantMdsService(); } - static int for_each_ls_in_tenant(const ObFunction &op); - static int for_each_tablet_in_ls(ObLS &ls, const ObFunction &op); - static int for_each_mds_table_in_ls(ObLS &ls, const ObFunction &op); share::ObTenantBufferCtxAllocator &get_buffer_ctx_allocator() { return buffer_ctx_allocator_; } TO_STRING_KV(KP(this), K_(is_inited)) public: @@ -80,6 +74,7 @@ class ObTenantMdsService private: static void try_recycle_mds_table_task(); static void dump_special_mds_table_status_task(); + static int for_each_mds_table_(ObLS &ls, const ObFunction &op); static int process_with_tablet_(ObTablet &tablet); static int get_tablet_oldest_scn_(ObTablet &tablet, share::SCN &oldest_scn); diff --git a/src/storage/ob_common_id_utils.cpp b/src/storage/ob_common_id_utils.cpp index 818cd2e2d..3f3093e52 100644 --- a/src/storage/ob_common_id_utils.cpp +++ b/src/storage/ob_common_id_utils.cpp @@ -53,7 +53,7 @@ int ObCommonIDUtils::gen_unique_id(ObCommonID &id) int ObCommonIDUtils::gen_unique_id_by_rpc(ObCommonID &id) { int ret = OB_SUCCESS; - // seekdb single-node: all LS leaders are local, just call gen_unique_id directly. + // seekdb single-node: the log stream leader is local, so generate the ID directly. // Switch tenant context so gen_unique_id's sys tenant check passes. MOD_SCOPE { if (OB_FAIL(gen_unique_id(id))) { diff --git a/src/storage/ob_common_id_utils.h b/src/storage/ob_common_id_utils.h index 8fe366889..d4f820bac 100644 --- a/src/storage/ob_common_id_utils.h +++ b/src/storage/ob_common_id_utils.h @@ -51,7 +51,7 @@ class ObCommonIDUtils // - OB_INVALID_ARGUMENT tenant is invalid or not matched with MTL_ID static int gen_unique_id(share::ObCommonID &id); - // Send rpc to the leader of sys LS of target tenant to execute gen_unique_id. + // Send RPC to the target tenant log stream to execute gen_unique_id. // // Use this one when target tenant doesn't exist on current machine. static int gen_unique_id_by_rpc(share::ObCommonID &id); diff --git a/src/storage/ob_direct_load_table_guard.cpp b/src/storage/ob_direct_load_table_guard.cpp index 6713fdaa1..8cab325e4 100644 --- a/src/storage/ob_direct_load_table_guard.cpp +++ b/src/storage/ob_direct_load_table_guard.cpp @@ -33,7 +33,6 @@ ObDirectLoadTableGuard::ObDirectLoadTableGuard(ObTablet &tablet, const share::SC table_handle_(), construct_timestamp_(0) { - ls_id_ = tablet.get_tablet_meta().ls_id_; tablet_id_ = tablet.get_tablet_meta().tablet_id_; construct_timestamp_ = ObClockGenerator::getClock(); } @@ -114,23 +113,20 @@ int ObDirectLoadTableGuard::prepare_memtable(ObDDLKV *&res_memtable) int ObDirectLoadTableGuard::acquire_memtable_once_() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; if (has_acquired_memtable_) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "direct load table guard cannot be inited twice", KR(ret), KPC(this)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { STORAGE_LOG(WARN, "failed to get log stream", K(ret), KPC(this)); - } else if (OB_UNLIKELY(!ls_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, invalid ls handle", K(ret), K(ls_handle), KPC(this)); } else { const int64_t start_time = ObClockGenerator::getClock(); bool need_create_new_memtable = false; do { if (need_create_new_memtable) { need_create_new_memtable = false; - if (OB_FAIL(do_create_memtable_(ls_handle))) { + if (OB_FAIL(do_create_memtable_(tenant_ls))) { STORAGE_LOG(WARN, "create direct load memtable failed", KR(ret), KPC(this)); } else if (is_write_filtered_) { break; @@ -143,7 +139,7 @@ int ObDirectLoadTableGuard::acquire_memtable_once_() ret = OB_SUCCESS; need_create_new_memtable = true; } - } else if (OB_FAIL(try_get_direct_load_memtable_for_write(ls_handle, need_create_new_memtable))) { + } else if (OB_FAIL(try_get_direct_load_memtable_for_write(tenant_ls, need_create_new_memtable))) { STORAGE_LOG(WARN, "get direct load memtable for write failed", KR(ret), K(start_time)); } } while (OB_SUCC(ret) && need_create_new_memtable); @@ -152,7 +148,7 @@ int ObDirectLoadTableGuard::acquire_memtable_once_() return ret; } -int ObDirectLoadTableGuard::do_create_memtable_(ObLSHandle &ls_handle) +int ObDirectLoadTableGuard::do_create_memtable_(ObLS *tenant_ls) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -161,7 +157,7 @@ int ObDirectLoadTableGuard::do_create_memtable_(ObLSHandle &ls_handle) CreateMemtableArg arg; arg.for_inc_direct_load_ = true; arg.for_replay_ = for_replay_; - if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( + if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tablet( tablet_id_, tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { STORAGE_LOG(WARN, "fail to get tablet", K(ret), KPC(this)); } else if (FALSE_IT(clog_checkpoint_scn = tablet_handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_)) { @@ -169,20 +165,20 @@ int ObDirectLoadTableGuard::do_create_memtable_(ObLSHandle &ls_handle) is_write_filtered_ = true; ret = OB_SUCCESS; } else if (FALSE_IT(arg.clog_checkpoint_scn_ = clog_checkpoint_scn)) { - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->create_memtable(tablet_id_, arg))) { + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->create_memtable(tablet_id_, arg))) { STORAGE_LOG(WARN, "fail to create a boundary memtable", K(ret), KPC(this)); } return ret; } -int ObDirectLoadTableGuard::try_get_direct_load_memtable_for_write(ObLSHandle &ls_handle, +int ObDirectLoadTableGuard::try_get_direct_load_memtable_for_write(ObLS *tenant_ls, bool &need_create_new_memtable) { int ret = OB_SUCCESS;; ObTabletHandle tablet_handle; ObSEArray local_table_handles; ObProtectedMemtableMgrHandle *protected_handle = nullptr; - if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet( + if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tablet( tablet_id_, tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { STORAGE_LOG(WARN, "fail to get tablet", K(ret), KPC(this)); } else if (OB_FAIL(tablet_handle.get_obj()->get_protected_memtable_mgr_handle(protected_handle))) { @@ -239,16 +235,12 @@ void ObDirectLoadTableGuard::clear_write_ref(ObIArray &table_ha void ObDirectLoadTableGuard::async_freeze_() { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - STORAGE_LOG(WARN, "failed to get ls", K(ret), "ls_id", ls_id_); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "ls should not be null", K(ret), KPC(this)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "failed to get ls", K(ret)); } else { const bool is_sync = false; - (void)ls->tablet_freeze(tablet_id_, + (void)tenant_ls->tablet_freeze(tablet_id_, is_sync, 0, /*timeout, useless for async one*/ false, /*need_rewrite_meta*/ diff --git a/src/storage/ob_direct_load_table_guard.h b/src/storage/ob_direct_load_table_guard.h index 2f2d0d526..6df652239 100644 --- a/src/storage/ob_direct_load_table_guard.h +++ b/src/storage/ob_direct_load_table_guard.h @@ -46,7 +46,6 @@ class ObDirectLoadTableGuard { K(has_acquired_memtable_), K(is_write_filtered_), K(for_replay_), - K(ls_id_), K(tablet_id_), K(ddl_redo_scn_), K(table_handle_), @@ -55,14 +54,13 @@ class ObDirectLoadTableGuard { private: void async_freeze_(); int acquire_memtable_once_(); - int do_create_memtable_(ObLSHandle &ls_handle); - int try_get_direct_load_memtable_for_write(ObLSHandle &ls_handle, bool &need_create_new_memtable); + int do_create_memtable_(ObLS *tenant_ls); + int try_get_direct_load_memtable_for_write(ObLS *tenant_ls, bool &need_create_new_memtable); private: bool has_acquired_memtable_; bool is_write_filtered_; const bool for_replay_; - share::ObLSID ls_id_; ObTabletID tablet_id_; const share::SCN &ddl_redo_scn_; ObTableHandleV2 table_handle_; diff --git a/src/storage/ob_disk_usage_reporter.cpp b/src/storage/ob_disk_usage_reporter.cpp index 56d753f10..4e19ecb76 100644 --- a/src/storage/ob_disk_usage_reporter.cpp +++ b/src/storage/ob_disk_usage_reporter.cpp @@ -20,6 +20,7 @@ #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/meta_store/ob_server_storage_meta_service.h" #include "storage/meta_store/ob_tenant_storage_meta_service.h" #include "logservice/ob_server_log_block_mgr.h" @@ -88,34 +89,19 @@ int ObDiskUsageReportTask::count_tenant_data() STORAGE_LOG(WARN, "failed to get tenant's meta block list", K(ret)); } else { ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObLSService *ls_service = share::g_mp->ls_service(); - if (OB_ISNULL(ls_service) || OB_ISNULL(t3m) ) { + if (OB_ISNULL(t3m)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error!!! ls_service and t3m must not nullptr", K(ret), KP(ls_service), KP(t3m)); + STORAGE_LOG(WARN, "tenant meta memory manager must not be null", K(ret), KP(t3m)); } else { ObTenantTabletPtrWithInMemObjIterator tablet_ptr_iter(*t3m); ObTabletPointerHandle pointer_handle; ObTabletHandle unused_tablet_handle; ObTabletMapKey tablet_map_key; const ObTabletPointer *tablet_pointer = nullptr; - ObLSHandle ls_handle; - common::ObRole ls_role; - ObLS *ls = NULL; - bool is_leader = false; - while (OB_SUCC(ret) && OB_SUCC(tablet_ptr_iter.get_next_tablet_pointer(tablet_map_key, pointer_handle, unused_tablet_handle))) { if (OB_UNLIKELY(!pointer_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "unexpected invalid tablet", K(ret), K(pointer_handle)); - } else if (OB_NOT_NULL(ls) && ls->get_ls_id() == tablet_map_key.ls_id_) { - // do not need get_ls again - } else if (OB_FAIL(ls_service->get_ls(tablet_map_key.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get_ls failed", K(tablet_map_key.ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error!!! ls must not nullptr", K(tablet_map_key.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(ls_role))) { - STORAGE_LOG(WARN, "fail to get ls_role", K(ret), KPC(ls)); } if (OB_FAIL(ret)) { @@ -246,4 +232,3 @@ int ObDiskUsageReportTask::delete_tenant_usage_stat() } // namespace storage } // namespace oceanbase - diff --git a/src/storage/ob_dml_running_ctx.cpp b/src/storage/ob_dml_running_ctx.cpp index 5f6d87f7a..738043fa0 100644 --- a/src/storage/ob_dml_running_ctx.cpp +++ b/src/storage/ob_dml_running_ctx.cpp @@ -20,7 +20,7 @@ #include "storage/ob_table_dml_param.h" #include "storage/tablet/ob_tablet.h" #include "storage/memtable/ob_memtable_context.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -110,7 +110,7 @@ int ObDMLRunningCtx::init( // Propagate async-index flag to the transaction context so that the log block header // carries HAS_ASYNC_INDEX, enabling Change Stream Fetcher fast filtering. if (OB_UNLIKELY(dml_param_.has_async_index_)) { - transaction::ObPartTransCtx *tx_ctx = store_ctx_.mvcc_acc_ctx_.mem_ctx_->get_trans_ctx(); + transaction::ObTxCtx *tx_ctx = store_ctx_.mvcc_acc_ctx_.mem_ctx_->get_trans_ctx(); if (OB_NOT_NULL(tx_ctx)) { tx_ctx->set_has_async_index_redo(); } diff --git a/src/storage/ob_gc_upper_trans_helper.cpp b/src/storage/ob_gc_upper_trans_helper.cpp index 17f24eafd..e4d2aa1af 100644 --- a/src/storage/ob_gc_upper_trans_helper.cpp +++ b/src/storage/ob_gc_upper_trans_helper.cpp @@ -28,7 +28,6 @@ int ObGCUpperTransHelper::try_get_sstable_upper_trans_version( int64_t &new_upper_trans_version) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); const ObTabletID &tablet_id = sstable.get_key().get_tablet_id(); new_upper_trans_version = INT64_MAX; if (INT64_MAX == sstable.get_upper_trans_version()) { @@ -42,9 +41,9 @@ int ObGCUpperTransHelper::try_get_sstable_upper_trans_version( LOG_WARN("max trans version should not be 0", K(sstable)); } else if (INT64_MAX != max_trans_version) { new_upper_trans_version = MAX(max_trans_version, sstable.get_max_merged_trans_version()); - FLOG_INFO("success to get new upper trans version", K(ret), K(ls_id), K(tablet_id), K(max_trans_version), K(sstable)); + FLOG_INFO("success to get new upper trans version", K(ret), K(tablet_id), K(max_trans_version), K(sstable)); } else { - LOG_TRACE("can not get upper trans version", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("can not get upper trans version", K(ret), K(tablet_id)); } } return ret; @@ -58,7 +57,6 @@ int ObGCUpperTransHelper::check_need_gc_or_update_upper_trans_version( bool &need_update) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; bool is_paused = false; // TODO(DanLing) get is_paused need_update = false; @@ -67,13 +65,13 @@ int ObGCUpperTransHelper::check_need_gc_or_update_upper_trans_version( ObIArray *new_upper_trans = upper_trans_param.new_upper_trans_; if (OB_UNLIKELY(new_upper_trans == nullptr || !new_upper_trans->empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("new_upper_trans is nullpr or empty", K(ret), K(ls_id), K(tablet_id), K(new_upper_trans)); + LOG_WARN("new_upper_trans is nullpr or empty", K(ret), K(tablet_id), K(new_upper_trans)); } else if (is_paused) { ret = OB_EAGAIN; - LOG_INFO("paused, cannot update trans version now", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("paused, cannot update trans version now", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); - } else if (tablet.get_tablet_meta().ha_status_.is_data_status_complete()) { + } else { ObITable *table = nullptr; ObSSTable *sstable = nullptr; int64_t new_upper_trans_version = INT64_MAX; diff --git a/src/storage/ob_i_memtable_mgr.cpp b/src/storage/ob_i_memtable_mgr.cpp index 5ec4a6441..8600c7971 100644 --- a/src/storage/ob_i_memtable_mgr.cpp +++ b/src/storage/ob_i_memtable_mgr.cpp @@ -202,33 +202,23 @@ int ObIMemtableMgr::release_memtables() } int ObIMemtableMgr::init( - const ObLSID &ls_id, const ObTabletID &tablet_id, lib::Worker::CompatMode compat_mode) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "failed to get ObLSService from MTL", KR(ret), KPC(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { - STORAGE_LOG(WARN, "failed to get ls", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls should not be NULL", KR(ret), KP(ls)); - } else if (OB_FAIL(init(tablet_id, ls_id, 0, 0, compat_mode, - ls->get_log_handler(), ls->get_freezer(), t3m))) { - STORAGE_LOG(WARN, "failed to init memtable mgr", KR(ret), K(tablet_id), K(ls_id)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "failed to get ls", KR(ret)); + } else if (OB_FAIL(init(tablet_id, 0, 0, compat_mode, + tenant_ls->get_log_handler(), tenant_ls->get_freezer(), t3m))) { + STORAGE_LOG(WARN, "failed to init memtable mgr", KR(ret), K(tablet_id)); } return ret; } int ObIMemtableMgr::init( const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const int64_t max_saved_schema_version, const int64_t max_saved_medium_scn, const lib::Worker::CompatMode compat_mode, @@ -238,10 +228,10 @@ int ObIMemtableMgr::init( { int ret = OB_SUCCESS; if (!tablet_id.is_special_merge_tablet() - && OB_FAIL(init_storage_recorder(tablet_id, ls_id, max_saved_schema_version, max_saved_medium_scn, compat_mode, log_handler))) { + && OB_FAIL(init_storage_recorder(tablet_id, max_saved_schema_version, max_saved_medium_scn, compat_mode, log_handler))) { TRANS_LOG(WARN, "failed to init schema recorder", K(ret), K(max_saved_schema_version), K(max_saved_medium_scn), K(compat_mode), KP(log_handler)); } else { - ret = init(tablet_id, ls_id, freezer, t3m); + ret = init(tablet_id, freezer, t3m); } return ret; } diff --git a/src/storage/ob_i_memtable_mgr.h b/src/storage/ob_i_memtable_mgr.h index 580d23a5b..45ebb92fe 100644 --- a/src/storage/ob_i_memtable_mgr.h +++ b/src/storage/ob_i_memtable_mgr.h @@ -233,13 +233,11 @@ class ObIMemtableMgr virtual ~ObIMemtableMgr(); int init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const lib::Worker::CompatMode compat_mode); int init( const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const int64_t max_saved_schema_version, const int64_t max_saved_medium_scn, const lib::Worker::CompatMode compat_mode, @@ -291,14 +289,12 @@ class ObIMemtableMgr virtual int init_storage_recorder( const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const int64_t max_saved_schema_version, const int64_t max_saved_medium_scn, const lib::Worker::CompatMode compat_mode, logservice::ObLogHandler *log_handler) { // do nothing UNUSED(tablet_id); - UNUSED(ls_id); UNUSED(max_saved_schema_version); UNUSED(max_saved_medium_scn); UNUSED(compat_mode); @@ -324,7 +320,6 @@ class ObIMemtableMgr void release_head_memtable(); void release_tail_memtable(); virtual int init(const ObTabletID &tablet_id, - const share::ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) = 0; protected: diff --git a/src/storage/ob_i_pre_warmer.cpp b/src/storage/ob_i_pre_warmer.cpp index bf3890039..c951251db 100644 --- a/src/storage/ob_i_pre_warmer.cpp +++ b/src/storage/ob_i_pre_warmer.cpp @@ -22,7 +22,7 @@ namespace oceanbase namespace share { -int ObPreWarmerParam::init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const bool use_fixed_percentage) +int ObPreWarmerParam::init(const common::ObTabletID &tablet_id, const bool use_fixed_percentage) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -34,15 +34,15 @@ int ObPreWarmerParam::init(const share::ObLSID &ls_id, const common::ObTabletID if (fixed_percentage_ > 0) { tmp_type = MEM_PRE_WARM; - LOG_INFO("use fixed percentage for prewarm", K(ls_id), K(tablet_id), K_(fixed_percentage), K(tmp_type)); + LOG_INFO("use fixed percentage for prewarm", K(tablet_id), K_(fixed_percentage), K(tmp_type)); } } if (PRE_WARM_TYPE_NONE == tmp_type) { storage::ObTabletStatAnalyzer tablet_analyzer; if (OB_TMP_FAIL(share::g_mp->tenant_tablet_stat_mgr() - ->get_tablet_analyzer(ls_id, tablet_id, tablet_analyzer))) { + ->get_tablet_analyzer(tablet_id, tablet_analyzer))) { if (OB_HASH_NOT_EXIST != tmp_ret) { - LOG_WARN_RET(tmp_ret, "Failed to get tablet stat analyzer", K(ls_id), K(tablet_id)); + LOG_WARN_RET(tmp_ret, "Failed to get tablet stat analyzer", K(tablet_id)); } } else { tmp_type = MEM_PRE_WARM; diff --git a/src/storage/ob_i_pre_warmer.h b/src/storage/ob_i_pre_warmer.h index 8088a6c7e..8ae3134ec 100644 --- a/src/storage/ob_i_pre_warmer.h +++ b/src/storage/ob_i_pre_warmer.h @@ -34,7 +34,6 @@ class ObTabletID; } namespace share { -class ObLSID; class ObIPreWarmer { public: @@ -72,7 +71,7 @@ struct ObPreWarmerParam {} virtual ~ObPreWarmerParam() { type_ = PRE_WARM_TYPE_MAX; } virtual bool is_valid() const { return type_ >= PRE_WARM_TYPE_NONE && type_ < PRE_WARM_TYPE_MAX; } - virtual int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const bool use_fixed_percentage = false); + virtual int init(const common::ObTabletID &tablet_id, const bool use_fixed_percentage = false); VIRTUAL_TO_STRING_KV(K_(type), K_(fixed_percentage)); ObPreWarmerType type_; int64_t fixed_percentage_; diff --git a/src/storage/ob_i_store.cpp b/src/storage/ob_i_store.cpp index 4ef40e52e..b04b2e997 100644 --- a/src/storage/ob_i_store.cpp +++ b/src/storage/ob_i_store.cpp @@ -18,7 +18,7 @@ #include "ob_i_store.h" #include "share/rc/ob_module_provider.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -49,7 +49,6 @@ int ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(ObColDescIArray &store_ou void ObStoreCtx::reset() { - ls_id_.reset(); ls_ = nullptr; branch_ = 0; tablet_id_.reset(); @@ -66,45 +65,36 @@ void ObStoreCtx::reset() check_seq_ = 0; } -int ObStoreCtx::init_for_read(const ObLSID &ls_id, - const common::ObTabletID tablet_id, +int ObStoreCtx::init_for_read(const common::ObTabletID tablet_id, const int64_t timeout, const int64_t tx_lock_timeout, const SCN &snapshot_version) { int ret = OB_SUCCESS; ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(ls_svr->get_ls(tenant_ls))) { STORAGE_LOG(WARN, "get_ls from ls service fail.", K(ret), K(*ls_svr)); } else { tablet_id_ = tablet_id; - ret = init_for_read(ls_handle, timeout, tx_lock_timeout, snapshot_version); + ret = init_for_read(tenant_ls, timeout, tx_lock_timeout, snapshot_version); } return ret; } -int ObStoreCtx::init_for_read(const ObLSHandle &ls_handle, +int ObStoreCtx::init_for_read(ObLS *tenant_ls, const int64_t timeout, const int64_t tx_lock_timeout, const SCN &snapshot_version) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObTxTable *tx_table = nullptr; - if (!ls_handle.is_valid() || timeout < 0 || !snapshot_version.is_valid()) { + if (OB_ISNULL(tenant_ls) || timeout < 0 || !snapshot_version.is_valid()) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "get invalid arguments", K(ret), K(ls_handle), K(timeout), K(tx_lock_timeout), K(snapshot_version)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_id_)); - } else if (OB_ISNULL(tx_table = ls->get_tx_table())) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "get_tx_table from log stream fail.", K(ret), K(*ls)); - } else if (OB_FAIL(mvcc_acc_ctx_.init_read(tx_table, snapshot_version, timeout, tx_lock_timeout))) { + STORAGE_LOG(WARN, "get invalid arguments", K(ret), K(tenant_ls), K(timeout), K(tx_lock_timeout), K(snapshot_version)); + } else if (OB_FAIL(mvcc_acc_ctx_.init_read(tenant_ls->get_tx_table(), + snapshot_version, timeout, tx_lock_timeout))) { STORAGE_LOG(WARN, "mvcc_acc_ctx init read fail", KR(ret), K(mvcc_acc_ctx_)); } else { - ls_id_ = ls->get_ls_id(); timeout_ = timeout; } return ret; diff --git a/src/storage/ob_i_store.h b/src/storage/ob_i_store.h index a484f2d30..876481516 100644 --- a/src/storage/ob_i_store.h +++ b/src/storage/ob_i_store.h @@ -47,7 +47,7 @@ class ObMvccRow; namespace transaction { -class ObPartTransCtx; +class ObTxCtx; } @@ -55,7 +55,6 @@ namespace storage { class ObTableStoreIterator; class ObLS; -class ObLSHandle; class ObTablet; class ObITable; struct ObStoreCtx; @@ -471,18 +470,17 @@ struct ObStoreCtx void reset(); bool is_valid() const { - return ls_id_.is_valid() && OB_NOT_NULL(ls_) && mvcc_acc_ctx_.is_valid(); + return OB_NOT_NULL(ls_) && mvcc_acc_ctx_.is_valid(); } bool is_read() const { return mvcc_acc_ctx_.is_read(); } bool is_write() const { return mvcc_acc_ctx_.is_write(); } bool is_replay() const { return mvcc_acc_ctx_.is_replay(); } bool is_read_store_ctx() const { return is_read_store_ctx_; } - int init_for_read(const share::ObLSID &ls_id, - const common::ObTabletID tablet_id, + int init_for_read(const common::ObTabletID tablet_id, const int64_t timeout, const int64_t lock_timeout_us, const share::SCN &snapshot_version); - int init_for_read(const storage::ObLSHandle &ls_handle, + int init_for_read(storage::ObLS *tenant_ls, const int64_t timeout, const int64_t lock_timeout_us, const share::SCN &snapshot_version); @@ -498,7 +496,6 @@ struct ObStoreCtx int get_all_tables(ObIArray &iter_tables); int get_fork_snapshot_scn(const common::ObTabletID &tablet_id, share::SCN &fork_snapshot_scn); TO_STRING_KV(KP(this), - K_(ls_id), KP_(ls), K_(branch), K_(timeout), @@ -510,7 +507,6 @@ struct ObStoreCtx K_(is_read_store_ctx), K_(update_full_column), K_(is_fork_ctx)); - share::ObLSID ls_id_; storage::ObLS *ls_; int16_t branch_; // parallel write id common::ObTabletID tablet_id_; diff --git a/src/storage/ob_i_tablet_memtable.cpp b/src/storage/ob_i_tablet_memtable.cpp index 1be802676..d151edbc5 100644 --- a/src/storage/ob_i_tablet_memtable.cpp +++ b/src/storage/ob_i_tablet_memtable.cpp @@ -30,12 +30,11 @@ namespace storage { int ObITabletMemtable::inc_unsubmitted_cnt() { int ret = OB_SUCCESS; - const share::ObLSID ls_id = get_ls_id(); int64_t unsubmitted_cnt = inc_unsubmitted_cnt_(); - TRANS_LOG(DEBUG, "inc_unsubmitted_cnt", K(ls_id), KPC(this), K(lbt())); + TRANS_LOG(DEBUG, "inc_unsubmitted_cnt", KPC(this), K(lbt())); if (OB_FAIL(get_unset_active_memtable_logging_blocked())) { - TRANS_LOG(WARN, "cannot inc unsubmitted_cnt", K(unsubmitted_cnt), K(ls_id), KPC(this)); + TRANS_LOG(WARN, "cannot inc unsubmitted_cnt", K(unsubmitted_cnt), KPC(this)); } return ret; @@ -44,7 +43,6 @@ int ObITabletMemtable::inc_unsubmitted_cnt() int ObITabletMemtable::dec_unsubmitted_cnt() { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); // fix issue 47021079 // To avoid the following case where logging_block cannot be unset: @@ -65,15 +63,15 @@ int ObITabletMemtable::dec_unsubmitted_cnt() bool is_frozen = is_frozen_memtable(); int64_t write_ref_cnt = get_write_ref(); int64_t new_unsubmitted_cnt = get_unsubmitted_cnt(); - TRANS_LOG(DEBUG, "dec_unsubmitted_cnt", K(ls_id), KPC(this), K(lbt())); + TRANS_LOG(DEBUG, "dec_unsubmitted_cnt", KPC(this), K(lbt())); if (OB_UNLIKELY(old_unsubmitted_cnt < 0)) { - TRANS_LOG(ERROR, "unsubmitted_cnt not match", K(ret), K(ls_id), KPC(this)); + TRANS_LOG(ERROR, "unsubmitted_cnt not match", K(ret), KPC(this)); } else if (is_frozen && 0 == write_ref_cnt && 0 == new_unsubmitted_cnt) { (void)unset_logging_blocked_for_active_memtable_(); - TRANS_LOG(INFO, "memtable log submitted", K(ret), K(ls_id), KPC(this)); + TRANS_LOG(INFO, "memtable log submitted", K(ret), KPC(this)); } return ret; @@ -88,7 +86,7 @@ void ObITabletMemtable::unset_logging_blocked_for_active_memtable_() if (OB_NOT_NULL(memtable_mgr)) { do { if (OB_FAIL(memtable_mgr->unset_logging_blocked_for_active_memtable(this))) { - TRANS_LOG(ERROR, "fail to unset logging blocked for active memtable", K(ret), K(ls_id_), KPC(this)); + TRANS_LOG(ERROR, "fail to unset logging blocked for active memtable", K(ret), KPC(this)); ob_usleep(100); } } while (OB_FAIL(ret)); @@ -102,7 +100,7 @@ int ObITabletMemtable::resolve_left_boundary_for_active_memtable_() if (OB_NOT_NULL(memtable_mgr)) { if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, get_end_scn()))) { - TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", K(ret), K(ls_id_), KPC(this)); + TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", K(ret), KPC(this)); } } @@ -148,7 +146,6 @@ int ObITabletMemtable::set_freezer(ObFreezer *handler) int ObITabletMemtable::set_rec_scn(const SCN rec_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; @@ -158,7 +155,7 @@ int ObITabletMemtable::set_rec_scn(const SCN rec_scn) TRANS_LOG(WARN, "invalid args", K(ret), K(rec_scn)); } else if (rec_scn <= get_start_scn()) { ret = OB_SCN_OUT_OF_BOUND; - TRANS_LOG(ERROR, "cannot set freeze log ts smaller to start log ts", K(ret), K(rec_scn), K(ls_id), KPC(this)); + TRANS_LOG(ERROR, "cannot set freeze log ts smaller to start log ts", K(ret), K(rec_scn), KPC(this)); } else { SCN old_rec_scn; SCN new_rec_scn = get_rec_scn(); @@ -175,7 +172,6 @@ int ObITabletMemtable::set_rec_scn(const SCN rec_scn) int ObITabletMemtable::set_start_scn(const share::SCN start_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; @@ -198,7 +194,7 @@ int ObITabletMemtable::set_start_scn(const share::SCN start_scn) // greater than the real log of its next memtable. || start_scn >= rec_scn_) { ret = OB_SCN_OUT_OF_BOUND; - TRANS_LOG(ERROR, "cannot set start ts now", K(ret), K(start_scn), K(ls_id), KPC(this)); + TRANS_LOG(ERROR, "cannot set start ts now", K(ret), K(start_scn), KPC(this)); } else { key_.scn_range_.start_scn_ = start_scn; } @@ -209,7 +205,6 @@ int ObITabletMemtable::set_start_scn(const share::SCN start_scn) int ObITabletMemtable::set_end_scn(const SCN freeze_scn) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; @@ -220,7 +215,7 @@ int ObITabletMemtable::set_end_scn(const SCN freeze_scn) } else if (freeze_scn < get_start_scn()) { ret = OB_SCN_OUT_OF_BOUND; TRANS_LOG(ERROR, "cannot set freeze log ts smaller to start log ts", - K(ret), K(freeze_scn), K(ls_id), KPC(this)); + K(ret), K(freeze_scn), KPC(this)); } else { SCN old_end_scn; SCN new_end_scn = get_end_scn(); @@ -240,7 +235,6 @@ int ObITabletMemtable::set_end_scn(const SCN freeze_scn) int ObITabletMemtable::set_max_end_scn(const SCN scn, bool allow_backoff) { int ret = OB_SUCCESS; - share::ObLSID ls_id = get_ls_id(); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; @@ -259,7 +253,7 @@ int ObITabletMemtable::set_max_end_scn(const SCN scn, bool allow_backoff) || scn > get_end_scn()) { ret = OB_SCN_OUT_OF_BOUND; TRANS_LOG(ERROR, "cannot set max end log ts smaller to start log ts", - K(ret), K(scn), K(ls_id), KPC(this)); + K(ret), K(scn), KPC(this)); } else if (allow_backoff) { set_has_backoffed(); TRANS_LOG(INFO, "set max_end_scn force", K(scn), K(max_end_scn_.atomic_get()), K(key_), KPC(this)); diff --git a/src/storage/ob_i_tablet_memtable.h b/src/storage/ob_i_tablet_memtable.h index 1be2fe3ab..a2b1fb581 100644 --- a/src/storage/ob_i_tablet_memtable.h +++ b/src/storage/ob_i_tablet_memtable.h @@ -247,7 +247,7 @@ class ObITabletMemtable : public ObIMemtable, public checkpoint::ObFreezeCheckpo // *************** pure virtual functions ***************** virtual bool is_inited() const = 0; virtual int init(const ObITable::TableKey &table_key, - ObLSHandle &ls_handle, + ObLS *tenant_ls, ObFreezer *freezer, ObTabletMemtableMgr *memtable_mgr, const int64_t schema_version, @@ -334,7 +334,6 @@ class ObITabletMemtable : public ObIMemtable, public checkpoint::ObFreezeCheckpo INHERIT_TO_STRING_KV("ObITable", ObITable, - K(ls_id_), K(allow_freeze_), K(is_flushed_), K(is_tablet_freeze_), diff --git a/src/storage/ob_order_perserving_encoder.cpp b/src/storage/ob_order_perserving_encoder.cpp index 78f587136..90d3fdf2a 100644 --- a/src/storage/ob_order_perserving_encoder.cpp +++ b/src/storage/ob_order_perserving_encoder.cpp @@ -402,7 +402,11 @@ int ObOrderPerservingEncoder::encode_from_string_varlen( && ((param.is_memcmp_ && specific::avx2::check_terminator_simd((unsigned char *)str.ptr(), str.length(), 0x00)) || (!param.is_memcmp_ && specific::avx2::check_terminator_simd((unsigned char *)str.ptr(), str.length(), 0x20)))) { MEMCPY(to, str.ptr(), str.length()); - str_toupper((char *)to, str.length()); + for (int64_t i = 0; i < str.length(); ++i) { + if (to[i] >= 'a' && to[i] <= 'z') { + to[i] -= 'a' - 'A'; + } + } to += str.length(); if (param.is_memcmp_) { *(to) = 0x00; diff --git a/src/storage/ob_partition_range_spliter.cpp b/src/storage/ob_partition_range_spliter.cpp index f544a2f88..a570b50b2 100644 --- a/src/storage/ob_partition_range_spliter.cpp +++ b/src/storage/ob_partition_range_spliter.cpp @@ -2286,17 +2286,15 @@ int ObPartitionIncrementalRangeSpliter::ObIncrementalIterator::prepare_table_acc int ObPartitionIncrementalRangeSpliter::ObIncrementalIterator::prepare_store_ctx() { int ret = OB_SUCCESS; - const ObLSID &ls_id = merge_ctx_.get_ls_id(); int64_t snapshot = merge_ctx_.get_snapshot(); SCN scn; if (OB_FAIL(scn.convert_for_tx(snapshot))) { - STORAGE_LOG(WARN, "convert for tx fail", K(ret), K(ls_id), K(snapshot)); - } else if (OB_FAIL(store_ctx_.init_for_read(ls_id, - merge_ctx_.get_tablet_id(), + STORAGE_LOG(WARN, "convert for tx fail", K(ret), K(snapshot)); + } else if (OB_FAIL(store_ctx_.init_for_read(merge_ctx_.get_tablet_id(), INT64_MAX, -1, scn))) { - STORAGE_LOG(WARN, "init store ctx fail", K(ret), K(ls_id), K(snapshot)); + STORAGE_LOG(WARN, "init store ctx fail", K(ret), K(snapshot)); } return ret; } diff --git a/src/storage/ob_partition_split_query.cpp b/src/storage/ob_partition_split_query.cpp index e6579ed63..68fd73b98 100644 --- a/src/storage/ob_partition_split_query.cpp +++ b/src/storage/ob_partition_split_query.cpp @@ -17,7 +17,6 @@ #define USING_LOG_PREFIX SHARE #include "ob_partition_split_query.h" -#include "src/storage/ls/ob_ls.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -28,12 +27,10 @@ using namespace oceanbase::storage; int ObPartitionSplitQuery::get_tablet_handle( const ObTabletID &tablet_id, - const ObLSID &ls_id, storage::ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); const int64_t snapshot_version = INT64_MAX; // MAX_TRANS_VERSION if (OB_FAIL(ObTabletCreateDeleteHelper::check_and_get_tablet(key, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, @@ -278,7 +275,7 @@ int ObPartitionSplitQuery::get_tablet_split_info( } else if (!split_info_.is_split_dst_with_partkey() && !split_info_.is_split_dst_without_partkey()) { ret = OB_SCHEMA_EAGAIN; LOG_WARN("maybe split dst finished, retry", K(ret), K(tablet.get_tablet_meta().tablet_id_), - K(tablet.get_tablet_meta().ls_id_)); + K(split_info_)); } return ret; } @@ -386,18 +383,17 @@ int ObPartitionSplitQuery::fill_auto_split_params( { int ret = OB_SUCCESS; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; ObTabletSplitTscInfo split_info; #ifdef ERRSIM DEBUG_SYNC(BEFORE_FILL_AUTO_SPLIT_PARAMS); #endif - if (!tablet_id.is_valid() || !ls_id.is_valid() - || OB_ISNULL(op) || OB_INVALID_ID == filter_type || OB_ISNULL(filter_params)) { + if (!tablet_id.is_valid() || OB_ISNULL(op) + || OB_INVALID_ID == filter_type || OB_ISNULL(filter_params)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", - K(ret), K(tablet_id), K(ls_id), KP(op), K(filter_type), KP(filter_params)); + K(ret), K(tablet_id), KP(op), K(filter_type), KP(filter_params)); } else if (OB_FAIL(filter_params->count() <= 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("auto split filter params is zero", K(ret)); @@ -419,7 +415,7 @@ int ObPartitionSplitQuery::fill_auto_split_params( } else if (!split_info.is_split_dst_without_partkey() && is_split_dst) { // newly fetched get_split_info's split mds says not split dst, but caller say is_split_dst ret = OB_SCHEMA_EAGAIN; - LOG_WARN("maybe split dst finished, retry", K(ret), K(tablet_id), K(ls_id)); + LOG_WARN("maybe split dst finished, retry", K(ret), K(tablet_id)); } else { // only need to fill bypass expr. if (OB_ISNULL(expr = filter_params->at(0))) { @@ -539,4 +535,3 @@ int ObPartitionSplitQuery::fill_range_filter_param( LOG_DEBUG("fill range filter param", K(ret), K(lower_bound), K(upper_bound)); return ret; } - diff --git a/src/storage/ob_partition_split_query.h b/src/storage/ob_partition_split_query.h index b1ea15535..0376155b2 100644 --- a/src/storage/ob_partition_split_query.h +++ b/src/storage/ob_partition_split_query.h @@ -35,7 +35,7 @@ class ObPushdownOperator; namespace storage { class ObTabletHandle; -class ObLSHandle; +class ObLS; struct ObTabletSplitTscInfo; } namespace share @@ -53,7 +53,6 @@ class ObPartitionSplitQuery final // FIXME:remove it int get_tablet_handle( const ObTabletID &tablet_id, - const ObLSID &ls_id, storage::ObTabletHandle &tablet_handle); int get_tablet_split_range( diff --git a/src/storage/ob_protected_memtable_mgr_handle.cpp b/src/storage/ob_protected_memtable_mgr_handle.cpp index fbb1b6975..ddb281829 100644 --- a/src/storage/ob_protected_memtable_mgr_handle.cpp +++ b/src/storage/ob_protected_memtable_mgr_handle.cpp @@ -26,8 +26,7 @@ namespace oceanbase namespace storage { -int ObProtectedMemtableMgrHandle::create_tablet_memtable_mgr_(const ObLSID &ls_id, - const ObTabletID &tablet_id, +int ObProtectedMemtableMgrHandle::create_tablet_memtable_mgr_(const ObTabletID &tablet_id, lib::Worker::CompatMode compat_mode) { int ret = OB_SUCCESS; @@ -36,10 +35,10 @@ int ObProtectedMemtableMgrHandle::create_tablet_memtable_mgr_(const ObLSID &ls_i if (memtable_mgr_handle_.is_valid()) { } else if (OB_ISNULL(mgr = pool->acquire())) { ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to memtable manager alloc", KR(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(mgr->ObIMemtableMgr::init(ls_id, tablet_id, compat_mode))) { + STORAGE_LOG(WARN, "failed to memtable manager alloc", KR(ret), K(tablet_id)); + } else if (OB_FAIL(mgr->ObIMemtableMgr::init(tablet_id, compat_mode))) { pool->release(mgr); - STORAGE_LOG(WARN, "failed to init memtable mgr", KR(ret), K(tablet_id), K(ls_id)); + STORAGE_LOG(WARN, "failed to init memtable mgr", KR(ret), K(tablet_id)); } else { memtable_mgr_handle_.set_memtable_mgr(mgr, pool); } diff --git a/src/storage/ob_protected_memtable_mgr_handle.h b/src/storage/ob_protected_memtable_mgr_handle.h index 4ca630f51..cb9829e85 100644 --- a/src/storage/ob_protected_memtable_mgr_handle.h +++ b/src/storage/ob_protected_memtable_mgr_handle.h @@ -34,9 +34,9 @@ } \ SpinWLockGuard guard(memtable_mgr_handle_lock_); \ if (!memtable_mgr_handle_.is_valid()) { \ - STORAGE_LOG(INFO, "memtable_mgr_handle_ is not exist, need create", K(tablet_meta.ls_id_), K(tablet_meta.tablet_id_)); \ - if (OB_FAIL(create_tablet_memtable_mgr_(tablet_meta.ls_id_, tablet_meta.tablet_id_, tablet_meta.compat_mode_))) { \ - STORAGE_LOG(WARN, "failed to create_tablet_memtable_mgr", K(tablet_meta.ls_id_), K(tablet_meta.tablet_id_), K(tablet_meta.compat_mode_)); \ + STORAGE_LOG(INFO, "memtable_mgr_handle_ is not exist, need create", K(tablet_meta.tablet_id_)); \ + if (OB_FAIL(create_tablet_memtable_mgr_(tablet_meta.tablet_id_, tablet_meta.compat_mode_))) { \ + STORAGE_LOG(WARN, "failed to create_tablet_memtable_mgr", K(tablet_meta.tablet_id_), K(tablet_meta.compat_mode_)); \ } \ } \ } while (OB_SUCC(ret)); @@ -176,8 +176,7 @@ class ObProtectedMemtableMgrHandle int try_reset_memtable_mgr_handle_(); bool need_reset_without_lock_(); bool need_reset_(); - int create_tablet_memtable_mgr_(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, + int create_tablet_memtable_mgr_(const ObTabletID &tablet_id, lib::Worker::CompatMode compat_mode); ObMemtableMgrHandle memtable_mgr_handle_; mutable common::SpinRWLock memtable_mgr_handle_lock_; diff --git a/src/storage/ob_storage_clog_recorder.cpp b/src/storage/ob_storage_clog_recorder.cpp index a8c149169..e232d1d1c 100644 --- a/src/storage/ob_storage_clog_recorder.cpp +++ b/src/storage/ob_storage_clog_recorder.cpp @@ -171,7 +171,7 @@ int ObIStorageClogRecorder::try_update_with_lock( return ret; } -int ObIStorageClogRecorder::try_update_for_leader( +int ObIStorageClogRecorder::try_update( const int64_t update_version, ObIAllocator *allocator, const int64_t timeout_ts) @@ -244,7 +244,7 @@ int ObIStorageClogRecorder::replay_clog( void ObIStorageClogRecorder::clog_update_fail() { - sync_clog_failed_for_leader(); + on_sync_clog_failure(); WEAK_BARRIER(); ATOMIC_STORE(&logcb_finish_flag_, true); } @@ -263,7 +263,7 @@ void ObIStorageClogRecorder::clog_update_succ( ret = OB_ERR_UNEXPECTED; LOG_WARN("clog ts is invalid", K(ret), K_(clog_scn)); } else { - if (OB_FAIL(sync_clog_succ_for_leader(update_version))) { + if (OB_FAIL(on_sync_clog_success(update_version))) { LOG_WARN("failed to save for leader", K(ret), KPC(this)); } else { finish_flag = true; @@ -297,40 +297,38 @@ int ObIStorageClogRecorder::write_clog( } int ObIStorageClogRecorder::get_tablet_handle( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_id)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, tablet_handle))) { + LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } return ret; } int ObIStorageClogRecorder::replay_get_tablet_handle( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::SCN &scn, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; const bool is_update_mds_table = false; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->replay_get_tablet(tablet_id, scn, is_update_mds_table, tablet_handle))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->replay_get_tablet(tablet_id, scn, is_update_mds_table, tablet_handle))) { if (OB_OBSOLETE_CLOG_NEED_SKIP == ret) { - LOG_INFO("clog is obsolete, should skip replay", K(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_INFO("clog is obsolete, should skip replay", K(ret), K(tablet_id), K(scn)); ret = OB_SUCCESS; } else if (OB_TIMEOUT == ret) { ret = OB_EAGAIN; - LOG_INFO("retry get tablet for timeout error", K(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_INFO("retry get tablet for timeout error", K(ret), K(tablet_id), K(scn)); } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_id), K(scn)); + LOG_WARN("failed to get tablet", K(ret), K(tablet_id), K(scn)); } } return ret; diff --git a/src/storage/ob_storage_clog_recorder.h b/src/storage/ob_storage_clog_recorder.h index c469b3837..2ba2dc795 100644 --- a/src/storage/ob_storage_clog_recorder.h +++ b/src/storage/ob_storage_clog_recorder.h @@ -31,7 +31,6 @@ class ObLogHandler; } // namespace palf namespace share { -class ObLSID; } namespace storage { @@ -80,7 +79,7 @@ class ObIStorageClogRecorder void reset(); // leader - int try_update_for_leader( + int try_update( const int64_t update_version, ObIAllocator *allocator, const int64_t timeout_ts = 1000000); @@ -109,8 +108,8 @@ class ObIStorageClogRecorder const char *buf, const int64_t size, int64_t &pos) = 0; - virtual int sync_clog_succ_for_leader(const int64_t update_version) = 0; - virtual void sync_clog_failed_for_leader() = 0; + virtual int on_sync_clog_success(const int64_t update_version) = 0; + virtual void on_sync_clog_failure() = 0; // call prepare struct only once virtual int prepare_struct_in_lock( int64_t &update_version, @@ -136,11 +135,9 @@ class ObIStorageClogRecorder share::SCN get_log_scn() const { return clog_scn_; } int get_tablet_handle( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, ObTabletHandle &tablet_handle); int replay_get_tablet_handle( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::SCN &scn, ObTabletHandle &tablet_handle); diff --git a/src/storage/ob_storage_ha_struct.cpp b/src/storage/ob_storage_ha_struct.cpp index 67e8d727f..7159874ad 100644 --- a/src/storage/ob_storage_ha_struct.cpp +++ b/src/storage/ob_storage_ha_struct.cpp @@ -19,379 +19,12 @@ #include "share/rc/ob_module_provider.h" #include "storage/tablet/ob_tablet_iterator.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { namespace storage { -ERRSIM_POINT_DEF(EN_REBUILD_FAILED_STATUS); -ERRSIM_POINT_DEF(ALLOW_MIGRATION_STATUS_CHANGED); - -/******************ObMigrationOpType*********************/ -static const char *migration_op_type_strs[] = { - "ADD_LS_OP", - "MIGRATE_LS_OP", - "REBUILD_LS_OP", - "CHANGE_LS_OP", - "REMOVE_LS_OP", - "RESTORE_STANDBY_LS_OP", - "REBUILD_TABLET_OP", - "REPLACE_LS_OP", -}; - -const char *ObMigrationOpType::get_str(const TYPE &type) -{ - const char *str = nullptr; - - if (type < 0 || type >= MAX_LS_OP) { - str = "UNKNOWN_OP"; - } else { - str = migration_op_type_strs[type]; - } - return str; -} - - - - -/******************ObMigrationStatusHelper*********************/ -int ObMigrationStatusHelper::trans_migration_op( - const ObMigrationOpType::TYPE &op_type, ObMigrationStatus &migration_status) -{ - int ret = OB_SUCCESS; - migration_status = OB_MIGRATION_STATUS_MAX; - - if (!ObMigrationOpType::is_valid(op_type)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(op_type)); - } else { - switch (op_type) { - case ObMigrationOpType::ADD_LS_OP: { - migration_status = OB_MIGRATION_STATUS_ADD; - break; - } - case ObMigrationOpType::MIGRATE_LS_OP: { - migration_status = OB_MIGRATION_STATUS_MIGRATE; - break; - } - case ObMigrationOpType::REBUILD_LS_OP: { - migration_status = OB_MIGRATION_STATUS_REBUILD; - break; - } - case ObMigrationOpType::CHANGE_LS_OP: { - migration_status = OB_MIGRATION_STATUS_CHANGE; - break; - } - case ObMigrationOpType::RESTORE_STANDBY_LS_OP: { - migration_status = OB_MIGRATION_STATUS_RESTORE_STANDBY; - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("unknown op type", K(ret), K(op_type)); - } - } - } - - return ret; -} - - -int ObMigrationStatusHelper::trans_reboot_status(const ObMigrationStatus &cur_status, ObMigrationStatus &reboot_status) -{ - int ret = OB_SUCCESS; - reboot_status = OB_MIGRATION_STATUS_MAX; - - if (cur_status < OB_MIGRATION_STATUS_NONE || cur_status >= OB_MIGRATION_STATUS_MAX) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(cur_status)); - } else { - switch (cur_status) { - case OB_MIGRATION_STATUS_NONE: { - reboot_status = OB_MIGRATION_STATUS_NONE; - break; - } - case OB_MIGRATION_STATUS_ADD: - case OB_MIGRATION_STATUS_ADD_FAIL: { - reboot_status = OB_MIGRATION_STATUS_ADD_FAIL; - break; - } - case OB_MIGRATION_STATUS_MIGRATE: - case OB_MIGRATION_STATUS_MIGRATE_FAIL: { - reboot_status = OB_MIGRATION_STATUS_MIGRATE_FAIL; - break; - } - case OB_MIGRATION_STATUS_REBUILD: { - reboot_status = OB_MIGRATION_STATUS_REBUILD; - break; - } - case OB_MIGRATION_STATUS_CHANGE: { - reboot_status = OB_MIGRATION_STATUS_NONE; - break; - } - case OB_MIGRATION_STATUS_RESTORE_STANDBY: { - reboot_status = OB_MIGRATION_STATUS_NONE; - break; - } - case OB_MIGRATION_STATUS_HOLD: { - reboot_status = OB_MIGRATION_STATUS_NONE; - break; - } - case OB_MIGRATION_STATUS_MIGRATE_WAIT : { - reboot_status = OB_MIGRATION_STATUS_MIGRATE_FAIL; - break; - } - case OB_MIGRATION_STATUS_ADD_WAIT : { - reboot_status = OB_MIGRATION_STATUS_ADD_FAIL; - break; - } - case OB_MIGRATION_STATUS_REBUILD_WAIT: { - reboot_status = OB_MIGRATION_STATUS_REBUILD; - break; - } - case OB_MIGRATION_STATUS_REBUILD_FAIL : { - reboot_status = OB_MIGRATION_STATUS_REBUILD_FAIL; - break; - } - case OB_MIGRATION_STATUS_GC: { - reboot_status = OB_MIGRATION_STATUS_GC; - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("invalid cur status for fail", K(ret), K(cur_status)); - } - } - } - return ret; -} - - - -bool ObMigrationStatusHelper::check_can_restore(const ObMigrationStatus &cur_status) -{ - return OB_MIGRATION_STATUS_NONE == cur_status; -} - -// The status of the log stream is OB_MIGRATION_STATUS_GC, which blocks log replay. -// Log stream that is not in the member_list will not be added to the member_list. -// If the log stream status modification fails, there is no need to online log_handler. -// After setting the LS GC flag and stopping log synchronization, the destination -// can be restored through rebuilding. -int ObMigrationStatusHelper::set_ls_migrate_gc_status_( - ObLS &ls, - bool &allow_gc) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ls.set_ls_migration_gc(allow_gc))) { - LOG_WARN("failed to set migration status", K(ret)); - } else if (!allow_gc) { - LOG_INFO("ls is not allow gc", K(ret), K(ls)); - } else if (OB_FAIL(ls.get_log_handler()->disable_sync())) { - LOG_WARN("failed to disable replay", K(ret)); - } - return ret; -} - - - -bool ObMigrationStatusHelper::check_migration_status_is_fail_(const ObMigrationStatus &cur_status) -{ - bool is_fail = false; - if (OB_MIGRATION_STATUS_ADD_FAIL == cur_status - || OB_MIGRATION_STATUS_MIGRATE_FAIL == cur_status - || OB_MIGRATION_STATUS_REBUILD_FAIL == cur_status) { - is_fail = true; - } - return is_fail; -} - -bool ObMigrationStatusHelper::need_online(const ObMigrationStatus &cur_status) -{ - return (OB_MIGRATION_STATUS_NONE == cur_status - || OB_MIGRATION_STATUS_GC == cur_status); -} - -bool ObMigrationStatusHelper::check_allow_gc_abandoned_ls(const ObMigrationStatus &cur_status) -{ - bool allow_gc = false; - if (check_migration_status_is_fail_(cur_status)) { - allow_gc = true; - } else if (OB_MIGRATION_STATUS_GC == cur_status) { - allow_gc = true; - } - return allow_gc; -} - - -bool ObMigrationStatusHelper::check_can_migrate_out(const ObMigrationStatus &cur_status) -{ - bool can_migrate_out = true; - if (OB_MIGRATION_STATUS_NONE != cur_status) { - can_migrate_out = false; - } - return can_migrate_out; -} - -int ObMigrationStatusHelper::check_can_change_status( - const ObMigrationStatus &cur_status, - const ObMigrationStatus &change_status, - bool &can_change) -{ - int ret = OB_SUCCESS; - can_change = false; - - if (!is_valid(cur_status) || !is_valid(change_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("check can change status get invalid argument", K(ret), K(cur_status), K(change_status)); - } else { - switch (cur_status) { - case OB_MIGRATION_STATUS_NONE: { - if (OB_MIGRATION_STATUS_ADD == change_status - || OB_MIGRATION_STATUS_MIGRATE == change_status - || OB_MIGRATION_STATUS_CHANGE == change_status - || OB_MIGRATION_STATUS_REBUILD == change_status - || OB_MIGRATION_STATUS_RESTORE_STANDBY == change_status - || OB_MIGRATION_STATUS_GC == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_ADD: { - if (OB_MIGRATION_STATUS_ADD == change_status - || OB_MIGRATION_STATUS_ADD_FAIL == change_status - || OB_MIGRATION_STATUS_ADD_WAIT == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_ADD_FAIL: { - if (OB_MIGRATION_STATUS_ADD_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_MIGRATE: { - if (OB_MIGRATION_STATUS_MIGRATE == change_status - || OB_MIGRATION_STATUS_MIGRATE_FAIL == change_status - || OB_MIGRATION_STATUS_MIGRATE_WAIT == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_MIGRATE_FAIL: { - if (OB_MIGRATION_STATUS_MIGRATE_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_REBUILD: { - if (OB_MIGRATION_STATUS_NONE == change_status - || OB_MIGRATION_STATUS_REBUILD == change_status - || OB_MIGRATION_STATUS_REBUILD_WAIT == change_status - || OB_MIGRATION_STATUS_REBUILD_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_CHANGE: { - if (OB_MIGRATION_STATUS_NONE == change_status - || OB_MIGRATION_STATUS_CHANGE == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_RESTORE_STANDBY: { - if (OB_MIGRATION_STATUS_NONE == change_status - || OB_MIGRATION_STATUS_RESTORE_STANDBY == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_HOLD: { - if (OB_MIGRATION_STATUS_HOLD == change_status - || OB_MIGRATION_STATUS_ADD_FAIL == change_status - || OB_MIGRATION_STATUS_MIGRATE_FAIL == change_status - || OB_MIGRATION_STATUS_NONE == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_MIGRATE_WAIT: { - if (OB_MIGRATION_STATUS_HOLD == change_status - || OB_MIGRATION_STATUS_MIGRATE_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_ADD_WAIT: { - if (OB_MIGRATION_STATUS_HOLD == change_status - || OB_MIGRATION_STATUS_ADD_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_REBUILD_WAIT: { - if (OB_MIGRATION_STATUS_NONE == change_status - || OB_MIGRATION_STATUS_REBUILD_WAIT == change_status - || OB_MIGRATION_STATUS_REBUILD == change_status - || OB_MIGRATION_STATUS_REBUILD_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_REBUILD_FAIL: { - if (OB_MIGRATION_STATUS_REBUILD_FAIL == change_status) { - can_change = true; - } - break; - } - case OB_MIGRATION_STATUS_GC: { - if (OB_MIGRATION_STATUS_GC == change_status) { - can_change = true; - } - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("invalid cur status for fail", K(ret), K(cur_status)); - } - } - } - -#ifdef ERRSIM - if (OB_SUCC(ret)) { - ret = ALLOW_MIGRATION_STATUS_CHANGED ? : OB_SUCCESS; - if (OB_FAIL(ret)) { - can_change = true; - ret = OB_SUCCESS; - } - } -#endif - - return ret; -} - -bool ObMigrationStatusHelper::is_valid(const ObMigrationStatus &status) -{ - return status >= ObMigrationStatus::OB_MIGRATION_STATUS_NONE - && status < ObMigrationStatus::OB_MIGRATION_STATUS_MAX; -} - - -bool ObMigrationStatusHelper::check_can_report_readable_scn( - const ObMigrationStatus &cur_status) -{ - bool can_report = false; - if (ObMigrationStatus::OB_MIGRATION_STATUS_HOLD == cur_status - || ObMigrationStatus::OB_MIGRATION_STATUS_NONE == cur_status) { - can_report = true; - } else { - can_report = false; - } - return can_report; -} - /******************ObStorageHASrcInfo*********************/ ObStorageHASrcInfo::ObStorageHASrcInfo() @@ -439,98 +72,6 @@ ObMacroBlockCopyArgInfo::~ObMacroBlockCopyArgInfo() -/******************ObCopyTabletSimpleInfo*********************/ -ObCopyTabletSimpleInfo::ObCopyTabletSimpleInfo() - : tablet_id_(), - status_(ObCopyTabletStatus::MAX_STATUS), - data_size_(0) -{ -} - -void ObCopyTabletSimpleInfo::reset() -{ - tablet_id_.reset(); - status_ = ObCopyTabletStatus::MAX_STATUS; - data_size_ = 0; -} - -bool ObCopyTabletSimpleInfo::is_valid() const -{ - return tablet_id_.is_valid() && ObCopyTabletStatus::is_valid(status_) && data_size_ >= 0; -} - -/******************ObMigrationFakeBlockID*********************/ -ObMigrationFakeBlockID::ObMigrationFakeBlockID() -{ - migration_fake_block_id_.reset(); - migration_fake_block_id_.set_block_index(FAKE_BLOCK_INDEX); -} - -/******************ObCopySSTableHelper*********************/ - -/******************ObMigrationUtils*********************/ -bool ObMigrationUtils::is_need_retry_error(const int err) -{ - bool bret = true; - switch (err) { - case OB_NOT_INIT : - case OB_INVALID_ARGUMENT : - case OB_ERR_UNEXPECTED : - case OB_ERR_SYS : - case OB_INIT_TWICE : - case OB_SRC_DO_NOT_ALLOWED_MIGRATE : - case OB_CANCELED : - case OB_NOT_SUPPORTED : - case OB_SERVER_OUTOF_DISK_SPACE : - case OB_LOG_NOT_SYNC : - case OB_INVALID_DATA : - case OB_CHECKSUM_ERROR : - case OB_DDL_SSTABLE_RANGE_CROSS : - case OB_TENANT_NOT_EXIST : - case OB_INVALID_TABLE_STORE : - case OB_UNEXPECTED_TABLET_STATUS : - case OB_MIGRATE_TX_DATA_NOT_CONTINUES : - bret = false; - break; - default: - break; - } - return bret; -} - - -int ObMigrationUtils::get_ls_rebuild_seq(const share::ObLSID &ls_id, int64_t &rebuild_seq) -{ - int ret = OB_SUCCESS; - rebuild_seq = 0; - storage::ObLS *ls = NULL; - ObLSService *ls_service = NULL; - ObLSHandle handle; - ObMigrationStatus status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid args", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_service = MTL_WITH_CHECK(ObLSService *))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream service is NULL", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::HA_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream not exist", K(ret), K(ls_id)); - } else if (OB_FAIL(ls->get_migration_status(status))) { - LOG_WARN("failed to get migration status", K(ret), KPC(ls)); - } else if (!ObMigrationStatusHelper::check_can_migrate_out(status) || ls->is_stopped() || ls->is_offline()) { - ret = OB_SRC_DO_NOT_ALLOWED_MIGRATE; - LOG_WARN("migration src ls migration status is not none or ls in stop status", - K(ret), KPC(ls), K(status)); - } else { - rebuild_seq = ls->get_rebuild_seq(); - } - return ret; -} - - /******************ObCopyTableKeyInfo*********************/ ObCopyTableKeyInfo::ObCopyTableKeyInfo() : src_table_key_(), @@ -729,7 +270,7 @@ void ObLSRebuildType::reset() bool ObLSRebuildType::is_valid() const { - return type_ >= NONE && type_ < MAX; + return NONE == type_ || CLOG == type_ || TABLET == type_; } @@ -1000,58 +541,6 @@ int ObMacroBlockReuseMgr::count(int64_t &count) return ret; } -ObLogicTabletID::ObLogicTabletID() - : tablet_id_() -{ -} - -int ObLogicTabletID::init( - const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (!tablet_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("init logic tablet id get invalid argument", K(ret), K(tablet_id)); - } else { - tablet_id_ = tablet_id; - } - return ret; -} - -void ObLogicTabletID::reset() -{ - tablet_id_.reset(); -} - - -bool ObLogicTabletID::operator == (const ObLogicTabletID &other) const -{ - bool is_same = true; - if (this == &other) { - // same - } else if (tablet_id_ != other.tablet_id_) { - is_same = false; - } else { - is_same = true; - } - return is_same; -} - - -ObLSMemberListInfo::ObLSMemberListInfo() - : learner_list_(), - leader_addr_(), - member_list_() -{ -} - - -bool ObLSMemberListInfo::is_valid() const -{ - return leader_addr_.is_valid() && !member_list_.empty(); -} - - int ObMacroBlockReuseMgr::get_macro_block_reuse_info( const ObITable::TableKey &table_key, const blocksstable::ObLogicMacroBlockId &logic_id, diff --git a/src/storage/ob_storage_ha_struct.h b/src/storage/ob_storage_ha_struct.h index d1225a2f7..29edad0bc 100644 --- a/src/storage/ob_storage_ha_struct.h +++ b/src/storage/ob_storage_ha_struct.h @@ -18,7 +18,6 @@ #define OCEABASE_STORAGE_HA_STRUCT_ #include "lib/ob_define.h" -#include "share/ob_ls_id.h" #include "common/ob_member.h" #include "common/ob_tablet_id.h" #include "lib/container/ob_array.h" @@ -27,83 +26,13 @@ #include "storage/blocksstable/ob_datum_rowkey.h" #include "storage/blocksstable/ob_logic_macro_id.h" #include "observer/scheduler/ob_dag_scheduler_config.h" -#include "common/ob_learner_list.h" -#include "storage/ob_tablet_ha_status.h" +#include "storage/ob_tablet_restore_state.h" namespace oceanbase { namespace storage { -enum ObMigrationStatus -{ - OB_MIGRATION_STATUS_NONE = 0, - OB_MIGRATION_STATUS_ADD = 1, - OB_MIGRATION_STATUS_ADD_FAIL = 2, - OB_MIGRATION_STATUS_MIGRATE = 3, - OB_MIGRATION_STATUS_MIGRATE_FAIL = 4, - OB_MIGRATION_STATUS_REBUILD = 5, - OB_MIGRATION_STATUS_REBUILD_FAIL = 6, - OB_MIGRATION_STATUS_CHANGE = 7, - OB_MIGRATION_STATUS_RESTORE_STANDBY = 8, - OB_MIGRATION_STATUS_HOLD = 9, - OB_MIGRATION_STATUS_MIGRATE_WAIT = 10, - OB_MIGRATION_STATUS_ADD_WAIT = 11, - OB_MIGRATION_STATUS_REBUILD_WAIT = 12, - OB_MIGRATION_STATUS_GC = 13, // ls wait allow gc - OB_MIGRATION_STATUS_MAX, -}; - -struct ObMigrationOpType -{ - enum TYPE - { - ADD_LS_OP = 0, - MIGRATE_LS_OP = 1, - REBUILD_LS_OP = 2, - CHANGE_LS_OP = 3, - REMOVE_LS_OP = 4, - RESTORE_STANDBY_LS_OP = 5, - REBUILD_TABLET_OP = 6, - REPLACE_LS_OP = 7, - MAX_LS_OP, - }; - static const char *get_str(const TYPE &status); - static OB_INLINE bool is_valid(const TYPE &type) { return type >= 0 && type < MAX_LS_OP; } -}; - -struct ObMigrationStatusHelper -{ -public: - static int trans_migration_op(const ObMigrationOpType::TYPE &op_type, ObMigrationStatus &migrate_status); - static int trans_reboot_status(const ObMigrationStatus &cur_status, ObMigrationStatus &reboot_status); - static bool check_can_restore(const ObMigrationStatus &cur_status); - // Check the migration status. The LS in the XXX_FAIL state is considered to be an abandoned LS, which can be judged to be directly GC when restarting - static bool need_online(const ObMigrationStatus &cur_status); - static bool check_allow_gc_abandoned_ls(const ObMigrationStatus &cur_status); - static bool check_can_migrate_out(const ObMigrationStatus &cur_status); - static int check_can_change_status( - const ObMigrationStatus &cur_status, - const ObMigrationStatus &change_status, - bool &can_change); - static bool is_valid(const ObMigrationStatus &status); - static bool check_can_report_readable_scn( - const ObMigrationStatus &cur_status); -private: - static bool check_migration_status_is_fail_(const ObMigrationStatus &cur_status); - static int set_ls_migrate_gc_status_( - ObLS &ls, - bool &allow_gc); -}; - -enum ObMigrationOpPriority -{ - PRIO_HIGH = 0, - PRIO_LOW = 1, - PRIO_MID = 2, - PRIO_INVALID -}; - struct ObStorageHASrcInfo { ObStorageHASrcInfo(); @@ -135,60 +64,6 @@ struct ObMacroBlockCopyArgInfo blocksstable::ObLogicMacroBlockId logic_macro_block_id_; }; -struct ObCopyTabletStatus -{ - enum STATUS - { - TABLET_EXIST = 0, - TABLET_NOT_EXIST = 1, - MAX_STATUS, - }; - static OB_INLINE bool is_valid(const STATUS &status) { return status >= 0 && status < MAX_STATUS; } -}; - -struct ObCopyTabletSimpleInfo -{ - ObCopyTabletSimpleInfo(); - virtual ~ObCopyTabletSimpleInfo() {} - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(tablet_id), K_(status)); - common::ObTabletID tablet_id_; - ObCopyTabletStatus::STATUS status_; - int64_t data_size_; -}; - -struct ObMigrationFakeBlockID -{ - ObMigrationFakeBlockID(); - virtual ~ObMigrationFakeBlockID() = default; - TO_STRING_KV(K_(migration_fake_block_id)); - static const int64_t FAKE_BLOCK_INDEX = INT64_MAX -1; - blocksstable::MacroBlockId migration_fake_block_id_; -}; - -struct ObCopySSTableHelper -{ -}; - -class ObIHAHandler -{ -public: - ObIHAHandler() {} - virtual ~ObIHAHandler() {} - virtual int process() = 0; -private: - DISALLOW_COPY_AND_ASSIGN(ObIHAHandler); -}; - -struct ObMigrationUtils -{ - static bool is_need_retry_error(const int err); - static int get_ls_rebuild_seq( - const share::ObLSID &ls_id, - int64_t &rebuild_seq); -}; - struct ObCopyTableKeyInfo final { OB_UNIS_VERSION(1); @@ -281,7 +156,6 @@ class ObLSRebuildType final { NONE = 0, CLOG = 1, - RESERVED_2 = 2, TABLET = 3, MAX }; @@ -477,34 +351,6 @@ class ObMacroBlockReuseMgr final DISALLOW_COPY_AND_ASSIGN(ObMacroBlockReuseMgr); }; -struct ObLogicTabletID final -{ -public: - ObLogicTabletID(); - ~ObLogicTabletID() = default; - int init(const common::ObTabletID &tablet_id); - void reset(); - bool operator == (const ObLogicTabletID &other) const; - TO_STRING_KV( - K_(tablet_id)); - common::ObTabletID tablet_id_; -}; - -struct ObLSMemberListInfo final -{ -public: - ObLSMemberListInfo(); - ~ObLSMemberListInfo() = default; - bool is_valid() const; - - TO_STRING_KV(K_(learner_list), K_(leader_addr), K_(member_list)); - common::GlobalLearnerList learner_list_; - common::ObAddr leader_addr_; - common::ObArray member_list_; -private: - DISALLOW_COPY_AND_ASSIGN(ObLSMemberListInfo); -}; - } } #endif diff --git a/src/storage/ob_storage_log_type.h b/src/storage/ob_storage_log_type.h index 17861f06e..9c845c0be 100644 --- a/src/storage/ob_storage_log_type.h +++ b/src/storage/ob_storage_log_type.h @@ -79,7 +79,6 @@ enum ObStorageLogType // for test OB_LOG_TEST = 30000, - OB_LOG_TRANS_LITE = 40006, OB_LOG_ADD_PARTITION_TO_PG = 40007, OB_LOG_REMOVE_PARTITION_FROM_PG = 40008, OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG = 40009, @@ -199,9 +198,6 @@ class ObStorageLogTypeToString case OB_LOG_TEST: log_type_str = "TEST"; break; - case OB_LOG_TRANS_LITE: - log_type_str = "TRANS_LITE"; - break; case OB_LOG_ADD_PARTITION_TO_PG: log_type_str = "ADD_PARTITION_TO_PG"; break; @@ -374,7 +370,6 @@ class ObStorageLogTypeChecker || is_test_log(log_type) || is_freeze_log(log_type) || is_offline_partition_log(log_type) - || OB_LOG_TRANS_LITE == log_type || is_split_log(log_type) || is_start_membership_log(log_type) || is_checkpoint_log(log_type) diff --git a/src/storage/ob_storage_rpc.cpp b/src/storage/ob_storage_rpc.cpp index 4ccf6ce4e..68bc7fe6d 100644 --- a/src/storage/ob_storage_rpc.cpp +++ b/src/storage/ob_storage_rpc.cpp @@ -38,551 +38,7 @@ namespace obcall { -ObCopyMacroBlockArg::ObCopyMacroBlockArg() - : logic_macro_block_id_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyMacroBlockArg, - logic_macro_block_id_); - - -ObCopyMacroBlockListArg::ObCopyMacroBlockListArg() - : ls_id_(), - table_key_(), - arg_list_() -{ -} - - -bool ObCopyMacroBlockListArg::is_valid() const -{ - return true - && ls_id_.is_valid() - && table_key_.is_valid() - && arg_list_.count() > 0; -} - - -OB_SERIALIZE_MEMBER(ObCopyMacroBlockListArg, ls_id_, table_key_, arg_list_); - -ObCopyMacroBlockInfo::ObCopyMacroBlockInfo() - : logical_id_(), - data_type_(ObCopyMacroBlockDataType::MAX) -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyMacroBlockInfo, logical_id_, data_type_); - -ObCopyMacroBlockRangeArg::ObCopyMacroBlockRangeArg() - : ls_id_(), - table_key_(), - data_version_(0), - backfill_tx_scn_(SCN::min_scn()), - copy_macro_range_info_(), - need_check_seq_(false), - ls_rebuild_seq_(-1) -{ -} - - -bool ObCopyMacroBlockRangeArg::is_valid() const -{ - return ls_id_.is_valid() - && table_key_.is_valid() - && data_version_ >= 0 - && backfill_tx_scn_ >= SCN::min_scn() - && copy_macro_range_info_.is_valid() - && ((need_check_seq_ && ls_rebuild_seq_ >= 0) || !need_check_seq_); -} - - -OB_SERIALIZE_MEMBER(ObCopyMacroBlockRangeArg, ls_id_, table_key_, data_version_, - backfill_tx_scn_, copy_macro_range_info_, need_check_seq_, ls_rebuild_seq_, copy_macro_block_infos_); - -ObCopyMacroBlockHeader::ObCopyMacroBlockHeader() - : is_reuse_macro_block_(false), - occupy_size_(0), - data_type_(ObCopyMacroBlockDataType::MACRO_DATA) // default value for compat, previous version won't contain data_type_ and will pass macro data all the time -{ -} - -void ObCopyMacroBlockHeader::reset() -{ - is_reuse_macro_block_ = false; - occupy_size_ = 0; - data_type_ = ObCopyMacroBlockDataType::MACRO_DATA; -} - -OB_SERIALIZE_MEMBER(ObCopyMacroBlockHeader, is_reuse_macro_block_, occupy_size_, data_type_); - -ObCopyTabletInfoArg::ObCopyTabletInfoArg() - : ls_id_(), - tablet_id_list_(), - need_check_seq_(false), - ls_rebuild_seq_(-1), - is_only_copy_major_(false), - version_(OB_INVALID_ID) -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyTabletInfoArg, - ls_id_, tablet_id_list_, need_check_seq_, ls_rebuild_seq_, - is_only_copy_major_, version_); - -ObCopyTabletInfo::ObCopyTabletInfo() - : tablet_id_(), - status_(ObCopyTabletStatus::MAX_STATUS), - param_(), - data_size_(0), - version_(OB_INVALID_ID) -{ -} - -void ObCopyTabletInfo::reset() -{ - tablet_id_.reset(); - status_ = ObCopyTabletStatus::MAX_STATUS; - param_.reset(); - data_size_ = 0; - version_ = OB_INVALID_ID; -} - -bool ObCopyTabletInfo::is_valid() const -{ - return tablet_id_.is_valid() - && ObCopyTabletStatus::is_valid(status_) - && ((ObCopyTabletStatus::TABLET_EXIST == status_ && param_.is_valid() && data_size_ >= 0) - || ObCopyTabletStatus::TABLET_NOT_EXIST == status_) - && version_ != OB_INVALID_ID; -} - - -OB_SERIALIZE_MEMBER(ObCopyTabletInfo, tablet_id_, status_, param_, data_size_, version_); - -/******************ObCopyTabletSSTableInfoArg*********************/ -ObCopyTabletSSTableInfoArg::ObCopyTabletSSTableInfoArg() - : tablet_id_(), - max_major_sstable_snapshot_(0), - minor_sstable_scn_range_(), - ddl_sstable_scn_range_() -{ -} - -ObCopyTabletSSTableInfoArg::~ObCopyTabletSSTableInfoArg() -{ -} - -void ObCopyTabletSSTableInfoArg::reset() -{ - tablet_id_.reset(); - max_major_sstable_snapshot_ = 0; - minor_sstable_scn_range_.reset(); - ddl_sstable_scn_range_.reset(); -} - -bool ObCopyTabletSSTableInfoArg::is_valid() const -{ - return tablet_id_.is_valid() - && max_major_sstable_snapshot_ >= 0 - && minor_sstable_scn_range_.is_valid() - && ddl_sstable_scn_range_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObCopyTabletSSTableInfoArg, - tablet_id_, max_major_sstable_snapshot_, minor_sstable_scn_range_, ddl_sstable_scn_range_); - -ObCopyTabletsSSTableInfoArg::ObCopyTabletsSSTableInfoArg() - : ls_id_(), - need_check_seq_(false), - ls_rebuild_seq_(-1), - is_only_copy_major_(false), - tablet_sstable_info_arg_list_(), - version_(OB_INVALID_ID) -{ -} - -ObCopyTabletsSSTableInfoArg::~ObCopyTabletsSSTableInfoArg() -{ - reset(); -} - -void ObCopyTabletsSSTableInfoArg::reset() -{ - ls_id_.reset(); - need_check_seq_ = false; - ls_rebuild_seq_ = -1; - is_only_copy_major_ = false; - tablet_sstable_info_arg_list_.reset(); - version_ = OB_INVALID_ID; -} - - -OB_SERIALIZE_MEMBER(ObCopyTabletsSSTableInfoArg, - ls_id_, need_check_seq_, ls_rebuild_seq_, is_only_copy_major_, - tablet_sstable_info_arg_list_, version_); - - -ObCopyTabletSSTableInfo::ObCopyTabletSSTableInfo() - : tablet_id_(), - table_key_(), - param_() -{ -} - -void ObCopyTabletSSTableInfo::reset() -{ - tablet_id_.reset(); - table_key_.reset(); - param_.reset(); -} - -bool ObCopyTabletSSTableInfo::is_valid() const -{ - return tablet_id_.is_valid() - && table_key_.is_valid() - && param_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObCopyTabletSSTableInfo, - tablet_id_, table_key_, param_); - - -ObCopyLSInfoArg::ObCopyLSInfoArg() - : ls_id_(), - version_(OB_INVALID_ID) -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyLSInfoArg, - ls_id_, version_); - - -ObCopyLSInfo::ObCopyLSInfo() - : ls_meta_package_(), - tablet_id_array_(), - is_log_sync_(false), - version_(OB_INVALID_ID) -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyLSInfo, - ls_meta_package_, tablet_id_array_, is_log_sync_, version_); - -ObFetchLSMetaInfoArg::ObFetchLSMetaInfoArg() - : ls_id_(), - version_(OB_INVALID_ID) -{ -} - - - -OB_SERIALIZE_MEMBER(ObFetchLSMetaInfoArg, ls_id_, version_); - - -ObFetchLSMetaInfoResp::ObFetchLSMetaInfoResp() - : ls_meta_package_(), - version_(OB_INVALID_ID) -{ -} - - -bool ObFetchLSMetaInfoResp::is_valid() const -{ - return ls_meta_package_.is_valid() - && version_ != OB_INVALID_ID; -} - -OB_SERIALIZE_MEMBER(ObFetchLSMetaInfoResp, ls_meta_package_, version_); - -ObFetchLSMemberListArg::ObFetchLSMemberListArg() - : ls_id_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObFetchLSMemberListArg, ls_id_); - -ObCheckRestorePreconditionResult::ObCheckRestorePreconditionResult() - : required_disk_size_(0), - total_tablet_size_(0), - cluster_version_(0) -{ -} - -OB_SERIALIZE_MEMBER(ObCheckRestorePreconditionResult, required_disk_size_, total_tablet_size_, cluster_version_); - -ObRestoreCopyTabletInfoArg::ObRestoreCopyTabletInfoArg() - : tablet_id_list_() -{ -} - -ObRestoreCopySSTableMacroRangeInfoArg::ObRestoreCopySSTableMacroRangeInfoArg() - : tablet_id_(), - copy_table_key_array_(), - macro_range_max_marco_count_(0) -{ -} - -ObRestoreCopySSTableMacroRangeInfoArg::~ObRestoreCopySSTableMacroRangeInfoArg() -{ -} - -bool ObRestoreCopySSTableMacroRangeInfoArg::is_valid() const -{ - return tablet_id_.is_valid() - && copy_table_key_array_.count() > 0 - && macro_range_max_marco_count_ > 0; -} - -int ObRestoreCopySSTableMacroRangeInfoArg::assign(const ObRestoreCopySSTableMacroRangeInfoArg &arg) -{ - int ret = OB_SUCCESS; - if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("copy sstable macro range info arg is invalid", K(ret), K(arg)); - } else if (OB_FAIL(copy_table_key_array_.assign(arg.copy_table_key_array_))) { - LOG_WARN("failed to assign copy table key array", K(ret), K(arg)); - } else { - tablet_id_ = arg.tablet_id_; - macro_range_max_marco_count_ = arg.macro_range_max_marco_count_; - } - return ret; -} - -ObRestoreCopyMacroBlockRangeArg::ObRestoreCopyMacroBlockRangeArg() - : table_key_(), - data_version_(0), - backfill_tx_scn_(SCN::min_scn()), - copy_macro_range_info_(), - copy_macro_block_infos_() -{ -} - -bool ObRestoreCopyMacroBlockRangeArg::is_valid() const -{ - return table_key_.is_valid() - && data_version_ >= 0 - && backfill_tx_scn_.is_valid() - && copy_macro_range_info_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObRestoreCopyTabletInfoArg, tablet_id_list_); -OB_SERIALIZE_MEMBER(ObRestoreCopySSTableMacroRangeInfoArg, tablet_id_, copy_table_key_array_, macro_range_max_marco_count_); -OB_SERIALIZE_MEMBER(ObRestoreCopyMacroBlockRangeArg, table_key_, data_version_, backfill_tx_scn_, copy_macro_range_info_, copy_macro_block_infos_); - -ObFetchLSMemberListInfo::ObFetchLSMemberListInfo() - : member_list_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObFetchLSMemberListInfo, member_list_); - -ObFetchLSMemberAndLearnerListArg::ObFetchLSMemberAndLearnerListArg() - : ls_id_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObFetchLSMemberAndLearnerListArg, ls_id_); - -ObFetchLSMemberAndLearnerListInfo::ObFetchLSMemberAndLearnerListInfo() - : member_list_(), - learner_list_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObFetchLSMemberAndLearnerListInfo, member_list_, learner_list_); - -ObCopySSTableMacroRangeInfoArg::ObCopySSTableMacroRangeInfoArg() - : ls_id_(), - tablet_id_(), - copy_table_key_array_(), - macro_range_max_marco_count_(0), - need_check_seq_(false), - ls_rebuild_seq_(0) -{ -} - -ObCopySSTableMacroRangeInfoArg::~ObCopySSTableMacroRangeInfoArg() -{ -} - - -bool ObCopySSTableMacroRangeInfoArg::is_valid() const -{ - return true - && ls_id_.is_valid() - && tablet_id_.is_valid() - && copy_table_key_array_.count() > 0 - && macro_range_max_marco_count_ > 0; -} - -int ObCopySSTableMacroRangeInfoArg::assign(const ObCopySSTableMacroRangeInfoArg &arg) -{ - int ret = OB_SUCCESS; - if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("copy sstable macro range info arg is invalid", K(ret), K(arg)); - } else if (OB_FAIL(copy_table_key_array_.assign(arg.copy_table_key_array_))) { - LOG_WARN("failed to assign src table array", K(ret), K(arg)); - } else { - ls_id_ = arg.ls_id_; - tablet_id_ = arg.tablet_id_; - macro_range_max_marco_count_ = arg.macro_range_max_marco_count_; - need_check_seq_ = arg.need_check_seq_; - ls_rebuild_seq_ = arg.ls_rebuild_seq_; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObCopySSTableMacroRangeInfoArg, ls_id_, - tablet_id_, copy_table_key_array_, macro_range_max_marco_count_, - need_check_seq_, ls_rebuild_seq_); - -ObCopySSTableMacroRangeInfoHeader::ObCopySSTableMacroRangeInfoHeader() - : copy_table_key_(), - macro_range_count_(0) -{ -} - -ObCopySSTableMacroRangeInfoHeader::~ObCopySSTableMacroRangeInfoHeader() -{ -} - -bool ObCopySSTableMacroRangeInfoHeader::is_valid() const -{ - return copy_table_key_.is_valid() && macro_range_count_ >= 0; -} - -void ObCopySSTableMacroRangeInfoHeader::reset() -{ - copy_table_key_.reset(); - macro_range_count_ = 0; -} - -OB_SERIALIZE_MEMBER(ObCopySSTableMacroRangeInfoHeader, - copy_table_key_, macro_range_count_); - -ObCopyTabletSSTableHeader::ObCopyTabletSSTableHeader() - : tablet_id_(), - status_(ObCopyTabletStatus::MAX_STATUS), - sstable_count_(0), - tablet_meta_(), - version_(OB_INVALID_ID) -{ -} - -void ObCopyTabletSSTableHeader::reset() -{ - tablet_id_.reset(); - status_ = ObCopyTabletStatus::MAX_STATUS; - sstable_count_ = 0; - tablet_meta_.reset(); - version_ = OB_INVALID_ID; -} - -bool ObCopyTabletSSTableHeader::is_valid() const -{ - return tablet_id_.is_valid() - && ObCopyTabletStatus::is_valid(status_) - && sstable_count_ >= 0 - && ((ObCopyTabletStatus::TABLET_EXIST == status_ && tablet_meta_.is_valid()) - || ObCopyTabletStatus::TABLET_NOT_EXIST == status_) - && version_ != OB_INVALID_ID; -} - -OB_SERIALIZE_MEMBER(ObCopyTabletSSTableHeader, - tablet_id_, status_, sstable_count_, tablet_meta_, version_); - -ObNotifyRestoreTabletsArg::ObNotifyRestoreTabletsArg() - : ls_id_(), tablet_id_array_(), restore_status_(), leader_proposal_id_(0) -{ -} - - -bool ObNotifyRestoreTabletsArg::is_valid() const -{ - return ls_id_.is_valid() - && restore_status_.is_valid() - && leader_proposal_id_ > 0; -} - -OB_SERIALIZE_MEMBER(ObNotifyRestoreTabletsArg, ls_id_, tablet_id_array_, restore_status_, leader_proposal_id_); - - -ObNotifyRestoreTabletsResp::ObNotifyRestoreTabletsResp() - : ls_id_(), restore_status_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObNotifyRestoreTabletsResp, ls_id_, restore_status_); - - -ObInquireRestoreArg::ObInquireRestoreArg() - : ls_id_(), restore_status_() -{ -} - - -bool ObInquireRestoreArg::is_valid() const -{ - return ls_id_.is_valid() - && restore_status_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObInquireRestoreArg, ls_id_, restore_status_); - -ObInquireRestoreResp::ObInquireRestoreResp() - : ls_id_(), is_leader_(false), restore_status_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObInquireRestoreResp, ls_id_, is_leader_, restore_status_); - - -ObRestoreUpdateLSMetaArg::ObRestoreUpdateLSMetaArg() - : ls_meta_package_() -{ -} - - -bool ObRestoreUpdateLSMetaArg::is_valid() const -{ - return ls_meta_package_.is_valid(); -} - -OB_SERIALIZE_MEMBER(ObRestoreUpdateLSMetaArg, ls_meta_package_); - - -ObCopyLSViewArg::ObCopyLSViewArg() - : ls_id_() -{ -} - - - -OB_SERIALIZE_MEMBER(ObCopyLSViewArg, ls_id_); +// Dead obcall copy/restore/LS RPC payload implementations removed. // ObStorageStreamRpcP<> obcall stream-RPC processor impls deleted — dead in seekdb. diff --git a/src/storage/ob_storage_rpc.h b/src/storage/ob_storage_rpc.h index ac6559b16..2c1fb1379 100644 --- a/src/storage/ob_storage_rpc.h +++ b/src/storage/ob_storage_rpc.h @@ -22,13 +22,11 @@ #include "storage/tx/ob_tx_result_struct.h" #include "lib/utility/ob_unify_serialize.h" #include "rpc/frame/ob_result_code.h" -#include "common/ob_member.h" #include "storage/ob_storage_struct.h" #include "observer/ob_server_struct.h" #include "storage/ob_storage_schema.h" #include "storage/ob_storage_ha_struct.h" #include "storage/blocksstable/ob_sstable_meta.h" -#include "storage/ls/ob_ls_meta_package.h" #include "tablet/ob_tablet_meta.h" #include "share/ls/ob_ls_restore_status.h" #include "storage/lob/ob_lob_rpc_struct.h" @@ -47,458 +45,7 @@ class ObICopySSTableMacroRangeObProducer; namespace obcall { -struct ObCopyMacroBlockArg -{ - OB_UNIS_VERSION(2); -public: - ObCopyMacroBlockArg(); - virtual ~ObCopyMacroBlockArg() {} - TO_STRING_KV(K_(logic_macro_block_id)); - blocksstable::ObLogicMacroBlockId logic_macro_block_id_; -}; - -struct ObCopyMacroBlockListArg -{ - OB_UNIS_VERSION(2); -public: - ObCopyMacroBlockListArg(); - virtual ~ObCopyMacroBlockListArg() {} - - bool is_valid() const; - - TO_STRING_KV(K_(ls_id), K_(table_key), "arg_count", arg_list_.count()); - share::ObLSID ls_id_; - storage::ObITable::TableKey table_key_; - common::ObSArray arg_list_; -}; - -enum ObCopyMacroBlockDataType { - MACRO_DATA = 0, - MACRO_META_ROW = 1, - MAX -}; - -struct ObCopyMacroBlockInfo final -{ - OB_UNIS_VERSION(1); -public: - ObCopyMacroBlockInfo(); - ~ObCopyMacroBlockInfo() {} - - TO_STRING_KV(K_(logical_id), K_(data_type)); -public: - ObLogicMacroBlockId logical_id_; - ObCopyMacroBlockDataType data_type_; -}; - -struct ObCopyMacroBlockRangeArg final -{ - OB_UNIS_VERSION(2); -public: - ObCopyMacroBlockRangeArg(); - ~ObCopyMacroBlockRangeArg() {} - - bool is_valid() const; - TO_STRING_KV(K_(ls_id), K_(table_key), K_(data_version), K_(backfill_tx_scn), K_(copy_macro_range_info)); - - share::ObLSID ls_id_; - storage::ObITable::TableKey table_key_; - int64_t data_version_; - share::SCN backfill_tx_scn_; - storage::ObCopyMacroRangeInfo copy_macro_range_info_; - bool need_check_seq_; - int64_t ls_rebuild_seq_; - ObSArray copy_macro_block_infos_; - DISALLOW_COPY_AND_ASSIGN(ObCopyMacroBlockRangeArg); -}; - -// Simplified version for single-replica scenario (no tenant/ls_id needed) -struct ObRestoreCopyMacroBlockRangeArg final -{ - OB_UNIS_VERSION(1); -public: - ObRestoreCopyMacroBlockRangeArg(); - ~ObRestoreCopyMacroBlockRangeArg() {} - - bool is_valid() const; - TO_STRING_KV(K_(table_key), K_(data_version), K_(backfill_tx_scn), K_(copy_macro_range_info)); - - storage::ObITable::TableKey table_key_; - int64_t data_version_; - share::SCN backfill_tx_scn_; - storage::ObCopyMacroRangeInfo copy_macro_range_info_; - ObSArray copy_macro_block_infos_; - DISALLOW_COPY_AND_ASSIGN(ObRestoreCopyMacroBlockRangeArg); -}; - -struct ObCopyMacroBlockHeader -{ - OB_UNIS_VERSION(2); -public: - ObCopyMacroBlockHeader(); - virtual ~ObCopyMacroBlockHeader() {} - void reset(); - - TO_STRING_KV(K_(is_reuse_macro_block), K_(occupy_size), K_(data_type)); - bool is_reuse_macro_block_; - int64_t occupy_size_; - ObCopyMacroBlockDataType data_type_; // FARM COMPAT WHITELIST FOR data_type_: renamed -}; - -struct ObCopyTabletInfoArg -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletInfoArg(); - virtual ~ObCopyTabletInfoArg() {} - - TO_STRING_KV(K_(ls_id), K_(tablet_id_list), K_(need_check_seq), - K_(ls_rebuild_seq), K_(is_only_copy_major), K_(version)); - share::ObLSID ls_id_; - common::ObSArray tablet_id_list_; - bool need_check_seq_; - int64_t ls_rebuild_seq_; - bool is_only_copy_major_; - uint64_t version_; -}; - -struct ObRestoreCopyTabletInfoArg final -{ - OB_UNIS_VERSION(1); -public: - ObRestoreCopyTabletInfoArg(); - virtual ~ObRestoreCopyTabletInfoArg() {} - - bool is_valid() const { return true; } - TO_STRING_KV(K_(tablet_id_list)); - common::ObSArray tablet_id_list_; -}; - -struct ObCopyTabletInfo -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletInfo(); - virtual ~ObCopyTabletInfo() {} - void reset(); - bool is_valid() const; - TO_STRING_KV(K_(tablet_id), K_(status), K_(param), K_(data_size), K_(version)); - - common::ObTabletID tablet_id_; - storage::ObCopyTabletStatus::STATUS status_; - storage::ObMigrationTabletParam param_; - int64_t data_size_; //need copy ssttablet size - uint64_t version_; -}; - -struct ObCopyTabletSSTableInfoArg final -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletSSTableInfoArg(); - ~ObCopyTabletSSTableInfoArg(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(tablet_id), K_(max_major_sstable_snapshot), K_(minor_sstable_scn_range), - K_(ddl_sstable_scn_range)); - - common::ObTabletID tablet_id_; - int64_t max_major_sstable_snapshot_; - share::ObScnRange minor_sstable_scn_range_; - share::ObScnRange ddl_sstable_scn_range_; -}; - -struct ObCopyTabletsSSTableInfoArg final -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletsSSTableInfoArg(); - ~ObCopyTabletsSSTableInfoArg(); - void reset(); - int assign(const ObCopyTabletsSSTableInfoArg &arg); - - TO_STRING_KV(K_(ls_id), K_(need_check_seq), - K_(ls_rebuild_seq), K_(is_only_copy_major), K_(tablet_sstable_info_arg_list), - K_(version)); - - share::ObLSID ls_id_; - bool need_check_seq_; - int64_t ls_rebuild_seq_; - bool is_only_copy_major_; - common::ObSArray tablet_sstable_info_arg_list_; - uint64_t version_; - DISALLOW_COPY_AND_ASSIGN(ObCopyTabletsSSTableInfoArg); -}; - -struct ObCopyTabletSSTableInfo -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletSSTableInfo(); - virtual ~ObCopyTabletSSTableInfo() {} - void reset(); - int assign(const ObCopyTabletSSTableInfo &info); - bool is_valid() const; - TO_STRING_KV(K_(tablet_id), K_(table_key), K_(param)); - - common::ObTabletID tablet_id_; - storage::ObITable::TableKey table_key_; - blocksstable::ObMigrationSSTableParam param_; -}; - -struct ObCopyLSInfoArg -{ - OB_UNIS_VERSION(2); -public: - ObCopyLSInfoArg(); - virtual ~ObCopyLSInfoArg() {} - - TO_STRING_KV(K_(ls_id)); - share::ObLSID ls_id_; - uint64_t version_; -}; - -struct ObCopyLSInfo -{ - OB_UNIS_VERSION(2); -public: - ObCopyLSInfo(); - virtual ~ObCopyLSInfo() {} - - TO_STRING_KV(K_(ls_meta_package), K_(tablet_id_array), K_(is_log_sync), K_(version)); - storage::ObLSMetaPackage ls_meta_package_; - common::ObSArray tablet_id_array_; - bool is_log_sync_; - uint64_t version_; -}; - -struct ObFetchLSMetaInfoArg -{ - OB_UNIS_VERSION(2); -public: - ObFetchLSMetaInfoArg(); - virtual ~ObFetchLSMetaInfoArg() {} - - TO_STRING_KV(K_(ls_id), K_(version)); - share::ObLSID ls_id_; - uint64_t version_; -}; - -struct ObFetchLSMetaInfoResp -{ - OB_UNIS_VERSION(3); -public: - ObFetchLSMetaInfoResp(); - virtual ~ObFetchLSMetaInfoResp() {} - bool is_valid() const; - - TO_STRING_KV(K_(ls_meta_package), K_(version)); - storage::ObLSMetaPackage ls_meta_package_; - uint64_t version_; -}; - -struct ObFetchLSMemberListArg -{ - OB_UNIS_VERSION(2); -public: - ObFetchLSMemberListArg(); - virtual ~ObFetchLSMemberListArg() {} - - TO_STRING_KV(K_(ls_id)); - share::ObLSID ls_id_; -}; - -struct ObCheckRestorePreconditionResult final -{ - OB_UNIS_VERSION(1); -public: - ObCheckRestorePreconditionResult(); - virtual ~ObCheckRestorePreconditionResult() {} - - TO_STRING_KV(K_(required_disk_size), K_(total_tablet_size), K_(cluster_version)); - int64_t required_disk_size_; // From ls_info.required_data_disk_size_ - int64_t total_tablet_size_; // Sum of all tablet sizes - uint64_t cluster_version_; -}; - -struct ObFetchLSMemberListInfo -{ - OB_UNIS_VERSION(2); -public: - ObFetchLSMemberListInfo(); - virtual ~ObFetchLSMemberListInfo() {} - - TO_STRING_KV(K_(member_list)); - common::ObMemberList member_list_; -}; - -struct ObFetchLSMemberAndLearnerListArg -{ - OB_UNIS_VERSION(2); -public: - ObFetchLSMemberAndLearnerListArg(); - virtual ~ObFetchLSMemberAndLearnerListArg() {} - - TO_STRING_KV(K_(ls_id)); - share::ObLSID ls_id_; -}; - -struct ObFetchLSMemberAndLearnerListInfo -{ - OB_UNIS_VERSION(2); -public: - ObFetchLSMemberAndLearnerListInfo(); - virtual ~ObFetchLSMemberAndLearnerListInfo() {} - - TO_STRING_KV(K_(member_list), K_(learner_list)); - common::ObMemberList member_list_; - common::GlobalLearnerList learner_list_; -}; - -struct ObCopySSTableMacroRangeInfoArg final -{ - OB_UNIS_VERSION(2); -public: - ObCopySSTableMacroRangeInfoArg(); - ~ObCopySSTableMacroRangeInfoArg(); - bool is_valid() const; - int assign(const ObCopySSTableMacroRangeInfoArg &arg); - - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(copy_table_key_array), K_(macro_range_max_marco_count)); - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObSArray copy_table_key_array_; - int64_t macro_range_max_marco_count_; - bool need_check_seq_; - int64_t ls_rebuild_seq_; - DISALLOW_COPY_AND_ASSIGN(ObCopySSTableMacroRangeInfoArg); -}; - -struct ObRestoreCopySSTableMacroRangeInfoArg final -{ - OB_UNIS_VERSION(1); -public: - ObRestoreCopySSTableMacroRangeInfoArg(); - ~ObRestoreCopySSTableMacroRangeInfoArg(); - bool is_valid() const; - int assign(const ObRestoreCopySSTableMacroRangeInfoArg &arg); - - TO_STRING_KV(K_(tablet_id), K_(copy_table_key_array), K_(macro_range_max_marco_count)); - common::ObTabletID tablet_id_; - common::ObSArray copy_table_key_array_; - int64_t macro_range_max_marco_count_; - DISALLOW_COPY_AND_ASSIGN(ObRestoreCopySSTableMacroRangeInfoArg); -}; - -struct ObCopySSTableMacroRangeInfoHeader final -{ - OB_UNIS_VERSION(2); -public: - ObCopySSTableMacroRangeInfoHeader(); - ~ObCopySSTableMacroRangeInfoHeader(); - bool is_valid() const; - void reset(); - TO_STRING_KV(K_(copy_table_key), K_(macro_range_count)); - - ObITable::TableKey copy_table_key_; - int64_t macro_range_count_; -}; - -struct ObCopyTabletSSTableHeader final -{ - OB_UNIS_VERSION(2); -public: - ObCopyTabletSSTableHeader(); - ~ObCopyTabletSSTableHeader() {} - void reset(); - bool is_valid() const; - TO_STRING_KV(K_(tablet_id), K_(status), K_(sstable_count), K_(tablet_meta), K_(version)); - - common::ObTabletID tablet_id_; - storage::ObCopyTabletStatus::STATUS status_; - int64_t sstable_count_; - ObMigrationTabletParam tablet_meta_; - uint64_t version_; // source observer version. -}; - -// Leader notify follower to restore some tablets. -struct ObNotifyRestoreTabletsArg -{ - OB_UNIS_VERSION(2); -public: - ObNotifyRestoreTabletsArg(); - virtual ~ObNotifyRestoreTabletsArg() {} - bool is_valid() const; - - TO_STRING_KV(K_(ls_id), K_(tablet_id_array), K_(restore_status), K_(leader_proposal_id)); - share::ObLSID ls_id_; - common::ObSArray tablet_id_array_; - share::ObLSRestoreStatus restore_status_; // indicate the type of data to restore - int64_t leader_proposal_id_; -}; - -struct ObNotifyRestoreTabletsResp -{ - OB_UNIS_VERSION(2); -public: - ObNotifyRestoreTabletsResp(); - virtual ~ObNotifyRestoreTabletsResp() {} - - TO_STRING_KV(K_(ls_id), K_(restore_status)); - share::ObLSID ls_id_; - ObRestoreStatus restore_status_; // restore status -}; - - -struct ObInquireRestoreResp -{ - OB_UNIS_VERSION(2); -public: - ObInquireRestoreResp(); - virtual ~ObInquireRestoreResp() {} - - TO_STRING_KV(K_(ls_id), K_(is_leader), K_(restore_status)); - share::ObLSID ls_id_; - bool is_leader_; - ObRestoreStatus restore_status_; // leader restore status -}; - -struct ObInquireRestoreArg -{ - OB_UNIS_VERSION(2); -public: - ObInquireRestoreArg(); - virtual ~ObInquireRestoreArg() {} - bool is_valid() const; - - TO_STRING_KV(K_(ls_id), K_(restore_status)); - share::ObLSID ls_id_; - share::ObLSRestoreStatus restore_status_; // restore status -}; - -struct ObRestoreUpdateLSMetaArg -{ - OB_UNIS_VERSION(2); -public: - ObRestoreUpdateLSMetaArg(); - virtual ~ObRestoreUpdateLSMetaArg() {} - bool is_valid() const; - - TO_STRING_KV(K_(ls_meta_package)); - storage::ObLSMetaPackage ls_meta_package_; -}; - -// Fetch ls meta and all tablet metas by stream reader. -struct ObCopyLSViewArg final -{ - OB_UNIS_VERSION(1); -public: - ObCopyLSViewArg(); - ~ObCopyLSViewArg() {} - - TO_STRING_KV(K_(ls_id)); - share::ObLSID ls_id_; -}; +// Dead obcall copy/restore/LS RPC payloads removed for single-LS seekdb. // Legacy shared-storage migrate-warmup obcall RPC arg/result structs removed // (ObGetMicroBlockCacheInfo{Arg,Res}, ObGetMigrationCacheJobInfo{Arg,Res}, @@ -568,7 +115,7 @@ class ObStorageRpc: public ObIStorageRpc common::ObAddr self_; }; -// ObStorageStreamRpcReader (obcall stream-RPC reader template) deleted — dead in seekdb. +// ObStorageStreamRpcReader (obcall stream-RPC reader template) deleted - dead in seekdb. } // storage } // oceanbase diff --git a/src/storage/ob_storage_rpc_arg.cpp b/src/storage/ob_storage_rpc_arg.cpp index 550c76fdd..01a1295fb 100644 --- a/src/storage/ob_storage_rpc_arg.cpp +++ b/src/storage/ob_storage_rpc_arg.cpp @@ -14,201 +14,17 @@ * limitations under the License. */ #define USING_LOG_PREFIX RPC -#include "logservice/logrpc/ob_log_rpc_arg.h" #include "storage/ob_storage_rpc_arg.h" -#include "storage/tx/ob_trans_service.h" // used by IncCommitLogArg::release -#include "share/rc/ob_tenant_base.h" // MTL_WITH_CHECK namespace oceanbase { namespace obcall { -using namespace oceanbase::transaction; using namespace oceanbase::storage; OB_SERIALIZE_MEMBER(ObBatchGetTabletBindingRes, binding_datas_); OB_SERIALIZE_MEMBER(ObBatchGetTabletSplitRes, split_datas_); -OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLCommitLogArg, ls_id_, table_key_, start_scn_, - table_id_, execution_id_, ddl_task_id_); - #ifdef OB_BUILD_SHARED_STORAGE -OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLFinishLogArg, log_info_); -#endif - -OB_DEF_SERIALIZE(ObRegisterTxDataArg) -{ - int ret = OB_SUCCESS; - OB_UNIS_ENCODE(*tx_desc_); - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_); - return ret; -} - -OB_DEF_DESERIALIZE(ObRegisterTxDataArg) -{ - int ret = OB_SUCCESS; - if (OB_SUCC(ret)) { - ObTransService *tx_svc = MTL_WITH_CHECK(ObTransService *); - if (OB_ISNULL(tx_svc)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null tx service ptr", KR(ret)); - } else if (OB_FAIL(tx_svc->acquire_tx(buf, data_len, pos, tx_desc_))) { - LOG_WARN("acquire tx by deserialize fail", K(data_len), K(pos), KR(ret)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_); - LOG_INFO("deserialize txDesc from session", KPC_(tx_desc), KPC(this)); - } - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObRegisterTxDataArg) -{ - int64_t len = 0; - OB_UNIS_ADD_LEN(*tx_desc_); - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_); - return len; -} -int ObRegisterTxDataArg::init(const ObTxDesc &tx_desc, - const ObLSID &ls_id, - const ObTxDataSourceType &type, - const ObString &buf, - const transaction::ObTxSEQ seq_no, - const int64_t base_request_id, - const transaction::ObRegisterMdsFlag ®ister_flag) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!tx_desc.is_valid() || !ls_id.is_valid() - || type == ObTxDataSourceType::UNKNOWN || !seq_no.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tx_desc), K(ls_id), K(type), K(seq_no)); - } else { - tx_desc_ = const_cast(&tx_desc); - ls_id_ = ls_id; - type_ = type; - buf_ = buf; - seq_no_ = seq_no; - request_id_ = base_request_id; - register_flag_ = register_flag; - } - return ret; -} - -void ObRegisterTxDataArg::inc_request_id(const int64_t base_request_id) -{ - if (-1 != base_request_id) { - request_id_ = base_request_id + 1; - } else { - request_id_++; - } -} - -int ObRpcRemoteWriteDDLCommitLogArg::init(const share::ObLSID &ls_id, - const storage::ObITable::TableKey &table_key, - const SCN &start_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !table_key.is_valid() || !start_scn.is_valid_and_not_min())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablet id is not valid", K(ret), K(ls_id), K(table_key), K(start_scn)); - } else { - ls_id_ = ls_id; - table_key_ = table_key; - start_scn_ = start_scn; - } - return ret; -} - -#ifdef OB_BUILD_SHARED_STORAGE -int ObRpcRemoteWriteDDLFinishLogArg::init(const storage::ObDDLFinishLogInfo &log) -{ - int ret = OB_SUCCESS; - if (!log.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(log)); - } else if (OB_FAIL(log_info_.assign(log))) { - LOG_WARN("fail to get assign log", K(ret), K(log)); - } - return ret; -} -#endif -int ObRpcRemoteWriteDDLRedoLogArg::init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, - const int64_t task_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(task_id == 0 || !ls_id.is_valid() || !redo_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("args are not valid", K(ret), K(task_id), K(ls_id), K(redo_info)); - } else { - ls_id_ = ls_id; - redo_info_ = redo_info; - task_id_ = task_id; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLRedoLogArg, ls_id_, redo_info_, task_id_); - -int ObRpcRemoteWriteDDLIncCommitLogArg::init(const share::ObLSID &ls_id, - const common::ObTabletID tablet_id, - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || - OB_ISNULL(tx_desc) || !tx_desc->is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablet id is not valid", K(ret), K(ls_id), K(tablet_id), K(lob_meta_tablet_id), KPC(tx_desc)); - } else if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } else { - ls_id_ = ls_id; - tablet_id_ = tablet_id; - lob_meta_tablet_id_ = lob_meta_tablet_id; - tx_desc_ = tx_desc; - } - return ret; -} - -int ObRpcRemoteWriteDDLIncCommitLogArg::release() -{ - int ret = OB_SUCCESS; - if (tx_desc_ != nullptr && need_release_) { - ObTransService *tx_svc = MTL_WITH_CHECK(ObTransService *); - if (OB_ISNULL(tx_svc)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null tx service ptr", K(ret)); - } else if (OB_FAIL(tx_svc->release_tx(*tx_desc_))) { - LOG_WARN("release tx fail", K(ret)); - } else { - need_release_ = false; - tx_desc_ = nullptr; - } - } - - return ret; -} - -ObRpcRemoteWriteDDLRedoLogArg::ObRpcRemoteWriteDDLRedoLogArg() - : ls_id_(), redo_info_(), task_id_(0) -{} - - - -ObRpcRemoteWriteDDLCommitLogArg::ObRpcRemoteWriteDDLCommitLogArg() - : ls_id_(), table_key_(), start_scn_(SCN::min_scn()), - table_id_(0), execution_id_(-1), ddl_task_id_(0) -{} - - - -#ifdef OB_BUILD_SHARED_STORAGE -ObRpcRemoteWriteDDLFinishLogArg::ObRpcRemoteWriteDDLFinishLogArg() - : log_info_() -{} - - - OB_SERIALIZE_MEMBER(ObGetSSMacroBlockArg, tenant_id_, macro_id_, offset_, size_); OB_DEF_SERIALIZE(ObGetSSMacroBlockResult) { @@ -219,129 +35,6 @@ OB_DEF_SERIALIZE(ObGetSSMacroBlockResult) } #endif - -ObRpcRemoteWriteDDLIncCommitLogArg::ObRpcRemoteWriteDDLIncCommitLogArg() - : ls_id_(), tablet_id_(), lob_meta_tablet_id_(), tx_desc_(nullptr), need_release_(false) -{} - -ObRpcRemoteWriteDDLIncCommitLogArg::~ObRpcRemoteWriteDDLIncCommitLogArg() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } -} - -OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLIncCommitLogRes, tx_result_); - -OB_DEF_SERIALIZE(ObRpcRemoteWriteDDLIncCommitLogArg) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (OB_SUCC(ret)) { - if (OB_ISNULL(tx_desc_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tx_desc_ is nullptr", K(ret)); - } else { - LST_DO_CODE(OB_UNIS_ENCODE, *tx_desc_); - } - } - return ret; -} - -OB_DEF_DESERIALIZE(ObRpcRemoteWriteDDLIncCommitLogArg) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (OB_SUCC(ret)) { - ObTransService *tx_svc = nullptr; - if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } else if (OB_ISNULL(tx_svc = MTL_WITH_CHECK(ObTransService *))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null tx service ptr", KR(ret)); - } else if (OB_FAIL(tx_svc->acquire_tx(buf, data_len, pos, tx_desc_))) { - LOG_WARN("acquire tx by deserialize fail", K(data_len), K(pos), KR(ret)); - } else { - need_release_ = true; - } - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObRpcRemoteWriteDDLIncCommitLogArg) -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (tx_desc_ != nullptr) { - LST_DO_CODE(OB_UNIS_ADD_LEN, *tx_desc_); - } - return len; -} -ObRegisterTxDataArg::ObRegisterTxDataArg() - : tx_desc_(nullptr), - ls_id_(), - type_(transaction::ObTxDataSourceType::UNKNOWN), - buf_(), - seq_no_(), - request_id_(0), - register_flag_() -{ -} -int ObFetchStableMemberListInfo::init(const common::ObMemberList &member_list, const palf::LogConfigVersion &config_version) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!member_list.is_valid() || !config_version.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", KR(ret), K(member_list), K(config_version)); - } else if (OB_FAIL(member_list_.deep_copy(member_list))) { - LOG_WARN("fail to assign memberlist", KR(ret), K(member_list)); - } else if (OB_FALSE_IT(config_version_ = config_version)) { - } - return ret; -} -int ObLSAccessModeInfo::init(const share::ObLSID &ls_idd, - const int64_t mode_version, - const palf::AccessMode &access_mode, - const share::SCN &ref_scn, - const share::SCN &sys_ls_end_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_idd.is_valid() - || palf::INVALID_PROPOSAL_ID == mode_version - || palf::AccessMode::INVALID_ACCESS_MODE == access_mode)) { - ret = OB_INVALID_ARGUMENT; - SHARE_LOG(WARN, "invalid argument", KR(ret), K(ls_idd), - K(mode_version), K(access_mode)); - } else { - ls_id_ = ls_idd; - mode_version_ = mode_version; - access_mode_ = access_mode; - ref_scn_ = ref_scn; - sys_ls_end_scn_ = sys_ls_end_scn; - } - return ret; -} -int ObLSAccessModeInfo::assign(const ObLSAccessModeInfo &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - ls_id_ = other.ls_id_; - mode_version_ = other.mode_version_; - access_mode_ = other.access_mode_; - ref_scn_ = other.ref_scn_; - addr_ = other.addr_; - sys_ls_end_scn_ = other.sys_ls_end_scn_; - } - return ret; -} -bool ObLSAccessModeInfo::is_valid() const -{ - return true - && ls_id_.is_valid() - && palf::INVALID_PROPOSAL_ID != mode_version_ - && palf::AccessMode::INVALID_ACCESS_MODE != access_mode_; -} #ifdef OB_BUILD_SHARED_STORAGE OB_DEF_DESERIALIZE(ObGetSSMacroBlockResult) { @@ -415,32 +108,31 @@ OB_SERIALIZE_MEMBER(ObSetSSCkptCompressorArg, tenant_id_, block_type_, compresso OB_SERIALIZE_MEMBER(ObSetSSCacheSizeRatioArg, tenant_id_, micro_cache_size_ratio_, macro_cache_size_ratio_); #endif #ifdef OB_BUILD_SHARED_STORAGE -OB_SERIALIZE_MEMBER(ObLSSyncHotMicroKeyArg, tenant_id_, ls_id_, leader_addr_, micro_keys_); -int ObLSSyncHotMicroKeyArg::assign(const ObLSSyncHotMicroKeyArg &other) +OB_SERIALIZE_MEMBER(ObSyncHotMicroKeyArg, tenant_id_, leader_addr_, micro_keys_); +int ObSyncHotMicroKeyArg::assign(const ObSyncHotMicroKeyArg &other) { int ret = OB_SUCCESS; if (OB_FAIL(micro_keys_.assign(other.micro_keys_))) { LOG_WARN("micro_keys_ assign failed", KR(ret)); } else { tenant_id_ = other.tenant_id_; - ls_id_ = other.ls_id_; leader_addr_ = other.leader_addr_; } return ret; } -bool ObLSSyncHotMicroKeyArg::is_valid() const +bool ObSyncHotMicroKeyArg::is_valid() const { - return is_valid_tenant_id(tenant_id_) && (micro_keys_.count() > 0) && (ls_id_ != ObLSID::INVALID_LS_ID) && leader_addr_.is_valid(); + return is_valid_tenant_id(tenant_id_) && (micro_keys_.count() > 0) && leader_addr_.is_valid(); } #endif OB_DEF_SERIALIZE(ObDDLBuildSingleReplicaRequestArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, source_tablet_id_, dest_tablet_id_, + LST_DO_CODE(OB_UNIS_ENCODE, source_tablet_id_, dest_tablet_id_, source_table_id_, dest_schema_id_, schema_version_, snapshot_version_, ddl_type_, task_id_, execution_id_, parallelism_, tablet_task_id_, data_format_version_, - dest_ls_id_, dest_schema_version_, + dest_schema_version_, compaction_scn_, can_reuse_macro_block_, split_sstable_type_, lob_col_idxs_, parallel_datum_rowkey_list_, is_no_logging_, min_split_start_scn_); @@ -449,13 +141,13 @@ OB_DEF_SERIALIZE(ObDDLBuildSingleReplicaRequestArg) OB_DEF_DESERIALIZE(ObDDLBuildSingleReplicaRequestArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, source_tablet_id_, dest_tablet_id_, + LST_DO_CODE(OB_UNIS_DECODE, source_tablet_id_, dest_tablet_id_, source_table_id_, dest_schema_id_, schema_version_, snapshot_version_, ddl_type_, task_id_, execution_id_, parallelism_, tablet_task_id_, data_format_version_, - dest_ls_id_, dest_schema_version_, + dest_schema_version_, compaction_scn_, can_reuse_macro_block_, split_sstable_type_, lob_col_idxs_); - if (FAILEDx(ObSplitUtil::deserializ_parallel_datum_rowkey( + if (FAILEDx(share::ObSplitUtil::deserializ_parallel_datum_rowkey( rowkey_allocator_, buf, data_len, pos, parallel_datum_rowkey_list_))) { LOG_WARN("deserialzie parallel info failed", K(ret)); } @@ -468,10 +160,10 @@ OB_DEF_DESERIALIZE(ObDDLBuildSingleReplicaRequestArg) OB_DEF_SERIALIZE_SIZE(ObDDLBuildSingleReplicaRequestArg) { int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, source_tablet_id_, dest_tablet_id_, + LST_DO_CODE(OB_UNIS_ADD_LEN, source_tablet_id_, dest_tablet_id_, source_table_id_, dest_schema_id_, schema_version_, snapshot_version_, ddl_type_, task_id_, execution_id_, parallelism_, tablet_task_id_, data_format_version_, - dest_ls_id_, dest_schema_version_, + dest_schema_version_, compaction_scn_, can_reuse_macro_block_, split_sstable_type_, lob_col_idxs_, parallel_datum_rowkey_list_, is_no_logging_, min_split_start_scn_); @@ -479,11 +171,11 @@ OB_DEF_SERIALIZE_SIZE(ObDDLBuildSingleReplicaRequestArg) } bool ObDDLBuildSingleReplicaRequestArg::is_valid() const { - bool is_valid = ls_id_.is_valid() && source_tablet_id_.is_valid() && dest_tablet_id_.is_valid() + bool is_valid = source_tablet_id_.is_valid() && dest_tablet_id_.is_valid() && OB_INVALID_ID != source_table_id_ && OB_INVALID_ID != dest_schema_id_ && schema_version_ > 0 && snapshot_version_ > 0 && task_id_ > 0 && parallelism_ > 0 && tablet_task_id_ > 0 && data_format_version_ > 0 - && dest_ls_id_.is_valid() && dest_schema_version_ > 0; + && dest_schema_version_ > 0; return is_valid; } int ObDDLBuildSingleReplicaRequestArg::assign(const ObDDLBuildSingleReplicaRequestArg &other) @@ -497,8 +189,6 @@ int ObDDLBuildSingleReplicaRequestArg::assign(const ObDDLBuildSingleReplicaReque } else if (OB_FAIL(parallel_datum_rowkey_list_.assign(other.parallel_datum_rowkey_list_))) { // shallow copy. LOG_WARN("assign failed", K(ret)); } else { - ls_id_ = other.ls_id_; - dest_ls_id_ = other.dest_ls_id_; source_tablet_id_ = other.source_tablet_id_; dest_tablet_id_ = other.dest_tablet_id_; source_table_id_ = other.source_table_id_; @@ -521,7 +211,7 @@ int ObDDLBuildSingleReplicaRequestArg::assign(const ObDDLBuildSingleReplicaReque return ret; } OB_SERIALIZE_MEMBER(ObDDLBuildSingleReplicaRequestResult, ret_code_, row_inserted_, row_scanned_, physical_row_count_); -OB_SERIALIZE_MEMBER(ObPrepareSplitRangesArg, ls_id_, tablet_id_, +OB_SERIALIZE_MEMBER(ObPrepareSplitRangesArg, tablet_id_, user_parallelism_, schema_tablet_size_, ddl_type_); OB_DEF_SERIALIZE(ObPrepareSplitRangesRes) { @@ -532,7 +222,7 @@ OB_DEF_SERIALIZE(ObPrepareSplitRangesRes) OB_DEF_DESERIALIZE(ObPrepareSplitRangesRes) { int ret = OB_SUCCESS; - if (OB_FAIL(ObSplitUtil::deserializ_parallel_datum_rowkey( + if (OB_FAIL(share::ObSplitUtil::deserializ_parallel_datum_rowkey( rowkey_allocator_, buf, data_len, pos, parallel_datum_rowkey_list_))) { LOG_WARN("deserialzie parallel info failed", K(ret)); } @@ -546,7 +236,7 @@ OB_DEF_SERIALIZE_SIZE(ObPrepareSplitRangesRes) } bool ObTabletSplitArg::is_valid() const { - bool is_valid = ls_id_.is_valid() && OB_INVALID_ID != table_id_ + bool is_valid = OB_INVALID_ID != table_id_ && schema_version_ > 0 && task_id_ > 0 && source_tablet_id_.is_valid() && dest_tablets_id_.count() > 0 && compaction_scn_ > 0 @@ -571,7 +261,6 @@ int ObTabletSplitArg::assign(const ObTabletSplitArg &other) } else if (OB_FAIL(parallel_datum_rowkey_list_.assign(other.parallel_datum_rowkey_list_))) { // shallow cpy. LOG_WARN("assign failed", K(ret), K(other)); } else { - ls_id_ = other.ls_id_; table_id_ = other.table_id_; lob_table_id_ = other.lob_table_id_; schema_version_ = other.schema_version_; @@ -588,7 +277,7 @@ int ObTabletSplitArg::assign(const ObTabletSplitArg &other) OB_DEF_SERIALIZE(ObTabletSplitArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, table_id_, lob_table_id_, + LST_DO_CODE(OB_UNIS_ENCODE, table_id_, lob_table_id_, schema_version_, task_id_, source_tablet_id_, dest_tablets_id_, compaction_scn_, data_format_version_, can_reuse_macro_block_, split_sstable_type_, @@ -598,12 +287,12 @@ OB_DEF_SERIALIZE(ObTabletSplitArg) OB_DEF_DESERIALIZE(ObTabletSplitArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, table_id_, lob_table_id_, + LST_DO_CODE(OB_UNIS_DECODE, table_id_, lob_table_id_, schema_version_, task_id_, source_tablet_id_, dest_tablets_id_, compaction_scn_, data_format_version_, can_reuse_macro_block_, split_sstable_type_, lob_col_idxs_); - if (FAILEDx(ObSplitUtil::deserializ_parallel_datum_rowkey( + if (FAILEDx(share::ObSplitUtil::deserializ_parallel_datum_rowkey( rowkey_allocator_, buf, data_len, pos, parallel_datum_rowkey_list_))) { LOG_WARN("deserialzie parallel info failed", K(ret)); } else { @@ -614,7 +303,7 @@ OB_DEF_DESERIALIZE(ObTabletSplitArg) OB_DEF_SERIALIZE_SIZE(ObTabletSplitArg) { int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, table_id_, lob_table_id_, + LST_DO_CODE(OB_UNIS_ADD_LEN, table_id_, lob_table_id_, schema_version_, task_id_, source_tablet_id_, dest_tablets_id_, compaction_scn_, data_format_version_, can_reuse_macro_block_, split_sstable_type_, @@ -642,151 +331,5 @@ OB_SERIALIZE_MEMBER(ObTabletSplitStartResult, ret_codes_, min_split_start_scn_); OB_SERIALIZE_MEMBER(ObTabletSplitFinishArg, split_info_array_); OB_SERIALIZE_MEMBER(ObTabletSplitFinishResult, ret_codes_); -ObCallRemoteWriteDDLRedoLogArg::ObCallRemoteWriteDDLRedoLogArg() - : ls_id_(), redo_info_(), task_id_(0) -{} - -int ObCallRemoteWriteDDLRedoLogArg::init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, - const int64_t task_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(task_id == 0 || !ls_id.is_valid() || !redo_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("args are not valid", K(ret), K(task_id), K(ls_id), K(redo_info)); - } else { - ls_id_ = ls_id; - redo_info_ = redo_info; - task_id_ = task_id; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObCallRemoteWriteDDLRedoLogArg, ls_id_, redo_info_, task_id_); - -ObCallRemoteWriteDDLCommitLogArg::ObCallRemoteWriteDDLCommitLogArg() - : ls_id_(), table_key_(), start_scn_(SCN::min_scn()), - table_id_(0), execution_id_(-1), ddl_task_id_(0) -{} - -int ObCallRemoteWriteDDLCommitLogArg::init(const share::ObLSID &ls_id, - const storage::ObITable::TableKey &table_key, - const SCN &start_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !table_key.is_valid() || !start_scn.is_valid_and_not_min())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablet id is not valid", K(ret), K(ls_id), K(table_key), K(start_scn)); - } else { - ls_id_ = ls_id; - table_key_ = table_key; - start_scn_ = start_scn; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObCallRemoteWriteDDLCommitLogArg, ls_id_, table_key_, start_scn_, - table_id_, execution_id_, ddl_task_id_); - - -ObCallRemoteWriteDDLIncCommitLogArg::ObCallRemoteWriteDDLIncCommitLogArg() - : ls_id_(), tablet_id_(), lob_meta_tablet_id_(), tx_desc_(nullptr), need_release_(false) -{} - -ObCallRemoteWriteDDLIncCommitLogArg::~ObCallRemoteWriteDDLIncCommitLogArg() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } -} - -int ObCallRemoteWriteDDLIncCommitLogArg::init(const share::ObLSID &ls_id, - const common::ObTabletID tablet_id, - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || - OB_ISNULL(tx_desc) || !tx_desc->is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tablet id is not valid", K(ret), K(ls_id), K(tablet_id), K(lob_meta_tablet_id), KPC(tx_desc)); - } else if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } else { - ls_id_ = ls_id; - tablet_id_ = tablet_id; - lob_meta_tablet_id_ = lob_meta_tablet_id; - tx_desc_ = tx_desc; - } - return ret; -} - -int ObCallRemoteWriteDDLIncCommitLogArg::release() -{ - int ret = OB_SUCCESS; - if (tx_desc_ != nullptr && need_release_) { - ObTransService *tx_svc = MTL_WITH_CHECK(ObTransService *); - if (OB_ISNULL(tx_svc)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null tx service ptr", K(ret)); - } else if (OB_FAIL(tx_svc->release_tx(*tx_desc_))) { - LOG_WARN("release tx fail", K(ret)); - } else { - need_release_ = false; - tx_desc_ = nullptr; - } - } - - return ret; -} - -OB_DEF_SERIALIZE(ObCallRemoteWriteDDLIncCommitLogArg) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (OB_SUCC(ret)) { - if (OB_ISNULL(tx_desc_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tx_desc_ is nullptr", K(ret)); - } else { - LST_DO_CODE(OB_UNIS_ENCODE, *tx_desc_); - } - } - return ret; -} - -OB_DEF_DESERIALIZE(ObCallRemoteWriteDDLIncCommitLogArg) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (OB_SUCC(ret)) { - ObTransService *tx_svc = nullptr; - if (OB_FAIL(release())) { - LOG_WARN("fail to release tx_desc", K(ret)); - } else if (OB_ISNULL(tx_svc = MTL_WITH_CHECK(ObTransService *))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null tx service ptr", KR(ret)); - } else if (OB_FAIL(tx_svc->acquire_tx(buf, data_len, pos, tx_desc_))) { - LOG_WARN("acquire tx by deserialize fail", K(data_len), K(pos), KR(ret)); - } else { - need_release_ = true; - } - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObCallRemoteWriteDDLIncCommitLogArg) -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, tablet_id_, lob_meta_tablet_id_); - if (tx_desc_ != nullptr) { - LST_DO_CODE(OB_UNIS_ADD_LEN, *tx_desc_); - } - return len; -} - -OB_SERIALIZE_MEMBER(ObCallRemoteWriteDDLIncCommitLogRes, tx_result_); - } // namespace obcall } // namespace oceanbase diff --git a/src/storage/ob_storage_rpc_arg.h b/src/storage/ob_storage_rpc_arg.h index c8a4f4c14..277a7c2fb 100644 --- a/src/storage/ob_storage_rpc_arg.h +++ b/src/storage/ob_storage_rpc_arg.h @@ -17,162 +17,18 @@ #define OCEANBASE_STORAGE_OB_STORAGE_RPC_ARG_H_ // moved from share/ob_rpc_struct.h:RPC arguments that embed storage types by value // (owner=storage;ns obrpc unchanged;RPC declaration remains in share proxy,vertical split for task 5) -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" #include "storage/tablet/ob_tablet_binding_mds_user_data.h" #include "storage/tablet/ob_tablet_create_delete_mds_user_data.h" #include "storage/tablet/ob_tablet_split_mds_user_data.h" #include "storage/tx/ob_multi_data_source.h" -#include "storage/tx/ob_trans_define.h" #include "storage/ddl/ob_ddl_struct.h" #include "storage/ob_i_table.h" #include "storage/blocksstable/ob_block_sstable_struct.h" namespace oceanbase { -namespace transaction { class ObTransService; } namespace obcall { -struct ObRpcRemoteWriteDDLCommitLogArg final -{ - OB_UNIS_VERSION(1); -public: - ObRpcRemoteWriteDDLCommitLogArg(); - ~ObRpcRemoteWriteDDLCommitLogArg() = default; - int init(const share::ObLSID &ls_id, - const storage::ObITable::TableKey &table_key, - const share::SCN &start_scn); - bool is_valid() const - { - return 1UL != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid_and_not_min(); - } - TO_STRING_KV(K_(ls_id), K_(table_key), K_(start_scn), K_(table_id), - K_(execution_id), K_(ddl_task_id)); -public: - share::ObLSID ls_id_; - storage::ObITable::TableKey table_key_; - share::SCN start_scn_; - int64_t table_id_; // depercated - int64_t execution_id_; // depercated - int64_t ddl_task_id_; // depercated -private: - DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLCommitLogArg); -}; - -struct ObRpcRemoteWriteDDLRedoLogArg -{ - OB_UNIS_VERSION(1); -public: - ObRpcRemoteWriteDDLRedoLogArg(); - ~ObRpcRemoteWriteDDLRedoLogArg() = default; - int init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, - const int64_t task_id); - bool is_valid() const { return 1UL != OB_INVALID_ID && ls_id_.is_valid() && redo_info_.is_valid() && task_id_ != 0; } - TO_STRING_KV(K(ls_id_), K_(redo_info), K(task_id_)); -public: - share::ObLSID ls_id_; - storage::ObDDLMacroBlockRedoInfo redo_info_; - int64_t task_id_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLRedoLogArg); -}; - -struct ObRpcRemoteWriteDDLIncCommitLogArg final -{ - OB_UNIS_VERSION(1); -public: - ObRpcRemoteWriteDDLIncCommitLogArg(); - ~ObRpcRemoteWriteDDLIncCommitLogArg(); - int init(const share::ObLSID &ls_id, - const common::ObTabletID tablet_id, - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc); - int release(); - bool is_valid() const - { - return 1UL != OB_INVALID_ID && ls_id_.is_valid() && tablet_id_.is_valid() && - OB_NOT_NULL(tx_desc_) && tx_desc_->is_valid(); - } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), KP_(tx_desc)); -public: - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObTabletID lob_meta_tablet_id_; - transaction::ObTxDesc *tx_desc_; - bool need_release_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLIncCommitLogArg); -}; - -struct ObRpcRemoteWriteDDLIncCommitLogRes final -{ - OB_UNIS_VERSION(1); -public: - ObRpcRemoteWriteDDLIncCommitLogRes() : tx_result_() {} - ~ObRpcRemoteWriteDDLIncCommitLogRes() {} - TO_STRING_KV(K_(tx_result)); -public: - transaction::ObTxExecResult tx_result_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLIncCommitLogRes); -}; - -#ifdef OB_BUILD_SHARED_STORAGE -struct ObRpcRemoteWriteDDLFinishLogArg final -{ - OB_UNIS_VERSION(1); -public: - ObRpcRemoteWriteDDLFinishLogArg(); - ~ObRpcRemoteWriteDDLFinishLogArg() = default; - int init(const storage::ObDDLFinishLogInfo &log); - bool is_valid() const - { - return log_info_.is_valid(); - } - TO_STRING_KV(K_(log_info)); -public: - storage::ObDDLFinishLogInfo log_info_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLFinishLogArg); -}; -#endif - -struct ObRegisterTxDataArg -{ - OB_UNIS_VERSION(1); - -public: - ObRegisterTxDataArg(); - ~ObRegisterTxDataArg() = default; - void inc_request_id(const int64_t base_request_id); - int init(const transaction::ObTxDesc &tx_desc, - const share::ObLSID &ls_id, - const transaction::ObTxDataSourceType &type, - const common::ObString &buf, - const transaction::ObTxSEQ seq_no, - const int64_t base_request_id, - const transaction::ObRegisterMdsFlag ®ister_flag); - TO_STRING_KV(KPC_(tx_desc), - K_(ls_id), - K_(type), - KP(buf_.length()), - K_(seq_no), - K_(request_id), - K_(register_flag)); - -public: - transaction::ObTxDesc *tx_desc_; - share::ObLSID ls_id_; - transaction::ObTxDataSourceType type_; - common::ObString buf_; - transaction::ObTxSEQ seq_no_; - int64_t request_id_; - - transaction::ObRegisterMdsFlag register_flag_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRegisterTxDataArg); -}; - struct ObBatchGetTabletBindingRes final { OB_UNIS_VERSION(1); @@ -548,23 +404,22 @@ struct ObCalibrateSSDiskSpaceArg final }; #endif #ifdef OB_BUILD_SHARED_STORAGE -struct ObLSSyncHotMicroKeyArg +struct ObSyncHotMicroKeyArg { OB_UNIS_VERSION(1); public: - ObLSSyncHotMicroKeyArg() : tenant_id_(OB_INVALID_TENANT_ID), ls_id_(share::ObLSID::INVALID_LS_ID), leader_addr_(), micro_keys_() {} - ~ObLSSyncHotMicroKeyArg() {} - int assign(const ObLSSyncHotMicroKeyArg &other); + ObSyncHotMicroKeyArg() : tenant_id_(OB_INVALID_TENANT_ID), leader_addr_(), micro_keys_() {} + ~ObSyncHotMicroKeyArg() {} + int assign(const ObSyncHotMicroKeyArg &other); bool is_valid() const; // return reserved serialize size besides ObSSMicroBlockCacheKeyMeta elements (including - // OB_UNIS_VERSION, tenant_id_, ls_id_, leader_addr_.get_serialize_size(), count of ObSArray, + // OB_UNIS_VERSION, tenant_id_, leader_addr_.get_serialize_size(), count of ObSArray, // and NS_::OB_SERIALIZE_SIZE_NEED_BYTES), which is smaller than 4KB. OB_INLINE int64_t get_reserved_serialize_size() const { return 4096; } - TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(leader_addr), "micro_keys_cnt", micro_keys_.count()); + TO_STRING_KV(K_(tenant_id), K_(leader_addr), "micro_keys_cnt", micro_keys_.count()); public: uint64_t tenant_id_; - int64_t ls_id_; ObAddr leader_addr_; ObSArray micro_keys_; }; @@ -575,25 +430,24 @@ struct ObDDLBuildSingleReplicaRequestArg final public: ObDDLBuildSingleReplicaRequestArg() : rowkey_allocator_("SplitRangeRPC"), - ls_id_(), source_tablet_id_(), dest_tablet_id_(), + source_tablet_id_(), dest_tablet_id_(), source_table_id_(OB_INVALID_ID), dest_schema_id_(OB_INVALID_ID), schema_version_(0), snapshot_version_(0), ddl_type_(0), task_id_(0), parallelism_(0), execution_id_(-1), tablet_task_id_(0), - data_format_version_(0), dest_ls_id_(), dest_schema_version_(0), + data_format_version_(0), dest_schema_version_(0), compaction_scn_(0), can_reuse_macro_block_(false), split_sstable_type_(share::ObSplitSSTableType::SPLIT_BOTH), lob_col_idxs_(), parallel_datum_rowkey_list_(), is_no_logging_(false), min_split_start_scn_() {} bool is_valid() const; int assign(const ObDDLBuildSingleReplicaRequestArg &other); - TO_STRING_KV(K_(ls_id), K_(source_tablet_id), K_(dest_tablet_id), + TO_STRING_KV(K_(source_tablet_id), K_(dest_tablet_id), K_(source_table_id), K_(dest_schema_id), K_(schema_version), K_(snapshot_version), K_(ddl_type), K_(task_id), K_(parallelism), K_(execution_id), K_(tablet_task_id), K_(data_format_version), - K_(dest_ls_id), K_(dest_schema_version), + K_(dest_schema_version), K_(compaction_scn), K_(can_reuse_macro_block), K_(split_sstable_type), K_(lob_col_idxs), K_(parallel_datum_rowkey_list), K_(is_no_logging), K_(min_split_start_scn)); public: common::ObArenaAllocator rowkey_allocator_; // alloc buf for datum rowkey. - share::ObLSID ls_id_; ObTabletID source_tablet_id_; ObTabletID dest_tablet_id_; int64_t source_table_id_; @@ -606,7 +460,6 @@ struct ObDDLBuildSingleReplicaRequestArg final int64_t execution_id_; int64_t tablet_task_id_; uint64_t data_format_version_; - share::ObLSID dest_ls_id_; int64_t dest_schema_version_; int64_t compaction_scn_; bool can_reuse_macro_block_; @@ -638,8 +491,7 @@ struct ObPrepareSplitRangesArg final OB_UNIS_VERSION(1); public: ObPrepareSplitRangesArg() - : ls_id_(), - tablet_id_(), + : tablet_id_(), user_parallelism_(0), schema_tablet_size_(0), ddl_type_(share::ObDDLType::DDL_INVALID) @@ -647,11 +499,10 @@ struct ObPrepareSplitRangesArg final ~ObPrepareSplitRangesArg() {} bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid() && share::ObDDLType::DDL_INVALID != ddl_type_; + return tablet_id_.is_valid() && share::ObDDLType::DDL_INVALID != ddl_type_; } - TO_STRING_KV(K(ls_id_), K(tablet_id_), K_(user_parallelism), K_(schema_tablet_size), K_(ddl_type)); + TO_STRING_KV(K(tablet_id_), K_(user_parallelism), K_(schema_tablet_size), K_(ddl_type)); public: - share::ObLSID ls_id_; ObTabletID tablet_id_; int64_t user_parallelism_; int64_t schema_tablet_size_; @@ -682,7 +533,7 @@ struct ObTabletSplitArg final public: ObTabletSplitArg() : rowkey_allocator_("SplitRangeRPC"), - ls_id_(), table_id_(OB_INVALID_ID), lob_table_id_(OB_INVALID_ID), + table_id_(OB_INVALID_ID), lob_table_id_(OB_INVALID_ID), schema_version_(0), task_id_(0), source_tablet_id_(), dest_tablets_id_(), compaction_scn_(0), data_format_version_(0), can_reuse_macro_block_(false), split_sstable_type_(share::ObSplitSSTableType::SPLIT_BOTH), @@ -691,18 +542,17 @@ struct ObTabletSplitArg final ~ObTabletSplitArg() = default; bool is_valid() const; int assign(const ObTabletSplitArg &other); - TO_STRING_KV(K_(ls_id), K_(table_id), K_(lob_table_id), + TO_STRING_KV(K_(table_id), K_(lob_table_id), K_(schema_version), K_(task_id), K_(source_tablet_id), K_(dest_tablets_id), K_(compaction_scn), K_(data_format_version), K_(can_reuse_macro_block), K_(split_sstable_type), K_(lob_col_idxs), K_(parallel_datum_rowkey_list), K_(min_split_start_scn)); public: common::ObArenaAllocator rowkey_allocator_; // alloc buf for datum rowkey. - share::ObLSID ls_id_; uint64_t table_id_; // scan rows needed. uint64_t lob_table_id_; // scan rows needed. - int64_t schema_version_; // report replica build status needed. - int64_t task_id_; // report replica build status needed. + int64_t schema_version_; // report DDL build status needed. + int64_t task_id_; // report DDL build status needed. common::ObTabletID source_tablet_id_; common::ObSArray dest_tablets_id_; int64_t compaction_scn_; @@ -767,95 +617,10 @@ struct ObTabletSplitFinishResult final public: common::ObSArray ret_codes_; uint64_t dest_tenant_id_; - share::ObLSID dest_ls_id_; int64_t dest_schema_version_; common::ObAddr server_addr_; }; -struct ObCallRemoteWriteDDLRedoLogArg -{ - OB_UNIS_VERSION(1); -public: - ObCallRemoteWriteDDLRedoLogArg(); - ~ObCallRemoteWriteDDLRedoLogArg() = default; - int init(const share::ObLSID &ls_id, - const storage::ObDDLMacroBlockRedoInfo &redo_info, - const int64_t task_id); - bool is_valid() const { return 1UL != OB_INVALID_ID && ls_id_.is_valid() && redo_info_.is_valid() && task_id_ != 0; } - TO_STRING_KV(K(ls_id_), K_(redo_info), K(task_id_)); -public: - share::ObLSID ls_id_; - storage::ObDDLMacroBlockRedoInfo redo_info_; - int64_t task_id_; -private: - DISALLOW_COPY_AND_ASSIGN(ObCallRemoteWriteDDLRedoLogArg); -}; - -struct ObCallRemoteWriteDDLCommitLogArg final -{ - OB_UNIS_VERSION(1); -public: - ObCallRemoteWriteDDLCommitLogArg(); - ~ObCallRemoteWriteDDLCommitLogArg() = default; - int init(const share::ObLSID &ls_id, - const storage::ObITable::TableKey &table_key, - const share::SCN &start_scn); - bool is_valid() const - { - return 1UL != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid_and_not_min(); - } - TO_STRING_KV(K_(ls_id), K_(table_key), K_(start_scn), K_(table_id), - K_(execution_id), K_(ddl_task_id)); -public: - share::ObLSID ls_id_; - storage::ObITable::TableKey table_key_; - share::SCN start_scn_; - int64_t table_id_; // depercated - int64_t execution_id_; // depercated - int64_t ddl_task_id_; // depercated -private: - DISALLOW_COPY_AND_ASSIGN(ObCallRemoteWriteDDLCommitLogArg); -}; -struct ObCallRemoteWriteDDLIncCommitLogArg final -{ - OB_UNIS_VERSION(1); -public: - ObCallRemoteWriteDDLIncCommitLogArg(); - ~ObCallRemoteWriteDDLIncCommitLogArg(); - int init(const share::ObLSID &ls_id, - const common::ObTabletID tablet_id, - const common::ObTabletID lob_meta_tablet_id, - transaction::ObTxDesc *tx_desc); - int release(); - bool is_valid() const - { - return 1UL != OB_INVALID_ID && ls_id_.is_valid() && tablet_id_.is_valid() && - OB_NOT_NULL(tx_desc_) && tx_desc_->is_valid(); - } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), KP_(tx_desc)); -public: - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObTabletID lob_meta_tablet_id_; - transaction::ObTxDesc *tx_desc_; - bool need_release_; -private: - DISALLOW_COPY_AND_ASSIGN(ObCallRemoteWriteDDLIncCommitLogArg); -}; - -struct ObCallRemoteWriteDDLIncCommitLogRes final -{ - OB_UNIS_VERSION(1); -public: - ObCallRemoteWriteDDLIncCommitLogRes() : tx_result_() {} - ~ObCallRemoteWriteDDLIncCommitLogRes() {} - TO_STRING_KV(K_(tx_result)); -public: - transaction::ObTxExecResult tx_result_; -private: - DISALLOW_COPY_AND_ASSIGN(ObCallRemoteWriteDDLIncCommitLogRes); -}; - } // namespace obcall } // namespace oceanbase #endif diff --git a/src/storage/ob_storage_schema_recorder.cpp b/src/storage/ob_storage_schema_recorder.cpp index 501d8c882..5c5b0ab0c 100644 --- a/src/storage/ob_storage_schema_recorder.cpp +++ b/src/storage/ob_storage_schema_recorder.cpp @@ -42,7 +42,6 @@ ObStorageSchemaRecorder::ObStorageSchemaRecorder() schema_guard_(nullptr), storage_schema_(nullptr), allocator_(nullptr), - ls_id_(), tablet_id_(), table_id_(0), max_column_cnt_(0) @@ -65,14 +64,12 @@ void ObStorageSchemaRecorder::destroy() ObIStorageClogRecorder::destroy(); free_allocated_info(); log_handler_ = NULL; - ls_id_.reset(); tablet_id_.reset(); table_id_ = 0; } int ObStorageSchemaRecorder::init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t saved_schema_version, const lib::Worker::CompatMode compat_mode, @@ -89,7 +86,6 @@ int ObStorageSchemaRecorder::init( LOG_WARN("failed to init ObIStorageClogRecorder", K(ret), K(saved_schema_version), K(log_handler)); } else { ignore_storage_schema_ = tablet_id.is_special_merge_tablet(); - ls_id_ = ls_id; tablet_id_ = tablet_id; compat_mode_ = compat_mode; is_inited_ = true; @@ -137,21 +133,21 @@ int ObStorageSchemaRecorder::inner_replay_clog( ObTabletHandle tmp_tablet_handle; int64_t stored_col_cnt = 0; - if (OB_FAIL(replay_get_tablet_handle(ls_id_, tablet_id_, scn, tmp_tablet_handle))) { + if (OB_FAIL(replay_get_tablet_handle(tablet_id_, scn, tmp_tablet_handle))) { if (OB_OBSOLETE_CLOG_NEED_SKIP == ret) { ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tablet handle", K(ret), K_(ls_id), K_(tablet_id), K(scn)); + LOG_WARN("failed to get tablet handle", K(ret), K_(tablet_id), K(scn)); } } else if (OB_FAIL(replay_storage_schema.deserialize(tmp_allocator, buf, size, pos))) { - LOG_WARN("fail to deserialize table schema", K(ret), K_(ls_id), K_(tablet_id)); + LOG_WARN("fail to deserialize table schema", K(ret), K_(tablet_id)); } else if (OB_FAIL(replay_storage_schema.get_store_column_count(stored_col_cnt, true/*full_col*/))) { LOG_WARN("failed to get store column count from replay schema", KR(ret),K(replay_storage_schema)); } else { // replay schema clog and update to ObStorageSchemaRecorder // need get column_cnt on schema_recorder to mini merge max_column_cnt_ = MAX(max_column_cnt_, stored_col_cnt); - FLOG_INFO("success to replay schema clog", K(ret), K_(ls_id), K_(tablet_id), K(replay_storage_schema), + FLOG_INFO("success to replay schema clog", K(ret), K_(tablet_id), K(replay_storage_schema), K(stored_col_cnt), K(max_column_cnt_)); } replay_storage_schema.reset(); @@ -186,11 +182,11 @@ int ObStorageSchemaRecorder::try_update_storage_schema( return ret; } -void ObStorageSchemaRecorder::sync_clog_failed_for_leader() +void ObStorageSchemaRecorder::on_sync_clog_failure() { } -int ObStorageSchemaRecorder::sync_clog_succ_for_leader(const int64_t update_version) +int ObStorageSchemaRecorder::on_sync_clog_success(const int64_t update_version) { int ret = OB_SUCCESS; if (OB_ISNULL(storage_schema_)) { @@ -200,7 +196,7 @@ int ObStorageSchemaRecorder::sync_clog_succ_for_leader(const int64_t update_vers ret = OB_ERR_UNEXPECTED; LOG_ERROR("schema version not match", K(storage_schema_), K(update_version)); } else { - LOG_INFO("success to update storage schema", K(ret), K_(ls_id), K_(tablet_id), K(storage_schema_), + LOG_INFO("success to update storage schema", K(ret), K_(tablet_id), K(storage_schema_), K(update_version), K_(clog_scn)); } return ret; @@ -233,8 +229,8 @@ int ObStorageSchemaRecorder::prepare_struct_in_lock( alloc_buf_offset += sizeof(ObSchemaGetterGuard); storage_schema_ = new (buf + alloc_buf_offset) ObStorageSchema(); } - if (FAILEDx(get_tablet_handle(ls_id_, tablet_id_, *tablet_handle_ptr_))) { - LOG_WARN("failed to get tablet handle", K(ret), K_(ls_id), K_(tablet_id)); + if (FAILEDx(get_tablet_handle(tablet_id_, *tablet_handle_ptr_))) { + LOG_WARN("failed to get tablet handle", K(ret), K_(tablet_id)); } else if (OB_FAIL(get_schema(update_version))) { LOG_WARN("fail to get expected schema", K(ret), K_(tablet_id), K(update_version)); } else if (OB_FAIL(generate_clog(clog_buf, clog_len))) { @@ -329,9 +325,9 @@ int ObStorageSchemaRecorder::submit_log( LOG_WARN("log handler or storage_schema is null", K(ret), KP(storage_schema_), KP(clog_buf), K(clog_len), K(tablet_handle_ptr_)); } else if (OB_FAIL(write_clog(clog_buf, clog_len))) { - LOG_WARN("fail to submit log", K(ret), K_(ls_id), K_(tablet_id)); + LOG_WARN("fail to submit log", K(ret), K_(tablet_id)); } else { - LOG_INFO("submit schema log succeed", K(ret), K_(ls_id), K_(tablet_id), K(clog_scn_), + LOG_INFO("submit schema log succeed", K(ret), K_(tablet_id), K(clog_scn_), "schema_version", storage_schema_->get_schema_version()); } diff --git a/src/storage/ob_storage_schema_recorder.h b/src/storage/ob_storage_schema_recorder.h index 55259cc61..b37055132 100644 --- a/src/storage/ob_storage_schema_recorder.h +++ b/src/storage/ob_storage_schema_recorder.h @@ -49,7 +49,6 @@ class ObStorageSchemaRecorder : public ObIStorageClogRecorder ~ObStorageSchemaRecorder(); int init( - const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t saved_schema_version, const lib::Worker::CompatMode compat_mode, @@ -69,7 +68,7 @@ class ObStorageSchemaRecorder : public ObIStorageClogRecorder ObStorageSchemaRecorder(const ObStorageSchemaRecorder&) = delete; ObStorageSchemaRecorder& operator=(const ObStorageSchemaRecorder&) = delete; - INHERIT_TO_STRING_KV("ObIStorageClogRecorder", ObIStorageClogRecorder, K_(ls_id), K_(tablet_id)); + INHERIT_TO_STRING_KV("ObIStorageClogRecorder", ObIStorageClogRecorder, K_(tablet_id)); private: virtual int inner_replay_clog( @@ -78,8 +77,8 @@ class ObStorageSchemaRecorder : public ObIStorageClogRecorder const char *buf, const int64_t size, int64_t &pos) override; - virtual int sync_clog_succ_for_leader(const int64_t update_version) override; - virtual void sync_clog_failed_for_leader() override; + virtual int on_sync_clog_success(const int64_t update_version) override; + virtual void on_sync_clog_failure() override; int get_schema(int64_t &table_version); @@ -113,7 +112,6 @@ class ObStorageSchemaRecorder : public ObIStorageClogRecorder ObStorageSchema *storage_schema_; ObIAllocator *allocator_; - share::ObLSID ls_id_; ObTabletID tablet_id_; int64_t table_id_; int64_t max_column_cnt_; diff --git a/src/storage/ob_storage_struct.cpp b/src/storage/ob_storage_struct.cpp index 1173473fc..df48939d4 100644 --- a/src/storage/ob_storage_struct.cpp +++ b/src/storage/ob_storage_struct.cpp @@ -407,7 +407,6 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam() snapshot_version_(ObVersionRange::MIN_VERSION), multi_version_start_(ObVersionRange::MIN_VERSION), storage_schema_(NULL), - rebuild_seq_(-1), sstable_(NULL), allow_duplicate_sstable_(false), upper_trans_param_() @@ -418,7 +417,6 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( const int64_t snapshot_version, const int64_t multi_version_start, const ObStorageSchema *storage_schema, - const int64_t rebuild_seq, const UpdateUpperTransParam upper_trans_param) : compaction_info_(), ddl_info_(), @@ -426,7 +424,6 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( snapshot_version_(snapshot_version), multi_version_start_(multi_version_start), storage_schema_(storage_schema), - rebuild_seq_(rebuild_seq), sstable_(NULL), allow_duplicate_sstable_(false), upper_trans_param_(upper_trans_param) @@ -437,7 +434,6 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( const int64_t snapshot_version, const int64_t multi_version_start, const ObStorageSchema *storage_schema, - const int64_t rebuild_seq, const blocksstable::ObSSTable *sstable, const bool allow_duplicate_sstable, const bool need_wait_check_flag) @@ -447,7 +443,6 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( snapshot_version_(snapshot_version), multi_version_start_(multi_version_start), storage_schema_(storage_schema), - rebuild_seq_(rebuild_seq), sstable_(sstable), allow_duplicate_sstable_(allow_duplicate_sstable), upper_trans_param_() @@ -461,7 +456,6 @@ bool ObUpdateTableStoreParam::is_valid() const && snapshot_version_ >= ObVersionRange::MIN_VERSION && nullptr != storage_schema_ && storage_schema_->is_valid() - && rebuild_seq_ >= 0 && compaction_info_.is_valid_with_sstable(NULL != sstable_/*have_sstable*/) && ha_info_.is_valid(); return bret; @@ -506,13 +500,9 @@ ObBatchUpdateTableStoreParam::ObBatchUpdateTableStoreParam() #ifdef ERRSIM errsim_point_info_(), #endif - rebuild_seq_(OB_INVALID_VERSION), - start_scn_(SCN::min_scn()), - tablet_meta_(nullptr), - restore_status_(ObTabletRestoreStatus::FULL), + storage_schema_(nullptr), tablet_split_param_(), tablet_fork_param_(), - need_replace_remote_sstable_(false), release_mds_scn_() { } @@ -520,21 +510,16 @@ ObBatchUpdateTableStoreParam::ObBatchUpdateTableStoreParam() void ObBatchUpdateTableStoreParam::reset() { tables_handle_.reset(); - rebuild_seq_ = OB_INVALID_VERSION; - start_scn_.set_min(); - tablet_meta_ = nullptr; - restore_status_ = ObTabletRestoreStatus::FULL; + storage_schema_ = nullptr; tablet_split_param_.reset(); tablet_fork_param_.reset(); - need_replace_remote_sstable_ = false; release_mds_scn_.reset(); } bool ObBatchUpdateTableStoreParam::is_valid() const { - return rebuild_seq_ > OB_INVALID_VERSION - && ObTabletRestoreStatus::is_valid(restore_status_) - && release_mds_scn_.is_valid(); + return release_mds_scn_.is_valid() + && (tablet_split_param_.is_valid() != tablet_fork_param_.is_valid()); } diff --git a/src/storage/ob_storage_struct.h b/src/storage/ob_storage_struct.h index dc57cb496..293ed5fe2 100644 --- a/src/storage/ob_storage_struct.h +++ b/src/storage/ob_storage_struct.h @@ -31,7 +31,7 @@ #include "storage/compaction/ob_compaction_util.h" #include "storage/compaction/ob_medium_compaction_mgr.h" #include "storage/ddl/ob_ddl_struct.h" -#include "storage/ob_tablet_ha_status.h" +#include "storage/ob_tablet_restore_state.h" #include "storage/blocksstable/ob_major_checksum_info.h" #include "storage/meta_mem/ob_tablet_handle.h" @@ -50,16 +50,12 @@ struct ObDiagnoseLocation; namespace storage { class ObStorageSchema; -struct ObMigrationTabletParam; typedef common::ObSEArray GetRowkeyArray; typedef common::ObSEArray ScanRangeArray; static const int64_t EXIST_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 1; static const int64_t MERGE_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 2; -// static const int64_t MV_LEFT_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 3; -// static const int64_t MV_RIGHT_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 4; -// static const int64_t MV_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 5; // static const int64_t BUILD_INDEX_READ_SNAPSHOT_VERSION = INT64_MAX - 6; // static const int64_t WARM_UP_READ_SNAPSHOT_VERSION = INT64_MAX - 7; static const int64_t GET_BATCH_ROWS_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 8; @@ -139,49 +135,6 @@ inline bool is_migrate_status_in_service(const ObMigrateStatus migrate_status) || OB_MIGRATE_STATUS_COPY_LOCAL_INDEX == migrate_status; } -enum ObReplicaOpType -{ - ADD_REPLICA_OP = 1, - MIGRATE_REPLICA_OP = 2, - REBUILD_REPLICA_OP = 3, - CHANGE_REPLICA_OP = 4, - REMOVE_REPLICA_OP = 5, - RESTORE_REPLICA_OP = 6, - COPY_GLOBAL_INDEX_OP = 7, - COPY_LOCAL_INDEX_OP = 8, - RESTORE_FOLLOWER_REPLICA_OP = 9, - BACKUP_REPLICA_OP = 10, - RESTORE_STANDBY_OP = 11, - VALIDATE_BACKUP_OP = 12, - FAST_MIGRATE_REPLICA_OP = 13, - LINK_SHARE_MAJOR_OP = 14, //share major only for read-only replica in ofs-mode. - BACKUP_BACKUPSET_OP = 15, - BACKUP_ARCHIVELOG_OP = 16, - UNKNOWN_REPLICA_OP, -}; - -inline bool is_replica_op_valid(const ObReplicaOpType replica_op) -{ - return replica_op >= ADD_REPLICA_OP && replica_op < UNKNOWN_REPLICA_OP; -} - -inline bool need_copy_split_state(const ObReplicaOpType replica_op) -{ - return COPY_GLOBAL_INDEX_OP != replica_op && COPY_LOCAL_INDEX_OP != replica_op; -} - -inline bool need_migrate_trans_table(const ObReplicaOpType replica_op) -{ - return REBUILD_REPLICA_OP == replica_op - || CHANGE_REPLICA_OP == replica_op - || ADD_REPLICA_OP == replica_op - || MIGRATE_REPLICA_OP == replica_op - || FAST_MIGRATE_REPLICA_OP == replica_op - || RESTORE_REPLICA_OP == replica_op - || RESTORE_FOLLOWER_REPLICA_OP == replica_op - || RESTORE_STANDBY_OP == replica_op; -} - struct ObTabletReportStatus { ObTabletReportStatus() @@ -413,7 +366,6 @@ struct ObUpdateTableStoreParam const int64_t snapshot_version, const int64_t multi_version_start, const ObStorageSchema *storage_schema, - const int64_t rebuild_seq, const blocksstable::ObSSTable *sstable = NULL, const bool allow_duplicate_sstable = false, const bool need_wait_check_flag = true); @@ -421,7 +373,6 @@ struct ObUpdateTableStoreParam const int64_t snapshot_version, const int64_t multi_version_start, const ObStorageSchema *storage_schema, - const int64_t rebuild_seq, const UpdateUpperTransParam upper_trans_param); int init_with_compaction_info( const ObCompactionTableStoreParam &comp_param, @@ -445,7 +396,7 @@ struct ObUpdateTableStoreParam #undef HA_PARAM_FUNC #undef PARAM_DEFINE_FUNC TO_STRING_KV(KP_(sstable), K_(snapshot_version), K_(multi_version_start), - KPC_(storage_schema), K_(rebuild_seq), + KPC_(storage_schema), K_(compaction_info), K_(ha_info), K_(ddl_info), K_(allow_duplicate_sstable), K_(upper_trans_param)); ObCompactionTableStoreParam compaction_info_; @@ -455,7 +406,6 @@ struct ObUpdateTableStoreParam int64_t snapshot_version_; int64_t multi_version_start_; const ObStorageSchema *storage_schema_; - int64_t rebuild_seq_; const blocksstable::ObSSTable *sstable_; bool allow_duplicate_sstable_; UpdateUpperTransParam upper_trans_param_; // set upper_trans_param_ only when update upper_trans_version @@ -501,21 +451,16 @@ struct ObBatchUpdateTableStoreParam final bool is_valid() const; void reset(); - TO_STRING_KV(K_(tables_handle), K_(rebuild_seq), - K_(start_scn), KP_(tablet_meta), K_(restore_status), K_(tablet_split_param), - K_(tablet_fork_param), K_(need_replace_remote_sstable), K_(release_mds_scn)); + TO_STRING_KV(K_(tables_handle), + KP_(storage_schema), K_(tablet_split_param), K_(tablet_fork_param), K_(release_mds_scn)); ObTablesHandleArray tables_handle_; #ifdef ERRSIM ObErrsimBackfillPoint errsim_point_info_; #endif - int64_t rebuild_seq_; - share::SCN start_scn_; - const ObMigrationTabletParam *tablet_meta_; - ObTabletRestoreStatus::STATUS restore_status_; + const ObStorageSchema *storage_schema_; ObSplitTableStoreParam tablet_split_param_; ObForkTableStoreParam tablet_fork_param_; - bool need_replace_remote_sstable_; share::SCN release_mds_scn_; DISALLOW_COPY_AND_ASSIGN(ObBatchUpdateTableStoreParam); diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index fcdc21ba1..984ac6469 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -73,13 +73,12 @@ void ObStorageTableGuard::throttle_if_needed_() // only do throttle on active memtable if (OB_NOT_NULL(memtable_) && memtable_->is_active_memtable()) { reset(); - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObLS *ls = nullptr; - const ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "get ls failed", KR(ret)); } else { + ls = tenant_ls; (void)TxShareMemThrottleUtil::do_throttle(for_replay_, store_ctx_.timeout_, share::memstore_throttled_alloc(), @@ -110,11 +109,10 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable { int ret = OB_SUCCESS; ObTabletTableIterator &iter = relative_table.tablet_iter_; - const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; if (OB_ISNULL(store_ctx_.ls_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id)); + LOG_WARN("ls is null", K(ret)); } while (OB_SUCC(ret) && need_to_refresh_table(*iter.table_iter())) { @@ -133,7 +131,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable relative_table.allow_not_ready(), true/*need_split_src_table*/, false/*need_split_dst_table*/))) { - LOG_WARN("fail to get read tables", K(ret), K(ls_id), K(remain_timeout), + LOG_WARN("fail to get read tables", K(ret), K(remain_timeout), "table_id", relative_table.get_table_id()); } else { // no worry. iter will hold tablet reference and its life cycle is longer than guard @@ -156,7 +154,6 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() bool need_retry = true; ObTableHandleV2 handle; ObITabletMemtable *tablet_memtable = nullptr; - const ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; const int64_t start_time = ObClockGenerator::getClock(); int64_t warn_interval = DEFAULT_REFRESH_WARN_INTERVAL; @@ -165,16 +162,16 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() if (OB_FAIL(tablet_->get_boundary_memtable_from_memtable_mgr(handle))) { // if there is no memtable, create a new one if (OB_ENTRY_NOT_EXIST == ret) { - ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(tablet_id, need_retry); } else { // OB_ENTRY_NOT_EXIST != ret - LOG_WARN("fail to get boundary memtable", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get boundary memtable", K(ret), K(tablet_id)); } } else if (OB_FAIL(handle.get_tablet_memtable(tablet_memtable))) { - LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(tablet_id)); } else if (tablet_memtable->is_direct_load_memtable()) { // set warn interval to 1 second because freeze direct load memtable is an async task warn_interval = FIND_DIRECT_LOAD_MT_WARN_INTERVAL; - ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(tablet_id, need_retry); } else if (OB_FAIL(check_freeze_to_inc_write_ref(static_cast(tablet_memtable), need_retry))) { if (OB_EAGAIN == ret) { } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { @@ -189,7 +186,6 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() OB_ERR_TOO_MUCH_TIME, "refresh replay table too much times", K(ret), - K(ls_id), K(tablet_id), K(cost_time)); } @@ -198,47 +194,39 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() if (OB_LS_OFFLINE == ret) { ret = OB_EAGAIN; - STORAGE_LOG(INFO, "reset ret code to OB_EAGAIN to avoid error log", KR(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(INFO, "reset ret code to OB_EAGAIN to avoid error log", KR(ret), K(tablet_id)); } return ret; } -int ObStorageTableGuard::create_data_memtable_for_replay_(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObStorageTableGuard::create_data_memtable_for_replay_(const common::ObTabletID &tablet_id, bool &need_retry) { int ret = OB_SUCCESS; - LOG_DEBUG("there is no boundary memtable", K(ret), K(ls_id), K(tablet_id)); - ObLSHandle ls_handle; + LOG_DEBUG("there is no boundary memtable", K(ret), K(tablet_id)); + ObLS *tenant_ls = nullptr; ObTabletHandle tmp_handle; SCN clog_checkpoint_scn; - ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_UNLIKELY(!ls_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, invalid ls handle", K(ret), K(ls_handle), K(ls_id), K(tablet_id)); - } else if (FALSE_IT(ls = ls_handle.get_ls())) { - } else if (ls->is_offline()) { - ret = OB_LS_OFFLINE; - FLOG_INFO("create data memtable failed because of ls offline", KR(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet(tablet_id, tmp_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("fail to get tablet", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret), K(tablet_id)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tablet( + tablet_id, tmp_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("fail to get tablet", K(ret), K(tablet_id)); } else if (FALSE_IT(clog_checkpoint_scn = tmp_handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_)) { } else if (replay_scn_ > clog_checkpoint_scn) { // TODO: get the newest schema_version from tablet CreateMemtableArg arg; arg.for_replay_ = for_replay_; arg.clog_checkpoint_scn_ = clog_checkpoint_scn; - if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->create_memtable(tablet_id, arg))) { - LOG_WARN("fail to create a boundary memtable", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(tenant_ls->get_tablet_svr()->create_memtable(tablet_id, arg))) { + LOG_WARN("fail to create a boundary memtable", K(ret), K(tablet_id)); } // In situation that replay_log_scn_ <= clog_checkpoint_scn, we have no need // to create the memtable. While we need double check to decide whether // another thread has created the memtable that we need replay. And if it // does, we must replay on the memtable. } else if (OB_FAIL(double_check_get_memtable_for_replay_(replay_scn_, need_retry))) { - LOG_WARN("fail to double check replay memtable", K(ret), K(ls_id), K(tablet_id), + LOG_WARN("fail to double check replay memtable", K(ret), K(tablet_id), K(replay_scn_), K(clog_checkpoint_scn)); } return ret; @@ -325,7 +313,6 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo { int ret = OB_SUCCESS; need_retry = true; - const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; // need to make sure the memtable is a right boundary memtable ObMemtable *old_memtable = memtable; @@ -336,7 +323,7 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo ObProtectedMemtableMgrHandle *protected_handle = NULL; if (OB_ISNULL(memtable)) { - LOG_INFO("table is null, need to refresh", K(need_retry), K(ls_id), K(tablet_id)); + LOG_INFO("table is null, need to refresh", K(need_retry), K(tablet_id)); } else if (FALSE_IT(old_freeze_flag = memtable->get_freeze_flag())) { } else if (FALSE_IT(is_tablet_freeze = memtable->get_is_tablet_freeze())) { } else if (memtable->is_active_memtable()) { @@ -353,10 +340,10 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo need_retry = false; ret = OB_SUCCESS; } else { - LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(tablet_id)); } } else if (OB_FAIL(handle.get_data_memtable(memtable))) { - LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(tablet_id)); } else { if (memtable != old_memtable) { is_tablet_freeze = memtable->get_is_tablet_freeze(); @@ -375,17 +362,14 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo SCN clog_checkpoint_scn; bool need_create_memtable = true; ObTabletHandle tmp_handle; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(need_retry), K(ls_id), K(tablet_id)); - } else if (OB_UNLIKELY(!ls_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, invalid ls handle", K(ret), K(need_retry), K(ls_handle), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret), K(need_retry), K(tablet_id)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->get_tablet(tablet_id, tmp_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("fail to get tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get tablet", K(ret), K(tablet_id)); } else if (FALSE_IT(clog_checkpoint_scn = tmp_handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_)) { } else if (for_replay_ && replay_scn_ <= clog_checkpoint_scn) { need_create_memtable = false; @@ -398,10 +382,10 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo arg.schema_version_ = memtable->get_max_schema_version(); arg.for_replay_ = for_replay_; arg.clog_checkpoint_scn_ = clog_checkpoint_scn; - if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->create_memtable(tablet_id, arg))) { + if (OB_FAIL(tenant_ls->get_tablet_svr()->create_memtable(tablet_id, arg))) { if (OB_EAGAIN == ret) { } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { - LOG_ERROR("fail to create new memtable for freeze", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + LOG_ERROR("fail to create new memtable for freeze", K(ret), K(need_retry), K(tablet_id)); } } } else if (for_replay_) { @@ -414,10 +398,10 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo need_retry = false; ret = OB_SUCCESS; } else { - LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(tablet_id)); } } else if (OB_FAIL(handle.get_data_memtable(memtable))) { - LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(tablet_id)); } else { if (memtable != old_memtable) { is_tablet_freeze = memtable->get_is_tablet_freeze(); @@ -441,20 +425,6 @@ bool ObStorageTableGuard::need_to_refresh_table(ObTableStoreIterator &iter) bool need_create_memtable = false; if (NULL == table || !table->is_data_memtable()) { need_create_memtable = true; - } else { - ObLSID ls_id = memtable->get_ls_id(); - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get memtable ls_id", K(ret), KPC(table)); - } else if (ls_id != store_ctx_.ls_->get_ls_id()) { - if (OB_UNLIKELY(!table->is_data_memtable())) { - ret =OB_ERR_UNEXPECTED; - ObLSID curr_ls_id = store_ctx_.ls_->get_ls_id(); - LOG_WARN("table is not data memtable, it does not allow ls_id to be different", K(ret), K(ls_id), K(curr_ls_id), KPC(table)); - } else { - need_create_memtable = true; - } - } } if (OB_SUCC(ret) && !need_create_memtable) { @@ -500,35 +470,32 @@ bool ObStorageTableGuard::need_to_refresh_table(ObTableStoreIterator &iter) bool need_log_error = false; check_if_need_log_(need_log, need_log_error); if (need_log) { - const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; if (need_log_error) { - LOG_ERROR_RET(OB_ERR_TOO_MUCH_TIME, "refresh table too much times", K(ret), K(exit_flag), K(ls_id), K(tablet_id), KP(table)); + LOG_ERROR_RET(OB_ERR_TOO_MUCH_TIME, "refresh table too much times", K(ret), K(exit_flag), K(tablet_id), KP(table)); LOG_DBA_ERROR_V2(OB_STORAGE_MEMTABLE_REFRESH_TIMEOUT, OB_ERR_TOO_MUCH_TIME, "refresh table too much times", - ", with ls_id=", ls_id, ", with tablet_id=", tablet_id, ", with exit_flag=", exit_flag); } else { - LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "refresh table too much times", K(ret), K(exit_flag), K(ls_id), K(tablet_id), KP(table)); + LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "refresh table too much times", K(ret), K(exit_flag), K(tablet_id), KP(table)); LOG_DBA_WARN_V2(OB_STORAGE_MEMTABLE_REFRESH_TOO_MUCH_TIME, OB_ERR_TOO_MUCH_TIME, "refresh table too much times", - ", with ls_id=", ls_id, ", with tablet_id=", tablet_id, ", with exit_flag=", exit_flag); } if (0 == exit_flag) { - LOG_WARN("table is null or not memtable", K(ret), K(ls_id), K(tablet_id), KP(table)); + LOG_WARN("table is null or not memtable", K(ret), K(tablet_id), KP(table)); } else if (1 == exit_flag) { - LOG_WARN("iterator store is expired", K(ret), K(ls_id), K(tablet_id), K(iter.check_store_expire()), K(iter.count()), K(iter)); + LOG_WARN("iterator store is expired", K(ret), K(tablet_id), K(iter.check_store_expire()), K(iter.count()), K(iter)); } else if (2 == exit_flag) { - LOG_WARN("failed to check_freeze_to_inc_write_ref", K(ret), K(ls_id), K(tablet_id), KPC(table)); + LOG_WARN("failed to check_freeze_to_inc_write_ref", K(ret), K(tablet_id), KPC(table)); } else if (3 == exit_flag) { - LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "check_freeze_to_inc_write_ref costs too much time", K(ret), K(ls_id), K(tablet_id), KPC(table)); + LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "check_freeze_to_inc_write_ref costs too much time", K(ret), K(tablet_id), KPC(table)); } else { - LOG_WARN("unexpect exit_flag", K(exit_flag), K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("unexpect exit_flag", K(exit_flag), K(ret), K(tablet_id)); } } } diff --git a/src/storage/ob_storage_table_guard.h b/src/storage/ob_storage_table_guard.h index 9f20bfcb7..a06e448ff 100644 --- a/src/storage/ob_storage_table_guard.h +++ b/src/storage/ob_storage_table_guard.h @@ -26,7 +26,6 @@ namespace oceanbase namespace share { class ObThrottleInfoGuard; -class ObLSID; } namespace common{ @@ -82,8 +81,7 @@ class ObStorageTableGuard memtable::ObMemtable *memtable, bool &bool_ret); int check_freeze_to_inc_write_ref(memtable::ObMemtable *table, bool &bool_ret); - int create_data_memtable_for_replay_(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, + int create_data_memtable_for_replay_(const common::ObTabletID &tablet_id, bool &no_need_create); bool need_to_refresh_table(ObTableStoreIterator &iter); void check_if_need_log_(bool &need_log, bool &need_log_error); diff --git a/src/storage/ob_super_block_struct.cpp b/src/storage/ob_super_block_struct.cpp index 56c5fc74e..9dbd5bace 100644 --- a/src/storage/ob_super_block_struct.cpp +++ b/src/storage/ob_super_block_struct.cpp @@ -300,8 +300,6 @@ bool ObTenantSnapshotMeta::is_valid() const // ========================== ObTenantSuperBlock ============================== -OB_SERIALIZE_MEMBER(ObLSItem, ls_id_, epoch_, status_, min_macro_seq_, max_macro_seq_); - ObTenantSuperBlock::ObTenantSuperBlock() { reset(); @@ -309,7 +307,7 @@ ObTenantSuperBlock::ObTenantSuperBlock() ObTenantSuperBlock::ObTenantSuperBlock(const bool is_hidden) : is_hidden_(is_hidden), version_(TENANT_SUPER_BLOCK_VERSION), - snapshot_cnt_(0), auto_inc_ls_epoch_(0), ls_cnt_(0) + snapshot_cnt_(0), auto_inc_ls_epoch_(0) { SET_FIRST_VALID_SLOG_CURSOR(replay_start_point_); tablet_meta_entry_ = ObServerSuperBlock::EMPTY_LIST_ENTRY_BLOCK; @@ -336,7 +334,6 @@ void ObTenantSuperBlock::reset() snapshot_cnt_ = 0; preallocated_seqs_.reset(); auto_inc_ls_epoch_ = 0; - ls_cnt_ = 0; } void ObTenantSuperBlock::copy_snapshots_from(const ObTenantSuperBlock &other) @@ -354,10 +351,9 @@ bool ObTenantSuperBlock::is_valid() const && ls_meta_entry_.is_valid() && tablet_meta_entry_.is_valid() && version_ > MIN_SUPER_BLOCK_VERSION - && (is_old_version() || IS_EMPTY_BLOCK_LIST(tablet_meta_entry_)) + && IS_EMPTY_BLOCK_LIST(tablet_meta_entry_) && snapshot_cnt_ >= 0 - && auto_inc_ls_epoch_ >= 0 - && ls_cnt_ >= 0; + && auto_inc_ls_epoch_ >= 0; return is_valid; } @@ -468,9 +464,7 @@ int ObTenantSuperBlock::serialize_(char *buf, const int64_t buf_len, int64_t &po tenant_snapshots_, snapshot_cnt_, preallocated_seqs_, - auto_inc_ls_epoch_, - ls_item_arr_, - ls_cnt_); + auto_inc_ls_epoch_); return ret; } @@ -481,7 +475,7 @@ int ObTenantSuperBlock::deserialize(const char *buf, const int64_t data_len, int OB_UNIS_DECODE(version_); OB_UNIS_DECODE(len); if (OB_SUCC(ret)) { - if (UNIS_VERSION < version_) { + if (UNIS_VERSION != version_) { ret = ::oceanbase::common::OB_NOT_SUPPORTED; LOG_WARN("ObTenantSuperBlock object version mismatch", K(ret), K_(version)); } else if (len < 0) { @@ -514,9 +508,7 @@ int ObTenantSuperBlock::deserialize_(const char *buf, const int64_t data_len, in tenant_snapshots_, snapshot_cnt_, preallocated_seqs_, - auto_inc_ls_epoch_, - ls_item_arr_, - ls_cnt_); + auto_inc_ls_epoch_); return ret; } @@ -538,40 +530,10 @@ int64_t ObTenantSuperBlock::get_serialize_size_(void) const tenant_snapshots_, snapshot_cnt_, preallocated_seqs_, - auto_inc_ls_epoch_, - ls_item_arr_, - ls_cnt_); + auto_inc_ls_epoch_); return len; } -OB_SERIALIZE_MEMBER(ObActiveTabletItem, tablet_id_, union_id_); - -ObActiveTabletItem::ObActiveTabletItem() : - tablet_id_(ObTabletID::INVALID_TABLET_ID), - union_id_(0) -{} -ObActiveTabletItem::ObActiveTabletItem(const common::ObTabletID tablet_id, const int64_t union_id) - : tablet_id_(tablet_id), union_id_(union_id) {} - - - -OB_SERIALIZE_MEMBER(ObLSActiveTabletArray, items_); - - -OB_SERIALIZE_MEMBER(ObPendingFreeTabletItem, tablet_id_, tablet_meta_version_, status_, free_time_, gc_type_); -OB_SERIALIZE_MEMBER(ObLSPendingFreeTabletArray, items_); - -int ObLSPendingFreeTabletArray::assign(const ObLSPendingFreeTabletArray &other) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(items_.assign(other.items_))) { - LOG_WARN("fail to assign items", K(ret)); - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObPrivateTabletCurrentVersion, tablet_addr_); - } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/ob_super_block_struct.h b/src/storage/ob_super_block_struct.h index d200af036..83b983f4c 100644 --- a/src/storage/ob_super_block_struct.h +++ b/src/storage/ob_super_block_struct.h @@ -20,7 +20,6 @@ #include "common/log/ob_log_cursor.h" #include "storage/blocksstable/ob_macro_block_id.h" #include "share/tenant_snapshot/ob_tenant_snapshot_id.h" -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "storage/meta_mem/ob_meta_obj_struct.h" #include "storage/meta_store/ob_tenant_seq_generator.h" @@ -151,60 +150,12 @@ struct ObTenantSnapshotMeta final share::ObTenantSnapshotID snapshot_id_; }; -enum class ObLSItemStatus : uint8_t -{ - CREATING = 0, - CREATED, // 1 - CREATE_ABORT, // 2 - DELETED, // 3 - MAX -}; - -struct ObLSItem -{ -public: - ObLSItem() : - ls_id_(), - epoch_(0), - status_(ObLSItemStatus::MAX), - min_macro_seq_(UINT64_MAX), - max_macro_seq_(UINT64_MAX) {} - virtual ~ObLSItem() { reset(); } - - void reset() - { - ls_id_.reset(); - epoch_ = 0; - status_ = ObLSItemStatus::MAX; - min_macro_seq_ = UINT64_MAX; - max_macro_seq_ = UINT64_MAX; - } - - bool is_valid() const - { - return ls_id_.is_valid() && epoch_ >= 0 && ObLSItemStatus::MAX != status_ && min_macro_seq_ < max_macro_seq_; - } - - TO_STRING_KV(K_(ls_id), K_(epoch), K_(status), K_(min_macro_seq), K_(max_macro_seq)); - OB_UNIS_VERSION_V(1); - -public: - share::ObLSID ls_id_; - int64_t epoch_; - ObLSItemStatus status_; - uint64_t min_macro_seq_; - uint64_t max_macro_seq_; -}; - struct ObTenantSuperBlock final { public: static const int64_t MAX_SNAPSHOT_NUM = 32; static const int64_t MIN_SUPER_BLOCK_VERSION = 0; - static const int64_t TENANT_SUPER_BLOCK_VERSION_V1 = 1; - static const int64_t TENANT_SUPER_BLOCK_VERSION_V3 = 3; - static const int64_t TENANT_SUPER_BLOCK_VERSION = 4; - static const int64_t MAX_LS_COUNT = 128; + static const int64_t TENANT_SUPER_BLOCK_VERSION = 5; ObTenantSuperBlock(); ObTenantSuperBlock(const bool is_hidden); ~ObTenantSuperBlock() = default; @@ -215,11 +166,9 @@ struct ObTenantSuperBlock final void reset(); bool is_valid() const; int get_snapshot(const share::ObTenantSnapshotID &snapshot_id, ObTenantSnapshotMeta &snapshot) const; - bool is_old_version() const { return version_ < TENANT_SUPER_BLOCK_VERSION; } int add_snapshot(const ObTenantSnapshotMeta &snapshot); int delete_snapshot(const share::ObTenantSnapshotID &snapshot_id); int check_new_snapshot(const share::ObTenantSnapshotID &snapshot_id) const; - bool is_trivial_version() const { return version_ == TENANT_SUPER_BLOCK_VERSION_V1; } TO_STRING_KV( K_(replay_start_point), @@ -229,8 +178,7 @@ struct ObTenantSuperBlock final K_(version), K_(snapshot_cnt), K_(preallocated_seqs), - K_(auto_inc_ls_epoch), - K_(ls_cnt)); + K_(auto_inc_ls_epoch)); OB_UNIS_VERSION(TENANT_SUPER_BLOCK_VERSION); public: @@ -247,8 +195,6 @@ struct ObTenantSuperBlock final // only meaningful for shared-storage ObTenantMonotonicIncSeqs preallocated_seqs_; int64_t auto_inc_ls_epoch_; - int64_t ls_cnt_; - ObLSItem ls_item_arr_[MAX_LS_COUNT]; }; #define IS_EMPTY_BLOCK_LIST(entry_block) (entry_block == oceanbase::storage::ObServerSuperBlock::EMPTY_LIST_ENTRY_BLOCK) @@ -256,134 +202,6 @@ struct ObTenantSuperBlock final #define SET_FIRST_VALID_SLOG_CURSOR(cursor) (set_cursor(cursor, 1/*file_id*/, 1/*log_id*/, 0/*offset*/)) -struct ObActiveTabletItem -{ -public: - ObActiveTabletItem(); - ObActiveTabletItem(const common::ObTabletID tablet_id, const int64_t union_id); - int64_t get_path_id() const { return meta_path_id_; } - uint64_t get_tablet_meta_version() const { return meta_version_id_; } - - TO_STRING_KV(K_(tablet_id), K_(meta_path_id), K_(meta_version_id)); - OB_UNIS_VERSION(1); - -public: - common::ObTabletID tablet_id_; - union { - int64_t union_id_; - // for PRIVATE_TABLET_META - struct { - int64_t meta_path_id_ : blocksstable::MacroBlockId::SF_BIT_PATH_ID; - uint64_t meta_version_id_ : blocksstable::MacroBlockId::SF_BIT_META_VERSION_ID; - }; - }; -}; - -struct ObLSActiveTabletArray -{ -public: - ObLSActiveTabletArray() - : items_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("ActiveItems")) {} - - ObLSActiveTabletArray(const ObLSActiveTabletArray &) = delete; - ObLSActiveTabletArray &operator=(const ObLSActiveTabletArray &) = delete; - - bool is_valid() const { return items_.count() >= 0; } - - TO_STRING_KV(K_(items)); - - OB_UNIS_VERSION(1); - -public: - common::ObSEArray items_; -}; - -enum class ObPendingFreeTabletStatus : uint8_t -{ - WAIT_GC = 0, - WAIT_VERIFY, // 1 - VERIFIED, // 2 - MAX -}; - -struct ObPendingFreeTabletItem -{ -public: - ObPendingFreeTabletItem() - : tablet_id_(), - tablet_meta_version_(0), - status_(ObPendingFreeTabletStatus::MAX), - free_time_(0), - gc_type_(GCTabletType::DropTablet) - {} - ObPendingFreeTabletItem( - const common::ObTabletID tablet_id, - const int64_t tablet_meta_version, - const ObPendingFreeTabletStatus status, - const int64_t free_time, - const GCTabletType gc_type) - : tablet_id_(tablet_id), tablet_meta_version_(tablet_meta_version), - status_(status), free_time_(free_time), - gc_type_(gc_type) - {} - - bool is_valid() const - { - return tablet_id_.is_valid() && tablet_meta_version_ > 0 && - ObPendingFreeTabletStatus::MAX != status_; - } - bool operator == (const ObPendingFreeTabletItem &other) const { - return tablet_id_ == other.tablet_id_ && - tablet_meta_version_ == other.tablet_meta_version_ && - status_ == other.status_; - } - - TO_STRING_KV(K_(tablet_id), K_(tablet_meta_version), K_(status)); - OB_UNIS_VERSION(1); - -public: - common::ObTabletID tablet_id_; - int64_t tablet_meta_version_; - ObPendingFreeTabletStatus status_; - // pending_free_items in pending_free_tablet_arr are incremented according to free time - int64_t free_time_; - GCTabletType gc_type_; -}; - -struct ObLSPendingFreeTabletArray -{ -public: - ObLSPendingFreeTabletArray() - : items_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("PendFreeItems")) {} - - ObLSPendingFreeTabletArray(const ObLSPendingFreeTabletArray &) = delete; - ObLSPendingFreeTabletArray &operator=(const ObLSPendingFreeTabletArray &) = delete; - - bool is_valid() const { return items_.count() >= 0; } - int assign(const ObLSPendingFreeTabletArray &other); - - TO_STRING_KV(K_(items)); - - OB_UNIS_VERSION(1); - -public: - common::ObSEArray items_; -}; - -struct ObPrivateTabletCurrentVersion -{ -public: - ObPrivateTabletCurrentVersion() : tablet_addr_() {} - - bool is_valid() const { return tablet_addr_.is_valid(); } - - TO_STRING_KV(K_(tablet_addr)); - OB_UNIS_VERSION(1); - -public: - ObMetaDiskAddr tablet_addr_; -}; - } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/ob_sync_tablet_seq_clog.cpp b/src/storage/ob_sync_tablet_seq_clog.cpp index ea563817c..d763ea765 100644 --- a/src/storage/ob_sync_tablet_seq_clog.cpp +++ b/src/storage/ob_sync_tablet_seq_clog.cpp @@ -102,24 +102,21 @@ ObSyncTabletSeqMdsLogCb::ObSyncTabletSeqMdsLogCb() { } -int ObSyncTabletSeqMdsLogCb::init(const ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t writer_id) +int ObSyncTabletSeqMdsLogCb::init(const ObTabletID &tablet_id, const int64_t writer_id) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObLSService *ls_srv = share::g_mp->ls_service(); - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ls id or tablet id", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls_srv->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls handle failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected null", K(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, + LOG_WARN("invalid tablet id", K(ret), K(tablet_id)); + } else if (OB_FAIL(ls_srv->get_ls(tenant_ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, tablet_handle_, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tablet", K(tablet_id)); } else if (OB_FAIL(mds_ctx_.set_writer(mds::MdsWriter{mds::WriterType::AUTO_INC_SEQ, writer_id}))) { LOG_WARN("fail to set writer", K(ret), K(writer_id)); } diff --git a/src/storage/ob_sync_tablet_seq_clog.h b/src/storage/ob_sync_tablet_seq_clog.h index c45be4ff9..78d8be95f 100644 --- a/src/storage/ob_sync_tablet_seq_clog.h +++ b/src/storage/ob_sync_tablet_seq_clog.h @@ -20,7 +20,6 @@ #include "common/ob_tablet_id.h" #include "logservice/palf/palf_callback.h" #include "logservice/ob_append_callback.h" -#include "share/ob_ls_id.h" #include "share/ob_tablet_autoincrement_param.h" #include "storage/ddl/ob_ddl_clog.h" #include "storage/multi_data_source/mds_ctx.h" @@ -59,7 +58,7 @@ class ObSyncTabletSeqMdsLogCb : public logservice::AppendCb ObSyncTabletSeqMdsLogCb(); virtual ~ObSyncTabletSeqMdsLogCb() = default; - int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t writer_id); + int init(const common::ObTabletID &tablet_id, const int64_t writer_id); virtual int on_success() override; virtual int on_failure() override; void try_release(); diff --git a/src/storage/ob_table_store_stat_mgr.h b/src/storage/ob_table_store_stat_mgr.h index a4321d1f3..cac005df5 100644 --- a/src/storage/ob_table_store_stat_mgr.h +++ b/src/storage/ob_table_store_stat_mgr.h @@ -25,7 +25,6 @@ #include "lib/hash/ob_hashmap.h" #include "lib/task/ob_timer.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -67,7 +66,7 @@ struct ObTableStoreStat void reset(); bool is_valid() const; - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(table_id), + TO_STRING_KV(K_(tablet_id), K_(table_id), K_(row_cache_hit_cnt), K_(row_cache_miss_cnt), K_(row_cache_put_cnt), K_(bf_filter_cnt), K_(bf_empty_read_cnt), K_(bf_access_cnt), K_(block_cache_hit_cnt), K_(block_cache_miss_cnt), @@ -82,7 +81,6 @@ struct ObTableStoreStat K_(sstable_bf_access_cnt), K_(rowkey_prefix), K_(logical_read_cnt), K_(physical_read_cnt)); - share::ObLSID ls_id_; common::ObTabletID tablet_id_; common::ObTableID table_id_; int64_t row_cache_hit_cnt_; diff --git a/src/storage/ob_tablet_autoinc_seq_rpc_handler.cpp b/src/storage/ob_tablet_autoinc_seq_rpc_handler.cpp index f8ae3fb8e..d835f4c1b 100644 --- a/src/storage/ob_tablet_autoinc_seq_rpc_handler.cpp +++ b/src/storage/ob_tablet_autoinc_seq_rpc_handler.cpp @@ -19,6 +19,7 @@ #include "ob_tablet_autoinc_seq_rpc_handler.h" #include "share/rc/ob_module_provider.h" #include "logservice/ob_log_service.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/multi_data_source/mds_ctx.h" using namespace oceanbase::share; @@ -152,27 +153,19 @@ int ObTabletAutoincSeqRpcHandler::fetch_tablet_autoinc_seq_cache( LOG_WARN("not inited", K(ret), K_(is_inited)); } else { MOD_SCOPE { - ObLSHandle ls_handle; - share::ObLSID ls_id = arg.ls_id_; - ObRole role = common::INVALID_ROLE; + ObLS *tenant_ls = nullptr; ObTabletHandle tablet_handle; ObTabletAutoincInterval autoinc_interval; const ObTabletID &tablet_id = arg.tablet_id_; - int64_t proposal_id = -1; ObTabletCreateDeleteMdsUserData user_data; mds::MdsWriter writer; mds::TwoPhaseCommitState trans_stat; share::SCN trans_version; bool is_committed = false; ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); - if (OB_FAIL(share::g_mp->log_service()->get_palf_role(ls_id, role, proposal_id))) { - LOG_WARN("get palf role failed", K(ret)); - } else if (!is_strong_leader(role)) { - ret = OB_NOT_MASTER; - LOG_WARN("follower received FetchTabletsSeq rpc", K(ret), K(ls_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle, THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : OB_DEFAULT_RPC_TIMEOUT))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, tablet_handle, THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : OB_DEFAULT_RPC_TIMEOUT))) { LOG_WARN("failed to get tablet", KR(ret), K(arg)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { LOG_WARN("fail to get latest tablet status", K(ret), K(arg)); @@ -205,17 +198,9 @@ int ObTabletAutoincSeqRpcHandler::batch_get_tablet_autoinc_seq( LOG_WARN("invalid argument", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSHandle ls_handle; - share::ObLSID ls_id = arg.ls_id_; - ObRole role = common::INVALID_ROLE; - int64_t proposal_id = -1; - if (OB_FAIL(share::g_mp->log_service()->get_palf_role(ls_id, role, proposal_id))) { - LOG_WARN("get palf role failed", K(ret)); - } else if (!is_strong_leader(role)) { - ret = OB_NOT_MASTER; - LOG_WARN("follower received FetchTabletsSeq rpc", K(ret), K(ls_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("get ls failed", K(ret)); } else if (OB_FAIL(res.autoinc_params_.reserve(arg.src_tablet_ids_.count()))) { LOG_WARN("failed to reserve autoinc param", K(ret)); } else { @@ -228,7 +213,7 @@ int ObTabletAutoincSeqRpcHandler::batch_get_tablet_autoinc_seq( autoinc_param.src_tablet_id_ = src_tablet_id; autoinc_param.dest_tablet_id_ = arg.dest_tablet_ids_.at(i); ObBucketHashRLockGuard lock_guard(bucket_lock_, src_tablet_id.hash()); - if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(src_tablet_id, tablet_handle))) { + if (OB_TMP_FAIL(tenant_ls->get_tablet(src_tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet", K(tmp_ret), K(src_tablet_id)); } else { ObTabletAutoincSeq autoinc_seq; @@ -265,30 +250,16 @@ int ObTabletAutoincSeqRpcHandler::batch_set_tablet_autoinc_seq( LOG_WARN("failed to assign autoinc params", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSHandle ls_handle; - ObLS *ls = nullptr; - share::ObLSID ls_id = arg.ls_id_; - ObRole role = common::INVALID_ROLE; - int64_t proposal_id = -1; - if (OB_FAIL(share::g_mp->log_service()->get_palf_role(ls_id, role, proposal_id))) { - LOG_WARN("get palf role failed", K(ret)); - } else if (!is_strong_leader(role)) { - ret = OB_NOT_MASTER; - LOG_WARN("follower received FetchTabletsSeq rpc", K(ret), K(ls_id)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { - ret = OB_NOT_MASTER; - LOG_WARN("ls not leader", K(ret), K(arg.ls_id_)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("get ls failed", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < res.autoinc_params_.count(); i++) { int tmp_ret = OB_SUCCESS; ObTabletHandle tablet_handle; share::ObMigrateTabletAutoincSeqParam &autoinc_param = res.autoinc_params_.at(i); ObBucketHashWLockGuard lock_guard(bucket_lock_, autoinc_param.dest_tablet_id_.hash()); - if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(autoinc_param.dest_tablet_id_, tablet_handle, + if (OB_TMP_FAIL(tenant_ls->get_tablet(autoinc_param.dest_tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("failed to get tablet", K(tmp_ret), K(autoinc_param)); } else if (OB_TMP_FAIL(tablet_handle.get_obj()->update_tablet_autoinc_seq(autoinc_param.autoinc_seq_, arg.is_tablet_creating_))) { @@ -319,7 +290,7 @@ int ObTabletAutoincSeqRpcHandler::replay_update_tablet_autoinc_seq( ObSyncTabletSeqReplayExecutor replay_executor; if (OB_FAIL(replay_executor.init(autoinc_seq, is_tablet_creating, replay_scn))) { LOG_WARN("failed to init tablet auto inc sequence replay executor", K(ret), K(autoinc_seq), K(is_tablet_creating), K(replay_scn)); - } else if (OB_FAIL(replay_executor.execute(replay_scn, ls->get_ls_id(), tablet_id))) { + } else if (OB_FAIL(replay_executor.execute(replay_scn, tablet_id))) { if (OB_TABLET_NOT_EXIST == ret) { LOG_INFO("tablet may be deleted, skip this log", K(ret), K(tablet_id), K(replay_scn)); ret = OB_SUCCESS; @@ -344,12 +315,7 @@ int ObTabletAutoincSeqRpcHandler::batch_set_tablet_autoinc_seq_in_trans( mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = arg.ls_id_; ObArenaAllocator allocator(common::ObMemAttr("SetAutoSeq")); - if (OB_UNLIKELY(ls_id != ls.get_ls_id())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid ls", K(ret), K(ls_id), K(ls.get_ls_id())); - } for (int64_t i = 0; OB_SUCC(ret) && i < arg.autoinc_params_.count(); i++) { allocator.reuse(); const ObTabletID &tablet_id = arg.autoinc_params_.at(i).dest_tablet_id_; @@ -357,9 +323,9 @@ int ObTabletAutoincSeqRpcHandler::batch_set_tablet_autoinc_seq_in_trans( ObTabletAutoincSeq data; ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); if (OB_FAIL(data.set_autoinc_seq_value(allocator, autoinc_seq))) { - LOG_WARN("failed to set autoinc seq value", K(ret), K(ls_id), K(tablet_id), K(autoinc_seq)); + LOG_WARN("failed to set autoinc seq value", K(ret), K(tablet_id), K(autoinc_seq)); } else if (OB_FAIL(set_tablet_autoinc_seq_in_trans(ls, tablet_id, data, replay_scn, ctx))) { - LOG_WARN("failed to set mds", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to set mds", K(ret), K(tablet_id)); } } return ret; @@ -374,9 +340,8 @@ int ObTabletAutoincSeqRpcHandler::set_tablet_autoinc_seq_in_trans( { MDS_TG(100_ms); int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls.get_ls_id(); if (!replay_scn.is_valid()) { - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; mds::MdsCtx &user_ctx = static_cast(ctx); @@ -390,7 +355,7 @@ int ObTabletAutoincSeqRpcHandler::set_tablet_autoinc_seq_in_trans( ObTabletAutoincSeqReplayExecutor replay_executor; if (CLICK_FAIL(replay_executor.init(ctx, replay_scn, data))) { LOG_ERROR("failed to init replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(replay_scn, ls_id, tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(replay_scn, tablet_id))) { if (OB_EAGAIN != ret) { LOG_ERROR("failed to replay mds", K(ret)); } diff --git a/src/storage/ob_tablet_autoincrement_service.cpp b/src/storage/ob_tablet_autoincrement_service.cpp index c9e2e2053..4480ab3d0 100644 --- a/src/storage/ob_tablet_autoincrement_service.cpp +++ b/src/storage/ob_tablet_autoincrement_service.cpp @@ -19,7 +19,6 @@ #include "ob_tablet_autoincrement_service.h" #include "storage/ob_storage_rpc.h" #include "share/rc/ob_module_provider.h" -#include "logservice/ob_log_service.h" #include "storage/ob_tablet_autoinc_seq_rpc_handler.h" namespace oceanbase @@ -69,56 +68,6 @@ int ObTabletAutoincMgr::set_interval(const ObTabletAutoincParam ¶m, ObTablet return ret; } -int ObTabletAutoincMgr::clear_cache_if_fallback_for_mlog( - const uint64_t current_value) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("tablet autoinc mgr is not inited", K(ret)); - } else { - const int64_t TRY_LOCK_INTERVAL = 1000L; // 1ms - uint64_t cache_value = 0; - - while (true) { - if (OB_SUCCESS != mutex_.trylock()) { - ob_usleep(TRY_LOCK_INTERVAL); - THIS_WORKER.sched_run(); - } else { - break; - } - } - - if (prefetch_node_.is_valid() && curr_node_.is_valid() - && curr_node_.cache_end_ + 1 != prefetch_node_.cache_start_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("curr_node_ and prefetch_node_ is invalid", KPC(this)); - } - - if (OB_FAIL(ret)) { - } else if (prefetch_node_.is_valid()) { - cache_value = prefetch_node_.cache_end_; - } else if (curr_node_.is_valid()) { - cache_value = curr_node_.cache_end_; - } - - if (OB_FAIL(ret)) { - } else if (0 == cache_value) { - LOG_INFO("inc cache is empty, skip check", KPC(this)); - } else if (cache_value + 1 < current_value) { - LOG_INFO("auto inc seq fallback, need clear cache", K(ret), KPC(this), K(current_value)); - curr_node_.reset(); - prefetch_node_.reset(); - next_value_ = 1; - } else if (cache_value + 1 > current_value) { - } - - mutex_.unlock(); - } - return ret; -} - int ObTabletAutoincMgr::fetch_interval(const ObTabletAutoincParam ¶m, ObTabletCacheInterval &interval) { int ret = OB_SUCCESS; @@ -176,34 +125,19 @@ int ObTabletAutoincMgr::fetch_new_range(const ObTabletAutoincParam ¶m, ObTabletCacheNode &node) { int ret = OB_SUCCESS; - share::ObLocationService *location_service = nullptr; - ObAddr leader_addr; - bool is_cache_hit = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("tablet auto increment service is not inited", K(ret), K(param), K(tablet_id)); - } else if (OB_ISNULL(location_service = GCTX.location_service_)) { - ret = OB_ERR_SYS; - LOG_WARN("location_cache is null", K(ret), KP(location_service)); } else { obcall::ObFetchTabletSeqArg arg; obcall::ObFetchTabletSeqRes res; arg.cache_size_ = MAX(cache_size_, param.auto_increment_cache_size_); // TODO(shuangcan): confirm this - arg.tablet_id_ = tablet_id; - // arg.ls_id_ will be filled by location_service->get bool finish = false; for (int64_t retry_times = 0; OB_SUCC(ret) && !finish; retry_times++) { const int64_t rpc_timeout = THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : OB_DEFAULT_RPC_TIMEOUT; - if (OB_FAIL(location_service->get(tablet_id, 0/*expire_renew_time*/, is_cache_hit, arg.ls_id_))) { - LOG_WARN("fail to get log stream id", K(ret), K(tablet_id)); - } else if (OB_FAIL(location_service->get_leader(GCONF.cluster_id, - arg.ls_id_, - false,/*force_renew*/ - leader_addr))) { - LOG_WARN("get leader failed", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().fetch_tablet_autoinc_seq_cache(arg, res))) { + if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().fetch_tablet_autoinc_seq_cache(arg, res))) { LOG_WARN("fail to fetch autoinc cache for tablets", K(ret), K(retry_times), K(arg), K(rpc_timeout)); } if (OB_SUCC(ret)) { @@ -323,95 +257,6 @@ int ObTabletAutoincrementService::get_autoinc_seq(const common::ObTabletID &tabl return ret; } -int ObTabletAutoincrementService::get_autoinc_seq_for_mlog(const ObLSID &ls_id, - const common::ObTabletID &tablet_id, - uint64_t &autoinc_seq) -{ - int ret = OB_SUCCESS; - const int64_t auto_increment_cache_size = ObTabletAutoincrementService::DEFAULT_CACHE_SIZE; - ObTabletAutoincParam param; - - ObTabletAutoincMgr *autoinc_mgr = nullptr; - - uint64_t current_value = 0; - ObLSHandle ls_handle; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLS *ls = nullptr; - ObTabletHandle tablet_handle; - bool is_committed = true; - bool seq_impossible_fallback = false; - - ObArenaAllocator allocator(common::ObMemAttr("FetchAutoSeq")); - ObTabletAutoincSeq tmp_autoinc_seq; - mds::MdsWriter writer;// will be removed later - mds::TwoPhaseCommitState trans_stat;// will be removed later - share::SCN trans_version;// will be removed later - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("tablet auto increment service is not inited", K(ret)); - } else if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls_service or log_service", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(ls_id)); - } - - if (OB_SUCC(ret)) { - bool need_retry = false; - const int64_t abs_timeout_us = THIS_WORKER.is_timeout_ts_valid() ? - THIS_WORKER.get_timeout_ts() : ObTimeUtility::current_time() + GCONF.rpc_timeout; - do { - need_retry = false; - if (OB_FAIL(tablet_handle.get_obj()->get_latest_autoinc_seq(tmp_autoinc_seq, allocator, writer, trans_stat, trans_version))) { - if (OB_EMPTY_RESULT == ret) { - seq_impossible_fallback = true; - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get auto inc seq", K(ret)); - } - } else if (OB_UNLIKELY(mds::TwoPhaseCommitState::ON_COMMIT != trans_stat)) { - if (ObTimeUtility::current_time() > abs_timeout_us) { - ret = OB_TIMEOUT; - LOG_WARN("get auto inc timeout", K(ret), K(abs_timeout_us)); - } else { - need_retry = true; - usleep(100); - } - } else if (OB_FAIL(tmp_autoinc_seq.get_autoinc_seq_value(current_value))) { - LOG_WARN("failed to get autoinc seq value", K(ret), K(tmp_autoinc_seq)); - } - } while (need_retry); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(acquire_mgr(tablet_id, auto_increment_cache_size, autoinc_mgr))) { - LOG_WARN("failed to acquire mgr", K(ret)); - } else { - ObTabletCacheInterval interval(tablet_id, 1/*cache size*/); - if (OB_ISNULL(autoinc_mgr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("autoinc mgr is unexpected null", K(ret)); - } else if (!seq_impossible_fallback - && OB_FAIL(autoinc_mgr->clear_cache_if_fallback_for_mlog(current_value))) { - LOG_WARN("failed to clear_cache_if_fallback_for_mlog", K(ret)); - } else if (OB_FAIL(autoinc_mgr->fetch_interval(param, interval))) { - LOG_WARN("fail to fetch interval", K(ret), K(param)); - } else if (OB_FAIL(interval.next_value(autoinc_seq))) { - LOG_WARN("fail to get next value", K(ret)); - } - } - if (nullptr != autoinc_mgr) { - release_mgr(autoinc_mgr); - } - return ret; -} - ObTabletAutoincrementService &ObTabletAutoincrementService::get_instance() { static ObTabletAutoincrementService autoinc_service; @@ -577,7 +422,6 @@ int ObTabletAutoincCacheCleaner::commit(const int64_t timeout_us) common::ObSEArray server_list; ObUnitInfoGetter ui_getter; obcall::ObClearTabletAutoincSeqCacheArg arg; - const ObLSID unused_ls_id = SYS_LS; int64_t abs_timeout_us = ObTimeUtility::current_time() + timeout_us; const ObTimeoutCtx &ctx = ObTimeoutCtx::get_ctx(); if (THIS_WORKER.is_timeout_ts_valid()) { @@ -597,7 +441,7 @@ int ObTabletAutoincCacheCleaner::commit(const int64_t timeout_us) LOG_WARN("init unit info getter failed", K(ret)); } else if (OB_FAIL(ui_getter.get_tenant_servers(server_list))) { LOG_WARN("get tenant servers failed", K(ret)); - } else if (OB_FAIL(arg.init(tablet_ids_, unused_ls_id))) { + } else if (OB_FAIL(arg.init(tablet_ids_))) { LOG_WARN("failed to init clear tablet autoinc arg", K(ret)); } else { // seekdb: all servers are local, call handler directly. diff --git a/src/storage/ob_tablet_ha_status.cpp b/src/storage/ob_tablet_ha_status.cpp deleted file mode 100644 index 29a5bdf79..000000000 --- a/src/storage/ob_tablet_ha_status.cpp +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX STORAGE - -#include "ob_tablet_ha_status.h" - - -namespace oceanbase -{ -namespace storage -{ - -bool ObTabletRestoreStatus::is_valid(const ObTabletRestoreStatus::STATUS &status) -{ - return status >= ObTabletRestoreStatus::FULL - && status < ObTabletRestoreStatus::RESTORE_STATUS_MAX; -} - -int ObTabletRestoreStatus::check_can_change_status( - const ObTabletRestoreStatus::STATUS &cur_status, - const ObTabletRestoreStatus::STATUS &change_status, - bool &can_change) -{ - int ret = OB_SUCCESS; - can_change = false; - - if (!ObTabletRestoreStatus::is_valid(cur_status) || !ObTabletRestoreStatus::is_valid(change_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("check can change status get invalid argument", K(ret), K(cur_status), K(change_status)); - } else { - switch (cur_status) { - case ObTabletRestoreStatus::PENDING: { - if (ObTabletRestoreStatus::UNDEFINED == change_status - || ObTabletRestoreStatus::EMPTY == change_status - || ObTabletRestoreStatus::PENDING == change_status) { - can_change = true; - } - break; - } - case ObTabletRestoreStatus::UNDEFINED: { - if (ObTabletRestoreStatus::PENDING == change_status - || ObTabletRestoreStatus::UNDEFINED == change_status) { - can_change = true; - } - break; - } - case ObTabletRestoreStatus::EMPTY: { - if (ObTabletRestoreStatus::MINOR_AND_MAJOR_META == change_status - || ObTabletRestoreStatus::REMOTE == change_status - || ObTabletRestoreStatus::FULL == change_status - || ObTabletRestoreStatus::EMPTY == change_status - || ObTabletRestoreStatus::UNDEFINED == change_status) { - can_change = true; - } - break; - } - case ObTabletRestoreStatus::MINOR_AND_MAJOR_META: { - if (ObTabletRestoreStatus::FULL == change_status - || ObTabletRestoreStatus::MINOR_AND_MAJOR_META == change_status) { - can_change = true; - } - break; - } - case ObTabletRestoreStatus::REMOTE: { - if (ObTabletRestoreStatus::FULL == change_status - || ObTabletRestoreStatus::REMOTE == change_status) { - can_change = true; - } - break; - } - case ObTabletRestoreStatus::FULL: { - if (ObTabletRestoreStatus::FULL == change_status - || ObTabletRestoreStatus::EMPTY == change_status) { - can_change = true; - } - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("invalid cur status for fail", K(ret), K(cur_status)); - } - } - - if (OB_SUCC(ret) && !can_change) { - FLOG_INFO("can not change tablet restore status", K(cur_status), K(change_status)); - } - - } - return ret; -} - -bool ObTabletDataStatus::is_valid(const ObTabletDataStatus::STATUS &status) -{ - return status >= ObTabletDataStatus::COMPLETE - && status < ObTabletDataStatus::DATA_STATUS_MAX; -} - -int ObTabletDataStatus::check_can_change_status( - const ObTabletDataStatus::STATUS &cur_status, - const ObTabletDataStatus::STATUS &change_status, - bool &can_change) -{ - int ret = OB_SUCCESS; - can_change = false; - - if (!ObTabletDataStatus::is_valid(cur_status) || !ObTabletDataStatus::is_valid(change_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("check can change status get invalid argument", K(ret), K(cur_status), K(change_status)); - } else { - switch (cur_status) { - case ObTabletDataStatus::INCOMPLETE: { - if (ObTabletDataStatus::COMPLETE == change_status - || ObTabletDataStatus::INCOMPLETE == change_status) { - can_change = true; - } - break; - } - case ObTabletDataStatus::COMPLETE: { - if (ObTabletDataStatus::COMPLETE == change_status) { - can_change = true; - } - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("invalid cur status for fail", K(ret), K(cur_status)); - } - } - - if (OB_SUCC(ret) && !can_change) { - FLOG_INFO("can not change tablet data status", K(cur_status), K(change_status)); - } - } - return ret; -} - -bool ObTabletExpectedStatus::is_valid(const ObTabletExpectedStatus::STATUS &status) -{ - return status >= ObTabletExpectedStatus::NORMAL - && status < ObTabletExpectedStatus::EXPECTED_STATUS_MAX; -} - -int ObTabletExpectedStatus::check_can_change_status( - const ObTabletExpectedStatus::STATUS &cur_status, - const ObTabletExpectedStatus::STATUS &change_status, - bool &can_change) -{ - int ret = OB_SUCCESS; - can_change = false; - - if (!ObTabletExpectedStatus::is_valid(cur_status) || !ObTabletExpectedStatus::is_valid(change_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("check can change status get invalid argument", K(ret), K(cur_status), K(change_status)); - } else if (ObTabletExpectedStatus::NORMAL == cur_status && ObTabletExpectedStatus::DELETED == change_status) { - can_change = true; - } else { - FLOG_INFO("can not change tablet meta status", K(cur_status), K(change_status)); - } - return ret; -} - -ObTabletHAStatus::ObTabletHAStatus() - : restore_status_(ObTabletRestoreStatus::RESTORE_STATUS_MAX), - data_status_(ObTabletDataStatus::DATA_STATUS_MAX), - reserved_(0) -{ -} - -bool ObTabletHAStatus::is_valid() const -{ - return ObTabletRestoreStatus::is_valid(restore_status_) - && ObTabletDataStatus::is_valid(data_status_); -} - -int ObTabletHAStatus::serialize(char *buf, const int64_t len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(len), K(pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, len, new_pos, ha_status_))) { - LOG_WARN("serialize ha status failed.", K(ret), K(new_pos), K(len), K(ha_status_), K(*this)); - } else { - pos = new_pos; - } - return ret; -} - -int ObTabletHAStatus::deserialize(const char *buf, const int64_t len, int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(len), K(pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ha_status_))) { - LOG_WARN("failed to deserialize ha status", K(ret), K(len), K(new_pos)); - } else { - pos = new_pos; - } - return ret; -} - -int64_t ObTabletHAStatus::get_serialize_size() const -{ - return serialization::encoded_length_i64(ha_status_); -} - -int ObTabletHAStatus::set_restore_status(const ObTabletRestoreStatus::STATUS &restore_status) -{ - int ret = OB_SUCCESS; - - if (!ObTabletRestoreStatus::is_valid(restore_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("set restore status get invalid argument", K(ret), K(restore_status)); - } else { - restore_status_ = restore_status; - } - return ret; -} - -int ObTabletHAStatus::get_restore_status(ObTabletRestoreStatus::STATUS &restore_status) const -{ - int ret = OB_SUCCESS; - if (!is_valid()) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ha status do not init", K(ret), KPC(this)); - } else { - restore_status = restore_status_; - } - return ret; -} - -int ObTabletHAStatus::set_data_status(const ObTabletDataStatus::STATUS &data_status) -{ - int ret = OB_SUCCESS; - - if (!ObTabletDataStatus::is_valid(data_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("set data status get invalid argument", K(ret), K(data_status)); - } else { - data_status_ = data_status; - } - return ret; -} - -int ObTabletHAStatus::get_data_status(ObTabletDataStatus::STATUS &data_status) const -{ - int ret = OB_SUCCESS; - if (!is_valid()) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ha status do not init", K(ret), KPC(this)); - } else { - data_status = data_status_; - } - return ret; -} - -int ObTabletHAStatus::set_expected_status(const ObTabletExpectedStatus::STATUS &expected_status) -{ - int ret = OB_SUCCESS; - if (!ObTabletExpectedStatus::is_valid(expected_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("set self status get invalid argument", K(ret), K(expected_status)); - } else { - expected_status_ = expected_status; - } - return ret; -} - -int ObTabletHAStatus::get_expected_status(ObTabletExpectedStatus::STATUS &expected_status) const -{ - int ret = OB_SUCCESS; - expected_status = ObTabletExpectedStatus::EXPECTED_STATUS_MAX; - if (!is_valid()) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ha status do not init", K(ret), KPC(this)); - } else { - expected_status = expected_status_; - } - return ret; -} - -int ObTabletHAStatus::init_status() -{ - int ret = OB_SUCCESS; - restore_status_ = ObTabletRestoreStatus::FULL; - data_status_ = ObTabletDataStatus::COMPLETE; - expected_status_ = ObTabletExpectedStatus::NORMAL; - return ret; -} - - -void ObTabletHAStatus::reset() -{ - restore_status_ = ObTabletRestoreStatus::STATUS::RESTORE_STATUS_MAX; - data_status_ = ObTabletDataStatus::STATUS::DATA_STATUS_MAX; - expected_status_ = ObTabletExpectedStatus::EXPECTED_STATUS_MAX; - reserved_ = 0; -} - - -} -} diff --git a/src/storage/ob_tablet_ha_status.h b/src/storage/ob_tablet_ha_status.h deleted file mode 100644 index 5720c1645..000000000 --- a/src/storage/ob_tablet_ha_status.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEABASE_STORAGE_TABLET_HA_STATUS_ -#define OCEABASE_STORAGE_TABLET_HA_STATUS_ - -#include -#include "lib/utility/ob_print_utils.h" -#include "share/ob_force_print_log.h" - -namespace oceanbase -{ -namespace storage -{ - -// status for tablet restore -class ObTabletRestoreStatus final -{ -public: - enum STATUS : uint8_t - { - FULL = 0, // both minor and major data are complete - EMPTY = 1, // both minor and major data are empty, wait restore - MINOR_AND_MAJOR_META = 2, // complete minor data with only major sst meta - PENDING = 3, // not sure if it is a valid tablet, need to confirm with backup media - UNDEFINED = 4, // invalid tablet, a placeholder - REMOTE = 5, - RESTORE_STATUS_MAX - }; -public: - ObTabletRestoreStatus() = default; - ~ObTabletRestoreStatus() = default; -public: - static bool is_valid(const ObTabletRestoreStatus::STATUS &status); - static bool is_full(const ObTabletRestoreStatus::STATUS &status) { return STATUS::FULL == status; } - static bool is_empty(const ObTabletRestoreStatus::STATUS &status) { return STATUS::EMPTY == status; } - static bool is_minor_and_major_meta(const ObTabletRestoreStatus::STATUS &status) { return STATUS::MINOR_AND_MAJOR_META == status; } - static bool is_pending(const ObTabletRestoreStatus::STATUS &status) { return STATUS::PENDING == status; } - static bool is_undefined(const ObTabletRestoreStatus::STATUS &status) { return STATUS::UNDEFINED == status; } - static bool is_remote(const ObTabletRestoreStatus::STATUS &status) { return STATUS::REMOTE == status; } - static int check_can_change_status( - const ObTabletRestoreStatus::STATUS &cur_status, - const ObTabletRestoreStatus::STATUS &change_status, - bool &can_change); -}; - -class ObTabletDataStatus final -{ -public: - enum STATUS : uint8_t - { - COMPLETE = 0, // minor and major data are complete - INCOMPLETE = 1, // minor or major data is incomplete - DATA_STATUS_MAX - }; -public: - ObTabletDataStatus() = default; - ~ObTabletDataStatus() = default; - - static bool is_valid(const ObTabletDataStatus::STATUS &status); - static bool is_complete(const ObTabletDataStatus::STATUS &status) { return STATUS::COMPLETE == status; } - static bool is_incomplete(const ObTabletDataStatus::STATUS &status) { return STATUS::INCOMPLETE == status; } - static int check_can_change_status( - const ObTabletDataStatus::STATUS &cur_status, - const ObTabletDataStatus::STATUS &change_status, - bool &can_change); -}; - -class ObTabletExpectedStatus final -{ -public: - enum STATUS : uint8_t - { - NORMAL = 0, - DELETED = 1, - EXPECTED_STATUS_MAX - }; -public: - ObTabletExpectedStatus() = default; - ~ObTabletExpectedStatus() = default; - static bool is_valid(const ObTabletExpectedStatus::STATUS &status); - static bool is_normal(const ObTabletExpectedStatus::STATUS &status) { return STATUS::NORMAL == status; } - static bool is_deleted(const ObTabletExpectedStatus::STATUS &status) { return STATUS::DELETED == status; } - static int check_can_change_status( - const ObTabletExpectedStatus::STATUS &cur_status, - const ObTabletExpectedStatus::STATUS &change_status, - bool &can_change); -}; - -class ObTabletHAStatus final -{ -public: - ObTabletHAStatus(); - ~ObTabletHAStatus() = default; - bool is_valid() const; - int serialize(char *buf, const int64_t len, int64_t &pos) const; - int deserialize(const char *buf, const int64_t len, int64_t &pos); - int64_t get_serialize_size() const; - void reset(); - int64_t get_ha_status() const { return ha_status_; } - void set_ha_status(int64_t ha_status) { ha_status_ = ha_status;} - bool is_none() const { return is_data_status_complete() && is_restore_status_full(); } - bool is_data_status_complete() const { return ObTabletDataStatus::is_complete(data_status_); } - bool is_restore_status_full() const { return ObTabletRestoreStatus::is_full(restore_status_); } - bool is_restore_status_pending() const { return ObTabletRestoreStatus::is_pending(restore_status_); } - bool is_restore_status_undefined() const { return ObTabletRestoreStatus::is_undefined(restore_status_); } - bool is_restore_status_empty() const { return ObTabletRestoreStatus::is_empty(restore_status_); } - bool is_restore_status_minor_and_major_meta() const { return ObTabletRestoreStatus::is_minor_and_major_meta(restore_status_); } - bool is_restore_status_remote() const { return ObTabletRestoreStatus::is_remote(restore_status_); } - bool is_expected_status_normal() const { return ObTabletExpectedStatus::is_normal(expected_status_); } - bool is_expected_status_deleted() const { return ObTabletExpectedStatus::is_deleted(expected_status_); } - - int set_restore_status(const ObTabletRestoreStatus::STATUS &restore_status); - int get_restore_status(ObTabletRestoreStatus::STATUS &restore_status) const; - int set_data_status(const ObTabletDataStatus::STATUS &data_status); - int get_data_status(ObTabletDataStatus::STATUS &data_status) const; - int set_expected_status(const ObTabletExpectedStatus::STATUS &expected_status); - int get_expected_status(ObTabletExpectedStatus::STATUS &expected_status) const; - int init_status(); - bool is_valid_for_backup() const { return ObTabletDataStatus::is_complete(data_status_) && ObTabletRestoreStatus::is_full(restore_status_); } - - inline bool check_allow_read() const - { - bool bool_ret = is_restore_status_full() || is_restore_status_remote(); - bool_ret &= is_data_status_complete(); - return bool_ret; - } - - TO_STRING_KV(K_(restore_status), K_(data_status), K_(expected_status), K_(reserved)); -public: - static const uint64_t SF_BIT_RESTORE_STATUS = 8; - static const uint64_t SF_BIT_DATA_STATUS = 8; - static const uint64_t SF_BIT_EXPECTED_STATUS = 8; - static const uint64_t SF_BIT_RESERVED = 40; -private: - union { - int64_t ha_status_; - struct { - ObTabletRestoreStatus::STATUS restore_status_ : SF_BIT_RESTORE_STATUS; - ObTabletDataStatus::STATUS data_status_ : SF_BIT_DATA_STATUS; - ObTabletExpectedStatus::STATUS expected_status_ : SF_BIT_EXPECTED_STATUS; - uint64_t reserved_ : SF_BIT_RESERVED; - }; - }; -}; - - - -} -} - -#endif diff --git a/src/storage/ob_tablet_restore_state.cpp b/src/storage/ob_tablet_restore_state.cpp new file mode 100644 index 000000000..0b75f9a07 --- /dev/null +++ b/src/storage/ob_tablet_restore_state.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/ob_tablet_restore_state.h" +#include "lib/utility/serialization.h" + +namespace oceanbase +{ +namespace storage +{ + +bool ObTabletRestoreStatus::is_valid(const STATUS status) +{ + return FULL <= status && status < RESTORE_STATUS_MAX; +} + +int ObTabletRestoreStatus::check_can_change_status( + const STATUS cur_status, + const STATUS change_status, + bool &can_change) +{ + int ret = OB_SUCCESS; + can_change = false; + + if (!is_valid(cur_status) || !is_valid(change_status)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid restore status", K(ret), K(cur_status), K(change_status)); + } else { + switch (cur_status) { + case PENDING: + can_change = UNDEFINED == change_status || EMPTY == change_status || PENDING == change_status; + break; + case UNDEFINED: + can_change = PENDING == change_status || UNDEFINED == change_status; + break; + case EMPTY: + can_change = MINOR_AND_MAJOR_META == change_status || REMOTE == change_status + || FULL == change_status || EMPTY == change_status || UNDEFINED == change_status; + break; + case MINOR_AND_MAJOR_META: + can_change = FULL == change_status || MINOR_AND_MAJOR_META == change_status; + break; + case REMOTE: + can_change = FULL == change_status || REMOTE == change_status; + break; + case FULL: + can_change = FULL == change_status || EMPTY == change_status; + break; + default: + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid restore status", K(ret), K(cur_status)); + break; + } + } + return ret; +} + +ObTabletRestoreState::ObTabletRestoreState() + : status_(ObTabletRestoreStatus::RESTORE_STATUS_MAX) +{ +} + +bool ObTabletRestoreState::is_valid() const +{ + return ObTabletRestoreStatus::is_valid(status_); +} + +int ObTabletRestoreState::serialize(char *buf, const int64_t len, int64_t &pos) const +{ + return serialization::encode_i8(buf, len, pos, static_cast(status_)); +} + +int ObTabletRestoreState::deserialize(const char *buf, const int64_t len, int64_t &pos) +{ + int ret = OB_SUCCESS; + int8_t status = 0; + if (OB_FAIL(serialization::decode_i8(buf, len, pos, &status))) { + LOG_WARN("failed to deserialize restore state", K(ret), K(len), K(pos)); + } else if (!ObTabletRestoreStatus::is_valid(static_cast(status))) { + ret = OB_INVALID_DATA; + LOG_WARN("invalid serialized restore state", K(ret), K(status)); + } else { + status_ = static_cast(status); + } + return ret; +} + +int64_t ObTabletRestoreState::get_serialize_size() const +{ + return serialization::encoded_length_i8(static_cast(status_)); +} + +void ObTabletRestoreState::reset() +{ + status_ = ObTabletRestoreStatus::RESTORE_STATUS_MAX; +} + +int ObTabletRestoreState::init_status() +{ + status_ = ObTabletRestoreStatus::FULL; + return OB_SUCCESS; +} + +int ObTabletRestoreState::set_restore_status(const ObTabletRestoreStatus::STATUS status) +{ + int ret = OB_SUCCESS; + if (!ObTabletRestoreStatus::is_valid(status)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid restore status", K(ret), K(status)); + } else { + status_ = status; + } + return ret; +} + +int ObTabletRestoreState::get_restore_status(ObTabletRestoreStatus::STATUS &status) const +{ + int ret = OB_SUCCESS; + if (!is_valid()) { + ret = OB_NOT_INIT; + LOG_WARN("restore state is not initialized", K(ret), KPC(this)); + } else { + status = status_; + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/ob_tablet_restore_state.h b/src/storage/ob_tablet_restore_state.h new file mode 100644 index 000000000..0f988610c --- /dev/null +++ b/src/storage/ob_tablet_restore_state.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_STORAGE_TABLET_RESTORE_STATE_ +#define OCEANBASE_STORAGE_TABLET_RESTORE_STATE_ + +#include +#include "lib/utility/ob_print_utils.h" + +namespace oceanbase +{ +namespace storage +{ + +class ObTabletRestoreStatus final +{ +public: + enum STATUS : uint8_t + { + FULL = 0, + EMPTY = 1, + MINOR_AND_MAJOR_META = 2, + PENDING = 3, + UNDEFINED = 4, + REMOTE = 5, + RESTORE_STATUS_MAX + }; + + static bool is_valid(const STATUS status); + static bool is_full(const STATUS status) { return FULL == status; } + static bool is_empty(const STATUS status) { return EMPTY == status; } + static bool is_minor_and_major_meta(const STATUS status) { return MINOR_AND_MAJOR_META == status; } + static bool is_pending(const STATUS status) { return PENDING == status; } + static bool is_undefined(const STATUS status) { return UNDEFINED == status; } + static bool is_remote(const STATUS status) { return REMOTE == status; } + static int check_can_change_status( + const STATUS cur_status, + const STATUS change_status, + bool &can_change); +}; + +class ObTabletRestoreState final +{ +public: + ObTabletRestoreState(); + ~ObTabletRestoreState() = default; + + bool is_valid() const; + int serialize(char *buf, const int64_t len, int64_t &pos) const; + int deserialize(const char *buf, const int64_t len, int64_t &pos); + int64_t get_serialize_size() const; + void reset(); + int init_status(); + + bool is_restore_status_full() const { return ObTabletRestoreStatus::is_full(status_); } + bool is_restore_status_pending() const { return ObTabletRestoreStatus::is_pending(status_); } + bool is_restore_status_undefined() const { return ObTabletRestoreStatus::is_undefined(status_); } + bool is_restore_status_empty() const { return ObTabletRestoreStatus::is_empty(status_); } + bool is_restore_status_minor_and_major_meta() const + { + return ObTabletRestoreStatus::is_minor_and_major_meta(status_); + } + bool is_restore_status_remote() const { return ObTabletRestoreStatus::is_remote(status_); } + bool check_allow_read() const + { + return is_restore_status_full() || is_restore_status_remote(); + } + + int set_restore_status(const ObTabletRestoreStatus::STATUS status); + int get_restore_status(ObTabletRestoreStatus::STATUS &status) const; + int64_t get_state_value() const { return static_cast(status_); } + + TO_STRING_KV(K_(status)); + +private: + ObTabletRestoreStatus::STATUS status_; +}; + +} // namespace storage +} // namespace oceanbase + +#endif diff --git a/src/storage/ob_tenant_tablet_stat_mgr.cpp b/src/storage/ob_tenant_tablet_stat_mgr.cpp index 44894354a..eb1933746 100644 --- a/src/storage/ob_tenant_tablet_stat_mgr.cpp +++ b/src/storage/ob_tenant_tablet_stat_mgr.cpp @@ -47,18 +47,14 @@ void ObTransNodeDMLStat::atomic_inc(const ObTransNodeDMLStat &other) /************************************* ObTabletStatKey *************************************/ ObTabletStatKey::ObTabletStatKey( - const int64_t ls_id, const uint64_t tablet_id) - : ls_id_(ls_id), - tablet_id_(tablet_id) + : tablet_id_(tablet_id) { } ObTabletStatKey::ObTabletStatKey( - const share::ObLSID ls_id, const ObTabletID tablet_id) - : ls_id_(ls_id), - tablet_id_(tablet_id) + : tablet_id_(tablet_id) { } @@ -68,15 +64,13 @@ ObTabletStatKey::~ObTabletStatKey() void ObTabletStatKey::reset() { - ls_id_.reset(); tablet_id_.reset(); } uint64_t ObTabletStatKey::hash() const { uint64_t hash_val = 0; - hash_val += ls_id_.hash(); - hash_val += tablet_id_.hash(); + hash_val = tablet_id_.hash(); return hash_val; } @@ -88,14 +82,14 @@ int ObTabletStatKey::hash(uint64_t &hash_val) const bool ObTabletStatKey::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } bool ObTabletStatKey::operator==(const ObTabletStatKey &other) const { bool bret = true; if (this == &other) { - } else if (ls_id_ != other.ls_id_ || tablet_id_ != other.tablet_id_) { + } else if (tablet_id_ != other.tablet_id_) { bret = false; } return bret; @@ -106,7 +100,7 @@ bool ObTabletStatKey::operator==(const ObTabletStatKey &other) const /************************************* ObTabletStat *************************************/ bool ObTabletStat::is_valid() const { - return ls_id_ > 0 && tablet_id_ > 0; + return tablet_id_ > 0; } bool ObTabletStat::check_need_report() const @@ -149,7 +143,6 @@ ObTabletStat& ObTabletStat::operator=(const ObTabletStat &other) ObTabletStat& ObTabletStat::operator+=(const ObTabletStat &other) { if (other.is_valid()) { - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; query_cnt_ += other.query_cnt_; merge_cnt_ += other.merge_cnt_; @@ -341,11 +334,10 @@ void ObTabletStream::clear_stat() void ObTabletStream::add_stat(const ObTabletStat &stat) { if (!key_.is_valid()) { - key_.ls_id_ = stat.ls_id_; key_.tablet_id_ = stat.tablet_id_; } - if (key_.ls_id_.id() == stat.ls_id_ && key_.tablet_id_.id() == stat.tablet_id_) { + if (key_.tablet_id_.id() == stat.tablet_id_) { curr_buckets_.add(stat); total_stat_ += stat; } @@ -744,7 +736,6 @@ int ObTenantTabletStatMgr::report_stat( } int ObTenantTabletStatMgr::get_latest_tablet_stat( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletStat &tablet_stat, ObTabletStat &total_tablet_stat, @@ -752,16 +743,15 @@ int ObTenantTabletStatMgr::get_latest_tablet_stat( { int ret = OB_SUCCESS; tablet_stat.reset(); - tablet_stat.ls_id_ = ls_id.id(); tablet_stat.tablet_id_ = tablet_id.id(); - const ObTabletStatKey key(ls_id, tablet_id); + const ObTabletStatKey key(tablet_id); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantTabletStatMgr not inited", K(ret)); } else if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get invalid arguments", K(ret), K(tablet_id)); } else { ObTabletStreamNode *stream_node = nullptr; ObBucketHashRLockGuard lock_guard(bucket_lock_, key.hash()); @@ -779,18 +769,17 @@ int ObTenantTabletStatMgr::get_latest_tablet_stat( } int ObTenantTabletStatMgr::clear_tablet_stat( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - const ObTabletStatKey key(ls_id, tablet_id); + const ObTabletStatKey key(tablet_id); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantTabletStatMgr not inited", K(ret)); } else if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get invalid arguments", K(ret), K(tablet_id)); } else { ObBucketHashWLockGuard lock_guard(bucket_lock_, key.hash()); if (OB_FAIL(inner_clear_tablet_stat(key))) { @@ -798,7 +787,7 @@ int ObTenantTabletStatMgr::clear_tablet_stat( } } if (OB_SUCC(ret)) { - FLOG_INFO("clear tablet stat", K(ret), K(ls_id), K(tablet_id)); + FLOG_INFO("clear tablet stat", K(ret), K(tablet_id)); } return ret; } @@ -831,15 +820,14 @@ int ObTenantTabletStatMgr::get_all_tablet_stats( int ObTenantTabletStatMgr::get_tablet_analyzer( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletStatAnalyzer &analyzer) { int ret = OB_SUCCESS; - if (OB_FAIL(get_latest_tablet_stat(ls_id, tablet_id, analyzer.tablet_stat_, analyzer.total_tablet_stat_, analyzer.mode_))) { + if (OB_FAIL(get_latest_tablet_stat(tablet_id, analyzer.tablet_stat_, analyzer.total_tablet_stat_, analyzer.mode_))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get latest tablet stat", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get latest tablet stat", K(ret), K(tablet_id)); } } else { const ObTableQueuingModeCfg &queuing_cfg = ObTableQueuingModeCfg::get_basic_config(analyzer.mode_); @@ -867,7 +855,6 @@ int ObTenantTabletStatMgr::inner_clear_tablet_stat(const ObTabletStatKey &key) } int ObTenantTabletStatMgr::batch_clear_tablet_stat( - const share::ObLSID &ls_id, const ObIArray &tablet_ids) { int ret = OB_SUCCESS; @@ -881,7 +868,6 @@ int ObTenantTabletStatMgr::batch_clear_tablet_stat( LOG_TRACE("tablet_ids empty, no need to clear"); } else { ObTabletStatKey key; - key.ls_id_ = ls_id; ObBucketWLockAllGuard lock_guard(bucket_lock_); for (int64_t idx = 0; idx < tablet_cnt; idx++) { key.tablet_id_ = tablet_ids.at(idx); @@ -895,7 +881,7 @@ int ObTenantTabletStatMgr::batch_clear_tablet_stat( } } if (OB_SUCC(ret)) { - FLOG_INFO("batch clear tablet stat in ls", K(ret), K(ls_id), K(tablet_cnt), K(clear_cnt)); + FLOG_INFO("batch clear tablet stat", K(ret), K(tablet_cnt), K(clear_cnt)); } } return ret; @@ -905,7 +891,7 @@ int ObTenantTabletStatMgr::update_tablet_stream(const ObTabletStat &report_stat) { int ret = OB_SUCCESS; ObTabletStreamNode *stream_node = nullptr; - ObTabletStatKey key(report_stat.ls_id_, report_stat.tablet_id_); + ObTabletStatKey key(report_stat.tablet_id_); { ObBucketHashRLockGuard lock_guard(bucket_lock_, key.hash()); ret = stream_map_.get_refactored(key, stream_node); @@ -1114,18 +1100,17 @@ void ObTenantTabletStatMgr::refresh_queuing_mode() } int ObTenantTabletStatMgr::get_queuing_cfg( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTableQueuingModeCfg& queuing_cfg) { int ret = OB_SUCCESS; - const ObTabletStatKey key(ls_id, tablet_id); + const ObTabletStatKey key(tablet_id); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantTabletStatMgr not inited", K(ret)); } else if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid arguments", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("get invalid arguments", K(ret), K(tablet_id)); } else { ObTabletStreamNode *stream_node = nullptr; ObBucketHashRLockGuard lock_guard(bucket_lock_, key.hash()); @@ -1137,7 +1122,7 @@ int ObTenantTabletStatMgr::get_queuing_cfg( } } else { queuing_cfg = ObTableQueuingModeCfg::get_basic_config(stream_node->mode_); - LOG_DEBUG("success get queuing cfg", K(ret), K(ls_id), K(tablet_id), K(queuing_cfg)); + LOG_DEBUG("success get queuing cfg", K(ret), K(tablet_id), K(queuing_cfg)); } } return ret; diff --git a/src/storage/ob_tenant_tablet_stat_mgr.h b/src/storage/ob_tenant_tablet_stat_mgr.h index 0342d0a32..a935c0a4f 100644 --- a/src/storage/ob_tenant_tablet_stat_mgr.h +++ b/src/storage/ob_tenant_tablet_stat_mgr.h @@ -18,7 +18,6 @@ #define OCEANBASE_STORAGE_TENANT_TABLET_STAT_MGR_H_ #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "lib/hash/ob_hashmap.h" #include "share/rc/ob_tenant_base.h" #include "lib/task/ob_timer.h" @@ -58,18 +57,17 @@ struct ObTransNodeDMLStat struct ObTabletStatKey { public: - ObTabletStatKey() : ls_id_(), tablet_id_() {} - ObTabletStatKey(const int64_t ls_id, const uint64_t tablet_id); - ObTabletStatKey(const share::ObLSID ls_id, const common::ObTabletID tablet_id); + ObTabletStatKey() : tablet_id_() {} + explicit ObTabletStatKey(const uint64_t tablet_id); + explicit ObTabletStatKey(const common::ObTabletID tablet_id); ~ObTabletStatKey(); void reset(); uint64_t hash() const; int hash(uint64_t &hash_val) const; bool is_valid() const; bool operator == (const ObTabletStatKey &other) const; - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; @@ -85,7 +83,7 @@ struct ObTabletStat int64_t get_total_merge_row_count() const { return insert_row_cnt_ + update_row_cnt_ + delete_row_cnt_; } ObTabletStat& operator=(const ObTabletStat &other); ObTabletStat& operator+=(const ObTabletStat &other); - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(query_cnt), K_(merge_cnt), K_(scan_logical_row_cnt), + TO_STRING_KV(K_(tablet_id), K_(query_cnt), K_(merge_cnt), K_(scan_logical_row_cnt), K_(scan_physical_row_cnt), K_(scan_micro_block_cnt), K_(pushdown_micro_block_cnt), K_(exist_row_total_table_cnt), K_(exist_row_read_table_cnt), K_(insert_row_cnt), K_(update_row_cnt), K_(delete_row_cnt)); @@ -94,7 +92,6 @@ struct ObTabletStat static constexpr int64_t QUERY_REPORT_INEFFICIENT_THRESHOLD = 3; static constexpr int64_t MERGE_REPORT_MIN_ROW_CNT = 1000; public: - int64_t ls_id_; uint64_t tablet_id_; uint32_t query_cnt_; uint32_t merge_cnt_; @@ -399,7 +396,6 @@ class ObTenantTabletStatMgr const ObTabletStat &stat, bool &succ_report); int get_latest_tablet_stat( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletStat &tablet_stat, ObTabletStat &total_tablet_stat, @@ -407,20 +403,16 @@ class ObTenantTabletStatMgr int get_all_tablet_stats( common::ObIArray &tablet_stats); int get_tablet_analyzer( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletStatAnalyzer &analyzer); int clear_tablet_stat( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); int batch_clear_tablet_stat( - const share::ObLSID &ls_id, const ObIArray &tablet_ids); void process_stats(); void refresh_all(const int64_t step); void refresh_queuing_mode(); int get_queuing_cfg( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTableQueuingModeCfg& queuing_cfg); int64_t get_last_update_time() { return report_stat_task_.last_update_time_; } diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index af3e2b6f2..5eeca400a 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -187,8 +187,8 @@ int ObRowGetter::init_dml_access_ctx( query_flag.scan_order_ = ObQueryFlag::Forward; query_flag.set_not_use_bloomfilter_cache(); query_flag.read_latest_ = ObQueryFlag::OBSF_MASK_READ_LATEST; - if (store_ctx.mvcc_acc_ctx_.write_flag_.is_plain_insert_gts_opt()) { - query_flag.set_plain_insert_gts_opt(); + if (store_ctx.mvcc_acc_ctx_.write_flag_.is_snapshot_opt()) { + query_flag.set_snapshot_opt(); } if (skip_read_lob) { query_flag.skip_read_lob_ = ObQueryFlag::OBSF_MASK_SKIP_READ_LOB; diff --git a/src/storage/retrieval/ob_block_stat_iter.cpp b/src/storage/retrieval/ob_block_stat_iter.cpp index c11a8c294..65e45e2a1 100644 --- a/src/storage/retrieval/ob_block_stat_iter.cpp +++ b/src/storage/retrieval/ob_block_stat_iter.cpp @@ -354,10 +354,7 @@ int ObBlockStatIterator::init_memtable_access_param( ObTableScanParam &scan_param) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(scan_param.is_mview_query())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("mview query not supported for block stat iterator", K(ret)); - } else if (OB_FAIL(main_table_param_.init(scan_param, &tablet_handle))) { + if (OB_FAIL(main_table_param_.init(scan_param, &tablet_handle))) { LOG_WARN("failed to init main table param", K(ret)); } else { ObVersionRange trans_version_range; @@ -409,23 +406,18 @@ int ObBlockStatIterator::refresh_tablet_iter() ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet iter is invalid", K(ret), K(get_table_param_.tablet_iter_)); } else { - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; rowkey_read_info_ = nullptr; main_table_param_.iter_param_.rowkey_read_info_ = nullptr; const int64_t remain_timeout = THIS_WORKER.get_timeout_remain(); - const share::ObLSID &ls_id = main_table_ctx_.ls_id_; const common::ObTabletID &tablet_id = get_table_param_.tablet_iter_.get_tablet()->get_tablet_meta().tablet_id_; const int64_t snapshot_version = main_table_ctx_.store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(); if (OB_UNLIKELY(remain_timeout <= 0)) { ret = OB_TIMEOUT; - LOG_WARN("timeout", K(ret), K(ls_id), K(tablet_id), K(remain_timeout)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null ptr to ls", K(ret), K(ls_handle)); - } else if (OB_FAIL(ls->get_tablet_svr()->get_read_tables( + LOG_WARN("timeout", K(ret), K(tablet_id), K(remain_timeout)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->get_read_tables( tablet_id, remain_timeout, snapshot_version, @@ -434,7 +426,7 @@ int ObBlockStatIterator::refresh_tablet_iter() false/*allow_not_ready*/, false/*need_split_src_table*/, false/*need_split_dst_table*/))) { - LOG_WARN("failed to refresh tablet iterator", K(ret), K(ls_id), K_(get_table_param)); + LOG_WARN("failed to refresh tablet iterator", K(ret), K_(get_table_param)); } else { rowkey_read_info_ = &get_table_param_.tablet_iter_.get_tablet_handle().get_obj()->get_rowkey_read_info(); main_table_param_.iter_param_.rowkey_read_info_ = rowkey_read_info_; @@ -997,4 +989,3 @@ bool ObBlockStatIterator::use_merged_range() const } // namespace storage } // namespace oceanbase - diff --git a/src/storage/retrieval/ob_spiv_daat_iter.h b/src/storage/retrieval/ob_spiv_daat_iter.h index 1d457ad19..77dc1e293 100644 --- a/src/storage/retrieval/ob_spiv_daat_iter.h +++ b/src/storage/retrieval/ob_spiv_daat_iter.h @@ -160,9 +160,6 @@ class ObSPIVDaaTIter : public ObSRDaaTIterImpl virtual int get_next_rows(const int64_t capacity, int64_t &count) override; virtual int inner_get_next_rows(const int64_t capacity, int64_t &count); virtual int process(); - virtual void set_ls_related_tablet_id(const share::ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids) - {} - protected: // for pre-filter common::hash::ObHashSet valid_docid_set_; diff --git a/src/storage/retrieval/ob_spiv_dim_iter.cpp b/src/storage/retrieval/ob_spiv_dim_iter.cpp index 912210f63..80c2d674d 100644 --- a/src/storage/retrieval/ob_spiv_dim_iter.cpp +++ b/src/storage/retrieval/ob_spiv_dim_iter.cpp @@ -39,7 +39,6 @@ int ObSPIVDaaTDimIter::init(const ObSPIVDimIterParam &iter_param) inv_idx_scan_rtdef_ = iter_param.inv_idx_scan_rtdef_; max_batch_size_ = OB_MAX(iter_param.eval_ctx_->max_batch_size_, 1); eval_ctx_ = iter_param.eval_ctx_; - ls_id_ = iter_param.ls_id_; dim_docid_value_tablet_id_ = iter_param.dim_docid_value_tablet_id_; dim_ = iter_param.dim_; cur_idx_ = -1; diff --git a/src/storage/retrieval/ob_spiv_dim_iter.h b/src/storage/retrieval/ob_spiv_dim_iter.h index 5e4bd839c..e1cf356d4 100644 --- a/src/storage/retrieval/ob_spiv_dim_iter.h +++ b/src/storage/retrieval/ob_spiv_dim_iter.h @@ -59,7 +59,6 @@ struct ObSPIVDimIterParam const ObDASScanCtDef *inv_idx_scan_ctdef_; ObDASScanRtDef *inv_idx_scan_rtdef_; sql::ObEvalCtx *eval_ctx_; - share::ObLSID ls_id_; ObTabletID dim_docid_value_tablet_id_; double query_value_; uint32_t dim_; @@ -79,7 +78,6 @@ class ObSPIVDaaTDimIter : public ObISRDaaTDimIter inv_scan_domain_id_expr_(nullptr), inv_scan_score_expr_(nullptr), eval_ctx_(nullptr), - ls_id_(), dim_docid_value_tablet_id_(), is_inited_(false), max_batch_size_(1), @@ -132,7 +130,6 @@ class ObSPIVDaaTDimIter : public ObISRDaaTDimIter const ObDASScanCtDef *inv_idx_scan_ctdef_; ObDASScanRtDef *inv_idx_scan_rtdef_; sql::ObEvalCtx *eval_ctx_; - share::ObLSID ls_id_; ObTabletID dim_docid_value_tablet_id_; bool is_inited_; int64_t max_batch_size_; diff --git a/src/storage/retrieval/ob_text_retrieval_token_iter.cpp b/src/storage/retrieval/ob_text_retrieval_token_iter.cpp index dc0bd3578..46081dceb 100644 --- a/src/storage/retrieval/ob_text_retrieval_token_iter.cpp +++ b/src/storage/retrieval/ob_text_retrieval_token_iter.cpp @@ -638,7 +638,6 @@ int ObTextRetrievalTokenIter::estimate_token_doc_cnt() est_param.index_id_ = inv_idx_agg_param_->index_id_; est_param.scan_flag_ = inv_idx_agg_param_->scan_flag_; est_param.tablet_id_ = inv_idx_agg_param_->tablet_id_; - est_param.ls_id_ = inv_idx_agg_param_->ls_id_; est_param.tx_id_ = inv_idx_agg_param_->tx_id_; est_param.schema_version_ = inv_idx_agg_param_->schema_version_; est_param.frozen_version_ = GET_BATCH_ROWS_READ_SNAPSHOT_VERSION; diff --git a/src/storage/slog/ob_storage_log.cpp b/src/storage/slog/ob_storage_log.cpp index 3883222f1..3c31e4c95 100644 --- a/src/storage/slog/ob_storage_log.cpp +++ b/src/storage/slog/ob_storage_log.cpp @@ -122,120 +122,70 @@ DEF_TO_STRING(ObLSMetaLog) OB_SERIALIZE_MEMBER(ObLSMetaLog, ls_meta_); -ObLSIDLog::ObLSIDLog(ObLSID &ls_id) - : ls_id_(ls_id) -{ -} - -bool ObLSIDLog::is_valid() const -{ - return ls_id_.is_valid(); -} - -DEF_TO_STRING(ObLSIDLog) +DEF_TO_STRING(ObLSMarkerLog) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(ls_id)); J_OBJ_END(); return pos; } -OB_SERIALIZE_MEMBER(ObLSIDLog, ls_id_); - -ObCreateTabletLog::ObCreateTabletLog(ObTablet *tablet) - : tablet_(tablet) -{ -} - -int ObCreateTabletLog::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - return tablet_->serialize(buf, buf_len, pos); -} - -int ObCreateTabletLog::deserialize(const char *buf, const int64_t data_len, int64_t &pos) -{ - // abandoned slog, skip deserialization - return OB_SUCCESS; -} - -int64_t ObCreateTabletLog::get_serialize_size() const -{ - return tablet_->get_serialize_size(); -} - -bool ObCreateTabletLog::is_valid() const -{ - return true; -} - -DEF_TO_STRING(ObCreateTabletLog) -{ - int64_t pos = 0; - J_OBJ_START(); - J_KV(KPC_(tablet)); - J_OBJ_END(); - return pos; -} +OB_SERIALIZE_MEMBER(ObLSMarkerLog); ObDeleteTabletLog::ObDeleteTabletLog() - : ls_id_(), tablet_id_() + : tablet_id_() { } -ObDeleteTabletLog::ObDeleteTabletLog(const ObLSID &ls_id, const ObTabletID &tablet_id) - : ls_id_(ls_id), tablet_id_(tablet_id) +ObDeleteTabletLog::ObDeleteTabletLog(const ObTabletID &tablet_id) + : tablet_id_(tablet_id) { } bool ObDeleteTabletLog::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } -OB_SERIALIZE_MEMBER(ObDeleteTabletLog, ls_id_, tablet_id_); +OB_SERIALIZE_MEMBER(ObDeleteTabletLog, tablet_id_); DEF_TO_STRING(ObDeleteTabletLog) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(ls_id), K_(tablet_id)); + J_KV(K_(tablet_id)); J_OBJ_END(); return pos; } ObUpdateTabletLog::ObUpdateTabletLog( - const ObLSID &ls_id, const ObTabletID &tablet_id, const ObMetaDiskAddr &disk_addr) - : ls_id_(ls_id), - tablet_id_(tablet_id), + : tablet_id_(tablet_id), disk_addr_(disk_addr) { } bool ObUpdateTabletLog::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid() && disk_addr_.is_valid(); + return tablet_id_.is_valid() && disk_addr_.is_valid(); } -OB_SERIALIZE_MEMBER(ObUpdateTabletLog, ls_id_, tablet_id_, disk_addr_); +OB_SERIALIZE_MEMBER(ObUpdateTabletLog, tablet_id_, disk_addr_); DEF_TO_STRING(ObUpdateTabletLog) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(ls_id), K_(tablet_id), K_(disk_addr)); + J_KV(K_(tablet_id), K_(disk_addr)); J_OBJ_END(); return pos; } ObEmptyShellTabletLog::ObEmptyShellTabletLog( - const ObLSID &ls_id, const ObTabletID &tablet_id, ObTablet *tablet) : version_(EMPTY_SHELL_SLOG_VERSION), - ls_id_(ls_id), tablet_id_(tablet_id), tablet_(tablet) { @@ -249,8 +199,6 @@ int ObEmptyShellTabletLog::serialize( int ret = OB_SUCCESS; if (OB_FAIL(serialization::encode(buf, data_len, pos, version_))) { STORAGE_LOG(WARN, "deserialize version_ failed", K(ret), KP(data_len), K(pos)); - } else if (OB_FAIL(ls_id_.serialize(buf, data_len, pos))) { - STORAGE_LOG(WARN, "deserialize ls_id_ failed", K(ret), KP(data_len), K(pos)); } else if (OB_FAIL(tablet_id_.serialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "deserialize tablet_id_ failed", K(ret), KP(data_len), K(pos)); } else if (OB_FAIL(tablet_->serialize(buf, data_len, pos))) { @@ -268,8 +216,6 @@ int ObEmptyShellTabletLog::deserialize_id( int ret = OB_SUCCESS; if (OB_FAIL(serialization::decode(buf, data_len, pos, version_))) { STORAGE_LOG(WARN, "deserialize version_ failed", K(ret), KP(data_len), K(pos)); - } else if (OB_FAIL(ls_id_.deserialize(buf, data_len, pos))) { - STORAGE_LOG(WARN, "deserialize ls_id_ failed", K(ret), KP(data_len), K(pos)); } else if (OB_FAIL(tablet_id_.deserialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "deserialize tablet_id_ failed", K(ret), KP(data_len), K(pos)); } @@ -285,8 +231,6 @@ int ObEmptyShellTabletLog::deserialize( int ret = OB_SUCCESS; if (OB_FAIL(serialization::decode(buf, data_len, pos, version_))) { STORAGE_LOG(WARN, "deserialize version_ failed", K(ret), KP(data_len), K(pos)); - } else if (OB_FAIL(ls_id_.deserialize(buf, data_len, pos))) { - STORAGE_LOG(WARN, "deserialize ls_id_ failed", K(ret), KP(data_len), K(pos)); } else if (OB_FAIL(tablet_id_.deserialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "deserialize tablet_id_ failed", K(ret), KP(data_len), K(pos)); } else if (OB_FAIL(tablet_->deserialize(buf, data_len, pos))) { @@ -300,7 +244,6 @@ int64_t ObEmptyShellTabletLog::get_serialize_size() const { int64_t size = 0; size += serialization::encoded_length(version_); - size += ls_id_.get_serialize_size(); size += tablet_id_.get_serialize_size(); size += tablet_->get_serialize_size(); return size; @@ -308,14 +251,14 @@ int64_t ObEmptyShellTabletLog::get_serialize_size() const bool ObEmptyShellTabletLog::is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } DEF_TO_STRING(ObEmptyShellTabletLog) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(ls_id), K_(tablet_id)); + J_KV(K_(tablet_id)); J_OBJ_END(); return pos; } diff --git a/src/storage/slog/ob_storage_log.h b/src/storage/slog/ob_storage_log.h index 4a54de6a6..aae73551f 100644 --- a/src/storage/slog/ob_storage_log.h +++ b/src/storage/slog/ob_storage_log.h @@ -29,7 +29,6 @@ namespace oceanbase namespace share { - class ObLSID; } namespace storage { @@ -132,47 +131,28 @@ struct ObLSMetaLog : public ObIBaseStorageLogEntry ObLSMeta ls_meta_; }; -struct ObLSIDLog : public ObIBaseStorageLogEntry +struct ObLSMarkerLog : public ObIBaseStorageLogEntry { public: - explicit ObLSIDLog(share::ObLSID &ls_id); - virtual ~ObLSIDLog() {} - virtual bool is_valid() const override; + ObLSMarkerLog() = default; + virtual ~ObLSMarkerLog() = default; + virtual bool is_valid() const override { return true; } DECLARE_TO_STRING; OB_UNIS_VERSION_V(1); -protected: - share::ObLSID &ls_id_; }; using ObCreateLSPrepareSlog = ObLSMetaLog; -using ObCreateLSAbortSLog = ObLSIDLog; -using ObCreateLSCommitSLog = ObLSIDLog; -using ObDeleteLSLog = ObLSIDLog; - -struct ObCreateTabletLog : public ObIBaseStorageLogEntry -{ -public: - ObCreateTabletLog() {} - explicit ObCreateTabletLog(ObTablet *tablet); - virtual ~ObCreateTabletLog() {} - virtual int serialize(char *buf, const int64_t buf_len, int64_t &pos) const override; - virtual int deserialize(const char *buf, const int64_t data_len, int64_t &pos) override; - virtual int64_t get_serialize_size() const override; - virtual bool is_valid() const override; - - DECLARE_TO_STRING; - -public: - ObTablet *tablet_; -}; +using ObCreateLSAbortSLog = ObLSMarkerLog; +using ObCreateLSCommitSLog = ObLSMarkerLog; +using ObDeleteLSLog = ObLSMarkerLog; struct ObDeleteTabletLog : public ObIBaseStorageLogEntry { public: ObDeleteTabletLog(); - ObDeleteTabletLog(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); + explicit ObDeleteTabletLog(const common::ObTabletID &tablet_id); virtual ~ObDeleteTabletLog() {} virtual bool is_valid() const override; @@ -180,7 +160,6 @@ struct ObDeleteTabletLog : public ObIBaseStorageLogEntry OB_UNIS_VERSION_V(1); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; @@ -189,7 +168,6 @@ struct ObUpdateTabletLog : public ObIBaseStorageLogEntry public: ObUpdateTabletLog() = default; ObUpdateTabletLog( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObMetaDiskAddr &disk_addr); virtual ~ObUpdateTabletLog() = default; @@ -197,7 +175,6 @@ struct ObUpdateTabletLog : public ObIBaseStorageLogEntry DECLARE_TO_STRING; OB_UNIS_VERSION_V(1); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; ObMetaDiskAddr disk_addr_; }; @@ -205,10 +182,10 @@ struct ObUpdateTabletLog : public ObIBaseStorageLogEntry struct ObEmptyShellTabletLog : public ObIBaseStorageLogEntry { public: - const int64_t EMPTY_SHELL_SLOG_VERSION = 1; + const int64_t EMPTY_SHELL_SLOG_VERSION = 2; public: ObEmptyShellTabletLog() = default; - explicit ObEmptyShellTabletLog(const ObLSID &ls_id_, const ObTabletID &tablet_id, ObTablet *tablet); + explicit ObEmptyShellTabletLog(const ObTabletID &tablet_id, ObTablet *tablet); virtual ~ObEmptyShellTabletLog() {} virtual bool is_valid() const override; virtual int serialize( @@ -228,7 +205,6 @@ struct ObEmptyShellTabletLog : public ObIBaseStorageLogEntry DECLARE_TO_STRING; public: int64_t version_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; ObTablet *tablet_; }; diff --git a/src/storage/slog/ob_storage_log_struct.h b/src/storage/slog/ob_storage_log_struct.h index 4afa4ad81..ef65e8901 100644 --- a/src/storage/slog/ob_storage_log_struct.h +++ b/src/storage/slog/ob_storage_log_struct.h @@ -52,12 +52,11 @@ enum class ObRedoLogSubType OB_REDO_LOG_UPDATE_LS = 12, OB_REDO_LOG_DELETE_LS = 13, - OB_REDO_LOG_PUT_OLD_TABLET = 14, // discard from 4.1 - OB_REDO_LOG_DELETE_TABLET = 15, - OB_REDO_LOG_UPDATE_TABLET = 16, - OB_REDO_LOG_EMPTY_SHELL_TABLET = 17, + OB_REDO_LOG_DELETE_TABLET = 14, + OB_REDO_LOG_UPDATE_TABLET = 15, + OB_REDO_LOG_EMPTY_SHELL_TABLET = 16, - OB_REDO_LOG_UPDATE_DUP_TABLE_LS = 18, + OB_REDO_LOG_UPDATE_DUP_TABLE_LS = 17, OB_REDO_LOG_MAX }; diff --git a/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.cpp b/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.cpp index 5d6bc875b..108d0efa5 100644 --- a/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.cpp +++ b/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.cpp @@ -160,8 +160,6 @@ int ObServerCheckpointSlogHandler::do_post_replay_work() LOG_WARN("not init", K(ret)); } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.first_mark_device())) { // mark must after finish replay slog LOG_WARN("fail to first mark device", K(ret)); - } else if (OB_FAIL(try_write_checkpoint_for_compat())) { - LOG_WARN("fail to try write checkpoint for compat", K(ret)); } else { tenant_meta_valid_for_replay_ = false; } @@ -184,47 +182,6 @@ void ObServerCheckpointSlogHandler::destroy() task_timer_.destroy(); } -int ObServerCheckpointSlogHandler::try_write_checkpoint_for_compat() -{ - int ret = OB_SUCCESS; - omt::ObTenantMeta meta; - bool exist = false; - omt::ObMultiTenant *omt = GCTX.omt_; - if (OB_ISNULL(omt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, omt is nullptr", K(ret)); - } else if (OB_FAIL(omt->get_tenant_meta_for_ckpt(meta, exist))) { - LOG_WARN("fail to get tenant meta", K(ret), KP(omt)); - } else { - bool need_svr_ckpt = false; - if (exist && meta.super_block_.is_old_version()) { - ObTenantSuperBlock &super_block = meta.super_block_; - { - MOD_SCOPE { - if (OB_FAIL(share::g_mp->tenant_storage_meta_service()->write_checkpoint(true/*is_force*/))) { - LOG_WARN("fail to write tenant slog checkpoint", K(ret)); - } else { - // we don't write checkpoint or update super_block for hidden tenant - // so it is necessary to update version here - if (super_block.is_hidden_) { - super_block.version_ = ObTenantSuperBlock::TENANT_SUPER_BLOCK_VERSION; - omt::ObTenant *tenant = static_cast(share::ObTenantEnv::get_tenant()); - tenant->set_tenant_super_block(super_block); - } - need_svr_ckpt = true; - } - } - } - } - if (OB_SUCC(ret)) { - if (need_svr_ckpt && OB_FAIL(write_checkpoint(true/*is_force*/))) { - LOG_ERROR("fail to write server checkpoint", K(ret)); - } - } - } - return ret; -} - int ObServerCheckpointSlogHandler::read_checkpoint(const ObServerSuperBlock &super_block) { int ret = OB_SUCCESS; diff --git a/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.h b/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.h index f65217804..65b4212e5 100644 --- a/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.h +++ b/src/storage/slog_ckpt/ob_server_checkpoint_slog_handler.h @@ -79,7 +79,6 @@ class ObServerCheckpointSlogHandler : public ObIRedoModule private: virtual int parse(const int32_t cmd, const char *buf, const int64_t len, FILE *stream) override; - int try_write_checkpoint_for_compat(); int read_checkpoint(const ObServerSuperBlock &super_block); int replay_and_apply_server_slog(const common::ObLogCursor &replay_start_point); int replay_server_slog(const common::ObLogCursor &replay_start_point, common::ObLogCursor &replay_finish_point); diff --git a/src/storage/slog_ckpt/ob_tablet_replay_create_handler.cpp b/src/storage/slog_ckpt/ob_tablet_replay_create_handler.cpp index 8b93ddf32..1a7faf6bd 100644 --- a/src/storage/slog_ckpt/ob_tablet_replay_create_handler.cpp +++ b/src/storage/slog_ckpt/ob_tablet_replay_create_handler.cpp @@ -238,8 +238,6 @@ int ObTabletReplayCreateHandler::concurrent_replay(ObStartupAccelTaskHandler* st } else { int64_t tablet_cnt_in_block = 0; int64_t valid_size_in_block = 0; - const bool is_old_version = static_cast(MTL_CTX())->get_super_block().is_old_version(); - // <1> ObMetaDiskAddr order is FILE < BLOCK < RAW_BLOCK, so handle FILE type addrs firstly int64_t i = 0; while (i < total_tablet_cnt_ && total_tablet_item_arr_[i].addr_.is_file()) { @@ -259,7 +257,7 @@ int ObTabletReplayCreateHandler::concurrent_replay(ObStartupAccelTaskHandler* st valid_size_in_block += upper_align(total_tablet_item_arr_[i].addr_.size(), 4096); } else { ADD_ITEM_RANGE_TO_TASK( - startup_accel_handler, i - tablet_cnt_in_block, i, is_old_version); // [start_item_idx, end_item_idx) + startup_accel_handler, i - tablet_cnt_in_block, i, false); // [start_item_idx, end_item_idx) pre_block_id = curr_block_id; tablet_cnt_in_block = 1; valid_size_in_block = upper_align(total_tablet_item_arr_[i].addr_.size(), 4096); @@ -267,7 +265,7 @@ int ObTabletReplayCreateHandler::concurrent_replay(ObStartupAccelTaskHandler* st } if (OB_SUCC(ret)) { // handle last range ADD_ITEM_RANGE_TO_TASK( - startup_accel_handler, total_tablet_cnt_ - tablet_cnt_in_block, total_tablet_cnt_, is_old_version); + startup_accel_handler, total_tablet_cnt_ - tablet_cnt_in_block, total_tablet_cnt_, false); } // handle last task ADD_LAST_TASK(startup_accel_handler, aggrgate_task_); @@ -358,20 +356,14 @@ int ObTabletReplayCreateHandler::add_task_(ObStartupAccelTaskHandler* startup_ac } int ObTabletReplayCreateHandler::get_tablet_svr_( - const ObLSID &ls_id, ObLSTabletService *&ls_tablet_svr, - ObLSHandle &ls_handle) + ObLS *&tenant_ls) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls handle", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_tablet_svr = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet service is null", K(ret), K(ls_id)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("fail to get ls", K(ret)); + } else { + ls_tablet_svr = tenant_ls->get_tablet_svr(); } return ret; } @@ -492,11 +484,11 @@ int ObTabletReplayCreateHandler::replay_create_tablet(const ObTabletReplayItem & { int ret = OB_SUCCESS; ObLSTabletService *ls_tablet_svr = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; const ObTabletMapKey &key = replay_item.key_; const ObMetaDiskAddr &addr = replay_item.addr_; - if (OB_FAIL(get_tablet_svr_(key.ls_id_, ls_tablet_svr, ls_handle))) { + if (OB_FAIL(get_tablet_svr_(ls_tablet_svr, tenant_ls))) { LOG_WARN("fail to get ls tablet service", K(ret)); } else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(addr, buf, buf_len, key.tablet_id_))) { LOG_WARN("fail to create tablet for replay", K(ret), K(key), K(addr)); @@ -526,11 +518,11 @@ int ObTabletReplayCreateHandler::replay_clone_tablet(const ObTabletReplayItem &r { int ret = OB_SUCCESS; ObLSTabletService *ls_tablet_svr = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; const ObTabletMapKey &key = replay_item.key_; const ObMetaDiskAddr &addr = replay_item.addr_; ObTabletHandle tablet_handle; - if (OB_FAIL(get_tablet_svr_(key.ls_id_, ls_tablet_svr, ls_handle))) { + if (OB_FAIL(get_tablet_svr_(ls_tablet_svr, tenant_ls))) { LOG_WARN("fail to get ls tablet service", K(ret)); } else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(addr, buf, buf_len, key.tablet_id_))) { LOG_WARN("fail to create tablet for replay", K(ret), K(key), K(addr)); diff --git a/src/storage/slog_ckpt/ob_tablet_replay_create_handler.h b/src/storage/slog_ckpt/ob_tablet_replay_create_handler.h index c5f467fe1..ecc837900 100644 --- a/src/storage/slog_ckpt/ob_tablet_replay_create_handler.h +++ b/src/storage/slog_ckpt/ob_tablet_replay_create_handler.h @@ -38,7 +38,7 @@ class SCN; namespace storage { -class ObLSHandle; +class ObLS; class ObTabletReplayCreateHandler; class ObLSTabletService; @@ -153,7 +153,7 @@ class ObTabletReplayCreateHandler const int64_t buf_len, ObArenaAllocator &allocator); static int replay_clone_tablet(const ObTabletReplayItem &replay_item, const char *buf, const int64_t buf_len); - static int get_tablet_svr_(const share::ObLSID &ls_id, ObLSTabletService *&ls_tablet_svr, ObLSHandle &ls_handle); + static int get_tablet_svr_(ObLSTabletService *&ls_tablet_svr, ObLS *&tenant_ls); int add_item_range_to_task_( observer::ObStartupAccelTaskHandler* startup_accel_handler, const ObTabletReplayCreateTask::Type type, diff --git a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp index 0348b0753..7736dcd20 100644 --- a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp +++ b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp @@ -280,94 +280,12 @@ int ObTenantCheckpointSlogHandler::replay_checkpoint_and_slog(const ObTenantSupe int ObTenantCheckpointSlogHandler::replay_checkpoint(const ObTenantSuperBlock &super_block) { - int ret = OB_SUCCESS; - const bool is_replay_old = super_block.is_trivial_version(); replay_tablet_disk_addr_map_.reuse(); - - if (OB_UNLIKELY(is_replay_old)) { - if (OB_FAIL(replay_old_checkpoint(super_block))) { - LOG_WARN("fail to replay old version checkpoint", K(ret), K(super_block)); - } - } else { - if (OB_FAIL(replay_new_checkpoint(super_block))) { - LOG_WARN("fail to replay new version checkpoint", K(ret), K(super_block)); - } + int ret = replay_new_checkpoint(super_block); + if (OB_FAIL(ret)) { + LOG_WARN("fail to replay checkpoint", K(ret), K(super_block)); } - LOG_INFO("finish replay tenant checkpoint", K(ret), K(super_block)); - - return ret; -} - -int ObTenantCheckpointSlogHandler::replay_old_checkpoint(const ObTenantSuperBlock &super_block) -{ - int ret = OB_SUCCESS; - - ObTenantStorageCheckpointReader tenant_storage_ckpt_reader; - ObArray meta_block_list; - - ObTenantStorageCheckpointReader::ObStorageMetaOp replay_ls_op = - std::bind(&ObTenantCheckpointSlogHandler::replay_ls_meta, - this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); - - ObTenantStorageCheckpointReader::ObStorageMetaOp replay_tablet_op = - std::bind(&ObTenantCheckpointSlogHandler::replay_tablet, - this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); - - if (!replay_ls_op.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replay_ls_op invalid", K(ret)); - } else if (!replay_tablet_op.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replay_tablet_op invalid", K(ret)); - } else if (OB_FAIL(tenant_storage_ckpt_reader.iter_read_meta_item( - super_block.ls_meta_entry_, replay_ls_op, meta_block_list))) { - LOG_WARN("fail to replay ls meta checkpoint", K(ret)); - } else if (OB_FAIL(ls_block_handle_.add_macro_blocks(meta_block_list))) { - LOG_WARN("fail to add_macro_blocks", K(ret)); - } else if (OB_FAIL(tenant_storage_ckpt_reader.iter_read_meta_item( - super_block.tablet_meta_entry_, replay_tablet_op, meta_block_list))) { - LOG_WARN("fail to replay tablet checkpoint", K(ret)); - } else if (OB_FAIL(tablet_block_handle_.add_macro_blocks(meta_block_list))) { - LOG_WARN("fail to add_macro_blocks", K(ret)); - } - - return ret; -} - -int ObTenantCheckpointSlogHandler::replay_ls_meta( - const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len) -{ - int ret = OB_SUCCESS; - UNUSEDx(addr); - ObLSMeta ls_meta; - int64_t pos = 0; - if (OB_ISNULL(buf)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret)); - } else if (OB_FAIL(ls_meta.deserialize(buf, buf_len, pos))) { - LOG_WARN("fail to deserialize", K(ret)); - } else if (OB_FAIL(share::g_mp->ls_service()->replay_create_ls(0/*ls_epoch*/, ls_meta))) { - LOG_WARN("fail to replay_put_ls", K(ret)); - } - - return ret; -} - -int ObTenantCheckpointSlogHandler::replay_tablet( - const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!addr.is_valid() || nullptr == buf || buf_len <= 0 || !addr.is_block())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(addr)); - } else if (OB_FAIL(inner_replay_old_deserialize( - addr, - buf, - buf_len, - false /* allow to overwrite the map's element or not */))) { - LOG_WARN("fail to replay old tablet", K(ret), K(addr), KP(buf), K(buf_len)); - } return ret; } @@ -397,9 +315,9 @@ int ObTenantCheckpointSlogHandler::do_replay_single_snapshot(const blocksstable: std::placeholders::_3); bool inc_ls_blocks_ref_succ = false; - if (OB_FAIL(ls_ckpt_reader.iter_read_meta_item( + if (OB_FAIL(ls_ckpt_reader.read_single_meta_item( ls_meta_entry, replay_snapshot_ls_op, ls_block_list))) { - LOG_WARN("fail to iter replay ls", K(ret), K(ls_meta_entry)); + LOG_WARN("fail to replay ls snapshot", K(ret), K(ls_meta_entry)); } else if (OB_FAIL(ObTenantMetaSnapshotHandler::inc_linked_block_ref( ls_block_list, inc_ls_blocks_ref_succ))) { LOG_WARN("fail to increase ls linked blocks' ref cnt", K(ret)); @@ -455,9 +373,9 @@ int ObTenantCheckpointSlogHandler::replay_new_checkpoint(const ObTenantSuperBloc std::placeholders::_3, std::ref(tablet_block_list)); - if (OB_FAIL(ls_ckpt_reader.iter_read_meta_item( + if (OB_FAIL(ls_ckpt_reader.read_single_meta_item( super_block.ls_meta_entry_, replay_ls_op, ls_block_list))) { - LOG_WARN("fail to iter replay ls", K(ret), K(super_block)); + LOG_WARN("fail to replay ls checkpoint", K(ret), K(super_block)); } else if (OB_FAIL(ls_block_handle_.add_macro_blocks(ls_block_list))) { LOG_WARN("fail to add ls linked blocks", K(ret), K(ls_block_list)); } else if (OB_FAIL(tablet_block_handle_.add_macro_blocks(tablet_block_list))) { @@ -604,7 +522,6 @@ int ObTenantCheckpointSlogHandler::clone_tablet( ret = OB_ERR_UNEXPECTED; LOG_WARN("slog is invalid", K(ret), K(slog)); } else { - tablet_key.ls_id_ = slog.ls_id_; tablet_key.tablet_id_ = slog.tablet_id_; if (OB_FAIL(replay_tablet_disk_addr_map_.set_refactored(tablet_key, slog.disk_addr_, /*allow_override*/ 0))) { LOG_WARN("fail to update tablet meta addr", K(ret), K(slog)); @@ -718,7 +635,6 @@ int ObTenantCheckpointSlogHandler::write_checkpoint(bool is_force) ret = OB_ERR_UNEXPECTED; LOG_WARN("ckpt_cursor_ is invalid", K(ret)); } else if (is_force // alter system command triggered - || last_super_block.is_old_version() // compat upgrade || (!is_major_doing && (ckpt_cursor_.file_id_ > last_super_block.replay_start_point_.file_id_)) || ((start_time > last_ckpt_time_ + min_interval) // slog is long && !is_major_doing @@ -998,12 +914,6 @@ int ObTenantCheckpointSlogHandler::replay(const ObRedoModuleReplayParam ¶m) } break; } - case ObRedoLogSubType::OB_REDO_LOG_PUT_OLD_TABLET: { - if (OB_FAIL(inner_replay_put_old_tablet(param))) { - LOG_WARN("fail to replay put old tablet slog", K(param)); - } - break; - } case ObRedoLogSubType::OB_REDO_LOG_EMPTY_SHELL_TABLET: { if (OB_FAIL(inner_replay_empty_shell_tablet(param))) { LOG_WARN("fail to replay put old tablet slog", K(param)); @@ -1061,13 +971,12 @@ int ObTenantCheckpointSlogHandler::inner_replay_create_ls_commit_slog( { int ret = OB_SUCCESS; - ObLSID ls_id; - ObCreateLSCommitSLog slog_entry(ls_id); + ObCreateLSCommitSLog slog_entry; int64_t pos = 0; const bool is_replay = true; if (OB_FAIL(slog_entry.deserialize(param.buf_, param.disk_addr_.size(), pos))) { LOG_WARN("fail to deserialize slog", K(ret), K(param), K(pos)); - } else if (OB_FAIL(share::g_mp->ls_service()->replay_create_ls_commit(ls_id))) { + } else if (OB_FAIL(share::g_mp->ls_service()->replay_create_ls_commit())) { LOG_WARN("fail to replay create ls commit slog", K(ret), K(param), K(pos)); } else { LOG_INFO("successfully replay create ls commit slog"); @@ -1080,41 +989,16 @@ int ObTenantCheckpointSlogHandler::inner_replay_delete_ls(const ObRedoModuleRepl { int ret = OB_SUCCESS; - ObLSID ls_id; - ObLSIDLog slog_entry(ls_id); + ObLSMarkerLog slog_entry; int64_t pos = 0; const bool is_replay = true; if (OB_FAIL(slog_entry.deserialize(param.buf_, param.disk_addr_.size(), pos))) { LOG_WARN("fail to deserialize remove log stream slog", K(param), K(pos)); - } else if (OB_FAIL(share::g_mp->ls_service()->replay_remove_ls(ls_id))) { + } else if (OB_FAIL(share::g_mp->ls_service()->replay_remove_ls())) { LOG_WARN("fail to remove log stream", K(param), K(pos)); - } else if (OB_FAIL(remove_tablets_from_replay_map_(ls_id))) { - LOG_WARN("fail to remove tablets", K(ret), K(ls_id)); } else { - LOG_INFO("successfully replay remove log stream", K(ret), K(ls_id)); - } - - return ret; -} - -int ObTenantCheckpointSlogHandler::remove_tablets_from_replay_map_(const ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - - ObArray need_removed_tablets; - ReplayTabletDiskAddrMap::iterator iter = replay_tablet_disk_addr_map_.begin(); - for (; OB_SUCC(ret) && iter != replay_tablet_disk_addr_map_.end(); ++iter) { - const ObTabletMapKey &map_key = iter->first; - if (ls_id == map_key.ls_id_) { - if (OB_FAIL(need_removed_tablets.push_back(map_key))) { - LOG_WARN("fail to push back", K(ret), K(map_key)); - } - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < need_removed_tablets.count(); i++) { - if (OB_FAIL(replay_tablet_disk_addr_map_.erase_refactored(need_removed_tablets.at(i)))) { - LOG_WARN("fail to erase tablet", K(ret), K(need_removed_tablets.at(i))); - } + replay_tablet_disk_addr_map_.reuse(); + LOG_INFO("successfully replay remove log stream", K(ret)); } return ret; @@ -1144,7 +1028,6 @@ int ObTenantCheckpointSlogHandler::inner_replay_deserialize( ret = OB_ERR_UNEXPECTED; LOG_WARN("slog is invalid", K(ret), K(slog)); } else { - tablet_key.ls_id_ = slog.ls_id_; tablet_key.tablet_id_ = slog.tablet_id_; if (OB_FAIL(replay_tablet_disk_addr_map_.set_refactored(tablet_key, slog.disk_addr_, allow_override ? 1 : 0))) { LOG_WARN("fail to update tablet meta addr", K(ret), K(slog)); @@ -1156,36 +1039,6 @@ int ObTenantCheckpointSlogHandler::inner_replay_deserialize( return ret; } -int ObTenantCheckpointSlogHandler::inner_replay_put_old_tablet(const ObRedoModuleReplayParam ¶m) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(inner_replay_old_deserialize( - param.disk_addr_, - param.buf_, - param.disk_addr_.size(), - true /* allow to overwrite the map's element or not */))) { - LOG_WARN("fail to replay old tablet", K(ret), K(param)); - } - return ret; -} - -int ObTenantCheckpointSlogHandler::inner_replay_old_deserialize( - const ObMetaDiskAddr &addr, - const char *buf, - const int64_t buf_len, - bool allow_override /* allow to overwrite the map's element or not */) -{ - int ret = OB_SUCCESS; - ObTabletMapKey map_key; - - if (OB_FAIL(ObTablet::deserialize_id(buf, buf_len, map_key.ls_id_, map_key.tablet_id_))) { - LOG_WARN("fail to deserialize log stream id and tablet id", K(ret)); - } else if (OB_FAIL(replay_tablet_disk_addr_map_.set_refactored(map_key, addr, allow_override ? 1 : 0))) { - LOG_WARN("update tablet meta addr fail", K(ret), K(map_key), K(addr)); - } - return ret; -} - int ObTenantCheckpointSlogHandler::inner_replay_delete_tablet(const ObRedoModuleReplayParam ¶m) { int ret = OB_SUCCESS; @@ -1196,7 +1049,7 @@ int ObTenantCheckpointSlogHandler::inner_replay_delete_tablet(const ObRedoModule if (OB_FAIL(slog_entry.deserialize(param.buf_, param.disk_addr_.size(), pos))) { LOG_WARN("fail to deserialize delete tablet slog", K(param), K(pos)); } else { - const ObTabletMapKey map_key(slog_entry.ls_id_, slog_entry.tablet_id_); + const ObTabletMapKey map_key(slog_entry.tablet_id_); if (OB_FAIL(replay_tablet_disk_addr_map_.erase_refactored(map_key)) && OB_HASH_NOT_EXIST != ret) { LOG_WARN("fail to erase tablet", K(ret), K(map_key), K(slog_entry)); } else { @@ -1217,8 +1070,7 @@ int ObTenantCheckpointSlogHandler::inner_replay_empty_shell_tablet(const ObRedoM if (OB_FAIL(slog.deserialize_id(param.buf_, param.disk_addr_.size(), pos))) { STORAGE_LOG(WARN, "failed to serialize tablet_id_", K(ret), K(param.disk_addr_.size()), K(pos)); } else { - const ObTabletMapKey map_key(slog.ls_id_, - slog.tablet_id_); + const ObTabletMapKey map_key(slog.tablet_id_); if (OB_FAIL(replay_tablet_disk_addr_map_.set_refactored(map_key, param.disk_addr_, 1))) { LOG_WARN("fail to set tablet", K(ret), K(map_key), K(param.disk_addr_)); } @@ -1249,8 +1101,7 @@ int ObTenantCheckpointSlogHandler::parse( } else { switch (sub_type) { case ObRedoLogSubType::OB_REDO_LOG_CREATE_LS_COMMIT: { - ObLSID ls_id; - ObLSIDLog slog_entry(ls_id); + ObLSMarkerLog slog_entry; snprintf(slog_name, ObStorageLogReplayer::MAX_SLOG_NAME_LEN, "create ls commit slog: "); if (OB_FAIL(ObStorageLogReplayer::print_slog(buf, len, slog_name, slog_entry, stream))) { LOG_WARN("fail to print slog", K(ret), KP(buf), K(len), K(slog_name), K(slog_entry)); @@ -1274,8 +1125,7 @@ int ObTenantCheckpointSlogHandler::parse( break; } case ObRedoLogSubType::OB_REDO_LOG_CREATE_LS_ABORT: { - ObLSID ls_id; - ObLSIDLog slog_entry(ls_id); + ObLSMarkerLog slog_entry; snprintf(slog_name, ObStorageLogReplayer::MAX_SLOG_NAME_LEN, "create ls abort slog: "); if (OB_FAIL(ObStorageLogReplayer::print_slog(buf, len, slog_name, slog_entry, stream))) { LOG_WARN("fail to print slog", K(ret), KP(buf), K(len), K(slog_name), K(slog_entry)); @@ -1283,8 +1133,7 @@ int ObTenantCheckpointSlogHandler::parse( break; } case ObRedoLogSubType::OB_REDO_LOG_DELETE_LS: { - ObLSID ls_id; - ObLSIDLog slog_entry(ls_id); + ObLSMarkerLog slog_entry; snprintf(slog_name, ObStorageLogReplayer::MAX_SLOG_NAME_LEN, "delete ls slog: "); if (OB_FAIL(ObStorageLogReplayer::print_slog(buf, len, slog_name, slog_entry, stream))) { LOG_WARN("fail to print slog", K(ret), KP(buf), K(len), K(slog_name), K(slog_entry)); @@ -1311,9 +1160,8 @@ int ObTenantCheckpointSlogHandler::parse( break; } case ObRedoLogSubType::OB_REDO_LOG_DELETE_TABLET: { - ObLSID ls_id; ObTabletID tablet_id; - ObDeleteTabletLog slog_entry(ls_id, tablet_id); + ObDeleteTabletLog slog_entry(tablet_id); snprintf(slog_name, ObStorageLogReplayer::MAX_SLOG_NAME_LEN, "delete tablet slog: "); if (OB_FAIL(ObStorageLogReplayer::print_slog(buf, len, slog_name, slog_entry, stream))) { LOG_WARN("fail to print slog", K(ret), KP(buf), K(len), K(slog_name), K(slog_entry)); diff --git a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h index 24b118760..8f091bd58 100644 --- a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h +++ b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h @@ -32,7 +32,6 @@ namespace oceanbase namespace share { -class ObLSID; class ObTenantSnapshotID; } namespace storage @@ -141,9 +140,6 @@ class ObTenantCheckpointSlogHandler : public ObIRedoModule int replay_snapshot(const ObTenantSuperBlock &super_block); int do_replay_single_snapshot(const blocksstable::MacroBlockId &ls_meta_entry); int replay_new_checkpoint(const ObTenantSuperBlock &super_block); - int replay_old_checkpoint(const ObTenantSuperBlock &super_block); - int replay_ls_meta(const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len); - int replay_tablet(const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len); int replay_snapshot_ls( const ObMetaDiskAddr &addr, const char *buf, @@ -160,7 +156,6 @@ class ObTenantCheckpointSlogHandler : public ObIRedoModule int inner_replay_create_ls_slog(const ObRedoModuleReplayParam ¶m); int inner_replay_create_ls_commit_slog(const ObRedoModuleReplayParam ¶m); int inner_replay_delete_ls(const ObRedoModuleReplayParam ¶m); - int inner_replay_put_old_tablet(const ObRedoModuleReplayParam ¶m); int inner_replay_update_tablet(const ObRedoModuleReplayParam ¶m); int inner_replay_delete_tablet(const ObRedoModuleReplayParam ¶m); int inner_replay_empty_shell_tablet(const ObRedoModuleReplayParam ¶m); @@ -169,18 +164,10 @@ class ObTenantCheckpointSlogHandler : public ObIRedoModule int inner_replay_das_record(const ObRedoModuleReplayParam ¶m); int read_from_ckpt(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, int64_t &r_len); int read_from_slog(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, int64_t &pos); - int remove_tablets_from_replay_map_(const share::ObLSID &ls_id); - int inner_replay_deserialize( const char *buf, const int64_t buf_len, bool allow_override /* allow to overwrite the map's element or not */); - int inner_replay_old_deserialize( - const ObMetaDiskAddr &addr, - const char *buf, - const int64_t buf_len, - bool allow_override /* allow to overwrite the map's element or not */); - private: const static int64_t BUCKET_NUM = 109; private: diff --git a/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.cpp b/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.cpp index bf35c959c..4ff28b49f 100644 --- a/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.cpp +++ b/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.cpp @@ -56,100 +56,6 @@ int ObTenantMetaSnapshotHandler::create_tenant_snapshot(const ObTenantSnapshotID return ret; } -int ObTenantMetaSnapshotHandler::create_single_ls_snapshot(const ObTenantSnapshotID &snapshot_id, - const ObLSID &ls_id, - share::SCN &clog_max_scn) -{ - int ret = OB_SUCCESS; - ObTenantStorageCheckpointWriter tenant_storage_meta_writer; - MacroBlockId orig_ls_meta_entry; - ObTenantSnapshotMeta snapshot; - ObSArray ls_block_list(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CreateSnapLS")); - bool inc_ls_blocks_ref_succ = false; - bool inc_tablet_blocks_ref_succ = false; - - if (OB_UNLIKELY(!snapshot_id.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(get_ls_meta_entry(snapshot_id, orig_ls_meta_entry))) { - LOG_WARN("fail to get ls meta entry", K(ret), K(snapshot_id)); - } else if (OB_FAIL(tenant_storage_meta_writer.init(ObTenantStorageMetaType::SNAPSHOT, nullptr))) { - LOG_WARN("fail to init tenant storage checkpoint writer", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(tenant_storage_meta_writer.record_single_ls_meta(orig_ls_meta_entry, - ls_id, - ls_block_list, - snapshot.ls_meta_entry_, - clog_max_scn))) { - LOG_WARN("fail to record_single_ls_meta", K(ret), K(orig_ls_meta_entry), K(snapshot_id), K(ls_id)); - } else if (FALSE_IT(snapshot.snapshot_id_ = snapshot_id)) { - } else if (OB_FAIL(inc_all_linked_block_ref(tenant_storage_meta_writer, - inc_ls_blocks_ref_succ, - inc_tablet_blocks_ref_succ))) { - LOG_WARN("fail to increase ref cnt for all linked blocks", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(share::g_mp->tenant_storage_meta_service()->swap_snapshot(snapshot))) { - LOG_WARN("fail to swap snapshot", K(ret), K(snapshot_id), K(ls_id), K(snapshot)); - } - - if (OB_FAIL(ret)) { - rollback_ref_cnt(inc_ls_blocks_ref_succ, inc_tablet_blocks_ref_succ, tenant_storage_meta_writer); - } else { - dec_meta_block_ref(ls_block_list); - } - - FLOG_INFO("finish create ls snapshot", K(ret), K(snapshot_id), K(ls_id), K(snapshot)); - return ret; -} - -int ObTenantMetaSnapshotHandler::delete_single_ls_snapshot(const ObTenantSnapshotID &snapshot_id, - const ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObTenantStorageCheckpointWriter tenant_storage_meta_writer; - MacroBlockId orig_ls_meta_entry; - MacroBlockId tablet_meta_entry; - ObTenantSnapshotMeta snapshot; - ObSArray ls_block_list(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CreateSnapLS")); - ObSArray deleted_tablet_addrs(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("DelSnapLS")); - ObSArray tablet_meta_block_list(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("DelSnapLS")); - bool inc_ls_blocks_ref_succ = false; - bool inc_tablet_blocks_ref_succ = false; - - if (OB_UNLIKELY(!snapshot_id.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(get_ls_meta_entry(snapshot_id, orig_ls_meta_entry))) { - LOG_WARN("fail to get ls meta entry", K(ret), K(snapshot_id)); - } else if (OB_FAIL(find_tablet_meta_entry(orig_ls_meta_entry, ls_id, tablet_meta_entry))) { - LOG_WARN("fail to get tablet meta entry", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(inner_delete_ls_snapshot(tablet_meta_entry, - deleted_tablet_addrs, - tablet_meta_block_list))) { - LOG_WARN("fail to exec inner_delete_ls_snapshot", K(ret), K(tablet_meta_entry), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(tenant_storage_meta_writer.init(ObTenantStorageMetaType::SNAPSHOT, nullptr))) { - LOG_WARN("fail to init tenant storage checkpoint writer", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(tenant_storage_meta_writer.delete_single_ls_meta(orig_ls_meta_entry, ls_id, ls_block_list, snapshot.ls_meta_entry_))) { - LOG_WARN("fail to delete_single_ls_meta", K(ret), K(orig_ls_meta_entry), K(snapshot_id), K(ls_id)); - } else if (FALSE_IT(snapshot.snapshot_id_ = snapshot_id)) { - } else if (OB_FAIL(inc_all_linked_block_ref(tenant_storage_meta_writer, inc_ls_blocks_ref_succ, inc_tablet_blocks_ref_succ))) { - LOG_WARN("fail to increase ref cnt for all linked blocks", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_FAIL(share::g_mp->tenant_storage_meta_service()->swap_snapshot(snapshot))) { - LOG_WARN("fail to swap snapshot", K(ret), K(snapshot_id), K(ls_id), K(snapshot)); - } - - if (OB_FAIL(ret)) { - rollback_ref_cnt(inc_ls_blocks_ref_succ, inc_tablet_blocks_ref_succ, tenant_storage_meta_writer); - } else { - dec_meta_block_ref(ls_block_list); - dec_meta_block_ref(tablet_meta_block_list); - if (OB_FAIL(inner_delete_tablet_by_addrs(deleted_tablet_addrs))) { - LOG_WARN("fail to inner_delete_tablet_by_addrs", K(ret), K(snapshot_id), K(ls_id)); - } - } - - FLOG_INFO("finish delete ls snapshot", K(ret), K(snapshot_id), K(ls_id), K(snapshot)); - return ret; -} - int ObTenantMetaSnapshotHandler::inc_all_linked_block_ref( ObTenantStorageCheckpointWriter &tenant_storage_meta_writer, bool &inc_ls_blocks_ref_succ, @@ -288,7 +194,7 @@ int ObTenantMetaSnapshotHandler::delete_tenant_snapshot(const ObTenantSnapshotID LOG_WARN("super block is invalid", K(ret), K(last_super_block)); } else if (OB_FAIL(last_super_block.get_snapshot(snapshot_id, snapshot))) { LOG_WARN("fail to get snapshot", K(ret), K(snapshot_id), K(last_super_block)); - } else if (OB_FAIL(ls_snapshot_reader.iter_read_meta_item( + } else if (OB_FAIL(ls_snapshot_reader.read_single_meta_item( snapshot.ls_meta_entry_, del_ls_snapshot_op, ls_meta_block_list))) { LOG_WARN("fail to delete ls snapshot", K(ret), K(snapshot)); } else if (OB_FAIL((share::g_mp->tenant_storage_meta_service()->delete_snapshot(snapshot_id)))) { @@ -433,131 +339,6 @@ int ObTenantMetaSnapshotHandler::get_all_tenant_snapshot(ObIArray &ls_ids) -{ - int ret = OB_SUCCESS; - omt::ObTenant *tenant = static_cast(share::ObTenantEnv::get_tenant()); - const ObTenantSuperBlock super_block = tenant->get_super_block(); - ObTenantSnapshotMeta snapshot; - ObTenantStorageCheckpointReader ls_ckpt_reader; - ObSArray meta_block_list(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("GetAllLS")); - ObTenantStorageCheckpointReader::ObStorageMetaOp push_ls_op = std::bind( - &ObTenantMetaSnapshotHandler::push_ls_snapshot, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3, - std::ref(ls_ids)); - - if (OB_UNLIKELY(!snapshot_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(snapshot_id)); - } else if (OB_UNLIKELY(tenant->is_hidden())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("can't get snapshot from hidden tenant", K(ret), K(snapshot_id)); - } else if (OB_UNLIKELY(!super_block.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("super block is invalid", K(ret), K(super_block), K(snapshot_id)); - } else if (OB_FAIL(super_block.get_snapshot(snapshot_id, snapshot))) { - LOG_WARN("fail to get snapshot", K(ret), K(snapshot_id)); - } else if (OB_FAIL(ls_ckpt_reader.iter_read_meta_item(snapshot.ls_meta_entry_, push_ls_op, meta_block_list))) { - LOG_WARN("fail to iter push ls", K(ret), K(snapshot)); - } - return ret; -} - -int ObTenantMetaSnapshotHandler::push_ls_snapshot( - const ObMetaDiskAddr &addr, - const char *buf, - const int64_t buf_len, - ObIArray &ls_ids) -{ - UNUSED(addr); - int ret = OB_SUCCESS; - int64_t pos = 0; - ObLSCkptMember ls_ckpt_member; - if (OB_FAIL(ls_ckpt_member.deserialize(buf, buf_len, pos))) { - LOG_WARN("fail to deserialize ls ckpt member", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(ls_ids.push_back(ls_ckpt_member.ls_meta_.ls_id_))) { - LOG_WARN("fail to push back ls id", K(ret), K(ls_ckpt_member)); - } - return ret; -} - -int ObTenantMetaSnapshotHandler::get_ls_snapshot( - const ObTenantSnapshotID &snapshot_id, - const ObLSID &ls_id, - blocksstable::MacroBlockId &tablet_meta_entry) -{ - int ret = OB_SUCCESS; - ObTenantSnapshotMeta snapshot; - omt::ObTenant *tenant = static_cast(share::ObTenantEnv::get_tenant()); - const ObTenantSuperBlock super_block = tenant->get_super_block(); - - if (OB_UNLIKELY(!snapshot_id.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_UNLIKELY(tenant->is_hidden())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("can't get ls snapshot from hidden tenant", K(ret), K(snapshot_id), K(ls_id)); - } else if (OB_UNLIKELY(!super_block.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("super block is invalid", K(ret), K(super_block)); - } else if (OB_FAIL(super_block.get_snapshot(snapshot_id, snapshot))) { - LOG_WARN("fail to get snapshot", K(ret), K(snapshot_id), K(super_block)); - } else if (OB_FAIL(find_tablet_meta_entry(snapshot.ls_meta_entry_, ls_id, tablet_meta_entry))) { - LOG_WARN("fail to get tablet meta entry", K(ret), K(snapshot), K(ls_id)); - } - return ret; -} - -int ObTenantMetaSnapshotHandler::find_tablet_meta_entry( - const blocksstable::MacroBlockId &ls_meta_entry, - const ObLSID &ls_id, - blocksstable::MacroBlockId &tablet_meta_entry) -{ - int ret = OB_SUCCESS; - ObLinkedMacroBlockItemReader ls_ckpt_reader; - ObMemAttr mem_attr("Snapshot"); - - if (OB_UNLIKELY(!ls_meta_entry.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(ls_meta_entry), K(ls_id)); - } else if (OB_UNLIKELY(IS_EMPTY_BLOCK_LIST(ls_meta_entry))) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("ls snapshot doesn't exist", K(ret), K(ls_meta_entry)); - } else if (OB_FAIL(ls_ckpt_reader.init(ls_meta_entry, mem_attr))) { - LOG_WARN("fail to init log stream item reader", K(ret), K(ls_meta_entry)); - } else { - char *item_buf = nullptr; - int64_t item_buf_len = 0; - ObLSCkptMember ls_ckpt_member; - int64_t pos = 0; - ObMetaDiskAddr addr; - while (OB_SUCC(ret)) { - item_buf = nullptr; - item_buf_len = 0; - pos = 0; - ls_ckpt_member.reset(); - if (OB_FAIL(ls_ckpt_reader.get_next_item(item_buf, item_buf_len, addr))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next log stream item", K(ret)); - } else { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("can't find target ls snapshot", K(ret)); - } - } else if (OB_FAIL(ls_ckpt_member.deserialize(item_buf, item_buf_len, pos))) { - LOG_WARN("fail to deserialize ls ckpt member", K(ret), KP(item_buf), K(item_buf_len), K(pos)); - } else if (ls_ckpt_member.ls_meta_.ls_id_ == ls_id) { - tablet_meta_entry = ls_ckpt_member.tablet_meta_entry_; - break; - } - } - } - return ret; -} - int ObTenantMetaSnapshotHandler::create_all_tablet(observer::ObStartupAccelTaskHandler* startup_accel_handler, const blocksstable::MacroBlockId &tablet_meta_entry) { diff --git a/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.h b/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.h index 1256afdef..1159a2b95 100644 --- a/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.h +++ b/src/storage/slog_ckpt/ob_tenant_meta_snapshot_handler.h @@ -50,36 +50,18 @@ class ObTenantMetaSnapshotHandler // create snapshot static int create_tenant_snapshot(const ObTenantSnapshotID &snapshot_id); - static int create_single_ls_snapshot(const ObTenantSnapshotID &snapshot_id, - const ObLSID &ls_id, - share::SCN &clog_max_scn); - // delete snapshot static int delete_tenant_snapshot(const ObTenantSnapshotID &snapshot_id); - static int delete_single_ls_snapshot(const ObTenantSnapshotID &snapshot_id, const ObLSID &ls_id); - - // clone snapshot - static int get_ls_snapshot( - const ObTenantSnapshotID &snapshot_id, - const ObLSID &ls_id, - blocksstable::MacroBlockId &tablet_meta_entry); static int create_all_tablet(observer::ObStartupAccelTaskHandler* startup_accel_handler, const blocksstable::MacroBlockId &tablet_meta_entry); static int get_ls_meta_entry(const ObTenantSnapshotID &snapshot_id, blocksstable::MacroBlockId &ls_meta_entry); // recover snapshot for restart static int get_all_tenant_snapshot(ObIArray &snapshot_ids); - static int get_all_ls_snapshot(const ObTenantSnapshotID &snapshot_id, ObIArray &ls_ids); - // increase ref cnt for linked blocks static int inc_linked_block_ref(const ObIArray &meta_block_list, bool &inc_success); private: - static int find_tablet_meta_entry( - const blocksstable::MacroBlockId &ls_meta_entry, - const ObLSID &ls_id, - blocksstable::MacroBlockId &tablet_meta_entry); - static int push_ls_snapshot(const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len, ObIArray &ls_ids); static int inc_all_linked_block_ref( ObTenantStorageCheckpointWriter &tenant_storage_meta_writer, bool &inc_ls_blocks_ref_succ, diff --git a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.cpp b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.cpp index f83dbb0fd..4dc1dcc6a 100644 --- a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.cpp +++ b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.cpp @@ -40,9 +40,9 @@ int ObTenantStorageCheckpointReader::iter_read_meta_item( ObMemAttr mem_attr(ObModIds::OB_CHECKPOINT); if (OB_UNLIKELY(IS_EMPTY_BLOCK_LIST(entry_block))) { - LOG_INFO("has no snapshot of log stream", K(ret)); + LOG_INFO("has no storage metadata checkpoint", K(ret)); } else if (OB_FAIL(item_reader.init(entry_block, mem_attr))) { - LOG_WARN("failed to init log stream item reader"); + LOG_WARN("failed to init storage metadata item reader"); } else { char *item_buf = nullptr; int64_t item_buf_len = 0; @@ -50,7 +50,7 @@ int ObTenantStorageCheckpointReader::iter_read_meta_item( while (OB_SUCC(ret)) { if (OB_FAIL(item_reader.get_next_item(item_buf, item_buf_len, addr))) { if (OB_ITER_END != ret) { - LOG_WARN("fail to get next log stream item", K(ret)); + LOG_WARN("fail to get next storage metadata item", K(ret)); } else { ret = OB_SUCCESS; break; @@ -74,6 +74,37 @@ int ObTenantStorageCheckpointReader::iter_read_meta_item( return ret; } +int ObTenantStorageCheckpointReader::read_single_meta_item( + const MacroBlockId &entry_block, + const ObStorageMetaOp &op, + ObIArray &block_list) +{ + int ret = OB_SUCCESS; + int64_t item_count = 0; + const ObStorageMetaOp count_item_op = [&item_count]( + const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len) -> int { + int ret = OB_SUCCESS; + UNUSED(addr); + UNUSED(buf); + UNUSED(buf_len); + if (OB_UNLIKELY(++item_count > 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("multiple metadata items found for singleton checkpoint", K(ret), K(item_count)); + } + return ret; + }; + + if (OB_FAIL(iter_read_meta_item(entry_block, count_item_op, block_list))) { + LOG_WARN("failed to validate singleton metadata checkpoint", K(ret), K(entry_block)); + } else if (OB_UNLIKELY(!IS_EMPTY_BLOCK_LIST(entry_block) && 1 != item_count)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("singleton metadata checkpoint contains no item", K(ret), K(entry_block)); + } else if (1 == item_count && OB_FAIL(iter_read_meta_item(entry_block, op, block_list))) { + LOG_WARN("failed to read singleton metadata checkpoint", K(ret), K(entry_block)); + } + return ret; +} + int ObTenantStorageCheckpointReader::read_tablet_checkpoint_by_addr( const ObIArray &block_list, const ObMetaDiskAddr &addr, diff --git a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.h b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.h index 11454e949..a04c1486a 100644 --- a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.h +++ b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_reader.h @@ -39,6 +39,9 @@ class ObTenantStorageCheckpointReader final int iter_read_meta_item(const blocksstable::MacroBlockId &entry_block, const ObStorageMetaOp &op, ObIArray &block_list); + int read_single_meta_item(const blocksstable::MacroBlockId &entry_block, + const ObStorageMetaOp &op, + ObIArray &block_list); static int read_tablet_checkpoint_by_addr(const ObIArray &block_list, const ObMetaDiskAddr &addr, char *item_buf, int64_t &item_buf_len); diff --git a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.cpp b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.cpp index f0bbfe5a9..14c5547c7 100644 --- a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.cpp +++ b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.cpp @@ -87,93 +87,22 @@ int ObTenantStorageCheckpointWriter::record_meta(MacroBlockId &ls_meta_entry) return ret; } -int ObTenantStorageCheckpointWriter::record_single_ls_meta( - const MacroBlockId &orig_ls_meta_entry, - const ObLSID &ls_id, - ObIArray &orig_linked_block_list, - blocksstable::MacroBlockId &ls_meta_entry, - share::SCN &clog_max_scn) -{ - int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObTenantStorageCheckpointReader ls_ckpt_reader; - ObTenantStorageCheckpointReader::ObStorageMetaOp copy_ls_meta_op = std::bind( - &ObTenantStorageCheckpointWriter::copy_ls_meta_for_creating, - this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3); - - if (OB_UNLIKELY(!orig_ls_meta_entry.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(orig_ls_meta_entry), K(ls_id)); - } else if (OB_FAIL(ls_ckpt_reader.iter_read_meta_item(orig_ls_meta_entry, copy_ls_meta_op, orig_linked_block_list))) { - LOG_WARN("fail to iter read and write ls snapshot", K(ret), K(orig_ls_meta_entry)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls", K(ret)); - } else if (OB_FAIL(do_record_ls_meta(*(ls_handle.get_ls()), clog_max_scn))) { - LOG_WARN("fail to record single ls meta", K(ret), K(ls_handle)); - } else if (OB_FAIL(close(ls_meta_entry))) { - LOG_WARN("fail to close tenant storage checkpoint writer", K(ret)); - } - return ret; -} - -int ObTenantStorageCheckpointWriter::delete_single_ls_meta( - const MacroBlockId &orig_ls_meta_entry, - const ObLSID &ls_id, - ObIArray &orig_linked_block_list, - blocksstable::MacroBlockId &ls_meta_entry) -{ - int ret = OB_SUCCESS; - ObTenantStorageCheckpointReader ls_ckpt_reader; - ObTenantStorageCheckpointReader::ObStorageMetaOp copy_ls_meta_op = std::bind( - &ObTenantStorageCheckpointWriter::copy_ls_meta_for_deleting, - this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3, - ls_id); - - if (OB_UNLIKELY(!orig_ls_meta_entry.is_valid() || !ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(orig_ls_meta_entry), K(ls_id)); - } else if (OB_FAIL(ls_ckpt_reader.iter_read_meta_item(orig_ls_meta_entry, copy_ls_meta_op, orig_linked_block_list))) { - LOG_WARN("fail to iter read and write ls snapshot", K(ret), K(orig_ls_meta_entry)); - } else if (OB_FAIL(close(ls_meta_entry))) { - LOG_WARN("fail to close tenant storage checkpoint writer", K(ret)); - } - return ret; -} - int ObTenantStorageCheckpointWriter::record_ls_meta(MacroBlockId &ls_entry_block) { int ret = OB_SUCCESS; - common::ObSharedGuard ls_iter; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; ls_item_writer_.reset(); tablet_item_writer_.reset(); ObMemAttr mem_attr(ObModIds::OB_CHECKPOINT); - if (OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get log stream iter", K(ret)); - } else if (OB_FAIL(ls_item_writer_.init(false /*whether need addr*/, mem_attr))) { + if (OB_FAIL(ls_item_writer_.init(false /*whether need addr*/, mem_attr))) { LOG_WARN("failed to init log stream item writer", K(ret)); } else { share::SCN unused_scn; - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("fail to get next log stream", K(ret)); - } - } - - if (OB_SUCC(ret) && OB_FAIL(do_record_ls_meta(*ls, unused_scn))) { - LOG_WARN("fail to do record storage meta", K(ret), KPC(ls)); - } + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(do_record_ls_meta(*tenant_ls, unused_scn))) { + LOG_WARN("fail to do record storage meta", K(ret)); } if (OB_FAIL(ret)) { @@ -235,41 +164,6 @@ int ObTenantStorageCheckpointWriter::write_item(const ObLSCkptMember &ls_ckpt_me return ret; } -int ObTenantStorageCheckpointWriter::copy_ls_meta_for_deleting( - const ObMetaDiskAddr &addr, - const char *buf, - const int64_t buf_len, - const ObLSID &ls_id) -{ - UNUSED(addr); - int ret = OB_SUCCESS; - ObLSCkptMember ls_ckpt_member; - int64_t pos = 0; - if (OB_FAIL(ls_ckpt_member.deserialize(buf, buf_len, pos))) { - LOG_WARN("fail to deserialize ls_ckpt_member", K(ret), KP(buf), K(buf_len)); - } else if (ls_id != ls_ckpt_member.ls_meta_.ls_id_ && OB_FAIL(write_item(ls_ckpt_member))) { - LOG_ERROR("fail to write ls snapshot", K(ret), K(ls_id), K(ls_ckpt_member)); - } - return ret; -} - -int ObTenantStorageCheckpointWriter::copy_ls_meta_for_creating( - const ObMetaDiskAddr &addr, - const char *buf, - const int64_t buf_len) -{ - UNUSED(addr); - int ret = OB_SUCCESS; - ObLSCkptMember ls_ckpt_member; - int64_t pos = 0; - if (OB_FAIL(ls_ckpt_member.deserialize(buf, buf_len, pos))) { - LOG_WARN("fail to deserialize ls_ckpt_member", K(ret), KP(buf), K(buf_len)); - } else if (OB_FAIL(write_item(ls_ckpt_member))) { - LOG_WARN("fail to write ls snapshot", K(ret), K(ls_ckpt_member)); - } - return ret; -} - int ObTenantStorageCheckpointWriter::close(blocksstable::MacroBlockId &ls_meta_entry) { int ret = OB_SUCCESS; @@ -327,7 +221,7 @@ int ObTenantStorageCheckpointWriter::record_tablet_meta(ObLS &ls, MacroBlockId & if (OB_SUCC(ret)) { ++processed_cnt; if (processed_cnt % 1000 == 0) { - FLOG_INFO("print compat processing procedure", K(ret), "ls_id", ls.get_ls_id(), K(processed_cnt), K(total_tablet_cnt)); + FLOG_INFO("print compat processing procedure", K(ret), K(processed_cnt), K(total_tablet_cnt)); } } } else if (ObTenantStorageMetaType::SNAPSHOT == meta_type_ && OB_FAIL(copy_tablet(tablet_key, slog_buf, clog_max_scn))) { @@ -363,10 +257,9 @@ int ObTenantStorageCheckpointWriter::persist_and_copy_tablet( int64_t slog_buf_pos = 0; MEMSET(slog_buf, 0, sizeof(ObUpdateTabletLog)); ObUpdateTabletLog slog; - slog.ls_id_ = tablet_key.ls_id_; slog.tablet_id_ = tablet_key.tablet_id_; bool has_slog = false; - const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_); + const ObTabletPersisterParam param(0, tablet_key.tablet_id_); if (OB_FAIL(OB_E(EventTable::EN_SLOG_CKPT_ERROR) OB_SUCCESS)) { } else if (OB_FAIL(ckpt_slog_handler_->check_slog(tablet_key, has_slog))) { @@ -438,10 +331,9 @@ int ObTenantStorageCheckpointWriter::copy_tablet( int64_t slog_buf_pos = 0; MEMSET(slog_buf, 0, sizeof(ObUpdateTabletLog)); ObUpdateTabletLog slog; - slog.ls_id_ = tablet_key.ls_id_; slog.tablet_id_ = tablet_key.tablet_id_; ObMetaDiskAddr old_addr; - const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_); + const ObTabletPersisterParam param(0, tablet_key.tablet_id_); if (OB_FAIL(share::g_mp->tenant_meta_mem_mgr()->get_tablet_with_allocator(WashTabletPriority::WTP_LOW, tablet_key, allocator, tablet_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -567,7 +459,7 @@ int ObTenantStorageCheckpointWriter::batch_compare_and_swap_tablet() } ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletHandle new_tablet_handle; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObLSService *ls_svr = nullptr; for (int64_t i = 0; OB_SUCC(ret) && i < tablet_item_addr_info_arr_.count(); i++) { @@ -584,7 +476,7 @@ int ObTenantStorageCheckpointWriter::batch_compare_and_swap_tablet() } else if (OB_ISNULL(ls_svr = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls service is null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(addr_info.tablet_key_.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(ls_svr->get_ls(tenant_ls))) { LOG_WARN("fail to get ls", K(ret), K(addr_info)); } else { if (OB_FAIL(get_tablet_with_addr(addr_info, new_tablet_handle))) { @@ -600,7 +492,7 @@ int ObTenantStorageCheckpointWriter::batch_compare_and_swap_tablet() LOG_INFO("the tablet has changed, skip the swap", K(tablet_addr), K(addr_info)); } else { do { - if (OB_FAIL(ls_handle.get_ls()->update_tablet_checkpoint( + if (OB_FAIL(tenant_ls->update_tablet_checkpoint( addr_info.tablet_key_, addr_info.old_addr_, addr_info.new_addr_, diff --git a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.h b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.h index 7fc896587..a44bdf50a 100644 --- a/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.h +++ b/src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.h @@ -25,11 +25,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -} - namespace storage { struct ObLSCkptMember; @@ -60,18 +55,6 @@ class ObTenantStorageCheckpointWriter final int record_meta(blocksstable::MacroBlockId &ls_meta_entry); // record meta for snapshot - int record_single_ls_meta( - const blocksstable::MacroBlockId &orig_ls_meta_entry, - const share::ObLSID &ls_id, - ObIArray &orig_linked_block_list, - blocksstable::MacroBlockId &ls_meta_entry, - share::SCN &clog_max_scn); - int delete_single_ls_meta( - const blocksstable::MacroBlockId &orig_ls_meta_entry, - const share::ObLSID &ls_id, - ObIArray &orig_linked_block_list, - blocksstable::MacroBlockId &ls_meta_entry); - int get_ls_block_list(common::ObIArray *&block_list); int get_tablet_block_list(common::ObIArray *&block_list); int batch_compare_and_swap_tablet(); @@ -96,9 +79,6 @@ class ObTenantStorageCheckpointWriter final TO_STRING_KV(K_(tablet_key), K_(old_addr), K_(new_addr), K_(tablet_pool_type), K_(need_rollback)); }; - int copy_ls_meta_for_deleting(const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len, - const share::ObLSID &ls_id); - int copy_ls_meta_for_creating(const ObMetaDiskAddr &addr, const char *buf, const int64_t buf_len); int write_item(const ObLSCkptMember &ls_ckpt_member); int close(blocksstable::MacroBlockId &ls_meta_entry); int do_record_ls_meta(ObLS &ls, share::SCN &clog_max_scn); diff --git a/src/storage/tablelock/ob_lock_memtable.cpp b/src/storage/tablelock/ob_lock_memtable.cpp index 27a44faa4..67cc7896b 100644 --- a/src/storage/tablelock/ob_lock_memtable.cpp +++ b/src/storage/tablelock/ob_lock_memtable.cpp @@ -21,7 +21,7 @@ #include "storage/tablelock/ob_table_lock_iterator.h" #include "storage/memtable/ob_lock_wait_mgr.h" #include "storage/tablelock/ob_table_lock_deadlock.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/compaction/ob_schedule_dag_func.h" #include "storage/tx_storage/ob_ls_service.h" @@ -89,18 +89,17 @@ int ObLockTableSplitLogCb::on_failure() return ret; } -int ObLockTableSplitLogCb::init(ObLockMemtable *memtable, const share::ObLSID &ls_id) +int ObLockTableSplitLogCb::init(ObLockMemtable *memtable) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("init ObLockTableSplitLogCb twice", KR(ret), K(memtable), K(ls_id)); - } else if (!is_valid_(memtable, ls_id)){ + LOG_WARN("init ObLockTableSplitLogCb twice", KR(ret), K(memtable)); + } else if (!is_valid_(memtable)){ ret = OB_INVALID_ARGUMENT; - LOG_WARN("the parameters is invalid", K(ret), K(memtable), K(ls_id)); + LOG_WARN("the parameters is invalid", K(ret), K(memtable)); } else { memtable_ = memtable; - ls_id_ = ls_id; is_inited_ = true; } return ret; @@ -136,7 +135,7 @@ int ObLockTableSplitLogCb::set(const ObTabletID &src_tablet_id, } -bool ObLockTableSplitLogCb::is_valid_(const ObLockMemtable *memtable, const ObLSID &ls_id) +bool ObLockTableSplitLogCb::is_valid_(const ObLockMemtable *memtable) { int ret = OB_SUCCESS; bool is_valid = true; @@ -144,10 +143,6 @@ bool ObLockTableSplitLogCb::is_valid_(const ObLockMemtable *memtable, const ObLS is_valid = false; LOG_WARN("ObLockMemtable in ObLockTableSplitLogCb is null"); } - if (!ls_id.is_valid()) { - is_valid = false; - LOG_WARN("ObLSID in ObLockTableSplitLogCb is invalid"); - } return is_valid; } @@ -186,7 +181,7 @@ ObLockMemtable::ObLockMemtable() max_committed_scn_(), is_frozen_(false), need_check_tablet_status_(false), - reserved_counter_(0), + tablet_status_check_counter_(0), freezer_(nullptr), flush_lock_(common::ObLatchIds::CLOG_CKPT_LOCK) {} @@ -198,29 +193,26 @@ ObLockMemtable::~ObLockMemtable() int ObLockMemtable::init( const ObITable::TableKey &table_key, - const ObLSID &ls_id, ObFreezer *freezer) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("init ObLockMemtable twice", KR(ret), K(ls_id)); + LOG_WARN("init ObLockMemtable twice", KR(ret)); } else if (OB_ISNULL(freezer) - || !ls_id.is_valid() || !table_key.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(freezer), K(table_key)); + LOG_WARN("invalid argument", K(ret), K(freezer), K(table_key)); } else { if (OB_FAIL(ObITable::init(table_key))) { LOG_WARN("ObITable::init fail", K(ret), K(table_key)); } else if (OB_FAIL(obj_lock_map_.init())) { LOG_WARN("lock map mgr init failed.", K(ret)); } else { - ls_id_ = ls_id; freezer_ = freezer; is_inited_ = true; - LOG_INFO("ObLockMemtable init successfully", K(ls_id), K(table_key), K(this)); + LOG_INFO("ObLockMemtable init successfully", K(table_key), K(this)); } if (OB_FAIL(ret)) { reset(); @@ -240,7 +232,7 @@ void ObLockMemtable::reset() flushed_scn_.set_min(); is_frozen_ = false; need_check_tablet_status_ = false; - reserved_counter_ = 0; + tablet_status_check_counter_ = 0; freezer_ = nullptr; is_inited_ = false; } @@ -261,8 +253,8 @@ int ObLockMemtable::lock_( bool register_to_deadlock = false; ObTxIDSet conflict_tx_set; const bool is_two_phase_lock = param.is_two_phase_lock_; - int64_t input_reserved_counter = -1; - int64_t output_reserved_counter = -1; + int64_t input_status_check_counter = -1; + int64_t output_status_check_counter = -1; // 1. record lock myself(check conflict). // 2. record lock at memtable ctx. @@ -279,8 +271,8 @@ int ObLockMemtable::lock_( if (OB_FAIL(guard.write_auth(ctx))) { LOG_WARN("not allow lock table.", K(ret), K(ctx)); } else if (OB_FAIL(check_tablet_write_allow_(lock_op, - input_reserved_counter, - output_reserved_counter))) { + input_status_check_counter, + output_status_check_counter))) { LOG_WARN("check tablet write allow failed", K(ret), K(lock_op)); } else { mem_ctx = static_cast(ctx.mvcc_acc_ctx_.mem_ctx_); @@ -314,10 +306,10 @@ int ObLockMemtable::lock_( LOG_WARN("record lock at mem_ctx failed.", K(ret), K(lock_op)); } else if (FALSE_IT(succ_step = STEP_IN_MEM_CTX)) { } else { - input_reserved_counter = output_reserved_counter; + input_status_check_counter = output_status_check_counter; ret = check_tablet_write_allow_(lock_op, - input_reserved_counter, - output_reserved_counter); + input_status_check_counter, + output_status_check_counter); } } if (OB_FAIL(ret) && succ_step == STEP_IN_LOCK_MGR) { @@ -430,13 +422,12 @@ int ObLockMemtable::lock_( } int ObLockMemtable::check_tablet_write_allow_(const ObTableLockOp &lock_op, - const int64_t input_reserved_counter, - int64_t &output_reserved_counter) + const int64_t input_status_check_counter, + int64_t &output_status_check_counter) { int ret = OB_SUCCESS; ObTabletID tablet_id; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; ObTabletHandle tablet_handle; ObTabletCreateDeleteMdsUserData data; mds::MdsWriter writer; @@ -445,30 +436,27 @@ int ObLockMemtable::check_tablet_write_allow_(const ObTableLockOp &lock_op, // the order must be ensured bool need_check_tablet_status = ATOMIC_LOAD(&need_check_tablet_status_); - output_reserved_counter = ATOMIC_LOAD(&reserved_counter_); + output_status_check_counter = ATOMIC_LOAD(&tablet_status_check_counter_); if (!need_check_tablet_status - && (input_reserved_counter < 0 - || input_reserved_counter == output_reserved_counter)) { + && (input_status_check_counter < 0 + || input_status_check_counter == output_status_check_counter)) { } else if (!lock_op.lock_id_.is_tablet_lock()) { } else if (OB_FAIL(lock_op.lock_id_.convert_to(tablet_id))) { LOG_WARN("convert lock id to tablet_id failed", K(ret), K(lock_op)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::TABLELOCK_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", K(ret), KP(ls)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, - tablet_handle, - 0, - ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet with timeout failed", K(ret), K(ls->get_ls_id()), K(tablet_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, + tablet_handle, + 0, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("get tablet with timeout failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_latest_tablet_status( data, writer, trans_stat, trans_version))) { LOG_WARN("failed to get CreateDeleteMdsUserData", KR(ret)); if (OB_EMPTY_RESULT == ret) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("failed to get CreateDeleteMdsUserData", KR(ret), K(ls->get_ls_id()), K(tablet_id)); + LOG_WARN("failed to get CreateDeleteMdsUserData", KR(ret), K(tablet_id)); } } else if (trans_stat == mds::TwoPhaseCommitState::ON_COMMIT && data.tablet_status_.is_writable_for_dml()) { // allow @@ -492,8 +480,8 @@ int ObLockMemtable::unlock_( bool need_retry = false; uint64_t unused_lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0}; ObLockStep succ_step = STEP_BEGIN; - int64_t input_reserved_counter = -1; - int64_t output_reserved_counter = -1; + int64_t input_status_check_counter = -1; + int64_t output_status_check_counter = -1; // 1. record unlock op myself(check conflict). // 2. record unlock op at memtable ctx. @@ -512,8 +500,8 @@ int ObLockMemtable::unlock_( } else if (OB_FAIL(guard.write_auth(ctx))) { LOG_WARN("not allow unlock table.", K(ret), K(ctx)); } else if (OB_FAIL(check_tablet_write_allow_(unlock_op, - input_reserved_counter, - output_reserved_counter))) { + input_status_check_counter, + output_status_check_counter))) { LOG_WARN("check tablet write allow failed", K(ret), K(unlock_op)); } else if (FALSE_IT(mem_ctx = static_cast(ctx.mvcc_acc_ctx_.mem_ctx_))) { // check whether the unlock op exist already @@ -538,10 +526,10 @@ int ObLockMemtable::unlock_( LOG_WARN("record lock at mem_ctx failed.", K(ret), K(unlock_op)); } else if (FALSE_IT(succ_step = STEP_IN_MEM_CTX)) { } else { - input_reserved_counter = output_reserved_counter; + input_status_check_counter = output_status_check_counter; ret = check_tablet_write_allow_(unlock_op, - input_reserved_counter, - output_reserved_counter); + input_status_check_counter, + output_status_check_counter); } if (OB_FAIL(ret) && succ_step == STEP_IN_LOCK_MGR) { obj_lock_map_.remove_lock_record(unlock_op); @@ -639,23 +627,20 @@ int ObLockMemtable::post_obj_lock_conflict_(ObMvccAccessCtx &acc_ctx, LOG_WARN("can not get tenant lock_wait_mgr MTL"); } else { int tmp_ret = OB_SUCCESS; - auto tx_ctx = acc_ctx.tx_ctx_; auto tx_id = acc_ctx.get_tx_id(); - bool remote_tx = tx_ctx->get_scheduler() != tx_ctx->get_addr(); // TODO: one thread only can wait at one lock now. // this may be not enough. if (OB_TMP_FAIL(lock_wait_mgr->post_lock(OB_TRY_LOCK_ROW_CONFLICT, LS_LOCK_TABLET, lock_id, lock_wait_expire_ts, - remote_tx, + false, -1, -1, // total_trans_node_cnt acc_ctx.tx_desc_->get_assoc_session_id(), tx_id, conflict_tx_id, lock_mode, - ls_id_, recheck_f))) { LOG_WARN("post_lock after tx conflict failed", K(tmp_ret), K(tx_id), K(conflict_tx_id)); @@ -676,8 +661,7 @@ void ObLockMemtable::update_rec_and_max_committed_scn(const share::SCN &commit_s K(commit_scn), K(rec_scn_), K(pre_rec_scn_), - K(freeze_scn_), - K(ls_id_)); + K(freeze_scn_)); pre_rec_scn_.dec_update(commit_scn); } else { rec_scn_.dec_update(commit_scn); @@ -935,7 +919,7 @@ int ObLockMemtable::update_lock_status( } else if ((OUT_TRANS_LOCK == op_info.op_type_ || OUT_TRANS_UNLOCK == op_info.op_type_) && LOCK_OP_COMPLETE == status) { update_rec_and_max_committed_scn(commit_scn); - LOG_INFO("out_trans update_lock_status", K(ret), K(op_info), K(commit_scn), K(status), K(rec_scn_), K(ls_id_)); + LOG_INFO("out_trans update_lock_status", K(ret), K(op_info), K(commit_scn), K(status), K(rec_scn_)); } LOG_DEBUG("ObLockMemtable::update_lock_status", K(ret), K(op_info), K(commit_scn), K(status)); return ret; @@ -1111,7 +1095,7 @@ SCN ObLockMemtable::get_rec_scn() // no need lock because rec_scn_ aesc except INT64_MAX LOG_INFO("rec_scn of ObLockMemtable is ", K(rec_scn_), K(flushed_scn_), K(pre_rec_scn_), - K(freeze_scn_), K(max_committed_scn_), K(is_frozen_), K(ls_id_)); + K(freeze_scn_), K(max_committed_scn_), K(is_frozen_)); // If pre_rec_scn_ is max, it means that previous memtable // has already been flushed. In ohter words, it means that // rec_scn_ is ready to work, so we can return rec_scn_. @@ -1130,7 +1114,7 @@ SCN ObLockMemtable::get_rec_scn() if (pre_rec_scn_ > rec_scn_) { LOG_INFO("prec_rec_scn_ is larger than rec_scn_!", K(pre_rec_scn_), K(rec_scn_), K(flushed_scn_), K(freeze_scn_), - K(max_committed_scn_), K(is_frozen_), K(ls_id_)); + K(max_committed_scn_), K(is_frozen_)); } return share::SCN::min(pre_rec_scn_, rec_scn_); } @@ -1156,17 +1140,17 @@ int ObLockMemtable::on_memtable_flushed() } else { ret = OB_ERR_UNEXPECTED; LOG_ERROR("freeze_scn_ should not smaller than flushed_scn_", K(ret), - K_(flushed_scn), K_(freeze_scn), K_(ls_id)); + K_(flushed_scn), K_(freeze_scn)); } ATOMIC_STORE(&is_frozen_, false); - LOG_INFO("lock memtable on_memtable_flushed success", K(ret), K(flushed_scn_), K(ls_id_)); + LOG_INFO("lock memtable on_memtable_flushed success", K(ret), K(flushed_scn_)); return ret; } bool ObLockMemtable::can_be_minor_merged() { bool bool_ret = is_frozen_memtable(); - LOG_INFO("ObLockMemtable::can_be_minor_merged", K(bool_ret), K(ls_id_)); + LOG_INFO("ObLockMemtable::can_be_minor_merged", K(bool_ret)); return bool_ret; } @@ -1189,11 +1173,11 @@ int ObLockMemtable::flush(SCN recycle_scn, SCN rec_scn = get_rec_scn(); if (rec_scn >= recycle_scn) { LOG_INFO("lock memtable no need to flush", K(rec_scn), K(recycle_scn), - K(is_frozen_), K(ls_id_)); + K(is_frozen_)); } else if (is_active_memtable()) { freeze_scn_.inc_update(max_committed_scn_); if (flushed_scn_ >= freeze_scn_) { - LOG_INFO("skip freeze because of flushed", K_(ls_id), K_(flushed_scn), K_(freeze_scn)); + LOG_INFO("skip freeze because of flushed", K_(flushed_scn), K_(freeze_scn)); } else { pre_rec_scn_ = rec_scn_; rec_scn_.set_max(); @@ -1212,7 +1196,7 @@ int ObLockMemtable::flush(SCN recycle_scn, if (is_frozen_memtable()) { SCN max_consequent_callbacked_scn = SCN::min_scn(); if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { - LOG_WARN("get_max_consequent_callbacked_scn failed", K(ret), K(ls_id_)); + LOG_WARN("get_max_consequent_callbacked_scn failed", K(ret)); } else if (max_consequent_callbacked_scn < freeze_scn_) { LOG_INFO("lock memtable not ready for flush", K(max_consequent_callbacked_scn), @@ -1221,7 +1205,6 @@ int ObLockMemtable::flush(SCN recycle_scn, // dependent to judging is_active_memtable() in dag // otherwise maybe merge active memtable compaction::ObTabletMergeDagParam param; - param.ls_id_ = ls_id_; param.tablet_id_ = LS_LOCK_TABLET; param.merge_type_ = compaction::MINI_MERGE; param.merge_version_ = ObVersion::MIN_VERSION; @@ -1230,7 +1213,7 @@ int ObLockMemtable::flush(SCN recycle_scn, LOG_WARN("failed to schedule lock_memtable merge dag", K(ret), K(this)); } } else { - LOG_INFO("schedule lock_memtable merge_dag successfully", K(ls_id_), K(freeze_scn_)); + LOG_INFO("schedule lock_memtable merge_dag successfully", K(freeze_scn_)); } } } @@ -1347,7 +1330,7 @@ int ObLockMemtable::replay_row( LOG_WARN("replay lock failed", K(ret), K(lock_op)); } } - LOG_DEBUG("ObMemtable::replay_row finish.", K(ret), K(lock_id), K(ls_id_)); + LOG_DEBUG("ObMemtable::replay_row finish.", K(ret), K(lock_id)); return ret; } @@ -1366,7 +1349,7 @@ int ObLockMemtable::replay_lock( } else if (OB_FAIL(replay_lock_(mem_ctx, lock_op, scn))) { LOG_WARN("replay lock failed", K(ret), K(lock_op)); } - LOG_DEBUG("ObMemtable::replay_lock finish.", K(ret), K(lock_op), K(ls_id_), K(scn)); + LOG_DEBUG("ObMemtable::replay_lock finish.", K(ret), K(lock_op), K(scn)); return ret; } @@ -1381,8 +1364,8 @@ int ObLockMemtable::table_lock_split(const ObTabletID &src_tablet_id, if (OB_FAIL(check_valid_for_table_lock_split_(src_tablet_id, dst_tablet_ids))) { LOG_WARN( "the parameters for table lock split is invlaid", K(ret), K(split_cb), K(src_tablet_id), K(dst_tablet_ids)); - } else if (OB_FAIL(split_cb.init(this, ls_id_))) { - LOG_WARN("init ObLockTableSplitLogCb failed", K(ret), K(ls_id_)); + } else if (OB_FAIL(split_cb.init(this))) { + LOG_WARN("init ObLockTableSplitLogCb failed", K(ret)); } else if (OB_FAIL(obj_lock_map_.table_lock_split(src_tablet_id, dst_tablet_ids, trans_id, split_cb))) { LOG_WARN("table lock split failed", K(ret), K(src_tablet_id), K(dst_tablet_ids), K(trans_id), K(split_cb)); } @@ -1401,24 +1384,16 @@ int ObLockMemtable::register_into_deadlock_detector_( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObTransLockPartID tx_lock_part_id; - ObAddr parent_addr; - const ObLSID &ls_id = ctx.ls_id_; const int64_t priority = ~(ctx.mvcc_acc_ctx_.tx_desc_->get_active_ts()); tx_lock_part_id.lock_id_ = lock_op.lock_id_; tx_lock_part_id.trans_id_ = lock_op.create_trans_id_; if (OB_FAIL(ObTableLockDeadlockDetectorHelper::register_trans_lock_part( - tx_lock_part_id, ls_id, priority))) { - LOG_WARN("register trans lock part failed", K(ret), K(tx_lock_part_id), - K(ls_id)); - } else if (OB_FAIL(ObTransDeadlockDetectorAdapter::get_trans_scheduler_info_on_participant( - tx_lock_part_id.trans_id_, ls_id, parent_addr))) { - LOG_WARN("get scheduler address failed", K(tx_lock_part_id), K(ls_id)); + tx_lock_part_id, priority))) { + LOG_WARN("register trans lock part failed", K(ret), K(tx_lock_part_id)); } else if (OB_FAIL(ObTableLockDeadlockDetectorHelper::add_parent( - tx_lock_part_id, parent_addr, lock_op.create_trans_id_))) { + tx_lock_part_id, GCTX.self_addr(), lock_op.create_trans_id_))) { LOG_WARN("add parent failed", K(ret), K(tx_lock_part_id)); - } else if (OB_FAIL(ObTableLockDeadlockDetectorHelper::block(tx_lock_part_id, - ls_id, - lock_op))) { + } else if (OB_FAIL(ObTableLockDeadlockDetectorHelper::block(tx_lock_part_id, lock_op))) { LOG_WARN("add dependency failed", K(ret), K(tx_lock_part_id)); } else { LOG_DEBUG("succeed register to the dead lock detector"); @@ -1588,7 +1563,7 @@ int ObLockMemtable::switch_to_follower() void ObLockMemtable::enable_check_tablet_status(const bool need_check) { // the order must be ensured - ATOMIC_INC(&reserved_counter_); + ATOMIC_INC(&tablet_status_check_counter_); ATOMIC_STORE(&need_check_tablet_status_, need_check); } diff --git a/src/storage/tablelock/ob_lock_memtable.h b/src/storage/tablelock/ob_lock_memtable.h index 0efe02479..b7c784027 100644 --- a/src/storage/tablelock/ob_lock_memtable.h +++ b/src/storage/tablelock/ob_lock_memtable.h @@ -80,7 +80,6 @@ friend class ObOBJLock; ObLockTableSplitLogCb() : is_inited_(false), memtable_(nullptr), - ls_id_(share::ObLSID::INVALID_LS_ID), src_tablet_id_(OB_INVALID_ID), dst_tablet_ids_(), is_logging_(false), @@ -88,7 +87,7 @@ friend class ObOBJLock; last_submit_log_ts_(OB_INVALID_TIMESTAMP), last_submit_scn_() {} virtual ~ObLockTableSplitLogCb() {} - int init(ObLockMemtable *memtable, const share::ObLSID &ls_id); + int init(ObLockMemtable *memtable); int set(const ObTabletID &src_tablet_id, const ObSArray &dst_tablet_ids); bool cb_success() { return cb_success_; }; // we should ensure that the lock memtable won't be released @@ -98,7 +97,6 @@ friend class ObOBJLock; const char *get_cb_name() const override { return "LockTableSplitLogCb"; } TO_STRING_KV(K(is_inited_), K(memtable_), - K(ls_id_), K(src_tablet_id_), K(dst_tablet_ids_), K(last_submit_log_ts_), @@ -106,12 +104,11 @@ friend class ObOBJLock; K(is_logging_), K(cb_success_)); private: - bool is_valid_(const ObLockMemtable *memtable, const share::ObLSID &ls_id); + bool is_valid_(const ObLockMemtable *memtable); private: bool is_inited_; ObLockMemtable *memtable_; - share::ObLSID ls_id_; ObTabletID src_tablet_id_; ObSArray dst_tablet_ids_; @@ -142,9 +139,7 @@ class ObLockMemtable ObLockMemtable(); ~ObLockMemtable(); - int init(const ObITable::TableKey &table_key, - const share::ObLSID &ls_id, - storage::ObFreezer *freezer); + int init(const ObITable::TableKey &table_key, storage::ObFreezer *freezer); void reset(); // =================== LOCK FUNCTIONS ===================== // try to lock a object. @@ -298,7 +293,7 @@ class ObLockMemtable void enable_check_tablet_status(const bool need_check); - INHERIT_TO_STRING_KV("ObITable", ObITable, KP(this), K_(snapshot_version), K_(ls_id)); + INHERIT_TO_STRING_KV("ObITable", ObITable, KP(this), K_(snapshot_version)); private: enum ObLockStep { STEP_BEGIN = 0, @@ -346,8 +341,8 @@ class ObLockMemtable int unregister_from_deadlock_detector_(const ObTableLockOp &lock_op); int check_tablet_write_allow_(const ObTableLockOp &lock_op, - const int64_t input_reserved_counter, - int64_t &output_reserved_counter); + const int64_t input_status_check_counter, + int64_t &output_status_check_counter); int get_lock_wait_expire_ts_(const int64_t lock_wait_start_ts); int check_and_set_tx_lock_timeout_(const memtable::ObMvccAccessCtx &acc_ctx); private: @@ -369,10 +364,9 @@ class ObLockMemtable share::SCN pre_rec_scn_; share::SCN max_committed_scn_; bool is_frozen_; - // enable tablet status check bool need_check_tablet_status_; - // detect status changes between table lock operations - int64_t reserved_counter_; + // Detect tablet status check toggles between two table-lock operations. + int64_t tablet_status_check_counter_; storage::ObFreezer *freezer_; RWLock flush_lock_; // lock before change ts diff --git a/src/storage/tablelock/ob_lock_memtable_mgr.cpp b/src/storage/tablelock/ob_lock_memtable_mgr.cpp index 85e6c5ef1..8d446b81c 100644 --- a/src/storage/tablelock/ob_lock_memtable_mgr.cpp +++ b/src/storage/tablelock/ob_lock_memtable_mgr.cpp @@ -32,8 +32,7 @@ namespace tablelock { ObLockMemtableMgr::ObLockMemtableMgr() - : ls_id_(), - lock_def_() + : lock_def_() { int ret = OB_SUCCESS; if (OB_FAIL(lock_def_.init(lib::ObMemAttr("LockMemtableMgr")))) { @@ -53,19 +52,17 @@ void ObLockMemtableMgr::destroy() int ret = OB_SUCCESS; const int64_t ref_cnt = get_ref(); if (OB_UNLIKELY(0 != ref_cnt)) { - LOG_ERROR("ref cnt is NOT 0", K(ret), K(ref_cnt), K_(ls_id), KPC(this)); + LOG_ERROR("ref cnt is NOT 0", K(ret), K(ref_cnt), KPC(this)); } MemMgrWLockGuard lock_guard(lock_); reset_tables(); - ls_id_.reset(); freezer_ = NULL; is_inited_ = false; } int ObLockMemtableMgr::init( const common::ObTabletID &tablet_id, - const ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) { @@ -73,21 +70,19 @@ int ObLockMemtableMgr::init( int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("lock memtable mgr init twice.", K(ret), K_(ls_id)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) || - OB_ISNULL(freezer) || + LOG_WARN("lock memtable mgr init twice.", K(ret)); + } else if (OB_ISNULL(freezer) || OB_ISNULL(t3m)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), KP(freezer), KP(t3m)); + LOG_WARN("invalid argument", K(ret), KP(freezer), KP(t3m)); } else if (!lock_def_.is_inited()) { ret = OB_NOT_INIT; - LOG_WARN("lock memtable mgr lock not init", K(ret), K(tablet_id), K(ls_id)); + LOG_WARN("lock memtable mgr lock not init", K(ret), K(tablet_id)); } else { - ls_id_ = ls_id; freezer_ = freezer; t3m_ = t3m; is_inited_ = true; - LOG_INFO("lock memtable mgr init successfully", K(ls_id), K(tablet_id), K(this)); + LOG_INFO("lock memtable mgr init successfully", K(tablet_id), K(this)); } return ret; } @@ -111,7 +106,7 @@ int ObLockMemtableMgr::create_memtable(const CreateMemtableArg &arg) if (get_memtable_count_() > 0) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("lock memtable already exists, should not create again", K(ret), K_(ls_id)); + LOG_WARN("lock memtable already exists, should not create again", K(ret)); } else if (OB_FAIL(t3m_->acquire_lock_memtable(handle))) { LOG_WARN("failed to create memtable", K(ret)); } else if (OB_ISNULL(table = handle.get_table())) { @@ -120,7 +115,7 @@ int ObLockMemtableMgr::create_memtable(const CreateMemtableArg &arg) } else if (OB_ISNULL(memtable = static_cast(table))) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("this is not lock memtable", K(ret), KPC(table)); - } else if (OB_FAIL(memtable->init(table_key, ls_id_, freezer_))) { + } else if (OB_FAIL(memtable->init(table_key, freezer_))) { LOG_WARN("memtable init fail.", K(ret)); } else if (OB_FAIL(add_memtable_(handle))) { LOG_WARN("add memtable fail.", K(ret)); @@ -128,7 +123,7 @@ int ObLockMemtableMgr::create_memtable(const CreateMemtableArg &arg) ret = OB_ERR_UNEXPECTED; LOG_WARN("ls_tx_svr is null", K(ret)); } else { - LOG_INFO("create lock memtable successfully", K_(ls_id), K(memtable), KPC(this)); + LOG_INFO("create lock memtable successfully", K(memtable), KPC(this)); } return ret; @@ -187,10 +182,10 @@ int ObLockMemtableMgr::release_head_memtable_(storage::ObIMemtable *imemtable, // for force const int64_t idx = get_memtable_idx(memtable_head_); if (nullptr != tables_[idx] && memtable == tables_[idx]) { - LOG_INFO("release head memtable", K(ret), K_(ls_id), KP(memtable)); + LOG_INFO("release head memtable", K(ret), KP(memtable)); release_head_memtable(); FLOG_INFO("succeed to release head lock table memtable", K(ret), - K_(ls_id), KP(imemtable), K(memtable_head_), K(memtable_tail_)); + KP(imemtable), K(memtable_head_), K(memtable_tail_)); } } else if (!force) { // just for flush diff --git a/src/storage/tablelock/ob_lock_memtable_mgr.h b/src/storage/tablelock/ob_lock_memtable_mgr.h index 1ab6fd68f..b280fa8b8 100644 --- a/src/storage/tablelock/ob_lock_memtable_mgr.h +++ b/src/storage/tablelock/ob_lock_memtable_mgr.h @@ -28,7 +28,6 @@ class ObTabletID; namespace share { -class ObLSID; } namespace memtable @@ -59,7 +58,6 @@ class ObLockMemtableMgr : public storage::ObIMemtableMgr // Init the memtable mgr, we use logstream id to fetch the ls_ctx_mgr and t3m // to alloc the memtable. virtual int init(const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, storage::ObFreezer *freezer, storage::ObTenantMetaMemMgr *t3m) override; virtual void destroy() override; @@ -74,7 +72,6 @@ class ObLockMemtableMgr : public storage::ObIMemtableMgr const bool force = false) override; private: - share::ObLSID ls_id_; common::ObQSyncLock lock_def_; }; diff --git a/src/storage/tablelock/ob_lock_table.cpp b/src/storage/tablelock/ob_lock_table.cpp index 2317b45e9..b1210f822 100644 --- a/src/storage/tablelock/ob_lock_table.cpp +++ b/src/storage/tablelock/ob_lock_table.cpp @@ -273,7 +273,7 @@ int ObLockTable::offline() { int ret = OB_SUCCESS; if (OB_NOT_NULL(parent_)) { - LOG_INFO("lock table offline", K(parent_->get_ls_id())); + LOG_INFO("lock table offline"); } // release all lock memtables before clean cache @@ -302,7 +302,7 @@ int ObLockTable::online() ObTabletMemberWrapper table_store_wrapper; ObLSTabletService *ls_tablet_svr = nullptr; if (OB_NOT_NULL(parent_)) { - LOG_INFO("online lock table", K(parent_->get_ls_id())); + LOG_INFO("online lock table"); } CreateMemtableArg arg; @@ -341,7 +341,6 @@ int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = parent_->get_ls_id(); share::schema::ObTableSchema table_schema; ObIMemtableMgr *memtable_mgr = nullptr; ObMemtableMgrHandle memtable_mgr_handle; @@ -355,12 +354,11 @@ int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, false/*skip_column_info*/, DATA_CURRENT_VERSION))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); - } else if (OB_FAIL(parent_->create_ls_inner_tablet(ls_id, - LS_LOCK_TABLET, + } else if (OB_FAIL(parent_->create_ls_inner_tablet(LS_LOCK_TABLET, ObLS::LS_INNER_TABLET_FROZEN_SCN, create_tablet_schema, create_scn))) { - LOG_WARN("failed to create lock tablet", K(ret), K(ls_id), K(LS_LOCK_TABLET), + LOG_WARN("failed to create lock tablet", K(ret), K(LS_LOCK_TABLET), K(table_schema), K(compat_mode), K(create_scn)); } else if (OB_FAIL(parent_->get_tablet_svr()-> get_lock_memtable_mgr(memtable_mgr_handle))) { @@ -377,11 +375,10 @@ int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const int ObLockTable::remove_tablet() { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = parent_->get_ls_id(); if (IS_NOT_INIT) { LOG_WARN("lock table does not inited, remove do nothing"); - } else if (OB_FAIL(parent_->remove_ls_inner_tablet(ls_id, LS_LOCK_TABLET))) { - LOG_ERROR("failed to remove ls inner tablet", K(ret), K(ls_id), K(LS_LOCK_TABLET)); + } else if (OB_FAIL(parent_->remove_ls_inner_tablet(LS_LOCK_TABLET))) { + LOG_ERROR("failed to remove ls inner tablet", K(ret), K(LS_LOCK_TABLET)); ob_usleep(1000 * 1000); ob_abort(); } @@ -823,7 +820,7 @@ int ObLockTable::add_lock_into_queue(storage::ObStoreCtx &ctx, const ObLockParam return ret; } -int ObLockTable::switch_to_leader() +int ObLockTable::activate() { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("switch_to_leader", 10 * 1000); @@ -833,8 +830,7 @@ int ObLockTable::switch_to_leader() } else { timeguard.click(); if (OB_NOT_NULL(parent_)) { - LOG_INFO("start to check and clear obj lock when switch to leader", K(ret), - K(parent_->get_ls_id())); + LOG_INFO("start to check and clear obj lock when switch to leader", K(ret)); } if (OB_FAIL(check_obj_lock_timer_.schedule(check_obj_lock_task_, 0 /* delay */, @@ -849,8 +845,7 @@ int ObLockTable::switch_to_leader() // ignore ret LOG_WARN("parent ls of ObLockTable is null", K(ret)); } else { - LOG_WARN("collect obj lock garbage when switch to leader failed", K(ret), - K(parent_->get_ls_id())); + LOG_WARN("collect obj lock garbage when switch to leader failed", K(ret)); } } else { // switch to leader for lock memtable @@ -868,7 +863,7 @@ int ObLockTable::switch_to_leader() return ret; } -void ObLockTable::switch_to_follower_forcedly() +void ObLockTable::deactivate() { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("switch_to_follower", 10 * 1000); @@ -881,20 +876,6 @@ void ObLockTable::switch_to_follower_forcedly() timeguard.click(); } -int ObLockTable::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("switch_to_follower", 10 * 1000); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLockTable is not inited", K(ret)); - } else if (OB_FAIL(switch_to_follower_())) { - LOG_WARN("switch to follower failed", K(ret)); - } - timeguard.click(); - return ret; -} - int ObLockTable::switch_to_follower_() { int ret = OB_SUCCESS; @@ -934,8 +915,6 @@ int ObLockTable::flush(share::SCN &scn) int ret = OB_SUCCESS; ObTableHandleV2 handle; ObLockMemtable *memtable = nullptr; - const share::ObLSID &ls_id = parent_->get_ls_id(); - if (IS_NOT_INIT) { ret = OB_NOT_INIT; TABLELOCK_LOG(WARN, "ObLockTable not inited", K(ret)); diff --git a/src/storage/tablelock/ob_lock_table.h b/src/storage/tablelock/ob_lock_table.h index c827c04b1..3bd7df10b 100644 --- a/src/storage/tablelock/ob_lock_table.h +++ b/src/storage/tablelock/ob_lock_table.h @@ -41,7 +41,6 @@ class ObTabletID; namespace share { -class ObLSID; namespace schema { class ObTableSchema; @@ -73,7 +72,7 @@ class ObLockMemtable; class ObLockMemtableMgr; class ObLockTable : public logservice::ObIReplaySubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::ObICheckpointSubHandler { public: @@ -165,10 +164,8 @@ class ObLockTable : public logservice::ObIReplaySubHandler, share::SCN get_rec_scn() override; int flush(share::SCN &rec_scn) override; // for role change - void switch_to_follower_forcedly() override; - int switch_to_leader() override; - int switch_to_follower_gracefully() override; - int resume_leader() override { return OB_SUCCESS; } + void deactivate() override; + int activate() override; // flush lock_memtable that flush had been failed diff --git a/src/storage/tablelock/ob_lock_utils.cpp b/src/storage/tablelock/ob_lock_utils.cpp index 16a832985..15bbdcab3 100644 --- a/src/storage/tablelock/ob_lock_utils.cpp +++ b/src/storage/tablelock/ob_lock_utils.cpp @@ -63,37 +63,6 @@ int ObInnerTableLockUtil::lock_inner_table_in_trans( return ret; } -int ObLSObjLockUtil::lock_ls_in_trans( - common::ObMySQLTransaction &trans, - const ObLSID &ls_id, - const transaction::tablelock::ObTableLockMode &lock_mode) -{ - int ret = OB_SUCCESS; - ObInnerSQLConnection *conn = NULL; - ObTimeoutCtx ctx; - const int64_t DEFAULT_TIMEOUT = GCONF.internal_sql_execute_timeout; - if (OB_UNLIKELY(!ls_id.is_valid_with_tenant() || !is_lock_mode_valid(lock_mode))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(ls_id), K(lock_mode)); - } else if (OB_ISNULL(conn = static_cast(trans.get_connection()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("connection is null", KR(ret), K(ls_id)); - } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_TIMEOUT))) { - LOG_WARN("fail to set default_timeout_ctx", KR(ret)); - } else { - ObLockObjRequest lock_arg; - lock_arg.lock_mode_ = lock_mode; - lock_arg.op_type_ = IN_TRANS_COMMON_LOCK; - lock_arg.timeout_us_ = ctx.get_timeout(); - lock_arg.obj_type_ = ObLockOBJType::OBJ_TYPE_LS; - lock_arg.obj_id_ = ls_id.id(); - if (OB_FAIL(ObInnerConnectionLockUtil::lock_obj(lock_arg, conn))) { - LOG_WARN("lock obj failed", KR(ret), K(lock_arg)); - } - } - return ret; -} - } // end namespace tablelock } // end namespace transaction } // end namespace oceanbase diff --git a/src/storage/tablelock/ob_lock_utils.h b/src/storage/tablelock/ob_lock_utils.h index 069326cc8..150a00849 100644 --- a/src/storage/tablelock/ob_lock_utils.h +++ b/src/storage/tablelock/ob_lock_utils.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_STORAGE_TABLELOCK_OB_LOCK_UTILS_H #define OCEANBASE_STORAGE_TABLELOCK_OB_LOCK_UTILS_H -#include "share/ob_ls_id.h" // ObLSID #include "share/inner_table/ob_inner_table_schema.h" #include "storage/tablelock/ob_table_lock_common.h" // ObTableLockMode @@ -64,26 +63,6 @@ class ObInnerTableLockUtil const bool is_from_sql); }; -class ObLSObjLockUtil -{ -public: - /* - * lock ls in trans with internal_sql_execute_timeout - * - * @param[in] trans: ObMySQLTransaction - * @param[in] ls_id: target log stream(ls) id - * @param[in] lock_mode: obj lock mode - * @return - * - OB_SUCCESS: lock ls successfully - * - OB_TRY_LOCK_ROW_CONFLICT: lock conflict - * - other: lock failed - */ - static int lock_ls_in_trans( - common::ObMySQLTransaction &trans, - const share::ObLSID &ls_id, - const ObTableLockMode &lock_mode); -}; - } // end namespace tablelock } // end namespace transaction } // end namespace oceanbase diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp index 2f9e40503..adab965bc 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp @@ -267,34 +267,6 @@ int ObLockMemCtx::get_table_lock_store_info(ObTableLockInfo &table_lock_info) return ret; } -int ObLockMemCtx::get_table_lock_for_reserved(ObTableLockInfo &table_lock_info, const ObIArray &tablet_list) -{ - int ret = OB_SUCCESS; - RDLockGuard guard(list_rwlock_); - DLIST_FOREACH(curr, lock_list_) { - if (OB_UNLIKELY(!curr->is_valid())) { - // no need dump to avoid been restored even if rollback - LOG_WARN("the table lock op no should not dump", K(curr->lock_op_)); - } else { - bool is_hit = false; - for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_list.count(); idx++) { - if (curr->lock_op_.is_tablet_lock(tablet_list.at(idx))) { - is_hit = true; - break; - } - } - if (OB_FAIL(ret)) { - } else if (!is_hit) { - } else if (OB_FAIL(table_lock_info.table_lock_ops_.push_back(curr->lock_op_))) { - LOG_WARN("fail to push back table_lock store info", K(ret)); - break; - } - } - } - table_lock_info.max_durable_scn_ = max_durable_scn_; - return ret; -} - int ObLockMemCtx::clear_table_lock( const bool is_committed, const SCN &commit_version, diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.h b/src/storage/tablelock/ob_mem_ctx_table_lock.h index e261b2893..94100b426 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.h +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.h @@ -129,7 +129,6 @@ class ObLockMemCtx int rollback_table_lock(const ObTxSEQ to_seq_no, const ObTxSEQ from_seq_no); int sync_log_succ(const share::SCN &scn); int get_table_lock_store_info(ObTableLockInfo &table_lock_info); - int get_table_lock_for_reserved(ObTableLockInfo &table_lock_info, const ObIArray &tablet_list); // used by deadlock detector to kill the trans. void set_killed() { is_killed_ = true; } diff --git a/src/storage/tablelock/ob_obj_lock.cpp b/src/storage/tablelock/ob_obj_lock.cpp index 9a18ee905..ac2bb3b88 100644 --- a/src/storage/tablelock/ob_obj_lock.cpp +++ b/src/storage/tablelock/ob_obj_lock.cpp @@ -2335,8 +2335,6 @@ int ObOBJLock::submit_log_(ObLockTableSplitLogCb &callback, const ObSArray &dst_tablet_ids) { int ret = OB_SUCCESS; - ObLSID ls_id; - if (!callback.is_logging_) { callback.is_logging_ = true; ObLockTableSplitLog split_log; @@ -2352,9 +2350,7 @@ int ObOBJLock::submit_log_(ObLockTableSplitLogCb &callback, logservice::ObLogBaseHeader base_header(logservice::ObLogBaseType::TABLE_LOCK_LOG_BASE_TYPE, logservice::ObReplayBarrierType::NO_NEED_BARRIER); buffer_size += base_header.get_serialize_size(); - ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; const bool need_nonblock = false; palf::LSN lsn; SCN scn; @@ -2366,15 +2362,10 @@ int ObOBJLock::submit_log_(ObLockTableSplitLogCb &callback, LOG_WARN("failed to serialize split log header", K(ret), K(buffer_size), K(pos)); } else if (OB_FAIL(split_log.serialize(buffer, buffer_size, pos))) { LOG_WARN("failed to serialize split log", K(ret), K(buffer_size), K(pos)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(callback.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", K(ret)); - } else if (OB_FAIL(ls->append(buffer, buffer_size, SCN::min_scn(), need_nonblock, false/*allow_compression*/, &callback, lsn, scn))) { + } else if (OB_FAIL(tenant_ls->append(buffer, buffer_size, SCN::min_scn(), + need_nonblock, false/*allow_compression*/, &callback, lsn, scn))) { LOG_WARN("failed to submit log", K(ret), K(buffer_size), K(pos)); } else { // These params should be gotten after append log to LS, diff --git a/src/storage/tablelock/ob_obj_lock.h b/src/storage/tablelock/ob_obj_lock.h index 0d41ce8c8..fe26327fd 100644 --- a/src/storage/tablelock/ob_obj_lock.h +++ b/src/storage/tablelock/ob_obj_lock.h @@ -32,7 +32,6 @@ namespace oceanbase namespace share { -class ObLSID; } namespace storage diff --git a/src/storage/tablelock/ob_table_lock_callback.cpp b/src/storage/tablelock/ob_table_lock_callback.cpp index 66326f145..ca236364f 100644 --- a/src/storage/tablelock/ob_table_lock_callback.cpp +++ b/src/storage/tablelock/ob_table_lock_callback.cpp @@ -18,7 +18,7 @@ #include "ob_table_lock_callback.h" #include "storage/tablelock/ob_lock_memtable.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { diff --git a/src/storage/tablelock/ob_table_lock_deadlock.cpp b/src/storage/tablelock/ob_table_lock_deadlock.cpp index 2102fa253..29c183e0c 100644 --- a/src/storage/tablelock/ob_table_lock_deadlock.cpp +++ b/src/storage/tablelock/ob_table_lock_deadlock.cpp @@ -19,7 +19,7 @@ #include "storage/tablelock/ob_table_lock_deadlock.h" #include "share/rc/ob_module_provider.h" #include "storage/tx/ob_trans_deadlock_adapter.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx_storage/ob_ls_service.h" namespace oceanbase @@ -36,7 +36,6 @@ OB_SERIALIZE_MEMBER(ObTransLockPartID, trans_id_, lock_id_); OB_SERIALIZE_MEMBER(ObTransLockPartBlockCallBack, - ls_id_, lock_op_); int ObTxLockPartOnDetectOp::operator() ( @@ -45,23 +44,14 @@ int ObTxLockPartOnDetectOp::operator() ( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSHandle handle; ObLS *ls = nullptr; - ObPartTransCtx *ctx = nullptr; - if (OB_UNLIKELY(!lock_part_id_.is_valid()) || - OB_UNLIKELY(!ls_id_.is_valid())) { + ObTxCtx *ctx = nullptr; + if (OB_UNLIKELY(!lock_part_id_.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid member", K(ret), K(lock_part_id_), K(ls_id_)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, - handle, - ObLSGetMod::TABLELOCK_MOD))) { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be null", K(ret)); - } else if (OB_FAIL(ls->get_tx_ctx(lock_part_id_.trans_id_, - true, /* does not check leader*/ - ctx))) { + LOG_WARN("invalid member", K(ret), K(lock_part_id_)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("get ls failed", K(ret)); + } else if (OB_FAIL(ls->get_tx_ctx(lock_part_id_.trans_id_, true, ctx))) { LOG_WARN("get tx ctx failed", K(ret), K(lock_part_id_)); } else { // tell the tx lock part it has been killed. @@ -75,17 +65,14 @@ int ObTxLockPartOnDetectOp::operator() ( } int ObTxLockPartOnDetectOp::set( - const ObTransLockPartID &lock_part_id, - const share::ObLSID &ls_id) + const ObTransLockPartID &lock_part_id) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!lock_part_id.is_valid()) || - OB_UNLIKELY(!ls_id.is_valid())) { + if (OB_UNLIKELY(!lock_part_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(lock_part_id), K(ls_id)); + LOG_WARN("invalid argument", K(lock_part_id)); } else { lock_part_id_ = lock_part_id; - ls_id_ = ls_id; } return ret; } @@ -93,7 +80,6 @@ int ObTxLockPartOnDetectOp::set( void ObTxLockPartOnDetectOp::reset() { lock_part_id_.reset(); - ls_id_.reset(); } static void release_buffers(char *&buffer, char *&buffer2) @@ -173,32 +159,24 @@ int ObTransLockPartBlockCallBack::operator()( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSHandle handle; ObLS *ls = nullptr; - ObPartTransCtx *ctx = nullptr; + ObTxCtx *ctx = nullptr; ObMemtableCtx *mem_ctx = nullptr; ObTxIDSet conflict_tx_set; if (OB_UNLIKELY(!is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("block callback is not valid", K(ret), KPC(this)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, - handle, - ObLSGetMod::TABLELOCK_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { if (OB_NOT_RUNNING == ret || OB_LS_NOT_EXIST == ret) { // ls is removing or removed. should never be here. LOG_ERROR("the ls need clean all the trans before it is removed", - K(ret), K_(ls_id)); + K(ret)); } else { - LOG_WARN("get ls failed", K(ret), K(ls_id_)); + LOG_WARN("get ls failed", K(ret)); } - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be null", K(ret)); - } else if (OB_FAIL(ls->get_tx_ctx(lock_op_.create_trans_id_, - true, /* does not check leader*/ - ctx))) { + } else if (OB_FAIL(ls->get_tx_ctx(lock_op_.create_trans_id_, true, ctx))) { LOG_WARN("get tx ctx failed", K(ret), K(lock_op_)); if (OB_TRANS_CTX_NOT_EXIST == ret) { // the tx may be killed, do not check conflict again, and remove it from @@ -210,10 +188,8 @@ int ObTransLockPartBlockCallBack::operator()( if (OB_ISNULL(mem_ctx = ctx->get_memtable_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("mem ctx should not be null", K(ret)); - } else if (OB_FAIL(ls->check_lock_conflict(mem_ctx, - lock_op_, - conflict_tx_set, - false /* does not include finished tx */))) { + } else if (OB_FAIL(ls->check_lock_conflict( + mem_ctx, lock_op_, conflict_tx_set, false /* does not include finished tx */))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("check lock conflict failed", K(ret), K_(lock_op)); } @@ -228,7 +204,6 @@ int ObTransLockPartBlockCallBack::operator()( // we should not remove the dependency because there may has new dependency now. // need_remove = (conflict_tx_set.count() == 0) ? true : false; ObTxIDSet::const_iterator_t it = conflict_tx_set.begin(); - ObAddr block_trans_addr; UserBinaryKey binary_key; for(;OB_SUCC(ret) && it != conflict_tx_set.end(); ++it) { @@ -238,18 +213,12 @@ int ObTransLockPartBlockCallBack::operator()( // we need get dependency next time. const ObTransID &block_trans_id = *it; binary_key.set_user_key(block_trans_id); - if (OB_FAIL(ObTransDeadlockDetectorAdapter::get_trans_scheduler_info_on_participant( - block_trans_id, ls_id_, block_trans_addr))) { - LOG_WARN("get block trans scheduler address failed", K(ret), K_(ls_id), K(block_trans_id)); - } else { - ObDependencyResource resource(block_trans_addr, - binary_key); - if (OB_FAIL(depend_res.push_back(resource))) { - LOG_WARN("push into array failed.", K(ret), K(resource)); - } + ObDependencyResource resource(GCTX.self_addr(), binary_key); + if (OB_FAIL(depend_res.push_back(resource))) { + LOG_WARN("push into array failed.", K(ret), K(resource)); } LOG_DEBUG("ObTransLockPartBlockCallBack get dependency", K(ret), K(lock_op_), - K(block_trans_id), K(block_trans_addr)); + K(block_trans_id)); } } if (OB_SUCCESS != (tmp_ret = ls->revert_tx_ctx(ctx))) { @@ -262,18 +231,16 @@ int ObTransLockPartBlockCallBack::operator()( } int ObTransLockPartBlockCallBack::init( - const share::ObLSID &ls_id, const ObTableLockOp &lock_op) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_valid())) { ret = OB_INIT_TWICE; LOG_WARN("call back has been inited", K(ret), KPC(this)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!lock_op.is_valid())) { + } else if (OB_UNLIKELY(!lock_op.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(lock_op)); + LOG_WARN("invalid argument", K(ret), K(lock_op)); } else { - ls_id_ = ls_id; lock_op_ = lock_op; } return ret; @@ -281,7 +248,6 @@ int ObTransLockPartBlockCallBack::init( int ObTableLockDeadlockDetectorHelper::register_trans_lock_part( const ObTransLockPartID &tx_lock_part_id, - const ObLSID &ls_id, const ObDetectorPriority priority) { int ret = OB_SUCCESS; @@ -291,12 +257,10 @@ int ObTableLockDeadlockDetectorHelper::register_trans_lock_part( CollectCallBack on_collect_callback(callback); - if (OB_UNLIKELY(!tx_lock_part_id.is_valid()) || - OB_UNLIKELY(!ls_id.is_valid())) { + if (OB_UNLIKELY(!tx_lock_part_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tx_lock_part_id), K(ls_id)); - } else if (OB_FAIL(on_detect_op.set(tx_lock_part_id, - ls_id))) { + LOG_WARN("invalid argument", K(ret), K(tx_lock_part_id)); + } else if (OB_FAIL(on_detect_op.set(tx_lock_part_id))) { LOG_WARN("set deadlock detect op failed", K(ret), K(tx_lock_part_id)); } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr() ->register_key(tx_lock_part_id, on_detect_op, @@ -305,7 +269,7 @@ int ObTableLockDeadlockDetectorHelper::register_trans_lock_part( LOG_WARN("register to deadlock detector failed.", K(ret), K(tx_lock_part_id), K(priority)); } - LOG_DEBUG("ObTableLockDeadlockDetectorHelper::register_trans_lock_part", K(ret), K(tx_lock_part_id), K(ls_id)); + LOG_DEBUG("ObTableLockDeadlockDetectorHelper::register_trans_lock_part", K(ret), K(tx_lock_part_id)); } return ret; } @@ -353,26 +317,23 @@ int ObTableLockDeadlockDetectorHelper::add_parent( int ObTableLockDeadlockDetectorHelper::block( const ObTransLockPartID &tx_lock_part_id, - const share::ObLSID &ls_id, const ObTableLockOp &lock_op) { int ret = OB_SUCCESS; if (ObDeadLockDetectorMgr::is_deadlock_enabled()) { ObTransLockPartBlockCallBack block_cb; if (OB_UNLIKELY(!tx_lock_part_id.is_valid()) || - OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!lock_op.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tx_lock_part_id), K(ls_id), K(lock_op)); - } else if (OB_FAIL(block_cb.init(ls_id, - lock_op))) { - LOG_WARN("block callback init failed", K(ret), K(ls_id), K(lock_op)); + LOG_WARN("invalid argument", K(ret), K(tx_lock_part_id), K(lock_op)); + } else if (OB_FAIL(block_cb.init(lock_op))) { + LOG_WARN("block callback init failed", K(ret), K(lock_op)); } else { // WARNING: be care for the BlockCallBack fn, it may use the wrong block interface. detector::BlockCallBack fn = block_cb; if (!fn.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("block callback invalid", K(ret), K(tx_lock_part_id), K(ls_id), K(lock_op)); + LOG_WARN("block callback invalid", K(ret), K(tx_lock_part_id), K(lock_op)); } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(tx_lock_part_id, fn))) { LOG_WARN("add block failed", K(ret), K(tx_lock_part_id), K(lock_op)); @@ -380,7 +341,7 @@ int ObTableLockDeadlockDetectorHelper::block( // do nothing } LOG_DEBUG("ObTableLockDeadlockDetectorHelper::block", K(ret), K(tx_lock_part_id), - K(ls_id), K(lock_op)); + K(lock_op)); } } return ret; diff --git a/src/storage/tablelock/ob_table_lock_deadlock.h b/src/storage/tablelock/ob_table_lock_deadlock.h index e24efa21d..bbed300c0 100644 --- a/src/storage/tablelock/ob_table_lock_deadlock.h +++ b/src/storage/tablelock/ob_table_lock_deadlock.h @@ -18,7 +18,6 @@ #define OCEANBASE_STORAGE_TABLELOCK_DEADLOCK_H_ #include "storage/deadlock/ob_deadlock_detector_common_define.h" -#include "share/ob_ls_id.h" #include "storage/tablelock/ob_table_lock_common.h" #include "storage/tx/ob_trans_define.h" @@ -86,12 +85,10 @@ class ObTxLockPartOnDetectOp int operator()( const common::ObIArray &info, const int64_t self_idx); - int set(const ObTransLockPartID &lock_part_id, - const share::ObLSID &ls_id); + int set(const ObTransLockPartID &lock_part_id); void reset(); private: ObTransLockPartID lock_part_id_; - share::ObLSID ls_id_; }; class ObTxLockPartCollectCallBack @@ -108,33 +105,19 @@ class ObTransLockPartBlockCallBack { OB_UNIS_VERSION(1); public: - ObTransLockPartBlockCallBack() - : ls_id_(), - lock_op_() - {} - ~ObTransLockPartBlockCallBack() - { - reset(); - } + ObTransLockPartBlockCallBack() : lock_op_() {} + ~ObTransLockPartBlockCallBack() = default; // @param [out] depend_res, a dependency list. // @param [out] need_remove, shall the callback be removed. int operator()(common::ObIArray &depend_res, bool &need_remove); - int init(const share::ObLSID &ls_id, - const ObTableLockOp &lock_op); + int init(const ObTableLockOp &lock_op); bool is_valid() const { - return ls_id_.is_valid() && lock_op_.is_valid(); - } - void reset() - { - ls_id_.reset(); - // lock_op_.reset(); + return lock_op_.is_valid(); } - TO_STRING_KV(K_(ls_id), K_(lock_op)); + TO_STRING_KV(K_(lock_op)); private: - // we need ls id to tell which ls we will try to find the obj lock - share::ObLSID ls_id_; // contain the info of which trans want to lock which obj with which lock mode. // we can get all the trans that will block this operation. ObTableLockOp lock_op_; @@ -150,7 +133,6 @@ class ObTableLockDeadlockDetectorHelper // all the trans lock part of one trans parent should use // the same ObTxLockPartDeadLockCtx. static int register_trans_lock_part(const ObTransLockPartID &tx_lock_part_id, - const share::ObLSID &ls_id, const share::detector::ObDetectorPriority priority); // unregister a trans lock part // @param [in] tx_lock_part_id, which trans lock part will be removed. @@ -164,10 +146,8 @@ class ObTableLockDeadlockDetectorHelper const ObTransID &parent_trans_id); // add the block dependency of trans lock part. // @param [in] tx_lock_part_id, whose dependency will be registered. - // @param [in] ls_id, which ls is the lock belong to. // @param [in] lock_op, the lock operation tx_lock_part_id wants to do. static int block(const ObTransLockPartID &tx_lock_part_id, - const share::ObLSID &ls_id, const ObTableLockOp &lock_op); }; diff --git a/src/storage/tablelock/ob_table_lock_local_executor.cpp b/src/storage/tablelock/ob_table_lock_local_executor.cpp index 42852c299..e4cb04f43 100644 --- a/src/storage/tablelock/ob_table_lock_local_executor.cpp +++ b/src/storage/tablelock/ob_table_lock_local_executor.cpp @@ -30,32 +30,12 @@ using namespace transaction::tablelock; namespace observer { -int check_exist(const share::ObLSID &ls_id, ObLSHandle &ls_handle) -{ - int ret = OB_SUCCESS; - ObLSService *ls_service = nullptr; - ObLS *ls = nullptr; - if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::TABLELOCK_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", K(ret), KP(ls)); - } else { - // do nothing - } - return ret; -} - template int check_exist(const ObLockTaskBatchRequest &arg, const common::ObTabletID &tablet_id, - ObLSHandle ls_handle) + ObLS * tenant_ls) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObTabletHandle tablet_handle; ObTabletStatus::Status tablet_status = ObTabletStatus::MAX; ObTabletCreateDeleteMdsUserData data; @@ -66,33 +46,33 @@ int check_exist(const ObLockTaskBatchRequest &arg, ObTableLockTaskType::UNLOCK_ALONE_TABLET == arg.task_type_ || ObTableLockTaskType::ADD_LOCK_INTO_QUEUE_WITHOUT_CHECK == arg.task_type_) { // alone tablet does not check exist - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", K(ret), KP(ls)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, - tablet_handle, - 0, - ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("get tablet with timeout failed", K(ret), "ls_id", ls->get_ls_id(), K(tablet_id)); + } else if (OB_FAIL(tenant_ls->get_tablet(tablet_id, + tablet_handle, + 0, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("get tablet with timeout failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->get_latest( data, unused_writer, unused_trans_stat, unused_trans_version))) { if (OB_EMPTY_RESULT == ret) { // tablet is creating ret = OB_TABLET_NOT_EXIST; } else { - LOG_WARN("failed to get latest tablet status", KR(ret), "ls_id", ls->get_ls_id(), K(tablet_id)); + LOG_WARN("failed to get latest tablet status", KR(ret), K(tablet_id)); } } else if (FALSE_IT(tablet_status = data.get_tablet_status())) { } else if (ObTabletStatus::NORMAL == tablet_status - || ObTabletStatus::RESERVED_4 == tablet_status - || ObTabletStatus::RESERVED_5 == tablet_status || ObTabletStatus::SPLIT_SRC == tablet_status || ObTabletStatus::SPLIT_DST == tablet_status) { // do nothing + } else if (ObTabletStatus::RESERVED_STATUS_4 == tablet_status + || ObTabletStatus::RESERVED_STATUS_5 == tablet_status + || ObTabletStatus::RESERVED_STATUS_6 == tablet_status) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("reserved tablet status is not supported", KR(ret), K(tablet_id), K(tablet_status)); } else if (OB_UNLIKELY(data.tablet_status_.is_deleted_for_gc())) { // tablet shell ret = OB_TABLET_NOT_EXIST; - LOG_INFO("tablet is already deleted", KR(ret), "ls_id", ls->get_ls_id(), K(tablet_id)); + LOG_INFO("tablet is already deleted", KR(ret), K(tablet_id)); } else { // do nothing } @@ -103,9 +83,9 @@ int check_exist(const ObLockTaskBatchRequest &arg, ({ \ int ret = OB_SUCCESS; \ ObAccessService *access_srv = share::g_mp->access_service(); \ - ObLSHandle ls_handle; \ + ObLS *tenant_ls = nullptr; \ common::ObTabletID tablet_id; \ - if (OB_FAIL(check_exist(arg.lsid_, ls_handle))) { \ + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { \ LOG_WARN("check ls failed", K(ret), K(arg)); \ if (OB_LS_NOT_EXIST == ret) { \ result.can_retry_ = true; \ @@ -118,25 +98,24 @@ int check_exist(const ObLockTaskBatchRequest &arg, K(arg.params_[i].lock_id_)); \ } else if (OB_FAIL(check_exist(arg, \ tablet_id, \ - ls_handle))) { \ + tenant_ls))) { \ LOG_WARN("check tablet failed", K(ret), K(tablet_id), \ - K(arg.params_[i].expired_time_), K(ls_handle)); \ + K(arg.params_[i].expired_time_), K(tenant_ls)); \ if (OB_TABLET_NOT_EXIST == ret) { \ result.can_retry_ = true; \ } \ } \ } \ if (OB_FAIL(ret)) { \ - } else if (OB_FAIL(access_srv->func_name(arg.lsid_, \ - *(arg.tx_desc_), \ + } else if (OB_FAIL(access_srv->func_name(*(arg.tx_desc_), \ arg.params_[i]))) { \ LOG_WARN("failed to exec", K(ret), K(arg.params_[i])); \ } else if (arg.params_[i].lock_id_.is_tablet_lock() && \ OB_FAIL(check_exist(arg, \ tablet_id, \ - ls_handle))) { \ + tenant_ls))) { \ LOG_WARN("check tablet failed", K(ret), K(tablet_id), \ - K(arg.params_[i].expired_time_), K(ls_handle)); \ + K(arg.params_[i].expired_time_), K(tenant_ls)); \ } else { \ result.success_pos_ = i; \ } \ @@ -210,8 +189,7 @@ int handle_batch_lock_task(const ObLockTaskBatchRequest &arg, static int process_for_replace_lock_table_(const ObLockTaskBatchRequest &arg, ObTableLockTaskResult &result); -static int replace_lock_for_tablet_in_table_(const share::ObLSID &ls_id, - transaction::ObTxDesc &tx_desc, +static int replace_lock_for_tablet_in_table_(transaction::ObTxDesc &tx_desc, const ObReplaceLockParam &lock_param); int handle_batch_replace_lock_task(const ObLockTaskBatchRequest &arg, @@ -274,9 +252,9 @@ static int process_for_replace_lock_table_(const ObLockTaskBatchRequestaccess_service(); - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; common::ObTabletID tablet_id; - if (OB_FAIL(check_exist(arg.lsid_, ls_handle))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("check ls failed", K(ret), K(arg)); if (OB_LS_NOT_EXIST == ret) { result.can_retry_ = true; @@ -286,19 +264,19 @@ static int process_for_replace_lock_table_(const ObLockTaskBatchRequestreplace_obj_lock(arg.lsid_, *(arg.tx_desc_), arg.params_[i]))) { + } else if (OB_FAIL(access_srv->replace_obj_lock(*(arg.tx_desc_), arg.params_[i]))) { LOG_WARN("failed to replace lock table", K(ret), K(arg.params_[i])); } } @@ -306,14 +284,13 @@ static int process_for_replace_lock_table_(const ObLockTaskBatchRequestaccess_service(); if (is_need_lock_tablet_mode(lock_param.lock_mode_) && !is_need_lock_tablet_mode(lock_param.new_lock_mode_)) { - ret = access_srv->unlock_obj(ls_id, tx_desc, lock_param); + ret = access_srv->unlock_obj(tx_desc, lock_param); } else if (!is_need_lock_tablet_mode(lock_param.lock_mode_) && is_need_lock_tablet_mode(lock_param.new_lock_mode_)) { ObLockParam new_lock_param; // we should set new_owner_id and new_lock_mode to owner_id and lock_mode in lock progress @@ -327,10 +304,10 @@ static int replace_lock_for_tablet_in_table_(const share::ObLSID &ls_id, lock_param.expired_time_))) { LOG_WARN("set lock_param for replace tablet lock failed", K(ret), K(lock_param)); } else { - ret = access_srv->lock_obj(ls_id, tx_desc, new_lock_param); + ret = access_srv->lock_obj(tx_desc, new_lock_param); } } else { - ret = access_srv->replace_obj_lock(ls_id, tx_desc, lock_param); + ret = access_srv->replace_obj_lock(tx_desc, lock_param); } return ret; } diff --git a/src/storage/tablelock/ob_table_lock_local_executor.h b/src/storage/tablelock/ob_table_lock_local_executor.h index 621d2b1e9..c31078fa4 100644 --- a/src/storage/tablelock/ob_table_lock_local_executor.h +++ b/src/storage/tablelock/ob_table_lock_local_executor.h @@ -17,19 +17,17 @@ #ifndef OCEANBASE_STORAGE_TABLELOCK_OB_TABLE_LOCK_LOCAL_EXECUTOR_H_ #define OCEANBASE_STORAGE_TABLELOCK_OB_TABLE_LOCK_LOCAL_EXECUTOR_H_ -#include "lib/container/ob_array.h" -#include "lib/net/ob_addr.h" -#include "lib/allocator/page_arena.h" +#include + #include "storage/tablelock/ob_table_lock_rpc_struct.h" namespace oceanbase { namespace observer { -// Local batch-lock handlers (single-replica): formerly the ObBatchLockTaskP / -// ObHighPriorityBatchLockTaskP / ObBatchReplaceLockTaskP processor bodies. They run in -// the current tenant MTL context (the table-lock service already runs there) and stuff -// the per-LS result; the return value is the "transport" code (OB_SUCCESS for local). +// Local batch-lock handlers run synchronously in the current tenant MTL context. +// The return value preserves the former transport-code channel, while result carries +// the task and transaction execution status. int handle_batch_lock_task( const transaction::tablelock::ObLockTaskBatchRequest &arg, transaction::tablelock::ObTableLockTaskResult &result); @@ -40,76 +38,31 @@ int handle_batch_replace_lock_task( const transaction::tablelock::ObLockTaskBatchRequest &arg, transaction::tablelock::ObTableLockTaskResult &result); -// Drop-in replacement for the removed ObBatchLockProxy / ObHighPriorityBatchLockProxy / -// ObBatchReplaceLockProxy. Mimics the subset of the async rpc-proxy interface that -// ObTableLockService uses (reuse/call/wait_all/check_return_cnt/get_results), but dispatches -// each per-LS request directly to the local handler instead of sending an RPC. template -class ObLocalBatchLockProxy +class ObLocalBatchLockExecutor { public: using Result = transaction::tablelock::ObTableLockTaskResult; using Handler = int (*)(const Request &, Result &); - explicit ObLocalBatchLockProxy(Handler handler) - : handler_(handler), allocator_("LocalLockRpc") {} - ~ObLocalBatchLockProxy() { destroy_results_(); } - void reuse() - { - destroy_results_(); - dests_.reuse(); - rcodes_.reuse(); - result_ptrs_.reuse(); - allocator_.reset(); - } - // Result is non-copyable (DISALLOW_COPY_AND_ASSIGN); allocate each from the arena and - // fill it in place, mirroring the old proxy which owned deserialized Result objects. - int call(const common::ObAddr &dest, - const int64_t timeout_us, - const int64_t cluster_id, - const int32_t group_id, - const Request &request) - { - UNUSED(timeout_us); - UNUSED(cluster_id); - UNUSED(group_id); - int ret = common::OB_SUCCESS; - void *buf = allocator_.alloc(sizeof(Result)); - if (OB_ISNULL(buf)) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - } else { - Result *result = new (buf) Result(); - const int rcode = handler_(request, *result); - if (OB_FAIL(dests_.push_back(dest))) { - } else if (OB_FAIL(rcodes_.push_back(rcode))) { - } else if (OB_FAIL(result_ptrs_.push_back(result))) { - } - } - return ret; - } - int wait_all(common::ObArray &return_code_array) - { - return return_code_array.assign(rcodes_); - } - int check_return_cnt(const int64_t return_cnt) const + explicit ObLocalBatchLockExecutor(Handler handler) + : handler_(handler), return_code_(common::OB_SUCCESS), result_() {} + ~ObLocalBatchLockExecutor() = default; + + int execute(const Request &request) { - return (return_cnt == result_ptrs_.count()) ? common::OB_SUCCESS : common::OB_ERR_UNEXPECTED; + result_.~Result(); + new (&result_) Result(); + return_code_ = handler_(request, result_); + return common::OB_SUCCESS; } - common::ObIArray &get_results() { return result_ptrs_; } - const common::ObIArray &get_dests() const { return dests_; } + + int get_return_code() const { return return_code_; } + const Result &get_result() const { return result_; } + private: - void destroy_results_() - { - for (int64_t i = 0; i < result_ptrs_.count(); ++i) { - if (OB_NOT_NULL(result_ptrs_.at(i))) { - result_ptrs_.at(i)->~Result(); - } - } - } Handler handler_; - common::ObArenaAllocator allocator_; - common::ObArray dests_; - common::ObArray rcodes_; - common::ObArray result_ptrs_; + int return_code_; + Result result_; }; } // namespace observer diff --git a/src/storage/tablelock/ob_table_lock_rpc_client.cpp b/src/storage/tablelock/ob_table_lock_rpc_client.cpp index 78234ec27..f0141771b 100644 --- a/src/storage/tablelock/ob_table_lock_rpc_client.cpp +++ b/src/storage/tablelock/ob_table_lock_rpc_client.cpp @@ -17,8 +17,6 @@ #define USING_LOG_PREFIX TABLELOCK #include "ob_table_lock_rpc_client.h" -#include "share/location_cache/ob_location_service.h" -#include "observer/ob_srv_network_frame.h" namespace oceanbase { @@ -29,31 +27,6 @@ namespace transaction namespace tablelock { -static inline int get_ls_leader( - const int64_t cluster_id, - const ObLSID &ls_id, - const int64_t abs_timeout_ts, - ObAddr &addr) -{ - int ret = OB_SUCCESS; - ObLocationService *location_service = GCTX.location_service_; - if (OB_ISNULL(location_service)) { - ret = OB_NOT_INIT; - LOG_WARN("location_service not inited", K(ret)); - } else if (OB_FAIL(location_service->get_leader_with_retry_until_timeout( - cluster_id, - ls_id, - addr, - abs_timeout_ts))) { - LOG_WARN("failed to get ls leader with retry until timeout", - K(ret), K(cluster_id), K(ls_id), K(addr), K(abs_timeout_ts)); - } else { - LOG_DEBUG("get ls leader from location_service", - K(ret), K(cluster_id), K(ls_id), K(addr), K(abs_timeout_ts)); - } - return ret; -} - int ObTableLockRpcClient::init() { // No-op: the table-lock RPC proxy was removed (single-replica; locking goes through the diff --git a/src/storage/tablelock/ob_table_lock_rpc_struct.cpp b/src/storage/tablelock/ob_table_lock_rpc_struct.cpp index 3a6c6fc09..57e432c56 100644 --- a/src/storage/tablelock/ob_table_lock_rpc_struct.cpp +++ b/src/storage/tablelock/ob_table_lock_rpc_struct.cpp @@ -85,8 +85,7 @@ OB_SERIALIZE_MEMBER_INHERIT(ObLockTabletRequest, ObLockTableRequest, OB_SERIALIZE_MEMBER_INHERIT(ObLockTabletsRequest, ObLockTableRequest, tablet_ids_); -OB_SERIALIZE_MEMBER_INHERIT(ObLockAloneTabletRequest, ObLockTabletsRequest, - ls_id_); +OB_SERIALIZE_MEMBER_INHERIT(ObLockAloneTabletRequest, ObLockTabletsRequest); OB_SERIALIZE_MEMBER(ObReplaceLockRequest, new_lock_mode_, @@ -110,7 +109,6 @@ OB_DEF_SERIALIZE_SIZE(ObTableLockTaskRequest) } else { LST_DO_CODE(OB_UNIS_ADD_LEN, task_type_, - lsid_, param_, *tx_desc_); } @@ -126,7 +124,6 @@ OB_DEF_SERIALIZE(ObTableLockTaskRequest) } else { LST_DO_CODE(OB_UNIS_ENCODE, task_type_, - lsid_, param_, *tx_desc_); } @@ -138,7 +135,6 @@ OB_DEF_DESERIALIZE(ObTableLockTaskRequest) int ret = OB_SUCCESS; LST_DO_CODE(OB_UNIS_DECODE, task_type_, - lsid_, param_); if (OB_FAIL(ret)) { // do nothing @@ -384,7 +380,6 @@ bool ObLockParam::is_valid() const || ObLockOBJType::OBJ_TYPE_DATABASE_NAME == lock_id_.obj_type_ || ObLockOBJType::OBJ_TYPE_OBJECT_NAME == lock_id_.obj_type_ || ObLockOBJType::OBJ_TYPE_DBMS_LOCK == lock_id_.obj_type_ - || ObLockOBJType::OBJ_TYPE_MATERIALIZED_VIEW == lock_id_.obj_type_ || ObLockOBJType::OBJ_TYPE_MYSQL_LOCK_FUNC == lock_id_.obj_type_ || ObLockOBJType::OBJ_TYPE_REFRESH_VECTOR_INDEX == lock_id_.obj_type_))); } @@ -679,15 +674,13 @@ bool ObUnLockTabletsRequest::is_valid() const void ObLockAloneTabletRequest::reset() { ObLockTabletsRequest::reset(); - ls_id_.reset(); } bool ObLockAloneTabletRequest::is_valid() const { bool is_valid = true; is_valid = (ObLockMsgType::LOCK_ALONE_TABLET_REQ == type_ && - ObLockRequest::is_valid() && - ls_id_.is_valid()); + ObLockRequest::is_valid()); for (int64_t i = 0; i < tablet_ids_.count() && is_valid; i++) { is_valid = is_valid && tablet_ids_.at(i).is_valid(); } @@ -704,8 +697,7 @@ bool ObUnLockAloneTabletRequest::is_valid() const bool is_valid = true; is_valid = ((ObLockMsgType::LOCK_ALONE_TABLET_REQ == type_ || ObLockMsgType::UNLOCK_ALONE_TABLET_REQ == type_) && - ObLockRequest::is_valid() && - ls_id_.is_valid()); + ObLockRequest::is_valid()); for (int64_t i = 0; i < tablet_ids_.count() && is_valid; i++) { is_valid = is_valid && tablet_ids_.at(i).is_valid(); } @@ -784,20 +776,17 @@ bool ObReplaceAllLocksRequest::is_valid() const int ObTableLockTaskRequest::set( const ObTableLockTaskType task_type, - const share::ObLSID &lsid, const ObLockParam ¶m, transaction::ObTxDesc *tx_desc) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!(task_type < MAX_TASK_TYPE)) || - OB_UNLIKELY(!lsid.is_valid()) || OB_UNLIKELY(!param.is_valid()) || OB_ISNULL(tx_desc)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(task_type), K(lsid), K(param), KP(tx_desc)); + LOG_WARN("invalid argument", K(ret), K(task_type), K(param), KP(tx_desc)); } else { task_type_ = task_type; - lsid_ = lsid; param_ = param; tx_desc_ = tx_desc; } @@ -819,7 +808,6 @@ void ObTableLockTaskRequest::reset() } } task_type_ = INVALID_LOCK_TASK_TYPE; - lsid_.reset(); param_.reset(); tx_desc_ = nullptr; need_release_tx_ = false; @@ -851,19 +839,15 @@ int ObInTransLockTableRequest::assign(const ObInTransLockTableRequest &arg) OB_SERIALIZE_MEMBER_INHERIT(ObInTransLockTabletRequest, ObInTransLockTableRequest, tablet_id_); -OB_SERIALIZE_MEMBER(ObAdminRemoveLockOpArg, ls_id_, lock_op_); +OB_SERIALIZE_MEMBER(ObAdminRemoveLockOpArg, lock_op_); -int ObAdminRemoveLockOpArg::set(const share::ObLSID &ls_id, - const ObTableLockOp &lock_op) +int ObAdminRemoveLockOpArg::set(const ObTableLockOp &lock_op) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(false || - !ls_id.is_valid() || - !lock_op.is_valid())) { + if (OB_UNLIKELY(!lock_op.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(lock_op)); + LOG_WARN("invalid argument", KR(ret), K(lock_op)); } else { - ls_id_ = ls_id; lock_op_ = lock_op; } return ret; @@ -871,25 +855,21 @@ int ObAdminRemoveLockOpArg::set(const share::ObLSID &ls_id, -OB_SERIALIZE_MEMBER(ObAdminUpdateLockOpArg, ls_id_, lock_op_, +OB_SERIALIZE_MEMBER(ObAdminUpdateLockOpArg, lock_op_, commit_version_, commit_scn_); -int ObAdminUpdateLockOpArg::set(const share::ObLSID &ls_id, - const ObTableLockOp &lock_op, +int ObAdminUpdateLockOpArg::set(const ObTableLockOp &lock_op, const share::SCN &commit_version, const share::SCN &commit_scn) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(false || - !ls_id.is_valid() || - !lock_op.is_valid() || + if (OB_UNLIKELY(!lock_op.is_valid() || !commit_version.is_valid() || !commit_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(ls_id), K(lock_op), + LOG_WARN("invalid argument", KR(ret), K(lock_op), K(commit_version), K(commit_scn)); } else { - ls_id_ = ls_id; lock_op_ = lock_op; commit_version_ = commit_version; commit_scn_ = commit_scn; diff --git a/src/storage/tablelock/ob_table_lock_rpc_struct.h b/src/storage/tablelock/ob_table_lock_rpc_struct.h index be69e9df3..a2d1af626 100644 --- a/src/storage/tablelock/ob_table_lock_rpc_struct.h +++ b/src/storage/tablelock/ob_table_lock_rpc_struct.h @@ -30,7 +30,6 @@ namespace oceanbase namespace share { -class ObLSID; } namespace transaction @@ -417,15 +416,12 @@ struct ObLockAloneTabletRequest : public ObLockTabletsRequest { OB_UNIS_VERSION_V(1); public: - ObLockAloneTabletRequest() : ls_id_() + ObLockAloneTabletRequest() { type_ = ObLockMsgType::LOCK_ALONE_TABLET_REQ; } virtual ~ObLockAloneTabletRequest() { reset(); } virtual void reset(); virtual bool is_valid() const; virtual ObTableLockTaskType get_task_type() const { return LOCK_ALONE_TABLET; } - INHERIT_TO_STRING_KV("ObLockTabletsRequest", ObLockTabletsRequest, K_(ls_id)); - public: - share::ObLSID ls_id_; }; struct ObUnLockAloneTabletRequest : public ObLockAloneTabletRequest @@ -483,7 +479,6 @@ class ObTableLockTaskRequest final public: ObTableLockTaskRequest() : task_type_(INVALID_LOCK_TASK_TYPE), - lsid_(), param_(), tx_desc_(nullptr), need_release_tx_(false) @@ -492,14 +487,12 @@ class ObTableLockTaskRequest final void reset(); int set( const ObTableLockTaskType task_type, - const share::ObLSID &lsid, const ObLockParam ¶m, transaction::ObTxDesc *tx_desc); bool is_valid() const { return (task_type_ < MAX_TASK_TYPE - && lsid_.is_valid() && param_.is_valid() && OB_NOT_NULL(tx_desc_) && tx_desc_->is_valid()); @@ -514,10 +507,9 @@ class ObTableLockTaskRequest final } bool is_timeout() const; - TO_STRING_KV(K(task_type_), K(lsid_), K(param_), KP(tx_desc_)); + TO_STRING_KV(K(task_type_), K(param_), KP(tx_desc_)); public: ObTableLockTaskType task_type_; - share::ObLSID lsid_; // go to which ls to lock. ObLockParam param_; transaction::ObTxDesc *tx_desc_; private: @@ -537,7 +529,6 @@ class ObLockTaskBatchRequest final public: ObLockTaskBatchRequest() : task_type_(INVALID_LOCK_TASK_TYPE), - lsid_(), params_(), tx_desc_(nullptr), need_release_tx_(false) @@ -552,25 +543,23 @@ class ObLockTaskBatchRequest final } } task_type_ = INVALID_LOCK_TASK_TYPE; - lsid_.reset(); tx_desc_ = nullptr; need_release_tx_ = false; params_.reset(); } - int init(const ObTableLockTaskType task_type, const share::ObLSID &lsid, transaction::ObTxDesc *tx_desc) + int init(const ObTableLockTaskType task_type, transaction::ObTxDesc *tx_desc) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!(task_type < MAX_TASK_TYPE)) || OB_UNLIKELY(!lsid.is_valid()) || OB_ISNULL(tx_desc)) { + if (OB_UNLIKELY(!(task_type < MAX_TASK_TYPE)) || OB_ISNULL(tx_desc)) { ret = OB_INVALID_ARGUMENT; - TABLELOCK_LOG(WARN, "invalid argument", K(ret), K(task_type), K(lsid), KP(tx_desc)); + TABLELOCK_LOG(WARN, "invalid argument", K(ret), K(task_type), KP(tx_desc)); } else { task_type_ = task_type; - lsid_ = lsid; tx_desc_ = tx_desc; } return ret; } - bool is_inited() const { return (task_type_ < MAX_TASK_TYPE && lsid_.is_valid() && OB_NOT_NULL(tx_desc_)); } + bool is_inited() const { return (task_type_ < MAX_TASK_TYPE && OB_NOT_NULL(tx_desc_)); } bool is_valid() const { bool valid = true; @@ -596,7 +585,6 @@ class ObLockTaskBatchRequest final TABLELOCK_LOG(WARN, "failed to assign params", KR(ret), K(arg)); } else { task_type_ = arg.task_type_; - lsid_ = arg.lsid_; tx_desc_ = arg.tx_desc_; } return ret; @@ -610,10 +598,9 @@ class ObLockTaskBatchRequest final return !is_unlock_request(); } - TO_STRING_KV(K(task_type_), K(lsid_), K(params_), KPC(tx_desc_)); + TO_STRING_KV(K(task_type_), K(params_), KPC(tx_desc_)); public: ObTableLockTaskType task_type_; - share::ObLSID lsid_; // go to which ls to lock. common::ObSArray params_; transaction::ObTxDesc *tx_desc_; private: @@ -631,7 +618,6 @@ OB_DEF_SERIALIZE_SIZE(ObLockTaskBatchRequest, template ) } else { LST_DO_CODE(OB_UNIS_ADD_LEN, task_type_, - lsid_, params_, *tx_desc_); } @@ -647,7 +633,6 @@ OB_DEF_SERIALIZE(ObLockTaskBatchRequest, template ) } else { LST_DO_CODE(OB_UNIS_ENCODE, task_type_, - lsid_, params_, *tx_desc_); } @@ -657,7 +642,7 @@ OB_DEF_SERIALIZE(ObLockTaskBatchRequest, template ) OB_DEF_DESERIALIZE(ObLockTaskBatchRequest, template ) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, task_type_, lsid_, params_); + LST_DO_CODE(OB_UNIS_DECODE, task_type_, params_); if (OB_FAIL(ret)) { // do nothing } else if (OB_FAIL(TxDescHelper::deserialize_tx_desc(buf, data_len, pos, tx_desc_))) { @@ -766,16 +751,11 @@ struct ObAdminRemoveLockOpArg { OB_UNIS_VERSION(1); public: - ObAdminRemoveLockOpArg() : ls_id_(), lock_op_() {} - ~ObAdminRemoveLockOpArg() - { - ls_id_.reset(); - } - int set(const share::ObLSID &ls_id, - const ObTableLockOp &lock_op); - TO_STRING_KV(K_(ls_id), K_(lock_op)); + ObAdminRemoveLockOpArg() : lock_op_() {} + ~ObAdminRemoveLockOpArg() {} + int set(const ObTableLockOp &lock_op); + TO_STRING_KV(K_(lock_op)); public: - share::ObLSID ls_id_; ObTableLockOp lock_op_; }; @@ -784,24 +764,20 @@ struct ObAdminUpdateLockOpArg OB_UNIS_VERSION(1); public: ObAdminUpdateLockOpArg() - : ls_id_(), - lock_op_(), + : lock_op_(), commit_version_(), commit_scn_() {} ~ObAdminUpdateLockOpArg() { - ls_id_.reset(); commit_version_.reset(); commit_scn_.reset(); } - int set(const share::ObLSID &ls_id, - const ObTableLockOp &lock_op, + int set(const ObTableLockOp &lock_op, const share::SCN &commit_version, const share::SCN &commit_scn); - TO_STRING_KV(K_(ls_id), K_(lock_op), K_(commit_version), K_(commit_scn)); + TO_STRING_KV(K_(lock_op), K_(commit_version), K_(commit_scn)); public: - share::ObLSID ls_id_; ObTableLockOp lock_op_; share::SCN commit_version_; share::SCN commit_scn_; diff --git a/src/storage/tablelock/ob_table_lock_service.cpp b/src/storage/tablelock/ob_table_lock_service.cpp index 83ef6d2b1..b6db02672 100644 --- a/src/storage/tablelock/ob_table_lock_service.cpp +++ b/src/storage/tablelock/ob_table_lock_service.cpp @@ -41,7 +41,6 @@ ObTableLockService::ObTableLockCtx::ObTableLockCtx() : is_in_trans_(false), table_id_(OB_INVALID_ID), partition_id_(OB_INVALID_ID), - ls_id_(ObLSID::INVALID_LS_ID), lock_op_type_(UNKNOWN_TYPE), origin_timeout_us_(-1), timeout_us_(-1), @@ -50,6 +49,7 @@ ObTableLockService::ObTableLockCtx::ObTableLockCtx() : tx_desc_(nullptr), tx_param_(), current_savepoint_(), + need_rollback_(false), tablet_list_(), obj_list_(), lock_mode_(MAX_LOCK_MODE), @@ -75,8 +75,8 @@ ObTableLockService::ObTableLockCtx::ObTableLockCtx() : void ObTableLockService::ObRetryCtx::reuse() { need_retry_ = false; - send_rpc_count_ = 0; - rpc_ls_array_.reuse(); + task_executed_ = false; + task_prepared_ = false; retry_lock_ids_.reuse(); } @@ -163,7 +163,6 @@ int ObTableLockService::ObTableLockCtx::set_by_lock_req(const ObLockRequest &arg task_type_ = LOCK_ALONE_TABLET; } table_id_ = lock_arg.table_id_; - ls_id_ = lock_arg.ls_id_; if (OB_FAIL(set_tablet_id(lock_arg.tablet_ids_))) { LOG_WARN("set tablet id failed", K(ret), K(lock_arg)); } @@ -304,7 +303,7 @@ void ObTableLockService::ObOBJLockGarbageCollector::run_gc_once_() { common::ObDIActionGuard ag("TableLockService", "OBJLockGC", "GCTimer"); int ret = OB_SUCCESS; - if (OB_FAIL(garbage_collect_for_all_ls_())) { + if (OB_FAIL(garbage_collect_())) { check_and_report_timeout_(); LOG_WARN("check and clear obj lock failed, will retry later", K(ret), K(last_success_timestamp_), KPC(this)); @@ -315,40 +314,15 @@ void ObTableLockService::ObOBJLockGarbageCollector::run_gc_once_() } } -int ObTableLockService::ObOBJLockGarbageCollector::garbage_collect_for_all_ls_() +int ObTableLockService::ObOBJLockGarbageCollector::garbage_collect_() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObSharedGuard ls_iter_guard; - ObLSService *ls_service = nullptr; - ObLS *ls = nullptr; - bool is_leader = false; if (!timer_.inited()) { ret = OB_NOT_INIT; LOG_WARN("timer of ObTableLockService::ObOBJLockGarbageCollector is not running", K(ret)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls_iter(ls_iter_guard, ObLSGetMod::TABLELOCK_MOD))) { - LOG_WARN("fail to get ls iterator", K(ret)); - } else { - do { - if (OB_FAIL(ls_iter_guard->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next iter failed", K(ret)); - } - } else if (ls->is_sys_ls()) { - if (OB_TMP_FAIL(check_is_leader_(ls, is_leader))) { - LOG_WARN("can not check whether this ls is leader", K(ret), K(tmp_ret), K(ls->get_ls_id())); - } else if (is_leader && OB_TMP_FAIL(ObTableLockDetector::do_detect_and_clear())) { - LOG_WARN("do_detect_and_clear failed", K(ret), K(tmp_ret), K(ls->get_ls_id())); - } - } else { - LOG_INFO("finish check and clear obj lock", K(ls->get_ls_id())); - } - } while (OB_SUCC(ret)); + } else if (OB_FAIL(ObTableLockDetector::do_detect_and_clear())) { + LOG_WARN("do_detect_and_clear failed", K(ret)); } - ret = OB_ITER_END == ret ? OB_SUCCESS : ret; return ret; } @@ -367,21 +341,6 @@ void ObTableLockService::ObOBJLockGarbageCollector::check_and_report_timeout_() } } -int ObTableLockService::ObOBJLockGarbageCollector::check_is_leader_(ObLS *ls, bool &is_leader) -{ - int ret = OB_SUCCESS; - ObRole role; - int64_t proposal_id = 0; - is_leader = false; - - if (OB_FAIL(ls->get_log_handler()->get_role(role, proposal_id))) { - STORAGE_LOG(WARN, "failed to get role", K(ret), K(ls->get_ls_id())); - } else { - is_leader = is_strong_leader(role); - } - return ret; -} - int ObTableLockService::ObTableLockCtx::set_tablet_id(const common::ObIArray &tablet_ids) { int ret = OB_SUCCESS; @@ -429,13 +388,6 @@ int64_t ObTableLockService::ObTableLockCtx::remain_timeoutus() const return remain_us > 0 ? remain_us : 0; } -int64_t ObTableLockService::ObTableLockCtx::get_rpc_timeoutus() const -{ - // rpc timeout should larger than stmt remain timeout us. - // we add 2 second now. - return (remain_timeoutus() + DEFAULT_RPC_TIMEOUT_US); -} - int64_t ObTableLockService::ObTableLockCtx::get_tablet_cnt() const { return tablet_list_.count(); @@ -446,28 +398,14 @@ const ObTabletID &ObTableLockService::ObTableLockCtx::get_tablet_id(const int64_ return tablet_list_.at(index); } -int ObTableLockService::ObTableLockCtx::add_touched_ls(const ObLSID &lsid) +void ObTableLockService::ObTableLockCtx::mark_need_rollback() { - int ret = OB_SUCCESS; - bool exist = false; - // check if the touched ls exist. - for (int64_t i = 0; OB_SUCC(ret) && i < need_rollback_ls_.count(); i++) { - ObLSID &curr = need_rollback_ls_.at(i); - if (curr == lsid) { - exist = true; - break; - } - } - // add if the touched ls not exist. - if (!exist && OB_FAIL(need_rollback_ls_.push_back(lsid))) { - LOG_ERROR("add touche ls failed", K(ret), K(lsid)); - } - return ret; + need_rollback_ = true; } -void ObTableLockService::ObTableLockCtx::clean_touched_ls() +void ObTableLockService::ObTableLockCtx::clear_need_rollback() { - need_rollback_ls_.reuse(); + need_rollback_ = false; } bool ObTableLockService::ObTableLockCtx::is_deadlock_avoid_enabled() const @@ -488,14 +426,11 @@ int ObTableLockService::init() ret = OB_INIT_TWICE; LOG_WARN("lock service init twice.", K(ret)); } else if (OB_UNLIKELY(!GCTX.self_addr().is_valid()) || - OB_ISNULL(GCTX.net_frame_) || - OB_ISNULL(GCTX.location_service_) || OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(GCTX.self_addr()), - KP(GCTX.net_frame_), KP(GCTX.location_service_), KP(GCTX.sql_proxy_)); + KP(GCTX.sql_proxy_)); } else { - location_service_ = GCTX.location_service_; sql_proxy_ = GCTX.sql_proxy_; is_inited_ = true; } @@ -526,7 +461,6 @@ void ObTableLockService::wait() void ObTableLockService::destroy() { obj_lock_garbage_collector_.destroy(); - location_service_ = nullptr; sql_proxy_ = nullptr; is_inited_ = false; } @@ -939,16 +873,16 @@ int ObTableLockService::process_lock_task_(ObTableLockCtx &ctx) int ObTableLockService::process_obj_lock_task_(ObTableLockCtx &ctx) { int ret = OB_SUCCESS; - ObLSLockMap ls_lock_map; + ObLockSet lock_set; if (ctx.obj_list_.empty()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("obj list is empty when lock obj", K(ret), K(ctx)); - } else if (OB_FAIL(get_ls_lock_map_(ctx, LOCK_SERVICE_LS, ctx.obj_list_, ls_lock_map))) { - LOG_WARN("get ls_lock_map failed", K(ret), K(ctx)); - } else if (ctx.is_enable_lock_priority_ && !ctx.is_unlock_task() && OB_FAIL(process_obj_lock_with_prio_(ctx, ls_lock_map))) { + } else if (OB_FAIL(get_lock_set_(ctx, ctx.obj_list_, lock_set))) { + LOG_WARN("get lock_set failed", K(ret), K(ctx)); + } else if (ctx.is_enable_lock_priority_ && !ctx.is_unlock_task() && OB_FAIL(process_obj_lock_with_prio_(ctx, lock_set))) { LOG_WARN("add obj lock into queue failed", K(ret), K(ctx)); - } else if (OB_FAIL(process_obj_lock_(ctx, ls_lock_map))) { + } else if (OB_FAIL(process_obj_lock_(ctx, lock_set))) { LOG_WARN("lock obj failed", K(ret), K(ctx)); } return ret; @@ -957,7 +891,7 @@ int ObTableLockService::process_obj_lock_task_(ObTableLockCtx &ctx) int ObTableLockService::process_table_lock_task_(ObTableLockCtx &ctx) { int ret = OB_SUCCESS; - ObLSLockMap table_ls_lock_map; + ObLockSet table_lock_set; ObLockID table_lock_id; ObTableLockMode table_lock_mode = ctx.lock_mode_; @@ -966,8 +900,8 @@ int ObTableLockService::process_table_lock_task_(ObTableLockCtx &ctx) } else if (is_part_table_lock_(ctx.task_type_) && OB_FAIL(get_table_lock_mode_(ctx.task_type_, ctx.lock_mode_, table_lock_mode))) { LOG_WARN("get table lock mode failed", K(ret), K(ctx), K(ctx.task_type_), K(ctx.lock_mode_)); - } else if (OB_FAIL(get_ls_lock_map_(ctx, LOCK_SERVICE_LS, table_lock_id, table_ls_lock_map))) { - LOG_WARN("get ls_lock_map failed", K(ret), K(ctx)); + } else if (OB_FAIL(get_lock_set_(ctx, table_lock_id, table_lock_set))) { + LOG_WARN("get lock_set failed", K(ret), K(ctx)); // NOTICE: // When lock_priority is enabled, we need to obtain the table's schema and tablets to enqueue the corresponding // tablets into the locking queue for prioritized locking. However, at this point, the table is not yet locked. @@ -981,12 +915,12 @@ int ObTableLockService::process_table_lock_task_(ObTableLockCtx &ctx) process_table_tablet_lock_with_prio_(ctx, ctx.lock_mode_, table_lock_mode, - table_ls_lock_map))) { + table_lock_set))) { LOG_WARN("add table and tablet lock into queue failed", K(ret), K(ctx)); } else if (OB_FAIL(process_table_tablet_lock_(ctx, ctx.lock_mode_, table_lock_mode, - table_ls_lock_map))) { + table_lock_set))) { LOG_WARN("lock table and tablet failed", K(ret), K(table_lock_mode), K(ctx)); } return ret; @@ -995,17 +929,17 @@ int ObTableLockService::process_table_lock_task_(ObTableLockCtx &ctx) int ObTableLockService::process_alone_tablet_lock_task_(ObTableLockCtx &ctx) { int ret = OB_SUCCESS; - ObLSLockMap ls_lock_map; + ObLockSet lock_set; // TODO: yanyuan.cxf we may need the right schema_version while lock/unlock alone tablet. ctx.schema_version_ = 0; - if (OB_FAIL(get_ls_lock_map_(ctx, ctx.tablet_list_, ls_lock_map))) { - LOG_WARN("fail to get ls lock map", K(ret), K(ctx.get_tablet_cnt())); + if (OB_FAIL(get_lock_set_(ctx, ctx.tablet_list_, lock_set))) { + LOG_WARN("fail to get lock set", K(ret), K(ctx.get_tablet_cnt())); } else if (ctx.is_enable_lock_priority_ && !ctx.is_unlock_task() - && OB_FAIL(process_obj_lock_with_prio_(ctx, ls_lock_map))) { + && OB_FAIL(process_obj_lock_with_prio_(ctx, lock_set))) { LOG_WARN("add alone tablet lock into queue failed", K(ret), K(ctx)); - } else if (OB_FAIL(process_obj_lock_(ctx, ls_lock_map))) { + } else if (OB_FAIL(process_obj_lock_(ctx, lock_set))) { LOG_WARN("lock alone tablet failed", K(ret), K(ctx)); } return ret; @@ -1059,320 +993,233 @@ int ObTableLockService::get_retry_lock_ids_(const ObLockIDArray &lock_ids, return ret; } -int ObTableLockService::get_retry_lock_ids_(const ObLSID &ls_id, - const ObLSLockMap &ls_lock_map, +int ObTableLockService::get_retry_lock_ids_(const ObLockSet &lock_set, const int64_t start_pos, ObLockIDArray &retry_lock_ids) { int ret = OB_SUCCESS; - const ObLockIDArray *lock_ids = nullptr; - // get the retry tablet list - if (OB_ISNULL(lock_ids = ls_lock_map.get(ls_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not exist at tablet map", K(ret), K(ls_id)); - } else if (OB_FAIL(get_retry_lock_ids_(*lock_ids, start_pos, retry_lock_ids))) { - // get the lock ids + if (OB_FAIL(get_retry_lock_ids_(lock_set.get_lock_ids(), start_pos, retry_lock_ids))) { LOG_WARN("get retry lock id list failed", K(ret)); } return ret; } -int ObTableLockService::collect_rollback_info_(const share::ObLSID &ls_id, - ObTableLockCtx &ctx) +int ObTableLockService::collect_rollback_info_(ObTableLockCtx &ctx) { int ret = OB_SUCCESS; - if (OB_FAIL(ctx.add_touched_ls(ls_id))) { - LOG_ERROR("add touched ls failed.", K(ret), K(ls_id)); - } + ctx.mark_need_rollback(); LOG_DEBUG("ObTableLockService::collect_rollback_info_", K(ret), K(ctx)); return ret; } -int ObTableLockService::collect_rollback_info_(const ObArray &ls_array, +int ObTableLockService::collect_rollback_info_(const ObRetryCtx &retry_ctx, ObTableLockCtx &ctx) { int ret = OB_SUCCESS; - // all rpcs treated as failed - for (int i = 0; i < ls_array.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(ctx.add_touched_ls(ls_array.at(i)))) { - LOG_ERROR("add touched ls failed.", K(ret), K(ls_array.at(i))); - } + if (retry_ctx.task_prepared_) { + ctx.mark_need_rollback(); } LOG_DEBUG("ObTableLockService::collect_rollback_info_", K(ret), K(ctx)); return ret; } -template -int ObTableLockService::collect_rollback_info_(const ObArray &ls_array, - RpcProxy &proxy_batch, - ObTableLockCtx &ctx) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - // 1. need wait rpcs that sent finish - // otherwise proxy reused or destructored will cause flying rpc core - // 2. don't use arg/dest here because call() may has failure. - // 3. return_array/result can be used only when wait_all() is success. - ObArray return_code_array; - if (OB_TMP_FAIL(proxy_batch.wait_all(return_code_array))) { - LOG_WARN("wait rpc failed", K(tmp_ret)); - } - - if (OB_FAIL(collect_rollback_info_(ls_array, ctx))) { - LOG_WARN("collect rollback info failed", K(ret)); - } - return ret; -} - -template -int ObTableLockService::handle_parallel_rpc_response_(RpcProxy &proxy_batch, +template +int ObTableLockService::handle_task_result_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, + const ObLockSet &lock_set, bool &can_retry, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObTransService *txs = share::g_mp->trans_service(); - ObLSID ls_id; can_retry = true; retry_ctx.need_retry_ = true; - // handle result - ObArray return_code_array; - if (OB_TMP_FAIL(proxy_batch.wait_all(return_code_array)) - || OB_TMP_FAIL(proxy_batch.check_return_cnt(return_code_array.count())) - || retry_ctx.send_rpc_count_ != return_code_array.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("rpc failed", KR(ret), KR(tmp_ret), K(retry_ctx.send_rpc_count_), K(return_code_array.count())); - // we need add the ls into touched to make rollback. - can_retry = false; - retry_ctx.need_retry_ = false; - (void) collect_rollback_info_(retry_ctx.rpc_ls_array_, ctx); - } else { - //check each ret of every rpc - const ObTableLockTaskResult *result = nullptr; - for (int64_t i = 0; i < return_code_array.count(); ++i) { - result = nullptr; - tmp_ret = return_code_array.at(i); - ls_id = retry_ctx.rpc_ls_array_.at(i); - if (need_retry_whole_rpc_task_(tmp_ret)) { - // rpc failed, but we need retry the whole rpc task. - LOG_WARN("lock rpc failed, but we need retry", KR(tmp_ret), K(i), K(ls_id)); - if (OB_TMP_FAIL(get_retry_lock_ids_(ls_id, - ls_lock_map, - 0, - retry_ctx.retry_lock_ids_))) { - can_retry = false; - retry_ctx.need_retry_ = false; - ret = tmp_ret; - LOG_WARN("get retry tablet list failed", KR(ret), K(ls_id)); - } - } else { - if (OB_TMP_FAIL(tmp_ret)) { - LOG_WARN("lock rpc failed", KR(tmp_ret), K(i), K(ls_id)); - } else if (OB_ISNULL(result = proxy_batch.get_results().at(i))) { - tmp_ret = OB_ERR_UNEXPECTED; - LOG_WARN("result is null", KR(tmp_ret), K(i), K(ls_id)); - } else if (OB_TMP_FAIL(result->get_tx_result_code())) { - LOG_WARN("get tx exec result failed", KR(tmp_ret), K(i), K(ls_id)); - } else if (OB_TMP_FAIL(txs->add_tx_exec_result(*ctx.tx_desc_, - result->tx_result_))) { - LOG_WARN("failed to add exec result", K(tmp_ret), K(ctx), K(result->tx_result_)); - } + if (retry_ctx.task_executed_) { + const ObTableLockTaskResult &result = executor.get_result(); + tmp_ret = executor.get_return_code(); + if (need_retry_whole_task_(tmp_ret)) { + LOG_WARN("lock task failed, but we need retry", KR(tmp_ret)); + if (OB_TMP_FAIL(get_retry_lock_ids_(lock_set, + 0, + retry_ctx.retry_lock_ids_))) { + can_retry = false; + retry_ctx.need_retry_ = false; + ret = tmp_ret; + LOG_WARN("get retry tablet list failed", KR(ret)); + } + } else { + if (OB_TMP_FAIL(tmp_ret)) { + LOG_WARN("lock task failed", KR(tmp_ret)); + } else if (OB_TMP_FAIL(result.get_tx_result_code())) { + LOG_WARN("get tx exec result failed", KR(tmp_ret)); + } else if (OB_TMP_FAIL(txs->add_tx_exec_result(*ctx.tx_desc_, + result.tx_result_))) { + LOG_WARN("failed to add exec result", K(tmp_ret), K(ctx), K(result.tx_result_)); + } - // rpc failed or we get tx exec result failed, - // we need add the ls into touched to make rollback. - if (OB_TMP_FAIL(tmp_ret)) { - ret = tmp_ret; - can_retry = false; - retry_ctx.need_retry_ = false; - (void) collect_rollback_info_(ls_id, ctx); - } else { - // if error codes are only OB_TRY_LOCK_ROW_CONFLICT, will retry - tmp_ret = result->get_ret_code(); - if (need_retry_part_rpc_task_(tmp_ret, result)) { - LOG_WARN("lock rpc failed, but we need retry", KR(tmp_ret), K(i), K(ls_id)); - if (OB_TMP_FAIL(get_retry_lock_ids_(ls_id, - ls_lock_map, - result->get_success_pos() + 1, - retry_ctx.retry_lock_ids_))) { - can_retry = false; - retry_ctx.need_retry_ = false; - ret = tmp_ret; - LOG_WARN("get retry tablet list failed", KR(ret), K(ls_id)); - } - } else if (OB_TRANS_KILLED == tmp_ret) { - // the trans need kill. - ctx.tx_is_killed_ = true; - can_retry = false; - } else if (OB_TMP_FAIL(tmp_ret)) { + // Execution or transaction-result failures require statement rollback. + if (OB_TMP_FAIL(tmp_ret)) { + ret = tmp_ret; + can_retry = false; + retry_ctx.need_retry_ = false; + (void) collect_rollback_info_(ctx); + } else { + tmp_ret = result.get_ret_code(); + if (need_retry_partial_task_(tmp_ret, &result)) { + LOG_WARN("lock task failed, but we need retry", KR(tmp_ret)); + if (OB_TMP_FAIL(get_retry_lock_ids_(lock_set, + result.get_success_pos() + 1, + retry_ctx.retry_lock_ids_))) { can_retry = false; retry_ctx.need_retry_ = false; + ret = tmp_ret; + LOG_WARN("get retry tablet list failed", KR(ret)); } - if (OB_TMP_FAIL(tmp_ret)) { - LOG_WARN("lock rpc wrong", K(tmp_ret), K(ls_id)); - if (OB_SUCC(ret) || ret == OB_TRY_LOCK_ROW_CONFLICT) { - ret = tmp_ret; - } + } else if (OB_TRANS_KILLED == tmp_ret) { + ctx.tx_is_killed_ = true; + can_retry = false; + } else if (OB_TMP_FAIL(tmp_ret)) { + retry_ctx.need_retry_ = false; + can_retry = false; + } + if (OB_TMP_FAIL(tmp_ret)) { + LOG_WARN("lock task failed", K(tmp_ret)); + if (OB_SUCC(ret) || ret == OB_TRY_LOCK_ROW_CONFLICT) { + ret = tmp_ret; } } } } } - LOG_DEBUG("ObTableLockService::handle_parallel_rpc_response_", K(ret), K(ctx)); + LOG_DEBUG("ObTableLockService::handle_task_result_", K(ret), K(ctx)); return ret; } -int ObTableLockService::pre_check_lock_(ObTableLockCtx &ctx, const ObLSLockMap &ls_lock_map) +int ObTableLockService::pre_check_lock_(ObTableLockCtx &ctx, const ObLockSet &lock_set) { int ret = OB_SUCCESS; - ret = batch_pre_check_lock_(ctx, ls_lock_map); + ret = batch_pre_check_lock_(ctx, lock_set); return ret; } // for 4.1 -template -int ObTableLockService::parallel_batch_rpc_handle_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_set_in_batches_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map) + const ObLockSet &lock_set) { int ret = OB_SUCCESS; - constexpr static int64_t MAP_NUM = 2; - ObLSLockMap maps[MAP_NUM]; - const ObLSLockMap *in_map = nullptr; - ObLSLockMap *retry_map = nullptr; - bool can_retry = true; // whether the whole rpc task can retry. + constexpr static int64_t RETRY_SET_NUM = 2; + ObLockSet retry_sets[RETRY_SET_NUM]; + const ObLockSet *input_set = nullptr; + ObLockSet *retry_set = nullptr; + bool can_retry = true; int64_t retry_times = 1; - for (int64_t i = 0; i < MAP_NUM && OB_SUCC(ret); i++) { - if (OB_FAIL(maps[i].create(10, lib::ObLabel("LSLockMap")))) { - LOG_WARN("ls lock map create failed", KR(ret), K(i)); - } - } if (OB_FAIL(ret)) { // do nothing } else { - retry_map = const_cast(&ls_lock_map); + retry_set = const_cast(&lock_set); do { - in_map = retry_map; - retry_map = &maps[retry_times % MAP_NUM]; - if (OB_FAIL(retry_map->reuse())) { - LOG_WARN("reuse retry map failed", K(ret)); - } else if (OB_FAIL(parallel_batch_rpc_handle_(proxy_batch, + input_set = retry_set; + retry_set = &retry_sets[retry_times % RETRY_SET_NUM]; + if (OB_FAIL(retry_set->reuse())) { + LOG_WARN("reuse retry set failed", K(ret)); + } else if (OB_FAIL(execute_lock_set_in_batches_(executor, ctx, - *in_map, + *input_set, can_retry, - *retry_map))) { - LOG_WARN("process rpc failed", KR(ret), K(can_retry), K(ctx), K(retry_times)); + *retry_set))) { + LOG_WARN("process lock task failed", KR(ret), K(can_retry), K(ctx), K(retry_times)); } - if (can_retry && !retry_map->empty()) { + if (can_retry && !retry_set->empty()) { retry_times++; } if (retry_times % 10 == 0) { LOG_WARN("retry too many times", K(retry_times), K(can_retry), K(ctx)); - FOREACH(data, ls_lock_map) { - const share::ObLSID &ls_id = data->first; - const ObLockIDArray &lock_ids = data->second; - LOG_WARN("retry data", K(ls_id), K(lock_ids)); - } + LOG_WARN("retry lock data", K(lock_set)); } - } while (can_retry && !retry_map->empty()); + } while (can_retry && !retry_set->empty()); } return ret; } -template -int ObTableLockService::parallel_send_rpc_task_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_set_once_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, + const ObLockSet &lock_set, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - bool has_retry = false; - - retry_ctx.send_rpc_count_ = 0; - // send async rpc parallel - FOREACH_X(data, ls_lock_map, OB_SUCC(ret)) { - const share::ObLSID &ls_id = data->first; - const ObLockIDArray &lock_ids = data->second; - if (!has_retry) { - if (OB_FAIL(send_rpc_task_(proxy_batch, - ctx, - ls_id, - lock_ids, - retry_ctx))) { - LOG_WARN("send rpc task failed", K(ret)); - } - } - if (retry_ctx.need_retry_ || has_retry) { - has_retry = true; - if (OB_TMP_FAIL(get_retry_lock_ids_(lock_ids, - 0, - retry_ctx.retry_lock_ids_))) { - LOG_WARN("get retry tablet failed", KR(ret)); - ret = tmp_ret; - }; + + retry_ctx.reuse(); + if (!lock_set.empty()) { + const ObLockIDArray &lock_ids = lock_set.get_lock_ids(); + if (OB_FAIL(execute_lock_task_(executor, + ctx, + lock_ids, + retry_ctx))) { + LOG_WARN("execute lock task failed", K(ret)); + } else if (retry_ctx.need_retry_ + && OB_FAIL(get_retry_lock_ids_(lock_ids, + 0, + retry_ctx.retry_lock_ids_))) { + LOG_WARN("get retry tablet failed", KR(ret)); } } return ret; } -template -int ObTableLockService::parallel_batch_rpc_handle_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_set_in_batches_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, + const ObLockSet &lock_set, bool &can_retry, - ObLSLockMap &retry_ls_lock_map) + ObLockSet &retry_lock_set) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; ObRetryCtx retry_ctx; - proxy_batch.reuse(); - // send async rpc parallel - if (OB_FAIL(parallel_send_rpc_task_(proxy_batch, + if (OB_FAIL(execute_lock_set_once_(executor, ctx, - ls_lock_map, + lock_set, retry_ctx))) { can_retry = false; - (void)collect_rollback_info_(retry_ctx.rpc_ls_array_, proxy_batch, ctx); - LOG_WARN("send rpc task failed", KR(ret)); + (void)collect_rollback_info_(retry_ctx, ctx); + LOG_WARN("execute lock task failed", KR(ret)); } else { - // process rpc response - ret = handle_parallel_rpc_response_(proxy_batch, + ret = handle_task_result_(executor, ctx, - ls_lock_map, + lock_set, can_retry, retry_ctx); } - // get the retry map + // get the retry set if (can_retry && retry_ctx.retry_lock_ids_.count() != 0) { - LOG_WARN("lock rpc failed, but we need retry", K(ret), K(can_retry), K(retry_ctx)); - if (OB_FAIL(fill_ls_lock_map_(ctx, + LOG_WARN("lock task failed, but we need retry", K(ret), K(can_retry), K(retry_ctx)); + if (OB_FAIL(fill_lock_set_(ctx, retry_ctx.retry_lock_ids_, - retry_ls_lock_map, - true /* force refresh location */))) { - LOG_WARN("refill ls lock map failed", KP(ret), K(ctx)); + retry_lock_set))) { + LOG_WARN("refill lock set failed", KP(ret), K(ctx)); can_retry = false; } } - LOG_DEBUG("ObTableLockService::parallel_batch_rpc_handle_", K(ret), K(ctx)); + LOG_DEBUG("ObTableLockService::execute_lock_set_in_batches_", K(ret), K(ctx)); return ret; } int ObTableLockService::batch_pre_check_lock_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map) + const ObLockSet &lock_set) { int ret = OB_SUCCESS; int last_ret = OB_SUCCESS; int64_t USLEEP_TIME = 100; // 0.1 ms bool need_retry = false; - observer::ObLocalBatchLockProxy> proxy_batch( + observer::ObLocalBatchLockExecutor> executor( observer::handle_batch_lock_task); // only used in LOCK_TABLE/LOCK_PARTITION if (LOCK_TABLE == ctx.task_type_ || @@ -1384,9 +1231,9 @@ int ObTableLockService::batch_pre_check_lock_(ObTableLockCtx &ctx, OB_ERR_EXCLUSIVE_LOCK_CONFLICT : OB_TIMEOUT; LOG_WARN("process obj lock timeout", K(ret), K(ctx)); } else { - ret = parallel_batch_rpc_handle_(proxy_batch, + ret = execute_lock_set_in_batches_(executor, ctx, - ls_lock_map); + lock_set); // the process process may be timeout because left time not enough, // just rewrite it to OB_ERR_EXCLUSIVE_LOCK_CONFLICT if (is_timeout_ret_code_(ret)) { @@ -1422,24 +1269,6 @@ int ObTableLockService::batch_pre_check_lock_(ObTableLockCtx &ctx, return ret; } -template -int ObTableLockService::rpc_call_(RpcProxy &proxy_batch, - const ObAddr &addr, - const int64_t timeout_us, - const LockRequest &request) -{ - int ret = OB_SUCCESS; - int32_t group_id = 0; - if (OB_FAIL(proxy_batch.call(addr, - timeout_us, - GCONF.cluster_id, - group_id, - request))) { - LOG_WARN("failed to all async rpc", KR(ret), K(addr), K(timeout_us), K(request)); - } - return ret; -} - int ObTableLockService::deal_with_deadlock_(ObTableLockCtx &ctx) { int ret = OB_SUCCESS; @@ -1484,15 +1313,14 @@ int ObTableLockService::get_table_partition_level_(const ObTableID table_id, int ObTableLockService::pack_batch_request_(ObTableLockCtx &ctx, const ObTableLockTaskType task_type, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObLockTaskBatchRequest &request) { int ret = OB_SUCCESS; ObLockParam lock_param; - if (OB_FAIL(request.init(task_type, ls_id, ctx.tx_desc_))) { - LOG_WARN("request init failed", K(ret), K(ctx), K(ls_id), KP(ctx.tx_desc_), K(lock_ids), K(task_type)); + if (OB_FAIL(request.init(task_type, ctx.tx_desc_))) { + LOG_WARN("request init failed", K(ret), K(ctx), KP(ctx.tx_desc_), K(lock_ids), K(task_type)); } else { for (int i = 0; i < lock_ids.count() && OB_SUCC(ret); ++i) { lock_param.reset(); @@ -1520,14 +1348,13 @@ int ObTableLockService::pack_batch_request_(ObTableLockCtx &ctx, int ObTableLockService::pack_batch_request_(ObTableLockCtx &ctx, const ObTableLockTaskType task_type, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObLockTaskBatchRequest &request) { int ret = OB_SUCCESS; ObReplaceLockParam lock_param; - if (OB_FAIL(request.init(task_type, ls_id, ctx.tx_desc_))) { - LOG_WARN("request init failed", K(ret), K(ctx), K(ls_id), KP(ctx.tx_desc_), K(lock_ids), K(task_type)); + if (OB_FAIL(request.init(task_type, ctx.tx_desc_))) { + LOG_WARN("request init failed", K(ret), K(ctx), KP(ctx.tx_desc_), K(lock_ids), K(task_type)); } else if (!ctx.is_replace_task()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("lock_param is not compatible with request", K(ret), K(ctx), K(task_type)); @@ -1546,76 +1373,65 @@ int ObTableLockService::pack_batch_request_(ObTableLockCtx &ctx, } -template -int ObTableLockService::batch_rpc_handle_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_set_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map) + const ObLockSet &lock_set) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - constexpr static int64_t MAP_NUM = 2; - ObLSLockMap maps[MAP_NUM]; - const ObLSLockMap *in_map = nullptr; - ObLSLockMap *retry_map = nullptr; - bool can_retry = true; // whether the whole rpc task can retry. + constexpr static int64_t RETRY_SET_NUM = 2; + ObLockSet retry_sets[RETRY_SET_NUM]; + const ObLockSet *input_set = nullptr; + ObLockSet *retry_set = nullptr; + bool can_retry = true; int64_t retry_times = 1; - for (int64_t i = 0; i < MAP_NUM && OB_SUCC(ret); i++) { - if (OB_FAIL(maps[i].create(10, lib::ObLabel("LSLockMap")))) { - LOG_WARN("ls lock map create failed", KR(ret), K(i)); - } - } if (OB_FAIL(ret)) { // do nothing } else { - retry_map = const_cast(&ls_lock_map); + retry_set = const_cast(&lock_set); do { - in_map = retry_map; - retry_map = &maps[retry_times % MAP_NUM]; - if (OB_FAIL(retry_map->reuse())) { - LOG_WARN("reuse retry map failed", K(ret)); - } else if (OB_FAIL(batch_rpc_handle_(proxy_batch, + input_set = retry_set; + retry_set = &retry_sets[retry_times % RETRY_SET_NUM]; + if (OB_FAIL(retry_set->reuse())) { + LOG_WARN("reuse retry set failed", K(ret)); + } else if (OB_FAIL(execute_lock_set_(executor, ctx, - *in_map, + *input_set, can_retry, - *retry_map))) { - LOG_WARN("process rpc failed", KR(ret), K(ctx), K(retry_times)); + *retry_set))) { + LOG_WARN("process lock task failed", KR(ret), K(ctx), K(retry_times)); } - if (can_retry && !retry_map->empty()) { + if (can_retry && !retry_set->empty()) { retry_times++; } if (retry_times % 10 == 0) { - LOG_WARN("retry too many times", K(retry_times), K(ctx), K(retry_map->size())); + LOG_WARN("retry too many times", K(retry_times), K(ctx), K(retry_set->size())); } - } while (can_retry && !retry_map->empty()); + } while (can_retry && !retry_set->empty()); } return ret; } -template -int ObTableLockService::send_rpc_task_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_task_(LocalExecutor &executor, ObTableLockCtx &ctx, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; - ObAddr addr; retry_ctx.need_retry_ = false; - if (OB_FAIL(retry_ctx.rpc_ls_array_.push_back(ls_id))) { - LOG_WARN("push_back lsid failed", K(ret), K(ls_id)); - } else if (OB_FAIL(get_ls_leader_( - ctx.tx_desc_->get_cluster_id(), ls_id, ctx.abs_timeout_ts_, addr))) { - if (need_renew_location_(ret)) { - retry_ctx.need_retry_ = true; - ret = OB_SUCCESS; - } - LOG_WARN("failed to get ls leader", K(ret), K(ctx), K(ls_id)); - } else if (ctx.is_timeout()) { - ret = OB_TIMEOUT; - LOG_WARN("process obj lock timeout", K(ret), K(ctx)); + if (OB_UNLIKELY(retry_ctx.task_prepared_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("single table lock task already prepared", K(ret), K(retry_ctx)); } else { - ret = pack_and_call_rpc_(proxy_batch, ctx, ls_id, lock_ids, addr, retry_ctx); + retry_ctx.task_prepared_ = true; + if (ctx.is_timeout()) { + ret = OB_TIMEOUT; + LOG_WARN("process obj lock timeout", K(ret), K(ctx)); + } else { + ret = pack_and_execute_task_(executor, ctx, lock_ids, retry_ctx); + } } if (OB_FAIL(ret)) { @@ -1624,109 +1440,106 @@ int ObTableLockService::send_rpc_task_(RpcProxy &proxy_batch, return ret; } -template -int ObTableLockService::pack_and_call_rpc_(RpcProxy &proxy_batch, +template +int ObTableLockService::pack_and_execute_task_(LocalExecutor &executor, ObTableLockCtx &ctx, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, - const ObAddr &addr, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; ObLockTaskBatchRequest request; - if (OB_FAIL(pack_batch_request_(ctx, ctx.task_type_, ls_id, lock_ids, request))) { - LOG_WARN("pack_batch_request_ failed", K(ret), K(ls_id), K(ctx), K(lock_ids)); - } else if (OB_FAIL(rpc_call_(proxy_batch, addr, ctx.get_rpc_timeoutus(), request))) { - LOG_WARN("failed to call async rpc", KR(ret), K(addr), K(ctx.abs_timeout_ts_), K(request)); + if (OB_FAIL(pack_batch_request_(ctx, ctx.task_type_, lock_ids, request))) { + LOG_WARN("pack_batch_request_ failed", K(ret), K(ctx), K(lock_ids)); + } else if (OB_UNLIKELY(retry_ctx.task_executed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("single table lock task already executed", K(ret), K(retry_ctx)); + } else if (OB_FAIL(executor.execute(request))) { + LOG_WARN("failed to execute local lock task", KR(ret), K(ctx.abs_timeout_ts_), K(request)); } else { - retry_ctx.send_rpc_count_++; + retry_ctx.task_executed_ = true; ALLOW_NEXT_LOG(); - LOG_INFO("send table lock rpc", KR(ret), K(retry_ctx.send_rpc_count_), K(addr), "request", request); + LOG_INFO("execute table lock task", KR(ret), K(retry_ctx), "request", request); } return ret; } template <> -int ObTableLockService::pack_and_call_rpc_(observer::ObLocalBatchLockProxy> &proxy_batch, +int ObTableLockService::pack_and_execute_task_(observer::ObLocalBatchLockExecutor> &executor, ObTableLockCtx &ctx, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, - const ObAddr &addr, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; ObLockTaskBatchRequest request; - if (OB_FAIL(pack_batch_request_(ctx, ctx.task_type_, ls_id, lock_ids, request))) { - LOG_WARN("pack_batch_request_ failed", K(ret), K(ls_id), K(ctx), K(lock_ids)); - } else if (OB_FAIL(rpc_call_(proxy_batch, addr, ctx.get_rpc_timeoutus(), request))) { - LOG_WARN("failed to call async rpc", KR(ret), K(addr), K(ctx.abs_timeout_ts_), K(request)); + if (OB_FAIL(pack_batch_request_(ctx, ctx.task_type_, lock_ids, request))) { + LOG_WARN("pack_batch_request_ failed", K(ret), K(ctx), K(lock_ids)); + } else if (OB_UNLIKELY(retry_ctx.task_executed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("single table lock task already executed", K(ret), K(retry_ctx)); + } else if (OB_FAIL(executor.execute(request))) { + LOG_WARN("failed to execute local lock task", KR(ret), K(ctx.abs_timeout_ts_), K(request)); } else { - retry_ctx.send_rpc_count_++; + retry_ctx.task_executed_ = true; ALLOW_NEXT_LOG(); - LOG_INFO("send table lock rpc", KR(ret), K(retry_ctx.send_rpc_count_), K(addr), "request", request); + LOG_INFO("execute table lock task", KR(ret), K(retry_ctx), "request", request); } return ret; } -template -int ObTableLockService::send_one_rpc_task_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_one_lock_task_(LocalExecutor &executor, ObTableLockCtx &ctx, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObRetryCtx &retry_ctx) { int ret = OB_SUCCESS; - retry_ctx.send_rpc_count_ = 0; - if (OB_FAIL(send_rpc_task_(proxy_batch, + retry_ctx.reuse(); + if (OB_FAIL(execute_lock_task_(executor, ctx, - ls_id, lock_ids, retry_ctx))) { - LOG_WARN("send rpc task failed", K(ret), K(ls_id)); + LOG_WARN("execute lock task failed", K(ret)); } else if (retry_ctx.need_retry_) { if (OB_FAIL(get_retry_lock_ids_(lock_ids, 0, retry_ctx.retry_lock_ids_))) { retry_ctx.need_retry_ = false; LOG_WARN("get retry tablet failed", KR(ret)); - }; + } } return ret; } -template -int ObTableLockService::batch_rpc_handle_(RpcProxy &proxy_batch, +template +int ObTableLockService::execute_lock_set_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, + const ObLockSet &lock_set, bool &can_retry, - ObLSLockMap &retry_ls_lock_map) + ObLockSet &retry_lock_set) { int ret = OB_SUCCESS; ObLockIDArray retry_lock_ids; ObRetryCtx retry_ctx; - FOREACH_X(data, ls_lock_map, OB_SUCC(ret) || can_retry) { - proxy_batch.reuse(); + if (!lock_set.empty()) { retry_ctx.reuse(); - const share::ObLSID &ls_id = data->first; - const ObLockIDArray &lock_ids = data->second; + const ObLockIDArray &lock_ids = lock_set.get_lock_ids(); - if (OB_FAIL(send_one_rpc_task_(proxy_batch, + if (OB_FAIL(execute_one_lock_task_(executor, ctx, - ls_id, lock_ids, retry_ctx))) { can_retry = false; - (void)collect_rollback_info_(retry_ctx.rpc_ls_array_, proxy_batch, ctx); - LOG_WARN("send rpc task failed", KR(ret)); + (void)collect_rollback_info_(retry_ctx, ctx); + LOG_WARN("execute lock task failed", KR(ret)); } else { - ret = handle_parallel_rpc_response_(proxy_batch, + ret = handle_task_result_(executor, ctx, - ls_lock_map, + lock_set, can_retry, retry_ctx); } - // collect one rpc's retry tablets. + // Collect retry tablets from the local task result. if (can_retry) { if (OB_FAIL(get_retry_lock_ids_(retry_ctx.retry_lock_ids_, 0, @@ -1736,13 +1549,12 @@ int ObTableLockService::batch_rpc_handle_(RpcProxy &proxy_batch, } } } - // get the retry map + // get the retry set if (can_retry && retry_lock_ids.count() != 0) { - if (OB_FAIL(fill_ls_lock_map_(ctx, - retry_lock_ids, - retry_ls_lock_map, - true /* force refresh location */))) { - LOG_WARN("refill ls lock map failed", KP(ret), K(ctx)); + if (OB_FAIL(fill_lock_set_(ctx, + retry_lock_ids, + retry_lock_set))) { + LOG_WARN("refill lock set failed", KP(ret), K(ctx)); can_retry = false; } } @@ -1750,21 +1562,21 @@ int ObTableLockService::batch_rpc_handle_(RpcProxy &proxy_batch, } int ObTableLockService::inner_process_obj_lock_batch_(ObTableLockCtx &ctx, - const ObLSLockMap &lock_map) + const ObLockSet &lock_map) { int ret = OB_SUCCESS; if (ctx.is_unlock_task()) { - observer::ObLocalBatchLockProxy> proxy_batch( + observer::ObLocalBatchLockExecutor> executor( observer::handle_high_priority_batch_lock_task); - ret = batch_rpc_handle_(proxy_batch, ctx, lock_map); + ret = execute_lock_set_(executor, ctx, lock_map); } else if (ctx.is_replace_task()) { - observer::ObLocalBatchLockProxy> proxy_batch( + observer::ObLocalBatchLockExecutor> executor( observer::handle_batch_replace_lock_task); - ret = batch_rpc_handle_(proxy_batch, ctx, lock_map); + ret = execute_lock_set_(executor, ctx, lock_map); } else { - observer::ObLocalBatchLockProxy> proxy_batch( + observer::ObLocalBatchLockExecutor> executor( observer::handle_batch_lock_task); - ret = batch_rpc_handle_(proxy_batch, ctx, lock_map); + ret = execute_lock_set_(executor, ctx, lock_map); } return ret; } @@ -1772,20 +1584,20 @@ int ObTableLockService::inner_process_obj_lock_batch_(ObTableLockCtx &ctx, int ObTableLockService::process_table_tablet_lock_with_prio_(ObTableLockCtx &ctx, const ObTableLockMode lock_mode, const ObTableLockMode table_lock_mode, - const ObLSLockMap &table_ls_lock_map) + const ObLockSet &table_lock_set) { int ret = OB_SUCCESS; - ObLSLockMap tablet_ls_lock_map; + ObLockSet tablet_lock_set; ctx.schema_version_ = 0; ctx.lock_mode_ = table_lock_mode; - if (OB_FAIL(process_obj_lock_with_prio_(ctx, table_ls_lock_map))) { + if (OB_FAIL(process_obj_lock_with_prio_(ctx, table_lock_set))) { LOG_WARN("lock table failed", K(ret), K(ctx), K(lock_mode)); - } else if (OB_FAIL(get_tablet_ls_lock_map_(lock_mode, ctx, tablet_ls_lock_map))) { - LOG_WARN("failed to get_tablet_ls_lock_map_", K(ret), K(ctx), K(lock_mode)); + } else if (OB_FAIL(get_tablet_lock_set_(lock_mode, ctx, tablet_lock_set))) { + LOG_WARN("failed to get_tablet_lock_set_", K(ret), K(ctx), K(lock_mode)); } else if (FALSE_IT(ctx.lock_mode_ = lock_mode)) { - } else if (OB_FAIL(process_obj_lock_with_prio_(ctx, tablet_ls_lock_map))) { + } else if (OB_FAIL(process_obj_lock_with_prio_(ctx, tablet_lock_set))) { LOG_WARN("lock tablet failed", K(ret), K(ctx)); } LOG_DEBUG("ObTableLockService::process_table_tablet_lock_with_prio_", @@ -1800,31 +1612,31 @@ int ObTableLockService::process_table_tablet_lock_with_prio_(ObTableLockCtx &ctx int ObTableLockService::process_table_tablet_lock_(ObTableLockCtx &ctx, const ObTableLockMode lock_mode, const ObTableLockMode table_lock_mode, - const ObLSLockMap &table_ls_lock_map) + const ObLockSet &table_lock_set) { int ret = OB_SUCCESS; - ObLSLockMap tablet_ls_lock_map; + ObLockSet tablet_lock_set; ctx.schema_version_ = 0; ctx.lock_mode_ = table_lock_mode; - if (OB_FAIL(process_obj_lock_(ctx, table_ls_lock_map))) { + if (OB_FAIL(process_obj_lock_(ctx, table_lock_set))) { LOG_WARN("lock table failed", K(ret), K(ctx), K(table_lock_mode)); } DEBUG_SYNC(TABLE_LOCK_AFTER_LOCK_TABLE_BEFORE_LOCK_TABLET); - if (FAILEDx(get_tablet_ls_lock_map_(lock_mode, ctx, tablet_ls_lock_map))) { - LOG_WARN("get tablet ls_lock_map failed", K(ret), K(ctx), K(lock_mode)); + if (FAILEDx(get_tablet_lock_set_(lock_mode, ctx, tablet_lock_set))) { + LOG_WARN("get tablet lock_set failed", K(ret), K(ctx), K(lock_mode)); } else if (FALSE_IT(ctx.lock_mode_ = lock_mode)) { - } else if (!ctx.is_enable_lock_priority_ && OB_FAIL(pre_check_lock_(ctx, tablet_ls_lock_map))) { + } else if (!ctx.is_enable_lock_priority_ && OB_FAIL(pre_check_lock_(ctx, tablet_lock_set))) { LOG_WARN("failed to pre_check_lock_", K(ret), K(ctx), K(lock_mode)); - } else if (OB_FAIL(process_obj_lock_(ctx, tablet_ls_lock_map))) { + } else if (OB_FAIL(process_obj_lock_(ctx, tablet_lock_set))) { LOG_WARN("lock tablet failed", K(ret), K(ctx), K(lock_mode)); } return ret; } int ObTableLockService::process_obj_lock_with_prio_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map) + const ObLockSet &lock_set) { int ret = OB_SUCCESS; const ObTableLockTaskType ori_task_type = ctx.task_type_; @@ -1838,7 +1650,7 @@ int ObTableLockService::process_obj_lock_with_prio_(ObTableLockCtx &ctx, } else { ctx.task_type_ = ObTableLockTaskType::ADD_LOCK_INTO_QUEUE; } - if (OB_FAIL(process_obj_lock_(ctx, ls_lock_map))) { + if (OB_FAIL(process_obj_lock_(ctx, lock_set))) { LOG_WARN("process_obj_lock_ failed", K(ret), K(ctx)); } ctx.task_type_ = ori_task_type; @@ -1849,7 +1661,7 @@ int ObTableLockService::process_obj_lock_with_prio_(ObTableLockCtx &ctx, } int ObTableLockService::process_obj_lock_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map) + const ObLockSet &lock_set) { int ret = OB_SUCCESS; @@ -1863,7 +1675,7 @@ int ObTableLockService::process_obj_lock_(ObTableLockCtx &ctx, } else if (OB_FAIL(start_sub_tx_(ctx))) { LOG_WARN("failed to start sub tx", K(ret), K(ctx)); } else if (OB_FAIL(inner_process_obj_lock_batch_(ctx, - ls_lock_map))) { + lock_set))) { LOG_WARN("fail to lock tablets", K(ret)); need_retry = need_retry_single_task_(ctx, ret); // rollback the sub tx and overwrite the ret code. @@ -1905,65 +1717,6 @@ int ObTableLockService::check_op_allowed_(const uint64_t table_id, return ret; } -int ObTableLockService::get_lock_id_ls_( - const ObTableLockCtx &ctx, - const ObLockID &lock_id, - ObLSID &ls_id, - bool force_refresh) -{ - int ret = OB_SUCCESS; - if (lock_id.is_tablet_lock()) { - ObTabletID tablet_id; - if (OB_FAIL(lock_id.convert_to(tablet_id))) { - LOG_WARN("convert tablet id failed", K(ret), K(lock_id)); - } else if (OB_FAIL(get_tablet_ls_(ctx, tablet_id, ls_id, force_refresh))) { - LOG_WARN("get tablet ls failed", K(ret), K(tablet_id), K(force_refresh)); - } - } else { - ls_id = LOCK_SERVICE_LS; - } - return ret; -} - -int ObTableLockService::get_tablet_ls_( - const ObTableLockCtx &ctx, - const ObTabletID &tablet_id, - ObLSID &ls_id, - bool force_refresh) -{ - int ret = OB_SUCCESS; - bool unused_cache_hit; - - if (LOCK_ALONE_TABLET == ctx.task_type_ || UNLOCK_ALONE_TABLET == ctx.task_type_ - || REPLACE_LOCK_ALONE_TABLET == ctx.task_type_) { - // we have specified the ls, just do lock and unlock at the specified ls - ls_id = ctx.ls_id_; - } else if (force_refresh) { - if (OB_FAIL(location_service_->get(tablet_id, - INT64_MAX, - unused_cache_hit, - ls_id))) { - LOG_WARN("failed to sync get ls by tablet failed.", - K(ret), K(1UL), K(tablet_id)); - } - } else if (OB_FAIL(location_service_->nonblock_get(tablet_id, ls_id))) { - if (OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST == ret && - OB_FAIL(location_service_->get(tablet_id, - INT64_MAX, - unused_cache_hit, - ls_id))) { - LOG_ERROR("failed to sync get ls by tablet failed.", - K(ret), K(1UL), K(tablet_id)); - } else if (OB_FAIL(ret)) { - LOG_WARN("failed to get ls by tablet", K(ret), K(1UL), - K(tablet_id)); - } - } - LOG_DEBUG("get tablet ls", K(ret), K(1UL), K(tablet_id), K(ls_id)); - - return ret; -} - int ObTableLockService::get_process_tablets_(const ObSimpleTableSchemaV2 *table_schema, ObTableLockCtx &ctx) { @@ -2019,83 +1772,55 @@ int ObTableLockService::get_process_tablets_(const ObSimpleTableSchemaV2 *table_ return ret; } -int ObTableLockService::fill_ls_lock_map_(ObTableLockCtx &ctx, +int ObTableLockService::fill_lock_set_(ObTableLockCtx &ctx, const ObLockIDArray &lock_ids, - ObLSLockMap &ls_lock_map, - bool force_refresh_location) + ObLockSet &lock_set) { int ret = OB_SUCCESS; - share::ObLSID ls_id; - ObLockIDArray lock_array; - ObLockIDArray *p = nullptr; - if (OB_FAIL(ls_lock_map.reuse())) { - LOG_WARN("fail to reuse ls_lock_map", KR(ret)); + UNUSED(ctx); + if (OB_FAIL(lock_set.reuse())) { + LOG_WARN("fail to reuse lock set", KR(ret)); } else { for (int64_t i = 0; i < lock_ids.count() && OB_SUCC(ret); ++i) { - ls_id.reset(); - lock_array.reuse(); - p = nullptr; const ObLockID &lock_id = lock_ids.at(i); - if (OB_FAIL(get_lock_id_ls_(ctx, lock_id, ls_id, force_refresh_location))) { - LOG_WARN("failed to get lock ls", K(ret), K(lock_id)); - } else if (OB_FAIL(ls_lock_map.set_refactored(ls_id, lock_array)) && - OB_ENTRY_EXIST != ret && OB_HASH_EXIST != ret) { - LOG_WARN("fail to set ls_lock_map", K(ret), K(ls_id)); - } else if (OB_ISNULL(p = ls_lock_map.get(ls_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not exist at ls lock map", K(ret), K(ls_id)); - } else if (OB_FAIL(p->push_back(lock_id))) { - LOG_WARN("push_back lock_id failed", K(ret), K(ls_id), K(lock_id)); + if (OB_FAIL(lock_set.push_back(lock_id))) { + LOG_WARN("push_back lock_id failed", K(ret), K(lock_id)); } - LOG_DEBUG("lock add to lock map", K(lock_id), K(i)); + LOG_DEBUG("lock added to lock set", K(lock_id), K(i)); } } return ret; } -int ObTableLockService::fill_ls_lock_map_(ObTableLockCtx &ctx, +int ObTableLockService::fill_lock_set_(ObTableLockCtx &ctx, const common::ObTabletIDArray &tablets, - ObLSLockMap &ls_lock_map) + ObLockSet &lock_set) { int ret = OB_SUCCESS; - share::ObLSID ls_id; - ObLockIDArray lock_array; - ObLockIDArray *p = nullptr; ObLockID lock_id; - if (OB_FAIL(ls_lock_map.reuse())) { - LOG_WARN("fail to reuse ls_lock_map", KR(ret)); + if (OB_FAIL(lock_set.reuse())) { + LOG_WARN("fail to reuse lock set", KR(ret)); } else { for (int64_t i = 0; i < tablets.count() && OB_SUCC(ret); ++i) { - ls_id.reset(); - lock_array.reuse(); - p = nullptr; lock_id.reset(); const ObTabletID &tablet_id = tablets.at(i); - if (OB_FAIL(get_tablet_ls_(ctx, tablet_id, ls_id))) { - LOG_WARN("failed to get tablet ls", K(ret), K(tablet_id)); - } else if (OB_FAIL(get_lock_id(tablet_id, + if (OB_FAIL(get_lock_id(tablet_id, lock_id))) { LOG_WARN("get lock id failed", K(ret), K(ctx)); - } else if (OB_FAIL(ls_lock_map.set_refactored(ls_id, lock_array)) && - OB_ENTRY_EXIST != ret && OB_HASH_EXIST != ret) { - LOG_WARN("fail to set ls_lock_map", K(ret), K(ls_id)); - } else if (OB_ISNULL(p = ls_lock_map.get(ls_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not exist at ls lock map", K(ret), K(ls_id)); - } else if (OB_FAIL(p->push_back(lock_id))) { - LOG_WARN("push_back lock_id failed", K(ret), K(ls_id), K(lock_id)); + } else if (OB_FAIL(lock_set.push_back(lock_id))) { + LOG_WARN("push_back lock_id failed", K(ret), K(lock_id)); } - LOG_DEBUG("tablet add to lock map", K(lock_id), K(tablet_id), K(i)); + LOG_DEBUG("tablet added to lock set", K(lock_id), K(tablet_id), K(i)); } } return ret; } -int ObTableLockService::get_tablet_ls_lock_map_(const ObTableLockMode lock_mode, +int ObTableLockService::get_tablet_lock_set_(const ObTableLockMode lock_mode, ObTableLockCtx &ctx, - ObLSLockMap &tablet_ls_lock_map) + ObLockSet &tablet_lock_set) { int ret = OB_SUCCESS; ObSimpleTableSchemaV2 *table_schema = nullptr; @@ -2112,67 +1837,47 @@ int ObTableLockService::get_tablet_ls_lock_map_(const ObTableLockMode lock_mode, } else if (FALSE_IT(ctx.schema_version_ = table_schema->get_schema_version())) { } else if (OB_FAIL(get_process_tablets_(table_schema, ctx))) { LOG_WARN("failed to get parts", K(ret), K(ctx)); - } else if (OB_FAIL(get_ls_lock_map_(ctx, ctx.tablet_list_, tablet_ls_lock_map))) { - LOG_WARN("fail to get ls lock map", K(ret), K(ctx.get_tablet_cnt())); + } else if (OB_FAIL(get_lock_set_(ctx, ctx.tablet_list_, tablet_lock_set))) { + LOG_WARN("fail to get lock set", K(ret), K(ctx.get_tablet_cnt())); } return ret; } -int ObTableLockService::get_ls_lock_map_(ObTableLockCtx &ctx, - const share::ObLSID &ls_id, +int ObTableLockService::get_lock_set_(ObTableLockCtx &ctx, const ObLockID &lock_id, - ObLSLockMap &ls_lock_map) + ObLockSet &lock_set) { int ret = OB_SUCCESS; - ObLockIDArray lock_array; - ObLockIDArray *p = nullptr; - if (OB_FAIL(ls_lock_map.create(10, lib::ObLabel("LSLockMap")))) { - LOG_WARN("ls_lock_map create failed"); - } else if (OB_FAIL(ls_lock_map.set_refactored(ls_id, lock_array)) && - OB_ENTRY_EXIST != ret && OB_HASH_EXIST != ret) { - LOG_WARN("fail to set tablet_map", K(ret), K(ls_id)); - } else if (OB_ISNULL(p = const_cast(ls_lock_map.get(ls_id)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not exist at tablet map", K(ret), K(ls_id)); - } else if (OB_FAIL(p->push_back(lock_id))) { - LOG_WARN("push_back lock_id failed", K(ret), K(ls_id), K(lock_id)); + if (OB_FAIL(lock_set.reuse())) { + LOG_WARN("fail to reuse lock set", KR(ret)); + } else if (OB_FAIL(lock_set.push_back(lock_id))) { + LOG_WARN("push_back lock_id failed", K(ret), K(lock_id)); } return ret; } -int ObTableLockService::get_ls_lock_map_(ObTableLockCtx &ctx, - const share::ObLSID &ls_id, +int ObTableLockService::get_lock_set_(ObTableLockCtx &ctx, const common::ObIArray &lock_ids, - ObLSLockMap &ls_lock_map) + ObLockSet &lock_set) { int ret = OB_SUCCESS; - ObLockIDArray lock_array; - ObLockIDArray *p = nullptr; - if (OB_FAIL(ls_lock_map.create(10, lib::ObLabel("LSLockMap")))) { - LOG_WARN("ls_lock_map create failed"); - } else if (OB_FAIL(ls_lock_map.set_refactored(ls_id, lock_array)) && - OB_ENTRY_EXIST != ret && OB_HASH_EXIST != ret) { - LOG_WARN("fail to set tablet_map", K(ret), K(ls_id)); - } else if (OB_ISNULL(p = const_cast(ls_lock_map.get(ls_id)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not exist at tablet map", K(ret), K(ls_id)); - } else if (OB_FAIL(p->assign(lock_ids))) { - LOG_WARN("assign lock_ids failed", K(ret), K(ls_id), K(lock_ids)); + if (OB_FAIL(lock_set.reuse())) { + LOG_WARN("fail to reuse lock set", KR(ret)); + } else if (OB_FAIL(lock_set.assign(lock_ids))) { + LOG_WARN("assign lock_ids failed", K(ret), K(lock_ids)); } return ret; } -int ObTableLockService::get_ls_lock_map_(ObTableLockCtx &ctx, +int ObTableLockService::get_lock_set_(ObTableLockCtx &ctx, const common::ObTabletIDArray &tablets, - ObLSLockMap &ls_lock_map) + ObLockSet &lock_set) { int ret = OB_SUCCESS; - if (OB_FAIL(ls_lock_map.create(10, lib::ObLabel("LSLockMap")))) { - LOG_WARN("ls_lock_map create failed", KR(ret)); - } else if (OB_FAIL(fill_ls_lock_map_(ctx, tablets, ls_lock_map))) { - LOG_WARN("fill ls lock map failed", KR(ret)); + if (OB_FAIL(fill_lock_set_(ctx, tablets, lock_set))) { + LOG_WARN("fill lock set failed", KR(ret)); } return ret; @@ -2190,9 +1895,7 @@ bool ObTableLockService::need_retry_trans_(const ObTableLockCtx &ctx, !ctx.is_try_lock() && !ctx.is_timeout()); // retry condition 2 - need_retry = need_retry || ((OB_NOT_MASTER == ret || - OB_LS_NOT_EXIST == ret || - OB_TABLET_NOT_EXIST == ret) && !ctx.is_timeout()); + need_retry = need_retry || (OB_TABLET_NOT_EXIST == ret && !ctx.is_timeout()); } return need_retry; } @@ -2202,36 +1905,28 @@ bool ObTableLockService::need_retry_single_task_(const ObTableLockCtx &ctx, { bool need_retry = false; if (ctx.is_in_trans_) { - need_retry = (OB_NOT_MASTER == ret || - OB_LS_NOT_EXIST == ret || - OB_TABLET_NOT_EXIST == ret); + need_retry = (OB_TABLET_NOT_EXIST == ret); } else { // TODO: yanyuan.cxf multi data source can not rollback, so we can not retry. } return need_retry; } -bool ObTableLockService::need_retry_whole_rpc_task_(const int ret) +bool ObTableLockService::need_retry_whole_task_(const int ret) { return (OB_TENANT_NOT_IN_SERVER == ret); } -bool ObTableLockService::need_retry_part_rpc_task_(const int ret, - const ObTableLockTaskResult *result) const +bool ObTableLockService::need_retry_partial_task_(const int ret, + const ObTableLockTaskResult *result) const { bool need_retry = false; - need_retry = (OB_LS_NOT_EXIST == ret || - OB_TABLET_NOT_EXIST == ret); + need_retry = (OB_TABLET_NOT_EXIST == ret); need_retry = need_retry && result->can_retry(); - // retry if OB_LS_NOT_EXIST/OB_TABLET_NOT_EXIST and lock task result is can retry. + // Retry if the tablet is being created and the lock task can retry. return need_retry; } -bool ObTableLockService::need_renew_location_(const int64_t ret) const -{ - return (OB_LS_LOCATION_NOT_EXIST == ret || OB_LS_LOCATION_LEADER_NOT_EXIST == ret); -} - int ObTableLockService::rewrite_return_code_(const int ret, const int ret_code_before_end_stmt_or_tx, const bool is_from_sql) const { int rewrite_rcode = ret; @@ -2254,43 +1949,14 @@ bool ObTableLockService::is_lock_conflict_ret_code_(const int ret) const bool ObTableLockService::is_timeout_ret_code_(const int ret) const { return (OB_TIMEOUT == ret || OB_TRANS_TIMEOUT == ret || - OB_TRANS_STMT_TIMEOUT == ret || OB_GET_LOCATION_TIME_OUT == ret); + OB_TRANS_STMT_TIMEOUT == ret); } bool ObTableLockService::is_can_retry_err_(const int ret) const { return (OB_TRANS_KILLED == ret || OB_OBJ_UNLOCK_CONFLICT == ret || OB_OBJ_LOCK_NOT_COMPLETED == ret - || OB_TRY_LOCK_ROW_CONFLICT == ret || OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret || OB_NOT_MASTER == ret - || OB_TIMEOUT == ret || OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_TRANS_CTX_NOT_EXIST == ret); -} - -int ObTableLockService::get_ls_leader_( - const int64_t cluster_id, - const ObLSID &ls_id, - const int64_t abs_timeout_ts, - ObAddr &addr) -{ - int ret = OB_SUCCESS; - // priority to set timeout_ctx: ctx > worker > default_timeout - ObTimeoutCtx ctx; - if (OB_FAIL(ctx.set_abs_timeout(abs_timeout_ts))) { - LOG_WARN("set abs timeout ts failed", KR(ret)); - } else if (OB_ISNULL(location_service_)) { - ret = OB_NOT_INIT; - LOG_WARN("table lock service not inited", K(ret)); - } else if (OB_FAIL(location_service_->get_leader(cluster_id, - ls_id, - true, /* force renew */ - addr))) { - LOG_WARN("failed to get ls leader with retry until timeout", - K(ret), K(cluster_id), K(ls_id), K(addr)); - } else { - LOG_DEBUG("get ls leader from location_service", - K(ret), K(cluster_id), K(ls_id), K(addr)); - } - LOG_DEBUG("ObTableLockService::process_obj_lock_", K(ret), K(ls_id), K(addr)); - - return ret; + || OB_TRY_LOCK_ROW_CONFLICT == ret || OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret + || OB_TIMEOUT == ret || OB_TRANS_CTX_NOT_EXIST == ret); } int ObTableLockService::start_tx_(ObTableLockCtx &ctx) @@ -2400,16 +2066,16 @@ int ObTableLockService::end_sub_tx_(ObTableLockCtx &ctx, const bool is_rollback) const auto &savepoint = ctx.current_savepoint_; const int64_t expire_ts = OB_MAX(ctx.abs_timeout_ts_, DEFAULT_TIMEOUT_US + ObTimeUtility::current_time()); ObTransService *txs = share::g_mp->trans_service(); - if (is_rollback && - OB_FAIL(txs->rollback_to_implicit_savepoint(*ctx.tx_desc_, - savepoint, - expire_ts, - &ctx.need_rollback_ls_))) { - LOG_WARN("fail to rollback sub tx", K(ret), K(ctx.tx_desc_), - K(ctx.need_rollback_ls_)); - } - - ctx.clean_touched_ls(); + if (is_rollback && + OB_FAIL(txs->rollback_to_implicit_savepoint(*ctx.tx_desc_, + savepoint, + expire_ts, + ctx.need_rollback()))) { + LOG_WARN("fail to rollback sub tx", K(ret), K(ctx.tx_desc_), + K(ctx.need_rollback_)); + } + + ctx.clear_need_rollback(); ctx.reset_savepoint(); } LOG_DEBUG("ObTableLockService::end_sub_tx_", K(ret), K(ctx)); @@ -2453,16 +2119,16 @@ int ObTableLockService::end_stmt_(ObTableLockCtx &ctx, const bool is_rollback) const int64_t expire_ts = OB_MAX(ctx.abs_timeout_ts_, DEFAULT_TIMEOUT_US + ObTimeUtility::current_time()); ObTransService *txs = share::g_mp->trans_service(); // just rollback the whole stmt, if it is needed. - if (is_rollback && - OB_FAIL(txs->rollback_to_implicit_savepoint(*ctx.tx_desc_, - savepoint, - expire_ts, - &ctx.need_rollback_ls_))) { - LOG_WARN("fail to rollback stmt", K(ret), K(ctx.tx_desc_), - K(ctx.need_rollback_ls_)); - } + if (is_rollback && + OB_FAIL(txs->rollback_to_implicit_savepoint(*ctx.tx_desc_, + savepoint, + expire_ts, + ctx.need_rollback()))) { + LOG_WARN("fail to rollback stmt", K(ret), K(ctx.tx_desc_), + K(ctx.need_rollback_)); + } LOG_DEBUG("ObTableLockService::end_stmt_", K(ret), K(ctx), K(is_rollback)); - ctx.clean_touched_ls(); + ctx.clear_need_rollback(); ctx.reset_stmt_savepoint(); } diff --git a/src/storage/tablelock/ob_table_lock_service.h b/src/storage/tablelock/ob_table_lock_service.h index 5dc0cb20e..b0320f4c0 100644 --- a/src/storage/tablelock/ob_table_lock_service.h +++ b/src/storage/tablelock/ob_table_lock_service.h @@ -21,7 +21,6 @@ #include "common/ob_tablet_id.h" #include "lib/task/ob_timer.h" -#include "share/ob_ls_id.h" #include "sql/ob_sql_trans_control.h" #include "storage/tablelock/ob_table_lock_common.h" #include "storage/tablelock/ob_table_lock_rpc_struct.h" @@ -29,14 +28,6 @@ namespace oceanbase { -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} - namespace share { namespace schema @@ -56,7 +47,24 @@ class ObTableLockService final private: static const int64_t OB_DEFAULT_LOCK_ID_COUNT = 10; typedef common::ObSEArray ObLockIDArray; - typedef hash::ObHashMap ObLSLockMap; + class ObLockSet + { + public: + ObLockSet() : lock_ids_() {} + ~ObLockSet() {} + int reuse() { lock_ids_.reuse(); return common::OB_SUCCESS; } + bool empty() const { return lock_ids_.empty(); } + int64_t size() const { return lock_ids_.count(); } + int64_t count() const { return lock_ids_.count(); } + int push_back(const ObLockID &lock_id) { return lock_ids_.push_back(lock_id); } + int assign(const common::ObIArray &lock_ids) { return lock_ids_.assign(lock_ids); } + const ObLockIDArray &get_lock_ids() const { return lock_ids_; } + ObLockIDArray &get_lock_ids() { return lock_ids_; } + TO_STRING_KV(K_(lock_ids)); + private: + ObLockIDArray lock_ids_; + DISALLOW_COPY_AND_ASSIGN(ObLockSet); + }; class ObTableLockCtx { public: @@ -71,11 +79,11 @@ class ObTableLockService final bool is_deadlock_avoid_enabled() const; bool is_timeout() const; int64_t remain_timeoutus() const; - int64_t get_rpc_timeoutus() const; int64_t get_tablet_cnt() const; const common::ObTabletID &get_tablet_id(const int64_t index) const; - int add_touched_ls(const share::ObLSID &lsid); - void clean_touched_ls(); + void mark_need_rollback(); + void clear_need_rollback(); + bool need_rollback() const { return need_rollback_; } bool is_savepoint_valid() { return current_savepoint_.is_valid(); } void reset_savepoint() { current_savepoint_.reset(); } @@ -101,7 +109,6 @@ class ObTableLockService final struct { uint64_t table_id_; uint64_t partition_id_; // set when lock or unlock specified partition - share::ObLSID ls_id_; // used for alone tablet lock and unlock }; }; @@ -114,17 +121,14 @@ class ObTableLockService final transaction::ObTxDesc *tx_desc_; ObTxParam tx_param_; // the tx param for current tx transaction::ObTxSEQ current_savepoint_; // used to rollback current sub tx. - share::ObLSArray need_rollback_ls_; // which ls has been modified after - // the current_savepoint_ created. + bool need_rollback_; // lock state changed after the savepoint was created common::ObTabletIDArray tablet_list_; // all the tablets need to be locked/unlocked ObLockIDArray obj_list_; ObTableLockMode lock_mode_; ObTableLockOwnerID lock_owner_; - // TODO: yanyuan.cxf we need better performance. - // share::ObLSArray ls_list_; // related ls list - int64_t schema_version_; // the schema version of the table to be locked + int64_t schema_version_; // the schema version of the table to be locked bool tx_is_killed_; // used to kill a trans. bool is_from_sql_; int ret_code_before_end_stmt_or_tx_; // used to mark this lock is still conflict while lock request exiting @@ -139,7 +143,7 @@ class ObTableLockService final K(tablet_list_), K(obj_list_), K(lock_op_type_), K(origin_timeout_us_), K(timeout_us_), K(abs_timeout_ts_), KPC(tx_desc_), K(tx_param_), - K(current_savepoint_), K(need_rollback_ls_), + K(current_savepoint_), K(need_rollback_), K(lock_mode_), K(lock_owner_), K(schema_version_), K(tx_is_killed_), K(is_from_sql_), K(ret_code_before_end_stmt_or_tx_), K(stmt_savepoint_), @@ -159,25 +163,25 @@ class ObTableLockService final ObTableLockOwnerID new_lock_owner_; }; - class ObRetryCtx - { - public: - ObRetryCtx() : need_retry_(false), - send_rpc_count_(0), - rpc_ls_array_(), - retry_lock_ids_() - {} - ~ObRetryCtx() - { reuse(); } - void reuse(); - public: - TO_STRING_KV(K_(need_retry), K_(send_rpc_count), K_(rpc_ls_array), - K_(retry_lock_ids)); - bool need_retry_; - int64_t send_rpc_count_; // how many rpc we have send. - ObArray rpc_ls_array_; - ObLockIDArray retry_lock_ids_; // the lock id need to be retry. - }; + class ObRetryCtx + { + public: + ObRetryCtx() : need_retry_(false), + task_executed_(false), + task_prepared_(false), + retry_lock_ids_() + {} + ~ObRetryCtx() + { reuse(); } + void reuse(); + public: + TO_STRING_KV(K_(need_retry), K_(task_executed), K_(task_prepared), + K_(retry_lock_ids)); + bool need_retry_; + bool task_executed_; + bool task_prepared_; + ObLockIDArray retry_lock_ids_; // the lock id need to be retry. + }; public: class ObOBJLockGarbageCollector { @@ -205,9 +209,8 @@ class ObTableLockService final }; private: void run_gc_once_(); - int garbage_collect_for_all_ls_(); + int garbage_collect_(); void check_and_report_timeout_(); - int check_is_leader_(ObLS *ls, bool &is_leader); public: static int64_t GARBAGE_COLLECT_EXEC_INTERVAL; static int64_t GARBAGE_COLLECT_TIMEOUT; @@ -217,12 +220,8 @@ class ObTableLockService final int64_t last_success_timestamp_; }; -public: - typedef hash::ObHashMap LockMap; - ObTableLockService() - : location_service_(nullptr), - sql_proxy_(nullptr), + : sql_proxy_(nullptr), obj_lock_garbage_collector_(), is_inited_(false) {} ~ObTableLockService() {} @@ -298,10 +297,9 @@ class ObTableLockService final const int64_t ret) const; bool need_retry_single_task_(const ObTableLockCtx &ctx, const int64_t ret) const; - bool need_retry_whole_rpc_task_(const int ret); - bool need_retry_part_rpc_task_(const int ret, - const ObTableLockTaskResult *result) const; - bool need_renew_location_(const int64_t ret) const; + bool need_retry_whole_task_(const int ret); + bool need_retry_partial_task_(const int ret, + const ObTableLockTaskResult *result) const; int rewrite_return_code_(const int ret, const int ret_code_before_end_stmt_or_tx = OB_SUCCESS, const bool is_from_sql = false) const; bool is_lock_conflict_ret_code_(const int ret) const; bool is_timeout_ret_code_(const int ret) const; @@ -323,130 +321,96 @@ class ObTableLockService final bool &is_allowed); int get_process_tablets_(const ObSimpleTableSchemaV2 *table_schema, ObTableLockCtx &ctx); - int get_tablet_ls_lock_map_(const ObTableLockMode lock_mode, ObTableLockCtx &ctx, ObLSLockMap &tablet_ls_lock_map); - int get_ls_lock_map_(ObTableLockCtx &ctx, + int get_tablet_lock_set_(const ObTableLockMode lock_mode, ObTableLockCtx &ctx, ObLockSet &tablet_lock_set); + int get_lock_set_(ObTableLockCtx &ctx, const common::ObTabletIDArray &tablets, - ObLSLockMap &ls_lock_map); - int get_ls_lock_map_(ObTableLockCtx &ctx, - const share::ObLSID &ls_id, + ObLockSet &lock_set); + int get_lock_set_(ObTableLockCtx &ctx, const ObLockID &lock_id, - ObLSLockMap &ls_lock_map); - int get_ls_lock_map_(ObTableLockCtx &ctx, - const share::ObLSID &ls_id, + ObLockSet &lock_set); + int get_lock_set_(ObTableLockCtx &ctx, const common::ObIArray &lock_ids, - ObLSLockMap &ls_lock_map); - int fill_ls_lock_map_(ObTableLockCtx &ctx, + ObLockSet &lock_set); + int fill_lock_set_(ObTableLockCtx &ctx, const ObLockIDArray &lock_ids, - ObLSLockMap &ls_lock_map, - bool force_refresh_location); - int fill_ls_lock_map_(ObTableLockCtx &ctx, + ObLockSet &lock_set); + int fill_lock_set_(ObTableLockCtx &ctx, const common::ObTabletIDArray &tablets, - ObLSLockMap &ls_lock_map); - int get_tablet_ls_(const ObTableLockCtx &ctx, - const ObTabletID &tablet_id, - share::ObLSID &ls_id, - bool force_refresh = false); - int get_lock_id_ls_(const ObTableLockCtx &ctx, - const ObLockID &lock_id, - share::ObLSID &ls_id, - bool force_refresh = false); - int get_ls_leader_(const int64_t cluster_id, - const share::ObLSID &ls_id, - const int64_t abs_timeout_ts, - ObAddr &addr); + ObLockSet &lock_set); int pack_batch_request_(ObTableLockCtx &ctx, const ObTableLockTaskType task_type, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObLockTaskBatchRequest &request); int pack_batch_request_(ObTableLockCtx &ctx, const ObTableLockTaskType task_type, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, ObLockTaskBatchRequest &request); - template - int batch_rpc_handle_(RpcProxy &proxy_batch, + template + int execute_lock_set_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &lock_map); - template - int batch_rpc_handle_(RpcProxy &proxy_batch, + const ObLockSet &lock_map); + template + int execute_lock_set_(LocalExecutor &executor, ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, + const ObLockSet &lock_set, bool &can_retry, - ObLSLockMap &retry_ls_lock_map); - template - int handle_parallel_rpc_response_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, - bool &can_retry, - ObRetryCtx &retry_ctx); - template - int parallel_batch_rpc_handle_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); - template - int parallel_batch_rpc_handle_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, - bool &can_retry, - ObLSLockMap &retry_ls_lock_map); - template - int parallel_send_rpc_task_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map, - ObRetryCtx &retry_ctx); - template - int send_one_rpc_task_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const share::ObLSID &ls_id, - const ObLockIDArray &lock_ids, - ObRetryCtx &retry_ctx); - template - int send_rpc_task_(RpcProxy &proxy_batch, - ObTableLockCtx &ctx, - const share::ObLSID &ls_id, - const ObLockIDArray &lock_ids, - ObRetryCtx &retry_ctx); - template - int rpc_call_(RpcProxy &proxy_batch, - const ObAddr &addr, - const int64_t timeout_us, - const LockRequest &request); - template - int pack_and_call_rpc_(RpcProxy &proxy_batch, + ObLockSet &retry_lock_set); + template + int handle_task_result_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockSet &lock_set, + bool &can_retry, + ObRetryCtx &retry_ctx); + template + int execute_lock_set_in_batches_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockSet &lock_set); + template + int execute_lock_set_in_batches_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockSet &lock_set, + bool &can_retry, + ObLockSet &retry_lock_set); + template + int execute_lock_set_once_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockSet &lock_set, + ObRetryCtx &retry_ctx); + template + int execute_one_lock_task_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockIDArray &lock_ids, + ObRetryCtx &retry_ctx); + template + int execute_lock_task_(LocalExecutor &executor, ObTableLockCtx &ctx, - const share::ObLSID &ls_id, const ObLockIDArray &lock_ids, - const ObAddr &addr, ObRetryCtx &retry_ctx); + template + int pack_and_execute_task_(LocalExecutor &executor, + ObTableLockCtx &ctx, + const ObLockIDArray &lock_ids, + ObRetryCtx &retry_ctx); template<> - int pack_and_call_rpc_(observer::ObLocalBatchLockProxy> &proxy_batch, - ObTableLockCtx &ctx, - const share::ObLSID &ls_id, - const ObLockIDArray &lock_ids, - const ObAddr &addr, - ObRetryCtx &retry_ctx); + int pack_and_execute_task_(observer::ObLocalBatchLockExecutor> &executor, + ObTableLockCtx &ctx, + const ObLockIDArray &lock_ids, + ObRetryCtx &retry_ctx); int get_retry_lock_ids_(const ObLockIDArray &lock_ids, const int64_t start_pos, ObLockIDArray &retry_lock_ids); - int get_retry_lock_ids_(const share::ObLSID &ls_id, - const ObLSLockMap &ls_lock_map, - const int64_t start_pos, - ObLockIDArray &retry_lock_ids); - int collect_rollback_info_(const share::ObLSID &ls_id, - ObTableLockCtx &ctx); - int collect_rollback_info_(const ObArray &ls_array, - ObTableLockCtx &ctx); - template - int collect_rollback_info_(const ObArray &ls_array, - RpcProxy &proxy_batch, - ObTableLockCtx &ctx); + int get_retry_lock_ids_(const ObLockSet &lock_set, + const int64_t start_pos, + ObLockIDArray &retry_lock_ids); + int collect_rollback_info_(ObTableLockCtx &ctx); + int collect_rollback_info_(const ObRetryCtx &retry_ctx, + ObTableLockCtx &ctx); int inner_process_obj_lock_batch_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); int process_obj_lock_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); int process_obj_lock_with_prio_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); static bool is_part_table_lock_(const ObTableLockTaskType task_type); int get_table_lock_mode_(const ObTableLockTaskType task_type, const ObTableLockMode part_lock_mode, @@ -454,16 +418,16 @@ class ObTableLockService final int process_table_tablet_lock_(ObTableLockCtx &ctx, const ObTableLockMode lock_mode, const ObTableLockMode table_lock_mode, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); int process_table_tablet_lock_with_prio_(ObTableLockCtx &ctx, const ObTableLockMode lock_mode, const ObTableLockMode table_lock_mode, - const ObLSLockMap &table_ls_lock_map); + const ObLockSet &table_lock_set); // only useful in LOCK_TABLE/LOCK_PARTITION int pre_check_lock_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); int batch_pre_check_lock_(ObTableLockCtx &ctx, - const ObLSLockMap &ls_lock_map); + const ObLockSet &lock_set); // used by deadlock detector. int deal_with_deadlock_(ObTableLockCtx &ctx); int get_table_partition_level_(const ObTableID table_id, ObPartitionLevel &part_level); @@ -473,11 +437,8 @@ class ObTableLockService final DISALLOW_COPY_AND_ASSIGN(ObTableLockService); private: - // TODO: yanyuan.cxf use parallel rpc and modify this to 5s. static const int64_t DEFAULT_TIMEOUT_US = 1500L * 1000L * 1000L; // 1500s - static const int64_t DEFAULT_RPC_TIMEOUT_US = 2L * 1000L * 1000L; // 2s - share::ObLocationService *location_service_; common::ObMySQLProxy *sql_proxy_; ObOBJLockGarbageCollector obj_lock_garbage_collector_; bool is_inited_; diff --git a/src/storage/tablet/ob_batch_create_tablet_arg.cpp b/src/storage/tablet/ob_batch_create_tablet_arg.cpp index 875e96d70..38d4b838a 100644 --- a/src/storage/tablet/ob_batch_create_tablet_arg.cpp +++ b/src/storage/tablet/ob_batch_create_tablet_arg.cpp @@ -26,7 +26,7 @@ namespace obcall { bool ObBatchCreateTabletArg::is_inited() const { - return id_.is_valid() && major_frozen_scn_.is_valid(); + return major_frozen_scn_.is_valid(); } bool ObBatchCreateTabletArg::is_valid() const @@ -55,12 +55,10 @@ bool ObBatchCreateTabletArg::is_valid() const void ObBatchCreateTabletArg::reset() { - id_.reset(); major_frozen_scn_.reset(); tablets_.reset(); table_schemas_.reset(); need_check_tablet_cnt_ = false; - is_old_mds_ = false; for (int64_t i = 0; i < create_tablet_schemas_.count(); ++i) { create_tablet_schemas_[i]->~ObCreateTabletSchema(); } @@ -112,10 +110,8 @@ int ObBatchCreateTabletArg::assign(const ObBatchCreateTabletArg &arg) if (OB_FAIL(ret)) { reset(); } else { - id_ = arg.id_; major_frozen_scn_ = arg.major_frozen_scn_; need_check_tablet_cnt_ = arg.need_check_tablet_cnt_; - is_old_mds_ = arg.is_old_mds_; clog_checkpoint_scn_ = arg.clog_checkpoint_scn_; mds_checkpoint_scn_ = arg.mds_checkpoint_scn_; create_type_ = arg.create_type_; @@ -138,16 +134,14 @@ bool ObBatchCreateTabletArg::set_binding_info_outside_create() const int ObBatchCreateTabletArg::init_create_tablet( - const share::ObLSID &id, const SCN &major_frozen_scn, const bool need_check_tablet_cnt) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!id.is_valid() || !major_frozen_scn.is_valid())) { + if (OB_UNLIKELY(!major_frozen_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(id), K(major_frozen_scn)); + LOG_WARN("invalid argument", KR(ret), K(major_frozen_scn)); } else { - id_ = id; /* To fix issue 2025022400107312907 major in new tablet should be larger than last freeze info @@ -192,33 +186,6 @@ int ObBatchCreateTabletArg::serialize_for_create_tablet_schemas(char *buf, return ret; } -int ObBatchCreateTabletArg::skip_unis_array_len(const char *buf, - int64_t data_len, - int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t count = 0; - if (pos > data_len) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos)); - } else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { - TRANS_LOG(WARN, "failed to decode array count", K(ret), KP(buf), K(data_len)); - } else if (count < 0) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos), K(count)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - int64_t version = 0; - int64_t len = 0; - OB_UNIS_DECODE(version); - OB_UNIS_DECODE(len); - CHECK_VERSION_LENGTH(1, version, len); - pos += len; - } - } - return ret; -} - int64_t ObBatchCreateTabletArg::get_serialize_size_for_create_tablet_schemas() const { int ret = OB_SUCCESS; @@ -278,52 +245,17 @@ int ObBatchCreateTabletArg::deserialize_create_tablet_schemas(const char *buf, return ret; } -int ObBatchCreateTabletArg::is_old_mds(const char *buf, - int64_t data_len, - bool &is_old_mds) -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - is_old_mds = false; - int64_t version = 0; - int64_t len = 0; - share::ObLSID id; - share::SCN major_frozen_scn; - bool need_check_tablet_cnt = false; - - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, version, len, id, major_frozen_scn); - if (OB_FAIL(ret)) { - } - // tablets array - else if (OB_FAIL(skip_unis_array_len(buf, data_len, pos))) { - TRANS_LOG(WARN, "failed to skip_unis_array_len", K(ret), KP(buf), K(data_len), K(pos)); - } - // schema array - else if (OB_FAIL(skip_unis_array_len(buf, data_len, pos))) { - TRANS_LOG(WARN, "failed to skip_unis_array_len", K(ret), KP(buf), K(data_len), K(pos)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, need_check_tablet_cnt, is_old_mds); - } - } - - return ret; -} - DEF_TO_STRING(ObBatchCreateTabletArg) { int64_t pos = 0; - J_KV(K_(id), K_(major_frozen_scn), K_(need_check_tablet_cnt), K_(is_old_mds), K_(tablets), K_(tablet_extra_infos), K_(clog_checkpoint_scn), K_(create_type), K_(mds_checkpoint_scn)); + J_KV(K_(major_frozen_scn), K_(need_check_tablet_cnt), K_(tablets), K_(tablet_extra_infos), K_(clog_checkpoint_scn), K_(create_type), K_(mds_checkpoint_scn)); return pos; } OB_DEF_SERIALIZE(ObBatchCreateTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, id_, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_, is_old_mds_); + LST_DO_CODE(OB_UNIS_ENCODE, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_); if (OB_FAIL(ret)) { } else if (OB_FAIL(serialize_for_create_tablet_schemas(buf, buf_len, pos))) { LOG_WARN("failed to serialize_for_create_tablet_schemas", KR(ret), KPC(this)); @@ -337,7 +269,7 @@ OB_DEF_SERIALIZE(ObBatchCreateTabletArg) OB_DEF_SERIALIZE_SIZE(ObBatchCreateTabletArg) { int len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, id_, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_, is_old_mds_); + LST_DO_CODE(OB_UNIS_ADD_LEN, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_); len += get_serialize_size_for_create_tablet_schemas(); OB_UNIS_ADD_LEN_ARRAY(tablet_extra_infos_, tablet_extra_infos_.count()); LST_DO_CODE(OB_UNIS_ADD_LEN, clog_checkpoint_scn_, create_type_, mds_checkpoint_scn_); @@ -347,41 +279,20 @@ OB_DEF_SERIALIZE_SIZE(ObBatchCreateTabletArg) OB_DEF_DESERIALIZE(ObBatchCreateTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, id_, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_); - if (OB_SUCC(ret)) { - if (pos == data_len) { - is_old_mds_ = true; - } else { - LST_DO_CODE(OB_UNIS_DECODE, is_old_mds_); - if (OB_FAIL(ret)) { - } else if (pos == data_len) { - } else if (OB_FAIL(deserialize_create_tablet_schemas(buf, data_len, pos))) { - LOG_WARN("failed to deserialize_for_create_tablet_schemas", KR(ret)); - } else { - int64_t tablet_extra_infos_count = 0; - OB_UNIS_DECODE(tablet_extra_infos_count); - if (tablet_extra_infos_count > 0 && OB_FAIL(tablet_extra_infos_.prepare_allocate(tablet_extra_infos_count))) { - LOG_WARN("prepare allocate failed", K(ret), K(tablet_extra_infos_count)); - } else { - OB_UNIS_DECODE_ARRAY(tablet_extra_infos_, tablet_extra_infos_count); - } - } - } - } - - if (OB_SUCC(ret) && tablet_extra_infos_.empty()) { - // process the compatibility of the ObCreateTabletExtraInfo. - const int64_t schemas_count = create_tablet_schemas_.empty() ? table_schemas_.count() : create_tablet_schemas_.count(); - if (OB_UNLIKELY(schemas_count <= 0)) { + LST_DO_CODE(OB_UNIS_DECODE, major_frozen_scn_, tablets_, table_schemas_, need_check_tablet_cnt_); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(deserialize_create_tablet_schemas(buf, data_len, pos))) { + LOG_WARN("failed to deserialize_for_create_tablet_schemas", KR(ret)); + } else { + int64_t tablet_extra_infos_count = 0; + OB_UNIS_DECODE(tablet_extra_infos_count); + if (tablet_extra_infos_count <= 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arg", K(ret), K(schemas_count), K(table_schemas_), K(create_tablet_schemas_)); + LOG_WARN("tablet extra infos are required", K(ret), K(tablet_extra_infos_count)); + } else if (OB_FAIL(tablet_extra_infos_.prepare_allocate(tablet_extra_infos_count))) { + LOG_WARN("prepare allocate failed", K(ret), K(tablet_extra_infos_count)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < schemas_count; i++) { - obcall::ObCreateTabletExtraInfo create_tablet_extra_info; // placeholder. - if (OB_FAIL(tablet_extra_infos_.push_back(create_tablet_extra_info))) { - LOG_WARN("failed to push back create tablet extra info", K(ret)); - } - } + OB_UNIS_DECODE_ARRAY(tablet_extra_infos_, tablet_extra_infos_count); } } LST_DO_CODE(OB_UNIS_DECODE, clog_checkpoint_scn_, create_type_, mds_checkpoint_scn_); diff --git a/src/storage/tablet/ob_batch_create_tablet_arg.h b/src/storage/tablet/ob_batch_create_tablet_arg.h index 8e174505a..92e860695 100644 --- a/src/storage/tablet/ob_batch_create_tablet_arg.h +++ b/src/storage/tablet/ob_batch_create_tablet_arg.h @@ -29,7 +29,7 @@ namespace obcall { struct ObBatchCreateTabletArg { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObBatchCreateTabletArg() { reset(); } @@ -38,8 +38,7 @@ struct ObBatchCreateTabletArg bool is_inited() const; void reset(); int assign(const ObBatchCreateTabletArg &arg); - int init_create_tablet(const share::ObLSID &id_, - const share::SCN &major_frozen_scn, + int init_create_tablet(const share::SCN &major_frozen_scn, const bool need_check_tablet_cnt); int64_t get_tablet_count() const; int serialize_for_create_tablet_schemas(char *buf, @@ -49,22 +48,14 @@ struct ObBatchCreateTabletArg int deserialize_create_tablet_schemas(const char *buf, const int64_t data_len, int64_t &pos); - static int is_old_mds(const char *buf, - int64_t data_len, - bool &is_old_mds); - static int skip_unis_array_len(const char *buf, - int64_t data_len, - int64_t &pos); bool set_binding_info_outside_create() const; DECLARE_TO_STRING; public: - share::ObLSID id_; share::SCN major_frozen_scn_; common::ObSArray table_schemas_; common::ObSArray tablets_; bool need_check_tablet_cnt_; - bool is_old_mds_; common::ObSArray create_tablet_schemas_; ObArenaAllocator allocator_; common::ObSArray tablet_extra_infos_; diff --git a/src/storage/tablet/ob_batch_create_tablet_pretty_arg.cpp b/src/storage/tablet/ob_batch_create_tablet_pretty_arg.cpp index a8e4b6d38..f5a48d85a 100644 --- a/src/storage/tablet/ob_batch_create_tablet_pretty_arg.cpp +++ b/src/storage/tablet/ob_batch_create_tablet_pretty_arg.cpp @@ -35,8 +35,7 @@ int64_t ObBatchCreateTabletPrettyArg::to_string(char *buf, const int64_t buf_len // do nothing } else { J_OBJ_START(); - J_KV("ls_id", arg_.id_, - "major_frozen_scn", arg_.major_frozen_scn_, + J_KV("major_frozen_scn", arg_.major_frozen_scn_, "total_tablet_cnt", arg_.get_tablet_count()); J_COMMA(); diff --git a/src/storage/tablet/ob_full_tablet_creator.cpp b/src/storage/tablet/ob_full_tablet_creator.cpp index 8b4104fe7..10bc03d42 100644 --- a/src/storage/tablet/ob_full_tablet_creator.cpp +++ b/src/storage/tablet/ob_full_tablet_creator.cpp @@ -16,7 +16,7 @@ #include "ob_full_tablet_creator.h" #include "share/rc/ob_module_provider.h" -#include "src/storage/tx_storage/ob_ls_map.h" +#include "storage/tablet/ob_tablet.h" #define USING_LOG_PREFIX STORAGE @@ -46,10 +46,10 @@ int ObFullTabletCreator::init() OB_MALLOC_NORMAL_BLOCK_SIZE/2, ObMemAttr("TinyAllocator", ObCtxIds::DEFAULT_CTX_ID)))) { LOG_WARN("fail to init tenant tiny allocator", K(ret)); } else { - ContextParam param; + lib::ContextParam param; param.set_mem_attr("MSTXCTX", common::ObCtxIds::DEFAULT_CTX_ID) .set_ablock_size(lib::INTACT_MIDDLE_AOBJECT_SIZE) - .set_properties(ALLOC_THREAD_SAFE); + .set_properties(lib::ALLOC_THREAD_SAFE); if (OB_FAIL(ROOT_CONTEXT->CREATE_CONTEXT(mstx_mem_ctx_, param))) { LOG_WARN("fail to create entity", K(ret)); } else if (nullptr == mstx_mem_ctx_) { @@ -94,7 +94,7 @@ int ObFullTabletCreator::throttle_tablet_creation() int ret = OB_SUCCESS; bool need_wait = false; - const int64_t limit_size = get_tenant_memory_limit() / 20; // 5% + const int64_t limit_size = lib::get_tenant_memory_limit() / 20; // 5% const int64_t timeout = 5 * 1000L; // 5ms for effective replay const int64_t log_timeout = 1 * 1000 * 1000L; // 1s const int64_t start_time = ObTimeUtility::fast_current_time(); diff --git a/src/storage/tablet/ob_i_tablet_mds_customized_interface.cpp b/src/storage/tablet/ob_i_tablet_mds_customized_interface.cpp index f970a009a..9428b16df 100644 --- a/src/storage/tablet/ob_i_tablet_mds_customized_interface.cpp +++ b/src/storage/tablet/ob_i_tablet_mds_customized_interface.cpp @@ -58,21 +58,14 @@ int ObITabletMdsCustomizedInterface::get_latest_split_data( if (OB_UNLIKELY(!check_is_inited_())) { ret = OB_NOT_INIT; MDS_LOG_GET(WARN, "not inited"); - } else { - ObITabletMdsInterface *src = nullptr; - ObTabletHandle src_tablet_handle; - - if (OB_FAIL(ret)) { - } else if (CLICK_FAIL((cross_ls_get_latest( - src, + } else if (CLICK_FAIL((get_latest( ReadSplitDataOp(data), writer, trans_stat, trans_version, read_seq)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG_GET(WARN, "fail to cross ls get latest", K(lbt())); - } + if (OB_EMPTY_RESULT != ret) { + MDS_LOG_GET(WARN, "fail to get latest", K(lbt())); } } return ret; @@ -93,21 +86,14 @@ int ObITabletMdsCustomizedInterface::get_latest_autoinc_seq( if (OB_UNLIKELY(!check_is_inited_())) { ret = OB_NOT_INIT; MDS_LOG_GET(WARN, "not inited"); - } else { - ObITabletMdsInterface *src = nullptr; - ObTabletHandle src_tablet_handle; - - if (OB_FAIL(ret)) { - } else if (CLICK_FAIL((cross_ls_get_latest( - src, + } else if (CLICK_FAIL((get_latest( ReadAutoIncSeqOp(allocator, data), writer, trans_stat, trans_version, read_seq)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG_GET(WARN, "fail to cross ls get latest", K(lbt())); - } + if (OB_EMPTY_RESULT != ret) { + MDS_LOG_GET(WARN, "fail to get latest", K(lbt())); } } return ret; diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.cpp b/src/storage/tablet/ob_i_tablet_mds_interface.cpp index d3b5f9ff3..a7a03d74c 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.cpp +++ b/src/storage/tablet/ob_i_tablet_mds_interface.cpp @@ -19,6 +19,7 @@ #include "share/rc/ob_module_provider.h" #include "storage/tablet/ob_mds_scan_param_helper.h" #include "storage/tablet/ob_mds_schema_helper.h" +#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" namespace oceanbase { @@ -183,7 +184,6 @@ int ObITabletMdsInterface::get_split_data( } else { // TODO(lihongqin.lhq): use get_latest_committed and block during 2pc const share::SCN snapshot = share::SCN::max_scn(); - if (CLICK_FAIL((get_snapshot(mds::DummyKey(), ReadSplitDataOp(data), snapshot, timeout)))) { if (OB_EMPTY_RESULT != ret) { @@ -214,7 +214,6 @@ int ObITabletMdsInterface::split_partkey_compare(const blocksstable::ObDatumRowk } else { // TODO(lihongqin.lhq): use get_latest_committed and block during 2pc const share::SCN snapshot = share::SCN::max_scn(); - if (CLICK_FAIL((get_snapshot(mds::DummyKey(), ReadSplitDataPartkeyCompareOp(rowkey, rowkey_read_info, partkey_projector, cmp_ret), snapshot, timeout)))) { if (OB_EMPTY_RESULT != ret) { @@ -236,14 +235,12 @@ int ObITabletMdsInterface::read_raw_data( mds::MdsDumpKV &kv) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = get_tablet_meta_().ls_id_; const common::ObTabletID &tablet_id = get_tablet_meta_().tablet_id_; const int64_t abs_timeout = timeout_us + ObClockGenerator::getClock(); ObMdsReadInfoCollector placeholder_collector; SMART_VARS_3((ObTableScanParam, scan_param), (ObStoreCtx, store_ctx), (ObMdsRowIterator, iter)) { if (OB_FAIL(ObMdsScanParamHelper::build_scan_param( allocator, - ls_id, tablet_id, ObMdsSchemaHelper::MDS_TABLE_ID, mds_unit_id, @@ -298,11 +295,10 @@ int ObITabletMdsInterface::get_tablet_handle_from_this( { int ret = OB_SUCCESS; const ObTablet *tablet = static_cast(this); - const share::ObLSID &ls_id = get_tablet_meta_().ls_id_; const common::ObTabletID &tablet_id = get_tablet_meta_().tablet_id_; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); if (OB_FAIL(t3m->build_tablet_handle_for_mds_scan(const_cast(tablet), tablet_handle))) { - MDS_LOG(WARN, "fail to build tablet handle", K(ret), K(ls_id), K(tablet_id)); + MDS_LOG(WARN, "fail to build tablet handle", K(ret), K(tablet_id)); } return ret; } diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.h b/src/storage/tablet/ob_i_tablet_mds_interface.h index 09cf007cc..df0b18474 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.h +++ b/src/storage/tablet/ob_i_tablet_mds_interface.h @@ -151,18 +151,10 @@ class ObITabletMdsInterface int &cmp_ret, const int64_t timeout) const; int fill_virtual_info(ObIArray &mds_node_info_array) const; - TO_STRING_KV(KP(this), "is_inited", check_is_inited_(), "ls_id", get_tablet_meta_().ls_id_, + TO_STRING_KV(KP(this), "is_inited", check_is_inited_(), "tablet_id", get_tablet_id_(), KP(get_tablet_pointer_())); int get_mds_table_rec_scn(share::SCN &rec_scn); int mds_table_flush(const share::SCN &recycle_scn); - // get tablet status from MDS, and check whether reserved state redo scn is valid. - // @param [in] written : if current tablet status is reserved, set true if redo_scn is valid, otherwise set fasle - // @return OB_STATE_NOT_MATCH : tablet status is not reserved. - // OB_EMPTY_RESULT : never has tablet status written. - // OB_LS_OFFLINE : read meet ls offline - // other error... - // CAUTIONS: this interface is only for reserved compatibility status. - int check_reserved_status_redo_written(bool &written); template int get_latest_committed_data(T &value, ObIAllocator *alloc = nullptr); protected:// implemented by ObTablet @@ -216,13 +208,6 @@ class ObITabletMdsInterface int replay(T &&mds, mds::MdsCtx &ctx, const share::SCN &scn); - template - int cross_ls_get_latest(const ObITabletMdsInterface *another, - OP &&read_op, - mds::MdsWriter &writer,// FIXME(zk250686): should not exposed, will be removed later - mds::TwoPhaseCommitState &trans_stat,// FIXME(zk250686): should not exposed, will be removed later - share::SCN &trans_version,// FIXME(zk250686): should not exposed, will be removed later - const int64_t read_seq = 0) const; template int replay(const Key &key, Value &&mds, @@ -233,16 +218,6 @@ class ObITabletMdsInterface int replay_remove(const Key &key, mds::MdsCtx &ctx, const share::SCN &scn);// called only by ObTabletReplayExecutor - template - int cross_ls_get_snapshot(const ObITabletMdsInterface *another, - const Key &key, - OP &&read_op, - const share::SCN snapshot, - const int64_t timeout_us) const; - template - int cross_ls_get_latest_committed(const ObITabletMdsInterface *another, - T &value, - ObIAllocator *alloc = nullptr) const; common::ObTabletID get_tablet_id_() const; template int obj_to_string_holder_(const T &obj, ObStringHolder &holder) const; diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.ipp b/src/storage/tablet/ob_i_tablet_mds_interface.ipp index 9924194e7..87d24883c 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.ipp +++ b/src/storage/tablet/ob_i_tablet_mds_interface.ipp @@ -436,7 +436,7 @@ int ObITabletMdsInterface::get_latest(OP &&read_op, MDS_LOG(WARN, "failed to check data completion"); } else if (!is_data_complete) { ret = OB_EAGAIN; - MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().ha_status_)); + MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().restore_state_)); } else if (OB_ISNULL(get_tablet_pointer_())) { ret = OB_BAD_NULL_ERROR; MDS_LOG(ERROR, "pointer on tablet should not be null"); @@ -519,7 +519,7 @@ int ObITabletMdsInterface::get_latest_committed(OP &&read_op) const MDS_LOG(WARN, "failed to check data completion"); } else if (!is_data_complete) { ret = OB_EAGAIN; - MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().ha_status_)); + MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().restore_state_)); } else if (OB_ISNULL(get_tablet_pointer_())) { ret = OB_BAD_NULL_ERROR; MDS_LOG(ERROR, "pointer on tablet should not be null"); @@ -606,7 +606,7 @@ int ObITabletMdsInterface::get_snapshot(const Key &key, MDS_LOG(WARN, "failed to check data completion"); } else if (!is_data_complete) { ret = OB_EAGAIN; - MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().ha_status_)); + MDS_LOG(INFO, "mds_data is not complete, try again later", K(ret), K(get_tablet_meta_().restore_state_)); } else if (OB_ISNULL(get_tablet_pointer_())) { ret = OB_BAD_NULL_ERROR; MDS_LOG(ERROR, "pointer on tablet should not be null"); @@ -696,7 +696,6 @@ int ObITabletMdsInterface::fill_virtual_info_from_mds_sstable(ObIArray>::value; - const share::ObLSID &ls_id = get_tablet_meta_().ls_id_; const common::ObTabletID &tablet_id = get_tablet_meta_().tablet_id_; const int64_t abs_timeout = ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US/*timeout_us*/ + ObClockGenerator::getClock(); @@ -704,7 +703,6 @@ int ObITabletMdsInterface::fill_virtual_info_from_mds_sstable(ObIArrayk_, cur_virtual_info->user_key_))) { MDS_LOG_GET(WARN, "fail to fill string holder"); - } else if (CLICK_FAIL(dump_kv->v_.convert_to_user_mds_node(user_mds_node, get_tablet_meta_().ls_id_, get_tablet_meta_().tablet_id_))) { + } else if (CLICK_FAIL(dump_kv->v_.convert_to_user_mds_node(user_mds_node))) { MDS_LOG_GET(WARN, "fail to convert tablet_status_node"); } else if (CLICK_FAIL(user_mds_node.fill_virtual_info(*cur_virtual_info))) { MDS_LOG_GET(WARN, "fail to fill virtual info"); } else { - cur_virtual_info->ls_id_ = get_tablet_meta_().ls_id_; cur_virtual_info->tablet_id_ = get_tablet_meta_().tablet_id_; cur_virtual_info->position_ = mds::NodePosition::DISK; cur_virtual_info->unit_id_ = dump_kv->v_.mds_unit_id_; @@ -778,7 +775,6 @@ int ObITabletMdsInterface::fill_virtual_info_by_obj_(const T &obj, if (CLICK_FAIL(obj_to_string_holder_(obj, cur_virtual_info->user_data_))) { MDS_LOG_GET(WARN, "fail to fill string holder"); } else { - cur_virtual_info->ls_id_ = get_tablet_meta_().ls_id_; cur_virtual_info->tablet_id_ = get_tablet_meta_().tablet_id_; cur_virtual_info->position_ = position; cur_virtual_info->unit_id_ = mds::TupleTypeIdx>::value; @@ -867,176 +863,13 @@ int ObITabletMdsInterface::get_latest_committed_data(T &value, ObIAllocator *all #undef PRINT_WRAPPER } - -template -int ObITabletMdsInterface::cross_ls_get_latest( - const ObITabletMdsInterface *another, - OP &&read_op, - mds::MdsWriter &writer, - mds::TwoPhaseCommitState &trans_stat, - share::SCN &trans_version, - const int64_t read_seq) const -{ - int ret = OB_SUCCESS; - - if (OB_FAIL((get_latest(std::forward(read_op), writer, trans_stat, trans_version, read_seq)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get latest", K(ret)); - } - } - - if (OB_EMPTY_RESULT == ret && nullptr != another) { - if (OB_FAIL((another->get_latest(std::forward(read_op), writer, trans_stat, trans_version, read_seq)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get latest", K(ret)); - } - } - } - - return ret; -} - -template -int ObITabletMdsInterface::cross_ls_get_latest_committed( - const ObITabletMdsInterface *another, - T &value, - ObIAllocator *alloc) const -{ - int ret = OB_SUCCESS; - - if (OB_FAIL((get_latest_committed(value, alloc)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get latest", K(ret)); - } - } - - if (OB_EMPTY_RESULT == ret && nullptr != another) { - if (OB_FAIL((another->get_latest_committed(value, alloc)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get latest", K(ret)); - } - } - } - - return ret; -} - -template -int ObITabletMdsInterface::cross_ls_get_snapshot( - const ObITabletMdsInterface *another, - const Key &key, - OP &&read_op, - const share::SCN snapshot, - const int64_t timeout_us) const -{ - int ret = OB_SUCCESS; - - if (OB_FAIL((get_snapshot(key, read_op, snapshot, timeout_us)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get snapshot", K(ret)); - } - } - - if (OB_EMPTY_RESULT == ret && nullptr != another) { - if (OB_FAIL((another->get_snapshot(key, read_op, snapshot, timeout_us)))) { - if (OB_EMPTY_RESULT != ret) { - MDS_LOG(WARN, "fail to get snapshot", K(ret)); - } - } - } - - return ret; -} - template ::type> int ObITabletMdsInterface::check_mds_data_complete_(bool &is_complete) const { int ret = OB_SUCCESS; - // For multi-source data (excluding tablet_status), during the migration of LS, - // reading is only permitted once the data is complete; - // otherwise, the data read will be incomplete. - is_complete = get_tablet_meta_().ha_status_.is_data_status_complete(); + is_complete = true; return ret; } -/********************************************this is special logic*****************************************************/ -inline int ObITabletMdsInterface::check_reserved_status_redo_written(bool &written) -{ - #define PRINT_WRAPPER KR(ret), K(*this), K(written) - MDS_TG(10_ms); - int ret = OB_SUCCESS; - bool is_online = false; - ObTabletCreateDeleteMdsUserData tablet_status; - share::SCN redo_scn; - if (OB_ISNULL(get_tablet_pointer_())) { - ret = OB_BAD_NULL_ERROR; - MDS_LOG(ERROR, "pointer on tablet should not be null"); - } else { - ObTabletMdsSharedLockGuard guard(get_tablet_pointer_()->get_mds_truncate_lock()); - if (OB_FAIL(guard.get_ret())) { - MDS_LOG_SET(WARN, "failed to add mds truncate lock"); - } else { - do { - mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; - if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { - if (OB_ENTRY_NOT_EXIST != ret) { - MDS_LOG_GET(WARN, "failed to get_mds_table"); - } else { - MDS_LOG_GET(TRACE, "failed to get_mds_table"); - } - } else if (!handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "mds cannot be NULL"); - } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_pointer_()->get_ls(), is_online))) { - MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); - } else if (CLICK_FAIL(handle.get_tablet_status_node(GetTabletStatusNodeFromMdsTableOp(tablet_status, redo_scn)))) { - if (OB_SNAPSHOT_DISCARDED != ret) { - MDS_LOG_GET(WARN, "failed to get mds data"); - } else { - MDS_LOG_GET(TRACE, "failed to get mds data"); - } - } else { - if (tablet_status.get_tablet_status() != ObTabletStatus::RESERVED_5) { - ret = OB_STATE_NOT_MATCH; - } else if (!redo_scn.is_valid() || redo_scn.is_max()) { - written = false; - MDS_LOG_GET(TRACE, "get reserved status on mds_table, but redo scn is not valid"); - } else { - written = true; - MDS_LOG_GET(TRACE, "get reserved status on mds_table, and redo scn is valid"); - } - } - if (CLICK_FAIL(ret)) { - if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { - if (CLICK_FAIL((get_mds_data_from_tablet( - mds::DummyKey(), - share::SCN::max_scn()/*snapshot*/, - ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US/*timeout_us*/, - ReadTabletStatusOp(tablet_status))))) { - MDS_LOG_GET(WARN, "failed to get latest data from tablet"); - } else if (tablet_status.get_tablet_status() != ObTabletStatus::RESERVED_5) { - ret = OB_STATE_NOT_MATCH; - } else { - written = true; - } - } - } - if (OB_SUCC(ret)) { - if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch(is_online))) { - if (!is_online) { - ret = OB_LS_OFFLINE; - } - MDS_LOG_GET(WARN, "failed to double check ls online"); - } else { - MDS_LOG_GET(TRACE, "success to check_reserved_status_redo_written"); - } - } - } while (ret == OB_VERSION_NOT_MATCH && is_online); - } - } - return ret; - #undef PRINT_WRAPPER -} } } diff --git a/src/storage/tablet/ob_mds_cl_range_query_iterator.h b/src/storage/tablet/ob_mds_cl_range_query_iterator.h index bd89f189c..1a2bff353 100644 --- a/src/storage/tablet/ob_mds_cl_range_query_iterator.h +++ b/src/storage/tablet/ob_mds_cl_range_query_iterator.h @@ -125,16 +125,14 @@ int ObMdsRangeQueryIterator::init( ObTableScanParam *src_scan_param = nullptr; char *buf = nullptr; - ObLSID ls_id; ObTabletID tablet_id; if (OB_ISNULL(buf = static_cast(scan_param.allocator_->alloc(sizeof(ObTableScanParam))))) { ret = OB_ALLOCATE_MEMORY_FAILED; MDS_LOG(WARN, "allocate mem failed", K(ret), KP(buf)); } else if (FALSE_IT(src_scan_param = new(buf) ObTableScanParam())) { - } else if (OB_FAIL(ObMdsRangeQueryIteratorHelper::get_tablet_ls_id_and_tablet_id(src_tablet_handle, ls_id, tablet_id))) { - MDS_LOG(WARN, "failed to get ls_id and tablet_id", K(ret), KPC(src_tablet_handle.get_obj())); + } else if (OB_FAIL(ObMdsRangeQueryIteratorHelper::get_tablet_id(src_tablet_handle, tablet_id))) { + MDS_LOG(WARN, "failed to get tablet_id", K(ret), KPC(src_tablet_handle.get_obj())); } else if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param(*scan_param.allocator_, - ls_id, tablet_id, scan_param.read_version_range_, *scan_param.mds_collector_, diff --git a/src/storage/tablet/ob_mds_range_query_iterator.cpp b/src/storage/tablet/ob_mds_range_query_iterator.cpp index 61cc4dcbb..339096447 100644 --- a/src/storage/tablet/ob_mds_range_query_iterator.cpp +++ b/src/storage/tablet/ob_mds_range_query_iterator.cpp @@ -53,15 +53,14 @@ int ObMdsRangeQueryIteratorHelper::check_mds_data_complete(const ObTabletHandle ret = OB_INVALID_ARGUMENT; MDS_LOG(WARN, "tablet is null", K(ret)); } else { - is_data_complete = tablet->get_tablet_meta().ha_status_.is_data_status_complete(); + is_data_complete = true; } return ret; } -int ObMdsRangeQueryIteratorHelper::get_tablet_ls_id_and_tablet_id( +int ObMdsRangeQueryIteratorHelper::get_tablet_id( const ObTabletHandle &tablet_handle, - ObLSID &ls_id, ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -69,7 +68,6 @@ int ObMdsRangeQueryIteratorHelper::get_tablet_ls_id_and_tablet_id( ret = OB_INVALID_ARGUMENT; MDS_LOG(WARN, "invalid argument", K(ret), K(tablet_handle)); } else { - ls_id = tablet_handle.get_obj()->get_ls_id(); tablet_id = tablet_handle.get_obj()->get_tablet_id(); } return ret; diff --git a/src/storage/tablet/ob_mds_range_query_iterator.h b/src/storage/tablet/ob_mds_range_query_iterator.h index 2e0976d53..de1ffafdd 100644 --- a/src/storage/tablet/ob_mds_range_query_iterator.h +++ b/src/storage/tablet/ob_mds_range_query_iterator.h @@ -52,9 +52,8 @@ class ObMdsRangeQueryIteratorHelper public: static int get_mds_table(const ObTabletHandle &tablet_handle, mds::MdsTableHandle &mds_table); static int check_mds_data_complete(const ObTabletHandle &tablet_handle, bool &is_data_complete); - static int get_tablet_ls_id_and_tablet_id( + static int get_tablet_id( const ObTabletHandle &tablet_handle, - ObLSID &ls_id, ObTabletID &tablet_id); }; diff --git a/src/storage/tablet/ob_mds_row_iterator.cpp b/src/storage/tablet/ob_mds_row_iterator.cpp index 0fd894fda..a0f476e48 100644 --- a/src/storage/tablet/ob_mds_row_iterator.cpp +++ b/src/storage/tablet/ob_mds_row_iterator.cpp @@ -81,7 +81,6 @@ int ObMdsRowIterator::init( table_scan_param_ = &scan_param; is_inited_ = true; LOG_DEBUG("succeed to init mds row iterator", K(ret), - "ls_id", tablet_handle.get_obj()->get_ls_id(), "tablet_id", tablet_handle.get_obj()->get_tablet_id(), K(scan_param)); } diff --git a/src/storage/tablet/ob_mds_scan_param_helper.cpp b/src/storage/tablet/ob_mds_scan_param_helper.cpp index 8a123c994..d455c071b 100644 --- a/src/storage/tablet/ob_mds_scan_param_helper.cpp +++ b/src/storage/tablet/ob_mds_scan_param_helper.cpp @@ -33,7 +33,6 @@ namespace storage { int ObMdsScanParamHelper::build_scan_param( common::ObIAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const uint64_t table_id, const uint8_t mds_unit_id, @@ -48,7 +47,6 @@ int ObMdsScanParamHelper::build_scan_param( const share::schema::ObTableSchema *table_schema = ObMdsSchemaHelper::get_instance().get_table_schema(); - scan_param.ls_id_ = ls_id; scan_param.tablet_id_ = tablet_id; scan_param.is_get_ = is_get; scan_param.is_for_foreign_check_ = false; @@ -72,7 +70,7 @@ int ObMdsScanParamHelper::build_scan_param( if (OB_FAIL(tx_snapshot.version_.convert_for_tx(read_version_range.snapshot_version_))) { LOG_WARN("failed to convert for tx", KR(ret), K(read_version_range)); } else { - scan_param.snapshot_.init_ls_read(ls_id, tx_snapshot); + scan_param.snapshot_.init_ls_read(tx_snapshot); scan_param.fb_snapshot_ = tx_snapshot.version_; } } diff --git a/src/storage/tablet/ob_mds_scan_param_helper.h b/src/storage/tablet/ob_mds_scan_param_helper.h index 93ff1b004..fad5d4ae9 100644 --- a/src/storage/tablet/ob_mds_scan_param_helper.h +++ b/src/storage/tablet/ob_mds_scan_param_helper.h @@ -35,7 +35,6 @@ class ObString; namespace share { -class ObLSID; class SCN; namespace schema { @@ -58,7 +57,6 @@ class ObMdsScanParamHelper public: static int build_scan_param( common::ObIAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const uint64_t table_id, const uint8_t mds_unit_id, @@ -71,7 +69,6 @@ class ObMdsScanParamHelper template static int build_customized_scan_param( common::ObIAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObVersionRange &read_version_range, ObMdsReadInfoCollector &collector, @@ -103,7 +100,6 @@ class ObMdsScanParamHelper template int ObMdsScanParamHelper::build_customized_scan_param( common::ObIAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObVersionRange &read_version_range, ObMdsReadInfoCollector &collector, @@ -119,7 +115,6 @@ int ObMdsScanParamHelper::build_customized_scan_param( if (OB_FAIL(build_scan_param( allocator, - ls_id, tablet_id, ObMdsSchemaHelper::MDS_TABLE_ID, mds_unit_id, diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 2a3593d69..7b988157b 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -27,6 +27,7 @@ #include "storage/tablet/ob_tablet_medium_info_reader.h" #include "storage/tablet/ob_tablet_mds_node_dump_operator.h" #include "storage/tx_storage/ob_ls_service.h" +#include "storage/tx/ob_ts_mgr.h" #include "storage/truncate_info/ob_tablet_truncate_info_reader.h" #include "storage/truncate_info/ob_truncate_info_array.h" #include "storage/tablet/ob_tablet_medium_info_reader.h" @@ -93,7 +94,6 @@ ObTableStoreCache::ObTableStoreCache() recycle_version_(0), last_major_column_count_(0), last_major_macro_block_cnt_(0), - is_tablet_referenced_by_collect_mv_(false), last_major_compressor_type_(ObCompressorType::INVALID_COMPRESSOR), last_major_latest_row_store_type_(ObRowStoreType::MAX_ROW_STORE) { @@ -107,18 +107,15 @@ void ObTableStoreCache::reset() recycle_version_ = 0; last_major_column_count_ = 0; last_major_macro_block_cnt_ = 0; - is_tablet_referenced_by_collect_mv_ = false; last_major_compressor_type_ = ObCompressorType::INVALID_COMPRESSOR; last_major_latest_row_store_type_ = ObRowStoreType::MAX_ROW_STORE; } int ObTableStoreCache::init( const ObSSTableArray &major_tables, - const ObSSTableArray &minor_tables, - const bool is_tablet_referenced_by_collect_mv) + const ObSSTableArray &minor_tables) { int ret = OB_SUCCESS; - is_tablet_referenced_by_collect_mv_ = is_tablet_referenced_by_collect_mv; major_table_cnt_ = major_tables.count(); minor_table_cnt_ = minor_tables.count(); last_major_snapshot_version_ = 0; @@ -201,7 +198,7 @@ ObTablet::ObTablet(const bool is_external_tablet) is_external_tablet_(is_external_tablet) { #if defined(__x86_64__) && !defined(ENABLE_OBJ_LEAK_CHECK) && !defined(_WIN32) - check_size(); + check_size(); #endif MEMSET(memtables_, 0x0, sizeof(memtables_)); } @@ -213,7 +210,7 @@ ObTablet::~ObTablet() void ObTablet::reset() { - FLOG_INFO("reset tablet", KP(this), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_, K(lbt())); + FLOG_INFO("reset tablet", KP(this), "tablet_id", tablet_meta_.tablet_id_, K(lbt())); reset_memtable(); reset_ddl_memtables(); @@ -268,7 +265,6 @@ int64_t ObTablet::get_try_cache_size() const int ObTablet::init_for_first_time_creation( common::ObArenaAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -295,8 +291,7 @@ int ObTablet::init_for_first_time_creation( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!data_tablet_id.is_valid()) //|| OB_UNLIKELY(create_scn <= OB_INVALID_TIMESTAMP) || OB_UNLIKELY(OB_INVALID_VERSION == snapshot_version) @@ -304,7 +299,7 @@ int ObTablet::init_for_first_time_creation( || OB_UNLIKELY(lib::Worker::CompatMode::INVALID == compat_mode) || OB_ISNULL(freezer)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), + LOG_WARN("invalid args", K(ret), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema), K(compat_mode), KP(freezer)); } else if (OB_UNLIKELY(!pointer_hdl_.is_valid()) || OB_ISNULL(log_handler_)) { @@ -312,15 +307,15 @@ int ObTablet::init_for_first_time_creation( LOG_WARN("tablet pointer handle is invalid", K(ret), K_(pointer_hdl), K_(log_handler)); } else if (!need_create_empty_major_sstable && FALSE_IT(table_store_flag.set_without_major_sstable())) { } else if (FALSE_IT(table_store_flag.set_is_user_data_table(create_tablet_schema.is_user_data_table()))) { - } else if (OB_FAIL(init_shared_params(ls_id, tablet_id, compat_mode))) { - LOG_WARN("failed to init shared params", K(ret), K(ls_id), K(tablet_id), K(compat_mode), KP(freezer)); + } else if (OB_FAIL(init_shared_params(tablet_id, compat_mode))) { + LOG_WARN("failed to init shared params", K(ret), K(tablet_id), K(compat_mode), KP(freezer)); } else if (is_split_dest_tablet && OB_FALSE_IT(split_info.set_data_incomplete(true))) { } else if (OB_FALSE_IT(split_info.set_split_src_tablet_id(split_src_tablet_id))) { - } else if (OB_FAIL(tablet_meta_.init(ls_id, tablet_id, data_tablet_id, + } else if (OB_FAIL(tablet_meta_.init(tablet_id, data_tablet_id, create_scn, snapshot_version, compat_mode, table_store_flag, create_tablet_schema.get_schema_version()/*create_schema_version*/, clog_checkpoint_scn, mds_checkpoint_scn, split_info, micro_index_clustered, false/*has_truncate_info*/, fork_info))) { - LOG_WARN("failed to init tablet meta", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), + LOG_WARN("failed to init tablet meta", K(ret), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(compat_mode), K(table_store_flag)); } else if (OB_FAIL(pull_memtables(allocator))) { LOG_WARN("fail to pull memtable", K(ret)); @@ -344,8 +339,7 @@ int ObTablet::init_for_first_time_creation( } else if (OB_FAIL(try_update_start_scn())) { LOG_WARN("failed to update start scn", K(ret), K(table_store_addr_)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(check_sstable_column_checksum())) { LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); @@ -447,8 +441,7 @@ int ObTablet::init_for_merge( } if (FAILEDx(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(try_update_start_scn())) { LOG_WARN("failed to update start scn", K(ret), K(param), K(table_store_addr_)); @@ -513,143 +506,6 @@ int ObTablet::init_for_merge( } -int ObTablet::init_with_migrate_param( - common::ObArenaAllocator &allocator, - const ObMigrationTabletParam ¶m, - const bool is_update, - ObFreezer *freezer) -{ - int ret = OB_SUCCESS; - const share::ObLSID &ls_id = param.ls_id_; - const common::ObTabletID &tablet_id = param.tablet_id_; - allocator_ = &allocator; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(!param.is_valid()) - || OB_ISNULL(freezer)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(param), KP(freezer)); - } else if (OB_UNLIKELY(!pointer_hdl_.is_valid()) - || OB_ISNULL(log_handler_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet pointer handle is invalid", K(ret), K_(pointer_hdl), K_(log_handler)); - } else if (!is_update && OB_FAIL(init_shared_params(ls_id, tablet_id, param.compat_mode_))) { - LOG_WARN("failed to init shared params", K(ret), K(ls_id), K(tablet_id), KP(freezer)); - } else if (OB_FAIL(tablet_meta_.init(param))) { - LOG_WARN("failed to init tablet meta", K(ret), K(param)); - } - - if (OB_SUCC(ret)) { - if (param.is_empty_shell()) { - if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, table_store_addr_.ptr_))) { - LOG_WARN("fail to allocate and new rowkey read info", K(ret)); - } else if (OB_FAIL(table_store_addr_.ptr_->init(allocator, *this))) { - LOG_WARN("fail to init table store", K(ret)); - } else if (OB_FAIL(try_update_start_scn())) { - LOG_WARN("failed to update start scn", K(ret), K(table_store_addr_)); - } else { - tablet_meta_.extra_medium_info_.reset(); - table_store_addr_.addr_.set_none_addr(); - storage_schema_addr_.addr_.set_none_addr(); - macro_info_addr_.addr_.set_none_addr(); - is_inited_ = true; - LOG_INFO("succeeded to init empty shell tablet", K(ret), K(param), KPC(this)); - } - } else { - ObTableHandleV2 mds_mini_sstable; - const blocksstable::ObSSTable *sstable = nullptr; - const bool need_compat = !tablet_id.is_ls_inner_tablet() && param.version_ < ObMigrationTabletParam::PARAM_VERSION_V3; - if (!need_compat) { - LOG_INFO("no need to build mds mini sstable", K(ret), - "ls_id", tablet_meta_.ls_id_, - "tablet_id", tablet_meta_.tablet_id_, - "migration_param_version", param.version_); - } else if (OB_FAIL(build_mds_mini_sstable_for_migration(allocator, param, mds_mini_sstable))) { - LOG_WARN("fail to build mds mini sstablr for migration", K(ret)); - } else if (OB_UNLIKELY(!mds_mini_sstable.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mds mini sstable is invalid, unexpected", K(ret), K(param), K(mds_mini_sstable)); - } else if (OB_FAIL(mds_mini_sstable.get_sstable(sstable))) { - LOG_WARN("failed to get sstable", K(ret), K(param), K(mds_mini_sstable)); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(pull_memtables(allocator))) { - LOG_WARN("fail to pull memtable", K(ret)); - } else if (!need_compat) { - const ObMajorChecksumInfo *ckm_info = ¶m.major_ckm_info_; - ALLOC_AND_INIT(allocator, table_store_addr_, (*this), - nullptr /*ObSSTable*/, - (ckm_info->is_empty() || - is_output_exec_mode(ckm_info->get_exec_mode())) - ? nullptr - : ckm_info); - } else if (OB_ISNULL(sstable)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sstable is null", K(ret), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_, K(param)); - } else { - ObTabletTableStore tmp_table_store; - ObUpdateTableStoreParam mds_param(param.snapshot_version_, - 1/*multi_version_start*/, - ObMdsSchemaHelper::get_instance().get_storage_schema(), - 0/*ls_rebuild_seq*/, - sstable, - false/*allow_duplicate_sstable*/); - if (OB_FAIL(mds_param.init_with_compaction_info(ObCompactionTableStoreParam( - compaction::ObMergeType::MDS_MINI_MERGE, - sstable->get_end_scn() /*clog_checkpoint_scn*/, - false /*need_report*/, false /*has_truncate_info*/)))) { - LOG_WARN("failed to init with compaction info", KR(ret)); - } else if (OB_FAIL(tmp_table_store.init(allocator, *this, nullptr/*sstable*/))) { - LOG_WARN("fail to init tmp table store", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, table_store_addr_.ptr_))) { - LOG_WARN("fail to alloc and new table store object", K(ret), K_(table_store_addr)); - } else if (OB_FAIL(table_store_addr_.ptr_->init(allocator, *this, mds_param, tmp_table_store))) { - LOG_WARN("fail to init table store", K(ret), K(mds_param)); - } else if (OB_FAIL(try_update_start_scn())) { - LOG_WARN("failed to update start scn", K(ret), K(mds_param), K(table_store_addr_)); - } - } - - if (OB_SUCC(ret)) { - ALLOC_AND_INIT(allocator, storage_schema_addr_, param.storage_schema_); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { - LOG_WARN("failed to init table store cache", K(ret), KPC(this)); - } else if (OB_FAIL(build_read_info(allocator, nullptr /*tablet*/))) { - LOG_WARN("fail to build read info", K(ret)); - } else if (OB_FAIL(check_medium_list())) { - LOG_WARN("failed to check medium list", K(ret), K(param)); - } else if (OB_FAIL(check_sstable_column_checksum())) { - LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); - } else if (OB_FAIL(init_aggregated_info(allocator, nullptr/* link_writer, tmp_tablet do no write */))) { - LOG_WARN("fail to init aggregated info", K(ret)); - } else if (FALSE_IT(set_initial_addr())) { - } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { - LOG_WARN("failed to increase macro ref cnt", K(ret)); - /* NOTICE!!! - * Subsequently, skipping `is_inited_ = true` is prohibited (i.e., OB_FAIL must not occur), otherwise - * it will lead to a macro block refcnt leak. */ - } else { - is_inited_ = true; - LOG_INFO("succeeded to init tablet with migration tablet param", K(ret), K(param), KPC(this)); - } - } - } - - if (OB_UNLIKELY(!is_inited_)) { - reset(); - } - - return ret; -} - int ObTablet::init_for_defragment( common::ObArenaAllocator &allocator, const ObIArray &tables, @@ -707,8 +563,7 @@ int ObTablet::init_for_defragment( LOG_WARN("fail to init aggregated info", K(ret)); } else if (FALSE_IT(set_initial_addr())) { } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(check_table_store_flag_match_with_table_store_(table_store_addr_.get_ptr()))) { LOG_WARN("failed to check table store flag match with table store", K(ret), K(old_tablet), K_(table_store_addr)); @@ -739,7 +594,7 @@ int ObTablet::update_restore_status_for_split_(const ObTabletTableStore &table_s int ret = OB_SUCCESS; ObTableStoreIterator iter; ObTabletRestoreStatus::STATUS meta_restore_status; - if (OB_FAIL(tablet_meta_.ha_status_.get_restore_status(meta_restore_status))) { + if (OB_FAIL(tablet_meta_.restore_state_.get_restore_status(meta_restore_status))) { LOG_WARN("get restore status failed", K(ret), K(tablet_meta_)); } else if (ObTabletRestoreStatus::is_remote(meta_restore_status)) { LOG_TRACE("already remote status", K_(tablet_meta)); @@ -767,7 +622,7 @@ int ObTablet::update_restore_status_for_split_(const ObTabletTableStore &table_s ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected sstable meta", K(ret), K(meta_handle), KPC(sstable)); } else if (meta_handle.get_sstable_meta().get_table_backup_flag().has_backup()) { - tablet_meta_.ha_status_.set_restore_status(ObTabletRestoreStatus::REMOTE); + tablet_meta_.restore_state_.set_restore_status(ObTabletRestoreStatus::REMOTE); break; } } @@ -818,7 +673,7 @@ int ObTablet::init_for_sstable_replace( LOG_WARN("failed to fetch old table store", K(ret), K(old_tablet)); } else if (OB_FAIL(old_table_store_wrapper.get_member(old_table_store))) { LOG_WARN("failed to get old table store", K(ret)); - } else if (FALSE_IT(storage_schema = OB_ISNULL(param.tablet_meta_) ? old_storage_schema : ¶m.tablet_meta_->storage_schema_)) { + } else if (FALSE_IT(storage_schema = OB_ISNULL(param.storage_schema_) ? old_storage_schema : param.storage_schema_)) { } else if (OB_FAIL(old_tablet.get_max_sync_storage_schema_version(max_sync_schema_version))) { LOG_WARN("failed to get max sync storage schema version", K(ret)); } else if (is_tablet_split && OB_FAIL(tablet_meta_.init(old_tablet.tablet_meta_, @@ -832,11 +687,6 @@ int ObTablet::init_for_sstable_replace( param.tablet_fork_param_.mds_checkpoint_scn_, fork_info))) { // init fork tablet meta. LOG_WARN("failed to init fork tablet meta", K(ret), K(old_tablet), K(param), K(max_sync_schema_version), K(fork_info)); - } else if (!is_tablet_split && !is_tablet_fork && OB_FAIL(tablet_meta_.init(old_tablet.tablet_meta_, param.tablet_meta_ - // this interface for migration to batch update table store - // use max schema to make sure sstable and schema match - ))) { - LOG_WARN("failed to init tablet meta", K(ret), K(old_tablet), K(param)); } else if (OB_FAIL(pull_memtables(allocator))){ LOG_WARN("fail to pull memtable", K(ret)); } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, table_store_addr_.ptr_))) { @@ -845,8 +695,6 @@ int ObTablet::init_for_sstable_replace( LOG_ERROR("failed to init split tablet table store", K(ret), K(old_tablet)); } else if (is_tablet_fork && OB_FAIL(table_store_addr_.ptr_->build_fork_new_table_store(allocator, *this, param, *old_table_store))) { LOG_WARN("failed to init fork tablet table store", K(ret), K(old_tablet)); - } else if (!is_tablet_split && !is_tablet_fork && OB_FAIL(table_store_addr_.ptr_->build_ha_new_table_store(allocator, *this, param, *old_table_store))) { - LOG_WARN("failed to init table store", K(ret), K(old_tablet)); } else if (OB_FAIL(ObStorageSchemaUtil::update_tablet_storage_schema( tablet_meta_.tablet_id_, *allocator_, *old_storage_schema, *storage_schema, storage_schema_addr_.ptr_))) { LOG_WARN("failed to choose and save storage schema", K(ret), K(old_tablet), K(param)); @@ -859,16 +707,11 @@ int ObTablet::init_for_sstable_replace( } else if (OB_FAIL(try_update_start_scn())) { LOG_WARN("failed to update start scn", K(ret), K(param), K(table_store_addr_)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (FALSE_IT(finish_medium_scn = get_last_major_snapshot_version())) { } else if (is_tablet_fork && FALSE_IT(tablet_meta_.extra_medium_info_.reset())) { // Fork table should not inherit source extra_medium_info; keep it consistent with forked major. - } else if (!is_tablet_fork && nullptr != param.tablet_meta_ && FALSE_IT(tablet_meta_.update_extra_medium_info( - old_tablet.tablet_meta_.extra_medium_info_, param.tablet_meta_->extra_medium_info_, finish_medium_scn))) { - } else if (!is_ls_inner_tablet() && !is_tablet_split && !is_tablet_fork && OB_FAIL(update_tablet_status_from_sstable(false/*expect_persist_status*/))) { - LOG_ERROR("fail to update tablet status from sstable", K(ret)); } else if (OB_FAIL(build_read_info(*allocator_, nullptr /*tablet*/))) { LOG_WARN("failed to build read info", K(ret)); } else if (OB_FAIL(check_medium_list())) { @@ -912,7 +755,7 @@ int ObTablet::init_for_sstable_replace( ObErrsimBackfillPointType point_type(ObErrsimBackfillPointType::TYPE::ERRSIM_REPLACE_SWAP_BEFORE); if (param.errsim_point_info_.is_errsim_point(point_type)) { ret = OB_EAGAIN; - LOG_WARN("[ERRSIM BACKFILL] errsim backfill swap tablet before", K(ret), K(param)); + LOG_WARN("[ERRSIM] errsim replace swap tablet before", K(ret), K(param)); } #endif return ret; @@ -984,7 +827,6 @@ int ObTablet::read_medium_info_list( const compaction::ObMediumCompactionInfoList *&medium_info_list) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; compaction::ObMediumCompactionInfoList *tmp_list = nullptr; common::ObSEArray medium_info_array; @@ -1000,7 +842,7 @@ int ObTablet::read_medium_info_list( LOG_WARN("fail to init medium info list", K(ret)); } else { medium_info_list = tmp_list; - LOG_TRACE("read medium info", K(ls_id), K(tablet_id), KPC(tmp_list)); + LOG_TRACE("read medium info", K(tablet_id), KPC(tmp_list)); } if (OB_FAIL(ret)) { @@ -1021,7 +863,6 @@ int ObTablet::read_medium_array( common::ObIArray &medium_info_array) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; const int64_t finish_medium_scn = get_last_major_snapshot_version(); ObMdsReadInfoCollector unused_collector; @@ -1029,12 +870,11 @@ int ObTablet::read_medium_array( SMART_VARS_2((ObTableScanParam, scan_param), (ObTabletMediumInfoReader, medium_info_reader)) { if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, tablet_id, ObMdsScanParamHelper::get_whole_read_version_range(), unused_collector, scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(tablet_id)); } else if (OB_FAIL(medium_info_reader.init(*this, scan_param))) { LOG_WARN("failed to init medium info reader", K(ret)); } else { @@ -1101,7 +941,7 @@ int ObTablet::get_truncate_info_newest_version(int64_t &newest_commit_version, i ObArenaAllocator tmp_allocator(ObMemAttr("TruncateInfoArr")); ObTruncateInfoArray tmp_array; SCN max_readable_scn; - if (OB_FAIL(OB_TS_MGR.get_ts_sync(GCONF.rpc_timeout, max_readable_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts_sync(GCONF.rpc_timeout, max_readable_scn))) { LOG_WARN("failed to get gts", K(ret)); } else if (OB_FAIL(read_truncate_info_array( tmp_allocator, @@ -1247,19 +1087,17 @@ int ObTablet::inner_read_truncate_info_array_from_mds( ObMdsReadInfoCollector &collector) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; SMART_VARS_2((ObTableScanParam, scan_param), (ObTabletTruncateInfoReader, truncate_info_reader)) { if (OB_FAIL(truncate_info_array.init_for_first_creation(allocator))) { LOG_WARN("failed to init truncate info array", KR(ret)); } else if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, tablet_id, read_version_range, collector, scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(tablet_id)); } else if (OB_FAIL(truncate_info_reader.init(*this, scan_param))) { LOG_WARN("failed to init truncate info reader", K(ret)); } else { @@ -1342,8 +1180,7 @@ int ObTablet::init_with_update_medium_info( } if (FAILEDx(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(init_aggregated_info(allocator, nullptr/* link_writer, tmp_tablet do no write */))) { LOG_WARN("fail to init aggregated info", K(ret)); @@ -1415,7 +1252,7 @@ int ObTablet::update_tablet_status_from_sstable(const bool expect_persist_status ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US/*timeout_us*/, ReadTabletStatusOp(last_tablet_status))))) { if (OB_ITER_END == ret) { - if (expect_persist_status && tablet_meta_.ha_status_.check_allow_read()) { + if (expect_persist_status && tablet_meta_.restore_state_.check_allow_read()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect none tablet status in sstables", K(ret), K(tablet_meta_)); } else { @@ -1428,8 +1265,8 @@ int ObTablet::update_tablet_status_from_sstable(const bool expect_persist_status } else if (OB_FAIL(tablet_meta_.last_persisted_committed_tablet_status_.assign(last_tablet_status))) { LOG_WARN("fail to assign last_persisted_committed_tablet_status", K(ret), K(last_tablet_status)); } else { - LOG_INFO("succeed to read last tablet status from sstable", K(ret), "ls_id", tablet_meta_.ls_id_, - "tablet_id", tablet_meta_.tablet_id_, "ha_status", tablet_meta_.ha_status_, + LOG_INFO("succeed to read last tablet status from sstable", K(ret), + "tablet_id", tablet_meta_.tablet_id_, "restore_state", tablet_meta_.restore_state_, "last_tablet_status", last_tablet_status, "last_persisted_commmited_tablet_status", tablet_meta_.last_persisted_committed_tablet_status_); } @@ -1438,11 +1275,11 @@ int ObTablet::update_tablet_status_from_sstable(const bool expect_persist_status return ret; } -int ObTablet::init_with_replace_members( +int ObTablet::init_with_updated_members( common::ObArenaAllocator &allocator, const ObTablet &old_tablet, - const int64_t snapshot_version, - const ObTabletDataStatus::STATUS &data_status) + const int64_t snapshot_version +) { int ret = OB_SUCCESS; ObTabletMemberWrapper table_store_wrapper; @@ -1454,7 +1291,7 @@ int ObTablet::init_with_replace_members( ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(is_inited_)); } else if (OB_UNLIKELY(!old_tablet.is_valid() || 0 >= snapshot_version - || !ObTabletDataStatus::is_valid(data_status))) { +)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(old_tablet), K(snapshot_version)); } else if (OB_UNLIKELY(!pointer_hdl_.is_valid()) @@ -1477,8 +1314,6 @@ int ObTablet::init_with_replace_members( } else if (OB_FAIL(tablet_meta_.assign(old_tablet_meta))) { LOG_WARN("failed to copy tablet meta", K(ret), K(old_tablet_meta)); } else if (FALSE_IT(tablet_meta_.snapshot_version_ = MAX(snapshot_version, tablet_meta_.snapshot_version_))) { - } else if (OB_FAIL(tablet_meta_.ha_status_.set_data_status(data_status))) { - LOG_WARN("failed to set data status", K(ret), K(data_status), K_(tablet_meta)); } if (FAILEDx(ObTabletObjLoadHelper::alloc_and_new(allocator, table_store_addr_.ptr_))) { @@ -1499,8 +1334,7 @@ int ObTablet::init_with_replace_members( if (FAILEDx(build_read_info(allocator, nullptr /*tablet*/))) { LOG_WARN("failed to build read info", K(ret)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { // full tablet only increases data macro blocks' ref cnt LOG_WARN("failed to increase sstables ref cnt", K(ret)); @@ -1511,7 +1345,7 @@ int ObTablet::init_with_replace_members( if (old_tablet.get_tablet_meta().has_next_tablet_) { set_next_tablet_guard(old_tablet.next_tablet_guard_); } - LOG_INFO("succeeded to init tablet with new snapshot", K(ret), K(snapshot_version), K(data_status), KPC(this), K(old_tablet)); + LOG_INFO("succeeded to init tablet with updated members", K(ret), K(snapshot_version), KPC(this), K(old_tablet)); is_inited_ = true; } @@ -1574,8 +1408,7 @@ int ObTablet::init_with_mds_sstable( } if (FAILEDx(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (CLICK() && FAILEDx(build_read_info(*allocator_, nullptr /*tablet*/))) { LOG_WARN("failed to build read info", K(ret)); @@ -1621,16 +1454,16 @@ int ObTablet::init_for_compat( } else if (OB_UNLIKELY(old_tablet.is_ls_inner_tablet() && mds_mini_sstable.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("ls inner tablet should NOT have valid mds sstable", K(ret), - "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); } else if (old_tablet.is_empty_shell()) { if (OB_FAIL(inner_init_compat_empty_shell(allocator, old_tablet, mds_mini_sstable))) { LOG_WARN("fail to compat empty shell tablet", K(ret), - "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); } } else { if (OB_FAIL(inner_init_compat_normal_tablet(allocator, has_tablet_status, old_tablet, mds_mini_sstable))) { LOG_WARN("fail to compat normal tablet", K(ret), K(has_tablet_status), - "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); } } @@ -1658,8 +1491,7 @@ int ObTablet::inner_init_compat_empty_shell( LOG_WARN("old tablet has null mds data", K(ret), K(old_tablet.get_tablet_meta())); } else { const ObTabletCreateDeleteMdsUserData &user_data = old_tablet.mds_data_->tablet_status_cache_; - if (ObTabletStatus::DELETED != user_data.tablet_status_ - && ObTabletStatus::RESERVED_6 != user_data.tablet_status_) { + if (ObTabletStatus::DELETED != user_data.tablet_status_) { ret = OB_STATE_NOT_MATCH; LOG_WARN("old tablet status is not deleted", K(ret), K(old_tablet.get_tablet_meta())); } else if (OB_FAIL(tablet_meta_.assign(old_tablet.tablet_meta_))) { @@ -1724,7 +1556,6 @@ int ObTablet::inner_init_compat_normal_tablet( ObUpdateTableStoreParam mds_param(old_tablet.tablet_meta_.snapshot_version_, 1/*multi_version_start*/, ObMdsSchemaHelper::get_instance().get_storage_schema(), - 0/*ls_rebuild_seq*/, sstable, false/*allow_duplicate_sstable*/); if (CLICK_FAIL(mds_param.init_with_compaction_info(ObCompactionTableStoreParam( @@ -1761,8 +1592,7 @@ int ObTablet::inner_init_compat_normal_tablet( if (OB_FAIL(ret)) { } else if (CLICK_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema_addr_.get_ptr()->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (CLICK_FAIL(build_read_info(*allocator_, nullptr /*tablet*/))) { LOG_WARN("failed to build read info", K(ret)); @@ -1897,7 +1727,7 @@ int ObTablet::check_no_backup_data() const int ret = OB_SUCCESS; ObTableStoreIterator iter; - if (!tablet_meta_.ha_status_.is_none()) { + if (!tablet_meta_.restore_state_.is_restore_status_full()) { // skip } else if (OB_FAIL(inner_get_all_sstables(iter))) { LOG_WARN("fail to get all sstables", K(ret)); @@ -1922,7 +1752,7 @@ int ObTablet::check_no_backup_data() const } else if (OB_UNLIKELY(meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("The sstable has backup data, but restore status is full.", - K(ret), KPC(cur), KPC(this), K(tablet_meta_.ha_status_)); + K(ret), KPC(cur), KPC(this), K(tablet_meta_.restore_state_)); } } } @@ -2216,7 +2046,7 @@ int ObTablet::load_deserialize( if (OB_FAIL(ret)) { } else if (tablet_meta_.has_next_tablet_) { - const ObTabletMapKey key(tablet_meta_.ls_id_, tablet_meta_.tablet_id_); + const ObTabletMapKey key(tablet_meta_.tablet_id_); if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, next_tablet_guard_))) { LOG_WARN("failed to acquire tablet", K(ret), K(key)); } else if (OB_ISNULL(next_tablet_guard_.get_obj())) { @@ -2262,8 +2092,7 @@ int ObTablet::deserialize_post_work(common::ObArenaAllocator &allocator) if (OB_FAIL(load_storage_schema(arena_allocator, schema))) { LOG_WARN("load storage schema failed", K(ret)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), - table_store_addr_.get_ptr()->get_minor_sstables(), - schema->is_tablet_referenced_by_collect_mv()))) { + table_store_addr_.get_ptr()->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (tablet_meta_.max_sync_storage_schema_version_ > schema->schema_version_) { LOG_INFO("tablet meta status is not right, upgrade may happened. fix max_sync_schema_version on purpose", @@ -2350,7 +2179,7 @@ int ObTablet::load_deserialize_v1( } else { tablet_meta_.extra_medium_info_ = info_list.get_extra_medium_info(); tablet_meta_.mds_checkpoint_scn_ = tablet_meta_.clog_checkpoint_scn_; - LOG_INFO("succeeded to build mds data", K(ret), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_, K_(mds_data)); + LOG_INFO("succeeded to build mds data", K(ret), "tablet_id", tablet_meta_.tablet_id_, K_(mds_data)); } if (OB_FAIL(ret)) { @@ -2359,8 +2188,7 @@ int ObTablet::load_deserialize_v1( } else if (OB_FAIL(pull_memtables(allocator))) { LOG_WARN("fail to pull memtable", K(ret), K(len), K(pos)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.ptr_->get_major_sstables(), - table_store_addr_.ptr_->get_minor_sstables(), - false))) { + table_store_addr_.ptr_->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else { set_initial_addr(); @@ -2402,8 +2230,6 @@ int ObTablet::deserialize_meta_v1( if (OB_UNLIKELY(tablet_meta_.length_ > len - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("buffer's length is not enough", K(ret), K(tablet_meta_.length_), K(len - new_pos)); - } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.ls_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.tablet_id_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize tablet id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.data_tablet_id_.deserialize(buf, len, new_pos))) { @@ -2428,7 +2254,7 @@ int ObTablet::deserialize_meta_v1( LOG_WARN("failed to deserialize compat mode", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(auto_inc_seq.deserialize(allocator, buf, len, new_pos))) { // get autoinc seq LOG_WARN("failed to deserialize auto inc seq", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.ha_status_.deserialize(buf, len, new_pos))) { + } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.restore_state_.deserialize(buf, len, new_pos))) { LOG_ERROR("failed to deserialize restore status", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < tablet_meta_.length_ && OB_FAIL(tablet_meta_.report_status_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize report status", K(ret), K(len), K(new_pos)); @@ -2740,15 +2566,10 @@ int ObTablet::deserialize( } } if (OB_SUCC(ret) && !storage_schema_addr_.addr_.is_none()) { - ObStorageSchema *storage_schema = nullptr; - if (OB_FAIL(load_storage_schema(allocator, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret)); - } else if (OB_FAIL(table_store_cache_.init(table_store->get_major_sstables(), - table_store->get_minor_sstables(), - storage_schema->is_tablet_referenced_by_collect_mv()))) { + if (OB_FAIL(table_store_cache_.init(table_store->get_major_sstables(), + table_store->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } - ObTabletObjLoadHelper::free(allocator, storage_schema); } } @@ -2825,7 +2646,7 @@ int ObTablet::deserialize( if (OB_SUCC(ret) && tablet_meta_.has_next_tablet_) { ObTabletHandle next_tablet_handle; - const ObTabletMapKey key(tablet_meta_.ls_id_, tablet_meta_.tablet_id_); + const ObTabletMapKey key(tablet_meta_.tablet_id_); if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tablet_from_pool(ObTabletPoolType::TP_NORMAL, key, next_tablet_handle))) { LOG_WARN("failed to acquire tablet", K(ret), K(key)); } else if (OB_ISNULL(next_tablet_handle.get_obj())) { @@ -3093,14 +2914,14 @@ int ObTablet::inc_ref_with_macro_iter(ObMacroInfoIterator ¯o_iter, bool &inc } else if (OB_TMP_FAIL(print_arr.push_back(block_info.macro_id_))) { LOG_WARN("fail to push macro id into print array", K(tmp_ret)); } else if (MAX_PRINT_COUNT == print_arr.size()) { - LOG_DEBUG("increase tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("increase tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif } #ifndef OB_BUILD_PACKAGE if (0 != print_arr.count()) { - LOG_DEBUG("increase tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("increase tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif @@ -3129,14 +2950,14 @@ int ObTablet::inc_ref_with_macro_iter(ObMacroInfoIterator ¯o_iter, bool &inc } else if (OB_TMP_FAIL(print_arr.push_back(block_info.macro_id_))) { LOG_WARN("fail to push macro id into print array", K(tmp_ret)); } else if (MAX_PRINT_COUNT == print_arr.size()) { - LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif } #ifndef OB_BUILD_PACKAGE if (0 != print_arr.count()) { - LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif @@ -3153,14 +2974,14 @@ int ObTablet::inc_ref_with_macro_iter(ObMacroInfoIterator ¯o_iter, bool &inc const ObTabletComplexAddr &auto_inc_seq_addr = mds_data_->auto_inc_seq_; FLOG_INFO("the tablet that inner increases ref cnt is", - K(ret), K(hold_ref_cnt_), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_, + K(ret), K(hold_ref_cnt_), "tablet_id", tablet_meta_.tablet_id_, K(table_store_addr_.addr_), K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_), K(tablet_addr_), KP(this), K(macro_iter), K(lbt())); } else { FLOG_INFO("the tablet that inner increases ref cnt is", - K(ret), K(hold_ref_cnt_), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_, + K(ret), K(hold_ref_cnt_), "tablet_id", tablet_meta_.tablet_id_, K(table_store_addr_.addr_), K(storage_schema_addr_.addr_), K(tablet_addr_), KP(this), K(macro_iter), K(lbt())); } @@ -3260,7 +3081,7 @@ void ObTablet::dec_macro_ref_cnt() int ret = OB_SUCCESS; if (OB_UNLIKELY(!hold_ref_cnt_)) { FLOG_INFO("tablet doesn't hold ref cnt, no need to dec ref cnt", - K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), + K(is_inited_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), K(tablet_addr_), KP(this), K(lbt())); } else if (OB_FAIL(check_meta_addr())) { LOG_ERROR("fail to check meta addrs", K(ret)); @@ -3334,14 +3155,14 @@ void ObTablet::dec_ref_with_aggregated_info() const ObTabletComplexAddr &medium_info_list_addr = mds_data_->medium_info_list_; const ObTabletComplexAddr &auto_inc_seq_addr = mds_data_->auto_inc_seq_; FLOG_INFO("the tablet that decreases ref cnt is", - K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), + K(is_inited_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_), K(tablet_addr_), KP(this), K(info_iter), K(lbt())); } else { FLOG_INFO("the tablet that decreases ref cnt is", - K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), + K(is_inited_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), K(storage_schema_addr_.addr_), K(tablet_addr_), KP(this), K(info_iter), K(lbt())); } } @@ -3375,7 +3196,7 @@ void ObTablet::dec_ref_with_macro_iter(ObMacroInfoIterator ¯o_iter) const if (OB_TMP_FAIL(print_arr.push_back(block_info.macro_id_))) { LOG_WARN("fail to push macro id into print array", K(tmp_ret)); } else if (MAX_PRINT_COUNT == print_arr.size()) { - LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif @@ -3384,7 +3205,7 @@ void ObTablet::dec_ref_with_macro_iter(ObMacroInfoIterator ¯o_iter) const } #ifndef OB_BUILD_PACKAGE if (0 != print_arr.size()) { - LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(tablet_meta_.ls_id_), K(print_arr)); + LOG_DEBUG("decrease tablet's macro ref", K(ret), K(tablet_meta_.tablet_id_), K(print_arr)); print_arr.reuse(); } #endif @@ -3555,7 +3376,6 @@ void ObTablet::set_initial_addr() int ObTablet::check_meta_addr() const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; if (nullptr != mds_data_) { @@ -3572,7 +3392,7 @@ int ObTablet::check_meta_addr() const || !aux_tablet_info_committed_kv_addr.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("meta addrs are invalid", K(ret), K(ls_id), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), + LOG_WARN("meta addrs are invalid", K(ret), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_)); @@ -3581,7 +3401,7 @@ int ObTablet::check_meta_addr() const || !storage_schema_addr_.addr_.is_valid() || !tablet_addr_.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("meta addrs are invalid", K(ret), K(ls_id), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), + LOG_WARN("meta addrs are invalid", K(ret), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), K(storage_schema_addr_.addr_)); } @@ -3591,7 +3411,7 @@ int ObTablet::check_meta_addr() const && ((tablet_addr_.is_block() ^ table_store_addr_.addr_.is_none()) || (tablet_addr_.is_block() ^ storage_schema_addr_.addr_.is_none()))) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("meta addrs are inconsistent", K(ret), K(ls_id), K(tablet_id), + LOG_ERROR("meta addrs are inconsistent", K(ret), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), K(storage_schema_addr_.addr_)); } @@ -3689,29 +3509,6 @@ void ObTablet::trim_tablet_list() next_tablet_guard_.reset(); } -int ObTablet::deserialize_id( - const char *buf, - const int64_t len, - share::ObLSID &ls_id, - common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - int32_t version = 0; - int32_t length = 0; - int64_t pos = 0; - if (OB_FAIL(serialization::decode_i32(buf, len, pos, (int32_t *)&version))) { - LOG_WARN("fail to deserialize tablet meta's version", K(ret), K(len), K(pos)); - } else if (OB_FAIL(serialization::decode_i32(buf, len, pos, (int32_t *)&length))) { - LOG_WARN("fail to deserialize tablet meta's length", K(ret), K(len), K(pos)); - } else if (ObTabletBlockHeader::TABLET_VERSION_V1 == version || ObTabletBlockHeader::TABLET_VERSION_V2 == version) { - if (OB_FAIL(ObTabletMeta::deserialize_id(buf, len, pos, ls_id, tablet_id))) { - LOG_WARN("fail to deserialize ls_id and tablet_id from tablet meta", K(ret), K(len)); - } - } - - return ret; -} - int ObTablet::get_max_sync_medium_scn(int64_t &max_medium_snapshot) const { int ret = OB_SUCCESS; @@ -4038,7 +3835,6 @@ int ObTablet::auto_get_read_tables( const bool need_split_dst_table) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; iter.table_store_iter_.reset(); bool succ_get_split_src_tables = false; @@ -4063,7 +3859,7 @@ int ObTablet::auto_get_read_tables( ObGetReadTablesMode mode = allow_not_ready ? ObGetReadTablesMode::ALLOW_NO_READY_READ : ObGetReadTablesMode::NORMAL; mode = fork_get_src_tables ? ObGetReadTablesMode::SKIP_MAJOR : mode; if (OB_FAIL(get_read_tables_(snapshot_version, iter.table_store_iter_, iter.table_store_iter_.table_store_handle_, mode))) { - LOG_WARN("failed to get read tables from table store", K(ret), K(ls_id), K(tablet_id), K(fork_get_src_tables), K(mode)); + LOG_WARN("failed to get read tables from table store", K(ret), K(tablet_id), K(fork_get_src_tables), K(mode)); } } if (OB_SUCC(ret) && !succ_get_split_src_tables && need_split_dst_table) { @@ -4078,8 +3874,7 @@ int ObTablet::auto_get_read_tables( #ifdef ENABLE_DEBUG_LOG if (OB_SUCC(ret) && (succ_get_split_src_tables || succ_get_split_dst_tables || fork_get_src_tables)) { - LOG_DEBUG("get read tables during tablet splitting or forking", - K(ls_id), K(tablet_id), K(snapshot_version), + LOG_DEBUG("get read tables during tablet splitting or forking", K(tablet_id), K(snapshot_version), "table_cnt", iter.table_store_iter_.table_ptr_array_.count(), K(succ_get_split_src_tables), K(succ_get_split_dst_tables), K(fork_get_src_tables)); } @@ -4158,26 +3953,25 @@ int ObTablet::get_split_src_read_table_if_need( const ObTabletID &split_src_tablet_id = tablet_meta_.split_info_.get_split_src_tablet_id(); if (OB_UNLIKELY(split_src_tablet_id.is_valid())) { if (OB_UNLIKELY(tablet_meta_.split_info_.is_data_incomplete() && !tablet_meta_.table_store_flag_.with_major_sstable())) { - const ObLSID &ls_id = tablet_meta_.ls_id_; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle src_tablet_handle; ObTablet *src_tablet = nullptr; ObStorageMetaHandle *meta_handle = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(split_src_tablet_id, src_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(split_src_tablet_id, src_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ret), K(split_src_tablet_id), K(ls_id)); + LOG_WARN("failed to get tablet", K(ret), K(split_src_tablet_id)); } else if (OB_ISNULL(src_tablet = src_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet handle obj is nullptr", K(ret)); } else if (OB_UNLIKELY(src_tablet->is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("split src tablet becomes empty shell", K(ret), K(ls_id), K(split_src_tablet_id)); + LOG_WARN("split src tablet becomes empty shell", K(ret), K(split_src_tablet_id)); } else if (OB_FAIL(iter.add_split_extra_tablet_handle(src_tablet_handle))) { LOG_WARN("fail to set split src tabelt handle", K(ret)); } else if (OB_FAIL(iter.table_store_iter_.alloc_split_extra_table_store_handle(meta_handle))) { @@ -4206,16 +4000,16 @@ int ObTablet::get_split_src_read_table_if_need( LOG_WARN("failed to get split data", K(ret)); } else if (data.is_split_dst()) { ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle src_tablet_handle; ObTablet *src_tablet = nullptr; ObStorageMetaHandle *meta_handle = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(tablet_meta_.ls_id_)); - } else if (OB_FAIL(data.get_split_src_tablet(*ls_handle.get_ls(), src_tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(data.get_split_src_tablet(*tenant_ls, src_tablet_handle))) { LOG_WARN("failed to get split src tablet", K(ret)); } else if (OB_ISNULL(src_tablet = src_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -4262,13 +4056,13 @@ int ObTablet::get_split_dst_read_table( if (OB_UNLIKELY(maybe_split_src)) { ObTabletSplitMdsUserData data; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; const int64_t timeout = ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(tablet_meta_.ls_id_)); + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else if (OB_FAIL(ObITabletMdsInterface::get_split_data(data, timeout))) { LOG_WARN("failed to get split data", K(ret)); } else if (data.is_split_src()) { @@ -4281,9 +4075,9 @@ int ObTablet::get_split_dst_read_table( const ObTabletID &dst_tablet_id = dst_tablet_ids.at(i); ObTablet *dst_tablet = nullptr; ObStorageMetaHandle *meta_handle = nullptr; - if (OB_FAIL(ls_handle.get_ls()->get_tablet(dst_tablet_ids.at(i), dst_tablet_handle, timeout, + if (OB_FAIL(tenant_ls->get_tablet(dst_tablet_ids.at(i), dst_tablet_handle, timeout, ObMDSGetTabletMode::READ_READABLE_COMMITED))) { - LOG_WARN("failed to get tablet", K(ret), K(dst_tablet_ids.at(i)), K(tablet_meta_.ls_id_)); + LOG_WARN("failed to get tablet", K(ret), K(dst_tablet_ids.at(i))); } else if (OB_FAIL(iter.add_split_extra_tablet_handle(dst_tablet_handle))) { LOG_WARN("fail to set split src tablet handle", K(ret)); } else if (OB_FAIL(iter.table_store_iter_.alloc_split_extra_table_store_handle(meta_handle))) { @@ -4332,15 +4126,15 @@ int ObTablet::get_split_src_major_table_if_need( LOG_WARN("failed to get split data", K(ret)); } else if (data.is_split_dst()) { ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle src_tablet_handle; ObTablet *src_tablet = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(tablet_meta_.ls_id_)); - } else if (OB_FAIL(data.get_split_src_tablet(*ls_handle.get_ls(), src_tablet_handle))) { + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(data.get_split_src_tablet(*tenant_ls, src_tablet_handle))) { LOG_WARN("failed to get split src tablet", K(ret)); } else if (OB_ISNULL(src_tablet = src_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -4364,25 +4158,24 @@ int ObTablet::get_fork_src_read_tables_( { int ret = OB_SUCCESS; const ObTabletID &src_tablet_id = tablet_meta_.fork_info_.get_fork_src_tablet_id(); - const ObLSID &ls_id = tablet_meta_.ls_id_; ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle src_tablet_handle; ObTablet *src_tablet = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(src_tablet_id, src_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet(src_tablet_id, src_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ret), K(src_tablet_id), K(ls_id)); + LOG_WARN("failed to get tablet", K(ret), K(src_tablet_id)); } else if (OB_ISNULL(src_tablet = src_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet handle obj is nullptr", K(ret)); } else if (OB_UNLIKELY(src_tablet->is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("src tablet becomes empty shell", K(ret), K(ls_id), K(src_tablet_id)); + LOG_WARN("src tablet becomes empty shell", K(ret), K(src_tablet_id)); } else if (OB_FAIL(iter.add_fork_tablet_handle(src_tablet_handle, tablet_meta_.fork_info_))) { LOG_WARN("fail to set fork src tabelt handle", K(ret)); } else if (OB_FAIL(src_tablet->auto_get_read_tables( @@ -4393,7 +4186,7 @@ int ObTablet::get_fork_src_read_tables_( false))) { LOG_WARN("failed to get read tables from src tablet", K(ret), KPC(src_tablet)); } else { - LOG_DEBUG("get fork src read tables success", K(tablet_meta_.fork_info_), "src_tablet_id", src_tablet_id, K(ls_id)); + LOG_DEBUG("get fork src read tables success", K(tablet_meta_.fork_info_), "src_tablet_id", src_tablet_id); } return ret; @@ -4788,7 +4581,6 @@ int ObTablet::prepare_memtable( int ObTablet::get_meta_disk_addr(ObMetaDiskAddr &addr) const { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet_meta_.ls_id_; const ObTabletID &tablet_id = tablet_meta_.tablet_id_; ObTabletPointer *tablet_ptr = nullptr; @@ -4803,7 +4595,7 @@ int ObTablet::get_meta_disk_addr(ObMetaDiskAddr &addr) const } } else if (OB_ISNULL(tablet_ptr = static_cast(pointer_hdl_.get_resource_ptr()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet pointer is null", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet pointer is null", K(ret), K(tablet_id)); } else { addr = tablet_ptr->get_addr(); } @@ -4980,7 +4772,6 @@ int ObTablet::get_active_memtable(ObTableHandleV2 &handle) const int ObTablet::create_memtable(CreateMemtableArg &arg) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; ObTimeGuard time_guard("ObTablet::create_memtable", 10 * 1000); common::SpinWLockGuard guard(memtables_lock_); @@ -5024,7 +4815,7 @@ int ObTablet::create_memtable(CreateMemtableArg &arg) LOG_WARN("table store update memtables fail", K(ret), K(memtable_array)); } else { time_guard.click("ts update mem"); - LOG_INFO("table store update memtable success", K(ret), K(ls_id), K(tablet_id), K_(table_store_addr), KP(this)); + LOG_INFO("table store update memtable success", K(ret), K(tablet_id), K_(table_store_addr), KP(this)); } } } @@ -5041,7 +4832,6 @@ int ObTablet::create_memtable(CreateMemtableArg &arg) int ObTablet::inner_create_memtable(CreateMemtableArg &arg) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; arg.new_clog_checkpoint_scn_ = tablet_meta_.clog_checkpoint_scn_; ObProtectedMemtableMgrHandle *protected_handle = NULL; @@ -5059,10 +4849,10 @@ int ObTablet::inner_create_memtable(CreateMemtableArg &arg) LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(this)); } else if (OB_FAIL(protected_handle->create_memtable(tablet_meta_, arg))) { if (OB_ENTRY_EXIST != ret && OB_MINOR_FREEZE_NOT_ALLOW != ret) { - LOG_WARN("failed to create memtable", K(ret), K(ls_id), K(tablet_id), KPC(this)); + LOG_WARN("failed to create memtable", K(ret), K(tablet_id), KPC(this)); } } else { - LOG_INFO("succeeded to create memtable for tablet", K(ret), K(tablet_id), K(ls_id), K(arg)); + LOG_INFO("succeeded to create memtable for tablet", K(ret), K(tablet_id), K(arg)); } return ret; @@ -5222,7 +5012,7 @@ int ObTablet::get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_creat ObTabletPointer *tablet_ptr = static_cast(pointer_hdl_.get_resource_ptr()); if (try_create) { bool is_created = false; - if (OB_FAIL(tablet_ptr->create_ddl_kv_mgr(tablet_meta_.ls_id_, tablet_meta_.tablet_id_, ddl_kv_mgr_handle, is_created))) { + if (OB_FAIL(tablet_ptr->create_ddl_kv_mgr(tablet_meta_.tablet_id_, ddl_kv_mgr_handle, is_created))) { LOG_WARN("create ddl kv mgr failed", K(ret), K(tablet_meta_)); } else if (is_created) { ddl_kv_mgr_handle.get_obj()->set_max_freeze_scn(tablet_meta_.ddl_checkpoint_scn_); @@ -5239,12 +5029,12 @@ int ObTablet::get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_creat } int ObTablet::init_shared_params( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const lib::Worker::CompatMode compat_mode) { int ret = OB_SUCCESS; ObProtectedMemtableMgrHandle *protected_handle = NULL; + ObLS *ls = NULL; if (OB_UNLIKELY(!pointer_hdl_.is_valid())) { ret = OB_ERR_UNEXPECTED; @@ -5254,11 +5044,21 @@ int ObTablet::init_shared_params( } } else if (!tablet_id.is_ls_inner_tablet()) { // tablet_memtable_mgr init in ObProtectedMemtableMgrHandle + } else if (OB_ISNULL(ls = static_cast( + pointer_hdl_.get_resource_ptr())->get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet has no log stream", K(ret), K(tablet_id)); } else { if (OB_FAIL(get_protected_memtable_mgr_handle(protected_handle))) { LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(this)); - } else if (OB_FAIL(protected_handle->init(ls_id, tablet_id, compat_mode))) { - LOG_WARN("failed to init memtable mgr", K(ret), K(tablet_id), K(ls_id)); + } else if (OB_FAIL(protected_handle->init(tablet_id, + 0 /* max_saved_schema_version */, + 0 /* max_saved_medium_scn */, + compat_mode, + ls->get_log_handler(), + ls->get_freezer(), + share::g_mp->tenant_meta_mem_mgr()))) { + LOG_WARN("failed to init memtable mgr", K(ret), K(tablet_id)); } } return ret; @@ -5323,151 +5123,9 @@ int ObTablet::try_update_table_store_flag(const bool with_major) return ret; } -int ObTablet::build_migration_tablet_param( - ObMigrationTabletParam &mig_tablet_param) const -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else { - // allocator - mig_tablet_param.allocator_.set_attr(ObMemAttr("MigTabletParam", ObCtxIds::DEFAULT_CTX_ID)); - - mig_tablet_param.ls_id_ = tablet_meta_.ls_id_; - mig_tablet_param.tablet_id_ = tablet_meta_.tablet_id_; - mig_tablet_param.data_tablet_id_ = tablet_meta_.data_tablet_id_; - mig_tablet_param.ref_tablet_id_ = tablet_meta_.ref_tablet_id_; - mig_tablet_param.create_scn_ = tablet_meta_.create_scn_; - mig_tablet_param.create_schema_version_ = tablet_meta_.create_schema_version_; - mig_tablet_param.micro_index_clustered_ = tablet_meta_.micro_index_clustered_; - mig_tablet_param.start_scn_ = tablet_meta_.start_scn_; - mig_tablet_param.clog_checkpoint_scn_ = tablet_meta_.clog_checkpoint_scn_; - mig_tablet_param.snapshot_version_ = tablet_meta_.snapshot_version_; - mig_tablet_param.multi_version_start_ = tablet_meta_.multi_version_start_; - mig_tablet_param.compat_mode_ = tablet_meta_.compat_mode_; - mig_tablet_param.ha_status_ = tablet_meta_.ha_status_; - mig_tablet_param.table_store_flag_ = tablet_meta_.table_store_flag_; - mig_tablet_param.ddl_checkpoint_scn_ = tablet_meta_.ddl_checkpoint_scn_; - mig_tablet_param.ddl_start_scn_ = tablet_meta_.ddl_start_scn_; - mig_tablet_param.ddl_snapshot_version_ = tablet_meta_.ddl_snapshot_version_; - mig_tablet_param.max_sync_storage_schema_version_ = tablet_meta_.max_sync_storage_schema_version_; - mig_tablet_param.max_serialized_medium_scn_ = tablet_meta_.max_serialized_medium_scn_; - // max_version on tablet meta is the latest serialized version - mig_tablet_param.ddl_execution_id_ = tablet_meta_.ddl_execution_id_; - mig_tablet_param.ddl_data_format_version_ = tablet_meta_.ddl_data_format_version_; - mig_tablet_param.ddl_commit_scn_ = tablet_meta_.ddl_commit_scn_; - mig_tablet_param.report_status_ = tablet_meta_.report_status_; - mig_tablet_param.mds_checkpoint_scn_ = tablet_meta_.mds_checkpoint_scn_; - mig_tablet_param.is_empty_shell_ = is_empty_shell(); - mig_tablet_param.split_info_ = tablet_meta_.split_info_; - mig_tablet_param.has_truncate_info_ = tablet_meta_.has_truncate_info_; - mig_tablet_param.fork_info_ = tablet_meta_.fork_info_; - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(build_migration_tablet_param_storage_schema(mig_tablet_param))) { - LOG_WARN("fail to build storage schema", K(ret)); - } else if (OB_FAIL(build_migration_tablet_param_major_ckm_info(mig_tablet_param))) { - LOG_WARN("fail to build major ckm info", K(ret)); - } - - if (OB_FAIL(ret)) { - } else { - mig_tablet_param.version_ = ObMigrationTabletParam::PARAM_VERSION_V3; - mig_tablet_param.extra_medium_info_.info_ = tablet_meta_.extra_medium_info_.info_; - mig_tablet_param.extra_medium_info_.last_medium_scn_ = tablet_meta_.extra_medium_info_.last_medium_scn_; - if (OB_FAIL(build_migration_tablet_param_last_tablet_status(mig_tablet_param))) { - LOG_WARN("fail to assign mig tablet param from tablet meta", - K(ret), K(tablet_meta_.last_persisted_committed_tablet_status_)); - } else { - LOG_INFO("build migration tablet param successfully", K(mig_tablet_param)); - } - } - } - return ret; -} - -int ObTablet::build_migration_tablet_param_last_tablet_status( - ObMigrationTabletParam &mig_tablet_param) const -{ - int ret = OB_SUCCESS; - mds::MdsWriter writer; - mds::TwoPhaseCommitState trans_stat; - share::SCN trans_version; - ObTabletCreateDeleteMdsUserData user_data; - if (OB_FAIL(ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { - LOG_WARN("failed to get tx data", K(ret), KPC(this)); - } else if (ObTabletStatus::RESERVED_5 == user_data.tablet_status_) { - if (trans_stat == mds::TwoPhaseCommitState::ON_COMMIT) { - if (OB_FAIL(mig_tablet_param.last_persisted_committed_tablet_status_.assign(user_data))) { - LOG_WARN("fail to assign mig tablet param from tablet meta", K(ret), K(user_data)); - } - } else { - mig_tablet_param.last_persisted_committed_tablet_status_.reset(); - } - } else { - if (OB_FAIL(mig_tablet_param.last_persisted_committed_tablet_status_.assign( - tablet_meta_.last_persisted_committed_tablet_status_))) { - LOG_WARN("fail to assign mig tablet param from tablet meta", - K(ret), K(tablet_meta_.last_persisted_committed_tablet_status_)); - } - } - return ret; -} - -int ObTablet::build_migration_tablet_param_storage_schema( - ObMigrationTabletParam &mig_tablet_param) const -{ - int ret = OB_SUCCESS; - ObArenaAllocator arena_allocator(common::ObMemAttr("BuildMigParam")); - ObStorageSchema *storage_schema = nullptr; - if (is_empty_shell()) { - // do nothing - } else if (OB_FAIL(load_storage_schema(arena_allocator, storage_schema))) { - LOG_WARN("fail to load storage schema", K(ret)); - } else if (OB_FAIL(mig_tablet_param.storage_schema_.init(mig_tablet_param.allocator_, *storage_schema))) { - LOG_WARN("failed to copy storage schema", K(ret), KPC(storage_schema)); - } - ObTabletObjLoadHelper::free(arena_allocator, storage_schema); - return ret; -} - -int ObTablet::build_migration_tablet_param_major_ckm_info( - ObMigrationTabletParam &mig_tablet_param) const -{ - int ret = OB_SUCCESS; - ObTabletMemberWrapper table_store_wrapper; - const ObTabletTableStore *table_store = nullptr; - const ObTabletID &tablet_id = tablet_meta_.tablet_id_; - - if (is_empty_shell()) { - // do nothing - } else if (OB_FAIL(fetch_table_store(table_store_wrapper))) { - LOG_WARN("fail to fetch table store", K(ret)); - } else if (OB_FAIL(table_store_wrapper.get_member(table_store))) { - LOG_WARN("fail to get table store", K(ret), K(tablet_id), K(table_store_wrapper)); - } else { - const ObMajorChecksumInfo &major_ckm_info = table_store->get_major_ckm_info(); - if (OB_UNLIKELY(!major_ckm_info.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("major ckm info on table store is invalid", K(ret), K(tablet_id), K(major_ckm_info)); - } else if (major_ckm_info.is_empty() || is_output_exec_mode(major_ckm_info.get_exec_mode())) { - // do not copy OUTPUT major ckm info to migrate or backup -#ifdef ERRSIM - SERVER_EVENT_SYNC_ADD("ss_merge_errsim", "src_is_exec_svr", "ls_id", tablet_meta_.ls_id_); -#endif - } else if (OB_FAIL(mig_tablet_param.major_ckm_info_.assign(major_ckm_info, &mig_tablet_param.allocator_))) { - LOG_WARN("failed to assign major ckm info from table store", K(ret), K(tablet_id), K(table_store)); - } - } - return ret; -} - -int ObTablet::build_migration_sstable_param( +int ObTablet::build_fork_sstable_param( const ObITable::TableKey &table_key, - blocksstable::ObMigrationSSTableParam &mig_sstable_param, - const bool is_fork_table) const + blocksstable::ObForkSSTableParam &fork_sstable_param) const { int ret = OB_SUCCESS; ObTableHandleV2 handle; @@ -5496,162 +5154,38 @@ int ObTablet::build_migration_sstable_param( LOG_WARN("fail to get sstable meta handle", K(ret), KPC(sstable)); } else { const ObSSTableMeta &sstable_meta = sstable_meta_handle.get_sstable_meta(); - mig_sstable_param.basic_meta_ = sstable_meta.get_basic_meta(); - mig_sstable_param.table_key_ = table_key; - mig_sstable_param.is_small_sstable_ = sstable->is_small_sstable(); - mig_sstable_param.table_key_ = sstable->get_key(); - mig_sstable_param.is_meta_root_ = sstable_meta.get_macro_info().is_meta_root(); + fork_sstable_param.basic_meta_ = sstable_meta.get_basic_meta(); + fork_sstable_param.is_meta_root_ = sstable_meta.get_macro_info().is_meta_root(); for (int64_t i = 0; OB_SUCC(ret) && i < sstable_meta.get_col_checksum_cnt(); ++i) { - if (OB_FAIL(mig_sstable_param.column_checksums_.push_back(sstable_meta.get_col_checksum()[i]))) { + if (OB_FAIL(fork_sstable_param.column_checksums_.push_back(sstable_meta.get_col_checksum()[i]))) { LOG_WARN("fail to push back column checksum", K(ret), K(i)); } } if (OB_SUCC(ret)) { - if (sstable_meta.is_shared_table() || is_fork_table) { - // data block info & meta block info - mig_sstable_param.is_meta_root_ = sstable_meta.get_macro_info().is_meta_root(); - if (OB_FAIL(build_migration_shared_table_addr_( - sstable_meta.get_root_info(), mig_sstable_param.allocator_, mig_sstable_param.root_block_addr_, - mig_sstable_param.root_block_buf_))) { - LOG_WARN("failed to build migration root info addr", K(ret), K(sstable_meta)); - } else if (OB_FAIL(build_migration_shared_table_addr_( - sstable_meta.get_macro_info().get_macro_meta_info(), mig_sstable_param.allocator_, - mig_sstable_param.data_block_macro_meta_addr_, mig_sstable_param.data_block_macro_meta_buf_))) { - LOG_WARN("failed to build migration macro info addr", K(ret), K(sstable_meta)); - } + fork_sstable_param.is_meta_root_ = sstable_meta.get_macro_info().is_meta_root(); + if (OB_FAIL(build_fork_shared_table_addr_( + sstable_meta.get_root_info(), fork_sstable_param.allocator_, fork_sstable_param.root_block_addr_, + fork_sstable_param.root_block_buf_))) { + LOG_WARN("failed to build fork root info addr", K(ret), K(sstable_meta)); + } else if (OB_FAIL(build_fork_shared_table_addr_( + sstable_meta.get_macro_info().get_macro_meta_info(), fork_sstable_param.allocator_, + fork_sstable_param.data_block_macro_meta_addr_, fork_sstable_param.data_block_macro_meta_buf_))) { + LOG_WARN("failed to build fork macro info addr", K(ret), K(sstable_meta)); } } } if (OB_FAIL(ret)) { - mig_sstable_param.reset(); - } else { - STORAGE_LOG(INFO, "succeed to build migration param", - K(mig_sstable_param), K(sstable_meta_handle.get_sstable_meta())); - } - return ret; -} - -int ObTablet::build_migration_sstable_param( - const ObITable::TableKey &table_key, - const blocksstable::ObSSTableMergeRes &res, - blocksstable::ObMigrationSSTableParam &mig_sstable_param) const -{ - int ret = OB_SUCCESS; - ObTableHandleV2 handle; - ObTabletMemberWrapper table_store_wrapper; - ObTableStoreIterator iter; - ObSSTableMetaHandle sstable_meta_handle; - ObSSTable *sstable = nullptr; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(!table_key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(table_key)); - } else if (OB_FAIL(fetch_table_store(table_store_wrapper))) { - LOG_WARN("fail to fetch table store", K(ret)); - } else if (OB_FAIL(table_store_wrapper.get_member()->get_table( - table_store_wrapper.get_meta_handle(), table_key, handle))) { - LOG_WARN("fail to get table from table store", K(ret), K(table_key)); - } else if (OB_FAIL(handle.get_sstable(sstable))) { - LOG_WARN("fail to get sstable", K(ret), KPC(sstable)); - } else if (OB_ISNULL(sstable)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", K(ret), KPC(sstable)); - } else if (OB_FAIL(sstable->get_meta(sstable_meta_handle))) { - LOG_WARN("fail to get sstable meta handle", K(ret), KPC(sstable)); - } else { - const ObSSTableMeta &sstable_meta = sstable_meta_handle.get_sstable_meta(); - - mig_sstable_param.basic_meta_ = sstable_meta.get_basic_meta(); - mig_sstable_param.table_key_ = table_key; - mig_sstable_param.is_small_sstable_ = sstable->is_small_sstable(); - mig_sstable_param.table_key_ = sstable->get_key(); - - // fill ObMigrationSSTableParam and update basic meta from merge res - ObSSTableBasicMeta &basic_meta = mig_sstable_param.basic_meta_; - mig_sstable_param.root_block_addr_ = res.root_desc_.addr_; - mig_sstable_param.root_block_buf_ = res.root_desc_.buf_; - mig_sstable_param.data_block_macro_meta_addr_ = res.data_root_desc_.addr_; - mig_sstable_param.data_block_macro_meta_buf_ = res.data_root_desc_.buf_; - mig_sstable_param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - basic_meta.root_row_store_type_ = res.root_row_store_type_; - basic_meta.data_index_tree_height_ = res.root_desc_.height_; - basic_meta.index_macro_block_count_ = res.index_blocks_cnt_; - basic_meta.data_macro_block_count_ = res.data_blocks_cnt_; - basic_meta.data_micro_block_count_ = res.micro_block_cnt_; - basic_meta.use_old_macro_block_count_ = res.use_old_macro_block_count_; - basic_meta.row_count_ = res.row_count_; - basic_meta.column_cnt_ = res.data_column_cnt_; - basic_meta.data_checksum_ = res.data_checksum_; - basic_meta.occupy_size_ = res.occupy_size_; - basic_meta.original_size_ = res.original_size_; - basic_meta.max_merged_trans_version_ = res.max_merged_trans_version_; - basic_meta.contain_uncommitted_row_ = res.contain_uncommitted_row_; - basic_meta.compressor_type_ = res.compressor_type_; - basic_meta.encrypt_id_ = res.encrypt_id_; - basic_meta.master_key_id_ = res.master_key_id_; - - for (int64_t i = 0; OB_SUCC(ret) && i < sstable_meta.get_col_checksum_cnt(); ++i) { - if (OB_FAIL(mig_sstable_param.column_checksums_.push_back(sstable_meta.get_col_checksum()[i]))) { - LOG_WARN("fail to push back column checksum", K(ret), K(i)); - } - } - if (FAILEDx(ObSSTableMetaChecker::check_sstable_basic_meta(sstable_meta.get_basic_meta(), - mig_sstable_param.basic_meta_))) { - STORAGE_LOG(WARN, "fail to check basic meta", - K(ret), K(sstable_meta.get_basic_meta()), K(mig_sstable_param.basic_meta_)); - } - } - if (OB_FAIL(ret)) { - mig_sstable_param.reset(); + fork_sstable_param.reset(); } else { - STORAGE_LOG(INFO, "succeed to build migration param with merge result", - K(mig_sstable_param), K(res), K(sstable_meta_handle.get_sstable_meta())); + STORAGE_LOG(INFO, "succeed to build fork sstable param", + K(fork_sstable_param), K(sstable_meta_handle.get_sstable_meta())); } return ret; } -int ObTablet::get_ha_sstable_size(int64_t &data_size) -{ - int ret = OB_SUCCESS; - data_size = 0; - ObTabletMemberWrapper table_store_wrapper; - ObTableStoreIterator iter; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(fetch_table_store(table_store_wrapper))) { - LOG_WARN("fail to fetch table store", K(ret)); - } else if (OB_FAIL(table_store_wrapper.get_member()->get_ha_tables(iter))) { - LOG_WARN("failed to get read tables", K(ret)); - } else { - while (OB_SUCC(ret)) { - ObITable *table = nullptr; - ObSSTable *sstable = nullptr; - if (OB_FAIL(iter.get_next(table))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get read tables", K(ret), K(*this)); - } - } else if (table->is_memtable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get migration get memtable", K(ret), KPC(table)); - } else if (FALSE_IT(sstable = static_cast (table))) { - } else { - data_size += sstable->get_occupy_size(); - } - } - } - return ret; -} int ObTablet::fetch_tablet_autoinc_seq_cache( const uint64_t cache_size, @@ -5711,7 +5245,6 @@ int ObTablet::get_kept_snapshot_info( int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; snapshot_info.reset(); - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; const int64_t last_major_snapshot_version = get_last_major_snapshot_version(); @@ -5730,12 +5263,11 @@ int ObTablet::get_kept_snapshot_info( SMART_VARS_2((ObTableScanParam, scan_param), (ObTabletMediumInfoReader, medium_info_reader)) { if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, tablet_id, ObMdsScanParamHelper::get_whole_read_version_range(), unused_collector, scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(tablet_id)); } else if (OB_FAIL(medium_info_reader.init(*this, scan_param))) { LOG_WARN("failed to init medium info reader", K(ret)); } else if (need_check_medium_info) { @@ -5773,11 +5305,7 @@ int ObTablet::get_kept_snapshot_info( old_snapshot_info = snapshot_info; bool use_multi_version_start_on_tablet = false; - if (!tablet_meta_.ha_status_.is_data_status_complete()) { - use_multi_version_start_on_tablet = true; - } else if (ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV == snapshot_info.snapshot_type_ && 0 == snapshot_info.snapshot_) { - use_multi_version_start_on_tablet = true; // if exist new mv in restore, use special snapshot info - } else if (min_reserved_snapshot_on_ls > 0) { + if (min_reserved_snapshot_on_ls > 0) { snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_LS_RESERVED, min_reserved_snapshot_on_ls); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_MIN_MEDIUM, min_medium_snapshot); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_SPLIT, min_split_snapshot); @@ -5798,7 +5326,7 @@ int ObTablet::get_kept_snapshot_info( if (current_time - (snapshot_info.snapshot_ / 1000 /*use microsecond here*/) > 40_min) { if (REACH_THREAD_TIME_INTERVAL(10_s)) { LOG_INFO("tablet multi version start not advance for a long time", K(ret), - "ls_id", get_tablet_meta().ls_id_, K(tablet_id), + K(tablet_id), K(snapshot_info), K(old_snapshot_info), K(min_medium_snapshot), K(min_reserved_snapshot_on_ls)); } @@ -5808,7 +5336,7 @@ int ObTablet::get_kept_snapshot_info( ret = OB_ERR_UNEXPECTED; LOG_WARN("snapshot info is invalid", KR(ret), K(snapshot_info)); } - LOG_TRACE("get multi version start", "ls_id", get_tablet_meta().ls_id_, K(tablet_id), + LOG_TRACE("get multi version start", K(tablet_id), K(snapshot_info), K(old_snapshot_info), K(min_medium_snapshot), K(min_reserved_snapshot_on_ls), K(max_merged_snapshot)); return ret; @@ -5817,7 +5345,6 @@ int ObTablet::get_kept_snapshot_info( int ObTablet::get_kept_snapshot_for_split(int64_t &min_split_snapshot) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; min_split_snapshot = INT64_MAX; if (!is_ls_inner_tablet()) { @@ -5829,7 +5356,7 @@ int ObTablet::get_kept_snapshot_for_split(int64_t &min_split_snapshot) const if (OB_EMPTY_RESULT == ret) { ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tx data", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tx data", K(ret), K(tablet_id)); } } else if (ObTabletStatus::SPLIT_SRC == user_data.get_tablet_status()) { if (mds::TwoPhaseCommitState::ON_PREPARE == trans_stat) { @@ -5850,7 +5377,6 @@ int ObTablet::write_sync_tablet_seq_log(ObTabletAutoincSeq &autoinc_seq, int ret = OB_SUCCESS; const int64_t WAIT_TIME = 1000; // 1ms const int64_t SYNC_TABLET_SEQ_LOG_TIMEOUT = 1000L * 1000L * 30L; // 30s - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; const enum ObReplayBarrierType replay_barrier_type = is_tablet_creating ? logservice::ObReplayBarrierType::PRE_BARRIER : logservice::ObReplayBarrierType::NO_NEED_BARRIER; @@ -5879,7 +5405,7 @@ int ObTablet::write_sync_tablet_seq_log(ObTabletAutoincSeq &autoinc_seq, LOG_WARN("failed to serialize log base header", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to serialize sync tablet seq log", K(ret)); - } else if (OB_FAIL(cb->init(tablet_meta_.ls_id_, tablet_id, static_cast(new_autoinc_seq)))) { + } else if (OB_FAIL(cb->init(tablet_id, static_cast(new_autoinc_seq)))) { LOG_WARN("failed to init cb", K(ret), K(tablet_meta_)); } else if (OB_FAIL(set(std::move(autoinc_seq), cb->get_mds_ctx(), THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_remain() : OB_DEFAULT_RPC_TIMEOUT))) { LOG_WARN("failed to set mds", K(ret)); @@ -5915,7 +5441,7 @@ int ObTablet::write_sync_tablet_seq_log(ObTabletAutoincSeq &autoinc_seq, LOG_WARN("submit sync tablet seq log failed", K(ret)); } else { int64_t wait_time = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("submit sync tablet seq log succeed", K(ret), K(ls_id), K(tablet_id), + LOG_INFO("submit sync tablet seq log succeed", K(ret), K(tablet_id), K(new_autoinc_seq), K(lsn), K(scn), K(wait_time)); } if (nullptr != cb) { @@ -5985,22 +5511,19 @@ int ObTablet::check_is_delete_insert_table(bool &is_delete_insert_table) const return ret; } -int ObTablet::start_direct_load_task_for_idem(ObLSHandle &ls_handle) +int ObTablet::start_direct_load_task_for_idem(ObLS *tenant_ls) { int ret = OB_SUCCESS; ObTabletMemberWrapper table_store_wrapper; ObDDLKvMgrHandle ddl_kv_mgr_handle; - if (!ls_handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls handle is not valid", K(ret)); - } else if (OB_FAIL(fetch_table_store(table_store_wrapper))) { + if (OB_FAIL(fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (nullptr != table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(false/*first*/)) { // the major sstable has already existed. LOG_DEBUG("no need to reload tablet direct load task", K(ret), K(tablet_meta_)); } else if (OB_FAIL(get_ddl_kv_mgr(ddl_kv_mgr_handle, true /* try create */))) { LOG_WARN("failed to get ddl kv mgr", K(ret), K(tablet_meta_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(tablet_meta_.tablet_id_))) { + } else if (OB_FAIL(tenant_ls->get_ddl_log_handler()->add_tablet(tablet_meta_.tablet_id_))) { LOG_WARN("add tablet id failed", K(ret), K(tablet_meta_.tablet_id_)); } return ret; @@ -6009,15 +5532,15 @@ int ObTablet::start_direct_load_task_for_idem(ObLSHandle &ls_handle) int ObTablet::start_direct_load_task_if_need() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; if (is_empty_shell()) { LOG_DEBUG("this tablet is empty shell, skip", K(ret), K(tablet_meta_)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), "ls_id", tablet_meta_.ls_id_); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { /* process for data format version > idem type */ if (DDL_IDEM_DATA_FORMAT_VERSION <= tablet_meta_.ddl_data_format_version_) { - if (OB_FAIL(start_direct_load_task_for_idem(ls_handle))) { + if (OB_FAIL(start_direct_load_task_for_idem(tenant_ls))) { LOG_WARN("failed to start direct load task for idem", K(ret), K(tablet_meta_)); } } @@ -6032,8 +5555,6 @@ int ObTablet::check_schema_version_elapsed( transaction::ObTransID &pending_tx_id) { int ret = OB_SUCCESS; - - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; ObMultiVersionSchemaService *schema_service = share::g_mp->tenant_schema_service()->get_schema_service(); SCN scn; @@ -6083,20 +5604,20 @@ int ObTablet::check_schema_version_elapsed( if (OB_SUCC(ret)) { transaction::ObTransService *txs = share::g_mp->trans_service(); ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; - if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { - LOG_WARN("failed to get ls", K(ret), "ls_id", tablet_meta_.ls_id_); - } else if (OB_FAIL(ls_handle.get_ls()->check_modify_schema_elapsed(tablet_id, schema_version, pending_tx_id))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(tenant_ls->check_modify_schema_elapsed(tablet_id, schema_version, pending_tx_id))) { if (OB_EAGAIN != ret) { - LOG_WARN("check schema version elapsed failed", K(ret), K(ls_id), K(tablet_id), K(schema_version)); + LOG_WARN("check schema version elapsed failed", K(ret), K(tablet_id), K(schema_version)); } else { - LOG_INFO("check schema version elapsed again", K(ret), K(ls_id), K(tablet_id), K(schema_version), K(refreshed_schema_ts)); + LOG_INFO("check schema version elapsed again", K(ret), K(tablet_id), K(schema_version), K(refreshed_schema_ts)); } } else if (OB_FAIL(txs->get_max_commit_version(max_commit_scn))) { LOG_WARN("fail to get max commit version", K(ret)); } else { max_commit_version = max_commit_scn.get_val_for_tx(); - LOG_INFO("check wait trans end", K(ret), K(ls_id), K(tablet_id), K(max_commit_version), K(max_commit_scn), K(refreshed_schema_ts)); + LOG_INFO("check wait trans end", K(ret), K(tablet_id), K(max_commit_version), K(max_commit_scn), K(refreshed_schema_ts)); } } } @@ -6108,7 +5629,6 @@ int ObTablet::write_tablet_schema_version_change_clog( SCN &scn) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; ObTabletSchemaVersionChangeLog log; if (OB_FAIL(log.init(tablet_id, schema_version))) { @@ -6170,12 +5690,12 @@ int ObTablet::write_tablet_schema_version_change_clog( } if (wait_timeout) { ret = OB_TIMEOUT; - LOG_WARN("submit schema version change log timeout", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("submit schema version change log timeout", K(ret), K(tablet_id)); } else if (tmp_cb->is_failed()) { ret = OB_NOT_MASTER; - LOG_WARN("submit schema version change log failed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("submit schema version change log failed", K(ret), K(tablet_id)); } else { - LOG_INFO("submit schema version change log succeed", K(ret), K(ls_id), K(tablet_id), K(schema_version)); + LOG_INFO("submit schema version change log succeed", K(ret), K(tablet_id), K(schema_version)); } tmp_cb->try_release(); // release the memory no matter succ or not } @@ -6260,7 +5780,6 @@ int ObTablet::get_tablet_report_info_by_ckm_info( const int64_t report_major_snapshot = major_ckm_info.get_compaction_scn(); if (OB_FAIL(tablet_replica.init( get_tablet_id(), - get_ls_id(), addr, report_major_snapshot, 0/*data_size*/, @@ -6283,7 +5802,6 @@ int ObTablet::get_tablet_report_info_by_ckm_info( LOG_WARN("failed to get freeze info", K(ret), K(report_major_snapshot)); } } else { - tablet_checksum.ls_id_ = get_ls_id(); tablet_checksum.tablet_id_ = get_tablet_id(); tablet_checksum.server_ = addr; tablet_checksum.row_count_ = major_ckm_info.get_row_count(); @@ -6334,7 +5852,6 @@ int ObTablet::get_tablet_report_info_by_sstable( } if (FAILEDx(tablet_replica.init( get_tablet_id(), - get_ls_id(), addr, report_major_snapshot, data_size, @@ -6357,7 +5874,6 @@ int ObTablet::get_tablet_report_info_by_sstable( LOG_WARN("failed to get freeze info", K(ret), K(report_major_snapshot)); } } else { - tablet_checksum.ls_id_ = get_ls_id(); tablet_checksum.tablet_id_ = get_tablet_id(); tablet_checksum.server_ = addr; tablet_checksum.row_count_ = get_tablet_meta().report_status_.row_count_; @@ -6479,25 +5995,6 @@ int ObTablet::get_recycle_version(const int64_t multi_version_start, int64_t &re return ret; } -int ObTablet::get_ha_tables(ObTableStoreIterator &iter) -{ - int ret = OB_SUCCESS; - ObTabletMemberWrapper table_store_wrapper; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(fetch_table_store(table_store_wrapper))) { - LOG_WARN("fail to fetch table store", K(ret)); - } else if (OB_FAIL(table_store_wrapper.get_member()->get_ha_tables(iter))) { - LOG_WARN("fail to get ha tables", K(ret)); - } else if (!table_store_addr_.is_memory_object() - && OB_FAIL(iter.set_handle(table_store_wrapper.get_meta_handle()))) { - LOG_WARN("fail to set storage meta handle", K(ret), K_(table_store_addr), K(table_store_wrapper)); - } - - return ret; -} - int ObTablet::update_ddl_info( const int64_t schema_version, const SCN &scn, @@ -6542,7 +6039,6 @@ int ObTablet::get_ddl_info(int64_t &schema_version, int64_t &schema_refreshed_ts int ObTablet::get_mds_table_rec_scn(SCN &rec_scn) const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; mds::MdsTableHandle mds_table; rec_scn = SCN::max_scn(); @@ -6559,9 +6055,9 @@ int ObTablet::get_mds_table_rec_scn(SCN &rec_scn) const } else if (OB_FAIL(inner_get_mds_table(mds_table))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; - LOG_DEBUG("mds_table does not exist", K(ret), K(ls_id), K(tablet_id)); + LOG_DEBUG("mds_table does not exist", K(ret), K(tablet_id)); } else { - LOG_WARN("failed to get mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get mds table", K(ret), K(tablet_id)); } } else if (OB_FAIL(mds_table.get_rec_scn(rec_scn))) { LOG_WARN("failed to get mds table rec scn", K(ret)); @@ -6664,7 +6160,7 @@ int ObTablet::calc_tablet_attr(ObTabletAttr &attr) const attr.reset(); attr.is_empty_shell_ = table_store_addr_.addr_.is_none(); attr.has_next_tablet_ = tablet_meta_.has_next_tablet_; - attr.ha_status_ = tablet_meta_.ha_status_.get_ha_status(); + attr.restore_state_ = tablet_meta_.restore_state_.get_state_value(); // calc space_usage attr.all_sstable_data_required_size_ = tablet_meta_.space_usage_.all_sstable_data_required_size_ + tablet_meta_.space_usage_.tablet_clustered_sstable_data_size_; @@ -6712,7 +6208,6 @@ int ObTablet::calc_tablet_attr(ObTabletAttr &attr) const int ObTablet::check_and_set_initial_state() { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet_meta_.ls_id_; const ObTabletID &tablet_id = tablet_meta_.tablet_id_; if (IS_NOT_INIT) { @@ -6734,7 +6229,7 @@ int ObTablet::check_and_set_initial_state() } else if (OB_FAIL(set_initial_state(false/*initial_state*/))) { LOG_WARN("failed to set initial state", K(ret)); } else { - LOG_DEBUG("set initial state to false", K(ret), K(ls_id), K(tablet_id)); + LOG_DEBUG("set initial state to false", K(ret), K(tablet_id)); } } @@ -6744,9 +6239,8 @@ int ObTablet::check_and_set_initial_state() int ObTablet::check_medium_list() const { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; - if (tablet_meta_.ha_status_.check_allow_read()) { + if (tablet_meta_.restore_state_.check_allow_read()) { ObTabletMemberWrapper table_store_wrapper; ObITable *last_major = nullptr; if (OB_FAIL(fetch_table_store(table_store_wrapper))) { @@ -6763,7 +6257,7 @@ int ObTablet::check_medium_list() const tablet_meta_.extra_medium_info_, &medium_info_array, last_major->get_snapshot_version()))) { - LOG_WARN("fail to validate medium info list", K(ret), K(ls_id), K(tablet_id), K(medium_info_array), KPC(last_major)); + LOG_WARN("fail to validate medium info list", K(ret), K(tablet_id), K(medium_info_array), KPC(last_major)); } // always free medium info @@ -6773,7 +6267,8 @@ int ObTablet::check_medium_list() const } } } else { - LOG_INFO("skip check medium list for non empty ha_status", KR(ret), K(ls_id), K(tablet_id), "ha_status", tablet_meta_.ha_status_); + LOG_INFO("skip medium list check while tablet restore is incomplete", KR(ret), K(tablet_id), + "restore_state", tablet_meta_.restore_state_); } return ret; } @@ -6786,7 +6281,6 @@ int ObTablet::prepare_param( param.table_id_ = relative_table.get_table_id(); param.tablet_id_ = tablet_meta_.tablet_id_; - param.ls_id_ = tablet_meta_.ls_id_; param.read_info_ = rowkey_read_info_; param.set_tablet_handle(relative_table.get_tablet_handle()); param.is_non_unique_local_index_ = relative_table.is_storage_index_table() && @@ -6797,7 +6291,6 @@ int ObTablet::prepare_param( int ObTablet::pre_check_empty_shell(const ObTablet &old_tablet, ObTabletCreateDeleteMdsUserData &user_data) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = old_tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = old_tablet.get_tablet_meta().tablet_id_; mds::MdsWriter writer;// will be removed later mds::TwoPhaseCommitState trans_stat;// will be removed later @@ -6808,22 +6301,22 @@ int ObTablet::pre_check_empty_shell(const ObTablet &old_tablet, ObTabletCreateDe mds::MdsTableHandle mds_table; share::SCN rec_scn; if (OB_FAIL(old_tablet.inner_get_mds_table(mds_table, false/*not_exist_create*/))) { - LOG_WARN("failed to get mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get mds table", K(ret), K(tablet_id)); } else if (OB_FAIL(mds_table.get_rec_scn(rec_scn))) { - LOG_WARN("failed to get rec scn from mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get rec scn from mds table", K(ret), K(tablet_id)); } else if (OB_UNLIKELY(rec_scn.is_max())) { ret = OB_STATE_NOT_MATCH; LOG_WARN("mds table exists, but rec scn is max, such kind tablet should be deleted instantly", - K(ret), K(ls_id), K(tablet_id)); + K(ret), K(tablet_id)); } else if (OB_FAIL(build_user_data_for_aborted_tx_tablet(rec_scn, user_data))) { - LOG_WARN("failed to build user data for aborted tx tablet", K(ret), K(ls_id), K(tablet_id), K(rec_scn)); + LOG_WARN("failed to build user data for aborted tx tablet", K(ret), K(tablet_id), K(rec_scn)); } } else { - LOG_WARN("failed to get latest tablet status", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get latest tablet status", K(ret), K(tablet_id)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT != trans_stat || !user_data.tablet_status_.is_deleted_for_gc()) { ret = OB_STATE_NOT_MATCH; - LOG_WARN("tablet is not under deleted tx committed status", K(ret), K(ls_id), K(tablet_id), K(trans_stat), K(user_data)); + LOG_WARN("tablet is not under deleted tx committed status", K(ret), K(tablet_id), K(trans_stat), K(user_data)); } return ret; @@ -7006,7 +6499,7 @@ int ObTablet::update_memtables() if (OB_FAIL(get_protected_memtable_mgr_handle(protected_handle))) { LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(this)); } else if (!protected_handle->has_memtable()) { - LOG_INFO("no memtable in memtable mgr", K(ret), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_); + LOG_INFO("no memtable in memtable mgr", K(ret), "tablet_id", tablet_meta_.tablet_id_); } else if (OB_FAIL(get_all_memtables_from_memtable_mgr(inc_memtables))) { LOG_WARN("failed to get all memtable", K(ret), KPC(this)); } else if (is_ls_inner_tablet() && OB_FAIL(rebuild_memtable(inc_memtables))) { @@ -7083,7 +6576,6 @@ int ObTablet::read_mds_table(common::ObIAllocator &allocator, { TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; mds_data.reset(); mds::MdsTableHandle mds_table_handle; @@ -7096,9 +6588,9 @@ int ObTablet::read_mds_table(common::ObIAllocator &allocator, } else if (CLICK_FAIL(inner_get_mds_table(mds_table_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_EMPTY_RESULT; - LOG_INFO("mds table does not exist, may be released", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("mds table does not exist, may be released", K(ret), K(tablet_id)); } else { - LOG_WARN("failed to get mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get mds table", K(ret), K(tablet_id)); } } else { ObTabletDumpMdsNodeOperator op(mds_data, allocator); @@ -7106,10 +6598,10 @@ int ObTablet::read_mds_table(common::ObIAllocator &allocator, mds::ScanNodeOrder::FROM_OLD_TO_NEW>(op, mds_construct_sequence, for_flush)))) { - LOG_WARN("failed to traverse mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to traverse mds table", K(ret), K(tablet_id)); } else if (!op.dumped()) { ret = OB_EMPTY_RESULT; - LOG_DEBUG("read nothing from mds table", K(ret), K(ls_id), K(tablet_id)); + LOG_DEBUG("read nothing from mds table", K(ret), K(tablet_id)); } } @@ -7126,9 +6618,9 @@ int ObTablet::get_mds_table_for_dump(mds::MdsTableHandle &mds_table) const if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_EMPTY_RESULT; LOG_INFO("mds table does not exist, may be released", - K(ret), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_)); + K(ret), K(tablet_meta_.tablet_id_)); } else { - LOG_WARN("failed to get mds table", K(ret), K(tablet_meta_.ls_id_), K(tablet_meta_.ls_id_)); + LOG_WARN("failed to get mds table", K(ret), K(tablet_meta_.tablet_id_)); } } return ret; @@ -7336,7 +6828,7 @@ int ObTablet::get_restore_status(ObTabletRestoreStatus::STATUS &restore_status) if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FALSE_IT(tablet_meta_.ha_status_.get_restore_status(restore_status))) { + } else if (OB_FALSE_IT(tablet_meta_.restore_state_.get_restore_status(restore_status))) { } else if (!ObTabletRestoreStatus::is_valid(restore_status)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("restore status is invalid", KR(ret), K(restore_status)); @@ -7354,9 +6846,9 @@ int ObTablet::get_mds_table_handle_(mds::MdsTableHandle &handle, LOG_TRACE("there is no mds table on ls inner tablet yet", KR(ret)); } else if (OB_FAIL(inner_get_mds_table(handle, create_if_not_exist))) { if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("inner get mds table failed", KR(ret), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_); + LOG_WARN("inner get mds table failed", KR(ret), "tablet_id", tablet_meta_.tablet_id_); } else if (REACH_THREAD_TIME_INTERVAL(10_s)) { - LOG_TRACE("inner get mds table failed", KR(ret), "ls_id", tablet_meta_.ls_id_, "tablet_id", tablet_meta_.tablet_id_); + LOG_TRACE("inner get mds table failed", KR(ret), "tablet_id", tablet_meta_.tablet_id_); } } return ret; @@ -7615,7 +7107,6 @@ int ObTablet::check_new_mds_with_cache( { int ret = OB_SUCCESS; const ObTabletID &tablet_id = get_tablet_meta().tablet_id_; - const ObLSID &ls_id = get_tablet_meta().ls_id_; bool r_valid = false; if (IS_NOT_INIT) { @@ -7643,7 +7134,7 @@ int ObTablet::check_new_mds_with_cache( } } else if (OB_FAIL(ObTabletCreateDeleteHelper::check_status_for_new_mds(*this, snapshot_version, tablet_status_cache_))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("failed to check status for new mds", KR(ret), K(ls_id), K(tablet_id), K(snapshot_version)); + LOG_WARN("failed to check status for new mds", KR(ret), K(tablet_id), K(snapshot_version)); } } } @@ -7656,7 +7147,6 @@ int ObTablet::check_tablet_status_for_read_all_committed() { int ret = OB_SUCCESS; const ObTabletID &tablet_id = get_tablet_meta().tablet_id_; - const ObLSID &ls_id = get_tablet_meta().ls_id_; ObTabletCreateDeleteMdsUserData user_data; mds::MdsWriter writer;// will be removed later mds::TwoPhaseCommitState trans_stat;// will be removed later @@ -7667,25 +7157,25 @@ int ObTablet::check_tablet_status_for_read_all_committed() if (OB_FAIL(get_latest_committed(user_data))) { if (OB_EMPTY_RESULT == ret) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet creation has not been committed, or has been roll backed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet creation has not been committed, or has been roll backed", K(ret), K(tablet_id)); } else if (OB_ERR_SHARED_LOCK_CONFLICT == ret) { if (OB_FAIL(get_latest(user_data, writer, trans_stat, trans_version))) { if (OB_EMPTY_RESULT == ret) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet creation has no been committed, or has been roll backed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet creation has no been committed, or has been roll backed", K(ret), K(tablet_id)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT != trans_stat) { if (transaction::ObTransVersion::INVALID_TRANS_VERSION == user_data.create_commit_version_) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("create commit version is invalid", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("create commit version is invalid", K(ret), K(tablet_id), K(user_data)); } } } else { - LOG_WARN("failed to get tablet status", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get tablet status", K(ret), K(tablet_id)); } } else if (OB_UNLIKELY(is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet become empty shell", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet become empty shell", K(ret), K(tablet_id)); } return ret; } @@ -7783,7 +7273,7 @@ int ObTablet::check_schema_version_with_cache(const int64_t schema_version) if (mds::TwoPhaseCommitState::ON_COMMIT == trans_stat) { ddl_data_cache_.set_value(tmp_ddl_data); candidate_cache = &ddl_data_cache_; - LOG_INFO("refresh ddl data cache", K(ret), K(tablet_meta_.ls_id_), K(tablet_id), K(ddl_data_cache_)); + LOG_INFO("refresh ddl data cache", K(ret), K(tablet_id), K(ddl_data_cache_)); } else { tmp_ddl_data_cache.set_value(tmp_ddl_data); candidate_cache = &tmp_ddl_data_cache; @@ -7792,9 +7282,9 @@ int ObTablet::check_schema_version_with_cache(const int64_t schema_version) if (OB_FAIL(ret)) { } else if (OB_ISNULL(candidate_cache)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl data cache is null", K(ret), KP(candidate_cache), K(tablet_meta_.ls_id_), K(tablet_id)); + LOG_WARN("ddl data cache is null", K(ret), KP(candidate_cache), K(tablet_id)); } else if (OB_FAIL(check_schema_version(*candidate_cache, schema_version))) { - LOG_WARN("fail to check schema version", K(ret), K(tablet_meta_.ls_id_), K(tablet_id)); + LOG_WARN("fail to check schema version", K(ret), K(tablet_id)); } } } @@ -7821,7 +7311,6 @@ int ObTablet::check_snapshot_readable_with_cache( bool &need_split_dst_table) { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet_meta_.ls_id_; const ObTabletID &tablet_id = tablet_meta_.tablet_id_; bool r_valid = false; need_split_dst_table = true; @@ -7879,7 +7368,7 @@ int ObTablet::check_snapshot_readable_with_cache( if (mds::TwoPhaseCommitState::ON_COMMIT == trans_stat) { ddl_data_cache_.set_value(tmp_ddl_data); candidate_cache = &ddl_data_cache_; - LOG_INFO("refresh ddl data cache", K(ret), K(ls_id), K(tablet_id), K(ddl_data_cache_)); + LOG_INFO("refresh ddl data cache", K(ret), K(tablet_id), K(ddl_data_cache_)); } else { tmp_ddl_data_cache.set_value(tmp_ddl_data); candidate_cache = &tmp_ddl_data_cache; @@ -7888,9 +7377,9 @@ int ObTablet::check_snapshot_readable_with_cache( if (OB_FAIL(ret)) { } else if (OB_ISNULL(candidate_cache)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl data cache is null", K(ret), KP(candidate_cache), K(ls_id), K(tablet_id)); + LOG_WARN("ddl data cache is null", K(ret), KP(candidate_cache), K(tablet_id)); } else if (OB_FAIL(check_snapshot_readable(*candidate_cache, snapshot_version, schema_version))) { - LOG_WARN("fail to check snapshot version", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(schema_version)); + LOG_WARN("fail to check snapshot version", K(ret), K(tablet_id), K(snapshot_version), K(schema_version)); } } } @@ -8055,28 +7544,27 @@ bool ObTablet::is_empty_shell() const bool ObTablet::is_data_complete() const { - return !is_empty_shell() - && tablet_meta_.ha_status_.is_data_status_complete(); + return !is_empty_shell(); } -int ObTablet::check_valid(const bool ignore_ha_status) const +int ObTablet::check_valid(const bool ignore_restore_state) const { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); } else { - ret = inner_check_valid(ignore_ha_status); + ret = inner_check_valid(ignore_restore_state); } return ret; } -// only check storage_schema & medium_list when ha_status is none -int ObTablet::inner_check_valid(const bool ignore_ha_status) const +// Medium info is complete only after tablet restore finishes. +int ObTablet::inner_check_valid(const bool ignore_restore_state) const { int ret = OB_SUCCESS; - const bool need_check_ha = tablet_meta_.ha_status_.check_allow_read() && !ignore_ha_status; - if (need_check_ha && OB_FAIL(check_medium_list())) { + const bool need_check_medium = tablet_meta_.restore_state_.check_allow_read() && !ignore_restore_state; + if (need_check_medium && OB_FAIL(check_medium_list())) { LOG_WARN("failed to check medium list", K(ret), KPC(this)); } else if (OB_FAIL(check_sstable_column_checksum())) { LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); @@ -8252,30 +7740,6 @@ int ObTablet::build_full_memory_mds_data( return ret; } -int ObTablet::build_mds_mini_sstable_for_migration( - common::ObArenaAllocator &allocator, - const ObMigrationTabletParam ¶m, - ObTableHandleV2 &mds_mini_sstable) -{ - int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet_meta_.ls_id_; - const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("already inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(ObMdsDataCompatHelper::generate_mds_mini_sstable(param, allocator, mds_mini_sstable))) { - if (OB_EMPTY_RESULT == ret) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("empty data from migration src", K(ret), K(ls_id), K(tablet_id), K(param)); - } else { - LOG_WARN("fail to generate mds mini sstable", K(ret), K(param)); - } - } - - return ret; -} - int ObTablet::get_memtables(common::ObIArray &memtables) const { int ret = OB_SUCCESS; @@ -8339,8 +7803,8 @@ int ObTablet::check_table_store_flag_match_with_table_store_(const ObTabletTable { int ret = OB_SUCCESS; ObTabletTableStoreFlag table_store_flag = tablet_meta_.table_store_flag_; - if (!tablet_meta_.ha_status_.is_none()) { - // tablet meta is not none, do not check it + if (!tablet_meta_.restore_state_.is_restore_status_full()) { + // The local table store is incomplete during restore. } else if (OB_ISNULL(table_store)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("table store should not be null", K(ret), KP(table_store)); @@ -8388,7 +7852,7 @@ int ObTablet::check_table_store_without_backup_table_(const ObSSTableArray &ssta return ret; } -int ObTablet::build_migration_shared_table_addr_( +int ObTablet::build_fork_shared_table_addr_( const ObRootBlockInfo &block_info, common::ObIAllocator &allocator, storage::ObMetaDiskAddr &addr, @@ -8400,7 +7864,7 @@ int ObTablet::build_migration_shared_table_addr_( if (!block_info.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("build migration shared table addr get invalid argument", K(ret)); + LOG_WARN("build fork shared table addr get invalid argument", K(ret)); } else if (!block_info.get_addr().is_memory()) { buf = nullptr; } else { diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 11e4fae37..39ef354df 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -47,7 +47,6 @@ namespace oceanbase { namespace share { -class ObLSID; struct ObTabletAutoincInterval; struct ObTabletReplicaChecksumItem; } @@ -90,6 +89,7 @@ class ObAllVirtualMdsNodeStat; namespace storage { +class ObLS; class ObIMemtable; class ObStoreCtx; class ObTableHandleV2; @@ -117,11 +117,10 @@ struct ObTableStoreCache void reset(); int init( const ObSSTableArray &major_tables, - const ObSSTableArray &minor_tables, - const bool is_tablet_referenced_by_collect_mv); + const ObSSTableArray &minor_tables); TO_STRING_KV(K_(last_major_snapshot_version), K_(major_table_cnt), K_(minor_table_cnt), K_(recycle_version), K_(last_major_column_count), - K_(last_major_macro_block_cnt), K_(is_tablet_referenced_by_collect_mv), + K_(last_major_macro_block_cnt), K_(last_major_compressor_type), K_(last_major_latest_row_store_type)); public: @@ -131,7 +130,6 @@ struct ObTableStoreCache int64_t recycle_version_; int64_t last_major_column_count_; int64_t last_major_macro_block_cnt_; - bool is_tablet_referenced_by_collect_mv_; //indicate tablet referenced by collect mv common::ObCompressorType last_major_compressor_type_; common::ObRowStoreType last_major_latest_row_store_type_; }; @@ -176,11 +174,9 @@ class ObTablet final : public ObITabletMdsCustomizedInterface inline int64_t get_last_major_total_macro_block_count() const { return table_store_cache_.last_major_macro_block_cnt_; } inline common::ObCompressorType get_last_major_compressor_type() const { return table_store_cache_.last_major_compressor_type_; } inline common::ObRowStoreType get_last_major_latest_row_store_type() const { return table_store_cache_.last_major_latest_row_store_type_; } - inline share::ObLSID get_ls_id() const { return tablet_meta_.ls_id_; } inline common::ObTabletID get_tablet_id() const { return tablet_meta_.tablet_id_; } inline common::ObTabletID get_data_tablet_id() const { return tablet_meta_.data_tablet_id_; } inline int64_t get_last_compaction_scn() const { return tablet_meta_.extra_medium_info_.last_medium_scn_; } - inline bool is_tablet_referenced_by_collect_mv() const { return table_store_cache_.is_tablet_referenced_by_collect_mv_; } inline bool is_user_tablet() const { return tablet_meta_.tablet_id_.is_user_tablet(); } inline bool is_user_data_table() const { return tablet_meta_.table_store_flag_.is_user_data_table(); } int get_mds_table_rec_scn(share::SCN &rec_scn) const; @@ -194,7 +190,6 @@ class ObTablet final : public ObITabletMdsCustomizedInterface // first time create tablet int init_for_first_time_creation( common::ObArenaAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN &create_scn, @@ -214,11 +209,6 @@ class ObTablet final : public ObITabletMdsCustomizedInterface const ObUpdateTableStoreParam ¶m, const ObTablet &old_tablet); bool is_external_tablet() const { return is_external_tablet_; } - int init_with_migrate_param( - common::ObArenaAllocator &allocator, - const ObMigrationTabletParam ¶m, - const bool is_update, - ObFreezer *freezer); //batch update table store with range cut int init_for_sstable_replace( common::ObArenaAllocator &allocator, @@ -230,11 +220,11 @@ class ObTablet final : public ObITabletMdsCustomizedInterface const ObTablet &old_tablet, const bool clear_wait_check_flag); - int init_with_replace_members( + int init_with_updated_members( common::ObArenaAllocator &allocator, const ObTablet &old_tablet, - const int64_t snapshot_version, - const ObTabletDataStatus::STATUS &data_status); + const int64_t snapshot_version +); // init for mds table mini merge int init_with_mds_sstable( common::ObArenaAllocator &allocator, @@ -412,8 +402,7 @@ class ObTablet final : public ObITabletMdsCustomizedInterface void reset_memtable(); // ATTENTION!!! The following two interfaces only release memtable from memtable manager. int release_memtables(const share::SCN scn); - // force release all memtables - // just for rebuild or migrate retry. + // Force release all memtables. int release_memtables(); int wait_release_memtables(); @@ -425,28 +414,9 @@ class ObTablet final : public ObITabletMdsCustomizedInterface int get_restore_status(ObTabletRestoreStatus::STATUS &restore_status) const; - // static help function - static int deserialize_id( - const char *buf, - const int64_t len, - share::ObLSID &ls_id, - common::ObTabletID &tablet_id); - - // migration section - // used for migration source generating create tablet rpc argument - int build_migration_tablet_param( - ObMigrationTabletParam &mig_tablet_param) const; - int build_migration_sstable_param( + int build_fork_sstable_param( const ObITable::TableKey &table_key, - blocksstable::ObMigrationSSTableParam &mig_sstable_param, - const bool is_fork_table) const; - int build_migration_sstable_param( - const ObITable::TableKey &table_key, - const blocksstable::ObSSTableMergeRes &res, - blocksstable::ObMigrationSSTableParam &mig_sstable_param) const; - int get_ha_tables( - ObTableStoreIterator &iter); - int get_ha_sstable_size(int64_t &data_size); + blocksstable::ObForkSSTableParam &fork_sstable_param) const; int get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_create = false); int start_direct_load_task_if_need(); int get_ddl_sstables(ObTableStoreIterator &table_store_iter) const; @@ -454,7 +424,6 @@ class ObTablet final : public ObITabletMdsCustomizedInterface int get_mini_minor_sstables(ObTableStoreIterator &table_store_iter) const; int get_table(const ObITable::TableKey &table_key, ObTableHandleV2 &handle) const; int get_recycle_version(const int64_t multi_version_start, int64_t &recycle_version) const; - int get_migration_sstable_size(int64_t &data_size); int check_is_delete_insert_table(bool &is_delete_insert_table) const; int check_table_store_without_backup_table_(const ObSSTableArray &sstable_array); @@ -568,7 +537,7 @@ class ObTablet final : public ObITabletMdsCustomizedInterface // different from the is_valid() function // typically used for check valid for migration or restore - int check_valid(const bool ignore_ha_status = false) const; + int check_valid(const bool ignore_restore_state = false) const; int64_t to_string(char *buf, const int64_t buf_len) const; int get_max_column_cnt_on_schema_recorder(int64_t &max_column_cnt); @@ -660,7 +629,7 @@ class ObTablet final : public ObITabletMdsCustomizedInterface ObMdsReadInfoCollector &collector) const; private: static bool ignore_ret(const int ret); - int inner_check_valid(const bool ignore_ha_status = false) const; + int inner_check_valid(const bool ignore_restore_state = false) const; int self_serialize(char *buf, const int64_t len, int64_t &pos) const; int64_t get_self_serialize_size() const; static int check_schema_version(const ObDDLInfoCache& ddl_info_cache, const int64_t schema_version); @@ -668,7 +637,6 @@ class ObTablet final : public ObITabletMdsCustomizedInterface logservice::ObLogHandler *get_log_handler() const { return log_handler_; } // TODO(gaishun.gs): get log handler from tablet pointer handle int init_shared_params( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const lib::Worker::CompatMode compat_mode); int build_read_info( @@ -848,27 +816,15 @@ class ObTablet final : public ObITabletMdsCustomizedInterface share::ObTabletReplicaChecksumItem &tablet_checksum, const bool need_checksums) const; - // mds mvs - int build_mds_mini_sstable_for_migration( - common::ObArenaAllocator &allocator, - const ObMigrationTabletParam ¶m, - ObTableHandleV2 &mds_mini_sstable); - int build_migration_tablet_param_storage_schema( - ObMigrationTabletParam &mig_tablet_param) const; - int build_migration_tablet_param_last_tablet_status( - ObMigrationTabletParam &mig_tablet_param) const; - int build_migration_tablet_param_major_ckm_info( - ObMigrationTabletParam &mig_tablet_param) const; - int clear_memtables_on_table_store(); // be careful to call this func, will destroy memtables array on table_store int check_table_store_flag_match_with_table_store_(const ObTabletTableStore *table_store); - int build_migration_shared_table_addr_( + int build_fork_shared_table_addr_( const ObRootBlockInfo &block_info, common::ObIAllocator &allocator, storage::ObMetaDiskAddr &addr, char *&buf) const; int get_kept_snapshot_for_split(int64_t &min_split_snapshot) const; - int start_direct_load_task_for_idem(ObLSHandle &ls_handle); + int start_direct_load_task_for_idem(ObLS *tenant_ls); public: static constexpr int32_t VERSION_V1 = 1; static constexpr int32_t VERSION_V2 = 2; @@ -911,9 +867,6 @@ class ObTablet final : public ObITabletMdsCustomizedInterface mutable common::SpinRWLock mds_cache_lock_; // size: 12B, alignment: 4B logservice::ObLogHandler *log_handler_; // size: 8B, alignment: 8B - //ATTENTION : Add a new variable need consider ObMigrationTabletParam - // and tablet meta init interface for migration. - // yuque : ObTablet *next_tablet_; // used in old_version_chain and tablet_gc_queue // size: 8B, alignment: 8B ObTabletStatusCache tablet_status_cache_; // size: 24B, alignment: 8B @@ -960,7 +913,7 @@ inline bool ObTablet::is_valid() const inline int ObTablet::allow_to_read_() { - return tablet_meta_.ha_status_.check_allow_read() ? common::OB_SUCCESS : common::OB_REPLICA_NOT_READABLE; + return tablet_meta_.restore_state_.check_allow_read() ? common::OB_SUCCESS : common::OB_REPLICA_NOT_READABLE; } inline void ObTablet::update_wash_score(const int64_t score) diff --git a/src/storage/tablet/ob_tablet_binding_helper.cpp b/src/storage/tablet/ob_tablet_binding_helper.cpp index f287f660b..258c1301d 100644 --- a/src/storage/tablet/ob_tablet_binding_helper.cpp +++ b/src/storage/tablet/ob_tablet_binding_helper.cpp @@ -20,7 +20,7 @@ #include "share/ob_ex_rpc.h" #include "share/rc/ob_module_provider.h" #include "storage/tablet/ob_tablet_binding_replay_executor.h" -#include "share/tablet/ob_tablet_to_ls_operator.h" +#include "share/tablet/ob_tablet_mapping_operator.h" #include "storage/tx_storage/ob_ls_service.h" #include "observer/ob_inner_sql_connection.h" #include "rootserver/ob_ddl_service.h" @@ -38,11 +38,9 @@ namespace storage { ObBatchUnbindTabletArg::ObBatchUnbindTabletArg() - : ls_id_(), - schema_version_(OB_INVALID_VERSION), + : schema_version_(OB_INVALID_VERSION), orig_tablet_ids_(), hidden_tablet_ids_(), - is_old_mds_(false), is_write_defensive_(false) { } @@ -55,104 +53,35 @@ int ObBatchUnbindTabletArg::assign(const ObBatchUnbindTabletArg &other) } else if (OB_FAIL(hidden_tablet_ids_.assign(other.hidden_tablet_ids_))) { LOG_WARN("failed to assign hidden tablet ids", K(ret)); } else { - - ls_id_ = other.ls_id_; schema_version_ = other.schema_version_; - is_old_mds_ = other.is_old_mds_; is_write_defensive_ = other.is_write_defensive_; } return ret; } -int ObBatchUnbindTabletArg::skip_array_len(const char *buf, - int64_t data_len, - int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t count = 0; - if (pos > data_len) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos)); - } else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { - TRANS_LOG(WARN, "failed to decode array count", K(ret), KP(buf), K(data_len)); - } else if (count < 0) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len), K(pos), K(count)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - ObTabletID tablet_id; - OB_UNIS_DECODE(tablet_id); - } - } - return ret; -} - -int ObBatchUnbindTabletArg::is_old_mds(const char *buf, - int64_t data_len, - bool &is_old_mds) -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - is_old_mds = false; - int64_t version = 0; - int64_t len = 0; - share::ObLSID id; - int64_t schema_version; - - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(data_len)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, version, len, id, schema_version); - if (OB_FAIL(ret)) { - } - // orig tablets array - else if (OB_FAIL(skip_array_len(buf, data_len, pos))) { - TRANS_LOG(WARN, "failed to skip_unis_array_len", K(ret), KP(buf), K(data_len), K(pos)); - } - // hidden tablets array - else if (OB_FAIL(skip_array_len(buf, data_len, pos))) { - TRANS_LOG(WARN, "failed to skip_unis_array_len", K(ret), KP(buf), K(data_len), K(pos)); - } else { - LST_DO_CODE(OB_UNIS_DECODE, is_old_mds); - } - } - - return ret; -} - OB_DEF_SERIALIZE(ObBatchUnbindTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, schema_version_, orig_tablet_ids_, hidden_tablet_ids_, is_old_mds_, is_write_defensive_); + LST_DO_CODE(OB_UNIS_ENCODE, schema_version_, orig_tablet_ids_, hidden_tablet_ids_, is_write_defensive_); return ret; } OB_DEF_SERIALIZE_SIZE(ObBatchUnbindTabletArg) { int len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, schema_version_, orig_tablet_ids_, hidden_tablet_ids_, is_old_mds_, is_write_defensive_); + LST_DO_CODE(OB_UNIS_ADD_LEN, schema_version_, orig_tablet_ids_, hidden_tablet_ids_, is_write_defensive_); return len; } OB_DEF_DESERIALIZE(ObBatchUnbindTabletArg) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, schema_version_, orig_tablet_ids_, hidden_tablet_ids_); - if (OB_SUCC(ret)) { - if (pos == data_len) { - is_old_mds_ = true; - } else { - LST_DO_CODE(OB_UNIS_DECODE, is_old_mds_); - } - } - LST_DO_CODE(OB_UNIS_DECODE, is_write_defensive_); + LST_DO_CODE(OB_UNIS_DECODE, schema_version_, orig_tablet_ids_, hidden_tablet_ids_, is_write_defensive_); return ret; } ObBatchUnbindLobTabletArg::ObBatchUnbindLobTabletArg() - : ls_id_(), - data_tablet_ids_() + : data_tablet_ids_() { } @@ -161,28 +90,17 @@ int ObBatchUnbindLobTabletArg::assign(const ObBatchUnbindLobTabletArg &other) int ret = OB_SUCCESS; if (OB_FAIL(data_tablet_ids_.assign(other.data_tablet_ids_))) { LOG_WARN("failed to assign data tablet ids", K(ret)); - } else { - - ls_id_ = other.ls_id_; } return ret; } -OB_SERIALIZE_MEMBER(ObBatchUnbindLobTabletArg, ls_id_, data_tablet_ids_); +OB_SERIALIZE_MEMBER(ObBatchUnbindLobTabletArg, data_tablet_ids_); -int ObTabletBindingHelper::get_ls(const ObLSID &ls_id, ObLSHandle &ls_handle) +int ObTabletBindingHelper::get_ls(ObLS *&tenant_ls) { int ret = OB_SUCCESS; - ObLSService* ls_srv = nullptr; - ObLS *ls = nullptr; - if (OB_ISNULL(ls_srv = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("share::g_mp->ls_service() fail, MTL not init?", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_ERROR("ls_srv->get_ls() fail", KR(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("invalid ls", KR(ret)); } return ret; } @@ -190,7 +108,7 @@ int ObTabletBindingHelper::get_ls(const ObLSID &ls_id, ObLSHandle &ls_handle) int ObTabletBindingHelper::get_tablet_for_new_mds(const ObLS &ls, const ObTabletID &tablet_id, const share::SCN &replay_scn, ObTabletHandle &handle) { int ret = OB_SUCCESS; - const ObTabletMapKey key(ls.get_ls_id(), tablet_id); + const ObTabletMapKey key(tablet_id); const bool for_replay = replay_scn.is_valid(); if (for_replay) { const bool replay_allow_tablet_not_exist = true; @@ -232,17 +150,17 @@ int ObTabletBindingHelper::modify_tablet_binding_for_new_mds_create(const ObBatc { MDS_TG(100_ms); int ret = OB_SUCCESS; - ObLSHandle ls_handle; - if (OB_FAIL(ObTabletBindingHelper::get_ls(arg.id_, ls_handle))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(ObTabletBindingHelper::get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret)); } else { const ObArray empty_array; - ObLS &ls = *ls_handle.get_ls(); + ObLS &ls = *tenant_ls; bool has_lob = false; for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablets_.count(); i++) { const ObCreateTabletInfo &info = arg.tablets_[i]; if (ObTabletCreateDeleteHelper::is_pure_hidden_tablets(info)) { - if (CLICK_FAIL(bind_hidden_tablet_to_orig_tablet(ls, info, replay_scn, ctx, arg.is_old_mds_))) { + if (CLICK_FAIL(bind_hidden_tablet_to_orig_tablet(ls, info, replay_scn, ctx))) { LOG_WARN("failed to add hidden tablet", K(ret)); } } else if (OB_FAIL(ObTabletBindingHelper::has_lob_tablets(arg, info, has_lob))) { @@ -286,11 +204,10 @@ int ObTabletBindingHelper::bind_hidden_tablet_to_orig_tablet( ObLS &ls, const ObCreateTabletInfo &info, const share::SCN &replay_scn, - mds::BufferCtx &ctx, - const bool for_old_mds) + mds::BufferCtx &ctx) { ObBindHiddenTabletToOrigTabletOp op(info); - return modify_tablet_binding_new_mds(ls, info.data_tablet_id_, replay_scn, ctx, for_old_mds, op); + return modify_tablet_binding_new_mds(ls, info.data_tablet_id_, replay_scn, ctx, op); } int ObBindLobTabletToDataTabletOp::assign(const ObBindLobTabletToDataTabletOp &other) @@ -337,7 +254,7 @@ int ObTabletBindingHelper::bind_lob_tablet_to_data_tablet( mds::BufferCtx &ctx) { ObBindLobTabletToDataTabletOp op(arg, info); - return modify_tablet_binding_new_mds(ls, info.data_tablet_id_, replay_scn, ctx, arg.is_old_mds_, op); + return modify_tablet_binding_new_mds(ls, info.data_tablet_id_, replay_scn, ctx, op); } int ObTabletBindingHelper::build_single_table_write_defensive( @@ -374,7 +291,6 @@ int ObTabletBindingHelper::modify_tablet_binding_new_mds( const ObTabletID &tablet_id, const share::SCN &replay_scn, mds::BufferCtx &ctx, - bool for_old_mds, F &&op) { MDS_TG(100_ms); @@ -401,9 +317,9 @@ int ObTabletBindingHelper::modify_tablet_binding_new_mds( } else { if (replay_scn.is_valid()) { ObTabletBindingReplayExecutor replay_executor; - if (CLICK_FAIL(replay_executor.init(ctx, data, replay_scn, for_old_mds))) { + if (CLICK_FAIL(replay_executor.init(ctx, data, replay_scn))) { LOG_ERROR("failed to init replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(replay_scn, ls.get_ls_id(), tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(replay_scn, tablet_id))) { if (OB_EAGAIN != ret) { LOG_ERROR("failed to replay tablet binding log", K(ret)); } @@ -439,15 +355,6 @@ int ObTabletUnbindMdsHelper::register_process( return ret; } -int ObTabletUnbindMdsHelper::on_commit_for_old_mds( - const char* buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg) -{ - mds::TLOCAL_MDS_INFO.reset();// disable runtime check - return ObTabletCreateDeleteHelper::process_for_old_mds(buf, len, notify_arg); -} - int ObTabletUnbindMdsHelper::on_register(const char* buf, const int64_t len, mds::BufferCtx &ctx) { int ret = OB_SUCCESS; @@ -455,9 +362,6 @@ int ObTabletUnbindMdsHelper::on_register(const char* buf, const int64_t len, mds ObBatchUnbindTabletArg arg; if (OB_FAIL(arg.deserialize(buf, len, pos))) { LOG_WARN("failed to deserialize arg", K(ret)); - } else if (arg.is_old_mds_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, arg is old mds", K(ret), K(arg)); } else if (OB_FAIL(register_process(arg, ctx))) { LOG_WARN("failed to register_process", K(ret)); } @@ -487,8 +391,6 @@ int ObTabletUnbindMdsHelper::on_replay(const char* buf, const int64_t len, const ObBatchUnbindTabletArg arg; if (OB_FAIL(arg.deserialize(buf, len, pos))) { LOG_WARN("failed to deserialize arg", K(ret)); - } else if (arg.is_old_mds_) { - LOG_INFO("skip unbind mds tablet for old mds", K(arg), K(scn)); } else if (OB_FAIL(replay_process(arg, scn, ctx))) { LOG_WARN("failed to replay_process", K(ret)); } @@ -517,7 +419,7 @@ int ObTabletUnbindMdsHelper::unbind_hidden_tablets_from_orig_tablets( for (int64_t i = 0; OB_SUCC(ret) && i < arg.orig_tablet_ids_.count(); i++) { const ObTabletID &orig_tablet = arg.orig_tablet_ids_.at(i); ObUnbindHiddenTabletFromOrigTabletOp op(arg.schema_version_); - if (OB_FAIL(ObTabletBindingHelper::modify_tablet_binding_new_mds(ls, orig_tablet, replay_scn, ctx, arg.is_old_mds_, op))) { + if (OB_FAIL(ObTabletBindingHelper::modify_tablet_binding_new_mds(ls, orig_tablet, replay_scn, ctx, op))) { LOG_WARN("failed to modify tablet binding", K(ret)); } } @@ -551,7 +453,7 @@ int ObTabletUnbindMdsHelper::set_redefined_versions_for_hidden_tablets( for (int64_t i = 0; OB_SUCC(ret) && i < arg.hidden_tablet_ids_.count(); i++) { const ObTabletID &hidden_tablet = arg.hidden_tablet_ids_.at(i); ObSetRwDefensiveOp op(arg.schema_version_); - if (OB_FAIL(ObTabletBindingHelper::modify_tablet_binding_new_mds(ls, hidden_tablet, replay_scn, ctx, arg.is_old_mds_, op))) { + if (OB_FAIL(ObTabletBindingHelper::modify_tablet_binding_new_mds(ls, hidden_tablet, replay_scn, ctx, op))) { LOG_WARN("failed to modify tablet binding", K(ret)); } } @@ -561,11 +463,11 @@ int ObTabletUnbindMdsHelper::set_redefined_versions_for_hidden_tablets( int ObTabletUnbindMdsHelper::modify_tablet_binding_for_unbind(const ObBatchUnbindTabletArg &arg, const share::SCN &replay_scn, mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - if (OB_FAIL(ObTabletBindingHelper::get_ls(arg.ls_id_, ls_handle))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(ObTabletBindingHelper::get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret)); } else { - ObLS &ls = *ls_handle.get_ls(); + ObLS &ls = *tenant_ls; if (OB_FAIL(unbind_hidden_tablets_from_orig_tablets(ls, arg, replay_scn, ctx))) { LOG_WARN("failed to unbind", K(ret)); } else if (arg.is_redefined()) { @@ -577,47 +479,46 @@ int ObTabletUnbindMdsHelper::modify_tablet_binding_for_unbind(const ObBatchUnbin return ret; } -ObTabletBindingMdsArg::ObTabletBindingMdsArg() : ls_id_(), tablet_ids_(), binding_datas_() +ObTabletBindingMdsArg::ObTabletBindingMdsArg() : tablet_ids_(), binding_datas_() { reset(); } bool ObTabletBindingMdsArg::is_valid() const { - return ls_id_.is_valid() && tablet_ids_.count() == binding_datas_.count(); + return !tablet_ids_.empty() && tablet_ids_.count() == binding_datas_.count(); } void ObTabletBindingMdsArg::reset() { - ls_id_.reset(); tablet_ids_.reset(); binding_datas_.reset(); } -OB_SERIALIZE_MEMBER(ObTabletBindingMdsArg, ls_id_, tablet_ids_, binding_datas_); +OB_SERIALIZE_MEMBER(ObTabletBindingMdsArg, tablet_ids_, binding_datas_); -int ObTabletBindingMdsHelper::get_sorted_ls_tablets(const ObIArray &tablet_ids, - ObArray> &ls_tablets, +int ObTabletBindingMdsHelper::get_sorted_tablets(const ObIArray &tablet_ids, + ObArray &sorted_tablet_ids, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - ls_tablets.reset(); - ObArray ls_ids; - if (OB_FAIL(ObTabletToLSTableOperator::batch_get_ls(trans, tablet_ids, ls_ids))) { - LOG_WARN("failed to batch get ls", K(ret)); - } else if (OB_UNLIKELY(tablet_ids.count() != ls_ids.count())) { + sorted_tablet_ids.reset(); + ObArray tablet_infos; + if (OB_FAIL(ObTabletMappingTableOperator::batch_get(trans, tablet_ids, tablet_infos))) { + LOG_WARN("failed to batch get tablet info", K(ret), K(tablet_ids)); + } else if (OB_UNLIKELY(tablet_ids.count() != tablet_infos.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet ids ls ids", K(ret), K(tablet_ids), K(ls_ids)); + LOG_WARN("invalid tablet info count", K(ret), K(tablet_ids), K(tablet_infos)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(ls_tablets.push_back(std::make_pair(ls_ids[i], tablet_ids.at(i))))) { - LOG_WARN("failed to push back tablet id and ls id", K(ret)); + if (OB_FAIL(sorted_tablet_ids.push_back(tablet_ids.at(i)))) { + LOG_WARN("failed to push back tablet id", K(ret), K(tablet_ids.at(i))); } } if (OB_SUCC(ret)) { - lib::ob_sort(ls_tablets.begin(), ls_tablets.end(), LSTabletCmp()); + lib::ob_sort(sorted_tablet_ids.begin(), sorted_tablet_ids.end(), TabletIDCmp()); } return ret; } @@ -634,30 +535,16 @@ int ObTabletBindingMdsHelper::batch_get_tablet_binding( LOG_WARN("invalid argument", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSService *ls_service = share::g_mp->ls_service(); - logservice::ObLogService *log_service = share::g_mp->log_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObRole role = INVALID_ROLE; - if (OB_ISNULL(ls_service) || OB_ISNULL(log_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls_service or log_service", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { - ret = OB_NOT_MASTER; - LOG_WARN("ls not leader", K(ret), K(arg.ls_id_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.tablet_ids_.at(i); ObTabletHandle tablet_handle; ObTabletBindingMdsUserData data; - if (OB_FAIL(ObTabletBindingHelper::get_tablet_for_new_mds(*ls, tablet_id, share::SCN::invalid_scn(), tablet_handle))) { + if (OB_FAIL(ObTabletBindingHelper::get_tablet_for_new_mds( + *tenant_ls, tablet_id, share::SCN::invalid_scn(), tablet_handle))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id), K(abs_timeout_us)); } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), data, abs_timeout_us - ObTimeUtility::current_time()))) { LOG_WARN("failed to update tablet autoinc seq", K(ret), K(abs_timeout_us)); @@ -679,7 +566,7 @@ int ObTabletBindingMdsHelper::batch_get_tablet_binding( } } else if (OB_UNLIKELY(trans_stat != mds::TwoPhaseCommitState::ON_COMMIT)) { ret = OB_EAGAIN; - LOG_WARN("check committed failed", K(ret), K(arg.ls_id_), K(tablet_id), K(tmp_data), K(trans_stat)); + LOG_WARN("check committed failed", K(ret), K(tablet_id), K(tmp_data), K(trans_stat)); } } } @@ -689,16 +576,15 @@ int ObTabletBindingMdsHelper::batch_get_tablet_binding( return ret; } -int ObTabletBindingMdsHelper::get_tablet_binding_mds_by_rpc(const ObLSID &ls_id, - const ObIArray &tablet_ids, +int ObTabletBindingMdsHelper::get_tablet_binding_mds_by_rpc(const ObIArray &tablet_ids, const int64_t abs_timeout_us, ObIArray &datas) { int ret = OB_SUCCESS; obcall::ObBatchGetTabletBindingArg arg; obcall::ObBatchGetTabletBindingRes res; - if (OB_FAIL(arg.init(ls_id, tablet_ids, true/*check_committed*/))) { - LOG_WARN("failed to init arg", K(ret), K(ls_id)); + if (OB_FAIL(arg.init(tablet_ids, true/*check_committed*/))) { + LOG_WARN("failed to init arg", K(ret)); } else if (OB_FAIL(ex_rpc::sync_call([&]{ return batch_get_tablet_binding(abs_timeout_us, arg, res); }))) { @@ -717,8 +603,6 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_for_create(const obcall::ObB ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - const ObLSID &ls_id = arg.id_; - if (OB_SUCC(ret)) { ObArray batch_tablet_ids; ObArray batch_ops; @@ -735,7 +619,7 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_for_create(const obcall::ObB } if (OB_FAIL(ret)) { } else if (is_last || batch_tablet_ids.count() >= ObTabletBindingMdsArg::BATCH_TABLET_CNT) { - if (OB_FAIL(modify_tablet_binding_(ls_id, batch_tablet_ids, abs_timeout_us, + if (OB_FAIL(modify_tablet_binding_batch_(batch_tablet_ids, abs_timeout_us, ModifyBindingByOps(batch_ops), trans))) { LOG_WARN("failed to modify tablet binding", K(ret)); } else { @@ -769,7 +653,7 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_for_create(const obcall::ObB } if (OB_FAIL(ret)) { } else if (is_last || batch_tablet_ids.count() >= ObTabletBindingMdsArg::BATCH_TABLET_CNT) { - if (OB_FAIL(modify_tablet_binding_(ls_id, batch_tablet_ids, abs_timeout_us, + if (OB_FAIL(modify_tablet_binding_batch_(batch_tablet_ids, abs_timeout_us, ModifyBindingByOps(batch_ops), trans))) { LOG_WARN("failed to modify tablet binding", K(ret)); } else { @@ -844,8 +728,7 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_for_write_defensive(const Ob } template -int ObTabletBindingMdsHelper::modify_tablet_binding_(const share::ObLSID &ls_id, - const ObIArray &tablet_ids, +int ObTabletBindingMdsHelper::modify_tablet_binding_batch_(const ObIArray &tablet_ids, const int64_t abs_timeout_us, F &&op, ObMySQLTransaction &trans) @@ -853,12 +736,10 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_(const share::ObLSID &ls_id, int ret = OB_SUCCESS; if (!tablet_ids.empty()) { ObArray old_datas; - if (OB_FAIL(get_tablet_binding_mds_by_rpc(ls_id, tablet_ids, abs_timeout_us, old_datas))) { + if (OB_FAIL(get_tablet_binding_mds_by_rpc(tablet_ids, abs_timeout_us, old_datas))) { LOG_WARN("failed to get tablet binding mds", K(ret)); } else { ObTabletBindingMdsArg arg; - - arg.ls_id_ = ls_id; if (OB_FAIL(arg.tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign", K(ret)); } @@ -890,19 +771,18 @@ int ObTabletBindingMdsHelper::modify_tablet_binding_(const ObIArray { int ret = OB_SUCCESS; if (!tablet_ids.empty()) { - ObArray> ls_tablets; + ObArray sorted_tablet_ids; ObArray this_batch_tablet_ids; - if (OB_FAIL(get_sorted_ls_tablets(tablet_ids, ls_tablets, trans))) { - LOG_WARN("failed to get sorted ls tablets", K(ret)); + if (OB_FAIL(get_sorted_tablets(tablet_ids, sorted_tablet_ids, trans))) { + LOG_WARN("failed to get sorted tablets", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablets.count(); i++) { - const ObLSID &ls_id = ls_tablets.at(i).first; - const ObTabletID &tablet_id = ls_tablets.at(i).second; - const bool is_last_or_next_ls_id_changed = i == ls_tablets.count() - 1 || ls_id != ls_tablets.at(i+1).first; + for (int64_t i = 0; OB_SUCC(ret) && i < sorted_tablet_ids.count(); i++) { + const ObTabletID &tablet_id = sorted_tablet_ids.at(i); + const bool is_last = i == sorted_tablet_ids.count() - 1; if (OB_FAIL(this_batch_tablet_ids.push_back(tablet_id))) { LOG_WARN("failed to push back", K(ret)); - } else if (is_last_or_next_ls_id_changed || this_batch_tablet_ids.count() >= ObTabletBindingMdsArg::BATCH_TABLET_CNT) { - if (OB_FAIL(modify_tablet_binding_(ls_id, this_batch_tablet_ids, abs_timeout_us, + } else if (is_last || this_batch_tablet_ids.count() >= ObTabletBindingMdsArg::BATCH_TABLET_CNT) { + if (OB_FAIL(modify_tablet_binding_batch_(this_batch_tablet_ids, abs_timeout_us, ModifyBindingByOp(op), trans))) { LOG_WARN("failed to modify tablet binding", K(ret)); } else { @@ -937,7 +817,7 @@ int ObTabletBindingMdsHelper::register_mds_( LOG_WARN("failed to allocate", K(ret)); } else if (OB_FAIL(arg.serialize(buf, size, pos))) { LOG_WARN("failed to serialize arg", K(ret)); - } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(arg.ls_id_, ObTxDataSourceType::TABLET_BINDING, buf, pos))) { + } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(ObTxDataSourceType::TABLET_BINDING, buf, pos))) { LOG_WARN("failed to register mds", K(ret)); } } @@ -976,22 +856,18 @@ int ObTabletBindingMdsHelper::modify_( mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = arg.ls_id_; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(arg)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("fail to get ls", KR(ret), K(arg)); - } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", KR(ret), K(arg), KP(ls)); } for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.tablet_ids_[i]; - if (OB_FAIL(set_tablet_binding_mds_(*ls, tablet_id, scn, arg.binding_datas_[i], ctx))) { - LOG_WARN("failed to set tablet binding mds", K(ret), K(ls_id), K(tablet_id), K(scn)); + if (OB_FAIL(set_tablet_binding_mds_( + *tenant_ls, tablet_id, scn, arg.binding_datas_[i], ctx))) { + LOG_WARN("failed to set tablet binding mds", K(ret), K(tablet_id), K(scn)); } } LOG_INFO("modify tablet binding data", K(ret), K(scn), K(ctx.get_writer()), K(arg)); @@ -1009,7 +885,6 @@ int ObTabletBindingMdsHelper::set_tablet_binding_mds_( int ret = OB_SUCCESS; if (!replay_scn.is_valid()) { mds::MdsCtx &user_ctx = static_cast(ctx); - ObLSHandle ls_handle; if (CLICK_FAIL(ls.get_tablet_svr()->set_ddl_info(tablet_id, data, user_ctx, 0/*lock_timeout_us*/))) { if (OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret) { ret = OB_EAGAIN; @@ -1019,9 +894,9 @@ int ObTabletBindingMdsHelper::set_tablet_binding_mds_( } } else { ObTabletBindingReplayExecutor replay_executor; - if (CLICK_FAIL(replay_executor.init(ctx, data, replay_scn, false/*for_old_mds*/))) { + if (CLICK_FAIL(replay_executor.init(ctx, data, replay_scn))) { LOG_ERROR("failed to init replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(replay_scn, ls.get_ls_id(), tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(replay_scn, tablet_id))) { if (OB_EAGAIN != ret) { LOG_ERROR("failed to replay tablet binding log", K(ret)); } @@ -1033,15 +908,15 @@ int ObTabletBindingMdsHelper::set_tablet_binding_mds_( int ObTabletUnbindLobMdsHelper::modify_tablet_binding_for_unbind_lob_(const ObBatchUnbindLobTabletArg &arg, const share::SCN &replay_scn, mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - if (OB_FAIL(ObTabletBindingHelper::get_ls(arg.ls_id_, ls_handle))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(ObTabletBindingHelper::get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret)); } else { - ObLS &ls = *ls_handle.get_ls(); + ObLS &ls = *tenant_ls; for (int64_t i = 0; OB_SUCC(ret) && i < arg.data_tablet_ids_.count(); i++) { const ObTabletID &data_tablet_id = arg.data_tablet_ids_.at(i); if (OB_FAIL(ObTabletBindingHelper::modify_tablet_binding_new_mds(ls, data_tablet_id, - replay_scn, ctx, false, ClearLobTabletId()))) { + replay_scn, ctx, ClearLobTabletId()))) { LOG_WARN("failed to modify tablet binding", K(ret), K(data_tablet_id)); } } diff --git a/src/storage/tablet/ob_tablet_binding_helper.h b/src/storage/tablet/ob_tablet_binding_helper.h index 05a41cbe4..4bce39bd8 100644 --- a/src/storage/tablet/ob_tablet_binding_helper.h +++ b/src/storage/tablet/ob_tablet_binding_helper.h @@ -24,7 +24,6 @@ #include "lib/container/ob_array_serialization.h" #include "common/mysqlclient/ob_mysql_transaction.h" #include "lib/ob_define.h" -#include "share/ob_ls_id.h" #include "storage/tablet/ob_tablet_binding_mds_user_data.h" namespace oceanbase @@ -63,7 +62,6 @@ class MdsCtx; } class ObLS; -class ObLSHandle; class ObTabletHandle; class ObTabletTxMultiSourceDataUnit; class ObTabletMapKey; @@ -76,19 +74,14 @@ class ObBatchUnbindTabletArg final ~ObBatchUnbindTabletArg() {} int assign(const ObBatchUnbindTabletArg &other); inline bool is_redefined() const { return schema_version_ != OB_INVALID_VERSION; } - TO_STRING_KV(K_(ls_id), K_(schema_version), K_(orig_tablet_ids), K_(hidden_tablet_ids), K_(is_write_defensive)); + TO_STRING_KV(K_(schema_version), K_(orig_tablet_ids), K_(hidden_tablet_ids), K_(is_write_defensive)); bool is_valid() { return true; } - static int is_old_mds(const char *buf, const int64_t len, bool &is_old_mds); - static int skip_array_len(const char *buf, int64_t data_len, int64_t &pos); - OB_UNIS_VERSION_V(1); + OB_UNIS_VERSION_V(2); public: - - share::ObLSID ls_id_; int64_t schema_version_; ObSArray orig_tablet_ids_; ObSArray hidden_tablet_ids_; - bool is_old_mds_; bool is_write_defensive_; private: DISALLOW_COPY_AND_ASSIGN(ObBatchUnbindTabletArg); @@ -100,13 +93,12 @@ class ObBatchUnbindLobTabletArg final ObBatchUnbindLobTabletArg(); ~ObBatchUnbindLobTabletArg() {} int assign(const ObBatchUnbindLobTabletArg &other); - TO_STRING_KV(K_(ls_id), K_(data_tablet_ids)); + TO_STRING_KV(K_(data_tablet_ids)); bool is_valid() { return true; } - OB_UNIS_VERSION_V(1); + OB_UNIS_VERSION_V(2); public: - share::ObLSID ls_id_; ObSArray data_tablet_ids_; private: DISALLOW_COPY_AND_ASSIGN(ObBatchUnbindLobTabletArg); @@ -122,16 +114,16 @@ class ObTabletBindingHelper final // create tablet by new mds static int modify_tablet_binding_for_new_mds_create(const obcall::ObBatchCreateTabletArg &arg, const share::SCN &replay_scn, mds::BufferCtx &ctx); - static int bind_hidden_tablet_to_orig_tablet(ObLS &ls, const obcall::ObCreateTabletInfo &info, const share::SCN &replay_scn, mds::BufferCtx &ctx, const bool for_old_mds); + static int bind_hidden_tablet_to_orig_tablet(ObLS &ls, const obcall::ObCreateTabletInfo &info, const share::SCN &replay_scn, mds::BufferCtx &ctx); static int bind_lob_tablet_to_data_tablet(ObLS &ls, const obcall::ObBatchCreateTabletArg &arg, const obcall::ObCreateTabletInfo &info, const share::SCN &replay_scn, mds::BufferCtx &ctx); // TODO (lihongqin.lhq) delete get_tablet_for_new_mds static int get_tablet_for_new_mds(const ObLS &ls, const ObTabletID &tablet_id, const share::SCN &replay_scn, ObTabletHandle &handle); // common template - static int modify_tablet_binding_new_mds(ObLS &ls, const ObTabletID &tablet_id, const share::SCN &replay_scn, mds::BufferCtx &ctx, const bool for_old_mds, F &&op); + static int modify_tablet_binding_new_mds(ObLS &ls, const ObTabletID &tablet_id, const share::SCN &replay_scn, mds::BufferCtx &ctx, F &&op); static int has_lob_tablets(const obcall::ObBatchCreateTabletArg &arg, const obcall::ObCreateTabletInfo &info, bool &has_lob); - static int get_ls(const share::ObLSID &ls_id, ObLSHandle &ls_handle); + static int get_ls(ObLS *&tenant_ls); static int build_single_table_write_defensive(const share::schema::ObTableSchema &table_schema, const int64_t schema_version, rootserver::ObDDLSQLTransaction &trans); @@ -148,7 +140,6 @@ class ObTabletUnbindMdsHelper public: static int on_register(const char* buf, const int64_t len, mds::BufferCtx &ctx); static int register_process(ObBatchUnbindTabletArg &arg, mds::BufferCtx &ctx); - static int on_commit_for_old_mds(const char* buf, const int64_t len, const transaction::ObMulSourceDataNotifyArg ¬ify_arg); static int replay_process(ObBatchUnbindTabletArg &arg, const share::SCN &scn, mds::BufferCtx &ctx); static int on_replay(const char* buf, const int64_t len, const share::SCN &scn, mds::BufferCtx &ctx); private: @@ -160,7 +151,7 @@ class ObTabletUnbindMdsHelper class ObTabletBindingMdsArg final { public: - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: // arg with such tablet cnt cannot be more than mds buffer limit (1.5M) const static int64_t BATCH_TABLET_CNT = 8192; @@ -168,11 +159,8 @@ class ObTabletBindingMdsArg final ~ObTabletBindingMdsArg() {} bool is_valid() const; void reset(); - TO_STRING_KV(K_(ls_id), K_(tablet_ids), K_(binding_datas)); + TO_STRING_KV(K_(tablet_ids), K_(binding_datas)); public: - - uint64_t tenant_data_version_; - share::ObLSID ls_id_; ObSArray tablet_ids_; ObSArray binding_datas_; private: @@ -266,10 +254,10 @@ struct ModifyBindingByOps final ObIArray &ops_; }; -struct LSTabletCmp final +struct TabletIDCmp final { - bool operator()(const std::pair &lhs, const std::pair &rhs) { - return lhs.first != rhs.first ? lhs.first < rhs.first : lhs.second < rhs.second; + bool operator()(const ObTabletID &lhs, const ObTabletID &rhs) { + return lhs < rhs; } }; @@ -280,15 +268,14 @@ class ObTabletBindingMdsHelper static int on_replay(const char* buf, const int64_t len, const share::SCN &scn, mds::BufferCtx &ctx); public: - static int get_sorted_ls_tablets(const ObIArray &tablet_ids, - ObArray> &ls_tablets, + static int get_sorted_tablets(const ObIArray &tablet_ids, + ObArray &sorted_tablet_ids, ObMySQLTransaction &trans); static int batch_get_tablet_binding( const int64_t abs_timeout_us, const obcall::ObBatchGetTabletBindingArg &arg, obcall::ObBatchGetTabletBindingRes &res); - static int get_tablet_binding_mds_by_rpc(const share::ObLSID &ls_id, - const ObIArray &tablet_ids, + static int get_tablet_binding_mds_by_rpc(const ObIArray &tablet_ids, const int64_t abs_timeout_us, ObIArray &datas); static int modify_tablet_binding_for_create(const obcall::ObBatchCreateTabletArg &arg, @@ -311,8 +298,7 @@ class ObTabletBindingMdsHelper private: template - static int modify_tablet_binding_(const share::ObLSID &ls_id, - const ObIArray &tablet_ids, + static int modify_tablet_binding_batch_(const ObIArray &tablet_ids, const int64_t abs_timeout_us, F &&op, ObMySQLTransaction &trans); diff --git a/src/storage/tablet/ob_tablet_binding_mds_user_data.h b/src/storage/tablet/ob_tablet_binding_mds_user_data.h index e9a71facd..48f171af8 100644 --- a/src/storage/tablet/ob_tablet_binding_mds_user_data.h +++ b/src/storage/tablet/ob_tablet_binding_mds_user_data.h @@ -64,8 +64,7 @@ class ObTabletBindingMdsUserData : public ObIStorageMetaObj K_(data_tablet_id), K_(hidden_tablet_id), K_(lob_meta_tablet_id), - K_(lob_piece_tablet_id), - K_(is_old_mds)); + K_(lob_piece_tablet_id)); public: int64_t snapshot_version_; // if redefined it is max readable snapshot, else it is min readable snapshot. @@ -75,7 +74,6 @@ class ObTabletBindingMdsUserData : public ObIStorageMetaObj common::ObTabletID lob_meta_tablet_id_; common::ObTabletID lob_piece_tablet_id_; bool redefined_; - bool is_old_mds_; }; } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_binding_replay_executor.cpp b/src/storage/tablet/ob_tablet_binding_replay_executor.cpp index 819c919e1..5d98a5907 100644 --- a/src/storage/tablet/ob_tablet_binding_replay_executor.cpp +++ b/src/storage/tablet/ob_tablet_binding_replay_executor.cpp @@ -30,8 +30,7 @@ ObTabletBindingReplayExecutor::ObTabletBindingReplayExecutor() int ObTabletBindingReplayExecutor::init( mds::BufferCtx &user_ctx, const ObTabletBindingMdsUserData &user_data, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -44,7 +43,6 @@ int ObTabletBindingReplayExecutor::init( user_ctx_ = &user_ctx; user_data_ = &user_data; scn_ = scn; - for_old_mds_ = for_old_mds; is_inited_ = true; } return ret; @@ -56,7 +54,7 @@ int ObTabletBindingReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) int ret = OB_SUCCESS; mds::MdsCtx &user_ctx = static_cast(*user_ctx_); - if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_, for_old_mds_))) { + if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_))) { LOG_WARN("failed to replay to tablet", K(ret)); } diff --git a/src/storage/tablet/ob_tablet_binding_replay_executor.h b/src/storage/tablet/ob_tablet_binding_replay_executor.h index d22d52da8..1fad16a34 100644 --- a/src/storage/tablet/ob_tablet_binding_replay_executor.h +++ b/src/storage/tablet/ob_tablet_binding_replay_executor.h @@ -35,8 +35,7 @@ class ObTabletBindingReplayExecutor final : public logservice::ObTabletReplayExe int init( mds::BufferCtx &user_ctx, const ObTabletBindingMdsUserData &user_data, - const share::SCN &scn, - const bool for_old_mds); + const share::SCN &scn); protected: bool is_replay_update_tablet_status_() const override @@ -57,7 +56,6 @@ class ObTabletBindingReplayExecutor final : public logservice::ObTabletReplayExe mds::BufferCtx *user_ctx_; const ObTabletBindingMdsUserData *user_data_; share::SCN scn_; - bool for_old_mds_; }; diff --git a/src/storage/tablet/ob_tablet_common.h b/src/storage/tablet/ob_tablet_common.h index b7e4681a4..7f3ba34c6 100644 --- a/src/storage/tablet/ob_tablet_common.h +++ b/src/storage/tablet/ob_tablet_common.h @@ -32,7 +32,7 @@ namespace storage READ_ALL_COMMITED: Read committed row after transaction committed, except empty shell. - Return tablet which has finished at least one MDS transaction(not including tablet in NORMAL not committed, reserved not committed status). + Return tablet which has finished at least one MDS transaction(not including tablet in NORMAL not committed, RESERVED_STATUS_5 not committed status). Not return CREATING and DELETING who was abandoned from 4.2. In addition, you should NOT pass read timeout under this mode. @@ -41,9 +41,8 @@ namespace storage READ_READABLE_COMMITED: Read committed row, not include deleted one. The most frequently used mode. Return - tablet in NORMAL or reserved status. Not return one in unreadable status. - If latest tablet status is reserved, we should check reserved scn to decide - whether it is legal to return the tablet. + tablet in NORMAL, RESERVED_STATUS_5 status. Not return one in unreadable status. + RESERVED_STATUS_4 is not readable in seekdb. If read operation reaches read timeout, you'll get OB_ERR_SHARED_LOCK_CONFLICT error. */ enum class ObMDSGetTabletMode diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.cpp b/src/storage/tablet/ob_tablet_create_delete_helper.cpp index 128992860..0ed280243 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_helper.cpp @@ -16,7 +16,7 @@ #include "storage/tablet/ob_tablet_create_delete_helper.h" #include "share/rc/ob_module_provider.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_trans_service.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/meta_store/ob_storage_meta_io_util.h" @@ -142,7 +142,6 @@ int ObTabletCreateDeleteHelper::check_status_for_new_mds( ObTabletStatusCache &tablet_status_cache) { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; ObTabletCreateDeleteMdsUserData user_data; mds::MdsWriter writer;// will be removed later @@ -151,13 +150,13 @@ int ObTabletCreateDeleteHelper::check_status_for_new_mds( if (OB_UNLIKELY(tablet.is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet is empty shell", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("tablet is empty shell", K(ret), K(tablet_id), K(user_data)); } else if (OB_FAIL(tablet.get_latest(user_data, writer, trans_state, trans_version))) { if (OB_EMPTY_RESULT == ret) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet creation has not been committed, or has been roll backed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("tablet creation has not been committed, or has been roll backed", K(ret), K(tablet_id)); } else { - LOG_WARN("failed to get snapshot", KR(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to get snapshot", KR(ret), K(tablet_id)); } } else { const ObTabletStatus::Status &status = user_data.tablet_status_.get_status(); @@ -166,15 +165,14 @@ int ObTabletCreateDeleteHelper::check_status_for_new_mds( case ObTabletStatus::SPLIT_DST: ret = check_read_snapshot_for_normal_or_split_dst(tablet, snapshot_version, user_data, writer, trans_state, trans_version); break; - case ObTabletStatus::RESERVED_5: - ret = check_read_snapshot_for_reserved_5(tablet, snapshot_version, user_data, writer, trans_state, trans_version); - break; case ObTabletStatus::DELETED: - case ObTabletStatus::RESERVED_4: - ret = check_read_snapshot_for_deleted_or_reserved_4(tablet, snapshot_version, user_data, writer, trans_state, trans_version); + ret = check_read_snapshot_for_deleted(tablet, snapshot_version, user_data, writer, trans_state, trans_version); break; - case ObTabletStatus::RESERVED_6: - ret = check_read_snapshot_for_reserved_6(tablet, snapshot_version, user_data); + case ObTabletStatus::RESERVED_STATUS_5: + case ObTabletStatus::RESERVED_STATUS_4: + case ObTabletStatus::RESERVED_STATUS_6: + ret = OB_NOT_SUPPORTED; + LOG_WARN("reserved tablet transfer status is not supported", K(ret), K(tablet_id), K(user_data)); break; case ObTabletStatus::SPLIT_SRC: ret = check_read_snapshot_for_split_src(tablet, snapshot_version, user_data, trans_state); @@ -184,14 +182,14 @@ int ObTabletCreateDeleteHelper::check_status_for_new_mds( break; default: ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected tablet status", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("unexpected tablet status", K(ret), K(tablet_id), K(user_data)); } if (OB_FAIL(ret)) { } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state && (ObTabletStatus::NORMAL == user_data.tablet_status_ || ObTabletStatus::SPLIT_DST == user_data.tablet_status_)) { tablet_status_cache.set_value(user_data); - LOG_INFO("refresh tablet status cache", K(ret), K(ls_id), K(tablet_id), K(tablet_status_cache), K(snapshot_version)); + LOG_INFO("refresh tablet status cache", K(ret), K(tablet_id), K(tablet_status_cache), K(snapshot_version)); } } @@ -206,7 +204,6 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_by_commit_version( const ObTabletStatus &tablet_status) { int ret = OB_SUCCESS; - const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; if (snapshot_version == ObTransVersion::MAX_TRANS_VERSION) { @@ -214,32 +211,32 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_by_commit_version( } else if (OB_UNLIKELY(create_commit_version == ObTransVersion::INVALID_TRANS_VERSION)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("create tablet trans version is invalid", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(create_commit_version)); + K(ret), K(tablet_id), K(snapshot_version), K(create_commit_version)); } else if (snapshot_version < create_commit_version) { // read snapshot is smaller than create tablet trans version, // no previous committed transaction ret= OB_SNAPSHOT_DISCARDED; LOG_INFO("tablet status is set to MAX because read snapshot is smaller than create trans version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(create_commit_version)); + K(ret), K(tablet_id), K(snapshot_version), K(create_commit_version)); } else if (delete_commit_version == ObTransVersion::INVALID_TRANS_VERSION) { // delete commit version is not valid, no delete transaction committed } else if (snapshot_version < delete_commit_version) { // read snapshot is smaller than delete tablet trans version, // previous transaction is create tablet, so tablet status is NORMAL LOG_INFO("tablet status is set to NORMAL because read snapshot is smaller than delete trans version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(delete_commit_version)); + K(ret), K(tablet_id), K(snapshot_version), K(delete_commit_version)); } else { // snapshot_version >= user_data.delete_commit_version_ - ret = ObTabletStatus::RESERVED_6 == tablet_status ? OB_TABLET_NOT_EXIST : OB_TABLE_NOT_EXIST; - LOG_INFO("tablet is deleted or in reserved deleted state", - K(ret), K(ls_id), K(tablet_id), K(tablet_status), K(snapshot_version), K(delete_commit_version)); + ret = OB_TABLE_NOT_EXIST; + LOG_INFO("tablet is deleted", + K(ret), K(tablet_id), K(tablet_status), K(snapshot_version), K(delete_commit_version)); } if (OB_FAIL(ret)) { - } else if (ObTabletStatus::NORMAL == tablet_status || ObTabletStatus::RESERVED_5 == tablet_status || ObTabletStatus::SPLIT_DST == tablet_status) { + } else if (ObTabletStatus::NORMAL == tablet_status || ObTabletStatus::SPLIT_DST == tablet_status) { if (OB_UNLIKELY(tablet.is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet is empty shell", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(create_commit_version)); + LOG_WARN("tablet is empty shell", K(ret), K(tablet_id), K(snapshot_version), K(create_commit_version)); } } else { ret = OB_TABLET_NOT_EXIST; @@ -257,47 +254,45 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_normal_or_split_dst( const share::SCN &trans_version) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObTabletStatus &tablet_status = user_data.tablet_status_; share::SCN read_snapshot; if (OB_UNLIKELY(ObTabletStatus::NORMAL != tablet_status && ObTabletStatus::SPLIT_DST != tablet_status)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(user_data)); } else if (user_data.create_commit_version_ == ObTransVersion::MAX_TRANS_VERSION) { ret = OB_ERR_UNEXPECTED; LOG_WARN("create commit version is max trans version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); } else if (user_data.create_commit_version_ != ObTransVersion::INVALID_TRANS_VERSION) { - LOG_INFO("tablet create transaction is committed, currently in reserved status transaction", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); - if (OB_FAIL(check_read_snapshot_for_reserved_status_tx(tablet, snapshot_version, user_data))) { - LOG_WARN("fail to check readsnapshot for reserved status tx", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); + LOG_INFO("tablet create transaction is committed", + K(ret), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); + if (OB_FAIL(check_read_snapshot_for_committed_create_tx(tablet, snapshot_version, user_data))) { + LOG_WARN("fail to check read snapshot for committed create tx", + K(ret), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); } } else if (OB_FAIL(check_read_snapshot_for_create_tx( tablet, snapshot_version, user_data, writer, trans_state, trans_version))) { LOG_WARN("fail to check read snapshot for create tx", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); } return ret; } -int ObTabletCreateDeleteHelper::check_read_snapshot_for_reserved_status_tx( +int ObTabletCreateDeleteHelper::check_read_snapshot_for_committed_create_tx( const ObTablet &tablet, const int64_t snapshot_version, const ObTabletCreateDeleteMdsUserData &user_data) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; if (snapshot_version < user_data.create_commit_version_) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(user_data)); } return ret; @@ -312,7 +307,6 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_create_tx( const share::SCN &trans_version) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObTabletStatus &tablet_status = user_data.tablet_status_; share::SCN read_snapshot; @@ -320,32 +314,32 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_create_tx( if (trans_state < mds::TwoPhaseCommitState::ON_PREPARE) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("tablet creation transaction has not entered 2pc procedure", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(trans_state), K(user_data)); } else if (OB_FAIL(read_snapshot.convert_for_tx(snapshot_version))) { LOG_WARN("failed to convert from int64_t to SCN", K(ret), K(snapshot_version)); } else if (trans_state >= mds::TwoPhaseCommitState::ON_PREPARE && trans_state < mds::TwoPhaseCommitState::ON_COMMIT) { if (read_snapshot < trans_version) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("read snapshot is smaller than prepare version", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); + K(ret), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); } else { // primary tenant ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("tablet creation transaction has not committed", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); + K(ret), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state) { if (snapshot_version < user_data.create_commit_version_) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(user_data)); } } return ret; } -int ObTabletCreateDeleteHelper::check_read_snapshot_for_reserved_5( +int ObTabletCreateDeleteHelper::check_read_snapshot_for_deleted( const ObTablet &tablet, const int64_t snapshot_version, const ObTabletCreateDeleteMdsUserData &user_data, @@ -354,124 +348,46 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_reserved_5( const share::SCN &trans_version) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObTabletStatus &tablet_status = user_data.tablet_status_; share::SCN read_snapshot; - if (OB_UNLIKELY(ObTabletStatus::RESERVED_5 != tablet_status)) { + if (OB_UNLIKELY(ObTabletStatus::DELETED != tablet_status)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(user_data)); } else if (snapshot_version < user_data.create_commit_version_) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); - } else if (trans_state < mds::TwoPhaseCommitState::ON_PREPARE) { - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("reserved tablet status transaction has not entered 2pc procedure, should retry", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(trans_state)); - } else if (OB_FAIL(read_snapshot.convert_for_tx(snapshot_version))) { - LOG_WARN("failed to convert from int64_t to SCN", K(ret), K(snapshot_version)); - } else if (trans_state >= mds::TwoPhaseCommitState::ON_PREPARE && trans_state < mds::TwoPhaseCommitState::ON_COMMIT) { - if (read_snapshot < trans_version) { - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is smaller than prepare version, should retry", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); - } else { - // primary tenant: not allowed to read, retry - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is no smaller than prepare version, primary tenant should retry", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); - } - } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state) { - if (read_snapshot < trans_version) { - // not allow to read - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is smaller than reserved status commit version, should retry", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); - } - } - - return ret; -} - -int ObTabletCreateDeleteHelper::check_read_snapshot_for_deleted_or_reserved_4( - const ObTablet &tablet, - const int64_t snapshot_version, - const ObTabletCreateDeleteMdsUserData &user_data, - const mds::MdsWriter &writer, - const mds::TwoPhaseCommitState &trans_state, - const share::SCN &trans_version) -{ - int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; - const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; - const ObTabletStatus &tablet_status = user_data.tablet_status_; - share::SCN read_snapshot; - - if (OB_UNLIKELY(ObTabletStatus::RESERVED_4 != tablet_status && ObTabletStatus::DELETED != tablet_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); - } else if (snapshot_version < user_data.create_commit_version_) { - ret = OB_SNAPSHOT_DISCARDED; - LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(user_data)); } else if (OB_FAIL(read_snapshot.convert_for_tx(snapshot_version))) { LOG_WARN("failed to convert from int64_t to SCN", K(ret), K(snapshot_version)); } else if (trans_state < mds::TwoPhaseCommitState::ON_PREPARE) { if (read_snapshot.is_max()) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is MAX, maybe this is a write request, should retry", - K(ret), K(ls_id), K(tablet_id), K(read_snapshot), K(user_data)); + LOG_WARN("read snapshot is MAX, maybe this is a write request", + K(ret), K(tablet_id), K(read_snapshot), K(user_data)); } } else if (trans_state >= mds::TwoPhaseCommitState::ON_PREPARE && trans_state < mds::TwoPhaseCommitState::ON_COMMIT) { if (read_snapshot < trans_version) { // allow to read } else { - // primary tenant: retry ret = OB_TABLET_NOT_EXIST; - LOG_INFO("read snapshot is no smaller than prepare version on primary tenant, should retry", - K(ret), K(ls_id), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); + LOG_INFO("read snapshot is no smaller than prepare version", + K(ret), K(tablet_id), K(trans_state), K(read_snapshot), K(trans_version)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state) { if (read_snapshot < trans_version) { // allow to read } else { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is no smaller than trans version, should retry", - K(ret), K(ls_id), K(tablet_id), K(read_snapshot), K(trans_version)); + LOG_WARN("read snapshot is no smaller than delete transaction commit version", + K(ret), K(tablet_id), K(read_snapshot), K(trans_version)); } } return ret; } -int ObTabletCreateDeleteHelper::check_read_snapshot_for_reserved_6( - const ObTablet &tablet, - const int64_t snapshot_version, - const ObTabletCreateDeleteMdsUserData &user_data) -{ - int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; - const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; - const ObTabletStatus &tablet_status = user_data.tablet_status_; - - if (OB_UNLIKELY(ObTabletStatus::RESERVED_6 != tablet_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); - } else if (snapshot_version < user_data.create_commit_version_) { - ret = OB_SNAPSHOT_DISCARDED; - LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); - } else if (snapshot_version >= user_data.reserved_commit_version_) { - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("read snapshot is no smaller than reserved status commit version, should retry", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); - } - - return ret; -} - int ObTabletCreateDeleteHelper::check_read_snapshot_for_split_src( const ObTablet &tablet, const int64_t snapshot_version, @@ -479,21 +395,20 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_split_src( const mds::TwoPhaseCommitState &trans_state) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObTabletStatus &tablet_status = user_data.tablet_status_; if (OB_UNLIKELY(ObTabletStatus::SPLIT_SRC != tablet_status)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(user_data)); } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state) { if (snapshot_version < user_data.create_commit_version_) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("read snapshot smaller than create commit version", - K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(user_data)); + K(ret), K(tablet_id), K(snapshot_version), K(user_data)); } else { ret = OB_TABLET_IS_SPLIT_SRC; - LOG_WARN("tablet is split src", K(ret), K(ls_id), K(tablet_id), K(common::lbt())); + LOG_WARN("tablet is split src", K(ret), K(tablet_id), K(common::lbt())); } } @@ -506,16 +421,15 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_split_src_deleted( const mds::TwoPhaseCommitState &trans_state) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObTabletStatus &tablet_status = user_data.tablet_status_; if (OB_UNLIKELY(ObTabletStatus::SPLIT_SRC_DELETED != tablet_status)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(user_data)); + LOG_WARN("invalid args", K(ret), K(tablet_id), K(user_data)); } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_state) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("split src deleted", K(ret), K(ls_id), K(tablet_id), K(common::lbt())); + LOG_WARN("split src deleted", K(ret), K(tablet_id), K(common::lbt())); } return ret; @@ -524,18 +438,16 @@ int ObTabletCreateDeleteHelper::check_read_snapshot_for_split_src_deleted( int ObTabletCreateDeleteHelper::create_tmp_tablet( const ObTabletMapKey &key, common::ObArenaAllocator &allocator, + ObLS &ls, ObTabletHandle &handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLSService *ls_service = share::g_mp->ls_service(); + ObLS *tenant_ls = &ls; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(key)); - } else if (OB_FAIL(ls_service->get_ls(key.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { - LOG_WARN("fail to get ls", K(ret), "ls_id", key.ls_id_); - } else if (OB_FAIL(t3m->create_tmp_tablet(WashTabletPriority::WTP_HIGH, key, allocator, ls_handle, handle))) { + } else if (OB_FAIL(t3m->create_tmp_tablet(WashTabletPriority::WTP_HIGH, key, allocator, tenant_ls, handle))) { LOG_WARN("fail to create temporary tablet", K(ret), K(key)); } else if (OB_ISNULL(handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -559,15 +471,15 @@ int ObTabletCreateDeleteHelper::create_msd_tablet( ObTabletHandle &handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); if (OB_UNLIKELY(!key.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(key)); - } else if (OB_FAIL(ls_service->get_ls(key.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { - LOG_WARN("fail to get ls", K(ret), "ls_id", key.ls_id_); - } else if (OB_FAIL(t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle))) { + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("fail to get ls", K(ret)); + } else if (OB_FAIL(t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, tenant_ls, handle))) { LOG_WARN("fail to create multi source data tablet", K(ret), K(key)); } else if (OB_ISNULL(handle.get_obj())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.h b/src/storage/tablet/ob_tablet_create_delete_helper.h index 08f8bbd1e..f8fa483f5 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.h +++ b/src/storage/tablet/ob_tablet_create_delete_helper.h @@ -45,6 +45,7 @@ class ObTransID; namespace storage { class ObTabletMapKey; +class ObLS; class ObTabletCreateSSTableParam; class ObTableHandleV2; class ObTablet; @@ -88,30 +89,12 @@ class ObTabletCreateDeleteHelper const mds::TwoPhaseCommitState &trans_state, const share::SCN &trans_version); static int check_read_snapshot_for_deleted( - ObTablet &tablet, - const int64_t snapshot_version, - const ObTabletCreateDeleteMdsUserData &user_data, - const mds::MdsWriter &writer, - const mds::TwoPhaseCommitState &trans_state, - const share::SCN &trans_version); - static int check_read_snapshot_for_reserved_5( const ObTablet &tablet, const int64_t snapshot_version, const ObTabletCreateDeleteMdsUserData &user_data, const mds::MdsWriter &writer, const mds::TwoPhaseCommitState &trans_state, const share::SCN &trans_version); - static int check_read_snapshot_for_deleted_or_reserved_4( - const ObTablet &tablet, - const int64_t snapshot_version, - const ObTabletCreateDeleteMdsUserData &user_data, - const mds::MdsWriter &writer, - const mds::TwoPhaseCommitState &trans_state, - const share::SCN &trans_version); - static int check_read_snapshot_for_reserved_6( - const ObTablet &tablet, - const int64_t snapshot_version, - const ObTabletCreateDeleteMdsUserData &user_data); static int check_read_snapshot_for_split_src( const ObTablet &tablet, const int64_t snapshot_version, @@ -124,7 +107,7 @@ class ObTabletCreateDeleteHelper static int check_read_snapshot_by_commit_version( const int64_t snapshot_version, const ObTabletCreateDeleteMdsUserData &user_data); - static int check_read_snapshot_for_reserved_status_tx( + static int check_read_snapshot_for_committed_create_tx( const ObTablet &tablet, const int64_t snapshot_version, const ObTabletCreateDeleteMdsUserData &user_data); @@ -139,6 +122,7 @@ class ObTabletCreateDeleteHelper static int create_tmp_tablet( const ObTabletMapKey &key, common::ObArenaAllocator &allocator, + ObLS &ls, ObTabletHandle &handle); static int prepare_create_msd_tablet(); static int create_msd_tablet( @@ -172,13 +156,6 @@ class ObTabletCreateDeleteHelper T &sstable); static bool is_pure_hidden_tablets(const obcall::ObCreateTabletInfo &info); - template - static int process_for_old_mds( - const char *buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg); -private: - private: class ReadMdsFunctor { @@ -194,70 +171,6 @@ class ObTabletCreateDeleteHelper }; }; -template -int ObTabletCreateDeleteHelper::process_for_old_mds( - const char *buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg) -{ - int ret = OB_SUCCESS; - Arg arg; - bool is_old_mds = false; - - if (OB_ISNULL(buf) || OB_UNLIKELY(len <= 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid args", K(ret), KP(buf), K(len)); - } else if (OB_FAIL(Arg::is_old_mds(buf, len, is_old_mds))) { - TRANS_LOG(WARN, "failed to is_old_mds", K(ret), KP(buf), K(len)); - } else if (is_old_mds) { - do { - int64_t pos = 0; - if (OB_FAIL(arg.deserialize(buf, len, pos))) { - TRANS_LOG(WARN, "failed to deserialize", KR(ret), K(notify_arg), K(pos)); - if (notify_arg.for_replay_) { - ret = OB_EAGAIN; - } else { - usleep(100 * 1000); - } - } - } while (OB_FAIL(ret) && !notify_arg.for_replay_); - - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(!arg.is_valid())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "arg is invalid", K(ret), K(arg)); - } else if (!arg.is_old_mds_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "arg is not old mds, but buf is old mds", K(ret), K(arg)); - } else { - mds::MdsCtx mds_ctx{mds::MdsWriter{notify_arg.tx_id_}}; - mds_ctx.set_binding_type_id(mds::TupleTypeIdx::value); - - if (notify_arg.for_replay_) { - if (OB_FAIL(Helper::replay_process(arg, notify_arg.scn_, mds_ctx))) { - ret = OB_EAGAIN; - TRANS_LOG(WARN, "failed to replay_process", K(ret), K(notify_arg), K(arg)); - } - } else { - do { - if (OB_FAIL(Helper::register_process(arg, mds_ctx))) { - TRANS_LOG(ERROR, "fail to register_process, retry", K(ret), K(arg), K(notify_arg)); - usleep(100 * 1000); - } - } while (OB_FAIL(ret)); - } - - if (OB_FAIL(ret)) { - } else { - mds_ctx.single_log_commit(notify_arg.trans_version_, notify_arg.scn_); - TRANS_LOG(INFO, "replay create commit for old_mds", KR(ret), K(arg)); - } - } - } - - return ret; -}; - template int ObTabletCreateDeleteHelper::create_sstable( const ObTabletCreateSSTableParam ¶m, diff --git a/src/storage/tablet/ob_tablet_create_delete_mds_user_data.cpp b/src/storage/tablet/ob_tablet_create_delete_mds_user_data.cpp index 2c6fe2ca2..eed1e739d 100644 --- a/src/storage/tablet/ob_tablet_create_delete_mds_user_data.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_mds_user_data.cpp @@ -30,7 +30,6 @@ namespace storage ObTabletCreateDeleteMdsUserData::ObTabletCreateDeleteMdsUserData() : tablet_status_(ObTabletStatus::NONE), reserved_scn_(share::SCN::invalid_scn()), - reserved_ls_id_(), data_type_(ObTabletMdsUserDataType::NONE), create_commit_scn_(share::SCN::invalid_scn()), create_commit_version_(ObTransVersion::INVALID_TRANS_VERSION), @@ -47,7 +46,6 @@ ObTabletCreateDeleteMdsUserData::ObTabletCreateDeleteMdsUserData( const int64_t create_commit_version) : tablet_status_(status), reserved_scn_(share::SCN::invalid_scn()), - reserved_ls_id_(), data_type_(type), create_commit_scn_(share::SCN::invalid_scn()), create_commit_version_(create_commit_version), @@ -63,7 +61,6 @@ int ObTabletCreateDeleteMdsUserData::assign(const ObTabletCreateDeleteMdsUserDat int ret = OB_SUCCESS; tablet_status_ = other.tablet_status_; reserved_scn_ = other.reserved_scn_; - reserved_ls_id_ = other.reserved_ls_id_; data_type_ = other.data_type_; create_commit_scn_ = other.create_commit_scn_; create_commit_version_ = other.create_commit_version_; @@ -78,7 +75,6 @@ void ObTabletCreateDeleteMdsUserData::reset() { tablet_status_ = ObTabletStatus::NONE; reserved_scn_.set_invalid(); - reserved_ls_id_.reset(); data_type_ = ObTabletMdsUserDataType::NONE; create_commit_scn_.set_invalid(); create_commit_version_ = ObTransVersion::INVALID_TRANS_VERSION; @@ -97,25 +93,23 @@ void ObTabletCreateDeleteMdsUserData::on_init() void ObTabletCreateDeleteMdsUserData::on_redo(const share::SCN &redo_scn) { + UNUSED(redo_scn); int ret = OB_SUCCESS; switch (data_type_) { case ObTabletMdsUserDataType::NONE : case ObTabletMdsUserDataType::CREATE_TABLET : case ObTabletMdsUserDataType::REMOVE_TABLET : - case ObTabletMdsUserDataType::RESERVED_7: - case ObTabletMdsUserDataType::RESERVED_4 : - case ObTabletMdsUserDataType::RESERVED_5 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_3 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_4 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_5 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_6 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_7 : case ObTabletMdsUserDataType::START_SPLIT_SRC : case ObTabletMdsUserDataType::START_SPLIT_DST : case ObTabletMdsUserDataType::FINISH_SPLIT_SRC : case ObTabletMdsUserDataType::FINISH_SPLIT_DST : { break; } - case ObTabletMdsUserDataType::RESERVED_3 : - case ObTabletMdsUserDataType::RESERVED_6 : { - reserved_scn_on_redo_(redo_scn); - break; - } default: { ret = OB_INVALID_ARGUMENT; LOG_ERROR("invalid cur status for fail", K(ret), KPC(this)); @@ -123,19 +117,16 @@ void ObTabletCreateDeleteMdsUserData::on_redo(const share::SCN &redo_scn) } } -void ObTabletCreateDeleteMdsUserData::reserved_scn_on_redo_(const share::SCN &redo_scn) -{ - reserved_scn_ = redo_scn; - LOG_INFO("reserved tablet status redo", KPC(this)); -} - void ObTabletCreateDeleteMdsUserData::on_commit(const share::SCN &commit_version, const share::SCN &commit_scn) { int ret = OB_SUCCESS; switch (data_type_) { case ObTabletMdsUserDataType::NONE : - case ObTabletMdsUserDataType::RESERVED_7: - case ObTabletMdsUserDataType::RESERVED_6 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_3 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_4 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_5 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_6 : + case ObTabletMdsUserDataType::RESERVED_MDS_TYPE_7 : case ObTabletMdsUserDataType::FINISH_SPLIT_DST : { break; } @@ -143,22 +134,10 @@ void ObTabletCreateDeleteMdsUserData::on_commit(const share::SCN &commit_version create_tablet_on_commit_(commit_version, commit_scn); break; } - case ObTabletMdsUserDataType::RESERVED_4 : { - reserved_start_on_commit_(commit_version); - break; - } case ObTabletMdsUserDataType::REMOVE_TABLET : { delete_tablet_on_commit_(commit_version, commit_scn); break; } - case ObTabletMdsUserDataType::RESERVED_5 : { - reserved_finish_on_commit_(commit_version, commit_scn); - break; - } - case ObTabletMdsUserDataType::RESERVED_3 : { - reserved_start_on_commit_(commit_version); - break; - } case ObTabletMdsUserDataType::START_SPLIT_SRC : { start_split_src_on_commit_(commit_version); break; @@ -196,22 +175,6 @@ void ObTabletCreateDeleteMdsUserData::delete_tablet_on_commit_( LOG_INFO("delete tablet commit", KPC(this)); } -void ObTabletCreateDeleteMdsUserData::reserved_start_on_commit_( - const share::SCN &commit_version) -{ - reserved_commit_version_ = commit_version.get_val_for_tx(); - LOG_INFO("reserved tablet status start commit", KPC(this)); -} - -void ObTabletCreateDeleteMdsUserData::reserved_finish_on_commit_( - const share::SCN &commit_version, - const share::SCN &commit_scn) -{ - delete_commit_scn_ = commit_scn; - delete_commit_version_ = commit_version.get_val_for_tx(); - LOG_INFO("reserved tablet status finish commit", KPC(this)); -} - void ObTabletCreateDeleteMdsUserData::start_split_src_on_commit_( const share::SCN &commit_version) { @@ -235,39 +198,28 @@ void ObTabletCreateDeleteMdsUserData::finish_split_src_on_commit_( LOG_INFO("split src delete tablet commit", KPC(this)); } -int ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger( - const share::ObLSID &ls_id) +int ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObLSService *ls_service = share::g_mp->ls_service(); - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { - ls->get_tablet_gc_handler()->set_tablet_gc_trigger(); + tenant_ls->get_tablet_gc_handler()->set_tablet_gc_trigger(); } return ret; } -int ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger( - const share::ObLSID &ls_id) +int ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObLSService *ls_service = share::g_mp->ls_service(); - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { - ls->get_tablet_empty_shell_handler()->set_empty_shell_trigger(true); - LOG_INFO("set tablet empty shell trigger", K(ret), K(ls_id), "handler", ls->get_tablet_empty_shell_handler()); + tenant_ls->get_tablet_empty_shell_handler()->set_empty_shell_trigger(true); + LOG_INFO("set tablet empty shell trigger", K(ret), + "handler", tenant_ls->get_tablet_empty_shell_handler()); } return ret; } @@ -276,7 +228,6 @@ OB_SERIALIZE_MEMBER( ObTabletCreateDeleteMdsUserData, tablet_status_, reserved_scn_, - reserved_ls_id_, data_type_, create_commit_scn_, create_commit_version_, diff --git a/src/storage/tablet/ob_tablet_create_delete_mds_user_data.h b/src/storage/tablet/ob_tablet_create_delete_mds_user_data.h index 5c0ddd387..e779f87ef 100644 --- a/src/storage/tablet/ob_tablet_create_delete_mds_user_data.h +++ b/src/storage/tablet/ob_tablet_create_delete_mds_user_data.h @@ -20,7 +20,6 @@ #include #include "lib/utility/ob_print_utils.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "storage/tablet/ob_tablet_status.h" #include "storage/tablet/ob_tablet_common.h" @@ -36,11 +35,11 @@ enum class ObTabletMdsUserDataType : int64_t CREATE_TABLET = 1, //for drop tablet REMOVE_TABLET = 2, - RESERVED_3 = 3, - RESERVED_4 = 4, - RESERVED_5 = 5, - RESERVED_6 = 6, - RESERVED_7 = 7, + RESERVED_MDS_TYPE_3 = 3, + RESERVED_MDS_TYPE_4 = 4, + RESERVED_MDS_TYPE_5 = 5, + RESERVED_MDS_TYPE_6 = 6, + RESERVED_MDS_TYPE_7 = 7, // for start split src START_SPLIT_SRC = 8, // for start split dst @@ -55,7 +54,7 @@ enum class ObTabletMdsUserDataType : int64_t class ObTabletCreateDeleteMdsUserData { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObTabletCreateDeleteMdsUserData(); ~ObTabletCreateDeleteMdsUserData() = default; @@ -72,20 +71,17 @@ class ObTabletCreateDeleteMdsUserData void on_redo(const share::SCN &redo_scn); void on_commit(const share::SCN &commit_version, const share::SCN &commit_scn); // todo(zk250686): tablet shell - static int set_tablet_gc_trigger(const share::ObLSID &ls_id); - static int set_tablet_empty_shell_trigger(const share::ObLSID &ls_id); + static int set_tablet_gc_trigger(); + static int set_tablet_empty_shell_trigger(); TO_STRING_KV(K_(tablet_status), K_(reserved_scn), - K_(reserved_ls_id), K_(data_type), + K_(data_type), K_(create_commit_scn), K_(create_commit_version), K_(delete_commit_scn), K_(delete_commit_version), K_(reserved_commit_version), K_(start_split_commit_version)); private: - void reserved_scn_on_redo_(const share::SCN &redo_scn); void create_tablet_on_commit_(const share::SCN &commit_version, const share::SCN &commit_scn); void delete_tablet_on_commit_(const share::SCN &commit_version, const share::SCN &commit_scn); - void reserved_start_on_commit_(const share::SCN &commit_version); - void reserved_finish_on_commit_(const share::SCN &commit_version, const share::SCN &commit_scn); void start_split_src_on_commit_(const share::SCN &commit_version); void start_split_dst_on_commit_(const share::SCN &commit_version); @@ -93,7 +89,6 @@ class ObTabletCreateDeleteMdsUserData public: ObTabletStatus tablet_status_; share::SCN reserved_scn_; - share::ObLSID reserved_ls_id_; ObTabletMdsUserDataType data_type_; // create_commit_scn_ remain unchanged throughout the entire tablet lifecycle diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.cpp b/src/storage/tablet/ob_tablet_create_mds_helper.cpp index 13f234ea0..6a2f9d008 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_mds_helper.cpp @@ -36,15 +36,6 @@ namespace storage ERRSIM_POINT_DEF(EN_CREATE_TABLET_FAILED); -int ObTabletCreateMdsHelper::on_commit_for_old_mds( - const char* buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg) -{ - mds::TLOCAL_MDS_INFO.reset();// disable runtime check - return ObTabletCreateDeleteHelper::process_for_old_mds(buf, len, notify_arg); -} - int ObTabletCreateMdsHelper::register_process( const ObBatchCreateTabletArg &arg, mds::BufferCtx &ctx) @@ -64,20 +55,17 @@ int ObTabletCreateMdsHelper::register_process( LOG_WARN("failed to create tablets", K(ret), K(arg)); } else if (!arg.set_binding_info_outside_create() && CLICK_FAIL(ObTabletBindingHelper::modify_tablet_binding_for_new_mds_create(arg, SCN::invalid_scn(), ctx))) { LOG_WARN("failed to modify tablet binding", K(ret)); - } else { - mds::ObTabletCreateMdsCtx &mds_ctx = static_cast(ctx); - mds_ctx.set_ls_id(arg.id_); } if (OB_FAIL(ret)) { // roll back int tmp_ret = OB_SUCCESS; - if (CLICK_TMP_FAIL(rollback_remove_tablets(arg.id_, tablet_id_array))) { + if (CLICK_TMP_FAIL(rollback_remove_tablets(tablet_id_array))) { LOG_ERROR("failed to roll back remove tablets", K(tmp_ret)); ob_usleep(1 * 1000 * 1000); ob_abort(); } - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(arg.id_))) { + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger())) { LOG_WARN("failed to set tablet gc trigger", K(ret)); } LOG_INFO("create tablet register", KR(ret), "arg", PRETTY_ARG(arg)); @@ -102,9 +90,6 @@ int ObTabletCreateMdsHelper::on_register( } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("arg is invalid", K(ret), "arg", PRETTY_ARG(arg)); - } else if (arg.is_old_mds_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, arg is old mds", K(ret), K(arg)); } else if (OB_FAIL(convert_schemas(arg))) { LOG_WARN("failed to convert_schemas", K(ret), "arg", PRETTY_ARG(arg)); } else if (CLICK_FAIL(check_create_new_tablets(arg, false/*is_replay*/))) { @@ -123,36 +108,32 @@ int ObTabletCreateMdsHelper::replay_process( MDS_TG(1_s); int ret = OB_SUCCESS; common::ObSArray tablet_id_array; - const ObLSID &ls_id = arg.id_; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; share::SCN tablet_change_checkpoint_scn; if (CLICK_FAIL(tablet_id_array.reserve(arg.get_tablet_count()))) { LOG_WARN("failed to reserve memory", K(ret), "capacity", arg.get_tablet_count()); - } else if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); - } else if (FALSE_IT(tablet_change_checkpoint_scn = ls->get_tablet_change_checkpoint_scn())) { - } else if (scn <= tablet_change_checkpoint_scn) { - LOG_INFO("current scn is smaller than ls tablet change check point scn, log replaying can be skipped", - K(ret), K(scn), K(tablet_change_checkpoint_scn)); - } else if (CLICK_FAIL(create_tablets(arg, true/*for_replay*/, scn, ctx, tablet_id_array))) { - LOG_WARN("failed to create tablets", K(ret), K(arg), K(scn)); - } else if (!arg.set_binding_info_outside_create() && CLICK_FAIL(ObTabletBindingHelper::modify_tablet_binding_for_new_mds_create(arg, scn, ctx))) { - LOG_WARN("failed to modify tablet binding", K(ret)); - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(ls_id))) { - LOG_WARN("failed to trigger tablet gc task", K(ret)); + } else if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { - mds::ObTabletCreateMdsCtx &mds_ctx = static_cast(ctx); - mds_ctx.set_ls_id(ls_id); + tablet_change_checkpoint_scn = tenant_ls->get_tablet_change_checkpoint_scn(); + if (scn <= tablet_change_checkpoint_scn) { + LOG_INFO("current scn is smaller than ls tablet change check point scn, log replaying can be skipped", + K(ret), K(scn), K(tablet_change_checkpoint_scn)); + } else if (CLICK_FAIL(create_tablets(arg, true/*for_replay*/, scn, ctx, tablet_id_array))) { + LOG_WARN("failed to create tablets", K(ret), K(arg), K(scn)); + } else if (!arg.set_binding_info_outside_create() + && CLICK_FAIL(ObTabletBindingHelper::modify_tablet_binding_for_new_mds_create( + arg, scn, ctx))) { + LOG_WARN("failed to modify tablet binding", K(ret)); + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger())) { + LOG_WARN("failed to trigger tablet gc task", K(ret)); + } } if (CLICK_FAIL(ret)) { // roll back int tmp_ret = OB_SUCCESS; - if (CLICK() && OB_TMP_FAIL(rollback_remove_tablets(ls_id, tablet_id_array))) { + if (CLICK() && OB_TMP_FAIL(rollback_remove_tablets(tablet_id_array))) { LOG_ERROR("failed to roll back remove tablets", K(tmp_ret)); ob_usleep(1 * 1000 * 1000); ob_abort(); @@ -181,8 +162,6 @@ int ObTabletCreateMdsHelper::on_replay( } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("arg is invalid", K(ret), "arg", PRETTY_ARG(arg)); - } else if (arg.is_old_mds_) { - LOG_INFO("skip replay create tablet for old mds", K(ret), K(scn), "arg", PRETTY_ARG(arg)); } else if (OB_FAIL(convert_schemas(arg))) { LOG_WARN("failed to convert_schemas", K(ret), "arg", PRETTY_ARG(arg)); } else if (CLICK_FAIL(check_create_new_tablets(arg, true/*is_replay*/))) { @@ -293,7 +272,6 @@ int ObTabletCreateMdsHelper::check_create_arg( { int ret = OB_SUCCESS; valid = true; - const ObLSID &ls_id = arg.id_; int64_t aux_info_idx = -1; ObSArray aux_info_idx_array; @@ -303,12 +281,12 @@ int ObTabletCreateMdsHelper::check_create_arg( if (is_contain(aux_info_idx_array, i)) { // do nothing } else if (is_pure_data_tablets(info) || is_mixed_tablets(info)) { - if (OB_FAIL(check_pure_data_or_mixed_tablets_info(ls_id, info, valid))) { - LOG_WARN("failed to check create tablet info", K(ret), K(ls_id), K(info)); + if (OB_FAIL(check_pure_data_or_mixed_tablets_info(info, valid))) { + LOG_WARN("failed to check create tablet info", K(ret), K(info)); } } else if (is_pure_aux_tablets(info)) { - if (OB_FAIL(check_pure_aux_tablets_info(ls_id, info, valid))) { - LOG_WARN("failed to check create tablet info", K(ret), K(ls_id), K(info)); + if (OB_FAIL(check_pure_aux_tablets_info(info, valid))) { + LOG_WARN("failed to check create tablet info", K(ret), K(info)); } } else if (is_bind_hidden_tablets(info)) { for (int64_t i = 0; OB_SUCC(ret) && i < info.tablet_ids_.count(); ++i) { @@ -316,13 +294,13 @@ int ObTabletCreateMdsHelper::check_create_arg( bool has_related_aux_info = find_aux_info_for_hidden_tablets(arg, tablet_id, aux_info_idx); if (has_related_aux_info) { const ObCreateTabletInfo &aux_info = arg.tablets_.at(aux_info_idx); - if (OB_FAIL(check_hidden_tablets_info(ls_id, info, &aux_info, valid))) { - LOG_WARN("failed to check create tablet info", K(ret), K(ls_id), K(info), K(aux_info)); + if (OB_FAIL(check_hidden_tablets_info(info, &aux_info, valid))) { + LOG_WARN("failed to check create tablet info", K(ret), K(info), K(aux_info)); } else if (OB_FAIL(aux_info_idx_array.push_back(aux_info_idx))) { LOG_WARN("failed to push back aux info idx", K(ret)); } - } else if (OB_FAIL(check_hidden_tablets_info(ls_id, info, nullptr/*aux_info*/, valid))) { - LOG_WARN("failed to check create tablet info", K(ret), K(ls_id), K(info)); + } else if (OB_FAIL(check_hidden_tablets_info(info, nullptr/*aux_info*/, valid))) { + LOG_WARN("failed to check create tablet info", K(ret), K(info)); } } } @@ -426,7 +404,6 @@ bool ObTabletCreateMdsHelper::is_bind_hidden_tablets(const obcall::ObCreateTable } int ObTabletCreateMdsHelper::check_pure_data_or_mixed_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &info, bool &valid) { @@ -434,7 +411,6 @@ int ObTabletCreateMdsHelper::check_pure_data_or_mixed_tablets_info( bool exist = false; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletMapKey key; - key.ls_id_ = ls_id; for (int64_t i = 0; OB_SUCC(ret) && !exist && i < info.tablet_ids_.count(); ++i) { const ObTabletID &tablet_id = info.tablet_ids_[i]; @@ -455,7 +431,6 @@ int ObTabletCreateMdsHelper::check_pure_data_or_mixed_tablets_info( } int ObTabletCreateMdsHelper::check_pure_aux_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &info, bool &valid) { @@ -463,7 +438,6 @@ int ObTabletCreateMdsHelper::check_pure_aux_tablets_info( bool exist = false; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletMapKey key; - key.ls_id_ = ls_id; for (int64_t i = 0; OB_SUCC(ret) && !exist && i < info.tablet_ids_.count(); ++i) { const ObTabletID &tablet_id = info.tablet_ids_[i]; @@ -494,7 +468,6 @@ int ObTabletCreateMdsHelper::check_pure_aux_tablets_info( } int ObTabletCreateMdsHelper::check_hidden_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &hidden_info, const obcall::ObCreateTabletInfo *aux_info, bool &valid) @@ -503,7 +476,6 @@ int ObTabletCreateMdsHelper::check_hidden_tablets_info( bool exist = false; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletMapKey key; - key.ls_id_ = ls_id; for (int64_t i = 0; OB_SUCC(ret) && !exist && i < hidden_info.tablet_ids_.count(); ++i) { const ObTabletID &tablet_id = hidden_info.tablet_ids_[i]; @@ -651,7 +623,6 @@ int ObTabletCreateMdsHelper::build_pure_data_tablet( { MDS_TG(5_ms); int ret = OB_SUCCESS; - const ObLSID &ls_id = arg.id_; const ObTabletID &data_tablet_id = info.data_tablet_id_; const ObCreateTabletSchema *create_tablet_schema = nullptr; const int64_t create_commit_version = info.create_commit_versions_.empty() ? ObTransVersion::INVALID_TRANS_VERSION @@ -665,25 +636,22 @@ int ObTabletCreateMdsHelper::build_pure_data_tablet( ObTabletHandle tablet_handle; bool exist = false; int64_t index = -1; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; bool need_create_empty_major_sstable = true; bool micro_index_clustered = false; ObTabletID split_src_tablet_id; share::ObForkTabletInfo fork_tablet_info; - if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else if (for_replay) { - const ObTabletMapKey key(ls_id, data_tablet_id); - if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet(key, ls, tablet_handle))) { + const ObTabletMapKey key(data_tablet_id); + if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet( + key, tenant_ls, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { exist = false; ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("failed to get tablet", K(ret), K(data_tablet_id)); } } else { exist = true; @@ -693,14 +661,14 @@ int ObTabletCreateMdsHelper::build_pure_data_tablet( if (CLICK_FAIL(ret)) { } else if (for_replay && exist) { LOG_INFO("create pure data tablet is already exist, skip it", K(ret), K(for_replay), K(exist), - K(ls_id), K(data_tablet_id)); + K(data_tablet_id)); } else if (CLICK_FAIL(get_table_schema_index(data_tablet_id, info.tablet_ids_, index))) { - LOG_WARN("failed to get table schema index", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("failed to get table schema index", K(ret), K(data_tablet_id)); } else if (OB_UNLIKELY(index < 0)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, table schema index is invalid", K(ret), K(ls_id), K(data_tablet_id), K(index)); + LOG_WARN("unexpected error, table schema index is invalid", K(ret), K(data_tablet_id), K(index)); } else if (CLICK_FAIL(tablet_id_array.push_back(data_tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("failed to push back tablet id", K(ret), K(data_tablet_id)); } else if (OB_FAIL(check_and_get_create_tablet_schema_info(create_tablet_schemas, create_tablet_extra_infos, info, index, create_tablet_schema, need_create_empty_major_sstable, micro_index_clustered, split_src_tablet_id))) { LOG_WARN("check and get create tablet schema_info failed", K(ret)); @@ -708,18 +676,18 @@ int ObTabletCreateMdsHelper::build_pure_data_tablet( // using need_create_empty_major_sstable to determine tablet build by the offline ddl } else if (OB_FAIL(info.get_fork_tablet_info(index, fork_tablet_info))) { LOG_WARN("failed to get fork tablet info", K(ret), K(index)); - } else if (CLICK_FAIL(ls->get_tablet_svr()->create_tablet(ls_id, data_tablet_id, data_tablet_id, + } else if (CLICK_FAIL(tenant_ls->get_tablet_svr()->create_tablet(data_tablet_id, data_tablet_id, scn, snapshot_version, *create_tablet_schema, compat_mode, need_create_empty_major_sstable, clog_checkpoint_scn, mds_checkpoint_scn, arg.create_type_, micro_index_clustered, split_src_tablet_id, data_format_version, tablet_handle, fork_tablet_info))) { - LOG_ERROR("failed to do create tablet", K(ret), K(ls_id), K(data_tablet_id), "arg", PRETTY_ARG(arg)); + LOG_ERROR("failed to do create tablet", K(ret), K(data_tablet_id), "arg", PRETTY_ARG(arg)); } if (OB_FAIL(ret)) { - } else if (CLICK_FAIL(set_tablet_status(ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, - arg.is_old_mds_, arg.create_type_, create_commit_version))) { - LOG_WARN("failed to set tablet normal status", K(ret), K(ls_id), K(data_tablet_id)); + } else if (CLICK_FAIL(set_tablet_status(tenant_ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, + arg.create_type_, create_commit_version))) { + LOG_WARN("failed to set tablet normal status", K(ret), K(data_tablet_id)); } return ret; @@ -735,7 +703,6 @@ int ObTabletCreateMdsHelper::build_mixed_tablets( { MDS_TG(10_ms); int ret = OB_SUCCESS; - const ObLSID &ls_id = arg.id_; const ObTabletID &data_tablet_id = info.data_tablet_id_; const ObSArray &tablet_ids = info.tablet_ids_; const ObIArray &create_commit_versions = info.create_commit_versions_; @@ -750,14 +717,10 @@ int ObTabletCreateMdsHelper::build_mixed_tablets( ObTabletID lob_meta_tablet_id; ObTabletID lob_piece_tablet_id; bool exist = false; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; - if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { @@ -782,13 +745,14 @@ int ObTabletCreateMdsHelper::build_mixed_tablets( if (OB_FAIL(ret)) { } else if (for_replay) { - const ObTabletMapKey key(ls_id, tablet_id); - if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet(key, ls, tablet_handle))) { + const ObTabletMapKey key(tablet_id); + if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet( + key, tenant_ls, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { exist = false; ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(data_tablet_id), K(tablet_id)); + LOG_WARN("failed to get tablet", K(ret), K(data_tablet_id), K(tablet_id)); } } else { exist = true; @@ -798,25 +762,25 @@ int ObTabletCreateMdsHelper::build_mixed_tablets( uint64_t data_format_version = 0; if (OB_FAIL(ret)) { } else if (for_replay && exist) { - LOG_INFO("tablet already exists in replay procedure, skip it", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("tablet already exists in replay procedure, skip it", K(ret), K(tablet_id)); } else if (CLICK_FAIL(tablet_id_array.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); } else if (FALSE_IT(data_format_version = create_tablet_extra_infos[info.table_schema_index_[i]].need_create_empty_major_ ? 0 : create_tablet_extra_infos[i].tenant_data_version_)) { // using need_create_empty_major_sstable to determine tablet build by the offline ddl } else if (OB_FAIL(info.get_fork_tablet_info(i, fork_tablet_info))) { LOG_WARN("failed to get fork tablet info", K(ret), K(i)); - } else if (CLICK_FAIL(ls->get_tablet_svr()->create_tablet(ls_id, tablet_id, data_tablet_id, + } else if (CLICK_FAIL(tenant_ls->get_tablet_svr()->create_tablet(tablet_id, data_tablet_id, scn, snapshot_version, *create_tablet_schema, compat_mode, need_create_empty_major_sstable, clog_checkpoint_scn, mds_checkpoint_scn, arg.create_type_, micro_index_clustered, split_src_tablet_id, data_format_version, tablet_handle, fork_tablet_info))) { - LOG_ERROR("failed to do create tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), "arg", PRETTY_ARG(arg)); + LOG_ERROR("failed to do create tablet", K(ret), K(tablet_id), K(data_tablet_id), "arg", PRETTY_ARG(arg)); } if (OB_FAIL(ret)) { - } else if (CLICK_FAIL(set_tablet_status(ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, - arg.is_old_mds_, arg.create_type_, create_commit_version))) { - LOG_WARN("failed to set tablet normal status", K(ret), K(ls_id), K(tablet_id)); + } else if (CLICK_FAIL(set_tablet_status(tenant_ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, + arg.create_type_, create_commit_version))) { + LOG_WARN("failed to set tablet normal status", K(ret), K(tablet_id)); } if (OB_FAIL(ret)) { @@ -831,7 +795,7 @@ int ObTabletCreateMdsHelper::build_mixed_tablets( ObTablet *data_tablet = data_tablet_handle.get_obj(); if (OB_ISNULL(data_tablet)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("data tablet is null", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("data tablet is null", K(ret), K(data_tablet_id)); } else { // binding info } @@ -850,7 +814,6 @@ int ObTabletCreateMdsHelper::build_pure_aux_tablets( { MDS_TG(10_ms); int ret = OB_SUCCESS; - const ObLSID &ls_id = arg.id_; const ObTabletID &data_tablet_id = info.data_tablet_id_; const ObSArray &tablet_ids = info.tablet_ids_; const ObIArray &create_commit_versions = info.create_commit_versions_; @@ -862,14 +825,10 @@ int ObTabletCreateMdsHelper::build_pure_aux_tablets( const share::SCN &mds_checkpoint_scn = arg.mds_checkpoint_scn_; ObTabletHandle tablet_handle; bool exist = false; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; - if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { @@ -884,13 +843,14 @@ int ObTabletCreateMdsHelper::build_pure_aux_tablets( ObTabletID split_src_tablet_id; share::ObForkTabletInfo fork_tablet_info; if (for_replay) { - const ObTabletMapKey key(ls_id, tablet_id); - if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet(key, ls, tablet_handle))) { + const ObTabletMapKey key(tablet_id); + if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet( + key, tenant_ls, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { exist = false; ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("failed to get tablet", K(ret), K(data_tablet_id)); } } else { exist = true; @@ -900,9 +860,9 @@ int ObTabletCreateMdsHelper::build_pure_aux_tablets( if (OB_FAIL(ret)) { } else if (for_replay && exist) { LOG_INFO("create pure aux tablet is already exist, skip it", K(ret), K(for_replay), K(exist), - K(ls_id), K(data_tablet_id), K(tablet_id)); + K(data_tablet_id), K(tablet_id)); } else if (CLICK_FAIL(tablet_id_array.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); } else if (OB_FAIL(check_and_get_create_tablet_schema_info(create_tablet_schemas, create_tablet_extra_infos, info, i, create_tablet_schema, need_create_empty_major_sstable, micro_index_clustered, split_src_tablet_id))) { LOG_WARN("check and get create tablet schema_info failed", K(ret)); @@ -910,18 +870,18 @@ int ObTabletCreateMdsHelper::build_pure_aux_tablets( // using need_create_empty_major_sstable to determine tablet build by the offline ddl } else if (OB_FAIL(info.get_fork_tablet_info(i, fork_tablet_info))) { LOG_WARN("failed to get fork tablet info", K(ret), K(i)); - } else if (CLICK_FAIL(ls->get_tablet_svr()->create_tablet(ls_id, tablet_id, data_tablet_id, + } else if (CLICK_FAIL(tenant_ls->get_tablet_svr()->create_tablet(tablet_id, data_tablet_id, scn, snapshot_version, *create_tablet_schema, compat_mode, need_create_empty_major_sstable, clog_checkpoint_scn, mds_checkpoint_scn, arg.create_type_, micro_index_clustered, split_src_tablet_id, data_format_version, tablet_handle, fork_tablet_info))) { - LOG_ERROR("failed to do create tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), "arg", PRETTY_ARG(arg)); + LOG_ERROR("failed to do create tablet", K(ret), K(tablet_id), K(data_tablet_id), "arg", PRETTY_ARG(arg)); } if (OB_FAIL(ret)) { - } else if (CLICK_FAIL(set_tablet_status(ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, - arg.is_old_mds_, arg.create_type_, create_commit_version))) { - LOG_WARN("failed to set tablet normal status", K(ret), K(ls_id), K(tablet_id)); + } else if (CLICK_FAIL(set_tablet_status(tenant_ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, + arg.create_type_, create_commit_version))) { + LOG_WARN("failed to set tablet normal status", K(ret), K(tablet_id)); } } @@ -940,7 +900,6 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( { MDS_TG(10_ms); int ret = OB_SUCCESS; - const ObLSID &ls_id = arg.id_; const ObTabletID &orig_tablet_id = info.data_tablet_id_; const ObSArray &tablet_ids = info.tablet_ids_; const ObIArray &create_commit_versions = info.create_commit_versions_; @@ -955,14 +914,10 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( ObTabletID lob_meta_tablet_id; ObTabletID lob_piece_tablet_id; bool exist = false; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; - if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { @@ -1000,13 +955,14 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( if (OB_FAIL(ret)) { } else if (for_replay) { - const ObTabletMapKey key(ls_id, tablet_id); - if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet(key, ls, tablet_handle))) { + const ObTabletMapKey key(tablet_id); + if (CLICK_FAIL(ObTabletCreateDeleteHelper::replay_mds_get_tablet( + key, tenant_ls, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { exist = false; ret = OB_SUCCESS; } else { - LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(orig_tablet_id), K(tablet_id)); + LOG_WARN("failed to get tablet", K(ret), K(orig_tablet_id), K(tablet_id)); } } else { exist = true; @@ -1016,25 +972,25 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( if (OB_FAIL(ret)) { } else if (for_replay && exist) { LOG_INFO("create hidden tablet is already exist, skip it", K(ret), K(for_replay), K(exist), - K(ls_id), K(orig_tablet_id), K(tablet_id)); + K(orig_tablet_id), K(tablet_id)); } else if (CLICK_FAIL(tablet_id_array.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); } else if (FALSE_IT(data_format_version = create_tablet_extra_infos[info.table_schema_index_[i]].need_create_empty_major_ ? 0 : create_tablet_extra_infos[i].tenant_data_version_)) { // using need_create_empty_major_sstable to determine tablet build by the offline ddl } else if (OB_FAIL(info.get_fork_tablet_info(i, fork_tablet_info))) { LOG_WARN("failed to get fork tablet info", K(ret), K(i)); - } else if (CLICK_FAIL(ls->get_tablet_svr()->create_tablet(ls_id, tablet_id, tablet_id, + } else if (CLICK_FAIL(tenant_ls->get_tablet_svr()->create_tablet(tablet_id, tablet_id, scn, snapshot_version, *create_tablet_schema, compat_mode, need_create_empty_major_sstable, clog_checkpoint_scn, mds_checkpoint_scn, arg.create_type_, micro_index_clustered, split_src_tablet_id, data_format_version, tablet_handle, fork_tablet_info))) { - LOG_ERROR("failed to do create tablet", K(ret), K(ls_id), K(tablet_id), K(orig_tablet_id), "arg", PRETTY_ARG(arg)); + LOG_ERROR("failed to do create tablet", K(ret), K(tablet_id), K(orig_tablet_id), "arg", PRETTY_ARG(arg)); } if (OB_FAIL(ret)) { - } else if (CLICK_FAIL(set_tablet_status(ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, - arg.is_old_mds_, arg.create_type_, create_commit_version))) { - LOG_WARN("failed to set tablet normal status", K(ret), K(ls_id), K(tablet_id)); + } else if (CLICK_FAIL(set_tablet_status(tenant_ls->get_tablet_svr(), tablet_handle, for_replay, scn, ctx, + arg.create_type_, create_commit_version))) { + LOG_WARN("failed to set tablet normal status", K(ret), K(tablet_id)); } // process lob meta/piece tablet @@ -1044,25 +1000,20 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( } int ObTabletCreateMdsHelper::rollback_remove_tablets( - const share::ObLSID &ls_id, const common::ObIArray &tablet_id_array) { MDS_TG(100_ms); int ret = OB_SUCCESS; - ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); + ObLS *tenant_ls = nullptr; + + if (CLICK_FAIL(get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) { MDS_TG(10_ms); const common::ObTabletID &tablet_id = tablet_id_array.at(i); - if (CLICK_FAIL(ls->get_tablet_svr()->rollback_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + if (CLICK_FAIL(tenant_ls->get_tablet_svr()->rollback_remove_tablet(tablet_id))) { + LOG_ERROR("failed to rollback remove tablet", K(ret), K(tablet_id)); } } } @@ -1070,15 +1021,13 @@ int ObTabletCreateMdsHelper::rollback_remove_tablets( return ret; } -int ObTabletCreateMdsHelper::get_ls( - const share::ObLSID &ls_id, - ObLSHandle &ls_handle) +int ObTabletCreateMdsHelper::get_ls(ObLS *&tenant_ls) { int ret = OB_SUCCESS; ObLSService *ls_service = share::g_mp->ls_service(); - if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); + if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } return ret; @@ -1090,7 +1039,6 @@ int ObTabletCreateMdsHelper::set_tablet_status( const bool for_replay, const share::SCN &scn, mds::BufferCtx &ctx, - const bool for_old_mds, const ObTabletMdsUserDataType &data_type, const int64_t create_commit_version) { @@ -1107,16 +1055,14 @@ int ObTabletCreateMdsHelper::set_tablet_status( } else if (OB_UNLIKELY(for_replay && !scn.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("scn is invalid", K(ret), - "ls_id", tablet->get_tablet_meta().ls_id_, "tablet_id", tablet->get_tablet_meta().tablet_id_, K(for_replay), K(scn)); } else if (for_replay) { ObTabletCreateReplayExecutor replay_executor; - const share::ObLSID &ls_id = tablet->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - if (CLICK_FAIL(replay_executor.init(ctx, scn, for_old_mds, data))) { + if (CLICK_FAIL(replay_executor.init(ctx, scn, data))) { LOG_ERROR("failed to init replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(scn, ls_id, tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(scn, tablet_id))) { LOG_ERROR("failed to replay mds data", K(ret)); } } else if (CLICK_FAIL(ls_tablet_service->set_tablet_status(tablet->get_tablet_meta().tablet_id_, data, user_ctx))) { diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.h b/src/storage/tablet/ob_tablet_create_mds_helper.h index 687576320..fadd7ed99 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.h +++ b/src/storage/tablet/ob_tablet_create_mds_helper.h @@ -34,7 +34,6 @@ namespace schema class ObTableSchema; } -class ObLSID; class SCN; } @@ -57,7 +56,7 @@ namespace mds struct BufferCtx; } -class ObLSHandle; +class ObLS; class ObTabletHandle; class ObLSTabletService; @@ -87,10 +86,6 @@ class ObTabletCreateMdsHelper const int64_t len, const share::SCN &scn, mds::BufferCtx &ctx); - static int on_commit_for_old_mds( - const char* buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg &arg); static int register_process( const obcall::ObBatchCreateTabletArg &arg, mds::BufferCtx &ctx); @@ -119,15 +114,12 @@ class ObTabletCreateMdsHelper static bool is_pure_aux_tablets(const obcall::ObCreateTabletInfo &info); static bool is_bind_hidden_tablets(const obcall::ObCreateTabletInfo &info); static int check_pure_data_or_mixed_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &info, bool &valid); static int check_pure_aux_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &info, bool &valid); static int check_hidden_tablets_info( - const share::ObLSID &ls_id, const obcall::ObCreateTabletInfo &hidden_info, const obcall::ObCreateTabletInfo *aux_info, bool &valid); @@ -164,18 +156,14 @@ class ObTabletCreateMdsHelper mds::BufferCtx &ctx, common::ObIArray &tablet_id_array); static int rollback_remove_tablets( - const share::ObLSID &ls_id, const common::ObIArray &tablet_id_array); - static int get_ls( - const share::ObLSID &ls_id, - ObLSHandle &ls_handle); + static int get_ls(ObLS *&tenant_ls); static int set_tablet_status( ObLSTabletService *ls_tablet_service, ObTabletHandle &tablet_handle, const bool for_replay, const share::SCN &scn, mds::BufferCtx &ctx, - const bool for_old_mds, const storage::ObTabletMdsUserDataType &create_type, const int64_t create_commit_version); static void handle_ret_for_replay(int &ret); diff --git a/src/storage/tablet/ob_tablet_create_replay_executor.cpp b/src/storage/tablet/ob_tablet_create_replay_executor.cpp index d32b2c3eb..8622e3434 100644 --- a/src/storage/tablet/ob_tablet_create_replay_executor.cpp +++ b/src/storage/tablet/ob_tablet_create_replay_executor.cpp @@ -31,7 +31,6 @@ ObTabletCreateReplayExecutor::ObTabletCreateReplayExecutor() int ObTabletCreateReplayExecutor::init( mds::BufferCtx &user_ctx, const share::SCN &scn, - const bool for_old_mds, const ObTabletCreateDeleteMdsUserData &user_data) { int ret = OB_SUCCESS; @@ -45,7 +44,6 @@ int ObTabletCreateReplayExecutor::init( user_ctx_ = &user_ctx; scn_ = scn; is_inited_ = true; - for_old_mds_ = for_old_mds; user_data_ = &user_data; } return ret; @@ -56,7 +54,7 @@ int ObTabletCreateReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) int ret = OB_SUCCESS; mds::MdsCtx &user_ctx = static_cast(*user_ctx_); - if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_, for_old_mds_))) { + if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_))) { LOG_WARN("failed to replay to tablet", K(ret)); } diff --git a/src/storage/tablet/ob_tablet_create_replay_executor.h b/src/storage/tablet/ob_tablet_create_replay_executor.h index 879be195b..b382548aa 100644 --- a/src/storage/tablet/ob_tablet_create_replay_executor.h +++ b/src/storage/tablet/ob_tablet_create_replay_executor.h @@ -37,7 +37,6 @@ class ObTabletCreateReplayExecutor final : public logservice::ObTabletReplayExec int init( mds::BufferCtx &user_ctx, const share::SCN &scn, - const bool for_old_mds, const ObTabletCreateDeleteMdsUserData &user_data); protected: @@ -56,7 +55,6 @@ class ObTabletCreateReplayExecutor final : public logservice::ObTabletReplayExec private: mds::BufferCtx *user_ctx_; share::SCN scn_; - bool for_old_mds_; const ObTabletCreateDeleteMdsUserData *user_data_; }; } // namespace storage diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.cpp b/src/storage/tablet/ob_tablet_create_sstable_param.cpp index 945da71d3..99c405390 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.cpp +++ b/src/storage/tablet/ob_tablet_create_sstable_param.cpp @@ -753,107 +753,10 @@ int ObTabletCreateSSTableParam::init_for_lob_split(const ObTabletID &new_tablet_ return ret; } -int ObTabletCreateSSTableParam::init_for_ha( - const blocksstable::ObMigrationSSTableParam &sstable_param, - const blocksstable::ObSSTableMergeRes &res) -{ - int ret = OB_SUCCESS; - table_key_ = sstable_param.table_key_; - sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; - schema_version_ = sstable_param.basic_meta_.schema_version_; - table_mode_ = sstable_param.basic_meta_.table_mode_; - index_type_ = static_cast(sstable_param.basic_meta_.index_type_); - create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; - progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; - progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; - latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; - column_cnt_ = res.data_column_cnt_; - nested_size_ = res.nested_size_; - nested_offset_ = res.nested_offset_; - max_merged_trans_version_ = res.max_merged_trans_version_; - rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; - ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; - table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_; - filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_; - tx_data_recycle_scn_ = sstable_param.basic_meta_.tx_data_recycle_scn_; - recycle_version_ = sstable_param.basic_meta_.recycle_version_; - if (OB_FAIL(inner_init_with_merge_res(res))) { - LOG_WARN("fail to inner init with merge res", K(ret), K(res)); - } else if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), K(sstable_param)); - } else if (OB_FAIL(blocksstable::ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(table_key_, filled_tx_scn_))) { - LOG_WARN("failed to fix filled tx scn value for compact", K(ret), K(table_key_), K(sstable_param)); - } else { - root_macro_seq_ = MAX(root_macro_seq_, sstable_param.basic_meta_.root_macro_seq_); - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("init for ha sstable get invalid argument", K(ret), K(sstable_param), KPC(this), - K(res)); - } - } - return ret; -} - -int ObTabletCreateSSTableParam::init_for_ha(const blocksstable::ObMigrationSSTableParam &sstable_param) -{ - int ret = OB_SUCCESS; - - table_key_ = sstable_param.table_key_; - sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; - schema_version_ = sstable_param.basic_meta_.schema_version_; - create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; - table_mode_ = sstable_param.basic_meta_.table_mode_; - index_type_ = static_cast(sstable_param.basic_meta_.index_type_); - progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; - progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; - is_ready_for_read_ = true; - root_row_store_type_ = sstable_param.basic_meta_.root_row_store_type_; - latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; - index_blocks_cnt_ = sstable_param.basic_meta_.index_macro_block_count_; - data_blocks_cnt_ = sstable_param.basic_meta_.data_macro_block_count_; - micro_block_cnt_ = sstable_param.basic_meta_.data_micro_block_count_; - use_old_macro_block_count_ = sstable_param.basic_meta_.use_old_macro_block_count_; - row_count_ = sstable_param.basic_meta_.row_count_; - column_cnt_ = sstable_param.basic_meta_.column_cnt_; - data_checksum_ = sstable_param.basic_meta_.data_checksum_; - occupy_size_ = sstable_param.basic_meta_.occupy_size_; - original_size_ = sstable_param.basic_meta_.original_size_; - max_merged_trans_version_ = sstable_param.basic_meta_.max_merged_trans_version_; - ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; - filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_; - tx_data_recycle_scn_ = sstable_param.basic_meta_.tx_data_recycle_scn_; - contain_uncommitted_row_ = sstable_param.basic_meta_.contain_uncommitted_row_; - compressor_type_ = sstable_param.basic_meta_.compressor_type_; - encrypt_id_ = sstable_param.basic_meta_.encrypt_id_; - master_key_id_ = sstable_param.basic_meta_.master_key_id_; - rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; - root_macro_seq_ = sstable_param.basic_meta_.root_macro_seq_; - MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - table_backup_flag_ = sstable_param.basic_meta_.table_backup_flag_; - table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_; - is_meta_root_ = sstable_param.is_meta_root_; - root_block_addr_.set_none_addr(); - data_block_macro_meta_addr_.set_none_addr(); - data_index_tree_height_ = sstable_param.basic_meta_.data_index_tree_height_; - recycle_version_ = sstable_param.basic_meta_.recycle_version_; - nested_offset_ = 0; - nested_size_ = 0; - if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { - LOG_WARN("fail to assign column checksums", K(ret), K(sstable_param)); - } else if (OB_FAIL(blocksstable::ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(table_key_, filled_tx_scn_))) { - LOG_WARN("failed to fix filled tx scn value for compact", K(ret), K(table_key_), K(sstable_param)); - } else if (sstable_param.is_shared_sstable() && OB_FAIL(inner_init_with_shared_sstable(sstable_param))) { - LOG_WARN("failed to inner init with shared sstable", K(ret), K(sstable_param)); - } else if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("init for ha sstable get invalid argument", K(ret), K(sstable_param), KPC(this)); - } - return ret; -} int ObTabletCreateSSTableParam::init_for_fork( - const blocksstable::ObMigrationSSTableParam &sstable_param, + const blocksstable::ObForkSSTableParam &sstable_param, const ObTabletID &dst_tablet_id, const ObITable::TableKey &src_table_key, const blocksstable::ObSSTableMeta &sstable_meta, @@ -1047,48 +950,9 @@ int ObTabletCreateSSTableParam::init_for_mds( return ret; } -int ObTabletCreateSSTableParam::inner_init_with_shared_sstable(const blocksstable::ObMigrationSSTableParam &sstable_param) -{ - int ret = OB_SUCCESS; - if (sstable_param.root_block_addr_.is_valid()) { - root_block_addr_ = sstable_param.root_block_addr_; - root_block_data_.buf_ = sstable_param.root_block_buf_; - root_block_data_.size_ = sstable_param.root_block_addr_.size(); - } - - if (sstable_param.data_block_macro_meta_addr_.is_valid()) { - data_block_macro_meta_addr_ = sstable_param.data_block_macro_meta_addr_; - data_block_macro_meta_.buf_ = sstable_param.data_block_macro_meta_buf_; - data_block_macro_meta_.size_ = sstable_param.data_block_macro_meta_addr_.size(); - } - - root_row_store_type_ = sstable_param.basic_meta_.root_row_store_type_; - data_index_tree_height_ = sstable_param.basic_meta_.data_index_tree_height_; - index_blocks_cnt_ = sstable_param.basic_meta_.index_macro_block_count_; - data_blocks_cnt_ = sstable_param.basic_meta_.data_macro_block_count_; - micro_block_cnt_ = sstable_param.basic_meta_.data_micro_block_count_; - use_old_macro_block_count_ = sstable_param.basic_meta_.use_old_macro_block_count_; - row_count_ = sstable_param.basic_meta_.row_count_; - data_checksum_ = sstable_param.basic_meta_.data_checksum_; - occupy_size_ = sstable_param.basic_meta_.occupy_size_; - original_size_ = sstable_param.basic_meta_.original_size_; - contain_uncommitted_row_ = sstable_param.basic_meta_.contain_uncommitted_row_; - compressor_type_ = sstable_param.basic_meta_.compressor_type_; - encrypt_id_ = sstable_param.basic_meta_.encrypt_id_; - master_key_id_ = sstable_param.basic_meta_.master_key_id_; - is_meta_root_ = sstable_param.is_meta_root_; - root_macro_seq_ = sstable_param.basic_meta_.root_macro_seq_; - STATIC_ASSERT(ARRAYSIZEOF(encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, - "ObTabletCreateSSTableParam encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); - STATIC_ASSERT(ARRAYSIZEOF(sstable_param.basic_meta_.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, - "ObSSTableMergeRes encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); - MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - table_backup_flag_ = sstable_param.basic_meta_.table_backup_flag_; - return ret; -} int ObTabletCreateSSTableParam::inner_init_with_shared_sstable( - const blocksstable::ObMigrationSSTableParam &sstable_param, + const blocksstable::ObForkSSTableParam &sstable_param, const common::ObIArray &data_block_ids, const common::ObIArray &other_block_ids) { diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.h b/src/storage/tablet/ob_tablet_create_sstable_param.h index dba2cdbe1..ee50e6084 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.h +++ b/src/storage/tablet/ob_tablet_create_sstable_param.h @@ -36,7 +36,7 @@ struct ObBlockInfo; struct ObSSTableBasicMeta; class ObSSTableMacroInfo; class ObSSTableMeta; -class ObMigrationSSTableParam; +class ObForkSSTableParam; class ObSSTable; class ObSSTableIndexBuilder; } @@ -123,20 +123,12 @@ struct ObTabletCreateSSTableParam final const int64_t sstable_logic_seq, const blocksstable::ObSSTableMergeRes &res); - int init_for_fork(const blocksstable::ObMigrationSSTableParam &sstable_param, + int init_for_fork(const blocksstable::ObForkSSTableParam &sstable_param, const ObTabletID &dst_tablet_id, const ObITable::TableKey &src_table_key, const blocksstable::ObSSTableMeta &sstable_meta, const share::SCN &max_end_scn = share::SCN()); - // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. - int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param, - const blocksstable::ObSSTableMergeRes &res); - - // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. - int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param); - - // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. int init_for_mds(const compaction::ObBasicTabletMergeCtx &ctx, const blocksstable::ObSSTableMergeRes &res, const ObStorageSchema &mds_schema); @@ -197,8 +189,7 @@ struct ObTabletCreateSSTableParam final private: static const int64_t DEFAULT_MACRO_BLOCK_CNT = 64; int inner_init_with_merge_res(const blocksstable::ObSSTableMergeRes &res); - int inner_init_with_shared_sstable(const blocksstable::ObMigrationSSTableParam &migration_param); - int inner_init_with_shared_sstable(const blocksstable::ObMigrationSSTableParam &sstable_param, + int inner_init_with_shared_sstable(const blocksstable::ObForkSSTableParam &sstable_param, const common::ObIArray &data_block_ids, const common::ObIArray &other_block_ids); int collect_macro_block_ids_from_meta( diff --git a/src/storage/tablet/ob_tablet_ddl_complete_mds_data.cpp b/src/storage/tablet/ob_tablet_ddl_complete_mds_data.cpp index 3a204eea4..b854445a9 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_mds_data.cpp +++ b/src/storage/tablet/ob_tablet_ddl_complete_mds_data.cpp @@ -18,7 +18,6 @@ #include "lib/utility/ob_unify_serialize.h" #include "share/ob_errno.h" #include "storage/tx/ob_trans_define.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/ddl/ob_direct_load_struct.h" diff --git a/src/storage/tablet/ob_tablet_ddl_complete_mds_data.h b/src/storage/tablet/ob_tablet_ddl_complete_mds_data.h index a7dfe6085..775a7f64b 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_mds_data.h +++ b/src/storage/tablet/ob_tablet_ddl_complete_mds_data.h @@ -19,7 +19,6 @@ #include #include "lib/utility/ob_print_utils.h" -#include "share/ob_ls_id.h" #include "share/scn.h" #include "common/ob_tablet_id.h" #include "storage/tablet/ob_tablet_common.h" diff --git a/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.cpp b/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.cpp index 4a3d9614b..2b4354062 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.cpp @@ -19,12 +19,9 @@ #include "share/rc/ob_module_provider.h" #include "common/ob_tablet_id.h" #include "share/scn.h" -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" -#include "storage/ls/ob_ls_get_mod.h" #include "storage/multi_data_source/buffer_ctx.h" #include "storage/multi_data_source/mds_ctx.h" -#include "storage/tx_storage/ob_ls_handle.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tablet/ob_tablet_ddl_complete_mds_helper.h" #include "storage/ddl/ob_direct_load_struct.h" @@ -46,7 +43,7 @@ namespace oceanbase namespace storage { ObTabletDDLCompleteArg::ObTabletDDLCompleteArg(): - has_complete_(false), ls_id_(), tablet_id_(), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), + has_complete_(false), tablet_id_(), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), rec_scn_(share::SCN::min_scn()), start_scn_(share::SCN::min_scn()),data_format_version_(0), snapshot_version_(0), table_key_(), storage_schema_(), allocator_() { } @@ -60,7 +57,6 @@ ObTabletDDLCompleteArg::~ObTabletDDLCompleteArg() void ObTabletDDLCompleteArg::reset() { has_complete_ = false; - ls_id_.reset(); tablet_id_.reset(); direct_load_type_ = ObDirectLoadType::DIRECT_LOAD_INVALID; rec_scn_ = share::SCN::min_scn(); @@ -74,8 +70,8 @@ void ObTabletDDLCompleteArg::reset() } bool ObTabletDDLCompleteArg::is_valid() const { - return (!has_complete_ && ls_id_.is_valid() && tablet_id_.is_valid()) || - (has_complete_ && ls_id_.is_valid() && tablet_id_.is_valid() && table_key_.is_valid() + return (!has_complete_ && tablet_id_.is_valid()) || + (has_complete_ && tablet_id_.is_valid() && table_key_.is_valid() && nullptr != storage_schema_ && storage_schema_->is_valid() && write_stat_.is_valid()) || (is_incremental_major_direct_load(direct_load_type_) && OB_NOT_NULL(storage_schema_) @@ -120,7 +116,6 @@ int ObTabletDDLCompleteArg::assign(const ObTabletDDLCompleteArg &other) LOG_WARN("failed to set write_stat", K(ret)); }else { has_complete_ = other.has_complete_; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; direct_load_type_ = other.direct_load_type_; rec_scn_ = other.rec_scn_; @@ -137,7 +132,7 @@ int64_t ObTabletDDLCompleteArg::get_serialize_size() const { int64_t len = 0; LST_DO_CODE(OB_UNIS_ADD_LEN, - has_complete_, ls_id_, tablet_id_, + has_complete_, tablet_id_, direct_load_type_, rec_scn_, start_scn_, data_format_version_, snapshot_version_, table_key_, write_stat_, trans_id_); @@ -151,7 +146,7 @@ int ObTabletDDLCompleteArg::serialize(char *buf, const int64_t buf_len, int64_t { int ret = OB_SUCCESS; LST_DO_CODE(OB_UNIS_ENCODE, - has_complete_, ls_id_, tablet_id_, direct_load_type_, rec_scn_, start_scn_, + has_complete_, tablet_id_, direct_load_type_, rec_scn_, start_scn_, data_format_version_, snapshot_version_, table_key_, write_stat_, trans_id_); if (OB_FAIL(ret)) { } else if (has_complete_) { @@ -168,7 +163,7 @@ int ObTabletDDLCompleteArg::serialize(char *buf, const int64_t buf_len, int64_t int ObTabletDDLCompleteArg::deserialize(const char *buf, const int64_t data_len, int64_t &pos) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, has_complete_, ls_id_, tablet_id_, direct_load_type_, rec_scn_, start_scn_, + LST_DO_CODE(OB_UNIS_DECODE, has_complete_, tablet_id_, direct_load_type_, rec_scn_, start_scn_, data_format_version_, snapshot_version_, table_key_, write_stat_, trans_id_); if (OB_FAIL(ret)) { } else if (has_complete_) { @@ -225,7 +220,7 @@ int ObTabletDDLCompleteMdsHelper::process(const char* buf, const int64_t len, co ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid arg", K(ret), K(arg)); } else { - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObLSService *ls_service = share::g_mp->ls_service(); common::ObArenaAllocator allocator(ObMemAttr("Ddl_Com_MdsH")); ObTabletDDLCompleteMdsUserData data; @@ -233,12 +228,12 @@ int ObTabletDDLCompleteMdsHelper::process(const char* buf, const int64_t len, co if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls_service is null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret), K(arg)); } else if (OB_FAIL(data.set_with_merge_arg(arg, allocator))) { LOG_WARN("failed to set with merge arg", K(ret)); } else { - if (OB_FAIL(process_ddl(ctx, ls_handle, arg.tablet_id_, data, scn, for_replay))) { + if (OB_FAIL(process_ddl(ctx, tenant_ls, arg.tablet_id_, data, scn, for_replay))) { LOG_WARN("failed to process ddl", KR(ret), K(arg), K(data), K(scn), K(for_replay)); } } @@ -249,7 +244,7 @@ int ObTabletDDLCompleteMdsHelper::process(const char* buf, const int64_t len, co int ObTabletDDLCompleteMdsHelper::process_ddl( mds::BufferCtx &ctx, - ObLSHandle &ls_handle, + ObLS *tenant_ls, const ObTabletID &tablet_id, const ObTabletDDLCompleteMdsUserData &data, const share::SCN &scn, @@ -258,21 +253,18 @@ int ObTabletDDLCompleteMdsHelper::process_ddl( int ret = OB_SUCCESS; MDS_TG(1_s); mds::MdsCtx &user_ctx = static_cast(ctx); - ObLS* ls = nullptr; ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!ls_handle.is_valid() || !tablet_id.is_valid() || !data.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || !data.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected invalid argument", KR(ret), K(ls_handle), K(tablet_id), K(data)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", KR(ret), K(ls_handle)); + LOG_WARN("unexpected invalid argument", KR(ret), K(tablet_id), K(data)); } else if (!for_replay) { - if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(tenant_ls, tablet_id, tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { } else if (OB_FAIL(ObTabletDDLCompleteReplayExecutor::freeze_ddl_kv(*tablet_handle.get_obj(), data))) { LOG_WARN("failed to freeze ddl kv", K(ret)); } else if (OB_FAIL(ObTabletDDLCompleteReplayExecutor::update_tablet_table_store(*tablet_handle.get_obj(), data))) { LOG_WARN("failed to update tablet table store", K(ret)); - } else if (CLICK_FAIL(ls->get_tablet_svr()->set_ddl_complete(tablet_id, mds::DummyKey(), data, user_ctx, 0/*lock_timeout_us*/))) { + } else if (CLICK_FAIL(tenant_ls->get_tablet_svr()->set_ddl_complete( + tablet_id, mds::DummyKey(), data, user_ctx, 0/*lock_timeout_us*/))) { if (OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret) { ret = OB_EAGAIN; } else { @@ -286,9 +278,9 @@ int ObTabletDDLCompleteMdsHelper::process_ddl( } } else { ObTabletDDLCompleteReplayExecutor replay_executor; - if (CLICK_FAIL(replay_executor.init(ctx, scn, false /* for old mds */, data))) { + if (CLICK_FAIL(replay_executor.init(ctx, scn, data))) { LOG_ERROR("failed to inti replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(scn, ls->get_ls_id(), tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(scn, tablet_id))) { LOG_ERROR("failed to execute replay ddl complete mds data", K(ret)); } } @@ -335,8 +327,8 @@ int ObTabletDDLCompleteMdsHelper::record_ddl_complete_arg_to_mds( } else if (OB_ISNULL(conn = static_cast(trans.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null connection", KR(ret), KP(conn)); - } else if (OB_FAIL(conn->register_multi_data_source(complete_arg.ls_id_, ObTxDataSourceType::DDL_COMPLETE_MDS, buf, buf_len))) { - LOG_WARN("failed to register multi data source", KR(ret), K(complete_arg.ls_id_)); + } else if (OB_FAIL(conn->register_multi_data_source(ObTxDataSourceType::DDL_COMPLETE_MDS, buf, buf_len))) { + LOG_WARN("failed to register multi data source", KR(ret)); } else if (OB_FAIL(trans.end(OB_SUCC(ret)))) { LOG_WARN("failed to end trans", KR(ret)); } else { diff --git a/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.h b/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.h index 526f9674b..961b4942d 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.h +++ b/src/storage/tablet/ob_tablet_ddl_complete_mds_helper.h @@ -28,7 +28,6 @@ namespace oceanbase { namespace share { -class ObLSID; class SCN; } @@ -45,6 +44,7 @@ namespace mds struct BufferCtx; } +class ObLS; class ObTabletDDLCompleteMdsUserData; class ObTabletDDLCompleteArg @@ -65,10 +65,9 @@ class ObTabletDDLCompleteArg int deserialize(const char *buf, const int64_t data_len, int64_t &pos); int64_t get_serialize_size() const; int from_mds_user_data(const ObTabletDDLCompleteMdsUserData &user_data); - TO_STRING_KV(K_(has_complete), K_(ls_id), K_(tablet_id), K_(direct_load_type), K_(rec_scn), K_(start_scn), K_(data_format_version), K_(snapshot_version), K_(table_key), KPC_(storage_schema), K_(write_stat), K_(trans_id)); + TO_STRING_KV(K_(has_complete), K_(tablet_id), K_(direct_load_type), K_(rec_scn), K_(start_scn), K_(data_format_version), K_(snapshot_version), K_(table_key), KPC_(storage_schema), K_(write_stat), K_(trans_id)); public: bool has_complete_; - share::ObLSID ls_id_; ObTabletID tablet_id_; /* ddl table merge param */ ObDirectLoadType direct_load_type_; @@ -94,7 +93,7 @@ class ObTabletDDLCompleteMdsHelper static int record_ddl_complete_arg_to_mds(const ObTabletDDLCompleteArg &complete_arg, common::ObIAllocator &allocator); static int process_ddl(mds::BufferCtx &ctx, - ObLSHandle &ls_handle, + ObLS *tenant_ls, const ObTabletID &tablet_id, const ObTabletDDLCompleteMdsUserData &data, const share::SCN &scn, diff --git a/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.cpp b/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.cpp index a246bcce8..78ec1033a 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.cpp +++ b/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.cpp @@ -42,7 +42,6 @@ ObTabletDDLCompleteReplayExecutor::ObTabletDDLCompleteReplayExecutor() int ObTabletDDLCompleteReplayExecutor::init( mds::BufferCtx &user_ctx, const share::SCN &scn, - const bool for_old_mds, const ObTabletDDLCompleteMdsUserData &user_data) { int ret = OB_SUCCESS; @@ -56,7 +55,6 @@ int ObTabletDDLCompleteReplayExecutor::init( user_ctx_ = &user_ctx; scn_ = scn; is_inited_ = true; - for_old_mds_ = for_old_mds; user_data_ = &user_data; } return ret; @@ -79,7 +77,7 @@ int ObTabletDDLCompleteReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) } if (OB_FAIL(ret)) { - } else if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_, for_old_mds_))) { + } else if (OB_FAIL(replay_to_mds_table_(tablet_handle, *user_data_, user_ctx, scn_))) { LOG_WARN("failed to replay to tablet", K(ret)); } else if (user_data_->has_complete_) { /* use tmp ret to avoid pending replay */ @@ -110,9 +108,7 @@ int ObTabletDDLCompleteReplayExecutor::freeze_ddl_kv(ObTablet &tablet, const ObT int ObTabletDDLCompleteReplayExecutor::update_tablet_table_store(ObTablet &tablet, const ObTabletDDLCompleteMdsUserData &user_data) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObLSHandle ls_handle; - ObLSService *ls_service = share::g_mp->ls_service(); + ObLS *tenant_ls = nullptr; ObTabletHandle new_tablet_handle; const ObSSTable *first_major_sstable = nullptr; @@ -125,21 +121,14 @@ int ObTabletDDLCompleteReplayExecutor::update_tablet_table_store(ObTablet &table } else if (nullptr != first_major_sstable) { /* do nothing */ LOG_INFO("first major sstable exist, do nothing", K(ret), K(user_data)); - } else if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls_service is null", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(tablet.get_ls_id(), ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret), K(user_data)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret)); } else { ObUpdateTableStoreParam param(user_data.snapshot_version_, ObVersionRange::MIN_VERSION, // multi_version_start - &user_data.storage_schema_, - ls_handle.get_ls()->get_rebuild_seq()); + &user_data.storage_schema_); param.ddl_info_.keep_old_ddl_sstable_ = true; - if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(user_data.table_key_.tablet_id_, param, new_tablet_handle))) { + if (OB_FAIL(tenant_ls->update_tablet_table_store(user_data.table_key_.tablet_id_, param, new_tablet_handle))) { LOG_WARN("failed to update table store", K(ret)); } } diff --git a/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.h b/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.h index e837bcc23..846f0a19a 100644 --- a/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.h +++ b/src/storage/tablet/ob_tablet_ddl_complete_replay_executor.h @@ -37,7 +37,6 @@ class ObTabletDDLCompleteReplayExecutor final : public logservice::ObTabletRepla int init( mds::BufferCtx &user_ctx, const share::SCN &scn, - const bool for_old_mds, const ObTabletDDLCompleteMdsUserData &user_data); static int freeze_ddl_kv(ObTablet &tablet, const ObTabletDDLCompleteMdsUserData &user_data); static int update_tablet_table_store(ObTablet &tablet, const ObTabletDDLCompleteMdsUserData &user_data); @@ -59,7 +58,6 @@ class ObTabletDDLCompleteReplayExecutor final : public logservice::ObTabletRepla private: mds::BufferCtx *user_ctx_; share::SCN scn_; - bool for_old_mds_; const ObTabletDDLCompleteMdsUserData *user_data_; }; } // namespace storage diff --git a/src/storage/tablet/ob_tablet_delete_mds_helper.cpp b/src/storage/tablet/ob_tablet_delete_mds_helper.cpp index b7d1f5b59..e5f4dcc1b 100644 --- a/src/storage/tablet/ob_tablet_delete_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_delete_mds_helper.cpp @@ -36,7 +36,7 @@ int ObTabletDeleteMdsHelper::register_process( if (CLICK_FAIL(delete_tablets(arg, ctx))) { LOG_WARN("failed to delete tablets", K(ret), K(arg)); - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(arg.id_))) { + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger())) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(arg)); } else { LOG_INFO("delete tablet register", KR(ret), K(arg)); @@ -45,15 +45,6 @@ int ObTabletDeleteMdsHelper::register_process( return ret; } -int ObTabletDeleteMdsHelper::on_commit_for_old_mds( - const char* buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg) -{ - mds::TLOCAL_MDS_INFO.reset();// disable runtime check - return ObTabletCreateDeleteHelper::process_for_old_mds(buf, len, notify_arg); -} - int ObTabletDeleteMdsHelper::on_register( const char* buf, const int64_t len, @@ -70,9 +61,6 @@ int ObTabletDeleteMdsHelper::on_register( LOG_WARN("invalid args", K(ret), KP(buf), K(len)); } else if (CLICK_FAIL(arg.deserialize(buf, len, pos))) { LOG_WARN("failed to deserialize", K(ret)); - } else if (arg.is_old_mds_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, arg is old mds", K(ret), K(arg)); } else if (CLICK_FAIL(register_process(arg, ctx))) { LOG_WARN("failed to register_process", K(ret), K(arg)); } @@ -90,7 +78,7 @@ int ObTabletDeleteMdsHelper::replay_process( if (CLICK_FAIL(replay_delete_tablets(arg, scn, ctx))) { LOG_WARN("failed to delete tablets", K(ret), K(arg), K(scn)); - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(arg.id_))) { + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger())) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(arg)); } else { LOG_INFO("delete tablet replay", KR(ret), K(scn), K(arg)); @@ -116,8 +104,6 @@ int ObTabletDeleteMdsHelper::on_replay( LOG_WARN("invalid args", K(ret), KP(buf), K(len)); } else if (CLICK_FAIL(arg.deserialize(buf, len, pos))) { LOG_WARN("failed to deserialize", K(ret)); - } else if (arg.is_old_mds_) { - LOG_INFO("skip replay delete tablet for old mds", K(arg), K(scn)); } else if (CLICK_FAIL(replay_process(arg, scn, ctx))) { LOG_WARN("failed to replay_process", K(ret), K(arg)); } @@ -135,15 +121,10 @@ int ObTabletDeleteMdsHelper::delete_tablets( bool exist = false; ObTabletHandle tablet_handle; ObTabletMapKey key; - key.ls_id_ = arg.id_; - - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - if (CLICK_FAIL(ls_service->get_ls(key.ls_id_, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(key.ls_id_)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; + + ObLS *tenant_ls = nullptr; + if (CLICK_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); } else { CLICK(); for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); ++i) { @@ -166,7 +147,8 @@ int ObTabletDeleteMdsHelper::delete_tablets( } else if (!exist) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet does not exist", K(ret), K(key)); - } else if (CLICK_FAIL(set_tablet_deleted_status(ls->get_tablet_svr(), tablet_handle, ctx))) { + } else if (CLICK_FAIL(set_tablet_deleted_status( + tenant_ls->get_tablet_svr(), tablet_handle, ctx))) { LOG_ERROR("failed to set tablet deleted status", K(ret), K(key)); } } @@ -186,16 +168,14 @@ int ObTabletDeleteMdsHelper::replay_delete_tablets( int ret = OB_SUCCESS; bool exist = false; ObRemoveTabletArg remove_tablet_arg; - remove_tablet_arg.ls_id_ = arg.id_; - for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); ++i) { MDS_TG(10_ms); exist = true; remove_tablet_arg.tablet_id_ = arg.tablet_ids_.at(i); ObTabletDeleteReplayExecutor replayer; - if (CLICK_FAIL(replayer.init(ctx, scn, arg.is_old_mds_))) { + if (CLICK_FAIL(replayer.init(ctx, scn))) { LOG_ERROR("failed to init tablet delete replay executor", K(ret), K(remove_tablet_arg)); - } else if (CLICK_FAIL(replayer.execute(scn, remove_tablet_arg.ls_id_, remove_tablet_arg.tablet_id_))) { + } else if (CLICK_FAIL(replayer.execute(scn, remove_tablet_arg.tablet_id_))) { if (OB_TABLET_NOT_EXIST == ret) { exist = false; ret = OB_SUCCESS; diff --git a/src/storage/tablet/ob_tablet_delete_mds_helper.h b/src/storage/tablet/ob_tablet_delete_mds_helper.h index ffe00d8a5..2547d5c17 100644 --- a/src/storage/tablet/ob_tablet_delete_mds_helper.h +++ b/src/storage/tablet/ob_tablet_delete_mds_helper.h @@ -52,10 +52,6 @@ class ObTabletDeleteMdsHelper static int register_process( obcall::ObBatchRemoveTabletArg &arg, mds::BufferCtx &ctx); - static int on_commit_for_old_mds( - const char* buf, - const int64_t len, - const transaction::ObMulSourceDataNotifyArg ¬ify_arg); static int on_replay( const char* buf, const int64_t len, diff --git a/src/storage/tablet/ob_tablet_delete_replay_executor.cpp b/src/storage/tablet/ob_tablet_delete_replay_executor.cpp index da4525dd0..91bf0942f 100644 --- a/src/storage/tablet/ob_tablet_delete_replay_executor.cpp +++ b/src/storage/tablet/ob_tablet_delete_replay_executor.cpp @@ -23,7 +23,7 @@ namespace oceanbase namespace storage { -OB_SERIALIZE_MEMBER(ObRemoveTabletArg, ls_id_, tablet_id_); +OB_SERIALIZE_MEMBER(ObRemoveTabletArg, tablet_id_); // ObTabletDeleteReplayExecutor @@ -33,8 +33,7 @@ ObTabletDeleteReplayExecutor::ObTabletDeleteReplayExecutor() int ObTabletDeleteReplayExecutor::init( mds::BufferCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -46,7 +45,6 @@ int ObTabletDeleteReplayExecutor::init( } else { ctx_ = &ctx; scn_ = scn; - for_old_mds_ = for_old_mds; is_inited_ = true; } return ret; @@ -69,7 +67,7 @@ int ObTabletDeleteReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) } else { data.tablet_status_ = ObTabletStatus::DELETED; data.data_type_ = ObTabletMdsUserDataType::REMOVE_TABLET; - if (CLICK_FAIL(replay_to_mds_table_(tablet_handle, data, user_ctx, scn_, for_old_mds_))) { + if (CLICK_FAIL(replay_to_mds_table_(tablet_handle, data, user_ctx, scn_))) { LOG_ERROR("failed to replay to tablet", K(ret)); } } diff --git a/src/storage/tablet/ob_tablet_delete_replay_executor.h b/src/storage/tablet/ob_tablet_delete_replay_executor.h index da1accb12..486a84d09 100644 --- a/src/storage/tablet/ob_tablet_delete_replay_executor.h +++ b/src/storage/tablet/ob_tablet_delete_replay_executor.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_STORAGE_OB_TABLET_DELETE_REPLAY_EXECUTOR #define OCEANBASE_STORAGE_OB_TABLET_DELETE_REPLAY_EXECUTOR -#include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" #include "logservice/replayservice/ob_tablet_replay_executor.h" @@ -33,13 +32,12 @@ struct ObRemoveTabletArg public: inline bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return tablet_id_.is_valid(); } - TO_STRING_KV(K_(ls_id), K_(tablet_id)); + TO_STRING_KV(K_(tablet_id)); public: - share::ObLSID ls_id_; common::ObTabletID tablet_id_; }; @@ -48,7 +46,7 @@ class ObTabletDeleteReplayExecutor final : public logservice::ObTabletReplayExec public: ObTabletDeleteReplayExecutor(); - int init(mds::BufferCtx &ctx, const share::SCN &scn, const bool for_old_mds); + int init(mds::BufferCtx &ctx, const share::SCN &scn); protected: bool is_replay_update_tablet_status_() const override @@ -64,10 +62,8 @@ class ObTabletDeleteReplayExecutor final : public logservice::ObTabletReplayExec } private: - const ObRemoveTabletArg *arg_; mds::BufferCtx *ctx_; share::SCN scn_; - bool for_old_mds_; }; diff --git a/src/storage/tablet/ob_tablet_fork_mds_helper.cpp b/src/storage/tablet/ob_tablet_fork_mds_helper.cpp index e00eb10d5..3a63e0481 100644 --- a/src/storage/tablet/ob_tablet_fork_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_fork_mds_helper.cpp @@ -37,8 +37,7 @@ namespace storage { ObTabletForkMdsArg::ObTabletForkMdsArg() - : ls_id_(), - autoinc_seq_arg_(), + : autoinc_seq_arg_(), truncate_arg_(), allocator_("ForkMdsArg") { @@ -52,7 +51,7 @@ ObTabletForkMdsArg::~ObTabletForkMdsArg() int ObTabletForkMdsArg::serialize(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, autoinc_seq_arg_); + LST_DO_CODE(OB_UNIS_ENCODE, autoinc_seq_arg_); if (OB_SUCC(ret)) { bool has_truncate_arg = truncate_arg_.is_valid(); if (OB_FAIL(serialization::encode_bool(buf, buf_len, pos, has_truncate_arg))) { @@ -69,7 +68,7 @@ int ObTabletForkMdsArg::serialize(char *buf, const int64_t buf_len, int64_t &pos int ObTabletForkMdsArg::deserialize(const char *buf, const int64_t data_len, int64_t &pos) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, ls_id_, autoinc_seq_arg_); + LST_DO_CODE(OB_UNIS_DECODE, autoinc_seq_arg_); if (OB_SUCC(ret)) { bool has_truncate_arg = false; if (OB_FAIL(serialization::decode_bool(buf, data_len, pos, &has_truncate_arg))) { @@ -91,7 +90,7 @@ int ObTabletForkMdsArg::deserialize(const char *buf, const int64_t data_len, int int64_t ObTabletForkMdsArg::get_serialize_size() const { int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, autoinc_seq_arg_); + LST_DO_CODE(OB_UNIS_ADD_LEN, autoinc_seq_arg_); len += serialization::encoded_length_bool(true); // has_truncate_arg flag if (truncate_arg_.is_valid()) { len += truncate_arg_.get_serialize_size(); @@ -101,15 +100,12 @@ int64_t ObTabletForkMdsArg::get_serialize_size() const bool ObTabletForkMdsArg::is_valid() const { - return true - && ls_id_.is_valid() - && (autoinc_seq_arg_.is_valid() || truncate_arg_.is_valid()); + return autoinc_seq_arg_.is_valid() || truncate_arg_.is_valid(); } void ObTabletForkMdsArg::reset() { - ls_id_.reset(); autoinc_seq_arg_.reset(); truncate_arg_.destroy(); allocator_.reset(); @@ -168,7 +164,7 @@ int ObTabletForkMdsHelper::register_mds( LOG_WARN("invalid tablet fork mds arg", KR(ret), K(arg)); } else if (OB_ISNULL(isql_conn = trans.get_connection())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid connection when register tablet fork mds", KR(ret), K(arg.ls_id_)); + LOG_WARN("invalid connection when register tablet fork mds", KR(ret)); } else { const int64_t size = arg.get_serialize_size(); ObArenaAllocator allocator; @@ -179,11 +175,11 @@ int ObTabletForkMdsHelper::register_mds( flag.mds_base_scn_.reset(); if (OB_ISNULL(buf = static_cast(allocator.alloc(size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate buffer for tablet fork mds", KR(ret), K(arg.ls_id_), K(size)); + LOG_WARN("failed to allocate buffer for tablet fork mds", KR(ret), K(size)); } else if (OB_FAIL(arg.serialize(buf, size, pos))) { - LOG_WARN("failed to serialize tablet fork mds arg", KR(ret), K(arg.ls_id_), K(size), K(pos)); - } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(arg.ls_id_, ObTxDataSourceType::TABLET_FORK, buf, pos, flag))) { - LOG_WARN("failed to register tablet fork mds", KR(ret), K(arg.ls_id_), K(need_flush_redo), K(pos)); + LOG_WARN("failed to serialize tablet fork mds arg", KR(ret), K(size), K(pos)); + } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(ObTxDataSourceType::TABLET_FORK, buf, pos, flag))) { + LOG_WARN("failed to register tablet fork mds", KR(ret), K(need_flush_redo), K(pos)); } } return ret; @@ -222,26 +218,22 @@ int ObTabletForkMdsHelper::modify( mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = arg.ls_id_; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(arg)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("fail to get ls", KR(ret), K(arg)); - } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", KR(ret), K(arg), KP(ls)); } if (OB_SUCC(ret) && arg.autoinc_seq_arg_.is_valid()) { - if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().batch_set_tablet_autoinc_seq_in_trans(*ls, arg.autoinc_seq_arg_, scn, ctx))) { - LOG_WARN("failed to batch set tablet autoinc seq", KR(ret), K(ls_id), K(scn), + if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().batch_set_tablet_autoinc_seq_in_trans( + *tenant_ls, arg.autoinc_seq_arg_, scn, ctx))) { + LOG_WARN("failed to batch set tablet autoinc seq", KR(ret), K(scn), "param_cnt", arg.autoinc_seq_arg_.autoinc_params_.count()); } else { - LOG_INFO("fork table: successfully set autoinc seq", K(ls_id), K(scn), + LOG_INFO("fork table: successfully set autoinc seq", K(scn), "param_cnt", arg.autoinc_seq_arg_.autoinc_params_.count()); } } @@ -249,7 +241,7 @@ int ObTabletForkMdsHelper::modify( if (OB_SUCC(ret) && arg.truncate_arg_.is_valid()) { const rootserver::ObTruncateTabletArg &truncate_arg = arg.truncate_arg_; ObTabletHandle tablet_handle; - const ObTabletMapKey key(ls_id, truncate_arg.index_tablet_id_); + const ObTabletMapKey key(truncate_arg.index_tablet_id_); if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, tablet_handle))) { LOG_WARN("failed to get tablet for truncate info", K(ret), K(key)); @@ -258,17 +250,17 @@ int ObTabletForkMdsHelper::modify( truncate_arg.truncate_info_, static_cast(ctx), 0/*lock_timeout_us*/))) { - LOG_WARN("failed to set truncate info", KR(ret), K(ls_id), K(scn), K(truncate_arg)); + LOG_WARN("failed to set truncate info", KR(ret), K(scn), K(truncate_arg)); } else { - LOG_INFO("fork table: successfully set truncate info", K(ls_id), K(scn), + LOG_INFO("fork table: successfully set truncate info", K(scn), K(truncate_arg.index_tablet_id_), K(truncate_arg.truncate_info_.key_)); } } if (OB_FAIL(ret)) { - LOG_WARN("failed to apply tablet fork mds", KR(ret), K(scn), K(arg.ls_id_), K(arg)); + LOG_WARN("failed to apply tablet fork mds", KR(ret), K(scn), K(arg)); } else { - LOG_DEBUG("tablet fork mds applied", K(scn), K(arg.ls_id_), + LOG_DEBUG("tablet fork mds applied", K(scn), "has_autoinc", arg.autoinc_seq_arg_.is_valid(), "autoinc_param_cnt", arg.autoinc_seq_arg_.is_valid() ? arg.autoinc_seq_arg_.autoinc_params_.count() : 0, "has_truncate", arg.truncate_arg_.is_valid()); @@ -278,4 +270,3 @@ int ObTabletForkMdsHelper::modify( } // namespace storage } // namespace oceanbase - diff --git a/src/storage/tablet/ob_tablet_fork_mds_helper.h b/src/storage/tablet/ob_tablet_fork_mds_helper.h index d8cbb8060..fbd87fe11 100644 --- a/src/storage/tablet/ob_tablet_fork_mds_helper.h +++ b/src/storage/tablet/ob_tablet_fork_mds_helper.h @@ -20,7 +20,6 @@ #include "common/ob_tablet_id.h" #include "lib/allocator/page_arena.h" #include "common/mysqlclient/ob_mysql_transaction.h" -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" #include "share/ob_tablet_autoincrement_param.h" #include "rootserver/truncate_info/ob_truncate_tablet_arg.h" @@ -48,7 +47,7 @@ struct BufferCtx; class ObTabletForkMdsArg final { -OB_UNIS_VERSION(1); +OB_UNIS_VERSION(2); public: ObTabletForkMdsArg(); ~ObTabletForkMdsArg(); @@ -57,11 +56,10 @@ OB_UNIS_VERSION(1); int set_autoinc_seq_arg(const obcall::ObBatchSetTabletAutoincSeqArg &arg); int set_truncate_arg(const rootserver::ObTruncateTabletArg &arg); - TO_STRING_KV(K_(ls_id), K_(autoinc_seq_arg), K_(truncate_arg)); + TO_STRING_KV(K_(autoinc_seq_arg), K_(truncate_arg)); public: - share::ObLSID ls_id_; obcall::ObBatchSetTabletAutoincSeqArg autoinc_seq_arg_; rootserver::ObTruncateTabletArg truncate_arg_; @@ -85,4 +83,3 @@ class ObTabletForkMdsHelper } // namespace oceanbase #endif // OCEANBASE_STORAGE_OB_TABLET_FORK_MDS_HELPER - diff --git a/src/storage/tablet/ob_tablet_iterator.cpp b/src/storage/tablet/ob_tablet_iterator.cpp index 98c78bc17..bb16baa5d 100644 --- a/src/storage/tablet/ob_tablet_iterator.cpp +++ b/src/storage/tablet/ob_tablet_iterator.cpp @@ -88,7 +88,6 @@ int ObLSTabletIterator::get_next_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle) ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); do { ObTabletMapKey key; - key.ls_id_ = ls_tablet_service_->ls_->get_ls_id(); if (OB_UNLIKELY(tablet_ids_.count() == idx_)) { ret = OB_ITER_END; } else { @@ -118,125 +117,6 @@ int ObLSTabletIterator::get_tablet_ids(ObIArray &ids) const return ret; } -ObHALSTabletIDIterator::ObHALSTabletIDIterator( - const share::ObLSID &ls_id, - const bool need_initial_state, - const bool need_sorted_tablet_id) - : ls_id_(ls_id), - tablet_ids_(), - idx_(0), - need_initial_state_(need_initial_state), - need_sorted_tablet_id_(need_sorted_tablet_id) -{ -} - -ObHALSTabletIDIterator::~ObHALSTabletIDIterator() -{ - reset(); -} - -bool ObHALSTabletIDIterator::is_valid() const -{ - return ls_id_.is_valid(); -} - -void ObHALSTabletIDIterator::reset() -{ - ls_id_.reset(); - tablet_ids_.reset(); - idx_ = 0; -} - -int ObHALSTabletIDIterator::sort_tablet_ids_if_need() -{ - int ret = OB_SUCCESS; - if (!need_sorted_tablet_id_) { - // do nothing - } else if (OB_UNLIKELY(0 != idx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get next tablet id before sort", K(ret), K_(idx)); - } else { - lib::ob_sort(tablet_ids_.begin(), tablet_ids_.end()); - LOG_INFO("sort tablet ids if need"); - } - return ret; -} - -int ObHALSTabletIDIterator::get_next_tablet_id(common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObTabletMapKey key; - key.ls_id_ = ls_id_; - - bool initial_state = true; - while (OB_SUCC(ret)) { - if (OB_UNLIKELY(tablet_ids_.count() == idx_)) { - ret = OB_ITER_END; - } else { - initial_state = true; - key.tablet_id_ = tablet_ids_.at(idx_); - if (OB_FAIL(t3m->get_tablet_pointer_initial_state(key, initial_state))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ++idx_; - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get tablet status from tablet pointer", K(ret), K(key)); - } - } else if (initial_state && !need_initial_state_) { - LOG_INFO("tablet is in initial state, should skip", K(ret), K(key)); - ++idx_; - } else { - ++idx_; - tablet_id = key.tablet_id_; - break; - } - } - } - - return ret; -} - - -ObHALSTabletIterator::ObHALSTabletIterator( - const share::ObLSID &ls_id, - const bool need_initial_state, - const bool need_sorted_tablet_id) - : ls_tablet_service_(nullptr), - tablet_id_iter_(ls_id, need_initial_state, need_sorted_tablet_id) -{} - - -ObHALSTabletIterator::~ObHALSTabletIterator() -{} - - -void ObHALSTabletIterator::reset() -{ - tablet_id_iter_.reset(); -} - -int ObHALSTabletIterator::get_next_tablet(ObTabletHandle &handle) -{ - int ret = OB_SUCCESS; - ObTabletID tablet_id; - handle.reset(); - if (OB_ISNULL(ls_tablet_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls tablet service is nullptr", K(ret), KP(ls_tablet_service_)); - } else if (OB_FAIL(tablet_id_iter_.get_next_tablet_id(tablet_id))) { - if (OB_ITER_END != ret) { - LOG_WARN("failed to get next tablet id", K(ret)); - } - } else if (OB_FAIL(ls_tablet_service_->get_tablet(tablet_id, handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); - } - - return ret; -} - - - ObLSTabletFastIter::ObLSTabletFastIter(ObITabletFilterOp &op, const ObMDSGetTabletMode mode) : ls_tablet_service_(nullptr), tablet_ids_(), @@ -281,7 +161,6 @@ int ObLSTabletAddrIterator::get_next_tablet_addr(ObTabletMapKey &key, ObMetaDisk ret = OB_ERR_UNEXPECTED; LOG_WARN("ls tablet service is nullptr", K(ret), KP(ls_tablet_service_)); } else { - key.ls_id_ = ls_tablet_service_->ls_->get_ls_id(); do { if (OB_UNLIKELY(tablet_ids_.count() == idx_)) { ret = OB_ITER_END; diff --git a/src/storage/tablet/ob_tablet_iterator.h b/src/storage/tablet/ob_tablet_iterator.h index 0af89e909..06094cb9b 100644 --- a/src/storage/tablet/ob_tablet_iterator.h +++ b/src/storage/tablet/ob_tablet_iterator.h @@ -21,7 +21,6 @@ #include "lib/container/ob_se_array.h" #include "lib/utility/ob_print_utils.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "storage/tablet/ob_tablet_common.h" #include "storage/meta_mem/ob_tablet_pointer.h" @@ -85,58 +84,6 @@ class ObLSTabletAddrIterator final int64_t idx_; }; -class ObHALSTabletIDIterator final -{ - friend class ObLSTabletService; -public: - ObHALSTabletIDIterator( - const share::ObLSID &ls_id, - const bool need_initial_state, - const bool need_sorted_tablet_id); - ~ObHALSTabletIDIterator(); - ObHALSTabletIDIterator(const ObHALSTabletIDIterator&) = delete; - ObHALSTabletIDIterator &operator=(const ObHALSTabletIDIterator&) = delete; -public: - int get_next_tablet_id(common::ObTabletID &tablet_id); - - void reset(); - bool is_valid() const; - - TO_STRING_KV(K_(ls_id), K_(tablet_ids), K_(idx)); - -private: - int sort_tablet_ids_if_need(); - -private: - share::ObLSID ls_id_; - common::ObSEArray tablet_ids_; - int64_t idx_; - const bool need_initial_state_; - const bool need_sorted_tablet_id_; -}; - - -class ObHALSTabletIterator final -{ - friend class ObLSTabletService; -public: - ObHALSTabletIterator(const share::ObLSID &ls_id, - const bool need_initial_state, - const bool need_sorted_tablet_id); - ~ObHALSTabletIterator(); - ObHALSTabletIterator(const ObHALSTabletIterator&) = delete; - ObHALSTabletIterator &operator=(const ObHALSTabletIterator&) = delete; -public: - int get_next_tablet(ObTabletHandle &handle); - - void reset(); - - TO_STRING_KV(KP_(ls_tablet_service), K_(tablet_id_iter)); -private: - ObLSTabletService *ls_tablet_service_; - ObHALSTabletIDIterator tablet_id_iter_; -}; - class ObLSTabletFastIter final { friend class ObLSTabletService; diff --git a/src/storage/tablet/ob_tablet_mds_data.cpp b/src/storage/tablet/ob_tablet_mds_data.cpp index f7116199f..9cc5dd415 100644 --- a/src/storage/tablet/ob_tablet_mds_data.cpp +++ b/src/storage/tablet/ob_tablet_mds_data.cpp @@ -959,7 +959,6 @@ int ObTabletMdsData::build_tablet_status( user_data.create_commit_scn_ = create_commit_scn; user_data.create_commit_version_ = tx_data.tx_scn_.get_val_for_tx(); user_data.reserved_scn_ = tx_data.reserved_scn_; - user_data.reserved_ls_id_ = tx_data.reserved_ls_id_; if (ObTabletStatus::DELETED == tx_data.tablet_status_) { //TODO(bizhu) check deleted trans scn user_data.delete_commit_scn_ = tx_data.tx_scn_; diff --git a/src/storage/tablet/ob_tablet_mds_table_mini_merger.cpp b/src/storage/tablet/ob_tablet_mds_table_mini_merger.cpp index a4b9fe3c2..59cef9975 100644 --- a/src/storage/tablet/ob_tablet_mds_table_mini_merger.cpp +++ b/src/storage/tablet/ob_tablet_mds_table_mini_merger.cpp @@ -285,43 +285,6 @@ int ObTabletDumpMds2MiniOperator::operator()(const mds::MdsDumpKV &kv) return ret; } -ObCrossLSMdsMiniMergeOperator::ObCrossLSMdsMiniMergeOperator(const share::SCN &scan_end_scn) - : ObMdsMiniMergeOperator(), - scan_end_scn_(scan_end_scn) -{ -} - -int ObCrossLSMdsMiniMergeOperator::operator()(const mds::MdsDumpKV &kv) -{ - int ret = OB_SUCCESS; - constexpr uint8_t tablet_status_mds_unit_id = mds::TupleTypeIdx>::value; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (tablet_status_mds_unit_id == kv.k_.mds_unit_id_) { - // filter tablet status mds kv - LOG_INFO("meet tablet status mds row, should skip", K(ret), K(tablet_status_mds_unit_id)); - } else if (kv.v_.end_scn_ > scan_end_scn_) { - LOG_INFO("node end scn is beyond scan end scn, should skip", K(ret), K_(scan_end_scn), K(kv)); - } else { - cur_row_.reuse(); - cur_allocator_.reuse(); - mds::MdsDumpKVStorageAdapter adapter(kv); - if (OB_FAIL(adapter.convert_to_mds_row(cur_allocator_, cur_row_))) { - LOG_WARN("fail to convert MdsDumpKVStorageAdapter to row", K(ret), K(adapter), K(cur_row_)); - } else if (OB_FAIL(row_store_.put_row_into_queue(cur_row_))) { - LOG_WARN("fail to put row into queue", K(ret)); - } else { - LOG_INFO("cross ls mds op succeed to add row", K(ret), K(adapter), K(cur_row_)); - } - } - - return ret; -} - int ObTabletDumpMediumMds2MiniOperator::operator()(const mds::MdsDumpKV &kv) { int ret = OB_SUCCESS; @@ -379,7 +342,6 @@ int ObMdsTableMiniMerger::init(compaction::ObTabletMergeCtx &ctx, ObMdsMiniMerge ret = OB_INIT_TWICE; MDS_LOG(WARN, "init twice", K(ret)); } else { - const share::ObLSID &ls_id = ctx.get_ls_id(); const common::ObTabletID &tablet_id = ctx.get_tablet_id(); const ObStorageSchema *storage_schema = ObMdsSchemaHelper::get_instance().get_storage_schema(); uint64_t data_version = 0; @@ -402,11 +364,11 @@ int ObMdsTableMiniMerger::init(compaction::ObTabletMergeCtx &ctx, ObMdsMiniMerge } else if (OB_UNLIKELY(!storage_schema->is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("mds storage schema is invalid", K(ret), KP(storage_schema), KPC(storage_schema)); - } else if (OB_FAIL(data_desc_.init(false/*is ddl*/, *storage_schema, ls_id, tablet_id, + } else if (OB_FAIL(data_desc_.init(false/*is ddl*/, *storage_schema, tablet_id, ctx.get_merge_type(), ctx.get_snapshot(), data_version, ctx.static_desc_.micro_index_clustered_, ctx.get_concurrent_cnt(), ctx.static_param_.scn_range_.end_scn_, ctx.get_exec_mode()))) { - LOG_WARN("fail to init whole desc", KR(ret), K(ctx), K(ls_id), K(tablet_id)); + LOG_WARN("fail to init whole desc", KR(ret), K(ctx), K(tablet_id)); } else if (OB_FAIL(macro_start_seq.set_parallel_degree(0))) { LOG_WARN("Failed to set parallel degree to macro start seq", K(ret)); } else if (OB_FAIL(macro_start_seq.set_sstable_seq(ctx.static_param_.sstable_logic_seq_))) { @@ -465,65 +427,9 @@ int ObMdsTableMiniMerger::generate_mds_mini_sstable( if (OB_FAIL(ret)) { FLOG_WARN("fail to generate mds mini sstable", K(ret)); } else { - const share::ObLSID &ls_id = ctx_->get_ls_id(); const common::ObTabletID &tablet_id = ctx_->get_tablet_id(); const blocksstable::ObSSTable *sstable = static_cast(table_handle.get_table()); - LOG_TRACE("succeed to generate mds mini sstable", K(ret), K(ls_id), K(tablet_id), KPC(sstable)); - } - return ret; -} - -int ObMdsDataCompatHelper::generate_mds_mini_sstable( - const ObMigrationTabletParam &mig_param, - common::ObArenaAllocator &allocator, - ObTableHandleV2 &table_handle) -{ - int ret = OB_SUCCESS; - TIMEGUARD_INIT(STORAGE, 10_ms); - compaction::ObTabletMergeDagParam param; - const share::ObLSID &ls_id = mig_param.ls_id_; - const common::ObTabletID &tablet_id = mig_param.tablet_id_; - compaction::ObTabletMergeCtx *ctx = nullptr; - void *buf = nullptr; - if (OB_UNLIKELY(!mig_param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid migration tablet param", K(ret), K(mig_param)); - } else if (OB_ISNULL((buf = allocator.alloc(sizeof(compaction::ObTabletMergeCtx))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObTabletMergeCtx", K(ret), K(allocator)); - } else { - // (start_scn, enc_scn] set (1, tablet_mds_checkpoint_scn], which means contains all mds data. - // After we set in this way, next mini merge, next mds sstable start scn will be tablet_mds_checkpoint_scn, sstables will be continuous. - param.ls_id_ = ls_id; - param.tablet_id_ = tablet_id; - param.merge_type_ = compaction::ObMergeType::MDS_MINI_MERGE; - param.merge_version_ = 0; - ctx = new (buf) compaction::ObTabletMergeCtx(param, allocator); - ctx->static_param_.start_time_ = common::ObTimeUtility::fast_current_time(); - ctx->static_param_.scn_range_.start_scn_ = share::SCN::plus(share::SCN::min_scn(), 1); - ctx->static_param_.scn_range_.end_scn_ = mig_param.mds_checkpoint_scn_; - ctx->static_param_.version_range_.snapshot_version_ = mig_param.mds_checkpoint_scn_.get_val_for_tx(); - ctx->static_param_.pre_warm_param_.type_ = ObPreWarmerType::MEM_PRE_WARM; - ctx->parallel_merge_ctx_.init_serial_merge(); // only use concurrent_cnt for small sstable temp space optimization - } - - if (OB_FAIL(ret)) { - } else { - SMART_VARS_2((ObMdsTableMiniMerger, mds_mini_merger), (ObTabletDumpMds2MiniOperator, op)) { - if (OB_FAIL(mds_mini_merger.init(*ctx, op))) { - LOG_WARN("fail to init mds mini merger", K(ret), KPC(ctx), K(ls_id), K(tablet_id)); - } else if (CLICK_FAIL((mig_param.mds_data_.scan_all_mds_data_with_op(mig_param.mds_checkpoint_scn_, op)))) { - LOG_WARN("failed to handle full memory mds data", K(ret), - K(ls_id), K(tablet_id), "mds_checkpoint_scn", mig_param.mds_checkpoint_scn_); - } else if (OB_FAIL(mds_mini_merger.generate_mds_mini_sstable(allocator, table_handle))) { - LOG_WARN("fail to generate mds mini sstable with mini merger", K(ret), K(mds_mini_merger)); - } - } - } - - // always destruct merge ctx - if (OB_NOT_NULL(ctx)) { - ctx->~ObTabletMergeCtx(); + LOG_TRACE("succeed to generate mds mini sstable", K(ret), K(tablet_id), KPC(sstable)); } return ret; } @@ -537,12 +443,11 @@ int ObMdsDataCompatHelper::generate_mds_mini_sstable( int ret = OB_SUCCESS; TIMEGUARD_INIT(STORAGE, 10_ms); compaction::ObTabletMergeDagParam param; - const share::ObLSID &ls_id = tablet.get_ls_id(); const common::ObTabletID &tablet_id = tablet.get_tablet_id(); void *buf = nullptr; if (tablet.is_ls_inner_tablet()) { ret = OB_NO_NEED_UPDATE; - LOG_INFO("no need to generate mds sstable for ls inner tablet", K(ret), K(ls_id), K(tablet_id)); + LOG_INFO("no need to generate mds sstable for ls inner tablet", K(ret), K(tablet_id)); } else if (OB_ISNULL(buf = allocator.alloc(sizeof(compaction::ObTabletMergeCtx)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc ObTabletMergeCtx", K(ret), K(allocator)); @@ -550,7 +455,6 @@ int ObMdsDataCompatHelper::generate_mds_mini_sstable( ObTabletFullMemoryMdsData data; // (start_scn, enc_scn] set (1, tablet_mds_checkpoint_scn], which means contains all mds data. // After we set in this way, next mini merge, next mds sstable start scn will be tablet_mds_checkpoint_scn, sstables will be continuous. - param.ls_id_ = ls_id; param.tablet_id_ = tablet_id; param.merge_type_ = compaction::ObMergeType::MDS_MINI_MERGE; param.merge_version_ = 0; @@ -567,15 +471,14 @@ int ObMdsDataCompatHelper::generate_mds_mini_sstable( } else { SMART_VARS_2((ObMdsTableMiniMerger, mds_mini_merger), (ObTabletDumpMds2MiniOperator, op)) { if (CLICK_FAIL(mds_mini_merger.init(*ctx, op))) { - LOG_ERROR("fail to init mds mini merger", K(ret), KPC(ctx), K(ls_id), K(tablet_id)); + LOG_ERROR("fail to init mds mini merger", K(ret), KPC(ctx), K(tablet_id)); } else if (CLICK_FAIL((data.scan_all_mds_data_with_op(tablet.get_mds_checkpoint_scn(), op)))) { - LOG_WARN("failed to handle full memory mds data", K(ret), - K(ls_id), K(tablet_id), "mds_checkpoint_scn", tablet.get_mds_checkpoint_scn()); + LOG_WARN("failed to handle full memory mds data", K(ret), K(tablet_id), "mds_checkpoint_scn", tablet.get_mds_checkpoint_scn()); } else if (CLICK_FAIL(mds_mini_merger.generate_mds_mini_sstable(allocator, table_handle))) { LOG_ERROR("fail to generate mds mini sstable with mini merger", K(ret), K(mds_mini_merger)); } else { has_tablet_status = !data.tablet_status_committed_kv_.v_.user_data_.empty() ? true : false; - LOG_INFO("succeed to generate mds mini sstable for compat", K(ret), K(ls_id), K(tablet_id), K(has_tablet_status), K(data)); + LOG_INFO("succeed to generate mds mini sstable for compat", K(ret), K(tablet_id), K(has_tablet_status), K(data)); } } } diff --git a/src/storage/tablet/ob_tablet_mds_table_mini_merger.h b/src/storage/tablet/ob_tablet_mds_table_mini_merger.h index 44e5d7893..75d552315 100644 --- a/src/storage/tablet/ob_tablet_mds_table_mini_merger.h +++ b/src/storage/tablet/ob_tablet_mds_table_mini_merger.h @@ -91,18 +91,6 @@ class ObTabletDumpMds2MiniOperator : public ObMdsMiniMergeOperator // last_rowkey same as cur_row | cur_row ... // last_rowkey not same as cur row | cur_row is First | last_row is Last -class ObCrossLSMdsMiniMergeOperator : public ObMdsMiniMergeOperator -{ -public: - explicit ObCrossLSMdsMiniMergeOperator(const share::SCN &scan_end_scn); - virtual ~ObCrossLSMdsMiniMergeOperator() = default; -protected: - virtual bool for_flush() override { return false; } - virtual int operator()(const mds::MdsDumpKV &kv) override; -private: - share::SCN scan_end_scn_; -}; - // to query all medium mds info, and dump them to minor sstable. class ObTabletDumpMediumMds2MiniOperator : public ObMdsMiniMergeOperator { @@ -140,10 +128,6 @@ class ObMdsTableMiniMerger class ObMdsDataCompatHelper { public: - static int generate_mds_mini_sstable( - const ObMigrationTabletParam &mig_param, - common::ObArenaAllocator &allocator, - ObTableHandleV2 &table_handle); static int generate_mds_mini_sstable( const ObTablet &tablet, common::ObArenaAllocator &allocator, diff --git a/src/storage/tablet/ob_tablet_medium_info_reader.cpp b/src/storage/tablet/ob_tablet_medium_info_reader.cpp index 6390c414b..8db6ee115 100644 --- a/src/storage/tablet/ob_tablet_medium_info_reader.cpp +++ b/src/storage/tablet/ob_tablet_medium_info_reader.cpp @@ -42,7 +42,6 @@ int ObTabletMediumInfoReader::init( ObTableScanParam &scan_param) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_id(); if (OB_UNLIKELY(is_inited_)) { @@ -51,7 +50,7 @@ int ObTabletMediumInfoReader::init( } else if (OB_FAIL((tablet.mds_range_query( scan_param, iter_)))) { - LOG_WARN("fail to do build query range iter", K(ret), K(ls_id), K(tablet_id), K(scan_param)); + LOG_WARN("fail to do build query range iter", K(ret), K(tablet_id), K(scan_param)); } else { is_inited_ = true; } @@ -156,7 +155,6 @@ int ObTabletMediumInfoReader::get_medium_info_with_merge_version( { int ret = OB_SUCCESS; medium_info = nullptr; - const share::ObLSID &ls_id = tablet.get_ls_id(); const ObTabletID &tablet_id = tablet.get_tablet_id(); ObMediumCompactionInfoKey medium_info_key(merge_version); if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info))) { @@ -166,12 +164,11 @@ int ObTabletMediumInfoReader::get_medium_info_with_merge_version( SMART_VARS_2((ObTableScanParam, scan_param), (ObTabletMediumInfoReader, medium_info_reader)) { if (OB_FAIL((ObMdsScanParamHelper::build_customized_scan_param( allocator, - ls_id, tablet_id, ObMdsScanParamHelper::get_whole_read_version_range(), unused_collector, scan_param)))) { - LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to build scan param", K(ret), K(tablet_id)); } else if (OB_FAIL(medium_info_reader.init(tablet, scan_param))) { LOG_WARN("fail to init medium info reader", K(ret)); } else if (OB_FAIL(medium_info_reader.get_specified_medium_info(allocator, medium_info_key, *medium_info))) { diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index d6945ff2e..1e66fee9b 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -84,13 +84,11 @@ void ObTabletMemtableMgr::destroy() } int ObTabletMemtableMgr::init(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) { int ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObMdsTableMgr *mds_table_mgr = nullptr; if (OB_UNLIKELY(is_inited_)) { @@ -101,23 +99,16 @@ int ObTabletMemtableMgr::init(const common::ObTabletID &tablet_id, || OB_ISNULL(freezer)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_id), KP(freezer), KP(t3m)); - } else if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ls service should not be NULL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, - ls_handle, - ObLSGetMod::TABLET_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("failed to get ls", K(ret)); - } else if (OB_ISNULL(ls_ = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ls should not be NULL", K(ret), KP(ls_)); } else { + ls_ = tenant_ls; tablet_id_ = tablet_id; t3m_ = t3m; freezer_ = freezer; retry_times_ = 0; is_inited_ = true; - TRANS_LOG(DEBUG, "succeeded to init tablet memtable mgr", K(ret), K(ls_id), K(tablet_id), KP(this), KPC(this)); + TRANS_LOG(DEBUG, "succeeded to init tablet memtable mgr", K(ret), K(tablet_id), KP(this), KPC(this)); } if (OB_UNLIKELY(!is_inited_)) { @@ -128,19 +119,18 @@ int ObTabletMemtableMgr::init(const common::ObTabletID &tablet_id, int ObTabletMemtableMgr::init_storage_recorder( const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const int64_t max_saved_schema_version, const int64_t max_saved_medium_scn, const lib::Worker::CompatMode compat_mode, logservice::ObLogHandler *log_handler) { int ret = OB_SUCCESS; - if (OB_FAIL(schema_recorder_.init(ls_id, tablet_id, max_saved_schema_version, compat_mode, log_handler))) { + if (OB_FAIL(schema_recorder_.init(tablet_id, max_saved_schema_version, compat_mode, log_handler))) { TRANS_LOG(WARN, "failed to init schema recorder", K(ret), K(max_saved_schema_version), KP(log_handler)); - } else if (OB_FAIL(medium_info_recorder_.init(ls_id, tablet_id, max_saved_medium_scn, log_handler))) { + } else if (OB_FAIL(medium_info_recorder_.init(tablet_id, max_saved_medium_scn, log_handler))) { TRANS_LOG(WARN, "failed to init medium info recorder", K(ret), K(max_saved_medium_scn), KP(log_handler)); } else { - TRANS_LOG(INFO, "success to init storage recorder", K(ret), K(ls_id), K(tablet_id), K(max_saved_schema_version), + TRANS_LOG(INFO, "success to init storage recorder", K(ret), K(tablet_id), K(max_saved_schema_version), K(max_saved_medium_scn), K(compat_mode)); } return ret; @@ -161,7 +151,7 @@ int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_for_leader_( if (0 == write_ref && 0 == unsubmitted_cnt) { share::SCN max_decided_scn; if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_decided_scn))) { - TRANS_LOG(WARN, "get max decided scn fail", K(ret), K(freezer_->get_ls_id())); + TRANS_LOG(WARN, "get max decided scn fail", K(ret)); } else if (max_decided_scn >= last_frozen_tablet_memtable->get_end_scn()) { // logstream's continous apply has pass frozen memtable's right boundary can_resolve = true; @@ -219,7 +209,6 @@ int ObTabletMemtableMgr::create_memtable(const CreateMemtableArg &arg) time_guard.click("lock"); int ret = OB_SUCCESS; - ObLSID ls_id; const uint32_t logstream_freeze_clock = freezer_->get_freeze_clock(); if (OB_UNLIKELY(!is_inited_)) { @@ -228,9 +217,8 @@ int ObTabletMemtableMgr::create_memtable(const CreateMemtableArg &arg) } else if (OB_ISNULL(ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is null", K(ret)); - } else if (FALSE_IT(ls_id = ls_->get_ls_id())) { } else if (has_memtable_() && OB_FAIL(check_boundary_memtable_(logstream_freeze_clock))) { - STORAGE_LOG(DEBUG, "check boundary memtable failed", KR(ret), K(arg), K(ls_id)); + STORAGE_LOG(DEBUG, "check boundary memtable failed", KR(ret), K(arg)); } else if (get_memtable_count_() >= MAX_MEMSTORE_CNT) { ret = OB_MINOR_FREEZE_NOT_ALLOW; ob_usleep(1 * 1000); @@ -239,7 +227,6 @@ int ObTabletMemtableMgr::create_memtable(const CreateMemtableArg &arg) get_first_frozen_memtable_(first_frozen_memtable); LOG_ERROR("cannot create more memtable", K(ret), - K(ls_id), K(tablet_id_), K(MAX_MEMSTORE_CNT), K(get_memtable_count_()), @@ -247,14 +234,14 @@ int ObTabletMemtableMgr::create_memtable(const CreateMemtableArg &arg) } } else if (arg.for_replay_ && arg.clog_checkpoint_scn_ != arg.new_clog_checkpoint_scn_) { ret = OB_EAGAIN; - LOG_INFO("clog_checkpoint_scn changed, need retry to replay", K(ls_id), K(tablet_id_), K(arg)); + LOG_INFO("clog_checkpoint_scn changed, need retry to replay", K(tablet_id_), K(arg)); } else if (MAX_MEMSTORE_CNT - 1 == get_memtable_count_() && arg.for_inc_direct_load_) { ret = OB_EAGAIN; if (REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL)) { STORAGE_LOG(INFO, "only data memtable can use the last slot in memtable mgr", K(ret), K(arg), KPC(this)); } } else if (OB_FAIL(create_memtable_(arg, logstream_freeze_clock, time_guard))) { - STORAGE_LOG(WARN, "create memtable failed", KR(ret), K(ls_id)); + STORAGE_LOG(WARN, "create memtable failed", KR(ret)); } return ret; @@ -301,31 +288,26 @@ int ObTabletMemtableMgr::create_memtable_(const CreateMemtableArg &arg, ObTableHandleV2 memtable_handle; ObITable::TableKey table_key; - const ObLSID ls_id = ls_->get_ls_id(); table_key.table_type_ = arg.for_inc_direct_load_ ? ObITable::DIRECT_LOAD_MEMTABLE : ObITable::DATA_MEMTABLE; table_key.tablet_id_ = tablet_id_; table_key.scn_range_.start_scn_ = arg.clog_checkpoint_scn_; table_key.scn_range_.end_scn_.set_max(); ObITabletMemtable *new_tablet_memtable = NULL; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; retry_times_ = 0; if (OB_FAIL(acquire_tablet_memtable_(arg.for_inc_direct_load_, memtable_handle))) { - LOG_WARN("failed to create memtable", K(ret), K(ls_id), K(tablet_id_)); + LOG_WARN("failed to create memtable", K(ret), K(tablet_id_)); } else if (FALSE_IT(tg.click("acquire_memtable"))) { } else if (OB_ISNULL(new_tablet_memtable = static_cast(memtable_handle.get_table()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get memtable", K(ret), K(ls_id), K(tablet_id_), K(memtable_handle)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::DATA_MEMTABLE_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id), K(tablet_id_)); - } else if (OB_UNLIKELY(!ls_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, invalid ls handle", K(ret), K(ls_handle), K(ls_id), K(tablet_id_)); + LOG_WARN("fail to get memtable", K(ret), K(tablet_id_), K(memtable_handle)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret), K(tablet_id_)); } else if (OB_FAIL(new_tablet_memtable->init( - table_key, ls_handle, freezer_, this, arg.schema_version_, logstream_freeze_clock))) { + table_key, tenant_ls, freezer_, this, arg.schema_version_, logstream_freeze_clock))) { LOG_WARN("failed to init memtable", K(ret), - K(ls_id), K(table_key), KP(freezer_), KP(this), @@ -333,18 +315,18 @@ int ObTabletMemtableMgr::create_memtable_(const CreateMemtableArg &arg, K(logstream_freeze_clock)); } else if (FALSE_IT(new_tablet_memtable->set_delete_insert_flag(arg.is_delete_insert_))) { } else if (OB_FAIL(resolve_boundary_(new_tablet_memtable, arg))) { - LOG_WARN("failed to add memtable", K(ret), K(ls_id), K(tablet_id_), K(memtable_handle)); + LOG_WARN("failed to add memtable", K(ret), K(tablet_id_), K(memtable_handle)); } else if (FALSE_IT(tg.click("init memtable"))) { } else if (FALSE_IT(block_freeze_if_memstore_full_(new_tablet_memtable))) { } else if (OB_FAIL(add_memtable_(memtable_handle))) { - LOG_WARN("failed to add memtable", K(ret), K(ls_id), K(tablet_id_), K(memtable_handle)); + LOG_WARN("failed to add memtable", K(ret), K(tablet_id_), K(memtable_handle)); } else if (FALSE_IT(tg.click("add memtable"))) { } else if (OB_FAIL(new_tablet_memtable->add_to_data_checkpoint(freezer_->get_ls_data_checkpoint()))) { - LOG_WARN("add to data_checkpoint failed", K(ret), K(ls_id), KPC(new_tablet_memtable)); + LOG_WARN("add to data_checkpoint failed", K(ret), KPC(new_tablet_memtable)); clean_tail_memtable_(); } else if (FALSE_IT(tg.click("add to data_checkpoint"))) { } else { - LOG_INFO("succeed to create memtable", K(arg), K(ret), K(ls_id), KPC(new_tablet_memtable), KPC(this)); + LOG_INFO("succeed to create memtable", K(arg), K(ret), KPC(new_tablet_memtable), KPC(this)); } return ret; @@ -711,7 +693,6 @@ int ObTabletMemtableMgr::get_memtable_for_replay(const SCN &replay_scn, ObTableH ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "ls is null", K(ret)); } else { - const share::ObLSID &ls_id = ls_->get_ls_id(); MemMgrRLockGuard lock_guard(lock_); int64_t i = 0; for (i = memtable_tail_ - 1; OB_SUCC(ret) && i >= memtable_head_; --i) { @@ -737,7 +718,7 @@ int ObTabletMemtableMgr::get_memtable_for_replay(const SCN &replay_scn, ObTableH ret = OB_NO_NEED_UPDATE; } else { ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("fail to get memtable for replay", K(ret), K(ls_id), K(tablet_id_), K(replay_scn), K(clog_checkpoint_scn), K(memtable_tail_ - memtable_head_)); + LOG_WARN("fail to get memtable for replay", K(ret), K(tablet_id_), K(replay_scn), K(clog_checkpoint_scn), K(memtable_tail_ - memtable_head_)); } } } @@ -784,7 +765,6 @@ int ObTabletMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "ls is null", K(ret)); } else { - const share::ObLSID &ls_id = ls_->get_ls_id(); const int64_t idx = get_memtable_idx(memtable_head_); int64_t occupy_size = 0; if (nullptr != tables_[idx] && memtable == tables_[idx]) { @@ -793,7 +773,7 @@ int ObTabletMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, if (0 == mt_stat.release_time_) { mt_stat.release_time_ = ObTimeUtility::current_time(); } else { - LOG_WARN("cannot set release_time twice", K(ls_id), KPC(memtable)); + LOG_WARN("cannot set release_time twice", KPC(memtable)); } if (!memtable->is_empty()) { memtable->set_read_barrier(); @@ -810,7 +790,7 @@ int ObTabletMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, ObITabletMemtable *active_memtable = get_active_memtable_(); if (OB_NOT_NULL(active_memtable) && !active_memtable->allow_freeze()) { active_memtable->set_allow_freeze(true); - FLOG_INFO("allow active memtable to be freezed", K(ls_id), KPC(active_memtable)); + FLOG_INFO("allow active memtable to be freezed", KPC(active_memtable)); } FLOG_INFO("succeed to release head data memtable", K(ret), K(occupy_size), KPC(memtable)); @@ -1018,15 +998,14 @@ int ObTabletMemtableMgr::set_frozen_for_all_memtables() ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "ls is null", K(ret)); } else { - const share::ObLSID &ls_id = ls_->get_ls_id(); for (int64_t i = memtable_head_; OB_SUCC(ret) && i < memtable_tail_; ++i) { // memtable that cannot be released will block memtables behind it ObITabletMemtable *memtable = static_cast(tables_[get_memtable_idx(i)]); if (OB_ISNULL(memtable)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "memtable is nullptr", K(ret), K(ls_id), KP(memtable), K(i)); + STORAGE_LOG(WARN, "memtable is nullptr", K(ret), KP(memtable), K(i)); } else { - STORAGE_LOG(INFO, "set frozen for offline", K(ls_id), K(i), KPC(memtable)); + STORAGE_LOG(INFO, "set frozen for offline", K(i), KPC(memtable)); memtable->set_offlined(); memtable->set_frozen(); } diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.h b/src/storage/tablet/ob_tablet_memtable_mgr.h index dce79eb9c..98af14e24 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.h +++ b/src/storage/tablet/ob_tablet_memtable_mgr.h @@ -66,7 +66,6 @@ class ObTabletMemtableMgr : public ObIMemtableMgr public: // derived from ObIMemtableMgr virtual int init(const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) override; @@ -80,7 +79,6 @@ class ObTabletMemtableMgr : public ObIMemtableMgr virtual int get_last_frozen_memtable(ObTableHandleV2 &handle) override; virtual int set_is_tablet_freeze_for_active_memtable(ObTableHandleV2 &handle); virtual int init_storage_recorder(const ObTabletID &tablet_id, - const share::ObLSID &ls_id, const int64_t max_saved_schema_version, const int64_t max_saved_medium_scn, const lib::Worker::CompatMode compat_mode, diff --git a/src/storage/tablet/ob_tablet_meta.cpp b/src/storage/tablet/ob_tablet_meta.cpp index cd3157e5b..a53884c57 100644 --- a/src/storage/tablet/ob_tablet_meta.cpp +++ b/src/storage/tablet/ob_tablet_meta.cpp @@ -37,7 +37,6 @@ const SCN ObTabletMeta::INIT_CREATE_SCN = SCN::invalid_scn(); ObTabletMeta::ObTabletMeta() : version_(TABLET_META_VERSION), length_(0), - ls_id_(), tablet_id_(), data_tablet_id_(), ref_tablet_id_(), @@ -47,7 +46,7 @@ ObTabletMeta::ObTabletMeta() ddl_checkpoint_scn_(SCN::min_scn()), snapshot_version_(OB_INVALID_TIMESTAMP), multi_version_start_(OB_INVALID_TIMESTAMP), - ha_status_(), + restore_state_(), report_status_(), table_store_flag_(), ddl_start_scn_(SCN::min_scn()), @@ -79,7 +78,6 @@ ObTabletMeta::~ObTabletMeta() } int ObTabletMeta::init( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN create_scn, @@ -99,20 +97,18 @@ int ObTabletMeta::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!data_tablet_id.is_valid()) //|| OB_UNLIKELY(create_scn <= OB_INVALID_TIMESTAMP) || OB_UNLIKELY(OB_INVALID_VERSION == snapshot_version) || OB_UNLIKELY(lib::Worker::CompatMode::INVALID == compat_mode)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), + LOG_WARN("invalid args", K(ret), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(clog_checkpoint_scn), K(compat_mode)); - } else if (OB_FAIL(ha_status_.init_status())) { + } else if (OB_FAIL(restore_state_.init_status())) { LOG_WARN("failed to init ha status", K(ret)); } else { version_ = TABLET_META_VERSION; - ls_id_ = ls_id; tablet_id_ = tablet_id; data_tablet_id_ = data_tablet_id; create_scn_ = create_scn; @@ -203,7 +199,6 @@ int ObTabletMeta::init( LOG_WARN("invalid args", K(ret), K(old_tablet_meta), K(max_sync_storage_schema_version)); } else { version_ = TABLET_META_VERSION; - ls_id_ = old_tablet_meta.ls_id_; tablet_id_ = old_tablet_meta.tablet_id_; data_tablet_id_ = old_tablet_meta.data_tablet_id_; ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; @@ -215,7 +210,7 @@ int ObTabletMeta::init( ddl_commit_scn_ = SCN::max(ddl_info.ddl_commit_scn_, old_tablet_meta.ddl_commit_scn_); clog_checkpoint_scn_ = SCN::max(clog_checkpoint_scn, old_tablet_meta.clog_checkpoint_scn_); compat_mode_ = old_tablet_meta.compat_mode_; - ha_status_ = old_tablet_meta.ha_status_; + restore_state_ = old_tablet_meta.restore_state_; report_status_ = old_tablet_meta.report_status_; snapshot_version_ = MAX(snapshot_version, old_tablet_meta.snapshot_version_); @@ -274,7 +269,6 @@ int ObTabletMeta::init( K(clog_checkpoint_scn), K(mds_checkpoint_scn)); } else { version_ = TABLET_META_VERSION; - ls_id_ = old_tablet_meta.ls_id_; tablet_id_ = old_tablet_meta.tablet_id_; data_tablet_id_ = old_tablet_meta.data_tablet_id_; ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; @@ -286,7 +280,7 @@ int ObTabletMeta::init( ddl_commit_scn_ = old_tablet_meta.ddl_commit_scn_; clog_checkpoint_scn_ = SCN::max(clog_checkpoint_scn, old_tablet_meta.clog_checkpoint_scn_); compat_mode_ = old_tablet_meta.compat_mode_; - ha_status_ = old_tablet_meta.ha_status_; + restore_state_ = old_tablet_meta.restore_state_; report_status_ = old_tablet_meta.report_status_; snapshot_version_ = MAX(snapshot_version, old_tablet_meta.snapshot_version_); @@ -335,7 +329,6 @@ int ObTabletMeta::init( LOG_WARN("invalid args", K(ret), K(old_tablet_meta), K(flush_scn)); } else { version_ = TABLET_META_VERSION; - ls_id_ = old_tablet_meta.ls_id_; tablet_id_ = old_tablet_meta.tablet_id_; data_tablet_id_ = old_tablet_meta.data_tablet_id_; ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; @@ -348,7 +341,7 @@ int ObTabletMeta::init( snapshot_version_ = old_tablet_meta.snapshot_version_; multi_version_start_ = old_tablet_meta.multi_version_start_; compat_mode_ = old_tablet_meta.compat_mode_; - ha_status_ = old_tablet_meta.ha_status_; + restore_state_ = old_tablet_meta.restore_state_; report_status_ = old_tablet_meta.report_status_; table_store_flag_ = old_tablet_meta.table_store_flag_; ddl_start_scn_ = old_tablet_meta.ddl_start_scn_; @@ -381,72 +374,6 @@ int ObTabletMeta::init( return ret; } -int ObTabletMeta::init( - const ObMigrationTabletParam ¶m) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(param)); - } else { - version_ = TABLET_META_VERSION; - ls_id_ = param.ls_id_; - tablet_id_ = param.tablet_id_; - data_tablet_id_ = param.data_tablet_id_; - ref_tablet_id_ = param.ref_tablet_id_; - create_scn_ = param.create_scn_; - create_schema_version_ = param.create_schema_version_; - micro_index_clustered_ = param.micro_index_clustered_; - start_scn_ = param.start_scn_; - clog_checkpoint_scn_ = param.clog_checkpoint_scn_; - ddl_checkpoint_scn_ = param.ddl_checkpoint_scn_; - snapshot_version_ = param.snapshot_version_; - multi_version_start_ = param.multi_version_start_; - compat_mode_ = param.compat_mode_; - report_status_.reset(); - table_store_flag_ = param.table_store_flag_; - ddl_start_scn_ = param.ddl_start_scn_; - ddl_commit_scn_ = param.ddl_commit_scn_; - ddl_snapshot_version_ = param.ddl_snapshot_version_; - max_sync_storage_schema_version_ = param.max_sync_storage_schema_version_; - ha_status_ = param.ha_status_; - max_serialized_medium_scn_ = param.max_serialized_medium_scn_; - ddl_execution_id_ = param.ddl_execution_id_; - ddl_data_format_version_ = param.ddl_data_format_version_; - mds_checkpoint_scn_ = param.mds_checkpoint_scn_; - is_empty_shell_ = param.is_empty_shell_; - extra_medium_info_ = param.extra_medium_info_; - split_info_ = param.split_info_; - has_truncate_info_ = param.has_truncate_info_; - fork_info_ = param.fork_info_; - if (param.version_ < ObMigrationTabletParam::PARAM_VERSION_V3) { - int64_t tmp_pos = 0; - const ObString &user_data = param.mds_data_.tablet_status_committed_kv_.v_.user_data_; - if (user_data.empty()) { - last_persisted_committed_tablet_status_.reset(); - } else if (OB_FAIL(last_persisted_committed_tablet_status_.deserialize(user_data.ptr(), user_data.length(), tmp_pos))) { - LOG_WARN("failed to deserialize user data", K(ret), K(tmp_pos), K(user_data)); - } - } else if (OB_FAIL(last_persisted_committed_tablet_status_.assign(param.last_persisted_committed_tablet_status_))) { - LOG_WARN("fail to init last_persisted_committed_tablet_status from mig param", K(ret), - "last_persisted_committed_tablet_status", - param.last_persisted_committed_tablet_status_); - } - if (OB_SUCC(ret)) { - is_inited_ = true; - } - } - - if (OB_UNLIKELY(!is_inited_)) { - reset(); - } - return ret; -} - int ObTabletMeta::assign(const ObTabletMeta &other) { int ret = OB_SUCCESS; @@ -460,7 +387,6 @@ int ObTabletMeta::assign(const ObTabletMeta &other) } else { version_ = other.version_; length_ = other.length_; - ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; data_tablet_id_ = other.data_tablet_id_; ref_tablet_id_ = other.ref_tablet_id_; @@ -470,7 +396,7 @@ int ObTabletMeta::assign(const ObTabletMeta &other) ddl_checkpoint_scn_ = other.ddl_checkpoint_scn_; snapshot_version_ = other.snapshot_version_; multi_version_start_ = other.multi_version_start_; - ha_status_ = other.ha_status_; + restore_state_ = other.restore_state_; report_status_ = other.report_status_; table_store_flag_ = other.table_store_flag_; ddl_start_scn_ = other.ddl_start_scn_; @@ -507,117 +433,9 @@ int ObTabletMeta::assign(const ObTabletMeta &other) return ret; } -int ObTabletMeta::init( - const ObTabletMeta &old_tablet_meta, - const ObMigrationTabletParam *tablet_meta) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!old_tablet_meta.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(old_tablet_meta)); - } else if (OB_FAIL(inner_check_(old_tablet_meta, tablet_meta))) { - LOG_WARN("failed to do inner check", K(ret), K(old_tablet_meta), KP(tablet_meta)); - } else { - const int64_t snapshot_version = OB_ISNULL(tablet_meta) ? - old_tablet_meta.snapshot_version_ : MAX(old_tablet_meta.snapshot_version_, tablet_meta->snapshot_version_); - const int64_t multi_version_start = OB_ISNULL(tablet_meta) ? - old_tablet_meta.multi_version_start_ : MAX(old_tablet_meta.multi_version_start_, tablet_meta->multi_version_start_); - const int64_t max_sync_storage_schema_version = OB_ISNULL(tablet_meta) ? - old_tablet_meta.max_sync_storage_schema_version_ : MIN(old_tablet_meta.max_sync_storage_schema_version_, - tablet_meta->max_sync_storage_schema_version_); - const SCN clog_checkpoint_scn = OB_ISNULL(tablet_meta) ? - old_tablet_meta.clog_checkpoint_scn_ : MAX(old_tablet_meta.clog_checkpoint_scn_, tablet_meta->clog_checkpoint_scn_); - - ObTabletTableStoreFlag table_store_flag = old_tablet_meta.table_store_flag_; - SCN ddl_checkpoint_scn = old_tablet_meta.ddl_checkpoint_scn_; - if (!table_store_flag.with_major_sstable()) { - if (OB_ISNULL(tablet_meta)) { - //do nothing - } else if (tablet_meta->table_store_flag_.with_major_sstable()) { - table_store_flag.set_with_major_sstable(); - ddl_checkpoint_scn = tablet_meta->ddl_checkpoint_scn_; - FLOG_INFO("update tablet table store flag with major", KPC(tablet_meta), K(table_store_flag), K(ddl_checkpoint_scn)); - } - } - const SCN mds_checkpoint_scn = OB_ISNULL(tablet_meta) ? - old_tablet_meta.mds_checkpoint_scn_ : MAX(old_tablet_meta.mds_checkpoint_scn_, tablet_meta->mds_checkpoint_scn_); - // fuse restore status during migration, consider the following timeline - // 1. SOURCE: tablet P0 was created with restore status FULL by replaying reserved status. - // 2. TARGET: rebuild was triggered, then create P0 with restore status FULL, and data status INCOMPLETE. - // 3. SOURCE: reserved status handler modified the restore status of P0 to EMPTY. - // 4. SOURCE: the minor of P0 was restored by restore handler, then set the restore status to MINOR_AND_MAJOR_META. - // 5. TARGET: the minor of P0 was restored by migration, then set data status COMPLETE. - // The result is P0 was FULL, but only exist minor sstables, with no major. - ObTabletHAStatus new_ha_status = old_tablet_meta.ha_status_; - if (!old_tablet_meta.ha_status_.is_data_status_complete() && OB_NOT_NULL(tablet_meta)) { - ObTabletRestoreStatus::STATUS src_restore_status; - if (OB_FAIL(tablet_meta->ha_status_.get_restore_status(src_restore_status))) { - LOG_WARN("failed to get restore status", K(ret), KPC(tablet_meta)); - } else if (OB_FAIL(new_ha_status.set_restore_status(src_restore_status))) { - LOG_WARN("failed to set new restore status", K(ret), K(new_ha_status), K(src_restore_status)); - } - } - - if (OB_SUCC(ret)) { - version_ = TABLET_META_VERSION; - ls_id_ = old_tablet_meta.ls_id_; - tablet_id_ = old_tablet_meta.tablet_id_; - data_tablet_id_ = old_tablet_meta.data_tablet_id_; - ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; - create_scn_ = old_tablet_meta.create_scn_; - start_scn_ = old_tablet_meta.start_scn_; - clog_checkpoint_scn_ = clog_checkpoint_scn; - ddl_checkpoint_scn_ = ddl_checkpoint_scn; - snapshot_version_ = snapshot_version; - multi_version_start_ = multi_version_start; - ha_status_ = new_ha_status; - report_status_ = old_tablet_meta.report_status_; //old tablet meta report status already reset - table_store_flag_ = table_store_flag; - ddl_start_scn_ = old_tablet_meta.ddl_start_scn_; - ddl_snapshot_version_ = old_tablet_meta.ddl_snapshot_version_; - max_sync_storage_schema_version_ = max_sync_storage_schema_version; - ddl_execution_id_ = old_tablet_meta.ddl_execution_id_; - ddl_data_format_version_ = old_tablet_meta.ddl_data_format_version_; - max_serialized_medium_scn_ = MAX(old_tablet_meta.max_serialized_medium_scn_, - OB_ISNULL(tablet_meta) ? 0 : tablet_meta->max_serialized_medium_scn_); - ddl_commit_scn_ = old_tablet_meta.ddl_commit_scn_; - mds_checkpoint_scn_ = mds_checkpoint_scn; - extra_medium_info_ = old_tablet_meta.extra_medium_info_; - space_usage_ = old_tablet_meta.space_usage_; - create_schema_version_ = old_tablet_meta.create_schema_version_; - compat_mode_ = old_tablet_meta.compat_mode_; - has_next_tablet_ = old_tablet_meta.has_next_tablet_; - micro_index_clustered_ = old_tablet_meta.micro_index_clustered_; - split_info_ = OB_ISNULL(tablet_meta) ? old_tablet_meta.split_info_ : tablet_meta->split_info_; // migration sstables replaces split status. - fork_info_ = OB_ISNULL(tablet_meta) ? old_tablet_meta.fork_info_ : tablet_meta->fork_info_; - if (old_tablet_meta.has_truncate_info_ || (OB_NOT_NULL(tablet_meta) && tablet_meta->has_truncate_info_)) { - has_truncate_info_ = true; - } - if (OB_FAIL(last_persisted_committed_tablet_status_.assign(old_tablet_meta.last_persisted_committed_tablet_status_))) { - LOG_WARN("fail to init last_persisted_committed_tablet_status from old tablet meta", K(ret), - "last_persisted_committed_tablet_status", old_tablet_meta.last_persisted_committed_tablet_status_); - } - } - } - - if (OB_SUCC(ret)) { - is_inited_ = true; - } - - if (OB_UNLIKELY(!is_inited_)) { - reset(); - } - return ret; -} - void ObTabletMeta::reset() { version_ = 0; - ls_id_.reset(); tablet_id_.reset(); data_tablet_id_.reset(); ref_tablet_id_.reset(); @@ -631,7 +449,7 @@ void ObTabletMeta::reset() snapshot_version_ = OB_INVALID_TIMESTAMP; multi_version_start_ = OB_INVALID_TIMESTAMP; compat_mode_ = lib::Worker::CompatMode::INVALID; - ha_status_.reset(); + restore_state_.reset(); report_status_.reset(); table_store_flag_.reset(); ddl_start_scn_.set_min(); @@ -655,8 +473,7 @@ void ObTabletMeta::reset() bool ObTabletMeta::is_valid() const { // TODO: add more check - return ls_id_.is_valid() - && tablet_id_.is_valid() + return tablet_id_.is_valid() && data_tablet_id_.is_valid() && create_scn_ != INVALID_CREATE_SCN && multi_version_start_ >= 0 @@ -666,10 +483,10 @@ bool ObTabletMeta::is_valid() const && compat_mode_ != lib::Worker::CompatMode::INVALID && max_sync_storage_schema_version_ >= 0 && max_serialized_medium_scn_ >= 0 - && ha_status_.is_valid() + && restore_state_.is_valid() && last_persisted_committed_tablet_status_.is_valid() - && (ha_status_.is_restore_status_pending() - || (!ha_status_.is_restore_status_pending() + && (restore_state_.is_restore_status_pending() + || (!restore_state_.is_restore_status_pending() && clog_checkpoint_scn_ >= INIT_CLOG_CHECKPOINT_SCN && start_scn_ >= INIT_CLOG_CHECKPOINT_SCN && start_scn_ <= clog_checkpoint_scn_)) @@ -704,8 +521,6 @@ int ObTabletMeta::serialize(char *buf, const int64_t len, int64_t &pos) const LOG_WARN("failed to serialize tablet meta's version", K(ret), K(len), K(new_pos), K_(version)); } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i32(buf, len, new_pos, length))) { LOG_WARN("failed to serialize tablet meta's length", K(ret), K(len), K(new_pos), K(length)); - } else if (new_pos - pos < length && OB_FAIL(ls_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ls id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length && OB_FAIL(tablet_id_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize tablet id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.serialize(buf, len, new_pos))) { @@ -728,7 +543,7 @@ int ObTabletMeta::serialize(char *buf, const int64_t len, int64_t &pos) const LOG_WARN("failed to serialize multi version start", K(ret), K(len), K(new_pos), K_(multi_version_start)); } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i8(buf, len, new_pos, static_cast(compat_mode_)))) { LOG_WARN("failed to serialize compat mode", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.serialize(buf, len, new_pos))) { + } else if (new_pos - pos < length && OB_FAIL(restore_state_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize ha status", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length && OB_FAIL(report_status_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize report status", K(ret), K(len), K(new_pos)); @@ -806,8 +621,6 @@ int ObTabletMeta::deserialize( if (OB_UNLIKELY(length_ > len - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("buffer's length is not enough", K(ret), K(length_), K(len - new_pos)); - } else if (new_pos - pos < length_ && OB_FAIL(ls_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(tablet_id_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize tablet id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(data_tablet_id_.deserialize(buf, len, new_pos))) { @@ -830,7 +643,7 @@ int ObTabletMeta::deserialize( LOG_WARN("failed to deserialize multi version start", K(ret), K(len)); } else if (new_pos - pos < length_ && OB_FAIL(serialization::decode_i8(buf, len, new_pos, &compat_mode))) { LOG_WARN("failed to deserialize compat mode", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length_ && OB_FAIL(ha_status_.deserialize(buf, len, new_pos))) { + } else if (new_pos - pos < length_ && OB_FAIL(restore_state_.deserialize(buf, len, new_pos))) { LOG_ERROR("failed to deserialize restore status", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(report_status_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize report status", K(ret), K(len), K(new_pos)); @@ -892,7 +705,6 @@ int64_t ObTabletMeta::get_serialize_size() const int64_t size = 0; size += serialization::encoded_length_i32(version_); size += serialization::encoded_length_i32(length_); - size += ls_id_.get_serialize_size(); size += tablet_id_.get_serialize_size(); size += data_tablet_id_.get_serialize_size(); size += ref_tablet_id_.get_serialize_size(); @@ -904,7 +716,7 @@ int64_t ObTabletMeta::get_serialize_size() const size += serialization::encoded_length_i64(snapshot_version_); size += serialization::encoded_length_i64(multi_version_start_); size += serialization::encoded_length_i8(static_cast(compat_mode_)); - size += ha_status_.get_serialize_size(); + size += restore_state_.get_serialize_size(); size += report_status_.get_serialize_size(); size += table_store_flag_.get_serialize_size(); size += ddl_start_scn_.get_fixed_serialize_size(); @@ -927,34 +739,6 @@ int64_t ObTabletMeta::get_serialize_size() const return size; } -int ObTabletMeta::deserialize_id( - const char *buf, - const int64_t len, - int64_t &pos, - share::ObLSID &ls_id, - common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - int32_t version = 0; - int32_t length = 0; - if (OB_FAIL(serialization::decode_i32(buf, len, pos, &version))) { - LOG_WARN("fail to deserialize tablet meta's version", K(ret), K(len), K(pos)); - } else if (OB_FAIL(serialization::decode_i32(buf, len, pos, &length))) { - LOG_WARN("fail to deserialize tablet meta's length", K(ret), K(len), K(pos)); - } else if (TABLET_META_VERSION == version) { - if (OB_FAIL(ls_id.deserialize(buf, len, pos))) { - LOG_WARN("fail to deserialize ls id", K(ret), K(len), K(pos)); - } else if (OB_FAIL(tablet_id.deserialize(buf, len, pos))) { - LOG_WARN("fail to deserialize tablet id", K(ret), K(len), K(pos)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("The version of tablet is not 1", K(ret), K(version)); - } - - return ret; -} - int ObTabletMeta::init_report_info( const blocksstable::ObSSTable *sstable, const int64_t report_version, @@ -1015,28 +799,6 @@ share::SCN ObTabletMeta::get_max_replayed_scn() const ddl_checkpoint_scn_); } -int ObTabletMeta::inner_check_( - const ObTabletMeta &old_tablet_meta, - const ObMigrationTabletParam *tablet_meta) -{ - int ret = OB_SUCCESS; - if (!old_tablet_meta.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("inner check get invalid argument", K(ret), K(old_tablet_meta)); - } else if (OB_ISNULL(tablet_meta)) { - //do nothing - } else if (old_tablet_meta.ls_id_ != tablet_meta->ls_id_ - || old_tablet_meta.tablet_id_ != tablet_meta->tablet_id_ - || old_tablet_meta.data_tablet_id_ != tablet_meta->data_tablet_id_ - || old_tablet_meta.ref_tablet_id_ != tablet_meta->ref_tablet_id_ - || old_tablet_meta.compat_mode_ != tablet_meta->compat_mode_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old tablet meta part variable is not same with migration tablet param", - K(ret), K(old_tablet_meta), KPC(tablet_meta)); - } - return ret; -} - void ObTabletMeta::update_extra_medium_info( const compaction::ObMergeType merge_type, const int64_t finish_medium_scn) @@ -1079,7 +841,6 @@ int ObTabletMeta::update_meta_last_persisted_committed_tablet_status( user_data.create_commit_scn_ = create_commit_scn; user_data.create_commit_version_ = tx_data.tx_scn_.get_val_for_tx(); user_data.reserved_scn_ = tx_data.reserved_scn_; - user_data.reserved_ls_id_ = tx_data.reserved_ls_id_; if (ObTabletStatus::DELETED == tx_data.tablet_status_) { //TODO(bizhu) check deleted trans scn user_data.delete_commit_scn_ = tx_data.tx_scn_; @@ -1092,712 +853,5 @@ int ObTabletMeta::update_meta_last_persisted_committed_tablet_status( return ret; } -ObMigrationTabletParam::ObMigrationTabletParam() - : magic_number_(MAGIC_NUM), - version_(PARAM_VERSION_V3), - is_empty_shell_(false), - ls_id_(), - tablet_id_(), - data_tablet_id_(), - ref_tablet_id_(), - create_scn_(ObTabletMeta::INVALID_CREATE_SCN), - start_scn_(), - clog_checkpoint_scn_(), - ddl_checkpoint_scn_(SCN::min_scn()), - snapshot_version_(OB_INVALID_TIMESTAMP), - multi_version_start_(OB_INVALID_TIMESTAMP), - compat_mode_(lib::Worker::CompatMode::INVALID), - ha_status_(), - report_status_(), - storage_schema_(), - medium_info_list_(), - extra_medium_info_(), - last_persisted_committed_tablet_status_(ObTabletStatus::MAX, ObTabletMdsUserDataType::MAX_TYPE, ObTransVersion::INVALID_TRANS_VERSION), - table_store_flag_(), - ddl_start_scn_(SCN::min_scn()), - ddl_snapshot_version_(OB_INVALID_TIMESTAMP), - max_sync_storage_schema_version_(0), - ddl_execution_id_(-1), - ddl_data_format_version_(0), - max_serialized_medium_scn_(0), - ddl_commit_scn_(SCN::min_scn()), - mds_checkpoint_scn_(), - mds_data_(), - create_schema_version_(0), - micro_index_clustered_(false), - major_ckm_info_(), - split_info_(), - has_truncate_info_(false), - allocator_("MigTblParam", OB_MALLOC_NORMAL_BLOCK_SIZE, ObCtxIds::DEFAULT_CTX_ID) -{ -} - -bool ObMigrationTabletParam::is_valid() const -{ - bool bool_ret = true; - - if (bool_ret) { - bool_ret = MAGIC_NUM == magic_number_ - && (PARAM_VERSION <= version_ && PARAM_VERSION_V3 >= version_) - && ls_id_.is_valid() - && tablet_id_.is_valid() - && data_tablet_id_.is_valid() - && create_scn_ != ObTabletMeta::INVALID_CREATE_SCN - && major_ckm_info_.is_valid(); - if (!bool_ret) { - LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid param", K_(ls_id), K_(tablet_id), K_(data_tablet_id), K_(create_scn), - K_(magic_number), K_(version), K_(major_ckm_info)); - } - } - - if (bool_ret) { - bool_ret = multi_version_start_ >= 0 - && multi_version_start_ <= snapshot_version_ - && compat_mode_ != lib::Worker::CompatMode::INVALID - && max_sync_storage_schema_version_ >= 0 - && max_serialized_medium_scn_ >= 0 - && last_persisted_committed_tablet_status_.is_valid(); - if (!bool_ret) { - LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid param", K_(multi_version_start), K_(snapshot_version), - K_(compat_mode), K_(max_sync_storage_schema_version), K_(max_serialized_medium_scn), - K_(clog_checkpoint_scn), K_(last_persisted_committed_tablet_status)); - } - } - - if (bool_ret) { - bool_ret = ha_status_.is_valid() - && (ha_status_.is_restore_status_pending() - || (start_scn_ >= SCN::base_scn() - && clog_checkpoint_scn_ >= SCN::base_scn() - && start_scn_ <= clog_checkpoint_scn_ - && (is_empty_shell() - || (!is_empty_shell() && storage_schema_.is_valid())))); - // no valid storage_schema_ for empty shell - if (!bool_ret) { - LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid param", K_(ha_status), - K_(start_scn), K_(clog_checkpoint_scn), K_(start_scn), K(is_empty_shell()), - "storage_schema", storage_schema_.is_valid()); - } - } - - return bool_ret; -} - -bool ObMigrationTabletParam::is_empty_shell() const -{ - return is_empty_shell_; -} - -SCN ObMigrationTabletParam::get_max_tablet_checkpoint_scn() const -{ - return MAX3(clog_checkpoint_scn_, mds_checkpoint_scn_, ddl_checkpoint_scn_); -} - -int ObMigrationTabletParam::serialize(char *buf, const int64_t len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - int64_t length = 0; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(len), K(pos)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("migration tablet param is invalid", K(ret), K(*this)); - } else if (FALSE_IT(length = get_serialize_size())) { - } else if (OB_UNLIKELY(length > len - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer's length is not enough", K(ret), K(length), K(len - new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, magic_number_))) { - LOG_WARN("failed to serialize magic number", K(ret), K(len), K(new_pos), K_(magic_number)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, version_))) { - LOG_WARN("failed to serialize version", K(ret), K(len), K(new_pos), K_(version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, length))) { - LOG_WARN("failed to serialize length", K(ret), K(len), K(new_pos), K(length)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_bool(buf, len, new_pos, is_empty_shell_))) { - LOG_WARN("failed to serialize is empty shell", K(ret), K(len), K(new_pos), K(is_empty_shell_)); - } else if (new_pos - pos < length && OB_FAIL(ls_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ls id", K(ret), K(len), K(new_pos), K_(ls_id)); - } else if (new_pos - pos < length && OB_FAIL(tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize tablet id", K(ret), K(len), K(new_pos), K_(tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize data tablet id", K(ret), K(len), K(new_pos), K_(data_tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(ref_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ref tablet id", K(ret), K(len), K(new_pos), K_(ref_tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(create_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(create_scn)); - } else if (new_pos - pos < length && OB_FAIL(start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize start scn", K(ret), K(len), K(new_pos), K_(start_scn)); - } else if (new_pos - pos < length && OB_FAIL(clog_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(clog_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(ddl_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize ddl checkpoint ts", K(ret), K(len), K(new_pos), K_(ddl_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, snapshot_version_))) { - LOG_ERROR("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, multi_version_start_))) { - LOG_ERROR("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(multi_version_start)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i8(buf, len, new_pos, static_cast(compat_mode_)))) { - LOG_WARN("failed to serialize compat mode", K(ret), K(len), K(new_pos), K_(compat_mode)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ha status", K(ret), K(len), K(new_pos), K_(ha_status)); - } else if (new_pos - pos < length && OB_FAIL(report_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize report status", K(ret), K(len), K(new_pos), K_(report_status)); - } else if (!is_empty_shell() && new_pos - pos < length && OB_FAIL(storage_schema_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize storage schema", K(ret), K(len), K(new_pos), K_(storage_schema)); - } else if (PARAM_VERSION_V2 <= version_ && !is_empty_shell() && new_pos - pos < length && OB_FAIL(medium_info_list_.serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize medium compaction list", K(ret), K(len), K(new_pos), K_(medium_info_list)); - } else if (new_pos - pos < length && OB_FAIL(table_store_flag_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize table store flag", K(ret), K(len), K(new_pos), K_(table_store_flag)); - } else if (new_pos - pos < length && OB_FAIL(ddl_start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize ddl start log ts", K(ret), K(len), K(new_pos), K_(ddl_start_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_snapshot_version_))) { - LOG_WARN("failed to serialize ddl snapshot version", K(ret), K(len), K(new_pos), K_(ddl_snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_sync_storage_schema_version_))) { - LOG_WARN("failed to serialize max_sync_storage_schema_version", K(ret), K(len), K(new_pos), K_(max_sync_storage_schema_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_execution_id_))) { - LOG_WARN("failed to serialize ddl execution id", K(ret), K(len), K(new_pos), K_(ddl_execution_id)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_data_format_version_))) { - LOG_WARN("failed to serialize ddl data format version", K(ret), K(len), K(new_pos), K_(ddl_data_format_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_serialized_medium_scn_))) { - LOG_WARN("failed to serialize max serialized medium scn", K(ret), K(len), K(new_pos), K_(max_serialized_medium_scn)); - } else if (new_pos - pos < length && OB_FAIL(ddl_commit_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize ddl commit scn", K(ret), K(len), K(new_pos), K_(ddl_commit_scn)); - } else if (new_pos - pos < length && OB_FAIL(mds_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_ERROR("failed to serialize mds checkpoint ts", K(ret), K(len), K(new_pos), K_(mds_checkpoint_scn)); - } else if (PARAM_VERSION_V2 == version_ && new_pos - pos < length && OB_FAIL(mds_data_.serialize(buf, len, new_pos))) { - //TODO(xianzhi) src tablet status will used to dest create tablet and set tablet status cache. check it - LOG_WARN("failed to serialize mds data", K(ret), K(len), K(new_pos), K_(mds_data)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, create_schema_version_))) { - LOG_ERROR("failed to serialize create schema version", K(ret), K(len), K(new_pos), K_(create_schema_version)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(extra_medium_info_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize extra_medium_info", K(ret), K(len), K(new_pos), K_(extra_medium_info)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(last_persisted_committed_tablet_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos), K_(last_persisted_committed_tablet_status)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::encode_bool(buf, len, new_pos, micro_index_clustered_))) { - LOG_WARN("failed to serialize micro_index_clustered", K(ret), K(len), K(new_pos), K_(micro_index_clustered)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(major_ckm_info_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize major ckm info", K(ret), K(len), K(new_pos), K_(major_ckm_info)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(split_info_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize split info", K(ret), K(len), K(new_pos), K_(split_info)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::encode_bool(buf, len, new_pos, has_truncate_info_))) { - LOG_WARN("failed to serialize has_truncate_info", K(ret), K(len), K(new_pos), K_(has_truncate_info)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(fork_info_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize fork info", K(ret), K(len), K(new_pos), K_(fork_info)); - } else if (OB_UNLIKELY(length != new_pos - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length)); - } else { - pos = new_pos; - - LOG_DEBUG("succeed to serialize migration tablet param", K(ret), KPC(this)); - } - - return ret; -} - -int ObMigrationTabletParam::deserialize_v2_v3(const char *buf, const int64_t len, int64_t &pos) -{ - int ret = OB_SUCCESS; - - int64_t new_pos = pos; - int8_t compat_mode = -1; - int64_t total_length = 0; // total serialize size - int64_t length = 0; // total serialize size expect for the first 2 fields. - - if (OB_FAIL(serialization::decode_i64(buf, len, new_pos, &total_length))) { - LOG_WARN("failed to deserialize length", K(ret), K(len), K(new_pos)); - } else if (OB_UNLIKELY(total_length - 24 > len - new_pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer's length is not enough", K(ret), K(total_length), K(len - new_pos)); - } else if (FALSE_IT(length = total_length - 16)) { - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_bool(buf, len, new_pos, &is_empty_shell_))) { - LOG_WARN("failed to serialize is empty shell", K(ret), K(len), K(new_pos), K(is_empty_shell_)); - } else if (new_pos - pos < length && OB_FAIL(ls_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize data tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ref_tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ref tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(create_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize create scn", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(start_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize start scn", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(clog_checkpoint_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_checkpoint_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl checkpoint ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &snapshot_version_))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &multi_version_start_))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i8(buf, len, new_pos, (int8_t*)(&compat_mode)))) { - LOG_WARN("failed to deserialize compat mode", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ha status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(report_status_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize report status", K(ret), K(len), K(new_pos)); - } else if (!is_empty_shell() && new_pos - pos < length && OB_FAIL(storage_schema_.deserialize(allocator_, buf, len, new_pos))) { - LOG_WARN("failed to deserialize storage schema", K(ret), K(len), K(new_pos)); - } else if (PARAM_VERSION_V2 <= version_ && !is_empty_shell() && new_pos - pos < length && OB_FAIL(medium_info_list_.deserialize(allocator_, buf, len, new_pos))) { - LOG_ERROR("failed to deserialize medium compaction list", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(table_store_flag_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize table store flag", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_start_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl start log ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_snapshot_version_))) { - LOG_WARN("failed to deserialize ddl snapshot version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &max_sync_storage_schema_version_))) { - LOG_ERROR("failed to deserialize max sync storage schema version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_execution_id_))) { - LOG_WARN("failed to deserialize ddl execution id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_data_format_version_))) { - LOG_WARN("failed to deserialize ddl data format version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &max_serialized_medium_scn_))) { - LOG_WARN("failed to deserialize max sync medium snapshot", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_commit_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl commit scn", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(mds_checkpoint_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize mds_checkpoint_scn", K(ret), K(len), K(new_pos)); - } else if (PARAM_VERSION_V2 == version_ && new_pos - pos < length && OB_FAIL(mds_data_.deserialize(allocator_, buf, len, new_pos))) { - LOG_WARN("failed to deserialize mds data", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &create_schema_version_))) { - LOG_ERROR("failed to deserialize create schema version", K(ret), K(len)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(extra_medium_info_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize extra_medium_info", K(ret), K(len), K(new_pos)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(last_persisted_committed_tablet_status_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::decode_bool(buf, len, new_pos, µ_index_clustered_))) { - LOG_WARN("failed to deserialize micro_index_clustered", K(ret), K(len)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(major_ckm_info_.deserialize(allocator_, buf, len, new_pos))) { - LOG_WARN("failed to deserialize major ckm info", K(ret), K(len)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(split_info_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize split info", K(ret), K(len)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::decode_bool(buf, len, new_pos, &has_truncate_info_))) { - LOG_WARN("failed to deserialize has_truncate_info", K(ret), K(len)); - } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(fork_info_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize fork info", K(ret), K(len)); - } else if (OB_UNLIKELY(length != new_pos - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet's length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length), KPC(this)); - } else { - if (PARAM_VERSION_V2 == version_) { - int64_t tmp_pos = 0; - extra_medium_info_ = mds_data_.medium_info_list_.extra_medium_info_; - const ObString &user_data = mds_data_.tablet_status_committed_kv_.v_.user_data_; - if (user_data.empty()) { - last_persisted_committed_tablet_status_.reset(); - } else if (OB_FAIL(last_persisted_committed_tablet_status_.deserialize(user_data.ptr(), user_data.length(), tmp_pos))) { - LOG_WARN("failed to deserialize user data", K(ret), K(tmp_pos), K(user_data)); - } - } - compat_mode_ = static_cast(compat_mode); - pos = new_pos; - LOG_DEBUG("succeed to deserialize migration tablet param v2_v3", K(ret), KPC(this)); - } - - return ret; -} - -int ObMigrationTabletParam::deserialize_v1(const char *buf, const int64_t len, int64_t &pos) -{ - int ret = OB_SUCCESS; - - int64_t new_pos = pos; - int8_t compat_mode = -1; - int64_t total_length = 0; // total serialize size - int64_t length = 0; // total serialize size expect for the first 2 fields. - ObArenaAllocator allocator("MigDeser"); - ObTabletAutoincSeq auto_inc_seq; - ObTabletTxMultiSourceDataUnit tx_data; - ObTabletBindingInfo ddl_data; - ObTabletMdsData mds_data; - - if (OB_FAIL(serialization::decode_i64(buf, len, new_pos, &total_length))) { - LOG_WARN("failed to deserialize length", K(ret), K(len), K(new_pos)); - } else if (OB_UNLIKELY(total_length - 24 > len - new_pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer's length is not enough", K(ret), K(total_length), K(len - new_pos)); - } else if (FALSE_IT(length = total_length - 16)) { - } else if (new_pos - pos < length && OB_FAIL(ls_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize data tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ref_tablet_id_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ref tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(create_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize create scn", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(start_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize start scn", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(clog_checkpoint_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_checkpoint_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl checkpoint ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &snapshot_version_))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &multi_version_start_))) { - LOG_ERROR("failed to deserialize clog checkpoint ts", K(ret), K(len)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i8(buf, len, new_pos, (int8_t*)(&compat_mode)))) { - LOG_WARN("failed to deserialize compat mode", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(auto_inc_seq.deserialize(allocator, buf, len, new_pos))) { - LOG_WARN("failed to deserialize auto inc seq", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ha status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(report_status_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize report status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(tx_data.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize multi source data", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_data.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize ddl data", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(storage_schema_.deserialize(allocator_, buf, len, new_pos))) { - LOG_WARN("failed to deserialize storage schema", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(medium_info_list_.deserialize(allocator_, buf, len, new_pos))) { - LOG_ERROR("failed to deserialize medium compaction list", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(table_store_flag_.deserialize(buf, len, new_pos))) { - LOG_WARN("failed to deserialize table store flag", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_start_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl start log ts", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_snapshot_version_))) { - LOG_WARN("failed to deserialize ddl snapshot version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &max_sync_storage_schema_version_))) { - LOG_ERROR("failed to deserialize max sync storage schema version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_execution_id_))) { - LOG_WARN("failed to deserialize ddl execution id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &ddl_data_format_version_))) { - LOG_WARN("failed to deserialize ddl data format version", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::decode_i64(buf, len, new_pos, &max_serialized_medium_scn_))) { - LOG_WARN("failed to deserialize max sync medium snapshot", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_commit_scn_.fixed_deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize ddl commit scn", K(ret), K(len), K(new_pos)); - } else if (OB_UNLIKELY(length != new_pos - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet's length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length), KPC(this)); - } else if (OB_FAIL(ObTabletMdsData::build_mds_data(allocator, auto_inc_seq, tx_data, create_scn_, ddl_data, - clog_checkpoint_scn_, medium_info_list_, mds_data))) { - LOG_WARN("failed to build mds data with version 1", K(ret), K(auto_inc_seq), K(tx_data), K_(create_scn), - K(ddl_data), K_(clog_checkpoint_scn), K_(medium_info_list)); - } else if (OB_FAIL(mds_data_.init(allocator_, mds_data))) { - LOG_WARN("failed to assign mds data", K(ret), K_(mds_data)); - } else if (OB_FAIL(ObTabletMeta::update_meta_last_persisted_committed_tablet_status( - tx_data, create_scn_, last_persisted_committed_tablet_status_))) { - LOG_WARN("fail to init last_persisted_committed_tablet_status from tx data", K(ret), - K(tx_data), K(create_scn_)); - } else { - // no need to fill split_info, use default. - extra_medium_info_ = medium_info_list_.get_extra_medium_info(); - is_empty_shell_ = false; - compat_mode_ = static_cast(compat_mode); - version_ = PARAM_VERSION_V2; - pos = new_pos; - micro_index_clustered_ = false; - LOG_DEBUG("succeed to deserialize migration tablet param v1", K(ret), KPC(this)); - } - - return ret; -} - -int ObMigrationTabletParam::deserialize(const char *buf, const int64_t len, int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - int64_t length; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(len <= 0) - || OB_UNLIKELY(pos < 0) - || OB_UNLIKELY(len < pos + 24)) { //at least 24 bytes for magic number, version and length - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(len), K(pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, len, new_pos, &magic_number_))) { - LOG_WARN("failed to deserialize magic_number", K(ret), K(len), K(new_pos)); - } else if (MAGIC_NUM == magic_number_) { - if (OB_FAIL(serialization::decode_i64(buf, len, new_pos, &version_))) { - LOG_WARN("failed to deserialize version", K(ret), K(len), K(new_pos)); - } else if (OB_UNLIKELY(PARAM_VERSION != version_ && PARAM_VERSION_V2 != version_ && PARAM_VERSION_V3 != version_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid param version", K(ret), K(version_)); - } else if ((PARAM_VERSION_V2 == version_ || PARAM_VERSION_V3 == version_) - && OB_FAIL(deserialize_v2_v3(buf, len, new_pos))) { - LOG_WARN("failed to deserialize v2/v3", K(ret), K(len), K(new_pos)); - } else if (PARAM_VERSION == version_ && OB_FAIL(deserialize_v1(buf, len, new_pos))) { - LOG_WARN("failed to deserialize v1", K(ret), K(len), K(new_pos)); - } else { - pos = new_pos; - - LOG_TRACE("succeed to deserialize migration tablet param", K(ret), KPC(this)); - } - } - - return ret; -} - -int64_t ObMigrationTabletParam::get_serialize_size() const -{ - int64_t size = 0; - int64_t length = 0; - size += serialization::encoded_length_i64(magic_number_); - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_i64(length); - size += serialization::encoded_length_bool(is_empty_shell_); - size += ls_id_.get_serialize_size(); - size += tablet_id_.get_serialize_size(); - size += data_tablet_id_.get_serialize_size(); - size += ref_tablet_id_.get_serialize_size(); - size += create_scn_.get_fixed_serialize_size(); - size += start_scn_.get_fixed_serialize_size(); - size += clog_checkpoint_scn_.get_fixed_serialize_size(); - size += ddl_checkpoint_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(snapshot_version_); - size += serialization::encoded_length_i64(multi_version_start_); - size += serialization::encoded_length_i8(static_cast(compat_mode_)); - size += ha_status_.get_serialize_size(); - size += report_status_.get_serialize_size(); - size += is_empty_shell() ? 0 : storage_schema_.get_serialize_size(); - if (PARAM_VERSION_V2 <= version_) { - size += is_empty_shell() ? 0 : medium_info_list_.get_serialize_size(); - } - size += table_store_flag_.get_serialize_size(); - size += ddl_start_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(ddl_snapshot_version_); - size += serialization::encoded_length_i64(max_sync_storage_schema_version_); - size += serialization::encoded_length_i64(ddl_execution_id_); - size += serialization::encoded_length_i64(ddl_data_format_version_); - if (PARAM_VERSION_V2 == version_) { - size += mds_data_.get_serialize_size(); - } - size += serialization::encoded_length_i64(max_serialized_medium_scn_); - size += ddl_commit_scn_.get_fixed_serialize_size(); - size += mds_checkpoint_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(create_schema_version_); - if (PARAM_VERSION_V3 <= version_) { - size += extra_medium_info_.get_serialize_size(); - size += last_persisted_committed_tablet_status_.get_serialize_size(); - size += serialization::encoded_length_bool(micro_index_clustered_); - size += major_ckm_info_.get_serialize_size(); - size += split_info_.get_serialize_size(); - size += serialization::encoded_length_bool(has_truncate_info_); - size += fork_info_.get_serialize_size(); - } - return size; -} - -void ObMigrationTabletParam::reset() -{ - is_empty_shell_ = false; - ls_id_.reset(); - tablet_id_.reset(); - data_tablet_id_.reset(); - ref_tablet_id_.reset(); - create_scn_ = ObTabletMeta::INVALID_CREATE_SCN; - start_scn_.reset(); - clog_checkpoint_scn_.reset(); - ddl_checkpoint_scn_.set_min(); - snapshot_version_ = OB_INVALID_TIMESTAMP; - multi_version_start_ = OB_INVALID_TIMESTAMP; - compat_mode_ = lib::Worker::CompatMode::INVALID; - ha_status_.reset(); - report_status_.reset(); - storage_schema_.reset(); - medium_info_list_.reset(); - extra_medium_info_.reset(); - // set max, which is invalid - last_persisted_committed_tablet_status_.reset(); - last_persisted_committed_tablet_status_.tablet_status_ = ObTabletStatus::MAX; - last_persisted_committed_tablet_status_.data_type_ = ObTabletMdsUserDataType::MAX_TYPE; - table_store_flag_.reset(); - ddl_start_scn_.set_min(); - ddl_snapshot_version_ = OB_INVALID_TIMESTAMP; - max_sync_storage_schema_version_ = 0; - ddl_execution_id_ = -1; - ddl_data_format_version_ = 0; - max_serialized_medium_scn_ = 0; - ddl_commit_scn_.set_min(); - mds_checkpoint_scn_.reset(); - mds_data_.reset(); - create_schema_version_ = 0; - micro_index_clustered_ = false; - major_ckm_info_.reset(); - split_info_.reset(); - has_truncate_info_ = false; - fork_info_.reset(); - allocator_.reset(); -} - -int ObMigrationTabletParam::assign(const ObMigrationTabletParam ¶m) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("migration tablet param is invalid", K(ret), K(param)); - } else { - // allocator - allocator_.set_attr(ObMemAttr("MigTabletParam", ObCtxIds::DEFAULT_CTX_ID)); - version_ = param.version_; - is_empty_shell_ = param.is_empty_shell_; - ls_id_ = param.ls_id_; - tablet_id_ = param.tablet_id_; - data_tablet_id_ = param.data_tablet_id_; - ref_tablet_id_ = param.ref_tablet_id_; - create_scn_ = param.create_scn_; - create_schema_version_ = param.create_schema_version_; - micro_index_clustered_ = param.micro_index_clustered_; - start_scn_ = param.start_scn_; - clog_checkpoint_scn_ = param.clog_checkpoint_scn_; - ddl_checkpoint_scn_ = param.ddl_checkpoint_scn_; - snapshot_version_ = param.snapshot_version_; - multi_version_start_ = param.multi_version_start_; - compat_mode_ = param.compat_mode_; - ha_status_ = param.ha_status_; - report_status_ = param.report_status_; - storage_schema_.reset(); // TODO: refactor? - medium_info_list_.reset(); - table_store_flag_ = param.table_store_flag_; - ddl_start_scn_ = param.ddl_start_scn_; - ddl_snapshot_version_ = param.ddl_snapshot_version_; - max_sync_storage_schema_version_ = param.max_sync_storage_schema_version_; - max_serialized_medium_scn_ = param.max_serialized_medium_scn_; - ddl_execution_id_ = param.ddl_execution_id_; - ddl_data_format_version_ = param.ddl_data_format_version_; - ddl_commit_scn_ = param.ddl_commit_scn_; - mds_checkpoint_scn_ = param.mds_checkpoint_scn_; - extra_medium_info_ = param.extra_medium_info_; - split_info_ = param.split_info_; - has_truncate_info_ = param.has_truncate_info_; - fork_info_ = param.fork_info_; - if (OB_FAIL(mds_data_.assign(param.mds_data_, allocator_))) { - LOG_WARN("failed to assign mds data", K(ret), K(param)); - } else if (OB_FAIL(last_persisted_committed_tablet_status_.assign( - param.last_persisted_committed_tablet_status_))) { - LOG_WARN("fail to init last_persisted_committed_tablet_status from mig param", K(ret)); - } else if (OB_FAIL(major_ckm_info_.assign(param.major_ckm_info_, &allocator_))) { - LOG_WARN("failed to assign major ckm info", KR(ret), K(param.major_ckm_info_)); - } - - if (OB_FAIL(ret)) { - } else if (is_empty_shell()) { - // do nothing - } else if (OB_FAIL(storage_schema_.init(allocator_, param.storage_schema_))) { - LOG_WARN("failed to assign storage schema", K(ret), K(param)); - } else if (OB_FAIL(medium_info_list_.init(allocator_, ¶m.medium_info_list_))) { - LOG_WARN("failed to assign medium info list", K(ret), K(param)); - } - } - return ret; -} - -int ObMigrationTabletParam::build_deleted_tablet_info(const share::ObLSID &ls_id, const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (!ls_id.is_valid() || !tablet_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid args", K(ret), K(ls_id), K(tablet_id)); - } else { - const ObTabletRestoreStatus::STATUS restore_status = ObTabletRestoreStatus::FULL; - const ObTabletDataStatus::STATUS data_status = ObTabletDataStatus::COMPLETE; - const ObTabletExpectedStatus::STATUS expected_status = ObTabletExpectedStatus::DELETED; - ls_id_ = ls_id; - tablet_id_ = tablet_id; - data_tablet_id_ = tablet_id; - create_scn_ = ObTabletMeta::INIT_CREATE_SCN; - create_schema_version_ = 0; - micro_index_clustered_ = false; - start_scn_ = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; - clog_checkpoint_scn_ = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; - compat_mode_ = lib::Worker::get_compatibility_mode(); - multi_version_start_ = 0; - snapshot_version_ = 0; - last_persisted_committed_tablet_status_.on_init(); - // no need to fill split_info, use default. - if (OB_FAIL(ha_status_.set_restore_status(restore_status))) { - LOG_WARN("failed to set restore status", K(ret), K(restore_status)); - } else if (OB_FAIL(ha_status_.set_data_status(data_status))) { - LOG_WARN("failed to set data status", K(ret), K(data_status)); - } else if (OB_FAIL(ha_status_.set_expected_status(expected_status))) { - LOG_WARN("failed to set expected status", K(ret), K(expected_status)); - } else if (OB_FAIL(ObMigrationTabletParam::construct_placeholder_storage_schema_and_medium( - allocator_, - storage_schema_, - medium_info_list_, - mds_data_))) { - LOG_WARN("failed to construct placeholder storage schema"); - } - } - return ret; -} - - -int ObMigrationTabletParam::construct_placeholder_storage_schema_and_medium( - common::ObArenaAllocator &allocator, - ObStorageSchema &storage_schema, - compaction::ObMediumCompactionInfoList &medium_info_list, - ObTabletFullMemoryMdsData &full_memory_mds_data) -{ - int ret = OB_SUCCESS; - storage_schema.reset(); - - storage_schema.allocator_ = &allocator; - storage_schema.rowkey_array_.set_allocator(&allocator); - storage_schema.column_array_.set_allocator(&allocator); - - storage_schema.storage_schema_version_ = ObStorageSchema::STORAGE_SCHEMA_VERSION_LATEST; - storage_schema.is_use_bloomfilter_ = false; - storage_schema.table_type_ = ObTableType::USER_TABLE; - //storage_schema.table_mode_ - storage_schema.index_type_ = ObIndexType::INDEX_TYPE_PRIMARY; - storage_schema.row_store_type_ = ObRowStoreType::FLAT_ROW_STORE; - storage_schema.schema_version_ = 1; - storage_schema.column_cnt_ = 1; - storage_schema.store_column_cnt_ = 1; - storage_schema.tablet_size_ = OB_DEFAULT_TABLET_SIZE; - storage_schema.pctfree_ = OB_DEFAULT_PCTFREE; - storage_schema.block_size_ = OB_DEFAULT_MACRO_BLOCK_SIZE; - storage_schema.progressive_merge_round_ = 0; - storage_schema.progressive_merge_num_ = 0; - storage_schema.master_key_id_ = OB_INVALID_ID; - storage_schema.compat_mode_ = static_cast(lib::Worker::get_compatibility_mode()); - ObStorageRowkeyColumnSchema rowkey_schema; - rowkey_schema.meta_type_.set_tinyint(); - rowkey_schema.column_idx_ = common::OB_APP_MIN_COLUMN_ID; - rowkey_schema.order_ = ObOrderType::ASC; - - ObStorageColumnSchema col_schema; - col_schema.is_rowkey_column_ = true; - col_schema.is_column_stored_in_sstable_ = true; - col_schema.meta_type_ = rowkey_schema.meta_type_; - col_schema.default_checksum_ = 0; - - if (OB_FAIL(storage_schema.rowkey_array_.reserve(storage_schema.column_cnt_))) { - STORAGE_LOG(WARN, "Fail to reserve rowkey column array", K(ret)); - } else if (OB_FAIL(storage_schema.column_array_.reserve(storage_schema.column_cnt_))) { - STORAGE_LOG(WARN, "Fail to reserve column array", K(ret)); - } else if (OB_FAIL(storage_schema.rowkey_array_.push_back(rowkey_schema))) { - STORAGE_LOG(WARN, "Fail to add rowkey column id to rowkey array", K(ret)); - } else if (OB_FAIL(storage_schema.column_array_.push_back(col_schema))) { - STORAGE_LOG(WARN, "Fail to push into column array", K(ret), K(col_schema)); - } else { - storage_schema.is_inited_ = true; - } - - if (FAILEDx(medium_info_list.init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } - - if (FAILEDx(full_memory_mds_data.init(allocator))) { - LOG_WARN("failed to init full memory mds data", K(ret)); - } - - if (OB_SUCC(ret) && OB_UNLIKELY(!storage_schema.is_valid() || !medium_info_list.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("placeholder storage schema or medium info list is not valid", K(ret), K(storage_schema), K(medium_info_list)); - } - return ret; -} - } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_meta.h b/src/storage/tablet/ob_tablet_meta.h index f0c4dc921..bd7b301d1 100644 --- a/src/storage/tablet/ob_tablet_meta.h +++ b/src/storage/tablet/ob_tablet_meta.h @@ -37,7 +37,7 @@ #include "storage/tablet/ob_tablet_mds_data.h" #include "storage/tablet/ob_tablet_full_memory_mds_data.h" #include "storage/tx/ob_trans_define.h" -#include "storage/ob_tablet_ha_status.h" +#include "storage/ob_tablet_restore_state.h" #include "storage/tablet/ob_tablet_table_store_flag.h" #include "share/scn.h" #include "storage/tablet/ob_tablet_mds_data.h" @@ -48,7 +48,6 @@ namespace oceanbase { namespace storage { -struct ObMigrationTabletParam; class ObTabletMeta final { friend class ObTablet; @@ -65,7 +64,6 @@ class ObTabletMeta final public: // first init func int init( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const share::SCN create_scn, @@ -99,12 +97,6 @@ class ObTabletMeta final int init( const ObTabletMeta &old_tablet_meta, const share::SCN &flush_scn); - int init( - const ObMigrationTabletParam ¶m); - int init( - const ObTabletMeta &old_tablet_meta, - const ObMigrationTabletParam *tablet_meta); - int assign(const ObTabletMeta &other); void reset(); bool is_valid() const; @@ -122,12 +114,6 @@ class ObTabletMeta final // Note, if a new type of checkpoint scn is added, donot forget to modify the returned scn. share::SCN get_max_replayed_scn() const; public: - static int deserialize_id( - const char *buf, - const int64_t len, - int64_t &pos, - share::ObLSID &ls_id, - common::ObTabletID &tablet_id); static int init_report_info( const blocksstable::ObSSTable *sstable, const int64_t report_version, @@ -142,7 +128,6 @@ class ObTabletMeta final ObTabletCreateDeleteMdsUserData &last_persisted_committed_tablet_status); public: TO_STRING_KV(K_(version), - K_(ls_id), K_(tablet_id), K_(data_tablet_id), K_(ref_tablet_id), @@ -154,7 +139,7 @@ class ObTabletMeta final K_(snapshot_version), K_(multi_version_start), K_(compat_mode), - K_(ha_status), + K_(restore_state), K_(report_status), K_(table_store_flag), K_(ddl_start_scn), @@ -177,7 +162,6 @@ class ObTabletMeta final public: int32_t version_; // alignment: 4B, size: 4B int32_t length_; // alignment: 4B, size: 4B - share::ObLSID ls_id_; // alignment: 8B, size: 8B common::ObTabletID tablet_id_; // alignment: 8B, size: 8B common::ObTabletID data_tablet_id_; // alignment: 8B, size: 8B common::ObTabletID ref_tablet_id_; // alignment: 8B, size: 8B @@ -187,7 +171,7 @@ class ObTabletMeta final share::SCN ddl_checkpoint_scn_; // alignment: 8B, size: 8B int64_t snapshot_version_; // alignment: 8B, size: 8B int64_t multi_version_start_; // alignment: 8B, size: 8B - ObTabletHAStatus ha_status_; // alignment: 8B, size: 8B + ObTabletRestoreState restore_state_; // alignment: 8B, size: 8B ObTabletReportStatus report_status_; // alignment: 8B, size: 40B ObTabletTableStoreFlag table_store_flag_; // alignment: 8B, size: 8B share::SCN ddl_start_scn_; // alignment: 8B, size: 8B @@ -205,12 +189,9 @@ class ObTabletMeta final share::SCN min_ss_tablet_version_; // alignment: 8B, size: 8B compaction::ObExtraMediumInfo extra_medium_info_; ObTabletCreateDeleteMdsUserData last_persisted_committed_tablet_status_; // quick access for tablet status in sstables - ObTabletSpaceUsage space_usage_; // calculated by tablet persist, ObMigrationTabletParam doesn't need it + ObTabletSpaceUsage space_usage_; // alignment: 8B, size: 48B int64_t create_schema_version_; // add after 4.2, record schema_version when first create tablet. NEED COMPAT - //ATTENTION : Add a new variable need consider ObMigrationTabletParam - // and tablet meta init interface for migration. - // yuque : lib::Worker::CompatMode compat_mode_; // alignment: 1B, size: 4B bool has_next_tablet_; // alignment: 1B, size: 2B bool is_empty_shell_; // alignment: 1B, size: 2B @@ -226,142 +207,13 @@ class ObTabletMeta final const compaction::ObExtraMediumInfo &src_addr_extra_info, const compaction::ObExtraMediumInfo &src_data_extra_info, const int64_t finish_medium_scn); - int inner_check_( - const ObTabletMeta &old_tablet_meta, - const ObMigrationTabletParam *tablet_meta); inline void set_space_usage_ (const ObTabletSpaceUsage &space_usage) { space_usage_ = space_usage; } private: - static const int32_t TABLET_META_VERSION = 1; + static const int32_t TABLET_META_VERSION = 2; private: bool is_inited_; }; -// decouple tablet meta from migration parameter -struct ObMigrationTabletParam final -{ -public: - ObMigrationTabletParam(); - ~ObMigrationTabletParam() { reset(); } - ObMigrationTabletParam(const ObMigrationTabletParam &) = delete; - ObMigrationTabletParam &operator=(const ObMigrationTabletParam &) = delete; -public: - bool is_valid() const; - bool is_empty_shell() const; - int serialize(char *buf, const int64_t len, int64_t &pos) const; - int deserialize(const char *buf, const int64_t len, int64_t &pos); - int64_t get_serialize_size() const; - void reset(); - int assign(const ObMigrationTabletParam ¶m); - int build_deleted_tablet_info(const share::ObLSID &ls_id, const ObTabletID &tablet_id); - - // Return the max tablet checkpoint scn which is the max scn among clog_checkpoint_scn, - // mds_checkpoint_scn and ddl_checkpoint_scn. - // Note, if a new type of checkpoint scn is added, donot forget to modify the returned scn. - share::SCN get_max_tablet_checkpoint_scn() const; - - // used for restore PENDING tablet, the placeholder tablet doesn't have storage schema to use - static int construct_placeholder_storage_schema_and_medium( - common::ObArenaAllocator &allocator, - ObStorageSchema &storage_schema, - compaction::ObMediumCompactionInfoList &medium_info_list, - ObTabletFullMemoryMdsData &full_memory_mds_data); - - TO_STRING_KV(K_(magic_number), - K_(version), - K_(is_empty_shell), - K_(ls_id), - K_(tablet_id), - K_(data_tablet_id), - K_(ref_tablet_id), - K_(create_scn), - K_(start_scn), - K_(clog_checkpoint_scn), - K_(ddl_checkpoint_scn), - K_(ddl_snapshot_version), - K_(ddl_start_scn), - K_(snapshot_version), - K_(multi_version_start), - K_(compat_mode), - K_(ha_status), - K_(report_status), - K_(storage_schema), - K_(medium_info_list), - K_(extra_medium_info), - K_(last_persisted_committed_tablet_status), - K_(table_store_flag), - K_(max_sync_storage_schema_version), - K_(ddl_execution_id), - K_(ddl_data_format_version), - K_(max_serialized_medium_scn), - K_(ddl_commit_scn), - K_(mds_checkpoint_scn), - K_(mds_data), - K_(create_schema_version), - K_(micro_index_clustered), - K_(major_ckm_info), - K_(split_info), - K_(has_truncate_info), - K_(fork_info)); -private: - int deserialize_v2_v3(const char *buf, const int64_t len, int64_t &pos); - int deserialize_v1(const char *buf, const int64_t len, int64_t &pos); - - // magic_number_ is added to support upgrade from old format(without version and length compatibility) - // The old format first member is ls_id_(also 8 bytes long), which is not possible be a negative number. - const static int64_t MAGIC_NUM = -20230111; -public: - const static int64_t PARAM_VERSION = 1; - const static int64_t PARAM_VERSION_V2 = 2; - const static int64_t PARAM_VERSION_V3 = 3; - int64_t magic_number_; - int64_t version_; - bool is_empty_shell_; - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObTabletID data_tablet_id_; - common::ObTabletID ref_tablet_id_; - share::SCN create_scn_; - share::SCN start_scn_; // for migration - share::SCN clog_checkpoint_scn_; - share::SCN ddl_checkpoint_scn_; - int64_t snapshot_version_; - int64_t multi_version_start_; - lib::Worker::CompatMode compat_mode_; - ObTabletHAStatus ha_status_; - ObTabletReportStatus report_status_; - ObStorageSchema storage_schema_; // not valid for empty shell - compaction::ObMediumCompactionInfoList medium_info_list_; // not valid for empty shell - compaction::ObExtraMediumInfo extra_medium_info_; - // Record source tablet last_persisted_committed_tablet_status_ for: - // 1. create migration shell tablet in case there's no valid status to get_tablet - // 2. create reserved shell tablet for mds table data in case there's no valid status to get_tablet - // In other scenes, latest status on destination tablet meta should be get from sstables. The one - // on param has no usage. - ObTabletCreateDeleteMdsUserData last_persisted_committed_tablet_status_; - ObTabletTableStoreFlag table_store_flag_; - share::SCN ddl_start_scn_; - int64_t ddl_snapshot_version_; - // max_sync_version may less than storage_schema.schema_version_ when major update schema - int64_t max_sync_storage_schema_version_; - int64_t ddl_execution_id_; - int64_t ddl_data_format_version_; - int64_t max_serialized_medium_scn_; - share::SCN ddl_commit_scn_; - share::SCN mds_checkpoint_scn_; - ObTabletFullMemoryMdsData mds_data_; - int64_t create_schema_version_; - bool micro_index_clustered_; - blocksstable::ObMajorChecksumInfo major_ckm_info_; // from table store - share::ObSplitTabletInfo split_info_; - // [since 4.3.5 bp2] be True after first major with truncate info - // will never be false even after truncate info recycled - bool has_truncate_info_; - share::ObForkTabletInfo fork_info_; - - // Add new serialization member before this line, below members won't serialize - common::ObArenaAllocator allocator_; // for storage schema -}; - } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_multi_source_data.cpp b/src/storage/tablet/ob_tablet_multi_source_data.cpp index 4ca9690fc..7b462aa86 100644 --- a/src/storage/tablet/ob_tablet_multi_source_data.cpp +++ b/src/storage/tablet/ob_tablet_multi_source_data.cpp @@ -32,7 +32,6 @@ ObTabletTxMultiSourceDataUnit::ObTabletTxMultiSourceDataUnit() length_(0), tx_id_(ObTabletCommon::FINAL_TX_ID), tablet_status_(), - reserved_ls_id_(), reserved_scn_() { tx_scn_.set_max(); @@ -50,7 +49,6 @@ ObTabletTxMultiSourceDataUnit::ObTabletTxMultiSourceDataUnit(const ObTabletTxMul tx_id_(other.tx_id_), tx_scn_(other.tx_scn_), tablet_status_(other.tablet_status_), - reserved_ls_id_(other.reserved_ls_id_), reserved_scn_(other.reserved_scn_) { } @@ -73,7 +71,6 @@ int ObTabletTxMultiSourceDataUnit::deep_copy(const ObIMultiSourceDataUnit *src, tx_id_ = data->tx_id_; tx_scn_ = data->tx_scn_; tablet_status_ = data->tablet_status_; - reserved_ls_id_ = data->reserved_ls_id_; reserved_scn_ = data->reserved_scn_; } @@ -88,7 +85,6 @@ void ObTabletTxMultiSourceDataUnit::reset() tx_scn_.reset(); tx_scn_.set_max(); tablet_status_ = ObTabletStatus::MAX; - reserved_ls_id_.reset(); reserved_scn_.reset(); } @@ -103,7 +99,6 @@ int64_t ObTabletTxMultiSourceDataUnit::to_string(char *buf, const int64_t buf_le K_(tablet_status), K_(is_tx_end), K_(unsynced_cnt_for_multi_data), - K_(reserved_ls_id), K_(reserved_scn)); J_OBJ_END(); return pos; @@ -146,8 +141,6 @@ int ObTabletTxMultiSourceDataUnit::serialize( LOG_WARN("failed to serialize tx scn", K(ret), K(len), K(new_pos), K_(tx_scn)); } else if (OB_FAIL(tablet_status_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize tablet status", K(ret), K(len), K(new_pos)); - } else if (OB_FAIL(reserved_ls_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize reserved ls id", K(ret), K(len), K(new_pos), K_(reserved_ls_id)); } else if (OB_FAIL(reserved_scn_.fixed_serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize reserved scn", K(ret), K(len), K(new_pos), K_(reserved_scn)); } else if (OB_UNLIKELY(pos + length_ != new_pos)) { @@ -184,8 +177,6 @@ int ObTabletTxMultiSourceDataUnit::deserialize( LOG_WARN("failed to deserialize tx scn", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(tablet_status_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize tablet status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length_ && OB_FAIL(reserved_ls_id_.deserialize(buf, len, new_pos))) { - LOG_ERROR("failed to deserialize reserved ls id", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(reserved_scn_.fixed_deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize reserved scn", K(ret), K(len), K(new_pos)); } @@ -210,7 +201,6 @@ int64_t ObTabletTxMultiSourceDataUnit::get_serialize_size() const size += tx_id_.get_serialize_size(); size += tx_scn_.get_fixed_serialize_size(); size += tablet_status_.get_serialize_size(); - size += reserved_ls_id_.get_serialize_size(); size += reserved_scn_.get_fixed_serialize_size(); return size; } diff --git a/src/storage/tablet/ob_tablet_multi_source_data.h b/src/storage/tablet/ob_tablet_multi_source_data.h index 253175c1a..c26ae3ca9 100644 --- a/src/storage/tablet/ob_tablet_multi_source_data.h +++ b/src/storage/tablet/ob_tablet_multi_source_data.h @@ -55,10 +55,9 @@ class ObTabletTxMultiSourceDataUnit : public memtable::ObIMultiSourceDataUnit transaction::ObTransID tx_id_; share::SCN tx_scn_; ObTabletStatus tablet_status_; - share::ObLSID reserved_ls_id_; share::SCN reserved_scn_; private: - static const int32_t TX_DATA_VERSION = 1; + static const int32_t TX_DATA_VERSION = 2; }; inline memtable::MultiSourceDataUnitType ObTabletTxMultiSourceDataUnit::type() const diff --git a/src/storage/tablet/ob_tablet_persister.cpp b/src/storage/tablet/ob_tablet_persister.cpp index 09d9a0556..1c8d36e66 100644 --- a/src/storage/tablet/ob_tablet_persister.cpp +++ b/src/storage/tablet/ob_tablet_persister.cpp @@ -60,7 +60,6 @@ ObTabletTransformArg::ObTabletTransformArg() table_store_addr_(), storage_schema_addr_(), tablet_macro_info_addr_(), - is_tablet_referenced_by_collect_mv_(false), ddl_kvs_(nullptr), ddl_kv_count_(0), memtable_count_(0) @@ -81,7 +80,6 @@ void ObTabletTransformArg::reset() table_store_addr_.reset(); storage_schema_addr_.reset(); tablet_macro_info_addr_.reset(); - is_tablet_referenced_by_collect_mv_ = false; ddl_kvs_ = nullptr; ddl_kv_count_ = 0; for (int64_t i = 0; i < MAX_MEMSTORE_CNT; ++i) { @@ -137,7 +135,7 @@ bool ObTabletPersisterParam::is_valid() const } else if (is_shared_object()) { // shared valid = 0 == ls_epoch_ && start_macro_seq_ >= 0; } else { // private - valid = ls_id_.is_valid() && ls_epoch_ >= 0 && 0 == start_macro_seq_; + valid = ls_epoch_ >= 0 && 0 == start_macro_seq_; } return valid; } @@ -406,7 +404,7 @@ int ObTabletPersister::modify_and_fill_tablet( { int ret = OB_SUCCESS; const ObTabletMeta &tablet_meta = old_tablet.get_tablet_meta(); - const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_); + const ObTabletMapKey key(tablet_meta.tablet_id_); const char* buf = reinterpret_cast(&old_tablet); const bool try_smaller_pool = old_tablet.get_try_cache_size() > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE ? false : true; @@ -451,7 +449,7 @@ int ObTabletPersister::modify_and_fill_tablet( LOG_WARN("old tablet doesn't hold ref cnt", K(ret), K(old_tablet)); } else { const ObTabletMeta &tablet_meta = old_tablet.get_tablet_meta(); - const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_); + const ObTabletMapKey key(tablet_meta.tablet_id_); const char* buf = reinterpret_cast(&old_tablet); const bool try_smaller_pool = old_tablet.get_try_cache_size() > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE ? false : true; @@ -503,7 +501,6 @@ int ObTabletPersister::convert_tablet_to_mem_arg( arg.rowkey_read_info_ptr_ = tablet.rowkey_read_info_; arg.table_store_addr_ = tablet.table_store_addr_.addr_; arg.storage_schema_addr_ = tablet.storage_schema_addr_.addr_; - arg.is_tablet_referenced_by_collect_mv_ = tablet.is_tablet_referenced_by_collect_mv(); arg.ddl_kvs_ = tablet.ddl_kvs_; arg.ddl_kv_count_ = tablet.ddl_kv_count_; MEMCPY(arg.memtables_, tablet.memtables_, sizeof(ObIMemtable*) * MAX_MEMSTORE_CNT); @@ -562,7 +559,6 @@ int ObTabletPersister::convert_tablet_to_disk_arg( if (try_cache_size > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE) { type = ObTabletPoolType::TP_LARGE; } - arg.is_tablet_referenced_by_collect_mv_ = tablet.is_tablet_referenced_by_collect_mv(); } return ret; @@ -583,7 +579,7 @@ int ObTabletPersister::persist_and_fill_tablet( ObMultiTimeStats::TimeStats *time_stats = nullptr; const ObTabletMeta &tablet_meta = old_tablet.get_tablet_meta(); - const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_); + const ObTabletMapKey key(tablet_meta.tablet_id_); ObTabletPoolType type = ObTabletPoolType::TP_NORMAL; bool try_smaller_pool = true; @@ -1030,8 +1026,7 @@ int ObTabletPersister::transform(const ObTabletTransformArg &arg, char *buf, con if (OB_SUCC(ret)) { if (OB_FAIL(tiny_tablet->table_store_cache_.init(table_store->get_major_sstables(), - table_store->get_minor_sstables(), - arg.is_tablet_referenced_by_collect_mv_))) { + table_store->get_minor_sstables()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(table_store), K(arg)); } else { time_stats->click("init_table_store_cache"); diff --git a/src/storage/tablet/ob_tablet_persister.h b/src/storage/tablet/ob_tablet_persister.h index d5e4311ed..739686189 100644 --- a/src/storage/tablet/ob_tablet_persister.h +++ b/src/storage/tablet/ob_tablet_persister.h @@ -81,8 +81,7 @@ class ObTabletTransformArg final K_(table_store_addr), K_(storage_schema_addr), K_(tablet_macro_info_addr), - KP_(tablet_macro_info_ptr), - K_(is_tablet_referenced_by_collect_mv)); + KP_(tablet_macro_info_ptr)); public: const ObRowkeyReadInfo *rowkey_read_info_ptr_; const ObTabletMacroInfo *tablet_macro_info_ptr_; @@ -90,7 +89,6 @@ class ObTabletTransformArg final ObMetaDiskAddr table_store_addr_; ObMetaDiskAddr storage_schema_addr_; ObMetaDiskAddr tablet_macro_info_addr_; - bool is_tablet_referenced_by_collect_mv_; ObDDLKV **ddl_kvs_; int64_t ddl_kv_count_; ObIMemtable *memtables_[MAX_MEMSTORE_CNT]; @@ -170,10 +168,9 @@ struct ObTabletPersisterParam final public: // private tablet meta persistence ObTabletPersisterParam( - const share::ObLSID ls_id, const int64_t ls_epoch, const ObTabletID tablet_id) - : ls_id_(ls_id), ls_epoch_(ls_epoch), tablet_id_(tablet_id), + : ls_epoch_(ls_epoch), tablet_id_(tablet_id), snapshot_version_(0), start_macro_seq_(0), ddl_redo_callback_(nullptr), ddl_finish_callback_(nullptr) {} @@ -185,7 +182,7 @@ struct ObTabletPersisterParam final const int64_t start_macro_seq, blocksstable::ObIMacroBlockFlushCallback *ddl_redo_callback = nullptr, blocksstable::ObIMacroBlockFlushCallback *ddl_finish_callback = nullptr) - : ls_id_(), ls_epoch_(0), tablet_id_(tablet_id), + : ls_epoch_(0), tablet_id_(tablet_id), snapshot_version_(snapshot_version), start_macro_seq_(start_macro_seq), ddl_redo_callback_(ddl_redo_callback), ddl_finish_callback_(ddl_finish_callback) {} @@ -195,10 +192,9 @@ struct ObTabletPersisterParam final bool is_shared_object() const { return snapshot_version_ > 0; } - TO_STRING_KV(K_(ls_id), K_(ls_epoch), K_(tablet_id), + TO_STRING_KV(K_(ls_epoch), K_(tablet_id), K_(snapshot_version), K_(start_macro_seq), KP_(ddl_redo_callback), KP_(ddl_finish_callback)); - share::ObLSID ls_id_; int64_t ls_epoch_; ObTabletID tablet_id_; int64_t snapshot_version_; diff --git a/src/storage/tablet/ob_tablet_split_mds_helper.cpp b/src/storage/tablet/ob_tablet_split_mds_helper.cpp index ad546a5fc..d3f2f077e 100644 --- a/src/storage/tablet/ob_tablet_split_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_split_mds_helper.cpp @@ -43,9 +43,7 @@ namespace storage bool ObTabletSplitMdsArg::is_valid() const { - return true - && ls_id_.is_valid() - && (split_data_tablet_ids_.count() == split_datas_.count() || 0 == split_datas_.count()) + return (split_data_tablet_ids_.count() == split_datas_.count() || 0 == split_datas_.count()) && ((tablet_status_ != ObTabletStatus::NONE) ^ tablet_status_tablet_ids_.empty()) && ((tablet_status_ != ObTabletStatus::NONE) == (tablet_status_data_type_ != ObTabletMdsUserDataType::NONE)); } @@ -53,7 +51,6 @@ bool ObTabletSplitMdsArg::is_valid() const void ObTabletSplitMdsArg::reset() { - ls_id_.reset(); split_data_tablet_ids_.reset(); split_datas_.reset(); tablet_status_tablet_ids_.reset(); @@ -218,7 +215,6 @@ int ObTabletSplitMdsArg::prepare_basic_args( } int ObTabletSplitMdsArg::init_split_start_src( - const ObLSID &ls_id, const ObIArray &new_table_schemas, const ObIArray &upd_table_schemas, const ObIArray &src_tablet_ids, @@ -226,7 +222,6 @@ int ObTabletSplitMdsArg::init_split_start_src( { int ret = OB_SUCCESS; reset(); - ls_id_ = ls_id; tablet_status_ = ObTabletStatus::SPLIT_SRC; tablet_status_data_type_ = ObTabletMdsUserDataType::START_SPLIT_SRC; if (OB_UNLIKELY(upd_table_schemas.count() != src_tablet_ids.count() @@ -261,20 +256,17 @@ int ObTabletSplitMdsArg::init_split_start_src( return ret; } -int ObTabletSplitMdsArg::init_set_freeze_flag(const ObLSID &ls_id, - const ObIArray &tablet_ids) +int ObTabletSplitMdsArg::init_set_freeze_flag(const ObIArray &tablet_ids) { int ret = OB_SUCCESS; reset(); - ls_id_ = ls_id; if (OB_FAIL(set_freeze_flag_tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign", K(ret)); } return ret; } -int ObTabletSplitMdsArg::init_split_start_dst(const ObLSID &ls_id, - const ObIArray &inc_table_schemas, +int ObTabletSplitMdsArg::init_split_start_dst(const ObIArray &inc_table_schemas, const ObIArray &src_tablet_ids, const ObIArray> &dst_tablet_ids, const ObIArray &dst_high_bound_vals) @@ -282,7 +274,6 @@ int ObTabletSplitMdsArg::init_split_start_dst(const ObLSID &ls_id, int ret = OB_SUCCESS; ObArenaAllocator allocator; reset(); - ls_id_ = ls_id; tablet_status_tablet_ids_.reset(); // split dst tablet status is NORMAL but without major, so no need to change its tablet status tablet_status_ = ObTabletStatus::NONE; tablet_status_data_type_ = ObTabletMdsUserDataType::NONE; @@ -352,14 +343,12 @@ int ObTabletSplitMdsArg::set_autoinc_seq_arg(const obcall::ObBatchSetTabletAutoi return ret; } -int ObTabletSplitMdsArg::init_split_end_src(const ObLSID &ls_id, - const ObTabletID &src_data_tablet_id, +int ObTabletSplitMdsArg::init_split_end_src(const ObTabletID &src_data_tablet_id, const ObIArray &src_local_index_tablet_ids, const ObIArray &src_lob_tablet_ids) { int ret = OB_SUCCESS; reset(); - ls_id_ = ls_id; split_data_tablet_ids_.reset(); split_datas_.reset(); tablet_status_tablet_ids_.reset(); @@ -380,8 +369,7 @@ int ObTabletSplitMdsArg::init_split_end_src(const ObLSID &ls_id, return ret; } -int ObTabletSplitMdsArg::init_split_end_dst(const ObLSID &ls_id, - const int64_t auto_part_size, +int ObTabletSplitMdsArg::init_split_end_dst(const int64_t auto_part_size, const ObIArray &dst_data_tablet_ids, const ObIArray> &dst_local_index_tablet_ids, const ObIArray> &dst_lob_tablet_ids) @@ -389,7 +377,6 @@ int ObTabletSplitMdsArg::init_split_end_dst(const ObLSID &ls_id, int ret = OB_SUCCESS; ObTabletSplitMdsUserData data; reset(); - ls_id_ = ls_id; tablet_status_tablet_ids_.reset(); tablet_status_ = ObTabletStatus::NORMAL; tablet_status_data_type_ = ObTabletMdsUserDataType::FINISH_SPLIT_DST; @@ -451,13 +438,11 @@ int ObTabletSplitMdsArg::init_split_end_dst(const ObLSID &ls_id, } int ObTabletSplitMdsArg::init( - const share::ObLSID &ls_id, const ObIArray &tablet_ids, const ObIArray &split_datas) { int ret = OB_SUCCESS; reset(); - ls_id_ = ls_id; if (OB_FAIL(split_data_tablet_ids_.assign(tablet_ids))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(split_datas_.assign(split_datas))) { @@ -470,7 +455,7 @@ int ObTabletSplitMdsArg::init( return ret; } -OB_SERIALIZE_MEMBER(ObTabletSplitMdsArg, ls_id_, split_data_tablet_ids_, split_datas_, tablet_status_tablet_ids_, tablet_status_, tablet_status_data_type_, set_freeze_flag_tablet_ids_, autoinc_seq_arg_); +OB_SERIALIZE_MEMBER(ObTabletSplitMdsArg, split_data_tablet_ids_, split_datas_, tablet_status_tablet_ids_, tablet_status_, tablet_status_data_type_, set_freeze_flag_tablet_ids_, autoinc_seq_arg_); int ObTabletSplitMdsArgPrepareSrcOp::operator()(const int64_t part_idx, ObBasePartition &part) { @@ -671,7 +656,6 @@ int ObTabletSplitMdsHelper::get_split_info(const ObTablet &tablet, ObIAllocator { int ret = OB_SUCCESS; const ObTabletMeta &tablet_meta = tablet.get_tablet_meta(); - const ObLSID &ls_id = tablet_meta.ls_id_; const ObTabletID &tablet_id = tablet_meta.tablet_id_; ObTabletSplitMdsUserData data; const int64_t abs_timeout_us = THIS_WORKER.is_timeout_ts_valid() ? THIS_WORKER.get_timeout_ts() : ObTimeUtility::current_time() + ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S; @@ -679,16 +663,13 @@ int ObTabletSplitMdsHelper::get_split_info(const ObTablet &tablet, ObIAllocator LOG_WARN("failed to get split data", K(ret)); } else if (data.is_split_dst()) { ObLSService *ls_service = nullptr; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(ls_id)); - } else if (OB_FAIL(data.get_tsc_split_info(tablet_id, *ls_handle.get_ls(), abs_timeout_us, allocator, split_info))) { + } else if (OB_FAIL(ls_service->get_ls(tenant_ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(data.get_tsc_split_info(tablet_id, *tenant_ls, abs_timeout_us, allocator, split_info))) { LOG_WARN("failed to get split info", K(ret)); } } else { @@ -728,31 +709,16 @@ int ObTabletSplitMdsHelper::batch_get_tablet_split( LOG_WARN("invalid argument", K(ret), K(arg)); } else { MOD_SCOPE { - ObLSService *ls_service = share::g_mp->ls_service(); - logservice::ObLogService *log_service = share::g_mp->log_service(); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObRole role = INVALID_ROLE; - if (OB_ISNULL(ls_service) || OB_ISNULL(log_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected ls_service or log_service", K(ret)); - } else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("get ls failed", K(ret), K(arg)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ls", K(ret), K(arg.ls_id_)); - } else if (OB_FAIL(ls->get_ls_role(role))) { - LOG_WARN("get role failed", K(ret), K(arg.ls_id_)); - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { - ret = OB_NOT_MASTER; - LOG_WARN("ls not leader", K(ret), K(arg.ls_id_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.tablet_ids_.at(i); ObTabletHandle tablet_handle; ObTabletSplitMdsUserData data; - if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { + if (OB_FAIL(tenant_ls->get_tablet(tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id), K(abs_timeout_us)); } else if (OB_FAIL(get_split_data_with_timeout(*tablet_handle.get_obj(), data, abs_timeout_us))) { LOG_WARN("failed to get split data", K(ret), K(abs_timeout_us)); @@ -774,7 +740,7 @@ int ObTabletSplitMdsHelper::batch_get_tablet_split( } } else if (OB_UNLIKELY(mds::TwoPhaseCommitState::ON_COMMIT != trans_stat)) { ret = OB_EAGAIN; - LOG_WARN("check committed failed", K(ret), K(arg.ls_id_), K(tablet_id), K(tmp_data)); + LOG_WARN("check committed failed", K(ret), K(tablet_id), K(tmp_data)); } } } @@ -784,16 +750,15 @@ int ObTabletSplitMdsHelper::batch_get_tablet_split( return ret; } -int ObTabletSplitMdsHelper::get_tablet_split_mds_by_rpc(const ObLSID &ls_id, - const ObIArray &tablet_ids, +int ObTabletSplitMdsHelper::get_tablet_split_mds_by_rpc(const ObIArray &tablet_ids, const int64_t abs_timeout_us, ObIArray &datas) { int ret = OB_SUCCESS; obcall::ObBatchGetTabletSplitArg arg; obcall::ObBatchGetTabletSplitRes res; - if (OB_FAIL(arg.init(ls_id, tablet_ids, true/*check_committed*/))) { - LOG_WARN("failed to init arg", K(ret), K(ls_id)); + if (OB_FAIL(arg.init(tablet_ids, true/*check_committed*/))) { + LOG_WARN("failed to init arg", K(ret)); } else if (OB_FAIL(ex_rpc::sync_call([&]{ return batch_get_tablet_split(abs_timeout_us, arg, res); }))) { @@ -814,7 +779,6 @@ int ObTabletSplitMdsHelper::set_auto_part_size_for_create(const obcall::ObBatchC { int ret = OB_SUCCESS; if (!auto_part_size_arr.empty()) { - const ObLSID &ls_id = create_arg.id_; ObTabletSplitMdsArg arg; ObArray split_data_tablet_ids; ObArray split_datas; @@ -847,7 +811,7 @@ int ObTabletSplitMdsHelper::set_auto_part_size_for_create(const obcall::ObBatchC } else if (OB_FAIL(split_datas.push_back(tmp_data))) { LOG_WARN("failed to push back", K(ret)); } else if (split_data_tablet_ids.count() >= ObTabletSplitMdsArg::BATCH_TABLET_CNT) { - if (OB_FAIL(arg.init(ls_id, split_data_tablet_ids, split_datas))) { + if (OB_FAIL(arg.init(split_data_tablet_ids, split_datas))) { LOG_WARN("failed to init tablet split mds arg", K(ret)); } else if (OB_FAIL(register_mds(arg, false/*need_flush_redo*/, trans))) { LOG_WARN("failed to register mds", K(ret)); @@ -861,7 +825,7 @@ int ObTabletSplitMdsHelper::set_auto_part_size_for_create(const obcall::ObBatchC } } if (OB_SUCC(ret) && !split_data_tablet_ids.empty()) { - if (OB_FAIL(arg.init(ls_id, split_data_tablet_ids, split_datas))) { + if (OB_FAIL(arg.init(split_data_tablet_ids, split_datas))) { LOG_WARN("failed to init tablet split mds arg", K(ret)); } else if (OB_FAIL(register_mds(arg, false/*need_flush_redo*/, trans))) { LOG_WARN("failed to register mds", K(ret)); @@ -899,28 +863,27 @@ int ObTabletSplitMdsHelper::modify_tablet_split_(const ObIArray &tab int ret = OB_SUCCESS; if (!tablet_ids.empty()) { ObTabletSplitMdsArg arg; - ObArray> ls_tablets; + ObArray sorted_tablet_ids; ObArray this_batch_tablet_ids; - if (OB_FAIL(ObTabletBindingMdsHelper::get_sorted_ls_tablets(tablet_ids, ls_tablets, trans))) { - LOG_WARN("failed to get sorted ls tablets", K(ret)); + if (OB_FAIL(ObTabletBindingMdsHelper::get_sorted_tablets(tablet_ids, sorted_tablet_ids, trans))) { + LOG_WARN("failed to get sorted tablets", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_tablets.count(); i++) { - const ObLSID &ls_id = ls_tablets.at(i).first; - const ObTabletID &tablet_id = ls_tablets.at(i).second; - const bool is_last_or_next_ls_id_changed = i == ls_tablets.count() - 1 || ls_id != ls_tablets.at(i+1).first; + for (int64_t i = 0; OB_SUCC(ret) && i < sorted_tablet_ids.count(); i++) { + const ObTabletID &tablet_id = sorted_tablet_ids.at(i); + const bool is_last = i == sorted_tablet_ids.count() - 1; if (OB_FAIL(this_batch_tablet_ids.push_back(tablet_id))) { LOG_WARN("failed to push back", K(ret)); - } else if (is_last_or_next_ls_id_changed || this_batch_tablet_ids.count() >= ObTabletSplitMdsArg::BATCH_TABLET_CNT) { + } else if (is_last || this_batch_tablet_ids.count() >= ObTabletSplitMdsArg::BATCH_TABLET_CNT) { ObArray datas; - if (OB_FAIL(get_tablet_split_mds_by_rpc(ls_id, this_batch_tablet_ids, abs_timeout_us, datas))) { + if (OB_FAIL(get_tablet_split_mds_by_rpc(this_batch_tablet_ids, abs_timeout_us, datas))) { LOG_WARN("failed to get tablet binding mds", K(ret)); } for (int64_t j = 0; OB_SUCC(ret) && j < datas.count(); j++) { ret = op(datas.at(j)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(arg.init(ls_id, this_batch_tablet_ids, datas))) { + } else if (OB_FAIL(arg.init(this_batch_tablet_ids, datas))) { LOG_WARN("failed to init", K(ret)); } else if (OB_FAIL(register_mds(arg, false/*need_flush_redo*/, trans))) { LOG_WARN("failed to register mds", K(ret)); @@ -962,7 +925,7 @@ int ObTabletSplitMdsHelper::register_mds( LOG_WARN("failed to allocate", K(ret)); } else if (OB_FAIL(arg.serialize(buf, size, pos))) { LOG_WARN("failed to serialize arg", K(ret)); - } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(arg.ls_id_, ObTxDataSourceType::TABLET_SPLIT, buf, pos, flag))) { + } else if (OB_FAIL(static_cast(isql_conn)->register_multi_data_source(ObTxDataSourceType::TABLET_SPLIT, buf, pos, flag))) { LOG_WARN("failed to register mds", K(ret)); } } @@ -1002,50 +965,47 @@ int ObTabletSplitMdsHelper::modify( { int ret = OB_SUCCESS; const bool is_reset_split_data = arg.split_datas_.empty(); - const share::ObLSID &ls_id = arg.ls_id_; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; bool need_empty_shell_trigger = false; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret), K(arg)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("fail to get ls", KR(ret), K(arg)); - } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls should not be NULL", KR(ret), K(arg), KP(ls)); } for (int64_t i = 0; OB_SUCC(ret) && i < arg.split_data_tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.split_data_tablet_ids_[i]; if (is_reset_split_data) { ObTabletSplitMdsUserData data; - if (OB_FAIL(set_tablet_split_mds(ls_id, tablet_id, scn, data, ctx))) { - LOG_WARN("failed to modify", K(ret), K(ls_id), K(tablet_id), K(scn)); + if (OB_FAIL(set_tablet_split_mds(tablet_id, scn, data, ctx))) { + LOG_WARN("failed to modify", K(ret), K(tablet_id), K(scn)); } } else { const ObTabletSplitMdsUserData &data = arg.split_datas_[i]; // index range is checked by is_valid() - if (OB_FAIL(set_tablet_split_mds(ls_id, tablet_id, scn, data, ctx))) { - LOG_WARN("failed to modify", K(ret), K(ls_id), K(tablet_id), K(scn)); + if (OB_FAIL(set_tablet_split_mds(tablet_id, scn, data, ctx))) { + LOG_WARN("failed to modify", K(ret), K(tablet_id), K(scn)); } } } for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_status_tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.tablet_status_tablet_ids_[i]; - if (OB_FAIL(set_tablet_status(*ls, tablet_id, arg.tablet_status_, arg.tablet_status_data_type_, scn, ctx))) { - LOG_WARN("failed to modify", K(ret), K(ls_id), K(tablet_id), K(scn)); + if (OB_FAIL(set_tablet_status(*tenant_ls, tablet_id, arg.tablet_status_, + arg.tablet_status_data_type_, scn, ctx))) { + LOG_WARN("failed to modify", K(ret), K(tablet_id), K(scn)); } else if (ObTabletStatus::SPLIT_SRC_DELETED == arg.tablet_status_) { need_empty_shell_trigger = true; } } if (OB_SUCC(ret) && arg.autoinc_seq_arg_.is_valid()) { - if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().batch_set_tablet_autoinc_seq_in_trans(*ls, arg.autoinc_seq_arg_, scn, ctx))) { + if (OB_FAIL(ObTabletAutoincSeqRpcHandler::get_instance().batch_set_tablet_autoinc_seq_in_trans( + *tenant_ls, arg.autoinc_seq_arg_, scn, ctx))) { LOG_WARN("failed to batch set tablet autoinc seq", K(ret), K(scn)); } } if (OB_SUCC(ret) && need_empty_shell_trigger) { - if (OB_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(ls_id))) { + if (OB_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger())) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(arg)); } } @@ -1053,7 +1013,7 @@ int ObTabletSplitMdsHelper::modify( if (scn.is_valid()) { // on_replay for (int64_t i = 0; OB_SUCC(ret) && i < arg.set_freeze_flag_tablet_ids_.count(); i++) { const ObTabletID &tablet_id = arg.set_freeze_flag_tablet_ids_[i]; - if (OB_FAIL(set_freeze_flag(*ls, tablet_id, scn))) { + if (OB_FAIL(set_freeze_flag(*tenant_ls, tablet_id, scn))) { LOG_WARN("failed to set freeze flag", K(ret), K(tablet_id), K(scn)); } } @@ -1063,7 +1023,6 @@ int ObTabletSplitMdsHelper::modify( } int ObTabletSplitMdsHelper::set_tablet_split_mds( - const ObLSID &ls_id, const ObTabletID &tablet_id, const share::SCN &replay_scn, const ObTabletSplitMdsUserData &data, @@ -1072,7 +1031,7 @@ int ObTabletSplitMdsHelper::set_tablet_split_mds( MDS_TG(100_ms); int ret = OB_SUCCESS; if (!replay_scn.is_valid()) { - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; mds::MdsCtx &user_ctx = static_cast(ctx); @@ -1086,7 +1045,7 @@ int ObTabletSplitMdsHelper::set_tablet_split_mds( ObTabletSplitReplayExecutor replay_executor; if (CLICK_FAIL(replay_executor.init(ctx, replay_scn, data))) { LOG_ERROR("failed to init replay executor", K(ret)); - } else if (CLICK_FAIL(replay_executor.execute(replay_scn, ls_id, tablet_id))) { + } else if (CLICK_FAIL(replay_executor.execute(replay_scn, tablet_id))) { if (OB_EAGAIN != ret) { LOG_ERROR("failed to replay mds", K(ret)); } @@ -1101,7 +1060,6 @@ int ObTabletSplitMdsHelper::set_freeze_flag( const share::SCN &replay_scn) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObArray memtables; @@ -1123,8 +1081,8 @@ int ObTabletSplitMdsHelper::set_freeze_flag( } else { ObMemtable *memtable = static_cast(table); if (memtable->is_active_memtable()) { - memtable->set_recommend_freeze(replay_scn); - LOG_INFO("succ set recommend freeze", K(ls_id), K(tablet_id), KP(memtable), K(replay_scn)); + memtable->set_recommend_snapshot_freeze(replay_scn); + LOG_INFO("succ set recommend snapshot freeze", K(tablet_id), KP(memtable), K(replay_scn)); } } } @@ -1141,7 +1099,6 @@ int ObTabletSplitMdsHelper::set_tablet_status( mds::BufferCtx &ctx) { int ret = OB_SUCCESS; - const ObLSID &ls_id = ls.get_ls_id(); ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; ObTabletCreateDeleteMdsUserData user_data; @@ -1156,7 +1113,7 @@ int ObTabletSplitMdsHelper::set_tablet_status( } } else if (OB_FALSE_IT(tablet = tablet_handle.get_obj())) { } else if (OB_FAIL(tablet->ObITabletMdsInterface::get_latest_tablet_status(user_data, writer, trans_stat, trans_version))) { - if (OB_EMPTY_RESULT == ret && !tablet->get_tablet_meta().ha_status_.check_allow_read()) { + if (OB_EMPTY_RESULT == ret && !tablet->get_tablet_meta().restore_state_.check_allow_read()) { ret = OB_EAGAIN; } else { LOG_WARN("failed to get tx data", K(ret), KPC(tablet)); diff --git a/src/storage/tablet/ob_tablet_split_mds_helper.h b/src/storage/tablet/ob_tablet_split_mds_helper.h index 3c7c5c538..18c0cfce8 100644 --- a/src/storage/tablet/ob_tablet_split_mds_helper.h +++ b/src/storage/tablet/ob_tablet_split_mds_helper.h @@ -24,7 +24,6 @@ #include "lib/container/ob_array.h" #include "lib/container/ob_array_serialization.h" #include "common/mysqlclient/ob_mysql_transaction.h" -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" #include "src/share/ob_tablet_autoincrement_param.h" #include "src/storage/tablet/ob_tablet_create_delete_mds_user_data.h" @@ -63,10 +62,10 @@ class ObTabletSplitMdsArg final // arg with such tablet cnt cannot be more than mds buffer limit (1.5M) const static int64_t BATCH_TABLET_CNT = 8192; - OB_UNIS_VERSION_V(1); + OB_UNIS_VERSION_V(2); public: - ObTabletSplitMdsArg() : ls_id_(), split_data_tablet_ids_(), split_datas_(), tablet_status_tablet_ids_(), tablet_status_(ObTabletStatus::NONE), tablet_status_data_type_(ObTabletMdsUserDataType::NONE), set_freeze_flag_tablet_ids_(), autoinc_seq_arg_() {} + ObTabletSplitMdsArg() : split_data_tablet_ids_(), split_datas_(), tablet_status_tablet_ids_(), tablet_status_(ObTabletStatus::NONE), tablet_status_data_type_(ObTabletMdsUserDataType::NONE), set_freeze_flag_tablet_ids_(), autoinc_seq_arg_() {} ~ObTabletSplitMdsArg() {} bool is_valid() const; void reset(); @@ -79,34 +78,28 @@ class ObTabletSplitMdsArg final ObIArray &dst_high_bound_vals); int init_split_start_src( - const share::ObLSID &ls_id, const ObIArray &new_table_schemas, const ObIArray &upd_table_schemas, const ObIArray &src_tablet_ids, const ObIArray> &dst_tablet_ids); - int init_set_freeze_flag(const share::ObLSID &ls_id, - const ObIArray &tablet_ids); - int init_split_start_dst(const share::ObLSID &ls_id, - const ObIArray &inc_table_schemas, + int init_set_freeze_flag(const ObIArray &tablet_ids); + int init_split_start_dst(const ObIArray &inc_table_schemas, const ObIArray &src_tablet_ids, const ObIArray> &dst_tablet_ids, const ObIArray &dst_high_bound_vals); int set_autoinc_seq_arg(const obcall::ObBatchSetTabletAutoincSeqArg &arg); - int init_split_end_src(const share::ObLSID &ls_id, - const ObTabletID &src_data_tablet_id, // or global index tablets + int init_split_end_src(const ObTabletID &src_data_tablet_id, // or global index tablets const ObIArray &src_local_index_tablet_ids, const ObIArray &src_lob_tablet_ids); - int init_split_end_dst(const share::ObLSID &ls_id, - const int64_t auto_part_size, + int init_split_end_dst(const int64_t auto_part_size, const ObIArray &dst_data_tablet_ids, // or global index tablets const ObIArray> &dst_local_index_tablet_ids, const ObIArray> &dst_lob_tablet_ids); int init( - const share::ObLSID &ls_id, const ObIArray &tablet_ids, const ObIArray &split_datas); - TO_STRING_KV(K_(ls_id), K_(split_data_tablet_ids), K_(split_datas), K_(tablet_status_tablet_ids), K_(tablet_status), K_(tablet_status_data_type), K_(set_freeze_flag_tablet_ids), K_(autoinc_seq_arg)); + TO_STRING_KV(K_(split_data_tablet_ids), K_(split_datas), K_(tablet_status_tablet_ids), K_(tablet_status), K_(tablet_status_data_type), K_(set_freeze_flag_tablet_ids), K_(autoinc_seq_arg)); private: static bool is_split_data_table(const share::schema::ObTableSchema &table_schema); @@ -118,13 +111,12 @@ class ObTabletSplitMdsArg final ObIArray &partkey_projector); public: - share::ObLSID ls_id_; ObSArray split_data_tablet_ids_; ObSArray split_datas_; ObSArray tablet_status_tablet_ids_; ObTabletStatus tablet_status_; ObTabletMdsUserDataType tablet_status_data_type_; - ObSArray set_freeze_flag_tablet_ids_; // set freeze flag on replay + ObSArray set_freeze_flag_tablet_ids_; // set recommended snapshot freeze flag on replay obcall::ObBatchSetTabletAutoincSeqArg autoinc_seq_arg_; private: DISALLOW_COPY_AND_ASSIGN(ObTabletSplitMdsArg); @@ -188,8 +180,7 @@ class ObTabletSplitMdsHelper const int64_t abs_timeout_us, const obcall::ObBatchGetTabletSplitArg &arg, obcall::ObBatchGetTabletSplitRes &res); - static int get_tablet_split_mds_by_rpc(const share::ObLSID &ls_id, - const ObIArray &tablet_ids, + static int get_tablet_split_mds_by_rpc(const ObIArray &tablet_ids, const int64_t abs_timeout_us, ObIArray &datas); @@ -209,7 +200,7 @@ class ObTabletSplitMdsHelper F &&op, ObMySQLTransaction &trans); static int modify(const ObTabletSplitMdsArg &arg, const share::SCN &scn, mds::BufferCtx &ctx); - static int set_tablet_split_mds(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::SCN &replay_scn, const ObTabletSplitMdsUserData &data, mds::BufferCtx &ctx); + static int set_tablet_split_mds(const ObTabletID &tablet_id, const share::SCN &replay_scn, const ObTabletSplitMdsUserData &data, mds::BufferCtx &ctx); static int set_freeze_flag(ObLS &ls, const ObTabletID &tablet_id, const share::SCN &replay_scn); static int set_tablet_status( ObLS &ls, diff --git a/src/storage/tablet/ob_tablet_split_mds_user_data.cpp b/src/storage/tablet/ob_tablet_split_mds_user_data.cpp index edecedd72..900664c33 100644 --- a/src/storage/tablet/ob_tablet_split_mds_user_data.cpp +++ b/src/storage/tablet/ob_tablet_split_mds_user_data.cpp @@ -161,7 +161,7 @@ int ObTabletSplitMdsUserData::calc_split_dst(ObLS &ls, const ObDatumRowkey &rowk LOG_WARN("fail to get dst_tablet_id", K(ret), K(dst_tablet_id_loc), K(dst_tablet_id)); } else if (OB_UNLIKELY(!dst_tablet_id.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("rowkey not in any split dst tablet", K(ret), K(ls.get_ls_id()), K(rowkey), K(*this)); + LOG_WARN("rowkey not in any split dst tablet", K(ret), K(rowkey), K(*this)); print_ref_tablets_split_data(ls); } } else { @@ -302,13 +302,13 @@ int ObTabletSplitMdsUserData::get_split_src_tablet(ObLS &ls, ObTabletHandle &src LOG_WARN("failed to get split src tablet id", K(ret)); } else if (OB_FAIL(ls.get_tablet(src_tablet_id, src_tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { - LOG_WARN("failed to get tablet", K(ret), K(src_tablet_id), K(ls.get_ls_id())); + LOG_WARN("failed to get tablet", K(ret), K(src_tablet_id)); } else if (OB_ISNULL(src_tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tablet", K(ret), K(ls.get_ls_id()), K(src_tablet_id)); + LOG_WARN("invalid tablet", K(ret), K(src_tablet_id)); } else if (OB_UNLIKELY(src_tablet_handle.get_obj()->is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("split src tablet becomes empty shell", K(ret), K(ls.get_ls_id()), K(src_tablet_id)); + LOG_WARN("split src tablet becomes empty shell", K(ret), K(src_tablet_id)); } return ret; } @@ -375,7 +375,7 @@ int ObTabletSplitMdsUserData::get_tsc_split_info( LOG_WARN("invalid tablet", K(ret)); } else if (OB_UNLIKELY(prev_dst_tablet_handle.get_obj()->is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("split dst tablet becomes empty shell", K(ret), K(ls.get_ls_id()), K(prev_dst_tablet_id)); + LOG_WARN("split dst tablet becomes empty shell", K(ret), K(prev_dst_tablet_id)); } else if (OB_FAIL(ObTabletSplitMdsHelper::get_valid_timeout(abs_timeout_us, timeout_us))) { LOG_WARN("failed to get valid timeout", K(ret), K(abs_timeout_us)); } else if (OB_FAIL(prev_dst_tablet_handle.get_obj()->ObITabletMdsInterface::get_split_data(prev_dst_data, timeout_us))) { @@ -423,7 +423,7 @@ int ObTabletSplitMdsUserData::get_tsc_split_info( if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(!(split_info.is_split_dst_with_partkey() || split_info.is_split_dst_without_partkey()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid tsc split info", K(ret), K(tablet_id), K(ls.get_ls_id()), K(split_info)); + LOG_WARN("invalid tsc split info", K(ret), K(tablet_id), K(split_info)); } return ret; } diff --git a/src/storage/tablet/ob_tablet_status.cpp b/src/storage/tablet/ob_tablet_status.cpp index 78b6dd97b..a4ff3d156 100644 --- a/src/storage/tablet/ob_tablet_status.cpp +++ b/src/storage/tablet/ob_tablet_status.cpp @@ -48,14 +48,12 @@ const char *ObTabletStatus::get_str(const ObTabletStatus &status) bool ObTabletStatus::is_deleted_for_gc() const { return ObTabletStatus::DELETED == status_ - || ObTabletStatus::RESERVED_6 == status_ || ObTabletStatus::SPLIT_SRC_DELETED == status_; } bool ObTabletStatus::is_writable_for_dml() const { return ObTabletStatus::NORMAL == status_ - || ObTabletStatus::RESERVED_5 == status_ || ObTabletStatus::SPLIT_DST == status_; } diff --git a/src/storage/tablet/ob_tablet_status.h b/src/storage/tablet/ob_tablet_status.h index 3ce248ac1..ade1c5efb 100644 --- a/src/storage/tablet/ob_tablet_status.h +++ b/src/storage/tablet/ob_tablet_status.h @@ -33,9 +33,9 @@ class ObTabletStatus NORMAL = 1, DELETING = 2, // deprecated after 4.1 DELETED = 3, - RESERVED_4 = 4, - RESERVED_5 = 5, - RESERVED_6 = 6, + RESERVED_STATUS_4 = 4, + RESERVED_STATUS_5 = 5, + RESERVED_STATUS_6 = 6, NONE = 7, SPLIT_SRC = 8, SPLIT_DST = 9, @@ -72,9 +72,9 @@ class ObTabletStatus "NORMAL", "DELETING", "DELETED", - "RESERVED_4", - "RESERVED_5", - "RESERVED_6", + "RESERVED_STATUS_4", + "RESERVED_STATUS_5", + "RESERVED_STATUS_6", "NONE", "SPLIT_SRC", "SPLIT_DST", diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index 4a7c0f39c..b437c0a84 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -109,7 +109,6 @@ int ObTabletTableStore::deserialize( int64_t &pos) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_ls_id(); const common::ObTabletID &tablet_id = tablet.get_tablet_id(); const int start_pos = pos; int64_t serialized_length = 0; @@ -153,7 +152,7 @@ int ObTabletTableStore::deserialize( LOG_WARN("failed to check major sstable ready for read", K(ret)); } if (!memtables_.empty()) { - FLOG_INFO("succeed to deserialize table store", K(ls_id), K(tablet_id), + FLOG_INFO("succeed to deserialize table store", K(tablet_id), K(major_tables_), K(minor_tables_), K(meta_major_tables_), K(mds_sstables_), K(memtables_), K(PRINT_TS(*this))); } } @@ -220,7 +219,7 @@ int ObTabletTableStore::init( { int ret = OB_SUCCESS; int64_t inc_base_snapshot_version = -1; - const ObTabletHAStatus &ha_status = tablet.get_tablet_meta().ha_status_; + const ObTabletRestoreState &restore_state = tablet.get_tablet_meta().restore_state_; const ObSSTable *new_sstable = param.sstable_; if (is_valid() || get_table_count() > 0) { @@ -234,7 +233,7 @@ int ObTabletTableStore::init( LOG_WARN("get unexpected table type", K(ret), KPC(new_sstable)); } else if (OB_FAIL(build_major_tables(allocator, param, old_store, inc_base_snapshot_version))) { LOG_WARN("failed to build major_tables", K(ret)); - } else if (OB_FAIL(build_minor_tables(allocator, param, old_store.minor_tables_, inc_base_snapshot_version, ha_status))) { + } else if (OB_FAIL(build_minor_tables(allocator, param, old_store.minor_tables_, inc_base_snapshot_version, restore_state))) { if (OB_UNLIKELY(OB_NO_NEED_MERGE != ret)) { LOG_WARN("failed to build minor_tables", K(ret)); } @@ -1046,29 +1045,6 @@ int ObTabletTableStore::get_mds_sstables(ObTableStoreIterator &iter) const #undef ADD_SSTABLES_TO_ITER -int ObTabletTableStore::get_ha_tables(ObTableStoreIterator &iter) const -{ - int ret = OB_SUCCESS; - iter.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("table store is not inited", K(ret)); - } else if (!major_tables_.empty() && OB_FAIL(iter.add_tables(major_tables_, 0, major_tables_.count()))) { - LOG_WARN("failed to add major table to iterator", K(ret)); - } else if (!minor_tables_.empty() && OB_FAIL(iter.add_tables(minor_tables_, 0, minor_tables_.count()))) { - LOG_WARN("failed to add minor table to iterator", K(ret)); - } else if (!ddl_sstables_.empty() && OB_FAIL(iter.add_tables(ddl_sstables_, 0, ddl_sstables_.count()))) { - LOG_WARN("failed to add ddl table to iterator", K(ret)); - } else if (!mds_sstables_.empty() && OB_FAIL(iter.add_tables(mds_sstables_, 0, mds_sstables_.count()))) { - LOG_WARN("failed to add mds table to iterator", K(ret)); - } else if (OB_FAIL(iter.set_retire_check())) { - LOG_WARN("failed to set retire check to iterator", K(ret)); - } else { - LOG_INFO("succeed to get ha tables", K(major_tables_), K(minor_tables_), K(ddl_sstables_), K(mds_sstables_)); - } - return ret; -} - int ObTabletTableStore::get_mini_minor_sstables(ObTableStoreIterator &iter) const { int ret = OB_SUCCESS; @@ -1320,7 +1296,7 @@ int ObTabletTableStore::build_minor_tables( const ObUpdateTableStoreParam ¶m, const ObSSTableArray &old_minor_tables, const int64_t inc_base_snapshot_version, - const ObTabletHAStatus &ha_status) + const ObTabletRestoreState &restore_state) { int ret = OB_SUCCESS; ObITable *new_table = const_cast(static_cast(param.sstable_)); //table can be null @@ -1343,8 +1319,8 @@ int ObTabletTableStore::build_minor_tables( LOG_WARN("failed to inner process tables", K(ret), K(old_minor_tables), KPC(new_table)); } else if (sstables.empty()) { // empty minor tables, do nothing - } else if (!ha_status.check_allow_read()) { - LOG_INFO("tablet in ha status, no need recycle minor sstable", K(ha_status)); + } else if (!restore_state.check_allow_read()) { + LOG_INFO("tablet restore is incomplete, skip recycling minor sstable", K(restore_state)); if (OB_FAIL(minor_tables_.init(allocator, sstables))) { LOG_WARN("failed to init minor_tables", K(ret)); } @@ -2112,27 +2088,6 @@ int ObTabletTableStore::need_remove_old_table( return ret; } -int ObTabletTableStore::build_ha_new_table_store( - common::ObArenaAllocator &allocator, - ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("double init", K(ret)); - } else if (OB_UNLIKELY(!param.is_valid() || !old_store.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("init tablet table store get invalid argument", K(ret), K(tablet), K(param), K(old_store)); - } else if (OB_FAIL(init(allocator, tablet))) { - LOG_WARN("failed to init a new empty table store", K(ret)); - } else if (OB_FAIL(build_ha_new_table_store_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to build new table store with old store", K(ret)); - } - return ret; -} - int ObTabletTableStore::check_new_sstable_can_be_accepted_( const ObSSTableArray &old_tables, ObITable *new_table) @@ -2166,613 +2121,19 @@ int ObTabletTableStore::check_new_sstable_can_be_accepted_( return ret; } -int ObTabletTableStore::build_ha_new_table_store_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - const ObSSTableArray &meta_major_table = old_store.meta_major_tables_; - const int64_t multi_version_start = tablet.get_multi_version_start(); - int64_t inc_base_snapshot_version = 0; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("table store is not inited", K(ret)); - } else if (OB_UNLIKELY(!param.is_valid() || !old_store.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("build ha new table store get invalid argument", K(ret), K(param), K(old_store)); - } else if (OB_FAIL(build_ha_major_tables_(allocator, param, old_store, multi_version_start, inc_base_snapshot_version))) { - LOG_WARN("failed to build ha major tables", K(ret), K(param), K(multi_version_start), K(old_store)); - } else if (OB_FAIL(build_ha_mds_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to build ha mds tables", K(ret), K(param), K(old_store)); - } else if (OB_FAIL(build_ha_minor_tables_(allocator, tablet, param, old_store, inc_base_snapshot_version))) { - LOG_WARN("failed to build ha minor tables", K(ret), K(param), K(old_store)); - } else if (OB_FAIL(build_ha_ddl_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to build ha ddl tables", K(ret), K(param), K(old_store)); - } else if (!meta_major_table.empty() && OB_FAIL(build_meta_major_table(allocator, nullptr/*new sstable*/, old_store.meta_major_tables_))) { - LOG_ERROR("failed to build meta major table", K(ret), K(old_store)); - } else if (OB_FAIL(build_major_checksum_info(old_store, NULL/*param*/, allocator))) { - LOG_WARN("failed to build major checksum info", KR(ret), K(param)); - } else if (OB_FAIL(build_memtable_array(tablet))) { - LOG_WARN("failed to pull memtable from memtable_mgr", K(ret)); - } else if (OB_FAIL(pull_ddl_memtables(allocator, tablet))) { - LOG_WARN("pull_ddl_memtables failed", K(ret)); - } else { - is_inited_ = true; - if (OB_FAIL(check_ready_for_read(tablet))) { - LOG_WARN("failed to check major sstable ready", K(ret)); - } else { - try_cache_local_sstables(allocator); - FLOG_INFO("succeed to build ha new table store", "tablet_id", tablet.get_tablet_meta().tablet_id_, - K(major_tables_), K(minor_tables_), K(memtables_), K(PRINT_TS(*this))); - } - } - - return ret; -} - -int ObTabletTableStore::build_ha_major_tables_( - common::ObArenaAllocator &allocator, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t &multi_version_start, - int64_t &inc_base_snapshot_version) -{ - int ret = OB_SUCCESS; - inc_base_snapshot_version = -1; - ObArray major_tables; - const bool allow_duplicate_sstable = true; - - if (!param.tables_handle_.empty() && OB_FAIL(param.tables_handle_.get_tables(major_tables))) { - LOG_WARN("failed to get major tables from param", K(ret)); - } else if (OB_FAIL(inner_build_major_tables_(allocator, old_store, major_tables, - multi_version_start, allow_duplicate_sstable, inc_base_snapshot_version))) { - LOG_WARN("failed to inner build major tables", K(ret), K(param)); - } - return ret; -} - -//migration will add tables with ddl and minor tables. this func will copy old minor -//tables when add ddl tables with param.update_ddl_sstable_ = true. -int ObTabletTableStore::replace_ha_minor_sstables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version) -{ - int ret = OB_SUCCESS; - ObArray new_minor_tables; - UNUSED(inc_base_snapshot_version); - ObArray need_add_minor_tables; - ObSEArray old_minor_tables; - const int64_t inc_pos = 0; - ObArray cut_minor_tables; - const SCN &clog_checkpoint_scn = tablet.get_tablet_meta().clog_checkpoint_scn_; - - if (OB_FAIL(param.tables_handle_.get_all_minor_sstables(need_add_minor_tables))) { - LOG_WARN("failed to add need add minor tables", K(ret), K(param)); - } else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(need_add_minor_tables))) { - LOG_WARN("failed to sort minor tables", K(ret)); - } else if (OB_FAIL(old_store.minor_tables_.get_all_tables(old_minor_tables))) { - LOG_WARN("failed to get old minor tables", K(ret), K(old_store)); - } else if (OB_FAIL(check_minor_tables_(old_minor_tables, true/*no remote table*/))) { - LOG_WARN("failed to check old store minor sstables", K(ret), K(old_minor_tables)); - } else if (OB_FAIL(combine_ha_multi_version_sstables_( - clog_checkpoint_scn, old_minor_tables, need_add_minor_tables, new_minor_tables))) { - LOG_WARN("failed to combin ha minor sstables", K(ret), K(old_store), K(param)); - } else if (new_minor_tables.empty()) { // no minor tables - LOG_INFO("minor tables is empty, skip it", K(ret), K(new_minor_tables)); - } else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(new_minor_tables))) { - LOG_WARN("failed to sort minor tables", K(ret)); - } else if (OB_FAIL(cut_ha_sstable_scn_range_(allocator, new_minor_tables, cut_minor_tables))) { - LOG_WARN("failed to cut ha sstable log ts range", K(ret), K(old_store), K(param)); - } else if (OB_FAIL(check_minor_tables_(cut_minor_tables))) { - LOG_WARN("minor tables is not continue", K(ret), K(param), K(cut_minor_tables), K(old_store)); - } else if (cut_minor_tables.at(cut_minor_tables.count() - 1)->get_end_scn() != clog_checkpoint_scn) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet meta is not match with minor sstables", K(ret), K(cut_minor_tables), K(param), K(old_store), "tablet_meta", tablet.get_tablet_meta()); - } else if (OB_FAIL(minor_tables_.init(allocator, cut_minor_tables, inc_pos))) { - LOG_WARN("failed to init minor_tables", K(ret)); - } else { - LOG_INFO("succeed build ha minor sstables", K(old_store), K(cut_minor_tables)); - } - return ret; -} - -int ObTabletTableStore::replace_ha_remote_minor_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version) -{ - UNUSED(inc_base_snapshot_version); - int ret = OB_SUCCESS; - ObArray new_minor_tables; - ObSEArray old_minor_tables; - const bool check_continue = true; - const int64_t inc_pos = 0; - if (OB_FAIL(old_store.minor_tables_.get_all_tables(old_minor_tables))) { - LOG_WARN("failed to get old minor tables", K(ret), K(old_store)); - } else if (OB_FAIL(replace_ha_remote_sstables_(old_minor_tables, - param.tables_handle_, - check_continue, - new_minor_tables))) { - LOG_WARN("failed to replace remote minor tables", K(ret), K(old_store), K(param)); - } else if (new_minor_tables.empty()) { - } else if (tablet.get_tablet_meta().ha_status_.is_data_status_complete() - && new_minor_tables.at(new_minor_tables.count() - 1)->get_end_scn() != tablet.get_tablet_meta().clog_checkpoint_scn_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet meta is not match with minor sstables", K(ret), K(old_store), K(param), K(new_minor_tables), K(tablet.get_tablet_meta())); - } else if (OB_FAIL(minor_tables_.init(allocator, new_minor_tables, inc_pos))) { - LOG_WARN("failed to init minor tables", K(ret)); - } else { - LOG_INFO("succeed replace ha remote minor sstables", K(old_store), K(new_minor_tables), K(param)); - } - - return ret; -} - -// TODO@wenqu: fix check continue, sort ddl sstables -int ObTabletTableStore::build_ha_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - if (param.need_replace_remote_sstable_) { - if (OB_FAIL(replace_ha_remote_ddl_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to replace ha remote ddl tables", K(ret), K(param), K(old_store)); - } - } else { - if (OB_FAIL(replace_ha_ddl_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to replace ha ddl tables", K(ret), K(param), K(old_store)); - } - } - return ret; -} - -int ObTabletTableStore::replace_ha_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - //TODO(muwei.ym) need reuse local minor sstable and cut sstable log ts 4.3 - int ret = OB_SUCCESS; - ObArray ddl_tables; - ObITable *new_table = nullptr; - ObITable *last_ddl_table = nullptr; - bool need_add_ddl_tables = true; - ObSSTableMetaHandle new_meta_handle; - - if (!old_store.major_tables_.empty() || tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { - need_add_ddl_tables = false; - } - - for (int64_t i = 0; OB_SUCC(ret) && i < param.tables_handle_.get_count() && need_add_ddl_tables; ++i) { - new_table = param.tables_handle_.get_table(i); - if (OB_ISNULL(new_table) || !new_table->is_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table is null or table type is unexpected", K(ret), KPC(new_table)); - } else if (new_table->is_major_sstable()) { - need_add_ddl_tables = false; - break; - } else if (!new_table->is_ddl_sstable()) { - //do nothing - } else if (OB_FAIL(static_cast(new_table)->get_meta(new_meta_handle))) { - LOG_WARN("get new table meta fail", K(ret), KPC(new_table)); - } else if (new_meta_handle.get_sstable_meta().get_basic_meta().ddl_scn_ < tablet.get_tablet_meta().ddl_start_scn_) { - // the ddl start scn is old, drop it - } else if (OB_NOT_NULL(last_ddl_table) && new_table->get_start_scn() != last_ddl_table->get_end_scn()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl table is not continue", K(ret), K(param), K(old_store)); - } else if (OB_FAIL(ddl_tables.push_back(new_table))) { - LOG_WARN("failed to push new table into array", K(ret), KPC(new_table)); - } else { - last_ddl_table = new_table; - } - } - - for (int64_t i = 0; OB_SUCC(ret) && i < old_store.ddl_sstables_.count() && need_add_ddl_tables; ++i) { - new_table = old_store.ddl_sstables_[i]; - if (OB_ISNULL(new_table) || (!new_table->is_ddl_sstable())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table is null or table type is unexpected", K(ret), KPC(new_table)); - } else if (OB_NOT_NULL(last_ddl_table) && new_table->get_start_scn() != last_ddl_table->get_end_scn()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddl table is not continue", K(ret), K(param), K(old_store)); - } else if (OB_FAIL(ddl_tables.push_back(new_table))) { - LOG_WARN("failed to push new table into array", K(ret), KPC(new_table)); - } else { - last_ddl_table = new_table; - } - } - - if (OB_SUCC(ret)) { - if (!need_add_ddl_tables) { - LOG_INFO("has major sstable ,no need add ddl sstable", K(param), K(old_store)); - } else if (ddl_tables.empty()) { // no minor tables - LOG_INFO("ddl tables is empty, skip it", K(ret), K(ddl_tables)); - } else if (OB_FAIL(ddl_sstables_.init(allocator, ddl_tables))) { - LOG_WARN("failed to init minor_tables", K(ret)); - } - } - return ret; -} - -int ObTabletTableStore::replace_ha_remote_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - ObArray new_ddl_tables; - ObSEArray old_ddl_tables; - ObITable *major_table = nullptr; - ObSSTableMetaHandle major_meta_handle; - const bool check_continue = true; - bool need_replace_ddl_tables = true; - for (int64_t i = 0; OB_SUCC(ret) && i < old_store.major_tables_.count(); ++i) { - major_table = old_store.major_tables_[i]; - if (OB_ISNULL(major_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("major table is null", K(ret), K(old_store)); - } else if (!major_table->is_major_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table type is unexpected", K(ret), KPC(major_table)); - } else if (OB_FAIL(static_cast(major_table)->get_meta(major_meta_handle))) { - LOG_WARN("get major table meta fail", K(ret), KPC(major_table)); - } else if (major_meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { - // This remote major is generated with ddl sstables by ddl commit. In this case, replace cannot - // continue, should schedule replace remote major action again. - ret = OB_NO_NEED_MERGE; - LOG_WARN("remote major table exist, replace cannot continue", K(ret), KPC(major_table)); - } else { - need_replace_ddl_tables = false; - } - } - - if (OB_FAIL(ret)) { - } else if (!need_replace_ddl_tables) { - LOG_INFO("major table exist, no need replace", K(old_store)); - } else if (OB_FAIL(old_store.ddl_sstables_.get_all_tables(old_ddl_tables))) { - LOG_WARN("failed to get old ddl tables", K(ret), K(old_store)); - } else if (OB_FAIL(replace_ha_remote_sstables_(old_ddl_tables, - param.tables_handle_, - check_continue, - new_ddl_tables))) { - LOG_WARN("failed to replace remote ddl tables", K(ret), K(old_store), K(param)); - } else if (new_ddl_tables.empty()) { - LOG_INFO("ddl tables is empty, skip it", K(old_store), K(param)); - } else if (OB_FAIL(ddl_sstables_.init(allocator, new_ddl_tables))) { - LOG_WARN("failed to init ddl tables", K(ret)); - } else { - LOG_INFO("succeed replace ha remote ddl sstables", K(old_store), K(new_ddl_tables), K(param)); - } - - return ret; -} - -int ObTabletTableStore::build_ha_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - if (param.need_replace_remote_sstable_) { - if (OB_FAIL(replace_ha_remote_mds_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to replace ha remote mds tables", K(ret), K(param), K(old_store)); - } - } else { - if (OB_FAIL(replace_ha_mds_tables_(allocator, tablet, param, old_store))) { - LOG_WARN("failed to replace ha mds tables", K(ret), K(param), K(old_store)); - } - } - return ret; -} - -int ObTabletTableStore::replace_ha_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - ObArray new_mds_tables; - ObArray need_add_mds_tables; - ObSEArray old_mds_tables; - const int64_t inc_pos = 0; - ObArray cut_mds_tables; - const SCN &mds_checkpoint_scn = tablet.get_tablet_meta().mds_checkpoint_scn_; - - if (OB_FAIL(param.tables_handle_.get_all_mds_sstables(need_add_mds_tables))) { - LOG_WARN("failed to add need add mds tables", K(ret), K(param)); - } else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(need_add_mds_tables))) { - LOG_WARN("failed to sort mds tables", K(ret), K(param), K(need_add_mds_tables)); - } else if (OB_FAIL(old_store.mds_sstables_.get_all_tables(old_mds_tables))) { - LOG_WARN("failed to get old mds tables", K(ret), K(old_store)); - } else if (OB_FAIL(check_minor_tables_(old_mds_tables, true/*no remote table*/))) { - LOG_WARN("failed to check old store minor sstables", K(ret), K(old_mds_tables)); - } else if (OB_FAIL(combine_ha_multi_version_sstables_( - mds_checkpoint_scn, old_mds_tables, need_add_mds_tables, new_mds_tables))) { - LOG_WARN("failed to combin ha mds sstables", K(ret), K(old_store), K(param)); - } else if (new_mds_tables.empty()) { // no mds tables - LOG_INFO("minor tables is empty, skip it", K(old_store), K(param)); - } else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(new_mds_tables))) { - LOG_WARN("failed to sort minor tables", K(ret)); - } else if (OB_FAIL(cut_ha_sstable_scn_range_(allocator, new_mds_tables, cut_mds_tables))) { - LOG_WARN("failed to cut ha sstable log ts range", K(ret), K(old_store), K(param)); - } else if (OB_FAIL(check_minor_tables_(cut_mds_tables))) { - LOG_WARN("minor tables is not continue", K(ret), K(param), K(cut_mds_tables), K(old_store)); - } else if (cut_mds_tables.at(cut_mds_tables.count() - 1)->get_end_scn() != mds_checkpoint_scn) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet meta is not match with minor sstables", K(ret), K(cut_mds_tables), K(param), K(old_store), "tablet_meta", tablet.get_tablet_meta()); - } else if (OB_FAIL(mds_sstables_.init(allocator, cut_mds_tables, inc_pos))) { - LOG_WARN("failed to init mds_sstables", K(ret)); - } else { - LOG_INFO("succeed build ha mds sstables", K(old_store), K(cut_mds_tables)); - } - return ret; -} - -int ObTabletTableStore::replace_ha_remote_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store) -{ - int ret = OB_SUCCESS; - ObArray new_mds_tables; - ObSEArray old_mds_tables; - const bool check_continue = true; - const int64_t inc_pos = 0; - const SCN &mds_checkpoint_scn = tablet.get_tablet_meta().mds_checkpoint_scn_; - - if (OB_FAIL(old_store.mds_sstables_.get_all_tables(old_mds_tables))) { - LOG_WARN("failed to get old minor tables", K(ret), K(old_store)); - } else if (OB_FAIL(replace_ha_remote_sstables_(old_mds_tables, - param.tables_handle_, - check_continue, - new_mds_tables))) { - LOG_WARN("failed to replace remote mds tables", K(ret), K(old_store), K(param)); - } else if (new_mds_tables.empty()) { - } else if (tablet.get_tablet_meta().ha_status_.is_data_status_complete() - && new_mds_tables.at(new_mds_tables.count() - 1)->get_end_scn() != mds_checkpoint_scn) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mds_checkpoint_scn is not match with mds sstables", K(ret), K(old_store), K(param), K(new_mds_tables), K(tablet.get_tablet_meta())); - } else if (OB_FAIL(mds_sstables_.init(allocator, new_mds_tables, inc_pos))) { - LOG_WARN("failed to init mds tables", K(ret)); - } else { - LOG_INFO("succeed replace ha remote mds sstables", K(old_store), K(new_mds_tables), K(param)); - } - - return ret; -} - -int ObTabletTableStore::cut_ha_sstable_scn_range_( - common::ObArenaAllocator &allocator, - common::ObIArray &orig_minor_sstables, - common::ObIArray &cut_minor_sstables) -{ - int ret = OB_SUCCESS; - SCN last_end_scn = SCN::min_scn(); - for (int64_t i = 0; OB_SUCC(ret) && i < orig_minor_sstables.count(); ++i) { - ObITable *table = orig_minor_sstables.at(i); - if (OB_ISNULL(table) || !table->is_multi_version_minor_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is null or table type is unexpected", K(ret), KPC(table)); - } else if (0 == i) { - last_end_scn = table->get_end_scn(); - } else if (last_end_scn < table->get_start_scn() || last_end_scn >= table->get_end_scn()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("minor sstable log ts is not continue or scn has overlap", K(ret), K(orig_minor_sstables)); - } else if (last_end_scn == table->get_start_scn()) { - last_end_scn = table->get_end_scn(); - } else { - ObSSTable *sstable = static_cast(table); - ObSSTable *orig_sstable = nullptr; - ObSSTable *copied_sstable = nullptr; - ObMetaDiskAddr addr; - addr.set_mem_addr(0, sizeof(ObSSTable)); - ObStorageMetaHandle sstable_handle; - if (sstable->is_loaded()) { - orig_sstable = sstable; - } else if (OB_FAIL(ObCacheSSTableHelper::load_sstable( - sstable->get_addr(), sstable_handle))) { - LOG_WARN("failed to load sstable", K(ret), KPC(sstable)); - } else if (OB_FAIL(sstable_handle.get_sstable(orig_sstable))) { - LOG_WARN("failed to get sstable from sstable handle", K(ret), K(sstable_handle)); - } - if (FAILEDx(orig_sstable->deep_copy(allocator, copied_sstable))) { - LOG_WARN("failed to deep copy sstable", K(ret), KPC(orig_sstable), KP(copied_sstable)); - } else if (OB_FAIL(copied_sstable->set_addr(addr))) { - LOG_WARN("failed to set sstable addr", K(ret), K(addr), KPC(copied_sstable)); - } else { - table = copied_sstable; - ObScnRange new_scn_range; - ObScnRange original_scn_range = table->get_scn_range(); - new_scn_range.start_scn_ = last_end_scn; - new_scn_range.end_scn_ = table->get_end_scn(); - - table->set_scn_range(new_scn_range); - last_end_scn = table->get_end_scn(); - LOG_INFO("cut ha sstable log ts range", KPC(orig_sstable), KPC(copied_sstable), - K(new_scn_range), K(original_scn_range)); - } - } - if (OB_SUCC(ret)) { - if (OB_ISNULL(table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table should not be NULL", K(ret), KP(table)); - } else if (OB_FAIL(cut_minor_sstables.push_back(table))) { - LOG_WARN("failed to add table into array", K(ret), KPC(table)); - } - } - } - return ret; -} -int ObTabletTableStore::combine_ha_multi_version_sstables_( - const share::SCN &scn, - common::ObIArray &old_store_sstables, - common::ObIArray &need_add_sstables, - common::ObIArray &new_sstables) -{ - int ret = OB_SUCCESS; - SCN max_copy_end_scn; - max_copy_end_scn.set_min(); - ObArray tmp_sstables; - for (int64_t i = 0; OB_SUCC(ret) && i < need_add_sstables.count(); ++i) { - ObITable *table = need_add_sstables.at(i); - if (OB_FAIL(tmp_sstables.push_back(table))) { - LOG_WARN("failed to push table into array", K(ret), KPC(table)); - } else { - max_copy_end_scn = table->get_end_scn(); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < old_store_sstables.count(); ++i) { - ObITable *table = old_store_sstables.at(i); - if (table->is_remote_logical_minor_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old store minor sstable contains logical sstable, unexpected", K(ret), K(old_store_sstables)); - } else if (table->get_end_scn() <= max_copy_end_scn) { - //do nothing - } else if (OB_FAIL(tmp_sstables.push_back(table))) { - LOG_WARN("failed to push table into array", K(ret), KPC(table)); - } - } - if (OB_SUCC(ret)) { - //TODO(muwei.ym) remove compare with clog checkpoint scn in 4.2 RC3 - if (tmp_sstables.empty()) { - //do nothing - } else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(tmp_sstables))) { - LOG_WARN("failed to sort sstables", K(ret), K(tmp_sstables)); - } else if (scn > tmp_sstables.at(tmp_sstables.count() - 1)->get_end_scn()) { - FLOG_INFO("scn is bigger than all sstables end scn, no need to keep it", - K(scn), K(tmp_sstables), K(major_tables_)); - } else if (OB_FAIL(new_sstables.assign(tmp_sstables))) { - LOG_WARN("failed to assign minor sstables", K(ret), K(tmp_sstables)); - } - } - return ret; -} -// restore replace remote sstables. -// All remote sstables must be replaced with local sstables at once. -int ObTabletTableStore::replace_ha_remote_sstables_( - const common::ObIArray &old_store_sstables, - const ObTablesHandleArray &new_tables_handle, - const bool check_continue, - common::ObIArray &out_sstables) -{ - int ret = OB_SUCCESS; - ObITable *old_table = nullptr; - ObITable *new_table = nullptr; - ObITable *last_table = nullptr; - ObSSTable *new_sstable = nullptr; - ObTableHandleV2 new_table_handle; - bool has_backup_macro = false; - for (int64_t i = 0; OB_SUCC(ret) && i < new_tables_handle.get_count(); ++i) { - new_table = new_tables_handle.get_table(i); - if (OB_ISNULL(new_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table is null", K(ret), KP(new_table)); - } else if (!new_table->is_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table is not sstable", K(ret), KPC(new_table)); - } else if (OB_FAIL(ObTableStoreUtil::check_has_backup_macro_block(new_table, has_backup_macro))) { - LOG_WARN("failed to check new table has backup macro block", K(ret), KPC(new_table)); - } else if (has_backup_macro) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table still has backup macro block", K(ret), KPC(new_table)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < old_store_sstables.count(); ++i) { - has_backup_macro = false; - new_table_handle.reset(); - old_table = old_store_sstables.at(i); - if (OB_ISNULL(old_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is null", K(ret), KP(old_table)); - } else if (!old_table->is_sstable()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("old table is not sstable", K(ret), KPC(old_table)); - } else if (OB_FAIL(ObTableStoreUtil::check_has_backup_macro_block(old_table, has_backup_macro))) { - LOG_WARN("failed to check old table has backup macro block", K(ret), KPC(old_table)); - } else if (!has_backup_macro) { - // this table does not has backup macro block, no need to be replaced. - if (check_continue && OB_NOT_NULL(last_table) && old_table->get_start_scn() != last_table->get_end_scn()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is not continue", K(ret), KPC(last_table), KPC(old_table)); - } else if (OB_FAIL(out_sstables.push_back(old_table))) { - LOG_WARN("failed to push old table into array", K(ret), KPC(old_table)); - } else { - last_table = old_table; - } - } else { - // this table needs to be replaced. - if (OB_FAIL(new_tables_handle.get_table(old_table->get_key(), new_table_handle))) { - LOG_WARN("failed to get new table", K(ret), KPC(old_table)); - if (OB_ENTRY_NOT_EXIST == ret) { - new_sstable = static_cast(old_table); - ret = OB_SUCCESS; - LOG_INFO("old table may have been merged, keep it", K(new_tables_handle), KPC(old_table)); - } - } else if (OB_FAIL(new_table_handle.get_sstable(new_sstable))) { - LOG_WARN("failed to get sstable", K(ret), K(new_table_handle)); - } - - if (OB_FAIL(ret)) { - } else if (check_continue && OB_NOT_NULL(last_table) && new_sstable->get_start_scn() != last_table->get_end_scn()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is not continue", K(ret), KPC(last_table), KPC(new_sstable), KPC(old_table)); - } else if (OB_FAIL(out_sstables.push_back(new_sstable))) { - LOG_WARN("failed to push new table into array", K(ret), KPC(new_sstable)); - } else { - last_table = new_sstable; - LOG_INFO("replace one remote sstable", KPC(old_table), KPC(new_sstable)); - } - } - } - return ret; -} -int ObTabletTableStore::build_ha_minor_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version) -{ - int ret = OB_SUCCESS; - if (param.need_replace_remote_sstable_) { - if (OB_FAIL(replace_ha_remote_minor_tables_(allocator, tablet, param, old_store, inc_base_snapshot_version))) { - LOG_WARN("failed to replace ha remote minor tables", K(ret), K(param), K(old_store)); - } - } else { - if (OB_FAIL(replace_ha_minor_sstables_(allocator, tablet, param, old_store, inc_base_snapshot_version))) { - LOG_WARN("failed to replace ha minor tables", K(ret), K(param), K(old_store)); - } - } - return ret; -} int ObTabletTableStore::get_mini_minor_sstables_(ObTableStoreIterator &iter) const { @@ -2877,7 +2238,7 @@ int ObTabletTableStore::build_split_new_table_store_( UpdateUpperTransParam unused_param; int64_t inc_base_snapshot_version = -1; ObSEArray batch_tables; - const ObTabletHAStatus &ha_status = tablet.get_tablet_meta().ha_status_; + const ObTabletRestoreState &restore_state = tablet.get_tablet_meta().restore_state_; ObSSTable *new_mds_sstable = nullptr; bool is_update_firstly = is_mds_merge(param.tablet_split_param_.merge_type_); if (OB_FAIL(param.tables_handle_.get_tables(batch_tables))) { @@ -2889,7 +2250,7 @@ int ObTabletTableStore::build_split_new_table_store_( LOG_WARN("failed to inner build major tables", K(ret), K(param), K(batch_tables)); } else if (OB_FAIL(build_split_minor_tables_(allocator, old_store, batch_tables, inc_base_snapshot_version, - ha_status))) { + restore_state))) { LOG_WARN("failed to inner build minor tables", K(ret), K(param), K(batch_tables)); } else { ObITable *last_major = nullptr; @@ -3139,7 +2500,7 @@ int ObTabletTableStore::build_split_minor_tables_( const ObTabletTableStore &old_store, const ObIArray &tables_array, const int64_t inc_base_snapshot_version, - const ObTabletHAStatus &ha_status) + const ObTabletRestoreState &restore_state) { int ret = OB_SUCCESS; ObArray minor_tables; @@ -3203,7 +2564,7 @@ int ObTabletTableStore::build_split_minor_tables_( ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected err", K(ret), K(old_store.minor_tables_)); } else if (start_add) { - } else if (!ha_status.check_allow_read()) { + } else if (!restore_state.check_allow_read()) { start_add = true; } else if (table->get_upper_trans_version() > inc_base_snapshot_version) { start_add = true; diff --git a/src/storage/tablet/ob_tablet_table_store.h b/src/storage/tablet/ob_tablet_table_store.h index 7357e1779..99a01013a 100644 --- a/src/storage/tablet/ob_tablet_table_store.h +++ b/src/storage/tablet/ob_tablet_table_store.h @@ -34,7 +34,7 @@ class ObTablet; class ObTableStoreIterator; class ObCachedTableHandle; class ObStorageMetaHandle; -struct ObTabletHAStatus; +struct ObTabletRestoreState; class ObLS; class ObIncMajorDDLAggregateCOSSTable; @@ -94,7 +94,6 @@ class ObTabletTableStore : public ObIStorageMetaObj ObArenaAllocator &allocator, const ObTablet &tablet, const blocksstable::ObSSTable *sstable = nullptr, - // when first create tablet in migration, will carry ObMajorChecksumInfo from src svr const ObMajorChecksumInfo *ckm_info = nullptr); // init for update int init( @@ -186,12 +185,6 @@ class ObTabletTableStore : public ObIStorageMetaObj int get_mini_minor_sstables( ObTableStoreIterator &iter) const; int get_recycle_version(const int64_t multi_version_start, int64_t &recycle_version) const; - int get_ha_tables(ObTableStoreIterator &iter) const; - int build_ha_new_table_store( - common::ObArenaAllocator &allocator, - ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); int check_ready_for_read(const ObTablet &tablet); int64_t to_string(char *buf, const int64_t buf_len) const; // ddl-split @@ -239,7 +232,7 @@ class ObTabletTableStore : public ObIStorageMetaObj const ObUpdateTableStoreParam ¶m, const ObSSTableArray &old_minor_tables, const int64_t inc_base_snapshot_version, - const ObTabletHAStatus &ha_status); + const ObTabletRestoreState &restore_state); int build_mds_minor_tables( common::ObArenaAllocator &allocator, const blocksstable::ObSSTable *new_sstable, @@ -272,21 +265,9 @@ class ObTabletTableStore : public ObIStorageMetaObj int check_continuous() const; template int check_minor_tables_(T &minor_tables, bool no_remote_table = false) const; - // ha int check_new_sstable_can_be_accepted_( const ObSSTableArray &old_tables, ObITable *new_table); - int build_ha_new_table_store_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int build_ha_major_tables_( - common::ObArenaAllocator &allocator, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t &multi_version_start, - int64_t &inc_base_snapshot_version); int inner_build_major_tables_for_ha_( common::ObArenaAllocator &allocator, const ObUpdateTableStoreParam ¶m, @@ -294,63 +275,6 @@ class ObTabletTableStore : public ObIStorageMetaObj const ObIArray &major_tables, int64_t &inc_base_snapshot_version); - int build_ha_minor_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version); - int build_ha_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int replace_ha_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int replace_ha_remote_ddl_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int build_ha_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int replace_ha_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int replace_ha_remote_mds_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store); - int cut_ha_sstable_scn_range_( - common::ObArenaAllocator &allocator, - common::ObIArray &orig_minor_sstables, - common::ObIArray &cut_minor_sstables); - int combine_ha_multi_version_sstables_( - const share::SCN &scn, - common::ObIArray &old_store_sstables, - common::ObIArray &need_add_sstables, - common::ObIArray &new_sstables); - int replace_ha_minor_sstables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version); - int replace_ha_remote_minor_tables_( - common::ObArenaAllocator &allocator, - const ObTablet &tablet, - const ObBatchUpdateTableStoreParam ¶m, - const ObTabletTableStore &old_store, - const int64_t inc_base_snapshot_version); // ddl int pull_ddl_memtables(common::ObArenaAllocator &allocator, const ObTablet &tablet); @@ -417,7 +341,7 @@ class ObTabletTableStore : public ObIStorageMetaObj const ObTabletTableStore &old_store, const ObIArray &tables_array, const int64_t inc_base_snapshot_version, - const ObTabletHAStatus &ha_status); + const ObTabletRestoreState &restore_state); int replace_sstables( common::ObArenaAllocator &allocator, @@ -428,11 +352,6 @@ class ObTabletTableStore : public ObIStorageMetaObj const ObTabletTableStore &old_store, const ObUpdateTableStoreParam *param, ObArenaAllocator &allocator); - int replace_ha_remote_sstables_( - const common::ObIArray &old_store_sstables, - const ObTablesHandleArray &new_tables_handle, - const bool check_continue, - common::ObIArray &out_sstables); int get_mini_minor_sstables_(ObTableStoreIterator &iter) const; int only_replace_major_( common::ObArenaAllocator &allocator, diff --git a/src/storage/tablet/ob_tablet_table_store_iterator.cpp b/src/storage/tablet/ob_tablet_table_store_iterator.cpp index f9945889a..b6d86ef0c 100644 --- a/src/storage/tablet/ob_tablet_table_store_iterator.cpp +++ b/src/storage/tablet/ob_tablet_table_store_iterator.cpp @@ -122,7 +122,7 @@ int ObTableStoreIterator::get_next(ObTableHandleV2 &table_handle) ObITable *table = nullptr; if (OB_UNLIKELY(!split_extra_table_store_handles_.empty())) { ret = OB_NOT_SUPPORTED; - LOG_ERROR("doesn't support cross tablet get table handl", K(ret), K(split_extra_table_store_handles_)); + LOG_ERROR("doesn't support cross tablet get table handle", K(ret), K(split_extra_table_store_handles_)); } else if (OB_FAIL(inner_move_idx_to_next())) { } else { if (OB_FAIL(get_ith_table(pos_, table))) { diff --git a/src/storage/tablet/ob_tablet_table_store_iterator.h b/src/storage/tablet/ob_tablet_table_store_iterator.h index 15cdb3c27..656d53bd3 100644 --- a/src/storage/tablet/ob_tablet_table_store_iterator.h +++ b/src/storage/tablet/ob_tablet_table_store_iterator.h @@ -94,8 +94,7 @@ class ObTableStoreIterator final return (NULL == memstore_retired_) ? false : ATOMIC_LOAD(memstore_retired_); } TO_STRING_KV(K_(table_ptr_array), K_(sstable_handle_array), K_(pos), K_(step), K_(memstore_retired), - K_(need_load_sstable), K_(table_store_handle), K_(split_extra_table_store_handles), - KP_(fork_infos)); + K_(need_load_sstable), K_(table_store_handle), K_(split_extra_table_store_handles), KP_(fork_infos)); private: int inner_move_idx_to_next(); int get_table_ptr_with_meta_handle( diff --git a/src/storage/truncate_info/ob_tablet_truncate_info_reader.cpp b/src/storage/truncate_info/ob_tablet_truncate_info_reader.cpp index ea03a017b..6014159b4 100644 --- a/src/storage/truncate_info/ob_tablet_truncate_info_reader.cpp +++ b/src/storage/truncate_info/ob_tablet_truncate_info_reader.cpp @@ -45,7 +45,6 @@ int ObTabletTruncateInfoReader::init( ObTableScanParam &scan_param) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_ls_id(); const common::ObTabletID &tablet_id = tablet.get_tablet_id(); if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; @@ -53,7 +52,7 @@ int ObTabletTruncateInfoReader::init( } else if (OB_FAIL((tablet.mds_range_query( scan_param, iter_)))) { - LOG_WARN("fail to do build query range iter", K(ret), K(ls_id), K(tablet_id), K(scan_param)); + LOG_WARN("fail to do build query range iter", K(ret), K(tablet_id), K(scan_param)); } else { is_inited_ = true; } diff --git a/src/storage/truncate_info/ob_truncate_info_mds_helper.cpp b/src/storage/truncate_info/ob_truncate_info_mds_helper.cpp index e37be88de..74f3fb502 100644 --- a/src/storage/truncate_info/ob_truncate_info_mds_helper.cpp +++ b/src/storage/truncate_info/ob_truncate_info_mds_helper.cpp @@ -58,7 +58,7 @@ int ObTruncateInfoMdsHelper::on_register( ObArenaAllocator tmp_allocator; ObTruncateTabletArg arg; int64_t pos = 0; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObTabletHandle tablet_handle; mds::MdsCtx &user_ctx = static_cast(ctx); @@ -70,10 +70,10 @@ int ObTruncateInfoMdsHelper::on_register( } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("arg is invalid", K(ret), K(arg)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { LOG_WARN("failed to get log stream", K(ret), K(arg)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(arg.index_tablet_id_, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(arg.ls_id_), K(arg.index_tablet_id_)); + } else if (OB_FAIL(tenant_ls->get_tablet(arg.index_tablet_id_, tablet_handle))) { + LOG_WARN("failed to get tablet", K(ret), K(arg.index_tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->set_truncate_info( arg.truncate_info_.key_, arg.truncate_info_, @@ -109,7 +109,7 @@ int ObTruncateInfoMdsHelper::on_replay( ObTruncateInfoClogReplayExecutor executor(arg); if (OB_FAIL(executor.init(ctx, scn))) { LOG_WARN("failed to init reply executor", K(ret), K(arg), K(ctx), K(scn)); - } else if (OB_FAIL(executor.execute(scn, arg.ls_id_, arg.index_tablet_id_))) { + } else if (OB_FAIL(executor.execute(scn, arg.index_tablet_id_))) { LOG_WARN("failed to executor", K(ret), K(arg), K(ctx), K(scn)); } else { LOG_INFO("[TRUNCATE INFO] on_replay for ObTruncateTabletArg", K(ret), K(arg)); diff --git a/src/storage/tx/ob_committer_define.h b/src/storage/tx/ob_committer_define.h index 4e6a76a3c..72124d99b 100644 --- a/src/storage/tx/ob_committer_define.h +++ b/src/storage/tx/ob_committer_define.h @@ -31,44 +31,6 @@ class ObICommitCallback }; // =================== NB: REMOVE DURING JOINT DEBUGGING =================== -enum class ObTwoPhaseCommitLogType : uint8_t -{ - OB_LOG_TX_INIT = 0, - OB_LOG_TX_COMMIT_INFO, - OB_LOG_TX_PREPARE, - OB_LOG_TX_PRE_COMMIT, - OB_LOG_TX_COMMIT, - OB_LOG_TX_ABORT, - OB_LOG_TX_CLEAR, - OB_LOG_TX_MAX, -}; - -enum class ObTwoPhaseCommitMsgType : uint8_t -{ - OB_MSG_TX_UNKNOWN = 0, - OB_MSG_TX_PREPARE_REQ, - OB_MSG_TX_PREPARE_RESP, - OB_MSG_TX_PRE_COMMIT_REQ, - OB_MSG_TX_PRE_COMMIT_RESP, - OB_MSG_TX_COMMIT_REQ, - OB_MSG_TX_COMMIT_RESP, - OB_MSG_TX_ABORT_REQ, - OB_MSG_TX_ABORT_RESP, - OB_MSG_TX_CLEAR_REQ, - OB_MSG_TX_CLEAR_RESP, - OB_MSG_TX_PREPARE_REDO_REQ, - OB_MSG_TX_PREPARE_REDO_RESP, - OB_MSG_TX_MAX, -}; - -enum class Ob2PCRole : int8_t -{ - UNKNOWN = -1, - ROOT = 0, - INTERNAL, - LEAF, -}; - enum class ObTxState : uint8_t { UNKNOWN = 0, @@ -82,26 +44,11 @@ enum class ObTxState : uint8_t MAX = 100 }; -const int64_t OB_C2PC_UPSTREAM_ID = INT64_MAX - 1; -const int64_t OB_C2PC_SENDER_ID = INT64_MAX - 2; - #define TRX_ENUM_CASE_TO_STR(class_name, src) \ case class_name::src: \ str = #src; \ break; -static const char *to_str_2pc_role(Ob2PCRole role) -{ - const char *str = "INVALID"; - switch (role) { - TRX_ENUM_CASE_TO_STR(Ob2PCRole, UNKNOWN) - TRX_ENUM_CASE_TO_STR(Ob2PCRole, ROOT) - TRX_ENUM_CASE_TO_STR(Ob2PCRole, INTERNAL) - TRX_ENUM_CASE_TO_STR(Ob2PCRole, LEAF) - }; - return str; -} - static const char *to_str_tx_state(ObTxState state) { const char *str = "INVALID"; diff --git a/src/storage/tx/ob_ctx_tx_data.cpp b/src/storage/tx/ob_ctx_tx_data.cpp index 2ad5db07c..867f60f92 100644 --- a/src/storage/tx/ob_ctx_tx_data.cpp +++ b/src/storage/tx/ob_ctx_tx_data.cpp @@ -35,7 +35,7 @@ namespace transaction TRANS_LOG(WARN, "get tx table guard without check failed", KR(ret), K(*this)); \ } else if (OB_ISNULL(tx_table = table_guard.get_tx_table())) { \ ret = OB_ERR_UNEXPECTED; \ - TRANS_LOG(WARN, "tx table is null", KR(ret), K(ctx_mgr_->get_ls_id()), K(*this)); \ + TRANS_LOG(WARN, "tx table is null", KR(ret), K(*this)); \ } int ObCtxTxData::init(const int64_t abs_expire_time, ObLSTxCtxMgr *ctx_mgr, int64_t tx_id) @@ -52,12 +52,12 @@ int ObCtxTxData::init(const int64_t abs_expire_time, ObLSTxCtxMgr *ctx_mgr, int6 TRANS_LOG(WARN, "ctx_mgr is null when get_tx_table", K(ret)); } else if (OB_ISNULL(tx_table = ctx_mgr_->get_tx_table())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx table is null", KR(ret), K(ctx_mgr_->get_ls_id()), K(*this)); + TRANS_LOG(WARN, "tx table is null", KR(ret), K(*this)); } else if (OB_FAIL(tx_table->alloc_tx_data(tx_data_guard_, true, abs_expire_time))) { - TRANS_LOG(WARN, "get tx data failed", KR(ret), K(ctx_mgr_->get_ls_id())); + TRANS_LOG(WARN, "get tx data failed", KR(ret)); } else if (OB_ISNULL(tx_data_guard_.tx_data())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx data is unexpected null", KR(ret), K(ctx_mgr_->get_ls_id())); + TRANS_LOG(WARN, "tx data is unexpected null", KR(ret)); } else { tx_data_guard_.tx_data()->tx_id_ = tx_id; } diff --git a/src/storage/tx/ob_direct_load_tx_ctx_define.h b/src/storage/tx/ob_direct_load_tx_ctx_define.h index 246decf87..9fd53e94c 100644 --- a/src/storage/tx/ob_direct_load_tx_ctx_define.h +++ b/src/storage/tx/ob_direct_load_tx_ctx_define.h @@ -14,7 +14,7 @@ * limitations under the License. */ -// class ObPartTransCtx +// class ObTxCtx // { // public: @@ -26,7 +26,6 @@ #include "lib/hash/ob_hashset.h" #include "share/scn.h" #include "storage/ddl/ob_ddl_inc_clog.h" -#include "storage/tx/ob_tx_serialization.h" namespace oceanbase { diff --git a/src/storage/tx/ob_gti_rpc.cpp b/src/storage/tx/ob_gti_rpc.cpp deleted file mode 100644 index b2c0d3d61..000000000 --- a/src/storage/tx/ob_gti_rpc.cpp +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gti_rpc.h" -#include "share/rc/ob_module_provider.h" -#include "ob_trans_id_service.h" -#include "ob_trans_service.h" -#include "share/ob_ex_rpc.h" - -namespace oceanbase -{ -using namespace share; -using namespace obcall; -namespace transaction -{ - -OB_SERIALIZE_MEMBER(ObGtiRequest, range_); - -int ObGtiRequest::init(const int64_t range) -{ - int ret = OB_SUCCESS; - if (!true || 0 >= range) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(range)); - } else { - range_ = range; - } - return ret; -} - -bool ObGtiRequest::is_valid() const -{ - return true && range_ > 0; -} - -int ObGtiRequestRpc::init(const ObAddr &self, ObGtiSource *gti_source) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "gti request rpc inited twice", KR(ret)); - } else if (!self.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(self)); - } else if (OB_SUCCESS != (ret = gti_request_cb_.init(gti_source))) { - TRANS_LOG(WARN, "gti request callback inited failed", KR(ret)); - } else { - self_ = self; - is_inited_ = true; - TRANS_LOG(INFO, "gti request rpc inited success", KP(this), K(self)); - } - return ret; -} - -int ObGtiRequestRpc::start() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gti request rpc not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gti request rpc already running", KR(ret)); - } else { - is_running_ = true; - TRANS_LOG(INFO, "gti request rpc start success"); - } - return ret; -} - -int ObGtiRequestRpc::stop() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gti request rpc not inited", KR(ret)); - } else { - is_running_ = false; - TRANS_LOG(INFO, "gti request rpc stop success"); - } - return ret; -} - -int ObGtiRequestRpc::wait() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gti request rpc not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gti request rpc is running", KR(ret)); - } else { - TRANS_LOG(INFO, "gti request rpc wait success"); - } - return ret; -} - -void ObGtiRequestRpc::destroy() -{ - int tmp_ret = OB_SUCCESS; - if (is_inited_) { - if (is_running_) { - if (OB_SUCCESS != (tmp_ret = stop())) { - TRANS_LOG_RET(WARN, tmp_ret, "gti request rpc stop error", K(tmp_ret)); - } else if (OB_SUCCESS != (tmp_ret = wait())) { - TRANS_LOG_RET(WARN, tmp_ret, "gti request rpc wait error", K(tmp_ret)); - } else { - // do nothing - } - } - is_inited_ = false; - self_.reset(); - TRANS_LOG(INFO, "gti request rpc destroy"); - } -} - -int ObGtiRequestRpc::post(const ObGtiRequest &msg) -{ - int ret = OB_SUCCESS; - ObAddr server; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gti request rpc not inited", KR(ret)); - } else if (!is_running_) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "gti request rpc not running", KR(ret)); - } else if (!msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(msg)); - } else if (OB_FAIL(share::g_mp->trans_service()->get_location_adapter()->nonblock_get_leader(GCONF.cluster_id, GTI_LS, server))) { - TRANS_LOG(WARN, "get leader failed", KR(ret), K(msg), K(GTI_LS)); - } else { - // single-replica: target is always local; dispatch async in-process (ex-RPC), - // restoring the original async .post(msg, >i_request_cb_) decoupling. msg is - // serialized; tenant context is restored on the worker via MTL_SWITCH. The cb is a - // plain value type now (no heap clone needed -- the result is passed to process() as - // an argument, not stored in the cb), so each async task captures its own value copy. - (void)ex_rpc::async_call(msg, - [cb = gti_request_cb_, server](const ObGtiRequest &m) mutable { - int ret = OB_SUCCESS; - MOD_SCOPE { - ObGtiRpcResult gti_rpc_result; - if (OB_FAIL(share::g_mp->trans_id_service()->handle_request(m, gti_rpc_result))) { - TRANS_LOG(WARN, "post local gti request failed", KR(ret), K(server), K(m)); - } else if (!gti_rpc_result.is_valid()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "post local gti request and gti_rpc_result is invalid", KR(ret), K(server), - K(m), K(gti_rpc_result)); - } else { - rpc::frame::ObResultCode rcode; - rcode.rcode_ = OB_SUCCESS; - - if (OB_FAIL(cb.process(gti_rpc_result, server, rcode))) { - TRANS_LOG(WARN, "post local gti request failed", KR(ret), K(server), K(m)); - } else { - TRANS_LOG(DEBUG, "post local gti request success", KR(ret), K(server), K(m)); - } - } - } - }); - } - return ret; -} - -} //transaction - -namespace obcall -{ - -OB_SERIALIZE_MEMBER(ObGtiRpcResult, status_, start_id_, end_id_); - -int ObGtiRpcResult::init(const int status, const int64_t start_id, - const int64_t end_id) -{ - int ret = OB_SUCCESS; - if (!true || - (OB_SUCCESS == status && (0 >= start_id || 0 >= end_id))) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), - K(status), K(start_id), K(end_id)); - } else { - status_ = status; - start_id_ = start_id; - end_id_ = end_id; - TRANS_LOG(INFO, "ObGtiRpcResult init", KR(ret), - K(status), K(start_id), K(end_id)); - } - return ret; -} - -void ObGtiRpcResult::reset() -{ - status_ = OB_SUCCESS; - start_id_ = 0; - end_id_ = 0; -} - -bool ObGtiRpcResult::is_valid() const -{ - return true && - (OB_SUCCESS != status_ || (start_id_ > 0 && end_id_ > 0)); -} - -} // obcall - -} // oceanbase diff --git a/src/storage/tx/ob_gti_rpc.h b/src/storage/tx/ob_gti_rpc.h deleted file mode 100644 index 3fd1d1173..000000000 --- a/src/storage/tx/ob_gti_rpc.h +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTI_RPC_ -#define OCEANBASE_TRANSACTION_OB_GTI_RPC_ - -#include "observer/ob_server_struct.h" -#include "rpc/frame/ob_result_code.h" -#include "storage/tx/ob_gti_source.h" - -namespace oceanbase -{ - -namespace transaction -{ - -class ObGtiRequest -{ - OB_UNIS_VERSION(1); -public: - ObGtiRequest() : range_(0) {} - ~ObGtiRequest() {} - int init(const int64_t range); - bool is_valid() const; -public: - - int64_t get_range() const { return range_; } - TO_STRING_KV(K_(range)); -private: - int64_t range_; -}; - -} //transaction - -namespace obcall -{ - -class ObGtiRpcResult -{ - OB_UNIS_VERSION(1); -public: - ObGtiRpcResult() { reset(); } - virtual ~ObGtiRpcResult() {} - int init(const int status, const int64_t start_id, const int64_t end_id); - - int get_status() const { return status_; } - int64_t get_start_id() const { return start_id_; } - int64_t get_end_id() const { return end_id_; } - void reset(); - bool is_valid() const; - TO_STRING_KV(K_(status), K_(start_id), K_(end_id)); -public: - static const int64_t OB_GTI_RPC_TIMEOUT = 1 * 1000 * 1000; -private: - int status_; - int64_t start_id_; - int64_t end_id_; -}; - -class ObGtiRPCCB -{ -public: - ObGtiRPCCB() : is_inited_(false), gti_source_(NULL) {} - ~ObGtiRPCCB() {} - int init(transaction::ObGtiSource *gti_source) - { - int ret = common::OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "ObGtiRPCCB inited twice", KR(ret)); - } else if (NULL == gti_source) { - ret = common::OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(gti_source)); - } else { - gti_source_ = gti_source; - is_inited_ = true; - } - return ret; - } - int process(const obcall::ObGtiRpcResult &result, const common::ObAddr &dst, - rpc::frame::ObResultCode &rcode) - { - return process_(result, dst, rcode); - } - -private: - int process_(const obcall::ObGtiRpcResult &result, const common::ObAddr &dst, - rpc::frame::ObResultCode &rcode) - { - int ret = OB_SUCCESS; - int status = OB_SUCCESS; - bool update = false; - - if (!is_inited_) { - TRANS_LOG(WARN, "ObGtiRPCCB not inited"); - ret = OB_NOT_INIT; - } else if (!true) { - TRANS_LOG(WARN, "invalid argument", K(dst)); - ret = OB_ERR_UNEXPECTED; - } else { - MOD_SCOPE { - if (OB_SUCCESS != rcode.rcode_) { - status = rcode.rcode_; - TRANS_LOG(WARN, "gti rpc error", K(rcode), K(dst)); - if (OB_NOT_MASTER == status - || OB_TENANT_NOT_IN_SERVER == status) { - if (OB_FAIL(gti_source_->refresh_gti_location())) { - TRANS_LOG(WARN, "refresh gti location fail", K(ret)); - } - } - } else { - status = result.get_status(); - if (OB_SUCCESS == status) { - if (OB_FAIL(gti_source_->update_trans_id(result.get_start_id(), - result.get_end_id()))) { - TRANS_LOG(WARN, "update trans id fail", K(ret)); - } - } else if (OB_NOT_MASTER == status) { - if (OB_FAIL(gti_source_->refresh_gti_location())) { - TRANS_LOG(WARN, "refresh gti location fail", K(ret)); - } - } - TRANS_LOG(INFO, "gti request callback", KR(ret), K(result), K(rcode)); - } - } else { - TRANS_LOG(WARN, "tenant switch fail", K(dst)); - } - } - return ret; - } - bool is_inited_; - transaction::ObGtiSource *gti_source_; -}; - -} // obcall - -namespace transaction -{ - -class ObGtiRequestRpc -{ -public: - ObGtiRequestRpc() : is_inited_(false), is_running_(false) {} - ~ObGtiRequestRpc() { destroy(); } - int init(const common::ObAddr &self, ObGtiSource *gti_source); - int start(); - int stop(); - int wait(); - void destroy(); -public: - int post(const ObGtiRequest &msg); -private: - bool is_inited_; - bool is_running_; - obcall::ObGtiRPCCB gti_request_cb_; - common::ObAddr self_; -}; - -} // transaction - -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_GTI_RPC_ diff --git a/src/storage/tx/ob_gti_source.cpp b/src/storage/tx/ob_gti_source.cpp index 92c437036..ca38a0562 100644 --- a/src/storage/tx/ob_gti_source.cpp +++ b/src/storage/tx/ob_gti_source.cpp @@ -15,8 +15,8 @@ */ #include "ob_gti_source.h" -#include "ob_trans_service.h" -#include "ob_gti_rpc.h" +#include "ob_trans_id_service.h" +#include "share/rc/ob_module_provider.h" namespace oceanbase { @@ -26,31 +26,16 @@ using namespace share; namespace transaction { -int ObGtiSource::init(const ObAddr &server) +int ObGtiSource::init() { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; TRANS_LOG(WARN, "init twice", KR(ret)); - } else if (OB_UNLIKELY(!server.is_valid())) { - TRANS_LOG(WARN, "invalid argument", KR(ret), K(server)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_ISNULL(gti_request_rpc_ = ObGtiRequestRpcFactory::alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc gti_reqeust_rpc fail", KR(ret)); - } else if (OB_FAIL(gti_request_rpc_->init(server, this))) { - TRANS_LOG(WARN, "response rpc init failed", KR(ret), K(server)); } else { - server_ = server; is_inited_ = true; - TRANS_LOG(INFO, "gti source init success", K(server), KP(this)); - } - if (OB_FAIL(ret)) { - if (NULL != gti_request_rpc_) { - ObGtiRequestRpcFactory::release(gti_request_rpc_); - gti_request_rpc_ = NULL; - } + TRANS_LOG(INFO, "gti source init success", KP(this)); } return ret; @@ -66,8 +51,6 @@ int ObGtiSource::start() } else if (is_running_) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "ObGtiSource is already running", KR(ret)); - } else if (OB_FAIL(gti_request_rpc_->start())) { - TRANS_LOG(WARN, "gti request rpc start", KR(ret)); } else { is_running_ = true; TRANS_LOG(INFO, "ObGtiSource start success"); @@ -77,34 +60,13 @@ int ObGtiSource::start() void ObGtiSource::stop() { - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObGtiSource is not inited", KR(ret)); - } else if (OB_FAIL(gti_request_rpc_->stop())) { - TRANS_LOG(WARN, "gti request rpc stop", KR(ret)); - } else { - is_running_ = false; - TRANS_LOG(INFO, "ObGtiSource stop success"); - } + is_running_ = false; + TRANS_LOG(INFO, "ObGtiSource stop success"); } void ObGtiSource::wait() { - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObGtiSource is not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ObGtiSource is running", KR(ret)); - } else if (OB_FAIL(gti_request_rpc_->wait())) { - TRANS_LOG(WARN, "gti request rpc wait", KR(ret)); - } else { - TRANS_LOG(INFO, "ObGtiSource wait success"); - } + TRANS_LOG(INFO, "ObGtiSource wait success"); } void ObGtiSource::destroy() @@ -116,10 +78,6 @@ void ObGtiSource::destroy() } is_inited_ = false; } - if (NULL != gti_request_rpc_) { - ObGtiRequestRpcFactory::release(gti_request_rpc_); - gti_request_rpc_ = NULL; - } TRANS_LOG(INFO, "ObGtiSource destroyed"); } @@ -134,21 +92,12 @@ void ObGtiSource::reset() } cur_idx_ = 0; cache_idx_ = 0; - server_.reset(); - gti_request_rpc_ = NULL; - gti_cache_leader_.reset(); retry_request_cnt_ = 0; last_request_ts_ = 0; preallocate_count_ = MIN_PREALLOCATE_COUNT; last_update_ts_ = ObTimeUtility::current_time(); } -int ObGtiSource::refresh_gti_location() -{ - int ret = OB_SUCCESS; - return ret; -} - int ObGtiSource::update_trans_id(const int64_t start_id, const int64_t end_id) { int ret = OB_SUCCESS; @@ -202,11 +151,13 @@ int ObGtiSource::get_trans_id(int64_t &trans_id) retry_interval = min(retry_request_cnt_ * RETRY_REQUEST_INTERVAL, MAX_RETRY_REQUEST_INTERVAL); } if (cur_ts - last_request_ts_ > retry_interval && ATOMIC_BCAS(&is_requesting_, false, true)) { - ObGtiRequest req; - if (OB_FAIL(req.init(get_preallocate_count_()))) { - TRANS_LOG(WARN, "ObGtiRequest init fail", KR(ret)); - } else if (OB_FAIL(gti_request_rpc_->post(req))) { - TRANS_LOG(WARN, "ObGtiRequest post fail", KR(ret)); + const int64_t range = get_preallocate_count_(); + int64_t start_id = 0; + int64_t end_id = 0; + if (OB_FAIL(share::g_mp->trans_id_service()->get_number(range, 0, start_id, end_id))) { + TRANS_LOG(WARN, "allocate transaction id range failed", KR(ret), K(range)); + } else if (OB_FAIL(update_trans_id(start_id, end_id))) { + TRANS_LOG(WARN, "cache transaction id range failed", KR(ret), K(start_id), K(end_id)); } else { last_request_ts_ = cur_ts; retry_request_cnt_++; diff --git a/src/storage/tx/ob_gti_source.h b/src/storage/tx/ob_gti_source.h index bf4a7d102..474ec8559 100644 --- a/src/storage/tx/ob_gti_source.h +++ b/src/storage/tx/ob_gti_source.h @@ -17,25 +17,13 @@ #ifndef OCEANBASE_TRANSACTION_OB_GTI_SOURCE_ #define OCEANBASE_TRANSACTION_OB_GTI_SOURCE_ -#include "lib/net/ob_addr.h" #include "lib/lock/ob_latch.h" namespace oceanbase { -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} - namespace transaction { -class ObGtiRequestRpc; -class ObGtiErrResponse; - struct IdCache { int64_t start_id; @@ -59,21 +47,19 @@ class ObGtiSource : public ObIGtiSource public: ObGtiSource() { reset(); } ~ObGtiSource() { destroy(); } - int init(const common::ObAddr &server); + int init(); virtual int start(); virtual void stop(); virtual void wait(); virtual void destroy(); virtual void reset(); - int refresh_gti_location(); int update_trans_id(const int64_t start_id, const int64_t end_id); virtual int get_trans_id(int64_t &trans_id); private: void update_preallocate_count_(); int64_t get_preallocate_count_(); public: - TO_STRING_KV(K_(is_inited), K_(is_running), K_(is_requesting), - K_(server), K_(gti_cache_leader)); + TO_STRING_KV(K_(is_inited), K_(is_running), K_(is_requesting)); public: static const int64_t MIN_PREALLOCATE_COUNT = 10000; static const int64_t MAX_PREALLOCATE_COUNT = 1000000; @@ -91,9 +77,6 @@ class ObGtiSource : public ObIGtiSource IdCache id_cache_[MAX_CACHE_NUM]; int64_t cur_idx_; int64_t cache_idx_; - common::ObAddr server_; - ObGtiRequestRpc *gti_request_rpc_; - common::ObAddr gti_cache_leader_; int64_t retry_request_cnt_; int64_t last_request_ts_; // lock for update trans id diff --git a/src/storage/tx/ob_gts_define.h b/src/storage/tx/ob_gts_define.h deleted file mode 100644 index c5ba7dc01..000000000 --- a/src/storage/tx/ob_gts_define.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_DEFINE_ -#define OCEANBASE_TRANSACTION_OB_GTS_DEFINE_ - -#include -#include "lib/utility/ob_print_utils.h" -#include "lib/utility/utility.h" -#include "share/ob_define.h" -#include "share/inner_table/ob_inner_table_schema_constants.h" -#include "ob_trans_define.h" - -namespace oceanbase -{ -namespace transaction -{ - -enum ObGTSCacheTaskType -{ - INVALID_GTS_TASK_TYPE = -1, - GET_GTS = 0, - WAIT_GTS_ELAPSING, -}; - -inline bool atomic_update(int64_t *v, const int64_t x) -{ - bool bool_ret = false; - int64_t ov = ATOMIC_LOAD(v); - while (ov < x) { - if (ATOMIC_BCAS(v, ov, x)) { - bool_ret = true; - break; - } else { - ov = ATOMIC_LOAD(v); - } - } - return bool_ret; -} - -} // transaction -} // oceanbase - -#endif // OCEANBASE_RANSACTION_OB_GTS_DEFINE_ diff --git a/src/storage/tx/ob_gts_local_cache.cpp b/src/storage/tx/ob_gts_local_cache.cpp deleted file mode 100644 index d08737055..000000000 --- a/src/storage/tx/ob_gts_local_cache.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gts_local_cache.h" - -namespace oceanbase -{ -using namespace common; - -namespace transaction -{ - -void ObGTSLocalCache::reset() -{ - srr_.reset(); - gts_ = 0; - latest_srr_.reset(); - receive_gts_ts_.reset(); -} - -//Due to network and other factors, it is impossible to guarantee that srr and gts maintain partial order, -//so the logic of this method: -//srr and gts and the locally saved values are all taken to the maximum value for storage -int ObGTSLocalCache::update_gts(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts, - bool &update) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(gts <= 0) - || gts >= INT64_MAX/2) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts)); - } else { - //The update sequence must be receive_gts_ts first, then gts, then srr - (void)atomic_update(&receive_gts_ts_.mts_, receive_gts_ts.mts_); - (void)atomic_update(>s_, gts); - update = atomic_update(&srr_.mts_, srr.mts_); - } - - return ret; -} - -// While updating srr and gts, it is also necessary to check whether gts crosses the barrier ts -int ObGTSLocalCache::update_gts_and_check_barrier(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(gts <= 0) - || gts >= INT64_MAX/2) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts)); - } else { - //The order of update must be receive_gts_ts first, then gts, then srr - (void)atomic_update(&receive_gts_ts_.mts_, receive_gts_ts.mts_); - (void)atomic_update(>s_, gts); - (void)atomic_update(&srr_.mts_, srr.mts_); - } - - return ret; -} - -int ObGTSLocalCache::update_gts(const int64_t gts, bool &update) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(gts < 0) || gts >= INT64_MAX/2) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(gts)); - } else { - update = atomic_update(>s_, gts); - } - - return ret; -} - -int ObGTSLocalCache::get_gts(int64_t >s) const -{ - int ret = OB_SUCCESS; - const int64_t tmp_gts = ATOMIC_LOAD(>s_); - if (OB_UNLIKELY(0 == tmp_gts)) { - ret = OB_EAGAIN; - } else { - //Here should not add 1 - gts = tmp_gts; - } - return ret; -} - -int ObGTSLocalCache::get_gts(const MonotonicTs stc, - int64_t >s, - MonotonicTs &receive_gts_ts, - bool &need_send_rpc) const -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!stc.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(stc)); - } else { - const int64_t srr = ATOMIC_LOAD(&srr_.mts_); - // Must get gts first, then receive_gts_ts - const int64_t tmp_gts = ATOMIC_LOAD(>s_); - const int64_t tmp_receive_gts_ts = ATOMIC_LOAD(&receive_gts_ts_.mts_); - if (0 == tmp_gts) { - ret = OB_EAGAIN; - need_send_rpc = true; - } else if (stc.mts_ > srr) { - ret = OB_EAGAIN; - need_send_rpc = (stc.mts_ > ATOMIC_LOAD(&latest_srr_.mts_)); - } else { - //Here should not add 1 - gts = tmp_gts; - receive_gts_ts = MonotonicTs(tmp_receive_gts_ts); - need_send_rpc = false; - } - } - - return ret; -} - -int ObGTSLocalCache::get_srr_and_gts_safe(MonotonicTs &srr, - int64_t >s, - MonotonicTs &receive_gts_ts) const -{ - // must set srr before gts - srr.mts_ = ATOMIC_LOAD(&srr_.mts_); - //Here should not add 1 - gts = ATOMIC_LOAD(>s_); - receive_gts_ts.mts_ = ATOMIC_LOAD(&receive_gts_ts_.mts_); - return OB_SUCCESS; -} - -int ObGTSLocalCache::update_latest_srr(const MonotonicTs latest_srr) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!latest_srr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(latest_srr)); - } else { - (void)atomic_update(&latest_srr_.mts_, latest_srr.mts_); - } - - return ret; -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_gts_local_cache.h b/src/storage/tx/ob_gts_local_cache.h deleted file mode 100644 index b177cc113..000000000 --- a/src/storage/tx/ob_gts_local_cache.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_LOCAL_CACHE_ -#define OCEANBASE_TRANSACTION_OB_GTS_LOCAL_CACHE_ - -#include "ob_gts_define.h" -#include "ob_gts_task_queue.h" -#include "share/ob_errno.h" -#include "lib/utility/ob_print_utils.h" -#include "lib/utility/utility.h" - -namespace oceanbase -{ -namespace transaction -{ - -class ObGTSLocalCache -{ -public: - ObGTSLocalCache() { reset(); } - ~ObGTSLocalCache() { destroy(); } - void reset(); - void destroy() { reset(); } - int update_gts(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts, - bool &update); - int update_gts_and_check_barrier(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts); - int update_gts(const int64_t gts, bool &update); - int get_gts(int64_t >s) const; - MonotonicTs get_latest_srr() const { return MonotonicTs(ATOMIC_LOAD(&latest_srr_.mts_)); } - MonotonicTs get_srr() const { return MonotonicTs(ATOMIC_LOAD(&srr_.mts_)); } - int get_gts(const MonotonicTs stc, int64_t >s, MonotonicTs &receive_gts_ts, bool &need_send_rpc) const; - int get_srr_and_gts_safe(MonotonicTs &srr, int64_t >s, MonotonicTs &receive_gts_ts) const; - int update_latest_srr(const MonotonicTs latest_srr); - bool no_rpc_on_road() const { return ATOMIC_LOAD(&latest_srr_.mts_) == ATOMIC_LOAD(&srr_.mts_); } - - TO_STRING_KV(K_(srr), K_(gts), K_(latest_srr)); -private: - // send rpc request timestamp - MonotonicTs srr_; - // The latest local gts value is always less than or equal to the gts leader - int64_t gts_; - MonotonicTs latest_srr_; - // receive gts - MonotonicTs receive_gts_ts_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_RANSACTION_OB_GTS_LOCAL_CACHE_ diff --git a/src/storage/tx/ob_gts_msg.cpp b/src/storage/tx/ob_gts_msg.cpp deleted file mode 100644 index a43ca1e7e..000000000 --- a/src/storage/tx/ob_gts_msg.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gts_msg.h" -#include "ob_gts_rpc.h" - -namespace oceanbase -{ -using namespace common; -using namespace obcall; - -namespace transaction -{ -// ObGtsRequest -OB_SERIALIZE_MEMBER(ObGtsRequest, srr_.mts_, range_size_, sender_); -// ObGtsErrResponse -OB_SERIALIZE_MEMBER(ObGtsErrResponse, srr_.mts_, status_, sender_); - -int ObGtsRequest::init(const MonotonicTs srr, const int64_t range_size, - const ObAddr &sender) -{ - int ret = OB_SUCCESS; - if (!true || !srr.is_valid() || 0 >= range_size || !sender.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(range_size), K(sender)); - } else { - srr_ = srr; - range_size_ = range_size; - sender_ = sender; - } - return ret; -} - -bool ObGtsRequest::is_valid() const -{ - return true && srr_.is_valid() && range_size_ > 0 && - sender_.is_valid(); -} - -//leader may be invalid, the validity check does not need to check this field -int ObGtsErrResponse::init(const MonotonicTs srr, const int status, - const ObAddr &sender) -{ - int ret = OB_SUCCESS; - if (!true || !srr.is_valid() || !sender.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(status), K(sender)); - } else { - srr_ = srr; - status_ = status; - sender_ = sender; - } - return ret; -} - -bool ObGtsErrResponse::is_valid() const -{ - return true && srr_.is_valid() && OB_SUCCESS != status_ && sender_.is_valid(); -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_gts_msg.h b/src/storage/tx/ob_gts_msg.h deleted file mode 100644 index 0d8cff7d0..000000000 --- a/src/storage/tx/ob_gts_msg.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_MSG_ -#define OCEANBASE_TRANSACTION_OB_GTS_MSG_ - -#include "share/ob_define.h" -#include "lib/utility/ob_unify_serialize.h" -#include "lib/net/ob_addr.h" -#include "ob_gts_define.h" - -namespace oceanbase -{ -namespace obcall -{ -class ObGtsRpcResult; -} -namespace transaction -{ -class ObGtsRequest -{ - OB_UNIS_VERSION(1); -public: - ObGtsRequest() : srr_(0), range_size_(0), sender_() {} - ~ObGtsRequest() {} - int init(const MonotonicTs srr, const int64_t range_size, - const common::ObAddr &sender); - bool is_valid() const; -public: - - MonotonicTs get_srr() const { return srr_; } - const common::ObAddr &get_sender() const { return sender_; } - TO_STRING_KV(K_(srr), K_(range_size), K_(sender)); -private: - MonotonicTs srr_; - int64_t range_size_; - common::ObAddr sender_; -}; - -class ObGtsErrResponse -{ - OB_UNIS_VERSION(1); -public: - ObGtsErrResponse() : srr_(0), status_(0), sender_() {} - ~ObGtsErrResponse() {} - int init(const MonotonicTs srr, const int status, - const common::ObAddr &sender); - bool is_valid() const; -public: - - MonotonicTs get_srr() const { return srr_; } - int get_status() const { return status_; } - const common::ObAddr &get_sender() const { return sender_; } - TO_STRING_KV(K_(srr), K_(status), K_(sender)); -private: - MonotonicTs srr_; - int status_; - common::ObAddr sender_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_GTS_MSG_ diff --git a/src/storage/tx/ob_gts_rpc.cpp b/src/storage/tx/ob_gts_rpc.cpp deleted file mode 100644 index 0d1aabd4e..000000000 --- a/src/storage/tx/ob_gts_rpc.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gts_rpc.h" -#include "share/rc/ob_module_provider.h" -#include "ob_timestamp_access.h" -#include "share/ob_ex_rpc.h" - -namespace oceanbase -{ - -using namespace common; -using namespace transaction; -using namespace obcall; -using namespace storage; -using namespace omt; -using namespace observer; -using namespace share; - -namespace obcall -{ - -OB_SERIALIZE_MEMBER(ObGtsRpcResult, status_, srr_.mts_, gts_start_, gts_end_); - -int ObGtsRpcResult::init(const int status, - const MonotonicTs srr, const int64_t gts_start, const int64_t gts_end) -{ - int ret = OB_SUCCESS; - if (!true || - (OB_SUCCESS == status && (!srr.is_valid() || 0 >= gts_start || 0 >= gts_end))) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), - K(status), K(srr), K(gts_start), K(gts_end)); - } else { - status_ = status; - srr_ = srr; - gts_start_ = gts_start; - gts_end_ = gts_end; - } - return ret; -} - -void ObGtsRpcResult::reset() -{ - status_ = OB_SUCCESS; - srr_.reset(); - gts_start_ = 0; - gts_end_ = 0; -} - -bool ObGtsRpcResult::is_valid() const -{ - return true && - (OB_SUCCESS != status_ || (srr_.is_valid() && gts_start_ > 0 && gts_end_ > 0)); -} - -} // obcall - -namespace transaction -{ -int ObGtsRequestRpc::init(const ObAddr &self, - ObTsMgr *ts_mgr, ObTsWorker *ts_worker) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "gts request rpc inited twice", KR(ret)); - } else if (!self.is_valid() || OB_ISNULL(ts_mgr) || OB_ISNULL(ts_worker)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(self), KP(ts_mgr), KP(ts_worker)); - } else if (OB_SUCCESS != (ret = gts_request_cb_.init(ts_mgr, ts_worker))) { - TRANS_LOG(WARN, "gts request callback inited failed", KR(ret)); - } else { - self_ = self; - is_inited_ = true; - ts_mgr_ = ts_mgr; - TRANS_LOG(INFO, "gts request rpc inited success", KP(this), K(self), KP(ts_mgr)); - } - return ret; -} - -int ObGtsRequestRpc::start() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gts request rpc not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gts request rpc already running", KR(ret)); - } else { - is_running_ = true; - TRANS_LOG(INFO, "gts request rpc start success"); - } - return ret; -} - -int ObGtsRequestRpc::stop() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gts request rpc not inited", KR(ret)); - } else { - is_running_ = false; - TRANS_LOG(INFO, "gts request rpc stop success"); - } - return ret; -} - -int ObGtsRequestRpc::wait() -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gts request rpc not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gts request rpc is running", KR(ret)); - } else { - TRANS_LOG(INFO, "gts request rpc wait success"); - } - return ret; -} - -void ObGtsRequestRpc::destroy() -{ - int tmp_ret = OB_SUCCESS; - if (is_inited_) { - if (is_running_) { - if (OB_SUCCESS != (tmp_ret = stop())) { - TRANS_LOG_RET(WARN, tmp_ret, "gts request rpc stop error", K(tmp_ret)); - } else if (OB_SUCCESS != (tmp_ret = wait())) { - TRANS_LOG_RET(WARN, tmp_ret, "gts request rpc wait error", K(tmp_ret)); - } else { - // do nothing - } - } - is_inited_ = false; - self_.reset(); - TRANS_LOG(INFO, "gts request rpc destroy"); - } -} - -int ObGtsRequestRpc::post(const ObAddr &server, - const ObGtsRequest &msg) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "gts request rpc not inited", KR(ret)); - } else if (!is_running_) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "gts request rpc not running", KR(ret)); - } else if (!true || !server.is_valid() || !msg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(server), K(msg)); - } else { - // single-replica: target is always local; dispatch async in-process (ex-RPC), - // restoring the original async .post(msg, >s_request_cb_) decoupling: the handler - // + response callback run on a worker thread, not the caller. msg is serialized - // (deep-copied). The cb is a plain value type now (config + tenant; unlike the old - // AsyncCB it no longer stores the pending result -- that is passed to process() as an - // argument), so a per-request value copy is the correct "clone": no heap clone(alloc) - // is needed in the in-process model, and each async task gets its own copy (own - // tenant) so they never race on the shared gts_request_cb_ member. - (void)ex_rpc::async_call(msg, - [cb = gts_request_cb_, server](const ObGtsRequest &m) mutable { - int ret = OB_SUCCESS; - ObGtsRpcResult gts_rpc_result; - MOD_SCOPE { - if (OB_FAIL(share::g_mp->timestamp_access()->handle_request(m, gts_rpc_result))) { - if (REACH_TIME_INTERVAL(100 * 1000)) { - TRANS_LOG(WARN, "post local gts request failed", KR(ret), K(server), K(m)); - } - } else if (!gts_rpc_result.is_valid()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "post local gts request and gts_rpc_result is invalid", KR(ret), K(server), - K(m), K(gts_rpc_result)); - } else { - rpc::frame::ObResultCode rcode; - rcode.rcode_ = OB_SUCCESS; - - if (OB_FAIL(cb.process(gts_rpc_result, server, rcode))) { - TRANS_LOG(WARN, "post local gts request failed", KR(ret), K(server), K(m)); - } else { - TRANS_LOG(DEBUG, "post local gts request success", KR(ret), K(server), K(m)); - } - } - } - }); - } - return ret; -} - -} // transaction - -} // oceanbase diff --git a/src/storage/tx/ob_gts_rpc.h b/src/storage/tx/ob_gts_rpc.h deleted file mode 100644 index ac3c92a34..000000000 --- a/src/storage/tx/ob_gts_rpc.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_RPC_ -#define OCEANBASE_TRANSACTION_OB_GTS_RPC_ - -#include "lib/thread/ob_queue_thread.h" -#include "lib/utility/ob_unify_serialize.h" -#include "lib/utility/utility.h" -#include "share/ob_define.h" -#include "rpc/frame/ob_result_code.h" -#include "share/ob_rpc_struct.h" -#include "storage/tx/ob_gts_msg.h" -#include "storage/tx/ob_ts_worker.h" -#include "storage/tx/ob_ts_response_handler.h" -#include "storage/tx/ob_ts_mgr.h" -#include "share/config/ob_server_config.h" -#include "observer/ob_server_struct.h" - -namespace oceanbase -{ - -namespace transaction -{ -class ObTsMgr; -class ObTsWorker; -} - -namespace obcall -{ -class ObGtsRpcResult -{ - OB_UNIS_VERSION(1); -public: - ObGtsRpcResult() { reset(); } - virtual ~ObGtsRpcResult() {} - int init(const int status, - const transaction::MonotonicTs srr, const int64_t gts_start, const int64_t gts_end); - - int get_status() const { return status_; } - transaction::MonotonicTs get_srr() const { return srr_; } - int64_t get_gts_start() const { return gts_start_; } - int64_t get_gts_end() const { return gts_end_; } - void reset(); - bool is_valid() const; - TO_STRING_KV(K_(status), K_(srr), K_(gts_start), K_(gts_end)); -public: - static const int64_t OB_GTS_RPC_TIMEOUT = 1 * 1000 * 1000; -private: - int status_; - transaction::MonotonicTs srr_; - int64_t gts_start_; - int64_t gts_end_; -}; - -class ObGtsRPCCB -{ -public: - ObGtsRPCCB() : is_inited_(false), ts_mgr_(NULL), ts_worker_(NULL) {} - ~ObGtsRPCCB() {} - int init(transaction::ObTsMgr *ts_mgr, - transaction::ObTsWorker *ts_worker) - { - int ret = common::OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "ObGtsRPCCB inited twice", KR(ret)); - } else if (NULL == ts_mgr || NULL == ts_worker) { - ret = common::OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(ts_mgr), KP(ts_worker)); - } else { - ts_mgr_ = ts_mgr; - ts_worker_ = ts_worker; - is_inited_ = true; - } - return ret; - } - int process(const obcall::ObGtsRpcResult &result, const common::ObAddr &dst, - rpc::frame::ObResultCode &rcode) - { - return process_(result, dst, rcode); - } - -private: - int process_(const obcall::ObGtsRpcResult &result, const common::ObAddr &dst, - rpc::frame::ObResultCode &rcode) - { - int ret = OB_SUCCESS; - int status = OB_SUCCESS; - bool update = false; - - if (!is_inited_) { - TRANS_LOG(WARN, "ObGtsRPCCB not inited"); - ret = OB_NOT_INIT; - } else if (!true) { - TRANS_LOG(WARN, "invalid argument", K(dst)); - ret = OB_ERR_UNEXPECTED; - } else if (OB_SUCCESS != rcode.rcode_) { - status = rcode.rcode_; - TRANS_LOG(WARN, "gts rpc error", K(rcode), K(dst)); - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "get gts need refresh gts location", K(status), K(result)); - } - if (NULL == ts_mgr_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gts local cache mgr is NULL", K(ret)); - } else if (OB_FAIL(ts_mgr_->refresh_gts_location())) { - TRANS_LOG(WARN, "refresh gts location fail", K(ret)); - } else { - // do nothing - } - } else { - status = result.get_status(); - if (OB_SUCCESS == status) { - if (NULL == ts_mgr_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gts local cache mgr is NULL", KR(ret)); - } else if (NULL == ts_worker_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "gts worker is NULL", KR(ret)); - } else if (OB_FAIL(ts_mgr_->update_gts(result.get_srr(), - result.get_gts_start(), - transaction::TS_SOURCE_GTS, - update))) { - } else if (!update) { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "gts local cache not updated", K(result)); - } - } else { - transaction::ObTsResponseTask *task = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < transaction::ObGtsSource::TOTAL_GTS_QUEUE_COUNT; ++i) { - if (NULL == (task = transaction::ObTsResponseTaskFactory::alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(ERROR, "alloc memory failed", KR(ret), KP(task)); - } else { - if (OB_FAIL(task->init(i, ts_mgr_, transaction::TS_SOURCE_GTS))) { - TRANS_LOG(WARN, "gts task init error", KR(ret), KP(task), K(i), K(result)); - } else if (OB_FAIL(ts_worker_->push_task(task))) { - TRANS_LOG(WARN, "push gts task failed", KR(ret), KP(task), K(result)); - } else { - TRANS_LOG(DEBUG, "push gts task success", KP(task), K(result)); - } - if (OB_SUCCESS != ret) { - transaction::ObTsResponseTaskFactory::free(task); - task = NULL; - } - } - } - } - } - TRANS_LOG(DEBUG, "gts request callback", KR(ret), K(result), K(rcode)); - } - return ret; - } - bool is_inited_; - transaction::ObTsMgr *ts_mgr_; - transaction::ObTsWorker *ts_worker_; -}; - -} // obcall - -namespace transaction -{ - -class ObIGtsRequestRpc -{ -public: - ObIGtsRequestRpc() {} - virtual ~ObIGtsRequestRpc() {} - virtual int start() = 0; - virtual int stop() = 0; - virtual int wait() = 0; - virtual void destroy() = 0; -public: - virtual int post(const common::ObAddr &server, - const ObGtsRequest &msg) = 0; -}; - -class ObGtsRequestRpc : public ObIGtsRequestRpc -{ -public: - ObGtsRequestRpc() : is_inited_(false), is_running_(false), ts_mgr_(NULL) {} - ~ObGtsRequestRpc() { destroy(); } - int init(const common::ObAddr &self, - transaction::ObTsMgr *ts_mgr, - transaction::ObTsWorker *ts_worker); - int start(); - int stop(); - int wait(); - void destroy(); -public: - int post(const common::ObAddr &server, const ObGtsRequest &msg); -private: - bool is_inited_; - bool is_running_; - obcall::ObGtsRPCCB gts_request_cb_; - common::ObAddr self_; - transaction::ObTsMgr *ts_mgr_; -}; - -} // transaction - -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_GTS_RPC_ diff --git a/src/storage/tx/ob_gts_source.cpp b/src/storage/tx/ob_gts_source.cpp deleted file mode 100644 index 4007c4c94..000000000 --- a/src/storage/tx/ob_gts_source.cpp +++ /dev/null @@ -1,646 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gts_source.h" -#include "share/rc/ob_module_provider.h" -#include "ob_trans_service.h" -#include "ob_timestamp_access.h" - -namespace oceanbase -{ -using namespace common; -using namespace share; - -namespace transaction -{ -/////////////////////Implementation of ObGtsStatistics///////////////////////// -void ObGtsStatistics::reset() -{ - last_stat_ts_ = 0; - gts_rpc_cnt_ = 0; - get_gts_cache_cnt_ = 0; - get_gts_with_stc_cnt_ = 0; - try_get_gts_cache_cnt_ = 0; - try_get_gts_with_stc_cnt_ = 0; - wait_gts_elapse_cnt_ = 0; - try_wait_gts_elapse_cnt_ = 0; -} - -int ObGtsStatistics::init() -{ - int ret = OB_SUCCESS; - - last_stat_ts_ = ObTimeUtility::current_time(); - - return ret; -} - -void ObGtsStatistics::statistics() -{ - const int64_t cur_ts = ObTimeUtility::current_time(); - const int64_t last_stat_ts = ATOMIC_LOAD(&last_stat_ts_); - if (cur_ts - last_stat_ts >= STAT_INTERVAL) { - if (ATOMIC_BCAS(&last_stat_ts_, last_stat_ts, cur_ts)) { - TRANS_LOG(INFO, "gts statistics", - "gts_rpc_cnt", ATOMIC_LOAD(>s_rpc_cnt_), - "get_gts_cache_cnt", ATOMIC_LOAD(&get_gts_cache_cnt_), - "get_gts_with_stc_cnt", ATOMIC_LOAD(&get_gts_with_stc_cnt_), - "try_get_gts_cache_cnt", ATOMIC_LOAD(&try_get_gts_cache_cnt_), - "try_get_gts_with_stc_cnt", ATOMIC_LOAD(&try_get_gts_with_stc_cnt_), - "wait_gts_elapse_cnt", ATOMIC_LOAD(&wait_gts_elapse_cnt_), - "try_wait_gts_elapse_cnt", ATOMIC_LOAD(&try_wait_gts_elapse_cnt_)); - ATOMIC_STORE(>s_rpc_cnt_, 0); - ATOMIC_STORE(&get_gts_cache_cnt_, 0); - ATOMIC_STORE(&get_gts_with_stc_cnt_, 0); - ATOMIC_STORE(&try_get_gts_cache_cnt_, 0); - ATOMIC_STORE(&try_get_gts_with_stc_cnt_, 0); - ATOMIC_STORE(&wait_gts_elapse_cnt_, 0); - ATOMIC_STORE(&try_wait_gts_elapse_cnt_, 0); - } - } - -} - -////////////////////////Implementation of ObGtsSource/////////////////////////////////// -void ObGtsSource::reset() -{ - is_inited_ = false; - - gts_local_cache_.reset(); - server_.reset(); - gts_request_rpc_ = NULL; - location_adapter_ = NULL; - for (int64_t i = 0; i < TOTAL_GTS_QUEUE_COUNT; ++i) { - queue_[i].reset(); - } - gts_cache_leader_.reset(); -} - - -int ObGtsSource::init(const ObAddr &server, - ObIGtsRequestRpc *gts_request_rpc, ObILocationAdapter *location_adapter) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "init twice", KR(ret)); - } else if (OB_UNLIKELY(!true) || - OB_UNLIKELY(false) || - OB_UNLIKELY(!server.is_valid()) || - OB_ISNULL(gts_request_rpc) || - OB_ISNULL(location_adapter)) { - TRANS_LOG(WARN, "invalid argument", KR(ret), K(server), - KP(gts_request_rpc), KP(location_adapter)); - ret = OB_INVALID_ARGUMENT; - } else { - for (int64_t i = 0; OB_SUCCESS == ret && i < TOTAL_GTS_QUEUE_COUNT; ++i) { - if (i < GET_GTS_QUEUE_COUNT) { - if (OB_FAIL(queue_[i].init(GET_GTS))) { - TRANS_LOG(WARN, "gts queue init error", KR(ret)); - } - } else if (i < GET_GTS_QUEUE_COUNT + WAIT_GTS_QUEUE_COUNT) { - if (OB_FAIL(queue_[i].init(WAIT_GTS_ELAPSING))) { - TRANS_LOG(WARN, "wait gts elapsing queue init error", KR(ret)); - } - } else { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpect error", KR(ret), K(i)); - } - } - if (OB_SUCCESS == ret) { - - server_ = server; - gts_request_rpc_ = gts_request_rpc; - location_adapter_ = location_adapter; - gts_statistics_.init(); - is_inited_ = true; - TRANS_LOG(INFO, "gts source init success", K(server), KP(this)); - } - } - - return ret; -} - -void ObGtsSource::destroy() -{ - if (is_inited_) { - for (int64_t i = 0; i < TOTAL_GTS_QUEUE_COUNT; i++) { - queue_[i].destroy(); - } - is_inited_ = false; - } -} - -//Get the value of gts cache, no need to be up-to-date -int ObGtsSource::get_gts(ObTsCbTask *task, int64_t >s) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int64_t tmp_gts = 0; - - if (OB_SUCCESS == (ret = gts_local_cache_.get_gts(tmp_gts))) { - //Able to find a suitable gts value - gts = tmp_gts; - } else if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "get gts error", KR(ret), KP(task)); - } else if (NULL == task) { - // do nothing - } else { - //Generate the latest gts value of the task into the queue - const int64_t queue_index = static_cast(task->hash() % GET_GTS_QUEUE_COUNT); - ObGTSTaskQueue *queue = &(queue_[queue_index]); - if (OB_SUCCESS != (tmp_ret = queue->push(task))) { - //The number of queues is sufficient, so failure is not allowed - TRANS_LOG(ERROR, "gts task push error", "ret", tmp_ret, KP(task)); - //overwrite retcode - ret = tmp_ret; - } else { - const bool need_refresh_gts_location = false; - if (OB_SUCCESS != (tmp_ret = refresh_gts_(need_refresh_gts_location))) { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "refresh gts failed", K(tmp_ret)); - } - } - } - } - gts_statistics_.inc_get_gts_cache_cnt(); - - return ret; -} - -int ObGtsSource::get_gts(const MonotonicTs stc, - ObTsCbTask *task, - int64_t >s, - MonotonicTs &receive_gts_ts) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int64_t tmp_gts = 0; - bool need_send_rpc = false; - ObAddr leader; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(!stc.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(stc), KP(task)); - } else if (OB_SUCCESS == (ret = gts_local_cache_.get_gts(stc, - tmp_gts, - receive_gts_ts, - need_send_rpc))) { - //Able to find a suitable gts value - gts = tmp_gts; - } else if (OB_UNLIKELY(OB_EAGAIN != ret)) { - TRANS_LOG(WARN, "get gts error", KR(ret), K(stc), KP(task)); - } else { - TRANS_LOG(DEBUG, "query_gts", KR(ret), K(need_send_rpc), K(stc), - K(gts_local_cache_.get_latest_srr())); - // Single-node lite: GTS leader is always the local timestamp service. Get gts directly. - leader = server_; - UNUSED(need_send_rpc); - MOD_SCOPE { - ret = OB_EAGAIN; - // Here the error code is overwritten by the result of the local call - if (OB_SUCCESS != (tmp_ret = get_gts_from_local_timestamp_service_(leader, gts, receive_gts_ts))) { - if (OB_EAGAIN != tmp_ret) { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "get_gts_from_local_timestamp_service fail", K(leader), K_(server), K(tmp_ret)); - } - } - } else { - ret = OB_SUCCESS; - } - } else { - ret = OB_EAGAIN; - } - // If ret is not OB_SUCCESS, it means that an asynchronous task needs to be added to wait for the subsequent gts value - if (OB_FAIL(ret) && NULL != task) { - // Generate a task to enter the queue, waiting for the latest gts value - const int64_t queue_index = static_cast(task->hash() % GET_GTS_QUEUE_COUNT); - ObGTSTaskQueue *queue = &(queue_[queue_index]); - if (OB_SUCCESS != (tmp_ret = queue->push(task))) { - //The number of queues is sufficient, so failure is not allowed - TRANS_LOG(ERROR, "gts task push error", "ret", tmp_ret, KP(task)); - //overwrite retcode - ret = tmp_ret; - } else { - if (EXECUTE_COUNT_PER_SEC(1)) { - TRANS_LOG(INFO, "push gts task success", K(*task)); - } - } - } - } - - gts_statistics_.inc_get_gts_with_stc_cnt(); - - return ret; -} - -// Get the timestamp from the local timestamp service -int ObGtsSource::get_gts_from_local_timestamp_service_(ObAddr &leader, - int64_t >s, - MonotonicTs &receive_gts_ts) -{ - int ret = OB_SUCCESS; - int64_t tmp_gts = 0; - const MonotonicTs cur_ts = MonotonicTs::current_time(); - - ObTimestampAccess *timestamp_access = share::g_mp->timestamp_access(); - if (OB_ISNULL(timestamp_access)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "timestamp access is null", KR(ret), KP(timestamp_access), K(leader)); - } else if (OB_FAIL(timestamp_access->get_number(tmp_gts))) { - if (EXECUTE_COUNT_PER_SEC(100)) { - TRANS_LOG(WARN, "global_timestamp_service get gts fail", K(leader), K(tmp_gts), KR(ret)); - } - if (OB_NOT_MASTER == ret) { - gts_cache_leader_.reset(); - } - } else { - if (gts_cache_leader_ != leader) { - gts_cache_leader_ = leader; - } - if (OB_FAIL(gts_local_cache_.update_gts_and_check_barrier(cur_ts, - tmp_gts, - cur_ts))) { - TRANS_LOG(WARN, "update gts fail", K(cur_ts), K(gts), K(receive_gts_ts), KR(ret)); - } else { - // get gts success, need to overwrite the OB_EAGAIN error code to OB_SUCCESS - gts = tmp_gts; - receive_gts_ts = cur_ts; - } - } - - return ret; -} - -int ObGtsSource::get_gts_from_local_timestamp_service_(ObAddr &leader, - int64_t >s) -{ - MonotonicTs unused_receive_gts_ts; - return get_gts_from_local_timestamp_service_(leader, gts, unused_receive_gts_ts); -} - - - -int64_t ObGtsSource::get_task_count() const -{ - int64_t task_count = 0; - for (int64_t i = 0; i < TOTAL_GTS_QUEUE_COUNT; i++) { - task_count += queue_[i].get_task_count(); - } - return task_count; -} - -int ObGtsSource::gts_callback_interrupted(const int errcode, const share::ObLSID ls_id) -{ - int ret = OB_SUCCESS; - int64_t task_count = 0; - for (int64_t i = 0; i < TOTAL_GTS_QUEUE_COUNT; i++) { - queue_[i].gts_callback_interrupted(errcode, ls_id); - task_count += queue_[i].get_task_count(); - } - if (OB_LS_OFFLINE != errcode) { - // in this case, all callbacck tasks of this tenant need to be cleared. - if (task_count > 0) { - ret = OB_EAGAIN; - } - } else { - // if OB_LS_OFFLINE, return OB_SUCCESS - } - return ret; -} - -int ObGtsSource::wait_gts_elapse(const int64_t ts, ObTsCbTask *task, bool &need_wait) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(0 >= ts) || OB_ISNULL(task)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(ts), KP(task)); - } else { - int64_t gts = 0; - bool tmp_need_wait = false; - ObAddr leader; - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(gts_local_cache_.get_gts(gts))) { - if (OB_UNLIKELY(OB_EAGAIN != ret)) { - TRANS_LOG(WARN, "get gts failed", K(ret)); - } else { - tmp_need_wait = true; - // rewrite ret - ret = OB_SUCCESS; - } - } else if (ts > gts) { - tmp_need_wait = true; - } else { - tmp_need_wait = false; - } - if (OB_SUCCESS == ret && tmp_need_wait) { - // Single-node lite: GTS leader is always the local timestamp service. - leader = server_; - if (OB_SUCCESS != (tmp_ret = get_gts_from_local_timestamp_service_(leader, gts))) { - if (OB_EAGAIN != tmp_ret && EXECUTE_COUNT_PER_SEC(100)) { - TRANS_LOG(WARN, "get_gts_from_local_timestamp_service fail", K(leader), K_(server), K(tmp_ret)); - } - } else if (ts <= gts) { - tmp_need_wait = false; - } else { - // do nothing - } - } - if (OB_SUCCESS == ret && tmp_need_wait) { - const int64_t index = WAIT_GTS_QUEUE_START_INDEX + task->hash() % WAIT_GTS_QUEUE_COUNT; - if (TOTAL_GTS_QUEUE_COUNT <= index) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "illegal gts queue index", KR(ret), K(index), KP(task)); - } else if (OB_FAIL(queue_[index].push(task))) { - TRANS_LOG(ERROR, "wait queue push task failed", KR(ret), KP(task)); - } else { - TRANS_LOG(INFO, "wait queue push task success", KP(task)); - } - if (OB_SUCCESS == ret) { - // ignore error code - const bool need_refresh_gts_location = false; - if (OB_SUCCESS != (tmp_ret = refresh_gts_(need_refresh_gts_location))) { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "refresh gts failed", K(tmp_ret), K(need_refresh_gts_location)); - } - } - } - } - if (OB_SUCCESS == ret) { - need_wait = tmp_need_wait; - } - } - - gts_statistics_.inc_wait_gts_elapse_cnt(); - - return ret; -} - -int ObGtsSource::wait_gts_elapse(const int64_t ts) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(0 >= ts)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(ts)); - } else { - int64_t gts = 0; - ObAddr leader; - if (OB_FAIL(gts_local_cache_.get_gts(gts))) { - if (OB_UNLIKELY(OB_EAGAIN != ret)) { - TRANS_LOG(WARN, "get gts failed", K(ret)); - } - } else if (ts > gts) { - ret = OB_EAGAIN; - } else { - //do nothing - } - // Local call optimization - if (OB_FAIL(ret)) { - int tmp_ret = OB_SUCCESS; - // Single-node lite: GTS leader is always the local timestamp service. - leader = server_; - if (OB_SUCCESS != (tmp_ret = get_gts_from_local_timestamp_service_(leader, gts))) { - if (OB_EAGAIN != tmp_ret) { - TRANS_LOG(WARN, "get_gts_from_local_timestamp_service fail", K(leader), K_(server), K(tmp_ret)); - } - } else if (ts <= gts) { - ret = OB_SUCCESS; - } else { - // do nothing - } - } - } - gts_statistics_.inc_try_wait_gts_elapse_cnt(); - - return ret; -} - -int ObGtsSource::refresh_gts(const bool need_refresh) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "not inited"); - ret = OB_NOT_INIT; - } else { - ret = refresh_gts_(need_refresh); - // Single-node lite: no gts RPC response drives wait-queue draining anymore. - // Drain here -- refresh_gts() is called ONLY by ObTsMgr::run1 on the - // dedicated T1_TsMgr thread (every 100ms), never on the log-apply thread, - // so firing get_gts / wait_gts_elapse callbacks here cannot re-enter or - // block log apply. foreach_task only pops satisfiable tasks; the rest stay - // queued for the next refresh. - int tmp_ret = OB_SUCCESS; - MonotonicTs drain_srr; - int64_t drain_gts = 0; - MonotonicTs drain_recv_ts; - if (OB_SUCCESS == (tmp_ret = gts_local_cache_.get_srr_and_gts_safe(drain_srr, drain_gts, drain_recv_ts))) { - for (int64_t i = 0; i < TOTAL_GTS_QUEUE_COUNT; ++i) { - (void)queue_[i].foreach_task(drain_srr, drain_gts, drain_recv_ts); - } - } - } - statistics_(); - if (log_interval_.reach()) { - TRANS_LOG(INFO, "refresh gts", KR(ret), K(need_refresh), K_(gts_local_cache)); - } - return ret; -} - -int ObGtsSource::get_gts_leader_(ObAddr &leader) -{ - int ret = OB_SUCCESS; - const int64_t cluster_id = GCONF.cluster_id; -#ifdef ERRSIM - ret = OB_E(EventTable::EN_GET_GTS_LEADER) OB_SUCCESS; - if (OB_LS_LOCATION_LEADER_NOT_EXIST == ret) { - ObClockGenerator::msleep(100); - return ret; - } -#endif - if (gts_cache_leader_.is_valid()) { - leader = gts_cache_leader_; - } else if (OB_FAIL(location_adapter_->nonblock_get_leader(cluster_id, GTS_LS, leader))) { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "gts nonblock get leader failed", K(ret), K(GTS_LS)); - } - } else { - gts_cache_leader_ = leader; - } - if (OB_SUCC(ret) && !leader.is_valid()) { - ret = OB_LS_LOCATION_LEADER_NOT_EXIST; - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "gts cache leader is invalid", KR(ret), K(leader), K(*this)); - } - } - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(sys_ls_leader_addr_, static_cast(leader.get_ipv4() << 31 | leader.get_port())); - - return ret; -} - -int ObGtsSource::query_gts_(const ObAddr &leader) -{ - // Single-node lite: refresh the local gts cache directly from the local - // timestamp service instead of posting a gts request RPC to a remote leader. - int ret = OB_SUCCESS; - int64_t gts = 0; - ObAddr local_leader = server_; - const MonotonicTs srr = MonotonicTs::current_time(); - UNUSED(leader); - if (OB_FAIL(gts_local_cache_.update_latest_srr(srr))) { - TRANS_LOG(WARN, "update latest srr error", KR(ret), K(srr)); - } else { - MOD_SCOPE { - if (OB_FAIL(get_gts_from_local_timestamp_service_(local_leader, gts))) { - if (OB_EAGAIN != ret && EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(WARN, "query gts from local timestamp service fail", KR(ret)); - } - } - } - } - return ret; -} - -int ObGtsSource::refresh_gts_location_() -{ - int ret = OB_SUCCESS; - return ret; -} - -int ObGtsSource::refresh_gts_(const bool need_refresh) -{ - // Single-node lite: just pull a fresh gts from the local timestamp service. - UNUSED(need_refresh); - ObAddr leader = server_; - return query_gts_(leader); -} - -void ObGtsSource::statistics_() -{ - gts_statistics_.statistics(); -} - -int ObGtsSource::update_gts(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts, - bool &update) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(0 >= gts) || OB_UNLIKELY(!receive_gts_ts.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts), K(receive_gts_ts)); - } else if (OB_FAIL(gts_local_cache_.update_gts(srr, gts, receive_gts_ts, update))) { - TRANS_LOG(WARN, "gts local cache update error", KR(ret), K(srr), K(gts), - K(receive_gts_ts), K(update)); - } else { - TRANS_LOG(DEBUG, "gts local cache update success", K(srr), K(gts)); - } - - return ret; -} - -int ObGtsSource::update_gts(const int64_t gts, bool &update) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(0 >= gts)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(gts)); - } else if (OB_FAIL(gts_local_cache_.update_gts(gts, update))) { - TRANS_LOG(WARN, "gts local cache update error", KR(ret), K(gts), K(update)); - } else { - TRANS_LOG(DEBUG, "gts local cache update success", K(gts)); - } - - return ret; -} - -int ObGtsSource::handle_gts_result(const int64_t queue_index) -{ - int ret = OB_SUCCESS; - MonotonicTs srr; - int64_t gts = 0; - MonotonicTs receive_gts_ts; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(!true)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret)); - } else if (OB_FAIL(gts_local_cache_.get_srr_and_gts_safe(srr, gts, receive_gts_ts))) { - TRANS_LOG(WARN, "get srr and gts failed", KR(ret)); - } else { - ObGTSTaskQueue *queue = &(queue_[queue_index]); - if (OB_FAIL(queue->foreach_task(srr, gts, receive_gts_ts))) { - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - if (gts_local_cache_.no_rpc_on_road()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = refresh_gts_(false))) { - TRANS_LOG(WARN, "refresh gts failed", K(tmp_ret)); - } - } - } else { - TRANS_LOG(WARN, "iterate task failed", KR(ret), K(queue_index)); - } - } - } - return ret; -} - -int ObGtsSource::handle_gts_err_response(const ObGtsErrResponse &err_msg) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(!err_msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(err_msg)); - } else { - if (OB_NOT_MASTER == err_msg.get_status()) { - gts_cache_leader_.reset(); - refresh_gts_location_(); - } - } - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "handle gts err response", KR(ret), K(err_msg), K(*this)); - } - - return ret; -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_gts_source.h b/src/storage/tx/ob_gts_source.h deleted file mode 100644 index b7ae6fbb4..000000000 --- a/src/storage/tx/ob_gts_source.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_SOURCE_ -#define OCEANBASE_TRANSACTION_OB_GTS_SOURCE_ - -#include "lib/net/ob_addr.h" -#include "lib/utility/utility.h" -#include "ob_gts_msg.h" -#include "ob_gts_local_cache.h" -#include "ob_gts_task_queue.h" -//#include "ob_ts_worker.h" -#include "ob_i_ts_source.h" - -namespace oceanbase -{ -namespace obcall -{ -class ObGtsRpcResult; -} -namespace transaction -{ -class ObILocationAdapter; -class ObTsCbTask; -class ObIGtsRequestRpc; -} -namespace transaction -{ - -class ObGtsStatistics -{ - static const int64_t STAT_INTERVAL = 5 * 1000 * 1000; -public: - ObGtsStatistics() { reset(); } - ~ObGtsStatistics() {} - int init(); - void reset(); - void inc_gts_rpc_cnt() { ATOMIC_INC(>s_rpc_cnt_); } - void inc_get_gts_cache_cnt() { ATOMIC_INC(&get_gts_cache_cnt_); } - void inc_get_gts_with_stc_cnt() { ATOMIC_INC(&get_gts_with_stc_cnt_); } - void inc_try_get_gts_cache_cnt() { ATOMIC_INC(&try_get_gts_cache_cnt_); } - void inc_try_get_gts_with_stc_cnt() { ATOMIC_INC(&try_get_gts_with_stc_cnt_); } - void inc_wait_gts_elapse_cnt() { ATOMIC_INC(&wait_gts_elapse_cnt_); } - void inc_try_wait_gts_elapse_cnt() { ATOMIC_INC(&try_wait_gts_elapse_cnt_); } - void statistics(); -private: - int64_t last_stat_ts_; - int64_t gts_rpc_cnt_; - - int64_t get_gts_cache_cnt_; - int64_t get_gts_with_stc_cnt_; - int64_t try_get_gts_cache_cnt_; - int64_t try_get_gts_with_stc_cnt_; - - int64_t wait_gts_elapse_cnt_; - int64_t try_wait_gts_elapse_cnt_; -}; - -class ObGtsSource -{ -public: - ObGtsSource() : log_interval_(3 * 1000 * 1000), refresh_location_interval_(100 * 1000) { reset(); } - ~ObGtsSource() { destroy(); } - int init(const common::ObAddr &server, ObIGtsRequestRpc *gts_request_rpc, - ObILocationAdapter *location_adapter); - void destroy(); - void reset(); - - int handle_gts_err_response(const ObGtsErrResponse &msg); - int handle_gts_result(const int64_t queue_index); - int update_gts(const MonotonicTs srr, const int64_t gts, const MonotonicTs receive_gts_ts, bool &update); - int64_t get_task_count() const; - int gts_callback_interrupted(const int errcode, const share::ObLSID ls_id); -public: - int update_gts(const int64_t gts, bool &update); - int get_gts(const MonotonicTs stc, ObTsCbTask *task, int64_t >s, MonotonicTs &receive_gts_ts); - int get_gts(ObTsCbTask *task, int64_t >s); - int wait_gts_elapse(const int64_t ts, ObTsCbTask *task, bool &need_wait); - int wait_gts_elapse(const int64_t ts); - int refresh_gts(const bool need_refresh); - bool is_external_consistent() { return true; } - int refresh_gts_location() { return refresh_gts_location_(); } - TO_STRING_KV(K_(gts_local_cache), K_(server), K_(gts_cache_leader)); -private: - int get_gts_leader_(common::ObAddr &leader); - int refresh_gts_location_(); - int refresh_gts_(const bool need_refresh); - int query_gts_(const common::ObAddr &leader); - void statistics_(); - int get_gts_from_local_timestamp_service_(common::ObAddr &leader, - int64_t >s, - MonotonicTs &receive_gts_ts); - int get_gts_from_local_timestamp_service_(common::ObAddr &leader, - int64_t >s); -public: - static const int64_t GET_GTS_QUEUE_COUNT = 1; - static const int64_t WAIT_GTS_QUEUE_COUNT = 1; - static const int64_t WAIT_GTS_QUEUE_START_INDEX = GET_GTS_QUEUE_COUNT; - static const int64_t TOTAL_GTS_QUEUE_COUNT = GET_GTS_QUEUE_COUNT + WAIT_GTS_QUEUE_COUNT; -private: - bool is_inited_; - - ObGTSLocalCache gts_local_cache_; - ObGTSTaskQueue queue_[TOTAL_GTS_QUEUE_COUNT]; - common::ObAddr server_; - ObIGtsRequestRpc *gts_request_rpc_; - ObILocationAdapter *location_adapter_; - // statistics - ObGtsStatistics gts_statistics_; - common::ObTimeInterval log_interval_; - common::ObAddr gts_cache_leader_; - common::ObTimeInterval refresh_location_interval_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_RANSACTION_OB_GTS_SOURCE_ diff --git a/src/storage/tx/ob_gts_task_queue.cpp b/src/storage/tx/ob_gts_task_queue.cpp deleted file mode 100644 index b9f1a92b3..000000000 --- a/src/storage/tx/ob_gts_task_queue.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_gts_task_queue.h" -#include "ob_ts_mgr.h" -#include "ob_trans_event.h" - -namespace oceanbase -{ - -using namespace common; -using namespace share; - -namespace transaction -{ - -int ObGTSTaskQueue::init(const ObGTSCacheTaskType &type) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "gts task queue init twice", KR(ret), K(type)); - } else if (GET_GTS != type && WAIT_GTS_ELAPSING != type) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid gts task type", KR(ret), K(type)); - } else { - task_type_ = type; - is_inited_ = true; - TRANS_LOG(INFO, "gts task queue init success", KP(this), K(type)); - } - return ret; -} - -void ObGTSTaskQueue::destroy() -{ - if (is_inited_) { - is_inited_ = false; - } -} - -void ObGTSTaskQueue::reset() -{ - is_inited_ = false; - task_type_ = INVALID_GTS_TASK_TYPE; -} - -int ObGTSTaskQueue::foreach_task(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - } else if (0 >= srr.mts_ || 0 >= gts) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts)); - } else { - - MAKE_TENANT_SWITCH_SCOPE_GUARD(ts_guard); - while (OB_SUCCESS == ret) { - common::ObLink *data = NULL; - (void)queue_.pop(data); - ObTsCbTask *task = static_cast(data); - if (NULL == task) { - break; - } else { - - const int64_t request_ts = 0/*task->get_request_ts()*/; - - if (OB_SUCC(ret)) { - SCN scn; - scn.convert_for_gts(gts); - if (GET_GTS == task_type_) { - if (OB_FAIL(task->get_gts_callback(srr, scn, receive_gts_ts))) { - if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "get gts callback failed", KR(ret), K(srr), K(gts), KP(task)); - } - } else { - TRANS_LOG(DEBUG, "get gts callback success", K(srr), K(gts), KP(task)); - } - } else if (WAIT_GTS_ELAPSING == task_type_) { - if (OB_FAIL(task->gts_elapse_callback(srr, scn))) { - if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "gts elapse callback failed", KR(ret), K(srr), K(gts), KP(task)); - } - } else { - TRANS_LOG(DEBUG, "gts elapse callback success", K(srr), K(gts), KP(task)); - } - } else { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unknown gts task type", KR(ret), K_(task_type)); - } - if (OB_EAGAIN == ret) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = queue_.push(task))) { - ret = tmp_ret; - TRANS_LOG(ERROR, "push gts task failed", KR(ret), KP(task)); - } else { - TRANS_LOG(DEBUG, "push back gts task", KP(task)); - break; - } - } else { - if (GET_GTS == task_type_) { - const int64_t total_used = ObTimeUtility::current_time() - request_ts; - ObTransStatistic::get_instance().add_gts_acquire_total_time( total_used); - ObTransStatistic::get_instance().add_gts_acquire_total_wait_count( 1); - } else if (WAIT_GTS_ELAPSING == task_type_) { - const int64_t total_used = ObTimeUtility::current_time() - request_ts; - ObTransStatistic::get_instance().add_gts_wait_elapse_total_time( total_used); - ObTransStatistic::get_instance().add_gts_wait_elapse_total_wait_count( 1); - } else { - // do nothing - } - } - } - } - } - } - return ret; -} - -int ObGTSTaskQueue::push(ObTsCbTask *task) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", KR(ret)); - } else if (NULL == task) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(task)); - } else if (OB_FAIL(queue_.push(task))) { - TRANS_LOG(ERROR, "push gts task failed", K(ret), KP(task)); - } else { - TRANS_LOG(DEBUG, "push gts task success", KP(task)); - } - return ret; -} - -int ObGTSTaskQueue::gts_callback_interrupted(const int errcode, const share::ObLSID ls_id) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - } else { - int64_t count = queue_.size(); - int64_t again_count = 0; - int64_t error_count = 0; - TRANS_LOG(INFO, "interrupt gts callback start", K(count), K(errcode), K(ls_id)); - while (OB_SUCC(ret) && count > 0) { - common::ObLink *data = NULL; - (void)queue_.pop(data); - count--; - ObTsCbTask *task = static_cast(data); - if (NULL == task) { - break; - } else { - - MOD_SCOPE { - if (OB_FAIL(task->gts_callback_interrupted(errcode, ls_id))) { - if (OB_EAGAIN != ret) { - error_count++; - // in this case, need restart the observer - TRANS_LOG(ERROR, "gts callback interrupted fail", KR(ret), KP(task)); - } else { - again_count++; - // case 1, errcode is OB_LS_OFFLINE and ls_id is not equal to ls_id in part ctx - // case 2, errcode is OB_LS_OFFLINE and task is an object of ObTsSyncGetTsCbTask - // return OB_EAGAIN - if (OB_FAIL(queue_.push(task))) { - // since task is not null, this failure is impossible - TRANS_LOG(ERROR, "push gts task failed", KR(ret), KP(task)); - } else { - TRANS_LOG(DEBUG, "push back gts task", KP(task)); - } - } - } else { - TRANS_LOG(DEBUG, "gts callback interrupted success", KP(task)); - } - } else { - error_count++; - TRANS_LOG(ERROR, "switch tenant failed", KR(ret), K(errcode), K(ls_id)); - } - } - } - TRANS_LOG(INFO, "interrupt gts callback end", K(again_count), K(error_count)); - } - return ret; -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_gts_task_queue.h b/src/storage/tx/ob_gts_task_queue.h deleted file mode 100644 index 16e2b5e6e..000000000 --- a/src/storage/tx/ob_gts_task_queue.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_GTS_TASK_QUEUE_ -#define OCEANBASE_TRANSACTION_OB_GTS_TASK_QUEUE_ - -#include "ob_gts_define.h" -#include "share/ob_errno.h" -#include "lib/utility/utility.h" -#include "lib/queue/ob_link_queue.h" -#include "lib/hash/ob_link_hashmap.h" - -namespace oceanbase -{ -namespace transaction -{ -class ObTsCbTask; - -class ObGTSTaskQueue -{ -public: - ObGTSTaskQueue() : is_inited_(false), task_type_(INVALID_GTS_TASK_TYPE) {} - ~ObGTSTaskQueue() { destroy(); } - int init(const ObGTSCacheTaskType &type); - void destroy(); - void reset(); - int foreach_task(const MonotonicTs srr, - const int64_t gts, - const MonotonicTs receive_gts_ts); - int push(ObTsCbTask *task); - int64_t get_task_count() const { return queue_.size(); } - int gts_callback_interrupted(const int errcode, const share::ObLSID ls_id); -private: - static const int64_t TOTAL_WAIT_TASK_NUM = 500 * 1000; -private: - bool is_inited_; - ObGTSCacheTaskType task_type_; - common::ObLinkQueue queue_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_GTS_TASK_QUEUE_ diff --git a/src/storage/tx/ob_i_ts_source.h b/src/storage/tx/ob_i_ts_source.h deleted file mode 100644 index f01dca240..000000000 --- a/src/storage/tx/ob_i_ts_source.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_I_TS_SOURCE_ -#define OCEANBASE_TRANSACTION_OB_I_TS_SOURCE_ - -#include -#include "lib/utility/ob_print_utils.h" -#include "ob_trans_define.h" - -namespace oceanbase -{ -namespace transaction -{ -class ObTsCbTask; - -enum -{ - TS_SOURCE_UNKNOWN = -1, - TS_SOURCE_GTS = 0, - MAX_TS_SOURCE, -}; - -OB_INLINE bool is_valid_ts_source(const int ts_type) -{ - return TS_SOURCE_UNKNOWN < ts_type && ts_type < MAX_TS_SOURCE; -} - -inline bool is_ts_type_external_consistent(const int64_t ts_type) -{ - return ts_type == TS_SOURCE_GTS; -} - -class ObTsParam -{ -public: - ObTsParam() { reset(); } - ~ObTsParam() { destroy(); } - void reset() { need_inc_ = true; } - void destroy() { reset(); } - void set_need_inc(const bool need_inc) { need_inc_ = need_inc; } - bool need_inc() const { return need_inc_; } -private: - // Whether the gts value needs to be +1, it is added by default in gts cache management - bool need_inc_; -}; - -} -}//end of namespace oceanbase - -#endif //OCEANBASE_TRANSACTION_OB_I_TS_SOURCE_ diff --git a/src/storage/tx/ob_id_service.cpp b/src/storage/tx/ob_id_service.cpp index 9c47d0a89..49b5e723b 100644 --- a/src/storage/tx/ob_id_service.cpp +++ b/src/storage/tx/ob_id_service.cpp @@ -19,7 +19,6 @@ #include "ob_timestamp_service.h" #include "ob_trans_id_service.h" #include "storage/tx_storage/ob_ls_service.h" -#include "sql/das/ob_das_id_service.h" namespace oceanbase { @@ -70,32 +69,18 @@ int ObIDService::check_and_fill_ls() { int ret = OB_SUCCESS; if (OB_ISNULL(ls_)) { - ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle handle; ObLS *ls = nullptr; - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "log stream service is NULL", K(ret), K(IDS_LS)); - } else if (OB_FAIL(ls_svr->get_ls(IDS_LS, handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get id service log stream failed", K(ret), K(IDS_LS)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "id service log stream not exist", K(ret), K(IDS_LS)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + TRANS_LOG(WARN, "get id service storage failed", K(ret)); } else { ls_ = ls; - TRANS_LOG(INFO, "ls set success", K(IDS_LS), KP(ls_)); + TRANS_LOG(INFO, "ls set success", KP(ls_)); } } return ret; } -void ObIDService::reset_ls() -{ - WLockGuard guard(rwlock_); - ls_ = NULL; -} - int ObIDService::submit_log_(const int64_t last_id, const int64_t limited_id) { int ret = OB_SUCCESS; @@ -173,8 +158,8 @@ int ObIDService::handle_submit_callback(const bool success, const int64_t limite (void)inc_update(&limited_id_, limited_id); rec_log_ts_.atomic_set(log_ts); latest_log_ts_.atomic_set(log_ts); - if (OB_FAIL(update_ls_id_meta(false))) { - TRANS_LOG(WARN, "update id meta of ls meta fail", K(ret), K(service_type_), K(limited_id), K(log_ts)); + if (OB_FAIL(update_id_meta(false))) { + TRANS_LOG(WARN, "update id meta fail", K(ret), K(service_type_), K(limited_id), K(log_ts)); } } else { // do nothing @@ -227,8 +212,8 @@ int ObIDService::handle_replay_result(const int64_t last_id, const int64_t limit if (log_ts > latest_log_ts_.atomic_get()) { rec_log_ts_.atomic_set(log_ts); latest_log_ts_.atomic_set(log_ts); - if (OB_FAIL(update_ls_id_meta(false))) { - TRANS_LOG(WARN, "update id meta of ls meta fail", K(ret), K(service_type_), K(last_id), K(limited_id), K(log_ts)); + if (OB_FAIL(update_id_meta(false))) { + TRANS_LOG(WARN, "update id meta fail", K(ret), K(service_type_), K(last_id), K(limited_id), K(log_ts)); } } } @@ -236,7 +221,7 @@ int ObIDService::handle_replay_result(const int64_t last_id, const int64_t limit return ret; } -int ObIDService::update_ls_id_meta(const bool write_slog) +int ObIDService::update_id_meta(const bool write_slog) { int ret = OB_SUCCESS; @@ -268,8 +253,8 @@ int ObIDService::flush(SCN &rec_scn) SCN latest_rec_log_ts = rec_log_ts_.atomic_get(); if (latest_rec_log_ts <= rec_scn) { latest_rec_log_ts = rec_log_ts_.atomic_get(); - if (OB_FAIL(update_ls_id_meta(true))) { - TRANS_LOG(WARN, "update id meta of ls meta fail", K(ret), K(service_type_)); + if (OB_FAIL(update_id_meta(true))) { + TRANS_LOG(WARN, "update id meta fail", K(ret), K(service_type_)); } else { rec_log_ts_.atomic_bcas(latest_rec_log_ts, SCN::max_scn()); } @@ -279,25 +264,6 @@ int ObIDService::flush(SCN &rec_scn) return ret; } -int ObIDService::check_leader(bool &leader) -{ - int ret = OB_SUCCESS; - common::ObRole role = common::ObRole::INVALID_ROLE; - int64_t proposal_id = 0; - - if (OB_FAIL(check_and_fill_ls())) { - TRANS_LOG(WARN, "ls set fail", K(ret)); - } else if (OB_FAIL(ls_->get_log_handler()->get_role(role, proposal_id))) { - TRANS_LOG(WARN, "get ls role fail", K(ret)); - } else if (common::ObRole::LEADER == role) { - leader = true; - } else { - leader = false; - } - - return ret; -} - SCN ObIDService::get_rec_scn() { const SCN rec_log_ts = rec_log_ts_.atomic_get(); @@ -305,84 +271,37 @@ SCN ObIDService::get_rec_scn() return rec_log_ts; } -int ObIDService::switch_to_follower_gracefully() -{ - // int ret = OB_SUCCESS; - // const int64_t expire_time = ObTimeUtility::current_time() + 100000; - // int locked = false; - // if (OB_SUCC(rwlock_.wrlock(expire_time))) { - // locked = true; - // } - // while (is_logging_ && locked) { - // rwlock_.unlock(); - // locked = false; - // ObClockGenerator::usleep(1000); - // if (ObTimeUtility::current_time() > expire_time) { - // break; - // } - // if (OB_SUCC(rwlock_.wrlock(expire_time))) { - // locked = true; - // } - // } - // if (locked) { - // const int64_t last_id = ATOMIC_LOAD(&last_id_); - // const int64_t limited_id = ATOMIC_LOAD(&limited_id_); - // // Caution: set limit id before submit log, make sure limit id <= last id - // ATOMIC_STORE(&limited_id_, last_id); - // //Submit log but do not require success to prevent blocking role switch - // submit_log_(ATOMIC_LOAD(&last_id_), limited_id); - // rwlock_.unlock(); - // locked = false; - // } - // TRANS_LOG(INFO, "switch to follower gracefully", K(ret), K(service_type_), K(locked)); - TRANS_LOG(INFO, "switch to follower gracefully", K(last_id_), K(limited_id_), K(service_type_)); - // There is no failure scenario for ObIDService to switch the follower, - // so return OB_SUCCESS directly. - return OB_SUCCESS; -} - int ObIDService::get_number(const int64_t range, const int64_t base_id, int64_t &start_id, int64_t &end_id) { int ret = OB_SUCCESS; - bool leader = false; - if (OB_FAIL(check_leader(leader))) { - TRANS_LOG(WARN, "check leader fail", K(ret)); - } else if (!leader) { - ret = OB_NOT_MASTER; - if (EXECUTE_COUNT_PER_SEC(10)) { - TRANS_LOG(WARN, "id service is not leader", K(ret), K_(service_type)); - } + int64_t tmp_id = 0; + const int64_t last_id = ATOMIC_LOAD(&last_id_); + int64_t limit_id = ATOMIC_LOAD(&limited_id_); + const int64_t allocated_range = min(min(limit_id - base_id, limit_id - last_id), range); + if (allocated_range <= 0) { + ret = OB_EAGAIN; } else { - int64_t tmp_id = 0; - const int64_t last_id = ATOMIC_LOAD(&last_id_); - int64_t limit_id = ATOMIC_LOAD(&limited_id_); - const int64_t allocated_range = min(min(limit_id - base_id, limit_id - last_id), range); - if (allocated_range <= 0) { - ret = OB_EAGAIN; - } else { - if (base_id > last_id) { - if (ATOMIC_BCAS(&last_id_, last_id, base_id + allocated_range)) { - tmp_id = base_id; - } else { - tmp_id = ATOMIC_FAA(&last_id_, allocated_range); - } + if (base_id > last_id) { + if (ATOMIC_BCAS(&last_id_, last_id, base_id + allocated_range)) { + tmp_id = base_id; } else { tmp_id = ATOMIC_FAA(&last_id_, allocated_range); } - // Caution: get limit id again, compete with switch_to_follower_gracefully - limit_id = ATOMIC_LOAD(&limited_id_); - if (tmp_id >= limit_id) { - ret = OB_EAGAIN; - } else { - start_id = tmp_id; - end_id = min(start_id + allocated_range, limit_id); - } + } else { + tmp_id = ATOMIC_FAA(&last_id_, allocated_range); } - if (OB_EAGAIN == ret || (limited_id_ - last_id_) < (pre_allocated_range_ * 2 / 3)) { - const int64_t pre_allocated_id = min(max_pre_allocated_id_(base_id), max(base_id, limited_id_) + max(range * 10, pre_allocated_range_)); - submit_log_with_lock_(pre_allocated_id, pre_allocated_id); + limit_id = ATOMIC_LOAD(&limited_id_); + if (tmp_id >= limit_id) { + ret = OB_EAGAIN; + } else { + start_id = tmp_id; + end_id = min(start_id + allocated_range, limit_id); } } + if (OB_EAGAIN == ret || (limited_id_ - last_id_) < (pre_allocated_range_ * 2 / 3)) { + const int64_t pre_allocated_id = min(max_pre_allocated_id_(base_id), max(base_id, limited_id_) + max(range * 10, pre_allocated_range_)); + submit_log_with_lock_(pre_allocated_id, pre_allocated_id); + } // thread local, 4 seconds // TODO, tenant level if (TC_REACH_TIME_INTERVAL(4000000)) { @@ -406,9 +325,8 @@ int64_t ObIDService::max_pre_allocated_id_(const int64_t base_id) void ObIDService::get_virtual_info(int64_t &last_id, int64_t &limited_id, SCN &rec_scn, SCN &latest_log_ts, int64_t &pre_allocated_range, - int64_t &submit_log_ts, bool &is_master) + int64_t &submit_log_ts) { - int ret = OB_SUCCESS; RLockGuard guard(rwlock_); last_id = last_id_; limited_id = limited_id_; @@ -416,11 +334,8 @@ void ObIDService::get_virtual_info(int64_t &last_id, int64_t &limited_id, SCN &r latest_log_ts = latest_log_ts_; pre_allocated_range = pre_allocated_range_; submit_log_ts = submit_log_ts_; - if (OB_FAIL(check_leader(is_master))) { - is_master = false; - } TRANS_LOG(INFO, "id service get virtual info", K_(last_id), K_(limited_id), K_(rec_log_ts), - K_(latest_log_ts), K_(pre_allocated_range), K_(submit_log_ts), K(is_master), K(ret)); + K_(latest_log_ts), K_(pre_allocated_range), K_(submit_log_ts)); } void ObIDService::get_all_id_service_type(int64_t *service_type) @@ -440,9 +355,6 @@ int ObIDService::get_id_service(const int64_t id_service_type, ObIDService *&id_ case transaction::ObIDService::TransIDService: id_service = (ObIDService *)share::g_mp->trans_id_service(); break; - case transaction::ObIDService::DASIDService: - id_service = (ObIDService *)share::g_mp->dasid_service(); - break; default: ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "get wrong id_service_type", K(ret), K(id_service_type)); @@ -518,16 +430,6 @@ int ObPresistIDLogCb::serialize_ls_log(ObPresistIDLog &ls_log, int64_t service_t } break; } - case ObIDService::ServiceType::DASIDService: { - logservice::ObLogBaseHeader - base_header(logservice::ObLogBaseType::DAS_ID_LOG_BASE_TYPE, - logservice::ObReplayBarrierType::NO_NEED_BARRIER, - service_type); - if (OB_FAIL(base_header.serialize(log_buf_, MAX_LOG_BUFF_SIZE, pos_))) { - TRANS_LOG(WARN, "ObPresistIDLogCb serialize base header error", KR(ret), KP(log_buf_), K(pos_)); - } - break; - } default: { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unknown service type", K(service_type)); @@ -575,20 +477,6 @@ int ObPresistIDLogCb::on_success() } break; } - case ObIDService::ServiceType::DASIDService: { - sql::ObDASIDService *das_id_service = nullptr; - if (OB_ISNULL(das_id_service = share::g_mp->dasid_service())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "das id service is null", K(ret)); - } else { - das_id_service->test_lock(); - if (OB_FAIL(das_id_service->handle_submit_callback(true, limited_id_, log_ts_))) { - TRANS_LOG(WARN, "das id service handle log callback fail", K(ret), K_(limited_id), K_(log_ts)); - } - TRANS_LOG(INFO, "das id service handle log callback", K(ret), K_(limited_id), K_(log_ts)); - } - break; - } default: { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unknown service type", K(*this)); @@ -630,19 +518,6 @@ int ObPresistIDLogCb::on_failure() } break; } - case ObIDService::ServiceType::DASIDService: { - sql::ObDASIDService *das_id_service = nullptr; - if (OB_ISNULL(das_id_service = share::g_mp->dasid_service())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "das id service is null", K(ret)); - } else { - if (OB_FAIL(das_id_service->handle_submit_callback(false, limited_id_, log_ts_))) { - TRANS_LOG(WARN, "das id service handle log callback fail", K(ret), K_(limited_id), K_(log_ts)); - } - TRANS_LOG(INFO, "das id service handle log callback", K(ret), K_(limited_id), K_(log_ts)); - } - break; - } default: { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unknown service type", K(*this)); diff --git a/src/storage/tx/ob_id_service.h b/src/storage/tx/ob_id_service.h index c350f0528..19e99b336 100644 --- a/src/storage/tx/ob_id_service.h +++ b/src/storage/tx/ob_id_service.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_TRANSACTION_OB_ID_SERVICE_ #define OCEANBASE_TRANSACTION_OB_ID_SERVICE_ -#include "share/ob_ls_id.h" #include "storage/slog/ob_storage_log_struct.h" #include "logservice/ob_append_callback.h" #include "logservice/ob_log_base_type.h" @@ -85,8 +84,7 @@ class ObPresistIDLogCb : public logservice::AppendCb }; class ObIDService : public logservice::ObIReplaySubHandler, - public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler + public logservice::ObICheckpointSubHandler { public: ObIDService() : rwlock_(ObLatchIds::ID_SOURCE_LOCK), log_interval_(100 * 1000) { reset(); } @@ -98,7 +96,6 @@ class ObIDService : public logservice::ObIReplaySubHandler, INVALID_ID_SERVICE_TYPE = -1, TimestampService, TransIDService, - DASIDService, MAX_SERVICE_TYPE, }; @@ -120,21 +117,13 @@ class ObIDService : public logservice::ObIReplaySubHandler, // for clog replay int replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, const share::SCN &log_ts); - // Switch main - int switch_to_follower_gracefully(); - void switch_to_follower_forcedly() {} - int resume_leader() { return OB_SUCCESS; } - int switch_to_leader() { return OB_SUCCESS; } - - int check_leader(bool &leader); int check_and_fill_ls(); - void reset_ls(); void update_limited_id(const int64_t limited_id, const share::SCN latest_log_ts); - int update_ls_id_meta(const bool write_slog); + int update_id_meta(const bool write_slog); // vtable void get_virtual_info(int64_t &last_id, int64_t &limited_id, share::SCN &rec_log_ts, share::SCN &latest_log_ts, int64_t &pre_allocated_range, - int64_t &submit_log_ts, bool &is_master); + int64_t &submit_log_ts); static const int64_t SUBMIT_LOG_ALARM_INTERVAL = 100 * 1000; static const int64_t MIN_LAST_ID = 1; protected: @@ -161,7 +150,7 @@ class ObIDService : public logservice::ObIReplaySubHandler, // current time when submit log int64_t submit_log_ts_; mutable common::SpinRWLock rwlock_; - ObLS *ls_; + storage::ObLS *ls_; ObPresistIDLogCb cb_; common::ObAddr self_; common::ObTimeInterval log_interval_; diff --git a/src/storage/tx/ob_keep_alive_ls_handler.cpp b/src/storage/tx/ob_keep_alive_ls_handler.cpp index 62d7f7a3c..b48362daf 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.cpp +++ b/src/storage/tx/ob_keep_alive_ls_handler.cpp @@ -38,8 +38,7 @@ int64_t ObKeepAliveLogBody::get_max_serialize_size() return max_log_body.get_serialize_size(); } -int ObKeepAliveLSHandler::init(const ObLSID &ls_id, - logservice::ObLogHandler *log_handler_ptr) +int ObKeepAliveLSHandler::init(logservice::ObLogHandler *log_handler_ptr) { int ret = OB_SUCCESS; logservice::ObLogBaseHeader base_header(ObLogBaseType::KEEP_ALIVE_LOG_BASE_TYPE, @@ -47,7 +46,7 @@ int ObKeepAliveLSHandler::init(const ObLSID &ls_id, submit_buf_len_ = base_header.get_serialize_size() + ObKeepAliveLogBody::get_max_serialize_size(); submit_buf_pos_ = 0; - if (OB_ISNULL(log_handler_ptr) || !ls_id.is_valid()) { + if (OB_ISNULL(log_handler_ptr)) { ret = OB_INVALID_ARGUMENT; } else if (OB_NOT_NULL(log_handler_ptr_)) { ret = OB_INIT_TWICE; @@ -57,9 +56,7 @@ int ObKeepAliveLSHandler::init(const ObLSID &ls_id, TRANS_LOG(WARN, "[Keep Alive] submit_buf alloc failed", K(ret), KP(submit_buf_), K(base_header)); } else { - ls_id_ = ls_id; is_busy_ = false; - is_master_ = false; is_stopped_ = false; log_handler_ptr_ = log_handler_ptr; stat_info_.reset(); @@ -86,11 +83,8 @@ void ObKeepAliveLSHandler::reset() submit_buf_pos_ = 0; log_handler_ptr_ = nullptr; is_busy_ = false; - is_master_ = false; is_stopped_ = false; last_gts_.set_min(); - sys_ls_end_scn_.set_min(); - ls_id_.reset(); tmp_keep_alive_info_.reset(); durable_keep_alive_info_.reset(); stat_info_.reset(); @@ -99,7 +93,7 @@ void ObKeepAliveLSHandler::reset() void ObKeepAliveLSHandler::clear_keep_alive_smaller_scn_info() { SpinWLockGuard guard(lock_); - FLOG_INFO("[Keep Alive] clear keep alive ls info", K(ls_id_), K(tmp_keep_alive_info_), K(durable_keep_alive_info_)); + FLOG_INFO("[Keep Alive] clear keep alive info", K(tmp_keep_alive_info_), K(durable_keep_alive_info_)); tmp_keep_alive_info_.reset(); durable_keep_alive_info_.reset(); } @@ -115,9 +109,6 @@ int ObKeepAliveLSHandler::try_submit_log(const SCN &min_start_scn, MinStartScnSt if (OB_ISNULL(log_handler_ptr_)) { stat_info_.other_error_cnt += 1; ret = OB_NOT_INIT; - } else if (!ATOMIC_LOAD(&is_master_)) { - stat_info_.not_master_cnt += 1; - // ret = OB_NOT_MASTER; } else if (ATOMIC_LOAD(&is_busy_)) { stat_info_.cb_busy_cnt += 1; // ret = OB_TX_NOLOGCB; @@ -126,25 +117,25 @@ int ObKeepAliveLSHandler::try_submit_log(const SCN &min_start_scn, MinStartScnSt // ret = OB_OP_NOT_ALLOW; } else { ATOMIC_STORE(&is_busy_, true); - share::SCN ref_scn = get_ref_scn_(); + const share::SCN ref_scn = last_gts_; if (ATOMIC_LOAD(&is_stopped_)) { ATOMIC_STORE(&is_busy_, false); TRANS_LOG(INFO, "ls hash stopped", K(ret)); } else if (OB_FAIL(serialize_keep_alive_log_(min_start_scn, min_start_status))) { ATOMIC_STORE(&is_busy_, false); - TRANS_LOG(WARN, "[Keep Alive] serialize keep alive log failed", K(ret), K(ls_id_)); + TRANS_LOG(WARN, "[Keep Alive] serialize keep alive log failed", K(ret)); } else if (OB_FAIL(log_handler_ptr_->append(submit_buf_, submit_buf_pos_, ref_scn, true, false/*allow_compression*/, this, lsn, scn))) { stat_info_.other_error_cnt += 1; ATOMIC_STORE(&is_busy_, false); - TRANS_LOG(WARN, "[Keep Alive] submit keep alive log failed", K(ret), K(ls_id_)); + TRANS_LOG(WARN, "[Keep Alive] submit keep alive log failed", K(ret)); } else { stat_info_.submit_succ_cnt += 1; tmp_keep_alive_info_.loop_job_succ_scn_ = scn; tmp_keep_alive_info_.lsn_ = lsn; tmp_keep_alive_info_.min_start_status_ = min_start_status; tmp_keep_alive_info_.min_start_scn_ = min_start_scn; - TRANS_LOG(DEBUG, "[Keep Alive] submit keep alive log success", K(ret), K(ls_id_), + TRANS_LOG(DEBUG, "[Keep Alive] submit keep alive log success", K(ret), K(tmp_keep_alive_info_), K(min_start_scn), K(min_start_status)); } } @@ -212,9 +203,7 @@ void ObKeepAliveLSHandler::print_stat_info() { SpinRLockGuard guard(lock_); ObCStringHelper helper; - TRANS_LOG(INFO, "[Keep Alive Stat] LS Keep Alive Info", - "LS_ID", ls_id_, - "Not_Master_Cnt", stat_info_.not_master_cnt, + TRANS_LOG(INFO, "[Keep Alive Stat] Keep Alive Info", "Near_To_GTS_Cnt", stat_info_.near_to_gts_cnt, "Other_Error_Cnt", stat_info_.other_error_cnt, "Submit_Succ_Cnt", stat_info_.submit_succ_cnt, @@ -222,8 +211,7 @@ void ObKeepAliveLSHandler::print_stat_info() "last_lsn", stat_info_.stat_keepalive_info_.lsn_, "last_gts", last_gts_, "min_start_scn", helper.convert(stat_info_.stat_keepalive_info_.min_start_scn_), - "min_start_status", stat_info_.stat_keepalive_info_.min_start_status_, - "sys_ls_end_scn", sys_ls_end_scn_); + "min_start_status", stat_info_.stat_keepalive_info_.min_start_status_); stat_info_.clear_cnt(); } @@ -243,25 +231,17 @@ bool ObKeepAliveLSHandler::check_gts_() bool need_submit = true; SCN gts; SCN max_scn; - SCN end_scn; int ret = OB_SUCCESS; - SCN sys_ls_end_scn = sys_ls_end_scn_.atomic_load(); if (OB_ISNULL(log_handler_ptr_)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid arguments", K(ret), KP(log_handler_ptr_)); - } else if (OB_FAIL(OB_TS_MGR.get_gts(nullptr, gts))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts(gts))) { TRANS_LOG(WARN, "get gts error", K(ret)); } else if (OB_FAIL(log_handler_ptr_->get_max_scn(max_scn))) { TRANS_LOG(WARN, "get max log_ts failed", K(ret)); - } else if (OB_FAIL(log_handler_ptr_->get_end_scn(end_scn))) { - TRANS_LOG(WARN, "get end scn failed", K(ret)); } else if (!last_gts_.is_valid_and_not_min() || last_gts_ == gts) { need_submit = true; - } else if (end_scn.is_valid_and_not_min() - && sys_ls_end_scn.is_valid_and_not_min() - && end_scn < sys_ls_end_scn) { - need_submit = true; } else if (gts.convert_to_ts() < max_scn.convert_to_ts() + KEEP_ALIVE_GTS_INTERVAL) { need_submit = false; } @@ -273,21 +253,11 @@ bool ObKeepAliveLSHandler::check_gts_() return need_submit; } -share::SCN ObKeepAliveLSHandler::get_ref_scn_() -{ - share::SCN ref_scn = last_gts_; - share::SCN sys_ls_end_scn = sys_ls_end_scn_.atomic_load(); - if (ref_scn.is_valid_and_not_min() && sys_ls_end_scn.is_valid_and_not_min()) { - ref_scn = sys_ls_end_scn > ref_scn ? sys_ls_end_scn : ref_scn; - } - return ref_scn; -} - int ObKeepAliveLSHandler::serialize_keep_alive_log_(const SCN &min_start_scn, MinStartScnStatus status) { int ret = OB_SUCCESS; - const int64_t replay_hint = ls_id_.hash(); + const int64_t replay_hint = 0; logservice::ObLogBaseHeader base_header(ObLogBaseType::KEEP_ALIVE_LOG_BASE_TYPE, ObReplayBarrierType::NO_NEED_BARRIER, replay_hint); ObKeepAliveLogBody log_body(1, min_start_scn, status); @@ -305,7 +275,7 @@ int ObKeepAliveLSHandler::serialize_keep_alive_log_(const SCN &min_start_scn, Mi K(log_body.get_serialize_size()), K(submit_buf_len_), K(submit_buf_pos_)); } - TRANS_LOG(DEBUG, "[Keep Alive] serialize keep alive log", K(ret), K(ls_id_), K(log_body)); + TRANS_LOG(DEBUG, "[Keep Alive] serialize keep alive log", K(ret), K(log_body)); return ret; } diff --git a/src/storage/tx/ob_keep_alive_ls_handler.h b/src/storage/tx/ob_keep_alive_ls_handler.h index cd5738c47..ea3edebe0 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.h +++ b/src/storage/tx/ob_keep_alive_ls_handler.h @@ -21,7 +21,6 @@ #include "logservice/ob_log_base_header.h" #include "logservice/ob_log_base_type.h" #include "logservice/ob_append_callback.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -111,7 +110,6 @@ class ObLSKeepAliveStatInfo void reset() { cb_busy_cnt = 0; - not_master_cnt = 0; near_to_gts_cnt = 0; other_error_cnt = 0; submit_succ_cnt = 0; @@ -121,14 +119,12 @@ class ObLSKeepAliveStatInfo void clear_cnt() { cb_busy_cnt = 0; - not_master_cnt = 0; near_to_gts_cnt = 0; other_error_cnt = 0; submit_succ_cnt = 0; } int64_t cb_busy_cnt; - int64_t not_master_cnt; int64_t near_to_gts_cnt; int64_t other_error_cnt; int64_t submit_succ_cnt; @@ -141,14 +137,14 @@ class ObLSKeepAliveStatInfo // after init , we should register in logservice class ObKeepAliveLSHandler : public logservice::ObIReplaySubHandler, public logservice::ObICheckpointSubHandler, - public logservice::ObIRoleChangeSubHandler, + public logservice::ObILocalLogHandler, public logservice::AppendCb { public: const int64_t KEEP_ALIVE_GTS_INTERVAL = 100 * 1000; public: ObKeepAliveLSHandler() : submit_buf_(nullptr) { reset(); } - int init(const share::ObLSID &ls_id,logservice::ObLogHandler * log_handler_ptr); + int init(logservice::ObLogHandler *log_handler_ptr); void stop(); // false - can not safe destroy @@ -167,33 +163,23 @@ class ObKeepAliveLSHandler : public logservice::ObIReplaySubHandler, int on_failure(); int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn); - void switch_to_follower_forcedly() - { - ATOMIC_STORE(&is_master_, false); - } - int switch_to_leader() { ATOMIC_STORE(&is_master_,true); return OB_SUCCESS;} - int switch_to_follower_gracefully() { ATOMIC_STORE(&is_master_,false); return OB_SUCCESS;} - int resume_leader() { ATOMIC_STORE(&is_master_,true);return OB_SUCCESS; } + void deactivate() override {} + int activate() override { return OB_SUCCESS; } share::SCN get_rec_scn() { return share::SCN::max_scn(); } int flush(share::SCN &rec_scn) { return OB_SUCCESS;} void get_min_start_scn(share::SCN &min_start_scn, share::SCN &keep_alive_scn, MinStartScnStatus &status); - void set_sys_ls_end_scn(const share::SCN &sys_ls_end_scn) { sys_ls_end_scn_.inc_update(sys_ls_end_scn);} - private: bool check_gts_(); int serialize_keep_alive_log_(const share::SCN &min_start_scn, MinStartScnStatus status); - share::SCN get_ref_scn_(); private : SpinRWLock lock_; bool is_busy_; - bool is_master_; bool is_stopped_; - share::ObLSID ls_id_; logservice::ObLogHandler * log_handler_ptr_; char *submit_buf_; @@ -201,7 +187,6 @@ private : int64_t submit_buf_pos_; share::SCN last_gts_; - share::SCN sys_ls_end_scn_; KeepAliveLsInfo tmp_keep_alive_info_; KeepAliveLsInfo durable_keep_alive_info_; diff --git a/src/storage/tx/ob_leak_checker.h b/src/storage/tx/ob_leak_checker.h index 4ee9da186..63a3c9668 100644 --- a/src/storage/tx/ob_leak_checker.h +++ b/src/storage/tx/ob_leak_checker.h @@ -83,7 +83,7 @@ class ObLeakChecker } bool is_equal(const Item &item) const { - return true && key_ == item.key_; + return key_ == item.key_; } void set_concerned() { is_concerned_ = true; } bool is_concerned() const { return is_concerned_; } diff --git a/src/storage/tx/ob_location_adapter.cpp b/src/storage/tx/ob_location_adapter.cpp deleted file mode 100644 index cb3fc7cec..000000000 --- a/src/storage/tx/ob_location_adapter.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_location_adapter.h" - -namespace oceanbase -{ -namespace transaction -{ - -using namespace common; -using namespace share; - -ObLocationAdapter::ObLocationAdapter() : is_inited_(false), - schema_service_(NULL), location_service_(NULL) -{ - reset_statistics(); -} - -int ObLocationAdapter::init(share::schema::ObMultiVersionSchemaService *schema_service, - share::ObLocationService *location_service) -{ - int ret = OB_SUCCESS; - - if (is_inited_) { - TRANS_LOG(WARN, "ob location adapter inited twice"); - ret = OB_INIT_TWICE; - } else if (OB_ISNULL(schema_service) || OB_ISNULL(location_service)) { - TRANS_LOG(WARN, "invalid argument", KP(schema_service), KP(location_service)); - ret = OB_INVALID_ARGUMENT; - } else { - schema_service_ = schema_service; - location_service_ = location_service; - is_inited_ = true; - TRANS_LOG(INFO, "ob location cache adapter inited success"); - } - - return ret; -} - -void ObLocationAdapter::destroy() -{ - if (is_inited_) { - is_inited_ = false; - TRANS_LOG(INFO, "ob location cache adapter destroyed"); - } -} - -void ObLocationAdapter::reset_statistics() -{ - renew_access_ = 0; - total_access_ = 0; - error_count_ = 0; -} - -void ObLocationAdapter::statistics() -{ - if (REACH_TIME_INTERVAL(TRANS_ACCESS_STAT_INTERVAL)) { - TRANS_LOG(INFO, "location adapter statistics", - K_(renew_access), K_(total_access), K_(error_count), - "renew_rate", static_cast(renew_access_) / static_cast(total_access_ + 1)); - reset_statistics(); - } -} - -int ObLocationAdapter::nonblock_get_leader(const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader) -{ - int ret = OB_SUCCESS; - const bool is_sync = false; -#ifdef TRANS_ERROR - int64_t random = ObRandom::rand(0, 100); - static int64_t total_alloc_cnt = 0; - static int64_t random_cnt = 0; - ++total_alloc_cnt; - if (0 == random % 50) { - ret = OB_LS_LOCATION_NOT_EXIST; - ++random_cnt; - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "get error for random", K(ls_id), - K(total_alloc_cnt), K(random_cnt)); - } - return ret; - } -#endif - - if (!is_inited_) { - TRANS_LOG(WARN, "ob location adapter not inited"); - ret = OB_NOT_INIT; - } else if (!ls_id.is_valid()) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else { - ret = get_leader_(cluster_id, ls_id, leader, is_sync); - } - - return ret; -} - -int ObLocationAdapter::get_leader_(const int64_t cluster_id, - const ObLSID &ls_id, - common::ObAddr &leader, - const bool is_sync) -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - TRANS_LOG(WARN, "ob location adapter not inited"); - ret = OB_NOT_INIT; - } else if (!ls_id.is_valid()) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (is_sync) { - bool force_renew = false; - if (OB_FAIL(location_service_->get_leader(cluster_id, ls_id, force_renew, leader))) { - TRANS_LOG(WARN, "get leader from locatition cache error", K(ret), K(ls_id)); - force_renew = true; - if (OB_SUCCESS != (ret = location_service_->get_leader(cluster_id, ls_id, force_renew, leader))) { - TRANS_LOG(WARN, "get leader from locatition cache error again", - KR(ret), K(ls_id), K(force_renew)); - } - renew_access_++; - } - } else { - if (OB_FAIL(location_service_->nonblock_get_leader(cluster_id, ls_id, leader))) { - TRANS_LOG(DEBUG, "nonblock get leader from locatition cache error", K(ret), K(ls_id)); - } - } - if (OB_SUCC(ret)) { - if (!leader.is_valid()) { - TRANS_LOG(WARN, "invalid server", K(ls_id), K(leader)); - ret = OB_ERR_UNEXPECTED; - } - } - // statistics - ++total_access_; - if (OB_FAIL(ret)) { - ++error_count_; - } - statistics(); - - return ret; -} - -int ObLocationAdapter::nonblock_get(const int64_t cluster_id, - const ObLSID &ls_id, - ObLSLocation &location) -{ - int ret = OB_SUCCESS; -#ifdef TRANS_ERROR - int64_t random = ObRandom::rand(0, 100); - static int64_t total_alloc_cnt = 0; - static int64_t random_cnt = 0; - ++total_alloc_cnt; - if (0 == random % 50) { - ret = OB_LS_LOCATION_NOT_EXIST; - ++random_cnt; - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "get error for random", K(ls_id), - K(total_alloc_cnt), K(random_cnt)); - } - return ret; - } -#endif - if (!is_inited_) { - TRANS_LOG(WARN, "ob location adapter not inited"); - ret = OB_NOT_INIT; - } else if (!(is_valid_cluster_id(cluster_id) && ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(cluster_id), K(ls_id)); - } else if (OB_FAIL(location_service_->nonblock_get(cluster_id, ls_id, location))) { - TRANS_LOG(WARN, "nonblock get failed", KR(ret), K(cluster_id), K(ls_id)); - } - return ret; -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_location_adapter.h b/src/storage/tx/ob_location_adapter.h deleted file mode 100644 index af8154cfd..000000000 --- a/src/storage/tx/ob_location_adapter.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_LOCATION_ADAPTER_H_ -#define OCEANBASE_TRANSACTION_OB_LOCATION_ADAPTER_H_ - -#include -#include "ob_trans_define.h" -#include "lib/net/ob_addr.h" -#include "share/ob_define.h" -#include "lib/container/ob_se_array.h" -#include "share/location_cache/ob_location_service.h" -#include "share/schema/ob_multi_version_schema_service.h" -#include "share/config/ob_server_config.h" - -#include "lib/time/ob_clock_generator.h" - -namespace oceanbase -{ - -namespace common -{ -class ObAddr; -} - -namespace share -{ -class ObLSID; -class ObLSLocation; -class ObLocationService; -} - -namespace transaction -{ - -class ObILocationAdapter -{ -public: - ObILocationAdapter() {} - virtual ~ObILocationAdapter() {} - virtual int init(share::schema::ObMultiVersionSchemaService *schema_service, - share::ObLocationService *location_service) = 0; - virtual void destroy() = 0; -public: - virtual int nonblock_get_leader(const int64_t cluster_id, const share::ObLSID &ls_id, - common::ObAddr &leader) = 0; - virtual int nonblock_get(const int64_t cluster_id, const share::ObLSID &ls_id, - share::ObLSLocation &location) = 0; -}; - -class ObLocationAdapter : public ObILocationAdapter -{ -public: - ObLocationAdapter(); - ~ObLocationAdapter() {} - - int init(share::schema::ObMultiVersionSchemaService *schema_service, - share::ObLocationService *location_service); - void destroy(); -public: - int nonblock_get_leader(const int64_t cluster_id, const share::ObLSID &ls_id, - common::ObAddr &leader); - int nonblock_get(const int64_t cluster_id, const share::ObLSID &ls_id, - share::ObLSLocation &location); -private: - int get_leader_(const int64_t cluster_id, const share::ObLSID &ls_id, - common::ObAddr &leader, const bool is_sync); - void reset_statistics(); - void statistics(); -private: - bool is_inited_; - share::schema::ObMultiVersionSchemaService *schema_service_; - share::ObLocationService *location_service_; - // for statistics - int64_t renew_access_; - int64_t total_access_; - int64_t error_count_; -}; - -} // transaction -} // oceanbase - -#endif diff --git a/src/storage/tx/ob_ls_tx_ctx_mgr_stat.cpp b/src/storage/tx/ob_ls_tx_ctx_mgr_stat.cpp index 0ba20162d..9c7b2eaf2 100644 --- a/src/storage/tx/ob_ls_tx_ctx_mgr_stat.cpp +++ b/src/storage/tx/ob_ls_tx_ctx_mgr_stat.cpp @@ -28,28 +28,28 @@ namespace transaction void ObLSTxCtxMgrStat::reset() { addr_.reset(); - ls_id_.reset(); - is_master_ = false; is_stopped_ = false; - state_ = -1; + block_tx_ = false; + block_normal_tx_ = false; + block_all_ = false; total_tx_ctx_count_ = 0; mgr_addr_ = 0; } //don't valid input arguments -int ObLSTxCtxMgrStat::init(const common::ObAddr &addr, const share::ObLSID &ls_id, - const bool is_master, const bool is_stopped, - const int64_t state, +int ObLSTxCtxMgrStat::init(const common::ObAddr &addr, + const bool is_stopped, const bool block_tx, + const bool block_normal_tx, const bool block_all, const int64_t total_tx_ctx_count, const int64_t mgr_addr) { int ret = OB_SUCCESS; addr_ = addr; - ls_id_ = ls_id; - is_master_ = is_master; is_stopped_ = is_stopped; - state_ = state; + block_tx_ = block_tx; + block_normal_tx_ = block_normal_tx; + block_all_ = block_all; total_tx_ctx_count_ = total_tx_ctx_count; mgr_addr_ = mgr_addr; diff --git a/src/storage/tx/ob_ls_tx_ctx_mgr_stat.h b/src/storage/tx/ob_ls_tx_ctx_mgr_stat.h index 033606847..0d8dd7140 100644 --- a/src/storage/tx/ob_ls_tx_ctx_mgr_stat.h +++ b/src/storage/tx/ob_ls_tx_ctx_mgr_stat.h @@ -30,28 +30,29 @@ class ObLSTxCtxMgrStat ObLSTxCtxMgrStat() { reset(); } virtual ~ObLSTxCtxMgrStat() { } void reset(); - int init(const common::ObAddr &addr, const share::ObLSID &ls_id, - const bool is_master, const bool is_stopped, - const int64_t state, + int init(const common::ObAddr &addr, + const bool is_stopped, const bool block_tx, + const bool block_normal_tx, const bool block_all, const int64_t total_tx_ctx_count, const int64_t mgr_addr); const common::ObAddr &get_addr() const { return addr_; } - const share::ObLSID &get_ls_id() const { return ls_id_; } - bool is_master() const { return is_master_; } bool is_stopped() const { return is_stopped_; } - int64_t get_state() const { return state_; } + bool is_tx_blocked() const { return block_tx_; } + bool is_normal_tx_blocked() const { return block_normal_tx_; } + bool is_all_blocked() const { return block_all_; } int64_t get_total_tx_ctx_count() const { return total_tx_ctx_count_; } int64_t get_mgr_addr() const { return mgr_addr_; } - TO_STRING_KV(K_(addr), K_(ls_id), K_(is_master), K_(is_stopped), K_(state), + TO_STRING_KV(K_(addr), K_(is_stopped), K_(block_tx), + K_(block_normal_tx), K_(block_all), K_(total_tx_ctx_count), K_(mgr_addr)); private: common::ObAddr addr_; - share::ObLSID ls_id_; - bool is_master_; bool is_stopped_; - int64_t state_; + bool block_tx_; + bool block_normal_tx_; + bool block_all_; int64_t total_tx_ctx_count_; int64_t mgr_addr_; }; diff --git a/src/storage/tx/ob_multi_data_source.cpp b/src/storage/tx/ob_multi_data_source.cpp index e1c28c717..ed30fae9e 100644 --- a/src/storage/tx/ob_multi_data_source.cpp +++ b/src/storage/tx/ob_multi_data_source.cpp @@ -16,7 +16,7 @@ #include "storage/tablet/ob_batch_create_tablet_arg.h" #include "ob_multi_data_source.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #define NEED_MDS_REGISTER_DEFINE #include "storage/multi_data_source/compile_utility/mds_register.h" #undef NEED_MDS_REGISTER_DEFINE @@ -39,7 +39,7 @@ namespace transaction int ObMulSourceTxDataNotifier::notify_table_lock(const ObTxBufferNodeArray &array, const ObMulSourceDataNotifyArg &arg, - ObPartTransCtx *part_ctx, + ObTxCtx *part_ctx, int64_t &total_time) { int ret = OB_SUCCESS; @@ -81,20 +81,16 @@ int ObMulSourceTxDataNotifier::notify_table_lock(const ObTxBufferNodeArray &arra int ObMulSourceTxDataNotifier::notify(const ObTxBufferNodeArray &array, const NotifyType notify_type, const ObMulSourceDataNotifyArg &arg, - ObPartTransCtx *part_ctx, + ObTxCtx *part_ctx, int64_t &total_time) { int ret = OB_SUCCESS; - ObMemtableCtx *mt_ctx = nullptr; const char *can_not_do_tx_end_reason = nullptr; bool can_do_tx_end = true; ObMulSourceDataNotifyArg tmp_notify_arg = arg; if (OB_ISNULL(part_ctx)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument", KR(ret), K(part_ctx)); - } else if (OB_ISNULL(mt_ctx = part_ctx->get_memtable_ctx())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "memtable ctx should not null", KR(ret), K(part_ctx)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { ObTimeGuard notify_time; @@ -116,16 +112,13 @@ int ObMulSourceTxDataNotifier::notify(const ObTxBufferNodeArray &array, } } - OB_ASSERT(node.type_ != ObTxDataSourceType::BEFORE_VERSION_4_1); if(OB_FAIL(ret)) { //do nothing - } else if (node.type_ < ObTxDataSourceType::BEFORE_VERSION_4_1 - && ObTxDataSourceType::CREATE_TABLET_NEW_MDS != node.type_ - && ObTxDataSourceType::DELETE_TABLET_NEW_MDS != node.type_ - && ObTxDataSourceType::UNBIND_TABLET_NEW_MDS != node.type_) { + } else if (uses_builtin_mds_notifier(node.type_)) { switch (node.type_) { case ObTxDataSourceType::TABLE_LOCK: { - ret = notify_table_lock(notify_type, buf, len, tmp_notify_arg, mt_ctx); + ret = notify_table_lock( + notify_type, buf, len, tmp_notify_arg, part_ctx->get_memtable_ctx()); break; } case ObTxDataSourceType::LS_TABLE: { @@ -140,10 +133,6 @@ int ObMulSourceTxDataNotifier::notify(const ObTxBufferNodeArray &array, ret = notify_ddl_barrier(notify_type, buf, len, tmp_notify_arg); break; } - case ObTxDataSourceType::STANDBY_UPGRADE: { - ret = notify_standby_upgrade(notify_type, buf, len, tmp_notify_arg); - break; - } default: { ret = OB_ERR_UNEXPECTED; break; @@ -212,12 +201,7 @@ int ObMulSourceTxDataNotifier::notify(const ObTxBufferNodeArray &array, break;\ case NotifyType::ON_COMMIT:\ MDS_LOG(INFO, "buffer ctx on_commit", K(node), KP(&buffer_ctx), K(buffer_ctx));\ - if (OB_FAIL(common::meta::MdsCommitForOldMdsWrapper::\ - on_commit_for_old_mds(buf,\ - len,\ - tmp_notify_arg))) {\ - MDS_LOG(WARN, "fail to on_commit_for_old_mds", KR(ret));\ - } else if (arg.for_replay_ && !common::meta::MdsCheckCanReplayWrapper::\ + if (arg.for_replay_ && !common::meta::MdsCheckCanReplayWrapper::\ check_can_replay_commit(buf,\ len,\ arg.scn_,\ @@ -337,30 +321,6 @@ int ObMulSourceTxDataNotifier::notify_ls_table(const NotifyType type, return ret; } -int ObMulSourceTxDataNotifier::notify_standby_upgrade(const NotifyType type, - const char *buf, const int64_t len, - const ObMulSourceDataNotifyArg &arg) -{ - int ret = OB_SUCCESS; - share::ObStandbyUpgrade data_version; - int64_t pos = 0; - - if (OB_ISNULL(buf) || OB_UNLIKELY(len <= 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(buf), K(len), K(type)); - } else if (OB_FAIL(data_version.deserialize(buf, len, pos))) { - TRANS_LOG(WARN, "failed to deserialize data_version", KR(ret), K(buf), K(len)); - } else if (pos > len) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "deserialize error", KR(ret), K(pos), K(len)); - } - TRANS_LOG(INFO, "standby upgrade notify", KR(ret), K(data_version), K(len), K(type)); - - ob_abort_log_cb_notify_(type, ret, arg.for_replay_); - - return ret; -} - int ObMulSourceTxDataNotifier::notify_ddl_trans(const NotifyType type, const char *buf, const int64_t len, const ObMulSourceDataNotifyArg &arg) @@ -497,7 +457,7 @@ OB_SERIALIZE_MEMBER(ObRegisterMdsFlag, need_flush_redo_instantly_, mds_base_scn_ // ObMDSStr //##################################################### -OB_SERIALIZE_MEMBER(ObMDSInnerSQLStr, mds_str_, type_, ls_id_, register_flag_); +OB_SERIALIZE_MEMBER(ObMDSInnerSQLStr, mds_str_, type_, register_flag_); ObMDSInnerSQLStr::ObMDSInnerSQLStr() { reset(); } @@ -507,26 +467,23 @@ void ObMDSInnerSQLStr::reset() { mds_str_.reset(); type_ = ObTxDataSourceType::UNKNOWN; - ls_id_.reset(); register_flag_.reset(); } int ObMDSInnerSQLStr::set(const char *msd_buf, const int64_t msd_buf_len, const ObTxDataSourceType &type, - const share::ObLSID ls_id, const ObRegisterMdsFlag ®ister_flag) { int ret = OB_SUCCESS; - if (OB_ISNULL(msd_buf) || 0 == msd_buf_len || ObTxDataSourceType::UNKNOWN == type || !ls_id.is_valid()) { + if (OB_ISNULL(msd_buf) || 0 == msd_buf_len || ObTxDataSourceType::UNKNOWN == type) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid arguments", K(ret), KP(msd_buf), K(msd_buf_len), K(type), K(ls_id)); + TRANS_LOG(WARN, "invalid arguments", K(ret), KP(msd_buf), K(msd_buf_len), K(type)); } else if (!mds_str_.empty()) { TRANS_LOG(WARN, "MSD str is not empty", K(ret), K(*this)); } else { mds_str_.assign_ptr(msd_buf, msd_buf_len); type_ = type; - ls_id_ = ls_id; register_flag_ = register_flag; } return ret; diff --git a/src/storage/tx/ob_multi_data_source.h b/src/storage/tx/ob_multi_data_source.h index 89512e256..4a4133dd9 100644 --- a/src/storage/tx/ob_multi_data_source.h +++ b/src/storage/tx/ob_multi_data_source.h @@ -22,7 +22,6 @@ #include "lib/list/ob_list.h" #include "lib/utility/ob_unify_serialize.h" #include "lib/string/ob_string.h" -#include "share/ob_ls_id.h" #include "share/scn.h" #include "storage/tx/ob_multi_data_source_printer.h" #include "storage/tx/ob_multi_data_source_tx_buffer_node.h" @@ -31,11 +30,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -} - namespace memtable { class ObMemtableCtx; @@ -51,7 +45,7 @@ class BufferCtxNode; namespace transaction { -class ObPartTransCtx; +class ObTxCtx; struct ObMulSourceDataNotifyArg; enum class ObTxDataSourceType : int64_t @@ -67,9 +61,6 @@ enum class ObTxDataSourceType : int64_t DDL_BARRIER = 5, // for all ddl trans(record incremental schema) DDL_TRANS = 6, - // for standby upgrade - STANDBY_UPGRADE = 8, - BEFORE_VERSION_4_1 = 13, #define NEED_GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION #define _GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION_(helper_class_name, buffer_ctx_type, ID, ENUM_NAME) ENUM_NAME = ID, #include "storage/multi_data_source/compile_utility/mds_register.h" @@ -78,6 +69,14 @@ enum class ObTxDataSourceType : int64_t MAX_TYPE = 100 }; +inline bool uses_builtin_mds_notifier(const ObTxDataSourceType type) +{ + return ObTxDataSourceType::TABLE_LOCK == type + || ObTxDataSourceType::LS_TABLE == type + || ObTxDataSourceType::DDL_BARRIER == type + || ObTxDataSourceType::DDL_TRANS == type; +} + enum class NotifyType : int64_t { UNKNOWN = -1, @@ -95,11 +94,11 @@ class ObMulSourceTxDataNotifier static int notify(const ObTxBufferNodeArray &array, const NotifyType type, const ObMulSourceDataNotifyArg &arg, - ObPartTransCtx *part_ctx, + ObTxCtx *part_ctx, int64_t &total_time); static int notify_table_lock(const ObTxBufferNodeArray &array, const ObMulSourceDataNotifyArg &arg, - ObPartTransCtx *part_ctx, + ObTxCtx *part_ctx, int64_t &total_time); private: static void ob_abort_log_cb_notify_(const NotifyType type, int err_code, bool for_replay); @@ -110,9 +109,6 @@ class ObMulSourceTxDataNotifier static int notify_ls_table(const NotifyType type, const char *buf, const int64_t len, const ObMulSourceDataNotifyArg &arg); - static int notify_standby_upgrade(const NotifyType type, - const char *buf, const int64_t len, - const ObMulSourceDataNotifyArg &arg); static int notify_ddl_trans(const NotifyType type, const char *buf, const int64_t len, const ObMulSourceDataNotifyArg &arg); @@ -162,16 +158,14 @@ class ObMDSInnerSQLStr int set(const char *mds_buf, const int64_t mds_buf_len, const ObTxDataSourceType &type, - const share::ObLSID ls_id, const ObRegisterMdsFlag ®ister_flag); const char *get_msd_buf() { return mds_str_.ptr(); } int64_t get_msd_buf_len() { return mds_str_.length(); } const ObTxDataSourceType &get_msd_type() { return type_; } - const share::ObLSID &get_ls_id() { return ls_id_; } const ObRegisterMdsFlag &get_register_flag() { return register_flag_; } - TO_STRING_KV(K(mds_str_), "type_", ObMultiDataSourcePrinter::to_str_mds_type(type_), K(ls_id_), K(register_flag_)); + TO_STRING_KV(K(mds_str_), "type_", ObMultiDataSourcePrinter::to_str_mds_type(type_), K(register_flag_)); private: // const char *msd_buf_; @@ -179,7 +173,6 @@ class ObMDSInnerSQLStr // bool from_copy_; common::ObString mds_str_; ObTxDataSourceType type_; - share::ObLSID ls_id_; ObRegisterMdsFlag register_flag_; }; diff --git a/src/storage/tx/ob_multi_data_source_printer.cpp b/src/storage/tx/ob_multi_data_source_printer.cpp index 38987a103..e06b7145c 100644 --- a/src/storage/tx/ob_multi_data_source_printer.cpp +++ b/src/storage/tx/ob_multi_data_source_printer.cpp @@ -33,10 +33,6 @@ const char *ObMultiDataSourcePrinter::to_str_mds_type(const ObTxDataSourceType & TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, LS_TABLE); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, DDL_BARRIER); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, DDL_TRANS); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, STANDBY_UPGRADE); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, STANDBY_UPGRADE_DATA_VERSION); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, BEFORE_VERSION_4_1); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TEST1); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TEST2); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TEST3); @@ -47,12 +43,7 @@ const char *ObMultiDataSourcePrinter::to_str_mds_type(const ObTxDataSourceType & TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, CHANGE_TABLET_TO_TABLE_MDS); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TABLET_SPLIT); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TABLET_BINDING); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_PUBLISH_SCN); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_NOTICE_SAFE); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_UPDATE_SCN); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, SYNC_TRUNCATE_INFO); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_MERGE_SCN); - TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MVIEW_MDS_OP); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, DDL_COMPLETE_MDS); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TABLET_FORK); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MAX_TYPE); diff --git a/src/storage/tx/ob_multi_data_source_tx_buffer_node.h b/src/storage/tx/ob_multi_data_source_tx_buffer_node.h index a119f2405..893e60994 100644 --- a/src/storage/tx/ob_multi_data_source_tx_buffer_node.h +++ b/src/storage/tx/ob_multi_data_source_tx_buffer_node.h @@ -40,7 +40,7 @@ enum class ObTxDataSourceType : int64_t; class ObTxBufferNode { - friend class ObPartTransCtx; + friend class ObTxCtx; friend class ObTxExecInfo; friend class ObMulSourceTxDataNotifier; friend class ObTxMDSCache; diff --git a/src/storage/tx/ob_tablet_to_ls_cache.cpp b/src/storage/tx/ob_tablet_to_ls_cache.cpp deleted file mode 100644 index 89f195780..000000000 --- a/src/storage/tx/ob_tablet_to_ls_cache.cpp +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "storage/tx/ob_tablet_to_ls_cache.h" - -namespace oceanbase -{ -namespace transaction -{ -int ObTabletToLSCache::init(ObTxCtxMgr *tx_ctx_mgr) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "ObTabletToLSCache init twice", KR(ret), K(tx_ctx_mgr)); - } else if (!true || OB_ISNULL(tx_ctx_mgr)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_ctx_mgr)); - } else if (OB_FAIL(map_.init(lib::ObMemAttr(lib::ObLabel("TabletToLS"))))) { - TRANS_LOG(WARN, "map init fail", KR(ret), K(tx_ctx_mgr)); - } else { - tx_ctx_mgr_ = tx_ctx_mgr; - is_inited_ = true; - TRANS_LOG(INFO, "ObTabletToLSCache init success", KR(ret), KPC(this)); - } - return ret; -} - -void ObTabletToLSCache::destroy() -{ - if (is_inited_) { - map_.destroy(); - tx_ctx_mgr_ = NULL; - is_inited_ = false; - TRANS_LOG(INFO, "ObTabletToLSCache destroy"); - } -} - - -int ObTabletToLSCache::create_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ObTabletLSNode *ls_cache = NULL; - ObTimeGuard tg("ObTabletToLSCache::create_tablet", 5 * 1000); // 5ms - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTabletToLSCache has not inited", KR(ret), K(tablet_id), K(ls_id), KPC(this), K(lbt())); - } else if (!tablet_id.is_valid() || !ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tablet_id), K(ls_id)); - } else if (OB_FAIL(tx_ctx_mgr_->get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls tx ctx mgr fail", KR(ret), K_(tx_ctx_mgr), K(tablet_id), K(ls_id)); - } else if (OB_ISNULL(ls_tx_ctx_mgr)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected ls_tx_ctx_mgr is null ", KR(ret), K(tablet_id), K(ls_id)); - } else { - if (OB_FAIL(map_.alloc_value(ls_cache))) { - TRANS_LOG(WARN, "alloc ls cache fail", KR(ret), K(tablet_id), K(ls_id)); - } else { - if (OB_FAIL(ls_cache->init(tablet_id, ls_tx_ctx_mgr, tx_ctx_mgr_))) { - TRANS_LOG(WARN, "init ls cache fail", KR(ret), K(tablet_id), K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (OB_FAIL(map_.insert(tablet_id, ls_cache))) { - tg.click(); - if (OB_ENTRY_EXIST == ret) { - // try to delete old and insert new again - static const int MAX_RETRY_CNT = 10; - int loop_cnt = 0; - ObTabletLSNode *ls_cache_old = NULL; - while (OB_ENTRY_EXIST == ret) { - tg.click(); - if (OB_FAIL(map_.get(tablet_id, ls_cache_old))) { - if (OB_ENTRY_NOT_EXIST == ret) { - // entry not exist: insert - ret = map_.insert(tablet_id, ls_cache); - } else { - // fail - } - } else { - // entry exist: delete old and insert new - if (OB_FAIL(map_.del(tablet_id, ls_cache_old)) && OB_ENTRY_NOT_EXIST != ret) { - // fail - } else { - // entry deleted or not exist: insert - ret = map_.insert(tablet_id, ls_cache); - } - // call revert after get from map - map_.revert(ls_cache_old); - } - if (++loop_cnt > MAX_RETRY_CNT) { - ret = OB_EAGAIN; - TRANS_LOG(WARN, "insert retry too much times", KR(ret), K(tablet_id), K(ls_cache), K(lbt())); - } - } - } - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "map insert fail", KR(ret), K(tablet_id), K(ls_cache), KP(ls_tx_ctx_mgr)); - // call clear after init if failed - ls_cache->clear(); - } - tg.click(); - } - if (OB_FAIL(ret)) { - // call free after alloc if failed - map_.free_value(ls_cache); - } - } - if (OB_FAIL(ret)) { - // call revert after get ls_tx_ctx_mgr if failed - tx_ctx_mgr_->revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - if (OB_ENTRY_EXIST == ret) { - ret = OB_SUCCESS; - } - } - } - TRANS_LOG(INFO, "create tablet cache", KR(ret), K(tablet_id), K(ls_id)); - - return ret; -} - -int ObTabletToLSCache::remove_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObTabletLSNode *ls_cache = NULL; - ObTimeGuard tg("ObTabletToLSCache::remove_tablet", 5 * 1000); // 5ms - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTabletToLSCache has not inited", KR(ret), K(tablet_id), K(ls_id), KPC(this), K(lbt())); - } else if (!tablet_id.is_valid() || !ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tablet_id), K(ls_id)); - } else if (OB_FAIL(map_.get(tablet_id, ls_cache))) { - // do nothing - } else { - if (ls_cache->get_ls_tx_ctx_mgr()->get_ls_id() == ls_id) { - ret = map_.del(tablet_id, ls_cache); - } - // call revert after get from map - map_.revert(ls_cache); - } - if (OB_FAIL(ret)) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - TRANS_LOG(ERROR, "remove tablet cache fail", KR(ret), K(tablet_id), K(ls_id)); - } - } - TRANS_LOG(INFO, "remove tablet cache", KR(ret), K(tablet_id), K(ls_id)); - - return ret; -} - -int ObTabletToLSCache::remove_ls_tablets(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - RemoveLSTabletFunctor functor(&map_, ls_id); - ObTimeGuard tg("ObTabletToLSCache::remove_ls_tablets", 1000 * 1000); // 1s - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTabletToLSCache has not inited", KR(ret), K(ls_id), KPC(this), K(lbt())); - } else if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(ls_id)); - } else if (OB_FAIL(map_.remove_if(functor))) { - TRANS_LOG(ERROR, "remove ls tablets cache fail", KR(ret), K(ls_id)); - } - TRANS_LOG(INFO, "remove ls tablets cache", KR(ret), K(ls_id)); - - return ret; -} - -int ObTabletToLSCache::check_and_get_ls_info(const common::ObTabletID &tablet_id, - share::ObLSID &ls_id, - bool &is_local_leader) -{ - int ret = OB_SUCCESS; - ObTabletLSNode *ls_cache = NULL; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTabletToLSCache has not inited", KR(ret), K(tablet_id), KPC(this), K(lbt())); - } else if (!tablet_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tablet_id)); - } else if (OB_FAIL(map_.get(tablet_id, ls_cache))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - TRANS_LOG(WARN, "get ls cache fail", KR(ret), K(tablet_id)); - } - } else { - if (OB_ISNULL(ls_tx_ctx_mgr = ls_cache->get_ls_tx_ctx_mgr())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected ls_tx_ctx_mgr is null", KR(ret), K(tablet_id)); - } else { - ls_id = ls_tx_ctx_mgr->get_ls_id(); - is_local_leader = ls_tx_ctx_mgr->is_master(); - } - // call revert after get from map - map_.revert(ls_cache); - } - TRANS_LOG(DEBUG, "check and get ls info", K(tablet_id), K(ls_id), K(is_local_leader), K(ret)); - return ret; -} - -int64_t ObTabletToLSCache::size() -{ - return map_.get_total_cnt(); -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_tablet_to_ls_cache.h b/src/storage/tx/ob_tablet_to_ls_cache.h deleted file mode 100644 index c21563b2d..000000000 --- a/src/storage/tx/ob_tablet_to_ls_cache.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TABLET_TO_LS_CACHE_ -#define OCEANBASE_TABLET_TO_LS_CACHE_ - -#include "storage/tx_storage/ob_ls_map.h" - -namespace oceanbase -{ -namespace transaction -{ - -class ObTabletLSNode: public ObLightHashLink -{ -public: - ObTabletLSNode() : ls_tx_ctx_mgr_(NULL), tx_ctx_mgr_(NULL) {} - ~ObTabletLSNode() { reset(); } - int init(const common::ObTabletID &tablet_id, ObLSTxCtxMgr *ls_tx_ctx_mgr, ObTxCtxMgr *tx_ctx_mgr) - { - int ret = OB_SUCCESS; - if (!tablet_id.is_valid() || OB_ISNULL(ls_tx_ctx_mgr) || OB_ISNULL(tx_ctx_mgr)) { - ret = OB_INVALID_ARGUMENT; - } else { - tablet_id_ = tablet_id; - ls_tx_ctx_mgr_ = ls_tx_ctx_mgr; - tx_ctx_mgr_ = tx_ctx_mgr; - } - return ret; - } - void reset() - { - if (OB_NOT_NULL(ls_tx_ctx_mgr_) && OB_NOT_NULL(tx_ctx_mgr_)) { - tx_ctx_mgr_->revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr_); - ls_tx_ctx_mgr_ = NULL; - tx_ctx_mgr_ = NULL; - } - tablet_id_.reset(); - } - void clear() - { - ls_tx_ctx_mgr_ = NULL; - tx_ctx_mgr_ = NULL; - tablet_id_.reset(); - } - ObLSTxCtxMgr *get_ls_tx_ctx_mgr() const - { - return ls_tx_ctx_mgr_; - } - bool contain(const common::ObTabletID &tablet_id) const - { - return tablet_id_ == tablet_id; - } - TO_STRING_KV(K_(tablet_id),KP_(ls_tx_ctx_mgr),KP_(tx_ctx_mgr),KP(this)); - -private: - common::ObTabletID tablet_id_; - ObLSTxCtxMgr *ls_tx_ctx_mgr_; - ObTxCtxMgr *tx_ctx_mgr_; -}; - -class ObTabletLSNodeAlloc -{ -public: - ObTabletLSNode* alloc_value() - { - return op_alloc(ObTabletLSNode); - } - void free_value(ObTabletLSNode *val) - { - if (OB_NOT_NULL(val)) { - op_free(val); - } - } -}; - -typedef ObLightHashMap TabletToLSMap; - -class RemoveLSTabletFunctor -{ -public: - RemoveLSTabletFunctor(TabletToLSMap *map, const share::ObLSID &ls_id) : map_(map), ls_id_(ls_id) {} - bool operator() (ObTabletLSNode *val) - { - bool need_remove = false; - if (OB_NOT_NULL(map_) && OB_NOT_NULL(val) && ls_id_ == val->get_ls_tx_ctx_mgr()->get_ls_id()) { - need_remove = true; - } - return need_remove; - } -private: - TabletToLSMap *map_; - share::ObLSID ls_id_; -}; - -class ObTabletToLSCache final -{ -public: - ObTabletToLSCache() : is_inited_(false), tx_ctx_mgr_(NULL), map_() { } - ~ObTabletToLSCache() { destroy(); } - - int init(ObTxCtxMgr *tx_ctx_mgr); - void destroy(); - int create_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id); - int remove_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id); - int remove_ls_tablets(const share::ObLSID &ls_id); - int check_and_get_ls_info(const common::ObTabletID &tablet_id, - share::ObLSID &ls_id, - bool &is_local_leader); - int64_t size(); - TO_STRING_KV(K_(is_inited),KP_(tx_ctx_mgr),KP(this)); - -private: - bool is_inited_; - ObTxCtxMgr *tx_ctx_mgr_; - TabletToLSMap map_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_TABLET_TO_LS_CACHE_ diff --git a/src/storage/tx/ob_timestamp_access.cpp b/src/storage/tx/ob_timestamp_access.cpp index ba26c65d5..5a2ec0caf 100644 --- a/src/storage/tx/ob_timestamp_access.cpp +++ b/src/storage/tx/ob_timestamp_access.cpp @@ -23,43 +23,14 @@ namespace oceanbase namespace transaction { -int ObTimestampAccess::handle_request(const ObGtsRequest &request, obcall::ObGtsRpcResult &result) -{ - int ret = OB_SUCCESS; - if (GTS_LEADER == service_type_) { - ret = share::g_mp->timestamp_service()->handle_request(request, result); - } else if (STS_LEADER == service_type_) { - } else { - ret = OB_NOT_MASTER; - if (EXECUTE_COUNT_PER_SEC(10)) { - TRANS_LOG(INFO, "ObTimestampAccess service type is FOLLOWER", K(ret), K_(service_type)); - } - } - return ret; -} - int ObTimestampAccess::get_number(int64_t >s) { - int ret = OB_SUCCESS; - if (GTS_LEADER == service_type_) { - ret = share::g_mp->timestamp_service()->get_timestamp(gts); - } else if (STS_LEADER == service_type_) { - } else { - ret = OB_NOT_MASTER; - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "ObTimestampAccess service type is FOLLOWER", K(ret), K_(service_type)); - } - } - return ret; + return share::g_mp->timestamp_service()->get_timestamp(gts); } -void ObTimestampAccess::get_virtual_info(int64_t &ts_value, - ServiceType &service_type, - common::ObRole &role, - int64_t &proposal_id) +void ObTimestampAccess::get_virtual_info(int64_t &ts_value) { - service_type = service_type_; - share::g_mp->timestamp_service()->get_virtual_info(ts_value, role, proposal_id); + share::g_mp->timestamp_service()->get_virtual_info(ts_value); } } diff --git a/src/storage/tx/ob_timestamp_access.h b/src/storage/tx/ob_timestamp_access.h index eafe596d0..95c24c06f 100644 --- a/src/storage/tx/ob_timestamp_access.h +++ b/src/storage/tx/ob_timestamp_access.h @@ -18,24 +18,16 @@ #define OCEANBASE_TRANSACTION_OB_TIMESTAMP_ACCESS_ #include "share/rc/ob_tenant_base.h" -#include "common/ob_role.h" namespace oceanbase { -namespace obcall -{ -class ObGtsRpcResult; -} - namespace transaction { -class ObGtsRequest; - class ObTimestampAccess { public: - ObTimestampAccess() : service_type_(FOLLOWER) {} + ObTimestampAccess() {} ~ObTimestampAccess() {} static int mtl_init(ObTimestampAccess *×tamp_access) { @@ -43,51 +35,9 @@ class ObTimestampAccess return OB_SUCCESS; } void destroy() { reset();} - void reset() { service_type_ = FOLLOWER; } - enum ServiceType { - FOLLOWER = 0, - GTS_LEADER, - STS_LEADER, - }; - void set_service_type(const ServiceType service_type) { service_type_ = service_type; } - ServiceType get_service_type() const { return service_type_; } - int handle_request(const ObGtsRequest &request, obcall::ObGtsRpcResult &result); + void reset() {} int get_number(int64_t >s); - void get_virtual_info(int64_t &ts_value, - ServiceType &service_type, - common::ObRole &role, - int64_t &proposal_id); - static const char *service_type_to_cstr(const ServiceType service_type) - { - const char *str; - switch (service_type) { - case ServiceType::FOLLOWER: - str = "FOLLOWER"; - break; - case ServiceType::GTS_LEADER: - str = "GTS_LEADER"; - break; - case ServiceType::STS_LEADER: - str = "STS_LEADER"; - break; - default: - str = "UNKNOWN"; - break; - } - return str; - } - static const char *ts_type_to_cstr(bool is_primary) - { - const char *str; - if (is_primary) { - str = "GTS"; - } else { - str = "STS"; - } - return str; - } -private: - ServiceType service_type_; + void get_virtual_info(int64_t &ts_value); }; diff --git a/src/storage/tx/ob_timestamp_service.cpp b/src/storage/tx/ob_timestamp_service.cpp index 50f85915b..c33a89ad2 100644 --- a/src/storage/tx/ob_timestamp_service.cpp +++ b/src/storage/tx/ob_timestamp_service.cpp @@ -15,10 +15,6 @@ */ #include "ob_timestamp_service.h" -#include "share/rc/ob_module_provider.h" -#include "observer/ob_srv_network_frame.h" -#include "ob_timestamp_access.h" -#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -48,11 +44,9 @@ int ObTimestampService::mtl_init(ObTimestampService *×tamp_service) // The interface for getting gts timestamp, actually a wrapper of ObIDService::get_number. // -// For most cases, the gts service uses the machine clock's time as gts timestamp, which means -// the gts service advances as fast as the machine clock. Howerver, when the gts service switches -// leader, the new leader will pre-allocate a range of timestamps, and this can lead to the gts -// timestamp becoming larger than the machine clock's time. Then the gts service will slow down to -// wait the machine clock. But we don't want the gts service to advance too slowly(when the request +// The timestamp service uses the machine clock as its base. Persisted preallocation can make the +// allocated range temporarily larger than the machine clock, so the service may need to slow down +// and wait for the machine clock. But we don't want the service to advance too slowly (when request // rate is low), since the observer may wait too long before the gts timestamp crosses log SCN. // So we periodically check the gts service's advancing speed, and if it's far slower than the // machine clock, we manually push the gts ahead. @@ -109,172 +103,10 @@ int ObTimestampService::get_timestamp(int64_t >s) return ret; } -int ObTimestampService::handle_request(const ObGtsRequest &request, ObGtsRpcResult &result) +void ObTimestampService::get_virtual_info(int64_t &ts_value) { - static int64_t total_cnt = 0; - static int64_t total_rt = 0; - static const int64_t STATISTICS_INTERVAL_US = 10000000; - const MonotonicTs start = MonotonicTs::current_time(); - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(request)); - } else { - TRANS_LOG(DEBUG, "handle gts request", K(request)); - int64_t gts = 0; - const MonotonicTs srr = request.get_srr(); - - const ObAddr &requester = request.get_sender(); - if (requester == self_) { - // Go local call to get gts - TRANS_LOG(DEBUG, "handle local gts request", K(requester)); - ret = handle_local_request_(request, result); - } else if (OB_FAIL(get_timestamp(gts))) { - // single-replica: remote requester path is dead; only local requests occur - if (EXECUTE_COUNT_PER_SEC(10)) { - TRANS_LOG(WARN, "get timestamp failed", KR(ret)); - } - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = result.init(ret, srr, 0, 0))) { - TRANS_LOG(WARN, "gts result init failed", K(tmp_ret), K(request)); - } - } else { - if (OB_FAIL(result.init(ret, srr, gts, gts))) { - TRANS_LOG(WARN, "gts result init failed", KR(ret), K(request)); - } - } - } - //obtain gts information: - //(1) How many gts requests are processed per second - //(2) How long does it take to process a request - const MonotonicTs end = MonotonicTs::current_time(); - const int64_t cost_us = request.get_srr().mts_ - end.mts_; - //Print the gts request that takes a long time for network transmission - if (cost_us > 500 * 1000) { - TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "gts request fly too much time", K(request), K(result), K(cost_us)); - } - ATOMIC_INC(&total_cnt); - ObTransStatistic::get_instance().add_gts_request_total_count( 1); - (void)ATOMIC_FAA(&total_rt, end.mts_ - start.mts_); - if (REACH_TIME_INTERVAL(STATISTICS_INTERVAL_US)) { - TRANS_LOG(INFO, "handle gts request statistics", K(total_rt), K(total_cnt), - "avg_rt", (double)total_rt / (double)(total_cnt + 1), - "avg_cnt", (double)total_cnt / (double)(STATISTICS_INTERVAL_US / 1000000)); - ATOMIC_STORE(&total_cnt, 0); - ATOMIC_STORE(&total_rt, 0); - } - return ret; -} - -#ifndef ERRSIM -ERRSIM_POINT_DEF(EN_GTS_HANDLE_REQUEST) -#endif - -int ObTimestampService::handle_local_request_(const ObGtsRequest &request, obcall::ObGtsRpcResult &result) -{ - int ret = OB_SUCCESS; - int64_t gts = 0; - - const MonotonicTs srr = request.get_srr(); -#ifndef ERRSIM - ret = EN_GTS_HANDLE_REQUEST; -#endif - // the fisrt case for errsim - if (OB_SUCCESS != ret) { - TRANS_LOG(WARN, "errsim for gts handle local request", KR(ret)); - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = result.init(ret, srr, 0, 0))) { - TRANS_LOG(WARN, "gts result init failed", K(tmp_ret), K(request)); - } - } else if (OB_FAIL(get_timestamp(gts))) { - if (EXECUTE_COUNT_PER_SEC(10)) { - TRANS_LOG(WARN, "get timestamp failed", KR(ret)); - } - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = result.init(ret, srr, 0, 0))) { - TRANS_LOG(WARN, "gts result init failed", K(tmp_ret), K(request)); - } - } else { - if (OB_FAIL(result.init(ret, srr, gts, gts))) { - TRANS_LOG(WARN, "local gts result init failed", KR(ret), K(request)); - } - } - return ret; -} - -int ObTimestampService::switch_to_follower_gracefully() -{ - (void)ObIDService::switch_to_follower_gracefully(); - int64_t type = share::g_mp->timestamp_access()->get_service_type(); - if (ObTimestampAccess::ServiceType::GTS_LEADER == type) { - share::g_mp->timestamp_access()->set_service_type(ObTimestampAccess::ServiceType::FOLLOWER); - } - TRANS_LOG(INFO, "ObTimestampService switch to follower gracefully success", K(type), "service_type", share::g_mp->timestamp_access()->get_service_type()); - return OB_SUCCESS; -} - -void ObTimestampService::switch_to_follower_forcedly() -{ - int64_t type = share::g_mp->timestamp_access()->get_service_type(); - if (ObTimestampAccess::ServiceType::GTS_LEADER == type) { - share::g_mp->timestamp_access()->set_service_type(ObTimestampAccess::ServiceType::FOLLOWER); - } - TRANS_LOG(INFO, "ObTimestampService switch to follower forcedly success", K(type), "service_type", share::g_mp->timestamp_access()->get_service_type()); -} - -int ObTimestampService::resume_leader() -{ - share::g_mp->timestamp_access()->set_service_type(ObTimestampAccess::ServiceType::GTS_LEADER); - TRANS_LOG(INFO, "ObTimestampService resume leader success", "service_type", share::g_mp->timestamp_access()->get_service_type()); - return OB_SUCCESS; -} - -int ObTimestampService::switch_to_leader() -{ - int ret = OB_SUCCESS; - TRANS_LOG(INFO, "ObTimestampService switch_to_leader called", K(ret)); - - if (OB_FAIL(check_and_fill_ls())) { - TRANS_LOG(WARN, "ls set fail", K(ret)); - } else { - SCN version; - if (OB_FAIL(ls_->get_log_handler()->get_max_scn(version))) { - TRANS_LOG(WARN, "get max ts fail", K(ret)); - } else { - int64_t version_val = version.is_valid() ? version.get_val_for_gts() : -1; - if (version_val >= ATOMIC_LOAD(&limited_id_)) { - inc_update(&last_id_, version_val); - ATOMIC_STORE(&tmp_last_id_, 0); - } else if (ATOMIC_LOAD(&tmp_last_id_) != 0 && version_val > ATOMIC_LOAD(&tmp_last_id_)) { - inc_update(&tmp_last_id_, version_val); - } else { - // do nothing - } - share::g_mp->timestamp_access()->set_service_type(ObTimestampAccess::ServiceType::GTS_LEADER); - TRANS_LOG(INFO, "ObTimestampService switch to leader success", K(ret), K(version), K(last_id_), K(limited_id_), - "service_type", share::g_mp->timestamp_access()->get_service_type()); - } - } - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "ObTimestampService switch_to_leader failed", K(ret), - "service_type", share::g_mp->timestamp_access()->get_service_type()); - } - - return ret; -} - -void ObTimestampService::get_virtual_info(int64_t &ts_value, common::ObRole &role, int64_t &proposal_id) -{ - int ret = OB_SUCCESS; ts_value = last_id_; - if (OB_FAIL(check_and_fill_ls())) { - TRANS_LOG(WARN, "ls set fail", K(ret)); - } else if (OB_FAIL(ls_->get_log_handler()->get_role(role, proposal_id))) { - TRANS_LOG(WARN, "get ls role fail", K(ret)); - } - TRANS_LOG(INFO, "gts get virtual info", K(ret), K_(last_id), K(ts_value), K(role), K(proposal_id)); + TRANS_LOG(INFO, "gts get virtual info", K_(last_id), K(ts_value)); } } diff --git a/src/storage/tx/ob_timestamp_service.h b/src/storage/tx/ob_timestamp_service.h index 9ad9e6785..b672963af 100644 --- a/src/storage/tx/ob_timestamp_service.h +++ b/src/storage/tx/ob_timestamp_service.h @@ -18,29 +18,12 @@ #define OCEANBASE_TRANSACTION_OB_TIMESTAMP_SERVICE_ #include "ob_id_service.h" -#include "ob_gts_rpc.h" -#include "logservice/palf/election_self.h" namespace oceanbase { -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} - -namespace obcall -{ -class ObGtsRpcResult; -} - namespace transaction { -class ObGtsRequest; - class ObTimestampService : public ObIDService { public: @@ -56,27 +39,21 @@ class ObTimestampService : public ObIDService reset(); } // nano second - static const int64_t TIMESTAMP_PREALLOCATED_RANGE = palf::election::MAX_LEASE_TIME * 1000; - static const int64_t PREALLOCATE_RANGE_FOR_SWITHOVER = 2 * TIMESTAMP_PREALLOCATED_RANGE; - int handle_request(const ObGtsRequest &request, obcall::ObGtsRpcResult &result); + static const int64_t TIMESTAMP_PREALLOCATED_RANGE = 10L * 1000L * 1000L * 1000L; + static const int64_t TIMESTAMP_RECOVERY_SAFETY_RANGE = 2 * TIMESTAMP_PREALLOCATED_RANGE; int get_timestamp(int64_t >s); - int switch_to_follower_gracefully(); - void switch_to_follower_forcedly(); - int resume_leader(); - int switch_to_leader(); int64_t get_limited_id() const { return limited_id_; } - static SCN get_sts_start_scn(const SCN &max_sys_ls_scn) - { return SCN::plus(max_sys_ls_scn, PREALLOCATE_RANGE_FOR_SWITHOVER); }; - void get_virtual_info(int64_t &ts_value, common::ObRole &role, int64_t &proposal_id); + static SCN get_sts_start_scn(const SCN &max_ls_scn) + { return SCN::plus(max_ls_scn, TIMESTAMP_RECOVERY_SAFETY_RANGE); }; + void get_virtual_info(int64_t &ts_value); private: - // last timestamp retrieved from gts leader, updated periodically, nanosecond + // last timestamp retrieved from the local timestamp service, in nanoseconds int64_t last_gts_; // the time of last request, updated periodically, nanosecond int64_t last_request_ts_; // the lock of checking the gts service's advancing speed, used in get_timestamp to avoid // concurrent threads all pushing the gts ahead int64_t check_gts_speed_lock_; - int handle_local_request_(const ObGtsRequest &request, obcall::ObGtsRpcResult &result); }; } diff --git a/src/storage/tx/ob_trans_ctx.cpp b/src/storage/tx/ob_trans_ctx.cpp index 4baa235b8..c19b43ab7 100644 --- a/src/storage/tx/ob_trans_ctx.cpp +++ b/src/storage/tx/ob_trans_ctx.cpp @@ -45,7 +45,6 @@ void ObTransCtx::print_trace_log_() void ObTransCtx::before_unlock(CtxLockArg &arg) { arg.trans_id_ = trans_id_; - arg.ls_id_ = ls_id_; opid_++; if (has_pending_callback_) { arg.commit_cb_ = commit_cb_; @@ -123,7 +122,7 @@ void ObTransCtx::print_trace_log_if_necessary_() static const int64_t SAMPLING_SEED = 128 * 64; // freectx if (!is_exiting_) { - TRANS_LOG_RET(WARN, OB_ERROR, "ObPartTransCtx not exiting", "context", *this, K(lbt())); + TRANS_LOG_RET(WARN, OB_ERROR, "ObTxCtx not exiting", "context", *this, K(lbt())); FORCE_PRINT_TRACE(tlog_, "[trans debug] "); } @@ -191,14 +190,13 @@ MonotonicTs ObTransCtx::get_stc_() -bool ObTransCtx::has_callback_scheduler_() +bool ObTransCtx::has_commit_callback_() { return !commit_cb_.is_enabled() // callback scheduler has accomplished by others || commit_cb_.is_inited(); // callback has been defered } -// callback scheduler commit result -int ObTransCtx::defer_callback_scheduler_(const int retcode, const SCN &commit_version) +int ObTransCtx::defer_commit_callback_(const int retcode, const SCN &commit_version) { int ret = OB_SUCCESS; if (!commit_cb_.is_enabled()) { @@ -294,17 +292,22 @@ int ObTransCtx::unregister_timeout_task_() return ret; } -void ObTransCtx::update_trans_2pc_timeout_() +void ObTransCtx::update_commit_retry_timeout_() { - const int64_t timeout_new = 2 * trans_2pc_timeout_; + const int64_t timeout_new = 2 * commit_retry_timeout_; - if (MAX_TRANS_2PC_TIMEOUT_US > timeout_new) { - trans_2pc_timeout_ = timeout_new; + if (MAX_TRANS_COMMIT_RETRY_TIMEOUT_US > timeout_new) { + commit_retry_timeout_ = timeout_new; } else { - trans_2pc_timeout_ = MAX_TRANS_2PC_TIMEOUT_US; + commit_retry_timeout_ = MAX_TRANS_COMMIT_RETRY_TIMEOUT_US; } } +int64_t ObTransCtx::get_commit_retry_interval_us_() +{ + return ObServerConfig::get_instance().trx_commit_retry_interval; +} + int ObTransCtx::set_app_trace_info_(const ObString &app_trace_info) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_trans_ctx.h b/src/storage/tx/ob_trans_ctx.h index d5c86bc43..d85c35f9f 100644 --- a/src/storage/tx/ob_trans_ctx.h +++ b/src/storage/tx/ob_trans_ctx.h @@ -25,9 +25,8 @@ #include "lib/list/ob_dlist.h" #include "lib/list/ob_dlink_node.h" #include "share/ob_cluster_version.h" -#include "ob_location_adapter.h" -#include "ob_trans_rpc.h" #include "ob_trans_define.h" +#include "ob_trans_factory.h" #include "ob_trans_timer.h" #include "ob_trans_result.h" #include "ob_trans_submit_log_cb.h" @@ -83,12 +82,12 @@ static inline void protocol_error(const int64_t state, const int64_t msg_type) // change one of the signature of `ObTransCtx`. // For Example: If you change the signature of the function `commit` in // `ObTransCtx`, you should also modify the signatore of function `commit` in -// `ObPartTransCtx`, `ObScheTransCtx` +// `ObTxCtx`, `ObScheTransCtx` class ObTransCtx: public share::ObLightHashLink { friend class CtxLock; public: - ObTransCtx(const char *ctx_type_str = "unknow", const int64_t ctx_type = ObTransCtxType::UNKNOWN) + ObTransCtx() : trans_expired_time_(0), ctx_create_time_(0), trans_service_(NULL), tlog_(NULL), cluster_version_(0), ls_tx_ctx_mgr_(NULL), @@ -96,7 +95,6 @@ class ObTransCtx: public share::ObLightHashLink client_sid_(UINT32_MAX), associated_session_id_(UINT32_MAX), stc_(0), part_trans_action_(ObPartTransAction::UNKNOWN), - callback_scheduler_on_clear_(false), pending_callback_param_(common::OB_SUCCESS), p_mt_ctx_(NULL), is_exiting_(false), for_replay_(false), has_pending_callback_(false), @@ -113,7 +111,6 @@ class ObTransCtx: public share::ObLightHashLink bool is_readonly() const { return false; } void set_for_replay(const bool for_replay) { for_replay_ = for_replay; } bool is_for_replay() const { return for_replay_; } - const share::ObLSID &get_ls_id() const { return ls_id_; } int set_ls_tx_ctx_mgr(ObLSTxCtxMgr *ls_tx_ctx_mgr) { ls_tx_ctx_mgr_ = ls_tx_ctx_mgr; @@ -121,11 +118,6 @@ class ObTransCtx: public share::ObLightHashLink } ObLSTxCtxMgr *get_ls_tx_ctx_mgr() { return ls_tx_ctx_mgr_; } ObTransService *get_trans_service() { return trans_service_; } - int set_ls_id(const share::ObLSID &ls_id) - { - ls_id_ = ls_id; - return common::OB_SUCCESS; - } int set_trans_id(const ObTransID &trans_id) { int ret = OB_SUCCESS; @@ -156,7 +148,6 @@ class ObTransCtx: public share::ObLightHashLink virtual int64_t get_part_trans_action() const { return part_trans_action_; } int acquire_ctx_ref() { return acquire_ctx_ref_(); } void release_ctx_ref(); - ObITransRpc *get_trans_rpc() const { return rpc_; } public: virtual bool is_inited() const = 0; virtual int handle_timeout(const int64_t delay) = 0; @@ -180,8 +171,8 @@ class ObTransCtx: public share::ObLightHashLink void set_stc_(const MonotonicTs stc); void set_stc_by_now_(); MonotonicTs get_stc_(); - bool has_callback_scheduler_(); - int defer_callback_scheduler_(const int ret, const share::SCN &commit_version); + bool has_commit_callback_(); + int defer_commit_callback_(const int ret, const share::SCN &commit_version); int prepare_commit_cb_for_role_change_(const int cb_ret, ObTxCommitCallback *&cb_list); int64_t get_remaining_wait_interval_us_() { @@ -207,7 +198,8 @@ class ObTransCtx: public share::ObLightHashLink virtual int register_timeout_task_(const int64_t interval_us); virtual int unregister_timeout_task_(); void generate_request_id_(); - void update_trans_2pc_timeout_(); + void update_commit_retry_timeout_(); + static int64_t get_commit_retry_interval_us_(); int set_app_trace_info_(const ObString &app_trace_info); int set_app_trace_id_(const ObString &app_trace_id); public: @@ -218,22 +210,18 @@ class ObTransCtx: public share::ObLightHashLink static const int64_t RP_TOTAL_NUM = 100 * 1000; // if 600 seconds after trans timeout, warn is required static const int64_t OB_TRANS_WARN_USE_TIME = 600 * 1000 * 1000; - static const int64_t MAX_TRANS_2PC_TIMEOUT_US = 3 * 1000 * 1000; // 3s + static const int64_t MAX_TRANS_COMMIT_RETRY_TIMEOUT_US = 3 * 1000 * 1000; // 3s protected: //0x0078746365657266 means freectx static const int64_t UNKNOWN_FREE_CTX_MAGIC_NUM = 0x0078746365657266; //0x006e776f6e6b6e75 means resetctx static const int64_t UNKNOWN_RESET_CTX_MAGIC_NUM = 0x006e776f6e6b6e75; static const int64_t CHECK_GC_PARTITION_INTERVAL = 10 * 1000 * 1000; - // the time interval of asking scheduler for participant - static const int64_t CHECK_SCHEDULER_STATUS_INTERVAL = 10 * 1000 * 1000; - // the time interval of asking scheduler for rs - static const int64_t CHECK_RS_SCHEDULER_STATUS_INTERVAL = 60 * 1000 * 1000; + static const int64_t CHECK_TX_STATUS_INTERVAL = 10 * 1000 * 1000; private: DISALLOW_COPY_AND_ASSIGN(ObTransCtx); protected: - share::ObLSID ls_id_; ObTransID trans_id_; common::ObAddr addr_; @@ -255,8 +243,6 @@ class ObTransCtx: public share::ObLightHashLink // the variable is used to record the action of the current transaction in the stmt execution int64_t part_trans_action_; ObTxCommitCallback commit_cb_; - // [only used by mysqltest]: will callback scheduler when clear log is persistented - bool callback_scheduler_on_clear_; ObTransNeedWaitWrap trans_need_wait_wrap_; int pending_callback_param_; // it is used to wake up the lock queue after submitting the log of elr trans @@ -270,8 +256,7 @@ class ObTransCtx: public share::ObLightHashLink int64_t opid_; int64_t request_id_; - ObITransRpc *rpc_; - int64_t trans_2pc_timeout_; + int64_t commit_retry_timeout_; ObTransTimeoutTask timeout_task_; ObITransTimer *timer_; ObTraceInfo trace_info_; @@ -287,7 +272,7 @@ class TransCtxAlloc void free_value(ObTransCtx* ctx) { if (NULL != ctx) { - ObTransCtxFactory::release(ctx); + ObTxCtxFactory::release(ctx); } } }; diff --git a/src/storage/tx/ob_trans_ctx_lock.cpp b/src/storage/tx/ob_trans_ctx_lock.cpp index e00d63f4b..dc1764118 100644 --- a/src/storage/tx/ob_trans_ctx_lock.cpp +++ b/src/storage/tx/ob_trans_ctx_lock.cpp @@ -168,7 +168,7 @@ void CtxLock::unlock_ctx() const int64_t lock_ts = ObClockGenerator::getClock() - lock_start_ts; if (lock_ts > WARN_LOCK_TS) { TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "ctx lock too much time", K(arg.trans_id_), - K(arg.ls_id_), K(lock_ts), K(lbt())); + K(lock_ts), K(lbt())); } } after_unlock(arg); diff --git a/src/storage/tx/ob_trans_ctx_lock.h b/src/storage/tx/ob_trans_ctx_lock.h index a04375b4b..0d8f6166c 100644 --- a/src/storage/tx/ob_trans_ctx_lock.h +++ b/src/storage/tx/ob_trans_ctx_lock.h @@ -46,11 +46,10 @@ typedef common::ObDList LocalTaskList; class CtxLockArg { public: - CtxLockArg() : ls_id_(), trans_id_(), task_list_(), commit_cb_(), + CtxLockArg() : trans_id_(), task_list_(), commit_cb_(), has_pending_callback_(false), need_retry_redo_sync_(false), p_mt_ctx_(NULL) {} public: - share::ObLSID ls_id_; ObTransID trans_id_; LocalTaskList task_list_; ObTxCommitCallback commit_cb_; diff --git a/src/storage/tx/ob_trans_ctx_mgr.h b/src/storage/tx/ob_trans_ctx_mgr.h index 81ef19972..8e0d3183d 100644 --- a/src/storage/tx/ob_trans_ctx_mgr.h +++ b/src/storage/tx/ob_trans_ctx_mgr.h @@ -24,13 +24,11 @@ #include "lib/hash/ob_link_hashmap.h" #include "lib/container/ob_se_array.h" #include "common/ob_simple_iterator.h" -#include "share/ob_ls_id.h" #include "share/ob_light_hashmap.h" #include "storage/memtable/ob_memtable_context.h" #include "ob_trans_ctx.h" #include "ob_trans_stat.h" -#include "storage/tx/ob_tx_ls_log_writer.h" #include "storage/tx_table/ob_tx_table_define.h" #include "storage/tx/ob_tx_log_adapter.h" #include "ob_trans_ctx_mgr_v4.h" diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index 3538171d1..f64fd0157 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -37,12 +37,6 @@ using namespace observer; namespace transaction { -#ifdef ERRSIM -ERRSIM_POINT_DEF(EN_SWITCH_TO_FOLLOWER_GRACEFULLY) -ERRSIM_POINT_DEF(EN_SUBMIT_START_WORKING_LOG) -ERRSIM_POINT_DEF(EN_APPLY_START_WORKING_LOG) -#endif - void ObLSTxCtxIterator::reset() { is_ready_ = false; current_bucket_pos_ = -1; @@ -66,7 +60,7 @@ int ObLSTxCtxIterator::set_ready(ObLSTxCtxMgr* ls_tx_ctx_mgr) return ret; } -int ObLSTxCtxIterator::get_next_tx_ctx(ObPartTransCtx *&tx_ctx) +int ObLSTxCtxIterator::get_next_tx_ctx(ObTxCtx *&tx_ctx) { TRANS_LOG(DEBUG, "ObLSTxCtxIterator::get_next_tx_ctx start"); @@ -101,7 +95,7 @@ int ObLSTxCtxIterator::get_next_tx_ctx(ObPartTransCtx *&tx_ctx) return ret; } -int ObLSTxCtxIterator::revert_tx_ctx(ObPartTransCtx *tx_ctx) +int ObLSTxCtxIterator::revert_tx_ctx(ObTxCtx *tx_ctx) { int ret = OB_SUCCESS; @@ -160,8 +154,7 @@ int ObLSTxCtxIterator::get_next_tx_id_(ObTransID& tx_id) return ret; } -OB_WEAK_SYMBOL int ObLSTxCtxMgr::init(const ObLSID &ls_id, - ObTxTable *tx_table, +OB_WEAK_SYMBOL int ObLSTxCtxMgr::init(ObTxTable *tx_table, ObLockTable *lock_table, ObTsMgr *ts_mgr, ObTransService *txs, @@ -173,8 +166,8 @@ OB_WEAK_SYMBOL int ObLSTxCtxMgr::init(const ObLSID &ls_id, if (IS_INIT) { TRANS_LOG(WARN, "ObLSTxCtxMgr inited twice"); ret = OB_INIT_TWICE; - } else if (OB_UNLIKELY(!ls_id.is_valid()) || OB_ISNULL(ts_mgr) || OB_ISNULL(txs)) { - TRANS_LOG(WARN, "invalid argument", K(ls_id), KP(ts_mgr), KP(txs)); + } else if (OB_ISNULL(ts_mgr) || OB_ISNULL(txs)) { + TRANS_LOG(WARN, "invalid argument", KP(ts_mgr), KP(txs)); ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(ls_tx_ctx_map_.init(lib::ObMemAttr("LSTxCtxMgr")))) { TRANS_LOG(WARN, "ctx_map_mgr init fail", KR(ret)); @@ -182,19 +175,14 @@ OB_WEAK_SYMBOL int ObLSTxCtxMgr::init(const ObLSID &ls_id, TRANS_LOG(WARN, "tx log adapter init error", KR(ret)); } else if (OB_NOT_NULL(log_adapter) && OB_FALSE_IT(tx_log_adapter_ = log_adapter)) { ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(log_cb_pool_mgr_.init(ls_id))) { - TRANS_LOG(WARN, "init log_cb_pool_mgr failed", K(ret), K(ls_id_), K(log_cb_pool_mgr_)); - } else if (OB_FAIL(ls_log_writer_.init(ls_id, tx_log_adapter_, this))) { - TRANS_LOG(WARN, "ls_log_writer init fail", KR(ret)); - } else if (OB_FAIL(tx_ls_state_mgr_.init(ls_id))) { - TRANS_LOG(WARN, "init tx_ls_state_mgr_ failed", KR(ret)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::START))) { - TRANS_LOG(WARN, "start ls_tx_ctx_mgr failed",K(ret),K(tx_ls_state_mgr_)); + } else if (OB_FAIL(log_cb_pool_mgr_.init())) { + TRANS_LOG(WARN, "init log_cb_pool_mgr failed", K(ret), K(log_cb_pool_mgr_)); } else { is_inited_ = true; - is_leader_serving_ = false; - - ls_id_ = ls_id; + stopped_ = false; + block_tx_ = false; + block_normal_tx_ = false; + block_all_ = false; tx_table_ = tx_table; lock_table_ = lock_table; txs_ = txs; @@ -202,7 +190,7 @@ OB_WEAK_SYMBOL int ObLSTxCtxMgr::init(const ObLSID &ls_id, aggre_rec_scn_.reset(); prev_aggre_rec_scn_.reset(); online_ts_ = 0; - TRANS_LOG(INFO, "ObLSTxCtxMgr inited success", KP(this), K(ls_id)); + TRANS_LOG(INFO, "ObLSTxCtxMgr inited success", KP(this)); } return ret; } @@ -212,9 +200,8 @@ void ObLSTxCtxMgr::destroy() WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (IS_INIT) { log_cb_pool_mgr_.destroy(); - ls_log_writer_.destroy(); is_inited_ = false; - TRANS_LOG(INFO, "ObLSTxCtxMgr destroyed", KP(this), K_(ls_id)); + TRANS_LOG(INFO, "ObLSTxCtxMgr destroyed", KP(this)); } } @@ -222,20 +209,20 @@ void ObLSTxCtxMgr::reset() { is_inited_ = false; - ls_id_.reset(); tx_table_ = NULL; lock_table_ = NULL; total_tx_ctx_count_ = 0; active_tx_count_ = 0; total_active_readonly_request_count_ = 0; - leader_takeover_ts_.reset(); - max_replay_commit_version_.reset(); + stopped_ = true; + block_tx_ = false; + block_normal_tx_ = false; + block_all_ = false; aggre_rec_scn_.reset(); prev_aggre_rec_scn_.reset(); online_ts_ = 0; txs_ = NULL; ts_mgr_ = NULL; - tx_ls_state_mgr_.reset(); ls_retain_ctx_mgr_.reset(); ObRemoveAllTxCtxFunctor fn; @@ -280,13 +267,13 @@ void ObLSTxCtxMgr::print_all_tx_ctx_(const int64_t max_print, const bool verbose int ObLSTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, bool& existed, - ObPartTransCtx *&ctx) + ObTxCtx *&ctx) { int ret = OB_SUCCESS; RLockGuard guard(rwlock_); if (OB_FAIL(create_tx_ctx_(arg, existed, ctx))) { - TRANS_LOG(DEBUG, "create tx ctx error", K(arg), K(existed), K_(ls_id)); + TRANS_LOG(DEBUG, "create tx ctx error", K(arg), K(existed)); } else { // do nothing } @@ -296,18 +283,17 @@ int ObLSTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, int ObLSTxCtxMgr::create_tx_ctx_(const ObTxCreateArg &arg, bool& exist, - ObPartTransCtx *&ctx) + ObTxCtx *&ctx) { int ret = OB_SUCCESS; ObTransCtx *tmp_ctx = NULL, *exist_ctx = NULL; - bool leader = false, insert_succ = false; - int64_t epoch = 0; + bool insert_succ = false; bool block = false; if (is_tx_blocked_()) { block = true; } else if (is_normal_blocked_()) { - if (arg.ctx_source_ != PartCtxSource::REGISTER_MDS) { + if (arg.ctx_source_ != TxCtxSource::REGISTER_MDS) { block = true; } } @@ -319,50 +305,26 @@ int ObLSTxCtxMgr::create_tx_ctx_(const ObTxCreateArg &arg, } else if (OB_UNLIKELY(!arg.is_valid())) { TRANS_LOG(WARN, "invalid argument", K(arg), KP(ts_mgr_)); ret = OB_INVALID_ARGUMENT; - } else if (!arg.for_replay_ && !is_master_()) { - ret = OB_NOT_MASTER; } else if (!arg.for_replay_ && block) { - TRANS_LOG(WARN, "ObLSTxCtxMgr is blocked", K(arg), K(tx_ls_state_mgr_)); + TRANS_LOG(WARN, "ObLSTxCtxMgr is blocked", K(arg)); ret = OB_PARTITION_IS_BLOCKED; } else if (is_stopped_()) { TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped", K(arg)); ret = OB_PARTITION_IS_STOPPED; - } else if (is_master_() && OB_FAIL(try_wait_gts_and_inc_max_commit_ts_())) { - TRANS_LOG(WARN, "switch_to_leader processing is not finished", KR(ret), K(ls_id_)); - } else if (!arg.for_replay_ && OB_FAIL(tx_log_adapter_->get_role(leader, epoch))) { - TRANS_LOG(WARN, "get replica role fail", K(ret)); - } else if (!arg.for_replay_ && !leader) { - ret = OB_NOT_MASTER; - TRANS_LOG(WARN, "replica not leader", K(ret)); - } else if (OB_ISNULL(tmp_ctx = ObTransCtxFactory::alloc(ObTransCtxType::PARTICIPANT))) { + } else if (OB_ISNULL(tmp_ctx = ObTxCtxFactory::alloc())) { TRANS_LOG(WARN, "alloc transaction context error", K(arg)); ret = OB_ALLOCATE_MEMORY_FAILED; } else { - int64_t epoch_v = 0; - if (arg.epoch_ > 0) { - epoch_v = arg.epoch_; - } else { - // For old-version compatibility, follower epoch can be 0, so no check is needed. - if (!arg.for_replay_) { - // pack `epoch(15bit) | ts_ns(48bit)` into int64_t, set most significant bit to zero - epoch_v = ~(1ULL << 63) & ((epoch << 48) | (ObTimeUtility::current_time_ns() & ~(0xFFFFULL << 48))); - } else { - epoch_v = -1; - } - } CtxLockGuard ctx_lock_guard; - ObPartTransCtx *tmp = static_cast(tmp_ctx); - if (OB_FAIL(tmp->init(arg.scheduler_, - arg.session_id_, + ObTxCtx *tmp = static_cast(tmp_ctx); + if (OB_FAIL(tmp->init(arg.session_id_, arg.client_sid_, arg.associated_session_id_, arg.tx_id_, arg.trans_expired_time_, - arg.ls_id_, arg.cluster_version_, arg.trans_service_, arg.cluster_id_, - epoch_v, this, arg.for_replay_, arg.ctx_source_, @@ -394,7 +356,7 @@ int ObLSTxCtxMgr::create_tx_ctx_(const ObTxCreateArg &arg, ls_tx_ctx_map_.revert(tmp_ctx); } else { tmp_ctx->set_exiting(); - ObTransCtxFactory::release(tmp_ctx); + ObTxCtxFactory::release(tmp_ctx); } } // if exist, wait ctx create done @@ -402,15 +364,15 @@ int ObLSTxCtxMgr::create_tx_ctx_(const ObTxCreateArg &arg, ret = OB_SUCCESS; CtxLockGuard ctx_guard; exist_ctx->get_ctx_guard(ctx_guard); - ctx = static_cast(exist_ctx); + ctx = static_cast(exist_ctx); } if (REACH_TIME_INTERVAL(OB_TRANS_STATISTICS_INTERVAL)) { - TRANS_LOG(INFO, "transaction statistics", K_(ls_id), "total_count", get_tx_ctx_count_()); + TRANS_LOG(INFO, "transaction statistics", "total_count", get_tx_ctx_count_()); } return ret; } -int ObLSTxCtxMgr::get_tx_ctx(const ObTransID &tx_id, const bool for_replay, ObPartTransCtx *&ctx) +int ObLSTxCtxMgr::get_tx_ctx(const ObTransID &tx_id, const bool for_replay, ObTxCtx *&ctx) { int ret = OB_SUCCESS; RLockGuard guard(rwlock_); @@ -425,7 +387,7 @@ int ObLSTxCtxMgr::get_tx_ctx(const ObTransID &tx_id, const bool for_replay, ObPa int ObLSTxCtxMgr::get_tx_ctx_with_timeout(const ObTransID &tx_id, const bool for_replay, - ObPartTransCtx *&tx_ctx, + ObTxCtx *&tx_ctx, const int64_t lock_timeout) { int ret = OB_SUCCESS; @@ -445,7 +407,7 @@ int ObLSTxCtxMgr::get_tx_ctx_with_timeout(const ObTransID &tx_id, return ret; } -int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObPartTransCtx *&ctx) +int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObTxCtx *&ctx) { int ret = OB_SUCCESS; ObTransCtx *tmp_ctx = NULL; @@ -457,10 +419,8 @@ int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObP TRANS_LOG(WARN, "ObLSTxCtxMgr not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(!tx_id.is_valid()) || OB_ISNULL(ts_mgr_)) { - TRANS_LOG(WARN, "invalid argument", K_(ls_id), K(tx_id), KP(ts_mgr_)); + TRANS_LOG(WARN, "invalid argument", K(tx_id), KP(ts_mgr_)); ret = OB_INVALID_ARGUMENT; - } else if (!for_replay && !is_master_()) { - ret = OB_NOT_MASTER; } else { if (OB_SUCC(ls_tx_ctx_map_.get(tx_id, tmp_ctx))) { if (OB_ISNULL(tmp_ctx)) { @@ -473,7 +433,7 @@ int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObP // ls_tx_ctx_map_.revert(tmp_ctx); // tmp_ctx = NULL; // } - ctx = static_cast(tmp_ctx); + ctx = static_cast(tmp_ctx); } } else if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_TRANS_CTX_NOT_EXIST; @@ -481,14 +441,12 @@ int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObP TRANS_LOG(ERROR, "get transaction context error", KR(ret), K(tx_id)); } if (REACH_TIME_INTERVAL(OB_TRANS_STATISTICS_INTERVAL)) { - TRANS_LOG(INFO, "transaction statistics", K_(ls_id), - "total_tx_ctx_count", get_tx_ctx_count_()); + TRANS_LOG(INFO, "transaction statistics", "total_tx_ctx_count", get_tx_ctx_count_()); } #ifdef ENABLE_DEBUG_LOG // ENABLE_DEBUG_LOG macro only defined in inner test environment if (REACH_TIME_INTERVAL(3 * 60 * 1000 * 1000 /*3 min*/)) { - TRANS_LOG(INFO, "transaction statistics", K_(ls_id), - "total_tx_ctx_count", get_tx_ctx_count_(), K(lbt())); + TRANS_LOG(INFO, "transaction statistics", "total_tx_ctx_count", get_tx_ctx_count_(), K(lbt())); } #endif } @@ -509,7 +467,7 @@ int ObLSTxCtxMgr::iterator_tx_id_in_one_bucket(ObTxIDIterator& iter, int bucket_ } -int ObLSTxCtxMgr::get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObPartTransCtx *&ctx) +int ObLSTxCtxMgr::get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObTxCtx *&ctx) { int ret = OB_SUCCESS; ObTransCtx *tmp_ctx = NULL; @@ -518,7 +476,7 @@ int ObLSTxCtxMgr::get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObPa TRANS_LOG(WARN, "ObLSTxCtxMgr not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(!tx_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K_(ls_id), K(tx_id)); + TRANS_LOG(WARN, "invalid argument", K(tx_id)); ret = OB_INVALID_ARGUMENT; } else { if (OB_FAIL(ls_tx_ctx_map_.get(tx_id, tmp_ctx))) { @@ -528,7 +486,7 @@ int ObLSTxCtxMgr::get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObPa TRANS_LOG(ERROR, "get transaction context error", KR(ret), K(tx_id)); } } else { - ctx = static_cast(tmp_ctx); + ctx = static_cast(tmp_ctx); } } return ret; @@ -540,11 +498,11 @@ int ObLSTxCtxMgr::remove_callback_for_uncommited_tx(const memtable::ObMemtableSe ObTimeGuard timeguard("remove callback for uncommited txn", 10L * 1000L); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObLSTxCtxMgr not inited", K_(ls_id)); + TRANS_LOG(WARN, "ObLSTxCtxMgr not inited"); ret = OB_NOT_INIT; } else if (OB_ISNULL(memtable_set)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "memtable is null", K_(ls_id)); + TRANS_LOG(WARN, "memtable is null"); } else { ObRemoveCallbackFunctor fn(memtable_set); if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { @@ -556,411 +514,6 @@ int ObLSTxCtxMgr::remove_callback_for_uncommited_tx(const memtable::ObMemtableSe return ret; } -int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, SCN start_working_ts) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - UNUSED(log); - - share::SCN tmp_applying_swl_scn; - if (OB_FAIL(retry_apply_start_working_log())) { - TRANS_LOG(WARN, "retry to apply start working log failed", K(ret), K(log), K(start_working_ts), - KPC(this)); - ret = OB_EAGAIN; - } else if (tx_ls_state_mgr_.need_retry_apply_SWL(tmp_applying_swl_scn)) - - { - if (tmp_applying_swl_scn < start_working_ts) { - ret = OB_EAGAIN; - TRANS_LOG(WARN, "need retry to apply prev start working log", K(ret), K(tmp_applying_swl_scn), - K(start_working_ts), K(log), KPC(this)); - } else { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "the applying SWL scn is larger than replaying SWL scn", K(ret), - K(tmp_applying_swl_scn), K(start_working_ts), K(log), KPC(this)); - } - - } else { - - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - ReplayTxStartWorkingLogFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - FLOG_WARN("[LsTxCtxMgr Role Change] replay start working log failed", KR(ret), - K(ls_id_)); - } else { - tx_ls_state_mgr_.replay_SWL_succ(start_working_ts); - FLOG_INFO("[LsTxCtxMgr Role Change] replay start working log success", - K(ls_id_)); - } - } - return ret; -} - -int ObLSTxCtxMgr::retry_apply_start_working_log() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - share::SCN retry_start_working_ts; - - if (tx_ls_state_mgr_.need_retry_apply_SWL(retry_start_working_ts)) { - ret = on_start_working_log_cb_succ(retry_start_working_ts); - } - - if (OB_TMP_FAIL(log_cb_pool_mgr_.clear_log_cb_pool(false /*for_replay*/))) { - TRANS_LOG(WARN, "clear log cb pool failed", K(ret), K(ls_id_), K(log_cb_pool_mgr_)); - } - return ret; -} - -int ObLSTxCtxMgr::on_start_working_log_cb_succ(SCN start_working_ts) -{ - int ret = OB_SUCCESS; - bool ignore_ret = false; - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - - share::SCN retry_start_working_ts; - if (!tx_ls_state_mgr_.waiting_SWL_cb()) { - TRANS_LOG(INFO, "This ls is not waiting start_working_cb. Skip the on_success operation", - K(ret), K(tx_ls_state_mgr_)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_SUCC, - start_working_ts))) { - TRANS_LOG(ERROR, "switch state fail", KR(ret), K(ls_id_)); - } - - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(errsim_apply_start_working_log())) { - TRANS_LOG(WARN, "errsim in on_start_working_log_cb_succ", K(ret), K(ls_id_)); - } - - if (!tx_ls_state_mgr_.need_retry_apply_SWL(retry_start_working_ts)) { - TRANS_LOG(INFO, "This ls need not retry apply start_working. Skip the apply operation", K(ret), - K(tx_ls_state_mgr_)); - } else { - if (OB_FAIL(ret)) { - // do nothing - } else if (is_t_pending_()) { - SwitchToLeaderFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "switch to leader failed", KR(ret), K(ls_id_)); - if (OB_NOT_MASTER == fn.get_ret()) { - // ignore ret - // PALF will switch to follower when submitting log return OB_NOT_MASTER - ignore_ret = true; - } - } - } else if (is_r_pending_()) { - ResumeLeaderFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(ls_id_)); - } - } else if (is_f_pending_()) { - TRANS_LOG(INFO, - "retry to apply start working on a follower", - KR(ret), - K(ls_id_), - K(tx_ls_state_mgr_)); - } else { - ret = OB_STATE_NOT_MATCH; - TRANS_LOG(ERROR, "unexpected state", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); - } - - if (OB_FAIL(ret)) { - if (ignore_ret) { - ret = OB_SUCCESS; - } - // TODO dingxi, takeover failed, notify palf to revoke itself - int tmp_ret = OB_SUCCESS; - // restore to follower - if (OB_TMP_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( - ObTxLSStateMgr::TxLSAction::APPLY_SWL_FAIL, start_working_ts))) { - TRANS_LOG(ERROR, "restore follower failed", KR(tmp_ret), K(ls_id_), K(tx_ls_state_mgr_)); - ret = tmp_ret; - } - } else { - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::APPLY_SWL_SUCC, - start_working_ts))) { - TRANS_LOG(ERROR, "switch state failed", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); - } else { - leader_takeover_ts_ = MonotonicTs::current_time(); - try_wait_gts_and_inc_max_commit_ts_(); - } - } - } - FLOG_INFO("[LsTxCtxMgr Role Change] on_start_working_log_cb_succ", K(ret), K(start_working_ts), - KPC(this)); - return ret; -} - -int ObLSTxCtxMgr::on_start_working_log_cb_fail() -{ - int ret = OB_SUCCESS; - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_FAIL))) { - TRANS_LOG(WARN, "switch state fail", KR(ret), K(ls_id_)); - } - FLOG_INFO("[LsTxCtxMgr Role Change] on_start_working_log_cb_fail", K(ret), KPC(this)); - return ret; -} - -int ObLSTxCtxMgr::submit_start_working_log_() -{ - int ret = OB_SUCCESS; - SCN scn; - const int64_t fake_epoch = 0xbaba; - if (OB_FAIL(errsim_submit_start_working_log())) { - TRANS_LOG(WARN, "errsim in submit start working log failed", K(ret), KPC(this)); - } else if (OB_FAIL(ls_log_writer_.submit_start_working_log(fake_epoch, scn))) { - TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); - } - return ret; -} - -int ObLSTxCtxMgr::switch_to_follower_forcedly() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObTimeGuard timeguard("ObLSTxCtxMgr::switch_to_follower_forcedly"); - ObTxCommitCallback *cb_list = NULL; - { - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (IS_NOT_INIT) { - TRANS_LOG(ERROR, "ObLSTxCtxMgr not inited", K(ls_id_)); - ret = OB_NOT_INIT; - } else if (is_follower_()) { - // already follower, do nothing - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::LEADER_REVOKE_FORCEDLLY))) { - TRANS_LOG(ERROR, "switch state error", KR(ret), "manager", *this); - } else { - SwitchToFollowerForcedlyFunctor fn(cb_list); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(ERROR, "for each transaction context error", KR(ret), "manager", *this); - } else { - is_leader_serving_ = false; - } - - if (OB_FAIL(ret)) { - tx_ls_state_mgr_.restore_tx_ls_state(); - } - } - } - if (OB_TMP_FAIL(log_cb_pool_mgr_.clear_log_cb_pool(false /*for_replay*/))) { - TRANS_LOG(WARN, "clear log cb pool failed", K(ret), K(ls_id_), K(log_cb_pool_mgr_)); - } - timeguard.click(); - // run callback out of lock, ignore ret - (void)process_callback_(cb_list); - if (timeguard.get_diff() > 3 * 1000000) { - TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "switch_to_follower_forcedly use too much time", K(timeguard), "manager", *this); - } - FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_follower_forcedly", K(ret), KPC(this)); - return ret; -} - -int ObLSTxCtxMgr::try_wait_gts_and_inc_max_commit_ts_() -{ - int ret = OB_SUCCESS; - if (!is_leader_serving_) { - SCN gts; - MonotonicTs receive_gts_ts(0); - - if (OB_FAIL(ts_mgr_->get_gts(leader_takeover_ts_, - nullptr, - gts, - receive_gts_ts))) { - if(OB_EAGAIN != ret) { - TRANS_LOG(WARN, "wait gts error", KR(ret), K_(ls_id), - K_(max_replay_commit_version)); - } else { - ret = OB_NOT_MASTER; - } - } else { - if (max_replay_commit_version_.is_valid() && max_replay_commit_version_ >= gts) { - ret = OB_NOT_MASTER; - } else { - is_leader_serving_ = true; - txs_->get_tx_version_mgr().update_max_commit_ts(gts, false); - TRANS_LOG(INFO, "skip waiting gts when takeover", - K(ls_id_), K(max_replay_commit_version_), K(gts)); - } - } - TRANS_LOG(INFO, "try wait gts", KR(ret), K_(ls_id), - K_(max_replay_commit_version), K(gts)); - } - return ret; -} - -// TODO dingxi, add dup table related logic -int ObLSTxCtxMgr::switch_to_leader() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - - if (OB_FAIL(retry_apply_start_working_log())) { - TRANS_LOG(WARN, "retry to apply prev start working log failed", K(ret), KPC(this)); - } else { - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( - ObTxLSStateMgr::TxLSAction::LEADER_TAKEOVER))) { - TRANS_LOG(WARN, "switch state failed", KR(ret), K(ls_id_)); - } else { - if (OB_FAIL(submit_start_working_log_())) { - TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(ls_id_)); - } - if (OB_FAIL(ret)) { - tx_ls_state_mgr_.restore_tx_ls_state(); - } - } - } - - if (OB_TMP_FAIL(log_cb_pool_mgr_.switch_to_leader(get_active_tx_count()))) { - TRANS_LOG(WARN, "switch to leader failed in log_cb_pool_mgr", K(ret), K(tmp_ret), K(ls_id_)); - } - - FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_leader", K(ret), KPC(this)); - return ret; -} - -int ObLSTxCtxMgr::switch_to_follower_gracefully() -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObTimeGuard timeguard("switch_to_follower_gracefully"); - int64_t start_time = ObTimeUtility::current_time(); - int64_t process_count = 0; - while (OB_SUCC(ret) && is_pending_()) { - if (ObTimeUtility::current_time() - start_time >= WAIT_SW_CB_TIMEOUT) { - ret = OB_TIMEOUT; - TRANS_LOG(WARN, "start working cb waiting timeout", K(ret), KPC(this)); - if (tx_ls_state_mgr_.is_start_working_apply_pending()) { - ret = OB_LS_NEED_REVOKE; - TRANS_LOG(WARN, "apply start working log failed with waiting timeout, need revoke", K(ret), KPC(this)); - } - } else { - ob_usleep(WAIT_SW_CB_INTERVAL); - } - } - timeguard.click(); - - ObTxCommitCallback *cb_list = NULL; - if (OB_SUCC(ret)) { - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - timeguard.click(); - - if (OB_FAIL(ret)) { - // do nothing - } else if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( - ObTxLSStateMgr::TxLSAction::LEADER_REVOKE_GRACEFULLY))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); - } else { - timeguard.click(); - // TODO - const int64_t abs_expired_time = INT64_MAX; - SwitchToFollowerGracefullyFunctor fn(abs_expired_time, cb_list); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); - ret = fn.get_ret(); - } else if (OB_FAIL(errsim_switch_to_followr_gracefully())) { - TRANS_LOG(WARN, "errsim in switch_to_follower_gracefully", K(ret), - K(ls_id_)); - } - process_count = fn.get_count(); - timeguard.click(); - if (OB_FAIL(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL( - tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::RESUME_LEADER))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); - } else if (OB_TMP_FAIL(submit_start_working_log_())) { - TRANS_LOG(WARN, "submit start working log failed", KR(tmp_ret), K(*this)); - } - if (OB_SUCCESS != tmp_ret) { - //Use a processing method that is compatible with the old code, - //treating the situation as the on_failure of a start working log. - tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_FAIL); - ret = OB_LS_NEED_REVOKE; - } - TRANS_LOG(WARN, "switch to follower failed", KR(ret), KR(tmp_ret), K(*this)); - } else { - is_leader_serving_ = false; - // TRANS_LOG(INFO, "switch to follower gracefully success", K(*this)); - } - timeguard.click(); - } - } - - if (OB_TMP_FAIL(log_cb_pool_mgr_.clear_log_cb_pool(false /*for_replay*/))) { - TRANS_LOG(WARN, "clear log cb pool failed", K(ret), K(tmp_ret), K(ls_id_), K(log_cb_pool_mgr_)); - } - - (void)process_callback_(cb_list); - timeguard.click(); - FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_follower_gracefully", K(ret), KPC(this), - K(process_count)); - if (timeguard.get_diff() > 1000000) { - TRANS_LOG(WARN, "use too much time", K(timeguard), K(process_count)); - } - return ret; -} - -// CALLED when RoleChangeService try to rollback after succeed calling of `switch_to_follower_gracefully` -int ObLSTxCtxMgr::resume_leader() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(retry_apply_start_working_log())) { - TRANS_LOG(WARN, "retry to apply prev start working log failed", K(ret), KPC(this)); - } else { - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( - ObTxLSStateMgr::TxLSAction::RESUME_LEADER))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); - } else { - // previous active info logs will be filter by start_working_ts in part_ctx - if (OB_FAIL(submit_start_working_log_())) { - TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); - } - if (OB_FAIL(ret)) { - tx_ls_state_mgr_.restore_tx_ls_state(); - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(*this)); - } else { - is_leader_serving_ = true; - TRANS_LOG(INFO, "resume leader success", K(*this)); - } - } - } - TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] resume_leader", K(ret), KPC(this)); - return ret; -} - -bool ObLSTxCtxMgr::in_leader_serving_state() -{ - bool bool_ret = false; - int ret = OB_SUCCESS; - RLockGuard guard(rwlock_); - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObLSTxCtxMgr not inited"); - } else if (!is_master_()) { - TRANS_LOG(DEBUG, "ObLSTxCtxMgr not master"); - } else if (OB_FAIL(try_wait_gts_and_inc_max_commit_ts_())) { - TRANS_LOG(WARN, "try_wait_gts_and_inc_max_commit_ts_ failed", K(ret), KPC(this)); - } else { - bool_ret = true; - } - return bool_ret; -} - int ObLSTxCtxMgr::stop(const bool graceful) { int ret = OB_SUCCESS; @@ -973,14 +526,10 @@ int ObLSTxCtxMgr::stop(const bool graceful) if (!graceful && total_active_readonly_request_count > 0) { ret = OB_EAGAIN; TRANS_LOG(WARN, "readonly requests are active", K(ret), K(total_active_readonly_request_count)); - } else if (OB_FAIL(ls_log_writer_.stop())) { - TRANS_LOG(WARN, "ls_log_writer_ stop error", KR(ret)); } else { { WLockGuard guard(minor_merge_lock_); - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::STOP))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); - } + ATOMIC_STORE(&stopped_, true); } if (OB_SUCC(ret)) { @@ -991,9 +540,6 @@ int ObLSTxCtxMgr::stop(const bool graceful) } else if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", *this); } - if (OB_FAIL(ret)) { - tx_ls_state_mgr_.restore_tx_ls_state(); - } } } } @@ -1037,9 +583,8 @@ int ObLSTxCtxMgr::block_tx(bool &is_all_tx_cleaned_up) if (is_stopped_()) { TRANS_LOG(WARN, "ls_tx_ctx_mgr is stopped, not need block"); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_TX))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { + ATOMIC_STORE(&block_tx_, true); is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); } TRANS_LOG(INFO, "block ls", K(ret), "manager", *this); @@ -1053,9 +598,8 @@ int ObLSTxCtxMgr::block_all(bool &is_all_tx_cleaned_up) if (is_stopped_()) { TRANS_LOG(WARN, "ls_tx_ctx_mgr is stopped, not need block"); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_WR))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { + ATOMIC_STORE(&block_all_, true); is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); } TRANS_LOG(INFO, "block ls", K(ret), "manager", *this); @@ -1067,11 +611,8 @@ int ObLSTxCtxMgr::block_normal(bool &is_all_tx_cleaned_up) int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_NORMAL_TX))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); - } else { - is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); - } + ATOMIC_STORE(&block_normal_tx_, true); + is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); TRANS_LOG(INFO, "block ls normally", K(ret), "manager", *this); return ret; } @@ -1081,11 +622,10 @@ int ObLSTxCtxMgr::online() int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::ONLINE))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); - } else { - online_ts_ = ObTimeUtility::current_time(); - } + ATOMIC_STORE(&stopped_, false); + ATOMIC_STORE(&block_tx_, false); + ATOMIC_STORE(&block_all_, false); + online_ts_ = ObTimeUtility::current_time(); TRANS_LOG(INFO, "online ls", K(ret), "manager", *this); return ret; } @@ -1095,14 +635,12 @@ int ObLSTxCtxMgr::unblock_normal() int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::UNBLOCK_NORMAL_TX))) { - TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); - } + ATOMIC_STORE(&block_normal_tx_, false); TRANS_LOG(INFO, "unblock ls normally", K(ret), "manager", *this); return ret; } -int ObLSTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(SCN &min_prepare_version) +int ObLSTxCtxMgr::get_min_uncommit_tx_prepare_version(SCN &min_prepare_version) { int ret = OB_SUCCESS; @@ -1132,12 +670,12 @@ int ObLSTxCtxMgr::get_min_undecided_scn(SCN &scn) return ret; } -int ObLSTxCtxMgr::check_scheduler_status(SCN &min_start_scn, MinStartScnStatus &status) +int ObLSTxCtxMgr::check_tx_status(SCN &min_start_scn, MinStartScnStatus &status) { int ret = OB_SUCCESS; - ObTimeGuard tg("ObLSTxCtxMgr::check_scheduler_status", 100000); + ObTimeGuard tg("ObLSTxCtxMgr::check_tx_status", 100000); - IteratePartCtxAskSchedulerStatusFunctor functor; + IterateTxCtxStatusFunctor functor; if (OB_FAIL(ls_tx_ctx_map_.for_each(functor))) { TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", *this); } else if (!min_start_scn.is_valid()) { @@ -1188,12 +726,13 @@ int ObLSTxCtxMgr::check_modify_schema_elapsed(const ObTabletID &tablet_id, // this interface is called by leader or follower } else { IterateCheckTabletModifySchema fn(tablet_id, schema_version); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", + const int for_each_ret = ls_tx_ctx_map_.for_each(fn); + if (OB_SUCCESS != for_each_ret) { + TRANS_LOG(WARN, "for each transaction context error", K(for_each_ret), "manager", *this); } // NOTE: for_each may return OB_EAGAIN if the iter break but not end. - ret = OB_SUCC(ret) ? fn.get_ret_code() : ret; + ret = OB_SUCCESS != fn.get_ret_code() ? fn.get_ret_code() : for_each_ret; block_tx_id = fn.get_tx_id(); } if (timeguard.get_diff() > 3 * 1000000) { @@ -1217,12 +756,13 @@ int ObLSTxCtxMgr::check_modify_time_elapsed(const ObTabletID &tablet_id, ret = OB_NOT_INIT; } else { IterateCheckTabletModifyTimestamp fn(tablet_id, timestamp); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", + const int for_each_ret = ls_tx_ctx_map_.for_each(fn); + if (OB_SUCCESS != for_each_ret) { + TRANS_LOG(WARN, "for each transaction context error", K(for_each_ret), "manager", *this); } // NOTE: for_each may return OB_EAGAIN if the iter break but not end. - ret = OB_SUCC(ret) ? fn.get_ret_code() : ret; + ret = OB_SUCCESS != fn.get_ret_code() ? fn.get_ret_code() : for_each_ret; block_tx_id = fn.get_tx_id(); } if (timeguard.get_diff() > 3 * 1000000) { @@ -1286,7 +826,7 @@ int ObLSTxCtxMgr::iterate_tx_ctx_stat(ObTxStatIterator &tx_stat_iter) return ret; } -int ObLSTxCtxMgr::revert_tx_ctx(ObPartTransCtx *ctx) +int ObLSTxCtxMgr::revert_tx_ctx(ObTxCtx *ctx) { return revert_tx_ctx_without_lock(ctx); } @@ -1320,7 +860,7 @@ int ObLSTxCtxMgr::del_tx_ctx(ObTransCtx *ctx) } else if (OB_ISNULL(ctx)) { TRANS_LOG(WARN, "invalid argument", KP(ctx)); ret = OB_INVALID_ARGUMENT; - } else if ((static_cast(ctx))->get_retain_cause() != RetainCause::UNKOWN) { + } else if ((static_cast(ctx))->get_retain_cause() != RetainCause::UNKOWN) { ret = OB_SUCCESS; TRANS_LOG(INFO, "can not del a retain ctx", KPC(ctx)); } else { @@ -1335,9 +875,7 @@ int ObLSTxCtxMgr::traverse_tx_to_submit_redo_log(ObTransID &fail_tx_id, const ui int ret = OB_SUCCESS; RLockGuard guard(rwlock_); ObTxSubmitLogFunctor fn(ObTxSubmitLogFunctor::SUBMIT_REDO_LOG, freeze_clock); - if (is_follower_()) { - // quit submit log because this is a follower - } else if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { if (OB_SUCCESS != fn.get_result()) { // get real ret code ret = fn.get_result(); @@ -1357,7 +895,7 @@ int ObLSTxCtxMgr::traverse_tx_to_submit_next_log() int ret = OB_SUCCESS; RLockGuard guard(rwlock_); ObTxSubmitLogFunctor fn(ObTxSubmitLogFunctor::SUBMIT_NEXT_LOG); - if (!is_follower_() && OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { if (OB_SUCCESS != fn.get_result()) { // get real ret code ret = fn.get_result(); @@ -1372,7 +910,7 @@ int ObLSTxCtxMgr::traverse_tx_to_submit_next_log() int ObLSTxCtxMgr::check_with_tx_data(const ObTransID& tx_id, ObITxDataCheckFunctor &fn) { int ret = OB_SUCCESS; - ObPartTransCtx *tx_ctx = NULL; + ObTxCtx *tx_ctx = NULL; if (OB_FAIL(get_tx_ctx_(tx_id, true, /*for_replay*/ @@ -1413,7 +951,7 @@ int ObLSTxCtxMgr::get_rec_scn(SCN &rec_scn) TRANS_LOG(WARN, "ObLSTxCtxMgr not inited", K(this)); ret = OB_NOT_INIT; } else if (is_stopped()) { - TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped", K(ls_id_)); + TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped"); ret = OB_STATE_NOT_MATCH; } else { GetRecLogTSFunctor fn; @@ -1441,7 +979,7 @@ int ObLSTxCtxMgr::on_tx_ctx_table_flushed() TRANS_LOG(WARN, "ObLSTxCtxMgr not inited", K(this)); ret = OB_NOT_INIT; } else if (is_stopped()) { - TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped", K(ls_id_)); + TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped"); ret = OB_STATE_NOT_MATCH; } else { OnTxCtxTableFlushedFunctor fn; @@ -1524,7 +1062,7 @@ int ObLSTxCtxMgr::update_aggre_log_ts_wo_lock(SCN rec_scn) if (rec_scn.is_valid()) { // we cannot lock here, because the lock order must be - // ObLSTxCtxMgr -> ObPartTransCtx, otherwise we may be + // ObLSTxCtxMgr -> ObTxCtx, otherwise we may be // deadlocked SCN old_v; SCN new_v; @@ -1545,7 +1083,7 @@ int ObLSTxCtxMgr::dump_single_tx_data_2_text(const int64_t tx_id_int, FILE *fd) { int ret = OB_SUCCESS; - ObPartTransCtx *tx_ctx = NULL; + ObTxCtx *tx_ctx = NULL; ObTransID tx_id(tx_id_int); if (OB_ISNULL(fd)) { @@ -1608,69 +1146,53 @@ int ObLSTxCtxMgr::end_readonly_request() return OB_SUCCESS; } -int ObTxCtxMgr::remove_all_ls_() +int ObTxCtxMgr::remove_context_manager_() { int ret = OB_SUCCESS; - - RemoveLSFunctor fn; - if (OB_FAIL(remove_if_(fn))) { - TRANS_LOG(WARN, "remove_if ls error", KR(ret)); + if (OB_NOT_NULL(tx_ctx_mgr_)) { + if (!tx_ctx_mgr_->is_stopped()) { + const int check_ret = OB_PARTITION_IS_NOT_STOPPED; + TRANS_LOG(WARN, "transaction context manager has not been stopped", + K(check_ret), KPC(tx_ctx_mgr_)); + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "remove transaction context manager failed", KR(ret), KPC(tx_ctx_mgr_)); + } else { + tx_ctx_mgr_->destroy(); + TRANS_LOG(INFO, "remove transaction context manager", KP(tx_ctx_mgr_)); + release_tx_ctx_mgr_(); + } } return ret; } -int ObTxCtxMgr::stop_ls_(const ObLSID &ls_id, const bool graceful) +int ObTxCtxMgr::stop_context_manager_(const bool graceful) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - ret = OB_PARTITION_NOT_EXIST; - TRANS_LOG(WARN, "ls not exist", K(ret), K(ls_id)); + if (OB_FAIL(tx_ctx_mgr_->stop(graceful))) { + TRANS_LOG(WARN, "stop transaction context manager failed", KR(ret)); } else { - if (OB_FAIL(ls_tx_ctx_mgr->stop(graceful))) { - TRANS_LOG(WARN, "stop ls error", KR(ret), K(ls_id)); - } else { - TRANS_LOG(INFO, "stop ls success", K(ls_id), "ctx_count", ls_tx_ctx_mgr->get_tx_ctx_count()); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); + TRANS_LOG(INFO, "stop transaction context manager success", "ctx_count", tx_ctx_mgr_->get_tx_ctx_count()); } return ret; } -int ObTxCtxMgr::wait_ls_(const ObLSID &ls_id) +int ObTxCtxMgr::wait_context_manager_() { int ret = OB_SUCCESS; int64_t count = 0; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; const int64_t PRINT_LOG_INTERVAL = 1000 * 1000; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - // check transaction context before removing ls - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - ret = OB_PARTITION_NOT_EXIST; - TRANS_LOG(WARN, "ls not exist", K(ret), K(ls_id)); - } else { - if (OB_UNLIKELY(!ls_tx_ctx_mgr->is_stopped())) { - ret = OB_PARTITION_IS_NOT_STOPPED; - TRANS_LOG(WARN, "ls has not been stopped", K(ret), K(ls_id)); - } else if ((count = ls_tx_ctx_mgr->get_tx_ctx_count()) > 0) { - if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - TRANS_LOG(WARN, "transaction context not empty, try again", KP(ls_tx_ctx_mgr), K(ls_id), K(count)); - } - ret = OB_EAGAIN; - } else if (OB_FAIL(ls_tx_ctx_mgr->get_ls_log_writer()->wait())) { - TRANS_LOG(WARN, "ls log_cbs are not empty", KR(ret), K(ls_id)); - } else { - TRANS_LOG(INFO, "wait ls success", K(ls_id)); + if (OB_UNLIKELY(!tx_ctx_mgr_->is_stopped())) { + ret = OB_PARTITION_IS_NOT_STOPPED; + TRANS_LOG(WARN, "transaction context manager has not been stopped", K(ret)); + } else if ((count = tx_ctx_mgr_->get_tx_ctx_count()) > 0) { + if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { + TRANS_LOG(WARN, "transaction context not empty, try again", KP(tx_ctx_mgr_), K(count)); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); + ret = OB_EAGAIN; + } else { + TRANS_LOG(INFO, "wait transaction context manager success"); } return ret; } @@ -1679,13 +1201,10 @@ int ObTxCtxMgr::init(ObTsMgr *ts_mgr, ObTransService *txs) { int ret = OB_SUCCESS; - int64_t i = 0; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; TRANS_LOG(WARN, "ObTxCtxMgr inited twice", K(*this)); - } else if (OB_FAIL(ls_tx_ctx_mgr_map_.init(lib::ObMemAttr("TxCtxMgr")))) { - TRANS_LOG(WARN, "ls_tx_ctx_mgr_map_ init error", KR(ret)); } else if (OB_ISNULL(ts_mgr)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "ts mgr is null"); @@ -1728,10 +1247,14 @@ int ObTxCtxMgr::stop() } else if (!is_running_) { ret = OB_NOT_RUNNING; TRANS_LOG(WARN, "ObTxCtxMgr already has been stopped", K(*this)); + } else if (OB_ISNULL(tx_ctx_mgr_)) { + is_running_ = false; } else { - StopLSFunctor fn; - if (OB_FAIL(foreach_ls_(fn))) { - TRANS_LOG(WARN, "foreach ls to stop error", KR(ret)); + const bool graceful = false; + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = stop_context_manager_(graceful))) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "stop transaction context manager failed", KR(ret), K(tmp_ret)); } else { is_running_ = false; TRANS_LOG(INFO, "ObTxCtxMgr stop success", K(*this)); @@ -1740,13 +1263,16 @@ int ObTxCtxMgr::stop() return ret; } -int ObTxCtxMgr::print_all_ls_tx_ctx_() +int ObTxCtxMgr::print_tx_ctx_() { int ret = OB_SUCCESS; + const bool verbose = true; - PrintAllLSTxCtxFunctor fn; - if (OB_FAIL(foreach_ls_(fn))) { - TRANS_LOG(WARN, "print ls transaction context error", KR(ret)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObTxCtxMgr is not inited", K(*this)); + } else if (OB_NOT_NULL(tx_ctx_mgr_)) { + tx_ctx_mgr_->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); } return ret; } @@ -1768,23 +1294,32 @@ int ObTxCtxMgr::wait() int64_t retry = 0; for (; OB_SUCCESS == ret && retry < MAX_WAIT_RETRY_TIMES; ++retry) { { - int64_t retry_count = 0; - - WaitLSFunctor fn(retry_count); - if (OB_FAIL(foreach_ls_(fn))) { - TRANS_LOG(WARN, "foreach ls error", KR(ret)); - } else if (retry_count > 0) { - TRANS_LOG(WARN, "ls need retry to wait", "ls_count", retry_count, "lbt", lbt()); - ret = OB_SUCCESS; - } else { + bool need_retry = false; + + if (OB_ISNULL(tx_ctx_mgr_)) { + break; + } else if (OB_FAIL(wait_context_manager_())) { + if (OB_EAGAIN == ret) { + // Unfinished transactions require another wait cycle. + need_retry = true; + ret = OB_SUCCESS; + } else { + const int wait_ret = ret; + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "wait transaction context manager failed", KR(ret), K(wait_ret)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (!need_retry) { break; } } ObTransCond::usleep(SLEEP_US); } if (OB_FAIL(ret) || MAX_WAIT_RETRY_TIMES == retry) { - if (OB_TMP_FAIL(print_all_ls_tx_ctx_())) { - TRANS_LOG(WARN, "print all ls tx ctx error", "ret", tmp_ret); + if (OB_TMP_FAIL(print_tx_ctx_())) { + TRANS_LOG(WARN, "print tx ctx error", "ret", tmp_ret); } } } @@ -1797,11 +1332,9 @@ void ObTxCtxMgr::destroy() int tmp_ret = OB_SUCCESS; if (is_inited_) { - if (OB_TMP_FAIL(remove_all_ls_())) { - TRANS_LOG_RET(WARN, tmp_ret, "remove all ls error", K(tmp_ret)); + if (OB_TMP_FAIL(remove_context_manager_())) { + TRANS_LOG_RET(WARN, tmp_ret, "remove transaction context manager error", K(tmp_ret)); } else { - - ls_tx_ctx_mgr_map_.destroy(); TRANS_LOG(INFO, "ObTxCtxMgr destroyed"); is_inited_ = false; } @@ -1811,91 +1344,53 @@ void ObTxCtxMgr::destroy() void ObTxCtxMgr::reset() { is_running_ = false; - - ls_tx_ctx_mgr_map_.reset(); + release_tx_ctx_mgr_(); ts_mgr_ = NULL; txs_ = NULL; - ls_alloc_cnt_ = 0; - ls_release_cnt_ = 0; is_inited_ = false; } -int ObTxCtxMgr::revert_ls_tx_ctx_mgr(ObLSTxCtxMgr *ls_tx_ctx_mgr) -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), KP(ls_tx_ctx_mgr)); - } else { - ls_tx_ctx_mgr_map_.revert(ls_tx_ctx_mgr); - } - - return ret; -} - -int ObTxCtxMgr::get_ls_tx_ctx_mgr(const ObLSID &ls_id, ObLSTxCtxMgr *&ls_tx_ctx_mgr) +void ObTxCtxMgr::release_tx_ctx_mgr_() { - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited", K(*this)); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(ls_tx_ctx_mgr_map_.get(ls_id, ls_tx_ctx_mgr))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_PARTITION_NOT_EXIST; - TRANS_LOG(TRACE, "get ls_tx_ctx_mgr error", KR(ret), K(ls_id)); - } else { - TRANS_LOG(WARN, "get ls_tx_ctx_mgr error", KR(ret), K(ls_id)); - } - ls_tx_ctx_mgr = NULL; + if (OB_NOT_NULL(tx_ctx_mgr_)) { + TRANS_LOG(INFO, "transaction context manager release", K(*tx_ctx_mgr_)); + ObLSTxCtxMgrFactory::release(tx_ctx_mgr_); + tx_ctx_mgr_ = NULL; } - - return ret; } -int ObTxCtxMgr::get_tx_ctx(const ObLSID &ls_id, - const ObTransID &tx_id, +int ObTxCtxMgr::get_tx_ctx(const ObTransID &tx_id, const bool for_replay, - ObPartTransCtx *&ctx) + ObTxCtx *&ctx) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited", K(*this)); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!tx_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id), K(tx_id)); + } else if (OB_UNLIKELY(!tx_id.is_valid())) { + TRANS_LOG(WARN, "invalid argument", K(tx_id)); ret = OB_INVALID_ARGUMENT; } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTxCtxMgr is not running"); ret = OB_NOT_RUNNING; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_id transaction context manager error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->get_tx_ctx(tx_id, for_replay, ctx))) { - TRANS_LOG(WARN, "get transaction context error", KR(ret), K(ls_id), K(tx_id)); + if (OB_FAIL(tx_ctx_mgr_->get_tx_ctx(tx_id, for_replay, ctx))) { + TRANS_LOG(WARN, "get transaction context error", KR(ret), K(tx_id)); } else if (OB_ISNULL(ctx)) { - TRANS_LOG(WARN, "transaction context is null", K(ls_id), K(tx_id)); + TRANS_LOG(WARN, "transaction context is null", K(tx_id)); ret = OB_ERR_UNEXPECTED; } else { - TRANS_LOG(DEBUG, "get transaction context success", K(ls_id), K(tx_id)); + TRANS_LOG(DEBUG, "get transaction context success", K(tx_id)); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } return ret; } int ObTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, bool& existed, - ObPartTransCtx *&ctx) { + ObTxCtx *&ctx) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited", K(*this)); @@ -1906,11 +1401,8 @@ int ObTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTxCtxMgr is not running"); ret = OB_NOT_RUNNING; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(arg.ls_id_, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_id transaction context manager error", K(arg)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->create_tx_ctx(arg, existed, ctx))) { + if (OB_FAIL(tx_ctx_mgr_->create_tx_ctx(arg, existed, ctx))) { TRANS_LOG(WARN, "get transaction context error", KR(ret), K(arg)); } else if (OB_ISNULL(ctx)) { TRANS_LOG(WARN, "transaction context is null", K(arg)); @@ -1918,15 +1410,13 @@ int ObTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, } else { TRANS_LOG(DEBUG, "get transaction context success", K(arg)); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } return ret; } -int ObTxCtxMgr::revert_tx_ctx(ObPartTransCtx *ctx) +int ObTxCtxMgr::revert_tx_ctx(ObTxCtx *ctx) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited", K(*this)); @@ -1935,102 +1425,51 @@ int ObTxCtxMgr::revert_tx_ctx(ObPartTransCtx *ctx) TRANS_LOG(WARN, "invalid argument", KP(ctx)); ret = OB_INVALID_ARGUMENT; } else { - // reference cannot be used here, otherwise context memory will be released - // and core dump may occur when printing ls_id const ObTransID tx_id = ctx->get_trans_id(); - const ObLSID ls_id = ctx->get_ls_id(); - if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_tx_ctx_mgr error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; + if (OB_FAIL(tx_ctx_mgr_->revert_tx_ctx(ctx))) { + TRANS_LOG(WARN, "revert tx ctx error", KR(ret), "context", *ctx); } else { - if (OB_FAIL(ls_tx_ctx_mgr->revert_tx_ctx(ctx))) { - TRANS_LOG(WARN, "revert tx ctx error", KR(ret), K(ls_id), "context", *ctx); - } else { - TRANS_LOG(DEBUG, "revert tx ctx success", K(ls_id), K(tx_id), K(ctx)); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); + TRANS_LOG(DEBUG, "revert tx ctx success", K(tx_id), K(ctx)); } } return ret; } -int ObTxCtxMgr::block_tx(const ObLSID &ls_id, bool &is_all_tx_cleaned_up) +int ObTxCtxMgr::block_tx(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "ls_tx_ctx_mgr not exist", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->block_tx(is_all_tx_cleaned_up))) { - TRANS_LOG(WARN, "block ls error", KR(ret), K(ls_id)); + if (OB_FAIL(tx_ctx_mgr_->block_tx(is_all_tx_cleaned_up))) { + TRANS_LOG(WARN, "block transaction context manager failed", KR(ret)); } else { - TRANS_LOG(INFO, "block ls success", K(ls_id), "ctx_count", ls_tx_ctx_mgr->get_tx_ctx_count()); + TRANS_LOG(INFO, "block transaction context manager success", "ctx_count", tx_ctx_mgr_->get_tx_ctx_count()); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } return ret; } -int ObTxCtxMgr::block_all(const ObLSID &ls_id, bool &is_all_tx_cleaned_up) +int ObTxCtxMgr::block_all(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "ls_tx_ctx_mgr not exist", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; - } else { - if (OB_FAIL(ls_tx_ctx_mgr->block_all(is_all_tx_cleaned_up))) { - TRANS_LOG(WARN, "block all error", KR(ret), K(ls_id)); - } else { - TRANS_LOG(INFO, "block all success", K(ls_id), "ctx_count", ls_tx_ctx_mgr->get_tx_ctx_count()); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - return ret; -} - -int ObTxCtxMgr::clear_all_tx(const ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - const KillTransArg arg(false); - bool is_all_tx_cleaned_up = false; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - - if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_tx_ctx_mgr error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->kill_all_tx(arg.graceful_, is_all_tx_cleaned_up))) { - TRANS_LOG(WARN, "kill all tx failed", KR(ret), K(ls_id), K(arg)); + if (OB_FAIL(tx_ctx_mgr_->block_all(is_all_tx_cleaned_up))) { + TRANS_LOG(WARN, "block all on transaction context manager failed", KR(ret)); } else { - TRANS_LOG(INFO, "ObTxCtxMgr clear_all_tx success", K(ls_id)); + TRANS_LOG(INFO, "block all on transaction context manager success", "ctx_count", tx_ctx_mgr_->get_tx_ctx_count()); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } - return ret; } - int ObTxCtxMgr::iterate_all_observer_tx_stat(ObTxStatIterator &tx_stat_iter) { int ret = OB_SUCCESS; @@ -2038,213 +1477,114 @@ int ObTxCtxMgr::iterate_all_observer_tx_stat(ObTxStatIterator &tx_stat_iter) if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else { - IterateAllLSTxStatFunctor fn(tx_stat_iter); - if (OB_FAIL(foreach_ls_(fn))) { - // rewrite eagain to real ret code + } else if (OB_NOT_NULL(tx_ctx_mgr_)) { + IterateObserverTxStatFunctor fn(tx_stat_iter); + if (!fn(tx_ctx_mgr_)) { ret = fn.get_ret(); - TRANS_LOG(WARN, "foreach_ls_ tx_stat error", KR(ret)); - } else { - // do nothing - } - } - return ret; -} - - -int ObTxCtxMgr::iterate_ls_tx_lock_stat(const ObLSID &ls_id, - ObTxLockStatIterator &tx_lock_stat_iter) -{ - int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_tx_ctx_mgr error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; - } else { - if (OB_FAIL(ls_tx_ctx_mgr->iterate_tx_lock_stat(tx_lock_stat_iter))) { - TRANS_LOG(WARN, "iterate_tx_lock_stat error", KR(ret), K(ls_id)); - } else { - TRANS_LOG(DEBUG, "ObTxLockStatIterator set ready success", K(ls_id)); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - - // ignore retcode when virtual table access error - if (OB_PARTITION_NOT_EXIST == ret || OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } - return ret; -} - -int ObTxCtxMgr::iterate_ls_id(ObLSIDIterator &ls_id_iter) -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else { - IterateLSIDFunctor fn(ls_id_iter); - if (OB_FAIL(foreach_ls_(fn))) { - TRANS_LOG(WARN, "for each all ls_id error", KR(ret)); - } else { - // do nothing + if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + } + TRANS_LOG(WARN, "iterate transaction stat failed", KR(ret)); } } - return ret; } -int ObTxCtxMgr::iterate_tx_ctx_mgr_stat(const ObAddr &addr, - ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter) +int ObTxCtxMgr::get_tx_ctx_mgr_stat(const ObAddr &addr, + ObLSTxCtxMgrStat &tx_ctx_mgr_stat) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else { - IterateLSTxCtxMgrStatFunctor fn(addr, tx_ctx_mgr_stat_iter); - if (OB_FAIL(foreach_ls_(fn))) { - TRANS_LOG(WARN, "for each all ls error", KR(ret)); - } + } else if (OB_ISNULL(tx_ctx_mgr_)) { + ret = OB_ENTRY_NOT_EXIST; + } else if (OB_FAIL(tx_ctx_mgr_stat.init(addr, + tx_ctx_mgr_->is_stopped(), + tx_ctx_mgr_->is_tx_blocked(), + tx_ctx_mgr_->is_normal_tx_blocked(), + tx_ctx_mgr_->is_all_blocked(), + tx_ctx_mgr_->get_tx_ctx_count(), + reinterpret_cast(tx_ctx_mgr_)))) { + TRANS_LOG(WARN, "transaction context manager stat init failed", + KR(ret), K(addr), KPC(tx_ctx_mgr_)); } return ret; } -int ObTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(const ObLSID &ls_id, SCN &min_prepare_version) +int ObTxCtxMgr::get_min_uncommit_tx_prepare_version(SCN &min_prepare_version) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get ls_id transaction context manager error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->get_ls_min_uncommit_tx_prepare_version(min_prepare_version))) { - TRANS_LOG(WARN, "ObTxCtxMgr get min uncommit prepare version error", KR(ret), K(ls_id)); + if (OB_FAIL(tx_ctx_mgr_->get_min_uncommit_tx_prepare_version(min_prepare_version))) { + TRANS_LOG(WARN, "ObTxCtxMgr get min uncommit prepare version error", KR(ret)); } else { TRANS_LOG(DEBUG, "ObTxCtxMgr get min uncommit prepare version success", - K(ls_id), K(min_prepare_version)); + K(min_prepare_version)); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } return ret; } -int ObTxCtxMgr::get_min_undecided_scn(const ObLSID &ls_id, SCN &scn) -{ - int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get participant transaction context mgr error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; - } else { - if (OB_FAIL(ls_tx_ctx_mgr->get_min_undecided_scn(scn))) { - TRANS_LOG(WARN, "get min_uncommit_log_id failed", KR(ret), K(ls_id)); - } else { - TRANS_LOG(DEBUG, "ObTxCtxMgr get min_uncommit_log_id success", K(ls_id), K(scn)); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - return ret; -} - -int ObTxCtxMgr::remove_callback_for_uncommited_tx( - const ObLSID ls_id, const memtable::ObMemtableSet *memtable_set) +int ObTxCtxMgr::remove_callback_for_uncommited_tx(const memtable::ObMemtableSet *memtable_set) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get participant transaction context mgr error", KP(memtable_set)); - ret = OB_PARTITION_NOT_EXIST; } else { - if (OB_FAIL(ls_tx_ctx_mgr->remove_callback_for_uncommited_tx(memtable_set))) { + if (OB_FAIL(tx_ctx_mgr_->remove_callback_for_uncommited_tx(memtable_set))) { TRANS_LOG(WARN, "get remove callback for uncommited txn failed", KR(ret), KP(memtable_set)); } else { TRANS_LOG(DEBUG, "get remove callback for uncommited txn succeed", KP(memtable_set)); } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); } return ret; } -int ObTxCtxMgr::create_ls(const ObLSID &ls_id, - ObTxTable *tx_table, - ObLockTable *lock_table, - ObLSTxService &ls_tx_svr, - ObITxLogParam *param, - ObITxLogAdapter *log_adapter) +int ObTxCtxMgr::create_context_manager(ObTxTable *tx_table, + ObLockTable *lock_table, + ObLSTxService &ls_tx_svr, + ObITxLogParam *param, + ObITxLogAdapter *log_adapter) { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; + const bool manager_existed = OB_NOT_NULL(tx_ctx_mgr_); if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_ISNULL(ls_tx_ctx_mgr = ObLSTxCtxMgrFactory::alloc())) { - TRANS_LOG(WARN, "alloc ls_id transaction context manager error", K(ls_id)); + } else if (OB_NOT_NULL(tx_ctx_mgr_)) { + ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "transaction context manager already exists", KR(ret)); + } else if (OB_ISNULL(tx_ctx_mgr_ = ObLSTxCtxMgrFactory::alloc())) { + TRANS_LOG(WARN, "alloc transaction context manager failed"); ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_FAIL(ls_tx_ctx_mgr->init(ls_id, tx_table, lock_table, - ts_mgr_, txs_, param, log_adapter))) { - TRANS_LOG(WARN, "ls_tx_ctx_mgr inited error", KR(ret), K(ls_id)); - ObLSTxCtxMgrFactory::release(ls_tx_ctx_mgr); - ls_tx_ctx_mgr = NULL; - } else if (OB_FAIL(ls_tx_svr.init(ls_id, ls_tx_ctx_mgr, txs_))) { - TRANS_LOG(WARN, "ls tx service init failed", K(ret), K(ls_id)); - ObLSTxCtxMgrFactory::release(ls_tx_ctx_mgr); - ls_tx_ctx_mgr = NULL; - } else if (OB_FAIL(ls_tx_ctx_mgr_map_.insert_and_get(ls_id, ls_tx_ctx_mgr, NULL))) { - TRANS_LOG(WARN, "ls_tx_ctx_mgr_map_ insert error", KR(ret), K(ls_id)); - ObLSTxCtxMgrFactory::release(ls_tx_ctx_mgr); - ls_tx_ctx_mgr = NULL; + } else if (OB_FAIL(tx_ctx_mgr_->init(tx_table, lock_table, + ts_mgr_, txs_, param, log_adapter))) { + TRANS_LOG(WARN, "transaction context manager init failed", KR(ret)); + } else if (OB_FAIL(ls_tx_svr.init(tx_ctx_mgr_, txs_))) { + TRANS_LOG(WARN, "transaction service init failed", K(ret)); } else { - ATOMIC_INC(&ls_alloc_cnt_); - // need to revert the trans ctx ref explicitly - ls_tx_ctx_mgr_map_.revert(ls_tx_ctx_mgr); - TRANS_LOG(INFO, "create ls success", - "total_alloc", ls_alloc_cnt_, - "total_release", ls_release_cnt_, - K(ls_id), KP(ls_tx_ctx_mgr)); + TRANS_LOG(INFO, "create transaction context manager success", KP(tx_ctx_mgr_)); + } + if (OB_FAIL(ret) && !manager_existed && OB_NOT_NULL(tx_ctx_mgr_)) { + release_tx_ctx_mgr_(); } return ret; } -int ObTxCtxMgr::remove_ls(const ObLSID &ls_id, const bool graceful) +int ObTxCtxMgr::remove_context_manager(const bool graceful) { int ret = OB_SUCCESS; const KillTransArg arg(graceful, false); @@ -2252,51 +1592,37 @@ int ObTxCtxMgr::remove_ls(const ObLSID &ls_id, const bool graceful) const int64_t SLEEP_US = 20000; //20ms const int64_t PRINT_LOG_INTERVAL = 1000 * 1000; // 1s const int64_t MAX_RETRY_NUM = 50; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTxCtxMgr not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(stop_ls_(ls_id, graceful))) { - if (OB_ENTRY_NOT_EXIST == ret || OB_PARTITION_NOT_EXIST == ret) { - TRANS_LOG(INFO, "ls not found", KR(ret), K(ls_id)); - ret = OB_SUCCESS; - } else { - TRANS_LOG(WARN, "stop ls failed", KR(ret), K(ls_id)); - } + } else if (OB_ISNULL(tx_ctx_mgr_)) { + // The transaction service may stop before storage initialization completes. + } else if (OB_FAIL(stop_context_manager_(graceful))) { + TRANS_LOG(WARN, "stop transaction context manager failed", KR(ret)); } else { - // there is no limit for retry times, tx ctx are required to be released + // Transaction contexts must drain before the log adapter is detached. for (int64_t retry = 0; need_retry && is_running_ && OB_SUCC(ret); ++retry) { need_retry = false; bool is_all_trans_cleaned_up = false; - // if ls_id has been removed, OB_SUCCESS will be returned - if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - need_retry = false; - } else { - if (OB_FAIL(wait_ls_(ls_id))) { - if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { - TRANS_LOG(WARN, "wait ls error", - KR(ret), K(retry), K(ls_id), K(*ls_tx_ctx_mgr)); - } - need_retry = (OB_EAGAIN == ret); - if (need_retry && MAX_RETRY_NUM == retry && NULL != ls_tx_ctx_mgr) { - // kill all trans if reach MAX_RETRY_NUM - if (OB_FAIL(ls_tx_ctx_mgr->kill_all_tx(arg.graceful_, is_all_trans_cleaned_up))) { - TRANS_LOG(WARN, "kill all transaction context error", KR(ret), K(arg)); - } else if (!is_all_trans_cleaned_up) { - const bool verbose = true; - ls_tx_ctx_mgr->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); - } else { - need_retry = false; - } + if (OB_FAIL(wait_context_manager_())) { + if (REACH_TIME_INTERVAL(PRINT_LOG_INTERVAL)) { + TRANS_LOG(WARN, "wait transaction context manager failed", + KR(ret), K(retry), K(*tx_ctx_mgr_)); + } + need_retry = (OB_EAGAIN == ret); + if (need_retry && MAX_RETRY_NUM == retry) { + if (OB_FAIL(tx_ctx_mgr_->kill_all_tx(arg.graceful_, is_all_trans_cleaned_up))) { + TRANS_LOG(WARN, "kill all transaction context error", KR(ret), K(arg)); + } else if (!is_all_trans_cleaned_up) { + const bool verbose = true; + tx_ctx_mgr_->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); + } else { + need_retry = false; } - } else { - need_retry = false; } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); + } else { + need_retry = false; } if (need_retry) { ret = OB_SUCCESS; @@ -2305,188 +1631,14 @@ int ObTxCtxMgr::remove_ls(const ObLSID &ls_id, const bool graceful) } if (OB_SUCC(ret)) { - // if ls_id has been removed, OB_SUCCESS is returned. - if (OB_SUCC(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - // remove ls_id transaction context from map - ls_tx_ctx_mgr->get_ls_log_adapter()->reset(); - if (OB_FAIL(ls_tx_ctx_mgr_map_.del(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "remove ls error", KR(ret), K(ls_id)); - } else { - ATOMIC_INC(&ls_release_cnt_); - TRANS_LOG(INFO, "remove ls success", KP(ls_tx_ctx_mgr), K(ls_id), - "total_alloc", ls_alloc_cnt_, - "total_release", ls_release_cnt_); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - } else if (OB_ENTRY_NOT_EXIST == ret || OB_PARTITION_NOT_EXIST == ret) { - TRANS_LOG(INFO, "ls not found", KR(ret), K(ls_id)); - ret = OB_SUCCESS; + tx_ctx_mgr_->get_ls_log_adapter()->reset(); } else if (OB_EAGAIN == ret) { - TRANS_LOG(WARN, "remove ls error, but return OB_SUCCESS", KR(ret), K(ls_id)); + TRANS_LOG(WARN, "remove transaction context manager timed out", KR(ret)); ret = OB_SUCCESS; - } else { - // do nothing - } - } - TRANS_LOG(INFO, "remove ls", KR(ret), K(ls_id), K(graceful)); - - return ret; -} - -int ObTxCtxMgr::check_scheduler_status(share::ObLSID ls_id) -{ - int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - SCN min_start_scn; - MinStartScnStatus min_status = MinStartScnStatus::UNKOWN; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get participant transaction context mgr error", K(ls_id)); - ret = OB_PARTITION_NOT_EXIST; - } else { - if (OB_FAIL(ls_tx_ctx_mgr->check_scheduler_status(min_start_scn, min_status))) { - TRANS_LOG(WARN, "check_scheduler_status failed", KR(ret), K(ls_id)); - } else { - TRANS_LOG(DEBUG, "check_scheduler_status success", K(ls_id)); - } - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - - return ret; -} - -// check ls status in trans layer -int ObTxCtxMgr::check_ls_status(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTxCtxMgr not inited", K(ret), K(ls_id)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - ret = OB_PARTITION_NOT_EXIST; - TRANS_LOG(WARN, "get ls_tx_ctx_mgr failed", K(ret), K(ls_id)); - } else if(ls_tx_ctx_mgr->is_stopped()) { - ret = OB_PARTITION_IS_BLOCKED; - TRANS_LOG(WARN, "ls_tx_ctx_mgr is stopped", K(ret), K(ls_id)); - } else if (ls_tx_ctx_mgr->is_all_blocked()) { - ret = OB_PARTITION_IS_BLOCKED; - TRANS_LOG(WARN, "logstream is blocked", K(ret), K(ls_id)); - } - if (OB_NOT_NULL(ls_tx_ctx_mgr)) { - revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - return ret; -} - -int ObTxCtxMgr::do_all_ls_standby_cleanup(ObTimeGuard &cleanup_timeguard) -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else { - StandbyCleanUpAllLSFunctor fn(cleanup_timeguard); - if (OB_FAIL(foreach_ls_(fn))) { - ret = fn.get_ret(); - TRANS_LOG(WARN, "foreach_ls standby cleanup error", KR(ret)); - } else { - // do nothing } } - return ret; -} + TRANS_LOG(INFO, "remove transaction context manager", KR(ret), K(graceful)); -int ObLSTxCtxMgr::do_standby_cleanup() -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObLSTxCtxMgr not inited"); - ret = OB_NOT_INIT; - } else { - StandbyCleanUpFunctor fn; - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - ret = fn.get_ret(); - TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", *this); - } - } - - return ret; -} - -OB_NOINLINE int ObLSTxCtxMgr::errsim_switch_to_followr_gracefully() -{ - int ret = OB_SUCCESS; - -#ifdef ERRSIM - ret = EN_SWITCH_TO_FOLLOWER_GRACEFULLY; -#endif - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "errsim for switch_to_follower_gracefully", K(ret), KPC(this)); - } - - return ret; -} - -OB_NOINLINE int ObLSTxCtxMgr::errsim_submit_start_working_log() -{ - int ret = OB_SUCCESS; - -#ifdef ERRSIM - ret = EN_SUBMIT_START_WORKING_LOG; -#endif - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "errsim for submit_start_working_log", K(ret), KPC(this)); - } - - return ret; -} - -OB_NOINLINE int ObLSTxCtxMgr::errsim_apply_start_working_log() -{ - int ret = OB_SUCCESS; - -#ifdef ERRSIM - ret = EN_APPLY_START_WORKING_LOG; -#endif - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "errsim for apply_start_working_log", K(ret), KPC(this)); - } - - return ret; -} - -int ObLSTxCtxMgr::wait_tx_write_end(ObTimeoutCtx &timeout_ctx) -{ - int ret = OB_SUCCESS; - int64_t active_tx_count = 0; - int64_t abs_expired_time = INT64_MAX; - if (timeout_ctx.get_abs_timeout() > 0) { - abs_expired_time = timeout_ctx.get_abs_timeout(); - } - WaitTxWriteEndFunctor fn(abs_expired_time); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); - ret = fn.get_ret(); - } else { - active_tx_count = fn.get_count(); - } - TRANS_LOG(INFO, "wait_tx_write_end", KR(ret), K(active_tx_count)); return ret; } diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.h b/src/storage/tx/ob_trans_ctx_mgr_v4.h index fc88ff408..623cb68d4 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.h +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.h @@ -21,9 +21,8 @@ #include "ob_tx_stat.h" #include "storage/tx_table/ob_tx_table_define.h" #include "common/ob_simple_iterator.h" +#include "lib/lock/ob_qsync_lock.h" #include "storage/tx/ob_trans_ctx.h" -#include "storage/tx/ob_tx_ls_log_writer.h" -#include "storage/tx/ob_tx_ls_state_mgr.h" #include "storage/tx/ob_tx_retain_ctx_mgr.h" #include "storage/tx/ob_tx_log_cb_mgr.h" #include "storage/tablelock/ob_lock_table.h" @@ -56,7 +55,7 @@ namespace transaction { class ObLSTxCtxMgrStat; class ObTransCtx; -class ObPartTransCtx; +class ObTxCtx; class ObTsMgr; class ObITxLogParam; class ObITxLogAdapter; @@ -72,91 +71,70 @@ class TestTxCtxTable; namespace transaction { -// Is used to store and traverse all ls id maintained -typedef common::ObSimpleIterator ObLSIDIterator; - // Is used to store and travserse all TxCtx's Stat information; typedef common::ObSimpleIterator ObTxStatIterator; -// Is used to store and travserse all ObLSTxCtxMgr's Stat information -typedef common::ObSimpleIterator ObTxCtxMgrStatIterator; - // Is used to travserse all TxCtx's lock information typedef common::ObSimpleIterator ObTxLockStatIterator; typedef share::ObLightHashMap ObLSTxCtxMap; -typedef common::LinkHashNode ObLSTxCtxMgrHashNode; -typedef common::LinkHashValue ObLSTxCtxMgrHashValue; - struct ObTxCreateArg { ObTxCreateArg(const bool for_replay, - const PartCtxSource ctx_source, + const TxCtxSource ctx_source, const ObTransID &trans_id, - const share::ObLSID &ls_id, const uint64_t cluster_id, const uint64_t cluster_version, const uint32_t session_id, const uint32_t client_sid, const uint32_t associated_session_id, - const common::ObAddr &scheduler, const int64_t trans_expired_time, ObTransService *trans_service, - ObXATransID xid = ObXATransID(), - int64_t epoch = -1) + ObXATransID xid = ObXATransID()) : for_replay_(for_replay), ctx_source_(ctx_source), tx_id_(trans_id), - ls_id_(ls_id), cluster_id_(cluster_id), cluster_version_(cluster_version), session_id_(session_id), client_sid_(client_sid), associated_session_id_(associated_session_id), - scheduler_(scheduler), trans_expired_time_(trans_expired_time), trans_service_(trans_service), - xid_(xid), - epoch_(epoch) {} + xid_(xid) {} bool is_valid() const { - return ls_id_.is_valid() - && tx_id_.is_valid() + return tx_id_.is_valid() && trans_expired_time_ > 0 && NULL != trans_service_; } TO_STRING_KV(K_(for_replay), "ctx_source", to_str_ctx_source(ctx_source_), K_(tx_id), - K_(ls_id), K_(cluster_id), K_(cluster_version), + K_(cluster_id), K_(cluster_version), K_(session_id),K_(client_sid), K_(associated_session_id), - K_(scheduler), K_(trans_expired_time), KP_(trans_service), - K_(epoch), K_(xid)); + K_(trans_expired_time), KP_(trans_service), + K_(xid)); bool for_replay_; - PartCtxSource ctx_source_; + TxCtxSource ctx_source_; ObTransID tx_id_; - share::ObLSID ls_id_; uint64_t cluster_id_; uint64_t cluster_version_; uint32_t session_id_; uint32_t client_sid_; uint32_t associated_session_id_; - const common::ObAddr &scheduler_; int64_t trans_expired_time_; ObTransService *trans_service_; ObXATransID xid_; - int64_t epoch_; }; // Is used to store and traverse ObTxID const static char OB_SIMPLE_ITERATOR_LABEL_FOR_TX_ID[] = "ObTxCtxMgr"; typedef common::ObSimpleIterator ObTxIDIterator; -// LogStream Transaction Context Manager -class ObLSTxCtxMgr: public share::ObLightHashLink +// Transaction Context Manager +class ObLSTxCtxMgr { // ut friend class unittest::TestTxCtxTable; @@ -165,7 +143,6 @@ class ObLSTxCtxMgr: public share::ObLightHashLink friend class ObTransCtx; friend class ObTransCtx; friend class ObTransTimer; - friend class IterateLSTxCtxMgrStatFunctor; public: typedef common::RWLock RWLock; @@ -184,12 +161,10 @@ class ObLSTxCtxMgr: public share::ObLightHashLink virtual ~ObLSTxCtxMgr() { destroy(); } // @param [in] tenant: ls's tenant, currently used by ts_mgr; - // @param [in] ls_id, associated ls_id; - // @param [in] ts_mgr: used to get gts, see: update_max_replay_commit_version function; + // @param [in] ts_mgr: transaction timestamp source; // @param [in] txs: transaction service which hold the ObTxCtxMgr; // @param [in] log_param: the params which is used to init ObLSTxCtxMgr's log_adapter_def_; - int init(const share::ObLSID &ls_id, - ObTxTable *tx_table, + int init(ObTxTable *tx_table, ObLockTable *lock_table, ObTsMgr *ts_mgr, ObTransService *txs, @@ -205,7 +180,6 @@ class ObLSTxCtxMgr: public share::ObLightHashLink // Offline the in-memory state of the ObLSTxCtxMgr int offline(); - int wait_tx_write_end(ObTimeoutCtx &timeout_ctx); public: // Create a TxCtx whose tx_id is specified // @param [in] tx_id: transaction ID @@ -214,36 +188,32 @@ class ObLSTxCtxMgr: public share::ObLightHashLink // been found, and the found TxCtx will be returned through the outgoing parameter tx_ctx; // TODO insert_and_get return existed object ptr; // @param [out] tx_ctx: newly allocated or already exsited transaction context - // Return Values That Need Attention: // @return OB_SUCCESS, if the tx_ctx newly allocated or already existed - // @return OB_NOT_MASTER, if this ls is a follower replica int create_tx_ctx(const ObTxCreateArg &arg, bool& existed, - ObPartTransCtx *&tx_ctx); + ObTxCtx *&tx_ctx); // Find specified TxCtx from the ObLSTxCtxMgr; // @param [in] tx_id: transaction ID // @param [in] for_replay: Identifies whether the TxCtx is used by replay processing; // @param [out] tx_ctx: context found through ObLSTxCtxMgr's hash table - // Return Values That Need Attention: - // @return OB_NOT_MASTER, if the LogStream is follower replica // @return OB_TRANS_CTX_NOT_EXIST, if the specified TxCtx is not found; - int get_tx_ctx(const ObTransID &tx_id, const bool for_replay, ObPartTransCtx *&tx_ctx); + int get_tx_ctx(const ObTransID &tx_id, const bool for_replay, ObTxCtx *&tx_ctx); int get_tx_ctx_with_timeout(const ObTransID &tx_id, const bool for_replay, - ObPartTransCtx *&tx_ctx, + ObTxCtx *&tx_ctx, const int64_t lock_timeout); // Find specified TxCtx directly from the ObLSTxCtxMgr's hash_map // @param [in] tx_id: transaction ID // @param [out] tx_ctx: context found through ObLSTxCtxMgr's hash table // @return OB_TRANS_CTX_NOT_EXIST, if the specified TxCtx is not found; - int get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObPartTransCtx *&ctx); + int get_tx_ctx_directly_from_hash_map(const ObTransID &tx_id, ObTxCtx *&ctx); // Decrease the specified tx_ctx's reference count // @param [in] tx_ctx: the TxCtx will be revert - int revert_tx_ctx(ObPartTransCtx *tx_ctx); + int revert_tx_ctx(ObTxCtx *tx_ctx); // Decrease the specified tx_ctx's reference count // @param [in] tx_ctx: the TxCtx will be revert @@ -261,15 +231,13 @@ class ObLSTxCtxMgr: public share::ObLightHashLink // Get the min prepare version of transaction module of current observer for slave read // @param [out] min_prepare_version: MIN(uncommitted tx_ctx's prepare version | ObLSTxCtxMgr) - int get_ls_min_uncommit_tx_prepare_version(share::SCN &min_prepare_version); + int get_min_uncommit_tx_prepare_version(share::SCN &min_prepare_version); // Check the ObLSTxCtxMgr's status; if it's stopped, the new transaction ctx will not be created; // otherwise, the tx_ctx is created normally bool is_stopped() const { return is_stopped_(); } - // Stop the ObLSTxCtxMgr, the function will - // 1. stop the ls_log_writer_; - // 2. kill all tx in the ls and callback sql; + // Stop the ObLSTxCtxMgr and kill all local transactions. // @param [in] graceful: indicate the kill behavior int stop(const bool graceful); @@ -321,13 +289,7 @@ class ObLSTxCtxMgr: public share::ObLightHashLink ObTransID &block_tx_id); // check schduler status for tx gc - int check_scheduler_status(share::SCN &min_start_scn, MinStartScnStatus &status); - - // Get this ObLSTxCtxMgr's ls_id_ - const share::ObLSID &get_ls_id() const { return ls_id_; } - - // Get this ObLSTxCtxMgr's ls_log_writer - ObTxLSLogWriter *get_ls_log_writer() { return &ls_log_writer_; } + int check_tx_status(share::SCN &min_start_scn, MinStartScnStatus &status); ObITxLogAdapter *get_ls_log_adapter() { return tx_log_adapter_; } @@ -344,9 +306,6 @@ class ObLSTxCtxMgr: public share::ObLightHashLink int get_lock_memtable(ObTableHandleV2 &handle) { return lock_table_->get_lock_memtable(handle); } - // check this ObLSTxCtxMgr is in leader serving state - bool in_leader_serving_state(); - // dump a single tx data to text // @param [in] tx_id // @param [in] fd @@ -354,12 +313,6 @@ class ObLSTxCtxMgr: public share::ObLightHashLink int start_readonly_request(); int end_readonly_request(); - // check this ObLSTxCtxMgr contains the specified ObLSID - bool contain(const share::ObLSID &ls_id) - { - return ls_id_ == ls_id; - } - public: // Increase this ObLSTxCtxMgr's total_tx_ctx_count void inc_total_tx_ctx_count() { (void)ATOMIC_AAF(&total_tx_ctx_count_, 1); } @@ -445,79 +398,17 @@ class ObLSTxCtxMgr: public share::ObLightHashLink int get_min_undecided_scn(share::SCN &scn); //1. During the minor merge process, the status of uncommitted transactions will be actively - // queried; if ObPartTransCtx is released due to Rebuild, and the query cannot be performed, + // queried; if ObTxCtx is released due to Rebuild, and the query cannot be performed, // the minor merge process will record an exception failure log and exit; //2. The high risk point is that in the process of minor merge memtable, it is necessary to // query the status of uncommitted transactions. At this time, the status of Trans is directly // queried through the pointer to TransCtx in ObMvccTransNode; the use of this pointer is not - // currently protected by reference counting. If ObPartTransCtx is released due to Rebuild + // currently protected by reference counting. If ObTxCtx is released due to Rebuild // during use, it will cause a wild pointer; //3. The minor_merge_lock_ is used to prevent this from happening int lock_minor_merge_lock() { return minor_merge_lock_.rdlock(); } int unlock_minor_merge_lock() { return minor_merge_lock_.rdunlock(); }; - // Through this interface, the PALF notifies this ObLSTxCtxMgr to forcibly switch to follower; - // When the PALF calls this interface, - // all log entries status associated with this ls has been confirmed; - // In all normal execution scenarios, the interface does not return failure; - // The PALF layer assumes that the operation must succeed, and does not handle the failure; - // TODO If this interface return failure, should set the replica to Manual Status; - int switch_to_follower_forcedly(); - - // Through this interface, the PALF notifies ObLSTxCtxMgr to switch to leader, then it will start - // executing switch_to_leader routine; - // 1. switch ObLSTxCtxMgr to LEADER_SWITCHING state, submit start_working log entry; - // 2. switch ObLSTxCtxMgr to L_WORKING state; - // Travesal ObLSTxCtxMgr and call the switch_to_leader interface of each TxCtx; - int switch_to_leader(); - - // Through this interface, the PALF notifies ObLSTxCtxMgr to *gracefully* switch to follower; - // This interface will be called when actively triggers the replica role switch action; - // It is necessary to ensure that the switching process does not kill the transaction; - // If the execution fails, the function will try to do the resume leader work by itself; - // If the function cannot complete the resume_leader action by itself, it needs to return - // OB_NEED_REVOKE, so that PALF will trigger the election process; - int switch_to_follower_gracefully(); - - // This interface will be called when other module switch_to_follower_gracefully executes failed; - // When this interface call is triggered, it can ensure that each TxCtx in ObLSTxCtxMgr - // is complete, and its status can be directly reset to Leader status; - int resume_leader(); - - // Replay the start working log entry. Traverse ObLSTxCtxMgr and set the data_complete_ of each - // ObTxCtx to false. Make all the preceding ACTIVE_INFO log entry invalid. - // @param [in] log: The start working log entry - int replay_start_working_log(const ObTxStartWorkingLog &log, share::SCN start_working_scn); - - // START_WORKING log entry is successfully written to the PALF; According to the current - // ObLSTxCtxMgr's status, drive ObLSTxCtxMgr to continue to execute the switch_to_leader routine - // or resume_leader routine; - int on_start_working_log_cb_succ(share::SCN start_working_scn); - int retry_apply_start_working_log(); - - // START_WORKING log entry failed to written to the PALF; - // Break the switch_to_leader routine or resume_leader routine; switch the ObLSTxCtxMgr's state to F_WORKING - int on_start_working_log_cb_fail(); - - // 1.In order to support slave-read, it is necessary to ensure the partial order relationship - // between log_id and scn; then before the transaction submits redolog to the PALF, - // The log entry's log id and scn ts will be generated by the PALF, and the partial order - // relationship between log_id and scn is guaranteed by the PALF; - // 2.So the redolog entry's commit version may be bigger than the GTS version; - // In the normal transaction execution process, you need to wait for GTS version >= commit version - // before executing the inc the publish_version and responding to the client commit success; - // 3.Distributed transactions can handle this situation normally in the recovery process; - // Single ls transactions require special handling; - // 4.When the single ls transaction's redolog entry is replayed, the largest - // batch_commit_version on the ls will be calculated - // When start executing swith_to_leader routine, wait_gts_elapse will be called to ensure that - // GTS has exceeded this batch_commit_version; - // @param [in] replay_commit_version : replayed single-ls-tx's log entry commit version; - void update_max_replay_commit_version(const share::SCN &replay_commit_version) - { - max_replay_commit_version_.inc_update(replay_commit_version); - } - // Iterate all tx ctx in this ls and get the min_start_scn int get_min_start_scn(share::SCN &min_start_scn); @@ -530,41 +421,32 @@ class ObLSTxCtxMgr: public share::ObLightHashLink // Get the tenant corresponding to this ObLSTxCtxMgr; - // check is master - bool is_master() const { return is_master_(); } - // check is blocked bool is_tx_blocked() const { return is_tx_blocked_(); } // check all blocked bool is_all_blocked() const { return is_all_blocked_(); } + bool is_normal_tx_blocked() const { return is_normal_blocked_(); } // Switch the prev_aggre_log_ts and aggre_log_ts during dump starts int refresh_aggre_rec_scn(); // Update aggre_log_ts without lock, because we canot lock using the order of - // ObPartTransCtx -> ObLSTxCtxMgr, It will be a deadlock with normal order. + // ObTxCtx -> ObLSTxCtxMgr, It will be a deadlock with normal order. int update_aggre_log_ts_wo_lock(share::SCN rec_log_ts); int get_max_decided_scn(share::SCN & scn); - int do_standby_cleanup(); - - int errsim_switch_to_followr_gracefully(); - int errsim_submit_start_working_log(); - int errsim_apply_start_working_log(); - TO_STRING_KV(KP(this), - K_(ls_id), - - K_(tx_ls_state_mgr), + K_(stopped), + K_(block_tx), + K_(block_normal_tx), + K_(block_all), K_(total_tx_ctx_count), K_(active_tx_count), K_(ls_retain_ctx_mgr), K_(aggre_rec_scn), - K_(prev_aggre_rec_scn), - "uref", - (!is_inited_ ? -1 : get_ref())); + K_(prev_aggre_rec_scn)); private: DISALLOW_COPY_AND_ASSIGN(ObLSTxCtxMgr); @@ -580,55 +462,34 @@ class ObLSTxCtxMgr: public share::ObLightHashLink int64_t get_tx_ctx_count_() const { return ATOMIC_LOAD(&total_tx_ctx_count_); } int create_tx_ctx_(const ObTxCreateArg &arg, bool &existed, - ObPartTransCtx *&ctx); - int get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObPartTransCtx *&tx_ctx); - int submit_start_working_log_(); - int try_wait_gts_and_inc_max_commit_ts_(); + ObTxCtx *&ctx); + int get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObTxCtx *&tx_ctx); share::SCN get_aggre_rec_scn_(); public: static const int64_t MAX_HASH_ITEM_PRINT = 16; - static const int64_t WAIT_SW_CB_TIMEOUT = 100 * 1000; // 100 ms - static const int64_t WAIT_SW_CB_INTERVAL = 10 * 1000; // 10 ms static const int64_t WAIT_READONLY_REQUEST_TIME = 10 * 1000 * 1000; private: - inline bool is_master_() const - { return tx_ls_state_mgr_.is_master(); } - - inline bool is_follower_() const - { return tx_ls_state_mgr_.is_follower(); } - inline bool is_tx_blocked_() const - { return tx_ls_state_mgr_.is_block_start_tx(); } + { return ATOMIC_LOAD(&block_tx_); } inline bool is_normal_blocked_() const - { return tx_ls_state_mgr_.is_block_start_normal_tx(); } + { return ATOMIC_LOAD(&block_normal_tx_); } inline bool is_all_blocked_() const - { return tx_ls_state_mgr_.is_block_WR(); } - - // check pending substate - inline bool is_t_pending_() const - { return tx_ls_state_mgr_.is_switch_leader_pending(); } - - inline bool is_r_pending_() const - { return tx_ls_state_mgr_.is_resume_leader_pending(); } - - inline bool is_f_pending_() const - { return tx_ls_state_mgr_.is_follower_swl_pending(); } - - inline bool is_pending_() const - { return tx_ls_state_mgr_.is_leader_takeover_pending(); } + { return ATOMIC_LOAD(&block_all_); } inline bool is_stopped_() const - { return tx_ls_state_mgr_.is_stopped(); } + { return ATOMIC_LOAD(&stopped_); } private: // Identifies this ObLSTxCtxMgr is inited or not; bool is_inited_; - // See the ObLSTxCtxMgr's internal class State - ObTxLSStateMgr tx_ls_state_mgr_; + bool stopped_; + bool block_tx_; + bool block_normal_tx_; + bool block_all_; // A thread-safe hashmap, used to find and traverse TxCtx in this ObLSTxCtxMgr ObLSTxCtxMap ls_tx_ctx_map_; @@ -636,16 +497,11 @@ class ObLSTxCtxMgr: public share::ObLightHashLink // The tenant ID to which this ObLSTxCtxMgr belongs - // The ls ID associated with this ObLSTxCtxMgr - share::ObLSID ls_id_; - // The tx table associated with this ObLSTxCtxMgr ObTxTable *tx_table_; // The Lock Table associate with this ObLSTxCtxMgr ObLockTable *lock_table_; - // Used to submit START_WORKING LogEntry - ObTxLSLogWriter ls_log_writer_; ObITxLogAdapter *tx_log_adapter_; ObLSTxLogAdapter log_adapter_def_; @@ -666,20 +522,12 @@ class ObLSTxCtxMgr: public share::ObLightHashLink int64_t active_tx_count_; - // It is used to record the time point of leader takeover - // gts must be refreshed to the newest before the leader provides services - MonotonicTs leader_takeover_ts_; - bool is_leader_serving_; - // Transaction service which hold the ObTxCtxMgr; ObTransService *txs_; // The time source ObTsMgr *ts_mgr_; - // See the ObLSTxCtxMgr's member function update_max_replay_commit_version - share::SCN max_replay_commit_version_; - // Recover log timestamp for aggregated tx ctx. For the purpose of Durability, // we need a space to store rec_log_ts for checkpoint[1]. Although tx ctx // itself can store rec_log_ts, it may release its ctx when txn itself @@ -717,8 +565,8 @@ class ObLSTxCtxIterator int set_ready(ObLSTxCtxMgr* ls_tx_ctx_mgr); bool is_ready() const {return is_ready_; } - int get_next_tx_ctx(ObPartTransCtx *&tx_ctx); - int revert_tx_ctx(ObPartTransCtx *tx_ctx); + int get_next_tx_ctx(ObTxCtx *&tx_ctx); + int revert_tx_ctx(ObTxCtx *tx_ctx); private: int get_next_tx_id_(ObTransID& tx_id); @@ -731,74 +579,50 @@ class ObLSTxCtxIterator ObTxIDIterator tx_id_iter_; }; -class ObLSTxCtxMgrAlloc +class ObTxCtxMgr { public: - static ObLSTxCtxMgr* alloc_value() { return NULL; } - static void free_value(ObLSTxCtxMgr* p) - { - if (NULL != p) { - TRANS_LOG(INFO, "ObLSTxCtxMgr release", K(*p)); - ObLSTxCtxMgrFactory::release(p); - } - } - static ObLSTxCtxMgrHashNode* alloc_node(ObLSTxCtxMgr* p) - { - UNUSED(p); - return op_alloc(ObLSTxCtxMgrHashNode); - } - static void free_node(ObLSTxCtxMgrHashNode* node) + ObTxCtxMgr() + : is_inited_(false), + is_running_(false), + tx_ctx_mgr_(NULL), + ts_mgr_(NULL), + txs_(NULL) { - if (NULL != node) { - op_free(node); - node = NULL; - } + reset(); } -}; - -typedef share::ObLightHashMap ObLSTxCtxMgrMap; - -class ObTxCtxMgr -{ -public: - ObTxCtxMgr() { reset(); } ~ObTxCtxMgr() { destroy(); } // @param [in] tenant: tenant id - // @param [in] ts_mgr: used to get gts, see: update_max_replay_commit_version function; + // @param [in] ts_mgr: transaction timestamp source; // @param [in] txs: transaction service which hold the ObTxCtxMgr; int init(ObTsMgr *ts_mgr, ObTransService *txs); // Mark ObTxCtxMgr as running int start(); - // Stop the ObTxCtxMgr; call the stop function of each ObLSTxCtxMgr and mark ObTxCtxMgr is not running; + // Stop the transaction context manager and mark ObTxCtxMgr not running. int stop(); // Destroy the ObTxCtxMgr; void destroy(); - // Wait until each ObLSTxCtxMgr has stopped and the number of TxCtx held is 0; + // Wait until the transaction context manager has stopped and holds no TxCtx. int wait(); // Reset the ObTxCtxMgr; void reset(); public: - // Find specified TxCtx from the specified ObLSTxCtxMgr; - // @param [in] ls_id: the specified ls ID + // Find a specified TxCtx. // @param [in] tx_id: transaction ID // @param [in] for_replay: Identifies whether the TxCtx is used by replay processing // @param [out] tx_ctx: context found through ls transaction context manager's hash table // @return OB_SUCCESS, if found the transaction tx_ctx - // @return OB_NOT_MASTER, if the LogStream is follower replica - int get_tx_ctx(const share::ObLSID &ls_id, - const ObTransID &tx_id, + int get_tx_ctx(const ObTransID &tx_id, const bool for_replay, - ObPartTransCtx *&tx_ctx); + ObTxCtx *&tx_ctx); - // Create a TxCtx whose tx_id is specified in the specified ObLSTxCtxMgr - // @param [in] ls_id: the specified ls ID + // Create a TxCtx with the specified transaction ID. // @param [in] tx_id: transaction ID // @param [in] for_replay: Identifies whether the TxCtx is created for replay processing; // @param [out] existed: @@ -806,159 +630,77 @@ class ObTxCtxMgr // TODO insert_and_get return existed object ptr; // @param [out] tx_ctx: newly allocated or already exsited transaction context // @return OB_SUCCESS, if the transaction tx_ctx newly allocated or already existed - // @return OB_NOT_MASTER, if this LogStream is a follower replica int create_tx_ctx(const ObTxCreateArg &arg, bool& existed, - ObPartTransCtx *&tx_ctx); + ObTxCtx *&tx_ctx); // Decrease the specified tx_ctx's reference count // @param [in] tx_ctx: the TxCtx will be revert - int revert_tx_ctx(ObPartTransCtx *tx_ctx); + int revert_tx_ctx(ObTxCtx *tx_ctx); - // Create ObLSTxCtxMgr; - // @param [in] tenant: the tenant ID; - // @param [in] ls_id: the specifiied ls ID - // @param [in] log_param: the params which is used to init ObLSTxCtxMgr's log_adapter_def_; - int create_ls(const share::ObLSID &ls_id, - ObTxTable *tx_table, - ObLockTable *lock_table, - storage::ObLSTxService &ls_tx_svr, /* TODO remove this argument*/ - ObITxLogParam *param, - ObITxLogAdapter *log_adapter); - - // Delete the specified ObLSTxCtxMgr, the stop_ls function will be called internally and ensure - // that there is no active TxCtx, then the ObLSTxCtxMgr will be deleted from the hash table - // of ObTxCtxMgr; - // @param [in] ls_id: the specifiied ls ID - // @param [in] graceful: indicate the kill behavior - int remove_ls(const share::ObLSID &ls_id, const bool graceful); + int create_context_manager(ObTxTable *tx_table, + ObLockTable *lock_table, + storage::ObLSTxService &ls_tx_svr, + ObITxLogParam *param, + ObITxLogAdapter *log_adapter); - // Block the specified ObLSTxCtxMgr, so that it can no longer create a new TxCtx, + // Delete the context manager after all active TxCtx objects are stopped. + int remove_context_manager(const bool graceful); + + // Block the context manager, so that it can no longer create a new TxCtx, // and returns whether there are still active transactions through out parameters; - // @param [in] ls_id: the specifiied ls ID // @param [out] is_all_tx_cleaned_up, set it to true, when all transactions are cleaned up; - int block_tx(const share::ObLSID &ls_id, bool &is_all_tx_cleaned_up); + int block_tx(bool &is_all_tx_cleaned_up); // block tx and readonly request - int block_all(const share::ObLSID &ls_id, bool &is_all_tx_cleaned_up); - - // Traverse the specified ObLSTxCtxMgr and kill all transactions it holds; - // @param [in] ls_id: the specifiied ls ID - int clear_all_tx(const share::ObLSID &ls_id); - - // kill the specified ls's transaction - // @param [in] ls_id: the specifiied ls ID - // @param [in] graceful: indicate the kill behavior - // @param [out] is_all_tx_cleaned_up: set it to true, when all transactions are killed - - //int get_store_ctx(const storage::ObStoreAccessType access_type, - // ObTxDesc &tx, - // const ObLSID &ls_id, - // storage::ObStoreCtx &store_ctx, - // const int64_t &snapshot_version); - - // revert_store_ctx - //int revert_store_ctx(ObTxDesc &ctx, storage::ObStoreCtx &store_ctx); + int block_all(bool &is_all_tx_cleaned_up); // Get the min prepare version of transaction module of current observer for slave read - // @param [in] indicates the specified ls; // @param [out] min_prepare_version: MIN(uncommitted transaction ctx's prepare version) - int get_ls_min_uncommit_tx_prepare_version(const share::ObLSID &ls_id, share::SCN &min_prepare_version); - - // TODO - int get_min_undecided_scn(const share::ObLSID &ls_id, share::SCN &scn); - - // Get all ls ID in the observer - // param [out] ls_id_iter: ObLSIDIterator is Used to return all ls id in the observer, - // which is subsequently used for iterative access of virtual table; - int iterate_ls_id(ObLSIDIterator &ls_id_iter); + int get_min_uncommit_tx_prepare_version(share::SCN &min_prepare_version); - // Get all ObLSTxCtxMgr's stat info in the observer - // @param [in] addr: the address of the observer, which is used to populate each line of output; - // @param [out] tx_ctx_mgr_stat_iter: Used to return all the collected ObLSTxCtxMgr's Stat - // information, and then used to iteratively output to the virtual table; - int iterate_tx_ctx_mgr_stat(const ObAddr &addr, - ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter); + // Get transaction context manager statistics. + // @param [in] addr: the address of the observer; + // @param [out] tx_ctx_mgr_stat: the collected context manager statistics; + int get_tx_ctx_mgr_stat(const ObAddr &addr, + ObLSTxCtxMgrStat &tx_ctx_mgr_stat); // Get all transaction information at the server level // @param [out] tx_stat_iter: Used to return all the collected TxCtx's Stat information, // and then used to iteratively output to the virtual table; int iterate_all_observer_tx_stat(ObTxStatIterator &tx_stat_iter); - // Print all TxCtx in the specific ObLSTxCtxMgr - // @param [in] ls_id: the specific ls ID; - - // Get specific ObLSTxCtxMgr's transaction lock stat - // @param [in] ls_id: the specified ls_id - // @param [out] tx_lock_stat_iter: Used to return all the collected TxLockStat information, - // and then used to iteratively output to the virtual table; - int iterate_ls_tx_lock_stat(const share::ObLSID &ls_id, - ObTxLockStatIterator &tx_lock_stat_iter); - // Before Deleting the memtable, remove its associated callback from the callback list of all the // active transactions; When the memtable has mini-merged, the commit_version of its associated // transaction may be undecided; - // @param [in] ls_id: the specified ls_id; // @param [in] mt: the memtable point which will be deleted; - int remove_callback_for_uncommited_tx( - const ObLSID ls_id, - const memtable::ObMemtableSet *memtable_set); + int remove_callback_for_uncommited_tx(const memtable::ObMemtableSet *memtable_set); TO_STRING_KV(K(is_inited_), KP(this)); - // Find specified ObLSTxCtxMgr from the ObTxCtxMgr and increase its reference count; - // @param [in] ls_id: the specified ls_id - // @param [out] ls_tx_ctx_mgr: the specified ls_tx_ctx_mgr - int get_ls_tx_ctx_mgr(const share::ObLSID &ls_id, ObLSTxCtxMgr *&ls_tx_ctx_mgr); + ObLSTxCtxMgr &get_tx_ctx_manager() { return *tx_ctx_mgr_; } - // Decrease specified ObLSTxCtxMgr's reference count - // @param [in] mgr: indicates the ObLSTxCtxMgr whose reference count needs to be decreased; - int revert_ls_tx_ctx_mgr(ObLSTxCtxMgr *ls_tx_ctx_mgr); - // @param [in] ls_id: the specified ls_id - int check_scheduler_status(share::ObLSID ls_id); - - int get_max_decided_scn(const share::ObLSID &ls_id, share::SCN & scn); - - int do_all_ls_standby_cleanup(ObTimeGuard &cleanup_timeguard); - - int check_ls_status(const share::ObLSID &ls_id); private: - int create_ls_(const share::ObLSID &ls_id, - ObLSTxService &ls_tx_svr, - ObITxLogParam *param); - - int remove_ls_(const share::ObLSID &ls_id); + int stop_context_manager_(const bool graceful); - int stop_ls_(const share::ObLSID &ls_id, const bool graceful); + int wait_context_manager_(); - int wait_ls_(const share::ObLSID &ls_id); + int remove_context_manager_(); - int remove_all_ls_(); - template int foreach_ls_(Fn &fn) { return ls_tx_ctx_mgr_map_.for_each(fn); } - template int remove_if_(Fn &fn) { return ls_tx_ctx_mgr_map_.remove_if(fn); } + int print_tx_ctx_(); - int print_all_ls_tx_ctx_(); + void release_tx_ctx_mgr_(); private: bool is_inited_; bool is_running_; - // The tenant ID to which this ObTxCtxMgr belongs - - - // A thread-safe hashmap, used to find and traverse ObLSTxCtxMgr - ObLSTxCtxMgrMap ls_tx_ctx_mgr_map_; + ObLSTxCtxMgr *tx_ctx_mgr_; // The time source ObTsMgr *ts_mgr_; - // Statistical variable that records the number of ObLSTxCtxMgr allocated; - int64_t ls_alloc_cnt_; - - // Statistical variable that records the number of ObLSTxCtxMgr released; - int64_t ls_release_cnt_; - // Transaction service which hold the ObTxCtxMgr; ObTransService *txs_; private: diff --git a/src/storage/tx/ob_trans_deadlock_adapter.cpp b/src/storage/tx/ob_trans_deadlock_adapter.cpp index a695c1163..a5c602774 100644 --- a/src/storage/tx/ob_trans_deadlock_adapter.cpp +++ b/src/storage/tx/ob_trans_deadlock_adapter.cpp @@ -286,19 +286,19 @@ class RemoteDeadLockCollectCallBack { const uint32_t sess_id_; }; -int ObTransDeadlockDetectorAdapter::gen_dependency_resource_array_(const ObIArray &blocked_trans_ids_and_addrs, +int ObTransDeadlockDetectorAdapter::gen_dependency_resource_array_(const ObIArray &blocked_trans_ids, ObIArray &dependency_resources) { int ret = OB_SUCCESS; UserBinaryKey binary_key; ObDependencyResource resource; - for (int64_t idx = 0; idx < blocked_trans_ids_and_addrs.count() && OB_SUCC(ret); idx++) { - if (!blocked_trans_ids_and_addrs.at(idx).is_valid()) { + for (int64_t idx = 0; idx < blocked_trans_ids.count() && OB_SUCC(ret); idx++) { + if (!blocked_trans_ids.at(idx).is_valid()) { ret = OB_ERR_UNEXPECTED; - DETECT_LOG(ERROR, "invalid trans id and addr"); - } else if (OB_FAIL(binary_key.set_user_key(blocked_trans_ids_and_addrs.at(idx).tx_id_))) { + DETECT_LOG(ERROR, "invalid trans id"); + } else if (OB_FAIL(binary_key.set_user_key(blocked_trans_ids.at(idx)))) { DETECT_LOG(ERROR, "fail to create key"); - } else if (OB_FAIL(resource.set_args(blocked_trans_ids_and_addrs.at(idx).scheduler_addr_, binary_key))) { + } else if (OB_FAIL(resource.set_args(GCTX.self_addr(), binary_key))) { DETECT_LOG(ERROR, "fail to create resource"); } else if (OB_FAIL(dependency_resources.push_back(resource))) { DETECT_LOG(ERROR, "fail to push resource"); @@ -309,13 +309,12 @@ int ObTransDeadlockDetectorAdapter::gen_dependency_resource_array_(const ObIArra int ObTransDeadlockDetectorAdapter::register_to_deadlock_detector_(const ObTransID self_tx_id, const uint32_t self_session_id, - const ObIArray &conflict_tx_ids, + const ObIArray &conflict_tx_ids, SessionGuard &session_guard) { - #define PRINT_WRAPPER KR(ret), K(self_tx_id), K(self_session_id), K(conflict_tx_ids), K(query_timeout), K(self_tx_scheduler) + #define PRINT_WRAPPER KR(ret), K(self_tx_id), K(self_session_id), K(conflict_tx_ids), K(query_timeout) int ret = OB_SUCCESS; int64_t query_timeout = 0; - ObAddr self_tx_scheduler; RemoteDeadLockCollectCallBack on_collect_op(self_session_id); ObTransOnDetectOperation on_detect_op(self_session_id, self_tx_id); ObSEArray blocked_resources; @@ -326,7 +325,6 @@ int ObTransDeadlockDetectorAdapter::register_to_deadlock_detector_(const ObTrans } else if (OB_ISNULL(session_guard->get_tx_desc())) { ret = OB_ERR_UNEXPECTED; DETECT_LOG(ERROR, "tx desc on session is NULL", PRINT_WRAPPER); - } else if (FALSE_IT(self_tx_scheduler = session_guard->get_tx_desc()->get_addr())) { } else if (OB_FAIL(gen_dependency_resource_array_(conflict_tx_ids, blocked_resources))) { DETECT_LOG(WARN, "fail to generate block resource", PRINT_WRAPPER); } else if (OB_ISNULL(share::g_mp->dead_lock_detector_mgr())) { @@ -343,14 +341,8 @@ int ObTransDeadlockDetectorAdapter::register_to_deadlock_detector_(const ObTrans share::g_mp->dead_lock_detector_mgr()->set_timeout(self_tx_id, query_timeout); if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(self_tx_id, blocked_resources))) { DETECT_LOG(WARN, "block on resource failed", PRINT_WRAPPER); - } else if (self_tx_scheduler != GCTX.self_addr()) { - if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->add_parent(self_tx_id, self_tx_scheduler, self_tx_id))) { - DETECT_LOG(WARN, "scheduler is not self, set parent failed", PRINT_WRAPPER); - } else { - DETECT_LOG(INFO, "remote execution register to deadlock detector success, scheduler is not self", PRINT_WRAPPER); - } } else { - DETECT_LOG(INFO, "remote execution register to deadlock detector success, scheduler is self", PRINT_WRAPPER); + DETECT_LOG(INFO, "register to deadlock detector success", PRINT_WRAPPER); } } return ret; @@ -358,7 +350,7 @@ int ObTransDeadlockDetectorAdapter::register_to_deadlock_detector_(const ObTrans } int ObTransDeadlockDetectorAdapter::replace_conflict_trans_ids_(const ObTransID self_tx_id, - const ObIArray &conflict_tx_ids, + const ObIArray &conflict_tx_ids, SessionGuard &session_guard) { #define PRINT_WRAPPER KR(ret), K(self_tx_id), K(conflict_tx_ids), K(current_blocked_resources) @@ -402,7 +394,7 @@ int ObTransDeadlockDetectorAdapter::replace_conflict_trans_ids_(const ObTransID int ObTransDeadlockDetectorAdapter::register_or_replace_conflict_trans_ids(const ObTransID self_tx_id, const uint32_t self_session_id, - const ObArray &conflict_tx_ids) + const ObArray &conflict_tx_ids) { #define PRINT_WRAPPER KR(ret), K(self_tx_id), K(self_session_id), K(conflict_tx_ids) CHECK_DEADLOCK_ENABLED(); @@ -456,78 +448,15 @@ int ObTransDeadlockDetectorAdapter::get_session_related_info_(const uint32_t ses } -int ObTransDeadlockDetectorAdapter::get_trans_scheduler_info_on_participant(const ObTransID trans_id, - const share::ObLSID ls_id, - ObAddr &scheduler_addr) -{ - #define PRINT_WRAPPER KR(ret), K(trans_id), K(ls_id), K(scheduler_addr) - int ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLSHandle ls_handle; - if (OB_ISNULL(ls_service)) { - ret = OB_BAD_NULL_ERROR; - DETECT_LOG(ERROR, "ls_service is NULL", PRINT_WRAPPER); - } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DEADLOCK_MOD))) { - DETECT_LOG(WARN, "fail to get ls", PRINT_WRAPPER); - } else if (OB_FAIL(ls_handle.get_ls()->get_tx_scheduler(trans_id, scheduler_addr))) { - DETECT_LOG(WARN, "fail to get tx scheduler", PRINT_WRAPPER); - } - return ret; - #undef PRINT_WRAPPER -} - -int ObTransDeadlockDetectorAdapter::get_conflict_trans_scheduler(const ObTransID &self_trans_id, - ObAddr &scheduler_addr) -{ - #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(scheduler_addr) - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObLSService *ls_service = share::g_mp->ls_service(); - ObLS *ls = nullptr; - ObSharedGuard iter; - scheduler_addr.reset(); - if (OB_UNLIKELY(ls_service == nullptr)) { - ret = OB_ERR_UNEXPECTED; - DETECT_LOG(WARN, "can not get ls service", PRINT_WRAPPER); - } else if (OB_FAIL(ls_service->get_ls_iter(iter, storage::ObLSGetMod::DEADLOCK_MOD))) { - DETECT_LOG(WARN, "fail to get ls iter", PRINT_WRAPPER); - } else { - do { - if (OB_FAIL(iter->get_next(ls))) { - DETECT_LOG(WARN, "get next iter failed", PRINT_WRAPPER); - } else if (OB_FAIL(ls->get_tx_scheduler(self_trans_id, scheduler_addr))) { - if (ret == OB_TRANS_CTX_NOT_EXIST) { - ret = OB_SUCCESS; - DETECT_LOG(TRACE, "ctx not exist on this logstream", K(ls->get_ls_id()), PRINT_WRAPPER); - } else { - DETECT_LOG(WARN, "get tx ctx failed", PRINT_WRAPPER); - } - } else { - DETECT_LOG(TRACE, "ctx on this logstream", K(ls->get_ls_id()), PRINT_WRAPPER); - break;// find ctx - } - } while (OB_SUCC(ret)); - if (OB_FAIL(ret) || !scheduler_addr.is_valid()) { - if (OB_SUCC(ret)) { - ret = OB_ENTRY_NOT_EXIST; - } - DETECT_LOG(WARN, "can't find trans ctx from all ls", PRINT_WRAPPER); - } - } - return ret; - #undef PRINT_WRAPPER -} - int ObTransDeadlockDetectorAdapter::create_detector_node_and_set_parent_if_needed_(CollectCallBack &on_collect_op, const ObTransID &self_trans_id, const uint32_t sess_id) { - #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(sess_id), K(query_timeout), K(trans_begin_ts), K(scheduler_addr) + #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(sess_id), K(query_timeout), K(trans_begin_ts) int ret = OB_SUCCESS; int64_t query_timeout = 0; int64_t trans_begin_ts = 0; ObTransOnDetectOperation on_detect_op(sess_id, self_trans_id); - ObAddr scheduler_addr; SessionGuard guard; if (OB_FAIL(get_session_related_info_(sess_id, query_timeout))) { DETECT_LOG(WARN, "fail to get session related info", PRINT_WRAPPER); @@ -539,7 +468,6 @@ int ObTransDeadlockDetectorAdapter::create_detector_node_and_set_parent_if_neede } else if (OB_ISNULL(guard->get_tx_desc())) { ret = OB_BAD_NULL_ERROR; DETECT_LOG(WARN, "tx desc is NULL", PRINT_WRAPPER); - } else if (FALSE_IT(scheduler_addr = guard->get_tx_desc()->get_addr())) { } else if (FALSE_IT(trans_begin_ts = guard->get_tx_desc()->get_active_ts())) { } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->register_key(self_trans_id, on_detect_op, @@ -548,11 +476,6 @@ int ObTransDeadlockDetectorAdapter::create_detector_node_and_set_parent_if_neede DETECT_LOG(WARN, "fail to register key", PRINT_WRAPPER); } else { share::g_mp->dead_lock_detector_mgr()->set_timeout(self_trans_id, query_timeout); - if (scheduler_addr != GCTX.self_addr()) { - if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->add_parent(self_trans_id, scheduler_addr, self_trans_id))) { - DETECT_LOG(WARN, "fail to get add parent", PRINT_WRAPPER); - } - } } return ret; #undef PRINT_WRAPPER @@ -579,7 +502,7 @@ int ObTransDeadlockDetectorAdapter::maintain_deadlock_info_when_end_stmt(sql::Ob memtable::ObLockWaitMgr::Node *node = nullptr; ObSQLSessionInfo *session = nullptr; ObTxDesc *desc = nullptr; - ObArray conflict_txs; + ObArray conflict_txs; if (++step && OB_ISNULL(session = GET_MY_SESSION(exec_ctx))) { ret = OB_BAD_NULL_ERROR; DETECT_LOG(ERROR, "session is NULL", PRINT_WRAPPER); @@ -683,10 +606,9 @@ int ObTransDeadlockDetectorAdapter::lock_wait_mgr_reconstruct_detector_waiting_f const ObTransID &self_trans_id, const uint32_t sess_id) { - #define PRINT_WRAPPER KR(ret), K(scheduler_addr), K(self_trans_id), K(sess_id), K(exist) + #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(sess_id), K(exist) CHECK_DEADLOCK_ENABLED(); int ret = OB_SUCCESS; - ObAddr scheduler_addr; bool exist = false; if (nullptr == (share::g_mp->dead_lock_detector_mgr())) { ret = OB_ERR_UNEXPECTED; @@ -701,11 +623,9 @@ int ObTransDeadlockDetectorAdapter::lock_wait_mgr_reconstruct_detector_waiting_f } if (OB_FAIL(ret)) { DETECT_LOG(WARN, "local execution register to deadlock detector waiting for row failed", PRINT_WRAPPER); - } else if (OB_FAIL(get_conflict_trans_scheduler(conflict_trans_id, scheduler_addr))) { - DETECT_LOG(WARN, "fail to get conflict trans scheduler addr", PRINT_WRAPPER); } else if (OB_FAIL(create_detector_node_and_set_parent_if_needed_(on_collect_op, self_trans_id, sess_id))) { DETECT_LOG(WARN, "fail to create detector node", PRINT_WRAPPER); - } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(self_trans_id, scheduler_addr, conflict_trans_id))) { + } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(self_trans_id, GCTX.self_addr(), conflict_trans_id))) { DETECT_LOG(WARN, "fail to block on conflict trans", PRINT_WRAPPER); } else { DETECT_LOG(TRACE, "local execution register to deadlock detector waiting for trans success", PRINT_WRAPPER); @@ -715,10 +635,9 @@ int ObTransDeadlockDetectorAdapter::lock_wait_mgr_reconstruct_detector_waiting_f } int ObTransDeadlockDetectorAdapter::change_detector_waiting_obj_from_row_to_trans(const ObTransID &self_trans_id, - const ObTransID &conflict_trans_id, - const ObAddr &scheduler_addr) + const ObTransID &conflict_trans_id) { - #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(scheduler_addr), K(conflict_trans_id) + #define PRINT_WRAPPER KR(ret), K(self_trans_id), K(conflict_trans_id) CHECK_DEADLOCK_ENABLED(); int ret = OB_SUCCESS; if (nullptr == (share::g_mp->dead_lock_detector_mgr())) { @@ -726,7 +645,7 @@ int ObTransDeadlockDetectorAdapter::change_detector_waiting_obj_from_row_to_tran DETECT_LOG(WARN, "fail to get ObDeadLockDetectorMgr", PRINT_WRAPPER); } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->activate_all(self_trans_id))) { DETECT_LOG(WARN, "fail to activate all", PRINT_WRAPPER); - } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(self_trans_id, scheduler_addr, conflict_trans_id))) { + } else if (OB_FAIL(share::g_mp->dead_lock_detector_mgr()->block(self_trans_id, GCTX.self_addr(), conflict_trans_id))) { DETECT_LOG(WARN, "fail to block on conflict trans", PRINT_WRAPPER); } else { DETECT_LOG(INFO, "change denpendency relationship from row to trnas", PRINT_WRAPPER); diff --git a/src/storage/tx/ob_trans_deadlock_adapter.h b/src/storage/tx/ob_trans_deadlock_adapter.h index 15cbe9771..1f326ae33 100644 --- a/src/storage/tx/ob_trans_deadlock_adapter.h +++ b/src/storage/tx/ob_trans_deadlock_adapter.h @@ -132,20 +132,15 @@ class ObTransDeadlockDetectorAdapter // if trans node on row removed(for example:1, dump trans. 2, a trans write too many row.) // change the dependency relationship from row to trans static int change_detector_waiting_obj_from_row_to_trans(const ObTransID &self_trans_id, - const ObTransID &conflict_trans_id, - const ObAddr &scheduler_addr); + const ObTransID &conflict_trans_id); // for all path static void unregister_from_deadlock_detector(const ObTransID &self_trans_id, const UnregisterPath path); /**********************************/ - static int get_conflict_trans_scheduler(const ObTransID &self_trans_id, ObAddr &scheduler_addr); static int kill_tx(const uint32_t sess_id); static int get_session_info(const uint32_t session_id, SessionGuard &guard); - static int get_trans_scheduler_info_on_participant(const ObTransID trans_id, - const share::ObLSID ls_id, - ObAddr &scheduler_addr); static int register_or_replace_conflict_trans_ids(const ObTransID self_tx_id, const uint32_t self_session_id, - const ObArray &conflict_tx_ids); + const ObArray &conflict_tx_ids); static int kill_stmt(const uint32_t sess_id); static void copy_str_and_translate_apostrophe(const char *src_ptr, const int64_t src_len, @@ -154,16 +149,16 @@ class ObTransDeadlockDetectorAdapter private: static int register_to_deadlock_detector_(const ObTransID self_tx_id, const uint32_t self_session_id, - const ObIArray &conflict_tx_ids, + const ObIArray &conflict_tx_ids, SessionGuard &session_guard); static int replace_conflict_trans_ids_(const ObTransID self_tx_id, - const ObIArray &conflict_tx_ids, + const ObIArray &conflict_tx_ids, SessionGuard &session_guard); static int create_detector_node_and_set_parent_if_needed_(CollectCallBack &on_collect_op, const ObTransID &self_trans_id, const uint32_t sess_id); static int get_session_related_info_(const uint32_t sess_id, int64_t &query_timeout); - static int gen_dependency_resource_array_(const ObIArray &blocked_trans_ids_and_addrs, + static int gen_dependency_resource_array_(const ObIArray &blocked_trans_ids, ObIArray &dependency_resources); static void try_unregister_deadlock_detector_(sql::ObSQLSessionInfo &session, const ObTransID &trans_id, UnregisterPath path); }; diff --git a/src/storage/tx/ob_trans_define.cpp b/src/storage/tx/ob_trans_define.cpp index 24e7a2061..2a43ddcba 100644 --- a/src/storage/tx/ob_trans_define.cpp +++ b/src/storage/tx/ob_trans_define.cpp @@ -48,13 +48,8 @@ OB_SERIALIZE_MEMBER(ObTransID, tx_id_); OB_SERIALIZE_MEMBER(ObStartTransParam, access_mode_, type_, isolation_, consistency_type_, cluster_version_, is_inner_trans_, read_snapshot_type_); OB_SERIALIZE_MEMBER(ObElrTransInfo, trans_id_, commit_version_, result_); -OB_SERIALIZE_MEMBER(ObLSLogInfo, id_, offset_); -OB_SERIALIZE_MEMBER(ObStateInfo, ls_id_, state_, version_, snapshot_version_, check_info_); OB_SERIALIZE_MEMBER(ObTransDesc, a_); -OB_SERIALIZE_MEMBER(ObTxExecPart, ls_id_, exec_epoch_); -OB_SERIALIZE_MEMBER(ObStandbyCheckInfo, check_info_ori_ls_id_, check_part_); - // class ObStartTransParam void ObStartTransParam::reset() { @@ -284,19 +279,6 @@ void ObCoreLocalPartitionAuditInfo::reset() } -bool ObStateInfo::need_update(const ObStateInfo &state_info) -{ - bool need_update = true; - if (ObTxState::PRE_COMMIT <= state_ && state_ <= ObTxState::CLEAR) { - need_update = false; - } else if (snapshot_version_ > state_info.snapshot_version_) { - need_update = false; - } else if (state_info.state_ < state_) { - need_update = false; - } - return need_update; -} - void ObAddrLogId::reset() { addr_.reset(); @@ -344,22 +326,13 @@ DEF_TO_STRING(ObLockForReadArg) return pos; } -DEFINE_TO_STRING_AND_YSON(ObTransKey, OB_ID(hash), hash_val_, - OB_ID(trans_id), trans_id_); - void ObTxExecInfo::reset() { state_ = ObTxState::INIT; - upstream_.reset(); - participants_.reset(); - incremental_participants_.reset(); - intermediate_participants_.reset(); - commit_parts_.reset(); + has_write_state_ = false; prev_record_lsn_.reset(); redo_lsns_.reset(); - scheduler_.reset(); prepare_version_.reset(); - trans_type_ = TransType::SP_TRANS; next_log_entry_no_ = 0; max_applied_log_ts_.reset(); max_applying_log_ts_.reset(); @@ -374,11 +347,8 @@ void ObTxExecInfo::reset() is_dup_tx_ = false; //touched_pkeys_.reset(); multi_data_source_.reset(); - prepare_log_info_arr_.reset(); xid_.reset(); need_checksum_ = true; - is_sub2pc_ = false; - exec_epoch_ = 0; serial_final_scn_.reset(); serial_final_seq_no_.reset(); dli_batch_set_.destroy(); @@ -456,33 +426,6 @@ void ObTxExecInfo::clear_buffer_ctx_in_multi_data_source() } } -int ObTxExecInfo::assign_commit_parts(const share::ObLSArray &participants, - const ObTxCommitParts &commit_parts) -{ - int ret = OB_SUCCESS; - - if (participants.count() != commit_parts.count()) { - // recover old version log, we need mock the commit parts - for (int64_t i = 0; OB_SUCC(ret) && i < participants.count(); i++) { - if (OB_FAIL(commit_parts_.push_back(ObTxExecPart(participants[i], - -1 /*exec_epoch*/)))) { - TRANS_LOG(WARN, "set commit parts error", K(ret), K(*this)); - } - } - - if (OB_FAIL(ret)) { - // reset on failure to ensure atomicity - commit_parts_.reset(); - } - } else { - if (OB_FAIL(commit_parts_.assign(commit_parts))) { - TRANS_LOG(WARN, "set commit parts error", K(ret), K(*this)); - } - } - - return ret; -} - int ObTxExecInfo::assign(const ObTxExecInfo &exec_info) { int ret = OB_SUCCESS; @@ -490,23 +433,12 @@ int ObTxExecInfo::assign(const ObTxExecInfo &exec_info) if (this == &exec_info) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "no need to assign the same object", KR(ret), K(exec_info)); - } else if (OB_FAIL(participants_.assign(exec_info.participants_))) { - TRANS_LOG(WARN, "participants assign error", KR(ret), K(exec_info)); - } else if (OB_FAIL(incremental_participants_.assign(exec_info.incremental_participants_))) { - TRANS_LOG(WARN, "incremental participants assign error", KR(ret), K(exec_info)); - } else if (OB_FAIL(intermediate_participants_.assign(exec_info.intermediate_participants_))) { - TRANS_LOG(WARN, "intermediate participants assign error", KR(ret), K(exec_info)); } else if (OB_FAIL(redo_lsns_.assign(exec_info.redo_lsns_))) { TRANS_LOG(WARN, "redo_lsns assign error", KR(ret), K(exec_info)); } else if (OB_FAIL(multi_data_source_.assign(exec_info.multi_data_source_))) { TRANS_LOG(WARN, "multi_data_source assign error", KR(ret), K(exec_info)); } else if (OB_FAIL(mds_buffer_ctx_array_.assign(exec_info.mds_buffer_ctx_array_))) { TRANS_LOG(WARN, "mds_buffer_ctx_array assign error", KR(ret), K(exec_info)); - } else if (OB_FAIL(prepare_log_info_arr_.assign(exec_info.prepare_log_info_arr_))) { - TRANS_LOG(WARN, "prepare log info array assign error", KR(ret), K(exec_info)); - } else if (OB_FAIL(assign_commit_parts(exec_info.participants_, - exec_info.commit_parts_))) { - TRANS_LOG(WARN, "commit parts assign error", KR(ret), K(exec_info)); } else if (OB_FAIL(dli_batch_set_.assign(exec_info.dli_batch_set_))) { TRANS_LOG(WARN, "direct load inc batch set assign error", K(ret), K(exec_info.dli_batch_set_)); } else { @@ -514,10 +446,8 @@ int ObTxExecInfo::assign(const ObTxExecInfo &exec_info) // for ObTransPartCtx::get_prepare_version_if_preapred(); prepare_version_.atomic_store(exec_info.prepare_version_); state_ = exec_info.state_; - upstream_ = exec_info.upstream_; + has_write_state_ = exec_info.has_write_state_; prev_record_lsn_ = exec_info.prev_record_lsn_; - scheduler_ = exec_info.scheduler_; - trans_type_ = exec_info.trans_type_; next_log_entry_no_ = exec_info.next_log_entry_no_; max_applied_log_ts_ = exec_info.max_applied_log_ts_; max_applying_log_ts_ = exec_info.max_applying_log_ts_; @@ -533,8 +463,6 @@ int ObTxExecInfo::assign(const ObTxExecInfo &exec_info) is_dup_tx_ = exec_info.is_dup_tx_; xid_ = exec_info.xid_; need_checksum_ = exec_info.need_checksum_; - is_sub2pc_ = exec_info.is_sub2pc_; - exec_epoch_ = exec_info.exec_epoch_; serial_final_scn_ = exec_info.serial_final_scn_; serial_final_seq_no_ = exec_info.serial_final_seq_no_; } @@ -543,15 +471,11 @@ int ObTxExecInfo::assign(const ObTxExecInfo &exec_info) OB_SERIALIZE_MEMBER(ObTxExecInfo, state_, - upstream_, - participants_, - incremental_participants_, + has_write_state_, prev_record_lsn_, redo_lsns_, multi_data_source_, - scheduler_, prepare_version_, - trans_type_, next_log_entry_no_, max_applying_log_ts_, max_applied_log_ts_, @@ -560,17 +484,12 @@ OB_SERIALIZE_MEMBER(ObTxExecInfo, checksum_[0], // FARM COMPAT WHITELIST checksum_scn_[0], // FARM COMPAT WHITELIST max_durable_lsn_, - data_complete_, - is_dup_tx_, + data_complete_, + is_dup_tx_, // touched_pkeys_, - prepare_log_info_arr_, xid_, need_checksum_, - is_sub2pc_, mds_buffer_ctx_array_, - intermediate_participants_, - commit_parts_, - exec_epoch_, checksum_, checksum_scn_, serial_final_scn_, @@ -596,64 +515,5 @@ void ObMulSourceDataNotifyArg::reset() -int RollbackMaskSet::merge_part(const share::ObLSID add_ls_id, const int64_t exec_epoch) -{ - int ret = OB_SUCCESS; - bool is_exist = false; - ObSpinLockGuard guard(lock_); - if (OB_ISNULL(rollback_parts_)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "rollback_parts is null", K(ret), K(add_ls_id)); - } else { - for (int64_t i = 0; i < rollback_parts_->count(); i++) { - if (rollback_parts_->at(i).ls_id_ == add_ls_id) { - is_exist = true; - if (OB_FAIL(mask_set_.unmask(rollback_parts_->at(i)))) { - TRANS_LOG(WARN, "unmask fail", KR(ret), K(add_ls_id)); - } - break; - } - } - if (!is_exist && OB_FAIL(rollback_parts_->push_back(ObTxExecPart(add_ls_id, - exec_epoch)))) { - TRANS_LOG(WARN, "push part to array failed", KR(ret), K(add_ls_id)); - } - } - return ret; -} - -int RollbackMaskSet::find_part(const share::ObLSID ls_id, - const int64_t orig_epoch, - ObTxExecPart &part) -{ - int ret = OB_SUCCESS; - bool is_exist = false; - ObSpinLockGuard guard(lock_); - if (OB_ISNULL(rollback_parts_)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "rollback_parts is null", K(ret), K(ls_id)); - } else { - for (int64_t idx = 0; idx < rollback_parts_->count(); idx++) { - if (rollback_parts_->at(idx).ls_id_ == ls_id) { - if (rollback_parts_->at(idx).exec_epoch_ != orig_epoch) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "check rollback part failed", K(ret), K(rollback_parts_), K(orig_epoch)); - } else { - part = rollback_parts_->at(idx); - is_exist = true; - } - break; - } - } - } - if (OB_SUCC(ret) && !is_exist) { - ret = OB_ENTRY_NOT_EXIST; - } - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "find part", K(ret), K(ls_id), K(orig_epoch), K(rollback_parts_)); - } - return ret; -} - } // transaction } // oceanbase diff --git a/src/storage/tx/ob_trans_define.h b/src/storage/tx/ob_trans_define.h index eb0f59f7f..cbed6ddad 100644 --- a/src/storage/tx/ob_trans_define.h +++ b/src/storage/tx/ob_trans_define.h @@ -30,7 +30,6 @@ #include "logservice/ob_log_base_header.h" #include "share/scn.h" #include "share/ob_cluster_version.h" -#include "share/ob_ls_id.h" #include "share/allocator/ob_reserve_arena.h" #include "sql/ob_sql_define.h" #include "sql/resolver/ob_stmt_type.h" @@ -90,7 +89,7 @@ namespace transaction { class ObLSTxCtxMgr; class ObTransCtx; -class ObPartTransCtx; +class ObTxCtx; class ObMemtableKeyInfo; class AggreLogTask; class ObXACtx; @@ -255,83 +254,10 @@ struct ObLockForReadArg ObTxSEQ data_sql_sequence_; bool read_latest_; bool read_uncommitted_; - // Compare with the source SCN: sstable is end_scn, and memtable is ObMvccTransNode scn. + // SCN used for visibility checks; sstable uses end_scn, memtable uses ObMvccTransNode scn. share::SCN scn_; }; -class ObTransKey final -{ -public: - ObTransKey() { /*reset();*/ } - ObTransKey(const ObTransKey &other) - { - ls_id_ = other.ls_id_; - trans_id_ = other.trans_id_; - hash_val_ = other.hash_val_; - } - explicit ObTransKey(const share::ObLSID &ls_id, const ObTransID &trans_id) - : ls_id_(ls_id), trans_id_(trans_id), hash_val_(0) { calc_hash_(); } -public: - bool is_valid() const { return ls_id_.is_valid() && trans_id_.is_valid(); } - ObTransKey &operator=(const ObTransKey &other) - { - ls_id_ = other.ls_id_; - trans_id_ = other.trans_id_; - hash_val_ = other.hash_val_; - return *this; - } - bool operator==(const ObTransKey &other) const - { return hash_val_ == other.hash_val_ && ls_id_ == other.ls_id_ && trans_id_ == other.trans_id_; } - bool operator!=(const ObTransKey &other) const - { return hash_val_ != other.hash_val_ || ls_id_ != other.ls_id_ || trans_id_ != other.trans_id_; } - OB_INLINE uint64_t hash() const { return hash_val_; } - int compare(const ObTransKey& other) const - { - int compare_ret = 0; - compare_ret = ls_id_.compare(other.ls_id_); - if (0 == compare_ret) { - compare_ret = trans_id_.compare(other.trans_id_); - } - return compare_ret; - } - int compare_trans_table(const ObTransKey& other) const - { - int compare_ret = 0; - if (hash_val_ == other.hash_val_) { - compare_ret = trans_id_.compare(other.trans_id_); - } else if (hash_val_ > other.hash_val_) { - compare_ret = 1; - } else { - compare_ret = -1; - } - return compare_ret; - } - void reset() - { - ls_id_.reset(); - trans_id_.reset(); - hash_val_ = 0; - } - const share::ObLSID &get_ls_id() const { return ls_id_; } - const ObTransID get_trans_id() const { return trans_id_; } - DECLARE_TO_STRING_AND_YSON; -private: - OB_INLINE uint64_t calc_hash_() - { - uint64_t hash_val = 0; - uint64_t trans_id_hash_val = 0; - uint64_t ls_val = 0; - trans_id_hash_val = trans_id_.hash(); - ls_val = ls_id_.hash(); - hash_val = common::murmurhash(&trans_id_hash_val, sizeof(trans_id_hash_val), ls_val); - return hash_val_ = hash_val; - } -private: - share::ObLSID ls_id_; - ObTransID trans_id_; - uint64_t hash_val_; -}; - class ObTransAccessMode { public: @@ -385,30 +311,6 @@ class ObTransSnapshotGeneType ~ObTransSnapshotGeneType() {} }; -enum TransType : int32_t -{ - UNKNOWN_TRANS = -1, - SP_TRANS = 0, - DIST_TRANS = 2 -}; - - -// class TransType -// { -// public: -// static const int32_t UNKNOWN = -1; -// static const int32_t SP_TRANS = 0; -// static const int32_t MINI_SP_TRANS = 1; -// static const int32_t DIST_TRANS = 2; -// static const int32_t EMPTY_TRANS = 3; -// public: -// static bool is_valid(const int32_t type) -// { return SP_TRANS == type || MINI_SP_TRANS == type || DIST_TRANS == type || EMPTY_TRANS == type; } -// private: -// TransType() {} -// ~TransType() {} -// }; -// class ObTransType { public: @@ -426,12 +328,6 @@ class ObTransType ~ObTransType() {} }; -enum class ObGlobalTxType : uint8_t -{ - PLAIN = 0, - XA_TRANS = 1, -}; - /* * +--------------------- +-------------------------------------+---------------------------------+ * | | CURRENT_READ | BOUNDED_STALENESS_READ | @@ -905,22 +801,18 @@ class ObRunningState ~ObRunningState() {} }; -// sub state is volatile state of tx ctx -class ObTxSubState +// runtime state is volatile state of tx ctx +class ObTxRuntimeState { public: - ObTxSubState() : flag_() {} - ~ObTxSubState() {} + ObTxRuntimeState() : flag_() {} + ~ObTxRuntimeState() {} void reset() { flag_.reset(); } bool is_info_log_submitted() const { return flag_.info_log_submitted_; } void set_info_log_submitted() { flag_.info_log_submitted_ = 1; } void clear_info_log_submitted() { flag_.info_log_submitted_ = 0; } - bool is_gts_waiting() const { return flag_.gts_waiting_; } - void set_gts_waiting() { flag_.gts_waiting_ = 1; } - void clear_gts_waiting() { flag_.gts_waiting_ = 0; } - bool is_state_log_submitting() const { return flag_.state_log_submitting_; } void set_state_log_submitting() { flag_.state_log_submitting_ = 1; } void clear_state_log_submitting() { flag_.state_log_submitting_ = 0; } @@ -929,80 +821,55 @@ class ObTxSubState void set_state_log_submitted() { flag_.state_log_submitted_ = 1; } void clear_state_log_submitted() { flag_.state_log_submitted_ = 0; } - bool is_prepare_notified() const { return flag_.prepare_notify_; } - void set_prepare_notified() { flag_.prepare_notify_ = 1; } - void clear_prepare_notified() { flag_.prepare_notify_ = 0; } - bool is_force_abort() const { return flag_.force_abort_; } void set_force_abort() { flag_.force_abort_ = 1; } void clear_force_abort() { flag_.force_abort_ = 0; } + bool is_gts_waiting() const { return flag_.gts_waiting_; } + void set_gts_waiting() { flag_.gts_waiting_ = 1; } + void clear_gts_waiting() { flag_.gts_waiting_ = 0; } + DECLARE_ON_DEMAND_TO_STRING TO_STRING_KV("info_log_submitted", flag_.info_log_submitted_, - "gts_waiting", - flag_.gts_waiting_, "state_log_submitting", flag_.state_log_submitting_, "state_log_submitted", flag_.state_log_submitted_, - // "prepare_notify", - // flag_.prepare_notify_, "force_abort", - flag_.force_abort_); + flag_.force_abort_, + "gts_waiting", + flag_.gts_waiting_); bool is_valid() const { return flag_.is_valid(); } private: struct BitFlag { unsigned int info_log_submitted_ : 1; - unsigned int gts_waiting_ : 1; unsigned int state_log_submitting_ : 1; unsigned int state_log_submitted_ : 1; - unsigned int prepare_notify_ : 1; unsigned int force_abort_ : 1; + unsigned int gts_waiting_ : 1; void reset() { info_log_submitted_ = 0; - gts_waiting_ = 0; state_log_submitting_ = 0; state_log_submitted_ = 0; - prepare_notify_ = 0; force_abort_ = 0; + gts_waiting_ = 0; } bool is_valid() const { - return info_log_submitted_ > 0 || gts_waiting_ > 0 || state_log_submitted_ > 0 - || state_log_submitting_ > 0 || prepare_notify_ > 0 || force_abort_ > 0; + return info_log_submitted_ > 0 || state_log_submitted_ > 0 + || state_log_submitting_ > 0 || force_abort_ > 0 || gts_waiting_ > 0; } BitFlag() { reset(); } } flag_; }; -class Ob2PCPrepareState -{ -public: - static const int64_t UNKNOWN = -1; - static const int64_t INIT = 0; - static const int64_t REDO_PREPARING = 11; - static const int64_t REDO_PREPARED = 12; - static const int64_t VERSION_PREPARING = 13; - static const int64_t VERSION_PREPARED = 14; -public: - static bool is_valid(const int64_t state) - { return state >= INIT && state <= VERSION_PREPARED; } - static bool in_state(const int64_t state) - { return state >= REDO_PREPARING && state <= VERSION_PREPARED; } - static bool is_redo_prepared(const int64_t state) - { return state >= REDO_PREPARED; } -private: - Ob2PCPrepareState() {} - ~Ob2PCPrepareState() {} -}; - class ObTransSubmitLogState { public: @@ -1022,30 +889,13 @@ class ObTransRetryTaskType static const int64_t UNKNOWN = -1; static const int64_t END_TRANS_CB_TASK = 0; static const int64_t ADVANCE_LS_CKPT_TASK = 1; - static const int64_t STANDBY_CLEANUP_TASK = 2; - static const int64_t DUP_TABLE_TX_REDO_SYNC_RETRY_TASK = 3; - static const int64_t MAX = 4; + static const int64_t DUP_TABLE_TX_REDO_SYNC_RETRY_TASK = 2; + static const int64_t MAX = 3; public: static bool is_valid(const int64_t task_type) { return task_type > UNKNOWN && task_type < MAX; } }; -class ObTransCtxType -{ -public: - static const int64_t UNKNOWN = -1; - static const int64_t SCHEDULER = 0; - static const int64_t COORDINATOR = 1; - static const int64_t PARTICIPANT = 2; - static const int64_t SLAVE_PARTICIPANT = 3; -public: - static bool is_valid(const int64_t type) - { return SCHEDULER == type || COORDINATOR == type || PARTICIPANT == type - || SLAVE_PARTICIPANT == type; } -}; - -typedef common::ObSArray ObLeaderEpochArray; - class ObMemtableKeyInfo { public: @@ -1137,17 +987,6 @@ class ObTransTask : public common::LinkTask int64_t task_type_; }; -class ObBatchCommitState -{ -public: - static const int INVALID = -1; - static const int INIT = 0; - static const int ALLOC_LOG_ID_TS = 1; - static const int GENERATE_PREPARE_LOG = 2; - static const int GENERATE_REDO_PREPARE_LOG = 3; - static const int BATCH_COMMITTED = 4; -}; - enum ObPartitionAuditOperator { PART_AUDIT_SET_BASE_ROW_COUNT = 0, @@ -1280,22 +1119,6 @@ class ObCoreLocalPartitionAuditInfo : public common::ObCoreLocalStorage STATEMENT_NOT_FINISH -// - If stmt info matches before leader switch and submit log: STATEMENT_NOT_FINISH -> NO_CHANGING_LEADER -// - if exists a on-the-fly log during submit STATE log: STATEMENT_NOT_FINISH -> LOGGING_NOT_FINISH -// If there exists a on-the-fly log during submit STATE log: NO_CHANGING_LEADER -> LOGGING_NOT_FINISH -// - If the prev log is synced before leader switch and submit log: LOGGING_NOT_FINISH -> NO_CHANGING_LEADER -// If the leader revokes: STATEMENT_NOT_FINISH/LOGGING_NOT_FINISH -> NO_CHANGING_LEADER -enum CHANGING_LEADER_STATE -{ - NO_CHANGING_LEADER = 0, - STATEMENT_NOT_FINISH = 1, - LOGGING_NOT_FINISH = 2 -}; - class ObAddrLogId { public: @@ -1370,121 +1193,9 @@ class ObUndoAction ObTxSEQ undo_to_; // exclusive }; -class ObLSLogInfo final -{ - OB_UNIS_VERSION(1); -public: - ObLSLogInfo() : id_(), offset_() {} - ObLSLogInfo(const share::ObLSID &id, const palf::LSN &offset) : id_(id), offset_(offset) {} - const share::ObLSID &get_ls_id() const { return id_; } - const palf::LSN &get_lsn() const { return offset_; } - bool is_valid() const - { - return id_.is_valid() && offset_.is_valid(); - } - - bool operator==(const ObLSLogInfo &log_info) const - { - return id_ == log_info.id_ && offset_ == log_info.offset_; - } - - TO_STRING_KV(K(id_), K(offset_)) -private: - share::ObLSID id_; - palf::LSN offset_; -}; - -struct ObTxExecPart -{ - OB_UNIS_VERSION(2); -public: - ObTxExecPart() : ls_id_(), - exec_epoch_(-1) {} - ObTxExecPart(share::ObLSID ls_id, int64_t epoch) - : ls_id_(ls_id), - exec_epoch_(epoch) {} - inline bool operator==(const ObTxExecPart &other) const { - return other.ls_id_ == ls_id_ && - other.exec_epoch_ == exec_epoch_; - } - bool is_valid() const { - return ls_id_.is_valid() - && exec_epoch_ > 0; - } - share::ObLSID ls_id_; - int64_t exec_epoch_; - - TO_STRING_KV(K_(ls_id), K_(exec_epoch)); -}; - -struct ObStandbyCheckInfo -{ - OB_UNIS_VERSION(1); -public: - ObStandbyCheckInfo() : - check_info_ori_ls_id_(-1), - check_part_() - {} - ~ObStandbyCheckInfo() {} - bool operator==(const ObStandbyCheckInfo &other) const { - bool bool_ret = check_info_ori_ls_id_ == other.check_info_ori_ls_id_ - && check_part_ == other.check_part_; - return bool_ret; - } - void operator=(const ObStandbyCheckInfo &other) { - check_info_ori_ls_id_ = other.check_info_ori_ls_id_; - check_part_ = other.check_part_; - } - bool is_valid() const { return check_info_ori_ls_id_.is_valid() - && check_part_.is_valid(); } - share::ObLSID check_info_ori_ls_id_; // those carrry check info origin ls id - ObTxExecPart check_part_; - TO_STRING_KV(K_(check_info_ori_ls_id), K_(check_part)); -}; - -class ObStateInfo -{ -public: - ObStateInfo() : state_(ObTxState::UNKNOWN), version_(), snapshot_version_(), check_info_() {} - ObStateInfo(const share::ObLSID &ls_id, - const ObTxState &state, - const share::SCN &version, - const share::SCN &snapshot_version) : - ls_id_(ls_id), state_(state) - { - version_ = version; - snapshot_version_ = snapshot_version; - } - ~ObStateInfo() {} - bool is_valid() const - { - return ls_id_.is_valid() && version_.is_valid() && snapshot_version_.is_valid(); } - void operator=(const ObStateInfo &state_info) - { - ls_id_ = state_info.ls_id_; - state_ = state_info.state_; - version_ = state_info.version_; - snapshot_version_ = state_info.snapshot_version_; - check_info_ = state_info.check_info_; - } - - bool need_update(const ObStateInfo &state_info); - TO_STRING_KV(K_(ls_id), K_(state), K_(version), K_(snapshot_version), K_(check_info)) - OB_UNIS_VERSION(1); -public: - share::ObLSID ls_id_; - ObTxState state_; - share::SCN version_; - share::SCN snapshot_version_; -// for epoch check - ObStandbyCheckInfo check_info_; -}; - typedef common::ObSEArray ObElrTransInfoArray; typedef common::ObSEArray ObRedoLogIdArray; typedef common::ObSEArray ObRedoLSNArray; -typedef common::ObSEArray ObLSLogInfoArray; -typedef common::ObSEArray ObStateInfoArray; struct CtxInfo final { @@ -1510,48 +1221,14 @@ static const int64_t MAX_ELR_TRANS_INTERVAL = 400 * 1000; #endif // max scheudler context in single server -static const int64_t MAX_PART_CTX_COUNT = 700 * 1000; +static const int64_t MAX_TX_CTX_COUNT = 700 * 1000; static const int DUP_TABLE_LEASE_LIST_MAX_COUNT = 8; #define TRANS_AGGRE_LOG_TIMESTAMP OB_INVALID_TIMESTAMP -typedef common::ObSEArray ObTxCommitParts; -typedef common::ObSEArray ObTxRollbackParts; - -#define CONVERT_COMMIT_PARTS_TO_PARTS(commit_parts, parts) \ - for (int64_t idx = 0; OB_SUCC(ret) && idx < commit_parts.count(); idx++) { \ - if (OB_FAIL(parts.push_back(commit_parts.at(idx).ls_id_))) { \ - TRANS_LOG(WARN, "parts push failed", K(ret)); \ - } \ - } \ - if (OB_FAIL(ret)) { \ - parts.reset(); \ - } -#define CONVERT_PARTS_TO_COMMIT_PARTS(parts, commit_parts) \ - for (int64_t idx = 0; OB_SUCC(ret) && idx < parts.count(); idx++) { \ - if (OB_FAIL(commit_parts.push_back(ObTxExecPart(parts.at(idx), -1)))) { \ - TRANS_LOG(WARN, "parts push failed", K(ret)); \ - } \ - } \ - if (OB_FAIL(ret)) { \ - commit_parts.reset(); \ - } - -class ObEndParticipantsRes -{ -public: - void reset() - { - blocked_trans_ids_.reset(); - } - TO_STRING_KV(K_(blocked_trans_ids)); -private: - ObBlockedTransArray blocked_trans_ids_; -}; - -enum class PartCtxSource +enum class TxCtxSource { UNKNOWN = 0, MVCC_WRITE = 1, @@ -1560,16 +1237,16 @@ enum class PartCtxSource RECOVER = 4, }; -static const char * to_str_ctx_source(const PartCtxSource & ctx_src) +static const char * to_str_ctx_source(const TxCtxSource & ctx_src) { const char * str = "INVALID"; switch(ctx_src) { - TRX_ENUM_CASE_TO_STR(PartCtxSource, UNKNOWN); - TRX_ENUM_CASE_TO_STR(PartCtxSource, MVCC_WRITE); - TRX_ENUM_CASE_TO_STR(PartCtxSource, REGISTER_MDS); - TRX_ENUM_CASE_TO_STR(PartCtxSource, REPLAY); - TRX_ENUM_CASE_TO_STR(PartCtxSource, RECOVER); + TRX_ENUM_CASE_TO_STR(TxCtxSource, UNKNOWN); + TRX_ENUM_CASE_TO_STR(TxCtxSource, MVCC_WRITE); + TRX_ENUM_CASE_TO_STR(TxCtxSource, REGISTER_MDS); + TRX_ENUM_CASE_TO_STR(TxCtxSource, REPLAY); + TRX_ENUM_CASE_TO_STR(TxCtxSource, RECOVER); } return str; } @@ -1586,36 +1263,16 @@ class ObTxMDSCache; static const int64_t MAX_TABLET_MODIFY_RECORD_COUNT = 16; // exec info need to be persisted by "trans context table" -template -struct ObIArrayPrintTrait { - const ObIArray &arr_; - ObIArrayPrintTrait(const ObIArray &arr): arr_(arr) {} - DECLARE_TO_STRING - { - int64_t pos = 0; - J_ARRAY_START(); - for(int i =0; i < arr_.count(); i++) { - BUF_PRINTO(arr_.at(i)); - if (i != arr_.count() - 1) { J_COMMA(); } - } - J_ARRAY_END(); - return pos; - } -}; struct ObTxExecInfo { OB_UNIS_VERSION(2); public: ObTxExecInfo() {} explicit ObTxExecInfo(TransModulePageAllocator &allocator) - : participants_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "PARTICIPANT")), - incremental_participants_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "INC_PART`")), - intermediate_participants_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "INTER_PART`")), - redo_lsns_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "REDO_LSNS")), + : redo_lsns_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "REDO_LSNS")), multi_data_source_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "MDS_ARRAY")), checksum_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "TX_CHECKSUM")), - checksum_scn_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "TX_CHECKSUM")), - prepare_log_info_arr_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "PREPARE_INFO")) + checksum_scn_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator, "TX_CHECKSUM")) { checksum_.push_back(0); checksum_scn_.push_back(share::SCN()); @@ -1628,11 +1285,11 @@ struct ObTxExecInfo // can not destroy in tx_ctx_table void destroy(ObTxMDSCache &mds_cache); int assign(const ObTxExecInfo &exec_info); + bool has_write_state() const { return has_write_state_; } + void mark_write_state() { has_write_state_ = true; } private: ObTxExecInfo &operator=(const ObTxExecInfo &info); - int assign_commit_parts(const share::ObLSArray &participants, - const ObTxCommitParts &commit_parts); public: DECLARE_ON_DEMAND_TO_STRING @@ -1642,17 +1299,12 @@ struct ObTxExecInfo // const_cast>(checksum_scn_).set_max_print_count(512); int64_t pos = 0; J_KV(K_(state), - K_(upstream), - K_(participants), - K_(incremental_participants), - K_(intermediate_participants), + K_(has_write_state), K_(prev_record_lsn), K_(redo_lsns), "redo_log_no", redo_lsns_.count(), K_(multi_data_source), - K_(scheduler), K_(prepare_version), - K_(trans_type), K_(next_log_entry_no), K_(max_applied_log_ts), K_(max_applying_log_ts), @@ -1664,50 +1316,33 @@ struct ObTxExecInfo K_(data_complete), K_(is_dup_tx), //K_(touched_pkeys), - K_(prepare_log_info_arr), K_(xid), K_(need_checksum), - K_(is_sub2pc), - K_(commit_parts), - K_(exec_epoch), K_(serial_final_scn), K_(serial_final_seq_no), K(dli_batch_set_.size())); return pos; } ObTxState state_; - share::ObLSID upstream_; - share::ObLSArray participants_; - ObTxCommitParts commit_parts_; - // for tree phase commit - share::ObLSArray incremental_participants_; - ObTxCommitParts intermediate_participants_; + bool has_write_state_; LogOffSet prev_record_lsn_; ObRedoLSNArray redo_lsns_; ObTxBufferNodeArray multi_data_source_; ObTxBufferCtxArray mds_buffer_ctx_array_; - common::ObAddr scheduler_; share::SCN prepare_version_; - int64_t trans_type_; int64_t next_log_entry_no_; share::SCN max_applied_log_ts_; share::SCN max_applying_log_ts_; - int64_t max_applying_part_log_no_; // start from 0 on follower and always be INT64_MAX on leader - ObTxSEQ max_submitted_seq_no_; // maintained on leader and sent to follower via ActiveInfoLog + int64_t max_applying_part_log_no_; + ObTxSEQ max_submitted_seq_no_; ObSEArray checksum_; ObSEArray checksum_scn_; palf::LSN max_durable_lsn_; bool data_complete_; bool is_dup_tx_; - // NB: Ensure where to put it the touched pkeys - //for liboblog - ObLSLogInfoArray prepare_log_info_arr_; // for xa ObXATransID xid_; bool need_checksum_; - bool is_sub2pc_; - int64_t exec_epoch_; - // if valid, this txCtx is logged by multi parallel thread // since this scn share::SCN serial_final_scn_; // the logic time of serial final log submitted diff --git a/src/storage/tx/ob_trans_define_v4.cpp b/src/storage/tx/ob_trans_define_v4.cpp index e067687de..d474cc22c 100644 --- a/src/storage/tx/ob_trans_define_v4.cpp +++ b/src/storage/tx/ob_trans_define_v4.cpp @@ -152,38 +152,23 @@ DEF_TO_STRING(ObTxSavePoint) J_OBJ_END(); return pos; } -OB_SERIALIZE_MEMBER(ObTxExecResult, incomplete_, parts_, +OB_SERIALIZE_MEMBER(ObTxExecResult, incomplete_, has_write_state_, write_state_, conflict_txs_,// FARM COMPAT WHITELIST for cflict_txs_ conflict_info_array_, - touched_ls_list_); + touched_storage_); OB_SERIALIZE_MEMBER(ObTxSnapshot, tx_id_, version_, scn_, elr_); OB_SERIALIZE_MEMBER(ObTxReadSnapshot, valid_, core_, source_, uncertain_bound_, - snapshot_lsid_, - parts_, - snapshot_ls_role_, - committed_, - snapshot_acquire_addr_); + has_write_state_, + committed_); - -// for lob aux table tx read snapshot -#define PREPARE_LOB_PARTS(parts, ls_id) \ - ObSEArray parts; \ - ARRAY_FOREACH_NORET(parts_, i) { \ - if (parts_.at(i).left_ == ls_id) { \ - parts.push_back(parts_.at(i)); \ - break; \ - } \ -} \ - -int ObTxReadSnapshot::serialize_for_lob(const share::ObLSID &ls_id, const share::SCN &fb_snapshot, SERIAL_PARAMS) const +int ObTxReadSnapshot::serialize_for_lob(const share::SCN &fb_snapshot, SERIAL_PARAMS) const { int ret = OB_SUCCESS; - PREPARE_LOB_PARTS(parts, ls_id); - LST_DO_CODE(OB_UNIS_ENCODE, core_, source_, snapshot_lsid_, snapshot_ls_role_, parts); + LST_DO_CODE(OB_UNIS_ENCODE, core_, source_, has_write_state_); OB_UNIS_ENCODE(fb_snapshot); return ret; } @@ -191,7 +176,7 @@ int ObTxReadSnapshot::serialize_for_lob(const share::ObLSID &ls_id, const share: int ObTxReadSnapshot::deserialize_for_lob(share::SCN &fb_snapshot, DESERIAL_PARAMS) { int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, core_, source_, snapshot_lsid_, snapshot_ls_role_, parts_); + LST_DO_CODE(OB_UNIS_DECODE, core_, source_, has_write_state_); OB_UNIS_DECODE(fb_snapshot); if (OB_SUCC(ret)) { valid_ = true; @@ -199,30 +184,27 @@ int ObTxReadSnapshot::deserialize_for_lob(share::SCN &fb_snapshot, DESERIAL_PARA return ret; } -int64_t ObTxReadSnapshot::get_serialize_size_for_lob(const share::ObLSID &ls_id, const share::SCN &fb_snapshot) const +int64_t ObTxReadSnapshot::get_serialize_size_for_lob(const share::SCN &fb_snapshot) const { int64_t len = 0; - PREPARE_LOB_PARTS(parts, ls_id); - LST_DO_CODE(OB_UNIS_ADD_LEN, core_, source_, snapshot_lsid_, snapshot_ls_role_, parts); + LST_DO_CODE(OB_UNIS_ADD_LEN, core_, source_, has_write_state_); OB_UNIS_ADD_LEN(fb_snapshot); return len; } -int ObTxReadSnapshot::build_snapshot_for_lob(const ObTxSnapshot &core, const share::ObLSID &ls_id) +int ObTxReadSnapshot::build_snapshot_for_lob(const ObTxSnapshot &core) { int ret = OB_SUCCESS; core_ = core; valid_= true; source_ = ObTxReadSnapshot::SRC::LS; - snapshot_lsid_ = ls_id; return ret; } int ObTxReadSnapshot::build_snapshot_for_lob( const int64_t snapshot_version, const int64_t snapshot_tx_id, - const int64_t snapshot_seq, - const share::ObLSID &ls_id) + const int64_t snapshot_seq) { int ret = OB_SUCCESS; core_.version_.convert_for_tx(snapshot_version); @@ -230,7 +212,6 @@ int ObTxReadSnapshot::build_snapshot_for_lob( core_.scn_ = ObTxSEQ::cast_from_int(snapshot_seq); valid_ = true; source_ = ObTxReadSnapshot::SRC::LS; - snapshot_lsid_ = ls_id; return ret; } @@ -248,86 +229,28 @@ int ObTxReadSnapshot::refresh_seq_no(const int64_t tx_seq_base) void ObTxReadSnapshot::convert_to_out_tx() { - parts_.reset(); + has_write_state_ = false; core_.tx_id_.reset(); core_.scn_.reset(); } -OB_SERIALIZE_MEMBER(ObTxPart, id_, addr_, epoch_, first_scn_, last_scn_, flag_.flag_val_); +OB_SERIALIZE_MEMBER(ObTxWriteState, first_scn_, last_scn_, flag_.flag_val_); -DEFINE_SERIALIZE(ObTxDesc::FLAG::FOR_FIXED_SER_VAL) +DEFINE_SERIALIZE(ObTxDesc::FLAG) { int ret = OB_SUCCESS; return serialization::encode_i64(buf, buf_len, pos, v_); } -DEFINE_DESERIALIZE(ObTxDesc::FLAG::FOR_FIXED_SER_VAL) +DEFINE_DESERIALIZE(ObTxDesc::FLAG) { int ret = OB_SUCCESS; return serialization::decode_i64(buf, data_len, pos, (int64_t*)&v_); } -DEFINE_GET_SERIALIZE_SIZE(ObTxDesc::FLAG::FOR_FIXED_SER_VAL) +DEFINE_GET_SERIALIZE_SIZE(ObTxDesc::FLAG) { return serialization::encoded_length_i64(v_); } -uint64_t ObTxDesc::FLAG::COMPAT_FOR_EXEC::get_serialize_v_() const -{ - FLAG ret_flag; - FLAG this_flag; - this_flag.compat_for_exec_.v_ = v_; - ret_flag.compat_for_exec_.v_ = 0; -#define _SET_FLAG_(x) ret_flag.x = this_flag.x - LST_DO(_SET_FLAG_, (;), - EXPLICIT_, - SHADOW_, - REPLICA_, - TRACING_); -#undef _SET_FLAG_ - return ret_flag.compat_for_exec_.v_; -} - -uint64_t ObTxDesc::FLAG::COMPAT_FOR_TX_ROUTE::get_serialize_v_() const -{ - FLAG ret_flag; - FLAG this_flag; - this_flag.compat_for_tx_route_.v_ = v_; - ret_flag.compat_for_tx_route_.v_ = 0; -#define _SET_FLAG_(x) ret_flag.x = this_flag.x - LST_DO(_SET_FLAG_, (;), - EXPLICIT_, - SHADOW_, - REPLICA_, - TRACING_, - RELEASED_, - PARTS_INCOMPLETE_, - PART_EPOCH_MISMATCH_, - WITH_TEMP_TABLE_, - DEFER_ABORT_); -#undef _SET_FLAG_ - return ret_flag.compat_for_tx_route_.v_; -} - -#define DEF_SERIALIZE_COMPAT_FOR_TX_DESC_FLAG(THE_TYPE) \ - DEFINE_SERIALIZE(ObTxDesc::FLAG::THE_TYPE) \ - { \ - int ret = OB_SUCCESS; \ - const uint64_t compat_v = get_serialize_v_(); \ - return serialization::encode_vi64(buf, buf_len, pos, compat_v); \ - } \ - DEFINE_DESERIALIZE(ObTxDesc::FLAG::THE_TYPE) \ - { \ - int ret = OB_SUCCESS; \ - return serialization::decode_vi64(buf, data_len, pos, (int64_t*)&v_); \ - } \ - DEFINE_GET_SERIALIZE_SIZE(ObTxDesc::FLAG::THE_TYPE) \ - { \ - const uint64_t compat_v = get_serialize_v_(); \ - return serialization::encoded_length_vi64(compat_v); \ - } - -DEF_SERIALIZE_COMPAT_FOR_TX_DESC_FLAG(COMPAT_FOR_EXEC) -DEF_SERIALIZE_COMPAT_FOR_TX_DESC_FLAG(COMPAT_FOR_TX_ROUTE) - OB_SERIALIZE_MEMBER(ObTxDesc, cluster_id_, @@ -339,15 +262,15 @@ OB_SERIALIZE_MEMBER(ObTxDesc, access_mode_, op_sn_, state_, - flags_.compat_for_exec_, + flags_, expire_ts_, active_ts_, timeout_us_, lock_timeout_us_, active_scn_, - parts_, + has_write_state_, + write_state_, xid_, - flags_.for_serialize_v_, seq_base_, client_sid_); OB_SERIALIZE_MEMBER(ObTxParam, @@ -363,33 +286,6 @@ OB_SERIALIZE_MEMBER(ObTxSavePoint, user_create_, name_); -OB_SERIALIZE_MEMBER(ObTxInfo, - - cluster_id_, - cluster_version_, - addr_, - tx_id_, - isolation_, - access_mode_, - snapshot_version_, - snapshot_uncertain_bound_, - op_sn_, - alloc_ts_, - active_ts_, - timeout_us_, - expire_ts_, - active_scn_, - parts_, - session_id_, - savepoints_, - seq_base_); -OB_SERIALIZE_MEMBER(ObTxStmtInfo, - tx_id_, - op_sn_, - parts_, - state_, - savepoints_); - ObTxDesc::ObTxDesc() : cluster_id_(-1), trace_info_(), @@ -408,7 +304,6 @@ ObTxDesc::ObTxDesc() sess_id_(0), assoc_sess_id_(0), client_sid_(0), - global_tx_type_(ObGlobalTxType::PLAIN), op_sn_(0), // default is from 0 state_(State::INVL), flags_({ 0 }), @@ -423,12 +318,11 @@ ObTxDesc::ObTxDesc() active_scn_(), min_implicit_savepoint_(), last_branch_id_(0), - parts_(), - savepoints_(), - conflict_txs_(), - coord_id_(), - commit_expire_ts_(0), - commit_parts_(), + has_write_state_(false), + write_state_(), + savepoints_(), + conflict_txs_(), + commit_expire_ts_(0), commit_version_(), commit_out_(-1), commit_times_(0), @@ -439,8 +333,6 @@ ObTxDesc::ObTxDesc() commit_cb_(NULL), cb_tid_(-1), exec_info_reap_ts_(0), - brpc_mask_set_(), - rpc_cond_(), commit_task_() #ifdef ENABLE_DEBUG_LOG , alloc_link_() @@ -471,10 +363,9 @@ int ObTxDesc::switch_to_idle() commit_ts_ = 0; finish_ts_ = 0; active_scn_.reset(); - parts_.reset(); - conflict_txs_.reset(); - coord_id_.reset(); - commit_parts_.reset(); + has_write_state_ = false; + write_state_ = ObTxWriteState(); + conflict_txs_.reset(); commit_version_.reset(); commit_out_ = 0; commit_times_ = 0; @@ -492,9 +383,7 @@ int ObTxDesc::switch_to_idle() inline void ObTxDesc::FLAG::switch_to_idle_() { - const FLAG sv = *this; v_ = 0; - REPLICA_ = sv.REPLICA_; } // this function helper will update current flag with the given @@ -535,7 +424,6 @@ void ObTxDesc::reset() last_rc_snapshot_version_.set_min(); snapshot_uncertain_bound_ = 0; snapshot_scn_.reset(); - global_tx_type_ = ObGlobalTxType::PLAIN; op_sn_ = -1; @@ -556,13 +444,12 @@ void ObTxDesc::reset() active_scn_.reset(); min_implicit_savepoint_.reset(); last_branch_id_ = 0; - parts_.reset(); - savepoints_.reset(); - conflict_txs_.reset(); + has_write_state_ = false; + write_state_ = ObTxWriteState(); + savepoints_.reset(); + conflict_txs_.reset(); - coord_id_.reset(); - commit_expire_ts_ = -1; - commit_parts_.reset(); + commit_expire_ts_ = -1; commit_version_.reset(); commit_out_ = -1; commit_times_ = 0; @@ -572,8 +459,6 @@ void ObTxDesc::reset() commit_cb_ = NULL; cb_tid_ = -1; exec_info_reap_ts_ = 0; - brpc_mask_set_.reset(); - rpc_cond_.reset(); commit_task_.reset(); tlog_.reset(); modified_tables_.reset(); @@ -696,44 +581,27 @@ bool ObTxDesc::contain_savepoint(const ObString &sp) return hit; } -int ObTxDesc::update_part_(ObTxPart &a, const bool append, const bool check_only_if_exist) +int ObTxDesc::merge_write_state_(ObTxWriteState &a, const bool append, const bool check_only_if_exist) { int ret = OB_SUCCESS; - bool hit = false; + const bool hit = has_write_state_; if (exec_info_reap_ts_ == 0) { exec_info_reap_ts_ = ObSequence::get_max_seq_no(); } - ARRAY_FOREACH_NORET(parts_, i) { - ObTxPart &p = parts_[i]; - if (p.id_ == a.id_) { - hit = true; - if (p.epoch_ == ObTxPart::EPOCH_DEAD) { - if (a.epoch_ > 0) { // unexpected: from dead to alive - flags_.PART_EPOCH_MISMATCH_ = true; - ret = OB_TRANS_NEED_ROLLBACK; - TRANS_LOG(WARN, "epoch missmatch", K(ret), K(a), K(p)); - } - } else if (p.epoch_ <= 0) { - p.epoch_ = a.epoch_; - } else if (a.epoch_ > 0 && p.epoch_ != a.epoch_) { - flags_.PART_EPOCH_MISMATCH_ = true; - ret = OB_TRANS_NEED_ROLLBACK; - TRANS_LOG(WARN, "tx-part epoch changed", K(ret), K(a), K(p)); - } - if (OB_SUCC(ret) && !check_only_if_exist) { - if (a.addr_.is_valid()) { p.addr_ = a.addr_; } - p.first_scn_ = MIN(a.first_scn_, p.first_scn_); - p.last_scn_ = p.last_scn_.is_max() ? a.last_scn_ : MAX(a.last_scn_, p.last_scn_); - p.last_touch_ts_ = exec_info_reap_ts_ + 1; - if (p.is_clean() && !a.is_clean()) { - p.flag_.set_dirty(); - } + + if (hit) { + ObTxWriteState &p = write_state_; + if (OB_SUCC(ret) && !check_only_if_exist) { + p.first_scn_ = MIN(a.first_scn_, p.first_scn_); + p.last_scn_ = p.last_scn_.is_max() ? a.last_scn_ : MAX(a.last_scn_, p.last_scn_); + p.last_touch_ts_ = exec_info_reap_ts_ + 1; + if (p.is_clean() && !a.is_clean()) { + p.flag_.set_dirty(); } - break; } } - if (ObTxDesc::State::IMPLICIT_ACTIVE == state_ && !active_scn_.is_valid()) { + if (OB_SUCC(ret) && ObTxDesc::State::IMPLICIT_ACTIVE == state_ && !active_scn_.is_valid()) { /* * it is a first stmt's retry, we should set active scn * to enable recognizing it is first stmt @@ -741,100 +609,74 @@ int ObTxDesc::update_part_(ObTxPart &a, const bool append, const bool check_only active_scn_ = get_tx_seq(); } - if (!hit) { + if (OB_FAIL(ret)) { + } else if (!hit) { if (append) { a.last_touch_ts_ = exec_info_reap_ts_ + 1; - if (OB_FAIL(parts_.push_back(a))) { - flags_.PARTS_INCOMPLETE_ = true; - TRANS_LOG(WARN, "push fail, set incomplete", K(ret), K_(parts), K(a)); - } else { - implicit_start_tx_(); - } + write_state_ = a; + has_write_state_ = true; + implicit_start_tx_(); } else { ret = OB_ENTRY_NOT_EXIST; } } - state_change_flags_.PARTS_CHANGED_ = true; + state_change_flags_.WRITE_STATE_CHANGED_ = true; return ret; } -void ObTxDesc::post_rb_savepoint_(ObTxPartRefList &parts, const ObTxSEQ &savepoint) +void ObTxDesc::finish_write_state_rollback_(ObTxWriteState *part, const ObTxSEQ &savepoint) { - ARRAY_FOREACH_NORET(parts, i) { - parts[i].last_scn_ = savepoint; + if (OB_NOT_NULL(part)) { + part->last_scn_ = savepoint; } - state_change_flags_.PARTS_CHANGED_ = true; + state_change_flags_.WRITE_STATE_CHANGED_ = true; } -int ObTxDesc::update_clean_part(const share::ObLSID &id, - const int64_t epoch, - const ObAddr &addr) +int ObTxDesc::update_clean_write_state() { - ObTxPart p; - p.id_ = id; - p.epoch_ = epoch; - p.addr_ = addr; + ObTxWriteState p; p.first_scn_ = ObTxSEQ::MAX_VAL(); p.last_scn_ = get_tx_seq(); - return update_part_(p, false); + return merge_write_state_(p, false); } -int ObTxDesc::add_clean_part_if_absent(const share::ObLSID &id, - const int64_t epoch, - const ObAddr &addr, - const bool is_dup) +int ObTxDesc::init_clean_write_state_if_absent() { int ret = OB_SUCCESS; ObTxSEQ cur_seq = get_tx_seq(); - ObTxPart p; - p.id_ = id; - p.epoch_ = epoch; - p.addr_ = addr; + ObTxWriteState p; p.first_scn_ = cur_seq; p.last_scn_ = cur_seq; p.flag_.set_clean(); - if (is_dup) { - p.flag_.set_dup_ls(); - } - return update_part_(p, true, true); + return merge_write_state_(p, true, true); } /* - * update_part - update txn participant info + * merge_write_state - update txn write state info * * if failed, txn was marked with PARTS_INCOMPLETE */ -int ObTxDesc::update_part(ObTxPart &p) +int ObTxDesc::merge_write_state(ObTxWriteState &p) { ObSpinLockGuard guard(lock_); - return update_part_(p, true); + return merge_write_state_(p, true); } -/* - * update_parts - update txn participants info - * - * if failed, txn was marked with PARTS_INCOMPLETE - */ -int ObTxDesc::update_parts(const share::ObLSArray &list) -{ - int ret = OB_SUCCESS, tmp_ret = ret; - ARRAY_FOREACH_NORET(list, i) { - const ObLSID &it = list[i]; - ObTxPart n; - n.id_ = it; - n.epoch_ = ObTxPart::EPOCH_UNKNOWN; - n.first_scn_ = ObTxSEQ::MAX_VAL(); - n.last_scn_ = ObTxSEQ::MAX_VAL(); - if (OB_TMP_FAIL(update_part_(n))) { - ret = tmp_ret; - } +int ObTxDesc::mark_write() +{ + int ret = OB_SUCCESS; + ObTxWriteState part; + part.first_scn_ = ObTxSEQ::MAX_VAL(); + part.last_scn_ = ObTxSEQ::MAX_VAL(); + if (OB_FAIL(merge_write_state_(part))) { + TRANS_LOG(WARN, "update single transaction write state failed", K(ret)); } return ret; } void ObTxDesc::implicit_start_tx_() { - if (parts_.count() > 0 && state_ == ObTxDesc::State::IDLE) { + if (has_write_state_ && state_ == ObTxDesc::State::IDLE) { state_ = ObTxDesc::State::IMPLICIT_ACTIVE; active_ts_ = ObClockGenerator::getClock(); expire_ts_ = get_expire_ts(); @@ -859,36 +701,12 @@ int64_t ObTxDesc::get_expire_ts() const return ret; } -bool ObTxDesc::is_dup_ls_modified() const -{ - int ret = OB_SUCCESS; - bool dup_ls_modified = false; - const bool self_locked = lock_.self_locked(); - if (!self_locked && OB_FAIL(lock_.lock())) { - TRANS_LOG(ERROR, "lock tx_desc failed", K(ret), K_(tx_id)); - } else { - ARRAY_FOREACH_NORET(parts_, i) - { - if(parts_[i].flag_.is_dup_ls()) - { - dup_ls_modified = true; - break; - } - } - if (!self_locked) { - lock_.unlock(); - } - } - return dup_ls_modified; -} - -int ObTxDesc::update_parts_(const ObTxPartList &list) +int ObTxDesc::merge_write_state_if_present_(const ObTxWriteState &part, const bool has_write_state) { int ret = OB_SUCCESS; - ARRAY_FOREACH(list, i) { - bool hit = false; - const ObTxPart &a = list[i]; - ret = update_part_(const_cast(a)); + if (has_write_state) { + ObTxWriteState copied_part = part; + ret = merge_write_state_(copied_part); } return ret; } @@ -897,11 +715,11 @@ int ObTxDesc::merge_exec_info_with(const ObTxDesc &src) { int ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); - if (OB_FAIL(update_parts_(src.parts_))) { - TRANS_LOG(WARN, "update parts failed", K(ret), KPC(this), K(src)); + if (OB_FAIL(merge_write_state_if_present_(src.write_state_, src.has_write_state_))) { + TRANS_LOG(WARN, "update write state failed", K(ret), KPC(this), K(src)); } - if (src.flags_.PARTS_INCOMPLETE_) { - flags_.PARTS_INCOMPLETE_ = true; + if (src.flags_.WRITE_STATE_INCOMPLETE_) { + flags_.WRITE_STATE_INCOMPLETE_ = true; TRANS_LOG(WARN, "src is incomplete, set dest incomplete also", K(ret), K(src)); } return ret; @@ -912,16 +730,14 @@ int ObTxDesc::get_inc_exec_info(ObTxExecResult &exec_info) int ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); if (exec_info_reap_ts_ >= 0) { - ARRAY_FOREACH(parts_, i) { - ObTxPart &p = parts_[i]; - if (p.last_touch_ts_ > exec_info_reap_ts_ && - OB_FAIL(exec_info.parts_.push_back(p))) { - TRANS_LOG(WARN, "push fail", K(ret), K(p), KPC(this), K(exec_info)); - } + if (has_write_state_ && + write_state_.last_touch_ts_ > exec_info_reap_ts_ && + OB_FAIL(exec_info.set_write_state(write_state_))) { + TRANS_LOG(WARN, "set exec write state failed", K(ret), K_(write_state), KPC(this), K(exec_info)); } - if (OB_FAIL(ret) || flags_.PARTS_INCOMPLETE_) { + if (OB_FAIL(ret) || flags_.WRITE_STATE_INCOMPLETE_) { exec_info.incomplete_ = true; - TRANS_LOG(WARN, "set incomplete", K(ret), K(flags_.PARTS_INCOMPLETE_)); + TRANS_LOG(WARN, "set incomplete", K(ret), K(flags_.WRITE_STATE_INCOMPLETE_)); } exec_info_reap_ts_ += 1; } @@ -936,11 +752,11 @@ int ObTxDesc::add_exec_info(const ObTxExecResult &exec_info) { int ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); - if (OB_FAIL(update_parts_(exec_info.parts_))) { - TRANS_LOG(WARN, "update parts failed", K(ret), KPC(this), K(exec_info)); + if (OB_FAIL(merge_write_state_if_present_(exec_info.write_state_, exec_info.has_write_state_))) { + TRANS_LOG(WARN, "update write state failed", K(ret), KPC(this), K(exec_info)); } if (exec_info.incomplete_) { - flags_.PARTS_INCOMPLETE_ = true; + flags_.WRITE_STATE_INCOMPLETE_ = true; TRANS_LOG(WARN, "exec_info is incomplete set incomplete also", K(ret), K(exec_info)); } (void) merge_conflict_txs_(exec_info.conflict_txs_); @@ -1016,7 +832,7 @@ bool ObTxDesc::execute_commit_cb() * pair with ObTransService::handle_tx_commit_result_ */ ATOMIC_LOAD_ACQ((int*)&state_); - if (is_tx_end() || is_xa_terminate_state_()) { + if (is_tx_end()) { ObTransID tx_id = tx_id_; ObITxCallback *cb = commit_cb_; int ret = OB_SUCCESS; @@ -1072,13 +888,6 @@ ObITxCallback *ObTxDesc::cancel_commit_cb() return commit_cb; } -bool ObTxDesc::is_xa_terminate_state_() const -{ - return state_ == State::SUB_PREPARED || - state_ == State::SUB_COMMITTED || - state_ == State::SUB_ROLLBACKED; -} - bool ObTxDesc::has_implicit_savepoint() const { return min_implicit_savepoint_.is_valid(); @@ -1106,7 +915,7 @@ void ObTxDesc::release_implicit_savepoint(const ObTxSEQ savepoint) } } -int ObTxDesc::fetch_conflict_txs(ObIArray &array) +int ObTxDesc::fetch_conflict_txs(ObIArray &array) { int ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); @@ -1117,12 +926,12 @@ int ObTxDesc::fetch_conflict_txs(ObIArray &array) return ret; } -int ObTxDesc::add_conflict_tx(const ObTransIDAndAddr conflict_tx) { +int ObTxDesc::add_conflict_tx(const ObTransID conflict_tx) { ObSpinLockGuard guard(lock_); return add_conflict_tx_(conflict_tx); } -int ObTxDesc::add_conflict_tx_(const ObTransIDAndAddr &conflict_tx) { +int ObTxDesc::add_conflict_tx_(const ObTransID &conflict_tx) { int ret = OB_SUCCESS; if (conflict_txs_.count() >= MAX_RESERVED_CONFLICT_TX_NUM) { ret = OB_SIZE_OVERFLOW; @@ -1136,13 +945,13 @@ int ObTxDesc::add_conflict_tx_(const ObTransIDAndAddr &conflict_tx) { return ret; } -int ObTxDesc::merge_conflict_txs(const ObIArray &conflict_txs) +int ObTxDesc::merge_conflict_txs(const ObIArray &conflict_txs) { ObSpinLockGuard guard(lock_); return merge_conflict_txs_(conflict_txs); } -int ObTxDesc::merge_conflict_txs_(const ObIArray &conflict_txs) +int ObTxDesc::merge_conflict_txs_(const ObIArray &conflict_txs) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -1158,19 +967,61 @@ int ObTxDesc::merge_conflict_txs_(const ObIArray &conflict_txs return ret; } -// get global trans type for session -// 1. if xid is empty or xa_ctx is null, PLAIN is returned. -ObGlobalTxType ObTxDesc::get_global_tx_type(const ObXATransID &xid) const +int ObTxDesc::get_write_state_copy(ObTxWriteState &write_state, bool &has_write_state) const +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + write_state = ObTxWriteState(); + has_write_state = false; + if (has_write_state_) { + write_state = write_state_; + has_write_state = true; + } + return ret; +} + +void ObTxDesc::reset_write_state() { - ObGlobalTxType tx_type = ObGlobalTxType::PLAIN; - return tx_type; + ObSpinLockGuard guard(lock_); + has_write_state_ = false; + write_state_ = ObTxWriteState(); + state_change_flags_.WRITE_STATE_CHANGED_ = true; } -int ObTxDesc::get_parts_copy(ObTxPartList ©_parts) + +int ObTxDesc::assign_write_state(const ObTxWriteState &participant) { int ret = OB_SUCCESS; ObSpinLockGuard guard(lock_); - if (OB_FAIL(copy_parts.assign(parts_))) { - TRANS_LOG(WARN, "TxDesc get participants copy error", K(ret), KPC(this)); + write_state_ = participant; + has_write_state_ = true; + state_change_flags_.WRITE_STATE_CHANGED_ = true; + return ret; +} + +int ObTxDesc::fill_read_snapshot_write_state(ObTxReadSnapshot &snapshot) const +{ + if (has_write_state_ && !write_state_.is_clean()) { + snapshot.mark_write_state(); + } + return OB_SUCCESS; +} + +int ObTxDesc::find_write_state_after(ObTxWriteState *&part, const ObTxSEQ scn) +{ + int ret = OB_SUCCESS; + part = NULL; + if (has_write_state_ && write_state_.last_scn_ > scn && !write_state_.is_clean()) { + part = &write_state_; + } + return ret; +} + +int ObTxDesc::get_abort_write_state(const ObTxWriteState *&part) const +{ + int ret = OB_SUCCESS; + part = NULL; + if (has_write_state_) { + part = &write_state_; } return ret; } @@ -1242,11 +1093,8 @@ ObTxReadSnapshot::ObTxReadSnapshot() committed_(false), core_(), source_(SRC::INVL), - snapshot_lsid_(), - snapshot_ls_role_(common::ObRole::INVALID_ROLE), - snapshot_acquire_addr_(), uncertain_bound_(0), - parts_() + has_write_state_(false) {} ObTxReadSnapshot::~ObTxReadSnapshot() @@ -1254,8 +1102,6 @@ ObTxReadSnapshot::~ObTxReadSnapshot() valid_ = false; committed_ = false; source_ = SRC::INVL; - snapshot_ls_role_ = common::INVALID_ROLE; - snapshot_acquire_addr_.reset(); uncertain_bound_ = 0; } @@ -1265,11 +1111,8 @@ void ObTxReadSnapshot::reset() committed_ = false; core_.reset(); source_ = SRC::INVL; - snapshot_lsid_.reset(); - snapshot_ls_role_ = common::INVALID_ROLE; - snapshot_acquire_addr_.reset(); uncertain_bound_ = 0; - parts_.reset(); + has_write_state_ = false; } int ObTxReadSnapshot::assign(const ObTxReadSnapshot &from) @@ -1279,16 +1122,16 @@ int ObTxReadSnapshot::assign(const ObTxReadSnapshot &from) committed_ = from.committed_; core_ = from.core_; source_ = from.source_; - snapshot_lsid_ = from.snapshot_lsid_; - snapshot_ls_role_ = from.snapshot_ls_role_; - snapshot_acquire_addr_ = from.snapshot_acquire_addr_; uncertain_bound_ = from.uncertain_bound_; - if (OB_FAIL(parts_.assign(from.parts_))) { - TRANS_LOG(WARN, "assign snapshot fail", K(ret), K(from)); - } + has_write_state_ = from.has_write_state_; return ret; } +void ObTxReadSnapshot::reset_write_state() +{ + has_write_state_ = false; +} + void ObTxReadSnapshot::init_weak_read(const SCN snapshot) { core_.version_ = snapshot; @@ -1296,16 +1139,15 @@ void ObTxReadSnapshot::init_weak_read(const SCN snapshot) core_.scn_.reset(); core_.elr_ = false; source_ = SRC::WEAK_READ_SERVICE; - parts_.reset(); + has_write_state_ = false; valid_ = true; } -void ObTxReadSnapshot::init_ls_read(const share::ObLSID &ls_id, const ObTxSnapshot &core) +void ObTxReadSnapshot::init_ls_read(const ObTxSnapshot &core) { core_ = core; source_ = SRC::LS; - snapshot_lsid_ = ls_id; valid_ = true; } @@ -1317,7 +1159,7 @@ void ObTxReadSnapshot::specify_snapshot_scn(const share::SCN snapshot) void ObTxReadSnapshot::try_set_read_elr() { - const bool can_read_elr = (source_ == SRC::LS && snapshot_ls_role_ == common::ObRole::LEADER); + const bool can_read_elr = source_ == SRC::LS; core_.set_elr(can_read_elr); } @@ -1328,22 +1170,12 @@ const char* ObTxReadSnapshot::get_source_name() const return SRC_NAME[(int)source_]; } -/* - * format snapshot info for sql audit display - * contains: src, ls_id, ls_role, parts - * 1. local select: - * "src:LOCAL;ls_id:1001;ls_role:LEADER;parts:[1001,1002]" - * 2. glocal select, with no ls and ls_role: "src:GTS;parts[1001]" - * 3. insert: "NONE" - * 4. longer than 128 chars, with "..." in the end - * "src:GLOBAL;ls_id:1001;ls_role:LEADER;parts:[1001,1002,102..." - */ - ObTxExecResult::ObTxExecResult() : allocator_("TxExecResult"), incomplete_(false), - touched_ls_list_(OB_MALLOC_NORMAL_BLOCK_SIZE, allocator_), - parts_(OB_MALLOC_NORMAL_BLOCK_SIZE, allocator_), + touched_storage_(false), + has_write_state_(false), + write_state_(), conflict_txs_(OB_MALLOC_NORMAL_BLOCK_SIZE, allocator_) {} @@ -1355,35 +1187,27 @@ ObTxExecResult::~ObTxExecResult() void ObTxExecResult::reset() { incomplete_ = false; - touched_ls_list_.reset(); - parts_.reset(); + touched_storage_ = false; + has_write_state_ = false; + write_state_ = ObTxWriteState(); conflict_txs_.reset(); allocator_.reset(); } -int ObTxExecResult::add_touched_ls(const share::ObLSID ls) +int ObTxExecResult::set_write_state(const ObTxWriteState &part) { int ret = OB_SUCCESS; - if (OB_FAIL(common::add_var_to_array_no_dup(touched_ls_list_, ls))) { - incomplete_ = true; - TRANS_LOG(WARN, "add touched ls failed, set incomplete", K(ret), K(ls)); - } + write_state_ = part; + has_write_state_ = true; return ret; } -int ObTxExecResult::add_touched_ls(const ObIArray &ls_list) +int ObTxExecResult::merge_write_state(const ObTxWriteState &part, const bool has_write_state) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ARRAY_FOREACH_NORET(ls_list, i) { - if (OB_TMP_FAIL(add_touched_ls(ls_list.at(i)))) { - TRANS_LOG(WARN, "add fail", K(ret)); - ret = tmp_ret; - } - } - if (OB_FAIL(ret)) { + if (has_write_state && OB_FAIL(set_write_state(part))) { incomplete_ = true; - TRANS_LOG(WARN, "add fail, set incomplete", K(ret), KPC(this)); + TRANS_LOG(WARN, "merge exec write state failed", K(ret), K(part), KPC(this)); } return ret; } @@ -1403,13 +1227,11 @@ int ObTxExecResult::merge_result(const ObTxExecResult &r) int ret = OB_SUCCESS; TRANS_LOG(TRACE, "txExecResult.merge with.start", K(r), KPC(this), K(lbt())); incomplete_ |= r.incomplete_; - if (OB_FAIL(append_dedup(parts_, r.parts_))) { + if (OB_FAIL(merge_write_state(r.write_state_, r.has_write_state_))) { incomplete_ = true; TRANS_LOG(WARN, "merge fail, set incomplete", K(ret), KPC(this)); - } else if (OB_FAIL(append_dedup(touched_ls_list_, r.touched_ls_list_))) { - incomplete_ = true; - TRANS_LOG(WARN, "merge touched_ls_list fail, set incomplete", K(ret), KPC(this)); } + touched_storage_ |= r.touched_storage_; if (OB_SUCC(ret)) { ret = merge_cflict_txs(r.conflict_txs_); } @@ -1421,7 +1243,7 @@ int ObTxExecResult::merge_result(const ObTxExecResult &r) return ret; } -int ObTxExecResult::merge_cflict_txs(const common::ObIArray &txs) +int ObTxExecResult::merge_cflict_txs(const common::ObIArray &txs) { int ret = OB_SUCCESS; if (OB_FAIL(append_dedup(conflict_txs_, txs))) { @@ -1434,32 +1256,28 @@ int ObTxExecResult::assign(const ObTxExecResult &r) { int ret = OB_SUCCESS; incomplete_ = r.incomplete_; - if (OB_FAIL(parts_.assign(r.parts_))) { + has_write_state_ = false; + write_state_ = ObTxWriteState(); + if (OB_FAIL(merge_write_state(r.write_state_, r.has_write_state_))) { incomplete_ = true; TRANS_LOG(WARN, "assign fail, set incomplete", K(ret), KPC(this)); - } else if (OB_FAIL(touched_ls_list_.assign(r.touched_ls_list_))) { - incomplete_ = true; - TRANS_LOG(WARN, "assign touched_ls_list fail, set incomplete", K(ret), KPC(this)); } + touched_storage_ = r.touched_storage_; conflict_txs_.assign(r.conflict_txs_); conflict_info_array_.assign(r.conflict_info_array_); return ret; } -ObTxPart::ObTxPart() - : id_(), - addr_(), - epoch_(-1), - first_scn_(), +ObTxWriteState::ObTxWriteState() + : first_scn_(), last_scn_(), last_touch_ts_(0), flag_() { } -ObTxPart::~ObTxPart() +ObTxWriteState::~ObTxWriteState() { - epoch_ = -1; first_scn_.reset(); last_scn_.reset(); last_touch_ts_ = 0; @@ -1716,43 +1534,6 @@ int ObTxDescMgr::iterate_tx_scheduler_stat(ObTxSchedulerStatIterator &tx_schedul return ret; } -// TAKEOVER REVOKE RESUME SWITCH_GRACEFUL -// LEADER N FOLLOWER N FOLLOWER -// FOLLOWER LEADER FOLLOWER LEADER FOLLOWER -int TxCtxStateHelper::switch_state(const int64_t op) -{ - int ret = OB_SUCCESS; - static const int64_t N = TxCtxRoleState::INVALID; - static const int64_t STATE_MAP[TxCtxRoleState::MAX][TxCtxOps::MAX] = { - {N, TxCtxRoleState::FOLLOWER, N, TxCtxRoleState::FOLLOWER}, - {TxCtxRoleState::LEADER, TxCtxRoleState::FOLLOWER, TxCtxRoleState::LEADER, TxCtxRoleState::FOLLOWER}}; - if (OB_UNLIKELY(!TxCtxOps::is_valid(op))) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(op)); - } else if (OB_UNLIKELY(!TxCtxRoleState::is_valid(state_))) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected state", K(state_)); - } else { - const int64_t new_state = STATE_MAP[state_][op]; - if (OB_UNLIKELY(!TxCtxRoleState::is_valid(new_state))) { - ret = OB_STATE_NOT_MATCH; - } else { - last_state_ = state_; - state_ = new_state; - is_switching_ = true; - } - } - return ret; -} - -void TxCtxStateHelper::restore_state() -{ - if (is_switching_) { - is_switching_ = false; - state_ = last_state_; - } -} - int ObTxDesc::alloc_branch_id(const int64_t count, int16_t &branch_id) { int ret = OB_SUCCESS; @@ -1766,33 +1547,15 @@ int ObTxDesc::alloc_branch_id(const int64_t count, int16_t &branch_id) } return ret; } -void ObTxDesc::mark_part_abort(const ObTransID tx_id, const int abort_cause) +void ObTxDesc::mark_write_state_aborted(const ObTransID tx_id, const int abort_cause) { ObSpinLockGuard guard(lock_); - if (tx_id == tx_id_ && state_ < State::IN_TERMINATE && !flags_.PART_ABORTED_) { - flags_.PART_ABORTED_ = true; + if (tx_id == tx_id_ && state_ < State::IN_TERMINATE && !flags_.WRITE_STATE_ABORTED_) { + flags_.WRITE_STATE_ABORTED_ = true; abort_cause_ = abort_cause; } } -int64_t ObTxDesc::get_coord_epoch() const -{ - int64_t epoch = -1; - - if (OB_UNLIKELY(!coord_id_.is_valid())) { - epoch = -1; - } else { - ARRAY_FOREACH_NORET(commit_parts_, i) { - const ObTxExecPart &part = commit_parts_[i]; - if (coord_id_ == part.ls_id_) { - epoch = part.exec_epoch_; - } - } - } - - return epoch; -} - // 1. clear transaction level snapshot // 2. clear savepoints int ObTxDesc::clear_state_for_autocommit_retry() @@ -1857,22 +1620,16 @@ bool ObTxDesc::has_modify_table(const uint64_t table_id) const return is_contain(modified_tables_, table_id); } -bool ObTxDesc::is_all_parts_clean() const +bool ObTxDesc::is_write_state_clean() const { - bool bret = true; - ARRAY_FOREACH_X(parts_, i, cnt, bret) { - bret = parts_[i].is_without_ctx() || parts_[i].is_clean(); - } - return bret; + return !has_write_state_ || write_state_.is_clean(); } -bool ObTxDesc::is_all_parts_without_valid_write() const +bool ObTxDesc::is_write_state_without_valid_write() const { - bool bret = true; - ARRAY_FOREACH_X(parts_, i, cnt, bret) { - bret = parts_[i].is_without_ctx() || parts_[i].is_clean() || parts_[i].is_without_valid_write(); - } - return bret; + return !has_write_state_ + || write_state_.is_clean() + || write_state_.is_without_valid_write(); } } // transaction } // oceanbase diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index 7bb300f43..94df6b434 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -20,7 +20,6 @@ #include #include #include "lib/container/ob_iarray.h" -#include "lib/container/ob_mask_set2.h" #include "lib/container/ob_se_array.h" #include "lib/list/ob_list.h" #include "lib/oblog/ob_log_module.h" @@ -29,7 +28,6 @@ #include "lib/container/ob_tuple.h" #include "common/ob_role.h" #include "share/ob_cluster_version.h" -#include "share/ob_ls_id.h" #include "share/ob_light_hashmap.h" #include "storage/tx/ob_trans_define.h" #include "common/ob_simple_iterator.h" @@ -43,65 +41,12 @@ namespace transaction class ObTxSchedulerStat; -struct ObTransIDAndAddr { // deadlock needed - OB_UNIS_VERSION(1); -public: - ObTransIDAndAddr() = default; - ObTransIDAndAddr(const ObTransID id, const common::ObAddr &addr) : tx_id_(id), scheduler_addr_(addr) {} - TO_STRING_KV(K_(tx_id), K_(scheduler_addr)); - bool operator==(const ObTransIDAndAddr &rhs) const { return tx_id_ == rhs.tx_id_ && - scheduler_addr_ == rhs.scheduler_addr_; } - bool is_valid() const { return tx_id_.is_valid() && scheduler_addr_.is_valid(); } - ObTransID tx_id_; - common::ObAddr scheduler_addr_; -}; -OB_SERIALIZE_MEMBER_TEMP(inline, ObTransIDAndAddr, tx_id_, scheduler_addr_); - class ObITxCallback { public: virtual void callback(int ret) = 0; }; -template -struct ObPair -{ - L left_; R right_; -public: - ObPair() : left_(), right_() {} - ObPair(const L &l, const R &r): left_(l), right_(r) {} - ObPair& operator=(const ObPair &b) - { left_ = b.left_; right_ = b.right_; return *this; } - inline bool operator==(const ObPair &b) const { - return b.left_ == left_ && b.right_ == right_; - } - TO_STRING_KV(K_(left), K_(right)); - NEED_SERIALIZE_AND_DESERIALIZE; -}; - -template -int ObPair::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, left_, right_); - return ret; -} -template -int ObPair::deserialize(const char *buf, int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, left_, right_); - return ret; -} - -template -int64_t ObPair::get_serialize_size() const -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, left_, right_); - return len; -} - template class ObRefList { @@ -126,19 +71,17 @@ class ObRefList #define OB_TX_ABORT_CAUSE_LIST \ - _XX(PARTICIPANT_IS_CLEAN) \ + _XX(WRITE_STATE_IS_CLEAN) \ _XX(TX_RESULT_INCOMPLETE) \ _XX(IN_CONSIST_STATE) \ _XX(SAVEPOINT_ROLLBACK_FAIL) \ _XX(IMPLICIT_ROLLBACK) \ _XX(SESSION_DISCONNECT) /*5*/ \ _XX(STOP) \ - _XX(PARTICIPANT_STATE_INCOMPLETE) \ - _XX(PARTICIPANTS_SET_INCOMPLETE) \ - _XX(PARTICIPANT_KILLED_FORCEDLY) \ - _XX(PARTICIPANT_KILLED_GRACEFULLY) /*10*/ \ - _XX(PARTICIPANT_SWITCH_FOLLOWER_FORCEDLY) \ - _XX(PARTICIPANT_SWITCH_LEADER_DATA_INCOMPLETE) \ + _XX(WRITE_STATE_STATE_INCOMPLETE) \ + _XX(WRITE_STATE_INCOMPLETE) \ + _XX(WRITE_STATE_KILLED_FORCEDLY) \ + _XX(WRITE_STATE_KILLED_GRACEFULLY) /*10*/ \ _XX(END_STMT_FAIL) \ _XX(EXPLICIT_ROLLBACK) \ _XX(CREATE_SAVEPOINT_FAIL) \ @@ -199,7 +142,7 @@ inline bool is_RC_isolevel(const ObTxIsolationLevel isolation) enum class ObTxAccessMode { - INVL = -1, RW = 0, RD_ONLY = 1, STANDBY_RD_ONLY = 2 + INVL = -1, RW = 0, RD_ONLY = 1 }; struct ObTxParam @@ -220,58 +163,42 @@ struct ObTxParam OB_UNIS_VERSION(1); }; -union ObTxPartFlag +union ObTxWriteStateFlag { int64_t flag_val_; struct FlagBit { - unsigned int is_dup_ls_ : 1; + unsigned int reserved_ : 1; bool is_clean_ : 1; // no Write happended, even rollbacked - TO_STRING_KV(K_(is_dup_ls), K_(is_clean)); + TO_STRING_KV(K_(is_clean)); } flag_bit_; - ObTxPartFlag() { reset(); } + ObTxWriteStateFlag() { reset(); } void reset() { flag_val_ = 0; } - bool is_dup_ls() const { return flag_bit_.is_dup_ls_ == 1; } - void set_dup_ls() { flag_bit_.is_dup_ls_ = 1; } bool is_clean() const { return flag_bit_.is_clean_; } void set_clean() { flag_bit_.is_clean_ = true; } void set_dirty() { flag_bit_.is_clean_ = false; } }; -struct ObTxPart +struct ObTxWriteState { - static const int64_t EPOCH_UNKNOWN = -1; - static const int64_t EPOCH_DEAD = -2; - ObTxPart(); - ~ObTxPart(); - share::ObLSID id_; // identifier, the logstream - ObAddr addr_; // its latest address - int64_t epoch_; // used to judge a ctx not revived + ObTxWriteState(); + ~ObTxWriteState(); ObTxSEQ first_scn_; // used to judge a ctx is clean in scheduler view ObTxSEQ last_scn_; // used to get rollback savepoint set int64_t last_touch_ts_; // used to judge a ctx retouched after a time point - ObTxPartFlag flag_; // used to describe some special attributes of a participant - bool operator==(const ObTxPart &rhs) const { return id_ == rhs.id_ && addr_ == rhs.addr_; } - bool operator!=(const ObTxPart &rhs) const { return !operator==(rhs); } + ObTxWriteStateFlag flag_; // used to describe some special attributes of the single write side bool is_clean() const { return flag_.is_clean(); } bool is_without_valid_write() const { return !first_scn_.is_valid() || last_scn_ < first_scn_; } - bool is_without_ctx() const { return is_without_ctx(epoch_); } - bool is_dup_ls() const { return flag_.is_dup_ls(); } - static bool is_without_ctx(int64_t epoch) { return EPOCH_DEAD == epoch; } - TO_STRING_KV(K_(id), K_(addr), K_(epoch), K_(first_scn), K_(last_scn), K_(last_touch_ts), K(flag_.flag_bit_)); + TO_STRING_KV(K_(first_scn), K_(last_scn), K_(last_touch_ts), K(flag_.flag_bit_)); OB_UNIS_VERSION(1); }; -typedef ObSEArray ObTxPartList; -typedef ObRefList ObTxPartRefList; -typedef ObPair ObTxLSEpochPair; - // internal core snapshot for read data class ObTxSnapshot { @@ -315,17 +242,16 @@ class ObTxReadSnapshot SPECIAL = 4, // user specify NONE = 5, // won't read, global snapshot not required } source_; - share::ObLSID snapshot_lsid_; // for source_ = LOCAL // - common::ObRole snapshot_ls_role_; // for source_ = LS, only can be used for dup_table with a - // max_commit_ts from the follower - ObAddr snapshot_acquire_addr_; // snapshot version acquired from which server int64_t uncertain_bound_; // for source_ GLOBAL - ObSEArray parts_; + bool has_write_state_; public: void init_weak_read(const share::SCN snapshot); void init_none_read() { valid_ = true; source_ = SRC::NONE; } - void init_ls_read(const share::ObLSID &ls_id, const ObTxSnapshot &core); + void init_ls_read(const ObTxSnapshot &core); void specify_snapshot_scn(const share::SCN snapshot); + void reset_write_state(); + void mark_write_state() { has_write_state_ = true; } + bool has_write_state() const { return has_write_state_; } const char* get_source_name() const; const ObTxSnapshot &snapshot() const { return core_; } const share::SCN &version() const { return core_.version(); } @@ -339,7 +265,6 @@ class ObTxReadSnapshot bool is_valid() const { return valid_; } void invalid() { valid_ = false; } bool is_committed() const { return committed_; } - const ObAddr get_snapshot_acquire_addr() const { return snapshot_acquire_addr_; } void reset(); int assign(const ObTxReadSnapshot &); void try_set_read_elr(); @@ -349,9 +274,9 @@ class ObTxReadSnapshot * * special serialize interface for lob to avoid lob locator too large */ - int serialize_for_lob(const share::ObLSID &ls_id, const share::SCN &fb_snapshot, SERIAL_PARAMS) const; + int serialize_for_lob(const share::SCN &fb_snapshot, SERIAL_PARAMS) const; int deserialize_for_lob(share::SCN &fb_snapshot, DESERIAL_PARAMS); - int64_t get_serialize_size_for_lob(const share::ObLSID &ls_id, const share::SCN &fb_snapshot) const; + int64_t get_serialize_size_for_lob(const share::SCN &fb_snapshot) const; /** * deprecated interface, DONOT use ! * @@ -361,7 +286,7 @@ class ObTxReadSnapshot * so provide one interface to build ObTxReadSnapshot from ObTxSnapshot * master no need use this */ - int build_snapshot_for_lob(const ObTxSnapshot &core, const share::ObLSID &ls_id); + int build_snapshot_for_lob(const ObTxSnapshot &core); /** * deprecated interface, DONOT use ! * @@ -373,8 +298,7 @@ class ObTxReadSnapshot int build_snapshot_for_lob( const int64_t snapshot_version, const int64_t snapshot_tx_id, - const int64_t snapshot_seq, - const share::ObLSID &ls_id); + const int64_t snapshot_seq); /** * only used for lob, other situation DONOT use * @@ -401,10 +325,7 @@ class ObTxReadSnapshot K_(source), K_(core), K_(uncertain_bound), - K_(snapshot_lsid), - K_(snapshot_ls_role), - K_(snapshot_acquire_addr), - K_(parts), + K_(has_write_state), K_(committed)); OB_UNIS_VERSION(1); DISABLE_COPY_ASSIGN(ObTxReadSnapshot); @@ -461,92 +382,43 @@ class ObTxExecResult friend class ObTransService; friend class ObTxDesc; OB_UNIS_VERSION(1); - TransModulePageAllocator allocator_; - bool incomplete_; - share::ObLSArray touched_ls_list_; - ObTxPartList parts_; - ObSArray conflict_txs_; // FARM COMPAT WHITELIST for cflict_txs_ - ObSArray conflict_info_array_; + TransModulePageAllocator allocator_; + bool incomplete_; + bool touched_storage_; + bool has_write_state_; + ObTxWriteState write_state_; + ObSArray conflict_txs_; + ObSArray conflict_info_array_; public: ObTxExecResult(); ~ObTxExecResult(); void reset(); - TO_STRING_KV(K_(incomplete), K_(parts), K_(touched_ls_list), K_(conflict_txs)); + TO_STRING_KV(K_(incomplete), K_(has_write_state), K_(write_state), K_(touched_storage), K_(conflict_txs)); void set_incomplete() { TRANS_LOG(TRACE, "tx result incomplete:", KP(this)); incomplete_ = true; } - int merge_cflict_txs(const common::ObIArray &txs); - bool is_incomplete() const { return incomplete_; } - int add_touched_ls(const share::ObLSID ls); - int add_touched_ls(const ObIArray &ls_list); - const share::ObLSArray &get_touched_ls() const { return touched_ls_list_; } - int merge_result(const ObTxExecResult &r); - int assign(const ObTxExecResult &r); - const ObSArray &get_conflict_txs() const { return conflict_txs_; } + int merge_cflict_txs(const common::ObIArray &txs); + bool is_incomplete() const { return incomplete_; } + bool touches_storage() const { return touched_storage_; } + void mark_touched_storage() { touched_storage_ = true; } + int set_write_state(const ObTxWriteState &part); + int merge_write_state(const ObTxWriteState &part, const bool has_write_state); + bool has_write_state() const { return has_write_state_; } + int merge_result(const ObTxExecResult &r); + int assign(const ObTxExecResult &r); + const ObSArray &get_conflict_txs() const { return conflict_txs_; } DISABLE_COPY_ASSIGN(ObTxExecResult); }; -class RollbackMaskSet -{ -public: - RollbackMaskSet() : rollback_parts_(NULL) {} - int init(share::ObCommonID tx_msg_id, ObTxRollbackParts &parts) { - ObSpinLockGuard guard(lock_); - tx_msg_id_ = tx_msg_id; - rollback_parts_ = &parts; - return mask_set_.init(&parts); - } - int get_not_mask(ObTxRollbackParts &remain) { - ObSpinLockGuard guard(lock_); - return mask_set_.get_not_mask(remain); - } - bool is_mask(const ObTxExecPart &part) { - ObSpinLockGuard guard(lock_); - return mask_set_.is_mask(part); - } - int mask(const ObTxExecPart &part) { - ObSpinLockGuard guard(lock_); - return mask_set_.mask(part); - } - int unmask(const ObTxExecPart &part) { - ObSpinLockGuard guard(lock_); - return mask_set_.unmask(part); - } - bool is_all_mask() { - ObSpinLockGuard guard(lock_); - return mask_set_.is_all_mask(); - } - share::ObCommonID get_tx_msg_id() const { - return tx_msg_id_; - } - void reset() { - ObSpinLockGuard guard(lock_); - tx_msg_id_.reset(); - rollback_parts_ = NULL; - mask_set_.reset(); - } - int merge_part(const share::ObLSID add_ls_id, - const int64_t exec_epoch); - int find_part(const share::ObLSID ls_id, - const int64_t orig_epoch, - ObTxExecPart &part); -private: - ObSpinLock lock_; - share::ObCommonID tx_msg_id_; - ObTxRollbackParts *rollback_parts_; - common::ObMaskSet2 mask_set_; -}; - class ObTxDesc final : public share::ObLightHashLink { static constexpr const char *OP_LABEL = "TX_DESC_VALUE"; static constexpr int64_t MAX_RESERVED_CONFLICT_TX_NUM = 30; friend class ObTransService; friend class ObTxDescMgr; - friend class ObPartTransCtx; + friend class ObTxCtx; friend class StopTxDescFunctor; - friend class ObTxStmtInfo; friend class IterateTxSchedulerFunctor; friend class ObTxnFreeRouteCtx; OB_UNIS_VERSION(1); @@ -574,7 +446,6 @@ class ObTxDesc final : public share::ObLightHashLink uint32_t sess_id_; // sesssion id of txn start, for XA it is XA_START session id uint32_t assoc_sess_id_; // the session which associated with uint32_t client_sid_; // client session id, which is produced by proxy - ObGlobalTxType global_tx_type_; // global trans type uint64_t op_sn_; // Tx level operation sequence No @@ -587,53 +458,29 @@ class ObTxDesc final : public share::ObLightHashLink ROLLBACK_SAVEPOINT, // rolling back to savepoint IN_TERMINATE, // committing, aborting ABORTED, // internal rolled back - ROLLED_BACK, // rolled back - COMMIT_TIMEOUT, // commit timeouted - COMMIT_UNKNOWN, // commit complted but result unknown, either committed or aborted - COMMITTED, // committed - SUB_PREPARING, // XA prepare started - SUB_PREPARED, // XA prepare response received - SUB_COMMITTING, // XA commit started - SUB_COMMITTED, // XA commit response received - SUB_ROLLBACKING, // XA rollback started - SUB_ROLLBACKED, // XA rollback response received - } state_; + ROLLED_BACK, // rolled back + COMMIT_TIMEOUT, // commit timeouted + COMMIT_UNKNOWN, // commit complted but result unknown, either committed or aborted + COMMITTED, // committed + } state_; union FLAG // flags { uint64_t v_; - struct FOR_FIXED_SER_VAL { - uint64_t v_; - TO_STRING_KV(K_(v)); - NEED_SERIALIZE_AND_DESERIALIZE; - } for_serialize_v_; - struct COMPAT_FOR_TX_ROUTE { - uint64_t v_; - uint64_t get_serialize_v_() const; - TO_STRING_KV(K_(v)); - NEED_SERIALIZE_AND_DESERIALIZE; - } compat_for_tx_route_; - struct COMPAT_FOR_EXEC { - uint64_t v_; - uint64_t get_serialize_v_() const; - TO_STRING_KV(K_(v)); - NEED_SERIALIZE_AND_DESERIALIZE; - } compat_for_exec_; struct { bool EXPLICIT_:1; // txn is explicted start bool SHADOW_:1; // this tx desc is a shadow copy, is not registered with tx_desc_mgr - bool REPLICA_:1; // a replica of primary/original, its state is transient, without whole lifecyle bool TRACING_:1; // tracing the Tx bool INTERRUPTED_: 1; // a single for blocking operation bool RELEASED_: 1; // after released, commit can give up bool BLOCK_: 1; // tx is blocking within some loop - bool PARTS_INCOMPLETE_: 1; // participants set incomplete (trans must abort) - bool PART_EPOCH_MISMATCH_: 1; // participant's born epoch mismatched + bool WRITE_STATE_INCOMPLETE_: 1; // write state state incomplete (trans must abort) bool WITH_TEMP_TABLE_: 1; // with txn level temporary table bool DEFER_ABORT_: 1; // need do abort in txn start node - bool PART_ABORTED_: 1; // some participant is aborted or in delay-abort state (trans must abort) + bool WRITE_STATE_ABORTED_: 1; // write state is aborted or in delay-abort state (trans must abort) }; + NEED_SERIALIZE_AND_DESERIALIZE; void switch_to_idle_(); } flags_; static_assert(sizeof(FLAG) == sizeof(int64_t), "ObTxDesc::FLAG should sizeof(int64_t)"); @@ -643,7 +490,7 @@ class ObTxDesc final : public share::ObLightHashLink struct { bool STATIC_CHANGED_:1; bool DYNAMIC_CHANGED_:1; - bool PARTS_CHANGED_:1; + bool WRITE_STATE_CHANGED_:1; bool EXTRA_CHANGED_:1; }; void reset() { v_ = 0;} @@ -661,20 +508,16 @@ class ObTxDesc final : public share::ObLightHashLink ObTxSEQ active_scn_; // logical time of ACTIVE | IMPLICIT_ACTIVE ObTxSEQ min_implicit_savepoint_; // mininum of implicit savepoints int16_t last_branch_id_; // branch_id allocator, reset when stmt start - ObTxPartList parts_; // participant list + bool has_write_state_; + ObTxWriteState write_state_; ObTxSavePointList savepoints_; // savepoints established // conflict_txs_ is used to store conflict trans id when try acquire row lock failed(meet lock conflict) // this information will used to detect deadlock - // conflict_txs_ is valid when transaction is not executed on local - // on scheduler, conflict_txs_ merges all participants executed results on remote - // on participant, conflict_txs_ temporary stores conflict information, and will be read by upper layers, bring back to scheduler - ObSArray conflict_txs_; // FARM COMPAT WHITELIST for cflict_txs_ + ObSArray conflict_txs_; ObSArray conflict_info_array_; - // used during commit - share::ObLSID coord_id_; // coordinator ID - int64_t commit_expire_ts_; // commit operation deadline - ObTxCommitParts commit_parts_; // participants to do commit + // used during commit + int64_t commit_expire_ts_; // commit operation deadline share::SCN commit_version_; // Tx commit version int commit_out_; // the commit result int commit_times_; // times of sent commit request @@ -691,9 +534,6 @@ class ObTxDesc final : public share::ObLightHashLink ObITxCallback *commit_cb_; // async commit callback int64_t cb_tid_; // commit callback thread id int64_t exec_info_reap_ts_; // the time reaping incremental tx exec info - RollbackMaskSet brpc_mask_set_; // used in message driven savepoint rollback - ObTransCond rpc_cond_; // used in message driven savepoint rollback - ObTxTimeoutTask commit_task_; // commit retry task ObTransTraceLog tlog_; #ifdef ENABLE_DEBUG_LOG @@ -721,25 +561,19 @@ class ObTxDesc final : public share::ObLightHashLink void reset(); void set_tx_id(const ObTransID &tx_id); void reset_tx_id(); - // udpate clean part's unknown field - int update_clean_part(const share::ObLSID &id, - const int64_t epoch, - const ObAddr &addr); - int add_clean_part_if_absent(const share::ObLSID &id, - const int64_t epoch, - const ObAddr &addr, - const bool is_dup); - int update_part(ObTxPart &p); - int update_parts(const share::ObLSArray &parts); + int update_clean_write_state(); + int init_clean_write_state_if_absent(); + int merge_write_state(ObTxWriteState &p); + int mark_write(); int switch_to_idle(); int set_commit_cb(ObITxCallback *cb); bool execute_commit_cb(); private: - int update_part_(ObTxPart &p, const bool append = true, const bool check_only_if_exist = false); - int add_conflict_tx_(const ObTransIDAndAddr &conflict_tx); - int merge_conflict_txs_(const ObIArray &conflict_ids); - int update_parts_(const ObTxPartList &list); - void post_rb_savepoint_(ObTxPartRefList &parts, const ObTxSEQ &savepoint); + int merge_write_state_(ObTxWriteState &p, const bool append = true, const bool check_only_if_exist = false); + int add_conflict_tx_(const ObTransID &conflict_tx); + int merge_conflict_txs_(const ObIArray &conflict_ids); + int merge_write_state_if_present_(const ObTxWriteState &part, const bool has_write_state); + void finish_write_state_rollback_(ObTxWriteState *part, const ObTxSEQ &savepoint); void implicit_start_tx_(); bool acq_commit_cb_lock_if_need_(); bool has_extra_state_() const; @@ -771,8 +605,8 @@ class ObTxDesc final : public share::ObLightHashLink K_(timeout_us), K_(lock_timeout_us), K_(expire_ts), - K_(coord_id), - K_(parts), + K_(has_write_state), + K_(write_state), K_(exec_info_reap_ts), K_(commit_version), K_(commit_times), @@ -783,7 +617,6 @@ class ObTxDesc final : public share::ObLightHashLink K_(flags_.SHADOW), K_(flags_.INTERRUPTED), K_(flags_.BLOCK), - K_(flags_.REPLICA), K_(conflict_txs), K_(abort_cause), K_(commit_expire_ts), @@ -791,17 +624,16 @@ class ObTxDesc final : public share::ObLightHashLink K_(modified_tables), K_(last_rc_snapshot_version), K_(ref)); - bool support_branch() const { return seq_base_ > 0; } // used by SQL alloc branch_id refer the min branch_id allowed // because branch_id bellow this is reserved for internal use static int branch_id_offset() { return MAX_CALLBACK_LIST_COUNT; } static bool is_alloced_branch_id(int branch_id) { return branch_id >= branch_id_offset(); } int alloc_branch_id(const int64_t count, int16_t &branch_id); - int fetch_conflict_txs(ObIArray &array); + int fetch_conflict_txs(ObIArray &array); void reset_conflict_txs() { ObSpinLockGuard guard(lock_); conflict_txs_.reset(); } - int add_conflict_tx(const ObTransIDAndAddr conflict_tx); - int merge_conflict_txs(const ObIArray &conflict_ids); + int add_conflict_tx(const ObTransID conflict_tx); + int merge_conflict_txs(const ObIArray &conflict_ids); bool has_conflict_txs() const { return conflict_txs_.count() > 0; } bool contain(const ObTransID &trans_id) const { return tx_id_ == trans_id; } /*used by TransHashMap*/ @@ -826,8 +658,8 @@ class ObTxDesc final : public share::ObLightHashLink const ObTransID &tid() const { return tx_id_; } bool is_valid() const { return !is_in_tx() || tx_id_.is_valid(); } ObTxAccessMode get_tx_access_mode() const { return access_mode_; } - bool is_rdonly() const { return access_mode_ == ObTxAccessMode::RD_ONLY || access_mode_ == ObTxAccessMode::STANDBY_RD_ONLY; } - bool is_clean() const { return parts_.empty(); } + bool is_rdonly() const { return access_mode_ == ObTxAccessMode::RD_ONLY; } + bool is_clean() const { return !has_write_state_; } bool is_shadow() const { return flags_.SHADOW_; } bool is_explicit() const { return flags_.EXPLICIT_; } void set_with_temporary_table() { flags_.WITH_TEMP_TABLE_ = true; } @@ -839,35 +671,26 @@ class ObTxDesc final : public share::ObLightHashLink bool is_tx_end() { return is_committed() || is_rollbacked(); } - bool is_committing() { - return state_ == State::IN_TERMINATE - || state_ == State::SUB_PREPARING - || state_ == State::SUB_COMMITTING - || state_ == State::SUB_ROLLBACKING; - } + bool is_committing() { + return state_ == State::IN_TERMINATE; + } bool is_terminated() { return state_ == State::ABORTED || is_tx_end(); } - bool is_committed() { - return state_ == State::COMMITTED - || state_ == State::COMMIT_TIMEOUT - || state_ == State::COMMIT_UNKNOWN - || state_ == State::SUB_COMMITTED; - } - bool is_rollbacked() { - return state_ == State::ROLLED_BACK - || state_ == State::SUB_ROLLBACKED; - } + bool is_committed() { + return state_ == State::COMMITTED + || state_ == State::COMMIT_TIMEOUT + || state_ == State::COMMIT_UNKNOWN; + } + bool is_rollbacked() { + return state_ == State::ROLLED_BACK; + } bool is_commit_unsucc() { return state_ == State::COMMIT_TIMEOUT || state_ == State::COMMIT_UNKNOWN || state_ == State::ROLLED_BACK; } - bool is_sub2pc() { - return state_ >= State::SUB_PREPARING - && state_ <= State::SUB_ROLLBACKED; - } - bool is_aborted() const { return state_ == State::ABORTED; } + bool is_aborted() const { return state_ == State::ABORTED; } bool is_tx_timeout() { return expire_ts_ > 0 && ObClockGenerator::getClock() > expire_ts_; } bool is_tx_commit_timeout() { return commit_expire_ts_ > 0 && ObClockGenerator::getClock() > commit_expire_ts_;} void set_xid(const ObXATransID &xid) { xid_ = xid; } @@ -881,7 +704,6 @@ class ObTxDesc final : public share::ObLightHashLink int64_t get_expire_ts() const; int64_t get_tx_lock_timeout() const { return lock_timeout_us_; } bool is_in_tx() const { return state_ > State::IDLE; } - bool is_dup_ls_modified() const; bool is_tx_active() const { return state_ >= State::ACTIVE && state_ < State::IN_TERMINATE; } void print_trace(); void dump_and_print_trace(); @@ -899,10 +721,6 @@ class ObTxDesc final : public share::ObLightHashLink void release_all_implicit_savepoint(); void release_implicit_savepoint(const ObTxSEQ savepoint); ObTransTraceLog &get_tlog() { return tlog_; } - bool is_xa_terminate_state_() const; - ObGlobalTxType get_global_tx_type(const ObXATransID &xid) const; - void set_global_tx_type(const ObGlobalTxType global_tx_type) - { global_tx_type_ = global_tx_type; } bool need_rollback() { return state_ == State::ABORTED; } int64_t get_timeout_us() const { return timeout_us_; } share::SCN get_tx_snapshot_version() { @@ -913,7 +731,13 @@ class ObTxDesc final : public share::ObLightHashLink } } ObITxCallback *cancel_commit_cb(); - int get_parts_copy(ObTxPartList ©_parts); + int get_write_state_copy(ObTxWriteState &write_state, bool &has_write_state) const; + void reset_write_state(); + int assign_write_state(const ObTxWriteState &participant); + int fill_read_snapshot_write_state(ObTxReadSnapshot &snapshot) const; + int find_write_state_after(ObTxWriteState *&part, const ObTxSEQ scn); + int get_abort_write_state(const ObTxWriteState *&part) const; + bool has_write_state() const { return has_write_state_; } int get_savepoints_copy(ObTxSavePointList ©_savepoints); // free route #define DEF_FREE_ROUTE_DECODE_(name) \ @@ -931,13 +755,12 @@ LST_DO(DEF_FREE_ROUTE_DECODE, (;), static, dynamic, parts, extra); int64_t estimate_state_size(); bool is_static_changed() { return state_change_flags_.STATIC_CHANGED_; } bool is_dynamic_changed() { return state_ > State::IDLE && state_change_flags_.DYNAMIC_CHANGED_; } - bool is_parts_changed() { return state_ > State::IDLE && state_change_flags_.PARTS_CHANGED_; }; + bool is_write_state_changed() { return state_ > State::IDLE && state_change_flags_.WRITE_STATE_CHANGED_; }; bool is_extra_changed() { return state_change_flags_.EXTRA_CHANGED_; }; void set_explicit() { flags_.EXPLICIT_ = true; } void clear_interrupt() { flags_.INTERRUPTED_ = false; } - void mark_part_abort(const ObTransID tx_id, const int abort_cause); - int64_t get_coord_epoch() const; - int get_and_inc_tx_seq(const int16_t branch, const int N, ObTxSEQ &tx_seq) const; + void mark_write_state_aborted(const ObTransID tx_id, const int abort_cause); + int get_and_inc_tx_seq(const int16_t branch, const int N, ObTxSEQ &tx_seq) const; ObTxSEQ inc_and_get_tx_seq(int16_t branch) const; int inc_and_get_tx_seq(const int16_t branch, const int N, ObTxSEQ &tx_seq) const; ObTxSEQ get_tx_seq(int64_t seq_abs = 0) const; @@ -947,8 +770,8 @@ LST_DO(DEF_FREE_ROUTE_DECODE, (;), static, dynamic, parts, extra); int add_modified_tables(const ObIArray &tables); bool has_modify_table(const uint64_t table_id) const; DISABLE_COPY_ASSIGN(ObTxDesc); - bool is_all_parts_clean() const; - bool is_all_parts_without_valid_write() const; + bool is_write_state_clean() const; + bool is_write_state_without_valid_write() const; }; // Is used to store and travserse all TxScheduler's Stat information; @@ -1058,115 +881,6 @@ class ObTxDescMgr final ObTransService &txs_; }; -class ObTxInfo -{ - friend class ObTransService; - OB_UNIS_VERSION(1); -protected: - - int64_t cluster_id_; - uint64_t cluster_version_; - int64_t seq_base_; - common::ObAddr addr_; - ObTransID tx_id_; - ObTxIsolationLevel isolation_; - ObTxAccessMode access_mode_; - share::SCN snapshot_version_; - int64_t snapshot_uncertain_bound_; - uint64_t op_sn_; - int64_t alloc_ts_; - int64_t active_ts_; - int64_t timeout_us_; - int64_t expire_ts_; - int64_t finish_ts_; - ObTxSEQ active_scn_; - ObTxPartList parts_; - uint32_t session_id_ = 0; - ObTxSavePointList savepoints_; -public: - ObTxInfo(): seq_base_(0) {} - TO_STRING_KV(K_(session_id), - K_(tx_id), - K_(access_mode), - K_(isolation), - K_(snapshot_version), - K_(active_scn), - K_(op_sn), - K_(alloc_ts), - K_(active_ts), - K_(timeout_us), - K_(expire_ts), - K_(seq_base), - K_(parts), - K_(cluster_id), - K_(cluster_version), - K_(savepoints)); - // TODO xa - bool is_valid() const { return tx_id_.is_valid(); } - const ObTransID &tid() const { return tx_id_; } -}; - -class ObTxStmtInfo -{ - friend class ObTransService; - OB_UNIS_VERSION(1); -protected: - ObTransID tx_id_; - uint64_t op_sn_; - ObTxPartList parts_; - ObTxDesc::State state_; - ObTxSavePointList savepoints_; -public: - TO_STRING_KV(K_(tx_id), - K_(op_sn), - K_(parts), - K_(state), - K_(savepoints)); - // TODO xa - bool is_valid() const { return tx_id_.is_valid(); } - const ObTransID &tid() const { return tx_id_; } - bool need_rollback() const { return state_ == ObTxDesc::State::ABORTED; } -}; - -class TxCtxRoleState -{ -public: - static const int64_t INVALID = -1; - static const int64_t LEADER = 0; - static const int64_t FOLLOWER = 1; - static const int64_t MAX = 2; - - static bool is_valid(const int64_t state) - { return state > INVALID && state < MAX; } -}; -class TxCtxOps -{ -public: - static const int64_t INVALID = -1; - static const int64_t TAKEOVER = 0; - static const int64_t REVOKE = 1; - static const int64_t RESUME = 2; - static const int64_t SWITCH_GRACEFUL = 3; - static const int64_t MAX = 4; - - static bool is_valid(const int64_t ops) - { return ops > INVALID && ops < MAX; } -}; -class TxCtxStateHelper -{ -public: - explicit TxCtxStateHelper(int64_t &state) : state_(state), - last_state_(TxCtxRoleState::INVALID), - is_switching_(false) {} - ~TxCtxStateHelper() {} - int switch_state(const int64_t op); - void restore_state(); -private: - int64_t &state_; - int64_t last_state_; - bool is_switching_; -}; - typedef lib::ObLockGuardWithTimeout ObSpinLockGuardWithTimeout; #define REC_TRANS_TRACE(recorder_ptr, trace_event) do { \ @@ -1190,24 +904,16 @@ typedef lib::ObLockGuardWithTimeout ObSpinLockGuardWithTimeout; inline ObTxSEQ ObTxDesc::get_tx_seq(int64_t seq_abs) const { int64_t seq = seq_abs > 0 ? seq_abs : ObSequence::get_max_seq_no(); - if (OB_LIKELY(support_branch())) { - if (seq < seq_base_) { - TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "seq_abs is less seq_base_", K(seq_abs), K(tx_id_), K(seq_base_)); - return ObTxSEQ::INVL(); - } - return ObTxSEQ(seq - seq_base_, 0); - } else { - return ObTxSEQ::mk_v0(seq); + if (seq < seq_base_) { + TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "seq_abs is less seq_base_", K(seq_abs), K(tx_id_), K(seq_base_)); + return ObTxSEQ::INVL(); } + return ObTxSEQ(seq - seq_base_, 0); } inline ObTxSEQ ObTxDesc::get_min_tx_seq() const { - if (OB_LIKELY(support_branch())) { - return ObTxSEQ(1, 0); - } else { - return ObTxSEQ::mk_v0(1); - } + return ObTxSEQ(1, 0); } inline int ObTxDesc::get_and_inc_tx_seq(const int16_t branch, @@ -1218,10 +924,8 @@ inline int ObTxDesc::get_and_inc_tx_seq(const int16_t branch, int64_t seq = 0; if (OB_FAIL(ObSequence::get_and_inc_max_seq_no(N, seq))) { TRANS_LOG(ERROR, "inc max seq no failed", K(ret), K(N)); - } else if (OB_LIKELY(support_branch())) { - tx_seq = ObTxSEQ(seq - seq_base_, branch); } else { - tx_seq = ObTxSEQ::mk_v0(seq); + tx_seq = ObTxSEQ(seq - seq_base_, branch); } return ret; } @@ -1229,11 +933,7 @@ inline int ObTxDesc::get_and_inc_tx_seq(const int16_t branch, inline ObTxSEQ ObTxDesc::inc_and_get_tx_seq(int16_t branch) const { int64_t seq = ObSequence::inc_and_get_max_seq_no(); - if (OB_LIKELY(support_branch())) { - return ObTxSEQ(seq - seq_base_, branch); - } else { - return ObTxSEQ::mk_v0(seq); - } + return ObTxSEQ(seq - seq_base_, branch); } inline int ObTxDesc::inc_and_get_tx_seq(const int16_t branch, @@ -1244,10 +944,8 @@ inline int ObTxDesc::inc_and_get_tx_seq(const int16_t branch, int64_t seq = 0; if (OB_FAIL(ObSequence::inc_and_get_max_seq_no(N, seq))) { TRANS_LOG(ERROR, "inc max seq no failed", K(ret), K(N)); - } else if (OB_LIKELY(support_branch())) { - tx_seq = ObTxSEQ(seq - seq_base_, branch); } else { - tx_seq = ObTxSEQ::mk_v0(seq); + tx_seq = ObTxSEQ(seq - seq_base_, branch); } return ret; } diff --git a/src/storage/tx/ob_trans_event.cpp b/src/storage/tx/ob_trans_event.cpp index 5c2f10bba..b3bdfd747 100644 --- a/src/storage/tx/ob_trans_event.cpp +++ b/src/storage/tx/ob_trans_event.cpp @@ -141,12 +141,6 @@ void ObTransStatistic::add_local_count(const int64_t value) EVENT_ADD(TRANS_LOCAL_COUNT, value); } -void ObTransStatistic::add_dist_count(const int64_t value) -{ - ObTenantDiagnosticInfoSummaryGuard g; - EVENT_ADD(TRANS_DIST_COUNT, value); -} - void ObTransStatistic::add_redo_log_replay_count(const int64_t value) { ObTenantDiagnosticInfoSummaryGuard g; @@ -159,18 +153,6 @@ void ObTransStatistic::add_redo_log_replay_time(const int64_t value) EVENT_ADD(REDO_LOG_REPLAY_TIME, value); } -void ObTransStatistic::add_prepare_log_replay_count(const int64_t value) -{ - ObTenantDiagnosticInfoSummaryGuard g; - EVENT_ADD(PREPARE_LOG_REPLAY_COUNT, value); -} - -void ObTransStatistic::add_prepare_log_replay_time(const int64_t value) -{ - ObTenantDiagnosticInfoSummaryGuard g; - EVENT_ADD(PREPARE_LOG_REPLAY_TIME, value); -} - void ObTransStatistic::add_commit_log_replay_count(const int64_t value) { ObTenantDiagnosticInfoSummaryGuard g; @@ -268,11 +250,5 @@ void ObTransStatistic::add_local_trans_total_used_time(const int64_t value) ObTenantDiagnosticInfoSummaryGuard g; EVENT_ADD(TRANS_LOCAL_TOTAL_USED_TIME, value); } -void ObTransStatistic::add_dist_trans_total_used_time(const int64_t value) -{ - ObTenantDiagnosticInfoSummaryGuard g; - EVENT_ADD(TRANS_DIST_TOTAL_USED_TIME, value); -} - } // transaction } // oceanbase diff --git a/src/storage/tx/ob_trans_event.h b/src/storage/tx/ob_trans_event.h index 673772f97..8831bdea8 100644 --- a/src/storage/tx/ob_trans_event.h +++ b/src/storage/tx/ob_trans_event.h @@ -70,12 +70,10 @@ class ObTransStatistic void add_trans_timeout_count(const int64_t value); // count how many transactions are started, via start_trans void add_trans_start_count(const int64_t value); - // count the total time of transactions, from start_trans to the end of receiving ack from coordinator + // count the total time of transactions, from start_trans to the final tx response void add_trans_total_used_time(const int64_t value); // count the total time of local trans void add_local_trans_total_used_time(const int64_t value); - // count the total time of distributed trans - void add_dist_trans_total_used_time(const int64_t value); // count the number of local stmt // count the number of remote stmt // count the number of distributed stmt @@ -91,20 +89,14 @@ class ObTransStatistic void add_trans_commit_time(const int64_t value); //Count the time from beginning rollback to receiving the abort response void add_trans_rollback_time(const int64_t value); - // count the number of transactions with 0 participant + // count the number of read-only transactions void add_readonly_count(const int64_t value); - // count the number of transactions with a single participant + // count the number of write transactions void add_local_count(const int64_t value); - // count the number of transactions with multiple participant - void add_dist_count(const int64_t value); // count how many redo logs are replayed void add_redo_log_replay_count(const int64_t value); // count the time spent on replaying redo logs void add_redo_log_replay_time(const int64_t value); - // count how many prepare logs are replayed - void add_prepare_log_replay_count(const int64_t value); - // count the time spent on replaying prepare logs - void add_prepare_log_replay_time(const int64_t value); // count how many commit logs are replayed void add_commit_log_replay_count(const int64_t value); // count the time spent on replaying commit logs @@ -123,8 +115,6 @@ class ObTransStatistic // count the callback time of sp commit log // count the number of callbacks of redo logs // count the callback time of redo log - // count the number of callbacks of prepare logs - // count the callback time of prepare log // count the number of callbacks of commit logs // count the callback time of commit log // count the number of callbacks of abort logs @@ -172,8 +162,7 @@ class ObTransStatistic trans_multi_partition_count_stat_("trans_single_partition_count"), distributed_stmt_stat_("distributed_stmt_count"), local_stmt_stat_("local_stmt_stat"), remote_stmt_stat_("remote_stmt_stat"), redo_replay_count_stat_("redo_replay_count"), - redo_replay_time_stat_("redo_replay_time"), prepare_replay_count_stat_("prepare_replay_count"), - prepare_replay_time_stat_("prepare_replay_time"), commit_replay_count_stat_("commit_replay_count"), + redo_replay_time_stat_("redo_replay_time"), commit_replay_count_stat_("commit_replay_count"), commit_replay_time_stat_("commit_replay_time"), abort_replay_count_stat_("abort_replay_count"), abort_replay_time_stat_("abort_replay_time"), clear_replay_count_stat_("clear_replay_count"), clear_replay_time_stat_("clear_replay_time"), stmt_timeout_count_stat_("stmt_timeout_count"), @@ -208,8 +197,6 @@ class ObTransStatistic ObTransStatItem remote_stmt_stat_; ObTransStatItem redo_replay_count_stat_; ObTransStatItem redo_replay_time_stat_; - ObTransStatItem prepare_replay_count_stat_; - ObTransStatItem prepare_replay_time_stat_; ObTransStatItem commit_replay_count_stat_; ObTransStatItem commit_replay_time_stat_; ObTransStatItem abort_replay_count_stat_; @@ -252,11 +239,9 @@ class ObTransStatistic #define TX_STAT_TIME_USED(time) ObTransStatistic::get_instance().add_trans_total_used_time( time); #define TX_STAT_COMMIT_TIME_USED(time) ObTransStatistic::get_instance().add_trans_commit_time( time); #define TX_STAT_ROLLBACK_TIME_USED(time) ObTransStatistic::get_instance().add_trans_rollback_time( time); -#define TX_STAT_DIST_INC ObTransStatistic::get_instance().add_dist_count( 1); #define TX_STAT_LOCAL_INC ObTransStatistic::get_instance().add_local_count( 1); #define TX_STAT_READONLY_INC ObTransStatistic::get_instance().add_readonly_count( 1); #define TX_STAT_READ_ELR_ROW_COUNT_INC transaction::ObTransStatistic::get_instance().add_read_elr_row_count( 1); #define TX_STAT_LOCAL_TOTAL_TIME_USED(time) ObTransStatistic::get_instance().add_local_trans_total_used_time( time); -#define TX_STAT_DIST_TOTAL_TIME_USED(time) ObTransStatistic::get_instance().add_dist_trans_total_used_time( time); #endif // OCEANABAE_TRANSACTION_OB_TRANS_EVENT_ diff --git a/src/storage/tx/ob_trans_factory.cpp b/src/storage/tx/ob_trans_factory.cpp index b2b31d66e..4dfbbe1b6 100644 --- a/src/storage/tx/ob_trans_factory.cpp +++ b/src/storage/tx/ob_trans_factory.cpp @@ -17,8 +17,7 @@ #include "ob_trans_factory.h" #include "share/rc/ob_module_provider.h" -#include "ob_trans_part_ctx.h" -#include "storage/tx/ob_gti_rpc.h" +#include "ob_tx_ctx.h" #include "storage/tx/ob_leak_checker.h" #include "observer/ob_server.h" @@ -30,10 +29,9 @@ using namespace obcall; namespace transaction { -int64_t ObTransCtxFactory::active_coord_ctx_count_ CACHE_ALIGNED = 0; -int64_t ObTransCtxFactory::active_part_ctx_count_ CACHE_ALIGNED = 0; -int64_t ObTransCtxFactory::total_release_part_ctx_count_ CACHE_ALIGNED = 0; -const char *ObTransCtxFactory::mod_type_ = "OB_TRANS_CTX"; +int64_t ObTxCtxFactory::active_tx_ctx_count_ CACHE_ALIGNED = 0; +int64_t ObTxCtxFactory::total_release_tx_ctx_count_ CACHE_ALIGNED = 0; +const char *ObTxCtxFactory::mod_type_ = "OB_TX_CTX"; int64_t ObLSTxCtxMgrFactory::alloc_count_ = 0; int64_t ObLSTxCtxMgrFactory::release_count_ = 0; @@ -102,60 +100,46 @@ const char *ObLSTxCtxMgrFactory::mod_type_ = "OB_PARTITION_TRANS_CTX_MGR"; #define MAKE_FACTORY_CLASS_IMPLEMENT_USE_OB_ALLOC(object_name, LABEL, arg...) MAKE_FACTORY_CLASS_IMPLEMENT(object_name, LABEL, OB, arg) #define MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(object_name, LABEL, arg...) MAKE_FACTORY_CLASS_IMPLEMENT(object_name, LABEL, RP, arg) -ObTransCtx *ObTransCtxFactory::alloc(const int64_t ctx_type) +ObTxCtx *ObTxCtxFactory::alloc() { int tmp_ret = OB_SUCCESS; - ObTransCtx *ctx = NULL; + ObTxCtx *ctx = NULL; if (OB_LIKELY(!ObTransErrsim::is_memory_errsim())) { - if (ObTransCtxType::PARTICIPANT == ctx_type) { - // During restart, the number of transaction contexts is relatively large - // and cannot be limited, otherwise there will be circular dependencies - if (ATOMIC_LOAD(&active_part_ctx_count_) > MAX_PART_CTX_COUNT && GCTX.status_ == ObServiceStatus::SS_SERVING) { - TRANS_LOG_RET(ERROR, tmp_ret, "participant context memory alloc failed", K_(active_part_ctx_count)); - tmp_ret = OB_TRANS_CTX_COUNT_REACH_LIMIT; - } else if (NULL != (ctx = mtl_sop_borrow(ObPartTransCtx))) { - (void)ATOMIC_FAA(&active_part_ctx_count_, 1); - TRANS_LOG(DEBUG, - "[Tx Ctx] alloc part_ctx success", - KP(ctx), - K(active_part_ctx_count_), - K(total_release_part_ctx_count_)); - } else { - // do nothing - } + // During restart, the number of transaction contexts is relatively large + // and cannot be limited, otherwise there will be circular dependencies. + if (ATOMIC_LOAD(&active_tx_ctx_count_) > MAX_TX_CTX_COUNT && GCTX.status_ == ObServiceStatus::SS_SERVING) { + TRANS_LOG_RET(ERROR, tmp_ret, "transaction context memory alloc failed", K_(active_tx_ctx_count)); + tmp_ret = OB_TRANS_CTX_COUNT_REACH_LIMIT; + } else if (NULL != (ctx = mtl_sop_borrow(ObTxCtx))) { + (void)ATOMIC_FAA(&active_tx_ctx_count_, 1); + TRANS_LOG(DEBUG, "alloc tx ctx success", KP(ctx), K(active_tx_ctx_count_), + K(total_release_tx_ctx_count_)); } else { - tmp_ret = OB_ERR_UNEXPECTED; - TRANS_LOG_RET(ERROR, tmp_ret, "unexpected error when context alloc", K(ctx_type)); + // do nothing } } if (REACH_TIME_INTERVAL(TRANS_MEM_STAT_INTERVAL)) { - TRANS_LOG(INFO, "ObTransCtx statistics", - K_(active_coord_ctx_count), - K_(active_part_ctx_count), - K_(total_release_part_ctx_count)); - (void)ATOMIC_STORE(&total_release_part_ctx_count_, 0); + TRANS_LOG(INFO, "ObTxCtx statistics", K_(active_tx_ctx_count), K_(total_release_tx_ctx_count)); + (void)ATOMIC_STORE(&total_release_tx_ctx_count_, 0); } (void) tmp_ret; // make compiler happy return ctx; } -void ObTransCtxFactory::release(ObTransCtx *ctx) +void ObTxCtxFactory::release(ObTransCtx *ctx) { if (OB_ISNULL(ctx)) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "context pointer is null when released", KP(ctx)); } else { - ObPartTransCtx *part_ctx = static_cast(ctx); - part_ctx->destroy(); - mtl_sop_return(ObPartTransCtx, part_ctx); - (void)ATOMIC_FAA(&active_part_ctx_count_, -1); - (void)ATOMIC_FAA(&total_release_part_ctx_count_, 1); - TRANS_LOG(DEBUG, - "[Tx Ctx] release part_ctx success", - KP(ctx), - K(active_part_ctx_count_), - K(total_release_part_ctx_count_)); + ObTxCtx *tx_ctx = static_cast(ctx); + tx_ctx->destroy(); + mtl_sop_return(ObTxCtx, tx_ctx); + (void)ATOMIC_FAA(&active_tx_ctx_count_, -1); + (void)ATOMIC_FAA(&total_release_tx_ctx_count_, 1); + TRANS_LOG(DEBUG, "release tx ctx success", KP(ctx), K(active_tx_ctx_count_), + K(total_release_tx_ctx_count_)); ctx = NULL; } } @@ -241,16 +225,12 @@ const char *TransRpcTaskFactory::get_mod_type() } */ -MAKE_OB_ALLOC(ObGtsRequestRpc, ObModIds::OB_GTS_REQUEST_RPC) -MAKE_OB_ALLOC(ObGtiRequestRpc, ObModIds::OB_GTI_REQUEST_RPC) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(ClogBuf, ObModIds::OB_TRANS_CLOG_BUF) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(MutatorBuf, ObModIds::OB_TRANS_MUTATOR_BUF) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(ObTransTraceLog, ObModIds::OB_TRANS_AUDIT_RECORD) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(ObPartitionAuditInfo, ObModIds::OB_PARTITION_AUDIT_INFO) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(ObCoreLocalPartitionAuditInfo, ObModIds::OB_CORE_LOCAL_STORAGE) -MAKE_FACTORY_CLASS_IMPLEMENT_USE_OB_ALLOC(ObGtsRequestRpc, ObModIds::OB_GTS_REQUEST_RPC) -MAKE_FACTORY_CLASS_IMPLEMENT_USE_OB_ALLOC(ObGtiRequestRpc, ObModIds::OB_GTI_REQUEST_RPC) MAKE_FACTORY_CLASS_IMPLEMENT_USE_RP_ALLOC(ObTxCommitCallbackTask, ObModIds::OB_END_TRANS_CB_TASK) void *MultiTxDataFactory::alloc(const int64_t len, const uint64_t arg1, const uint64_t arg2) diff --git a/src/storage/tx/ob_trans_factory.h b/src/storage/tx/ob_trans_factory.h index a93d543ad..f7766b559 100644 --- a/src/storage/tx/ob_trans_factory.h +++ b/src/storage/tx/ob_trans_factory.h @@ -31,7 +31,7 @@ namespace obcall namespace transaction { class ObTransCtx; -class ObPartTransCtx; +class ObTxCtx; //class ObPartitionTransCtxMgr; class ObLSTxCtxMgr; //class TransRpcTask; @@ -42,25 +42,20 @@ class RollbackTransTask; class CallbackTransTask; class WaitTransEndTask; class ObCoreLocalPartitionAuditInfo; -class ObGtsRequestRpc; -class ObGtiRequestRpc; class ObTxCommitCallbackTask; -class ObTransCtxFactory +class ObTxCtxFactory { public: - static ObTransCtx *alloc(const int64_t ctx_type); + static ObTxCtx *alloc(); static void release(ObTransCtx *ctx); - static int64_t get_alloc_count() { return ATOMIC_LOAD(&active_part_ctx_count_); } + static int64_t get_alloc_count() { return ATOMIC_LOAD(&active_tx_ctx_count_); } static int64_t get_release_count() { return 0; } static const char *get_mod_type() { return mod_type_; } - static int64_t get_active_part_ctx_cunt() { return ATOMIC_LOAD(&active_part_ctx_count_); } private: static const char *mod_type_; - static int64_t active_sche_ctx_count_; - static int64_t active_coord_ctx_count_; - static int64_t active_part_ctx_count_; - static int64_t total_release_part_ctx_count_; + static int64_t active_tx_ctx_count_; + static int64_t total_release_tx_ctx_count_; }; template @@ -178,8 +173,6 @@ MAKE_FACTORY_CLASS_DEFINE(CallbackTransTask) MAKE_FACTORY_CLASS_DEFINE(ObTransTraceLog) MAKE_FACTORY_CLASS_DEFINE(ObPartitionAuditInfo) MAKE_FACTORY_CLASS_DEFINE(ObCoreLocalPartitionAuditInfo) -MAKE_FACTORY_CLASS_DEFINE(ObGtsRequestRpc) -MAKE_FACTORY_CLASS_DEFINE(ObGtiRequestRpc) MAKE_FACTORY_CLASS_DEFINE(ObTxCommitCallbackTask) class MultiTxDataFactory diff --git a/src/storage/tx/ob_trans_functor.h b/src/storage/tx/ob_trans_functor.h index c70c68649..4930dbb9e 100644 --- a/src/storage/tx/ob_trans_functor.h +++ b/src/storage/tx/ob_trans_functor.h @@ -19,10 +19,9 @@ #include "common/ob_simple_iterator.h" #include "ob_trans_ctx.h" -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" #include "ob_trans_stat.h" -#include "storage/tx/ob_ls_tx_ctx_mgr_stat.h" #include "ob_trans_version_mgr.h" #include "storage/blocksstable/ob_macro_block_writer.h" #include "share/ob_force_print_log.h" @@ -82,15 +81,14 @@ class TxFunctorStat iter_cnt_++; } - void finish_iter_single(const char *func_name, const ObTransID &tx_id, const share::ObLSID &ls_id) + void finish_iter_single(const char *func_name, const ObTransID &tx_id) { functor_name_ = func_name; - ls_id_ = ls_id; if (INT64_MAX != single_expired_limit_) { int64_t cur_time = ObTimeUtility::fast_current_time(); if (cur_time - single_begin_time_ > single_expired_limit_) { single_expired_cnt_++; - TRANS_LOG(INFO, "single tx cost too much time", K_(functor_name), K(tx_id), K(ls_id), + TRANS_LOG(INFO, "single tx cost too much time", K_(functor_name), K(tx_id), "cost_time", cur_time - single_begin_time_, K(single_begin_time_)); } } @@ -102,10 +100,10 @@ class TxFunctorStat } if (iter_cnt_ > 0) { if (force_print) { - TRANS_LOG(INFO, "ls trans functor stat", K_(functor_name), K_(ls_id), KPC(this)); + TRANS_LOG(INFO, "trans functor stat", K_(functor_name), KPC(this)); } else if (total_expired_limit_ != INT_MAX && finish_time_ - begin_time_ >= total_expired_limit_) { - TRANS_LOG(INFO, "ls trans functor stat", K_(functor_name), K_(ls_id), KPC(this)); + TRANS_LOG(INFO, "trans functor stat", K_(functor_name), KPC(this)); } } } @@ -122,7 +120,6 @@ class TxFunctorStat private: const char *functor_name_; - share::ObLSID ls_id_; int64_t single_begin_time_; @@ -135,7 +132,7 @@ class TxFunctorStat }; // XXX TMP_CODE -// In the future, ObTransCtx will no longer be stored in the hashmap, but ObPartTransCtx directly; +// In the future, ObTransCtx will no longer be stored in the hashmap, but ObTxCtx directly; // there are too many changes in this commit, so they will be processed in the next commit; TODO senchen; #define OPERATOR_V4(FUNC_NAME) \ private: \ @@ -143,15 +140,14 @@ class TxFunctorStat public: \ bool operator()(ObTransCtx *tx_ctx_base) { \ bool bool_ret = false; \ - ObPartTransCtx *tx_ctx = static_cast(tx_ctx_base); \ + ObTxCtx *tx_ctx = static_cast(tx_ctx_base); \ ObTransID tx_id = tx_ctx->get_trans_id(); \ - share::ObLSID ls_id = tx_ctx->get_ls_id(); \ func_stat_.begin_iter_single(); \ bool_ret = internal_operator(tx_id, tx_ctx); \ - func_stat_.finish_iter_single(#FUNC_NAME, tx_id, ls_id);\ + func_stat_.finish_iter_single(#FUNC_NAME, tx_id);\ return bool_ret; \ }; \ - bool internal_operator(const ObTransID &tx_id, ObPartTransCtx *tx_ctx) + bool internal_operator(const ObTransID &tx_id, ObTxCtx *tx_ctx) #define SET_EXPIRED_LIMIT(SINGLE_LIMIT, TOTAL_LIMIT) \ func_stat_.set_expired_limit(SINGLE_LIMIT, TOTAL_LIMIT); @@ -159,165 +155,6 @@ class TxFunctorStat #define PRINT_FUNC_STAT func_stat_.print_stat_(); #define FORCE_PRINT_FUNC_STAT func_stat_.print_stat_(true); -class SwitchToFollowerForcedlyFunctor -{ -public: - SwitchToFollowerForcedlyFunctor(ObTxCommitCallback *&cb_list) : cb_list_(cb_list) - { - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/) - } - ~SwitchToFollowerForcedlyFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(SwitchToFollowerForcedlyFunctor) - { - int tmp_ret = common::OB_SUCCESS; - - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - tmp_ret = common::OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - } else if (common::OB_SUCCESS != (tmp_ret = tx_ctx->switch_to_follower_forcedly(cb_list_))) { - TRANS_LOG_RET(ERROR, tmp_ret, "leader revoke failed", K(tx_id), K(*tx_ctx)); - } - - return true; - } - -private: - ObTxCommitCallback *&cb_list_; -}; - -class SwitchToLeaderFunctor -{ -public: - explicit SwitchToLeaderFunctor(share::SCN &start_working_ts) : ret_(common::OB_SUCCESS) - { - start_working_ts_ = start_working_ts; - - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); - } - ~SwitchToLeaderFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(SwitchToLeaderFunctor) - { - bool bool_ret = false; - int tmp_ret = common::OB_SUCCESS; - - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - TRANS_LOG_RET(WARN, common::OB_INVALID_ARGUMENT, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - } else if (OB_TMP_FAIL(tx_ctx->switch_to_leader(start_working_ts_))) { - TRANS_LOG_RET(WARN, tmp_ret, "switch_to_leader error", "ret", tmp_ret, K(*tx_ctx)); - ret_ = tmp_ret; - } else { - bool_ret = true; - } - return bool_ret; - } - int get_ret() const { return ret_; } -private: - share::SCN start_working_ts_; - int ret_; -}; - -class SwitchToFollowerGracefullyFunctor -{ -public: - SwitchToFollowerGracefullyFunctor(const int64_t abs_expired_time, ObTxCommitCallback *&cb_list) - : abs_expired_time_(abs_expired_time), count_(0), ret_(OB_SUCCESS), cb_list_(cb_list) - { - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); - } - ~SwitchToFollowerGracefullyFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(SwitchToFollowerGracefullyFunctor) - { - bool bool_ret = false; - int ret = OB_SUCCESS; - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - ret_ = ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - } else { - ++count_; - if ((count_ % BATCH_CHECK_COUNT) == 0) { - const int64_t now = ObTimeUtility::current_time(); - if (now >= abs_expired_time_) { - ret_ = ret = OB_TIMEOUT; - TRANS_LOG(WARN, "switch to follower gracefully timeout"); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(tx_ctx->switch_to_follower_gracefully(cb_list_))) { - TRANS_LOG(WARN, "switch to follower gracefully failed", KR(ret), K(*tx_ctx)); - ret_ = ret; - } else { - bool_ret = true; - } - } - return bool_ret; - } - int get_ret() const { return ret_; } - int64_t get_count() const { return count_; } - -private: - static const int64_t BATCH_CHECK_COUNT = 100; - int64_t abs_expired_time_; - int64_t count_; - int ret_; - ObTxCommitCallback *&cb_list_; -}; - -class ResumeLeaderFunctor -{ -public: - ResumeLeaderFunctor(share::SCN &start_working_ts) - { - start_working_ts_ = start_working_ts; - - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); - } - ~ResumeLeaderFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(ResumeLeaderFunctor) - { - bool bool_ret = false; - int ret = OB_SUCCESS; - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - } else if (OB_FAIL(tx_ctx->resume_leader(start_working_ts_))) { - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(*tx_ctx)); - } else { - bool_ret = true; - } - return bool_ret; - } - -private: - share::SCN start_working_ts_; -}; - -class ReplayTxStartWorkingLogFunctor -{ -public: - ReplayTxStartWorkingLogFunctor(share::SCN &start_working_ts) - { - start_working_ts_ = start_working_ts; - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); - } - ~ReplayTxStartWorkingLogFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(ReplayTxStartWorkingLogFunctor) - { - int ret = OB_SUCCESS; - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - ret = common::OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_id), "ctx", OB_P(tx_ctx)); - } else { - if (OB_FAIL(tx_ctx->replay_start_working_log(start_working_ts_))) { - TRANS_LOG(WARN, "replay start working log error", KR(ret), K(tx_id)); - } - } - return true; - } - -private: - share::SCN start_working_ts_; -}; - class KillTxCtxFunctor { public: @@ -359,236 +196,6 @@ class KillTxCtxFunctor ObTxCommitCallback *&cb_list_; }; -class WaitTxWriteEndFunctor -{ -public: - WaitTxWriteEndFunctor(const int64_t abs_expired_time) - : abs_expired_time_(abs_expired_time), count_(0), ret_(OB_SUCCESS) - { - - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); - } - ~WaitTxWriteEndFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(WaitTxWriteEndFunctor) - { - bool bool_ret = false; - int ret = OB_SUCCESS; - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - ret_ = ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - } else { - ++count_; - if ((count_ % BATCH_CHECK_COUNT) == 0) { - const int64_t now = ObTimeUtility::current_time(); - if (now >= abs_expired_time_) { - ret_ = ret = OB_TIMEOUT; - TRANS_LOG(WARN, "wait tx write end timeout", K(count_)); - } - } - } - if (OB_FAIL(ret)) { - } else { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "wait tx write end failed", KR(ret), K(*tx_ctx)); - ret_ = ret; - } - return bool_ret; - } - int get_ret() const { return ret_; } - int64_t get_count() const { return count_; } -private: - static const int64_t BATCH_CHECK_COUNT = 100; - int64_t abs_expired_time_; - int64_t count_; - int ret_; -}; - -class StopLSFunctor -{ -public: - StopLSFunctor() {} - ~StopLSFunctor() {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = false; - const bool graceful = false; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (OB_TMP_FAIL(ls_tx_ctx_mgr->stop(graceful))) { - TRANS_LOG_RET(WARN, tmp_ret, "ObLSTxCtxMgr stop error", K(tmp_ret), K(ls_id)); - } else { - bool_ret = true; - } - } - - return bool_ret; - } -}; - -class WaitLSFunctor -{ -public: - explicit WaitLSFunctor(int64_t &retry_count) : retry_count_(retry_count) {} - ~WaitLSFunctor() {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = true; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (!ls_tx_ctx_mgr->is_stopped()) { - tmp_ret = OB_PARTITION_IS_NOT_STOPPED; - TRANS_LOG_RET(WARN, tmp_ret, "ls_id has not been stopped", K(ls_id)); - } else if (ls_tx_ctx_mgr->get_tx_ctx_count() > 0) { - // if there are unfinished transactions at the ls_id, - // increase retry_count by 1 - ++retry_count_; - } else { - // do nothing - } - } - - if (common::OB_SUCCESS != tmp_ret) { - bool_ret = false; - } - return bool_ret; - } - -private: - int64_t &retry_count_; -}; - -class RemoveLSFunctor -{ -public: - RemoveLSFunctor() {} - ~RemoveLSFunctor() {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = false; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (!ls_tx_ctx_mgr->is_stopped()) { - tmp_ret = OB_PARTITION_IS_NOT_STOPPED; - TRANS_LOG_RET(WARN, tmp_ret, "ls_tx_ctx_mgr has not been stopped", K(ls_id)); - } else { - // Release all ctx memory on the ls_id - ls_tx_ctx_mgr->destroy(); - ls_tx_ctx_mgr = NULL; - bool_ret = true; - } - TRANS_LOG_RET(INFO, tmp_ret, "remove ls", K(ls_id), KP(ls_tx_ctx_mgr)); - } - return bool_ret; - } -}; - -class IterateLSIDFunctor -{ -public: - explicit IterateLSIDFunctor(ObLSIDIterator &ls_id_iter) : ls_id_iter_(ls_id_iter) {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = false; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (OB_TMP_FAIL(ls_id_iter_.push(ls_id))) { - TRANS_LOG_RET(WARN, tmp_ret, "ObLSIDIterator push ls_id error", K(tmp_ret), K(ls_id)); - } else { - bool_ret = true; - } - } - return bool_ret; - } -private: - ObLSIDIterator &ls_id_iter_; -}; - -class IterateLSTxCtxMgrStatFunctor -{ -public: - IterateLSTxCtxMgrStatFunctor(const ObAddr &addr, ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter) - : tx_ctx_mgr_stat_iter_(tx_ctx_mgr_stat_iter), addr_(addr) {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = false; - ObLSTxCtxMgrStat ls_tx_ctx_mgr_stat; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else { - uint64_t mgr_state; - bool is_master = false; - bool is_stopped = true; - ls_tx_ctx_mgr->tx_ls_state_mgr_.iter_ctx_mgr_stat_info(mgr_state, is_master, is_stopped); - tmp_ret = ls_tx_ctx_mgr_stat.init(addr_, - ls_tx_ctx_mgr->ls_id_, - is_master, - is_stopped, - mgr_state, - ls_tx_ctx_mgr->total_tx_ctx_count_, - (int64_t)(&(*ls_tx_ctx_mgr))); - if (OB_SUCCESS != tmp_ret) { - TRANS_LOG_RET(WARN, tmp_ret, "ObLSTxCtxMgrStat init error", K_(addr), "ls_tx_ctx_mgr", *ls_tx_ctx_mgr); - } else if (OB_TMP_FAIL(tx_ctx_mgr_stat_iter_.push(ls_tx_ctx_mgr_stat))) { - TRANS_LOG_RET(WARN, tmp_ret, "ObTxCtxMgrStatIterator push error", - K(tmp_ret), K(ls_id), "ls_tx_ctx_mgr", *ls_tx_ctx_mgr); - } else { - bool_ret = true; - } - } - } - - return bool_ret; - } -private: - ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter_; - const ObAddr &addr_; -}; - class IterateCheckTabletModifySchema { public: @@ -737,11 +344,11 @@ class ObGetMinUndecidedLogTsFunctor share::SCN log_ts_; }; -class IterateAllLSTxStatFunctor +class IterateObserverTxStatFunctor { public: - explicit IterateAllLSTxStatFunctor(ObTxStatIterator &tx_stat_iter): tx_stat_iter_(tx_stat_iter), - ret_(OB_SUCCESS) {} + explicit IterateObserverTxStatFunctor(ObTxStatIterator &tx_stat_iter): tx_stat_iter_(tx_stat_iter), + ret_(OB_SUCCESS) {} bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) { int ret = common::OB_SUCCESS; @@ -751,13 +358,8 @@ class IterateAllLSTxStatFunctor ret = OB_INVALID_ARGUMENT; TRANS_LOG_RET(WARN, ret, "invalid argument", KP(ls_tx_ctx_mgr)); } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (OB_FAIL(ls_tx_ctx_mgr->iterate_tx_ctx_stat(tx_stat_iter_))) { - TRANS_LOG_RET(WARN, ret, "iterate_tx_ctx_stat error", K(ret), K(ls_id)); + if (OB_FAIL(ls_tx_ctx_mgr->iterate_tx_ctx_stat(tx_stat_iter_))) { + TRANS_LOG_RET(WARN, ret, "iterate_tx_ctx_stat error", K(ret)); } else { bool_ret = true; } @@ -833,14 +435,13 @@ class IterateTxStatFunctor } if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; - share::ObLSArray participants_arr; + bool has_write_state = false; ObTxData *tx_data = NULL; int busy_cbs_cnt = -1; tx_ctx->ctx_tx_data_.get_tx_data_ptr(tx_data); - if (OB_TMP_FAIL(tx_ctx->get_stat_for_virtual_table(participants_arr, busy_cbs_cnt))) { - TRANS_LOG_RET(WARN, tmp_ret, "ObTxStat get participants copy error", K(tmp_ret)); - // push an invalid ls id to hint the failure - participants_arr.push_back(share::ObLSID()); + if (OB_TMP_FAIL(tx_ctx->get_stat_for_virtual_table(has_write_state, busy_cbs_cnt))) { + TRANS_LOG_RET(WARN, tmp_ret, "ObTxStat get write state copy error", K(tmp_ret)); + has_write_state = false; } if (OB_TMP_FAIL(tx_ctx->mt_ctx_.get_callback_list_stat(tx_stat.callback_list_stats_))) { TRANS_LOG_RET(WARN, tmp_ret, "ObTxStat get callback lists stat error", K(tmp_ret)); @@ -848,26 +449,21 @@ class IterateTxStatFunctor if (OB_FAIL(tx_stat.init(tx_ctx->addr_, tx_id, has_decided, - tx_ctx->ls_id_, - participants_arr, + has_write_state, tx_ctx->ctx_create_time_, tx_ctx->trans_expired_time_, tx_ctx->ref_, tx_ctx->last_op_sn_, tx_ctx->pending_write_, (int64_t)tx_ctx->exec_info_.state_, - tx_ctx->exec_info_.trans_type_, tx_ctx->part_trans_action_, tx_ctx, tx_ctx->get_pending_log_size(), tx_ctx->get_flushed_log_size(), - tx_ctx->role_state_, tx_ctx->session_id_, tx_ctx->client_sid_, - tx_ctx->exec_info_.scheduler_, tx_ctx->is_exiting_, tx_ctx->exec_info_.xid_, - tx_ctx->exec_info_.upstream_, tx_ctx->last_request_ts_, OB_NOT_NULL(tx_data) ? tx_data->start_scn_.atomic_load() : SCN::invalid_scn(), OB_NOT_NULL(tx_data) ? tx_data->end_scn_.atomic_load() : SCN::invalid_scn(), @@ -1019,7 +615,6 @@ class IterateTxLockStatFunctor for (int i = 0; OB_SUCC(ret) && i < count; i++) { ObTxLockStat tx_lock_stat; if (OB_FAIL(tx_lock_stat.init(tx_ctx->get_addr(), - tx_ctx->get_ls_id(), memtable_key_info_arr.at(i), tx_ctx->get_session_id(), tx_ctx->get_client_sid(), @@ -1081,36 +676,6 @@ class PrintFunctor bool verbose_; }; -class PrintAllLSTxCtxFunctor -{ -public: - PrintAllLSTxCtxFunctor() {} - ~PrintAllLSTxCtxFunctor() {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int tmp_ret = common::OB_SUCCESS; - bool bool_ret = false; - const bool verbose = true; - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - tmp_ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else { - ls_tx_ctx_mgr->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); - bool_ret = true; - } - } - UNUSED(tmp_ret); - return bool_ret; - } -}; - class ObRemoveAllTxCtxFunctor { public: @@ -1241,10 +806,10 @@ class GetMinStartSCNFunctor share::SCN min_start_scn_; }; -class IteratePartCtxAskSchedulerStatusFunctor +class IterateTxCtxStatusFunctor { public: - IteratePartCtxAskSchedulerStatusFunctor() + IterateTxCtxStatusFunctor() { SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); first_err_code_ = OB_SUCCESS; @@ -1252,8 +817,8 @@ class IteratePartCtxAskSchedulerStatusFunctor min_start_scn_.set_max(); } - ~IteratePartCtxAskSchedulerStatusFunctor() { PRINT_FUNC_STAT; } - OPERATOR_V4(IteratePartCtxAskSchedulerStatusFunctor) + ~IterateTxCtxStatusFunctor() { PRINT_FUNC_STAT; } + OPERATOR_V4(IterateTxCtxStatusFunctor) { int ret = OB_SUCCESS; @@ -1271,8 +836,8 @@ class IteratePartCtxAskSchedulerStatusFunctor // logic for gc tx ctx int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(tx_ctx->check_scheduler_status())) { - TRANS_LOG(WARN, "check scheduler status error", KR(tmp_ret), "ctx", *tx_ctx); + if (OB_TMP_FAIL(tx_ctx->check_tx_status())) { + TRANS_LOG(WARN, "check transaction status error", KR(tmp_ret), "ctx", *tx_ctx); } } @@ -1342,10 +907,11 @@ class IterateTxSchedulerFunctor if (OB_SUCCESS == tmp_ret) { ObTxSchedulerStat tx_scheduler_stat; - ObTxPartList copy_parts; + ObTxWriteState write_state; + bool has_write_state = false; ObTxSavePointList copy_savepoints; - if (OB_TMP_FAIL(tx_desc->get_parts_copy(copy_parts))) { - TRANS_LOG_RET(WARN, tmp_ret, "ObTxSchedulerStat get participants copy error", K(tmp_ret)); + if (OB_TMP_FAIL(tx_desc->get_write_state_copy(write_state, has_write_state))) { + TRANS_LOG_RET(WARN, tmp_ret, "ObTxSchedulerStat get write state copy error", K(tmp_ret)); } else if (OB_TMP_FAIL(tx_desc->get_savepoints_copy(copy_savepoints))) { TRANS_LOG_RET(WARN, tmp_ret, "ObTxSchedulerStat get savepoints copy error", K(tmp_ret)); } else if (OB_TMP_FAIL(tx_scheduler_stat.init(tx_desc->addr_, @@ -1355,8 +921,8 @@ class IterateTxSchedulerFunctor (int64_t)tx_desc->state_, tx_desc->cluster_id_, tx_desc->xid_, - tx_desc->coord_id_, - copy_parts, + has_write_state, + write_state, tx_desc->isolation_, tx_desc->snapshot_version_, tx_desc->access_mode_, @@ -1388,75 +954,6 @@ class IterateTxSchedulerFunctor }; -class StandbyCleanUpAllLSFunctor -{ -public: - StandbyCleanUpAllLSFunctor(ObTimeGuard &cleanup_timeguard) - : ret_(OB_SUCCESS), cleanup_timeguard_(cleanup_timeguard) {} - ~StandbyCleanUpAllLSFunctor() {} - bool operator()(ObLSTxCtxMgr *ls_tx_ctx_mgr) - { - int ret = common::OB_SUCCESS; - bool bool_ret = false; - cleanup_timeguard_.click(); - - if (OB_ISNULL(ls_tx_ctx_mgr)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, ret, "invalid argument", KP(ls_tx_ctx_mgr)); - } else { - const share::ObLSID &ls_id = ls_tx_ctx_mgr->get_ls_id(); - - if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); - } else if (OB_FAIL(ls_tx_ctx_mgr->do_standby_cleanup())) { - TRANS_LOG_RET(WARN, ret, "iterate_standby_cleanup error", K(ret), K(ls_id)); - } else { - bool_ret = true; - } - } - if (OB_FAIL(ret)) { - ret_ = ret; - } - return bool_ret; - } - int get_ret() const { return ret_; } -private: - int ret_; - ObTimeGuard &cleanup_timeguard_; -}; - -class StandbyCleanUpFunctor -{ -public: - StandbyCleanUpFunctor() {} - ~StandbyCleanUpFunctor() {} - OPERATOR_V4(StandbyCleanUpFunctor) - { - int ret = common::OB_SUCCESS; - bool bool_ret = false; - - if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG_RET(WARN, ret, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); - // If you encounter a situation where tx_ctx has not been init yet, - // skip it directly, there will be a background thread retry - } else if (!tx_ctx->is_inited()) { - // not inited, don't need to traverse - } - if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret || OB_SUCC(ret)) { - bool_ret = true; - } else { - ret_ = ret; - } - - return bool_ret; - } - int get_ret() const { return ret_; } -private: - int ret_; -}; - } // transaction } // oceanbase diff --git a/src/storage/tx/ob_trans_id.h b/src/storage/tx/ob_trans_id.h index 9d7fa67f1..202df1692 100644 --- a/src/storage/tx/ob_trans_id.h +++ b/src/storage/tx/ob_trans_id.h @@ -20,6 +20,7 @@ #include #include "lib/utility/ob_unify_serialize.h" #include "lib/utility/ob_print_utils.h" +#include "lib/json/ob_yson.h" #include "lib/ob_define.h" #include "lib/time/ob_time_utility.h" // ObMonotonicTs namespace oceanbase diff --git a/src/storage/tx/ob_trans_id_service.cpp b/src/storage/tx/ob_trans_id_service.cpp index 7f22c08db..c061f7215 100644 --- a/src/storage/tx/ob_trans_id_service.cpp +++ b/src/storage/tx/ob_trans_id_service.cpp @@ -35,29 +35,5 @@ int ObTransIDService::init() return OB_SUCCESS; } -int ObTransIDService::handle_request(const ObGtiRequest &request, obcall::ObGtiRpcResult &result) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(request)); - } else { - TRANS_LOG(DEBUG, "handle gti request", K(request)); - - const int64_t range = request.get_range(); - int64_t start_id = 0; - int64_t end_id = 0; - if (OB_FAIL(get_number(range, 0, start_id, end_id))) { - TRANS_LOG(WARN, "get trans id failed", KR(ret)); - } - if (OB_FAIL(result.init(ret, start_id, end_id))) { - TRANS_LOG(WARN, "gti result init failed", KR(ret), K(request)); - } - } - //todo zhaoxing:ObTransStatistic - return ret; -} - } } diff --git a/src/storage/tx/ob_trans_id_service.h b/src/storage/tx/ob_trans_id_service.h index 9603f854b..64c0bbb4c 100644 --- a/src/storage/tx/ob_trans_id_service.h +++ b/src/storage/tx/ob_trans_id_service.h @@ -18,7 +18,6 @@ #define OCEANBASE_TRANSACTION_OB_TRANS_ID_SERVICE_ #include "ob_id_service.h" -#include "ob_gti_rpc.h" namespace oceanbase { @@ -35,7 +34,6 @@ class ObTransIDService : public ObIDService static int mtl_init(ObTransIDService *&trans_id_service); void destroy() { reset(); } static const int64_t TRANS_ID_PREALLOCATED_RANGE = 1000000; // TransID default preallocated size - int handle_request(const ObGtiRequest &request, obcall::ObGtiRpcResult &result); }; } diff --git a/src/storage/tx/ob_trans_rpc.cpp b/src/storage/tx/ob_trans_rpc.cpp deleted file mode 100644 index e7b217806..000000000 --- a/src/storage/tx/ob_trans_rpc.cpp +++ /dev/null @@ -1,625 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_trans_rpc.h" -#include "share/rc/ob_module_provider.h" -#include "ob_trans_service.h" -#include "share/ob_ex_rpc.h" - -namespace oceanbase -{ - -using namespace common; -using namespace transaction; -using namespace obcall; -using namespace storage; -using namespace share; - -namespace obcall -{ -OB_SERIALIZE_MEMBER(ObTransRpcResult, status_, send_timestamp_, private_data_); -OB_SERIALIZE_MEMBER(ObTxRpcRollbackSPResult, status_, send_timestamp_, addr_, - born_epoch_, ignore_); - -bool need_refresh_location_cache_(const int ret) -{ - return (common::OB_NOT_MASTER == ret || - common::OB_PARTITION_IS_BLOCKED == ret || - common::OB_REPLICA_NOT_READABLE == ret || - common::OB_LS_NOT_EXIST == ret || - common::OB_PARTITION_NOT_EXIST == ret || - common::OB_TENANT_NOT_EXIST == ret || - common::OB_TENANT_NOT_IN_SERVER == ret); -} - -int refresh_location_cache(const share::ObLSID ls) -{ - return share::g_mp->trans_service()->refresh_location_cache(ls); -} - -int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, - const share::ObLSID &receiver_ls_id, - const transaction::ObTransID &tx_id, - const int16_t msg_type, - const int status, - const ObAddr &addr, - const int64_t request_id, - const SCN &private_data) -{ - return share::g_mp->trans_service()->handle_trans_msg_callback(sender_ls_id, - receiver_ls_id, - tx_id, - msg_type, - status, - addr, - request_id, - private_data); -} - - -int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, - const int64_t epoch, - const transaction::ObTransID &tx_id, - const int status, - const int64_t request_id, - const ObTxRpcRollbackSPResult &result) -{ - if (result.ignore_) { - return OB_SUCCESS; - } - return share::g_mp->trans_service()->handle_sp_rollback_resp(receiver_ls_id, - epoch, - tx_id, - status, - request_id, - result.born_epoch_, - result.addr_); -} - -void ObTransRpcResult::reset() -{ - status_ = OB_SUCCESS; - send_timestamp_ = 0L; - private_data_.reset(); -} - -void ObTransRpcResult::init(const int status, const int64_t timestamp) -{ - status_ = status; - send_timestamp_ = timestamp; -} - - -} // obcall - -namespace -{ -// Local in-process dispatch of a transaction message, replacing the async RPC -// post + ObTxRPCCB completion. Runs fire-and-forget on the ASYNC_CALL worker -// pool, switches into the target tenant, invokes the same handler the ObTx*P -// processor used, then drives the sender-side callback. The local transport -// never fails, so the callback status comes from result.get_status() (exactly -// what ObTxRPCCB::process used when rcode == OB_SUCCESS). Single-replica: the -// receiver LS leader is always local, so dst == the resolved local server. -template -void dispatch_tx_msg_async_(const common::ObAddr &dst, - const MsgType &msg, Handler handler) -{ - (void)ex_rpc::async_call(msg, [dst, handler](MsgType &m) { - int ret = OB_SUCCESS; - MOD_SCOPE { - transaction::ObTransService *txs = share::g_mp->trans_service(); - obcall::ObTransRpcResult result; - if (OB_ISNULL(txs)) { - ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "get tx service fail", K(ret)); - } else { - if (!m.is_valid()) { - ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "msg is invalid", K(ret), K(m)); - } else if (OB_FAIL((txs->*handler)(m, result)) && OB_TRANS_COMMITED != ret) { - TRANS_LOG(WARN, "handle txn message fail", K(ret), "msg", m); - } - (void)obcall::handle_trans_msg_callback(m.get_sender(), m.get_receiver(), - m.get_trans_id(), m.get_msg_type(), result.get_status(), dst, - m.get_request_id(), result.private_data_); - } - } - }); -} - -// rollback-savepoint request has its own result type and completion path -// (handle_sp_rollback_resp, which no-ops when result.ignore_ is set for the -// async-resp protocol). Otherwise identical to dispatch_tx_msg_async_. -void dispatch_rollback_sp_async_(const transaction::ObTxRollbackSPMsg &msg) -{ - (void)ex_rpc::async_call(msg, [](transaction::ObTxRollbackSPMsg &m) { - int ret = OB_SUCCESS; - MOD_SCOPE { - transaction::ObTransService *txs = share::g_mp->trans_service(); - obcall::ObTxRpcRollbackSPResult result; - if (OB_ISNULL(txs)) { - ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "get tx service fail", K(ret)); - } else { - if (!m.is_valid()) { - ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "msg is invalid", K(ret), K(m)); - } else if (OB_FAIL(txs->handle_sp_rollback_request(m, result))) { - TRANS_LOG(WARN, "handle txn message fail", K(ret), "msg", m); - } - (void)obcall::handle_sp_rollback_resp(m.get_receiver(), m.get_epoch(), - m.get_trans_id(), result.get_status(), m.get_request_id(), result); - } - } - }); -} - -// 2PC distributed messages (TX_2PC_*). The legacy transport coalesced these -// through obcall::ObBatchRpc -> ObBatchP::handle_tx_req -> handle_tx_batch_req. -// Single-replica: the receiver LS leader is always local, so we serialize the -// concrete msg (exactly as ObIFill::fill_buffer did) and dispatch it in-process -// + async into the same sink (ObTransService::handle_tx_batch_req), preserving -// the original async, fire-and-forget delivery semantics. -void dispatch_tx_2pc_async_(const transaction::ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int16_t msg_type = msg.get_msg_type(); - const int64_t size = msg.get_req_size(); - char *buf = NULL; - int64_t filled = 0; - if (size <= 0) { - TRANS_LOG(WARN, "invalid 2pc msg size", K(size), K(msg_type)); - } else if (OB_ISNULL(buf = static_cast(ob_malloc(size, SET_USE_500("TxRpc2pc"))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc 2pc msg buffer fail", K(ret), K(size), K(msg_type)); - } else if (OB_FAIL(msg.fill_buffer(buf, size, filled))) { - TRANS_LOG(WARN, "serialize 2pc msg fail", K(ret), K(size), K(msg_type)); - ob_free(buf); - buf = NULL; - } else { - const int32_t data_size = static_cast(filled); - (void)ex_rpc::async_call([msg_type, buf, data_size]() { - int ret = OB_SUCCESS; - MOD_SCOPE { - transaction::ObTransService *txs = share::g_mp->trans_service(); - if (OB_ISNULL(txs)) { - ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "get tx service fail", K(ret)); - } else if (OB_FAIL(txs->handle_tx_batch_req(msg_type, buf, data_size))) { - TRANS_LOG(WARN, "handle 2pc msg fail", K(ret), K(msg_type)); - } - } - ob_free(buf); - }); - } -} -} // anonymous namespace - -namespace transaction -{ -int ObTransRpc::init(ObTransService *trans_service, - const common::ObAddr &self) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - TRANS_LOG(WARN, "ObTransRpc inited twice"); - ret = OB_INIT_TWICE; - } else if (OB_ISNULL(trans_service) - || !self.is_valid()) { - TRANS_LOG(WARN, "invalid argument", KP(trans_service), K(self)); - ret = OB_INVALID_ARGUMENT; - } else { - - trans_service_ = trans_service; - last_stat_ts_ = ObTimeUtility::current_time(); - is_inited_ = true; - TRANS_LOG(INFO, "transaction rpc inited success"); - } - return ret; -} - -int ObTransRpc::start() -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - TRANS_LOG(WARN, "ObTransRpc is not inited"); - ret = OB_NOT_INIT; - } else if (is_running_) { - TRANS_LOG(WARN, "ObTransRpc is already running"); - ret = OB_ERR_UNEXPECTED; - } else { - is_running_ = true; - TRANS_LOG(INFO, "ObTransRpc start success"); - } - - return ret; -} - -void ObTransRpc::stop() -{ - if (!is_inited_) { - TRANS_LOG_RET(WARN, OB_NOT_INIT, "ObTransRpc is not inited"); - } else if (!is_running_) { - TRANS_LOG_RET(WARN, OB_IN_STOP_STATE, "ObTransRpc already has been stopped"); - } else { - is_running_ = false; - TRANS_LOG(INFO, "ObTransRpc stop success"); - } -} - -void ObTransRpc::wait() -{ - if (!is_inited_) { - TRANS_LOG_RET(WARN, OB_NOT_INIT, "ObTransRpc is not inited"); - } else if (is_running_) { - TRANS_LOG_RET(WARN, OB_IN_STOP_STATE, "ObTransRpc is already running"); - } else { - TRANS_LOG(INFO, "ObTransRpc wait success"); - } -} - -void ObTransRpc::destroy() -{ - if (is_inited_) { - if (is_running_) { - stop(); - wait(); - } - is_inited_ = false; - trans_service_ = NULL; - TRANS_LOG(INFO, "transaction rpc destroyed"); - } -} -int ObTransRpc::post_commit_msg_(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int64_t msg_type = msg.get_msg_type(); - - switch (msg_type) - { - case TX_COMMIT: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_commit_request); - break; - } - case TX_COMMIT_RESP: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_commit_response); - break; - } - case TX_ABORT: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_abort_request); - break; - } - default: - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "rpc proxy not supported", K(server), K(msg)); - break; - } - return ret; -} - -int ObTransRpc::post_(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int64_t msg_type = msg.get_msg_type(); - - switch (msg_type) - { - case ROLLBACK_SAVEPOINT: - { - dispatch_rollback_sp_async_(static_cast(msg)); - break; - } - case ROLLBACK_SAVEPOINT_RESP: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sp_rollback_response); - break; - } - case KEEPALIVE: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_keepalive); - break; - } - case KEEPALIVE_RESP: - { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_keepalive_response); - break; - } - case TX_COMMIT: - case TX_COMMIT_RESP: - case TX_ABORT: - { - // Why we shoud set a new mehtod : post_commit_msg ? - // Method stack size is overflow (max size = 10KB) because of rpc_proxy deep_copy - ret = post_commit_msg_(server, msg); - break; - } - case SUBPREPARE: - case SUBCOMMIT: - case SUBROLLBACK: - { - ret = post_sub_request_msg_(server, msg); - break; - } - case SUBPREPARE_RESP: - case SUBCOMMIT_RESP: - case SUBROLLBACK_RESP: - { - ret = post_sub_response_msg_(server, msg); - break; - } - case ASK_STATE: - case ASK_STATE_RESP: - case COLLECT_STATE: - case COLLECT_STATE_RESP: - { - ret = post_standby_msg_(server, msg); - break; - } - default: - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "rpc proxy not supported", K(server), K(msg)); - break; - } - return ret; -} - -int ObTransRpc::post_msg(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - - - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "ObTransRpc not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransRpc is not running"); - ret = OB_NOT_RUNNING; - } else if (OB_UNLIKELY(!true) || - OB_UNLIKELY(!server.is_valid()) || OB_UNLIKELY(!msg.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(server), K(msg)); - ret = OB_INVALID_ARGUMENT; - } else if (ObTxMsgTypeChecker::is_2pc_msg_type(msg.get_msg_type())) { - dispatch_tx_2pc_async_(msg); - } else if (OB_FAIL(post_(server, msg))) { - TRANS_LOG(WARN, "post msg error", K(ret), K(server), K(msg)); - } else { - // do nothing - } - - if (OB_SUCC(ret)) { - total_trans_msg_count_++; - statistics_(); - TRANS_LOG(DEBUG, "post transaction message success", K(msg)); - } - - return ret; -} - -int ObTransRpc::post_msg(const ObLSID &ls_id, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; -#ifdef TRANS_ERROR - const int64_t random = ObRandom::rand(1, 100); - if (0 == random % 20) { - //mock package drop: 5% - TRANS_LOG(INFO, "post trans msg failed for random error (discard msg)", K(server), K(msg)); - return ret; - } else if (0 == random % 50) { - TRANS_LOG(INFO, "post trans msg failed for random error (delayed msg)", K(server), K(msg)); - } else { - // do nothing - } -#endif - - - int64_t cluster_id = GCONF.cluster_id; - ObAddr server; - - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "ObTransRpc not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransRpc is not running"); - ret = OB_NOT_RUNNING; - } else if (OB_UNLIKELY(!true) || OB_UNLIKELY(!msg.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(msg)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(trans_service_->get_location_adapter()->nonblock_get_leader(cluster_id, ls_id, server))) { - TRANS_LOG(WARN, "get leader failed", KR(ret), K(msg), K(cluster_id), K(ls_id)); - if (ObTxMsgTypeChecker::is_2pc_msg_type(msg.get_msg_type())) { - if (OB_LS_IS_DELETED == ret) { - int tmp_ret = trans_service_->handle_ls_deleted(msg); - if (OB_SUCCESS == tmp_ret) { - ret = OB_SUCCESS; - } - } - } - } else if (ObTxMsgTypeChecker::is_2pc_msg_type(msg.get_msg_type())) { - // 2pc msg: in-process async dispatch (single-replica leader is local) - dispatch_tx_2pc_async_(msg); - } else if (OB_FAIL(post_(server, msg))) { - TRANS_LOG(WARN, "post msg error", K(ret), K(server), K(msg)); - } else { - // do nothing - } - - if (OB_SUCC(ret)) { - total_trans_msg_count_++; - statistics_(); - TRANS_LOG(DEBUG, "post transaction message success", K(msg)); - } - - return ret; -} - -int ObTransRpc::post_sub_request_msg_(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int64_t msg_type = msg.get_msg_type(); - - switch (msg_type) { - case SUBPREPARE: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_prepare_request); - break; - } - case SUBCOMMIT: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_commit_request); - break; - } - case SUBROLLBACK: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_rollback_request); - break; - } - default: { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "rpc proxy not supported", K(server), K(msg)); - break; - } - } - return ret; -} - -int ObTransRpc::post_sub_response_msg_(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int64_t msg_type = msg.get_msg_type(); - - switch (msg_type) { - case SUBPREPARE_RESP: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_prepare_response); - break; - } - case SUBCOMMIT_RESP: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_commit_response); - break; - } - case SUBROLLBACK_RESP: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_sub_rollback_response); - break; - } - default: { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "rpc proxy not supported", K(server), K(msg)); - break; - } - } - return ret; -} - -int ObTransRpc::ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - ObAskTxStateFor4377RespMsg &resp) -{ - int ret = OB_SUCCESS; - - - int64_t cluster_id = GCONF.cluster_id; - ObAddr server; - - if (OB_UNLIKELY(!is_inited_)) { - TRANS_LOG(WARN, "ObTransRpc not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransRpc is not running"); - ret = OB_NOT_RUNNING; - } else if (OB_UNLIKELY(!true) - || OB_UNLIKELY(!msg.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(msg)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(trans_service_->get_location_adapter()->nonblock_get_leader(cluster_id, - msg.ls_id_, - server))) { - TRANS_LOG(WARN, "get leader failed", KR(ret), K(msg), K(cluster_id)); - } else { - // single-replica: the target LS leader is local, dispatch in-process - // (mirrors ObAskTxStateFor4377P::process: the handler status is carried back - // in resp.ret_, the call itself returns the transport code OB_SUCCESS). - ret = ex_rpc::sync_call([this, &msg, &resp]() -> int { - int ret = OB_SUCCESS; - bool is_alive = false; - if (OB_FAIL(trans_service_->handle_ask_tx_state_for_4377(msg, is_alive))) { - TRANS_LOG(WARN, "handle ask tx state for 4377 failed", K(ret), K(msg)); - } - resp.is_alive_ = is_alive; - resp.ret_ = ret; - return OB_SUCCESS; - }); - TRANS_LOG(WARN, "ask tx state for 4377 finished", KR(ret), K(msg), K(cluster_id)); - } - - return ret; -} - -int ObTransRpc::post_standby_msg_(const ObAddr &server, ObTxMsg &msg) -{ - int ret = OB_SUCCESS; - const int64_t msg_type = msg.get_msg_type(); - - switch (msg_type) { - case ASK_STATE: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_ask_state); - break; - } - case ASK_STATE_RESP: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_ask_state_response); - break; - } - case COLLECT_STATE: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_collect_state); - break; - } - case COLLECT_STATE_RESP: { - dispatch_tx_msg_async_(server, static_cast(msg), - &ObTransService::handle_trans_collect_state_response); - break; - } - default: { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "rpc proxy not supported", K(server), K(msg)); - break; - } - } - return ret; -} - -void ObTransRpc::statistics_() -{ - const int64_t cur_ts = ObTimeUtility::current_time(); - if (cur_ts - last_stat_ts_ > STAT_INTERVAL) { - TRANS_LOG(INFO, "rpc statistics", K_(total_trans_msg_count)); - total_trans_msg_count_ = 0; - last_stat_ts_ = cur_ts; - } -} - - -} // transaction - -} // oceanbase diff --git a/src/storage/tx/ob_trans_rpc.h b/src/storage/tx/ob_trans_rpc.h deleted file mode 100644 index 97c5f391c..000000000 --- a/src/storage/tx/ob_trans_rpc.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TRANS_RPC_ -#define OCEANBASE_TRANSACTION_OB_TRANS_RPC_ - -#include "lib/thread/ob_queue_thread.h" -#include "lib/utility/ob_unify_serialize.h" -#include "lib/utility/utility.h" -#include "share/ob_define.h" -#include "rpc/frame/ob_result_code.h" -#include "rpc/frame/ob_req_transport.h" -#include "share/ob_rpc_struct.h" -#include "ob_trans_define.h" -#include "ob_trans_factory.h" -#include "ob_tx_msg.h" -#include "share/config/ob_server_config.h" -#include "observer/ob_server_struct.h" - -namespace oceanbase -{ -namespace transaction -{ -class ObTransService; -class ObTxMsg; -} - -namespace obcall -{ -class ObTransRpcResult -{ - OB_UNIS_VERSION(1); -public: - ObTransRpcResult() - { reset(); } - virtual ~ObTransRpcResult() {} - - void init(const int status, const int64_t timestamp); - int get_status() const { return status_; } - int64_t get_timestamp() const { return send_timestamp_; } - void reset(); - TO_STRING_KV(K_(status), K_(send_timestamp), K_(private_data)); -private: - int status_; - int64_t send_timestamp_; -public: - // for ObTxCommitReqMsg, it is commit version - share::SCN private_data_; -}; - -struct ObTxRpcRollbackSPResult -{ - OB_UNIS_VERSION(1); -public: - ObTxRpcRollbackSPResult(): ignore_(false) {} - int status_; - int64_t send_timestamp_; - int64_t born_epoch_; - ObAddr addr_; - // rollback response has changed to use ObTxRollbackSPRespMsg - // use this field to indicate handler ignore handle by this msg - bool ignore_; -public: - int get_status() const { return status_; } - TO_STRING_KV(K_(status), K_(send_timestamp), K_(born_epoch), K_(addr), - K_(ignore)); -}; - -// publich method -bool need_refresh_location_cache_(const int status); -int refresh_location_cache(const share::ObLSID ls); -int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, - const share::ObLSID &receiver_ls_id, - const transaction::ObTransID &tx_id, - const int16_t msg_type, - const int status, - const ObAddr &receiver_addr, - const int64_t request_id, - const share::SCN &private_data); - -int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, - const int64_t epoch, - const transaction::ObTransID &tx_id, - const int status, - const int64_t request_id, - const ObTxRpcRollbackSPResult &result); -} // obcall - -namespace transaction -{ - -class ObITransRpc -{ -public: - ObITransRpc() {} - virtual ~ObITransRpc() {} - virtual int start() = 0; - virtual void stop() = 0; - virtual void wait() = 0; - virtual void destroy() = 0; -public: - virtual int post_msg(const ObAddr &server, ObTxMsg &msg) = 0; - virtual int post_msg(const share::ObLSID &p, ObTxMsg &msg) = 0; - virtual int ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - ObAskTxStateFor4377RespMsg &resp) = 0; - -}; - -/* - * transaction msg rpc class - * - * single-replica: every transaction message (including 2PC) is delivered - * in-process and async via ex_rpc::async_call; there is no inter-server - * transport. - */ -class ObTransRpc : public ObITransRpc -{ -public: - ObTransRpc() : is_inited_(false), - is_running_(false), - trans_service_(NULL), - total_trans_msg_count_(0), - last_stat_ts_(0) {} - ~ObTransRpc() { destroy(); } - int init(ObTransService *trans_service, - const common::ObAddr &self); - int start(); - void stop(); - void wait(); - void destroy(); -public: - int post_msg(const ObAddr &server, ObTxMsg &msg); - int post_msg(const share::ObLSID &p, ObTxMsg &msg); - int ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - ObAskTxStateFor4377RespMsg &resp); - private: - int post_(const ObAddr &server, ObTxMsg &msg); - int post_commit_msg_(const ObAddr &server, ObTxMsg &msg); - int post_sub_request_msg_(const ObAddr &server, ObTxMsg &msg); - int post_sub_response_msg_(const ObAddr &server, ObTxMsg &msg); - int post_standby_msg_(const ObAddr &server, ObTxMsg &msg); - void statistics_(); -private: - static const int64_t STAT_INTERVAL = 1 * 1000 * 1000; - bool is_inited_; - bool is_running_; - // common info - - ObTransService *trans_service_; - // statistic info - int64_t total_trans_msg_count_ CACHE_ALIGNED; - int64_t last_stat_ts_ CACHE_ALIGNED; -}; - -} // transaction - -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_TRANS_RPC_ diff --git a/src/storage/tx/ob_trans_service.cpp b/src/storage/tx/ob_trans_service.cpp index 00f3e9666..d9c002c99 100644 --- a/src/storage/tx/ob_trans_service.cpp +++ b/src/storage/tx/ob_trans_service.cpp @@ -22,7 +22,6 @@ #include "ob_trans_functor.h" #include "storage/tx_storage/ob_ls_service.h" #include "observer/ob_srv_network_frame.h" -#include "storage/tx/ob_tx_result_struct.h" namespace oceanbase { @@ -41,9 +40,6 @@ namespace transaction ObTransService::ObTransService() : is_inited_(false), is_running_(false), - use_def_(true), - rpc_(&rpc_def_), - location_adapter_(&location_adapter_def_), schema_service_(NULL), ts_mgr_(NULL), input_queue_count_(0), @@ -62,17 +58,10 @@ int ObTransService::mtl_init(ObTransService *&it) { int ret = OB_SUCCESS; const ObAddr &self = GCTX.self_addr(); - share::ObLocationService *location_service = GCTX.location_service_; share::schema::ObMultiVersionSchemaService *schema_service = GCTX.schema_service_; - if (OB_FAIL(it->rpc_def_.init(it, self))) { - TRANS_LOG(ERROR, "rpc init error", KR(ret)); - } else if (OB_FAIL(it->location_adapter_def_.init(schema_service, location_service))) { - TRANS_LOG(ERROR, "location adapter init error", KR(ret)); - } else if (OB_FAIL(it->gti_source_def_.init(self))) { + if (OB_FAIL(it->gti_source_def_.init())) { TRANS_LOG(ERROR, "gti source init error", KR(ret)); } else if (OB_FAIL(it->init(self, - &it->rpc_def_, - &it->location_adapter_def_, &it->gti_source_def_, &OB_TS_MGR, schema_service))) { @@ -82,8 +71,6 @@ int ObTransService::mtl_init(ObTransService *&it) } int ObTransService::init(const ObAddr &self, - ObITransRpc *rpc, - ObILocationAdapter *location_adapter, ObIGtiSource *gti_source, ObTsMgr *ts_mgr, share::schema::ObMultiVersionSchemaService *schema_service) @@ -103,14 +90,11 @@ int ObTransService::init(const ObAddr &self, TRANS_LOG(WARN, "ObTransService inited twice", KPC(this)); ret = OB_INIT_TWICE; } else if (OB_UNLIKELY(!self.is_valid()) - || OB_ISNULL(rpc) - || OB_ISNULL(location_adapter) || OB_ISNULL(gti_source) || OB_ISNULL(ts_mgr) || OB_ISNULL(schema_service)) { TRANS_LOG(WARN, "invalid argument", K(self), - KP(location_adapter), KP(rpc), - KP(location_adapter), KP(ts_mgr), + KP(gti_source), KP(ts_mgr), KP(schema_service)); ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(timer_.init("TransTimeWheel"))) { @@ -123,23 +107,18 @@ int ObTransService::init(const ObAddr &self, TRANS_LOG(WARN, "ObTxDescMgr init error", K(ret)); } else if (OB_FAIL(tx_ctx_mgr_.init(ts_mgr, this))) { TRANS_LOG(WARN, "tx_ctx_mgr_ init error", KR(ret)); - } else if (OB_FAIL(rollback_sp_msg_mgr_.init(lib::ObMemAttr("RollbackSPMgr")))) { - TRANS_LOG(WARN, "init rollback msg map failed", KR(ret)); - } else if (OB_FAIL(tablet_to_ls_cache_.init(&tx_ctx_mgr_))) { - TRANS_LOG(WARN, "init tablet to ls cache failed", K(ret)); + } else if (OB_FAIL(tx_timestamp_waiter_.init(ts_mgr))) { + TRANS_LOG(WARN, "tx timestamp waiter init error", KR(ret)); } else if (OB_FAIL(read_only_checker_.init())) { TRANS_LOG(WARN, "read only checker init failed", K(ret)); } else { self_ = self; gti_source_ = gti_source; - rpc_ = rpc; - location_adapter_ = location_adapter; schema_service_ = schema_service; ts_mgr_ = ts_mgr; - rollback_sp_msg_sequence_ = ObTimeUtil::current_time(); is_inited_ = true; - TRANS_LOG(INFO, "transaction service inited success", KPC(this), K(tenant_memory_limit), K_(tablet_to_ls_cache)); + TRANS_LOG(INFO, "transaction service inited success", KPC(this), K(tenant_memory_limit)); } if (OB_SUCC(ret)) { #ifdef ENABLE_DEBUG_LOG @@ -180,12 +159,12 @@ int ObTransService::start() ret = OB_ERR_UNEXPECTED; } else if (OB_FAIL(timer_.start())) { TRANS_LOG(WARN, "ObTransTimer start error", K(ret)); - } else if (OB_FAIL(rpc_->start())) { - TRANS_LOG(WARN, "ObTransRpc start error", KR(ret)); } else if (OB_FAIL(gti_source_->start())) { TRANS_LOG(WARN, "ObGtiSource start error", KR(ret)); } else if (OB_FAIL(tx_ctx_mgr_.start())) { TRANS_LOG(WARN, "tx_ctx_mgr_ start error", KR(ret)); + } else if (OB_FAIL(tx_timestamp_waiter_.start())) { + TRANS_LOG(WARN, "tx timestamp waiter start error", KR(ret)); } else if (OB_FAIL(tx_desc_mgr_.start())) { TRANS_LOG(WARN, "tx_desc_mgr_ start error", KR(ret)); } else { @@ -213,10 +192,8 @@ void ObTransService::stop() TRANS_LOG(WARN, "tx_desc_mgr stop error", KR(ret)); } else if (OB_FAIL(timer_.stop())) { TRANS_LOG(WARN, "ObTransTimer stop error", K(ret)); - } else if (OB_FAIL(ts_mgr_->interrupt_gts_callbacks())) { - TRANS_LOG(WARN, "gts_mgr stop error", K(ret)); } else { - rpc_->stop(); + tx_timestamp_waiter_.stop(); gti_source_->stop(); ObLinkQueueThreadPool::stop(); is_running_ = false; @@ -234,16 +211,18 @@ int ObTransService::wait_() } else if (OB_UNLIKELY(is_running_)) { TRANS_LOG(WARN, "ObTransService is running"); ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(tx_ctx_mgr_.wait())) { - TRANS_LOG(WARN, "tx_ctx_mgr_ wait error", KR(ret)); - } else if (OB_FAIL(tx_desc_mgr_.wait())) { - TRANS_LOG(WARN, "tx_desc_mgr_ wait error", KR(ret)); - } else if (OB_FAIL(timer_.wait())) { - TRANS_LOG(WARN, "ObTransTimer wait error", K(ret)); } else { - rpc_->wait(); - gti_source_->wait(); - TRANS_LOG(INFO, "transaction service wait success", KPC(this)); + ObLinkQueueThreadPool::wait(); + if (OB_FAIL(tx_ctx_mgr_.wait())) { + TRANS_LOG(WARN, "tx_ctx_mgr_ wait error", KR(ret)); + } else if (OB_FAIL(tx_desc_mgr_.wait())) { + TRANS_LOG(WARN, "tx_desc_mgr_ wait error", KR(ret)); + } else if (OB_FAIL(timer_.wait())) { + TRANS_LOG(WARN, "ObTransTimer wait error", K(ret)); + } else { + gti_source_->wait(); + TRANS_LOG(INFO, "transaction service wait success", KPC(this)); + } } return ret; } @@ -256,13 +235,8 @@ void ObTransService::destroy() wait(); } timer_.destroy(); - if (use_def_) { - rpc_->destroy(); - location_adapter_->destroy(); - use_def_ = false; - } gti_source_->destroy(); - tablet_to_ls_cache_.destroy(); + tx_timestamp_waiter_.destroy(); tx_ctx_mgr_.destroy(); tx_desc_mgr_.destroy(); #ifdef ENABLE_DEBUG_LOG @@ -273,17 +247,17 @@ void ObTransService::destroy() } #endif is_inited_ = false; - TRANS_LOG(INFO, "transaction service destroyed", KPC(this), K_(tablet_to_ls_cache)); + TRANS_LOG(INFO, "transaction service destroyed", KPC(this)); } } -int ObTransService::get_trans_start_session_id(const share::ObLSID &ls_id, const ObTransID &tx_id, uint32_t &session_id) +int ObTransService::get_trans_start_session_id(const ObTransID &tx_id, uint32_t &session_id) { int ret = OB_SUCCESS; - transaction::ObPartTransCtx *part_ctx = nullptr; - ObLSHandle ls_handle; + transaction::ObTxCtx *part_ctx = nullptr; + ObLS *tenant_ls = nullptr; session_id = ObBasicSessionInfo::INVALID_SESSID; if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObTransService not inited"); @@ -291,13 +265,10 @@ int ObTransService::get_trans_start_session_id(const share::ObLSID &ls_id, const } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTransService is not running"); ret = OB_NOT_RUNNING; - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get ls failed", K(ret), K(ls_id), K(tx_id)); - } else if (!ls_handle.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "ls is null in ObLSHandle", K(ret), K(ls_id), K(tx_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tx_start_session_id(tx_id, session_id))) { - TRANS_LOG(WARN, "get ObPartTransCtx by ls_id and tx_id failed", K(ls_id), K(tx_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + TRANS_LOG(WARN, "get transaction storage failed", K(ret), K(tx_id)); + } else if (OB_FAIL(tenant_ls->get_tx_start_session_id(tx_id, session_id))) { + TRANS_LOG(WARN, "get ObTxCtx by tx_id failed", K(tx_id)); } return ret; } @@ -361,18 +332,6 @@ void ObTransService::handle(LinkTask *task) mtl_free(advance_ckpt_task); advance_ckpt_task = nullptr; } - } else if (ObTransRetryTaskType::STANDBY_CLEANUP_TASK == trans_task->get_task_type()) { - ObTxStandbyCleanupTask *standby_cleanup_task = static_cast(trans_task); - if (OB_ISNULL(standby_cleanup_task)) { - // ignore ret - TRANS_LOG(WARN, "standby cleanup task is null"); - } else if (OB_FAIL(do_standby_cleanup())) { - TRANS_LOG(WARN, "do standby cleanup failed", K(ret)); - } - if (OB_NOT_NULL(standby_cleanup_task)) { - mtl_free(standby_cleanup_task); - standby_cleanup_task = nullptr; - } } else { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexpected trans task type!!!", KR(ret), K(*trans_task)); @@ -392,7 +351,7 @@ void ObTransService::handle(LinkTask *task) UNUSED(ret); //make compiler happy } -int ObTransService::get_ls_min_uncommit_prepare_version(const ObLSID &ls_id, SCN &min_prepare_version) +int ObTransService::get_min_uncommit_prepare_version(SCN &min_prepare_version) { int ret = OB_SUCCESS; @@ -402,48 +361,19 @@ int ObTransService::get_ls_min_uncommit_prepare_version(const ObLSID &ls_id, SCN } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTransService is not running"); ret = OB_NOT_RUNNING; - } else if (!ls_id.is_valid()) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(tx_ctx_mgr_.get_ls_min_uncommit_tx_prepare_version(ls_id, min_prepare_version))) { - TRANS_LOG(WARN, "ObPartTransCtxMgr set memstore version error", KR(ret), K(ls_id)); + } else if (OB_FAIL(tx_ctx_mgr_.get_min_uncommit_tx_prepare_version(min_prepare_version))) { + TRANS_LOG(WARN, "ObTxCtxMgr set memstore version error", KR(ret)); } else if (!min_prepare_version.is_valid()) { - TRANS_LOG(ERROR, "invalid min prepare version, unexpected error", K(ls_id), K(min_prepare_version)); + TRANS_LOG(ERROR, "invalid min prepare version, unexpected error", K(min_prepare_version)); ret = OB_ERR_UNEXPECTED; } else { - TRANS_LOG(DEBUG, "get min uncommit prepare version success", K(ls_id), K(min_prepare_version)); + TRANS_LOG(DEBUG, "get min uncommit prepare version success", K(min_prepare_version)); } return ret; } -int ObTransService::get_max_decided_scn(const share::ObLSID &ls_id, share::SCN &scn) -{ - int ret = OB_SUCCESS; - - ObLSTxCtxMgr *ls_tx_mgr_ptr = nullptr; - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTransService not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransService is not running"); - ret = OB_NOT_RUNNING; - } else if (!ls_id.is_valid()) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id, ls_tx_mgr_ptr))) { - TRANS_LOG(WARN, "get ls tx ctx mgr failed", K(ret)); - } else { - if (OB_FAIL(ls_tx_mgr_ptr->get_max_decided_scn(scn))) { - TRANS_LOG(WARN, "get max decided scn failed", K(ret)); - } - tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_mgr_ptr); - } - return ret; -} - -int ObTransService::remove_callback_for_uncommited_txn( - const ObLSID ls_id, const memtable::ObMemtableSet *memtable_set) +int ObTransService::remove_callback_for_uncommited_txn(const memtable::ObMemtableSet *memtable_set) { int ret = OB_SUCCESS; @@ -455,13 +385,10 @@ int ObTransService::remove_callback_for_uncommited_txn( } else if (OB_ISNULL(memtable_set)) { TRANS_LOG(WARN, "memtable is NULL"); ret = OB_INVALID_ARGUMENT; - } else if (!ls_id.is_valid()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected ls id", KR(ret), K(ls_id), KPC(memtable_set)); - } else if (OB_FAIL(tx_ctx_mgr_.remove_callback_for_uncommited_tx(ls_id, memtable_set))) { - TRANS_LOG(WARN, "participant remove callback for uncommitt txn failed", KR(ret), K(ls_id), KP(memtable_set)); + } else if (OB_FAIL(tx_ctx_mgr_.remove_callback_for_uncommited_tx(memtable_set))) { + TRANS_LOG(WARN, "participant remove callback for uncommitt txn failed", KR(ret), KP(memtable_set)); } else { - TRANS_LOG(DEBUG, "participant remove callback for uncommitt txn success", K(ls_id), KP(memtable_set)); + TRANS_LOG(DEBUG, "participant remove callback for uncommitt txn success", KP(memtable_set)); } return ret; @@ -486,30 +413,14 @@ int ObTransService::remove_callback_for_uncommited_txn( // only for ob_admin use int ObTransService::register_mds_into_tx(ObTxDesc &tx_desc, - const ObLSID &ls_id, const ObTxDataSourceType &type, const char *buf, const int64_t buf_len, - const int64_t request_id, const ObRegisterMdsFlag ®ister_flag, transaction::ObTxSEQ seq_no) { - const int64_t MAX_RETRY_CNT = 5; - const int64_t RETRY_INTERVAL = 400 * 1000; - - ObTxPrintTimeGuard tx_print_guard; - // ObTimeGuard time_guard("register mds", 1 * 1000 * 1000); - int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObString str; - obcall::ObRegisterTxDataArg arg; - obcall::ObRegisterTxDataResult result; - ObAddr ls_leader_addr; - str.assign_ptr(buf, buf_len); - int64_t local_retry_cnt = 0; - int64_t retry_cnt = 0; - int64_t remain_timeout_us = 0; ObTxExecResult tx_result; ObTxParam tx_param; tx_param.cluster_id_ = tx_desc.cluster_id_; @@ -517,12 +428,11 @@ int ObTransService::register_mds_into_tx(ObTxDesc &tx_desc, tx_param.isolation_ = tx_desc.isolation_; tx_param.timeout_us_ = tx_desc.timeout_us_; ObTxSEQ savepoint; - - if (OB_UNLIKELY(!tx_desc.is_valid() || !ls_id.is_valid() || type <= ObTxDataSourceType::UNKNOWN - || type >= ObTxDataSourceType::MAX_TYPE || OB_ISNULL(buf) || buf_len < 0)) { + + if (OB_UNLIKELY(!tx_desc.is_valid() || type <= ObTxDataSourceType::UNKNOWN + || type >= ObTxDataSourceType::MAX_TYPE || OB_ISNULL(buf) || buf_len <= 0)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_desc), K(ls_id), K(type), KP(buf), - K(buf_len)); + TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_desc), K(type), KP(buf), K(buf_len)); } else if (!tx_desc.is_tx_active()) { ret = OB_TRANS_IS_EXITING; TRANS_LOG(WARN, "txn must in active for register", K(ret)); @@ -531,117 +441,39 @@ int ObTransService::register_mds_into_tx(ObTxDesc &tx_desc, } else if (!seq_no.is_valid()) { seq_no = tx_desc.inc_and_get_tx_seq(0); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(arg.init(tx_desc, - ls_id, - type, - str, - seq_no, - request_id, - register_flag))) { - TRANS_LOG(WARN, "rpc arg init failed", KR(ret), K(tx_desc), K(ls_id), K(type)); - } else { - tx_print_guard.click_start("total register", 0); - do { - result.reset(); - tx_result.reset(); - if (OB_NOT_MASTER == ret) { - ob_usleep(RETRY_INTERVAL); - retry_cnt += 1; - } - if (ObTimeUtil::current_time() >= tx_desc.expire_ts_) { + if (OB_SUCC(ret)) { + do { + ret = register_mds_into_ctx_(tx_desc, type, buf, buf_len, seq_no, register_flag); + if (OB_EAGAIN == ret && ObTimeUtil::current_time() >= tx_desc.expire_ts_) { ret = OB_TIMEOUT; - TRANS_LOG(WARN, "register tx data timeout", KR(ret), K(tx_desc), K(ls_id), K(type), - K(retry_cnt)); - } else if (OB_ISNULL(location_adapter_) - || OB_FAIL(location_adapter_->nonblock_get_leader( - tx_desc.cluster_id_, ls_id, ls_leader_addr))) { - TRANS_LOG(WARN, "get leader failed", KR(ret), K(ls_id)); - } else if (ls_leader_addr == self_) { - local_retry_cnt = 0; - - tx_print_guard.click_start("register_in_ctx", 1); - do { - if (OB_FAIL(register_mds_into_ctx_(*(arg.tx_desc_), ls_id, type, buf, buf_len, seq_no, register_flag))) { - TRANS_LOG(WARN, "register msd into ctx failed", K(ret)); - if (OB_EAGAIN == ret) { - if (ObTimeUtil::current_time() >= tx_desc.expire_ts_) { - ret = OB_TIMEOUT; - TRANS_LOG(WARN, "register tx data timeout in this participant", KR(ret), K(tx_desc), - K(ls_id), K(type), K(retry_cnt), K(local_retry_cnt)); - } else if (local_retry_cnt > MAX_RETRY_CNT) { - ret = OB_NOT_MASTER; - TRANS_LOG(WARN, "local retry too many times, need retry by the scheduler", K(ret), - K(local_retry_cnt), K(retry_cnt)); - } else { - local_retry_cnt++; - } - } - } - } while (OB_EAGAIN == ret); - tx_print_guard.click_end(1); - - // collect participants regardless of register error - if (OB_TMP_FAIL(collect_tx_exec_result(*(arg.tx_desc_), result.tx_result_))) { - TRANS_LOG(WARN, "collect tx exec result failed", K(ret)); - } else if (OB_TMP_FAIL(tx_result.merge_result(result.tx_result_))) { - TRANS_LOG(WARN, "merge tx result failed", K(ret), K(result)); - } - - if (OB_TMP_FAIL(tmp_ret)) { - if (OB_SUCC(ret)) { - ret = tmp_ret; - } - TRANS_LOG(ERROR, "set exec result failed when register mds", K(ret), K(tmp_ret), - K(*(arg.tx_desc_))); - } - } else if (this->self_ != tx_desc.addr_) { - ret = OB_NOT_MASTER; - TRANS_LOG(INFO, - "The follower receive a register request. we will return err_code to scheduler", - K(ret), K(tx_desc), K(ls_id), K(type), K(buf_len), K(request_id)); - } else if (OB_FALSE_IT(arg.inc_request_id(-1))) { - } else if (OB_FALSE_IT(tx_print_guard.click_start("register_by_rpc", 2))) { - } else if (OB_FALSE_IT(remain_timeout_us = tx_desc.expire_ts_ - ObTimeUtil::fast_current_time())) { - } else if (OB_FAIL(OB_NOT_SUPPORTED /*single-replica: LS leader is local; remote register path dead*/)) { - TRANS_LOG(WARN, "register_tx_fata failed", KR(ret), K(ls_leader_addr), K(arg), K(tx_desc), - K(ls_id), K(result)); - tx_print_guard.click_end(2); - } else if (OB_FALSE_IT(tx_print_guard.click_end(2))) { - } else if (OB_FAIL(result.result_)) { - TRANS_LOG(WARN, "register tx data failed in remote", KR(ret), K(tx_desc), K(ls_id), - K(type)); - } else if (OB_FAIL(tx_result.merge_result(result.tx_result_))) { - TRANS_LOG(WARN, "merge tx result failed", KR(ret), K(result)); + TRANS_LOG(WARN, "register tx data timeout", KR(ret), K(tx_desc), K(type)); + } else if (OB_EAGAIN == ret) { + ob_usleep(1000); } - } while (OB_NOT_MASTER == ret && this->self_ == tx_desc.addr_); + } while (OB_EAGAIN == ret); - tx_print_guard.click_start("handle_tx_result", 3); - if (OB_SUCC(ret)) { - if (OB_FAIL(add_tx_exec_result(tx_desc, tx_result))) { - TRANS_LOG(WARN, "add tx exec result failed", K(ret), K(tx_desc), K(tx_result)); - } + if (OB_TMP_FAIL(collect_tx_exec_result(tx_desc, tx_result))) { + TRANS_LOG(WARN, "collect tx exec result failed", K(tmp_ret), K(tx_desc)); } - if (OB_FAIL(ret) && OB_NOT_MASTER != ret) { - share::ObLSArray ls_list; - ls_list.push_back(ls_id); - int tmp_ret = OB_SUCCESS; - int64_t expire_ts = tx_desc.expire_ts_; // FIXME: use a rational expire_ts - if (OB_TMP_FAIL(rollback_to_implicit_savepoint(tx_desc, savepoint, expire_ts, &ls_list))) { - TRANS_LOG(WARN, "rollback to savepoint fail", K(tmp_ret), K(savepoint), K(expire_ts)); + if (OB_SUCC(ret) && OB_SUCCESS != tmp_ret) { + ret = tmp_ret; + } else if (OB_SUCC(ret) && OB_FAIL(add_tx_exec_result(tx_desc, tx_result))) { + TRANS_LOG(WARN, "add tx exec result failed", K(ret), K(tx_desc), K(tx_result)); + } + if (OB_FAIL(ret)) { + tmp_ret = rollback_to_implicit_savepoint(tx_desc, savepoint, tx_desc.expire_ts_, true); + if (OB_SUCCESS != tmp_ret) { + TRANS_LOG(WARN, "rollback to savepoint fail", K(tmp_ret), K(savepoint), K(tx_desc)); } } - tx_print_guard.click_end(3); } - TRANS_LOG(INFO, "register multi data source result", KR(ret), K(arg), K(result), K(tx_desc), - K(local_retry_cnt), K(retry_cnt), K(request_id), K(tx_print_guard)); + TRANS_LOG(INFO, "register multi data source result", KR(ret), K(tx_desc), K(type), K(seq_no)); return ret; } int ObTransService::register_mds_into_ctx_(ObTxDesc &tx_desc, - const ObLSID &ls_id, const ObTxDataSourceType &type, const char *buf, const int64_t buf_len, @@ -649,47 +481,47 @@ int ObTransService::register_mds_into_ctx_(ObTxDesc &tx_desc, const ObRegisterMdsFlag ®ister_flag) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *tenant_ls = nullptr; ObStoreCtx store_ctx; ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; write_flag.set_is_mds(); - if (OB_UNLIKELY(!tx_desc.is_valid() || !ls_id.is_valid() || OB_ISNULL(buf) || buf_len <= 0)) { + if (OB_UNLIKELY(!tx_desc.is_valid() || OB_ISNULL(buf) || buf_len <= 0)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_desc), K(ls_id), KP(buf), K(buf_len)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get ls handle fail", K(ret), K(ls_id)); - } else if (FALSE_IT(store_ctx.ls_id_ = ls_id)) { - } else if (FALSE_IT(store_ctx.ls_ = ls_handle.get_ls())) { - } else if (OB_FAIL(get_write_store_ctx(tx_desc, snapshot, write_flag, store_ctx, ObTxSEQ::INVL(), true))) { - TRANS_LOG(WARN, "get store ctx failed", KR(ret), K(tx_desc), K(ls_id)); + TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_desc), KP(buf), K(buf_len)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + TRANS_LOG(WARN, "get ls fail", K(ret)); } else { - ObPartTransCtx *ctx = store_ctx.mvcc_acc_ctx_.tx_ctx_; - ObMdsThrottleGuard mds_throttle_guard(ls_id, false /* for_replay */, ctx->get_trans_expired_time()); - if (OB_ISNULL(ctx)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected null ptr", KR(ret), K(tx_desc), K(ls_id), K(type)); - } else if (OB_FAIL(ctx->register_multi_data_source(type, - buf, - buf_len, - false /*try lock*/, - seq_no, - register_flag))) { - TRANS_LOG(WARN, "register multi source data failed", KR(ret), K(tx_desc), K(ls_id), K(type), K(register_flag)); - } else if (ObTxDataSourceType::DDL_TRANS == type) { - // Change Stream Fetcher filters by has_async_index; DDL logs must carry it. - // MDS is registered before commit, and commit block writes MDS before redo. - ctx->set_has_async_index_redo(); - } - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = revert_store_ctx(store_ctx))) { - TRANS_LOG(WARN, "revert store ctx failed", KR(tmp_ret), K(tx_desc), K(ls_id), K(type)); + store_ctx.ls_ = tenant_ls; + if (OB_FAIL(get_write_store_ctx( + tx_desc, snapshot, write_flag, store_ctx, ObTxSEQ::INVL(), true))) { + TRANS_LOG(WARN, "get store ctx failed", KR(ret), K(tx_desc)); } else { - store_ctx.reset(); + ObTxCtx *ctx = store_ctx.mvcc_acc_ctx_.tx_ctx_; + ObMdsThrottleGuard mds_throttle_guard( + false /* for_replay */, ctx->get_trans_expired_time()); + if (OB_FAIL(ctx->register_multi_data_source(type, + buf, + buf_len, + false /*try lock*/, + seq_no, + register_flag))) { + TRANS_LOG(WARN, "register multi source data failed", KR(ret), K(tx_desc), K(type), K(register_flag)); + } else if (ObTxDataSourceType::DDL_TRANS == type) { + // Change Stream Fetcher filters by has_async_index; DDL logs must carry it. + // MDS is registered before commit, and commit block writes MDS before redo. + ctx->set_has_async_index_redo(); + } + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = revert_store_ctx(store_ctx))) { + TRANS_LOG(WARN, "revert store ctx failed", KR(tmp_ret), K(tx_desc), K(type)); + } else { + store_ctx.reset(); + } } } - TRANS_LOG(DEBUG, "register multi source data on participant", KR(ret), K(tx_desc), K(ls_id), K(type)); + TRANS_LOG(DEBUG, "register multi source data", KR(ret), K(tx_desc), K(type)); return ret; } diff --git a/src/storage/tx/ob_trans_service.h b/src/storage/tx/ob_trans_service.h index 2f99495aa..73ce9f82d 100644 --- a/src/storage/tx/ob_trans_service.h +++ b/src/storage/tx/ob_trans_service.h @@ -29,21 +29,17 @@ #include "lib/utility/utility.h" #include "ob_trans_define.h" #include "ob_trans_timer.h" -#include "ob_location_adapter.h" -#include "ob_trans_rpc.h" +#include "ob_tx_timestamp_waiter.h" #include "ob_trans_ctx_mgr.h" #include "ob_trans_memory_stat.h" #include "ob_trans_event.h" -#include "ob_gts_rpc.h" #include "ob_gti_source.h" #include "ob_tx_version_mgr.h" -#include "ob_tx_standby_cleanup.h" #include "lib/utility/ob_tracepoint.h" #include "lib/container/ob_iarray.h" #include "observer/ob_server_struct.h" #include "common/storage/ob_sequence.h" #include "ob_tx_elr_util.h" -#include "ob_tablet_to_ls_cache.h" #include "src/storage/tx_storage/ob_tx_leak_checker.h" #define MAX_REDO_SYNC_TASK_COUNT 10 @@ -51,11 +47,6 @@ namespace oceanbase { -namespace obcall -{ -class ObTransRpcResult; -} - namespace common { class ObAddr; @@ -83,14 +74,13 @@ typedef common::ObSimpleIterator -{ -public: - ObRollbackSPMsgGuard(ObCommonID tx_msg_id, ObTxDesc &tx_desc, ObTxDescMgr &tx_desc_mgr) - : tx_msg_id_(tx_msg_id), tx_desc_(tx_desc), tx_desc_mgr_(tx_desc_mgr) { - tx_desc_.inc_ref(1); - } - ~ObRollbackSPMsgGuard() { - if (0 == tx_desc_.dec_ref(1)) { - tx_desc_mgr_.free(&tx_desc_); - } - tx_msg_id_.reset(); - } - ObTxDesc &get_tx_desc() { return tx_desc_; } - bool contain(ObCommonID tx_msg_id) { return tx_msg_id == tx_msg_id_; } -private: - ObCommonID tx_msg_id_; - ObTxDesc &tx_desc_; - ObTxDescMgr &tx_desc_mgr_; -}; - -class ObRollbackSPMsgGuardAlloc -{ -public: - static ObRollbackSPMsgGuard* alloc_value() - { - return (ObRollbackSPMsgGuard*)ob_malloc(sizeof(ObRollbackSPMsgGuard), "RollbackSPMsg"); - } - static void free_value(ObRollbackSPMsgGuard *p) - { - if (NULL != p) { - p->~ObRollbackSPMsgGuard(); - ob_free(p); - p = NULL; - } - } -}; - class ObTransService : public common::ObLinkQueueThreadPool { public: @@ -163,8 +115,6 @@ class ObTransService : public common::ObLinkQueueThreadPool virtual ~ObTransService() { destroy(); } static int mtl_init(ObTransService* &trans_service); int init(const ObAddr &self, - ObITransRpc *rpc, - ObILocationAdapter *location_adapter, ObIGtiSource *gti_source, ObTsMgr *ts_mgr, share::schema::ObMultiVersionSchemaService *schema_service); @@ -179,59 +129,33 @@ class ObTransService : public common::ObLinkQueueThreadPool ObReadOnlyTxChecker &get_read_tx_checker() { return read_only_checker_; } int64_t get_unique_seq() { return ATOMIC_AAF(&tx_debug_seq_, 1); } - int check_trans_partition_leader_unsafe(const share::ObLSID &ls_id, bool &is_leader); int calculate_trans_cost(const ObTransID &tid, uint64_t &cost); - int get_ls_min_uncommit_prepare_version(const share::ObLSID &ls_id, share::SCN &min_prepare_version); + int get_min_uncommit_prepare_version(share::SCN &min_prepare_version); //get the memory used condition of transaction module - int get_trans_start_session_id(const share::ObLSID &ls_id, const ObTransID &tx_id, uint32_t &session_id); - int remove_callback_for_uncommited_txn( - const share::ObLSID ls_id, - const memtable::ObMemtableSet *memtable_set); + int get_trans_start_session_id(const ObTransID &tx_id, uint32_t &session_id); + int remove_callback_for_uncommited_txn(const memtable::ObMemtableSet *memtable_set); const common::ObAddr &get_server() { return self_; } ObTransTimer &get_trans_timer() { return timer_; } - ObITransRpc *get_trans_rpc() { return rpc_; } - ObILocationAdapter *get_location_adapter() { return location_adapter_; } common::ObMySQLProxy *get_mysql_proxy() { return GCTX.sql_proxy_; } bool is_running() const { return is_running_; } ObTsMgr *get_ts_mgr() { return ts_mgr_; } + ObTxTimestampWaiter &get_tx_timestamp_waiter() { return tx_timestamp_waiter_; } share::schema::ObMultiVersionSchemaService *get_schema_service() { return schema_service_; } ObTxVersionMgr &get_tx_version_mgr() { return tx_version_mgr_; } int register_mds_into_tx(ObTxDesc &tx_desc, - const share::ObLSID &ls_id, const ObTxDataSourceType &type, const char *buf, const int64_t buf_len, - const int64_t request_id = 0, const ObRegisterMdsFlag ®ister_flag = ObRegisterMdsFlag(), const transaction::ObTxSEQ seq_no = transaction::ObTxSEQ()); ObTxELRUtil &get_tx_elr_util() { return elr_util_; } - int create_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id) - { - return tablet_to_ls_cache_.create_tablet(tablet_id, ls_id); - } - int remove_tablet(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id) - { - return tablet_to_ls_cache_.remove_tablet(tablet_id, ls_id); - } - int remove_tablet(const share::ObLSID &ls_id) - { - return tablet_to_ls_cache_.remove_ls_tablets(ls_id); - } - int check_and_get_ls_info(const common::ObTabletID &tablet_id, - share::ObLSID &ls_id, - bool &is_local_leader) - { - return tablet_to_ls_cache_.check_and_get_ls_info(tablet_id, ls_id, is_local_leader); - } #ifdef ENABLE_DEBUG_LOG transaction::ObDefensiveCheckMgr *get_defensive_check_mgr() { return defensive_check_mgr_; } #endif private: void check_env_(); - bool can_create_ctx_(const int64_t trx_start_ts, const common::ObTsWindows &changing_leader_windows); int register_mds_into_ctx_(ObTxDesc &tx_desc, - const share::ObLSID &ls_id, const ObTxDataSourceType &type, const char *buf, const int64_t buf_len, @@ -239,10 +163,8 @@ class ObTransService : public common::ObLinkQueueThreadPool const ObRegisterMdsFlag ®ister_flag); private: int handle_batch_msg_(const int type, const char *buf, const int32_t size); - int64_t fetch_rollback_sp_sequence_() { return ATOMIC_AAF(&rollback_sp_msg_sequence_, 1); } public: int get_max_commit_version(share::SCN &commit_version) const; - int get_max_decided_scn(const share::ObLSID &ls_id, share::SCN & scn); #include "ob_trans_service_v4.h" private: static const int64_t END_STMT_MORE_TIME_US = 100 * 1000; @@ -263,16 +185,11 @@ class ObTransService : public common::ObLinkQueueThreadPool // for ObTransID common::ObAddr self_; - ObTransRpc rpc_def_; - ObLocationAdapter location_adapter_def_; // transaction timer ObTransTimer timer_; + ObTxTimestampWaiter tx_timestamp_waiter_; ObTxVersionMgr tx_version_mgr_; protected: - bool use_def_; - ObITransRpc *rpc_; - // the adapter between transaction and location cache - ObILocationAdapter *location_adapter_; // the adapter between transaction and clog share::schema::ObMultiVersionSchemaService *schema_service_; private: @@ -283,17 +200,10 @@ class ObTransService : public common::ObLinkQueueThreadPool #ifdef ENABLE_DEBUG_LOG transaction::ObDefensiveCheckMgr *defensive_check_mgr_; #endif - // in order to pass the mittest, tablet_to_ls_cache_ must be declared before tx_desc_mgr_ - ObTabletToLSCache tablet_to_ls_cache_; // txDesc's manager ObTxDescMgr tx_desc_mgr_; ObTxELRUtil elr_util_; - // for rollback-savepoint request-id - int64_t rollback_sp_msg_sequence_; - // for rollback-savepoint msg resp callback to find tx_desc - share::ObLightHashMap rollback_sp_msg_mgr_; - // tenant level atomic inc seq, just for debug int64_t tx_debug_seq_; ObReadOnlyTxChecker read_only_checker_; diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 1f3cf8dc8..c7d9fd802 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "src/storage/tx/ob_trans_part_ctx.h" +#include "src/storage/tx/ob_tx_ctx.h" #include "share/rc/ob_module_provider.h" #include "observer/ob_server.h" #include "storage/tx_storage/ob_ls_service.h" @@ -28,8 +28,7 @@ using namespace share; static const int64_t POST_COMMIT_REQ_RETRY_INTERVAL = 100 * 1000; // 100msg -int ObTransService::create_ls(const share::ObLSID &ls_id, - ObLS &ls, +int ObTransService::create_ls(ObLS &ls, ObITxLogParam *param, ObITxLogAdapter *log_adapter) { @@ -42,32 +41,28 @@ int ObTransService::create_ls(const share::ObLSID &ls_id, } else if (OB_UNLIKELY(!is_running_)) { ret = OB_NOT_RUNNING; TRANS_LOG(WARN, "ObTransService is not running", K(ret), K(*this)); - } else if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); } else if (OB_ISNULL(tx_table = ls.get_tx_table())) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "get tx table fail", K(ret), K(ls_id)); - } else if (OB_FAIL(tx_ctx_mgr_.create_ls(ls_id, - tx_table, - ls.get_lock_table(), - *ls.get_tx_svr(), - param, - log_adapter))) { - TRANS_LOG(WARN, "create ls failed", K(ret), K(*this)); + TRANS_LOG(WARN, "get tx table fail", K(ret)); + } else if (OB_FAIL(tx_ctx_mgr_.create_context_manager(tx_table, + ls.get_lock_table(), + *ls.get_tx_svr(), + param, + log_adapter))) { + TRANS_LOG(WARN, "create transaction context manager failed", K(ret), K(*this)); } else { // do nothing } if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "create ls failed", K(ret), K(ls_id)); + TRANS_LOG(WARN, "create transaction context manager failed", K(ret)); } else { - TRANS_LOG(INFO, "create ls success", K(ls_id)); + TRANS_LOG(INFO, "create transaction context manager success"); } return ret; } -int ObTransService::remove_ls(const share::ObLSID &ls_id, const bool graceful) +int ObTransService::remove_ls(const bool graceful) { int ret = OB_SUCCESS; @@ -77,45 +72,15 @@ int ObTransService::remove_ls(const share::ObLSID &ls_id, const bool graceful) } else if (OB_UNLIKELY(!is_running_)) { ret = OB_NOT_RUNNING; TRANS_LOG(WARN, "ObTransService is not running", K(ret)); - } else if (!ls_id.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(tx_ctx_mgr_.remove_ls(ls_id, graceful))) { - TRANS_LOG(WARN, "participant remove ls_id error", K(ret), K(ls_id), K(graceful)); - } else { - TRANS_LOG(INFO, "remove ls success", K(ls_id), K(graceful)); - } - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "remove ls failed", K(ret), K(ls_id), K(graceful)); + } else if (OB_FAIL(tx_ctx_mgr_.remove_context_manager(graceful))) { + TRANS_LOG(WARN, "remove transaction context manager failed", K(ret), K(graceful)); } else { - TRANS_LOG(INFO, "remove ls success", K(ls_id), K(graceful)); + TRANS_LOG(INFO, "remove transaction context manager success", K(graceful)); } return ret; } -#ifdef CHECK_TX_PARTS_CONTAIN_ -#error "redefine CHECK_TX_PARTS_CONTAIN_" -#else -#define CHECK_TX_PARTS_CONTAIN_(parts, id, epoch, ls_id, ret_epoch, exist) \ - if (OB_SUCC(ret)) { \ - exist = false; \ - ARRAY_FOREACH_NORET(parts, idx) { \ - if (parts.at(idx).id == ls_id) { \ - exist = true; \ - if (ObTxPart::is_without_ctx(parts.at(idx).epoch)) { \ - /* target LS was dropped */ \ - /* can not accept access any more */ \ - ret = OB_PARTITION_IS_BLOCKED; \ - } else { \ - ret_epoch = parts.at(idx).epoch; \ - } \ - break; \ - } \ - } \ - } -#endif - int ObTransService::acquire_tx(const char* buf, const int64_t len, int64_t &pos, @@ -145,9 +110,9 @@ int ObTransService::acquire_tx(const char* buf, * do_commit_tx_ - the real work of commit tx * * steps: - * 1. decide coordinator + * 1. prepare commit info * 2. try local call optimization, if fail fallback to step 3 - * 3. post commit message to coordinator + * 3. post commit message to the transaction context * * If any failures occurred: * - if no message has been sent, state can be revert to @@ -157,9 +122,9 @@ int ObTransService::acquire_tx(const char* buf, * * Return: * OB_SUCCESS - either local commit started or - * remote commit retry task has been registred + * commit retry task has been registred * OB_XXX - try local commit failed and can not been - * fallback to remote commit via send message + * fallback to commit retry via send message */ int ObTransService::do_commit_tx_(ObTxDesc &tx, const int64_t expire_ts, @@ -167,40 +132,31 @@ int ObTransService::do_commit_tx_(ObTxDesc &tx, SCN &commit_version) { int ret = OB_SUCCESS; - ObTxPart *coord = NULL; tx.set_commit_cb(&cb); tx.commit_expire_ts_ = expire_ts; - if (OB_FAIL(decide_tx_commit_info_(tx, coord))) { - TRANS_LOG(WARN, "decide tx coordinator fail, tx will abort", K(ret), K(tx)); - } else if (OB_FAIL(tx.commit_task_.init(&tx, this))) { + if (OB_FAIL(tx.commit_task_.init(&tx, this))) { TRANS_LOG(WARN, "init timeout task fail", K(ret), K(tx)); - } else if (coord->addr_ == self_ && ( - OB_SUCC(local_ls_commit_tx_(tx.tx_id_, - tx.coord_id_, - tx.commit_parts_, + } else if (OB_SUCC(local_ls_commit_tx_(tx.tx_id_, expire_ts, tx.trace_info_.get_app_trace_info(), tx.op_sn_, SCN::max_scn(), - tx.get_coord_epoch(), - commit_version, - self_)) - || !commit_need_retry_(ret))) { + commit_version)) + || !commit_need_retry_(ret)) { if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "local ls commit tx fail", K(ret), K_(tx.coord_id), K(tx)); + TRANS_LOG(WARN, "local ls commit tx fail", K(ret), K(tx)); } else { TRANS_LOG(TRACE, "local ls commit tx started", K(tx)); } } else { // get gts cache as commit start scn - if (OB_FAIL(ts_mgr_->get_gts(NULL, tx.commit_start_scn_))) { + if (OB_FAIL(ts_mgr_->get_gts(tx.commit_start_scn_))) { TRANS_LOG(WARN, "get gts cache fail", K(ret)); } if (OB_FAIL(do_commit_tx_slowpath_(tx))) { - TRANS_LOG(WARN, "commit tx slowpath fail", K(ret), - K_(tx.coord_id), K_(tx.commit_parts), K(tx)); + TRANS_LOG(WARN, "commit tx slowpath fail", K(ret), K(tx)); } else { - TRANS_LOG(TRACE, "remote commit started", K(tx), K_(self)); + TRANS_LOG(TRACE, "commit retry started", K(tx), K_(self)); } } // start commit fail @@ -210,38 +166,31 @@ int ObTransService::do_commit_tx_(ObTxDesc &tx, return ret; } -#define DELETED_UNRETRYABLE_ERROR(ret) (OB_LS_IS_DELETED == ret) /* - * try send commit msg to coordinator, and register retry task + * try send commit msg to the transaction context, and register retry task * if msg send fail, the retry task will retry later * if both register task fail and send are failed, the commit failed */ int ObTransService::do_commit_tx_slowpath_(ObTxDesc &tx) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObTxCommitMsg commit_msg; - bool post_succ = false; - if (OB_TMP_FAIL(build_tx_commit_msg_(tx, commit_msg))) { - TRANS_LOG(WARN, "build tx commit msg fail", K(tmp_ret), K(tx)); - // build msg fail won't cause commit fail, later driven by retry timer - } else if (OB_TMP_FAIL(rpc_->post_msg(tx.coord_id_, commit_msg))) { - TRANS_LOG(WARN, "post tx commit msg fail", K(tmp_ret), K(tx), K(commit_msg)); - if (DELETED_UNRETRYABLE_ERROR(tmp_ret)) { - ret = tx.commit_times_ > 0 ? OB_TRANS_UNKNOWN : OB_TRANS_KILLED; - } else { - // retryable error : location incorrect, server shutdown etc. + SCN commit_version; + const int commit_ret = local_ls_commit_tx_(tx.tx_id_, + tx.commit_expire_ts_, + tx.trace_info_.get_app_trace_info(), + tx.op_sn_, + tx.commit_start_scn_, + commit_version); + if (OB_SUCCESS == commit_ret) { + ++tx.commit_times_; + } else if (commit_need_retry_(commit_ret)) { + if (OB_FAIL(register_commit_retry_task_(tx, POST_COMMIT_REQ_RETRY_INTERVAL))) { + TRANS_LOG(WARN, "register retry commit task fail", K(ret), K(commit_ret), K(tx)); } } else { - post_succ = true; - ++tx.commit_times_; - } - - if (OB_SUCC(ret) && - OB_FAIL(register_commit_retry_task_(tx, post_succ ? INT64_MAX : POST_COMMIT_REQ_RETRY_INTERVAL))) { - TRANS_LOG(WARN, "register retry commit task fail", K(ret), K(post_succ), K(tx)); + ret = handle_tx_commit_result_(tx, commit_ret, commit_version); } - TRANS_LOG(TRACE, "do commit tx slowpath", K(ret), K(post_succ), K(tx)); + TRANS_LOG(TRACE, "retry local tx commit", K(ret), K(commit_ret), K(tx)); return ret; } @@ -250,7 +199,7 @@ int ObTransService::register_commit_retry_task_(ObTxDesc &tx, int64_t max_delay) const int64_t MIN_DELAY = 50 * 1000;// 50ms int ret = OB_SUCCESS; int saved_ret = OB_SUCCESS; - max_delay = max_delay == INT64_MAX ? ObTransCtx::MAX_TRANS_2PC_TIMEOUT_US : max_delay; + max_delay = max_delay == INT64_MAX ? ObTransCtx::MAX_TRANS_COMMIT_RETRY_TIMEOUT_US : max_delay; int64_t now = ObClockGenerator::getClock(); int64_t expire_after = std::min(tx.expire_ts_ - now, tx.commit_expire_ts_ - now); int64_t delay = std::min(max_delay, tx.commit_task_.get_delay() * 2); @@ -320,7 +269,7 @@ int ObTransService::unregister_commit_retry_task_(ObTxDesc &tx) /* * retry tx commit * 1. if tx already terminated, ignore - * 2. send commit msg to coordinator + * 2. send commit msg to the transaction context * 3. register retry task again */ int ObTransService::handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay) @@ -370,7 +319,7 @@ int ObTransService::handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay) } /* - * handle_tx_commit_result - callback from coordinator + * handle_tx_commit_result - commit result callback */ int ObTransService::handle_tx_commit_result(const ObTransID &tx_id, const int result, @@ -412,8 +361,7 @@ int ObTransService::handle_tx_commit_result(const ObTransID &tx_id, * result was fall into three categories: * 1) finished and finalized: * COMMITTED / ABORTED / NOT_FOUND / TIME_OUT - * 2) special error hint a retry is expected: - * eg. NOT_MASTER | SWITCH_TO_FOLLOWER | FROZEN_BLOKING etc. + * 2) local transient errors that should retry, such as freeze blocking * 3) other errors : should be ignored and retry */ int ObTransService::handle_tx_commit_result_(ObTxDesc &tx, @@ -428,33 +376,14 @@ int ObTransService::handle_tx_commit_result_(ObTxDesc &tx, switch (result) { case OB_EAGAIN: case OB_BLOCK_FROZEN: - // for single log stream trans, the leader is freezing - // and is not able to submit log right now, - // return this result to drive and try again later. - case OB_SWITCHING_TO_FOLLOWER_GRACEFULLY: - // 1. callback from switch_to_follower_gracefully on local - case OB_NOT_MASTER: - // 1. callback from switch_to_follower_forcedly on local - // 2. callback from commit_response (from remote) commit_fin = false; if (tx.commit_task_.is_registered()) { - // the task maybe already registred: - // 1. location cache stale: leader on local actually - // 2. L--(regier)-->F-->L--(here)-->F if (OB_FAIL(unregister_commit_retry_task_(tx))) { TRANS_LOG(ERROR, "deregister timeout task fail", K(tx)); } } if (OB_SUCC(ret)) { - int64_t max_delay = INT64_MAX; - if (OB_SWITCHING_TO_FOLLOWER_GRACEFULLY == result) { - max_delay = 300 * 1000; - } - - if (OB_EAGAIN == result) { - max_delay = 300 * 1000; - } - + const int64_t max_delay = OB_EAGAIN == result ? 300 * 1000 : INT64_MAX; if (OB_FAIL(register_commit_retry_task_(tx, max_delay))) { commit_fin = true; state = ObTxDesc::State::ROLLED_BACK; @@ -539,7 +468,7 @@ int ObTransService::handle_tx_commit_result_(ObTxDesc &tx, void ObTransService::abort_tx__(ObTxDesc &tx, const bool cleanup) { - abort_participants_(tx); + abort_write_state_(tx); if (!cleanup) { invalid_registered_snapshot_(tx); } else { @@ -565,7 +494,7 @@ int ObTransService::abort_tx_(ObTxDesc &tx, const int cause, const bool cleanup) if (tx.addr_ == self_) { abort_tx__(tx, cleanup); } else { - abort_participants_(tx); + abort_write_state_(tx); tx.flags_.DEFER_ABORT_ = true; } tx.state_ = ObTxDesc::State::ABORTED; @@ -590,125 +519,19 @@ void ObTransService::invalid_registered_snapshot_(ObTxDesc &tx) void ObTransService::process_registered_snapshot_on_commit_(ObTxDesc &tx) { - // cleanup snapshot's participant info, so that they will skip + // cleanup snapshot's write state info, so that they will skip // verify participant txn ctx, which cause false negative, // because txn ctx has quit when txn committed. int ret = OB_SUCCESS; ARRAY_FOREACH(tx.savepoints_, i) { ObTxSavePoint &p = tx.savepoints_[i]; if (p.is_snapshot() && p.snapshot_->valid_) { - p.snapshot_->parts_.reset(); + p.snapshot_->reset_write_state(); p.snapshot_->committed_ = true; } } } -/* - * decide tx commit coordinator and participants - * - * choice local participant as coordinator preferentially - */ -int ObTransService::decide_tx_commit_info_(ObTxDesc &tx, ObTxPart *&coord) -{ - int ret = OB_SUCCESS; - ObTxPartList &parts = tx.parts_; - coord = NULL; - tx.coord_id_.reset(); - tx.commit_parts_.reset(); - ARRAY_FOREACH(parts, i) { - if (parts[i].is_without_ctx()) { - // skip participant, without ctx created - } else if (OB_FAIL(tx.commit_parts_.push_back(ObTxExecPart(parts[i].id_, - parts[i].epoch_)))) { - TRANS_LOG(WARN, "part id push fail", K(ret), K(tx)); - } else if (!tx.coord_id_.is_valid() && parts[i].addr_ == self_) { - tx.coord_id_ = parts[i].id_; - coord = &parts[i]; - } else if (OB_ISNULL(coord)) { - coord = &parts[i]; - } - } - if (OB_SUCC(ret) && !tx.coord_id_.is_valid() && OB_NOT_NULL(coord)) { - tx.coord_id_ = coord->id_; - } - - TRANS_LOG(DEBUG, "decide tx coord", K(ret), K_(tx.coord_id), K(*this), K(tx)); - return ret; -} - -/* - * get coordinator id for 2pc caller - * it's need to remember coordinaotr in phase 2 of 2PC - * it's required to remember coordinaotr in phase 2 of 2PC - * case 1: xa trans gets its coord before xa prepare - */ - -/* - * phase one of 2pc, i.e., prepare phase - * persist log and trans state to ensure recoverablity - * case 1: xa prepare - */ -#define OB_TRANS_RDONLY 0 - -int ObTransService::build_tx_sub_prepare_msg_(const ObTxDesc &tx, ObTxSubPrepareMsg &msg) -{ - int ret = OB_SUCCESS; - msg.cluster_version_ = tx.cluster_version_; - - msg.tx_id_ = tx.tx_id_; - msg.expire_ts_ = tx.commit_expire_ts_; - msg.receiver_ = tx.coord_id_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = tx.cluster_id_; - msg.request_id_ = tx.op_sn_; - msg.xid_ = tx.xid_; - CONVERT_COMMIT_PARTS_TO_PARTS(tx.commit_parts_, msg.parts_); - if (FAILEDx(msg.commit_parts_.assign(tx.commit_parts_))) { - TRANS_LOG(WARN, "assign commit parts fail", K(ret), K(tx)); - } - return ret; -} - -/* - * phase two of 2pc - * alloc trxDesc and register transMgr - * if exist ? get and use - * case 1) coordinator is local, direct function call - * case 2) send RPC - * finially, deregister and release trxDesc - */ - -int ObTransService::build_tx_sub_commit_msg_(const ObTxDesc &tx, ObTxSubCommitMsg &msg) -{ - int ret = OB_SUCCESS; - - msg.tx_id_ = tx.tx_id_; - msg.receiver_ = tx.coord_id_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.xid_ = tx.xid_; - msg.cluster_version_ = GET_MIN_CLUSTER_VERSION(); - msg.cluster_id_ = GCONF.cluster_id; - msg.request_id_ = tx.op_sn_; - return ret; -} - -int ObTransService::build_tx_sub_rollback_msg_(const ObTxDesc &tx, ObTxSubRollbackMsg &msg) -{ - int ret = OB_SUCCESS; - - msg.tx_id_ = tx.tx_id_; - msg.receiver_ = tx.coord_id_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.xid_ = tx.xid_; - msg.cluster_version_ = GET_MIN_CLUSTER_VERSION(); - msg.cluster_id_ = GCONF.cluster_id; - msg.request_id_ = tx.op_sn_; - return ret; -} - int ObTransService::interrupt(ObTxDesc &tx, int cause) { int ret = OB_SUCCESS; @@ -735,100 +558,52 @@ int ObTransService::interrupt(ObTxDesc &tx, int cause) return ret; } -/* - * participant keepalive - * this has two effects: - * 1) GC participant: if tx terminated, participant will abort it self - * 2) fast abort transaction: if participant report itself failure, - * whole transaction will terminated from top to bottom - */ - -int ObTransService::handle_trans_keepalive(const ObTxKeepaliveMsg &msg, ObTransRpcResult &result) +int ObTransService::report_write_ctx_status(const ObTransID &tx_id, + const int status, + int &tx_status) { int ret = OB_SUCCESS; - int ret_status = OB_SUCCESS; - const ObTransID &tx_id = msg.tx_id_; + tx_status = OB_SUCCESS; ObTxDesc *tx = NULL; - bool do_response = true; - if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx)) && - OB_ENTRY_NOT_EXIST != ret) { - TRANS_LOG(WARN, "get tx fail", K(ret), K(tx_id), K(msg)); - ret_status = ret; + if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + tx_status = OB_TRANS_CTX_NOT_EXIST; + } else { + TRANS_LOG(WARN, "get tx fail", K(ret), K(tx_id)); + tx_status = ret; + } } else if (OB_ISNULL(tx)) { - ret_status = OB_TRANS_CTX_NOT_EXIST; + tx_status = OB_TRANS_CTX_NOT_EXIST; } else if (tx->is_committed() && tx_id == tx->tx_id_) { - ret_status = OB_TRANS_COMMITED; + tx_status = OB_TRANS_COMMITED; } else if (tx->is_rollbacked() && tx_id == tx->tx_id_) { - ret_status = OB_TRANS_ROLLBACKED; + tx_status = OB_TRANS_ROLLBACKED; } else if (tx->is_aborted() && tx_id == tx->tx_id_) { - ret_status = OB_TRANS_KILLED; - } else if (OB_SUCCESS != msg.status_) { - TRANS_LOG(WARN, "tx participant in failed status", K(msg)); - if (OB_TRANS_KILLED == msg.status_) { - TRANS_LOG(INFO, "participant was killed, mark tx should abort", K(tx_id), K(msg.sender_)); - tx->mark_part_abort(tx_id, OB_TRANS_KILLED); - ret_status = OB_TRANS_NEED_ROLLBACK; - } else if (msg.status_ > 0) { - TRANS_LOG(INFO, "participant failed, mark tx should abort", K(tx_id), K(msg.status_), K(msg.sender_)); - tx->mark_part_abort(tx_id, msg.status_); - ret_status = OB_TRANS_NEED_ROLLBACK; + tx_status = OB_TRANS_KILLED; + } else if (OB_SUCCESS != status) { + TRANS_LOG(WARN, "write ctx reported failure", K(tx_id), K(status)); + if (OB_TRANS_KILLED == status) { + tx->mark_write_state_aborted(tx_id, OB_TRANS_KILLED); + tx_status = OB_TRANS_NEED_ROLLBACK; + } else if (status > 0) { + tx->mark_write_state_aborted(tx_id, status); + tx_status = OB_TRANS_NEED_ROLLBACK; } } - ObTxKeepaliveRespMsg resp; - resp.cluster_version_ = GET_MIN_CLUSTER_VERSION(); - - resp.cluster_id_ = GCONF.cluster_id; - resp.request_id_ = ObClockGenerator::getClock(); - resp.tx_id_ = tx_id; - resp.sender_addr_ = self_; - resp.sender_ = share::SCHEDULER_LS; - resp.receiver_ = msg.sender_; - resp.status_ = ret_status; - if (OB_FAIL(rpc_->post_msg(msg.sender_addr_, resp))) { - TRANS_LOG(WARN, "post tx keepalive resp fail", K(ret), K(resp), KPC(this)); - } - result.reset(); - result.init(ret, resp.get_timestamp()); if (OB_NOT_NULL(tx)) { tx_desc_mgr_.revert(*tx); } - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - TRANS_LOG(INFO, "handle trans keepalive", K(ret), K(msg)); - } - return ret; -} - -int ObTransService::handle_trans_keepalive_response(const ObTxKeepaliveRespMsg &msg, obcall::ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - ObPartTransCtx *ctx = NULL; - const ObTransID &tx_id = msg.tx_id_; - const share::ObLSID &ls_id = msg.receiver_; - if (OB_FAIL(get_tx_ctx_(ls_id, tx_id, ctx))) { - TRANS_LOG(WARN, "get tx ctx fail", K(tx_id), K(ls_id)); - } else { - (void)ctx->handle_tx_keepalive_response(msg.status_); - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - result.reset(); - result.init(ret, msg.get_timestamp()); return ret; } -int ObTransService::find_parts_after_sp_(ObTxDesc &tx, - ObTxPartRefList &parts, - const ObTxSEQ scn) +int ObTransService::find_write_state_after_savepoint_(ObTxDesc &tx, + ObTxWriteState *&part, + const ObTxSEQ scn) { int ret = OB_SUCCESS; - ARRAY_FOREACH(tx.parts_, i) { - if (tx.parts_.at(i).last_scn_ > scn && - !tx.parts_.at(i).is_clean()) { - if (OB_FAIL(parts.push_back(tx.parts_.at(i)))) { - TRANS_LOG(WARN, "push back participant failed", K(ret)); - } - } + if (OB_FAIL(tx.find_write_state_after(part, scn))) { + TRANS_LOG(WARN, "collect rollback write state failed", K(ret), K(tx), K(scn)); } return ret; } @@ -840,50 +615,35 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, ObTxDesc *tx_desc) { int ret = OB_SUCCESS; - ObLSID ls_id = store_ctx.ls_id_; if (OB_UNLIKELY(store_ctx.timeout_ < 0)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "store_ctx.timeout_ is invalid", K(ret), K(store_ctx), K(lbt())); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !snapshot.valid_ || OB_ISNULL(store_ctx.ls_))) { + } else if (OB_UNLIKELY(!snapshot.valid_ || OB_ISNULL(store_ctx.ls_))) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid ls_id, ls or invalid snapshot store_ctx", K(ret), K(snapshot), K(store_ctx), K(lbt())); + TRANS_LOG(WARN, "invalid ls or invalid snapshot store_ctx", K(ret), K(snapshot), K(store_ctx), K(lbt())); } else if (snapshot.is_special()) { if (OB_FAIL(validate_snapshot_version_(snapshot.core_.version_, store_ctx.timeout_, *store_ctx.ls_))) { TRANS_LOG(WARN, "invalid speficied snapshot", K(ret), K(snapshot), K(store_ctx)); } - } else if (snapshot.is_ls_snapshot() && snapshot.snapshot_lsid_ != ls_id) { - // For single-tablet operations, a local snapshot cannot be used to access another log stream. - ret = OB_SNAPSHOT_DISCARDED; - TRANS_LOG(WARN, "use a local snapshot to access other logstream, need retry", - K(ret), K(store_ctx), K(snapshot)); } - bool check_readable_ok = false; ObTransID snap_tx_id = snapshot.core_.tx_id_; - ObPartTransCtx *tx_ctx = NULL; + ObTxCtx *tx_ctx = NULL; if (OB_SUCC(ret) && snap_tx_id.is_valid()) { // inner tx read, we verify txCtx's status - bool exist = false; - int64_t part_epoch = 0; - CHECK_TX_PARTS_CONTAIN_(snapshot.parts_, left_, right_, ls_id, part_epoch, exist); - if (OB_SUCC(ret) && (exist || read_latest)) { - if (OB_FAIL(get_tx_ctx_(ls_id, store_ctx.ls_, snap_tx_id, tx_ctx))) { + const bool exist = snapshot.has_write_state(); + if (exist || read_latest) { + if (OB_FAIL(get_tx_ctx_(store_ctx.ls_, snap_tx_id, tx_ctx))) { if (OB_TRANS_CTX_NOT_EXIST == ret && !exist) { ret = OB_SUCCESS; } else { TRANS_LOG(WARN, "get tx ctx fail", - K(ret), K(store_ctx), K(snapshot), K(ls_id), K(exist), K(read_latest)); + K(ret), K(store_ctx), K(snapshot), K(exist), K(read_latest)); } - } else if (exist && tx_ctx->epoch_ != part_epoch) { - ret = OB_TRANS_CTX_NOT_EXIST; - TRANS_LOG(WARN, "exist txCtx epoch mismatch within snapshot", K(ret), - K(part_epoch), K(tx_ctx->epoch_), K(ls_id), KPC(tx_ctx), K(snapshot)); } else if (OB_FAIL(tx_ctx->check_status())) { TRANS_LOG(WARN, "check status fail", K(ret), K(store_ctx), KPC(tx_ctx)); - } else { - check_readable_ok = true; } if (OB_FAIL(ret) && OB_NOT_NULL(tx_ctx)) { revert_tx_ctx_(store_ctx.ls_, tx_ctx); @@ -892,29 +652,6 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, } } - // need continue to check replica's readability - if (OB_SUCC(ret) && !check_readable_ok && - OB_FAIL(check_replica_readable_(snapshot, - snapshot.core_.elr_, - snapshot.source_, - ls_id, - store_ctx.timeout_, - store_ctx.tablet_id_, - *store_ctx.ls_))) { - TRANS_LOG(WARN, "replica not readable", K(ret), - K(snapshot), - K(ls_id), - K(store_ctx), - "ls_weak_read_ts", store_ctx.ls_->get_ls_wrs_handler()->get_ls_weak_read_ts()); - } - - if (OB_SUCC(ret)) { - if (snapshot.snapshot_ls_role_ == common::ObRole::FOLLOWER - && snapshot.snapshot_acquire_addr_ != GCTX.self_addr()) { - TRANS_LOG(INFO, "get read store_ctx by a follower's max_commit_ts", K(ret), K(snapshot), - K(ls_id), K(store_ctx)); - } - } bool create_tx_ctx = false; if (OB_SUCC(ret) && !tx_ctx && snapshot.read_elr()) { if (!tx_desc) { @@ -922,10 +659,10 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, } else { int tmp_ret = OB_SUCCESS; bool exist = false; - if (OB_TMP_FAIL(acquire_tx_ctx(ls_id, *tx_desc, tx_ctx, store_ctx.ls_, false, false, exist))) { - TRANS_LOG(WARN, "try elr read fail, can not acquire tx ctx", K(tmp_ret), "tx_id", tx_desc->get_tx_id(), K(ls_id)); + if (OB_TMP_FAIL(acquire_tx_ctx(*tx_desc, tx_ctx, store_ctx.ls_, false, false, exist))) { + TRANS_LOG(WARN, "try elr read fail, can not acquire tx ctx", K(tmp_ret), "tx_id", tx_desc->get_tx_id()); } else { - TRANS_LOG(DEBUG, "use elr read, create tx ctx success", K(ls_id), KPC(tx_desc), KPC(tx_ctx)); + TRANS_LOG(DEBUG, "use elr read, create tx ctx success", KPC(tx_desc), KPC(tx_ctx)); create_tx_ctx = !exist; } } @@ -951,7 +688,7 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, tx_ctx = NULL; } } else { - update_max_read_ts_(ls_id, snapshot.core_.version_); + update_max_read_ts_(snapshot.core_.version_); } TRANS_LOG(DEBUG, "get-read-store-ctx", K(ret), K(store_ctx), K(read_latest), K(snapshot)); @@ -985,44 +722,33 @@ int ObTransService::get_write_store_ctx(ObTxDesc &tx, const bool special) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = store_ctx.ls_id_; - ObPartTransCtx *tx_ctx = NULL; + ObTxCtx *tx_ctx = NULL; const int16_t branch = store_ctx.branch_; ObTxSEQ data_scn = spec_seq_no; // for LOB aux table, spec_seq_no is valid ObTxSnapshot snap = snapshot.core_; bool access_started = false; - ObRole role = common::ObRole::INVALID_ROLE; bool ctx_exist = false; ObTxTable *tx_table = nullptr; if (tx.access_mode_ == ObTxAccessMode::RD_ONLY) { ret = OB_ERR_READ_ONLY_TRANSACTION; - TRANS_LOG(WARN, "tx is readonly", K(ret), K(ls_id), K(tx), KPC(this)); - } else if (tx.access_mode_ == ObTxAccessMode::STANDBY_RD_ONLY) { - ret = OB_STANDBY_READ_ONLY; - TRANS_LOG(WARN, "tx is standby readonly", K(ret), K(ls_id), K(tx), KPC(this)); + TRANS_LOG(WARN, "tx is readonly", K(ret), K(tx), KPC(this)); } else if (OB_UNLIKELY(!snapshot.valid_)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "snapshot invalid", K(ret), K(snapshot), K(lbt())); } else if (OB_UNLIKELY(store_ctx.timeout_ < 0)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "store_ctx.timeout_ is invalid", K(ret), K(store_ctx), K(lbt())); - } else if (OB_ISNULL(store_ctx.ls_) || !store_ctx.ls_id_.is_valid()) { + } else if (OB_ISNULL(store_ctx.ls_)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "store_ctx's ls_ or ls_id_ is invalid", K(ret), K(store_ctx), K(lbt())); + TRANS_LOG(WARN, "store_ctx's ls_ is invalid", K(ret), K(store_ctx), K(lbt())); } else if (snapshot.is_none_read() - && OB_FAIL(acquire_local_snapshot_(ls_id, - snap.version_, - false /*is_read_only*/, - role))) { - TRANS_LOG(WARN, "acquire ls snapshot for mvcc write fail", K(ret), K(ls_id)); - } else if (snapshot.is_ls_snapshot() && snapshot.snapshot_lsid_ != ls_id) { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "use ls snapshot access another ls", K(ret), K(snapshot), K(ls_id)); - } else if (OB_FAIL(acquire_tx_ctx(ls_id, tx, tx_ctx, store_ctx.ls_, special, snapshot.read_elr(), ctx_exist))) { - TRANS_LOG(WARN, "acquire tx ctx fail", K(ret), K(tx), K(ls_id), KPC(this)); + && OB_FAIL(acquire_local_snapshot_(snap.version_))) { + TRANS_LOG(WARN, "acquire ls snapshot for mvcc write fail", K(ret)); + } else if (OB_FAIL(acquire_tx_ctx(tx, tx_ctx, store_ctx.ls_, special, snapshot.read_elr(), ctx_exist))) { + TRANS_LOG(WARN, "acquire tx ctx fail", K(ret), K(tx), KPC(this)); } else if (OB_FAIL(tx_ctx->start_access(tx, data_scn, branch, write_flag))) { - TRANS_LOG(WARN, "tx ctx start access fail", K(ret), K(tx_ctx), K(ls_id), KPC(this)); + TRANS_LOG(WARN, "tx ctx start access fail", K(ret), K(tx_ctx), KPC(this)); } if (OB_FAIL(ret)) { } else if (FALSE_IT(access_started = true)) { @@ -1058,7 +784,7 @@ int ObTransService::get_write_store_ctx(ObTxDesc &tx, * * so it's required to update `max_read_ts` for these write */ - update_max_read_ts_(ls_id, snap.version_); + update_max_read_ts_(snap.version_); } TRANS_LOG(DEBUG, "get-write-store-ctx", K(ret), K(store_ctx), KPC(this), K(tx), K(snapshot), K(lbt())); @@ -1067,67 +793,54 @@ int ObTransService::get_write_store_ctx(ObTxDesc &tx, /* * the get here imply `get if exist` or `create if should` - * create predication: - * the create must ensure current replica is leader - * at the time of create finish + * A missing local context is created directly in the unique log stream. */ -int ObTransService::acquire_tx_ctx(const share::ObLSID &ls_id, - const ObTxDesc &tx, - ObPartTransCtx *&ctx, +int ObTransService::acquire_tx_ctx(const ObTxDesc &tx, + ObTxCtx *&ctx, ObLS *ls, const bool special, const bool try_get, bool &exist) { int ret = OB_SUCCESS; - int64_t part_epoch = 0; - CHECK_TX_PARTS_CONTAIN_(tx.parts_, id_, epoch_, ls_id, part_epoch, exist); - if (OB_FAIL(ret)) { - } else if (exist) { - if (OB_FAIL(get_tx_ctx_(ls_id, ls, tx.tx_id_, ctx))) { - TRANS_LOG(WARN, "get tx ctx fail", K(ret), K(ls_id), K(tx)); + exist = tx.has_write_state(); + if (exist) { + if (OB_FAIL(get_tx_ctx_(ls, tx.tx_id_, ctx))) { + TRANS_LOG(WARN, "get tx ctx fail", K(ret), K(tx)); if (ret == OB_TRANS_CTX_NOT_EXIST) { - TRANS_LOG(WARN, "participant lost update", K(ls_id), K_(tx.tx_id)); + TRANS_LOG(WARN, "write state lost update", K_(tx.tx_id)); } - } else if (ctx->epoch_ != part_epoch) { - ret = OB_TRANS_CTX_NOT_EXIST; - TRANS_LOG(WARN, "exist txCtx epoch mismatch within txDesc", K(ret), - K(part_epoch), K(ctx->epoch_), K(ls_id), K(ctx), K(tx)); - revert_tx_ctx_(ls, ctx); - ctx = NULL; } - } else if (try_get && OB_SUCC(get_tx_ctx_(ls_id, ls, tx.tx_id_, ctx))) { - } else if (OB_FAIL(create_tx_ctx_(ls_id, ls, tx, ctx, special, exist))) { - TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls_id), K(tx), K(special)); + } else if (try_get && OB_SUCC(get_tx_ctx_(ls, tx.tx_id_, ctx))) { + } else if (OB_FAIL(create_tx_ctx_(ls, tx, ctx, special, exist))) { + TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(tx), K(special)); } - TRANS_LOG(TRACE, "acquire tx ctx", K(ret), K(*this), K(ls_id), K(try_get), K(exist), K(tx), KP(ctx), K(special)); + TRANS_LOG(TRACE, "acquire tx ctx", K(ret), K(*this), K(try_get), K(exist), K(tx), KP(ctx), K(special)); return ret; } // plain create -int ObTransService::get_tx_ctx_(const share::ObLSID &ls_id, - ObLS *ls, +int ObTransService::get_tx_ctx_(ObLS *ls, const ObTransID &tx_id, - ObPartTransCtx *&ctx) + ObTxCtx *&ctx) { int ret = OB_SUCCESS; if (OB_NOT_NULL(ls)) { ret = ls->get_tx_ctx(tx_id, false, ctx); } else { - ret = tx_ctx_mgr_.get_tx_ctx(ls_id, tx_id, false, ctx); + ret = tx_ctx_mgr_.get_tx_ctx(tx_id, false, ctx); } - TRANS_LOG(TRACE, "get tx ctx", K(ret), K(tx_id), K(ls_id), KP(ctx), KP(ls)); + TRANS_LOG(TRACE, "get tx ctx", K(ret), K(tx_id), KP(ctx), KP(ls)); return ret; } -int ObTransService::get_tx_ctx_(const share::ObLSID &ls_id, - const ObTransID &tx_id, - ObPartTransCtx *&ctx) -{ return get_tx_ctx_(ls_id, NULL, tx_id, ctx); } +int ObTransService::get_tx_ctx_(const ObTransID &tx_id, + ObTxCtx *&ctx) +{ return get_tx_ctx_(NULL, tx_id, ctx); } -int ObTransService::revert_tx_ctx_(ObLS* ls, ObPartTransCtx *ctx) +int ObTransService::revert_tx_ctx_(ObLS* ls, ObTxCtx *ctx) { int ret = OB_SUCCESS; if (OB_NOT_NULL(ls)) { @@ -1140,7 +853,7 @@ int ObTransService::revert_tx_ctx_(ObLS* ls, ObPartTransCtx *ctx) return ret; } -int ObTransService::revert_tx_ctx_(ObPartTransCtx *ctx) +int ObTransService::revert_tx_ctx_(ObTxCtx *ctx) { return revert_tx_ctx_(NULL, ctx); } /* @@ -1150,29 +863,26 @@ int ObTransService::revert_tx_ctx_(ObPartTransCtx *ctx) * * NB: special tx_ctx would not blocked when in block_normal state */ -int ObTransService::create_tx_ctx_(const share::ObLSID &ls_id, - ObLS *ls, +int ObTransService::create_tx_ctx_(ObLS *ls, const ObTxDesc &tx, - ObPartTransCtx *&ctx, + ObTxCtx *&ctx, const bool special, bool &exist) { int ret = OB_SUCCESS; int64_t epoch = 0; - PartCtxSource ctx_source = PartCtxSource::MVCC_WRITE; + TxCtxSource ctx_source = TxCtxSource::MVCC_WRITE; if(special) { - ctx_source = PartCtxSource::REGISTER_MDS; + ctx_source = TxCtxSource::REGISTER_MDS; } ObTxCreateArg arg(false, /* for_replay */ ctx_source, tx.tx_id_, - ls_id, tx.cluster_id_, tx.cluster_version_, tx.sess_id_, /*session_id*/ tx.client_sid_, tx.assoc_sess_id_, /*associated_session_id*/ - tx.addr_, tx.get_expire_ts(), this, tx.xid_); @@ -1180,24 +890,23 @@ int ObTransService::create_tx_ctx_(const share::ObLSID &ls_id, ls->create_tx_ctx(arg, exist, ctx) : tx_ctx_mgr_.create_tx_ctx(arg, exist, ctx); if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "get tx ctx from mgr fail", K(ret), K(tx.tx_id_), K(ls_id), K(exist), K(tx), K(arg)); + TRANS_LOG(WARN, "get tx ctx from mgr fail", K(ret), K(tx.tx_id_), K(exist), K(tx), K(arg)); ctx = NULL; } - TRANS_LOG(TRACE, "create tx ctx", K(ret), K(ls_id), K(exist), K(tx)); + TRANS_LOG(TRACE, "create tx ctx", K(ret), K(exist), K(tx)); return ret; } -int ObTransService::create_tx_ctx_(const share::ObLSID &ls_id, - const ObTxDesc &tx, - ObPartTransCtx *&ctx, +int ObTransService::create_tx_ctx_(const ObTxDesc &tx, + ObTxCtx *&ctx, bool &exist) -{ return create_tx_ctx_(ls_id, NULL, tx, ctx, false, exist); } +{ return create_tx_ctx_(NULL, tx, ctx, false, exist); } void ObTransService::fetch_cflict_tx_ids_from_mem_ctx_to_desc_(ObMvccAccessCtx &acc_ctx)// for deadlock { // merge all ctx(in every logstream)'s conflict trans ids to trans_desc int ret = OB_SUCCESS; - common::ObArray array; + common::ObArray array; if (OB_ISNULL(acc_ctx.mem_ctx_)) { ret = OB_BAD_NULL_ERROR; DETECT_LOG(ERROR, "mem_ctx_ on acc_ctx is null", KR(ret), K(array)); @@ -1215,16 +924,15 @@ int ObTransService::revert_store_ctx(storage::ObStoreCtx &store_ctx) { int ret = OB_SUCCESS; ObMvccAccessCtx &acc_ctx = store_ctx.mvcc_acc_ctx_; - ObPartTransCtx *tx_ctx = acc_ctx.tx_ctx_; + ObTxCtx *tx_ctx = acc_ctx.tx_ctx_; if (acc_ctx.is_read()) { if (OB_NOT_NULL(tx_ctx)) { if (acc_ctx.has_create_tx_ctx_) { // elr read will try to create tx ctx ObTxDesc *tx_desc = acc_ctx.tx_desc_; - const bool is_dup = false; if (OB_ISNULL(tx_desc)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "tx desc is null", K(ret), K(store_ctx)); - } else if (OB_FAIL(tx_desc->add_clean_part_if_absent(tx_ctx->ls_id_, tx_ctx->epoch_, self_, is_dup))) { + } else if (OB_FAIL(tx_desc->init_clean_write_state_if_absent())) { TRANS_LOG(WARN, "append part fail", K(ret), KPC(tx_ctx)); } } @@ -1237,17 +945,14 @@ int ObTransService::revert_store_ctx(storage::ObStoreCtx &store_ctx) TRANS_LOG(ERROR, "write access but tx ctx is NULL", K(ret), K(store_ctx)); } else { /* - * record transaction participant info + * record transaction write state info */ ObTxDesc *tx = acc_ctx.tx_desc_; acc_ctx.tx_ctx_ = NULL; - ObTxPart p; - p.id_ = tx_ctx->ls_id_; - p.addr_ = self_; - p.epoch_ = tx_ctx->epoch_; + ObTxWriteState p; p.first_scn_ = tx_ctx->first_scn_; p.last_scn_ = tx_ctx->last_scn_; - if (OB_FAIL(tx->update_part(p))) { + if (OB_FAIL(tx->merge_write_state(p))) { TRANS_LOG(WARN, "append part fail", K(ret), K(p), KPC(tx_ctx)); } (void) fetch_cflict_tx_ids_from_mem_ctx_to_desc_(acc_ctx); @@ -1259,14 +964,6 @@ int ObTransService::revert_store_ctx(storage::ObStoreCtx &store_ctx) TRANS_LOG(ERROR, "unexpected store ctx type", K(ret), K(store_ctx)); } - if (OB_SUCC(ret) && (acc_ctx.is_read())) { - // just for warning and report the errors - if (acc_ctx.tx_table_guards_.check_ls_offline()) { - int tmp_ret = OB_LS_OFFLINE; - STORAGE_LOG(WARN, "ls offline during the read operation", K(tmp_ret), K(acc_ctx.snapshot_)); - } - } - TRANS_LOG(DEBUG, "revert store ctx", K(ret), K(*this), K(lbt())); return ret; } @@ -1290,7 +987,7 @@ int ObTransService::validate_snapshot_version_(const SCN snapshot, const MonotonicTs stc_ahead = get_req_receive_mts_() - MonotonicTs(GET_GTS_AHEAD_INTERVAL); MonotonicTs tmp_receive_gts_ts(0); do { - ret = ts_mgr_->get_gts(stc_ahead, NULL, gts, tmp_receive_gts_ts); + ret = ts_mgr_->get_gts(stc_ahead, gts, tmp_receive_gts_ts); if (ret == OB_EAGAIN) { if (expire_ts <= ObClockGenerator::getClock()) { ret = OB_TIMEOUT; @@ -1315,116 +1012,6 @@ int ObTransService::validate_snapshot_version_(const SCN snapshot, return ret; } -/* - * check ls's readable snapshot - * - * here introduce a concept named 'replica_readable_version' - * - * it was updated by: - * 1. tx commit: - * on Leader: on pre_commit - * on Sync Replica: on pre_commit - * 2. transaction log replay: - * on Follower replica - * 3. and by read on Leader or Sync Replica - * - * with this concept, we can verify replica readable as - * compare with replica_readable_snapshot: - * 1. v = my_read_snapshot_version - * 2. if v <= replica_readble_snapshot return OK, otherwise - * 3. check is_leader or sync_replica of ls - * if so, update replica_readable_snapshot = v and return OK, otherwise - * 4. return OB_REPLICA_NOT_READABLE - */ -int ObTransService::check_replica_readable_(const ObTxReadSnapshot &snapshot, - const bool elr, - const ObTxReadSnapshot::SRC src, - const share::ObLSID &ls_id, - const int64_t expire_ts, - const ObTabletID &tablet_id, - ObLS &ls) -{ - int ret = OB_SUCCESS; - bool leader = false; - int64_t epoch = 0; - share::SCN max_replayed_scn; - max_replayed_scn.reset(); - - if (MTL_TENANT_ROLE_CACHE_IS_PRIMARY() - && snapshot.source_ == ObTxReadSnapshot::SRC::LS - && snapshot.snapshot_lsid_ == ls_id - && snapshot.snapshot_acquire_addr_ == GCTX.self_addr() - && snapshot.snapshot_ls_role_ == common::ObRole::LEADER) { - // for the single ls statement obtained from the leader, avoid a high-cost get_role operation. - // when taking a snapshot locally, skip checking the leader which has been checked when getting snapshot. - } else if (!check_ls_readable_(ls, snapshot.core_.version_, src)) { - if (OB_FAIL(ls.get_tx_svr()->get_tx_ls_log_adapter()->get_role(leader, epoch))) { - TRANS_LOG(WARN, "get replica status fail", K(ls_id)); - } else if (leader || is_sync_replica_(ls_id)) { - ret = OB_SUCCESS; - } else if (ObTxReadSnapshot::SRC::SPECIAL == src || - ObTxReadSnapshot::SRC::WEAK_READ_SERVICE == src) { - // to compatible with SQL's retry-logic, trigger re-choose replica - ret = OB_REPLICA_NOT_READABLE; - } else { - if (OB_SUCC(wait_follower_readable_(ls, expire_ts, snapshot.core_.version_, src))) { - TRANS_LOG(INFO, "read from follower", K(snapshot), K(snapshot), K(ls)); - } else { - ret = OB_NOT_MASTER; - } - } - } - - TRANS_LOG(TRACE, "check replica readable", K(ret), K(snapshot), K(ls_id)); - return ret; -} - -bool ObTransService::check_ls_readable_(ObLS &ls, - const SCN &snapshot, - const ObTxReadSnapshot::SRC src) -{ - int ret = OB_SUCCESS; - bool readable = false; - readable = snapshot <= ls.get_ls_wrs_handler()->get_ls_weak_read_ts(); - return readable; -} - -int ObTransService::wait_follower_readable_(ObLS &ls, - const int64_t expire_ts, - const SCN &snapshot, - const ObTxReadSnapshot::SRC src) -{ - int ret = OB_REPLICA_NOT_READABLE; - int64_t compare_timeout = 0; - - - compare_timeout = GCONF._follower_snapshot_read_retry_duration; - - if (compare_timeout > 0) { - int64_t compare_expired_time = ObClockGenerator::getClock() + compare_timeout; - int64_t stmt_timeout = expire_ts - ObClockGenerator::getClock(); - int64_t retry_interval = 0; - do { - if (OB_UNLIKELY(ObClockGenerator::getClock() >= expire_ts)) { - ret = OB_TIMEOUT; - } else if (check_ls_readable_(ls, snapshot, src)) { - TRANS_LOG(WARN, "read from follower", K(snapshot), K(ls.get_ls_id())); - ret = OB_SUCCESS; - } else if (ObClockGenerator::getClock() >= compare_expired_time) { - break; - } else if (OB_REPLICA_NOT_READABLE == ret) { - stmt_timeout = expire_ts - ObClockGenerator::getClock(); - compare_timeout = compare_expired_time - ObClockGenerator::getClock(); - retry_interval = MIN(MIN3(GCONF.weak_read_version_refresh_interval, compare_timeout, stmt_timeout), 100000); - ob_usleep(static_cast(retry_interval)); - } else { - // do nothing - } - } while (OB_REPLICA_NOT_READABLE == ret); - } - return ret; -} - MonotonicTs ObTransService::get_req_receive_mts_() { /* @@ -1452,323 +1039,78 @@ int ObTransService::collect_tx_exec_result(ObTxDesc &tx, return ret; } -int ObTransService::build_tx_commit_msg_(const ObTxDesc &tx, ObTxCommitMsg &msg) -{ - int ret = OB_SUCCESS; - msg.cluster_version_ = tx.cluster_version_; - - msg.tx_id_ = tx.tx_id_; - msg.expire_ts_ = tx.commit_expire_ts_; - msg.receiver_ = tx.coord_id_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = tx.cluster_id_; - msg.app_trace_info_ = tx.trace_info_.get_app_trace_info(); - msg.request_id_ = tx.op_sn_; - msg.epoch_ = tx.get_coord_epoch(); - msg.commit_start_scn_ = tx.commit_start_scn_; - CONVERT_COMMIT_PARTS_TO_PARTS(tx.commit_parts_, msg.parts_); - if (FAILEDx(msg.commit_parts_.assign(tx.commit_parts_))) { - TRANS_LOG(WARN, "assign part epochs fail", K(ret), K(tx)); - } - return ret; -} - -int ObTransService::abort_participants_(const ObTxDesc &tx_desc) +int ObTransService::abort_write_state_(const ObTxDesc &tx_desc) { int ret = OB_SUCCESS; - const ObTxPartList &parts = tx_desc.parts_; - // ignore ret - ARRAY_FOREACH_NORET(parts, idx) { - const ObTxPart &p = parts.at(idx); - if (p.is_without_ctx()) { - } else if (OB_FAIL(post_tx_abort_part_msg_(tx_desc, p))) { - TRANS_LOG(WARN, "post tx abort part msg", K(ret), K(tx_desc), K(p)); - } + const ObTxWriteState *part = NULL; + if (OB_FAIL(tx_desc.get_abort_write_state(part))) { + TRANS_LOG(WARN, "get abort write state failed", K(ret), K(tx_desc)); + } else if (OB_NOT_NULL(part) && OB_FAIL(abort_write_ctx_(tx_desc))) { + TRANS_LOG(WARN, "abort write context failed", K(ret), K(tx_desc), KPC(part)); } return ret; } -OB_NOINLINE int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, - SCN &snapshot, - const bool is_read_only, - ObRole &role) +OB_NOINLINE int ObTransService::acquire_local_snapshot_(SCN &snapshot) { int ret = OB_SUCCESS; - int64_t epoch = 0; - bool leader = false; - bool is_leader_serving = false; SCN snapshot0; SCN snapshot1; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; const bool can_elr = MTL_TENANT_ROLE_CACHE_IS_PRIMARY() ? true : false; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get ls fail", K(ret), K(ls_id)); - } else if (!ls_handle.is_valid() || OB_ISNULL(ls_handle.get_ls())) { - ret = OB_NOT_MASTER; - TRANS_LOG(WARN, "invalid ls, acquire gts for snapshot", K(ret), K(ls_id), K(ls_handle)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tx_svr()->get_tx_ls_log_adapter()->get_role(leader, - epoch))) { - TRANS_LOG(WARN, "get replica role fail", K(ret), K(ls_id)); - } else if (!leader) { - ret = OB_NOT_MASTER; - } else if (OB_FAIL(ls_handle.get_ls()->get_tx_svr()->check_in_leader_serving_state( - is_leader_serving))) { - ret = OB_NOT_MASTER; - // XXX In standby cluster mode, the failure to call acquire_local_snapshot_ is an - // normal situation, no error log needs to be printed - // TRANS_LOG(WARN, "check ls tx service leader serving state fail", K(ret), K(ls_id), K(ret)); - } else if (!is_leader_serving) { - ret = OB_NOT_MASTER; - } else { - role = LEADER; - } - - if (OB_FAIL(ret)) { - // do nothing - } else if (FALSE_IT(snapshot0 = tx_version_mgr_.get_max_commit_ts(can_elr))) { + if (FALSE_IT(snapshot0 = tx_version_mgr_.get_max_commit_ts(can_elr))) { } else if (!snapshot0.is_valid_and_not_min()) { ret = OB_EAGAIN; - } else if (OB_FAIL(ts_mgr_->get_gts(NULL, snapshot1))) { + } else if (OB_FAIL(ts_mgr_->get_gts(snapshot1))) { } else { snapshot = SCN::max(snapshot0, snapshot1); } #ifdef ENABLE_DEBUG_LOG - TRANS_LOG(TRACE, "acquire local snapshot", K(ret), K(ls_id), K(snapshot)); + TRANS_LOG(TRACE, "acquire local snapshot", K(ret), K(snapshot)); #endif return ret; } -int ObTransService::sync_acquire_global_snapshot_(ObTxDesc &tx, - const int64_t expire_ts, - SCN &snapshot, - int64_t &uncertain_bound) -{ - int ret = OB_SUCCESS; - uint64_t op_sn = tx.op_sn_; - const int64_t GET_GTS_AHEAD_INTERVAL = 0; - tx.flags_.BLOCK_ = true; - tx.lock_.unlock(); - ret = acquire_global_snapshot__(expire_ts, - GET_GTS_AHEAD_INTERVAL, - snapshot, - uncertain_bound); - tx.lock_.lock(); - bool interrupted = tx.flags_.INTERRUPTED_; - if (interrupted) { - ret = OB_ERR_INTERRUPTED; - TRANS_LOG(WARN, "acquiring global snapshot has been interrupted", KR(ret), K(tx)); - } - tx.clear_interrupt(); - tx.flags_.BLOCK_ = false; - if (op_sn != tx.op_sn_) { - if (tx.is_aborted()) { - ret = tx.abort_cause_ == OB_DEAD_LOCK ? OB_DEAD_LOCK : OB_TRANS_KILLED; - TRANS_LOG(WARN, "txn has been aborted", KR(ret), K(tx.abort_cause_)); - } else if (tx.is_rollbacked()) { - ret = OB_TRANS_ROLLBACKED; - TRANS_LOG(WARN, "txn has been rollbacked", KR(ret), K(tx)); - } else if (OB_FAIL(ret)) { - } else { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "txn has been disturbed", KR(ret), K(tx)); - } - } - return ret; -} - -int ObTransService::acquire_global_snapshot__(const int64_t expire_ts, - const int64_t gts_ahead, - SCN &snapshot, - int64_t &uncertain_bound) -{ - int ret = OB_SUCCESS; - const MonotonicTs request_time_base = get_req_receive_mts_(); - const MonotonicTs request_time = request_time_base - MonotonicTs(gts_ahead); - const int64_t current_time = ObClockGenerator::getClock(); - // occupy current worker thread for at most 1s - const int64_t MAX_WAIT_TIME_US = 1 * 1000 * 1000; - MonotonicTs gts_receive_ts(0); - const int64_t timeout_us = min(MAX_WAIT_TIME_US, expire_ts - current_time); - if (current_time >= expire_ts) { - ret = OB_TIMEOUT; - TRANS_LOG(WARN, "get gts timeout", K(ret), K(expire_ts), K(current_time)); - } else if (OB_FAIL(ts_mgr_->get_gts_sync( request_time, timeout_us, snapshot, gts_receive_ts))) { - TRANS_LOG(WARN, "get gts fail", K(ret), K(timeout_us), K(request_time)); - if (OB_TIMEOUT == ret) { - ret = OB_GTS_NOT_READY; - } - } else if (OB_UNLIKELY(!snapshot.is_valid())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "invalid snapshot from gts", K(ret), K(snapshot)); - } else { - uncertain_bound = gts_receive_ts.mts_ + gts_ahead; - } - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "acquire global snapshot fail", K(ret), K(gts_ahead), K(expire_ts), K(request_time)); - } - return ret; -} - - /******************************************************************** * * RPC and Message Handle * ********************************************************************/ -int ObTransService::batch_post_rollback_savepoint_msg_(ObTxDesc &tx, - ObTxRollbackSPMsg &msg, - const ObTxRollbackParts &list, - int &post_succ_num) +int ObTransService::abort_write_ctx_(const ObTxDesc &tx_desc) { int ret = OB_SUCCESS; - int last_ret = OB_SUCCESS; - post_succ_num = 0; - const ObTxDesc *msg_tx_ptr = msg.tx_ptr_; - ARRAY_FOREACH_NORET(list, idx) { - const ObTxExecPart &p = list.at(idx); - msg.receiver_ = p.ls_id_; - msg.epoch_ = p.exec_epoch_; - if (msg.epoch_ > 0) { - msg.tx_ptr_ = NULL; - } - if (OB_FAIL(rpc_->post_msg(msg.receiver_, msg))) { - if (OB_LS_IS_DELETED == ret) { - ObSpinLockGuard lock(tx.lock_); - ObAddr fake_addr; - on_sp_rollback_succ_(p, tx, ObTxPart::EPOCH_DEAD, fake_addr); - ret = OB_SUCCESS; - } else { - TRANS_LOG(WARN, "post msg falied", K(ret), K(msg), K(p)); - last_ret = ret; - } - } else { ++post_succ_num; } - msg.tx_ptr_ = msg_tx_ptr; - } - return last_ret; -} - -int ObTransService::post_tx_abort_part_msg_(const ObTxDesc &tx_desc, - const ObTxPart &p) -{ - int ret = OB_SUCCESS; - ObTxAbortMsg msg; - msg.cluster_version_ = tx_desc.cluster_version_; - - msg.tx_id_ = tx_desc.tx_id_; - msg.receiver_ = p.id_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = tx_desc.cluster_id_; - msg.request_id_ = tx_desc.op_sn_; - msg.reason_ = tx_desc.abort_cause_; - bool local_opt = false; - if (p.addr_ == self_) { - ObTransRpcResult r; - if (OB_SUCC(handle_trans_abort_request(msg, r))) { - local_opt = true; - } + ObTxCtx *ctx = NULL; + if (OB_FAIL(get_tx_ctx_(tx_desc.tx_id_, ctx))) { + TRANS_LOG(WARN, "get transaction context error", KR(ret), K_(tx_desc.tx_id)); + } else if (OB_FAIL(ctx->abort(tx_desc.abort_cause_))) { + TRANS_LOG(WARN, "abort local transaction context error", KR(ret), K(tx_desc)); } - if (!local_opt) { - ret = rpc_->post_msg(p.id_, msg); + if (OB_NOT_NULL(ctx)) { + revert_tx_ctx_(ctx); } return ret; } -bool ObTransService::is_sync_replica_(const share::ObLSID &ls_id) -{ - UNUSED(ls_id); - // FIXME: - /* - * 1. ls.props.is_for_dup_table = true - * 2. replica's in lease - */ - return false; -} - -int ObTransService::handle_trans_commit_response(ObTxCommitRespMsg &resp, ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - ret = handle_tx_commit_result(resp.tx_id_, resp.ret_, resp.commit_version_); - result.reset(); - result.init(ret, resp.get_timestamp()); -#ifndef NDEBUG - TRANS_LOG(INFO, "handle trans commit response", K(ret), K(resp)); -#else - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "handle trans commit response fail", K(ret), K(resp)); - } -#endif - return ret; -} - -/* - * handle tx commit request - * 1. get txCtx and call its commit - * 2. if txCtx not exist, get txState from txTable - * 3. if both of txTable and txCtx not exist, replay with TRANS_UNKNOWN - */ -int ObTransService::handle_trans_commit_request(ObTxCommitMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - SCN commit_version; - if (msg.commit_parts_.count() == 0) { - // for compatible - CONVERT_PARTS_TO_COMMIT_PARTS(msg.parts_, msg.commit_parts_); - } - if (OB_SUCC(ret)) { - ret = local_ls_commit_tx_(msg.tx_id_, - msg.receiver_, - msg.commit_parts_, - msg.expire_ts_, - msg.app_trace_info_, - msg.request_id_, - msg.commit_start_scn_, - msg.epoch_, - commit_version, - msg.sender_addr_); - } - result.reset(); - result.init(ret, msg.get_timestamp()); - result.private_data_ = commit_version; -#ifndef NDEBUG - TRANS_LOG(INFO, "handle trans commit request", K(ret), K(msg)); -#else - if (OB_FAIL(ret) && OB_TRANS_COMMITED != ret) { - TRANS_LOG(WARN, "handle trans commit request failed", K(ret), K(msg)); - } -#endif - return ret; -} - int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, - const share::ObLSID &coord, - const ObTxCommitParts &parts, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id, const SCN commit_start_scn, - const int64_t epoch, - SCN &commit_version, - const common::ObAddr &caller) + SCN &commit_version) { int ret = OB_SUCCESS; MonotonicTs commit_time = get_req_receive_mts_(); - ObPartTransCtx *ctx = NULL; - if (OB_FAIL(get_tx_ctx_(coord, tx_id, ctx))) { - TRANS_LOG(WARN, "get coordinator tx context fail", K(ret), K(tx_id), K(coord)); + ObTxCtx *ctx = NULL; + if (OB_FAIL(get_tx_ctx_(tx_id, ctx))) { + TRANS_LOG(WARN, "get transaction context fail", K(ret), K(tx_id)); if (OB_TRANS_CTX_NOT_EXIST == ret) { int64_t tx_state = ObTxData::RUNNING; share::SCN recycle_scn; - if (OB_FAIL(get_tx_state_from_tx_table_(coord, tx_id, tx_state, commit_version, recycle_scn))) { - TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(coord), K(tx_id)); + if (OB_FAIL(get_tx_state_from_tx_table_(tx_id, tx_state, commit_version, recycle_scn))) { + TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(tx_id)); if (OB_TRANS_CTX_NOT_EXIST == ret) { if (commit_start_scn > recycle_scn) { ret = OB_TRANS_KILLED; // abort without persistent @@ -1779,22 +1121,7 @@ int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, } else { switch (tx_state) { case ObTxData::COMMIT: - { - ObLSTxCtxMgr *ls_tx_mgr = NULL; - if (OB_FAIL(tx_ctx_mgr_.get_ls_tx_ctx_mgr(coord, ls_tx_mgr))) { - TRANS_LOG(WARN, "can not get ls_tx_mgr", K(ret), "ls_id", coord); - } else if (OB_ISNULL(ls_tx_mgr)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ls_tx_mgr is NULL", K(ret), "ls_id", coord); - } else if (ls_tx_mgr->in_leader_serving_state()) { - ret = OB_TRANS_COMMITED; - } else { - ret = OB_NOT_MASTER; - } - if (OB_NOT_NULL(ls_tx_mgr)) { - tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_mgr); - } - } + ret = OB_TRANS_COMMITED; break; case ObTxData::ABORT: ret = OB_TRANS_KILLED; @@ -1802,15 +1129,12 @@ int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, case ObTxData::RUNNING: default: ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx in-progress but ctx miss", K(ret), K(tx_state), K(tx_id), K(coord)); + TRANS_LOG(WARN, "tx in-progress but ctx miss", K(ret), K(tx_state), K(tx_id)); } } } - } else if (ctx->get_scheduler() != caller) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "receive commit from not scheduler", K(ret), K(caller), K(ctx->get_scheduler())); - } else if (OB_FAIL(ctx->commit(parts, commit_time, expire_ts, app_trace_info, request_id))) { - TRANS_LOG(WARN, "commit fail", K(ret), K(coord), K(tx_id)); + } else if (OB_FAIL(ctx->commit(commit_time, expire_ts, app_trace_info, request_id))) { + TRANS_LOG(WARN, "commit fail", K(ret), K(tx_id)); } if (OB_NOT_NULL(ctx)) { revert_tx_ctx_(ctx); @@ -1818,447 +1142,78 @@ int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, return ret; } -int ObTransService::get_tx_state_from_tx_table_(const share::ObLSID &ls_id, - const ObTransID &tx_id, +int ObTransService::get_tx_state_from_tx_table_(const ObTransID &tx_id, int64_t &state, SCN &commit_version, SCN &recycled_scn) { int ret = OB_SUCCESS; ObTxTableGuard tx_table_guard; - ObLSHandle ls_handle; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get ls handle fail", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tx_table()->get_tx_table_guard(tx_table_guard))) { - TRANS_LOG(WARN, "get tx table guard failed", KR(ret), K(ls_id), KPC(this)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + TRANS_LOG(WARN, "get ls fail", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tx_table()->get_tx_table_guard(tx_table_guard))) { + TRANS_LOG(WARN, "get tx table guard failed", KR(ret), KPC(this)); } else if (OB_FAIL(tx_table_guard.try_get_tx_state(tx_id, state, commit_version, recycled_scn))) { - TRANS_LOG(WARN, "get tx state failed", KR(ret), K(ls_id), K(tx_id), KPC(this)); + TRANS_LOG(WARN, "get tx state failed", KR(ret), K(tx_id), KPC(this)); } return ret; } -int ObTransService::handle_trans_abort_request(ObTxAbortMsg &abort_req, ObTransRpcResult &result) +bool ObTransService::common_retryable_error_(const int ret) { + return (OB_EAGAIN == ret + || OB_NEED_RETRY == ret + || OB_TX_NOLOGCB == ret + || OB_PARTITION_IS_BLOCKED == ret + ); +} + +int ObTransService::update_max_read_ts_(const SCN ts) { int ret = OB_SUCCESS; - UNUSED(result); - ObPartTransCtx *ctx = NULL; - if (OB_FAIL(get_tx_ctx_(abort_req.get_receiver(), abort_req.get_trans_id(), ctx))) { - // We donot respond with the abort response, because we think the abort is - // eventually always successful if we have never send the commit request - TRANS_LOG(WARN, "get transaction context error", KR(ret), K(abort_req.get_trans_id())); - } else if (OB_FAIL(ctx->abort(abort_req.reason_))) { - TRANS_LOG(WARN, "trans rollback error", KR(ret), K(abort_req)); - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - if (ObTxAbortCause::IMPLICIT_ROLLBACK != abort_req.reason_) { - TRANS_LOG(INFO, "handle trans abort request", K(ret), K(abort_req)); - } + tx_version_mgr_.update_max_read_ts(ts); + TRANS_LOG(TRACE, "update max read ts", K(ret), K(ts)); return ret; } -int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, - obcall::ObTxRpcRollbackSPResult &result) +int ObTransService::gen_trans_id(ObTransID &trans_id) { int ret = OB_SUCCESS; - int64_t ctx_born_epoch = -1; - ret = ls_rollback_to_savepoint_(msg.tx_id_, - msg.receiver_, - msg.epoch_, - msg.op_sn_, - msg.savepoint_, - msg.tx_seq_base_, - ctx_born_epoch, - msg.tx_ptr_, - msg.specified_from_scn_); - if (msg.use_async_resp()) { - ObTxRollbackSPRespMsg resp; - resp.cluster_version_ = msg.cluster_version_; - - resp.sender_addr_ = self_; - resp.sender_ = msg.receiver_; - resp.receiver_ = msg.sender_; - resp.cluster_id_ = msg.cluster_id_; - resp.tx_id_ = msg.tx_id_; - resp.request_id_ = msg.request_id_; - resp.ret_ = ret; - resp.orig_epoch_ = msg.epoch_, - resp.epoch_ = ctx_born_epoch; - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(rpc_->post_msg(msg.sender_addr_, resp))) { - TRANS_LOG(WARN, "pos rollback sp resp fail", K(tmp_ret), K(resp)); + + int retry_times = 0; + { + const int MAX_RETRY_TIMES = 50; + int64_t tx_id = 0; + do { + if (OB_SUCC(gti_source_->get_trans_id(tx_id))) { + } else if (OB_EAGAIN == ret) { + if (retry_times++ > MAX_RETRY_TIMES) { + ret = OB_GTI_NOT_READY; + TRANS_LOG(WARN, "get trans id not ready", K(ret), K(retry_times), KPC(this)); + } else { + ob_usleep(1000); + } + } else { + TRANS_LOG(WARN, "get trans id fail", KR(ret)); + } + } while (OB_EAGAIN == ret); + if (OB_SUCC(ret)) { + trans_id = ObTransID(tx_id); } } - result.status_ = ret; - result.addr_ = self_; - result.born_epoch_ = ctx_born_epoch; - result.send_timestamp_ = msg.get_timestamp(); - result.ignore_ = msg.use_async_resp(); -#ifndef NDEBUG - TRANS_LOG(INFO, "handle savepoint rollback request", K(ret), K(msg), K(result)); -#else - if (OB_FAIL(ret)) { - TRANS_LOG(INFO, "handle savepoint rollback request fail", K(ret), K(msg), K(result)); - } -#endif + TRANS_LOG(TRACE, "gen trans id", K(ret), K(trans_id), K(retry_times)); return ret; } -int ObTransService::handle_sp_rollback_response(ObTxRollbackSPRespMsg &msg, - obcall::ObTransRpcResult &result) +bool ObTransService::commit_need_retry_(const int ret) { - int ret = OB_SUCCESS; - ret = handle_sp_rollback_resp(msg.sender_, - msg.orig_epoch_, - msg.tx_id_, - msg.ret_, - msg.request_id_, - msg.epoch_, - msg.sender_addr_); - result.reset(); - result.init(ret, msg.get_timestamp()); - return ret; + return OB_BLOCK_FROZEN == ret + || common_retryable_error_(ret); } -// check ls status in trans layer -int ObTransService::check_ls_status_(const share::ObLSID &ls_id, bool &leader) -{ - int ret = OB_SUCCESS; - int64_t epoch = 0; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - if (OB_FAIL(tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { - TRANS_LOG(WARN, "get id service log stream failed"); - } else if (OB_ISNULL(ls_tx_ctx_mgr)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ls ctx mgr is null", K(ls_id), KPC(this)); - } else if (OB_FAIL(ls_tx_ctx_mgr->get_ls_log_adapter()->get_role(leader, epoch))) { - TRANS_LOG(WARN, "get ls role fail", K(ret)); - } - if (ls_tx_ctx_mgr) { - tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr); - } - return ret; -} -// need_check_leader : for unittest, set to false -int ObTransService::handle_tx_batch_req(int msg_type, - const char *buf, - int32_t size, - const bool need_check_leader) -{ - int ret = OB_SUCCESS; - bool leader = false; - int64_t UNUSED = 0; -#define CASE__(msg_type__, msg_class__, msg_handler__) \ - case msg_type__: \ - { \ - int64_t pos = 0; \ - ObPartTransCtx *ctx = NULL; \ - msg_class__ msg; \ - if (OB_FAIL(msg.deserialize(buf, size, pos))) { \ - TRANS_LOG(WARN, "deserialize msg failed", K(ret), K(msg_type), K(size)); \ - } else if (!msg.is_valid()) { \ - ret = OB_INVALID_ARGUMENT; \ - TRANS_LOG(ERROR, "msg is invalid", K(ret), K(msg_type), K(msg)); \ - } else if (OB_FAIL(get_tx_ctx_(msg.get_receiver(), msg.get_trans_id(), ctx))) { \ - TRANS_LOG(WARN, "get tx context fail", K(ret), K(msg)); \ - if (OB_TRANS_CTX_NOT_EXIST == ret || \ - OB_PARTITION_NOT_EXIST == ret || \ - OB_LS_NOT_EXIST == ret) { \ - (void)handle_orphan_2pc_msg_(msg, need_check_leader, false); \ - } \ - } else if (OB_FAIL(ctx->get_ls_tx_ctx_mgr() \ - ->get_ls_log_adapter()->get_role(leader, UNUSED))) { \ - TRANS_LOG(WARN, "check ls leader status error", K(ret), K(msg)); \ - } else if (!leader) { \ - ret = OB_NOT_MASTER; \ - TRANS_LOG(WARN, "ls not master", K(ret), K(msg)); \ - } else if (ctx->is_exiting()) { \ - ret = OB_TRANS_CTX_NOT_EXIST; \ - TRANS_LOG(INFO, "tx context is exiting",K(ret),K(msg)); \ - (void)handle_orphan_2pc_msg_(msg, false, false); \ - } else if (msg_type == TX_2PC_PREPARE_REDO_REQ || \ - msg_type == TX_2PC_PREPARE_REQ) { \ - ret = OB_TRANS_CTX_NOT_EXIST; \ - TRANS_LOG(WARN, "tx exec not complete",K(ret), K(msg)); \ - (void)handle_orphan_2pc_msg_(msg, false, false); \ - } else if (OB_FAIL(ctx->msg_handler__(msg))) { \ - TRANS_LOG(WARN, "handle 2pc request fail", K(ret), K(msg)); \ - } \ - if (OB_NOT_NULL(ctx)) { \ - revert_tx_ctx_(ctx); \ - } \ - break; \ - } - - switch (msg_type) { - default: { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "unsupported msg type", K(ret), K(msg_type)); - break; - } - } -#undef CASE__ - return ret; -} - -bool ObTransService::common_retryable_error_(const int ret) { - return (OB_NOT_MASTER == ret - || OB_EAGAIN == ret - || OB_NEED_RETRY == ret - || OB_TX_NOLOGCB == ret - || OB_PARTITION_IS_BLOCKED == ret - || OB_REPLICA_NOT_READABLE == ret - || OB_LS_NOT_EXIST == ret - || OB_PARTITION_NOT_EXIST == ret - || OB_TENANT_NOT_EXIST == ret - || OB_TENANT_NOT_IN_SERVER == ret - || is_location_service_renew_error(ret) - ); -} - -void ObTransService::on_sp_rollback_succ_(const ObTxExecPart &part, - ObTxDesc &tx, - const int64_t born_epoch, - const ObAddr &addr) -{ - if (tx.brpc_mask_set_.is_mask(part)) { - TRANS_LOG(DEBUG, "has marked received", K(part)); - } else { - if (part.exec_epoch_ <= 0) { - tx.update_clean_part(part.ls_id_, born_epoch, addr); - } - (void)tx.brpc_mask_set_.mask(part); - } -} - -int ObTransService::handle_sp_rollback_resp(const share::ObLSID &ls_id, - const int64_t orig_epoch, - const transaction::ObTransID &tx_id, - const int status, - const int64_t request_id, - const int64_t ret_epoch, - const ObAddr &ret_addr) -{ - int ret = OB_SUCCESS; - TRANS_LOG(INFO, "handle_sp_rollback_resp", K(tx_id), K(ls_id), K(status)); - ObRollbackSPMsgGuard *rollback_sp_msg_guard = NULL; - ObTxDesc *tx = NULL; - // find tx_msg by request_id - ObCommonID msg_id(request_id); - if (request_id <= 0) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "rollback sp resp request_id is invalid", KR(ret), K(tx_id), K(request_id)); - } else if (OB_FAIL(rollback_sp_msg_mgr_.get(msg_id, rollback_sp_msg_guard))) { - TRANS_LOG(WARN, "get trans_desc fail", K(ret), K(tx_id)); - } else if (FALSE_IT(tx = &rollback_sp_msg_guard->get_tx_desc())) { - } else if (tx->tx_id_ != tx_id || tx->state_ != ObTxDesc::State::ROLLBACK_SAVEPOINT) { // fast fail - TRANS_LOG(WARN, "receive stale rollback response message", - K(status), K(request_id), K(ret_epoch), K(ret_addr), K(tx_id), K(tx->tx_id_), K(tx->op_sn_)); - } else if (status == OB_TRANS_RPC_TIMEOUT || common_retryable_error_(status)) { - TRANS_LOG(WARN, "rollback savepoint on ls return an retryable error", K(status), K(ls_id), K(tx_id), K(request_id)); - } else if (OB_FAIL(tx->lock_.lock(10_ms))) { - TRANS_LOG(WARN, "lock fail", K(ret), K(ls_id), K(tx_id), K(request_id), K(status)); - } else { - // must compare tx_msg_id in tx lock - if (tx->brpc_mask_set_.get_tx_msg_id() != msg_id) { - TRANS_LOG(WARN, "receive stale rollback response message", K(tx_id), K(tx->brpc_mask_set_.get_tx_msg_id()), K(msg_id)); - } else if (tx->state_ != ObTxDesc::State::ROLLBACK_SAVEPOINT) { - TRANS_LOG(WARN, "receive stale rollback response message", K(status), K(request_id), KPC(tx)); - } else if (tx->tx_id_ != tx_id) { - TRANS_LOG(WARN, "receive old rpc result msg", K(ret), K_(tx->op_sn), K(request_id), K(tx_id), K(tx->tx_id_)); - } else if (status == OB_SUCCESS) { - ObTxExecPart p; - if (OB_FAIL(tx->brpc_mask_set_.find_part(ls_id, orig_epoch, p))) { - TRANS_LOG(WARN, "find part failed", K(ret), K(ls_id), K(tx_id)); - } else { - // find rollback part by ls_id - (void)on_sp_rollback_succ_(p, *tx, ret_epoch, ret_addr); - if (tx->brpc_mask_set_.is_all_mask()) { - tx->rpc_cond_.notify(OB_SUCCESS); - } - } - } else { // other failure - // notify waiter, cause the savepoint rollback fail - TRANS_LOG(WARN, "rollback_sp response an error", K(status), - K(tx_id), K(tx->tx_id_), K(ret_epoch), - K(request_id), K(ls_id), K(ret_addr)); - tx->rpc_cond_.notify(status); - } - - tx->lock_.unlock(); - } - if (OB_NOT_NULL(tx)) { - rollback_sp_msg_mgr_.revert(rollback_sp_msg_guard); - } - return ret; -} - -int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, - const share::ObLSID &receiver_ls_id, - const ObTransID &tx_id, - const int16_t msg_type, - const int status, - const ObAddr &receiver_addr, - const int64_t request_id, - const SCN &private_data) -{ - int64_t start_ts = ObClockGenerator::getClock(); - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTransService is not running", K(ret)); - } else if (!tx_id.is_valid() - || !ObTxMsgTypeChecker::is_valid_msg_type(msg_type) - || !receiver_addr.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(tx_id), - K(msg_type), K(status), K(receiver_addr), K(request_id)); - } else if (KEEPALIVE == msg_type && common::OB_TENANT_NOT_IN_SERVER == status) { - ObPartTransCtx *ctx = NULL; - if (OB_FAIL(get_tx_ctx_(sender_ls_id, tx_id, ctx))) { - TRANS_LOG(WARN, "get tx ctx fail", K(tx_id), K(sender_ls_id)); - } else { - (void)ctx->handle_tx_keepalive_response(status); - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - } else if (common::OB_TENANT_NOT_IN_SERVER == status - || common::OB_TRANS_RPC_TIMEOUT == status) { - // upper layer do retry - } else if (TX_COMMIT == msg_type) { - switch(status) { - case OB_NOT_MASTER: - case OB_SUCCESS: break; - default: - share::SCN commit_version = private_data; - if (OB_FAIL(handle_tx_commit_result(tx_id, status, commit_version))) { - TRANS_LOG(WARN, "handle tx commit fail", K(ret), K(tx_id)); - } - } - } else if (SUBPREPARE == msg_type) { - switch (status) { - case OB_NOT_MASTER: - case OB_SUCCESS: break; - default: - if (OB_FAIL(handle_sub_prepare_result(tx_id, status))) { - TRANS_LOG(WARN, "handle tx commit fail", K(ret), K(tx_id)); - } - } - } else if (SUBCOMMIT == msg_type) { - switch (status) { - case OB_NOT_MASTER: - case OB_SUCCESS: break; - default: - if (OB_FAIL(handle_sub_commit_result(tx_id, status))) { - TRANS_LOG(WARN, "handle tx commit fail", K(ret), K(tx_id)); - } - } - } else if (SUBROLLBACK == msg_type) { - switch (status) { - case OB_NOT_MASTER: - case OB_SUCCESS: break; - default: - if (OB_FAIL(handle_sub_rollback_result(tx_id, status))) { - TRANS_LOG(WARN, "handle tx commit fail", K(ret), K(tx_id)); - } - } - } - int64_t elapsed_ts = ObClockGenerator::getClock() - start_ts; -#ifndef NDEBUG - TRANS_LOG(INFO, "handle trans msg callback", K(ret), K(elapsed_ts), - K(tx_id), K(sender_ls_id), K(receiver_ls_id), - K(msg_type), K(status), K(receiver_addr), K(request_id)); -#else - if (OB_FAIL(ret) || OB_SUCCESS != status) { - TRANS_LOG(WARN, "handle trans msg callback", K(ret), K(elapsed_ts), - K(tx_id), K(sender_ls_id), K(receiver_ls_id), - K(msg_type), K(status), K(receiver_addr), K(request_id)); - } -#endif - return ret; -} - -int ObTransService::update_max_read_ts_(const share::ObLSID &lsid, - const SCN ts) -{ - int ret = OB_SUCCESS; - tx_version_mgr_.update_max_read_ts(ts); - TRANS_LOG(TRACE, "update max read ts", K(ret), K(lsid), K(ts)); - return ret; -} - -// need_check_leader : just for unittest case -int ObTransService::handle_orphan_2pc_msg_(const ObTxMsg &msg, const bool need_check_leader, const bool ls_deleted) -{ - int ret = OB_NOT_SUPPORTED; - - TRANS_LOG(WARN, "2pc is not supported", K(ret), K(msg)); - - return ret; -} - -int ObTransService::refresh_location_cache(const share::ObLSID ls) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTransService not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTransService is not running", K(ret)); - } else if (!ls.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls)); - } else { - if (EXECUTE_COUNT_PER_SEC(16)) { - TRANS_LOG(INFO, "refresh location cache success", K(ls), K(lbt())); - } - } - return ret; -} - -int ObTransService::gen_trans_id(ObTransID &trans_id) -{ - int ret = OB_SUCCESS; - - int retry_times = 0; - { - const int MAX_RETRY_TIMES = 50; - int64_t tx_id = 0; - do { - if (OB_SUCC(gti_source_->get_trans_id(tx_id))) { - } else if (OB_EAGAIN == ret) { - if (retry_times++ > MAX_RETRY_TIMES) { - ret = OB_GTI_NOT_READY; - TRANS_LOG(WARN, "get trans id not ready", K(ret), K(retry_times), KPC(this)); - } else { - ob_usleep(1000); - } - } else { - TRANS_LOG(WARN, "get trans id fail", KR(ret)); - } - } while (OB_EAGAIN == ret); - if (OB_SUCC(ret)) { - trans_id = ObTransID(tx_id); - } - } - TRANS_LOG(TRACE, "gen trans id", K(ret), K(trans_id), K(retry_times)); - return ret; -} - -bool ObTransService::commit_need_retry_(const int ret) -{ - return OB_BLOCK_FROZEN == ret - || common_retryable_error_(ret); -} - - - -int ObTransService::block_tx(const share::ObLSID &ls_id, bool &is_all_tx_cleaned_up) +int ObTransService::block_tx(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; @@ -2268,19 +1223,16 @@ int ObTransService::block_tx(const share::ObLSID &ls_id, bool &is_all_tx_cleaned } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTransService is not running"); ret = OB_NOT_RUNNING; - } else if (!ls_id.is_valid()) { - TRANS_LOG(WARN, "invalid argument", K(ls_id)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(tx_ctx_mgr_.block_tx(ls_id, is_all_tx_cleaned_up))) { - TRANS_LOG(WARN, "block ls error", KR(ret), K(ls_id)); + } else if (OB_FAIL(tx_ctx_mgr_.block_tx(is_all_tx_cleaned_up))) { + TRANS_LOG(WARN, "block transaction context manager failed", KR(ret)); } else { - TRANS_LOG(INFO, "block ls_id success", K(ls_id), K(is_all_tx_cleaned_up)); + TRANS_LOG(INFO, "block transaction context manager success", K(is_all_tx_cleaned_up)); } return ret; } -int ObTransService::iterate_tx_ctx_mgr_stat(ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter) +int ObTransService::get_tx_ctx_mgr_stat(ObLSTxCtxMgrStat &tx_ctx_mgr_stat) { int ret = OB_SUCCESS; @@ -2290,8 +1242,8 @@ int ObTransService::iterate_tx_ctx_mgr_stat(ObTxCtxMgrStatIterator &tx_ctx_mgr_s } else if (OB_UNLIKELY(!is_running_)) { TRANS_LOG(WARN, "ObTransService is not running"); ret = OB_NOT_RUNNING; - } else if (OB_FAIL(tx_ctx_mgr_.iterate_tx_ctx_mgr_stat(self_, tx_ctx_mgr_stat_iter))) { - TRANS_LOG(WARN, "iterate_tx_ctx_mgr_stat error", KR(ret), K_(self)); + } else if (OB_FAIL(tx_ctx_mgr_.get_tx_ctx_mgr_stat(self_, tx_ctx_mgr_stat))) { + TRANS_LOG(WARN, "get_tx_ctx_mgr_stat error", KR(ret), K_(self)); } else { // do nothing } @@ -2381,609 +1333,21 @@ int ObTransService::update_user_savepoint_(ObTxDesc &tx, const ObTxSavePointList -int ObTransService::handle_timeout_for_xa(ObTxDesc &tx, const int64_t delay) -{ - int ret = OB_SUCCESS; - int64_t now = ObClockGenerator::getClock(); - bool cb_executed = false; - ObTransID tx_id; - bool need_cb = false; - { - ObSpinLockGuardWithTimeout guard(tx.lock_, 5000000); - if (OB_FAIL(guard.get_ret())) { - TRANS_LOG(WARN, "failed to acquire lock in specified time", K(tx)); - // FIXME: how to handle it without lock protection - // according to handle_tx_commit_timeout - need_cb = false; - } else { - tx_id = tx.tx_id_; - if (!tx.commit_task_.is_registered()){ - TRANS_LOG(INFO, "task canceled", K(tx)); - } else if(OB_FAIL(unregister_commit_retry_task_(tx))) { - TRANS_LOG(ERROR, "deregister timeout task fail", K(tx), K(ret)); - } else if (tx.flags_.RELEASED_) { - TRANS_LOG(INFO, "tx released, cancel commit retry", K(tx)); - } else { - if (ObTxDesc::State::SUB_PREPARING == tx.state_) { - ret = handle_sub_prepare_timeout_(tx, delay); - } else if (ObTxDesc::State::SUB_COMMITTING == tx.state_) { - ret = handle_sub_commit_timeout_(tx, delay); - } else if (ObTxDesc::State::SUB_ROLLBACKING == tx.state_) { - ret = handle_sub_rollback_timeout_(tx, delay); - } else { - } - } - need_cb = true; - } - } - if (need_cb) { - cb_executed = tx.execute_commit_cb(); - } - TRANS_LOG(INFO, "handle tx commit timeout", K(ret), K(tx_id), K(cb_executed)); - return ret; -} - -int ObTransService::handle_sub_prepare_timeout_(ObTxDesc &tx, const int64_t delay) +/* + * create_in_txn_implicit_savepoint - create an implicit savepoint when txn is active + */ +int ObTransService::create_in_txn_implicit_savepoint(ObTxDesc &tx, ObTxSEQ &savepoint) { int ret = OB_SUCCESS; - int64_t now = ObClockGenerator::getClock(); - if (tx.state_ != ObTxDesc::State::SUB_PREPARING) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpect trans state", K(ret), K_(tx.state), K(tx)); - } else if (tx.commit_expire_ts_ <= now) { - TRANS_LOG(WARN, "sub prepare timeout", K_(tx.commit_expire_ts), K(tx)); - ret = handle_sub_prepare_result_(tx, OB_TRANS_STMT_TIMEOUT); - } else { - ObTxSubPrepareMsg sub_prepare_msg; - if (OB_FAIL(build_tx_sub_prepare_msg_(tx, sub_prepare_msg))) { - TRANS_LOG(WARN, "build tx commit msg fail", K(ret), K(tx)); - } else if (OB_FAIL(rpc_->post_msg(tx.coord_id_, sub_prepare_msg))) { - TRANS_LOG(WARN, "post commit msg fail", K(ret), K(tx)); - } - if (OB_FAIL(register_commit_retry_task_(tx))) { - TRANS_LOG(WARN, "reregister task fail", K(ret), K(tx)); - } - } - TRANS_LOG(INFO, "handle sub prepare timeout", K(ret), K(tx)); - return ret; -} -int ObTransService::handle_sub_rollback_timeout_(ObTxDesc &tx, const int64_t delay) -{ - int ret = OB_SUCCESS; - int64_t now = ObClockGenerator::getClock(); - if (tx.state_ != ObTxDesc::State::SUB_ROLLBACKING) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpect trans state", K(ret), K_(tx.state), K(tx)); - } else if (tx.commit_expire_ts_ <= now) { - TRANS_LOG(WARN, "sub rollback timeout", K_(tx.commit_expire_ts), K(tx)); - const bool is_rollback = true; - ret = handle_sub_end_tx_result_(tx, is_rollback, OB_TRANS_STMT_TIMEOUT); - } else { - ObTxSubRollbackMsg sub_rollback_msg; - if (OB_FAIL(build_tx_sub_rollback_msg_(tx, sub_rollback_msg))) { - TRANS_LOG(WARN, "build tx commit msg fail", K(ret), K(tx)); - } else if (OB_FAIL(rpc_->post_msg(tx.coord_id_, sub_rollback_msg))) { - TRANS_LOG(WARN, "post commit msg fail", K(ret), K(tx)); - } - if (OB_FAIL(register_commit_retry_task_(tx))) { - TRANS_LOG(WARN, "reregister task fail", K(ret), K(tx)); - } - } - TRANS_LOG(INFO, "handle sub rollback timeout", K(ret), K(tx)); - return ret; -} - -int ObTransService::handle_sub_commit_timeout_(ObTxDesc &tx, const int64_t delay) -{ - int ret = OB_SUCCESS; - int64_t now = ObClockGenerator::getClock(); - if (tx.state_ != ObTxDesc::State::SUB_COMMITTING) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpect trans state", K(ret), K_(tx.state), K(tx)); - } else if (tx.commit_expire_ts_ <= now) { - TRANS_LOG(WARN, "sub commit timeout", K_(tx.commit_expire_ts), K(tx)); - const bool is_rollback = false; - ret = handle_sub_end_tx_result_(tx, is_rollback, OB_TRANS_STMT_TIMEOUT); - } else { - ObTxSubCommitMsg sub_commit_msg; - if (OB_FAIL(build_tx_sub_commit_msg_(tx, sub_commit_msg))) { - TRANS_LOG(WARN, "build tx commit msg fail", K(ret), K(tx)); - } else if (OB_FAIL(rpc_->post_msg(tx.coord_id_, sub_commit_msg))) { - TRANS_LOG(WARN, "post commit msg fail", K(ret), K(tx)); - } - if (OB_FAIL(register_commit_retry_task_(tx))) { - TRANS_LOG(WARN, "reregister task fail", K(ret), K(tx)); - } - } - TRANS_LOG(INFO, "handle sub commit timeout", K(ret), K(tx)); - return ret; -} - -int ObTransService::handle_sub_prepare_request(const ObTxSubPrepareMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - if (msg.commit_parts_.count () > 0) { - if (OB_FAIL(sub_prepare_local_ls_(msg.tx_id_, - msg.receiver_, - msg.commit_parts_, - msg.expire_ts_, - msg.app_trace_info_, - msg.request_id_, - msg.xid_))) { - TRANS_LOG(WARN, "handle tx commit request fail", K(ret), K(msg)); - } - } else { - // for compatible - ObTxCommitParts commit_parts; - CONVERT_PARTS_TO_COMMIT_PARTS(msg.parts_, commit_parts); - if (FAILEDx(sub_prepare_local_ls_(msg.tx_id_, - msg.receiver_, - commit_parts, - msg.expire_ts_, - msg.app_trace_info_, - msg.request_id_, - msg.xid_))) { - TRANS_LOG(WARN, "handle tx commit request fail", K(ret), K(msg)); - } - } - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub prepare request", K(ret), K(msg)); - return ret; -} - -int ObTransService::sub_prepare_local_ls_(const ObTransID &tx_id, - const share::ObLSID &coord, - const ObTxCommitParts &parts, - const int64_t &expire_ts, - const common::ObString &app_trace_info, - const int64_t &request_id, - const ObXATransID &xid) -{ - int ret = OB_SUCCESS; - MonotonicTs commit_time = get_req_receive_mts_(); - ObPartTransCtx *ctx = NULL; - if (OB_FAIL(get_tx_ctx_(coord, tx_id, ctx))) { - TRANS_LOG(WARN, "get coordinator context fail", K(ret), K(tx_id), K(coord)); - if (OB_TRANS_CTX_NOT_EXIST == ret) { - int64_t tx_state; - SCN commit_version; - if (OB_FAIL(get_tx_state_from_tx_table_(coord, tx_id, tx_state, commit_version))) { - TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(coord), K(tx_id)); - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_TRANS_KILLED; // presume abort - } - } else { - switch (tx_state) { - case ObTxData::COMMIT: - ret = OB_TRANS_COMMITED; - break; - case ObTxData::ABORT: - ret = OB_TRANS_KILLED; - break; - case ObTxData::RUNNING: - default: - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected non-existence status of trans ctx", K(ret), - K(tx_state), K(tx_id), K(coord)); - } - } - } - } else { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "commit fail", K(ret), K(coord), K(tx_id)); - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - return ret; -} - -int ObTransService::handle_sub_prepare_response(const ObTxSubPrepareRespMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - ret = handle_sub_prepare_result(msg.tx_id_, msg.ret_); - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub prepare response", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_sub_prepare_result(const ObTransID &tx_id, - const int result) -{ - int ret = OB_SUCCESS; - ObTxDesc *tx = NULL; - if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx))) { - TRANS_LOG(WARN, "cannot found tx by id", K(ret), K(tx_id), K(result)); - } else { - bool need_cb = false; - tx->lock_.lock(); - if (ObTxDesc::State::IN_TERMINATE >= tx->state_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected tx state", K(ret), - K_(tx->state), K(tx_id), K(result), KPC(tx)); - tx->print_trace_(); - } else if (ObTxDesc::State::SUB_PREPARED == tx->state_) { - TRANS_LOG(WARN, "tx has been prepared", K_(tx->state), - K(tx_id), K(result), KPC(tx)); - } else if (ObTxDesc::State::ROLLED_BACK == tx->state_) { - TRANS_LOG(WARN, "tx has been rollbacked", K_(tx->state), - K(tx_id), K(result), KPC(tx)); - } else if (ObTxDesc::State::SUB_PREPARING != tx->state_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected tx state", K_(tx->state), - K(tx_id), K(result), KPC(tx)); - } else { - need_cb = true; - ret = handle_sub_prepare_result_(*tx, result); - } - tx->lock_.unlock(); - if (need_cb) { tx->execute_commit_cb(); } - } - if (OB_NOT_NULL(tx)) { - tx_desc_mgr_.revert(*tx); - } - return ret; -} - -int ObTransService::handle_sub_prepare_result_(ObTxDesc &tx, - const int result) -{ - int ret = OB_SUCCESS; - bool commit_fin = true; - int commit_out = OB_SUCCESS; - switch (result) { - case OB_EAGAIN: - case OB_BLOCK_FROZEN: - case OB_SWITCHING_TO_FOLLOWER_GRACEFULLY: - case OB_NOT_MASTER: - commit_fin = false; - if (tx.commit_task_.is_registered()) { - // the task maybe already registred: - // 1. location cache stale: leader on local actually - // 2. L--(regier)-->F-->L--(here)-->F - if (OB_FAIL(unregister_commit_retry_task_(tx))) { - TRANS_LOG(ERROR, "deregister timeout task fail", K(tx)); - } - } - if (OB_SUCC(ret)) { - int64_t max_delay = INT64_MAX; - if (OB_SWITCHING_TO_FOLLOWER_GRACEFULLY == result) { - max_delay = 300 * 1000; - } - - if (OB_FAIL(register_commit_retry_task_(tx, max_delay))) { - commit_fin = true; - tx.state_ = ObTxDesc::State::ROLLED_BACK; - tx.commit_out_ = OB_TRANS_ROLLBACKED; - } - } - break; - case OB_SUCCESS: - // success of sub prepare - tx.state_ = ObTxDesc::State::SUB_PREPARED; - commit_out = OB_SUCCESS; - break; - case OB_TRANS_COMMITED: - commit_fin = true; - tx.state_ = ObTxDesc::State::COMMITTED; - commit_out = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected tx result", K(tx), K(result)); - break; - case OB_TRANS_KILLED: - case OB_TRANS_ROLLBACKED: - default: - tx.state_ = ObTxDesc::State::ROLLED_BACK; - commit_out = result; - break; - } - // commit finished, cleanup - if (commit_fin) { - if (tx.finish_ts_ <= 0) { // maybe aborted early - tx.finish_ts_ = ObClockGenerator::getClock(); - } - tx.commit_out_ = commit_out; - if (tx.commit_task_.is_registered()) { - if (OB_FAIL(unregister_commit_retry_task_(tx))) { - TRANS_LOG(ERROR, "deregister timeout task fail", K(tx)); - } - } - tx_post_terminate_(tx); - } - TRANS_LOG(INFO, "handle sub prepare result", K(ret), K(tx), K(commit_fin), K(result)); - return ret; -} - -int ObTransService::handle_sub_commit_request(const ObTxSubCommitMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - const bool is_rollback = false; - if (OB_FAIL(sub_end_tx_local_ls_(msg.tx_id_, - msg.receiver_, - msg.request_id_, - msg.xid_, - msg.sender_addr_, - is_rollback))) { - TRANS_LOG(WARN, "fail to handle tx commit request", K(ret), K(msg)); - } - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub commit request", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_sub_rollback_request(const ObTxSubRollbackMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - const bool is_rollback = true; - if (OB_FAIL(sub_end_tx_local_ls_(msg.tx_id_, - msg.receiver_, - msg.request_id_, - msg.xid_, - msg.sender_addr_, - is_rollback))) { - TRANS_LOG(WARN, "fail to handle tx rollback request", K(ret), K(msg)); - } - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub rollback request", K(ret), K(msg)); - return ret; -} - -int ObTransService::sub_end_tx_local_ls_(const ObTransID &tx_id, - const share::ObLSID &coord, - const int64_t &request_id, - const ObXATransID &xid, - const ObAddr &sender_addr, - const bool is_rollback) -{ - int ret = OB_SUCCESS; - MonotonicTs commit_time = get_req_receive_mts_(); - ObPartTransCtx *ctx = NULL; - if (OB_FAIL(get_tx_ctx_(coord, tx_id, ctx))) { - TRANS_LOG(WARN, "fail to get coordinator tx context", K(ret), K(tx_id), K(coord)); - if (OB_TRANS_CTX_NOT_EXIST == ret) { - int64_t tx_state = ObTxData::RUNNING; - SCN commit_version; - if (OB_FAIL(get_tx_state_from_tx_table_(coord, tx_id, tx_state, commit_version))) { - TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(coord), K(tx_id)); - if (OB_TRANS_CTX_NOT_EXIST == ret) { - ret = OB_TRANS_KILLED; // presume abort - } - } else { - switch (tx_state) { - case ObTxData::COMMIT: - ret = OB_TRANS_COMMITED; - break; - case ObTxData::ABORT: - ret = OB_TRANS_KILLED; - break; - case ObTxData::RUNNING: - default: - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected non-existence status of trans ctx", K(ret), - K(tx_state), K(tx_id), K(coord)); - } - } - } - } else { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "fail to end trans", K(ret), K(coord), K(tx_id)); - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - return ret; -} - -int ObTransService::handle_sub_commit_response(const ObTxSubCommitRespMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - ret = handle_sub_commit_result(msg.tx_id_, msg.ret_); - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub commit response", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_sub_commit_result(const ObTransID &tx_id, - const int result) -{ - int ret = OB_SUCCESS; - ObTxDesc *tx = NULL; - if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx))) { - TRANS_LOG(WARN, "fail to get trans desc by trans id", K(ret), K(tx_id), K(result)); - } else if (NULL == tx) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected trans desc", K(ret), K(tx_id), K(result)); - } else { - bool need_cb = false; - tx->lock_.lock(); - // TODO, check state - if (ObTxDesc::State::SUB_COMMITTING != tx->state_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected trans state", K_(tx->state), - K(tx_id), K(result), KPC(tx)); - } else { - int final_result = result; - const bool is_rollback = false; - if (OB_TRANS_COMMITED == result) { - final_result = OB_SUCCESS; - } - need_cb = true; - ret = handle_sub_end_tx_result_(*tx, is_rollback, final_result); - } - tx->lock_.unlock(); - if (need_cb) { tx->execute_commit_cb(); } - } - if (OB_NOT_NULL(tx)) { - tx_desc_mgr_.revert(*tx); - } - return ret; -} - -int ObTransService::handle_sub_rollback_response(const ObTxSubRollbackRespMsg &msg, - ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - ret = handle_sub_rollback_result(msg.tx_id_, msg.ret_); - result.reset(); - result.init(ret, msg.get_timestamp()); - TRANS_LOG(INFO, "handle sub rollback response", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_sub_rollback_result(const ObTransID &tx_id, - const int result) -{ - int ret = OB_SUCCESS; - ObTxDesc *tx = NULL; - if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx))) { - TRANS_LOG(WARN, "fail to get trans desc by trans id", K(ret), K(tx_id), K(result)); - } else if (NULL == tx) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected trans desc", K(ret), K(tx_id), K(result)); - } else { - bool need_cb = false; - tx->lock_.lock(); - // TODO, check state - if (ObTxDesc::State::SUB_ROLLBACKING != tx->state_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected trans state", K_(tx->state), - K(tx_id), K(result), KPC(tx)); - } else { - int final_result = result; - const bool is_rollback = true; - if (OB_TRANS_KILLED == result) { - final_result = OB_SUCCESS; - } - need_cb = true; - ret = handle_sub_end_tx_result_(*tx, is_rollback, final_result); - } - tx->lock_.unlock(); - if (need_cb) { tx->execute_commit_cb(); } - } - if (OB_NOT_NULL(tx)) { - tx_desc_mgr_.revert(*tx); - } - return ret; -} - -int ObTransService::handle_sub_end_tx_result_(ObTxDesc &tx, - const bool is_rollback, - const int result) -{ - int ret = OB_SUCCESS; - bool commit_fin = true; - switch (result) { - case OB_EAGAIN: - case OB_BLOCK_FROZEN: - case OB_SWITCHING_TO_FOLLOWER_GRACEFULLY: - case OB_NOT_MASTER: { - commit_fin = false; - if (tx.commit_task_.is_registered()) { - // the task maybe already registred: - // 1. location cache stale: leader on local actually - // 2. L--(regier)-->F-->L--(here)-->F - if (OB_FAIL(unregister_commit_retry_task_(tx))) { - TRANS_LOG(ERROR, "deregister timeout task fail", K(tx)); - } - } - if (OB_SUCC(ret)) { - int64_t max_delay = INT64_MAX; - if (OB_SWITCHING_TO_FOLLOWER_GRACEFULLY == result) { - max_delay = 300 * 1000; - } - - if (OB_FAIL(register_commit_retry_task_(tx, max_delay))) { - commit_fin = true; - tx.state_ = ObTxDesc::State::ROLLED_BACK; - tx.commit_out_ = OB_TRANS_ROLLBACKED; - } - } - break; - } - case OB_TRANS_STMT_TIMEOUT: { - commit_fin = true; - tx.commit_out_ = OB_TRANS_STMT_TIMEOUT; - // TODO, use other state to denote timeout - tx.state_ = ObTxDesc::State::ROLLED_BACK; - TRANS_LOG(WARN, "stmt timeout of sub end trans", K(tx), K(result)); - break; - } - case OB_TRANS_COMMITED: { - commit_fin = true; - // NOTE that success is returned if xa rollback - tx.commit_out_ = OB_SUCCESS; - tx.state_ = ObTxDesc::State::SUB_COMMITTED; - break; - } - case OB_TRANS_ROLLBACKED: - case OB_TRANS_KILLED: { - commit_fin = true; - tx.state_ = ObTxDesc::State::SUB_ROLLBACKED; - if (is_rollback) { - tx.commit_out_ = OB_SUCCESS; - } else { - tx.commit_out_ = result; - } - break; - } - case OB_SUCCESS: { - commit_fin = true; - tx.commit_out_ = OB_SUCCESS; - if (is_rollback) { - tx.state_ = ObTxDesc::State::SUB_ROLLBACKED; - } else { - tx.state_ = ObTxDesc::State::SUB_COMMITTED; - } - break; - } - case OB_TRANS_UNKNOWN: { - commit_fin = true; - tx.state_ = ObTxDesc::State::COMMIT_UNKNOWN; - tx.commit_out_ = result; - break; - } - default: { - commit_fin = false; - TRANS_LOG(WARN, "recv unrecongized commit result, just ignore", K(result), K(tx)); - break; - } - } - if (commit_fin) { - if (tx.commit_task_.is_registered()) { - if (OB_FAIL(unregister_commit_retry_task_(tx))) { - TRANS_LOG(ERROR, "deregister timeout task fail", K(tx)); - } - } - } - return ret; -} - - -/* - * create_in_txn_implicit_savepoint - create an implicit savepoint when txn is active - */ -int ObTransService::create_in_txn_implicit_savepoint(ObTxDesc &tx, ObTxSEQ &savepoint) -{ - int ret = OB_SUCCESS; - - ObTxParam tx_param; - tx_param.timeout_us_ = tx.timeout_us_; - tx_param.lock_timeout_us_ = tx.lock_timeout_us_; - tx_param.access_mode_ = tx.access_mode_; - tx_param.isolation_ = tx.isolation_; - tx_param.cluster_id_ = tx.cluster_id_; - if (tx_param.is_valid()) { - ret = create_implicit_savepoint(tx, tx_param, savepoint); + ObTxParam tx_param; + tx_param.timeout_us_ = tx.timeout_us_; + tx_param.lock_timeout_us_ = tx.lock_timeout_us_; + tx_param.access_mode_ = tx.access_mode_; + tx_param.isolation_ = tx.isolation_; + tx_param.cluster_id_ = tx.cluster_id_; + if (tx_param.is_valid()) { + ret = create_implicit_savepoint(tx, tx_param, savepoint); } else { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "create in txn implicit savepoint, but txn not in txn", K(ret), K(tx)); @@ -2991,267 +1355,6 @@ int ObTransService::create_in_txn_implicit_savepoint(ObTxDesc &tx, ObTxSEQ &save return ret; } -// for standby -int ObTransService::get_tx_ctx_for_standby_(const share::ObLSID &ls_id, - const ObTransID &tx_id, - ObPartTransCtx *&ctx) -{ - return tx_ctx_mgr_.get_tx_ctx(ls_id, tx_id, true, ctx); -} - -int ObTransService::check_for_standby(const share::ObLSID &ls_id, - const ObTransID &tx_id, - const SCN &snapshot, - bool &can_read, - SCN &trans_version) -{ - int ret = OB_SUCCESS; - - ret = OB_NOT_SUPPORTED; - - TRANS_LOG(ERROR, "2pc standby read is not supported", K(ret), K(tx_id), K(ls_id), K(snapshot)); - - return ret; -} - -int ObTransService::mds_infer_standby_trx_state(const ObLS *ls_ptr, - const ObLSID &ls_id, - const ObTransID &tx_id, - const SCN &snapshot, - ObTxCommitData::TxDataState &tx_data_state, - share::SCN &commit_version) -{ - int ret = OB_SUCCESS; - - ret = OB_NOT_SUPPORTED; - TRANS_LOG(ERROR, "2pc standby read is not supported", K(ret), K(tx_id), K(ls_id), K(snapshot)); - - return ret; -} - -int ObTransService::handle_trans_ask_state(const ObAskStateMsg &msg, - obcall::ObTransRpcResult &result) -{ - int ret = OB_SUCCESS; - - ret = OB_NOT_SUPPORTED; - TRANS_LOG(ERROR, "2pc standby read is not supported", K(ret), K(msg)); - - return ret; -} - -int ObTransService::check_and_fill_state_info(const ObTransID &tx_id, ObStateInfo &state_info) -{ - int ret = OB_SUCCESS; - int64_t tx_state = ObTxData::RUNNING; - SCN version; - if (OB_FAIL(get_tx_state_from_tx_table_(state_info.ls_id_, tx_id, tx_state, version))) { - TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(state_info), K(tx_id), K(version)); - } else { - switch (tx_state) { - case ObTxData::COMMIT: - state_info.state_ = ObTxState::COMMIT; - break; - case ObTxData::ABORT: - state_info.state_ = ObTxState::ABORT; - break; - case ObTxData::RUNNING: - default: - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx in-progress but ctx miss", K(ret), K(tx_state), K(tx_id), K(state_info)); - } - state_info.version_ = version; - } - return ret; -} - -void ObTransService::build_tx_ask_state_resp_(ObAskStateRespMsg &resp, const ObAskStateMsg &msg) -{ - resp.cluster_version_ = msg.cluster_version_; - - resp.tx_id_ = msg.tx_id_; - resp.sender_addr_ = self_; - resp.sender_ = msg.receiver_; - resp.request_id_ = ObTimeUtility::current_time(); - resp.cluster_id_ = msg.cluster_id_; - if (msg.ori_ls_id_.is_valid()) { // for msg compat - resp.receiver_ = msg.ori_ls_id_; - } else { - resp.receiver_ = msg.sender_; - } -} - -int ObTransService::handle_trans_ask_state_response(const ObAskStateRespMsg &msg, - obcall::ObTransRpcResult &result) -{ - int ret = OB_NOT_SUPPORTED; - TRANS_LOG(ERROR, "handle trans ask state resp state", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_trans_collect_state(const ObCollectStateMsg &msg, - obcall::ObTransRpcResult &result) -{ - int ret = OB_NOT_SUPPORTED; - TRANS_LOG(INFO, "handle trans collect state", K(ret), K(msg)); - return ret; -} - -void ObTransService::build_tx_collect_state_resp_(ObCollectStateRespMsg &resp, const ObCollectStateMsg &msg) -{ - resp.cluster_version_ = msg.cluster_version_; - - resp.tx_id_ = msg.tx_id_; - resp.sender_addr_ = self_; - resp.sender_ = msg.receiver_; - resp.request_id_ = ObTimeUtility::current_time(); - resp.cluster_id_ = msg.cluster_id_; - resp.receiver_ = msg.sender_; - resp.state_info_.snapshot_version_ = msg.snapshot_; -} - -int ObTransService::handle_trans_collect_state_response(const ObCollectStateRespMsg &msg, - obcall::ObTransRpcResult &result) -{ - int ret = OB_NOT_SUPPORTED; - TRANS_LOG(INFO, "handle trans collect state", K(ret), K(msg)); - return ret; -} - -int ObTransService::handle_ls_deleted(const ObTxMsg &msg) -{ - TRANS_LOG(INFO, "handle ls deleted", K(msg)); - return handle_orphan_2pc_msg_(msg, false, true); -} -void ObTransService::register_standby_cleanup_task() -{ - int ret = OB_SUCCESS; - ObTxStandbyCleanupTask *task = nullptr; - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTransService not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransService is not running"); - ret = OB_NOT_RUNNING; - } else if (OB_ISNULL(task = static_cast( - share::mtl_malloc(sizeof(ObTxStandbyCleanupTask), "standby_cleanup")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc ObTxStandbyCleanupTask failed", K(ret)); - } else if (OB_FALSE_IT(new (task) ObTxStandbyCleanupTask())) { - } else if (OB_FAIL(push(task))) { - TRANS_LOG(WARN, "push ObTxStandbyCleanupTask failed", K(ret)); - } else { - TRANS_LOG(INFO, "push ObTxStandbyCleanupTask success"); - } -} - -int ObTransService::do_standby_cleanup() -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("do standby cleanup", 1); - - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObTransService not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_running_)) { - TRANS_LOG(WARN, "ObTransService is not running"); - ret = OB_NOT_RUNNING; - } else if (OB_FAIL(tx_ctx_mgr_.do_all_ls_standby_cleanup(timeguard))) { - TRANS_LOG(WARN, "iterate tx stat error", KR(ret)); - } else { - // do nothing - } - - return ret; -} - -bool ObTransService::is_ls_dropped_(const share::ObLSID ls_id) { - int ret = OB_SUCCESS; - bool bret = false; - ObLSService *ls_svr = share::g_mp->ls_service(); - storage::ObLSHandle handle; - ObLS *ls; - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "log stream service is NULL", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get id service log stream failed"); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "id service log stream not exist"); - } - return bret; -} - -int ObTransService::ask_tx_state_for_4377(const ObLSID ls_id, - const ObTransID tx_id, - bool &is_alive) -{ - int ret = OB_SUCCESS; - - static const int64_t MAX_ALLOWED_ASK_STATE_FOR_4377_TIMES = 10 * 1000 * 1000; //10s - static const int64_t SLEEP_DURATION_FOR_ASK_STATE_FOR_4377 = 100 * 1000; //100ms - const int64_t start_ts = ObTimeUtility::current_time(); - ObAskTxStateFor4377Msg msg; - ObAskTxStateFor4377RespMsg resp; - msg.tx_id_ = tx_id; - msg.ls_id_ = ls_id; - - do { - if (OB_FAIL(rpc_->ask_tx_state_for_4377(msg, resp))) { - TRANS_LOG(WARN, "ask tx state for 4377 failed", K(ret)); - if (OB_LS_IS_DELETED == ret) { - is_alive = false; - ret = OB_SUCCESS; - TRANS_LOG(WARN, "ls is deleted during ask tx state", K(ret), K(msg)); - } - } else if (OB_FAIL(resp.ret_)) { - ret = resp.ret_; - } else { - is_alive = resp.is_alive_; - } - - if (OB_FAIL(ret)) { - ob_usleep(SLEEP_DURATION_FOR_ASK_STATE_FOR_4377); - } - - if (OB_FAIL(ret) && ObTimeUtility::current_time() - start_ts >= MAX_ALLOWED_ASK_STATE_FOR_4377_TIMES) { - TRANS_LOG(WARN, "timeout for 4377 check", K(ret), K(ls_id), K(tx_id), K(start_ts)); - ret = OB_TIMEOUT; - } - } while (OB_FAIL(ret) && OB_TIMEOUT != ret); - - TRANS_LOG(INFO, "tx state check for 4377 finished", K(ls_id), K(tx_id), K(ret), K(is_alive)); - - return ret; -} - -int ObTransService::handle_ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - bool &is_alive) -{ - int ret = OB_SUCCESS; - ObPartTransCtx *ctx = NULL; - is_alive = false; - - if (OB_FAIL(get_tx_ctx_(msg.ls_id_, msg.tx_id_, ctx))) { - TRANS_LOG(WARN, "fail to get tx context", K(ret), K(msg)); - if (OB_TRANS_CTX_NOT_EXIST == ret || OB_PARTITION_NOT_EXIST == ret) { - ret = OB_SUCCESS; - is_alive = false; - TRANS_LOG(WARN, "tx state is not exist for 4377", K(ret), K(msg)); - } - } else if (OB_FAIL(ctx->handle_ask_tx_state_for_4377(is_alive))) { - TRANS_LOG(WARN, "fail to handle trans ask state resp", K(ret), K(msg)); - } - - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } - TRANS_LOG(INFO, "handle ask tx state for 4377", K(ret), K(msg), K(is_alive)); - return ret; -} - void ObTransService::force_release_tx_when_session_destroy(ObTxDesc &tx) { { diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index 8ef73a6e7..a5f03b9e7 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -14,12 +14,10 @@ * limitations under the License. */ -int create_ls(const share::ObLSID &ls_id, - ObLS &ls, +int create_ls(ObLS &ls, ObITxLogParam *param, ObITxLogAdapter * log_adapter); -int remove_ls(const share::ObLSID &ls_id, - const bool graceful = true); +int remove_ls(const bool graceful = true); /* * acquire a transaction descriptor by deserialize from buffer */ @@ -50,12 +48,12 @@ int create_in_txn_implicit_savepoint(ObTxDesc &tx, ObTxSEQ &savepoint); */ /* - * acquire transaction's coordinator - * return READ_ONLY if transaction not contain any participants + * prepare transaction commit target + * return READ_ONLY if transaction does not contain a write state */ /* - * collect transaction participant info after transactional data access + * collect transaction write state info after transactional data access */ int collect_tx_exec_result(ObTxDesc &tx, ObTxExecResult &result); @@ -82,38 +80,13 @@ int get_write_store_ctx(ObTxDesc &tx, const bool special = false); int revert_store_ctx(storage::ObStoreCtx &store_ctx); -int acquire_tx_ctx(const share::ObLSID &ls_id, - const ObTxDesc &tx, - ObPartTransCtx *&ctx, +int acquire_tx_ctx(const ObTxDesc &tx, + ObTxCtx *&ctx, ObLS *ls, const bool special, const bool try_get, bool &exist); -//handle msg -int handle_trans_commit_request(ObTxCommitMsg &commit_req, obcall::ObTransRpcResult &result); -int handle_trans_commit_response(ObTxCommitRespMsg &commit_resp, obcall::ObTransRpcResult &result); -int handle_trans_abort_request(ObTxAbortMsg &abort_req, obcall::ObTransRpcResult &result); -int handle_sp_rollback_request(ObTxRollbackSPMsg &sp_rollbacck_req, obcall::ObTxRpcRollbackSPResult &result); -int handle_sp_rollback_response(ObTxRollbackSPRespMsg &sp_rollbacck_resp, obcall::ObTransRpcResult &result); -int handle_sp_rollback_resp(const share::ObLSID &ls_id, - const int64_t orig_epoch, - const transaction::ObTransID &tx_id, - const int status, - const int64_t request_id, - const int64_t ret_epoch, - const ObAddr &ret_addr); -int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, - const share::ObLSID &receiver_ls_id, - const ObTransID &tx_id, - const int16_t msg_type, - const int status, - const ObAddr &receiver_addr, - const int64_t request_id, - const share::SCN &private_data); -int handle_trans_keepalive(const ObTxKeepaliveMsg &msg, obcall::ObTransRpcResult &result); -int handle_trans_keepalive_response(const ObTxKeepaliveRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_tx_batch_req(int type, const char* buf, int32_t size, const bool need_check_leader = true); -int refresh_location_cache(const share::ObLSID ls); +int report_write_ctx_status(const ObTransID &tx_id, const int status, int &tx_status); int handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay); int handle_tx_commit_result(const ObTransID &tx_id, const int result, @@ -123,67 +96,22 @@ ObTxCtxMgr &get_tx_ctx_mgr() { return tx_ctx_mgr_; } -int block_tx(const share::ObLSID &ls_id, bool &is_all_tx_cleaned_up); +int block_tx(bool &is_all_tx_cleaned_up); // block tx and readonly request -int iterate_tx_ctx_mgr_stat(ObTxCtxMgrStatIterator &tx_ctx_mgr_stat_iter); +int get_tx_ctx_mgr_stat(ObLSTxCtxMgrStat &tx_ctx_mgr_stat); int iterate_all_observer_tx_stat(ObTxStatIterator &tx_stat_iter); int iterate_tx_scheduler_stat(ObTxSchedulerStatIterator &tx_scheduler_stat_iter); -// for xa -/* - * recover transaction descriptor with tx info - */ -int handle_timeout_for_xa(ObTxDesc &tx, const int64_t delay); -int handle_sub_prepare_request(const ObTxSubPrepareMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_prepare_response(const ObTxSubPrepareRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_prepare_result(const ObTransID &tx_id, const int result); -int handle_sub_commit_request(const ObTxSubCommitMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_commit_response(const ObTxSubCommitRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_commit_result(const ObTransID &tx_id, const int result); -int handle_sub_rollback_request(const ObTxSubRollbackMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_rollback_response(const ObTxSubRollbackRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_sub_rollback_result(const ObTransID &tx_id, const int result); int gen_trans_id(ObTransID &trans_id); -//for standby -int check_and_fill_state_info(const ObTransID &tx_id, ObStateInfo &state_info); -int handle_trans_ask_state(const ObAskStateMsg &msg, obcall::ObTransRpcResult &result); -int handle_trans_ask_state_response(const ObAskStateRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_trans_collect_state(const ObCollectStateMsg &msg, obcall::ObTransRpcResult &result); -int handle_trans_collect_state_response(const ObCollectStateRespMsg &msg, obcall::ObTransRpcResult &result); -int handle_ls_deleted(const ObTxMsg &msg); -void build_tx_collect_state_resp_(ObCollectStateRespMsg &resp, const ObCollectStateMsg &msg); -void build_tx_ask_state_resp_(ObAskStateRespMsg &resp, const ObAskStateMsg &msg); -int check_for_standby(const share::ObLSID &ls_id, - const ObTransID &tx_id, - const SCN &snapshot, - bool &can_read, - SCN &trans_version); -int mds_infer_standby_trx_state(const ObLS *ls_ptr, - const ObLSID &ls_id, - const ObTransID &tx_id, - const SCN &snapshot, - ObTxCommitData::TxDataState &tx_data_state, - share::SCN &commit_version); -void register_standby_cleanup_task(); -int do_standby_cleanup(); - -// tx state check for 4377 -int ask_tx_state_for_4377(const ObLSID ls_id, - const ObTransID tx_id, - bool &is_alive); -int handle_ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - bool &is_alive); - TO_STRING_KV(K(is_inited_), KP(this)); private: -int check_ls_status_(const share::ObLSID &ls_id, bool &leader); int init_tx_(ObTxDesc &tx, const uint32_t session_id, const uint32_t client_sid, @@ -193,87 +121,37 @@ int start_tx_(ObTxDesc &tx); int abort_tx_(ObTxDesc &tx, const int cause, bool cleanup = true); void abort_tx__(ObTxDesc &tx, const bool cleanup); int finalize_tx_(ObTxDesc &tx); -int find_parts_after_sp_(ObTxDesc &tx, - ObTxPartRefList &parts, - const ObTxSEQ scn); +int find_write_state_after_savepoint_(ObTxDesc &tx, + ObTxWriteState *&part, + const ObTxSEQ scn); int rollback_savepoint_(ObTxDesc &tx, - ObTxPartRefList &parts, + ObTxWriteState *part, const ObTxSEQ savepoint, int64_t expire_ts); -int rollback_savepoint_slowpath_(ObTxDesc &tx, - ObTxRollbackParts &rollback_parts, - const ObTxSEQ specified_from_scn, - const ObTxSEQ scn, - const int64_t expire_ts); -void on_sp_rollback_succ_(const ObTxExecPart &part, - ObTxDesc &tx, - const int64_t born_epoch, - const ObAddr &addr); -int create_tx_ctx_(const share::ObLSID &ls_id, - const ObTxDesc &tx, - ObPartTransCtx *&ctx, +int create_tx_ctx_(const ObTxDesc &tx, + ObTxCtx *&ctx, bool &exist); -int create_tx_ctx_(const share::ObLSID &ls_id, - ObLS *ls, +int create_tx_ctx_(ObLS *ls, const ObTxDesc &tx, - ObPartTransCtx *&ctx, + ObTxCtx *&ctx, const bool special, bool &exist); -int get_tx_ctx_(const share::ObLSID &ls_id, - ObLS *ls, +int get_tx_ctx_(ObLS *ls, const ObTransID &tx_id, - ObPartTransCtx *&ctx); + ObTxCtx *&ctx); -int get_tx_ctx_(const share::ObLSID &ls_id, - const ObTransID &tx_id, - ObPartTransCtx *&ctx); -int get_tx_ctx_for_standby_(const share::ObLSID &ls_id, - const ObTransID &tx_id, - ObPartTransCtx *&ctx); -int revert_tx_ctx_(ObLS *ls, ObPartTransCtx *ctx); -int revert_tx_ctx_(ObPartTransCtx *ctx); +int get_tx_ctx_(const ObTransID &tx_id, + ObTxCtx *&ctx); +int revert_tx_ctx_(ObLS *ls, ObTxCtx *ctx); +int revert_tx_ctx_(ObTxCtx *ctx); int validate_snapshot_version_(const share::SCN snapshot, const int64_t expire_ts, ObLS &ls); -int check_replica_readable_(const ObTxReadSnapshot &snapshot, - const bool elr, - const ObTxReadSnapshot::SRC src, - const share::ObLSID &ls_id, - const int64_t expired_ts, - const ObTabletID &tablet_id, - ObLS &ls); -bool check_ls_readable_(ObLS &ls, - const share::SCN &snapshot, - const ObTxReadSnapshot::SRC src); -int build_tx_commit_msg_(const ObTxDesc &tx, ObTxCommitMsg &msg); -int abort_participants_(const ObTxDesc &tx_desc); -int acquire_local_snapshot_(const share::ObLSID &ls_id, - share::SCN &snapshot, - const bool is_read_only, - ObRole &role); -int sync_acquire_global_snapshot_(ObTxDesc &tx, - const int64_t expire_ts, - share::SCN &snapshot, - int64_t &uncertain_bound); -int acquire_global_snapshot__(const int64_t expire_ts, - const int64_t gts_ahead, - share::SCN &snapshot, - int64_t &uncertain_bound); -int batch_post_rollback_savepoint_msg_(ObTxDesc &tx, - ObTxRollbackSPMsg &msg, - const ObTxRollbackParts &list, - int &post_succ_num); -int post_tx_commit_msg_(ObTxDesc &tx_desc, - ObTxCommitMsg &msg, - ObITxCallback *cb); -int post_tx_abort_part_msg_(const ObTxDesc &tx_desc, - const ObTxPart &p); -bool is_sync_replica_(const share::ObLSID &ls_id); - -int handle_orphan_2pc_msg_(const ObTxMsg &msg, const bool need_check_leader, const bool ls_deleted); +int abort_write_state_(const ObTxDesc &tx_desc); +int acquire_local_snapshot_(share::SCN &snapshot); +int abort_write_ctx_(const ObTxDesc &tx_desc); -int update_max_read_ts_(const share::ObLSID &lsid, - const share::SCN ts); +int update_max_read_ts_(const share::SCN ts); int do_commit_tx_(ObTxDesc &tx, const int64_t expire_ts, ObITxCallback &cb, @@ -284,54 +162,25 @@ int unregister_commit_retry_task_(ObTxDesc &tx); int handle_tx_commit_result_(ObTxDesc &tx, const int result, const share::SCN commit_version = share::SCN()); -int decide_tx_commit_info_(ObTxDesc &tx, ObTxPart *&coord); int local_ls_commit_tx_(const ObTransID &tx_id, - const share::ObLSID &coord, - const ObTxCommitParts &parts, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id, const share::SCN commit_start_scn, - const int64_t epoch, - share::SCN &commit_version, - const common::ObAddr &caller); -int get_tx_state_from_tx_table_(const share::ObLSID &lsid, - const ObTransID &tx_id, + share::SCN &commit_version); +int get_tx_state_from_tx_table_(const ObTransID &tx_id, int64_t &state, share::SCN &commit_version) { share::SCN recycle_scn; - return get_tx_state_from_tx_table_(lsid, tx_id, state, commit_version, recycle_scn); + return get_tx_state_from_tx_table_(tx_id, state, commit_version, recycle_scn); } -int get_tx_state_from_tx_table_(const share::ObLSID &lsid, - const ObTransID &tx_id, +int get_tx_state_from_tx_table_(const ObTransID &tx_id, int64_t &state, share::SCN &commit_version, share::SCN &recycle_scn); OB_NOINLINE int gen_trans_id_(ObTransID &trans_id); bool commit_need_retry_(const int ret); -// for xa -int build_tx_sub_prepare_msg_(const ObTxDesc &tx, ObTxSubPrepareMsg &msg); -int build_tx_sub_commit_msg_(const ObTxDesc &tx, ObTxSubCommitMsg &msg); -int build_tx_sub_rollback_msg_(const ObTxDesc &tx, ObTxSubRollbackMsg &msg); -int sub_prepare_local_ls_(const ObTransID &tx_id, - const share::ObLSID &coord, - const ObTxCommitParts &parts, - const int64_t &expire_ts, - const common::ObString & app_trace_info, - const int64_t &request_id, - const ObXATransID &xid); -int handle_sub_prepare_timeout_(ObTxDesc &tx, const int64_t delay); -int handle_sub_rollback_timeout_(ObTxDesc &tx, const int64_t delay); -int handle_sub_commit_timeout_(ObTxDesc &tx, const int64_t delay); -int handle_sub_prepare_result_(ObTxDesc &tx, const int result); -int handle_sub_end_tx_result_(ObTxDesc &tx, const bool is_rollback, const int result); -int sub_end_tx_local_ls_(const ObTransID &tx_id, - const share::ObLSID &coord, - const int64_t &request_id, - const ObXATransID &xid, - const common::ObAddr &sender_addr, - const bool is_rollback); int assign_user_savepoint_(ObTxDesc &tx, ObTxSavePointList &savepoints); int update_user_savepoint_(ObTxDesc &tx, const ObTxSavePointList &savepoints); @@ -339,22 +188,13 @@ int update_user_savepoint_(ObTxDesc &tx, const ObTxSavePointList &savepoints); ObTxCtxMgr tx_ctx_mgr_; void invalid_registered_snapshot_(ObTxDesc &tx); void process_registered_snapshot_on_commit_(ObTxDesc &tx); -int ls_rollback_to_savepoint_(const ObTransID &tx_id, - const share::ObLSID &ls, - const int64_t verify_epoch, +int rollback_tx_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const ObTxSEQ savepoint, const int64_t tx_seq_base, - int64_t &ctx_born_epoch, - const ObTxDesc *tx, + const ObTxDesc &tx, const ObTxSEQ from_scn, int64_t expire_ts = -1); -int sync_rollback_savepoint__(ObTxDesc &tx, - ObTxRollbackSPMsg &msg, - RollbackMaskSet &mask_set, - int64_t expire_ts, - const int64_t max_retry_interval, - int &retries); int create_local_implicit_savepoint_(ObTxDesc &tx, ObTxSEQ &savepoint); int create_global_implicit_savepoint_(ObTxDesc &tx, @@ -368,7 +208,7 @@ int rollback_to_global_implicit_savepoint_(ObTxDesc &tx, const ObTxSEQ savepoint, const int64_t expire_ts, const ObTxCleanPolicy clean_policy); -int ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, +int sync_rollback_to_savepoint_(ObTxCtx *part_ctx, const ObTxSEQ savepoint, const int64_t op_sn, const int64_t tx_seq_base, @@ -378,12 +218,7 @@ void tx_post_terminate_(ObTxDesc &tx); int start_epoch_(ObTxDesc &tx); int tx_sanity_check_(ObTxDesc &tx); void fetch_cflict_tx_ids_from_mem_ctx_to_desc_(memtable::ObMvccAccessCtx &acc_ctx); -int wait_follower_readable_(ObLS &ls, - const int64_t expire_ts, - const share::SCN &snapshot, - const ObTxReadSnapshot::SRC src); MonotonicTs get_req_receive_mts_(); -bool is_ls_dropped_(const share::ObLSID ls_id); static bool common_retryable_error_(const int ret); void direct_execute_commit_cb_(ObTxDesc &tx); void adjust_tx_snapshot_(ObTxDesc &tx, ObTxReadSnapshot &snapshot); diff --git a/src/storage/tx/ob_trans_stat.cpp b/src/storage/tx/ob_trans_stat.cpp index b33646c31..424238555 100644 --- a/src/storage/tx/ob_trans_stat.cpp +++ b/src/storage/tx/ob_trans_stat.cpp @@ -41,7 +41,6 @@ void ObTransStat::reset() state_ = static_cast(ObTxState::UNKNOWN); session_id_ = 0; proxy_session_id_ = 0; - trans_type_ = TransType::UNKNOWN_TRANS; part_trans_action_ = ObPartTransAction::UNKNOWN; lock_for_read_retry_count_ = 0; ctx_addr_ = 0; diff --git a/src/storage/tx/ob_trans_stat.h b/src/storage/tx/ob_trans_stat.h index 205f39715..53184ac6a 100644 --- a/src/storage/tx/ob_trans_stat.h +++ b/src/storage/tx/ob_trans_stat.h @@ -46,7 +46,6 @@ class ObTransStat int64_t get_trans_refer_cnt() const { return refer_; } int64_t get_sql_no() const { return sql_no_; } int64_t get_state() const { return state_; } - int get_trans_type() const { return trans_type_; } int64_t get_part_trans_action() const { return part_trans_action_; } uint64_t get_lock_for_read_retry_count() const { return lock_for_read_retry_count_; } int64_t get_ctx_addr() const { return ctx_addr_; } @@ -60,7 +59,7 @@ class ObTransStat TO_STRING_KV(K_(addr), K_(trans_id), K_(is_exiting), K_(is_readonly), K_(has_decided), K_(is_dirty), K_(active_memstore_version), K_(trans_param), K_(ctx_create_time), K_(expired_time), K_(refer), - K_(sql_no), K_(state), K_(session_id), K_(proxy_session_id), K_(trans_type), + K_(sql_no), K_(state), K_(session_id), K_(proxy_session_id), K_(ctx_addr), K_(prev_trans_arr), K_(next_trans_arr), K_(prev_trans_commit_count), K_(ctx_id), K_(pending_log_size), K_(flushed_log_size)); private: @@ -80,8 +79,6 @@ class ObTransStat int64_t state_; uint32_t session_id_; uint64_t proxy_session_id_; - //SP_TRANS,MINI_SP_TRANS,DIST_TRANS - int trans_type_; int64_t part_trans_action_; uint64_t lock_for_read_retry_count_; int64_t ctx_addr_; diff --git a/src/storage/tx/ob_trans_submit_log_cb.cpp b/src/storage/tx/ob_trans_submit_log_cb.cpp index 5a71fa0ab..a8b289227 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.cpp +++ b/src/storage/tx/ob_trans_submit_log_cb.cpp @@ -16,7 +16,7 @@ #include "ob_trans_submit_log_cb.h" #include "share/rc/ob_module_provider.h" -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" #include "storage/allocator/ob_shared_memory_allocator_mgr.h" namespace oceanbase @@ -182,16 +182,15 @@ int ObTxLogCb::on_success() const int64_t bk_log_size = log_size_; const bool bk_is_reserved = group_ptr_->is_reserved(); ObTxLogCbGroup *bk_group_ptr = group_ptr_; - ObPartTransCtx *part_ctx = group_ptr_->get_tx_ctx(); + ObTxCtx *part_ctx = group_ptr_->get_tx_ctx(); const ObTransID tx_id = part_ctx->get_trans_id(); - const ObLSID ls_id = part_ctx->get_ls_id(); const share::SCN log_ts = log_ts_; if (NULL == part_ctx) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "ctx is null", K(ret), KPC(part_ctx)); } else { if (OB_FAIL(part_ctx->on_success(this))) { - TRANS_LOG(WARN, "sync log success callback error", K(ret), K(tx_id), K(ls_id), K(log_ts)); + TRANS_LOG(WARN, "sync log success callback error", K(ret), K(tx_id), K(log_ts)); } } } @@ -210,19 +209,18 @@ int ObTxLogCb::on_failure() const int64_t bk_log_size = log_size_; const bool bk_is_reserved = group_ptr_->is_reserved(); ObTxLogCbGroup *bk_group_ptr = group_ptr_; - ObPartTransCtx *part_ctx = group_ptr_->get_tx_ctx(); + ObTxCtx *part_ctx = group_ptr_->get_tx_ctx(); const ObTransID tx_id = part_ctx->get_trans_id(); - const ObLSID ls_id = part_ctx->get_ls_id(); const share::SCN log_ts = log_ts_; if (NULL == part_ctx) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "ctx is null", KR(ret), K(*this)); } else { if (OB_FAIL(part_ctx->on_failure(this))) { - TRANS_LOG(WARN, "sync log success callback error", KR(ret), K(tx_id), K(ls_id), K(log_ts)); + TRANS_LOG(WARN, "sync log success callback error", KR(ret), K(tx_id), K(log_ts)); } } - TRANS_LOG(INFO, "ObTxLogCb::on_failure end", KR(ret), K(tx_id), K(ls_id), K(log_ts)); + TRANS_LOG(INFO, "ObTxLogCb::on_failure end", KR(ret), K(tx_id), K(log_ts)); } return ret; } diff --git a/src/storage/tx/ob_trans_submit_log_cb.h b/src/storage/tx/ob_trans_submit_log_cb.h index cfa0b5c7f..49658ec59 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.h +++ b/src/storage/tx/ob_trans_submit_log_cb.h @@ -25,7 +25,6 @@ #include "ob_trans_define.h" #include "ob_tx_ctx_mds.h" #include "ob_trans_event.h" -#include "share/ob_ls_id.h" #include "logservice/ob_log_handler.h" #include "logservice/ob_append_callback.h" #include "lib/list/ob_dlink_node.h" @@ -45,7 +44,7 @@ class ObTxData; namespace transaction { class ObTransService; -class ObPartTransCtx; +class ObTxCtx; class ObTxLogCbGroup; } diff --git a/src/storage/tx/ob_trans_timer.cpp b/src/storage/tx/ob_trans_timer.cpp index 50506d564..515bb6acb 100644 --- a/src/storage/tx/ob_trans_timer.cpp +++ b/src/storage/tx/ob_trans_timer.cpp @@ -165,14 +165,8 @@ void ObTxTimeoutTask::runTimerTask() ObTransService *txs = txs_; ObTxDesc *tx_desc = tx_desc_; DEFER({ txs->release_tx_ref(*tx_desc); }); - if (tx_desc_->is_xa_trans() && tx_desc_->is_sub2pc()) { - if (OB_FAIL(txs_->handle_timeout_for_xa(*tx_desc_, delay_))) { - TRANS_LOG(WARN, "fail to handle timeout", K(ret), KPC_(tx_desc)); - } - } else { - if (OB_FAIL(txs_->handle_tx_commit_timeout(*tx_desc_, delay_))) { - TRANS_LOG(WARN, "handle timeout fail", K(ret), KPC_(tx_desc)); - } + if (OB_FAIL(txs_->handle_tx_commit_timeout(*tx_desc_, delay_))) { + TRANS_LOG(WARN, "handle timeout fail", K(ret), KPC_(tx_desc)); } } } diff --git a/src/storage/tx/ob_ts_mgr.cpp b/src/storage/tx/ob_ts_mgr.cpp index 1e7b178d6..d58eff463 100644 --- a/src/storage/tx/ob_ts_mgr.cpp +++ b/src/storage/tx/ob_ts_mgr.cpp @@ -15,464 +15,53 @@ */ #include "ob_ts_mgr.h" -#include "ob_gts_rpc.h" -#include "share/ob_share_util.h" // relocated-definition owner +#include "ob_timestamp_access.h" +#include "share/config/ob_server_config.h" +#include "share/rc/ob_module_provider.h" namespace oceanbase { using namespace common; using namespace share; -using namespace share::schema; -using namespace obcall; namespace transaction { -////////////////////////ObTsMgr implementation/////////////////////////////////// -int ObTsMgr::init(const ObAddr &server, - share::schema::ObMultiVersionSchemaService &schema_service, - share::ObLocationService &location_service) +int ObTsMgr::get_gts(SCN &scn) { int ret = OB_SUCCESS; - - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "ObTsMgr inited twice", KR(ret)); - } else if (!server.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(server)); - } else if (OB_FAIL(location_adapter_def_.init(&schema_service, &location_service))) { - TRANS_LOG(ERROR, "location adapter init error", KR(ret)); - } else if (OB_ISNULL(gts_request_rpc_ = ObGtsRequestRpcFactory::alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc gts_reqeust_rpc fail", KR(ret)); - } else if (OB_FAIL(ts_worker_.init(this, true))) { - TRANS_LOG(WARN, "ts worker init failed", KR(ret)); - } else if (OB_FAIL(gts_request_rpc_->init(server, this, &ts_worker_))) { - TRANS_LOG(WARN, "response rpc init failed", KR(ret), K(server)); - } else if (FALSE_IT(location_adapter_ = &location_adapter_def_)) { - } else if (OB_FAIL(ts_source_.init(server, gts_request_rpc_, location_adapter_))) { - TRANS_LOG(WARN, "ts source init failed", KR(ret)); - } else { - server_ = server; - is_inited_ = true; - TRANS_LOG(INFO, "ObTsMgr inited success", KP(this), K(server)); - } - - if (OB_FAIL(ret)) { - if (NULL != gts_request_rpc_) { - ObGtsRequestRpcFactory::release(gts_request_rpc_); - gts_request_rpc_ = NULL; - } - } - - return ret; -} - -void ObTsMgr::reset() -{ - is_inited_ = false; - is_running_ = false; - ts_source_.reset(); - server_.reset(); - location_adapter_ = NULL; - gts_request_rpc_ = NULL; -} - -int ObTsMgr::start() -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ObTsMgr is already running", KR(ret)); - } else if (OB_FAIL(gts_request_rpc_->start())) { - TRANS_LOG(WARN, "gts request rpc start", KR(ret)); - // Start gts task refresh thread - } else if (OB_FAIL(share::ObThreadPool::start())) { - TRANS_LOG(ERROR, "GTS local cache manager refresh worker thread start error", KR(ret)); - } else { - is_running_ = true; - TRANS_LOG(INFO, "ObTsMgr start success"); - } - return ret; -} - -void ObTsMgr::stop() -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", KR(ret)); - } else if (OB_FAIL(gts_request_rpc_->stop())) { - TRANS_LOG(WARN, "gts request rpc stop", KR(ret)); - } else { - (void)share::ObThreadPool::stop(); - (void)ts_worker_.stop(); - is_running_ = false; - TRANS_LOG(INFO, "ObTsMgr stop success"); - } -} - -void ObTsMgr::wait() -{ - int ret = OB_SUCCESS; - - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", KR(ret)); - } else if (is_running_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ObTsMgr is running", KR(ret)); - } else if (OB_FAIL(gts_request_rpc_->wait())) { - TRANS_LOG(WARN, "gts request rpc wait", KR(ret)); - } else { - (void)share::ObThreadPool::wait(); - (void)ts_worker_.wait(); - TRANS_LOG(INFO, "ObTsMgr wait success"); - } -} - -void ObTsMgr::destroy() -{ - if (is_inited_) { - if (is_running_) { - stop(); - wait(); - } - (void)share::ObThreadPool::destroy(); - (void)ts_worker_.destroy(); - - location_adapter_def_.destroy(); - server_.reset(); - location_adapter_ = NULL; - is_running_ = false; - is_inited_ = false; - TRANS_LOG(INFO, "ObTsMgr destroyed"); - } - if (NULL != gts_request_rpc_) { - ObGtsRequestRpcFactory::release(gts_request_rpc_); - gts_request_rpc_ = NULL; - } -} -// Execute gts task refresh, by a dedicated thread to be responsible -void ObTsMgr::run1() -{ - int ret = OB_SUCCESS; - // cluster version less than 2.0 will not update gts - lib::set_thread_name("TsMgr"); - while (!has_set_stop()) { - int tmp_ret = OB_SUCCESS; - // sleep 100 * 1000 us - ob_usleep(REFRESH_GTS_INTERVEL_US, true/*is_idle_sleep*/); - if (OB_SUCCESS != (tmp_ret = ts_source_.refresh_gts(false))) { - if (EXECUTE_COUNT_PER_SEC(1)) { - TRANS_LOG(WARN, "refresh gts failed", K(tmp_ret)); - } - } - if (EXECUTE_COUNT_PER_SEC(1)) { - TRANS_LOG(INFO, "refresh gts", KR(ret)); - } - } -} - -int ObTsMgr::handle_gts_err_response(const ObGtsErrResponse &msg) -{ - int ret = OB_SUCCESS; - ObTimeGuard timeguard("handle_gts_err_response", 100000); - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(!msg.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(msg)); - } else if (OB_FAIL(ts_source_.handle_gts_err_response(msg))) { - TRANS_LOG(WARN, "handle gts err response error", KR(ret), K(msg)); - } else { - // do nothing - } - - return ret; -} - -int ObTsMgr::refresh_gts_location() -{ - int ret = OB_SUCCESS; - ObTimeGuard timeguard("refresh_gts_location", 100000); - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_FAIL(ts_source_.refresh_gts_location())) { - TRANS_LOG(WARN, "refresh gts location error", K(ret)); - } else { - // do nothing - } - - return ret; -} - -int ObTsMgr::handle_gts_result(const int64_t queue_index, const int ts_type) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_FAIL(ts_source_.handle_gts_result(queue_index))) { - TRANS_LOG(WARN, "handle gts result error", KR(ret)); - } else { - // do nothing - } - return ret; -} - -int ObTsMgr::update_gts(const MonotonicTs srr, - const int64_t gts, - const int ts_type, - bool &update) -{ - int ret = OB_SUCCESS; - const MonotonicTs receive_gts_ts = MonotonicTs::current_time(); - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(0 >= gts)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts)); - } else if (OB_FAIL(ts_source_.update_gts(srr, gts, receive_gts_ts, update))) { - TRANS_LOG(WARN, "update gts cache failed", KR(ret), K(srr), K(gts)); - } else { - // do nothing - } - - return ret; -} - - -int ObTsMgr::interrupt_gts_callbacks() -{ - int ret = OB_SUCCESS; - share::ObLSID ls_id; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else { - const int64_t task_count = ts_source_.get_task_count(); - if (0 != task_count) { - ret = ts_source_.gts_callback_interrupted(OB_TENANT_NOT_EXIST, ls_id); - } - if (OB_SUCCESS != ret) { - TRANS_LOG(WARN, "interrupt gts callbacks failed", KR(ret), K(ls_id)); - } else { - TRANS_LOG(INFO, "interrupt gts callbacks success", K(ls_id)); - } - } - return ret; -} - -int ObTsMgr::update_gts(const int64_t gts, bool &update) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(0 >= gts) || - OB_UNLIKELY(gts > ObTimeUtility::current_time_ns() + 86400000000000L)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(gts)); - } else if (OB_FAIL(ts_source_.update_gts(gts, update))) { - TRANS_LOG(WARN, "update gts cache failed", K(ret), K(gts)); - } - - return ret; -} - -int ObTsMgr::get_gts(ObTsCbTask *task, SCN &scn) -{ - int ret = OB_SUCCESS; - int64_t gts = 0;//need be invalid value for SCN - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_FAIL(ts_source_.get_gts(task, gts))) { + int64_t gts = 0; + if (OB_FAIL(share::g_mp->timestamp_access()->get_number(gts))) { if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "get gts error", K(ret), KP(task)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(scn.convert_for_gts(gts))) { - TRANS_LOG(WARN, "failed to convert_for_gts", K(ret), K(gts)); + TRANS_LOG(WARN, "get local timestamp failed", KR(ret)); } + } else if (OB_FAIL(scn.convert_for_gts(gts))) { + TRANS_LOG(WARN, "convert local timestamp failed", KR(ret), K(gts)); } - return ret; } int ObTsMgr::get_gts(const MonotonicTs stc, - ObTsCbTask *task, SCN &scn, MonotonicTs &receive_gts_ts) { - int ret = OB_SUCCESS; - int64_t gts = 0;//need be invalid value for SCN - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(!stc.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(stc), KP(task)); - } else if (OB_FAIL(ts_source_.get_gts(stc, task, gts, receive_gts_ts))) { - if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "get gts error", K(ret), K(stc), KP(task)); - } - } - + int ret = get_gts(scn); if (OB_SUCC(ret)) { - if (OB_FAIL(scn.convert_for_gts(gts))) { - TRANS_LOG(WARN, "failed to convert_for_gts", K(ret), K(gts)); - } + receive_gts_ts = MonotonicTs::current_time(); + TRANS_LOG(DEBUG, "get local timestamp", K(stc), K(scn), K(receive_gts_ts)); } return ret; } -int ObTsMgr::get_ts_sync(const int64_t timeout_us, share::SCN &scn) +int ObTsMgr::get_gts_sync(const int64_t timeout_us, SCN &scn) { - bool unused_is_external_consistent = false; - return get_ts_sync(timeout_us, scn, unused_is_external_consistent); -} - -int ObTsMgr::get_gts_sync(const MonotonicTs stc, - const int64_t timeout_us, - share::SCN &scn, - MonotonicTs &receive_gts_ts) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(!stc.is_valid()) - || OB_UNLIKELY(timeout_us < 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(stc), K(timeout_us)); - } else { - int64_t expire_ts = ObClockGenerator::getClock() + timeout_us; - int retry_times = 0; - const int64_t SLEEP_TIME_US = 500; - do { - const int64_t now = ObClockGenerator::getClock(); - int64_t gts_result = 0; - if (now >= expire_ts) { - ret = OB_TIMEOUT; - } else if (OB_FAIL(ts_source_.get_gts(stc, NULL, gts_result, receive_gts_ts))) { - if (OB_EAGAIN == ret) { - ob_usleep(SLEEP_TIME_US); - } else { - TRANS_LOG(WARN, "get gts fail", K(ret), K(now)); - } - } else { - scn.convert_for_gts(gts_result); - } - } while (OB_EAGAIN == ret); - } - - return ret; -} - -int ObTsMgr::get_ts_sync(const int64_t timeout_us, - SCN &scn, - bool &is_external_consistent) -{ - int ret = OB_SUCCESS; - const int64_t start = ObTimeUtility::current_time(); - const MonotonicTs stc = MonotonicTs::current_time(); - MonotonicTs receive_gts_ts; - int64_t sleep_us = 100 * 1000; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); - } else if (OB_UNLIKELY(timeout_us < 0)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(timeout_us)); - } else { - do { - int64_t ts = 0; - if (OB_FAIL(ts_source_.get_gts(stc, NULL, ts, receive_gts_ts))) { - if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "get gts error", K(ret), K(stc)); - } else { - ob_usleep(sleep_us); - sleep_us = sleep_us * 2; - sleep_us = (sleep_us >= 1000000 ? 1000000 : sleep_us); - // rewrite ret - ret = OB_SUCCESS; - } - } else { - scn.convert_for_gts(ts); - is_external_consistent = true; - break; - } - } while (OB_SUCCESS == ret); - } - - return ret; -} - -int ObTsMgr::wait_gts_elapse(const SCN &scn, - ObTsCbTask *task, bool &need_wait) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - // } else if (OB_UNLIKELY(!is_running_)) { - // ret = OB_NOT_RUNNING; - // TRANS_LOG(WARN, "ObTsMgr not running", K(ret)); - } else if (OB_UNLIKELY(!scn.is_valid()) - || OB_ISNULL(task)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(scn), KP(task)); - } else { - const int64_t ts = scn.get_val_for_gts(); - if (OB_FAIL(ts_source_.wait_gts_elapse(ts, task, need_wait))) { - TRANS_LOG(WARN, "wait gts elapse failed", K(ret), K(ts), KP(task)); + int ret = OB_EAGAIN; + const int64_t expire_ts = ObClockGenerator::getClock() + timeout_us; + while (OB_EAGAIN == ret) { + if (ObClockGenerator::getClock() >= expire_ts) { + ret = OB_TIMEOUT; + } else if (OB_EAGAIN == (ret = get_gts(scn))) { + ob_usleep(500); } } return ret; @@ -480,157 +69,29 @@ int ObTsMgr::wait_gts_elapse(const SCN &scn, int ObTsMgr::wait_gts_elapse(const SCN &scn) { - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!is_running_)) { - ret = OB_NOT_RUNNING; - TRANS_LOG(WARN, "ObTsMgr not running", K(ret)); - } else if (OB_UNLIKELY(!scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(scn)); - } else { - const int64_t ts = scn.get_val_for_gts(); - if (OB_FAIL(ts_source_.wait_gts_elapse(ts))) { - if (OB_EAGAIN != ret) { - TRANS_LOG(WARN, "wait gts elapse fail", K(ret), K(ts)); + int ret = OB_EAGAIN; + const int64_t expire_ts = ObClockGenerator::getClock() + GCONF.rpc_timeout; + while (OB_EAGAIN == ret) { + SCN current_scn; + if (ObClockGenerator::getClock() >= expire_ts) { + ret = OB_TIMEOUT; + } else if (OB_FAIL(get_gts(current_scn))) { + if (OB_EAGAIN == ret) { + ob_usleep(500); } + } else if (current_scn < scn) { + ret = OB_EAGAIN; + ob_usleep(500); } } - return ret; } -ObTsMgr *&ObTsMgr::get_instance_inner() -{ - static ObTsMgr instance; - static ObTsMgr *instance2 = &instance; - return instance2; -} - ObTsMgr &ObTsMgr::get_instance() { - return *get_instance_inner(); -} - -int ObTsMgr::interrupt_gts_callback_for_ls_offline(const share::ObLSID ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); - } else { - const int64_t task_count = ts_source_.get_task_count(); - if (0 != task_count) { - ret = ts_source_.gts_callback_interrupted(OB_LS_OFFLINE, ls_id); - } - - if (OB_SUCCESS != ret) { - TRANS_LOG(WARN, "interrupt gts callback failed", KR(ret), K(ls_id)); - } else { - TRANS_LOG(INFO, "interrupt gts callback success", K(ls_id)); - } - } - return ret; -} - -} // transaction -} // oceanbase - -#define USING_LOG_PREFIX SHARE // needed by LOG_* macros inside the relocated block; the file did not define it originally -// ===== definition moved from share/ob_share_util.cpp ===== -// removes share→storage/tx inverted include; declaration remains in share/ob_share_util.h, resolved at link time(transitional state, final state should split the class) -namespace oceanbase -{ -namespace share -{ - -// get_tenant_gts has been demoted to storage::free function(see end of file) - - -} // namespace share -} // namespace oceanbase - -// ===== definition moved from src/share/backup/ob_backup_struct.cpp (ObBackupUtils -> ObTsMgr) ===== -namespace oceanbase -{ -namespace transaction -{ - -int ObTsMgr::get_backup_scn(const uint64_t &tenant_id, share::SCN &scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else { - ret = OB_EAGAIN; - const transaction::MonotonicTs stc = transaction::MonotonicTs::current_time(); - transaction::MonotonicTs unused_ts(0); - const int64_t start_time = ObTimeUtility::fast_current_time(); - const int64_t TIMEOUT = 10 * 1000 * 1000; //10s - while (OB_EAGAIN == ret) { - if (ObTimeUtility::fast_current_time() - start_time > TIMEOUT) { - ret = OB_TIMEOUT; - LOG_WARN("stmt is timeout", KR(ret), K(start_time), K(TIMEOUT)); - } else if (OB_FAIL(OB_TS_MGR.get_gts(stc, NULL, - scn, unused_ts))) { - if (OB_EAGAIN != ret) { - LOG_WARN("failed to get gts", KR(ret), K(tenant_id)); - } else { - // waiting 10ms - ob_usleep(10L * 1000L); - } - } - } - } - LOG_INFO("get tenant gts", KR(ret), K(tenant_id), K(scn)); - return ret; + static ObTsMgr instance; + return instance; } -} // namespace transaction -} // namespace oceanbase -#undef USING_LOG_PREFIX // prevent unity-build leakage into later TUs - -// from share::ObShareUtil demoted(A-setmember split) -#define USING_LOG_PREFIX SHARE -namespace oceanbase -{ -namespace transaction -{ -using namespace oceanbase::share; -int get_tenant_gts(SCN >s_scn) -{ - int ret = OB_SUCCESS; - { - ret = OB_EAGAIN; - const transaction::MonotonicTs stc = transaction::MonotonicTs::current_time(); - transaction::MonotonicTs unused_ts(0); - const int64_t start_time = ObTimeUtility::fast_current_time(); - const int64_t TIMEOUT = GCONF.rpc_timeout; - while (OB_EAGAIN == ret) { - if (ObTimeUtility::fast_current_time() - start_time > TIMEOUT) { - ret = OB_TIMEOUT; - LOG_WARN("stmt is timeout", KR(ret), K(start_time), K(TIMEOUT)); - } else if (OB_FAIL(OB_TS_MGR.get_gts(stc, NULL, - gts_scn, unused_ts))) { - if (OB_EAGAIN != ret) { - LOG_WARN("failed to get gts", KR(ret)); - } else { - // waiting 10ms - ob_usleep(10L * 1000L); - } - } - } - } - LOG_INFO("get tenant gts", KR(ret), K(gts_scn)); - return ret; -} -} -} -#undef USING_LOG_PREFIX +} // namespace transaction +} // namespace oceanbase diff --git a/src/storage/tx/ob_ts_mgr.h b/src/storage/tx/ob_ts_mgr.h index 71b56a230..bc993b14e 100644 --- a/src/storage/tx/ob_ts_mgr.h +++ b/src/storage/tx/ob_ts_mgr.h @@ -17,181 +17,36 @@ #ifndef OCEANBASE_TRANSACTION_OB_TS_MGR_ #define OCEANBASE_TRANSACTION_OB_TS_MGR_ -#include -#include "lib/utility/ob_print_utils.h" -#include "lib/lock/ob_drw_lock.h" -#include "lib/hash/ob_link_hashmap.h" -#include "lib/atomic/atomic128.h" -#include "lib/net/ob_addr.h" -#include "lib/queue/ob_link_queue.h" -#include "lib/container/ob_iarray.h" -#include "share/ob_errno.h" -#include "share/ob_thread_pool.h" -#include "lib/lock/ob_qsync_lock.h" -#include "ob_gts_source.h" -#include "ob_gts_define.h" -#include "ob_ts_worker.h" -#include "ob_location_adapter.h" - -#define REFRESH_GTS_INTERVEL_US (500 * 1000) +#include "ob_trans_define.h" namespace oceanbase { namespace share { -class ObLocationService; -namespace schema -{ -class ObSchemaGetterGuard; -class ObMultiVersionSchemaService; -} -} -namespace obcall -{ -} -namespace rpc -{ -namespace frame -{ -class ObReqTransport; -} -} -namespace share -{ class SCN; } namespace transaction { -class ObLocationAdapter; -class ObGtsRequestRpc; -class ObIGlobalTimestampService; -class ObTsCbTask : public common::ObLink +class ObTsMgr { public: - ObTsCbTask() {} - virtual ~ObTsCbTask() {} - virtual int gts_callback_interrupted(const int errcode, const share::ObLSID ls_id) = 0; - virtual int get_gts_callback(const MonotonicTs srr, const share::SCN >s, const MonotonicTs receive_gts_ts) = 0; - virtual int gts_elapse_callback(const MonotonicTs srr, const share::SCN >s) = 0; - virtual MonotonicTs get_stc() const = 0; - virtual uint64_t hash() const = 0; - - VIRTUAL_TO_STRING_KV("", ""); -}; + ObTsMgr() = default; + virtual ~ObTsMgr() = default; -class ObITsMgr -{ -public: - virtual int update_gts(const int64_t gts, bool &update) = 0; + virtual int get_gts(share::SCN &scn); virtual int get_gts(const MonotonicTs stc, - ObTsCbTask *task, share::SCN &scn, - MonotonicTs &receive_gts_ts) = 0; - virtual int get_gts_sync(const MonotonicTs stc, - const int64_t timeout_us, - share::SCN &scn, - MonotonicTs &receive_gts_ts) = 0; - - virtual int get_gts(ObTsCbTask *task, share::SCN &scn) = 0; - virtual int get_ts_sync(const int64_t timeout_ts, - share::SCN &scn, bool &is_external_consistent) = 0; - virtual int get_ts_sync(const int64_t timeout_ts, share::SCN &scn) = 0; - virtual int wait_gts_elapse(const share::SCN &scn, ObTsCbTask *task, - bool &need_wait) = 0; - virtual int wait_gts_elapse(const share::SCN &scn) = 0; - virtual bool is_external_consistent() = 0; - virtual int interrupt_gts_callbacks() = 0; - virtual int interrupt_gts_callback_for_ls_offline(const share::ObLSID ls_id) = 0; -public: - VIRTUAL_TO_STRING_KV("", ""); -}; - -class ObTsMgr; -class ObTsMgr : public share::ObThreadPool, public ObITsMgr -{ -public: - ObTsMgr() { reset(); } - ~ObTsMgr() { destroy(); } - int init(const common::ObAddr &server, - share::schema::ObMultiVersionSchemaService &schema_service, - share::ObLocationService &location_service); - void reset(); - int start(); - void stop(); - void wait(); - void destroy(); - void run1(); + MonotonicTs &receive_gts_ts); + virtual int get_gts_sync(const int64_t timeout_us, share::SCN &scn); + virtual int wait_gts_elapse(const share::SCN &scn); - int handle_gts_err_response(const ObGtsErrResponse &msg); - int handle_gts_result(const int64_t queue_index, const int ts_type); - int update_gts(const MonotonicTs srr, const int64_t gts, const int ts_type, bool &update); - int interrupt_gts_callback_for_ls_offline(const share::ObLSID ls_id); -public: - int update_gts(const int64_t gts, bool &update); - // According to stc get the appropriate gts value, if the conditions are not met, need to register gts task, wait for asynchronous callback - int get_gts(const MonotonicTs stc, - ObTsCbTask *task, - share::SCN &scn, - MonotonicTs &receive_gts_ts); - /** - * The synchronous interface corresponding to `get_gts`, used for synchronously obtaining an appropriate GTS timestamp, with a timeout parameter to avoid long waits. - * Compared to the original synchronous interface `get_ts_sync`, this interface has better performance. - * @param[in] stc: The point in time to obtain the GTS, generally current time - * @param[in] timeout_us: Timeout duration, unit us - * @param[out] scn: The result of the obtained GTS timestamp - * @param[out] receive_gts_ts: The point in time when the GTS response was received - */ - int get_gts_sync(const MonotonicTs stc, - const int64_t timeout_us, - share::SCN &scn, - MonotonicTs &receive_gts_ts); - //Only get the latest value from local gts cache, but it may fail, failure handling logic as follows: - //1. If task == NULL, it means the caller does not need asynchronous callbacks, directly return the error, and let the caller handle it - //2. If task != NULL, need to register asynchronous callback task - int get_gts(ObTsCbTask *task, share::SCN &scn); - int get_ts_sync(const int64_t timeout_us, - share::SCN &scn, bool &is_external_consistent); - int get_ts_sync(const int64_t timeout_us, share::SCN &scn); - int wait_gts_elapse(const share::SCN &scn, ObTsCbTask *task, - bool &need_wait); - int wait_gts_elapse(const share::SCN &scn); - bool is_external_consistent() { return true; } - int refresh_gts_location(); - int interrupt_gts_callbacks(); -public: - TO_STRING_KV("ts_source", "GTS"); -public: - // get current tenant GTS as archive start snapshot (retry up to 10s). - // relocated from share::ObBackupUtils::get_backup_scn (module-boundary: share must not depend on storage/tx) - static int get_backup_scn(const uint64_t &tenant_id, share::SCN &scn); static ObTsMgr &get_instance(); -private: -private: - static ObTsMgr* &get_instance_inner(); -private: - bool is_inited_; - bool is_running_; - common::ObAddr server_; - ObGtsRequestRpc *gts_request_rpc_; - ObLocationAdapter *location_adapter_; - ObLocationAdapter location_adapter_def_; - ObTsWorker ts_worker_; - ObGtsSource ts_source_; }; #define OB_TS_MGR (::oceanbase::transaction::ObTsMgr::get_instance()) -} -}//end of namespace oceanbase - -namespace oceanbase -{ -namespace transaction -{ -// demoted from share::ObShareUtil(GTS query convenience wrapper, uses OB_TS_MGR.get_gts plus waiting internally) -int get_tenant_gts(share::SCN >s_scn); -} -} +} // namespace transaction +} // namespace oceanbase -#endif //OCEANBASE_TRANSACTION_OB_TS_MGR_ +#endif // OCEANBASE_TRANSACTION_OB_TS_MGR_ diff --git a/src/storage/tx/ob_ts_response_handler.cpp b/src/storage/tx/ob_ts_response_handler.cpp deleted file mode 100644 index 08e86b147..000000000 --- a/src/storage/tx/ob_ts_response_handler.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX TRANS - -#include "ob_ts_response_handler.h" -#include "ob_ts_mgr.h" - -using namespace oceanbase::transaction; -using namespace oceanbase::observer; -using namespace oceanbase::common; -using namespace oceanbase::obcall; - -int64_t ObTsResponseTaskFactory::alloc_count_; -int64_t ObTsResponseTaskFactory::free_count_; - -void ObTsResponseHandler::reset() -{ - task_ = NULL; - ts_mgr_ = NULL; -} - -int ObTsResponseHandler::init(observer::ObSrvTask *task, ObTsMgr *ts_mgr) -{ - int ret = OB_SUCCESS; - - if (NULL == task || NULL == ts_mgr) { - ret = OB_INVALID_ARGUMENT;; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(task), KP(ts_mgr)); - } else { - //Indicates that the task is a task generated internally by the Observer - task_ = task; - ts_mgr_ = ts_mgr; - } - - return ret; -} - -int ObTsResponseHandler::run() -{ - int ret = OB_SUCCESS; - if (NULL == task_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "task is null, unexpected error", KR(ret), KP_(task)); - } else if (NULL == ts_mgr_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ts mgr is null, unexpected error", KR(ret), KP_(ts_mgr)); - } else { - ObTsResponseTask *task = static_cast(task_); - if (OB_FAIL(ts_mgr_->handle_gts_result(task->get_arg1(), task->get_ts_type()))) { - TRANS_LOG(WARN, "handle gts result failed", KR(ret), K(*task)); - } - //op_reclaim_free(task); - //task = NULL; - } - return ret; -} - -void ObTsResponseTask::reset() -{ - arg1_ = 0; - handler_.reset(); - ts_type_ = TS_SOURCE_UNKNOWN; -} - -int ObTsResponseTask::init(const int64_t arg1, - ObTsMgr *ts_mgr, - int ts_type) -{ - int ret = OB_SUCCESS; - - if (!true - || NULL == ts_mgr - || !is_valid_ts_source(ts_type)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(arg1), KP(ts_mgr), K(ts_type)); - } else if (OB_FAIL(handler_.init(this, ts_mgr))) { - TRANS_LOG(WARN, "ObTsResponseHandler init error", KR(ret)); - } else { - //Different from the task of sql disconnection, it is used for memory release - set_type(ObRequest::OB_TS_TASK); - arg1_ = arg1; - ts_type_ = ts_type; - } - - return ret; -} - -ObTsResponseTask *ObTsResponseTaskFactory::alloc() -{ - ObTsResponseTask *task = NULL; - if (NULL != (task = op_reclaim_alloc(ObTsResponseTask))) { - (void)ATOMIC_FAA(&alloc_count_, 1); - alloc_count_++; - if (REACH_TIME_INTERVAL(3 * 1000 * 1000)) { - TRANS_LOG(INFO, "ts response task statistics", K_(alloc_count), K_(free_count)); - } - } - return task; -} - -void ObTsResponseTaskFactory::free(ObTsResponseTask *task) -{ - if (NULL != task) { - op_reclaim_free(task); - task = NULL; - (void)ATOMIC_FAA(&free_count_, 1); - } -} - diff --git a/src/storage/tx/ob_ts_response_handler.h b/src/storage/tx/ob_ts_response_handler.h deleted file mode 100644 index d626a37c5..000000000 --- a/src/storage/tx/ob_ts_response_handler.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TS_RESPONSE_HANDLER_ -#define OCEANBASE_TRANSACTION_OB_TS_RESPONSE_HANDLER_ - -#include "rpc/frame/ob_req_processor.h" -#include "observer/ob_srv_task.h" -#include "ob_gts_msg.h" -#include "ob_gts_define.h" - -namespace oceanbase -{ -namespace transaction -{ -class ObTsMgr; -class ObTsResponseHandler : public rpc::frame::ObReqProcessor -{ -public: - ObTsResponseHandler() { reset(); } - ~ObTsResponseHandler() {} - int init(observer::ObSrvTask *task, ObTsMgr *ts_mgr); - void reset(); -protected: - int run(); -private: - DISALLOW_COPY_AND_ASSIGN(ObTsResponseHandler); - observer::ObSrvTask *task_; - ObTsMgr *ts_mgr_; -}; // end of class ObTsResponseHandler - -class ObTsResponseTask : public observer::ObSrvTask -{ -public: - ObTsResponseTask() { reset(); } - ~ObTsResponseTask() {} - int init(const int64_t arg1, - ObTsMgr *ts_mgr, - int ts_type); - void reset(); - - int64_t get_arg1() const { return arg1_; } - rpc::frame::ObReqProcessor &get_processor() { return handler_; } - int get_ts_type() const { return ts_type_; } - TO_STRING_KV(KP(this), K_(arg1), K_(ts_type)); -private: - int64_t arg1_; - ObTsResponseHandler handler_; - int ts_type_; -}; - -class ObTsResponseTaskFactory -{ -public: - static ObTsResponseTask *alloc(); - static void free(ObTsResponseTask *task); -private: - static int64_t alloc_count_; - static int64_t free_count_; -}; - -} // transaction -} // oceanbase - -#endif /* OCEANBASE_TRANSACTION_OB_TS_RESPONSE_HANDLER_*/ diff --git a/src/storage/tx/ob_ts_worker.cpp b/src/storage/tx/ob_ts_worker.cpp deleted file mode 100644 index f180ba5fc..000000000 --- a/src/storage/tx/ob_ts_worker.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_ts_worker.h" -#include "lib/ob_running_mode.h" -#include "observer/omt/ob_multi_tenant.h" // previously hidden behind a transitive include -#include "ob_ts_response_handler.h" -#include "ob_ts_mgr.h" -#include "observer/omt/ob_tenant.h" - -namespace oceanbase -{ -using namespace common; -using namespace omt; -using namespace observer; - -namespace transaction -{ -int64_t ObTsWorker::get_thread_count_() const -{ - return lib::is_mini_mode() ? 1 : MAX(common::get_cpu_count() / 12, 1L); -} - -int ObTsWorker::init(ObTsMgr *ts_mgr, const bool use_local_worker) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "init twice", KR(ret)); - } else if (NULL == ts_mgr) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(ts_mgr)); - } else if (use_local_worker && OB_FAIL(common::ObSimpleThreadPool::init(get_thread_count_(), - MAX_TASK_NUM, - "TSWorker"))) { - TRANS_LOG(WARN, "ts worker thread pool init failed", K(ret), K(get_thread_count_())); - } else { - TRANS_LOG(INFO, "ts worker thread pool init success", K(use_local_worker), K(get_thread_count_())); - } - if (OB_SUCCESS == ret) { - use_local_worker_ = use_local_worker; - ts_mgr_ = ts_mgr; - is_inited_ = true; - TRANS_LOG(INFO, "ts worker init success", KP(this), KP(ts_mgr), K(use_local_worker)); - } else { - TRANS_LOG(WARN, "ts worker init failed", KR(ret), KP(this), KP(ts_mgr), K(use_local_worker)); - } - return ret; -} - -void ObTsWorker::stop() -{ - if (is_inited_ && use_local_worker_) { - common::ObSimpleThreadPool::stop(); - } -} - -void ObTsWorker::wait() -{ - if (is_inited_ && use_local_worker_) { - common::ObSimpleThreadPool::wait(); - } -} - -void ObTsWorker::destroy() -{ - stop(); - wait(); - if (is_inited_ && use_local_worker_) { - common::ObSimpleThreadPool::destroy(); - } - is_inited_ = false; - use_local_worker_ = false; - ts_mgr_ = NULL; -} - -int ObTsWorker::push_task(ObTsResponseTask *task) -{ - int ret = OB_SUCCESS; - if (!is_inited_) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ts worker not init", KR(ret)); - } else if (!true || NULL == task) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(task)); - } else if (use_local_worker_) { - if (OB_FAIL(common::ObSimpleThreadPool::push(task))) { - TRANS_LOG(WARN, "push task to local worker failed", K(ret), KP(task)); - } - } else { - ObMultiTenant *omt = GCTX.omt_; - if (NULL == omt) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected error, omt is null", KR(ret), KP(omt)); - } else { - ObTenant *tenant = nullptr; - if (OB_FAIL(omt->get_tenant(tenant))) { - TRANS_LOG(WARN, "get tenant failed", KR(ret)); - } else if (OB_ISNULL(tenant)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tenant is null", KR(ret)); - } else if (OB_FAIL(tenant->recv_request(*task))) { - TRANS_LOG(WARN, "recv request failed", KR(ret), KP(task)); - } - } - } - return ret; -} - -void ObTsWorker::handle(void *task) -{ - int ret = OB_SUCCESS; - if (NULL != task) { - ObTsResponseTask *ts_task = reinterpret_cast(task); - if (NULL == ts_mgr_) { - TRANS_LOG(WARN, "ts mgr is NULL", KP_(ts_mgr)); - } else if (OB_FAIL(ts_mgr_->handle_gts_result(ts_task->get_arg1(), - ts_task->get_ts_type()))) { - TRANS_LOG(WARN, "handle gts result failed", KR(ret), K(*ts_task)); - } else { - TRANS_LOG(DEBUG, "handle gts result success", K(*ts_task)); - } - ObTsResponseTaskFactory::free(ts_task); - ts_task = NULL; - } -} - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_ts_worker.h b/src/storage/tx/ob_ts_worker.h deleted file mode 100644 index e3c977d92..000000000 --- a/src/storage/tx/ob_ts_worker.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TS_WORKER_ -#define OCEANBASE_TRANSACTION_OB_TS_WORKER_ - -#include "lib/utility/utility.h" -#include "lib/thread/ob_simple_thread_pool.h" -#include "share/ob_define.h" - -namespace oceanbase -{ -namespace transaction -{ -class ObTsResponseTask; -class ObTsMgr; -class ObTsWorker : public common::ObSimpleThreadPool -{ -public: - ObTsWorker() : is_inited_(false), use_local_worker_(false), ts_mgr_(NULL) {} - ~ObTsWorker() {} - int init(ObTsMgr *ts_mgr, const bool use_local_worker = false); - void stop(); - void wait(); - void destroy(); - int push_task(ObTsResponseTask *task); -protected: - void handle(void *task); -public: - static const int64_t MAX_TASK_NUM = 10240; -private: - int64_t get_thread_count_() const; -private: - bool is_inited_; - bool use_local_worker_; - ObTsMgr *ts_mgr_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_TS_WORKER_ diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index d1ef0b4c2..af1fb9f70 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -16,7 +16,7 @@ #include "ob_trans_service.h" #include "share/rc/ob_module_provider.h" -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" #include "storage/tx/ob_weak_read_util.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/ls/ob_ls.h" @@ -111,7 +111,7 @@ int ObTransService::finalize_tx_(ObTxDesc &tx) ObSpinLockGuard guard(tx.lock_); if (!tx.flags_.RELEASED_) { tx.flags_.RELEASED_ = true; - if (tx.is_tx_active() && !tx.flags_.REPLICA_) { + if (tx.is_tx_active()) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "release tx when tx is active", K(ret), KPC(this), K(tx)); tx.print_trace_(); @@ -346,7 +346,7 @@ int ObTransService::rollback_tx(ObTxDesc &tx) case ObTxDesc::State::IMPLICIT_ACTIVE: tx.state_ = ObTxDesc::State::IN_TERMINATE; tx.abort_cause_ = OB_TRANS_ROLLBACKED; - abort_participants_(tx); + abort_write_state_(tx); case ObTxDesc::State::IDLE: tx.state_ = ObTxDesc::State::ROLLED_BACK; tx.finish_ts_ = ObClockGenerator::getClock(); @@ -501,25 +501,20 @@ int ObTransService::submit_commit_tx(ObTxDesc &tx, tx.print_trace_(); ret = OB_TRANS_ROLLBACKED; handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); - } else if (tx.flags_.PARTS_INCOMPLETE_) { - TRANS_LOG(WARN, "txn participants set incomplete, can not commit", K(ret), K(tx)); - abort_tx_(tx, ObTxAbortCause::PARTICIPANTS_SET_INCOMPLETE); + } else if (tx.flags_.WRITE_STATE_INCOMPLETE_) { + TRANS_LOG(WARN, "txn write state state incomplete, can not commit", K(ret), K(tx)); + abort_tx_(tx, ObTxAbortCause::WRITE_STATE_INCOMPLETE); handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); ret = OB_TRANS_ROLLBACKED; - } else if (tx.flags_.PART_EPOCH_MISMATCH_) { - TRANS_LOG(WARN, "txn participant state incomplete, can not commit", K(ret), K(tx)); - abort_tx_(tx, ObTxAbortCause::PARTICIPANT_STATE_INCOMPLETE); - handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); - ret = OB_TRANS_ROLLBACKED; - } else if (tx.flags_.PART_ABORTED_) { - TRANS_LOG(WARN, "txn participant aborted, can not commit", K(ret), K(tx)); + } else if (tx.flags_.WRITE_STATE_ABORTED_) { + TRANS_LOG(WARN, "txn write state aborted, can not commit", K(ret), K(tx)); abort_tx_(tx, OB_TRANS_ROLLBACKED); handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); ret = OB_TRANS_ROLLBACKED; - } else if (tx.is_all_parts_without_valid_write()) { + } else if (tx.is_write_state_without_valid_write()) { // explicit savepoint rollback cause empty valid-part-set tx.set_commit_cb(&cb); - abort_participants_(tx); // let part ctx quit + abort_write_state_(tx); // let write state ctx quit handle_tx_commit_result_(tx, OB_SUCCESS); // commit success ret = OB_SUCCESS; } @@ -621,26 +616,18 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, } if (OB_FAIL(ret)) { } else if (is_RR_or_SERIAL_isolevel(isolation)) { - // only acquire snapshot once in these isolation level + // only acquire snapshot once in these isolation level if (tx.isolation_ != isolation /*change isolation*/ || !tx.snapshot_version_.is_valid()/*version invalid*/) { SCN version; - int64_t uncertain_bound = 0; - if (OB_FAIL(sync_acquire_global_snapshot_(tx, expire_ts, version, uncertain_bound))) { - TRANS_LOG(WARN, "acquire global snapshot fail", K(ret), K(tx)); - } else if (tx.access_mode_ != ObTxAccessMode::STANDBY_RD_ONLY && - !tx.tx_id_.is_valid() - && OB_FAIL(tx_desc_mgr_.add(tx))) { - if (OB_STANDBY_READ_ONLY == ret) { - tx.access_mode_ = ObTxAccessMode::STANDBY_RD_ONLY; - ret = OB_SUCCESS; - } else { - TRANS_LOG(WARN, "add tx to mgr fail", K(ret), K(tx)); - } + if (OB_FAIL(acquire_local_snapshot_(version))) { + TRANS_LOG(WARN, "acquire local snapshot fail", K(ret), K(tx)); + } else if (!tx.tx_id_.is_valid() && OB_FAIL(tx_desc_mgr_.add(tx))) { + TRANS_LOG(WARN, "add tx to mgr fail", K(ret), K(tx)); } if (OB_SUCC(ret)) { tx.snapshot_version_ = version; - tx.snapshot_uncertain_bound_ = uncertain_bound; + tx.snapshot_uncertain_bound_ = 0; tx.snapshot_scn_ = tx.get_tx_seq(ObSequence::get_max_seq_no() + 1); tx.state_change_flags_.EXTRA_CHANGED_ = true; } @@ -651,35 +638,26 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, snapshot.uncertain_bound_ = tx.snapshot_uncertain_bound_; } } else { // RC isolation level - if (OB_FAIL(sync_acquire_global_snapshot_(tx, - expire_ts, - snapshot.core_.version_, - snapshot.uncertain_bound_))) { - TRANS_LOG(WARN, "acquire global snapshot fail", K(ret), K(tx)); + if (OB_FAIL(acquire_local_snapshot_(snapshot.core_.version_))) { + TRANS_LOG(WARN, "acquire local snapshot fail", K(ret), K(tx)); } else { + snapshot.uncertain_bound_ = 0; adjust_tx_snapshot_(tx, snapshot); } } if (OB_SUCC(ret)) { - snapshot.source_ = ObTxReadSnapshot::SRC::GLOBAL; - snapshot.parts_.reset(); - snapshot.snapshot_ls_role_ = common::ObRole::INVALID_ROLE; - snapshot.snapshot_acquire_addr_ = GCTX.self_addr(); + snapshot.source_ = ObTxReadSnapshot::SRC::LS; + snapshot.uncertain_bound_ = 0; + snapshot.reset_write_state(); // If tx id is valid , record tx_id and scn if (tx.tx_id_.is_valid()) { snapshot.core_.tx_id_ = tx.tx_id_; snapshot.core_.scn_ = tx.get_tx_seq(); } - if (tx.state_ != ObTxDesc::State::IDLE) { - ARRAY_FOREACH(tx.parts_, i) { - if (!tx.parts_[i].is_clean()) { - ObTxLSEpochPair p(tx.parts_[i].id_, tx.parts_[i].epoch_); - if (OB_FAIL(snapshot.parts_.push_back(p))) { - TRANS_LOG(WARN, "push snapshot parts fail", K(ret), K(tx), K(p), K(snapshot)); - } - } - } + if (tx.state_ != ObTxDesc::State::IDLE && + OB_FAIL(tx.fill_read_snapshot_write_state(snapshot))) { + TRANS_LOG(WARN, "fill snapshot write state failed", K(ret), K(tx), K(snapshot)); } snapshot.valid_ = true; } @@ -699,128 +677,40 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, return ret; } -int ObTransService::get_ls_read_snapshot(ObTxDesc &tx, - const ObTxIsolationLevel iso_level, - const share::ObLSID &lsid, - const int64_t expire_ts, - ObTxReadSnapshot &snapshot) -{ - int ret = OB_SUCCESS; - bool fallback_get_global_snapshot = false; - ObRole role = common::ObRole::INVALID_ROLE; - // if txn is active use txn's isolation instead - ObTxIsolationLevel isolation = tx.is_in_tx() ? tx.isolation_ : iso_level; - if (is_RR_or_SERIAL_isolevel(isolation)) { - ret = get_read_snapshot(tx, isolation, expire_ts, snapshot); - } else if (!lsid.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid lsid", K(ret), K(lsid)); - } else { - ObSpinLockGuard guard(tx.lock_); - if (OB_FAIL(tx_sanity_check_(tx))) { - } else if (OB_SUCC(acquire_local_snapshot_(lsid, - snapshot.core_.version_, - true /*is_read_only*/, - role))) { - adjust_tx_snapshot_(tx, snapshot); - snapshot.source_ = ObTxReadSnapshot::SRC::LS; - snapshot.snapshot_lsid_ = lsid; - snapshot.uncertain_bound_ = 0; - snapshot.parts_.reset(); - snapshot.snapshot_acquire_addr_ = GCTX.self_addr(); - snapshot.snapshot_ls_role_ = role; - // If tx id is valid , record tx_id and scn - if (tx.tx_id_.is_valid()) { - snapshot.core_.tx_id_ = tx.tx_id_; - snapshot.core_.scn_ = tx.get_tx_seq(); - } - if (tx.state_ != ObTxDesc::State::IDLE) { - ARRAY_FOREACH(tx.parts_, i) { - if (tx.parts_[i].id_ == lsid && !tx.parts_[i].is_clean()) { - if (OB_FAIL(snapshot.parts_.push_back(ObTxLSEpochPair(lsid, tx.parts_[i].epoch_)))) { - TRANS_LOG(WARN, "push lsid to snapshot.parts fail", K(ret), K(lsid), K(tx)); - } - } - } - } - snapshot.valid_ = true; - } else { - // XXX In standby cluster mode, the failure to call acquire_local_snapshot_ is an - // normal situation, no error log needs to be printed - // TRANS_LOG(WARN, "acquire local snapshot fail, fallback gts", K(ret), K(lsid)); - fallback_get_global_snapshot = true; - } - } - if (fallback_get_global_snapshot) { - ret = get_read_snapshot(tx, isolation, expire_ts, snapshot); - } - ObTransTraceLog &tlog = tx.get_tlog(); - common::ObTraceIdAdaptor trace_id; - trace_id.set(ObCurTraceId::get()); - REC_TRANS_TRACE_EXT(&tlog, get_ls_read_snapshot, OB_Y(ret), OB_Y(expire_ts), - OB_ID(ls_id), lsid.id(), - OB_ID(isolation_level), (int)isolation, - OB_ID(snapshot_source), (int)snapshot.source_, - OB_ID(snapshot_version), snapshot.core_.version_, - OB_ID(snapshot_txid), snapshot.core_.tx_id_.get_id(), - OB_ID(snapshot_scn), snapshot.core_.scn_.cast_to_int(), - OB_ID(trace_id), trace_id, - OB_ID(ref), tx.get_ref(), - OB_ID(thread_id), GETTID()); - return ret; -} - int ObTransService::get_read_snapshot_version(const int64_t expire_ts, SCN &snapshot_version) { int ret = OB_SUCCESS; - int64_t uncertain_bound = 0; - ret = acquire_global_snapshot__(expire_ts, - 0, - snapshot_version, - uncertain_bound); + UNUSED(expire_ts); + ret = acquire_local_snapshot_(snapshot_version); return ret; } int ObTransService::get_weak_read_snapshot_version(const int64_t max_read_stale_us_for_user, - const bool local_single_ls, SCN &snapshot) { int ret = OB_SUCCESS; - // Single-node lite: the weak-read snapshot is the local readable timestamp - // (min ls_weak_read_ts_ across local log streams), maintained by the tx loop - // worker. There is no cluster/server WRS aggregation anymore. + // The weak-read snapshot is the local readable timestamp maintained by the + // transaction loop worker. bool monotinic_read = ObWeakReadUtil::enable_monotonic_weak_read(); - UNUSED(local_single_ls); SCN wrs_scn = SCN::max_scn(); { storage::ObLSService *ls_svr = share::g_mp->ls_service(); - common::ObSharedGuard ls_iter; + storage::ObLS *tenant_ls = nullptr; storage::ObLS *ls = nullptr; if (OB_ISNULL(ls_svr)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "ls service is null", K(ret), KPC(this)); - } else if (OB_FAIL(ls_svr->get_ls_iter(ls_iter, ObLSGetMod::TRANS_MOD))) { - TRANS_LOG(WARN, "get ls iter fail", K(ret), KPC(this)); + } else if (OB_FAIL(ls_svr->get_ls(tenant_ls))) { + TRANS_LOG(WARN, "get transaction storage fail", K(ret), KPC(this)); } else { - int64_t ls_cnt = 0; - while (OB_SUCC(ret) && OB_SUCC(ls_iter->get_next(ls))) { - if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "ls is null", K(ret), KPC(this)); - } else { - wrs_scn = SCN::min(wrs_scn, ls->get_ls_wrs_handler()->get_ls_weak_read_ts()); - ls_cnt++; - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - if (OB_SUCC(ret) && (0 == ls_cnt || !wrs_scn.is_valid_and_not_min())) { - // No readable ls weak-read ts yet: fall back to gts-derived min version. - ret = ObWeakReadUtil::generate_min_weak_read_version(wrs_scn); - } + ls = tenant_ls; + wrs_scn = ls->get_ls_wrs_handler()->get_ls_weak_read_ts(); + } + if (OB_SUCC(ret) && !wrs_scn.is_valid_and_not_min()) { + // No readable ls weak-read ts yet: fall back to gts-derived min version. + ret = ObWeakReadUtil::generate_min_weak_read_version(wrs_scn); } } if (OB_SUCC(ret)) { @@ -832,7 +722,7 @@ int ObTransService::get_weak_read_snapshot_version(const int64_t max_read_stale_ // check snapshot version barrier which is setted by user system variable SCN gts_cache; SCN current_scn; - if (OB_FAIL(OB_TS_MGR.get_gts(NULL, gts_cache))) { + if (OB_FAIL(OB_TS_MGR.get_gts(gts_cache))) { TRANS_LOG(WARN, "get ts sync error", K(ret), K(max_read_stale_us_for_user)); } else { const int64_t current_time_us = std::max(ObTimeUtility::current_time(), gts_cache.convert_to_ts()); @@ -1031,7 +921,7 @@ int ObTransService::create_global_implicit_savepoint_(ObTxDesc &tx, int ObTransService::rollback_to_implicit_savepoint(ObTxDesc &tx, const ObTxSEQ savepoint, const int64_t expire_ts, - const share::ObLSArray *extra_touched_ls, + const bool touched_storage, const ObTxCleanPolicy clean_policy) { int ret = OB_SUCCESS; @@ -1040,7 +930,7 @@ int ObTransService::rollback_to_implicit_savepoint(ObTxDesc &tx, if (savepoint.get_branch() // NOTE: branch savepoint only support local rollback || tx.flags_.SHADOW_) { if (OB_FAIL(tx_sanity_check_(tx))) { - } else if (OB_NOT_NULL(extra_touched_ls)) { + } else if (touched_storage) { ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "rollback on remote only suport collected tx parts", K(ret), K(savepoint), K(tx)); @@ -1049,12 +939,12 @@ int ObTransService::rollback_to_implicit_savepoint(ObTxDesc &tx, } } else { if (tx.state_ < ObTxDesc::State::IN_TERMINATE) { - if (OB_NOT_NULL(extra_touched_ls) && !extra_touched_ls->empty()) { - if (OB_FAIL(tx.update_parts(*extra_touched_ls))) { - TRANS_LOG(WARN, "add tx part with extra_touched_ls fail", K(ret), K(tx), KPC(extra_touched_ls)); + if (touched_storage) { + if (OB_FAIL(tx.mark_write())) { + TRANS_LOG(WARN, "mark transaction write failed", K(ret), K(tx)); abort_tx_(tx, ret); } else { - TRANS_LOG(INFO, "add tx part with extra_touched_ls", KPC(extra_touched_ls), K_(tx.tx_id)); + TRANS_LOG(INFO, "mark transaction write", K_(tx.tx_id)); } } } @@ -1077,33 +967,28 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, const int64_t expire_ts) { int ret = OB_SUCCESS; - ObTxPartRefList parts; + ObTxWriteState *part = NULL; int64_t start_ts = ObTimeUtility::current_time(); - // when rollback local we use this from_scn to all downstream participants + // when rollback local we use this from_scn for the single write state ObTxSEQ from_scn = savepoint.clone_with_seq(ObSequence::inc_and_get_max_seq_no(), tx.seq_base_); - if (OB_FAIL(find_parts_after_sp_(tx, parts, savepoint))) { - TRANS_LOG(WARN, "find rollback parts fail", K(ret), K(savepoint), K(tx)); - } else { - ARRAY_FOREACH(parts, i) { - ObPartTransCtx *ctx = NULL; - ObTxPart &p = parts[i]; - if (OB_FAIL(get_tx_ctx_(p.id_, tx.tx_id_, ctx))) { - TRANS_LOG(WARN, "get tx ctx fail", K(ret), K_(p.id), K(tx)); - } else if (p.epoch_ != ctx->epoch_) { - ret = OB_TRANS_CTX_NOT_EXIST; // FIXME more decent errno - } else if (OB_FAIL(ls_sync_rollback_savepoint__(ctx, - savepoint, - tx.op_sn_, - tx.seq_base_, - expire_ts, - from_scn))) { - TRANS_LOG(WARN, "LS rollback savepoint fail", K(ret), K(savepoint), K(tx)); - } else { - p.last_scn_ = savepoint; - } - if (OB_NOT_NULL(ctx)) { - revert_tx_ctx_(ctx); - } + if (OB_FAIL(find_write_state_after_savepoint_(tx, part, savepoint))) { + TRANS_LOG(WARN, "find rollback write state fail", K(ret), K(savepoint), K(tx)); + } else if (OB_NOT_NULL(part)) { + ObTxCtx *ctx = NULL; + if (OB_FAIL(get_tx_ctx_(tx.tx_id_, ctx))) { + TRANS_LOG(WARN, "get tx ctx fail", K(ret), KPC(part), K(tx)); + } else if (OB_FAIL(sync_rollback_to_savepoint_(ctx, + savepoint, + tx.op_sn_, + tx.seq_base_, + expire_ts, + from_scn))) { + TRANS_LOG(WARN, "LS rollback savepoint fail", K(ret), K(savepoint), K(tx)); + } else { + part->last_scn_ = savepoint; + } + if (OB_NOT_NULL(ctx)) { + revert_tx_ctx_(ctx); } } @@ -1156,8 +1041,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, break; case ObTxDesc::State::IMPLICIT_ACTIVE: tx.release_implicit_savepoint(savepoint); - if (!tx.flags_.REPLICA_ // on tx start node - && !tx.has_implicit_savepoint() // to first savepoint + if (!tx.has_implicit_savepoint() // to first savepoint && tx.active_scn_ >= savepoint // rollback all dirty state && !tx.has_extra_state_()) { // hasn't explicit savepoint or serializable snapshot reset_tx = need_rollback_(clean_policy); @@ -1173,18 +1057,15 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, } if (normal_rollback) { - ObTxPartRefList parts; + ObTxWriteState *part = NULL; if (OB_FAIL(ret)) { - } else if (tx.flags_.PART_EPOCH_MISMATCH_) { + } else if (tx.flags_.WRITE_STATE_INCOMPLETE_) { ret = OB_TRANS_NEED_ROLLBACK; - TRANS_LOG(WARN, "some participant born epoch mismatch, txn will rollback internally", K(ret)); - } else if (tx.flags_.PARTS_INCOMPLETE_) { - ret = OB_TRANS_NEED_ROLLBACK; - TRANS_LOG(WARN, "txn participants set incomplete, txn will rollback internally", K(ret)); - } else if (OB_FAIL(find_parts_after_sp_(tx, parts, savepoint))) { + TRANS_LOG(WARN, "txn write state state incomplete, txn will rollback internally", K(ret)); + } else if (OB_FAIL(find_write_state_after_savepoint_(tx, part, savepoint))) { TRANS_LOG(WARN, "find rollback parts fail", K(ret), K(tx), K(savepoint)); } else if (OB_FAIL(rollback_savepoint_(tx, - parts, + part, savepoint, expire_ts))) { TRANS_LOG(WARN, "do savepoint rollback fail", K(ret)); @@ -1193,7 +1074,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, if (reset_tx) { } else if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback savepoint fail, abort tx", - K(ret), K(savepoint), K(parts), K(tx)); + K(ret), K(savepoint), KPC(part), K(tx)); // advance op_sequence to reject further rollback resp messsages tx.inc_op_sn(); abort_tx_(tx, ObTxAbortCause::SAVEPOINT_ROLLBACK_FAIL); @@ -1241,7 +1122,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, return ret; } -int ObTransService::ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, +int ObTransService::sync_rollback_to_savepoint_(ObTxCtx *part_ctx, const ObTxSEQ savepoint, const int64_t op_sn, const int64_t tx_seq_base, @@ -1271,12 +1152,12 @@ int ObTransService::ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, } while ((OB_NEED_RETRY == ret || OB_EAGAIN == ret || OB_TX_NOLOGCB == ret) && blockable); #ifndef NDEBUG TRANS_LOG(INFO, "rollback to savepoint sync", K(ret), - K(part_ctx->get_trans_id()), K(part_ctx->get_ls_id()), K(retry_cnt), + K(part_ctx->get_trans_id()), K(retry_cnt), K(op_sn), K(savepoint), K(expire_ts)); #else if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback to savepoint sync fail", K(ret), - K(part_ctx->get_trans_id()), K(part_ctx->get_ls_id()), K(retry_cnt), + K(part_ctx->get_trans_id()), K(retry_cnt), K(op_sn), K(savepoint), K(expire_ts)); } #endif @@ -1354,18 +1235,18 @@ int ObTransService::rollback_to_explicit_savepoint(ObTxDesc &tx, } } if (OB_SUCC(ret)) { - ObTxPartRefList parts; - if (OB_FAIL(find_parts_after_sp_(tx, parts, sp_scn))) { + ObTxWriteState *part = NULL; + if (OB_FAIL(find_write_state_after_savepoint_(tx, part, sp_scn))) { TRANS_LOG(WARN, "find rollback parts fail", K(ret), K(tx), K(savepoint)); } else if (OB_FAIL(rollback_savepoint_(tx, - parts, + part, sp_scn, expire_ts))) { TRANS_LOG(WARN, "do savepoint rollback fail", K(ret)); } if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback savepoint fail, abort tx", - K(ret), K(savepoint), K(sp_scn), K(parts), K(tx)); + K(ret), K(savepoint), K(sp_scn), KPC(part), K(tx)); abort_tx_(tx, ObTxAbortCause::SAVEPOINT_ROLLBACK_FAIL); } } @@ -1458,133 +1339,123 @@ int ObTransService::create_stash_savepoint(ObTxDesc &tx, const ObString &name) return ret; } -// impl note -// fastpath: -// local call, if NOT_MASTER, fallback slowpath -// slowpath: -// post msg, -// abort tx if rollback failed int ObTransService::rollback_savepoint_(ObTxDesc &tx, - ObTxPartRefList &parts, + ObTxWriteState *part, const ObTxSEQ savepoint, int64_t expire_ts) { int ret = OB_SUCCESS; - bool slowpath = true; expire_ts = std::min(expire_ts, tx.get_expire_ts()); - if (parts.count() == 0) { - slowpath = false; - TRANS_LOG(INFO, "empty rollback participant set", K(tx), K(savepoint)); - } else if (parts.count() == 1 && parts[0].addr_ == self_) { - slowpath = false; - ObTxPart &p = parts[0]; - int64_t born_epoch = 0; - if (OB_FAIL(ls_rollback_to_savepoint_(tx.tx_id_, - p.id_, - p.epoch_, - tx.op_sn_, + if (OB_ISNULL(part)) { + TRANS_LOG(INFO, "empty rollback write state set", K(tx), K(savepoint)); + } else { + const ObTransID tx_id = tx.tx_id_; + const int64_t op_sn = tx.op_sn_; + const int64_t seq_base = tx.seq_base_; + ret = rollback_tx_to_savepoint_(tx_id, + op_sn, + savepoint, + seq_base, + tx, + ObTxSEQ::INVL(), + -1/*non-blocking*/); + if (common_retryable_error_(ret)) { + TRANS_LOG(INFO, "fallback to blocking rollback", K(ret), K(savepoint), KPC(part), K(tx)); + if (OB_UNLIKELY(tx.flags_.INTERRUPTED_)) { + ret = OB_ERR_INTERRUPTED; + tx.clear_interrupt(); + TRANS_LOG(WARN, "interrupted", K(ret), K(tx)); + } else { + const ObTxDesc::State save_state = tx.state_; + tx.state_ = ObTxDesc::State::ROLLBACK_SAVEPOINT; + tx.flags_.BLOCK_ = true; + tx.lock_.unlock(); + const int64_t retry_expire_ts = std::max(ObTimeUtility::current_time() + 200 * 1000, expire_ts); + int64_t retry_cnt = 0; + do { + ret = rollback_tx_to_savepoint_(tx_id, + op_sn, savepoint, - tx.seq_base_, - born_epoch, - &tx, + seq_base, + tx, ObTxSEQ::INVL(), - -1/*non-blocking*/))) { - if (common_retryable_error_(ret)) { - slowpath = true; - TRANS_LOG(INFO, "fallback to msg driven rollback", K(ret), K(savepoint), K(p), K(tx)); - } else { - TRANS_LOG(WARN, "rollback savepoint fail", K(ret), K(savepoint), K(p), K(tx)); + -1/*non-blocking*/); + if (common_retryable_error_(ret)) { + if (tx.flags_.INTERRUPTED_) { + ret = OB_ERR_INTERRUPTED; + TRANS_LOG(WARN, "rollback was interrupted", K(ret), K(tx_id), K(retry_cnt)); + } else if (ObTimeUtility::current_time() >= retry_expire_ts) { + ret = OB_TIMEOUT; + TRANS_LOG(WARN, "can not retry rollback_to because of timeout", K(ret), K(tx_id), K(retry_cnt)); + } else { + if (retry_cnt % 5 == 0) { + TRANS_LOG(WARN, "retry blocking rollback", K(ret), K(tx_id), K(retry_cnt)); + } + ++retry_cnt; + const int64_t retry_interval = std::min(10 * 1000 * retry_cnt, 50 * 1000L); + ob_usleep(retry_interval); + } + } + } while (common_retryable_error_(ret)); + tx.lock_.lock(); + if (OB_SUCC(ret) && tx.is_tx_active()) { + tx.state_ = save_state; + } + if (OB_SUCC(ret) && tx.flags_.INTERRUPTED_) { + ret = OB_ERR_INTERRUPTED; + TRANS_LOG(WARN, "rollback savepoint was interrupted", K(ret)); + } + tx.clear_interrupt(); + tx.flags_.BLOCK_ = false; } - } else { - if (p.epoch_ <= 0) { tx.update_clean_part(p.id_, born_epoch, self_); } - TRANS_LOG(TRACE, "succ to rollback on participant", K(p), K(tx), K(savepoint)); } - } - if (slowpath) { - ObTxRollbackParts rollback_parts; - if (OB_FAIL(rollback_parts.reserve(parts.count()))) { - TRANS_LOG(WARN, "reserve space fail", K(ret), K(parts), K(tx)); + if (OB_FAIL(ret)) { + TRANS_LOG(WARN, "rollback savepoint fail", K(ret), K(savepoint), KPC(part), K(tx)); } else { - ARRAY_FOREACH(parts, i) { - rollback_parts.push_back(ObTxExecPart(parts[i].id_, - parts[i].epoch_)); - } - } - if (FAILEDx(rollback_savepoint_slowpath_(tx, - rollback_parts, - ObTxSEQ::INVL(), - savepoint, - expire_ts))) { - TRANS_LOG(WARN, "rollback slowpath fail", K(ret), - K(parts), K(savepoint), K(expire_ts), K(tx)); + tx.update_clean_write_state(); + TRANS_LOG(TRACE, "succ to rollback on write state", KPC(part), K(tx), K(savepoint)); } } if (OB_TIMEOUT == ret && ObTimeUtility::current_time() >= tx.get_expire_ts()) { ret = OB_TRANS_TIMEOUT; } - if (OB_SUCC(ret) && parts.count() > 0) { - tx.post_rb_savepoint_(parts, savepoint); + if (OB_SUCC(ret) && OB_NOT_NULL(part)) { + tx.finish_write_state_rollback_(part, savepoint); } return ret; } /** - * ls_rollback_to_savepoint - rollback savepoint on LogStream - * @verify_epoch: used verify tx_ctx's epoch, prevent - * create-crash-recreate + * rollback_tx_to_savepoint - rollback a transaction to a savepoint * @op_sn: the operator sequence inner transaction * used to keep operation order correctly - * @ctx_born_epoch: return the ctx's born epoch if created(born) - * @tx: tnx descriptor used to create participant txn ctx + * @tx: transaction descriptor used to create the local tx ctx * @expire_ts: an expire_ts used if retry was required * -1 if non-blocking desired * - * Impl Note: - * if verify_epoch = -1, which means first time to touch the LS - * if the tx_ctx not exist, one will be created, and its epoch returned - * otherwise, tx_ctx's epoch was verified with @verify_epoch */ -int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, - const share::ObLSID &ls, - const int64_t verify_epoch, +int ObTransService::rollback_tx_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const ObTxSEQ savepoint, const int64_t tx_seq_base, - int64_t &ctx_born_epoch, - const ObTxDesc *tx, + const ObTxDesc &tx, const ObTxSEQ from_scn, int64_t expire_ts) { int ret = OB_SUCCESS; - int64_t retry_cnt = 0; - ObPartTransCtx *ctx = NULL; - bool is_leader = false; - // check ls is leader, to fast fail when lease expired but role change has not armed - if (OB_FAIL(check_ls_status_(ls, is_leader))) { - TRANS_LOG(WARN, "check ls leader failed", K(ret), K(ls)); - } else if (!is_leader) { - ret = OB_NOT_MASTER; - } else if (OB_FAIL(get_tx_ctx_(ls, tx_id, ctx))) { - if (OB_NOT_MASTER == ret) { - } else if (OB_TRANS_CTX_NOT_EXIST == ret && verify_epoch <= 0) { + ObTxCtx *ctx = NULL; + if (OB_FAIL(get_tx_ctx_(tx_id, ctx))) { + if (OB_TRANS_CTX_NOT_EXIST == ret) { int64_t tx_state = ObTxData::RUNNING; share::SCN commit_version; - if (OB_FAIL(get_tx_state_from_tx_table_(ls, tx_id, tx_state, commit_version))) { + if (OB_FAIL(get_tx_state_from_tx_table_(tx_id, tx_state, commit_version))) { if (OB_TRANS_CTX_NOT_EXIST == ret) { bool ctx_exist = false; - if (OB_FAIL(create_tx_ctx_(ls, *tx, ctx, ctx_exist))) { - if ((OB_PARTITION_IS_BLOCKED == ret || OB_PARTITION_IS_STOPPED == ret) && is_ls_dropped_(ls)) { - ctx_born_epoch = ObTxPart::EPOCH_DEAD; - ret = OB_SUCCESS; - } else { - TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls), KPC(tx)); - } + if (OB_FAIL(create_tx_ctx_(tx, ctx, ctx_exist))) { + TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(tx)); } - } else if (OB_REPLICA_NOT_READABLE == ret && is_ls_dropped_(ls)) { - ctx_born_epoch = ObTxPart::EPOCH_DEAD; - ret = OB_SUCCESS; } else { - TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(ls), K(tx_id)); + TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(tx_id)); } } else { switch (tx_state) { @@ -1597,26 +1468,21 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, case ObTxData::RUNNING: default: ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx in-progress but ctx miss", K(ret), K(tx_state), K(tx_id), K(ls)); + TRANS_LOG(WARN, "tx in-progress but ctx miss", K(ret), K(tx_state), K(tx_id)); } } } else { - TRANS_LOG(WARN, "get transaction context error", K(ret), K(tx_id), K(ls)); + TRANS_LOG(WARN, "get transaction context error", K(ret), K(tx_id)); } } if (OB_SUCC(ret) && OB_NOT_NULL(ctx)) { - ctx_born_epoch = ctx->epoch_; - if (verify_epoch > 0 && ctx->epoch_ != verify_epoch) { - ret = OB_TRANS_CTX_NOT_EXIST; - TRANS_LOG(WARN, "current ctx illegal, born epoch not match", K(ret), K(ls), K(tx_id), - K(verify_epoch), K(ctx_born_epoch), KPC(ctx)); - } else if(OB_FAIL(ls_sync_rollback_savepoint__(ctx, - savepoint, - op_sn, - tx_seq_base, - expire_ts, - from_scn))) { - TRANS_LOG(WARN, "LS rollback to savepoint fail", K(ret), K(tx_id), K(ls), K(op_sn), K(savepoint), KPC(ctx)); + if (OB_FAIL(sync_rollback_to_savepoint_(ctx, + savepoint, + op_sn, + tx_seq_base, + expire_ts, + from_scn))) { + TRANS_LOG(WARN, "rollback transaction to savepoint fail", K(ret), K(tx_id), K(op_sn), K(savepoint), KPC(ctx)); } } if (OB_NOT_NULL(ctx)) { @@ -1625,222 +1491,6 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, return ret; } -inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, - ObTxRollbackParts &rollback_parts, - const ObTxSEQ specified_from_scn, - const ObTxSEQ savepoint, - const int64_t expire_ts) -{ - int ret = OB_SUCCESS; - int64_t max_retry_intval = GCONF._ob_trans_rpc_timeout; - int64_t tx_msg_id = fetch_rollback_sp_sequence_(); - if (rollback_parts.count() == 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "rollback parts is empty", K(ret), K(tx)); - } else { - tx.brpc_mask_set_.reset(); - if (OB_FAIL(tx.brpc_mask_set_.init(ObCommonID(tx_msg_id), rollback_parts))) { - TRANS_LOG(WARN, "init rpc mask set fail", K(ret), K(tx)); - } - } - ObTxRollbackSPMsg msg; - msg.cluster_version_ = tx.cluster_version_; - - msg.sender_addr_ = self_; - msg.sender_ = SCHEDULER_LS; - msg.cluster_id_ = tx.cluster_id_; - msg.tx_id_ = tx.tx_id_; - msg.savepoint_ = savepoint; - msg.op_sn_ = tx.op_sn_; - msg.tx_seq_base_ = tx.seq_base_; - msg.epoch_ = -1; - msg.request_id_ = tx_msg_id; - msg.specified_from_scn_ = specified_from_scn; - // prepare msg.tx_ptr_ if required - // TODO(yunxing.cyx) : in 4.1 rework here, won't serialize txDesc - ObTxDesc *tmp_tx_desc = NULL; - ARRAY_FOREACH_NORET(rollback_parts, i) { - if (rollback_parts.at(i).exec_epoch_ <= 0) { - int64_t len = tx.get_serialize_size() + sizeof(ObTxDesc); - char *buf = (char*)ob_malloc(len, "TxDesc"); - int64_t pos = sizeof(ObTxDesc); - if (OB_FAIL(tx.serialize(buf, len, pos))) { - TRANS_LOG(WARN, "serialize tx fail", KR(ret), K(tx)); - ob_free(buf); - } else { - tmp_tx_desc = new(buf)ObTxDesc(); - pos = sizeof(ObTxDesc); - if (OB_FAIL(tmp_tx_desc->deserialize(buf, len, pos))) { - TRANS_LOG(WARN, "deserialize tx fail", KR(ret)); - } else { - tmp_tx_desc->parts_.reset(); - msg.tx_ptr_ = tmp_tx_desc; - } - } - break; - } - } - int64_t start_ts = ObTimeUtility::current_time(); - int retries = 0; - if (OB_UNLIKELY(tx.flags_.INTERRUPTED_)) { - ret = OB_ERR_INTERRUPTED; - tx.clear_interrupt(); - TRANS_LOG(WARN, "interrupted", K(ret), K(tx)); - } - if (OB_SUCC(ret)) { - // setup state before release lock - ObTxDesc::State save_state = tx.state_; - tx.state_ = ObTxDesc::State::ROLLBACK_SAVEPOINT; - tx.flags_.BLOCK_ = true; - // release lock before blocking - tx.lock_.unlock(); - ret = sync_rollback_savepoint__(tx, msg, tx.brpc_mask_set_, - expire_ts, max_retry_intval, retries); - tx.lock_.lock(); - // restore state - if (OB_SUCC(ret) && tx.is_tx_active()) { - tx.state_ = save_state; - } - // mask_set need clear - tx.brpc_mask_set_.reset(); - // check interrupt - if (OB_SUCC(ret) && tx.flags_.INTERRUPTED_) { - ret = OB_ERR_INTERRUPTED; - TRANS_LOG(WARN, "rollback savepoint was interrupted", K(ret)); - } - // clear interrupt flag - tx.clear_interrupt(); - tx.flags_.BLOCK_ = false; - } - if (OB_NOT_NULL(tmp_tx_desc)) { - msg.tx_ptr_ = NULL; - tmp_tx_desc->~ObTxDesc(); - ob_free(tmp_tx_desc); - } - int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; - TRANS_LOG(INFO, "rollback savepoint slowpath", K(ret), - K_(tx.tx_id), K(start_ts), K(retries), - K(savepoint), K(expire_ts), K(tx), K(rollback_parts.count())); - ObTransTraceLog &tlog = tx.get_tlog(); - REC_TRANS_TRACE_EXT(&tlog, rollback_savepoint_slowpath, OB_Y(ret), - OB_ID(savepoint), savepoint.cast_to_int(), OB_Y(expire_ts), - OB_ID(retry_cnt), retries, - OB_ID(time_used), elapsed_us); - return ret; -} - -inline int ObTransService::sync_rollback_savepoint__(ObTxDesc &tx, - ObTxRollbackSPMsg &msg, - RollbackMaskSet &mask_set, - int64_t expire_ts, - const int64_t max_retry_intval, - int &retries) -{ - const int64_t MIN_WAIT_TIME = 200 * 1000; // 200 ms - int ret = OB_SUCCESS; - int64_t start_ts = ObClockGenerator::getClock(); - retries = 0; - int64_t min_retry_intval = 10 * 1000; // 10 ms - expire_ts = std::max(ObTimeUtility::current_time() + MIN_WAIT_TIME, expire_ts); - ObCommonID msg_id(msg.request_id_); - ObRollbackSPMsgGuard *rollback_sp_msg_guard = NULL; - bool insert_mgr = false; - share::ObTenantBase *tenant_base = MTL_CTX(); - omt::ObTenant *tenant = static_cast(tenant_base); - - if (OB_ISNULL(tenant_base)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "get tenant is null", K(ret)); - } else if (OB_ISNULL(rollback_sp_msg_guard = ObRollbackSPMsgGuardAlloc::alloc_value())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc tx_msg_guard failed", KR(ret), K(msg.tx_id_)); - } else if (FALSE_IT(new (rollback_sp_msg_guard) ObRollbackSPMsgGuard(msg_id, tx, tx_desc_mgr_))) { - } else if (OB_FAIL(rollback_sp_msg_mgr_.insert(msg_id, rollback_sp_msg_guard))) { - TRANS_LOG(WARN, "insert tx_desc to holder failed", KR(ret), K(msg.tx_id_)); - } else { - insert_mgr = true; - } - while (OB_SUCC(ret)) { - int64_t retry_intval = std::min(min_retry_intval * (1 + retries), max_retry_intval); - int64_t waittime = std::min(expire_ts - ObTimeUtility::current_time(), retry_intval); - if (waittime <=0) { - ret = OB_TIMEOUT; - TRANS_LOG(WARN, "tx rpc wait result timeout", K(ret), K(expire_ts), K(retries)); - } else { - ObTxRollbackParts remain; - mask_set.get_not_mask(remain); - int64_t remain_cnt = remain.count(); - TRANS_LOG(DEBUG, "unmasked parts", K(remain), K(tx), K(retries)); - // post msg to participants - if (remain_cnt > 0) { - tx.rpc_cond_.reset(); /* reset rpc_cond */ - int post_succ_num = 0; - if (OB_FAIL(batch_post_rollback_savepoint_msg_(tx, msg, remain, post_succ_num))) { - TRANS_LOG(WARN, "batch post tx msg fail", K(msg), K(remain), K(retries)); - if (is_location_service_renew_error(ret)) { - // ignore ret - ret = OB_SUCCESS; - } - } else { remain_cnt = post_succ_num; } - } - // if post failed, wait awhile and retry - // if post succ, wait responses - if (OB_SUCC(ret) && remain_cnt > 0) { - // wait result - int rpc_ret = OB_SUCCESS; - if (OB_FAIL(tx.rpc_cond_.wait(waittime, rpc_ret))) { - TRANS_LOG(WARN, "tx rpc condition wakeup", K(ret), - K(tx.tx_id_), K(waittime), K(rpc_ret), K(expire_ts), K(remain), K(remain_cnt), K(retries), - K_(tx.state)); - // if trans is terminated, rollback savepoint should be terminated - // NOTE that this case is only for xa trans - // EXAMPLE, tx desc is shared by branch 1 and branch 2 - // 1. branch 1 starts to rollback savepoint - // 2. branch 2 is terminated - // 3. branch 1 receives callback of rollback savepoint - if (tx.is_terminated()) { - ret = OB_TRANS_HAS_DECIDED; - } else if (retries > 10 && tenant->has_stopped()) { - ret = OB_TENANT_NOT_IN_SERVER; - TRANS_LOG(WARN, "tenant has been stopped", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - if (OB_SUCCESS != rpc_ret) { - TRANS_LOG(WARN, "tx rpc fail", K(rpc_ret), K_(tx.tx_id), K(waittime), K(remain), K(remain_cnt), K(retries)); - if (rpc_ret == OB_TRANS_CTX_NOT_EXIST) { - // participant has quit, may be txn is timeout or other failure occured - // txn need abort - ret = tx.is_tx_timeout() ? OB_TRANS_TIMEOUT : OB_TRANS_KILLED; - } else { - ret = rpc_ret; - } - } - } - // check request complete - if (OB_SUCC(ret) && mask_set.is_all_mask()) { - TRANS_LOG(INFO, "all savepoint rollback succeed", K_(tx.tx_id), - K(remain_cnt), K(waittime), K(retries)); - break; - } - // interrupted, fail fastly - if (tx.flags_.INTERRUPTED_) { - ret = OB_ERR_INTERRUPTED; - TRANS_LOG(WARN, "rollback was interrupted", "caused_by", ObTxAbortCauseNames::of(tx.abort_cause_), - "trans_id", tx.tx_id_, - K(remain_cnt), K(waittime), K(retries)); - } - } - ++retries; - } - if (insert_mgr) { - // remove msg from mgr - rollback_sp_msg_mgr_.del(msg_id, rollback_sp_msg_guard); - } - return ret; -} - int ObTransService::merge_tx_state(ObTxDesc &to, const ObTxDesc &from) { TRANS_LOG(TRACE, "merge_tx_state", K(to), K(from)); @@ -1865,7 +1515,6 @@ int ObTransService::add_tx_exec_result(ObTxDesc &tx, const ObTxExecResult &exec_ int ret = tx.add_exec_info(exec_info); ObTransTraceLog &tlog = tx.get_tlog(); REC_TRANS_TRACE_EXT(&tlog, add_tx_exec_result, OB_ID(opid), tx.op_sn_, - OB_ID(num), exec_info.parts_.count(), OB_ID(flag), exec_info.incomplete_, OB_ID(thread_id), GETTID()); return ret; @@ -1908,15 +1557,11 @@ void ObTransService::tx_post_terminate_(ObTxDesc &tx) // TODO: COMMIT_UNKNOWN, COMMIT_TIMEOUT } } - switch(tx.parts_.count()) { - case 0: TX_STAT_READONLY_INC break; - case 1: + if (!tx.has_write_state()) { + TX_STAT_READONLY_INC; + } else { TX_STAT_LOCAL_INC; TX_STAT_LOCAL_TOTAL_TIME_USED(trans_used_time_us); - break; - default: - TX_STAT_DIST_INC; - TX_STAT_DIST_TOTAL_TIME_USED(trans_used_time_us); } } // release all savepoints @@ -1980,8 +1625,8 @@ OB_INLINE int ObTransService::tx_sanity_check_(ObTxDesc &tx) case ObTxDesc::State::IDLE: case ObTxDesc::State::ACTIVE: case ObTxDesc::State::IMPLICIT_ACTIVE: - if (tx.flags_.PART_ABORTED_) { - TRANS_LOG(WARN, "some participant was aborted, abort tx now"); + if (tx.flags_.WRITE_STATE_ABORTED_) { + TRANS_LOG(WARN, "write state was aborted, abort tx now"); abort_tx_(tx, tx.abort_cause_); // go through } else { diff --git a/src/storage/tx/ob_tx_api.h b/src/storage/tx/ob_tx_api.h index 446aac54f..54b9989e5 100644 --- a/src/storage/tx/ob_tx_api.h +++ b/src/storage/tx/ob_tx_api.h @@ -167,29 +167,6 @@ int get_read_snapshot(ObTxDesc &tx, const int64_t expire_ts, ObTxReadSnapshot &snapshot); -/** - * get_ls_read_snapshot - get a read snapshot which can be used to read - * a consistency view of current state of data - * of the specified local LogStream - * - * @tx: the tx in which snapshot stationed - * @isolation_level: the Isolation Level setting - * @local_ls_id: the local LogStream's id - * @expire_ts: microseconds of timestamp after which acquire - * action timeout - * @snapshot: the snapshot acquired - * - * Return: - * OB_SUCCESS - OK - * OB_NOT_MASTER - if local replica of @local_ls_id not leader - * OB_TIMEOUT - if expire_ts hit - */ -int get_ls_read_snapshot(ObTxDesc &tx, - const ObTxIsolationLevel isolation_level, - const share::ObLSID &local_ls_id, - const int64_t expire_ts, - ObTxReadSnapshot &snapshot); - // ------------------------------------------------------------------ // get snapshot version for out of transaction read special case // ------------------------------------------------------------------ @@ -210,24 +187,6 @@ int get_ls_read_snapshot(ObTxDesc &tx, */ int get_read_snapshot_version(const int64_t expire_ts, share::SCN &snapshot_version); - -/** - * get_ls_read_snapshot_version - get a read snapshot of specified - * local LogStream - * - * the snasphot can be used to read a consistency view of the state - * of specified log stream - * - * @local_ls_id: the local LogStream's id - * @expire_ts: microseconds of timestamp after - * which acquire action timeout - * @snapshot_version: the snapshot acquired - * - * Return: - * OB_SUCCESS - OK - * OB_NOT_MASTER - if local replica of @local_ls_id not leader - * OB_TIMEOUT - if expire_ts hit - */ /** * get_weak_read_snapshot_version - get snapshot version for weak read * @@ -239,7 +198,6 @@ int get_read_snapshot_version(const int64_t expire_ts, * OB_REPLICA_NOT_READABLE - snapshot is too stale */ int get_weak_read_snapshot_version(const int64_t max_read_stale_time, - const bool local_single_ls, share::SCN &snapshot_version); /** * refresh_read_snapshot_tx_state - update snapshot's tx state part @@ -397,19 +355,19 @@ int create_explicit_savepoint(ObTxDesc &tx, * @tx: the target transaction's descriptor * @savepoint: the target savepoint * @expire_ts: microseconds after which timeouted - * @extra_touched_ls: indicate the LogStreams which may - * touched by this transaction after - * the savepoint but not sensed by this - * transaction for some reason - * (eg. network partition, OutOfMemory) + * @touched_storage: indicate storage may be touched by this + * transaction after the savepoint but not + * sensed by this transaction for some + * reason (eg. network partition, + * OutOfMemory) * @clean_policy: control how to process when transaction * is wholly rollbacked, default is FAST_ROLLBACK * * When transaction wholly rollbacked, there are three policy: - * - FAST_ROLLBACK: only rollback transaction, the participants maybe in dirty + * - FAST_ROLLBACK: only rollback transaction, the write state may be dirty * state if the rollback msg has not been received successfully - * - KEEP: do not rollback transaction, but rollback write-set on participants - * - ROLLBACK: clean write-set on participants and rollback transaction + * - KEEP: do not rollback transaction, but rollback write-set on write state + * - ROLLBACK: clean write-set on write state and rollback transaction * * Return: * OB_SUCCESS - OK @@ -418,11 +376,11 @@ int create_explicit_savepoint(ObTxDesc &tx, * OB_TRANS_NEED_ROLLBACK - transaction internal error, need user explicit * rollback */ -int rollback_to_implicit_savepoint(ObTxDesc &tx, - const ObTxSEQ savepoint, - const int64_t expire_ts, - const share::ObLSArray *extra_touched_ls, - const ObTxCleanPolicy = ObTxCleanPolicy::FAST_ROLLBACK); + int rollback_to_implicit_savepoint(ObTxDesc &tx, + const ObTxSEQ savepoint, + const int64_t expire_ts, + const bool touched_storage, + const ObTxCleanPolicy = ObTxCleanPolicy::FAST_ROLLBACK); /** * rollback_to_explicit_savepoint - rollback to a explicit savepoint @@ -508,7 +466,7 @@ int create_stash_savepoint(ObTxDesc &tx, const ObString &name); int stash_pop_savepoints(ObTxDesc &tx, ObString &name); // ------------------------------------------------------------------ -// distributed transaction execution participant info collection +// transaction write state info collection // ------------------------------------------------------------------ /** @@ -539,7 +497,7 @@ int merge_tx_state(ObTxDesc &to, const ObTxDesc &from); int get_tx_exec_result(ObTxDesc &tx, ObTxExecResult &exec_info); /** - * add_tx_exec_result - report touched participant to transaction mananger + * add_tx_exec_result - report touched write state to transaction mananger * * * used in PX: @@ -560,7 +518,7 @@ int get_tx_exec_result(ObTxDesc &tx, ObTxExecResult &exec_info); * txs.add_tx_exec_result(session.tx_desc_, remoteResult.tx_exec_info) * * @tx: the transaction descriptor - * @exec_info: the transaction participants information, + * @exec_info: the transaction write state information, * generated by transaction manager during access * transactional storage, and gathered by calling: * get_tx_exec_result(tx, exec_info) diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_tx_ctx.cpp similarity index 71% rename from src/storage/tx/ob_trans_part_ctx.cpp rename to src/storage/tx/ob_tx_ctx.cpp index 3da58b08d..2a85d8218 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_tx_ctx.cpp @@ -16,7 +16,7 @@ #define USING_LOG_PREFIX TRANS -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" #include "share/rc/ob_module_provider.h" #include "ob_tx_redo_submitter.h" #include "storage/tx/ob_trans_service.h" @@ -27,6 +27,7 @@ #include "logservice/ob_log_service.h" #include "storage/ddl/ob_ddl_inc_clog_callback.h" #include "storage/tx/ob_tx_log_operator.h" +#include "storage/tx/ob_tx_print_time_guard.h" namespace oceanbase { @@ -44,34 +45,19 @@ using namespace tablelock; do { \ } while (0); -bool ObPartTransCtx::is_inited() const { return ATOMIC_LOAD(&is_inited_); } +bool ObTxCtx::is_inited() const { return ATOMIC_LOAD(&is_inited_); } -static const int64_t STANDBY_STAT_ARRAY_SIZE = 10; -// cache line aligned -static int64_t STANDBY_STAT[STANDBY_STAT_ARRAY_SIZE][8]; - -static void statistics_for_standby() -{ - for (int64_t i = 0; i < STANDBY_STAT_ARRAY_SIZE; i++) { - LOG_INFO("standby checking statistics", K(STANDBY_STAT[i][0])); - STANDBY_STAT[i][0] = 0; - } -} - -int ObPartTransCtx::init(const common::ObAddr &scheduler, - const uint32_t session_id, +int ObTxCtx::init(const uint32_t session_id, const uint32_t client_sid, const uint32_t associated_session_id, const ObTransID &trans_id, const int64_t trans_expired_time, - const ObLSID &ls_id, const uint64_t cluster_version, ObTransService *trans_service, const uint64_t cluster_id, - const int64_t epoch, ObLSTxCtxMgr *ls_ctx_mgr, const bool for_replay, - const PartCtxSource ctx_source, + const TxCtxSource ctx_source, ObXATransID xid) { int ret = OB_SUCCESS; @@ -82,34 +68,32 @@ int ObPartTransCtx::init(const common::ObAddr &scheduler, // specified init : initialize with specified value if (OB_UNLIKELY(is_inited_)) { - TRANS_LOG(WARN, "ObPartTransCtx inited twice"); + TRANS_LOG(WARN, "ObTxCtx inited twice"); ret = OB_INIT_TWICE; - } else if (OB_UNLIKELY(!true) || OB_UNLIKELY(!trans_id.is_valid()) + } else if (OB_UNLIKELY(!trans_id.is_valid()) || OB_UNLIKELY(trans_expired_time <= 0) || OB_UNLIKELY(cluster_version < 0) - || OB_ISNULL(trans_service) || OB_UNLIKELY(!ls_id.is_valid())) { - TRANS_LOG(WARN, "invalid argument", K(trans_id), K(epoch), K(cluster_version), - KP(trans_service), K(ls_id)); + || OB_ISNULL(trans_service)) { + TRANS_LOG(WARN, "invalid argument", K(trans_id), K(cluster_version), + KP(trans_service)); ret = OB_INVALID_ARGUMENT; } else { - ls_id_ = ls_id; ls_tx_ctx_mgr_ = ls_ctx_mgr; trans_service_ = trans_service; if (OB_FAIL(lock_.init(this))) { TRANS_LOG(WARN, "init lock error", KR(ret)); - } else if (OB_ISNULL(rpc_ = trans_service->get_trans_rpc()) - || OB_ISNULL(timer_ = &(trans_service->get_trans_timer()))) { - TRANS_LOG(ERROR, "ObTransService is invalid, unexpected error", KP(rpc_)); + } else if (OB_ISNULL(timer_ = &(trans_service->get_trans_timer()))) { + TRANS_LOG(ERROR, "ObTransService is invalid, unexpected error"); ret = OB_ERR_UNEXPECTED; } else if (OB_FAIL(timeout_task_.init(this))) { TRANS_LOG(WARN, "timeout task init error", KR(ret)); - } else if (OB_FAIL(init_memtable_ctx_(ls_id))) { - TRANS_LOG(WARN, "ObPartTransCtx init memtable context error", KR(ret), K(trans_id), K(ls_id)); - } else if (OB_FAIL(init_log_cbs_(ls_id, trans_id))) { - TRANS_LOG(WARN, "init log cbs failed", KR(ret), K(trans_id), K(ls_id)); + } else if (OB_FAIL(init_memtable_ctx_())) { + TRANS_LOG(WARN, "ObTxCtx init memtable context error", KR(ret), K(trans_id)); + } else if (OB_FAIL(init_log_cbs_(trans_id))) { + TRANS_LOG(WARN, "init log cbs failed", KR(ret), K(trans_id)); } else if (OB_FAIL(ctx_tx_data_.init(trans_expired_time, ls_ctx_mgr, trans_id))) { - TRANS_LOG(WARN, "init ctx tx data failed",K(ret), K(trans_id), K(ls_id)); - } else if (OB_FAIL(mds_cache_.init(ls_id, trans_id))) { - TRANS_LOG(WARN, "init mds cache failed", K(ret), K(trans_id), K(ls_id)); + TRANS_LOG(WARN, "init ctx tx data failed",K(ret), K(trans_id)); + } else if (OB_FAIL(mds_cache_.init(trans_id))) { + TRANS_LOG(WARN, "init mds cache failed", K(ret), K(trans_id)); } } @@ -120,36 +104,33 @@ int ObPartTransCtx::init(const common::ObAddr &scheduler, associated_session_id_ = associated_session_id; addr_ = trans_service->get_server(); trans_id_ = trans_id; + for_replay_ = for_replay; trans_expired_time_ = trans_expired_time; ctx_create_time_ = ObClockGenerator::getClock(); ctx_source_ = ctx_source; cluster_version_accurate_ = cluster_version > 0; cluster_version_ = cluster_version ?: LAST_BARRIER_DATA_VERSION; part_trans_action_ = ObPartTransAction::INIT; - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; + commit_retry_timeout_ = get_commit_retry_interval_us_(); last_request_ts_ = ctx_create_time_; - exec_info_.scheduler_ = scheduler; - exec_info_.trans_type_ = TransType::SP_TRANS; - last_ask_scheduler_status_ts_ = ObClockGenerator::getClock(); + last_check_tx_status_ts_ = ObClockGenerator::getClock(); cluster_id_ = cluster_id; - epoch_ = epoch; pending_write_ = 0; - set_role_state(for_replay); block_frozen_memtable_ = nullptr; if (!xid.empty()) { exec_info_.xid_ = xid; } - if (is_follower_()) { + if (is_for_replay()) { mt_ctx_.trans_replay_begin(); } else { mt_ctx_.trans_begin(); } mt_ctx_.set_trans_ctx(this); - mt_ctx_.set_for_replay(is_follower_()); + mt_ctx_.set_for_replay(is_for_replay()); if (!GCONF.enable_record_trace_log) { tlog_ = NULL; } else { @@ -166,30 +147,29 @@ int ObPartTransCtx::init(const common::ObAddr &scheduler, REC_TRANS_TRACE_EXT2(tlog_, init, OB_ID(addr), (void*)this, - OB_ID(id), ls_id_.id(), OB_ID(trans_id), trans_id, OB_ID(ref), get_ref()); - TRANS_LOG(TRACE, "part trans ctx init", K(ret), K(trans_id), K(trans_expired_time), - K(ls_id), K(cluster_version), KP(trans_service), K(cluster_id), K(epoch)); + TRANS_LOG(TRACE, "tx ctx init", K(ret), K(trans_id), K(trans_expired_time), + K(cluster_version), KP(trans_service), K(cluster_id)); return ret; } -int ObPartTransCtx::init_memtable_ctx_(const ObLSID &ls_id) +int ObTxCtx::init_memtable_ctx_() { int ret = OB_SUCCESS; if (OB_FAIL(mt_ctx_.init())) { TRANS_LOG(WARN, "memtable context init fail", KR(ret)); } else if (OB_FAIL(mt_ctx_.enable_lock_table(ls_tx_ctx_mgr_))) { - TRANS_LOG(WARN, "enable_lock_table fail", K(ret), K(ls_id)); + TRANS_LOG(WARN, "enable_lock_table fail", K(ret)); } else { // the elr_handler.mt_ctx_ is used to notify the lock_wait_mgr for early lock release txn elr_handler_.set_memtable_ctx(&mt_ctx_); } - TRANS_LOG(DEBUG, "init memtable ctx", KR(ret), K(ls_id), K(trans_id_)); + TRANS_LOG(DEBUG, "init memtable ctx", KR(ret), K(trans_id_)); return ret; } -void ObPartTransCtx::destroy() +void ObTxCtx::destroy() { int ret = OB_SUCCESS; if (OB_LIKELY(is_inited_)) { @@ -221,7 +201,7 @@ void ObPartTransCtx::destroy() ls_tx_ctx_mgr_->dec_total_tx_ctx_count(); } // Defensive Check 3 : missing to callback scheduler - if (!is_follower_() && need_callback_scheduler_()) { + if (!is_for_replay() && need_commit_callback_()) { int tx_result = OB_TRANS_UNKNOWN; switch (ctx_tx_data_.get_state()) { case ObTxCommitData::COMMIT: tx_result = OB_TRANS_COMMITED; break; @@ -229,11 +209,7 @@ void ObPartTransCtx::destroy() default: TRANS_LOG(ERROR, "oops! unexpected tx_state in tx data", K(ctx_tx_data_.get_state())); } - if (SCN::min_scn() == start_working_log_ts_) { - TRANS_LOG(ERROR, "missing callback scheduler, do callback", K(tx_result), KPC(this)); - } else { - TRANS_LOG(WARN, "missing callback scheduler maybe, do callback", K(tx_result), KPC(this)); - } + TRANS_LOG(ERROR, "missing commit callback, do callback", K(tx_result), KPC(this)); // NOTE: callback scheduler may introduce deadlock, need take care trans_service_->handle_tx_commit_result(trans_id_, tx_result, SCN()); FORCE_PRINT_TRACE(tlog_, "[missing callback scheduler] "); @@ -250,8 +226,7 @@ void ObPartTransCtx::destroy() mds_cache_.destroy(); if (mds_cache_.is_mem_leak()) { - TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "mds memory leak!", K(trans_id_), K(ls_id_), - K(mds_cache_), K(exec_info_), K(ctx_tx_data_), K(create_ctx_scn_), + TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "mds memory leak!", K(trans_id_), K(mds_cache_), K(exec_info_), K(ctx_tx_data_), K(create_ctx_scn_), K(ctx_source_), K(ctx_create_time_)); FORCE_PRINT_TRACE(tlog_, "[check mds mem leak] "); } @@ -275,13 +250,12 @@ void ObPartTransCtx::destroy() } } -void ObPartTransCtx::default_init_() +void ObTxCtx::default_init_() { // TODO for ObTransCtx // lock_.reset(); stc_.reset(); commit_cb_.reset(); - callback_scheduler_on_clear_ = false; pending_callback_param_ = OB_SUCCESS; trans_need_wait_wrap_.reset(); is_exiting_ = false; @@ -303,10 +277,10 @@ void ObPartTransCtx::default_init_() cur_query_start_time_ = 0; clean_retain_cause_(); - upstream_state_ = ObTxState::INIT; + target_state_ = ObTxState::INIT; exec_info_.reset(); ctx_tx_data_.reset(); - sub_state_.reset(); + runtime_state_.reset(); reset_log_cbs_(); last_op_sn_ = 0; last_scn_.reset(); @@ -318,12 +292,9 @@ void ObPartTransCtx::default_init_() mds_cache_.reset(); retain_ctx_func_ptr_ = nullptr; create_ctx_scn_.reset(); - ctx_source_ = PartCtxSource::UNKNOWN; + ctx_source_ = TxCtxSource::UNKNOWN; replay_completeness_.reset(); is_submitting_redo_log_for_freeze_ = false; - start_working_log_ts_ = SCN::min_scn(); - max_2pc_commit_scn_.reset(); - coord_prepare_info_arr_.reset(); reserve_allocator_.reset(); elr_handler_.reset(); trace_log_.reset(); @@ -331,7 +302,7 @@ void ObPartTransCtx::default_init_() } // thread-unsafe -int ObPartTransCtx::start_trans() +int ObTxCtx::start_trans() { int ret = OB_SUCCESS; // first register task timeout = 10s, @@ -340,14 +311,13 @@ int ObPartTransCtx::start_trans() const int64_t left_time = trans_expired_time_ - ObClockGenerator::getClock(); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(is_exiting_)) { TRANS_LOG(WARN, "transaction is exiting", "context", *this); ret = OB_TRANS_IS_EXITING; - // leader expired before init ctx - } else if (OB_UNLIKELY(is_follower_())) { - ret = OB_NOT_MASTER; + } else if (OB_UNLIKELY(is_for_replay())) { + ret = OB_STATE_NOT_MATCH; TRANS_LOG(WARN, "invalid state, transaction is replaying", KR(ret), KPC(this)); } else if (OB_UNLIKELY(left_time <= 0)) { ret = OB_TRANS_TIMEOUT; @@ -372,7 +342,7 @@ int ObPartTransCtx::start_trans() return ret; } -int ObPartTransCtx::trans_kill_() +int ObTxCtx::trans_kill_() { int ret = OB_SUCCESS; TRANS_LOG(INFO, "trans killed", K(trans_id_)); @@ -390,7 +360,7 @@ int ObPartTransCtx::trans_kill_() return ret; } -int ObPartTransCtx::trans_clear_(const share::SCN log_ts) +int ObTxCtx::trans_clear_(const share::SCN log_ts) { int ret = OB_SUCCESS; @@ -420,7 +390,7 @@ int ObPartTransCtx::trans_clear_(const share::SCN log_ts) return ret; } -int ObPartTransCtx::handle_timeout(const int64_t delay) +int ObTxCtx::handle_timeout(const int64_t delay) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -432,7 +402,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) CtxLockGuard guard(lock_, false); timeguard.click(); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(is_exiting_)) { TRANS_LOG(WARN, "transaction is exiting", "context", *this); @@ -442,7 +412,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) } else { timeguard.click(); (void)unregister_timeout_task_(); - update_trans_2pc_timeout_(); + update_commit_retry_timeout_(); timeout_task_.set_running(true); if (trans_expired_time_ > 0 && trans_expired_time_ < INT64_MAX) { @@ -451,7 +421,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) tmp_ret = OB_TRANS_COMMIT_TOO_MUCH_TIME; LOG_DBA_ERROR_V2(OB_TRANS_COMMIT_COST_TOO_MUCH_TIME, tmp_ret, "Transaction commit cost too much time. ", - "trans_id is ", trans_id_, ". ls_id is ", ls_id_, ". ", + "trans_id is ", trans_id_, ". ", "[suggestion] You can query GV$OB_PROCESSLIST view to get more information."); } } else { @@ -460,7 +430,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) tmp_ret = OB_TRANS_LIVE_TOO_MUCH_TIME; LOG_DBA_WARN_V2(OB_TRANS_LIVE_TOO_LONG, tmp_ret, "Transaction live too long without commit or abort. ", - "trans_id is ", trans_id_, ". ls_id is ", ls_id_, ". ", + "trans_id is ", trans_id_, ". ", "[suggestion] This may be normal and simply because the client hasn't executed the 'commit' command yet. ", "You can query GV$OB_PROCESSLIST view to get more information ", "and confirm whether you need to submit this transaction."); @@ -477,7 +447,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) } // handle commit timeout on root node - if (!is_follower_() && part_trans_action_ == ObPartTransAction::COMMIT && is_local_tx_()) { + if (!is_for_replay() && part_trans_action_ == ObPartTransAction::COMMIT) { if (tx_expired) { tmp_ret = post_tx_commit_resp_(OB_TRANS_TIMEOUT); TRANS_LOG(INFO, "callback scheduler txn has timeout", K(tmp_ret), KPC(this)); @@ -491,55 +461,52 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) if (OB_FAIL(logservice::check_clog_disk_full_or_hang(clog_is_full, clog_is_hang))) { TRANS_LOG(WARN, "fail to check clog disk status", KR(ret)); } else if (clog_is_full || clog_is_hang) { - tmp_ret = post_tx_commit_resp_(OB_NOT_MASTER); + tmp_ret = post_tx_commit_resp_(OB_EAGAIN); TRANS_LOG(ERROR, "clog disk has fatal error, make scheduler retry commit", K(tmp_ret), KPC(this)); } } } // go to preapre state when recover from redo complete - if (!is_follower_()) { + if (!is_for_replay()) { if (ObTxState::REDO_COMPLETE == get_downstream_state()) { - if (is_local_tx_()) { - if (!is_logging_()) { - if (OB_FAIL(one_phase_commit_())) { - TRANS_LOG(WARN, "two phase commit failed", KR(ret), KPC(this)); - } else { - part_trans_action_ = ObPartTransAction::COMMIT; - } + if (!is_logging_()) { + if (OB_FAIL(one_phase_commit_())) { + TRANS_LOG(WARN, "one phase commit failed", KR(ret), KPC(this)); + } else { + part_trans_action_ = ObPartTransAction::COMMIT; } } } } // retry commiting for every node - if (!is_follower_() && is_committing_()) { + if (!is_for_replay() && is_committing_()) { try_submit_next_log_(); } // retry submit abort log for local tx abort // - // NOTE: due to some local tx may set to dist-trans if it request 2pc msg - // while choose abort itself forcedly, hence need add the second condition - if (!is_follower_() && (is_local_tx_() || sub_state_.is_force_abort()) - && get_upstream_state() == ObTxState::ABORT - && get_upstream_state() != get_downstream_state()) { + // Force-abort may set a target state before the durable state catches up. + if (!is_for_replay() + && get_target_state() == ObTxState::ABORT + && get_target_state() != get_downstream_state()) { if (OB_FAIL(compensate_abort_log_())) { TRANS_LOG(WARN, "compensate abort log failed", KR(ret), KPC(this)); } } // if not committing, abort txn if it was expired - if (!is_follower_() && !is_committing_() && tx_expired) { + if (!is_for_replay() && !is_committing_() && tx_expired) { if (OB_FAIL(abort_(OB_TRANS_TIMEOUT))) { TRANS_LOG(WARN, "abort failed", KR(ret), KPC(this)); } } // register timeout task again if need - if (!is_follower_() && !is_exiting_) { - const int64_t timeout_left = is_committing_() ? trans_2pc_timeout_ : - MIN(MAX_TRANS_2PC_TIMEOUT_US, MAX(trans_expired_time_ - now, 1000 * 1000)); + if (!is_for_replay() && !is_exiting_) { + const int64_t timeout_left = is_committing_() ? commit_retry_timeout_ : + MIN(MAX_TRANS_COMMIT_RETRY_TIMEOUT_US, MAX(trans_expired_time_ - now, 1000 * 1000)); if (OB_FAIL(register_timeout_task_(timeout_left))) { TRANS_LOG(WARN, "register timeout task failed", KR(ret), KPC(this)); } @@ -571,7 +538,7 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) return ret; } -int ObPartTransCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) +int ObTxCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -581,14 +548,14 @@ int ObPartTransCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) CtxLockGuard guard(lock_); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (is_exiting_) { TRANS_LOG(INFO, "trans is existing when being killed", K(*this), K(arg)); } else { bool notify_scheduler_tx_killed = false; if (arg.graceful_) { - if (FALSE_IT(notify_scheduler_tx_killed = !is_follower_() && part_trans_action_ == ObPartTransAction::START)) { + if (FALSE_IT(notify_scheduler_tx_killed = !is_for_replay() && part_trans_action_ == ObPartTransAction::START)) { } else if (has_persisted_log_() || is_logging_()) { // submit abort_log and wait success if (OB_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { @@ -610,25 +577,23 @@ int ObPartTransCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) if (OB_FAIL(do_local_tx_end_(TxEndAction::KILL_TX_FORCEDLY))) { TRANS_LOG(WARN, "kill trx forcedly failed", "context", *this, K(arg)); } - notify_scheduler_tx_killed = !is_follower_() && part_trans_action_ == ObPartTransAction::START; + notify_scheduler_tx_killed = !is_for_replay() && part_trans_action_ == ObPartTransAction::START; // if ctx was killed gracefully or forcely killed // notify scheduler commit result, if in committing - if (!is_follower_() && part_trans_action_ == ObPartTransAction::COMMIT && is_local_tx_()) { + if (!is_for_replay() && part_trans_action_ == ObPartTransAction::COMMIT) { // notify scheduler only if commit callback has not been armed if (commit_cb_.is_enabled() && !commit_cb_.is_inited()) { - if (exec_info_.scheduler_ == addr_) { - if (OB_TMP_FAIL(prepare_commit_cb_for_role_change_(cb_param, cb_list))) { - TRANS_LOG(WARN, "prepare commit cb fail", K(tmp_ret), K(cb_param), KPC(this)); - ret = (ret == OB_SUCCESS) ? tmp_ret : ret; - } - } else { - post_tx_commit_resp_(cb_param); + if (OB_TMP_FAIL(prepare_commit_cb_for_role_change_(cb_param, cb_list))) { + TRANS_LOG(WARN, "prepare commit cb fail", K(tmp_ret), K(cb_param), KPC(this)); + ret = (ret == OB_SUCCESS) ? tmp_ret : ret; } } } } if (notify_scheduler_tx_killed) { - notify_scheduler_tx_killed_(arg.graceful_ ? ObTxAbortCause::PARTICIPANT_KILLED_GRACEFULLY : ObTxAbortCause::PARTICIPANT_KILLED_FORCEDLY); + notify_tx_killed_(arg.graceful_ + ? ObTxAbortCause::WRITE_STATE_KILLED_GRACEFULLY + : ObTxAbortCause::WRITE_STATE_KILLED_FORCEDLY); } } TRANS_LOG(WARN, "trans is killed", K(ret), K(arg), K(cb_param), KPC(this)); @@ -642,7 +607,6 @@ int ObPartTransCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) * * the commiting is asynchronous * - * @parts: participant list * @commit_time: STC reference * @expire_ts: timestamp in micorseconds after which * commit result is not concerned for the caller @@ -659,24 +623,23 @@ int ObPartTransCtx::kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list) * OB_ERR_XXX - the request was rejected, can not be handle * caller can retry commit or choice to abort txn */ -int ObPartTransCtx::commit(const ObTxCommitParts &parts, - const MonotonicTs &commit_time, +int ObTxCtx::commit(const MonotonicTs &commit_time, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id) { - TRANS_LOG(DEBUG, "tx.commit", K(parts), K(trans_id_), K(ls_id_)); + TRANS_LOG(DEBUG, "tx.commit", K(trans_id_)); int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; CtxLockGuard guard(lock_); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(0 >= expire_ts)) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument", K(ret), K(expire_ts), KPC(this)); - } else if (OB_UNLIKELY(is_follower_())) { - ret = OB_NOT_MASTER; + } else if (OB_UNLIKELY(is_for_replay())) { + ret = OB_STATE_NOT_MATCH; TRANS_LOG(WARN, "transaction is replaying", KR(ret), KPC(this)); } else if (!(ObTxState::INIT == get_downstream_state() || (ObTxState::REDO_COMPLETE == get_downstream_state() && part_trans_action_ < ObPartTransAction::COMMIT))) { @@ -701,32 +664,21 @@ int ObPartTransCtx::commit(const ObTxCommitParts &parts, } else if (OB_UNLIKELY(pending_write_)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "access in progress", K(ret), K_(pending_write), KPC(this)); - } else if (OB_FAIL(set_2pc_participants_(parts))) { - TRANS_LOG(WARN, "set participants failed", K(ret), KPC(this)); - } else if (OB_FAIL(set_2pc_request_id_(request_id))) { + } else if (OB_FAIL(set_commit_request_id_(request_id))) { TRANS_LOG(WARN, "set request id failed", K(ret), K(request_id), KPC(this)); } else if (OB_FAIL(set_app_trace_info_(app_trace_info))) { TRANS_LOG(WARN, "set app trace info error", K(ret), K(app_trace_info), KPC(this)); } else if (FALSE_IT(stmt_expired_time_ = expire_ts)) { } else { + exec_info_.mark_write_state(); if (commit_time.is_valid()) { set_stc_(commit_time); } else { set_stc_by_now_(); } - if (exec_info_.participants_.count() <= 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "the size of participant is 0 when commit", KPC(this)); - } else if (exec_info_.participants_.count() == 1 && - exec_info_.participants_[0] == ls_id_) { - exec_info_.trans_type_ = TransType::SP_TRANS; - can_elr_ = (trans_service_->get_tx_elr_util().is_can_tenant_elr() ? true : false); - if (OB_FAIL(one_phase_commit_())) { - TRANS_LOG(WARN, "start sp coimit fail", K(ret), KPC(this)); - } - } else { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(ERROR, "2pc is not supported", K(ret), KPC(this)); + can_elr_ = (trans_service_->get_tx_elr_util().is_can_tenant_elr() ? true : false); + if (OB_FAIL(one_phase_commit_())) { + TRANS_LOG(WARN, "start sp coimit fail", K(ret), KPC(this)); } } if (OB_SUCC(ret)) { @@ -743,7 +695,7 @@ int ObPartTransCtx::commit(const ObTxCommitParts &parts, return ret; } -int ObPartTransCtx::one_phase_commit_() +int ObTxCtx::one_phase_commit_() { int ret = OB_SUCCESS; @@ -754,7 +706,7 @@ int ObPartTransCtx::one_phase_commit_() return ret; } -int ObPartTransCtx::check_modify_schema_elapsed( +int ObTxCtx::check_modify_schema_elapsed( const ObTabletID &tablet_id, const int64_t schema_version) { @@ -763,7 +715,7 @@ int ObPartTransCtx::check_modify_schema_elapsed( CtxLockGuard guard(lock_, false); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(schema_version <= 0) || OB_UNLIKELY(!tablet_id.is_valid())) { @@ -772,9 +724,9 @@ int ObPartTransCtx::check_modify_schema_elapsed( ret = OB_INVALID_ARGUMENT; } else if (is_exiting_) { // do nothing - } else if (is_follower_()) { - ret = OB_EAGAIN; - TRANS_LOG(WARN, "current transaction not master, need retry later", K(ret), + } else if (is_for_replay()) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(WARN, "cannot check modify schema on replay context", K(ret), K(tablet_id), K(schema_version), "context", *this); } else if (OB_FAIL(mt_ctx_.check_modify_schema_elapsed(tablet_id, schema_version))) { @@ -796,7 +748,7 @@ int ObPartTransCtx::check_modify_schema_elapsed( return ret; } -int ObPartTransCtx::check_modify_time_elapsed( +int ObTxCtx::check_modify_time_elapsed( const ObTabletID &tablet_id, const int64_t timestamp) { @@ -805,7 +757,7 @@ int ObPartTransCtx::check_modify_time_elapsed( CtxLockGuard guard(lock_, false); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_UNLIKELY(timestamp <= 0) || OB_UNLIKELY(!tablet_id.is_valid())) { @@ -814,9 +766,9 @@ int ObPartTransCtx::check_modify_time_elapsed( ret = OB_INVALID_ARGUMENT; } else if (is_exiting_) { // do nothing - } else if (is_follower_()) { - ret = OB_NOT_MASTER; - TRANS_LOG(WARN, "current transaction not master, need retry later", K(ret), + } else if (is_for_replay()) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(WARN, "cannot check modify time on replay context", K(ret), K(tablet_id), K(timestamp), "context", *this); } else if (OB_FAIL(mt_ctx_.check_modify_time_elapsed(tablet_id, timestamp))) { @@ -838,11 +790,11 @@ int ObPartTransCtx::check_modify_time_elapsed( return ret; } -int ObPartTransCtx::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const +int ObTxCtx::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (is_exiting_) { // do nothing @@ -858,11 +810,11 @@ int ObPartTransCtx::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const return ret; } -int ObPartTransCtx::iterate_tx_lock_priority_list(ObPrioOpIterator &iter) const +int ObTxCtx::iterate_tx_lock_priority_list(ObPrioOpIterator &iter) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (is_exiting_) { // do nothing @@ -878,13 +830,13 @@ int ObPartTransCtx::iterate_tx_lock_priority_list(ObPrioOpIterator &iter) const return ret; } -int ObPartTransCtx::iterate_tx_lock_stat(ObTxLockStatIterator &iter) +int ObTxCtx::iterate_tx_lock_stat(ObTxLockStatIterator &iter) { int ret = OB_SUCCESS; ObMemtableKeyArray memtable_key_info_arr; if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_FAIL(get_memtable_key_arr(memtable_key_info_arr))) { TRANS_LOG(WARN, "get memtable key arr fail", KR(ret), K(memtable_key_info_arr)); @@ -902,7 +854,6 @@ int ObPartTransCtx::iterate_tx_lock_stat(ObTxLockStatIterator &iter) for (int i = 0; OB_SUCC(ret) && i < count; i++) { ObTxLockStat tx_lock_stat; if (OB_FAIL(tx_lock_stat.init(get_addr(), - get_ls_id(), memtable_key_info_arr.at(i), get_session_id(), get_client_sid(), @@ -922,7 +873,7 @@ int ObPartTransCtx::iterate_tx_lock_stat(ObTxLockStatIterator &iter) return ret; } -int ObPartTransCtx::trans_replay_abort_(const SCN &final_log_ts) +int ObTxCtx::trans_replay_abort_(const SCN &final_log_ts) { int ret = OB_SUCCESS; @@ -935,7 +886,7 @@ int ObPartTransCtx::trans_replay_abort_(const SCN &final_log_ts) return ret; } -int ObPartTransCtx::trans_replay_commit_(const SCN &commit_version, +int ObTxCtx::trans_replay_commit_(const SCN &commit_version, const SCN &final_log_ts, const uint64_t log_cluster_version, const uint64_t checksum) @@ -961,7 +912,7 @@ int ObPartTransCtx::trans_replay_commit_(const SCN &commit_version, return ret; } -int ObPartTransCtx::update_publish_version_(const SCN &publish_version, const bool for_replay) +int ObTxCtx::update_publish_version_(const SCN &publish_version, const bool for_replay) { int ret = OB_SUCCESS; if (!publish_version.is_valid()) { @@ -978,208 +929,29 @@ int ObPartTransCtx::update_publish_version_(const SCN &publish_version, const bo return ret; } -int ObPartTransCtx::replay_start_working_log(const SCN start_working_ts) -{ - int ret = OB_SUCCESS; - - CtxLockGuard guard(lock_); - exec_info_.data_complete_ = false; - start_working_log_ts_ = start_working_ts; - if (!has_persisted_log_()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "trx has no persisten log", K(ret), K(*this)); - } - - return ret; -} -bool ObPartTransCtx::is_logging_() const { return !busy_cbs_.is_empty(); } +bool ObTxCtx::is_logging_() const { return !busy_cbs_.is_empty(); } -bool ObPartTransCtx::need_force_abort_() const +bool ObTxCtx::need_force_abort_() const { - return sub_state_.is_force_abort() && !sub_state_.is_state_log_submitted(); + return runtime_state_.is_force_abort() && !runtime_state_.is_state_log_submitted(); } -bool ObPartTransCtx::is_force_abort_logging_() const +bool ObTxCtx::is_force_abort_logging_() const { - return sub_state_.is_force_abort() && sub_state_.is_state_log_submitting(); + return runtime_state_.is_force_abort() && runtime_state_.is_state_log_submitting(); } -bool ObPartTransCtx::has_persisted_log_() const +bool ObTxCtx::has_persisted_log_() const { return exec_info_.max_applying_log_ts_.is_valid() || // for ctx created by parallel replay, and no serial log replayed // the max_applying_log_ts_ has not been set, in this case use // replay_completeness to distinguish with situations on leader - (is_follower_() && replay_completeness_.is_unknown()); -} - -int ObPartTransCtx::gts_callback_interrupted(const int errcode, - const share::ObLSID target_ls_id) -{ - int ret = OB_SUCCESS; - - bool need_revert_ctx = false; - { - CtxLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(ERROR, "ObPartTransCtx not inited", K(ret)); - } else if (OB_LS_OFFLINE == errcode) { - if (target_ls_id != ls_id_) { - ret = OB_EAGAIN; - } else { - need_revert_ctx = true; - sub_state_.clear_gts_waiting(); - TRANS_LOG(INFO, "transaction is interruputed gts callback", KR(ret), K(errcode), "context", *this); - } - } else { - // for OB_TENANT_NOT_EXIST - need_revert_ctx = true; - sub_state_.clear_gts_waiting(); - TRANS_LOG(INFO, "transaction is interruputed gts callback", KR(ret), K(errcode), "context", *this); - } - } - if (need_revert_ctx) { - ret = ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this); - } - - return ret; -} - -int ObPartTransCtx::get_gts_callback(const MonotonicTs srr, - const SCN >s, - const MonotonicTs receive_gts_ts) -{ - int ret = OB_SUCCESS; - bool need_revert_ctx = false; - { - CtxLockGuard guard(lock_); - TRANS_LOG(DEBUG, "ObPartTransCtx get_gts_callback begin", KPC(this)); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObPartTransCtx not inited", K(ret)); - } else if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(!gts.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts), "context", *this); - } else if (OB_UNLIKELY(is_exiting_)) { - ret = OB_TRANS_IS_EXITING; - need_revert_ctx = true; - TRANS_LOG(WARN, "transaction is exiting", KR(ret), "context", *this); - } else if (OB_UNLIKELY(is_follower_())) { - sub_state_.clear_gts_waiting(); - need_revert_ctx = true; - } else if (srr < get_stc_()) { - ret = OB_EAGAIN; - } else { - if (OB_LIKELY(sub_state_.is_gts_waiting())) { - sub_state_.clear_gts_waiting(); - } else { - TRANS_LOG(ERROR, "unexpected sub state", K(*this)); - } - const int64_t GET_GTS_AHEAD_INTERVAL = 0; //GCONF._ob_get_gts_ahead_interval; - set_trans_need_wait_wrap_(receive_gts_ts, GET_GTS_AHEAD_INTERVAL); - // the same as before prepare - mt_ctx_.set_trans_version(gts); - const SCN max_read_ts = trans_service_->get_tx_version_mgr().get_max_read_ts(); - // TRANS_LOG(INFO, "get_gts_callback mid", K(*this), K(log_type)); - - if (is_local_tx_()) { - if (OB_FAIL(ctx_tx_data_.set_commit_version(SCN::max(gts, max_read_ts)))) { - TRANS_LOG(WARN, "set commit_version failed", K(ret)); - } else if (part_trans_action_ != ObPartTransAction::COMMIT) { - // one phase commit failed or abort - TRANS_LOG(INFO, "one phase commit has not been successful, need retry", K(ret), KPC(this)); - } else if (sub_state_.is_state_log_submitted()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "the commit log has been submitted", K(ret), KPC(this)); - } else if (OB_FAIL(submit_log_impl_(ObTxLogType::TX_COMMIT_LOG))) { - TRANS_LOG(WARN, "submit commit log in gts callback failed", K(ret), KPC(this)); - } - } - - need_revert_ctx = true; - } - REC_TRANS_TRACE_EXT2(tlog_, get_gts_callback, OB_Y(ret), OB_ID(srr), srr.mts_, OB_Y(gts), OB_ID(ref), get_ref()); - - // before revert self - if (OB_FAIL(ret)) { - if (OB_EAGAIN == ret) { - TRANS_LOG(DEBUG, "ObPartTransCtx::get_gts_callback - retry gts task by TsMgr", KR(ret), K(*this), - K(gts)); - } else { - TRANS_LOG(WARN, "ObPartTransCtx::get_gts_callback", KR(ret), K(*this), K(gts)); - if (sub_state_.is_gts_waiting()) { - sub_state_.clear_gts_waiting(); - } - } - } - TRANS_LOG(DEBUG, "ObPartTransCtx get_gts_callback end", KPC(this)); - } - - if (need_revert_ctx) { - ret = ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this); - } - - return ret; -} - -int ObPartTransCtx::gts_elapse_callback(const MonotonicTs srr, const SCN >s) -{ - int ret = OB_SUCCESS; - bool need_revert_ctx = false; - { - CtxLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "ObPartTransCtx not inited", K(ret)); - } else if (OB_UNLIKELY(!srr.is_valid()) || OB_UNLIKELY(!gts.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(srr), K(gts), "context", *this); - } else if (OB_UNLIKELY(is_exiting_)) { - need_revert_ctx = true; - ret = OB_TRANS_IS_EXITING; - TRANS_LOG(WARN, "transaction is exiting", KR(ret), "context", *this); - } else if (ctx_tx_data_.get_commit_version() > gts) { - ret = OB_EAGAIN; - TRANS_LOG(WARN, "the commit version is still larger than gts", K(ret), K(this)); - } else { - if (OB_UNLIKELY(!sub_state_.is_gts_waiting())) { - TRANS_LOG(ERROR, "unexpected gts waiting flag", K(*this)); - } else { - sub_state_.clear_gts_waiting(); - } - - if (is_local_tx_()) { - if (OB_FAIL(after_local_commit_succ_())) { - TRANS_LOG(WARN, "terminate trx after local commit failed", KR(ret), KPC(this)); - } - } - need_revert_ctx = true; - } - REC_TRANS_TRACE_EXT2(tlog_, gts_elapse_callback, OB_Y(ret), OB_ID(srr), srr.mts_, OB_Y(gts), OB_ID(ref), get_ref()); - - // before revert self - if (OB_FAIL(ret)) { - if (OB_EAGAIN == ret) { - TRANS_LOG(WARN, "ObPartTransCtx::gts_elapse_callback - retry gts task by TsMgr", KR(ret), K(*this), - K(gts)); - } else { - TRANS_LOG(WARN, "ObPartTransCtx::gts_elapse_callback", KR(ret), K(*this), K(gts)); - if (sub_state_.is_gts_waiting()) { - sub_state_.clear_gts_waiting(); - } - } - } - } - - if (need_revert_ctx) { - ret = ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this); - } - return ret; + (is_for_replay() && replay_completeness_.is_unknown()); } -int ObPartTransCtx::get_prepare_version_if_prepared(bool &is_prepared, SCN &prepare_version) +int ObTxCtx::get_prepare_version_if_prepared(bool &is_prepared, SCN &prepare_version) { int ret = OB_SUCCESS; ObTxState cur_state = exec_info_.state_; @@ -1206,11 +978,11 @@ int ObPartTransCtx::get_prepare_version_if_prepared(bool &is_prepared, SCN &prep return ret; } -int ObPartTransCtx::get_memtable_key_arr(ObMemtableKeyArray &memtable_key_arr) +int ObTxCtx::get_memtable_key_arr(ObMemtableKeyArray &memtable_key_arr) { int ret = OB_SUCCESS; if (OB_SUCCESS == lock_.try_lock()) { - if (IS_NOT_INIT || is_follower_() || is_exiting_) { + if (IS_NOT_INIT || is_for_replay() || is_exiting_) { TRANS_LOG(DEBUG, "part_ctx not need to get memtable key"); } else if (OB_FAIL(mt_ctx_.get_memtable_key_arr(memtable_key_arr))) { TRANS_LOG(WARN, "get_memtable_key_arr fail", K(ret), K(memtable_key_arr), K(mt_ctx_)); @@ -1227,16 +999,16 @@ int ObPartTransCtx::get_memtable_key_arr(ObMemtableKeyArray &memtable_key_arr) return ret; } -bool ObPartTransCtx::can_be_recycled_() +bool ObTxCtx::can_be_recycled_() { bool bool_ret = true; if (IS_NOT_INIT) { bool_ret = false; } else if (is_exiting_) { bool_ret = false; - } else if (is_follower_()) { + } else if (is_for_replay()) { bool_ret = false; - } else if (ObTxState::REDO_COMPLETE < get_upstream_state()) { + } else if (ObTxState::REDO_COMPLETE < get_target_state()) { bool_ret = false; } else if (is_logging_()) { // FIXME. xiaoshi bool_ret = false; @@ -1248,12 +1020,12 @@ bool ObPartTransCtx::can_be_recycled_() return bool_ret; } -bool ObPartTransCtx::need_to_ask_scheduler_status_() +bool ObTxCtx::need_to_check_tx_status_() { bool bool_ret = false; if (can_be_recycled_()) { - if (ObTimeUtility::current_time() - last_ask_scheduler_status_ts_ - < CHECK_SCHEDULER_STATUS_INTERVAL) { + if (ObTimeUtility::current_time() - last_check_tx_status_ts_ + < CHECK_TX_STATUS_INTERVAL) { bool_ret = false; } else { bool_ret = true; @@ -1262,40 +1034,17 @@ bool ObPartTransCtx::need_to_ask_scheduler_status_() return bool_ret; } -int ObPartTransCtx::check_rs_scheduler_is_alive_(bool &is_alive) -{ - int ret = OB_SUCCESS; - int64_t trace_time = 0; - int64_t cur_time = ObTimeUtility::current_time(); - is_alive = true; - if (OB_ISNULL(trans_service_)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "trans service is NULL", KR(ret), K(*this)); - } else if (need_force_abort_() && !exec_info_.scheduler_.is_valid()) { - is_alive = false; - TRANS_LOG(WARN, "a aborting trans will be gc with invalid scheduler", K(ret), K(is_alive), KPC(this)); - } else if (exec_info_.scheduler_ != addr_) { - is_alive = false; - TRANS_LOG(INFO, "scheduler is not local address, consider it unreachable", - K(exec_info_.scheduler_), K(addr_), KPC(this)); - } else { - is_alive = true; - } - - return ret; -} - -int ObPartTransCtx::gc_ctx_() +int ObTxCtx::gc_ctx_() { int ret = OB_SUCCESS; bool has_redo_log = false; if (OB_FAIL(prepare_mul_data_source_tx_end_(false))) { - TRANS_LOG(WARN, "trans gc need retry", K(ret), K(trans_id_), K(ls_id_)); + TRANS_LOG(WARN, "trans gc need retry", K(ret), K(trans_id_)); } else { TRANS_LOG(INFO, "[TRANS GC] participant will **abort** itself due to scheduler has quit", KPC(this)); REC_TRANS_TRACE_EXT2(tlog_, tx_ctx_gc, OB_ID(ref), get_ref()); - if (need_callback_scheduler_()) { - TRANS_LOG(INFO, "[TRANS GC] scheduler has down, skip callback scheduler", KP(this), + if (need_commit_callback_()) { + TRANS_LOG(INFO, "[TRANS GC] transaction owner has quit, skip commit callback", KP(this), K_(trans_id)); commit_cb_.disable(); } @@ -1306,37 +1055,14 @@ int ObPartTransCtx::gc_ctx_() return ret; } -int ObPartTransCtx::check_scheduler_status() +int ObTxCtx::check_tx_status() { - int ret = OB_SUCCESS; if (OB_SUCCESS == lock_.try_lock()) { CtxLockGuard guard(lock_, false); - // 1. check the status of scheduler on rs - bool is_alive = true; - bool need_check_scheduler = need_to_ask_scheduler_status_(); - if (!need_check_scheduler) { - TRANS_LOG(DEBUG, "don't need ask scheduler status", K(ret), KPC(this)); - } else if (exec_info_.scheduler_ != addr_ && OB_FAIL(check_rs_scheduler_is_alive_(is_alive))) { - TRANS_LOG(WARN, "check rs scheduler is alive error", KR(ret), K(is_alive), KPC(this)); - // scheduler has crashed - } else if (!is_alive) { - TRANS_LOG(WARN, "[TRANS GC] scheduler server is not alive, participant will GC", KPC(this)); - if (OB_FAIL(gc_ctx_())) { - TRANS_LOG(WARN, "force kill part_ctx error", KR(ret), KPC(this)); - } - } else { - // do nothing - } - // 2. Ask for the status of scheduler - ret = OB_SUCCESS; - int ctx_status = OB_SUCCESS; - if (sub_state_.is_force_abort()) { - ctx_status = OB_TRANS_KILLED; - } - - if (is_alive && need_check_scheduler) { - post_keepalive_msg_(ctx_status); - last_ask_scheduler_status_ts_ = ObClockGenerator::getClock(); + if (need_to_check_tx_status_()) { + const int ctx_status = runtime_state_.is_force_abort() ? OB_TRANS_KILLED : OB_SUCCESS; + report_write_ctx_status_(ctx_status, true); + last_check_tx_status_ts_ = ObClockGenerator::getClock(); } if (is_committing_() || ObClockGenerator::getClock() > trans_expired_time_) { @@ -1346,7 +1072,7 @@ int ObPartTransCtx::check_scheduler_status() return OB_SUCCESS; } -int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) +int ObTxCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); @@ -1355,8 +1081,6 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) if (!ctx_info.is_valid()) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument", K(ctx_info)); - // } else if (OB_FAIL(init_tx_data_(ctx_info.ls_id_, ctx_info.tx_id_))) { - // TRANS_LOG(WARN, "unexpected null ptr", K(*this)); } else if (OB_FAIL(mt_ctx_.recover_from_table_lock_durable_info(ctx_info.table_lock_info_))) { TRANS_LOG(ERROR, "recover_from_table_lock_durable_info failed", K(ret)); } else if (OB_FAIL(ctx_tx_data_.recover_tx_data(ctx_info.tx_data_guard_.tx_data()))) { @@ -1365,12 +1089,10 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) TRANS_LOG(WARN, "exec_info assign error", K(ret), K(ctx_info)); } else { trans_id_ = ctx_info.tx_id_; - ls_id_ = ctx_info.ls_id_; - // set upstream state when recover tx ctx table - set_upstream_state(get_downstream_state()); + set_target_state(get_downstream_state()); if (ObTxState::REDO_COMPLETE == get_downstream_state()) { - sub_state_.set_info_log_submitted(); + runtime_state_.set_info_log_submitted(); } if (exec_info_.prepare_version_.is_valid()) { mt_ctx_.set_trans_version(exec_info_.prepare_version_); @@ -1394,14 +1116,6 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) mt_ctx_.set_skip_checksum_calc(); } - if (OB_FAIL(ret)) { - } else { - if (exec_info_.exec_epoch_ > 0) { - epoch_ = exec_info_.exec_epoch_; - } - } - - // insert into retain ctx mgr if it will not replay commit or abort log if (OB_FAIL(ret)) { // do nothing @@ -1416,16 +1130,13 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) TRANS_LOG(WARN, "insert into retain ctx mgr failed", K(ret), KPC(this)); } if (OB_SUCC(ret)) { - TRANS_LOG(INFO, "recover retain ctx into mgr success", K(ret), K(trans_id_), K(ls_id_)); + TRANS_LOG(INFO, "recover retain ctx into mgr success", K(ret), K(trans_id_)); } } if (OB_FAIL(ret)) { // do nothing - } else if ((exec_info_.trans_type_ == TransType::SP_TRANS && ObTxState::COMMIT == exec_info_.state_) - || (exec_info_.trans_type_ == TransType::DIST_TRANS - && ObTxState::CLEAR == exec_info_.state_) - || (ObTxState::ABORT == exec_info_.state_)) { + } else if (ObTxState::COMMIT == exec_info_.state_ || ObTxState::ABORT == exec_info_.state_) { set_exiting_(); TRANS_LOG(INFO, "set exiting with a finished trx in recover", K(ret), KPC(this)); } @@ -1437,7 +1148,6 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) if ((ObTxState::COMMIT == exec_info_.state_ && exec_info_.max_applying_log_ts_ == exec_info_.max_applied_log_ts_) || ObTxState::CLEAR == exec_info_.state_) { - ls_tx_ctx_mgr_->update_max_replay_commit_version(ctx_tx_data_.get_commit_version()); //update max_commit_ts for dup table because of migrate or recover share::g_mp->trans_service() ->get_tx_version_mgr() @@ -1461,7 +1171,7 @@ int ObPartTransCtx::recover_tx_ctx_table_info(ObTxCtxTableInfo &ctx_info) } // Checkpoint the tx ctx table -int ObPartTransCtx::serialize_tx_ctx_to_buffer(ObTxLocalBuffer &buffer, int64_t &serialize_size) +int ObTxCtx::serialize_tx_ctx_to_buffer(ObTxLocalBuffer &buffer, int64_t &serialize_size) { int ret = OB_SUCCESS; ObTxCtxTableInfo ctx_info; @@ -1521,18 +1231,18 @@ int ObPartTransCtx::serialize_tx_ctx_to_buffer(ObTxLocalBuffer &buffer, int64_t serialize_size = pos; } } - TRANS_LOG(INFO, "[TRANS CHECKPOINT] checkpoint trans ctx", K(ret), K_(trans_id), K_(ls_id), KP(this)); + TRANS_LOG(INFO, "[TRANS CHECKPOINT] checkpoint trans ctx", K(ret), K_(trans_id), KP(this)); return ret; } -const SCN ObPartTransCtx::get_rec_log_ts() const +const SCN ObTxCtx::get_rec_log_ts() const { return get_rec_log_ts_(); } -const SCN ObPartTransCtx::get_rec_log_ts_() const +const SCN ObTxCtx::get_rec_log_ts_() const { - share::SCN log_ts = SCN::max_scn();; + share::SCN log_ts = SCN::max_scn(); share::SCN rec_log_ts = rec_log_ts_.atomic_load(); share::SCN prev_rec_log_ts = prev_rec_log_ts_.atomic_load(); @@ -1551,7 +1261,7 @@ const SCN ObPartTransCtx::get_rec_log_ts_() const return log_ts; } -int ObPartTransCtx::on_tx_ctx_table_flushed() +int ObTxCtx::on_tx_ctx_table_flushed() { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); @@ -1561,7 +1271,7 @@ int ObPartTransCtx::on_tx_ctx_table_flushed() return ret; } -int64_t ObPartTransCtx::to_string(char* buf, const int64_t buf_len) const +int64_t ObTxCtx::to_string(char* buf, const int64_t buf_len) const { int64_t len1 = 0; int64_t len2 = 0; @@ -1572,14 +1282,14 @@ int64_t ObPartTransCtx::to_string(char* buf, const int64_t buf_len) const return len1 + len2; } -int ObPartTransCtx::remove_callback_for_uncommited_txn( +int ObTxCtx::remove_callback_for_uncommited_txn( const memtable::ObMemtableSet *memtable_set) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; } else if (OB_ISNULL(memtable_set)) { ret = OB_INVALID_ARGUMENT; @@ -1595,10 +1305,10 @@ int ObPartTransCtx::remove_callback_for_uncommited_txn( // the semantic of submit redo for freeze is // should flush all redos bellow specified freeze_clock (inclusive) // otherwise, need return some error to caller to indicate need retry -int ObPartTransCtx::submit_redo_log_for_freeze(const uint32_t freeze_clock) +int ObTxCtx::submit_redo_log_for_freeze(const uint32_t freeze_clock) { int ret = OB_SUCCESS; - TRANS_LOG(TRACE, "", K_(trans_id), K_(ls_id)); + TRANS_LOG(TRACE, "", K_(trans_id)); ObTimeGuard tg("submit_redo_for_freeze_log", 100000); bool submitted = false; bool need_submit = fast_check_need_submit_redo_for_freeze_(); @@ -1619,11 +1329,10 @@ int ObPartTransCtx::submit_redo_log_for_freeze(const uint32_t freeze_clock) return ret; } -int ObPartTransCtx::submit_redo_after_write(const bool force, const ObTxSEQ &write_seq_no) +int ObTxCtx::submit_redo_after_write(const bool force, const ObTxSEQ &write_seq_no) { int ret = OB_SUCCESS; - TRANS_LOG(TRACE, "submit_redo_after_write", K(force), K(write_seq_no), K_(trans_id), K_(ls_id), - K(mt_ctx_.get_pending_log_size())); + TRANS_LOG(TRACE, "submit_redo_after_write", K(force), K(write_seq_no), K_(trans_id), K(mt_ctx_.get_pending_log_size())); ObTimeGuard tg("submit_redo_for_after_write", 100000); if (force || mt_ctx_.pending_log_size_too_large(write_seq_no)) { bool parallel_logging = false; @@ -1673,7 +1382,7 @@ int ObPartTransCtx::submit_redo_after_write(const bool force, const ObTxSEQ &wri return ret; } -int ObPartTransCtx::serial_submit_redo_after_write_(int &submitted_cnt) +int ObTxCtx::serial_submit_redo_after_write_(int &submitted_cnt) { int ret = OB_SUCCESS; if (OB_SUCC(check_can_submit_redo_())) { @@ -1685,8 +1394,8 @@ int ObPartTransCtx::serial_submit_redo_after_write_(int &submitted_cnt) if (should_switch && submitted_cnt > 0) { const share::SCN serial_final_scn = submitter.get_submitted_scn(); int tmp_ret = switch_to_parallel_logging_(serial_final_scn, exec_info_.max_submitted_seq_no_); - TRANS_LOG(INFO, "**leader switch to parallel logging**", K(tmp_ret), - K_(ls_id), K_(trans_id), + TRANS_LOG(INFO, "switch to parallel logging", K(tmp_ret), + K_(trans_id), K(serial_final_scn), "serial_final_seq_no", exec_info_.serial_final_seq_no_, K(before_submit_pending_size), @@ -1696,7 +1405,7 @@ int ObPartTransCtx::serial_submit_redo_after_write_(int &submitted_cnt) return ret; } -bool ObPartTransCtx::should_switch_to_parallel_logging_() +bool ObTxCtx::should_switch_to_parallel_logging_() { bool ok = false; if (GCONF._enable_parallel_redo_logging) { @@ -1711,12 +1420,12 @@ bool ObPartTransCtx::should_switch_to_parallel_logging_() return ok; } -int ObPartTransCtx::check_can_submit_redo_() +int ObTxCtx::check_can_submit_redo_() { int ret = OB_SUCCESS; bool is_tx_committing = ObTxState::INIT != get_downstream_state(); bool final_log_submitting = - sub_state_.is_state_log_submitting() || sub_state_.is_state_log_submitted(); + runtime_state_.is_state_log_submitting() || runtime_state_.is_state_log_submitted(); if (is_tx_committing ||final_log_submitting || is_force_abort_logging_()) { @@ -1728,7 +1437,7 @@ int ObPartTransCtx::check_can_submit_redo_() // Concurrency safe annotation: // init log_block_ is an local operation // prepare_log_cb_ is protected by `log_cb_lock_` -int ObPartTransCtx::prepare_for_submit_redo(ObTxLogCb *&log_cb, +int ObTxCtx::prepare_for_submit_redo(ObTxLogCb *&log_cb, ObTxLogBlock &log_block, const bool serial_final) { @@ -1736,12 +1445,12 @@ int ObPartTransCtx::prepare_for_submit_redo(ObTxLogCb *&log_cb, if (!log_block.is_inited() && OB_FAIL(init_log_block_(log_block, ObTxAdaptiveLogBuf::NORMAL_LOG_BUF_SIZE, serial_final))) { TRANS_LOG(WARN, "init log block fail", K(ret)); } else if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb)) && OB_TX_NOLOGCB != ret) { - TRANS_LOG(WARN, "prepare log_cb fail", K(ret)); + TRANS_LOG(WARN, "alloc log_cb fail", K(ret)); } return ret; } -int ObPartTransCtx::submit_redo_log_for_freeze_(bool &submitted, const uint32_t freeze_clock) +int ObTxCtx::submit_redo_log_for_freeze_(bool &submitted, const uint32_t freeze_clock) { int ret = OB_SUCCESS; ATOMIC_STORE(&is_submitting_redo_log_for_freeze_, true); @@ -1766,7 +1475,7 @@ int ObPartTransCtx::submit_redo_log_for_freeze_(bool &submitted, const uint32_t return ret; } -bool ObPartTransCtx::fast_check_need_submit_redo_for_freeze_() const +bool ObTxCtx::fast_check_need_submit_redo_for_freeze_() const { bool has_pending_log = true; bool blocked = false; @@ -1778,12 +1487,12 @@ bool ObPartTransCtx::fast_check_need_submit_redo_for_freeze_() const } -int64_t ObPartTransCtx::get_part_trans_action() const +int64_t ObTxCtx::get_part_trans_action() const { return part_trans_action_; } -bool ObPartTransCtx::is_table_lock_killed() const +bool ObTxCtx::is_table_lock_killed() const { bool is_killed = false; is_killed = (mt_ctx_.is_table_lock_killed() || @@ -1791,17 +1500,17 @@ bool ObPartTransCtx::is_table_lock_killed() const return is_killed; } -int ObPartTransCtx::compensate_abort_log_() +int ObTxCtx::compensate_abort_log_() { int ret = OB_SUCCESS; if (is_force_abort_logging_()) { // do nothing - } else if(OB_FALSE_IT(sub_state_.set_force_abort())) { + } else if(OB_FALSE_IT(runtime_state_.set_force_abort())) { } else if (OB_FAIL(submit_log_impl_(ObTxLogType::TX_ABORT_LOG))) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(restart_2pc_trans_timer_())) { - TRANS_LOG(WARN, "restart_2pc_trans_timer_ for submit abort log fail", + if (OB_TMP_FAIL(restart_commit_retry_timer_())) { + TRANS_LOG(WARN, "restart_commit_retry_timer_ for submit abort log fail", KR(ret), KR(tmp_ret), KPC(this)); } TRANS_LOG(WARN, "submit abort log failed", KR(ret), K(*this)); @@ -1811,7 +1520,7 @@ int ObPartTransCtx::compensate_abort_log_() return ret; } -int ObPartTransCtx::abort_(int reason) +int ObTxCtx::abort_(int reason) { int ret = OB_SUCCESS; REC_TRANS_TRACE_EXT2(tlog_, abort_, OB_ID(reason), reason); @@ -1826,7 +1535,7 @@ int ObPartTransCtx::abort_(int reason) return ret; } -int ObPartTransCtx::update_max_commit_version_() +int ObTxCtx::update_max_commit_version_() { int ret = OB_SUCCESS; trans_service_->get_tx_version_mgr().update_max_commit_ts( @@ -1855,7 +1564,7 @@ int ObPartTransCtx::update_max_commit_version_() // // NB: You need pay much attention to the order of the following steps // TODO: Integrate trans_kill and trans_replay_end into the same function -int ObPartTransCtx::tx_end_(const bool commit) +int ObTxCtx::tx_end_(const bool commit) { int ret = OB_SUCCESS; @@ -1914,7 +1623,7 @@ int ObPartTransCtx::tx_end_(const bool commit) ERRSIM_POINT_DEF(EN_TX_ON_SUCCESS_DELAY) #endif -int ObPartTransCtx::on_success(ObTxLogCb *log_cb) +int ObTxCtx::on_success(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -1978,7 +1687,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) skip_on_succ_cnt++; // the TxCtx maybe has been killed forcedly by background GC thread // the log_cb process has been skipped - if (sub_state_.is_force_abort()) { + if (runtime_state_.is_force_abort()) { TRANS_LOG(WARN, "ctx has been aborted forcedly before log sync successfully", KPC(this)); print_trace_log_(); busy_cbs_.remove(log_cb); @@ -2020,7 +1729,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) max_cost_cb_scn = log_cb->get_log_ts(); } if (after_invoke_ts - before_invoke_ts > LOG_CB_ON_SUCC_TIME_LIMIT) { - TRANS_LOG(WARN, "invoke on_succ cost too much time", K(ret), K(trans_id_), K(ls_id_), K(cur_cb), + TRANS_LOG(WARN, "invoke on_succ cost too much time", K(ret), K(trans_id_), K(cur_cb), K(log_cb)); } invoke_on_succ_time += (after_invoke_ts - before_invoke_ts); @@ -2048,7 +1757,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) } submit_record_log_time = ObTimeUtility::fast_current_time() - before_submit_record_ts; } - handle_fast_commit = !(sub_state_.is_state_log_submitted() || log_cb->get_callbacks().count() == 0); + handle_fast_commit = !(runtime_state_.is_state_log_submitted() || log_cb->get_callbacks().count() == 0); try_submit_next_log = !ObTxLogTypeChecker::is_state_log(log_cb->get_last_log_type()) && is_committing_(); retry_submit_mds = ObTxLogTypeChecker::is_mds_log(log_cb->get_last_log_type()); busy_cbs_.remove(log_cb); @@ -2086,15 +1795,14 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) CtxLockGuard guard(lock_, CtxLockGuard::MODE::CTX); if (OB_TMP_FAIL(submit_log_impl_(ObTxLogType::TX_MULTI_DATA_SOURCE_LOG))) { TRANS_LOG(WARN, "submit mds log failed in on_success", K(ret), K(tmp_ret), K(trans_id_), - K(ls_id_), K(exec_info_.state_), K(upstream_state_), K(ctx_tx_data_), K(mds_cache_), + K(exec_info_.state_), K(target_state_), K(ctx_tx_data_), K(mds_cache_), K(exec_info_.multi_data_source_), K(busy_cbs_.get_size()), K(free_cbs_.get_size()), KP(log_cb)); } } if (ObTimeUtility::fast_current_time() - cur_ts > LOG_CB_ON_SUCC_TIME_LIMIT) { - TRANS_LOG(WARN, "on_success cost too much time", K(ret), K(trans_id_), K(ls_id_), - K(max_cost_cb_scn), K(max_cost_cb_time), K(skip_on_succ_cnt), K(invoke_on_succ_cnt), + TRANS_LOG(WARN, "on_success cost too much time", K(ret), K(trans_id_), K(max_cost_cb_scn), K(max_cost_cb_time), K(skip_on_succ_cnt), K(invoke_on_succ_cnt), K(invoke_on_succ_time), K(submit_record_log_time), K(fast_commit_time), K(on_succ_ctx_lock_hold_time), K(try_submit_next_log_cost_time), K(log_sync_used_time), K(ctx_lock_wait_time)); @@ -2107,7 +1815,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_array, +int ObTxCtx::replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_array, const share::SCN op_scn) { int ret = OB_SUCCESS; @@ -2131,7 +1839,7 @@ int ObPartTransCtx::replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_ true))) { TRANS_LOG(WARN, "preapre mds tx_op failed", KR(ret)); } else if (OB_FAIL(tx_data_guard.tx_data()->op_guard_->add_tx_op_batch(trans_id_, - ls_id_, op_scn, tx_op_batch))) { + op_scn, tx_op_batch))) { TRANS_LOG(WARN, "add_tx_op_batch failed", KR(ret)); } // tx_op_batch not put into tx_data, need to release @@ -2153,11 +1861,11 @@ int ObPartTransCtx::replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_ TRANS_LOG(WARN, "insert tx data failed", KR(ret)); } } - TRANS_LOG(INFO, "replay mds to tx_table", KR(ret), K(mds_node_array.count()), K(trans_id_), K(ls_id_), K(op_scn), K(op_exist)); + TRANS_LOG(INFO, "replay mds to tx_table", KR(ret), K(mds_node_array.count()), K(trans_id_), K(op_scn), K(op_exist)); return ret; } -int ObPartTransCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) +int ObTxCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) { int ret = OB_SUCCESS; const ObTxBufferNodeArray &node_array = log_cb.get_mds_range().get_range_array(); @@ -2171,7 +1879,7 @@ int ObPartTransCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) } if (OB_FAIL(ret)) { } else if (all_big_segment) { - TRANS_LOG(INFO, "MDS big_segment not support tx_op just skip", K(trans_id_), K(ls_id_), KP(this)); + TRANS_LOG(INFO, "MDS big_segment not support tx_op just skip", K(trans_id_), KP(this)); // big segment not support tx_op if (OB_NOT_NULL(log_cb.get_tx_op_array()) && log_cb.get_tx_op_array()->count() > 0) { TRANS_LOG(WARN, "MDS big_segment log_cb pre_alloc is not null", KPC(this), K(log_cb)); @@ -2216,7 +1924,7 @@ int ObPartTransCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { TRANS_LOG(WARN, "init tx op failed", KR(ret)); } else if (OB_FAIL(tx_data_guard.tx_data()->op_guard_->add_tx_op_batch(trans_id_, - ls_id_, op_scn, tx_op_array))) { + op_scn, tx_op_array))) { TRANS_LOG(WARN, "add_tx_op_batch failed", KR(ret)); } else { *log_cb.get_tx_data_guard().tx_data() = *tx_data_guard.tx_data(); @@ -2229,11 +1937,11 @@ int ObPartTransCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) } } } - TRANS_LOG(INFO, "insert mds to tx_table", KR(ret), K(trans_id_), K(ls_id_), K(exec_info_.multi_data_source_.count()), K(log_cb)); + TRANS_LOG(INFO, "insert mds to tx_table", KR(ret), K(trans_id_), K(exec_info_.multi_data_source_.count()), K(log_cb)); return ret; } -int ObPartTransCtx::insert_undo_action_to_tx_table_(ObUndoAction &undo_action, +int ObTxCtx::insert_undo_action_to_tx_table_(ObUndoAction &undo_action, ObTxDataGuard &new_tx_data_guard, storage::ObUndoStatusNode *&undo_node, const share::SCN op_scn) @@ -2255,11 +1963,11 @@ int ObPartTransCtx::insert_undo_action_to_tx_table_(ObUndoAction &undo_action, TRANS_LOG(WARN, "insert tx data failed", KR(ret)); } } - TRANS_LOG(INFO, "insert undo_action to tx_table", KR(ret), K(undo_action), K(trans_id_), K(ls_id_), K(op_scn), KP(undo_node)); + TRANS_LOG(INFO, "insert undo_action to tx_table", KR(ret), K(undo_action), K(trans_id_), K(op_scn), KP(undo_node)); return ret; } -int ObPartTransCtx::replay_undo_action_to_tx_table_(ObUndoAction &undo_action, +int ObTxCtx::replay_undo_action_to_tx_table_(ObUndoAction &undo_action, const share::SCN op_scn) { int ret = OB_SUCCESS; @@ -2288,11 +1996,11 @@ int ObPartTransCtx::replay_undo_action_to_tx_table_(ObUndoAction &undo_action, } } TRANS_LOG(INFO, "replay undo_action to tx_table", KR(ret), K(undo_action), K(trans_id_), - K(ls_id_), K(op_scn), KP(tx_data_op), K(tx_data_op_ref)); + K(op_scn), KP(tx_data_op), K(tx_data_op_ref)); return ret; } -int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) +int ObTxCtx::on_success_ops_(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; const SCN log_ts = log_cb->get_log_ts(); @@ -2334,12 +2042,7 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) } } else if (ObTxLogType::TX_BIG_SEGMENT_LOG == log_type) { remove_unsynced_segment_cb_(log_cb->get_log_ts()); - } else if (ObTxLogType::TX_ACTIVE_INFO_LOG == log_type) { - if (log_ts > start_working_log_ts_) { - exec_info_.data_complete_ = true; - } } else if (ObTxLogType::TX_COMMIT_INFO_LOG == log_type) { - ObTwoPhaseCommitLogType two_phase_log_type; set_durable_state_(ObTxState::REDO_COMPLETE); } else if (ObTxLogType::TX_ROLLBACK_TO_LOG == log_type) { if (OB_FAIL(insert_undo_action_to_tx_table_(log_cb->get_undo_action(), @@ -2352,11 +2055,8 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) log_cb->reset_undo_node(); } } else if (ObTxLogTypeChecker::is_state_log(log_type)) { - sub_state_.clear_state_log_submitting(); - if (ObTxLogType::TX_PREPARE_LOG == log_type) { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "2pc is not supported", K(ret), K(log_type), KPC(this)); - } else if (ObTxLogType::TX_COMMIT_LOG == log_type) { + runtime_state_.clear_state_log_submitting(); + if (ObTxLogType::TX_COMMIT_LOG == log_type) { if (OB_SUCC(ret)) { if (exec_info_.multi_data_source_.count() > 0 && get_retain_cause() == RetainCause::UNKOWN && OB_FAIL(insert_into_retain_ctx_mgr_(RetainCause::MDS_WAIT_GC_COMMIT_LOG, log_ts, log_lsn, @@ -2365,20 +2065,11 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) } } if (OB_SUCC(ret)) { - if (is_local_tx_()) { - if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { - TRANS_LOG(WARN, "set end log ts failed", K(ret)); - } else { - if (OB_FAIL(on_local_commit_tx_())) { - TRANS_LOG(WARN, "on local commit failed", KR(ret), K(*this)); - } - } + if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { + TRANS_LOG(WARN, "set end log ts failed", K(ret)); } else { - const NotifyType type = NotifyType::ON_COMMIT; - if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { - TRANS_LOG(WARN, "set end log ts failed", K(ret)); - } else if (OB_FAIL(notify_data_source_(type, log_ts, false, exec_info_.multi_data_source_))) { - TRANS_LOG(WARN, "notify data source failed", KR(ret), K(*this)); + if (OB_FAIL(on_local_commit_tx_())) { + TRANS_LOG(WARN, "on local commit failed", KR(ret), K(*this)); } } } @@ -2391,24 +2082,10 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) } } if (OB_SUCC(ret)) { - if (is_local_tx_() || sub_state_.is_force_abort()) { - if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { - TRANS_LOG(WARN, "set end log ts failed", K(ret)); - } else if (OB_FAIL(on_local_abort_tx_())) { - TRANS_LOG(WARN, "on local abort failed", KR(ret), K(*this)); - } - } else { - const NotifyType type = NotifyType::ON_ABORT; - if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { - TRANS_LOG(WARN, "set end log ts failed", K(ret)); - } else if (OB_FAIL(mds_cache_.generate_final_notify_array(exec_info_.multi_data_source_, - true /*need_merge_cache*/, - true /*allow_log_overflow*/))) { - TRANS_LOG(WARN, "gen total mds array failed", K(ret)); - } else if (OB_FAIL(notify_data_source_(type, log_ts, false, - mds_cache_.get_final_notify_array(), false/*willing_to_commit*/))) { - TRANS_LOG(WARN, "notify data source failed", KR(ret), K(*this)); - } + if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { + TRANS_LOG(WARN, "set end log ts failed", K(ret)); + } else if (OB_FAIL(on_local_abort_tx_())) { + TRANS_LOG(WARN, "on local abort failed", KR(ret), K(*this)); } } } else if (ObTxLogType::TX_CLEAR_LOG == log_type) { @@ -2427,7 +2104,7 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::common_on_success_(ObTxLogCb *log_cb) +int ObTxCtx::common_on_success_(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -2453,35 +2130,32 @@ int ObPartTransCtx::common_on_success_(ObTxLogCb *log_cb) return ret; } -void ObPartTransCtx::check_and_register_timeout_task_() +void ObTxCtx::check_and_register_timeout_task_() { int64_t tmp_ret = OB_SUCCESS; if (!timeout_task_.is_running() && !timeout_task_.is_registered() - && !is_follower_() + && !is_for_replay() && !is_exiting_) { - const int64_t timeout_left = is_committing_() ? trans_2pc_timeout_ - : MIN(MAX_TRANS_2PC_TIMEOUT_US, MAX(trans_expired_time_ - ObClockGenerator::getClock(), 1000 * 1000)); + const int64_t timeout_left = is_committing_() ? commit_retry_timeout_ + : MIN(MAX_TRANS_COMMIT_RETRY_TIMEOUT_US, MAX(trans_expired_time_ - ObClockGenerator::getClock(), 1000 * 1000)); if (OB_SUCCESS != (tmp_ret = register_timeout_task_(timeout_left))) { TRANS_LOG_RET(WARN, tmp_ret, "register timeout task failed", KR(tmp_ret), KPC(this)); } } } -int ObPartTransCtx::try_submit_next_log_(const bool for_freeze) +int ObTxCtx::try_submit_next_log_(const bool for_freeze) { int ret = OB_SUCCESS; - ObTxLogType log_type = ObTxLogType::UNKNOWN; + ObTxLogType log_type = ObTxLogType::TX_COMMIT_LOG; if (ObPartTransAction::COMMIT == part_trans_action_ && !need_force_abort_() - && !sub_state_.is_state_log_submitted() && !sub_state_.is_state_log_submitting() - && !sub_state_.is_gts_waiting() && exec_info_.state_ < ObTxState::PREPARE) { - if (is_follower_()) { - ret = OB_NOT_MASTER; + && !runtime_state_.is_state_log_submitted() && !runtime_state_.is_state_log_submitting() + && exec_info_.state_ < ObTxState::PREPARE) { + if (is_for_replay()) { + ret = OB_STATE_NOT_MATCH; } else { - if (is_local_tx_()) { - log_type = ObTxLogType::TX_COMMIT_LOG; - } if (OB_FAIL(submit_log_impl_(log_type))) { TRANS_LOG(WARN, "submit log for commit failed", K(ret), K(log_type), KPC(this)); } else { @@ -2498,7 +2172,7 @@ int ObPartTransCtx::try_submit_next_log_(const bool for_freeze) return ret; } -int ObPartTransCtx::fix_redo_lsns_(const ObTxLogCb *log_cb) +int ObTxCtx::fix_redo_lsns_(const ObTxLogCb *log_cb) { int ret = OB_SUCCESS; LSN lsn; @@ -2514,11 +2188,11 @@ int ObPartTransCtx::fix_redo_lsns_(const ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) +int ObTxCtx::on_failure(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; share::SCN max_committed_scn; - if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->get_palf_committed_max_scn(max_committed_scn))) { + if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->get_max_decided_scn(max_committed_scn))) { TRANS_LOG(ERROR, "get palf max committed scn fail, need retry", K(ret), KPC(this)); OB_SAFE_ABORT(); } else { @@ -2549,7 +2223,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) ctx_tx_data_.set_start_log_ts(SCN()); } if (ObTxLogTypeChecker::is_state_log(log_type)) { - sub_state_.clear_state_log_submitting(); + runtime_state_.clear_state_log_submitting(); } if (OB_FAIL(fix_redo_lsns_(log_cb))) { TRANS_LOG(ERROR, "fix redo lsns failed", KR(ret), K(*this)); @@ -2571,11 +2245,9 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) log_cb->set_tx_data(nullptr); } } - if (ObTxLogType::TX_PREPARE_LOG == log_type) { - } if (ObTxLogType::TX_COMMIT_LOG == log_type) { // if local tx commit log callback on failure, reset trans_version to make standby read skip this - if (is_local_tx_() && !mt_ctx_.get_trans_version().is_max()) { + if (!mt_ctx_.get_trans_version().is_max()) { mt_ctx_.set_trans_version(SCN::max_scn()); TRANS_LOG(INFO, "clear local trans version when commit log on failure", K(ret), KPC(this)); } @@ -2594,10 +2266,10 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) return_log_cb_(log_cb, true); log_cb = NULL; if (ObTxLogType::TX_COMMIT_INFO_LOG == log_type) { - sub_state_.clear_info_log_submitted(); + runtime_state_.clear_info_log_submitted(); } if (busy_cbs_.is_empty() && get_downstream_state() < ObTxState::PREPARE) { - sub_state_.clear_state_log_submitted(); + runtime_state_.clear_state_log_submitted(); } if (busy_cbs_.is_empty() && !has_persisted_log_()) { // busy callback array is empty and trx has not persisted any log, exit here @@ -2605,9 +2277,9 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) if (OB_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { TRANS_LOG(WARN, "do local tx abort failed", K(ret)); } - if (need_callback_scheduler_()) { + if (need_commit_callback_()) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(defer_callback_scheduler_(OB_TRANS_KILLED, SCN::invalid_scn()))) { + if (OB_TMP_FAIL(defer_commit_callback_(OB_TRANS_KILLED, SCN::invalid_scn()))) { TRANS_LOG(WARN, "notify scheduler txn killed fail", K(tmp_ret), K_(trans_id)); } else { commit_cb_.disable(); @@ -2621,7 +2293,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) OB_ID(log_type), (void*)log_type, OB_ID(t), log_ts, OB_ID(ref), get_ref()); - TRANS_LOG(INFO, "ObPartTransCtx::on_failure end", KR(ret), K(*this), KPC(log_cb)); + TRANS_LOG(INFO, "ObTxCtx::on_failure end", KR(ret), K(*this), KPC(log_cb)); } int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this))) { @@ -2631,7 +2303,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::get_local_max_read_version_(SCN &local_max_read_version) +int ObTxCtx::get_local_max_read_version_(SCN &local_max_read_version) { int ret = OB_SUCCESS; local_max_read_version = trans_service_->get_tx_version_mgr().get_max_read_ts(); @@ -2641,7 +2313,7 @@ int ObPartTransCtx::get_local_max_read_version_(SCN &local_max_read_version) return ret; } -int ObPartTransCtx::get_gts_(SCN >s) +int ObTxCtx::get_gts_(SCN >s) { int ret = OB_SUCCESS; MonotonicTs receive_gts_ts; @@ -2649,19 +2321,8 @@ int ObPartTransCtx::get_gts_(SCN >s) const MonotonicTs stc_ahead = get_stc_() - MonotonicTs(GET_GTS_AHEAD_INTERVAL); ObTsMgr *ts_mgr = trans_service_->get_ts_mgr(); - if (sub_state_.is_gts_waiting()) { - ret = OB_EAGAIN; - TRANS_LOG(INFO, "gts is waiting", K(ret), K(*this)); - } else if (OB_FAIL(ts_mgr->get_gts(stc_ahead, this, gts, receive_gts_ts))) { - if (OB_EAGAIN == ret) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(tmp_ret), K(*this)); - } else { - // REC_TRANS_TRACE_EXT2(tlog_, wait_get_gts, OB_Y(log_type)); - } - sub_state_.set_gts_waiting(); - } else { + if (OB_FAIL(ts_mgr->get_gts(stc_ahead, gts, receive_gts_ts))) { + if (OB_EAGAIN != ret) { TRANS_LOG(WARN, "get gts failed", KR(ret), K(*this), K(stc_ahead)); } } else { @@ -2672,30 +2333,98 @@ int ObPartTransCtx::get_gts_(SCN >s) return ret; } -int ObPartTransCtx::wait_gts_elapse_commit_version_(bool &need_wait) +int ObTxCtx::gts_callback_interrupted() { int ret = OB_SUCCESS; - need_wait = false; + bool need_revert_ctx = false; - ObTsMgr *ts_mgr = trans_service_->get_ts_mgr(); + { + CtxLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(ERROR, "ObTxCtx not inited", KR(ret)); + } else { + runtime_state_.clear_gts_waiting(); + need_revert_ctx = true; + TRANS_LOG(INFO, "transaction gts callback interrupted", KPC(this)); + } + } + + if (need_revert_ctx) { + if (OB_FAIL(ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this))) { + TRANS_LOG(ERROR, "release ctx ref failed", KR(ret)); + } + } + return ret; +} + +int ObTxCtx::gts_elapse_callback(const SCN >s) +{ + int ret = OB_SUCCESS; + bool need_revert_ctx = false; + + { + CtxLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObTxCtx not inited", KR(ret)); + } else if (OB_UNLIKELY(!gts.is_valid())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid gts", KR(ret), K(gts), KPC(this)); + } else if (OB_UNLIKELY(is_exiting_)) { + ret = OB_TRANS_IS_EXITING; + need_revert_ctx = true; + TRANS_LOG(WARN, "transaction is exiting", KR(ret), KPC(this)); + } else if (ctx_tx_data_.get_commit_version() > gts) { + ret = OB_EAGAIN; + } else { + if (OB_UNLIKELY(!runtime_state_.is_gts_waiting())) { + TRANS_LOG(ERROR, "unexpected gts waiting flag", KPC(this)); + } else { + runtime_state_.clear_gts_waiting(); + } + if (OB_FAIL(after_local_commit_succ_())) { + TRANS_LOG(WARN, "terminate trx after local commit failed", KR(ret), KPC(this)); + } + need_revert_ctx = true; + } + REC_TRANS_TRACE_EXT2(tlog_, gts_elapse_callback, + OB_Y(ret), OB_Y(gts), OB_ID(ref), get_ref()); + + if (OB_FAIL(ret) && OB_EAGAIN != ret && runtime_state_.is_gts_waiting()) { + runtime_state_.clear_gts_waiting(); + } + } + + if (need_revert_ctx) { + if (OB_FAIL(ls_tx_ctx_mgr_->revert_tx_ctx_without_lock(this))) { + TRANS_LOG(ERROR, "release ctx ref failed", KR(ret)); + } + } + return ret; +} + +int ObTxCtx::wait_gts_elapse_commit_version_(bool &need_wait) +{ + int ret = OB_SUCCESS; + need_wait = false; - if (OB_FAIL(ts_mgr->wait_gts_elapse(ctx_tx_data_.get_commit_version(), - this, - need_wait))) { - TRANS_LOG(WARN, "wait gts elapse failed", KR(ret), K(*this)); + if (OB_FAIL(trans_service_->get_tx_timestamp_waiter().wait_gts_elapse( + ctx_tx_data_.get_commit_version(), this, need_wait))) { + TRANS_LOG(WARN, "wait gts elapse failed", KR(ret), KPC(this)); } else if (need_wait) { - sub_state_.set_gts_waiting(); + runtime_state_.set_gts_waiting(); if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(WARN, "get trans ctx error", KR(ret), K(*this)); + TRANS_LOG(WARN, "get trans ctx ref failed", KR(ret), KPC(this)); } - TRANS_LOG(INFO, "need wait gts elapse", KR(ret), K(*this)); + TRANS_LOG(INFO, "need wait gts elapse", KR(ret), KPC(this)); REC_TRANS_TRACE_EXT2(tlog_, wait_gts_elapse, OB_ID(ref), get_ref()); } return ret; } -int ObPartTransCtx::generate_prepare_version_() +int ObTxCtx::generate_prepare_version_() { int ret = OB_SUCCESS; @@ -2732,7 +2461,7 @@ int ObPartTransCtx::generate_prepare_version_() if (OB_FAIL(get_local_max_read_version_(local_max_read_version))) { TRANS_LOG(WARN, "get local max read version failed", KR(ret), K(*this)); } else { - // should not overwrite the prepare version of other participants + // should not overwrite the prepare version of another participant context exec_info_.prepare_version_ = SCN::max(SCN::max(gts, local_max_read_version), exec_info_.prepare_version_); if (exec_info_.prepare_version_ > gts) { @@ -2746,7 +2475,7 @@ int ObPartTransCtx::generate_prepare_version_() } // for single-ls transaction, commit version may later be updated by log ts -int ObPartTransCtx::generate_commit_version_() +int ObTxCtx::generate_commit_version_() { int ret = OB_SUCCESS; if (!ctx_tx_data_.get_commit_version().is_valid()) { @@ -2769,14 +2498,14 @@ int ObPartTransCtx::generate_commit_version_() } // starting from 0 -int64_t ObPartTransCtx::get_redo_log_no_() const +int64_t ObTxCtx::get_redo_log_no_() const { return exec_info_.redo_lsns_.count(); } inline -int ObPartTransCtx::submit_redo_if_serial_logging_(ObTxLogBlock &log_block, +int ObTxCtx::submit_redo_if_serial_logging_(ObTxLogBlock &log_block, bool &has_redo, ObRedoLogSubmitHelper &helper) { @@ -2796,7 +2525,7 @@ int ObPartTransCtx::submit_redo_if_serial_logging_(ObTxLogBlock &log_block, // when parallel logging, redo need submitted seperate with other txn's log inline -int ObPartTransCtx::submit_redo_if_parallel_logging_() +int ObTxCtx::submit_redo_if_parallel_logging_() { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_parallel_logging())) { @@ -2810,27 +2539,27 @@ int ObPartTransCtx::submit_redo_if_parallel_logging_() // this function is thread safe, not need other lock's protection inline -int ObPartTransCtx::init_log_block_(ObTxLogBlock &log_block, +int ObTxCtx::init_log_block_(ObTxLogBlock &log_block, const int64_t suggested_buf_size, const bool serial_final) { ObTxLogBlockHeader &header = log_block.get_header(); // the log_entry_no will be backfill before log-block to be submitted - header.init(cluster_id_, cluster_version_, -1 /*log_entry_no*/, trans_id_, exec_info_.scheduler_); + header.init(cluster_id_, cluster_version_, -1 /*log_entry_no*/, trans_id_); if (OB_UNLIKELY(serial_final)) { header.set_serial_final(); } if (OB_UNLIKELY(has_async_index_redo_)) { header.set_has_async_index(); } return log_block.init_for_fill(suggested_buf_size); } -inline int ObPartTransCtx::reuse_log_block_(ObTxLogBlock &log_block) +inline int ObTxCtx::reuse_log_block_(ObTxLogBlock &log_block) { ObTxLogBlockHeader &header = log_block.get_header(); - header.init(cluster_id_, cluster_version_, exec_info_.next_log_entry_no_, trans_id_, exec_info_.scheduler_); + header.init(cluster_id_, cluster_version_, exec_info_.next_log_entry_no_, trans_id_); if (OB_UNLIKELY(has_async_index_redo_)) { header.set_has_async_index(); } return log_block.reuse_for_fill(); } -int ObPartTransCtx::submit_redo_commit_info_log_() +int ObTxCtx::submit_redo_commit_info_log_() { int ret = OB_SUCCESS; ObTxLogBlock log_block; @@ -2843,8 +2572,8 @@ int ObPartTransCtx::submit_redo_commit_info_log_() if (need_force_abort_() || is_force_abort_logging_() || get_downstream_state() == ObTxState::ABORT) { ret = OB_TRANS_KILLED; - TRANS_LOG(ERROR, "tx has been aborting, can not submit prepare log", K(ret)); - } else if (sub_state_.is_info_log_submitted()) { + TRANS_LOG(ERROR, "tx has been aborting, can not submit commit info log", K(ret)); + } else if (runtime_state_.is_info_log_submitted()) { // state log already submitted, do nothing } else if (OB_FAIL(submit_redo_if_parallel_logging_())) { } else if (OB_FAIL(init_log_block_(log_block))) { @@ -2883,12 +2612,11 @@ int ObPartTransCtx::submit_redo_commit_info_log_() return ret; } -int ObPartTransCtx::validate_commit_info_log_(const ObTxCommitInfoLog &commit_info_log) +int ObTxCtx::validate_commit_info_log_(const ObTxCommitInfoLog &commit_info_log) { int ret = OB_SUCCESS; - if (!commit_info_log.get_scheduler().is_valid() - || commit_info_log.get_redo_lsns().count() != exec_info_.redo_lsns_.count()) { + if (commit_info_log.get_redo_lsns().count() != exec_info_.redo_lsns_.count()) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "invalid commit info log", K(ret), K(commit_info_log), KPC(this)); } @@ -2896,7 +2624,7 @@ int ObPartTransCtx::validate_commit_info_log_(const ObTxCommitInfoLog &commit_in return ret; } -int ObPartTransCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, +int ObTxCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, bool &has_redo, ObRedoLogSubmitHelper &helper, logservice::ObReplayBarrierType &barrier) @@ -2906,7 +2634,7 @@ int ObPartTransCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, const int64_t replay_hint = trans_id_.get_id(); barrier = logservice::ObReplayBarrierType::NO_NEED_BARRIER; - if (sub_state_.is_info_log_submitted()) { + if (runtime_state_.is_info_log_submitted()) { // state log already submitted, do nothing } else if (OB_FAIL(submit_redo_if_serial_logging_(log_block, has_redo, helper))) { TRANS_LOG(WARN, "submit redo log failed", KR(ret), K(*this)); @@ -2916,16 +2644,12 @@ int ObPartTransCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, TRANS_LOG(WARN, "decide commit info log barrier failed", K(ret), K(barrier), KPC(this)); } else { ObTxCommitInfoLog commit_info_log( - exec_info_.scheduler_, exec_info_.participants_, exec_info_.upstream_, - exec_info_.is_sub2pc_, exec_info_.is_dup_tx_, can_elr_, trace_info_.get_app_trace_id(), + exec_info_.is_dup_tx_, can_elr_, trace_info_.get_app_trace_id(), trace_info_.get_app_trace_info(), exec_info_.prev_record_lsn_, exec_info_.redo_lsns_, - exec_info_.incremental_participants_, cluster_version_, exec_info_.xid_, exec_info_.commit_parts_, epoch_); + exec_info_.xid_); - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(validate_commit_info_log_(commit_info_log))) { - TRANS_LOG(WARN, "invalid commit info log", K(ret), K(commit_info_log), K(trans_id_), - K(ls_id_)); + if (OB_FAIL(validate_commit_info_log_(commit_info_log))) { + TRANS_LOG(WARN, "invalid commit info log", K(ret), K(commit_info_log), K(trans_id_)); } else if (OB_FAIL(log_block.add_new_log(commit_info_log))) { if (OB_BUF_NOT_ENOUGH == ret) { // TRANS_LOG(WARN, "buf not enough", K(ret), K(commit_info_log), KPC(this)); @@ -2935,283 +2659,66 @@ int ObPartTransCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, } } else if (log_block.get_cb_arg_array().count() == 0) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(log_cb->reserve_callbacks(helper.callbacks_.count()))) { - TRANS_LOG(WARN, "resolve callbacks failed", K(ret), KPC(this)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb))) { - TRANS_LOG(ERROR, "submit log failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - release_ctx_ref_(); - } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { - } else { - log_cb = NULL; - if (OB_FAIL(validate_commit_info_log_(commit_info_log))) { - TRANS_LOG(WARN, "invalid commit info log", K(ret), K(commit_info_log), K(trans_id_), - K(ls_id_)); - } else if (OB_FAIL(log_block.add_new_log(commit_info_log))) { - TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); - //} else if (commit_info_log_barrier_type - // != logservice::ObReplayBarrierType::NO_NEED_BARRIER - // && OB_FAIL(log_block.rewrite_barrier_log_block( - // trans_id_.get_id(), commit_info_log_barrier_type))) { - // TRANS_LOG(WARN, "rewrite commit info log barrier type failed", K(ret), - // K(commit_info_log_barrier_type), KPC(this)); + TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); + return_log_cb_(log_cb); + log_cb = NULL; + } else if (OB_FAIL(log_cb->reserve_callbacks(helper.callbacks_.count()))) { + TRANS_LOG(WARN, "resolve callbacks failed", K(ret), KPC(this)); + return_log_cb_(log_cb); + log_cb = NULL; + } else if (OB_FAIL(acquire_ctx_ref_())) { + TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); + } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb))) { + TRANS_LOG(ERROR, "submit log failed", KR(ret), K(*this)); + return_log_cb_(log_cb); + log_cb = NULL; + release_ctx_ref_(); + } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { + } else { + log_cb = NULL; + if (OB_FAIL(validate_commit_info_log_(commit_info_log))) { + TRANS_LOG(WARN, "invalid commit info log", K(ret), K(commit_info_log), K(trans_id_)); + } else if (OB_FAIL(log_block.add_new_log(commit_info_log))) { + TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); + //} else if (commit_info_log_barrier_type + // != logservice::ObReplayBarrierType::NO_NEED_BARRIER + // && OB_FAIL(log_block.rewrite_barrier_log_block( + // trans_id_.get_id(), commit_info_log_barrier_type))) { + // TRANS_LOG(WARN, "rewrite commit info log barrier type failed", K(ret), + // K(commit_info_log_barrier_type), KPC(this)); + } + has_redo = false; } - has_redo = false; + } else { + TRANS_LOG(WARN, "add new log failed", KR(ret), K(this)); } - } else { - TRANS_LOG(WARN, "add new log failed", KR(ret), K(this)); } - //} else if (commit_info_log_barrier_type != logservice::ObReplayBarrierType::NO_NEED_BARRIER - // && OB_FAIL(log_block.rewrite_barrier_log_block(trans_id_.get_id(), - // commit_info_log_barrier_type))) { - // TRANS_LOG(WARN, "rewrite commit info log barrier type failed", K(ret), - // K(commit_info_log_barrier_type), KPC(this)); + //} else if (commit_info_log_barrier_type != logservice::ObReplayBarrierType::NO_NEED_BARRIER + // && OB_FAIL(log_block.rewrite_barrier_log_block(trans_id_.get_id(), + // commit_info_log_barrier_type))) { + // TRANS_LOG(WARN, "rewrite commit info log barrier type failed", K(ret), + // K(commit_info_log_barrier_type), KPC(this)); } - } return ret; } -int ObPartTransCtx::submit_redo_active_info_log_() +// For liboblog, the log_ts of commit log must be larger than commit_version_ +int ObTxCtx::submit_commit_log_() { int ret = OB_SUCCESS; ObTxLogBlock log_block; + palf::LSN prev_lsn; bool has_redo = false; ObRedoLogSubmitHelper helper; - ObTableLockPrioOpArray prio_op_array; - if (OB_FAIL(submit_redo_if_parallel_logging_())) { - TRANS_LOG(WARN, "submit redo failed", KR(ret)); - } else if (OB_FAIL(init_log_block_(log_block))) { - TRANS_LOG(WARN, "init log block failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_multi_data_source_(log_block))) { - TRANS_LOG(WARN, "submit multi source data failed", K(ret), K(*this)); - } else if (OB_FAIL(submit_redo_if_serial_logging_(log_block, has_redo, helper))) { - TRANS_LOG(WARN, "submit redo log failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_pending_log_block_(log_block, helper, logservice::ObReplayBarrierType::NO_NEED_BARRIER))) { - TRANS_LOG(WARN, "submit pending log failed", KR(ret), K(*this)); - } else if (exec_info_.redo_lsns_.count() > 0 && OB_FAIL(submit_record_log_())) { - TRANS_LOG(WARN, "submit record log failed", KR(ret), K(*this)); - } else if (OB_FAIL(reuse_log_block_(log_block))) { - TRANS_LOG(WARN, "reuse log block failed", KR(ret), K(*this)); - } else { - // get table lock priority info of this trans - (void)mt_ctx_.get_prio_op_array(prio_op_array); - ObTxActiveInfoLog active_info_log(exec_info_.scheduler_, exec_info_.trans_type_, session_id_, - associated_session_id_, - trace_info_.get_app_trace_id(), - mt_ctx_.get_min_table_version(), can_elr_, - addr_, // - cur_query_start_time_, // - false, // sub2pc - exec_info_.is_dup_tx_, - trans_expired_time_, epoch_, last_op_sn_, first_scn_, - last_scn_, exec_info_.max_submitted_seq_no_, - cluster_version_, - exec_info_.xid_, - exec_info_.serial_final_seq_no_, - prio_op_array); - ObTxLogCb *log_cb = nullptr; - if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { - TRANS_LOG(WARN, "get log cb failed", KR(ret), KP(log_cb), K(*this)); - } else if (OB_FAIL(log_block.add_new_log(active_info_log))) { - TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = nullptr; - } else if (log_block.get_cb_arg_array().count() == 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb))) { - TRANS_LOG(ERROR, "submit log to clog adapter failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = nullptr; - release_ctx_ref_(); - } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { - } else { - TRANS_LOG(INFO, "submit active info success", K(*log_cb)); - log_cb = nullptr; - } - } - - return ret; -} - -int ObPartTransCtx::submit_prepare_log_() -{ - int ret = OB_SUCCESS; - ObTxLogBlock log_block; - palf::LSN prev_lsn; - bool has_redo = false; - bool redo_log_submitted = false; - ObRedoLogSubmitHelper helper; const int64_t replay_hint = trans_id_.get_id(); - using ReplayBarrier = logservice::ObReplayBarrierType; - ReplayBarrier commit_info_log_barrier = ReplayBarrier::NO_NEED_BARRIER; - if (need_force_abort_() || is_force_abort_logging_() - || get_downstream_state() == ObTxState::ABORT) { - ret = OB_TRANS_KILLED; - TRANS_LOG(ERROR, "tx has been aborting, can not submit prepare log", K(ret)); - } - - bool contain_commit_info = false; - if (OB_SUCC(ret)) { - if (!sub_state_.is_info_log_submitted()) { - prev_lsn.reset(); - if (OB_FAIL(submit_redo_if_parallel_logging_())) { - } else if (OB_FAIL(init_log_block_(log_block))) { - TRANS_LOG(WARN, "init log block failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_multi_data_source_(log_block))) { - TRANS_LOG(WARN, "submit multi source data failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_redo_commit_info_log_(log_block, has_redo, helper, - commit_info_log_barrier))) { - TRANS_LOG(WARN, "submit redo commit state log failed", KR(ret), K(*this)); - } else { - // do nothing - contain_commit_info = true; - } - // init log_block for prepare log - } else if (OB_FAIL(init_log_block_(log_block))) { - TRANS_LOG(WARN, "init log block failed", K(ret)); - } - } - - const ReplayBarrier compound_log_barrier = - contain_commit_info ? commit_info_log_barrier : ReplayBarrier::NO_NEED_BARRIER; - - if (OB_SUCC(ret)) { - if (OB_FAIL(errism_submit_prepare_log_())) { - TRANS_LOG(WARN, "errsim for submit prepare log", K(ret), KPC(this)); - } - } - if (OB_SUCC(ret)) { - ObTxLogCb *log_cb = NULL; - ObTxPrevLogType prev_log_type(ObTxPrevLogType::TypeEnum::COMMIT_INFO); - - if (OB_FAIL(get_prev_log_lsn_(log_block, prev_log_type, prev_lsn))) { - TRANS_LOG(WARN, "get prev log lsn failed", K(ret), K(*this)); - } - - ObTxPrepareLog prepare_log(exec_info_.incremental_participants_, prev_lsn, prev_log_type); - - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { - if (OB_UNLIKELY(OB_TX_NOLOGCB != ret)) { - TRANS_LOG(WARN, "get log cb failed", KR(ret), K(*this)); - } - } else if (OB_FAIL(log_block.add_new_log(prepare_log))) { - if (OB_BUF_NOT_ENOUGH == ret) { - // TRANS_LOG(WARN, "buf not enough", K(ret), K(prepare_log)); - if (log_block.get_cb_arg_array().count() == 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(log_cb->reserve_callbacks(helper.callbacks_.count()))) { - TRANS_LOG(WARN, "resolve callbacks failed", K(ret), KPC(this)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb, replay_hint, - compound_log_barrier))) { - TRANS_LOG(ERROR, "submit log to clog adapter failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - release_ctx_ref_(); - } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { - } else { - redo_log_submitted = true; - prepare_log.set_prev_lsn(log_cb->get_lsn()); - // TRANS_LOG(INFO, "submit redo and commit_info log success", K(*log_cb)); - log_cb = NULL; - if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { - if (OB_UNLIKELY(OB_TX_NOLOGCB != ret)) { - TRANS_LOG(WARN, "get log cb failed", KR(ret), K(*this)); - } - } else if (OB_FAIL(log_block.add_new_log(prepare_log))) { - TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (log_block.get_cb_arg_array().count() == 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, exec_info_.prepare_version_, log_cb))) { - TRANS_LOG(ERROR, "submit log to clog adapter failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - release_ctx_ref_(); - } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { - } else { - // TRANS_LOG(INFO, "submit prepare log in clog adapter success", K(*log_cb)); - log_cb = NULL; - } - } - } else { - TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - } - } else if (log_block.get_cb_arg_array().count() == 0) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(log_cb->reserve_callbacks(helper.callbacks_.count()))) { - TRANS_LOG(WARN, "resolve callbacks failed", K(ret), KPC(this)); - return_log_cb_(log_cb); - log_cb = NULL; - } else if (OB_FAIL(acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, exec_info_.prepare_version_, log_cb, replay_hint, compound_log_barrier))) { - TRANS_LOG(ERROR, "submit log to clog adapter failed", KR(ret), K(*this)); - return_log_cb_(log_cb); - log_cb = NULL; - release_ctx_ref_(); - } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { - } else { - redo_log_submitted = true; - // TRANS_LOG(INFO, "submit redo,commit_info and prepare log in clog adapter success", K(*log_cb)); - log_cb = NULL; - } - } - - return ret; -} - -// For liboblog, the log_ts of commit log must be larger than commit_version_ -int ObPartTransCtx::submit_commit_log_() -{ - int ret = OB_SUCCESS; - ObTxLogBlock log_block; - palf::LSN prev_lsn; - bool has_redo = false; - ObRedoLogSubmitHelper helper; - const int64_t replay_hint = trans_id_.get_id(); - - const bool local_tx = is_local_tx_(); using LogBarrierType = logservice::ObReplayBarrierType; LogBarrierType commit_info_log_barrier = LogBarrierType::NO_NEED_BARRIER; if (need_force_abort_() || is_force_abort_logging_() || get_downstream_state() == ObTxState::ABORT) { ret = OB_TRANS_KILLED; - TRANS_LOG(ERROR, "tx has been aborting, can not submit prepare log", K(ret)); + TRANS_LOG(ERROR, "tx has been aborting, can not submit commit log", K(ret)); } else if (OB_FAIL(mds_cache_.reserve_final_notify_array(exec_info_.multi_data_source_))) { TRANS_LOG(WARN, "reserve mds cache memory failed", KR(ret), K(*this)); } else if (OB_FAIL(mds_cache_.generate_final_notify_array(exec_info_.multi_data_source_, @@ -3221,29 +2728,26 @@ int ObPartTransCtx::submit_commit_log_() } else { bool log_block_inited = false; int64_t suggested_buf_size = ObTxAdaptiveLogBuf::NORMAL_LOG_BUF_SIZE; - if (local_tx && - mds_cache_.get_final_notify_array().count() == 0 && + if (mds_cache_.get_final_notify_array().count() == 0 && // 512B ((mt_ctx_.get_pending_log_size() < ObTxAdaptiveLogBuf::MIN_LOG_BUF_SIZE / 4) || // for corner case test IS_CORNER(10000))) { suggested_buf_size = ObTxAdaptiveLogBuf::MIN_LOG_BUF_SIZE; } - if (local_tx) { - if (!sub_state_.is_info_log_submitted()) { - prev_lsn.reset(); - if (OB_FAIL(submit_redo_if_parallel_logging_())) { - } else if (OB_FAIL(init_log_block_(log_block, suggested_buf_size))) { - TRANS_LOG(WARN, "init log block failed", KR(ret), K(*this)); - } else if (FALSE_IT(log_block_inited = true)) { - } else if (OB_FAIL(submit_multi_data_source_(log_block))) { - TRANS_LOG(WARN, "submit multi source data failed", KR(ret), K(*this)); - } else if (OB_SUCC(submit_redo_commit_info_log_(log_block, has_redo, helper, - commit_info_log_barrier))) { - // do nothing - } else { - TRANS_LOG(WARN, "submit redo commit state log failed", KR(ret), K(*this)); - } + if (!runtime_state_.is_info_log_submitted()) { + prev_lsn.reset(); + if (OB_FAIL(submit_redo_if_parallel_logging_())) { + } else if (OB_FAIL(init_log_block_(log_block, suggested_buf_size))) { + TRANS_LOG(WARN, "init log block failed", KR(ret), K(*this)); + } else if (FALSE_IT(log_block_inited = true)) { + } else if (OB_FAIL(submit_multi_data_source_(log_block))) { + TRANS_LOG(WARN, "submit multi source data failed", KR(ret), K(*this)); + } else if (OB_SUCC(submit_redo_commit_info_log_(log_block, has_redo, helper, + commit_info_log_barrier))) { + // do nothing + } else { + TRANS_LOG(WARN, "submit redo commit state log failed", KR(ret), K(*this)); } } // init log_block for commit log @@ -3252,15 +2756,6 @@ int ObPartTransCtx::submit_commit_log_() } } - if (OB_SUCC(ret)) { - if (is_local_tx_() && - (exec_info_.participants_.count() > 1 || - exec_info_.intermediate_participants_.count() > 0)) { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(WARN, "2pc is not supported", K(ret), KPC(this)); - } - } - if (OB_SUCC(ret)) { SCN log_commit_version; ObSEArray checksum_arr; @@ -3269,12 +2764,6 @@ int ObPartTransCtx::submit_commit_log_() && replay_completeness_.is_complete() && OB_FAIL(mt_ctx_.calc_checksum_all(checksum_arr))) { TRANS_LOG(ERROR, "calc checksum failed", K(ret)); - } else if (!local_tx) { - log_commit_version = ctx_tx_data_.get_commit_version(); - prev_log_type.set_prepare(); - if (OB_FAIL(get_prev_log_lsn_(log_block, prev_log_type, prev_lsn))) { - TRANS_LOG(WARN, "get prev log lsn failed", K(ret), K(*this)); - } } else { prev_log_type.set_commit_info(); if (OB_FAIL(get_prev_log_lsn_(log_block, prev_log_type, prev_lsn))) { @@ -3288,9 +2777,8 @@ int ObPartTransCtx::submit_commit_log_() ObTxCommitLog commit_log(log_commit_version, collapsed_checksum, checksum_sig, - exec_info_.incremental_participants_, - mds_cache_.get_final_notify_array(), exec_info_.trans_type_, prev_lsn, - coord_prepare_info_arr_, prev_log_type); + mds_cache_.get_final_notify_array(), prev_lsn, + prev_log_type); ObTxLogCb *log_cb = NULL; bool redo_log_submitted = false; LogBarrierType commit_log_barrier_type = LogBarrierType::NO_NEED_BARRIER; @@ -3313,9 +2801,6 @@ int ObPartTransCtx::submit_commit_log_() if (OB_FAIL(ret)) { //do nothing - } else if (is_local_tx_() && exec_info_.upstream_.is_valid()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected trans type with valid upstream", K(ret), KPC(this)); } else if (OB_FAIL(log_block.add_new_log(commit_log))) { if (OB_BUF_NOT_ENOUGH == ret) { TRANS_LOG(WARN, "buf not enough", K(ret), K(commit_log)); @@ -3380,8 +2865,8 @@ int ObPartTransCtx::submit_commit_log_() log_cb = NULL; release_ctx_ref_(); } else { - // sp trans update it's commit version - if (OB_SUCC(ret) && is_local_tx_()) { + // The transaction updates its commit version from the commit log. + if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = ctx_tx_data_.set_commit_version(log_cb->get_log_ts()))) { @@ -3423,8 +2908,8 @@ int ObPartTransCtx::submit_commit_log_() return_log_cb_(log_cb); log_cb = NULL; } else { - // sp trans update it's commit version - if (OB_SUCC(ret) && is_local_tx_()) { + // The transaction updates its commit version from the commit log. + if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = ctx_tx_data_.set_commit_version(log_cb->get_log_ts()))) { TRANS_LOG(WARN, "set commit version failed", K(tmp_ret)); @@ -3441,10 +2926,10 @@ int ObPartTransCtx::submit_commit_log_() return ret; } -int ObPartTransCtx::submit_abort_log_() +int ObTxCtx::submit_abort_log_() { int ret = OB_SUCCESS; - set_upstream_state(ObTxState::ABORT); + set_target_state(ObTxState::ABORT); ObTxLogCb *log_cb = NULL; ObTxLogBlock log_block; const int64_t replay_hint = trans_id_.get_id(); @@ -3523,14 +3008,16 @@ int ObPartTransCtx::submit_abort_log_() return ret; } -int ObPartTransCtx::submit_clear_log_() +int ObTxCtx::submit_clear_log_() { int ret = OB_SUCCESS; ObTxLogBlock log_block; - ObTxClearLog clear_log(exec_info_.incremental_participants_); + ObTxClearLog clear_log; const int64_t replay_hint = trans_id_.get_id(); ObTxLogCb *log_cb = NULL; - if (OB_FAIL(init_log_block_(log_block))) { + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(init_log_block_(log_block))) { TRANS_LOG(WARN, "init log block failed", KR(ret), K(*this)); } else if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { if (OB_UNLIKELY(OB_TX_NOLOGCB != ret)) { @@ -3547,9 +3034,7 @@ int ObPartTransCtx::submit_clear_log_() log_cb = NULL; } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); - } else if (OB_FAIL(submit_log_block_out_(log_block, - share::SCN::max(ctx_tx_data_.get_end_log_ts(), max_2pc_commit_scn_), - log_cb))) { + } else if (OB_FAIL(submit_log_block_out_(log_block, ctx_tx_data_.get_end_log_ts(), log_cb))) { TRANS_LOG(ERROR, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -3563,7 +3048,7 @@ int ObPartTransCtx::submit_clear_log_() return ret; } -int ObPartTransCtx::submit_record_log_() +int ObTxCtx::submit_record_log_() { int ret = OB_SUCCESS; ObTxLogBlock log_block; @@ -3604,7 +3089,7 @@ int ObPartTransCtx::submit_record_log_() } template -int ObPartTransCtx::submit_direct_load_inc_log_( +int ObTxCtx::submit_direct_load_inc_log_( DLI_LOG &dli_log, const ObTxDirectLoadIncLog::DirectLoadIncLogType dli_log_type, const ObDDLIncLogBasic &batch_key, @@ -3650,18 +3135,18 @@ int ObPartTransCtx::submit_direct_load_inc_log_( } TRANS_LOG(DEBUG, " submit direct load inc log", K(ret), K(get_trans_id()), - K(get_ls_id()), K(dli_log), K(dli_log_type), K(batch_key), K(replay_barrier), + K(dli_log), K(dli_log_type), K(batch_key), K(replay_barrier), K(replay_hint), K(scn)); return ret; } -int ObPartTransCtx::submit_direct_load_inc_redo_log(storage::ObDDLRedoLog &ddl_redo_log, +int ObTxCtx::submit_direct_load_inc_redo_log(storage::ObDDLRedoLog &ddl_redo_log, logservice::AppendCb *extra_cb, const int64_t replay_hint, share::SCN &scn) { int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("ObPartTransCtx::submit_direct_load_inc_redo_log", + common::ObTimeGuard timeguard("ObTxCtx::submit_direct_load_inc_redo_log", 1 * 1000 * 1000); // 1s ObDDLIncLogBasic inc_log_basic; if (OB_FAIL(check_status_())) { @@ -3675,12 +3160,12 @@ int ObPartTransCtx::submit_direct_load_inc_redo_log(storage::ObDDLRedoLog &ddl_r return ret; } -int ObPartTransCtx::submit_direct_load_inc_start_log(storage::ObDDLIncStartLog &ddl_start_log, +int ObTxCtx::submit_direct_load_inc_start_log(storage::ObDDLIncStartLog &ddl_start_log, logservice::AppendCb *extra_cb, share::SCN &scn) { int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("ObPartTransCtx::submit_direct_load_inc_start_log", + common::ObTimeGuard timeguard("ObTxCtx::submit_direct_load_inc_start_log", 1 * 1000 * 1000); // 1s ObDDLIncLogBasic inc_log_basic = ddl_start_log.get_log_basic(); if (OB_FAIL(check_status_())) { @@ -3694,13 +3179,13 @@ int ObPartTransCtx::submit_direct_load_inc_start_log(storage::ObDDLIncStartLog & return ret; } -int ObPartTransCtx::submit_direct_load_inc_commit_log(storage::ObDDLIncCommitLog &ddl_commit_log, +int ObTxCtx::submit_direct_load_inc_commit_log(storage::ObDDLIncCommitLog &ddl_commit_log, logservice::AppendCb *extra_cb, share::SCN &scn, bool need_free_extra_cb) { int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("ObPartTransCtx::submit_direct_load_inc_commit_log", + common::ObTimeGuard timeguard("ObTxCtx::submit_direct_load_inc_commit_log", 1 * 1000 * 1000); // 1s ObDDLIncLogBasic inc_log_basic = ddl_commit_log.get_log_basic(); if (OB_FAIL(check_status_())) { @@ -3715,7 +3200,7 @@ int ObPartTransCtx::submit_direct_load_inc_commit_log(storage::ObDDLIncCommitLog return ret; } -int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) +int ObTxCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -3731,7 +3216,7 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) iter != exec_info_.dli_batch_set_.end(); iter++) { if (iter->first.need_compensate_ddl_end()) { TRANS_LOG(DEBUG, " need compensate ddl end log", K(ret), K(tmp_ret), - K(submit_log_type), K(trans_id_), K(ls_id_), K(iter->first)); + K(submit_log_type), K(trans_id_), K(iter->first)); if (compensate_ddl_end_fail_cnt <= MAX_COMPENSATE_FAIL_CNT && (ObTxLogType::TX_ABORT_LOG == submit_log_type || ObTxLogType::TX_COMMIT_INFO_LOG == submit_log_type)) { @@ -3742,7 +3227,7 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) share::SCN submitted_scn; if (OB_ISNULL(extra_cb)) { ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc memory failed", K(ret), K(ls_id_), K(trans_id_), + TRANS_LOG(WARN, "alloc memory failed", K(ret), K(trans_id_), K(submit_log_type)); } else { new (extra_cb) ObDDLIncCommitClogCb(); @@ -3750,14 +3235,13 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) if (OB_TMP_FAIL(ret)) { // do nothing - } else if (OB_TMP_FAIL(extra_cb->init(ls_id_, iter->first.get_batch_key()))) { - TRANS_LOG(WARN, "init extra cb failed", K(ret), K(iter->first), K(trans_id_), - K(ls_id_)); + } else if (OB_TMP_FAIL(extra_cb->init(iter->first.get_batch_key()))) { + TRANS_LOG(WARN, "init extra cb failed", K(ret), K(iter->first), K(trans_id_)); extra_cb->~ObDDLIncCommitClogCb(); mtl_free(extra_cb); } else if (OB_TMP_FAIL(inc_commit_log.init(iter->first.get_batch_key()))) { TRANS_LOG(WARN, "init inc commit log failed", K(ret), K(inc_commit_log), K(iter->first), - K(trans_id_), K(ls_id_)); + K(trans_id_)); extra_cb->~ObDDLIncCommitClogCb(); mtl_free(extra_cb); } else if (OB_TMP_FAIL( @@ -3771,7 +3255,7 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) true))) { if (tmp_ret != OB_TX_NOLOGCB) { TRANS_LOG(WARN, "submit direct load inc commit log failed", K(ret), K(inc_commit_log), - KPC(extra_cb), K(submitted_scn), K(trans_id_), K(ls_id_)); + KPC(extra_cb), K(submitted_scn), K(trans_id_)); } extra_cb->~ObDDLIncCommitClogCb(); mtl_free(extra_cb); @@ -3788,23 +3272,23 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) } else if (iter->first.is_ddl_start_logging()) { need_ddl_end_count += 1; TRANS_LOG(INFO, "the DDL inc start is logging", K(ret), K(tmp_ret), K(iter->first), - K(trans_id_), K(ls_id_)); + K(trans_id_)); } else if (iter->first.is_ddl_end_logging()) { TRANS_LOG(INFO, "the DDL inc end is logging", K(ret), K(tmp_ret), K(iter->first), - K(trans_id_), K(ls_id_)); + K(trans_id_)); } else if (OB_TMP_FAIL(unused_batch_key_array.push_back(iter->first.get_batch_key()))) { TRANS_LOG(WARN, "push back unused batch key failed", K(ret), K(tmp_ret), K(submit_log_type), - K(trans_id_), K(ls_id_), K(iter->first)); + K(trans_id_), K(iter->first)); } else { TRANS_LOG(INFO, " Try to remove unused batch info", K(ret), - K(tmp_ret), K(submit_log_type), K(trans_id_), K(ls_id_), K(iter->first), + K(tmp_ret), K(submit_log_type), K(trans_id_), K(iter->first), K(unused_batch_key_array.count())); } } if (OB_TMP_FAIL(exec_info_.dli_batch_set_.remove_unlog_batch_info(unused_batch_key_array))) { TRANS_LOG(WARN, "remove unlog batch_info failed", K(ret), K(tmp_ret), K(submit_log_type), - K(trans_id_), K(ls_id_), K(unused_batch_key_array)); + K(trans_id_), K(unused_batch_key_array)); } if (OB_SUCC(ret) && need_ddl_end_count > 0) { @@ -3815,22 +3299,20 @@ int ObPartTransCtx::check_dli_batch_completed_(ObTxLogType submit_log_type) ret = OB_TRANS_NEED_ROLLBACK; TRANS_LOG(ERROR, " incompleted direct load inc batch info with state log", - K(ret), K(tmp_ret), K(submit_log_type), K(trans_id_), K(ls_id_), - K(need_ddl_end_count), K(ObTimeUtility::fast_current_time() - start_ts)); + K(ret), K(tmp_ret), K(submit_log_type), K(trans_id_), K(need_ddl_end_count), K(ObTimeUtility::fast_current_time() - start_ts)); } else { ret = OB_TRANS_CANNOT_BE_KILLED; TRANS_LOG(WARN, " The trx can not be finished because of a incompleted " "direct load inc batch info", - K(ret), K(tmp_ret), K(submit_log_type), K(trans_id_), K(ls_id_), - K(need_ddl_end_count), K(ObTimeUtility::fast_current_time() - start_ts)); + K(ret), K(tmp_ret), K(submit_log_type), K(trans_id_), K(need_ddl_end_count), K(ObTimeUtility::fast_current_time() - start_ts)); } } } return ret; } -int ObPartTransCtx::submit_big_segment_log_() +int ObTxCtx::submit_big_segment_log_() { int ret = OB_SUCCESS; @@ -3890,7 +3372,7 @@ int ObPartTransCtx::submit_big_segment_log_() return ret; } -int ObPartTransCtx::prepare_big_segment_submit_(ObTxLogCb *segment_cb, +int ObTxCtx::prepare_big_segment_submit_(ObTxLogCb *segment_cb, const share::SCN &base_scn, logservice::ObReplayBarrierType barrier_type, const ObTxLogType &segment_log_type) @@ -3929,15 +3411,15 @@ int ObPartTransCtx::prepare_big_segment_submit_(ObTxLogCb *segment_cb, return ret; } -// void ObPartTransCtx::after_segment_part_submit_(ObTxLogCb * submit_log_cb,share::SCN log_scn) +// void ObTxCtx::after_segment_part_submit_(ObTxLogCb * submit_log_cb,share::SCN log_scn) // { // // } -int ObPartTransCtx::add_unsynced_segment_cb_(ObTxLogCb *log_cb) +int ObTxCtx::add_unsynced_segment_cb_(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; ObTxLogCbRecord cb_record(*log_cb); - if (!is_follower_()) { + if (!is_for_replay()) { big_segment_info_.segment_buf_.set_prev_part_id(log_cb->get_log_ts().get_val_for_gts()); } @@ -3947,7 +3429,7 @@ int ObPartTransCtx::add_unsynced_segment_cb_(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::remove_unsynced_segment_cb_(const share::SCN &remove_scn) +int ObTxCtx::remove_unsynced_segment_cb_(const share::SCN &remove_scn) { int ret = OB_SUCCESS; // big_segment_info_.unsynced_segment_part_cbs_.remove(log_cb); @@ -3967,7 +3449,7 @@ int ObPartTransCtx::remove_unsynced_segment_cb_(const share::SCN &remove_scn) return ret; } -share::SCN ObPartTransCtx::get_min_unsyncd_segment_scn_() +share::SCN ObTxCtx::get_min_unsyncd_segment_scn_() { share::SCN min_scn; min_scn.invalid_scn(); @@ -3992,7 +3474,7 @@ share::SCN ObPartTransCtx::get_min_unsyncd_segment_scn_() } inline -int ObPartTransCtx::submit_log_block_out_(ObTxLogBlock &log_block, +int ObTxCtx::submit_log_block_out_(ObTxLogBlock &log_block, const share::SCN &base_scn, ObTxLogCb *&log_cb, const int64_t replay_hint, @@ -4000,9 +3482,7 @@ int ObPartTransCtx::submit_log_block_out_(ObTxLogBlock &log_block, const int64_t retry_timeout_us) { int ret = OB_SUCCESS; - bool is_2pc_state_log = false; - if (is_contain_stat_log(log_block.get_cb_arg_array()) && FALSE_IT(is_2pc_state_log = true)) { - } else if ((!is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_ABORT_LOG) + if ((!is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_ABORT_LOG) && !is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_CLEAR_LOG)) && (is_force_abort_logging_() || get_downstream_state() == ObTxState::ABORT)) { @@ -4033,9 +3513,7 @@ int ObPartTransCtx::submit_log_block_out_(ObTxLogBlock &log_block, return ret; } -ERRSIM_POINT_DEF(ERRSIM_DELAY_TX_SUBMIT_LOG); - -int ObPartTransCtx::submit_log_impl_(const ObTxLogType log_type) +int ObTxCtx::submit_log_impl_(const ObTxLogType log_type) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -4044,45 +3522,19 @@ int ObPartTransCtx::submit_log_impl_(const ObTxLogType log_type) if (big_segment_info_.segment_buf_.is_active()) { ret = OB_LOG_TOO_LARGE; TRANS_LOG(INFO, "can not submit any log before all big log submittted", K(ret), K(log_type), - K(trans_id_), K(ls_id_), K(big_segment_info_)); + K(trans_id_), K(big_segment_info_)); } else { switch (log_type) { - // leader switch - case ObTxLogType::TX_ACTIVE_INFO_LOG: { - ret = submit_redo_active_info_log_(); - break; - } // for xa and duplicate table // TODO. need submit log in sync log callback case ObTxLogType::TX_COMMIT_INFO_LOG: { ret = submit_redo_commit_info_log_(); break; } - case ObTxLogType::TX_PREPARE_LOG: { - ret = OB_NOT_SUPPORTED; - TRANS_LOG(ERROR, "2pc is not supported", K(ret), KPC(this)); - break; - } case ObTxLogType::TX_COMMIT_LOG: { if (!mt_ctx_.is_prepared()) { - if (!is_local_tx_()) { - int tmp_ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "not set prepare verison into mt_ctx, unexpected error", "ret", tmp_ret, K(*this)); - print_trace_log_(); - } else { - // try generate commit version - ret = generate_commit_version_(); - } + ret = generate_commit_version_(); } - if (OB_SWITCHING_TO_FOLLOWER_GRACEFULLY == ERRSIM_DELAY_TX_SUBMIT_LOG && 1001 == ls_id_.id()) { - ret = ERRSIM_DELAY_TX_SUBMIT_LOG; - } else if (ERRSIM_DELAY_TX_SUBMIT_LOG < int(-9999)) { - // reject trans_id and tenant id to contorl trans sate - // ret = process_errsim_for_standby_read_(int(ERRSIM_DELAY_TX_SUBMIT_LOG), - // OB_SWITCHING_TO_FOLLOWER_GRACEFULLY); - TRANS_LOG(INFO, "reject delay tx submit log when submit commit log", - K(ret), K(ERRSIM_DELAY_TX_SUBMIT_LOG)); - } if (OB_SUCC(ret) && mt_ctx_.is_prepared()) { ret = submit_commit_log_(); } @@ -4120,15 +3572,14 @@ int ObPartTransCtx::submit_log_impl_(const ObTxLogType log_type) return ret; } -void ObPartTransCtx::handle_submit_log_err_(const ObTxLogType log_type, int &ret) +void ObTxCtx::handle_submit_log_err_(const ObTxLogType log_type, int &ret) { if (OB_TX_NOLOGCB == ret) { if (REACH_COUNT_PER_SEC(10) && REACH_TIME_INTERVAL(100 * 1000)) { TRANS_LOG(INFO, "can not get log_cb when submit_log", KR(ret), K(log_type), "busy_cbs.first", PC(busy_cbs_.is_empty() ? NULL : busy_cbs_.get_first())); } - if (ObTxLogType::TX_PREPARE_LOG == log_type || ObTxLogType::TX_COMMIT_LOG == log_type - || ObTxLogType::TX_COMMIT_INFO_LOG == log_type) { + if (ObTxLogType::TX_COMMIT_LOG == log_type || ObTxLogType::TX_COMMIT_INFO_LOG == log_type) { // need submit log in log sync callback // rewrite ret ret = OB_SUCCESS; @@ -4141,7 +3592,7 @@ void ObPartTransCtx::handle_submit_log_err_(const ObTxLogType log_type, int &ret } else if (OB_ERR_TOO_BIG_ROWSIZE == ret) { int tmp_ret = OB_SUCCESS; if (ObPartTransAction::COMMIT == part_trans_action_ - || get_upstream_state() >= ObTxState::REDO_COMPLETE) { + || get_target_state() >= ObTxState::REDO_COMPLETE) { if (OB_TMP_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { TRANS_LOG(WARN, "abort tx failed", KR(ret), KPC(this)); } else { @@ -4159,17 +3610,17 @@ void ObPartTransCtx::handle_submit_log_err_(const ObTxLogType log_type, int &ret } } -void ObPartTransCtx::reset_redo_lsns_() +void ObTxCtx::reset_redo_lsns_() { exec_info_.redo_lsns_.reset(); } -void ObPartTransCtx::set_prev_record_lsn_(const LogOffSet &prev_record_lsn) +void ObTxCtx::set_prev_record_lsn_(const LogOffSet &prev_record_lsn) { exec_info_.prev_record_lsn_ = prev_record_lsn; } -bool ObPartTransCtx::need_record_log_() const +bool ObTxCtx::need_record_log_() const { // Record Log will be generated if the number of log ids // is no less than the max size of prev_redo_log_ids_ @@ -4182,13 +3633,13 @@ bool ObPartTransCtx::need_record_log_() const TRANS_LOG(INFO, "need_record_log: ", K(prev_redo_lsns_count)); } #endif - return get_redo_log_no_() >= prev_redo_lsns_count && !sub_state_.is_info_log_submitted(); + return get_redo_log_no_() >= prev_redo_lsns_count && !runtime_state_.is_info_log_submitted(); } #define bitmap_is_contain(X) (bitmap & (uint64_t)X) //big row may use a unused log_block to invoke after_submit_log_ -int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, +int ObTxCtx::after_submit_log_(ObTxLogBlock &log_block, ObTxLogCb *log_cb, ObRedoLogSubmitHelper *helper) { @@ -4243,20 +3694,12 @@ int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, big_segment_info_.reuse(); } } - if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_ACTIVE_INFO_LOG)) { - // do nothing - } if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_COMMIT_INFO_LOG)) { - sub_state_.set_info_log_submitted(); - } - if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_PREPARE_LOG)) { - sub_state_.set_state_log_submitting(); - sub_state_.set_state_log_submitted(); - exec_info_.prepare_version_ = SCN::max(log_cb->get_log_ts(), exec_info_.prepare_version_); + runtime_state_.set_info_log_submitted(); } if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_COMMIT_LOG)) { - sub_state_.set_state_log_submitting(); - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitting(); + runtime_state_.set_state_log_submitted(); // elr const bool has_row_updated = mt_ctx_.has_row_updated(); if (can_elr_ && has_row_updated) { @@ -4267,12 +3710,12 @@ int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, } } if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_ABORT_LOG)) { - sub_state_.set_state_log_submitting(); - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitting(); + runtime_state_.set_state_log_submitted(); } if (OB_SUCC(ret) && bitmap_is_contain(ObTxLogType::TX_CLEAR_LOG)) { - sub_state_.set_state_log_submitting(); - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitting(); + runtime_state_.set_state_log_submitted(); } if (OB_SUCC(ret)) { if (!ctx_tx_data_.get_start_log_ts().is_valid()) { @@ -4282,10 +3725,10 @@ int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, } } if(OB_FAIL(ret)) { - TRANS_LOG(WARN, "after submit log failed", K(ret), K(trans_id_), K(ls_id_), K(exec_info_), K(*log_cb)); + TRANS_LOG(WARN, "after submit log failed", K(ret), K(trans_id_), K(exec_info_), K(*log_cb)); } else { #ifndef NDEBUG - TRANS_LOG(INFO, "after submit log success", K(ret), K(trans_id_), K(ls_id_), K(exec_info_), K(*log_cb), KPC(this)); + TRANS_LOG(INFO, "after submit log success", K(ret), K(trans_id_), K(exec_info_), K(*log_cb), KPC(this)); #endif } REC_TRANS_TRACE_EXT(tlog_, @@ -4306,7 +3749,7 @@ int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, return ret; } -int ObPartTransCtx::get_max_submitting_log_info_(palf::LSN &lsn, SCN &log_ts) +int ObTxCtx::get_max_submitting_log_info_(palf::LSN &lsn, SCN &log_ts) { int ret = OB_SUCCESS; ObTxLogCb *log_cb = NULL; @@ -4332,7 +3775,7 @@ int ObPartTransCtx::get_max_submitting_log_info_(palf::LSN &lsn, SCN &log_ts) return ret; } -int ObPartTransCtx::get_prev_log_lsn_(const ObTxLogBlock &log_block, +int ObTxCtx::get_prev_log_lsn_(const ObTxLogBlock &log_block, ObTxPrevLogType &prev_log_type, palf::LSN &lsn) { @@ -4366,7 +3809,7 @@ int ObPartTransCtx::get_prev_log_lsn_(const ObTxLogBlock &log_block, return ret; } -int ObPartTransCtx::set_start_scn_in_commit_log_(ObTxCommitLog &commit_log) +int ObTxCtx::set_start_scn_in_commit_log_(ObTxCommitLog &commit_log) { int ret = OB_SUCCESS; @@ -4383,14 +3826,14 @@ int ObPartTransCtx::set_start_scn_in_commit_log_(ObTxCommitLog &commit_log) } -int ObPartTransCtx::try_submit_next_log() +int ObTxCtx::try_submit_next_log() { CtxLockGuard guard(lock_); return try_submit_next_log_(true); } //***************************** for 4.0 -int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, +int ObTxCtx::check_replay_avaliable_(const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no, bool &need_replay) @@ -4398,9 +3841,9 @@ int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, int ret = OB_SUCCESS; need_replay = true; if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); + TRANS_LOG(WARN, "ObTxCtx not inited"); ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(!is_follower_())) { + } else if (OB_UNLIKELY(!is_for_replay())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "invalid state, transaction is not replaying", KR(ret), "context", *this); } else if (OB_UNLIKELY(!timestamp.is_valid())) { @@ -4409,7 +3852,7 @@ int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, // } else if (is_exiting_) { // // ret = OB_TRANS_CTX_NOT_EXIST; } else { - // check state like ObPartTransCtx::is_trans_valid_for_replay_ + // check state like ObTxCtx::is_trans_valid_for_replay_ if (!exec_info_.max_applying_log_ts_.is_valid()) { // do nothing } else if (exec_info_.max_applying_log_ts_ > timestamp) { @@ -4431,7 +3874,7 @@ int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, return ret; } -int ObPartTransCtx::push_replaying_log_ts(const SCN log_ts_ns, const int64_t log_entry_no) +int ObTxCtx::push_replaying_log_ts(const SCN log_ts_ns, const int64_t log_entry_no) { int ret = OB_SUCCESS; @@ -4452,14 +3895,14 @@ int ObPartTransCtx::push_replaying_log_ts(const SCN log_ts_ns, const int64_t log if (OB_UNLIKELY(replay_completeness_.is_unknown())) { const bool replay_continous = exec_info_.next_log_entry_no_ == log_entry_no; if (OB_FAIL(set_replay_completeness_(replay_continous, log_ts_ns))) { - TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(ls_id_), K(trans_id_), KP(this)); + TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(trans_id_), KP(this)); } } } return ret; } -int ObPartTransCtx::push_replayed_log_ts(const SCN log_ts_ns, +int ObTxCtx::push_replayed_log_ts(const SCN log_ts_ns, const palf::LSN &offset, const int64_t log_entry_no) { @@ -4501,7 +3944,7 @@ int ObPartTransCtx::push_replayed_log_ts(const SCN log_ts_ns, return ret; } -int ObPartTransCtx::iter_next_log_for_replay(ObTxLogBlock &log_block, +int ObTxCtx::iter_next_log_for_replay(ObTxLogBlock &log_block, ObTxLogHeader &log_header, const share::SCN log_scn) { @@ -4547,7 +3990,7 @@ int ObPartTransCtx::iter_next_log_for_replay(ObTxLogBlock &log_block, return ret; } -int ObPartTransCtx::replay_one_part_of_big_segment(const palf::LSN &offset, +int ObTxCtx::replay_one_part_of_big_segment(const palf::LSN &offset, const share::SCN ×tamp, const int64_t &part_log_no) { @@ -4570,7 +4013,7 @@ int ObPartTransCtx::replay_one_part_of_big_segment(const palf::LSN &offset, return ret; } -int ObPartTransCtx::update_replaying_log_no_(const SCN &log_ts_ns, int64_t part_log_no) +int ObTxCtx::update_replaying_log_no_(const SCN &log_ts_ns, int64_t part_log_no) { int ret = OB_SUCCESS; @@ -4590,22 +4033,7 @@ int ObPartTransCtx::update_replaying_log_no_(const SCN &log_ts_ns, int64_t part_ return ret; } -int ObPartTransCtx::check_trans_type_for_replay_(const int32_t &trans_type, - const SCN &commit_log_ts) -{ - int ret = OB_SUCCESS; - - if (create_ctx_scn_ == commit_log_ts) { - // TRANS_LOG(INFO, "start replay from commit log", K(trans_type), K(commit_log_ts)); - exec_info_.trans_type_ = trans_type; - } else if (exec_info_.trans_type_ != trans_type) { - // ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "trans type not match", K(trans_type), K(commit_log_ts), K(*this)); - } - return ret; -} - -int ObPartTransCtx::replay_update_tx_data_(const bool commit, +int ObTxCtx::replay_update_tx_data_(const bool commit, const SCN &log_ts, const SCN &commit_version) { @@ -4614,19 +4042,13 @@ int ObPartTransCtx::replay_update_tx_data_(const bool commit, if (OB_FAIL(ctx_tx_data_.set_end_log_ts(log_ts))) { TRANS_LOG(WARN, "set end log ts failed", K(ret)); } else if (commit) { - if (TransType::SP_TRANS == exec_info_.trans_type_ - && !commit_version.is_valid()) { + if (!commit_version.is_valid()) { if (OB_FAIL(ctx_tx_data_.set_commit_version(log_ts))) { TRANS_LOG(WARN, "set commit_version failed", K(ret)); } - } else if (TransType::DIST_TRANS == exec_info_.trans_type_ - && commit_version.is_valid()) { - if (OB_FAIL(ctx_tx_data_.set_commit_version(commit_version))) { - TRANS_LOG(WARN, "set commit_version failed", K(ret)); - } } else { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "Unkown trans type or commit_version", K(ret), K(log_ts), K(commit_version)); + TRANS_LOG(ERROR, "unexpected trans type or commit_version", K(ret), K(log_ts), K(commit_version)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(ctx_tx_data_.set_state(ObTxData::COMMIT))) { @@ -4641,7 +4063,7 @@ int ObPartTransCtx::replay_update_tx_data_(const bool commit, return ret; } -int ObPartTransCtx::replace_tx_data_with_backup_(const ObTxDataBackup &backup, SCN log_ts) +int ObTxCtx::replace_tx_data_with_backup_(const ObTxDataBackup &backup, SCN log_ts) { int ret = OB_SUCCESS; share::SCN tmp_log_ts = log_ts; @@ -4663,7 +4085,7 @@ int ObPartTransCtx::replace_tx_data_with_backup_(const ObTxDataBackup &backup, S return ret; } -void ObPartTransCtx::force_no_need_replay_checksum(const bool parallel_replay, +void ObTxCtx::force_no_need_replay_checksum(const bool parallel_replay, const share::SCN &log_ts) { if (ATOMIC_LOAD(&exec_info_.need_checksum_)) { @@ -4672,11 +4094,11 @@ void ObPartTransCtx::force_no_need_replay_checksum(const bool parallel_replay, } } -void ObPartTransCtx::force_no_need_replay_checksum_(const bool parallel_replay, +void ObTxCtx::force_no_need_replay_checksum_(const bool parallel_replay, const share::SCN &log_ts) { if (ATOMIC_LOAD(&exec_info_.need_checksum_)) { - TRANS_LOG(INFO, "set skip calc checksum", K_(trans_id), K_(ls_id), KP(this), K(parallel_replay), K(log_ts)); + TRANS_LOG(INFO, "set skip calc checksum", K_(trans_id), KP(this), K(parallel_replay), K(log_ts)); if (parallel_replay) { update_rec_log_ts_(true/*for_replay*/, log_ts); } @@ -4685,7 +4107,7 @@ void ObPartTransCtx::force_no_need_replay_checksum_(const bool parallel_replay, } } -void ObPartTransCtx::check_no_need_replay_checksum(const SCN &log_ts, const int index) +void ObTxCtx::check_no_need_replay_checksum(const SCN &log_ts, const int index) { // TODO(handora.qc): How to lock the tx_ctx @@ -4737,7 +4159,7 @@ void ObPartTransCtx::check_no_need_replay_checksum(const SCN &log_ts, const int * maintained on leader and when switch to follower, it will persistent * redo_lsns with `RecordLog` */ -int ObPartTransCtx::check_and_merge_redo_lsns_(const palf::LSN &offset) +int ObTxCtx::check_and_merge_redo_lsns_(const palf::LSN &offset) { int ret = OB_SUCCESS; if (!is_support_parallel_replay_()) { @@ -4760,10 +4182,8 @@ int ObPartTransCtx::check_and_merge_redo_lsns_(const palf::LSN &offset) * memtable (and locktable), and only Txn's Log will replay into Tx ctx * and affect the Tx ctx's state * - * for compatible with old version, redo belongs to `tx_log_queue` will - * still handle same as Txn's Log */ -int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, +int ObTxCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no, @@ -4775,18 +4195,11 @@ int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, common::ObTimeGuard timeguard("replay_redo_in_ctx", 10 * 1000); { CtxLockGuard guard(lock_); - // before 4.2.4, cluster_version is in RedoLog, and - // the cluster_version_ is initialized to CURRENT_CLUSTER_VERSION when ctx created - // it should be correct with in redo - // since 4.2.4, cluster_version is in LogBlockHeader, so the cluster_version is correct - // when created, and cluster_version in RedoLog is 0 - if (redo_log.get_cluster_version() > 0) { - ret = correct_cluster_version_(redo_log.get_cluster_version()); - } - - // if this is serial final redo log - // change the logging to parallel logging - if (OB_SUCC(ret) && serial_final) { + if (is_tx_log_queue && OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { + TRANS_LOG(WARN, "update replaying log no for redo failed", K(ret), K(timestamp), + K(part_log_no), K_(trans_id)); + } else if (serial_final) { + // A serial-final redo switches subsequent logging to parallel mode. if (!is_tx_log_queue) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "serial final redo must be in tx_log_queue", K(ret), KPC(this), K(timestamp)); @@ -4800,10 +4213,6 @@ int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, ObTransStatistic::get_instance().add_redo_log_replay_count( 1); ObTransStatistic::get_instance().add_redo_log_replay_time( timeguard.get_diff()); - // for redo compound with Txn Log, go with old ways, this can also handle replay of old version - if (is_tx_log_queue) { - replay_redo_in_ctx_compat_(redo_log, offset, timestamp, part_log_no); - } } #ifndef NDEBUG @@ -4813,41 +4222,6 @@ int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, return ret; } -int ObPartTransCtx::replay_redo_in_ctx_compat_(const ObTxRedoLog &redo_log, - const palf::LSN &offset, - const SCN ×tamp, - const int64_t &part_log_no) -{ - int ret = OB_SUCCESS; - lib::Worker::CompatMode mode = lib::Worker::CompatMode::INVALID; - bool need_replay = true; - - CtxLockGuard guard(lock_); - - if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { - TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); - } else if (!need_replay) { - TRANS_LOG(INFO, "need not replay log", K(redo_log), K(timestamp), K(offset), K(*this)); - // no need to replay - } else if (OB_FAIL((update_replaying_log_no_(timestamp, part_log_no)))) { - TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp)); - } else { - // TODO add clog_encrypt_info_ - UNUSED(set_upstream_state(ObTxState::INIT)); - if (OB_SUCC(ret) && OB_FAIL(check_and_merge_redo_lsns_(offset))) { - TRANS_LOG(WARN, "check and merge redo lsns failed", K(ret), K(trans_id_), K(timestamp), K(offset)); - } - } - REC_TRANS_TRACE_EXT(tlog_, replay_redo, OB_ID(ret), ret, - OB_Y(need_replay), OB_ID(offset), offset.val_, - OB_ID(t), timestamp, OB_ID(ref), get_ref()); - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[Replay Tx] Replay Redo in TxCtx Failed", K(ret), K(timestamp), K(offset), - K(need_replay), K(redo_log), K(*this)); - } - return ret; -} - // // Replay RollbackToLog // the RollbackToLog operate on memtable, its replay seperate two step @@ -4863,7 +4237,7 @@ int ObPartTransCtx::replay_redo_in_ctx_compat_(const ObTxRedoLog &redo_log, // is possible. // // -int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, +int ObTxCtx::replay_rollback_to(const ObTxRollbackToLog &log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no, @@ -4911,12 +4285,12 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, // or the txn must been marked not `need_checksum` ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "replay branch savepoint hit bug", K(ret), KP(this), - K(from), K(to), K(pre_barrier), K_(trans_id), K_(ls_id), K_(replay_completeness), K_(exec_info)); + K(from), K(to), K(pre_barrier), K_(trans_id), K_(replay_completeness), K_(exec_info)); } else if (replay_completeness_.is_complete()) { ret = OB_EAGAIN; if (TC_REACH_TIME_INTERVAL(1_s)) { TRANS_LOG(INFO, "branch savepoint should wait replay serial final because of calc checksum", - K(ret), K(from), K(to), K(timestamp), KP(this), K_(trans_id), K_(ls_id), K_(exec_info)); + K(ret), K(from), K(to), K(timestamp), KP(this), K_(trans_id), K_(exec_info)); } } else if (replay_completeness_.is_unknown()) { // try to fetch the replay position of replay-queue of txn-log @@ -4928,14 +4302,14 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, if (OB_ISNULL(log_service)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "tenant logservice is null", K(ret), K(timestamp), K_(trans_id)); - } else if (OB_FAIL(log_service->get_log_replay_service()->get_min_unreplayed_scn(ls_id_, min_unreplayed_scn))) { + } else if (OB_FAIL(log_service->get_log_replay_service()->get_min_unreplayed_scn(min_unreplayed_scn))) { TRANS_LOG(WARN, "get min unreplayed scn fail", K(ret), K(timestamp), K_(trans_id)); } else if (min_unreplayed_scn == timestamp) { // all previous log replayed // the txn must not replay from its first log, aka. incomplete-replay - TRANS_LOG(INFO, "detect txn replayed from middle", K(ret), K(timestamp), K_(trans_id), K_(ls_id), K_(exec_info)); + TRANS_LOG(INFO, "detect txn replayed from middle", K(ret), K(timestamp), K_(trans_id), K_(exec_info)); if (OB_FAIL(set_replay_completeness_(false, timestamp))) { - TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(ls_id_), K(trans_id_), KP(this)); + TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(trans_id_), KP(this)); } } else if (min_unreplayed_scn > timestamp) { ret = OB_ERR_UNEXPECTED; @@ -4944,7 +4318,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, ret = OB_EAGAIN; if (TC_REACH_TIME_INTERVAL(1_s)) { TRANS_LOG(INFO, "branch savepoint should wait replay serial final because of calc checksum", - K(ret), K(from), K(to), K(timestamp), K(min_unreplayed_scn), KP(this), K_(trans_id), K_(ls_id), K_(exec_info)); + K(ret), K(from), K(to), K(timestamp), K(min_unreplayed_scn), KP(this), K_(trans_id), K_(exec_info)); } } } else { @@ -4969,11 +4343,6 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, KPC(this)); } - // this is compatible code, since 4.2.4, redo_lsn not collect during replay - if (OB_SUCC(ret) && OB_FAIL(check_and_merge_redo_lsns_(offset))) { - TRANS_LOG(WARN, "check and merge redo lsns failed", K(ret), K(trans_id_), K(timestamp), K(offset)); - } - // // Step2, remove TxNode(s) // @@ -5012,95 +4381,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, return ret; } -int ObPartTransCtx::replay_active_info(const ObTxActiveInfoLog &log, - const palf::LSN &offset, - const SCN ×tamp, - const int64_t &part_log_no) -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("replay_active_info", 10 * 1000); - bool need_replay = true; - // int64_t start = ObTimeUtility::fast_current_time(); - CtxLockGuard guard(lock_); - - if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { - TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); - } else if (!need_replay) { - TRANS_LOG(INFO, "need not replay log", K(log), K(timestamp), K(offset), K(*this)); - // no need to replay - } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { - TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); - } else if (OB_FAIL(set_scheduler_(log.get_scheduler()))) { - TRANS_LOG(WARN, "set scheduler error", K(ret), K(log), K(*this)); - } else if (OB_FAIL(set_app_trace_id_(log.get_app_trace_id()))) { - TRANS_LOG(WARN, "set app trace id error", K(ret), K(log), K(*this)); - } else if (log.get_cluster_version() > 0 && - OB_FAIL(correct_cluster_version_(log.get_cluster_version()))) { - } else { - exec_info_.trans_type_ = log.get_trans_type(); - trans_expired_time_ = log.get_tx_expired_time(); - session_id_ = log.get_session_id(); - // schema_version - can_elr_ = log.is_elr(); - // cur_query_start_time - exec_info_.is_sub2pc_ = log.is_sub2pc(); - exec_info_.xid_ = log.get_xid(); - epoch_ = log.get_epoch(); - last_op_sn_ = log.get_last_op_sn(); - first_scn_ = log.get_first_seq_no(); - last_scn_ = log.get_last_seq_no(); - // for table lock priority array - // ignore the result - const tablelock::ObTableLockPrioOpArray &prio_op_array = log.get_prio_op_array(); - (void)mt_ctx_.prepare_prio_op_list(prio_op_array); - - exec_info_.max_submitted_seq_no_.inc_update(log.get_max_submitted_seq_no()); - exec_info_.serial_final_seq_no_ = log.get_serial_final_seq_no(); - exec_info_.data_complete_ = true; - } - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[Replay Tx] Replay Active Info in TxCtx Failed", K(timestamp), K(offset), K(ret), - K(need_replay), K(log), K(*this)); - } else { -#ifndef NDEBUG - TRANS_LOG(INFO, "[Replay Tx] Replay Active Info in TxCtx", K(timestamp), K(offset), K(ret), - K(need_replay), K(log), K(*this)); -#endif - } - REC_TRANS_TRACE_EXT(tlog_, replay_active_info, OB_ID(ret), ret, - OB_ID(used), timeguard.get_diff(), - OB_ID(ref), get_ref()); - return ret; -} - -int ObPartTransCtx::assign_commit_parts(const share::ObLSArray &log_participants, - const ObTxCommitParts &log_commit_parts) -{ - int ret = OB_SUCCESS; - - if (log_participants.count() != log_commit_parts.count()) { - // replay old version log, we need mock the commit parts - for (int64_t i = 0; OB_SUCC(ret) && i < log_participants.count(); i++) { - if (OB_FAIL(exec_info_.commit_parts_.push_back(ObTxExecPart(log_participants[i], - -1 /*exec_epoch*/)))) { - TRANS_LOG(WARN, "set commit parts error", K(ret), K(*this)); - } - } - - if (OB_FAIL(ret)) { - // reset on failure to ensure atomicity - exec_info_.commit_parts_.reset(); - } - } else { - if (OB_FAIL(exec_info_.commit_parts_.assign(log_commit_parts))) { - TRANS_LOG(WARN, "set commit parts error", K(ret), K(*this)); - } - } - - return ret; -} - -int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, +int ObTxCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no, @@ -5125,52 +4406,18 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); } else if (OB_FAIL(exec_info_.redo_lsns_.assign(commit_info_log.get_redo_lsns()))) { TRANS_LOG(WARN, "set redo log offsets error", K(commit_info_log), K(*this)); - } else if (OB_FAIL(set_scheduler_(commit_info_log.get_scheduler()))) { - TRANS_LOG(WARN, "set scheduler error", K(ret), K(commit_info_log), K(*this)); - } else if (OB_FAIL(exec_info_.participants_.assign(commit_info_log.get_participants()))) { - TRANS_LOG(WARN, "set participants error", K(ret), K(commit_info_log), K(*this)); - } else if (OB_FAIL(exec_info_.incremental_participants_.assign( - commit_info_log.get_incremental_participants()))) { - TRANS_LOG(WARN, "set incremental_participants error", K(ret), K(commit_info_log), K(*this)); - } else if (OB_FAIL(assign_commit_parts(commit_info_log.get_participants(), - commit_info_log.get_commit_parts()))) { - TRANS_LOG(WARN, "set commit parts error", K(ret), K(commit_info_log), K(*this)); } else if (OB_FAIL(set_app_trace_info_(commit_info_log.get_app_trace_info()))) { TRANS_LOG(WARN, "set app trace info error", K(ret), K(commit_info_log), K(*this)); } else if (OB_FAIL(set_app_trace_id_(commit_info_log.get_app_trace_id()))) { TRANS_LOG(WARN, "set app trace id error", K(ret), K(commit_info_log), K(*this)); } else { - // NOTE that set xa variables before set trans type - set_2pc_upstream_(commit_info_log.get_upstream()); + exec_info_.mark_write_state(); exec_info_.xid_ = commit_info_log.get_xid(); - exec_info_.is_sub2pc_ = commit_info_log.is_sub2pc(); - if (exec_info_.participants_.count() > 1) { - exec_info_.trans_type_ = TransType::DIST_TRANS; - } else if (exec_info_.upstream_.is_valid()) { - exec_info_.trans_type_ = TransType::DIST_TRANS; - } else { - exec_info_.trans_type_ = TransType::SP_TRANS; - } - can_elr_ = commit_info_log.is_elr(); - if (commit_info_log.get_cluster_version() > 0) { - ret = correct_cluster_version_(commit_info_log.get_cluster_version()); - } - sub_state_.set_info_log_submitted(); - epoch_ = commit_info_log.get_epoch(); + runtime_state_.set_info_log_submitted(); reset_redo_lsns_(); - ObTwoPhaseCommitLogType two_phase_log_type = ObTwoPhaseCommitLogType::OB_LOG_TX_MAX; - if (OB_FAIL(ret)) { - // do nothing - } else if (is_local_tx_()) { - set_durable_state_(ObTxState::REDO_COMPLETE); - set_upstream_state(ObTxState::REDO_COMPLETE); - } else if (replay_completeness_.is_incomplete()) { - set_durable_state_(ObTxState::REDO_COMPLETE); - set_upstream_state(ObTxState::REDO_COMPLETE); - // incomplete replay ctx will exiting by replay commit/abort/clear, no need to depend on 2PC - } - + set_durable_state_(ObTxState::REDO_COMPLETE); + set_target_state(ObTxState::REDO_COMPLETE); } if (OB_FAIL(ret)) { @@ -5203,73 +4450,11 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, return ret; } -int ObPartTransCtx::replay_prepare(const ObTxPrepareLog &prepare_log, - const palf::LSN &offset, - const SCN ×tamp, - const int64_t &part_log_no) -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("replay_prepare", 10 * 1000); - // const int64_t start = ObTimeUtility::fast_current_time(); - bool need_replay = true; - - CtxLockGuard guard(lock_); - - if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { - TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); - } else if (!need_replay) { - TRANS_LOG(INFO, "need not replay log", K(prepare_log), K(timestamp), K(offset), K(*this)); - // no need to replay - } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { - TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); - } else if (OB_FAIL(exec_info_.incremental_participants_.assign( - prepare_log.get_incremental_participants()))) { - TRANS_LOG(WARN, "set incremental_participants error", K(ret), K(*this)); - } else { - exec_info_.prepare_version_ = timestamp; - exec_info_.trans_type_ = TransType::DIST_TRANS; - mt_ctx_.set_prepare_version(timestamp); - ObTwoPhaseCommitLogType two_phase_log_type = ObTwoPhaseCommitLogType::OB_LOG_TX_MAX; - if (replay_completeness_.is_incomplete()) { - set_durable_state_(ObTxState::PREPARE); - set_upstream_state(ObTxState::PREPARE); - // incomplete replay ctx will exiting by replay commit/abort/clear, no need to depend on 2PC - } - } - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "replay prepare log error", KR(ret), "context", *this, K(prepare_log)); - } else if (OB_FAIL(notify_data_source_(NotifyType::ON_PREPARE, timestamp, true, - exec_info_.multi_data_source_))) { - - TRANS_LOG(WARN, "notify data source for ON_PREPARE failed", KR(ret), K(*this)); - } else { - sub_state_.set_state_log_submitted(); - } - - const int64_t used_time = timeguard.get_diff(); - REC_TRANS_TRACE_EXT2(tlog_, replay_prepare, OB_ID(ret), ret, OB_ID(used), - used_time, OB_ID(offset), offset.val_, - OB_ID(t), timestamp, OB_ID(ref), get_ref()); - ObTransStatistic::get_instance().add_prepare_log_replay_count( 1); - ObTransStatistic::get_instance().add_prepare_log_replay_time( used_time); - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[Replay Tx] replay prepare log failed", K(ret), K(used_time), K(timestamp), - K(offset), K(prepare_log), K(*this)); - } else { -#ifndef NDEBUG - TRANS_LOG(INFO, "[Replay Tx] replay prepare log", K(ret), K(used_time), K(timestamp), K(offset), - K(prepare_log), K(*this)); -#endif - } - return ret; -} - ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_BEFORE_NOTIFY_TABLELOCK) ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_TABLELOCK) ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_ON_COMMIT) ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_COMMIT) -int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, +int ObTxCtx::replay_commit(const ObTxCommitLog &commit_log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no, @@ -5284,9 +4469,6 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); - } else if (OB_FAIL(check_trans_type_for_replay_(commit_log.get_trans_type(), timestamp))) { - TRANS_LOG(WARN, "check trans type for replay failed", K(ret), K(timestamp), K(commit_log), - K(*this)); } else if (OB_FAIL(replay_update_tx_data_(true, timestamp, commit_version))) { TRANS_LOG(WARN, "replay update tx data failed", KR(ret), K(*this)); } else if (OB_FAIL(replace_tx_data_with_backup_(commit_log.get_tx_data_backup(), timestamp))) { @@ -5297,9 +4479,6 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, // no need to replay } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); - } else if (OB_FAIL(exec_info_.incremental_participants_.assign( - commit_log.get_incremental_participants()))) { - TRANS_LOG(WARN, "set incremental_participants error", K(ret), K(*this)); } else { if (OB_SUCC(ret)) { if ((!commit_log.get_multi_source_data().empty() || !exec_info_.multi_data_source_.empty()) @@ -5321,18 +4500,8 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, } } if (OB_SUCC(ret)) { - if (is_local_tx_()) { - set_durable_state_(ObTxState::COMMIT); - set_upstream_state(ObTxState::COMMIT); - } else { - ObTwoPhaseCommitLogType two_phase_log_type = ObTwoPhaseCommitLogType::OB_LOG_TX_MAX; - if (replay_completeness_.is_incomplete()) { - set_durable_state_(ObTxState::COMMIT); - set_upstream_state(ObTxState::COMMIT); - // incomplete replay ctx will exiting by replay commit/abort/clear, no need to depend on - // 2PC - } - } + set_durable_state_(ObTxState::COMMIT); + set_target_state(ObTxState::COMMIT); } } @@ -5366,17 +4535,18 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, TRANS_LOG(WARN, "errsim error", K(ret)); } else if ((!ctx_tx_data_.is_read_only()) && OB_FAIL(ctx_tx_data_.insert_into_tx_table())) { TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); - } else if (is_local_tx_()) { + } else { if (OB_FAIL(trans_clear_(timestamp))) { - TRANS_LOG(WARN, "transaction clear error or trans_type is sp_trans", KR(ret), "context", *this); + TRANS_LOG(WARN, "transaction clear error", KR(ret), "context", *this); } else { set_exiting_(); } } - ls_tx_ctx_mgr_->update_max_replay_commit_version(ctx_tx_data_.get_commit_version()); + trans_service_->get_tx_version_mgr().update_max_commit_ts( + ctx_tx_data_.get_commit_version(), false); } if (OB_SUCC(ret)) { - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitted(); } const int64_t used_time = timeguard.get_diff(); @@ -5397,7 +4567,7 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, return ret; } -int ObPartTransCtx::replay_clear(const ObTxClearLog &clear_log, +int ObTxCtx::replay_clear(const ObTxClearLog &clear_log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no) @@ -5417,31 +4587,19 @@ int ObPartTransCtx::replay_clear(const ObTxClearLog &clear_log, } } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); - } else if (OB_FAIL(exec_info_.incremental_participants_.assign( - clear_log.get_incremental_participants()))) { - TRANS_LOG(WARN, "set incremental_participants error", K(ret), K(*this)); } else if (OB_FAIL(trans_clear_(timestamp))) { TRANS_LOG(WARN, "transaction clear error", KR(ret), "context", *this); } else { - if (is_local_tx_()) { - //TODO ignore err_code when replay from a middle log without tx_ctx_table_info - TRANS_LOG(WARN, "unexpected clear log", KPC(this), K(clear_log)); - print_trace_log_(); - // ret = OB_ERR_UNEXPECTED; - } else { - ObTwoPhaseCommitLogType two_phase_log_type = ObTwoPhaseCommitLogType::OB_LOG_TX_MAX; - if (replay_completeness_.is_incomplete()) { - set_durable_state_(ObTxState::CLEAR); - set_upstream_state(ObTxState::CLEAR); - // incomplete replay ctx will exiting by replay commit/abort/clear, no need to depend on 2PC - } - } + //TODO ignore err_code when replay from a middle log without tx_ctx_table_info + TRANS_LOG(WARN, "unexpected clear log", KPC(this), K(clear_log)); + print_trace_log_(); + // ret = OB_ERR_UNEXPECTED; if (OB_SUCC(ret)) { set_exiting_(); } } if (OB_SUCC(ret)) { - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitted(); } if (OB_FAIL(ret)) { TRANS_LOG(WARN, "replay clear log error", KR(ret), "context", *this, K(clear_log)); @@ -5471,7 +4629,7 @@ ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_TX_END) ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_ABORT) ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_CLEAR) ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_ON_ABORT) -int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, +int ObTxCtx::replay_abort(const ObTxAbortLog &abort_log, const palf::LSN &offset, const SCN ×tamp, const int64_t &part_log_no) @@ -5516,17 +4674,8 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, } } if (OB_SUCC(ret)) { - if (is_local_tx_()) { - set_durable_state_(ObTxState::ABORT); - set_upstream_state(ObTxState::ABORT); - } else { - ObTwoPhaseCommitLogType two_phase_log_type = ObTwoPhaseCommitLogType::OB_LOG_TX_MAX; - if (replay_completeness_.is_incomplete()) { - set_durable_state_(ObTxState::ABORT); - set_upstream_state(ObTxState::ABORT); - // incomplete replay ctx will exiting by replay commit/abort/clear, no need to depend on 2PC - } - } + set_durable_state_(ObTxState::ABORT); + set_target_state(ObTxState::ABORT); } } if (OB_SUCC(ret)) { @@ -5565,7 +4714,7 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, } } if (OB_SUCC(ret)) { - sub_state_.set_state_log_submitted(); + runtime_state_.set_state_log_submitted(); } const int64_t used_time = timeguard.get_diff(); REC_TRANS_TRACE_EXT2(tlog_, replay_abort, OB_ID(ret), ret, OB_ID(used), @@ -5587,7 +4736,7 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, return ret; } -int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, +int ObTxCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, const palf::LSN &lsn, const SCN ×tamp, const int64_t &part_log_no) @@ -5630,7 +4779,7 @@ int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, : timestamp; if (OB_FAIL(ret)) { - // do nothing + // do nothing //TODO & ATTENTION: deep copy a part of the mds array in the log twice after recovered from the tx_ctx_table } else if (OB_FAIL(deep_copy_mds_array_(log.get_data(), increamental_array))) { TRANS_LOG(WARN, "deep copy mds array failed", K(ret)); @@ -5642,502 +4791,114 @@ int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, } else if (OB_FAIL(notify_data_source_(NotifyType::ON_REDO, timestamp, true, - increamental_array))) { - TRANS_LOG(WARN, "notify data source for ON_REDO failed", K(ret)); - } else if (OB_FAIL(replay_mds_to_tx_table_(increamental_array, timestamp))) { - TRANS_LOG(WARN, "insert mds_op to tx_table failed", K(ret)); - } - - if (OB_SUCC(ret) && OB_FAIL(check_and_merge_redo_lsns_(lsn))) { - TRANS_LOG(WARN, "check and merge redo lsns failed", K(ret), K(trans_id_), K(timestamp), K(lsn)); - } - - // rollback mds log replay - if (OB_FAIL(ret)) { - int tmp_ret = OB_SUCCESS; - for (int64_t i = additional_index; - i < exec_info_.multi_data_source_.count() && OB_SUCCESS == tmp_ret; i++) { - - ObTxBufferNode &node = exec_info_.multi_data_source_.at(i); - if (nullptr != node.data_.ptr()) { - mds_cache_.free_mds_node(node.data_, node.get_register_no()); - node.get_buffer_ctx_node().destroy_ctx(); - } - } - for (int64_t i = exec_info_.multi_data_source_.count() - 1; - i >= additional_index && OB_SUCCESS == tmp_ret; i--) { - if (OB_TMP_FAIL(exec_info_.multi_data_source_.remove(i))) { - TRANS_LOG(WARN, "remove mds node in exec_info failed", K(ret)); - } - } - } - REC_TRANS_TRACE_EXT2(tlog_, replay_multi_data_source, OB_ID(ret), ret, OB_ID(used), - timeguard.get_diff(), OB_ID(offset), lsn.val_, OB_ID(t), timestamp, - OB_ID(ref), get_ref()); - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[Replay Tx] Replay MSD Redo in TxCtx Failed", K(ret), K(timestamp), K(lsn), - K(need_replay), K(log), K(*this)); - } else { -#ifndef NDEBUG - TRANS_LOG(INFO, "[Replay Tx] Replay MSD Redo in TxCtx", K(ret), K(timestamp), K(lsn), - K(need_replay), K(log), K(*this)); -#endif - } - return ret; -} - -int ObPartTransCtx::replay_record(const ObTxRecordLog &log, - const palf::LSN &lsn, - const SCN ×tamp, - const int64_t &part_log_no) -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("replay_record", 10 * 1000); - bool need_replay = false; - CtxLockGuard guard(lock_); - - if (OB_FAIL(check_replay_avaliable_(lsn, timestamp, part_log_no, need_replay))) { - TRANS_LOG(WARN, "check replay available failed", KR(ret), K(lsn), K(timestamp), K(*this)); - } else if (!need_replay) { - TRANS_LOG(INFO, "need not replay log", K(log), K(timestamp), K(lsn), K(*this)); - // no need to replay - } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { - TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); - } else { - reset_redo_lsns_(); - set_prev_record_lsn_(lsn); - } - - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[Replay Tx] Replay Record in TxCtx Failed", K(ret), K(timestamp), K(lsn), - K(need_replay), K(log), K(*this)); - } else { - TRANS_LOG(INFO, "[Replay Tx] Replay Record in TxCtx", K(ret), K(timestamp), K(lsn), - K(need_replay), K(log), K(*this)); - } - - return ret; -} - -const SCN ObPartTransCtx::get_min_undecided_log_ts() const -{ - SCN log_ts; - CtxLockGuard guard(lock_); - if (!busy_cbs_.is_empty()) { - const ObTxLogCb *log_cb = busy_cbs_.get_first(); - if (OB_ISNULL(log_cb)) { - TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected null ptr", K(*this)); - } else { - log_ts = log_cb->get_log_ts(); - } - } - return log_ts; -} - - -bool ObPartTransCtx::is_committing_() const -{ - return ObTxState::INIT != exec_info_.state_ || ObPartTransAction::COMMIT == part_trans_action_ - || ObPartTransAction::ABORT == part_trans_action_; -} - -int ObPartTransCtx::switch_to_leader(const SCN &start_working_ts) -{ - int ret = OB_SUCCESS; - - share::SCN append_mode_initial_scn; - append_mode_initial_scn.set_invalid(); - - common::ObTimeGuard timeguard("switch to leader", 10 * 1000); - CtxLockGuard guard(lock_); - TxCtxStateHelper state_helper(role_state_); - timeguard.click(); - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); - ret = OB_NOT_INIT; - } else if (OB_UNLIKELY(is_exiting_)) { - TRANS_LOG(DEBUG, "transaction is exiting", "context", *this); - } else if (OB_UNLIKELY(replay_completeness_.is_incomplete()) && (get_retain_cause() == RetainCause::UNKOWN)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "incomplete replayed ctx should not switch to leader", K(ret), KPC(this)); - print_trace_log_(); - } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->get_append_mode_initial_scn( - append_mode_initial_scn))) { - /* We can not ensure whether there are some redo logs after the append_mode_initial_scn. - * All running trans must be be killed by the append_mode_initial_scn.*/ - TRANS_LOG(WARN, "get append mode initial scn from the palf failed", K(ret), KPC(this)); - } else if (OB_FAIL(state_helper.switch_state(TxCtxOps::TAKEOVER))) { - TRANS_LOG(WARN, "switch role state error", KR(ret), K(*this)); - } else { - const bool contain_mds_table_lock = is_contain_mds_type_(ObTxDataSourceType::TABLE_LOCK); - const bool contain_mds_tablet_split = is_contain_mds_type_(ObTxDataSourceType::TABLET_SPLIT); - const bool need_kill_tx = contain_mds_table_lock - || contain_mds_tablet_split; - bool kill_by_append_mode_initial_scn = false; - if (append_mode_initial_scn.is_valid()) { - kill_by_append_mode_initial_scn = exec_info_.max_applying_log_ts_ <= append_mode_initial_scn; - } - - if (ObTxState::INIT == exec_info_.state_) { - if (exec_info_.data_complete_ && !contain_mds_table_lock - && !contain_mds_tablet_split && !kill_by_append_mode_initial_scn) { - if (OB_FAIL(mt_ctx_.replay_to_commit(false /*is_resume*/))) { - TRANS_LOG(WARN, "replay to commit failed", KR(ret), K(*this)); - } - } else { - TRANS_LOG(WARN, "txn data incomplete, will be aborted", K(contain_mds_table_lock), - K(contain_mds_tablet_split), - K(kill_by_append_mode_initial_scn), K(append_mode_initial_scn), KPC(this)); - if (has_persisted_log_()) { - if (ObPartTransAction::COMMIT == part_trans_action_ - || get_upstream_state() >= ObTxState::REDO_COMPLETE) { - - TRANS_LOG(WARN, "abort self instantly with a tx_commit request", - K(contain_mds_table_lock), K(contain_mds_tablet_split), - K(need_kill_tx), K(kill_by_append_mode_initial_scn), - K(append_mode_initial_scn), KPC(this)); - if (OB_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { - //Temporary fix: - //The transaction cannot be killed temporarily, waiting for handle_timeout to retry abort. - //Do not block the rest of the transactions from taking over. - if (OB_TRANS_CANNOT_BE_KILLED == ret) { - TRANS_LOG( - INFO, - "The transaction cannot be killed temporarily, waiting for handle_timeout to retry abort.", - K(ret), KPC(this)); - ret = OB_SUCCESS; - } - TRANS_LOG(WARN, "abort tx failed", KR(ret), KPC(this)); - - } - } else { - if (OB_FAIL(do_local_tx_end_(TxEndAction::DELAY_ABORT_TX))) { - TRANS_LOG(WARN, "abort tx failed", KR(ret), K(*this)); - } - notify_scheduler_tx_killed_(ObTxAbortCause::PARTICIPANT_SWITCH_LEADER_DATA_INCOMPLETE); - } - } else { - TRANS_LOG(ERROR, "unexpected trx which has not persisted log", K(*this)); - } - } - } else { - if (OB_FAIL(mt_ctx_.replay_to_commit(false /*is_resume*/))) { - TRANS_LOG(WARN, "replay to commit failed", KR(ret), K(*this)); - } - } - timeguard.click(); - - // The txn with is_exiting_ = true only need to maintain the state - if (OB_SUCC(ret) && !is_exiting_) { - // The request_id_ should be initialized to prevent the 2pc cannot be - // driven if all participants are switching leader - generate_request_id_(); - if (ObTxState::INIT == exec_info_.state_ && exec_info_.data_complete_) { - const int64_t left_time = trans_expired_time_ - ObClockGenerator::getRealClock(); - if (left_time < ObServerConfig::get_instance().trx_2pc_retry_interval) { - trans_2pc_timeout_ = (left_time > 0 ? left_time : 100 * 1000); - } else { - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; - } - } else { - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; - } - // do not post transaction message to avoid deadlock, bug#8257026 - // just register timeout task - (void)unregister_timeout_task_(); - if (OB_FAIL(register_timeout_task_(trans_2pc_timeout_))) { - TRANS_LOG(WARN, "register timeout handler error", KR(ret), "context", *this); - } - } - if (OB_SUCC(ret)) { - exec_info_.data_complete_ = false; - start_working_log_ts_ = start_working_ts; - } else { - state_helper.restore_state(); - } - timeguard.click(); - } - if (OB_SUCC(ret)) { - (void)try_submit_next_log_(); - big_segment_info_.reuse(); - } - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "switch to leader failed", KR(ret), K(ret), KPC(this)); - } else { - last_request_ts_ = ObClockGenerator::getClock(); -#ifndef NDEBUG - TRANS_LOG(INFO, "switch to leader succeed", KPC(this)); -#endif - } - REC_TRANS_TRACE_EXT2(tlog_, switch_to_leader, OB_ID(ret), ret, OB_ID(used), timeguard.get_diff(), - OB_ID(ref), get_ref()); - return ret; -} - -inline bool ObPartTransCtx::need_callback_scheduler_() { - return is_local_tx_() - && ObPartTransAction::COMMIT == part_trans_action_ - && addr_ == exec_info_.scheduler_ - && commit_cb_.is_enabled() - && !commit_cb_.is_inited(); -} - -// It's possile that a follower-state ctx experiences switch_to_follower_forcedly -// 1. in leader state -// 2. switch graceful and turned into follower state, but another sub-handler fails -// 3. resume_leader is called -// 4. resume_leader fails -// 5. revoke, and still in follower state -int ObPartTransCtx::switch_to_follower_forcedly(ObTxCommitCallback *&cb_list_head) -{ - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("switch_to_follower_forcely", 10 * 1000); - CtxLockGuard guard(lock_); - TxCtxStateHelper state_helper(role_state_); + increamental_array))) { + TRANS_LOG(WARN, "notify data source for ON_REDO failed", K(ret)); + } else if (OB_FAIL(replay_mds_to_tx_table_(increamental_array, timestamp))) { + TRANS_LOG(WARN, "insert mds_op to tx_table failed", K(ret)); + } - if (IS_NOT_INIT) { - TRANS_LOG(WARN, "ObPartTransCtx not inited"); - ret = OB_NOT_INIT; - } else if (is_exiting_) { - // do nothing - } else if (is_follower_()) { - TRANS_LOG(INFO, "current tx already follower", K(*this)); - } else if (OB_FAIL(state_helper.switch_state(TxCtxOps::REVOKE))) { - TRANS_LOG(WARN, "switch role state error", KR(ret), K(*this)); - } else { - (void)unregister_timeout_task_(); - if (!has_persisted_log_() && !is_logging_()) { - // has not persisted any log, exit here - bool need_cb_scheduler = need_callback_scheduler_(); - if (need_cb_scheduler) { - // disable commit_cb to prevent do_local_tx_end_ callback scheduler, - // which cause deadlock - commit_cb_.disable(); - } - if (OB_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { - TRANS_LOG(WARN, "do local tx abort failed", K(ret)); - } else if (need_cb_scheduler) { - if (OB_FAIL(prepare_commit_cb_for_role_change_(OB_TRANS_KILLED, cb_list_head))) { - TRANS_LOG(WARN, "prepare commit cb fail", K(ret), KPC(this)); - } - } - if (OB_FAIL(ret) && need_cb_scheduler) { - // recover commit_cb_ when failed - commit_cb_.enable(); - } - if (!need_cb_scheduler) { - notify_scheduler_tx_killed_(ObTxAbortCause::PARTICIPANT_SWITCH_FOLLOWER_FORCEDLY); - } - TRANS_LOG(INFO, "switch to follower forcely, txn aborted without persisted log", KPC(this)); - } else { - // has persisted log, wait new leader to advance it - if (pending_write_ && OB_FALSE_IT(mt_ctx_.wait_pending_write())) { - // do nothing - } else if (OB_FALSE_IT(mt_ctx_.commit_to_replay())) { - // do nothing - } else if (OB_FAIL(mt_ctx_.clean_unlog_callbacks())) { - TRANS_LOG(WARN, "clear unlog callbacks", KR(ret), K(*this)); - } + if (OB_SUCC(ret) && OB_FAIL(check_and_merge_redo_lsns_(lsn))) { + TRANS_LOG(WARN, "check and merge redo lsns failed", K(ret), K(trans_id_), K(timestamp), K(lsn)); + } - if (is_local_tx_() && - !is_logging_() && - !sub_state_.is_state_log_submitted()) { - mt_ctx_.set_trans_version(SCN::max_scn()); - if (OB_FAIL(ctx_tx_data_.set_commit_version(share::SCN::invalid_scn()))) { - TRANS_LOG(ERROR, "reset commit version failed", KR(ret), K(*this)); - } else { - TRANS_LOG(INFO, "clear local tx trans version when switch to follower forcely", KP(this)); - } - } + // rollback mds log replay + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + for (int64_t i = additional_index; + i < exec_info_.multi_data_source_.count() && OB_SUCCESS == tmp_ret; i++) { - // special handle commit triggered by local call: coordinator colocate with scheduler - // let scheduler retry commit with RPC if required - if (need_callback_scheduler_()) { - int commit_ret = OB_SUCCESS; - // no CommitInfoLog has been submitted, txn must abort - if (exec_info_.state_ == ObTxState::INIT && !sub_state_.is_info_log_submitted()) { - commit_ret = OB_TRANS_KILLED; - } else { - // otherwise, txn either continue commit or abort, need retry to get final result - commit_ret = OB_NOT_MASTER; - } - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(prepare_commit_cb_for_role_change_(commit_ret, cb_list_head))) { - TRANS_LOG(WARN, "prepare commit cb fail", K(tmp_ret), KPC(this)); - ret = (OB_SUCCESS == ret) ? tmp_ret : ret; - } else { - TRANS_LOG(INFO, "switch to follower forcely, notify txn commit result to scheduler", - "commit_result", commit_ret, KPC(this)); - } + ObTxBufferNode &node = exec_info_.multi_data_source_.at(i); + if (nullptr != node.data_.ptr()) { + mds_cache_.free_mds_node(node.data_, node.get_register_no()); + node.get_buffer_ctx_node().destroy_ctx(); } } - if (OB_FAIL(ret)) { - state_helper.restore_state(); + for (int64_t i = exec_info_.multi_data_source_.count() - 1; + i >= additional_index && OB_SUCCESS == tmp_ret; i--) { + if (OB_TMP_FAIL(exec_info_.multi_data_source_.remove(i))) { + TRANS_LOG(WARN, "remove mds node in exec_info failed", K(ret)); + } } } + REC_TRANS_TRACE_EXT2(tlog_, replay_multi_data_source, OB_ID(ret), ret, OB_ID(used), + timeguard.get_diff(), OB_ID(offset), lsn.val_, OB_ID(t), timestamp, + OB_ID(ref), get_ref()); + if (OB_FAIL(ret)) { - TRANS_LOG(ERROR, "switch to follower forcedly failed", KR(ret), KPC(this)); + TRANS_LOG(WARN, "[Replay Tx] Replay MSD Redo in TxCtx Failed", K(ret), K(timestamp), K(lsn), + K(need_replay), K(log), K(*this)); } else { - big_segment_info_.reuse(); - TRANS_LOG(INFO, "switch to follower forcedly success", KPC(this)); +#ifndef NDEBUG + TRANS_LOG(INFO, "[Replay Tx] Replay MSD Redo in TxCtx", K(ret), K(timestamp), K(lsn), + K(need_replay), K(log), K(*this)); +#endif } - REC_TRANS_TRACE_EXT2(tlog_, switch_to_follower_forcely, - OB_ID(ret), ret, - OB_ID(used), timeguard.get_diff(), - OB_ID(ref), get_ref()); return ret; } -int ObPartTransCtx::switch_to_follower_gracefully(ObTxCommitCallback *&cb_list_head) +int ObTxCtx::replay_record(const ObTxRecordLog &log, + const palf::LSN &lsn, + const SCN ×tamp, + const int64_t &part_log_no) { int ret = OB_SUCCESS; - bool need_submit_log = false; - ObTxLogType log_type = ObTxLogType::TX_ACTIVE_INFO_LOG; - common::ObTimeGuard timeguard("switch_to_follower_gracefully", 10 * 1000); + common::ObTimeGuard timeguard("replay_record", 10 * 1000); + bool need_replay = false; CtxLockGuard guard(lock_); - timeguard.click(); - TxCtxStateHelper state_helper(role_state_); - if (is_exiting_) { - // do nothing - } else if (is_force_abort_logging_()) { - // is aborting, skip - } else if (is_follower_()) { - TRANS_LOG(INFO, "current tx already follower", K(*this)); - } else if (OB_FAIL(state_helper.switch_state(TxCtxOps::SWITCH_GRACEFUL))) { - TRANS_LOG(WARN, "switch role state error", KR(ret), K(*this)); - } else { - if (pending_write_) { - TRANS_LOG(INFO, "current tx is executing stmt", K(*this)); - mt_ctx_.wait_pending_write(); - timeguard.click(); - need_submit_log = true; - } else if (!is_committing_()) { - need_submit_log = true; - } else if (ObPartTransAction::COMMIT == part_trans_action_) { - /* NOTE: - * - If commitInfoLog has submitted, txn can continue on the new Leader - * the scheduler should retry commit, in order to: - * - detect commit timeout - * - query commit result when commit response message was lost - * - If commitInfoLog hasn't submitted, commitInfoLog will be submitted - * the scheduler should retry commit, in order to: - * - found the txn has aborted or survived on the new Leader - * - detect commit timeout - */ - if (exec_info_.state_ < ObTxState::REDO_COMPLETE && !sub_state_.is_info_log_submitted()) { - need_submit_log = true; - log_type = ObTxLogType::TX_COMMIT_INFO_LOG; - } - if (need_callback_scheduler_()) { - if (OB_FAIL(prepare_commit_cb_for_role_change_(OB_SWITCHING_TO_FOLLOWER_GRACEFULLY, cb_list_head))) { - TRANS_LOG(WARN, "prepare commit cb fail", K(ret), KPC(this)); - } else { - TRANS_LOG(INFO, "swtich to follower gracefully, notify scheduler retry", KPC(this)); - } - } - } - timeguard.click(); - if (OB_SUCC(ret) && need_submit_log && !need_force_abort_()) { - if (ObTxLogType::TX_COMMIT_INFO_LOG == log_type || ObTxLogType::TX_ACTIVE_INFO_LOG == log_type) { - if (OB_FAIL(submit_log_impl_(log_type))) { - // currently, if there is no log callback, switch leader would fail, - // and resume leader would be called. - TRANS_LOG(WARN, "submit active/commit info log failed", KR(ret), K(*this)); - } - } else { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected log type", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(mt_ctx_.commit_to_replay())) { - TRANS_LOG(WARN, "commit to replay failed", KR(ret), K(*this)); - } else if (OB_FAIL(unregister_timeout_task_())) { - TRANS_LOG(WARN, "unregister timeout handler error", KR(ret), KPC(this)); - } - } - - timeguard.click(); - if (OB_FAIL(ret)) { - state_helper.restore_state(); - } + if (OB_FAIL(check_replay_avaliable_(lsn, timestamp, part_log_no, need_replay))) { + TRANS_LOG(WARN, "check replay available failed", KR(ret), K(lsn), K(timestamp), K(*this)); + } else if (!need_replay) { + TRANS_LOG(INFO, "need not replay log", K(log), K(timestamp), K(lsn), K(*this)); + // no need to replay + } else if (OB_FAIL(update_replaying_log_no_(timestamp, part_log_no))) { + TRANS_LOG(WARN, "update replaying log no failed", K(ret), K(timestamp), K(part_log_no)); + } else { + reset_redo_lsns_(); + set_prev_record_lsn_(lsn); } - REC_TRANS_TRACE_EXT2(tlog_, switch_to_follower_gracefully, - OB_ID(ret), ret, - OB_ID(log_type), log_type, - OB_ID(used), timeguard.get_diff(), - OB_ID(ref), get_ref()); + if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "switch to follower gracefully failed", KR(ret), KPC(this)); + TRANS_LOG(WARN, "[Replay Tx] Replay Record in TxCtx Failed", K(ret), K(timestamp), K(lsn), + K(need_replay), K(log), K(*this)); } else { -#ifndef NDEBUG - TRANS_LOG(INFO, "switch to follower gracefully succeed", KPC(this)); -#endif + TRANS_LOG(INFO, "[Replay Tx] Replay Record in TxCtx", K(ret), K(timestamp), K(lsn), + K(need_replay), K(log), K(*this)); } return ret; } -// a ctx may be in leader state when resume_leader is called. -// assume 100 ctx in ctx mgr, -// 1. switch_to_follower_gracefully, traverse 100 ctx -// and 50 of them switch to follower state, but the 51th fails, -// at this time, there are 50 ctx in leader state and 50 ctx in follower state. -// 2. rollback the change, and would traverse all the 100 ctx, including -// the 50 ctx in leader state, for such ctx, do nothing. -int ObPartTransCtx::resume_leader(const SCN &start_working_ts) +const SCN ObTxCtx::get_min_undecided_log_ts() const { - int ret = OB_SUCCESS; - common::ObTimeGuard timeguard("resume_leader", 10 * 1000); + SCN log_ts; CtxLockGuard guard(lock_); - TxCtxStateHelper state_helper(role_state_); - if (is_exiting_) { - // do nothing - } else if (is_leader_()) { - // do nothing - } else if (OB_FAIL(state_helper.switch_state(TxCtxOps::RESUME))) { - TRANS_LOG(WARN, "switch role state error", KR(ret), K(*this)); - } else { - if (OB_FAIL(mt_ctx_.replay_to_commit(true /*is_resume*/))) { - TRANS_LOG(WARN, "replay to commit failed", KR(ret), K(*this)); + if (!busy_cbs_.is_empty()) { + const ObTxLogCb *log_cb = busy_cbs_.get_first(); + if (OB_ISNULL(log_cb)) { + TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected null ptr", K(*this)); } else { - if (ObTxState::INIT == exec_info_.state_) { - const int64_t left_time = trans_expired_time_ - ObClockGenerator::getRealClock(); - if (left_time < ObServerConfig::get_instance().trx_2pc_retry_interval) { - trans_2pc_timeout_ = (left_time > 0 ? left_time : 100 * 1000); - } else { - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; - } - } else { - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; - } - (void)unregister_timeout_task_(); - if (OB_FAIL(register_timeout_task_(trans_2pc_timeout_))) { - TRANS_LOG(WARN, "register timeout handler error", KR(ret), KPC(this)); - } - } - if (OB_SUCC(ret)) { - exec_info_.data_complete_ = false; - start_working_log_ts_ = start_working_ts; - } - if (OB_FAIL(ret)) { - state_helper.restore_state(); + log_ts = log_cb->get_log_ts(); } } - if (OB_SUCC(ret)) { - (void)try_submit_next_log_(); - } - REC_TRANS_TRACE_EXT2(tlog_, resume_leader, OB_ID(ret), ret, - OB_ID(used), timeguard.get_diff(), - OB_ID(ref), get_ref()); - TRANS_LOG(INFO, "resume leader", KR(ret), K(ret), KPC(this)); - return ret; + return log_ts; +} + + +bool ObTxCtx::is_committing_() const +{ + return ObTxState::INIT != exec_info_.state_ || ObPartTransAction::COMMIT == part_trans_action_ + || ObPartTransAction::ABORT == part_trans_action_; +} + +inline bool ObTxCtx::need_commit_callback_() { + return ObPartTransAction::COMMIT == part_trans_action_ + && commit_cb_.is_enabled() + && !commit_cb_.is_inited(); } -//***************************** -int ObPartTransCtx::check_with_tx_data(ObITxDataCheckFunctor &fn) + +int ObTxCtx::check_with_tx_data(ObITxDataCheckFunctor &fn) { // NB: You need notice the lock is not acquired during check int ret = OB_SUCCESS; @@ -6160,7 +4921,7 @@ int ObPartTransCtx::check_with_tx_data(ObITxDataCheckFunctor &fn) return ret; } -int ObPartTransCtx::update_rec_log_ts_for_parallel_replay(const SCN &rec_scn) +int ObTxCtx::update_rec_log_ts_for_parallel_replay(const SCN &rec_scn) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); @@ -6191,7 +4952,7 @@ int ObPartTransCtx::update_rec_log_ts_for_parallel_replay(const SCN &rec_scn) return ret; } -int ObPartTransCtx::update_rec_log_ts_(bool for_replay, const SCN &rec_log_ts) +int ObTxCtx::update_rec_log_ts_(bool for_replay, const SCN &rec_log_ts) { int ret = OB_SUCCESS; @@ -6242,7 +5003,7 @@ int ObPartTransCtx::update_rec_log_ts_(bool for_replay, const SCN &rec_log_ts) // next checkpoint. While we shouldnot return the refreshed rec_log_ts before // the checkpoint of the tx ctx table is succeed. So we need remember the // rec_log_ts using prev_rec_log_ts before successfully checkpointing -int ObPartTransCtx::refresh_rec_log_ts_() +int ObTxCtx::refresh_rec_log_ts_() { int ret = OB_SUCCESS; @@ -6251,45 +5012,10 @@ int ObPartTransCtx::refresh_rec_log_ts_() // checkpointed prev_rec_log_ts_.atomic_store(rec_log_ts_); - if (is_follower_()) { - // Case 1: As follower, the replay may involve both serial and concurrent - // replay. While regardless of how the replay occurs, we update the - // rec_scn after the replay is completed. Therefore, as long as we ensure - // that the state of the txn of the rec_scn is complete before updating - // the rec_scn(indicated by ObTxReplayExecutor::finish_replay_), it is - // safe to clear the rec_scn during the refresh_rec_scn. - if (busy_cbs_.is_empty()) { - rec_log_ts_.atomic_store(share::SCN::invalid_scn()); - } else { - // Case 1.1: As follower, there may also exist log which is proposed - // while not committed because of the current leader's switch mechinism - // (leader in txn layer will be switched from follower even before all - // proposed log has been synced). So we need refer to the log ts in the - // current log_cb. - const ObTxLogCb *log_cb = busy_cbs_.get_first(); - if (OB_ISNULL(log_cb)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected null ptr", K(*this)); - } else { - rec_log_ts_.atomic_store(log_cb->get_log_ts()); - } - } + if (busy_cbs_.is_empty()) { + rec_log_ts_.atomic_store(share::SCN::invalid_scn()); } else { - // Case 2: As leader, the application is discrete and not in order, so we - // rely on the first lo(we call it FCL later)g hasnot been applied as - // rec_log_ts if exists or reset it if not because all log of the txn with - // its log ts in front of the FCL must be contained in the checkpoint. - if (busy_cbs_.is_empty()) { - rec_log_ts_.atomic_store(share::SCN::invalid_scn()); - } else { - const ObTxLogCb *log_cb = busy_cbs_.get_first(); - if (OB_ISNULL(log_cb)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected null ptr", K(*this)); - } else { - rec_log_ts_.atomic_store(log_cb->get_log_ts()); - } - } + rec_log_ts_.atomic_store(busy_cbs_.get_first()->get_log_ts()); } } else { TRANS_LOG(WARN, "we should not allow concurrent merge of tx ctx table", K(*this)); @@ -6298,10 +5024,9 @@ int ObPartTransCtx::refresh_rec_log_ts_() return ret; } -int ObPartTransCtx::get_tx_ctx_table_info_(ObTxCtxTableInfo &info) +int ObTxCtx::get_tx_ctx_table_info_(ObTxCtxTableInfo &info) { int ret = OB_SUCCESS; - exec_info_.exec_epoch_ = epoch_; // leave target_scn to MAX and the callee will choose the greatest // calculable scn, especially when parallel replay, the max scn of // a parallel replayed callback-list will be carefully choosen to @@ -6319,7 +5044,6 @@ int ObPartTransCtx::get_tx_ctx_table_info_(ObTxCtxTableInfo &info) TRANS_LOG(WARN, "fail to assign exec_info", K(ret), KPC(this)); } else { info.tx_id_ = trans_id_; - info.ls_id_ = ls_id_; info.cluster_id_ = cluster_id_; if (cluster_version_accurate_) { info.cluster_version_ = cluster_version_; @@ -6337,24 +5061,7 @@ int ObPartTransCtx::get_tx_ctx_table_info_(ObTxCtxTableInfo &info) return ret; } -int ObPartTransCtx::set_scheduler_(const common::ObAddr &scheduler) -{ - int ret = OB_SUCCESS; - - exec_info_.scheduler_ = scheduler; - - return ret; -} - -const common::ObAddr &ObPartTransCtx::get_scheduler() const -{ - if (exec_info_.data_complete_ && !exec_info_.scheduler_.is_valid()) { - TRANS_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "invalid scheduler addr", K(trans_id_)); - } - return exec_info_.scheduler_; -} - -int ObPartTransCtx::gen_total_mds_array_(ObTxBufferNodeArray &mds_array) +int ObTxCtx::gen_total_mds_array_(ObTxBufferNodeArray &mds_array) { int ret = OB_SUCCESS; @@ -6367,7 +5074,7 @@ int ObPartTransCtx::gen_total_mds_array_(ObTxBufferNodeArray &mds_array) return ret; } -int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, +int ObTxCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, ObTxBufferNodeArray &incremental_array, bool need_replace) { @@ -6378,11 +5085,8 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, || old_node.get_data_source_type() >= ObTxDataSourceType::MAX_TYPE) { ret = OB_ERR_UNDEFINED; TRANS_LOG(ERROR, "unexpected mds type", KR(ret), K(*this)); - } else if (old_node.get_data_source_type() <= ObTxDataSourceType::BEFORE_VERSION_4_1 - && ObTxDataSourceType::CREATE_TABLET_NEW_MDS != old_node.get_data_source_type() - && ObTxDataSourceType::DELETE_TABLET_NEW_MDS != old_node.get_data_source_type() - && ObTxDataSourceType::UNBIND_TABLET_NEW_MDS != old_node.get_data_source_type()) { - TRANS_LOG(DEBUG, "old mds type, no need process with buffer ctx", + } else if (uses_builtin_mds_notifier(old_node.get_data_source_type())) { + TRANS_LOG(DEBUG, "built-in mds type has no buffer ctx", K(old_node.get_data_source_type()), K(*this)); } else { if (OB_ISNULL(old_node.get_buffer_ctx_node().get_ctx())) { // this is replay path, create ctx @@ -6514,7 +5218,7 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, int64_t ctx_array_start_index = 0; for (int64_t i = 0; OB_SUCC(ret) && i < tmp_buf_array_cnt; ++i) { - if (is_follower_()) { + if (is_for_replay()) { tmp_buf_arr[i].set_submitted(); tmp_buf_arr[i].set_synced(); } @@ -6538,7 +5242,7 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, TRANS_LOG(WARN, "push back incremental_array failed", K(ret)); } TRANS_LOG(INFO, "filter mds node replay by the register_no", K(ret), K(trans_id_), - K(ls_id_), K(i), K(ctx_array_start_index), K(tmp_buf_arr[i].get_register_no()), + K(i), K(ctx_array_start_index), K(tmp_buf_arr[i].get_register_no()), K(exec_info_.multi_data_source_[ctx_array_start_index])); } else { ret = OB_ERR_UNEXPECTED; @@ -6559,7 +5263,7 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, return ret; } -int ObPartTransCtx::prepare_mds_tx_op_(const ObTxBufferNodeArray &mds_array, +int ObTxCtx::prepare_mds_tx_op_(const ObTxBufferNodeArray &mds_array, SCN op_scn, ObTenantTxDataOpAllocator &tx_op_allocator, ObTxOpArray &tx_op_array, @@ -6595,11 +5299,11 @@ int ObPartTransCtx::prepare_mds_tx_op_(const ObTxBufferNodeArray &mds_array, tx_op_allocator.free(new_node_wrapper); } } - TRANS_LOG(INFO, "prepare_mds_tx_op", K(ret), K(trans_id_), K(ls_id_), K(mds_array), K(tx_op_array), K(op_scn)); + TRANS_LOG(INFO, "prepare_mds_tx_op", K(ret), K(trans_id_), K(mds_array), K(tx_op_array), K(op_scn)); return ret; } -int ObPartTransCtx::decide_state_log_barrier_type_( +int ObTxCtx::decide_state_log_barrier_type_( const ObTxLogType &state_log_type, logservice::ObReplayBarrierType &final_barrier_type) { @@ -6666,7 +5370,7 @@ int ObPartTransCtx::decide_state_log_barrier_type_( return ret; } -bool ObPartTransCtx::is_contain_mds_type_(const ObTxDataSourceType target_type) +bool ObTxCtx::is_contain_mds_type_(const ObTxDataSourceType target_type) { bool is_contain = false; @@ -6683,7 +5387,7 @@ bool ObPartTransCtx::is_contain_mds_type_(const ObTxDataSourceType target_type) return is_contain; } -int ObPartTransCtx::submit_multi_data_source_() +int ObTxCtx::submit_multi_data_source_() { int ret = OB_SUCCESS; ObTxLogBlock log_block; @@ -6695,7 +5399,7 @@ int ObPartTransCtx::submit_multi_data_source_() return ret; } -int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) +int ObTxCtx::submit_multi_data_source_(ObTxLogBlock &log_block) { int ret = OB_SUCCESS; @@ -6707,8 +5411,8 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) if (is_force_abort_logging_() || get_downstream_state() == ObTxState::ABORT) { ret = OB_TRANS_KILLED; - TRANS_LOG(ERROR, "tx has been aborting, can not submit prepare log", K(ret)); - } else if (sub_state_.is_info_log_submitted()) { + TRANS_LOG(ERROR, "tx has been aborting, can not submit multi data source log", K(ret)); + } else if (runtime_state_.is_info_log_submitted()) { // state log already submitted, do nothing } else if (mds_cache_.count() > 0) { ObTxMultiDataSourceLog log; @@ -6749,7 +5453,7 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) TRANS_LOG(WARN, "prepare big segment failed", K(ret), KPC(this)); } else { ret = OB_LOG_TOO_LARGE; - TRANS_LOG(INFO, "construct big multi data source",K(ret),K(trans_id_),K(ls_id_),K(log)); + TRANS_LOG(INFO, "construct big multi data source",K(ret),K(trans_id_),K(log)); } } } else if (log_block.get_cb_arg_array().count() == 0) { @@ -6788,7 +5492,7 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) } else { if (barrier_type != logservice::ObReplayBarrierType::NO_NEED_BARRIER || !mds_base_scn.is_min()) { TRANS_LOG(INFO, "submit MDS redo with barrier or base_scn successfully", K(ret), K(trans_id_), - K(ls_id_), KPC(log_cb), K(mds_cache_), K(exec_info_.multi_data_source_), + KPC(log_cb), K(mds_cache_), K(exec_info_.multi_data_source_), K(mds_base_scn), K(barrier_type)); } log_cb = nullptr; @@ -6809,7 +5513,7 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) return ret; } -int ObPartTransCtx::prepare_mul_data_source_tx_end_(bool is_commit) +int ObTxCtx::prepare_mul_data_source_tx_end_(bool is_commit) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -6820,8 +5524,8 @@ int ObPartTransCtx::prepare_mul_data_source_tx_end_(bool is_commit) && OB_FAIL(submit_log_impl_(ObTxLogType::TX_MULTI_DATA_SOURCE_LOG))) { TRANS_LOG(WARN, "submit multi data souce log failed", K(ret)); - if (OB_TMP_FAIL(restart_2pc_trans_timer_())) { - TRANS_LOG(WARN, "restart_2pc_trans_timer_ error", KR(ret), KR(tmp_ret), KPC(this)); + if (OB_TMP_FAIL(restart_commit_retry_timer_())) { + TRANS_LOG(WARN, "restart_commit_retry_timer_ error", KR(ret), KR(tmp_ret), KPC(this)); } } else if (OB_FAIL(mds_cache_.generate_final_notify_array(exec_info_.multi_data_source_, true /*need_merge_cache*/, @@ -6837,36 +5541,10 @@ int ObPartTransCtx::prepare_mul_data_source_tx_end_(bool is_commit) } #ifdef ERRSIM -ERRSIM_POINT_DEF(EN_SUBMIT_TX_PREPARE_LOG) ERRSIM_POINT_DEF(EN_NOTIFY_MDS) #endif -OB_NOINLINE int ObPartTransCtx::errism_submit_prepare_log_() -{ - - int ret = OB_SUCCESS; - - // if (OB_NOT_MASTER == ERRSIM_DELAY_TX_SUBMIT_LOG && is_root()) { - // ret = ERRSIM_DELAY_TX_SUBMIT_LOG; - // } else if (OB_BLOCK_FROZEN == ERRSIM_DELAY_TX_SUBMIT_LOG && !is_root()) { - // ret = ERRSIM_DELAY_TX_SUBMIT_LOG; - // } else if (ERRSIM_DELAY_TX_SUBMIT_LOG < int(-9999)) { - // // reject trans_id and tenant id to contorl trans sate - // // ret = process_errsim_for_standby_read_(int(ERRSIM_DELAY_TX_SUBMIT_LOG), OB_NOT_MASTER); - // TRANS_LOG(INFO, "reject delay tx submit log", K(ret), K(ERRSIM_DELAY_TX_SUBMIT_LOG)); - // } - -#ifdef ERRSIM - ret = EN_SUBMIT_TX_PREPARE_LOG; - if(OB_TRANS_NEED_ROLLBACK == ret) { - sub_state_.set_force_abort(); - } -#endif - - return ret; -} - -OB_NOINLINE OB_WEAK_SYMBOL int ObPartTransCtx::errsim_notify_mds_() +OB_NOINLINE OB_WEAK_SYMBOL int ObTxCtx::errsim_notify_mds_() { int ret = OB_SUCCESS; @@ -6881,13 +5559,13 @@ OB_NOINLINE OB_WEAK_SYMBOL int ObPartTransCtx::errsim_notify_mds_() return ret; } -int ObPartTransCtx::notify_table_lock_(const SCN &log_ts, +int ObTxCtx::notify_table_lock_(const SCN &log_ts, const bool for_replay, const ObTxBufferNodeArray ¬ify_array, const bool is_force_kill) { int ret = OB_SUCCESS; - if (is_exiting_ && sub_state_.is_force_abort()) { + if (is_exiting_ && runtime_state_.is_force_abort()) { // do nothing } else { ObMulSourceDataNotifyArg arg; @@ -6908,14 +5586,13 @@ int ObPartTransCtx::notify_table_lock_(const SCN &log_ts, TRANS_LOG(WARN, "notify data source failed", K(ret), K(arg)); } if (notify_array.count() > 0) { - TRANS_LOG(INFO, "notify MDS table lock", K(ret), K(trans_id_), K(ls_id_), - K(log_ts), K(notify_array.count()), K(notify_array), K(total_time)); + TRANS_LOG(INFO, "notify MDS table lock", K(ret), K(trans_id_), K(log_ts), K(notify_array.count()), K(notify_array), K(total_time)); } } return ret; } -int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, +int ObTxCtx::notify_data_source_(const NotifyType notify_type, const SCN &log_ts, const bool for_replay, const ObTxBufferNodeArray ¬ify_array, @@ -6925,13 +5602,13 @@ int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, int ret = OB_SUCCESS; if (OB_FAIL(errsim_notify_mds_())) { - TRANS_LOG(WARN, "notify mds errsim", K(ret), K(ls_id_), K(trans_id_), K(notify_type), K(log_ts), + TRANS_LOG(WARN, "notify mds errsim", K(ret), K(trans_id_), K(notify_type), K(log_ts), K(for_replay), K(notify_array), K(is_force_kill)); } if (OB_FAIL(ret)) { // do nothing - } else if (is_exiting_ && sub_state_.is_force_abort()) { + } else if (is_exiting_ && runtime_state_.is_force_abort()) { // do nothing } else { ObMulSourceDataNotifyArg arg; @@ -6951,8 +5628,7 @@ int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, TRANS_LOG(WARN, "notify data source failed", K(ret), K(arg)); } if (notify_array.count() > 0) { - TRANS_LOG(INFO, "notify MDS", K(ret), K(trans_id_), K(ls_id_), - "notify_type", ObMultiDataSourcePrinter::to_str_notify_type(notify_type), + TRANS_LOG(INFO, "notify MDS", K(ret), K(trans_id_), "notify_type", ObMultiDataSourcePrinter::to_str_notify_type(notify_type), K(log_ts), K(notify_array.count()), K(notify_array), K(total_time)); } @@ -6961,7 +5637,7 @@ int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, } ERRSIM_POINT_DEF(TX_FORCE_WRITE_CLOG) -int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_source_type, +int ObTxCtx::register_multi_data_source(const ObTxDataSourceType data_source_type, const char *buf, const int64_t len, const bool try_lock, @@ -7007,12 +5683,12 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou ret = OB_TRANS_KILLED; } TRANS_LOG(WARN, "tx has decided", K(ret), KPC(this)); - } else if (OB_UNLIKELY(sub_state_.is_force_abort())) { + } else if (OB_UNLIKELY(runtime_state_.is_force_abort())) { ret = OB_TRANS_KILLED; TRANS_LOG(WARN, "tx force aborted due to data incomplete", K(ret), KPC(this)); - } else if (is_follower_()) { - ret = OB_NOT_MASTER; - TRANS_LOG(ERROR, "can not register mds on a follower", K(ret), K(data_source_type), K(len), + } else if (is_for_replay()) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(ERROR, "can not register mds on a replay context", K(ret), K(data_source_type), K(len), KPC(this)); } else if (is_committing_()) { ret = OB_TRANS_HAS_DECIDED; @@ -7025,10 +5701,7 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou TRANS_LOG(WARN, "alloc mds node from the mds_cache_ failed", K(ret), K(mds_cache_), KPC(this)); } else { mds::BufferCtx *buffer_ctx = nullptr; - if (data_source_type > ObTxDataSourceType::BEFORE_VERSION_4_1 - || ObTxDataSourceType::CREATE_TABLET_NEW_MDS == data_source_type - || ObTxDataSourceType::DELETE_TABLET_NEW_MDS == data_source_type - || ObTxDataSourceType::UNBIND_TABLET_NEW_MDS == data_source_type) { + if (!uses_builtin_mds_notifier(data_source_type)) { ret = mds::MdsFactory::create_buffer_ctx(data_source_type, trans_id_, buffer_ctx); } if (OB_FAIL(ret)) { @@ -7071,13 +5744,13 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou // do nothing } else if (OB_SUCCESS != (tmp_ret = submit_log_impl_(ObTxLogType::TX_MULTI_DATA_SOURCE_LOG))) { - if (tmp_ret == OB_NOT_MASTER) { - ret = OB_TRANS_NEED_ROLLBACK; - } else if (tmp_ret == OB_TX_NOLOGCB || tmp_ret == OB_EAGAIN) { + if (tmp_ret == OB_TX_NOLOGCB || tmp_ret == OB_EAGAIN) { ret = OB_SUCCESS; if (register_flag.need_flush_redo_instantly_) { mds_cache_.set_need_retry_submit_mds(true); } + } else { + ret = tmp_ret; } TRANS_LOG(WARN, "submit mds log failed", K(tmp_ret), K(ret), K(register_flag), K(data_source_type), KPC(this)); @@ -7091,11 +5764,10 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou if (OB_FAIL(ret)) { tx_print_guard.get_diff(); - TRANS_LOG(WARN, "register MDS redo in part_ctx failed", K(ret), K(trans_id_), K(ls_id_), - K(data_source_type), K(len), K(register_flag), K(mds_cache_), K(*this), + TRANS_LOG(WARN, "register MDS redo in part_ctx failed", K(ret), K(trans_id_), K(data_source_type), K(len), K(register_flag), K(mds_cache_), K(*this), K(tx_print_guard), K(lbt())); } else if (tx_print_guard.get_diff() > 1 * 1000 * 1000) { - TRANS_LOG(INFO, "register MDS redo in ctx", K(ret), K(trans_id_), K(ls_id_), K(data_source_type), + TRANS_LOG(INFO, "register MDS redo in ctx", K(ret), K(trans_id_), K(data_source_type), K(len), K(register_flag), K(tx_print_guard)); } @@ -7105,7 +5777,7 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou return ret; } -int ObPartTransCtx::del_retain_ctx() +int ObTxCtx::del_retain_ctx() { int ret = OB_SUCCESS; bool need_del = false; @@ -7174,7 +5846,7 @@ int ObPartTransCtx::del_retain_ctx() } -int ObPartTransCtx::submit_pending_log_block_(ObTxLogBlock &log_block, +int ObTxCtx::submit_pending_log_block_(ObTxLogBlock &log_block, memtable::ObRedoLogSubmitHelper &helper, const logservice::ObReplayBarrierType &barrier) { @@ -7182,11 +5854,12 @@ int ObPartTransCtx::submit_pending_log_block_(ObTxLogBlock &log_block, if (log_block.get_cb_arg_array().empty()) { TRANS_LOG(INFO, "no need to submit pending log block because of empty", K(ret), K(trans_id_), - K(ls_id_), K(log_block)); + K(log_block)); } else { bool need_final_cb = false; if (is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_COMMIT_LOG) - || is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_COMMIT_LOG)) { + || is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_ABORT_LOG) + || is_contain(log_block.get_cb_arg_array(), ObTxLogType::TX_CLEAR_LOG)) { need_final_cb = true; } const int64_t replay_hint = trans_id_.get_id(); @@ -7213,7 +5886,7 @@ int ObPartTransCtx::submit_pending_log_block_(ObTxLogBlock &log_block, release_ctx_ref_(); } else if (OB_FAIL(after_submit_log_(log_block, log_cb, &helper))) { } else { - // TRANS_LOG(INFO, "submit prepare log in clog adapter success", K(*log_cb)); + // TRANS_LOG(INFO, "submit pending log block in clog adapter success", K(*log_cb)); log_cb = NULL; } } @@ -7221,7 +5894,7 @@ int ObPartTransCtx::submit_pending_log_block_(ObTxLogBlock &log_block, return ret; } -int ObPartTransCtx::check_status() +int ObTxCtx::check_status() { CtxLockGuard guard(lock_, CtxLockGuard::MODE::ACCESS); return check_status_(); @@ -7243,7 +5916,7 @@ int ObPartTransCtx::check_status() * in order to reuse this routine, do `check txn is active` at first, thus even if * an active txn which has switched into follower also pass these check */ -inline int ObPartTransCtx::check_status_() +inline int ObTxCtx::check_status_() { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_committing_())) { @@ -7257,7 +5930,7 @@ inline int ObPartTransCtx::check_status_() ret = OB_TRANS_KILLED; } TRANS_LOG(WARN, "tx has decided", K(ret), KPC(this)); - } else if (OB_UNLIKELY(sub_state_.is_force_abort())) { + } else if (OB_UNLIKELY(runtime_state_.is_force_abort())) { if (is_trans_expired_()) { ret = OB_TRANS_TIMEOUT; TRANS_LOG(WARN, "tx has decided", K(ret), KPC(this)); @@ -7265,8 +5938,8 @@ inline int ObPartTransCtx::check_status_() ret = OB_TRANS_KILLED; TRANS_LOG(WARN, "tx force aborted due to data incomplete", K(ret), KPC(this)); } - } else if (OB_UNLIKELY(is_follower_())) { - ret = OB_NOT_MASTER; + } else if (OB_UNLIKELY(is_for_replay())) { + ret = OB_STATE_NOT_MATCH; } else if (OB_UNLIKELY(is_exiting_)) { ret = OB_TRANS_IS_EXITING; TRANS_LOG(WARN, "tx is exiting", K(ret), KPC(this)); @@ -7285,7 +5958,7 @@ inline int ObPartTransCtx::check_status_() * 2) acquire memtable ctx's ref * 3) alloc data_scn if not specified */ -int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, +int ObTxCtx::start_access(const ObTxDesc &tx_desc, ObTxSEQ &data_scn, const int16_t branch, const concurrent_control::ObWriteFlag &write_flag) @@ -7324,7 +5997,7 @@ int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, pending_write = ATOMIC_AAF(&pending_write_, 1); // others must wait the first thread of parallel open the write epoch // hence this must be done in lock - if (data_scn.support_branch() && pending_write == 1) { + if (pending_write == 1) { callback_list_idx = mt_ctx_.acquire_callback_list(true); } } @@ -7333,21 +6006,17 @@ int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, // other operations are allowed to out of lock if (OB_SUCC(ret)) { mt_ctx_.inc_ref(); - if (data_scn.support_branch()) { - if (pending_write != 1) { - callback_list_idx = mt_ctx_.acquire_callback_list(false); - } - // remember selected callback_list idx into seq_no - if (data_scn.get_branch() == 0 && alloc && callback_list_idx != 0) { - data_scn.set_branch(callback_list_idx); - } - } else { // OLD version < 4.2.4 - ret = OB_ERR_UNEXPECTED; + if (pending_write != 1) { + callback_list_idx = mt_ctx_.acquire_callback_list(false); + } + // remember selected callback_list idx into seq_no + if (data_scn.get_branch() == 0 && alloc && callback_list_idx != 0) { + data_scn.set_branch(callback_list_idx); } } last_request_ts_ = ObClockGenerator::getClock(); - TRANS_LOG(TRACE, "start_access", K(ret), K(data_scn.support_branch()), K(data_scn), KPC(this)); + TRANS_LOG(TRACE, "start_access", K(ret), K(data_scn), KPC(this)); common::ObTraceIdAdaptor trace_id; trace_id.set(ObCurTraceId::get()); REC_TRANS_TRACE_EXT(tlog_, start_access, @@ -7369,7 +6038,7 @@ int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, * dec ref of memtable context * merge provisional write's callbacks into the total final callback list */ -int ObPartTransCtx::end_access() +int ObTxCtx::end_access() { int ret = OB_SUCCESS; // to reduce lock contention, these operation is out of lock @@ -7385,7 +6054,7 @@ int ObPartTransCtx::end_access() return ret; } -int ObPartTransCtx::check_pending_log_overflow(const int64_t stmt_timeout) +int ObTxCtx::check_pending_log_overflow(const int64_t stmt_timeout) { int ret = OB_SUCCESS; const int64_t MAX_LOCAL_RETRY_US = 1 * 1000 * 1000; // 1s @@ -7469,7 +6138,7 @@ int ObPartTransCtx::check_pending_log_overflow(const int64_t stmt_timeout) * than the sequence passed to start_access. and the last_scn was only set * when start_access was called */ -int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, +int ObTxCtx::rollback_to_savepoint(const int64_t op_sn, ObTxSEQ from_scn, const ObTxSEQ to_scn, const int64_t seq_base) @@ -7480,14 +6149,7 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, if (OB_FAIL(check_status_())) { } else if(is_logging_()) { ret = OB_NEED_RETRY; - // check cur leader and fast fail - int tmp_ret = OB_SUCCESS; bool leader = false; int64_t epoch = 0; - if (OB_TMP_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->get_role(leader, epoch))) { - TRANS_LOG(WARN, "get ls role failed", K(tmp_ret), K(trans_id_), K(ls_id_)); - } else if (!leader) { - ret = OB_NOT_MASTER; - } - TRANS_LOG(WARN, "rollback_to need retry because of logging", K(ret), K(trans_id_), K(ls_id_), K(busy_cbs_.get_size())); + TRANS_LOG(WARN, "rollback_to need retry because of logging", K(ret), K(trans_id_), K(busy_cbs_.get_size())); } else if (op_sn < last_op_sn_) { ret = OB_TRANS_SQL_SEQUENCE_ILLEGAL; } else if (FALSE_IT(last_op_sn_ = op_sn)) { @@ -7496,7 +6158,7 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, ret = OB_NEED_RETRY; TRANS_LOG(WARN, "has pending write, rollback blocked", K(ret), K(to_scn), K(pending_write_), KPC(this)); } else if (last_scn_ <= to_scn) { - TRANS_LOG(INFO, "rollback succeed trivially", K_(trans_id), K_(ls_id), K(op_sn), K(to_scn), K_(last_scn)); + TRANS_LOG(INFO, "rollback succeed trivially", K_(trans_id), K(op_sn), K(to_scn), K_(last_scn)); } else if (!from_scn.is_valid() && // generate from if not specified FALSE_IT(from_scn = to_scn.clone_with_seq(ObSequence::inc_and_get_max_seq_no(), seq_base))) { @@ -7520,7 +6182,7 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, return ret; } -int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, +int ObTxCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, const ObTxSEQ to_scn, const share::SCN replay_scn) { @@ -7541,7 +6203,7 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, ObTxDataGuard update_tx_data_guard; tmp_tx_data_guard.reset(); update_tx_data_guard.reset(); - if (is_follower_()) { /* Follower */ + if (is_for_replay()) { /* Follower */ ObUndoAction undo_action(from_scn, to_scn); // _NOTICE_ must load Undo(s) from TxDataTable before overwriten if (replay_completeness_.is_unknown() && @@ -7551,7 +6213,7 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, } else if (OB_FAIL(replay_undo_action_to_tx_table_(undo_action, replay_scn))) { TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); } - } else if (OB_UNLIKELY(exec_info_.max_submitted_seq_no_ > to_scn)) { /* Leader */ + } else if (OB_UNLIKELY(exec_info_.max_submitted_seq_no_ > to_scn)) { ObTxDataGuard tx_data_guard; ObTxTable *tx_table = nullptr; ctx_tx_data_.get_tx_table(tx_table); @@ -7580,7 +6242,7 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, return ret; } -int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, +int ObTxCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, const ObTxSEQ to_scn) { int ret = OB_SUCCESS; @@ -7590,7 +6252,7 @@ int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, int64_t replay_hint = trans_id_.get_id(); ObUndoStatusNode *undo_node = NULL; logservice::ObReplayBarrierType barrier = logservice::ObReplayBarrierType::NO_NEED_BARRIER; - if (to_scn.support_branch() && is_parallel_logging()) { + if (is_parallel_logging()) { const int16_t branch_id = to_scn.get_branch(); if (branch_id != 0 && to_scn.get_seq() > exec_info_.serial_final_seq_no_.get_seq()) { replay_hint += mt_ctx_.get_tx_seq_replay_idx(to_scn); @@ -7642,14 +6304,14 @@ int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, return ret; } -int ObPartTransCtx::abort(const int reason) +int ObTxCtx::abort(const int reason) { UNUSED(reason); int ret = OB_SUCCESS; CtxLockGuard guard(lock_); - if (OB_UNLIKELY(is_follower_())) { - ret = OB_NOT_MASTER; - TRANS_LOG(WARN, "not master", KR(ret), KPC(this)); + if (OB_UNLIKELY(is_for_replay())) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(WARN, "cannot abort replay context", KR(ret), KPC(this)); } else if (OB_UNLIKELY(is_committing_())) { if (part_trans_action_ == ObPartTransAction::ABORT) { TRANS_LOG(INFO, "tx already aborting", KPC(this)); @@ -7661,7 +6323,7 @@ int ObPartTransCtx::abort(const int reason) TRANS_LOG(INFO, "tx already aborting", KPC(this)); } // } else if (OB_FAIL(prepare_mul_data_source_tx_end_(false))) { - // TRANS_LOG(WARN, "trans abort need retry", K(ret), K(trans_id_), K(ls_id_), K(reason)); + // TRANS_LOG(WARN, "trans abort need retry", K(ret), K(trans_id_), K(reason)); } else { if (OB_FAIL(abort_(reason))) { TRANS_LOG(WARN, "abort_ failed", KR(ret), K(*this)); @@ -7671,7 +6333,7 @@ int ObPartTransCtx::abort(const int reason) return ret; } -int ObPartTransCtx::handle_tx_keepalive_response(const int64_t status) +int ObTxCtx::handle_tx_keepalive_response(const int64_t status) { int ret = OB_SUCCESS; @@ -7683,23 +6345,10 @@ int ObPartTransCtx::handle_tx_keepalive_response(const int64_t status) return ret; } -int ObPartTransCtx::tx_keepalive_response_(const int64_t status) +int ObTxCtx::tx_keepalive_response_(const int64_t status) { int ret = OB_SUCCESS; - if (OB_SWITCHING_TO_FOLLOWER_GRACEFULLY == ERRSIM_DELAY_TX_SUBMIT_LOG) { - return ret; - } else if (ERRSIM_DELAY_TX_SUBMIT_LOG < int(-9999)) { - // reject trans_id and tenant id to contorl trans sate - // ret = process_errsim_for_standby_read_(int(ERRSIM_DELAY_TX_SUBMIT_LOG), - // OB_SWITCHING_TO_FOLLOWER_GRACEFULLY); - if (OB_FAIL(ret)) { - return ret; - TRANS_LOG(INFO, "reject delay tx submit log when response to keepalive", - K(ret), K(ERRSIM_DELAY_TX_SUBMIT_LOG)); - } - } - if ((OB_TRANS_CTX_NOT_EXIST == status || OB_TRANS_ROLLBACKED == status || OB_TRANS_KILLED == status || common::OB_TENANT_NOT_IN_SERVER == status) && can_be_recycled_()) { if (REACH_TIME_INTERVAL(5 * 1000 * 1000)) { @@ -7722,7 +6371,7 @@ int ObPartTransCtx::tx_keepalive_response_(const int64_t status) return ret; } -int ObPartTransCtx::try_alloc_retain_ctx_func_() +int ObTxCtx::try_alloc_retain_ctx_func_() { int ret = OB_SUCCESS; @@ -7738,7 +6387,7 @@ int ObPartTransCtx::try_alloc_retain_ctx_func_() } else if (OB_FALSE_IT(new (retain_ctx_func_ptr_) ObMDSRetainCtxFunctor())) { } - } else if (!is_follower_()) { + } else if (!is_for_replay()) { TRANS_LOG(INFO, "no need to alloc a new retain_ctx_func", K(ret), KP(retain_ctx_func_ptr_), KPC(retain_ctx_func_ptr_), KPC(this)); } @@ -7746,7 +6395,7 @@ int ObPartTransCtx::try_alloc_retain_ctx_func_() return ret; } -int ObPartTransCtx::try_gc_retain_ctx_func_() +int ObTxCtx::try_gc_retain_ctx_func_() { int ret = OB_SUCCESS; @@ -7759,7 +6408,7 @@ int ObPartTransCtx::try_gc_retain_ctx_func_() return ret; } -int ObPartTransCtx::insert_into_retain_ctx_mgr_(RetainCause cause, +int ObTxCtx::insert_into_retain_ctx_mgr_(RetainCause cause, const SCN &log_ts, palf::LSN lsn, bool for_replay) @@ -7814,7 +6463,7 @@ int ObPartTransCtx::insert_into_retain_ctx_mgr_(RetainCause cause, return ret; } -int ObPartTransCtx::do_local_tx_end_(TxEndAction tx_end_action) +int ObTxCtx::do_local_tx_end_(TxEndAction tx_end_action) { int ret = OB_SUCCESS; @@ -7825,17 +6474,17 @@ int ObPartTransCtx::do_local_tx_end_(TxEndAction tx_end_action) switch (tx_end_action) { case TxEndAction::COMMIT_TX: { - if (sub_state_.is_force_abort()) { + if (runtime_state_.is_force_abort()) { if (OB_FAIL(compensate_abort_log_())) { - TRANS_LOG(WARN, "compensate abort log failed", K(ret), K(ls_id_), K(trans_id_), - K(tx_end_action), K(sub_state_)); + TRANS_LOG(WARN, "compensate abort log failed", K(ret), K(trans_id_), + K(tx_end_action), K(runtime_state_)); } else { ret = OB_TRANS_KILLED; } } else { ret = do_local_commit_tx_(); } - // part_trans_action_ will be set as commit in ObPartTransCtx::commit function + // part_trans_action_ will be set as commit in ObTxCtx::commit function break; } case TxEndAction::ABORT_TX: { @@ -7850,7 +6499,7 @@ int ObPartTransCtx::do_local_tx_end_(TxEndAction tx_end_action) break; } case TxEndAction::DELAY_ABORT_TX: { - sub_state_.set_force_abort(); + runtime_state_.set_force_abort(); // NOTE: clean unlog callbacks is requried: // if mvcc-row is too large and can't be serialized, freeze thread // will delay_abort the txn, if don't clean unlog_callbacks @@ -7868,7 +6517,7 @@ int ObPartTransCtx::do_local_tx_end_(TxEndAction tx_end_action) return ret; } -int ObPartTransCtx::do_local_commit_tx_() +int ObTxCtx::do_local_commit_tx_() { int ret = OB_SUCCESS; @@ -7881,8 +6530,8 @@ int ObPartTransCtx::do_local_commit_tx_() } else if (OB_FAIL(submit_log_impl_(ObTxLogType::TX_COMMIT_LOG))) { // log submitting will retry in handle_timeout int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(restart_2pc_trans_timer_())) { - TRANS_LOG(WARN, "restart_2pc_trans_timer_ error", KR(ret), KR(tmp_ret), KPC(this)); + if (OB_TMP_FAIL(restart_commit_retry_timer_())) { + TRANS_LOG(WARN, "restart_commit_retry_timer_ error", KR(ret), KR(tmp_ret), KPC(this)); ret = OB_EAGAIN; } else { ret = OB_SUCCESS; @@ -7892,7 +6541,7 @@ int ObPartTransCtx::do_local_commit_tx_() return ret; } -int ObPartTransCtx::do_local_abort_tx_() +int ObTxCtx::do_local_abort_tx_() { int ret = OB_SUCCESS; @@ -7907,7 +6556,7 @@ int ObPartTransCtx::do_local_abort_tx_() // if (part_trans_action_ < ObPartTransAction::COMMIT) { // part_trans_action_ = ObPartTransAction::ABORT; // } - sub_state_.set_force_abort(); + runtime_state_.set_force_abort(); if (OB_FAIL(on_local_abort_tx_())) { TRANS_LOG(WARN, "local tx abort failed", KR(ret), K(*this)); } @@ -7915,7 +6564,7 @@ int ObPartTransCtx::do_local_abort_tx_() return ret; } -int ObPartTransCtx::do_force_kill_tx_() +int ObTxCtx::do_force_kill_tx_() { int ret = OB_SUCCESS; @@ -7940,7 +6589,7 @@ int ObPartTransCtx::do_force_kill_tx_() if (OB_FAIL(unregister_timeout_task_())) { TRANS_LOG(WARN, "unregister timer task error", KR(ret), "context", *this); } - sub_state_.set_force_abort(); + runtime_state_.set_force_abort(); // Ignore ret set_exiting_(); TRANS_LOG(INFO, "transaction killed success", "context", *this); @@ -7948,14 +6597,14 @@ int ObPartTransCtx::do_force_kill_tx_() return ret; } -int ObPartTransCtx::on_local_commit_tx_() +int ObTxCtx::on_local_commit_tx_() { int ret = OB_SUCCESS; bool need_wait = false; - if (sub_state_.is_gts_waiting()) { + if (runtime_state_.is_gts_waiting()) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected gts waiting flag", K(ret), KPC(this)); + TRANS_LOG(WARN, "unexpected gts waiting flag", KR(ret), KPC(this)); } else if (!OB_UNLIKELY(ctx_tx_data_.get_commit_version().is_valid())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "invalid commit version", K(ret), KPC(this)); @@ -7972,7 +6621,6 @@ int ObPartTransCtx::on_local_commit_tx_() } else if (FALSE_IT(set_durable_state_(ObTxState::COMMIT))) { } else if (FALSE_IT(unregister_timeout_task_())) { - } else if (need_wait) { REC_TRANS_TRACE_EXT2(tlog_, wait_gts_elapse, OB_ID(ref), get_ref()); } @@ -7986,7 +6634,7 @@ int ObPartTransCtx::on_local_commit_tx_() return ret; } -int ObPartTransCtx::after_local_commit_succ_() +int ObTxCtx::after_local_commit_succ_() { int ret = OB_SUCCESS; @@ -8000,7 +6648,7 @@ int ObPartTransCtx::after_local_commit_succ_() return ret; } -int ObPartTransCtx::on_local_abort_tx_() +int ObTxCtx::on_local_abort_tx_() { int ret = OB_SUCCESS; @@ -8031,7 +6679,7 @@ int ObPartTransCtx::on_local_abort_tx_() return ret; } -int ObPartTransCtx::dump_2_text(FILE *fd) +int ObTxCtx::dump_2_text(FILE *fd) { int ret = OB_SUCCESS; @@ -8042,7 +6690,7 @@ int ObPartTransCtx::dump_2_text(FILE *fd) int64_t str_len = to_string(buf, buf_len); - fprintf(fd, "********** ObPartTransCtx ***********\n\n"); + fprintf(fd, "********** ObTxCtx ***********\n\n"); fprintf(fd, "%s\n", buf); ObTxDataGuard tx_data_guard; ctx_tx_data_.get_tx_data(tx_data_guard); @@ -8053,32 +6701,12 @@ int ObPartTransCtx::dump_2_text(FILE *fd) tx_data_ptr->dump_2_text(fd); } - fprintf(fd, "\n********** ObPartTransCtx ***********\n"); - return ret; -} - -int ObPartTransCtx::handle_ask_tx_state_for_4377(bool &is_alive) -{ - int ret = OB_SUCCESS; - CtxLockGuard guard(lock_); - - if (IS_NOT_INIT) { - is_alive = false; - TRANS_LOG(WARN, "ObPartTransCtx not inited"); - } else { - if (exec_info_.state_ == ObTxState::ABORT) { - // Treat aborted txn as not alive to avoid reporting a stale 4377 conflict. - is_alive = false; - } else { - is_alive = true; - } - } - - TRANS_LOG(INFO, "handle ask tx state for 4377", K(ret), KPC(this), K(is_alive)); + fprintf(fd, "\n********** ObTxCtx ***********\n"); return ret; } -int ObPartTransCtx::check_is_aborted_in_tx_data_(const ObTransID tx_id, +// Check whether an old transaction has already aborted in tx data. +int ObTxCtx::check_is_aborted_in_tx_data_(const ObTransID tx_id, bool &is_aborted) { int ret = OB_SUCCESS; @@ -8101,9 +6729,6 @@ int ObPartTransCtx::check_is_aborted_in_tx_data_(const ObTransID tx_id, if (OB_TRANS_CTX_NOT_EXIST == ret) { is_aborted = false; ret = OB_SUCCESS; - } else if (OB_REPLICA_NOT_READABLE == ret) { - is_aborted = false; - ret = OB_SUCCESS; } else { TRANS_LOG(WARN, "get tx state from tx data failed", K(ret), KPC(this)); } @@ -8118,41 +6743,31 @@ int ObPartTransCtx::check_is_aborted_in_tx_data_(const ObTransID tx_id, return ret; } -void ObPartTransCtx::print_first_mvcc_callback_() +void ObTxCtx::print_first_mvcc_callback_() { mt_ctx_.print_first_mvcc_callback(); } -void ObPartTransCtx::post_keepalive_msg_(const int status) +void ObTxCtx::report_write_ctx_status_(const int status, const bool check_tx_status) { - ObTxKeepaliveMsg msg; - msg.cluster_version_ = cluster_version_; - - msg.cluster_id_ = cluster_id_; - msg.request_id_ = ObClockGenerator::getClock(); - msg.tx_id_ = trans_id_; - msg.sender_addr_ = addr_; - msg.sender_ = ls_id_; - msg.receiver_ = share::SCHEDULER_LS; // this just used to pass validation - msg.epoch_ = epoch_; - msg.status_ = status; int ret = OB_SUCCESS; - if (OB_FAIL(rpc_->post_msg(exec_info_.scheduler_, msg))) { - TRANS_LOG(WARN, "post tx keepalive msg fail", K(ret), K(msg), KPC(this)); + int tx_status = OB_SUCCESS; + if (OB_ISNULL(trans_service_)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "trans service is null", K(ret), KPC(this)); + } else if (OB_FAIL(trans_service_->report_write_ctx_status(trans_id_, status, tx_status))) { + TRANS_LOG(WARN, "report write ctx status fail", K(ret), K(status), KPC(this)); + } else if (check_tx_status && OB_FAIL(tx_keepalive_response_(tx_status))) { + TRANS_LOG(WARN, "handle tx status fail", K(ret), K(tx_status), KPC(this)); } } -void ObPartTransCtx::notify_scheduler_tx_killed_(const int kill_reason) +void ObTxCtx::notify_tx_killed_(const int kill_reason) { - if (exec_info_.scheduler_ != addr_) { - TRANS_LOG(INFO, "scheduler is not local address, skip notify scheduler tx killed", - K(exec_info_.scheduler_), K(addr_), K(kill_reason), KPC(this)); - } else { - post_keepalive_msg_(kill_reason); - } + report_write_ctx_status_(kill_reason, false); } -int ObPartTransCtx::submit_redo_log_out(ObTxLogBlock &log_block, +int ObTxCtx::submit_redo_log_out(ObTxLogBlock &log_block, ObTxLogCb *&log_cb, ObRedoLogSubmitHelper &helper, const int64_t replay_hint, @@ -8199,27 +6814,27 @@ int ObPartTransCtx::submit_redo_log_out(ObTxLogBlock &log_block, return ret; } -bool ObPartTransCtx::is_parallel_logging() const +bool ObTxCtx::is_parallel_logging() const { return exec_info_.serial_final_scn_.is_valid(); } -inline bool ObPartTransCtx::has_replay_serial_final_() const +inline bool ObTxCtx::has_replay_serial_final_() const { return exec_info_.serial_final_scn_.is_valid() && exec_info_.max_applied_log_ts_ >= exec_info_.serial_final_scn_; } -int ObPartTransCtx::set_replay_incomplete(const share::SCN log_ts) { +int ObTxCtx::set_replay_incomplete(const share::SCN log_ts) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); if (OB_FAIL(set_replay_completeness_(false, log_ts))) { - TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(ls_id_), K(trans_id_), KP(this)); + TRANS_LOG(WARN, "set replay completeness failed", KR(ret), K(trans_id_), KP(this)); } return ret; } -int ObPartTransCtx::set_replay_completeness_(const bool complete, const SCN replay_scn) +int ObTxCtx::set_replay_completeness_(const bool complete, const SCN replay_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(replay_completeness_.is_unknown())) { @@ -8228,26 +6843,26 @@ int ObPartTransCtx::set_replay_completeness_(const bool complete, const SCN repl TRANS_LOG(WARN, "load Undo(s) from tx-table fail", K(ret), KPC(this)); } else { TRANS_LOG(INFO, "replay from middle, load Undo(s) from tx-table succuess", - K(ret), K_(ls_id), K_(trans_id)); + K(ret), K_(trans_id)); } } if (OB_SUCC(ret)) { replay_completeness_.set(complete); if (!complete) { force_no_need_replay_checksum_(false, share::SCN::invalid_scn()); - TRANS_LOG(INFO, "incomplete replay, set skip checksum", K_(trans_id), K_(ls_id)); + TRANS_LOG(INFO, "incomplete replay, set skip checksum", K_(trans_id)); } } } return ret; } -inline bool ObPartTransCtx::is_support_parallel_replay_() const +inline bool ObTxCtx::is_support_parallel_replay_() const { return cluster_version_accurate_; } -inline int ObPartTransCtx::switch_to_parallel_logging_(const share::SCN serial_final_scn, +inline int ObTxCtx::switch_to_parallel_logging_(const share::SCN serial_final_scn, const ObTxSEQ max_seq_no) { int ret = OB_SUCCESS; @@ -8278,7 +6893,7 @@ inline int ObPartTransCtx::switch_to_parallel_logging_(const share::SCN serial_f return ret; } -inline void ObPartTransCtx::recovery_parallel_logging_() +inline void ObTxCtx::recovery_parallel_logging_() { mt_ctx_.set_parallel_logging(exec_info_.serial_final_scn_, exec_info_.serial_final_seq_no_); if (exec_info_.max_applied_log_ts_ >= exec_info_.serial_final_scn_) { @@ -8289,7 +6904,7 @@ inline void ObPartTransCtx::recovery_parallel_logging_() } } -inline int ObPartTransCtx::correct_cluster_version_(uint64_t cluster_version_in_log) +inline int ObTxCtx::correct_cluster_version_(uint64_t cluster_version_in_log) { int ret = OB_SUCCESS; if (cluster_version_in_log > 0 && cluster_version_ != cluster_version_in_log) { @@ -8304,119 +6919,55 @@ inline int ObPartTransCtx::correct_cluster_version_(uint64_t cluster_version_in_ return ret; } -int ObPartTransCtx::get_stat_for_virtual_table(share::ObLSArray &participants, int &busy_cbs_cnt) +int ObTxCtx::get_stat_for_virtual_table(bool &has_write_state, int &busy_cbs_cnt) { int ret = OB_SUCCESS; if (OB_SUCC(lock_.try_rdlock_ctx())) { - participants.assign(exec_info_.participants_); + has_write_state = exec_info_.has_write_state(); busy_cbs_cnt = busy_cbs_.get_size(); lock_.unlock_ctx(); } return ret; } -int ObPartTransCtx::collect_mview_mds_op(bool &need_collect, ObMViewOpArg &arg) -{ - int ret = OB_SUCCESS; - need_collect = false; - CtxLockGuard guard(lock_); - const int64_t cnt = exec_info_.multi_data_source_.count(); - if (cnt > 0) { - for (int64_t i = cnt - 1; i >= 0; i--) { - if (exec_info_.multi_data_source_[i].get_data_source_type() == ObTxDataSourceType::MVIEW_MDS_OP) { - need_collect = true; - const ObMViewMdsOpCtx *user_ctx = static_cast(exec_info_.multi_data_source_[i].get_buffer_ctx_node().get_ctx()); - if (OB_ISNULL(user_ctx)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "mview mds is null", KR(ret), KPC(this)); - } else if (OB_FAIL(arg.assign(user_ctx->get_arg_const()))) { - TRANS_LOG(WARN, "assign mview op arg failed", KR(ret), KPC(this)); - } - break; - } - } - } - return ret; -} - -int ObPartTransCtx::post_tx_commit_resp_(const int status) +int ObTxCtx::post_tx_commit_resp_(const int status) { int ret = OB_SUCCESS; - bool has_skip = false, use_rpc = true; + bool has_skip = false; const share::SCN commit_version = ctx_tx_data_.get_commit_version(); - // scheduler on this server, direct call - if (exec_info_.scheduler_ == addr_) { - use_rpc = false; - if (!has_callback_scheduler_()) { - if (OB_FAIL(defer_callback_scheduler_(status, commit_version))) { - TRANS_LOG(WARN, "report tx commit result fail", K(ret), K(status), KPC(this)); - } else { -#ifndef NDEBUG - TRANS_LOG(INFO, "report tx commit result to local scheduler succeed", K(status), KP(this)); -#endif - } - } else if (commit_cb_.get_cb_ret() == status) { - has_skip = true; - } else { - // maybe has been callbacked due to switch to follower - // the callback status is not final commit status: - // either OB_NOT_MASTER or OB_SWITCH_TO_FOLLOWER_GRACEFULLY - // in these case should callback the scheduler with final status again - use_rpc = true; - } - } - if (use_rpc) { - ObTxCommitRespMsg msg; - build_tx_common_msg_(SCHEDULER_LS, msg); - msg.commit_version_ = commit_version; - msg.ret_ = status; - if (OB_FAIL(post_msg_(exec_info_.scheduler_, msg))) { - TRANS_LOG(WARN, "rpc post msg failed", K(*this), K(msg)); + if (!has_commit_callback_()) { + if (OB_FAIL(defer_commit_callback_(status, commit_version))) { + TRANS_LOG(WARN, "report tx commit result fail", K(ret), K(status), KPC(this)); } else { #ifndef NDEBUG - TRANS_LOG(INFO, "post tx commit resp successfully", K(status), - K(msg), K(exec_info_.scheduler_), K(*this)); + TRANS_LOG(INFO, "report tx commit result succeed", K(status), KP(this)); #endif } + } else { + has_skip = true; } REC_TRANS_TRACE_EXT(tlog_, response_scheduler, OB_ID(ret), ret, OB_ID(tag1), has_skip, - OB_ID(tag2), use_rpc, OB_ID(status), status, OB_ID(commit_version), commit_version); return ret; } -int ObPartTransCtx::restart_2pc_trans_timer_() +int ObTxCtx::restart_commit_retry_timer_() { int ret = OB_SUCCESS; - trans_2pc_timeout_ = ObServerConfig::get_instance().trx_2pc_retry_interval; + commit_retry_timeout_ = get_commit_retry_interval_us_(); (void)unregister_timeout_task_(); - if (OB_FAIL(register_timeout_task_(trans_2pc_timeout_))) { + if (OB_FAIL(register_timeout_task_(commit_retry_timeout_))) { TRANS_LOG(WARN, "register timeout handler error", KR(ret), "context", *this); } return ret; } -int ObPartTransCtx::set_2pc_participants_(const ObTxCommitParts& participants) -{ - int ret = OB_SUCCESS; - if (exec_info_.participants_.count() > 0) { - TRANS_LOG(WARN, "participants has set before", KPC(this)); - } else { - CONVERT_COMMIT_PARTS_TO_PARTS(participants, exec_info_.participants_); - if (FAILEDx(assign_commit_parts(exec_info_.participants_, - participants))) { - TRANS_LOG(WARN, "set participants error", K(ret), K(participants), KPC(this)); - } - } - return ret; -} - -int ObPartTransCtx::set_2pc_request_id_(const int64_t request_id) +int ObTxCtx::set_commit_request_id_(const int64_t request_id) { int ret = OB_SUCCESS; @@ -8425,82 +6976,7 @@ int ObPartTransCtx::set_2pc_request_id_(const int64_t request_id) return ret; } -int ObPartTransCtx::set_2pc_upstream_(const ObLSID &upstream) -{ - int ret = OB_SUCCESS; - - if (!exec_info_.upstream_.is_valid()) { - // upstream should be fixed during each state in 2pc in order to prevent - // the deadlock in the cycle based tree phase commit. - exec_info_.upstream_ = upstream; - } - - return ret; -} - -int ObPartTransCtx::post_msg_(const ObLSID &receiver, - ObTxMsg &msg) -{ - TRANS_LOG(DEBUG, "post msg", K(receiver), K(msg)); - return rpc_->post_msg(receiver, msg); -} - -int ObPartTransCtx::post_msg_(const ObAddr &receiver, - ObTxMsg &msg) -{ - return rpc_->post_msg(receiver, msg); -} - -void ObPartTransCtx::build_tx_common_msg_(const ObLSID &receiver, - ObTxMsg &msg) -{ - build_tx_common_msg_(receiver, - cluster_version_, - trans_id_, - addr_, - ls_id_, - cluster_id_, - msg); - // fill exec_epoch - for (int64_t idx = 0; idx < exec_info_.commit_parts_.count(); idx++) { - if (exec_info_.commit_parts_.at(idx).ls_id_ == receiver) { - msg.epoch_ = exec_info_.commit_parts_.at(idx).exec_epoch_; - } - } -} - -void ObPartTransCtx::build_tx_common_msg_(const ObTxMsg &recv_msg, - const common::ObAddr &self_addr, - ObTxMsg &msg) -{ - build_tx_common_msg_(recv_msg.get_sender(), - recv_msg.get_cluster_version(), - recv_msg.get_trans_id(), - self_addr, - recv_msg.get_receiver(), - recv_msg.get_cluster_id(), - msg); -} - -void ObPartTransCtx::build_tx_common_msg_(const ObLSID &receiver, - const int64_t cluster_version, - const int64_t tx_id, - const common::ObAddr& self_addr, - const ObLSID &self_ls_id, - const int64_t cluster_id, - ObTxMsg &msg) -{ - msg.receiver_ = receiver; - msg.cluster_version_ = cluster_version; - - msg.tx_id_ = tx_id; - msg.sender_addr_ = self_addr; - msg.sender_ = self_ls_id; - msg.request_id_ = ObTimeUtility::current_time(); - msg.cluster_id_ = cluster_id; -} - -int ObPartTransCtx::supplement_tx_op_if_exist_(const bool for_replay, const SCN replay_scn) +int ObTxCtx::supplement_tx_op_if_exist_(const bool for_replay, const SCN replay_scn) { int ret = OB_SUCCESS; @@ -8526,11 +7002,11 @@ int ObPartTransCtx::supplement_tx_op_if_exist_(const bool for_replay, const SCN OB_FAIL(recover_tx_ctx_from_tx_op_(tmp_tx_data_guard.tx_data()->op_guard_->get_tx_op_list(), replay_scn))) { TRANS_LOG(WARN, "recover tx_ctx from tx_op failed", KR(ret)); } - TRANS_LOG(INFO, "supplement_tx_op_if_exist_", KR(ret), K(trans_id_), K(ls_id_), K(ctx_tx_data_)); + TRANS_LOG(INFO, "supplement_tx_op_if_exist_", KR(ret), K(trans_id_), K(ctx_tx_data_)); return ret; } -int ObPartTransCtx::recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const SCN replay_scn) +int ObTxCtx::recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const SCN replay_scn) { TRANS_LOG(INFO, "recover tx_ctx from_tx_op begin", K(tx_op_list.get_count()), K(replay_scn), KPC(this)); int ret = OB_SUCCESS; @@ -8587,14 +7063,14 @@ int ObPartTransCtx::recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const S } -int ObPartTransCtx::update_local_max_commit_version_(const SCN &commit_version) +int ObTxCtx::update_local_max_commit_version_(const SCN &commit_version) { int ret = OB_SUCCESS; trans_service_->get_tx_version_mgr().update_max_commit_ts(commit_version, false); return ret; } -inline bool ObPartTransCtx::is_support_tx_op_() const +inline bool ObTxCtx::is_support_tx_op_() const { return cluster_version_accurate_; } diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_tx_ctx.h similarity index 77% rename from src/storage/tx/ob_trans_part_ctx.h rename to src/storage/tx/ob_tx_ctx.h index b0b132aa3..0cb81317a 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_tx_ctx.h @@ -14,16 +14,15 @@ * limitations under the License. */ -#ifndef OCEANBASE_TRANSACTION_OB_TRANS_PART_CTX_ -#define OCEANBASE_TRANSACTION_OB_TRANS_PART_CTX_ +#ifndef OCEANBASE_TRANSACTION_OB_TX_CTX_ +#define OCEANBASE_TRANSACTION_OB_TX_CTX_ #include "ob_trans_ctx.h" #include "ob_ts_mgr.h" #include "ob_trans_ctx_mgr.h" #include "ob_ctx_tx_data.h" -#include "lib/container/ob_mask_set2.h" #include "lib/list/ob_dlist.h" -#include "share/ob_ls_id.h" +#include "lib/queue/ob_link.h" #include "logservice/palf/lsn.h" #include "ob_tx_ctx_mds.h" #include @@ -53,12 +52,10 @@ class ObTxLogBlock; enum class ObTxLogType : int64_t; class ObTxRedoLog; class ObTxCommitInfoLog; -class ObTxPrepareLog; class ObTxCommitLog; class ObTxAbortLog; class ObTxClearLog; class ObIRetainCtxCheckFunctor; -struct ObTxMsg; } namespace palf { @@ -103,77 +100,45 @@ int64_t search(const ObIArray &array, fn &equal_func) return search_index; } -template -class EqualToStateInfoFunctor -{ -public: - EqualToStateInfoFunctor(const T &tmp_info) : - tmp_info_(tmp_info) - {} - bool operator()(const T& item) { - bool bool_ret = false; - if (tmp_info_.ls_id_ == item.ls_id_) { - if (tmp_info_.check_info_.is_valid()) { - if (tmp_info_.check_info_ == item.check_info_) { - bool_ret = true; - } - } else { // for old version msg compat - bool_ret = true; - } - } - return bool_ret; - } -private: - const T &tmp_info_; -}; - template class ObTxCtxLogOperator; // participant transaction context -class ObPartTransCtx : public ObTransCtx, - public ObTsCbTask +class ObTxCtx : public ObTransCtx, public common::ObLink { friend class ObTransService; friend class IterateTxStatFunctor; friend class IterateTransStatForKeyFunctor; - friend class StandbyCleanUpFunctor; - friend class MockObTxCtx; friend class ObTxELRHandler; friend class ObIRetainCtxCheckFunctor; friend class memtable::ObRedoLogGenerator; template friend class ObTxCtxLogOperator; public: - ObPartTransCtx() - : ObTransCtx("participant", ObTransCtxType::PARTICIPANT), ObTsCbTask(), + ObTxCtx() + : ObTransCtx(), + common::ObLink(), is_inited_(false), mt_ctx_(), reserve_allocator_("PartCtx"), exec_info_(reserve_allocator_), mds_cache_(reserve_allocator_), has_extra_log_cb_group_(false), has_async_index_redo_(false), reserve_log_cb_group_(true/*is_reserve*/), - extra_cb_group_list_(), - role_state_(TxCtxRoleState::FOLLOWER), - coord_prepare_info_arr_(OB_MALLOC_NORMAL_BLOCK_SIZE, - ModulePageAllocator(reserve_allocator_, "PREPARE_INFO")) + extra_cb_group_list_() { /*reset();*/ } - ~ObPartTransCtx() { destroy(); } + ~ObTxCtx() { destroy(); } void destroy(); - int init(const common::ObAddr &scheduler, - const uint32_t session_id, + int init(const uint32_t session_id, const uint32_t client_sid, const uint32_t associated_session_id, const ObTransID &trans_id, const int64_t trans_expired_time, - const share::ObLSID &ls_id, const uint64_t cluster_version, ObTransService *trans_service, const uint64_t cluster_id, - const int64_t epoch, ObLSTxCtxMgr *ls_ctx_mgr, const bool for_replay, - const PartCtxSource ctx_source, + const TxCtxSource ctx_source, ObXATransID xid); void reset() { } int construct_context(const ObTransMsg &msg); @@ -181,13 +146,14 @@ class ObPartTransCtx : public ObTransCtx, int start_trans(); bool is_inited() const; int handle_timeout(const int64_t delay); + int gts_callback_interrupted(); + int gts_elapse_callback(const share::SCN >s); /* * graceful kill: wait trx finish logging */ int kill(const KillTransArg &arg, ObTxCommitCallback *&cb_list); memtable::ObMemtableCtx *get_memtable_ctx() { return &mt_ctx_; } - int commit(const ObTxCommitParts &parts, - const MonotonicTs &commit_time, + int commit(const MonotonicTs &commit_time, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id); @@ -195,17 +161,8 @@ class ObPartTransCtx : public ObTransCtx, int one_phase_commit_(); int get_prepare_version_if_prepared(bool &is_prepared, share::SCN &prepare_version); const share::SCN get_commit_version() const { return ctx_tx_data_.get_commit_version(); } - uint64_t hash() const { return trans_id_.hash(); } - int gts_callback_interrupted(const int errcode, const share::ObLSID target_ls_id); - int get_gts_callback(const MonotonicTs srr, const share::SCN >s, const MonotonicTs receive_gts_ts); - int gts_elapse_callback(const MonotonicTs srr, const share::SCN >s); - MonotonicTs get_stc() const { return stc_; } - - int64_t get_role_state() const { return role_state_; } - int dump_2_text(FILE *fd); public: - int replay_start_working_log(const share::SCN start_working_ts); int set_trans_app_trace_id_str(const ObString &app_trace_id_str); const ObString &get_trans_app_trace_id_str() const { return trace_info_.get_app_trace_id(); } int check_modify_schema_elapsed(const ObTabletID &tablet_id, @@ -218,7 +175,7 @@ class ObPartTransCtx : public ObTransCtx, int get_memtable_key_arr(ObMemtableKeyArray &memtable_key_arr); uint64_t get_lock_for_read_retry_count() const { return mt_ctx_.get_lock_for_read_retry_count(); } - int check_scheduler_status(); + int check_tx_status(); int remove_callback_for_uncommited_txn(const memtable::ObMemtableSet *memtable_set); int64_t get_trans_mem_total_size() const { return mt_ctx_.get_trans_mem_total_size(); } int check_with_tx_data(ObITxDataCheckFunctor &fn); @@ -240,42 +197,34 @@ class ObPartTransCtx : public ObTransCtx, { mt_ctx_.set_table_lock_killed(); } bool is_table_lock_killed() const; - share::ObLSID get_ls_id() const { return ls_id_; } uint32_t get_session_id() const { return session_id_; } uint32_t get_client_sid() const { return client_sid_; } // for elr bool is_can_elr() const { return can_elr_; } - // tx state check for 4377 - int handle_ask_tx_state_for_4377(bool &is_alive); public: // thread safe int64_t to_string(char* buf, const int64_t buf_len) const; private: // thread unsafe - ON_DEMAND_TO_STRING_KV_("self_ls_id", - ls_id_, - K_(session_id), + ON_DEMAND_TO_STRING_KV_(K_(session_id), K_(associated_session_id), K_(part_trans_action), K_(pending_write), K(extra_cb_group_list_.get_size()), K(busy_cbs_.get_size()), K(ctx_tx_data_), - K(role_state_), K(create_ctx_scn_), "ctx_source", ctx_source_, - K(epoch_), K(replay_completeness_), - "upstream_state", - to_str_tx_state(upstream_state_), + "target_state", + to_str_tx_state(target_state_), K_(rec_log_ts), K_(prev_rec_log_ts), K_(last_request_ts), KP_(block_frozen_memtable), - K_(max_2pc_commit_scn), K(mt_ctx_)); public: @@ -283,15 +232,9 @@ class ObPartTransCtx : public ObTransCtx, static const int64_t RESERVED_MEM_SIZE = 256; private: void default_init_(); - int init_memtable_ctx_(const share::ObLSID &ls_id); - // Please use it carefully, because it only refer to the downstream_state_ - bool is_in_durable_2pc_() const; + int init_memtable_ctx_(); bool is_logging_() const; - // It is decided based on both durable 2pc state and on the fly logging. - // So it can be used safely at any time. - bool is_in_2pc_() const; - // force abort but not submit abort log bool need_force_abort_() const; // force abort but wait abort log_cb @@ -310,8 +253,7 @@ class ObPartTransCtx : public ObTransCtx, int trans_replay_abort_(const share::SCN &final_log_ts); int update_publish_version_(const share::SCN &publish_version, const bool for_replay); bool can_be_recycled_(); - bool need_to_ask_scheduler_status_(); - int check_rs_scheduler_is_alive_(bool &is_alive); + bool need_to_check_tx_status_(); int gc_ctx_(); int common_on_success_(ObTxLogCb * log_cb); @@ -323,15 +265,10 @@ class ObPartTransCtx : public ObTransCtx, // ======================================================== // newly added for 4.0 - int post_msg_(const share::ObLSID &receiver, ObTxMsg &msg); - int post_msg_(const ObAddr &server, ObTxMsg &msg); - bool is_decided() const { return ctx_tx_data_.is_decided(); } // check data is rollbacked either partially or totally // in which case the reader should be failed bool is_data_rollbacked() const { return mt_ctx_.is_tx_rollbacked(); } - int set_scheduler(const common::ObAddr &scheduler); - const common::ObAddr &get_scheduler() const; int on_success(ObTxLogCb *log_cb); int on_failure(ObTxLogCb *log_cb); @@ -375,19 +312,11 @@ class ObPartTransCtx : public ObTransCtx, const int64_t &part_log_no, const bool is_tx_log_queue, const bool pre_barrier); - int replay_active_info(const ObTxActiveInfoLog &active_info_log, - const palf::LSN &offset, - const share::SCN ×tamp, - const int64_t &part_log_no); int replay_commit_info(const ObTxCommitInfoLog &commit_info_log, const palf::LSN &offset, const share::SCN ×tamp, const int64_t &part_log_no, const bool pre_barrier); - int replay_prepare(const ObTxPrepareLog &prepare_log, - const palf::LSN &offset, - const share::SCN ×tamp, - const int64_t &part_log_no); int replay_commit(const ObTxCommitLog &commit_log, const palf::LSN &offset, const share::SCN ×tamp, @@ -430,20 +359,10 @@ class ObPartTransCtx : public ObTransCtx, int correct_cluster_version_(uint64_t cluster_version_in_log); - // leader switch related - bool need_callback_scheduler_(); - int switch_to_follower_forcedly(ObTxCommitCallback *&cb_list); - int switch_to_leader(const share::SCN &start_working_ts); - int switch_to_follower_gracefully(ObTxCommitCallback *&cb_list); - int resume_leader(const share::SCN &start_working_ts); + bool need_commit_callback_(); int supplement_tx_op_if_exist_(const bool for_replay, const share::SCN replay_scn); int recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const share::SCN replay_scn); - void set_role_state(const bool for_replay) - { - role_state_ = for_replay ? TxCtxRoleState::FOLLOWER : TxCtxRoleState::LEADER; - } - int register_multi_data_source(const ObTxDataSourceType type, const char *buf, const int64_t len, @@ -470,8 +389,6 @@ class ObPartTransCtx : public ObTransCtx, int del_retain_ctx(); - int collect_mview_mds_op(bool &need_collect, ObMViewOpArg &arg); - // ======================================================== private: // ======================================================== @@ -490,11 +407,9 @@ class ObPartTransCtx : public ObTransCtx, memtable::ObRedoLogSubmitHelper *redo_helper); int submit_commit_log_(); int submit_abort_log_(); - int submit_prepare_log_(); int submit_clear_log_(); int submit_record_log_(); int submit_redo_commit_info_log_(); - int submit_redo_active_info_log_(); int submit_redo_if_serial_logging_(ObTxLogBlock &log_block, bool &has_redo, memtable::ObRedoLogSubmitHelper &helper); @@ -538,10 +453,6 @@ class ObPartTransCtx : public ObTransCtx, int compensate_abort_log_(); int validate_commit_info_log_(const ObTxCommitInfoLog &commit_info_log); - int switch_log_type_(const ObTwoPhaseCommitLogType &log_type, - ObTxLogType &ret_log_type); - int switch_log_type_(const ObTxLogType ret_log_type, - ObTwoPhaseCommitLogType &log_type); int64_t get_redo_log_no_() const; bool has_persisted_log_() const; @@ -557,7 +468,6 @@ class ObPartTransCtx : public ObTransCtx, void set_dup_table_tx_() { exec_info_.is_dup_tx_ = true; - exec_info_.trans_type_ = TransType::DIST_TRANS; } int do_local_tx_end_(TxEndAction tx_end_action); @@ -583,15 +493,11 @@ class ObPartTransCtx : public ObTransCtx, const share::SCN &log_ts, const share::SCN &commit_version); int replace_tx_data_with_backup_(const ObTxDataBackup &backup, share::SCN log_ts_ns); - int check_trans_type_for_replay_(const int32_t &trans_type, const share::SCN &commit_log_ts); void set_durable_state_(const ObTxState state) { exec_info_.state_ = state; } ObTxState get_durable_state_() const { return exec_info_.state_; } - bool is_2pc_logging_() const - { return sub_state_.is_state_log_submitting() || sub_state_.is_gts_waiting(); } - int notify_table_lock_(const SCN &log_ts, const bool for_replay, const ObTxBufferNodeArray ¬ify_array, @@ -637,11 +543,6 @@ class ObPartTransCtx : public ObTransCtx, int prepare_mul_data_source_tx_end_(bool is_commit); - int errism_submit_prepare_log_(); - int replay_redo_in_ctx_compat_(const ObTxRedoLog &redo_log, - const palf::LSN &offset, - const share::SCN ×tamp, - const int64_t &part_log_no); bool is_support_parallel_replay_() const; int set_replay_completeness_(const bool complete, const share::SCN replay_scn); int errsim_notify_mds_(); @@ -653,7 +554,7 @@ class ObPartTransCtx : public ObTransCtx, virtual int update_local_max_commit_version_(const share::SCN &commit_version); private: - int init_log_cbs_(const share::ObLSID&ls_id, const ObTransID &tx_id); + int init_log_cbs_(const ObTransID &tx_id); int extend_log_cbs_(ObTxLogCb *&log_cb); int extend_log_cb_group_(); void reset_log_cb_list_(common::ObDList &cb_list); @@ -665,18 +566,12 @@ class ObPartTransCtx : public ObTransCtx, int get_prev_log_lsn_(const ObTxLogBlock &log_block, ObTxPrevLogType &prev_log_type, palf::LSN &lsn); int set_start_scn_in_commit_log_(ObTxCommitLog &commit_log); - // int init_tx_data_(const share::ObLSID&ls_id, const ObTransID &tx_id); - bool is_local_tx_() const { return TransType::SP_TRANS == exec_info_.trans_type_; } - void set_trans_type_(int64_t trans_type) { exec_info_.trans_type_ = trans_type; } - int set_scheduler_(const common::ObAddr &scheduler); int check_replay_avaliable_(const palf::LSN &offset, const share::SCN ×tamp, const int64_t &part_log_no, bool &need_replay); - bool is_leader_() const { return TxCtxRoleState::LEADER == role_state_; } // inaccurate state when switch leader - bool is_follower_() const { return TxCtxRoleState::FOLLOWER == role_state_; } //inaccurate state when switch leader int update_rec_log_ts_(bool for_replay, const share::SCN &rec_log_ts); int refresh_rec_log_ts_(); int get_tx_ctx_table_info_(ObTxCtxTableInfo &info); @@ -688,36 +583,10 @@ class ObPartTransCtx : public ObTransCtx, // ======================================================== private: - int set_2pc_upstream_(const share::ObLSID&upstream); - int set_2pc_participants_(const ObTxCommitParts &participants); - int set_2pc_request_id_(const int64_t request_id); - int update_2pc_prepare_version_(const share::SCN &prepare_version); - int merge_prepare_log_info_(const ObLSLogInfoArray &info_array); - int merge_prepare_log_info_(const ObLSLogInfo &prepare_info); - int set_2pc_commit_version_(const share::SCN &commit_version); - int find_participant_id_(const share::ObLSID&participant, - int64_t &participant_id); + int set_commit_request_id_(const int64_t request_id); int post_tx_commit_resp_(const int status); - void build_tx_common_msg_(const share::ObLSID&receiver, - ObTxMsg &msg); - static void build_tx_common_msg_(const ObTxMsg &recv_msg, - const common::ObAddr &self_addr, - ObTxMsg &msg); - static void build_tx_common_msg_(const share::ObLSID &receiver, - const int64_t cluster_version, - const int64_t tx_id, - const common::ObAddr& self_addr, - const share::ObLSID &self_ls_id, - const int64_t cluster_id, - ObTxMsg &msg); - static int post_orphan_msg_(const ObTwoPhaseCommitMsgType &msg_type, - const ObTxMsg &recv_msg, - const common::ObAddr &self_addr, - ObITransRpc* rpc, - const bool ls_deleted); - static int get_max_decided_scn_(const share::ObLSID &ls_id, share::SCN &scn); - int get_stat_for_virtual_table(share::ObLSArray &participants, int &busy_cbs_cnt); + int get_stat_for_virtual_table(bool &has_write_state, int &busy_cbs_cnt); int submit_log_if_allow(const char *buf, const int64_t size, @@ -728,15 +597,13 @@ class ObPartTransCtx : public ObTransCtx, private: // int tx_end_(const bool commit, const int64_t commit_version); - void register_gts_callback_(); - int restart_2pc_trans_timer_(); + int restart_commit_retry_timer_(); // ============================ TX COMMITTER END ============================ public: /* * check_status - check txn ctx is ready to acccept access * - * the precoditions include: - * is leader, un-terminated, not changing leader, etc. + * the preconditions include an active, writable local context. */ int check_status(); /* @@ -765,18 +632,17 @@ class ObPartTransCtx : public ObTransCtx, bool fast_check_need_submit_redo_for_freeze_() const; int check_status_(); int tx_keepalive_response_(const int64_t status); - void post_keepalive_msg_(const int status); - void notify_scheduler_tx_killed_(const int kill_reason); + void report_write_ctx_status_(const int status, const bool check_tx_status); + void notify_tx_killed_(const int kill_reason); int rollback_to_savepoint_(const ObTxSEQ from_scn, const ObTxSEQ to_scn, const share::SCN replay_scn = share::SCN::invalid_scn()); int submit_rollback_to_log_(const ObTxSEQ from_scn, const ObTxSEQ to_scn); - int check_ls_state_(const SCN &snapshot, const ObLSID &ls_id, const ObStandbyCheckInfo &check_info); int submit_redo_log_for_freeze_(bool &try_submit, const uint32_t freeze_clock); void print_first_mvcc_callback_(); - void set_upstream_state(ObTxState state) { upstream_state_ = state; } + void set_target_state(ObTxState state) { target_state_ = state; } public: int prepare_for_submit_redo(ObTxLogCb *&log_cb, @@ -789,20 +655,17 @@ class ObPartTransCtx : public ObTransCtx, const bool has_hold_ctx_lock, share::SCN &submitted_scn); bool is_parallel_logging() const; - int assign_commit_parts(const share::ObLSArray &log_participants, - const ObTxCommitParts &log_commit_parts); - ObTxState get_upstream_state() const { return upstream_state_; } + ObTxState get_target_state() const { return target_state_; } ObTxState get_downstream_state() const { return exec_info_.state_; } private: - DISALLOW_COPY_AND_ASSIGN(ObPartTransCtx); + DISALLOW_COPY_AND_ASSIGN(ObTxCtx); private: //0x0078746374726170 means reset partctx static const int64_t PART_CTX_MAGIC_NUM = 0x0078746374726170; static const int64_t REPLAY_PRINT_TRACE_THRESHOLD = 10 * 1000; // 10 ms static const int64_t REDO_SYNC_TASK_RETRY_INTERVAL_US = 10 * 1000; // 10ms - static const int64_t PRE_COMMIT_TASK_RETRY_INTERVAL_US = 5 * 1000; // 5ms static const int64_t END_STMT_SLEEP_US = 10 * 1000; // 10ms static const int64_t MAX_END_STMT_RETRY_TIMES = 100; static const uint64_t MAX_PREV_LOG_IDS_COUNT = 1024; @@ -814,7 +677,7 @@ class ObPartTransCtx : public ObTransCtx, share::SCN end_log_ts_; int64_t stmt_expired_time_; - int64_t last_ask_scheduler_status_ts_; + int64_t last_check_tx_status_ts_; int64_t cur_query_start_time_; // when cluster_version is unknown at ctx created time, will choice // CLUSTER_CURRENT_VERSION, which may not the real cluster_version @@ -827,9 +690,6 @@ class ObPartTransCtx : public ObTransCtx, */ // number of in-progress access int pending_write_; - // LogHandler's epoch at which this part_ctx was created, used to detect - // participant amnesia: crash and then recreated by obsolete message or operation - int64_t epoch_; // latest operation sequence no, used to detect duplicate operation int64_t last_op_sn_; // data sequence no of latest access @@ -848,8 +708,8 @@ class ObPartTransCtx : public ObTransCtx, // redo log callbacked. ObTxMDSCache mds_cache_; ObIRetainCtxCheckFunctor *retain_ctx_func_ptr_; - // sub_state_ is volatile - ObTxSubState sub_state_; + // runtime_state_ is volatile + ObTxRuntimeState runtime_state_; bool has_extra_log_cb_group_; // Set when DML writes to a table that has async indexes. @@ -882,9 +742,6 @@ class ObPartTransCtx : public ObTransCtx, share::SCN rec_log_ts_; share::SCN prev_rec_log_ts_; bool is_ctx_table_merged_; - // trace_info_ - int64_t role_state_; - // +-------------------+ +---------------------------------+ +-------+ +-----------------+ +----------------------+ // | tx_ctx A exiting | | | | | | replay from | | | // | start_log_ts = n | recover_ts = n | remove from tx_ctx_table & dump | recover_ts = n+10 | crash | | min_ckpt_ts n+m | | tx_ctx is incomplete | @@ -903,20 +760,13 @@ class ObPartTransCtx : public ObTransCtx, // set true when submitting redo log for freezing and reset after freezing bool is_submitting_redo_log_for_freeze_; share::SCN create_ctx_scn_; // replay or recover debug - PartCtxSource ctx_source_; // For CDC - prev_lsn - - share::SCN start_working_log_ts_; + TxCtxSource ctx_source_; // For CDC - prev_lsn int16_t retain_cause_; - ObTxState upstream_state_; - share::SCN max_2pc_commit_scn_; - ObLSLogInfoArray coord_prepare_info_arr_; - // tmp scheduler addr is used to post response for the second phase of xa commit/rollback - common::ObAddr tmp_scheduler_; + ObTxState target_state_; // this is used to denote the time of last request including start_access, commit, rollback // this is a tempoary variable which is set to now by default - // therefore, if a follower switchs to leader, the variable is set to now int64_t last_request_ts_; // ======================================================== @@ -929,13 +779,13 @@ STATIC_ASSERT(OB_TX_MAX_LOG_CBS >= PREALLOC_LOG_CALLBACK_COUNT && #if defined(__x86_64__) /* uncomment this block to error messaging real size template struct size_of_xxx_; -static size_of_xxx_ _x; +static size_of_xxx_ _x; */ // orinally 11264 -> 10000 -> 11264(for 32 log_cbs) -//STATIC_ASSERT(sizeof(ObPartTransCtx) < 15000, "ObPartTransCtx is too big "); +//STATIC_ASSERT(sizeof(ObTxCtx) < 15000, "ObTxCtx is too big "); #endif } // transaction } // oceanbase -#endif // OCEANBASE_TRANSACTION_OB_TRANS_PART_CTX_ +#endif // OCEANBASE_TRANSACTION_OB_TX_CTX_ diff --git a/src/storage/tx/ob_tx_ctx_log_cb_helper.cpp b/src/storage/tx/ob_tx_ctx_log_cb_helper.cpp index aadc22c13..acec87495 100644 --- a/src/storage/tx/ob_tx_ctx_log_cb_helper.cpp +++ b/src/storage/tx/ob_tx_ctx_log_cb_helper.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase @@ -23,25 +23,25 @@ namespace oceanbase namespace transaction { -int ObPartTransCtx::init_log_cbs_(const share::ObLSID &ls_id, const ObTransID &tx_id) +int ObTxCtx::init_log_cbs_(const ObTransID &tx_id) { int ret = OB_SUCCESS; if (OB_FAIL(reserve_log_cb_group_.check_and_reset_log_cbs(false))) { TRANS_LOG(ERROR, "reset reserve_log_cb_group failed in the init function", K(ret), K(tx_id), - K(ls_id_), K(reserve_log_cb_group_)); + K(reserve_log_cb_group_)); } else if (OB_FAIL(reserve_log_cb_group_.init(ObTxLogCbGroup::RESERVED_LOG_CB_GROUP_NO))) { - TRANS_LOG(WARN, "init a log cb group failed", K(ret), K(ls_id), K(tx_id), + TRANS_LOG(WARN, "init a log cb group failed", K(ret), K(tx_id), K(reserve_log_cb_group_)); } else if (OB_FAIL(reserve_log_cb_group_.occupy_by_tx(this))) { - TRANS_LOG(WARN, "set tx id in log cb group failed", K(ret), K(ls_id), K(tx_id), + TRANS_LOG(WARN, "set tx id in log cb group failed", K(ret), K(tx_id), K(reserve_log_cb_group_)); } else { ATOMIC_STORE(&has_extra_log_cb_group_, false); ObSpinLockGuard guard(log_cb_lock_); for (int i = 0; i < ObTxLogCbGroup::MAX_LOG_CB_COUNT_IN_GROUP; i++) { if (i != ObTxLogCbGroup::FREEZE_LOG_CB_INDEX) { - TRANS_LOG(DEBUG, "init reserved log cb into free_list", K(ret), K(ls_id), K(tx_id), K(i), + TRANS_LOG(DEBUG, "init reserved log cb into free_list", K(ret), K(tx_id), K(i), KPC(reserve_log_cb_group_.get_log_cb_by_index(i))); free_cbs_.add_last(reserve_log_cb_group_.get_log_cb_by_index(i)); } @@ -51,7 +51,7 @@ int ObPartTransCtx::init_log_cbs_(const share::ObLSID &ls_id, const ObTransID &t return ret; } -int ObPartTransCtx::extend_log_cb_group_() +int ObTxCtx::extend_log_cb_group_() { int ret = OB_SUCCESS; @@ -59,12 +59,11 @@ int ObPartTransCtx::extend_log_cb_group_() ObTxLogCbGroup *group_ptr = nullptr; if (OB_FAIL( get_ls_tx_ctx_mgr()->get_log_cb_pool_mgr().acquire_idle_log_cb_group(group_ptr, this))) { - TRANS_LOG(WARN, "acquire a idle log cb group failed", K(ret), KPC(group_ptr), K(trans_id_), - K(ls_id_)); + TRANS_LOG(WARN, "acquire a idle log cb group failed", K(ret), KPC(group_ptr), K(trans_id_)); } else if (false == (extra_cb_group_list_.add_last(group_ptr))) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "insert into extra_cb_group_list_ failed", K(ret), KPC(group_ptr), - K(trans_id_), K(ls_id_)); + K(trans_id_)); } else { ATOMIC_STORE(&has_extra_log_cb_group_, true); for (int i = 0; i < ObTxLogCbGroup::MAX_LOG_CB_COUNT_IN_GROUP; i++) { @@ -72,14 +71,14 @@ int ObPartTransCtx::extend_log_cb_group_() } } - TRANS_LOG(INFO, "extend a log cb group", K(ret), K(trans_id_), K(ls_id_), KPC(group_ptr)); + TRANS_LOG(INFO, "extend a log cb group", K(ret), K(trans_id_), KPC(group_ptr)); return ret; } -void ObPartTransCtx::reset_log_cb_list_(common::ObDList &cb_list) { cb_list.clear(); } +void ObTxCtx::reset_log_cb_list_(common::ObDList &cb_list) { cb_list.clear(); } -void ObPartTransCtx::reset_log_cbs_() +void ObTxCtx::reset_log_cbs_() { int tmp_ret = OB_SUCCESS; @@ -99,12 +98,12 @@ void ObPartTransCtx::reset_log_cbs_() KPC(this)); } if (OB_SUCCESS == tmp_ret) { - get_ls_tx_ctx_mgr()->get_log_cb_pool_mgr().dec_ls_occupying_cnt(); + get_ls_tx_ctx_mgr()->get_log_cb_pool_mgr().dec_occupying_cnt(); } } } -int ObPartTransCtx::prepare_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) +int ObTxCtx::prepare_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) { int ret = OB_SUCCESS; if (OB_FAIL(get_log_cb_(need_freeze_cb, log_cb)) && REACH_TIME_INTERVAL(100 * 1000)) { @@ -113,15 +112,14 @@ int ObPartTransCtx::prepare_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_c return ret; } -int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) +int ObTxCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; if (OB_NOT_NULL(log_cb)) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid log cb", K(ret), K(need_freeze_cb), KP(log_cb), K(trans_id_), - K(ls_id_)); + TRANS_LOG(WARN, "invalid log cb", K(ret), K(need_freeze_cb), KP(log_cb), K(trans_id_)); } else { if (need_freeze_cb) { @@ -135,7 +133,7 @@ int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) TRANS_LOG(WARN, "the freeze log cb is busy", K(ret), KPC(tmp_cb), K(reserve_log_cb_group_)); } else { log_cb = tmp_cb; - TRANS_LOG(DEBUG, "get a freeze log cb", K(ret), K(trans_id_), K(ls_id_), KPC(log_cb), + TRANS_LOG(DEBUG, "get a freeze log cb", K(ret), K(trans_id_), KPC(log_cb), K(reserve_log_cb_group_)); } } @@ -148,15 +146,14 @@ int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) true ? GCONF._trx_max_log_cb_limit : 16; if (busy_cbs_cnt < trx_max_log_cb_limit || trx_max_log_cb_limit <= 0) { if (OB_TMP_FAIL(extend_log_cb_group_())) { - TRANS_LOG(WARN, "extend a log cb group failed", K(ret), K(tmp_ret), K(trans_id_), - K(ls_id_)); + TRANS_LOG(WARN, "extend a log cb group failed", K(ret), K(tmp_ret), K(trans_id_)); } else { TRANS_LOG(INFO, "extend log cb group success", K(ret), K(tmp_ret), K(trans_id_), - K(ls_id_), K(busy_cbs_cnt), K(busy_cbs_.get_size()), K(free_cbs_.get_size())); + K(busy_cbs_cnt), K(busy_cbs_.get_size()), K(free_cbs_.get_size())); } } else if (EXECUTE_COUNT_PER_SEC(10)) { TRANS_LOG(INFO, "The configured limit of log_cbs has been reached", K(ret), K(tmp_ret), - K(trans_id_), K(ls_id_), K(busy_cbs_cnt), K(trx_max_log_cb_limit), + K(trans_id_), K(busy_cbs_cnt), K(trx_max_log_cb_limit), K(free_cbs_.get_size())); } } @@ -165,7 +162,7 @@ int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) ret = OB_TX_NOLOGCB; TRANS_LOG(WARN, "no free cbs in ctx", KR(ret), K(free_cbs_.get_size()), K(*this)); } else { - TRANS_LOG(DEBUG, "get a extra log cb", K(ret), K(trans_id_), K(ls_id_), KPC(log_cb), + TRANS_LOG(DEBUG, "get a extra log cb", K(ret), K(trans_id_), KPC(log_cb), K(reserve_log_cb_group_)); } } @@ -173,7 +170,7 @@ int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) if (OB_SUCC(ret)) { if (OB_ISNULL(log_cb)) { ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected log callback", K(ret), K(trans_id_), K(ls_id_), KPC(log_cb)); + TRANS_LOG(WARN, "unexpected log callback", K(ret), K(trans_id_), KPC(log_cb)); } else { log_cb->reuse(); log_cb->set_busy(); @@ -183,14 +180,14 @@ int ObPartTransCtx::get_log_cb_(const bool need_freeze_cb, ObTxLogCb *&log_cb) return ret; } -int ObPartTransCtx::return_redo_log_cb(ObTxLogCb *log_cb) +int ObTxCtx::return_redo_log_cb(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; ret = return_log_cb_(log_cb); return ret; } -int ObPartTransCtx::return_log_cb_(ObTxLogCb *log_cb, bool release_final_cb) +int ObTxCtx::return_log_cb_(ObTxLogCb *log_cb, bool release_final_cb) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_ctx_mds.cpp b/src/storage/tx/ob_tx_ctx_mds.cpp index 28d65afb1..aac5303c8 100644 --- a/src/storage/tx/ob_tx_ctx_mds.cpp +++ b/src/storage/tx/ob_tx_ctx_mds.cpp @@ -15,20 +15,19 @@ */ #include "ob_tx_ctx_mds.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { namespace transaction { -int ObTxMDSCache::init(const share::ObLSID ls_id, const ObTransID tx_id) +int ObTxMDSCache::init(const ObTransID tx_id) { int ret = OB_SUCCESS; #ifdef ENABLE_DEBUG_LOG owner_id_ = 1UL; - ls_id_ = ls_id; tx_id_ = tx_id; record_mem_ret_ = OB_SUCCESS; @@ -47,7 +46,6 @@ void ObTxMDSCache::reset() max_register_no_ = 0; #ifdef ENABLE_DEBUG_LOG owner_id_ = 0; - ls_id_.reset(); tx_id_.reset(); mem_stat_hash_.destroy(); record_mem_ret_ = OB_SUCCESS; @@ -68,14 +66,13 @@ void ObTxMDSCache::destroy() #ifdef ENABLE_DEBUG_LOG owner_id_ = 0; - ls_id_.reset(); tx_id_.reset(); mem_stat_hash_.destroy(); record_mem_ret_ = OB_SUCCESS; #endif } -int ObTxMDSCache::alloc_mds_node(const ObPartTransCtx *tx_ctx, +int ObTxMDSCache::alloc_mds_node(const ObTxCtx *tx_ctx, const char *buf, const int64_t buf_len, common::ObString &data, @@ -109,15 +106,14 @@ int ObTxMDSCache::alloc_mds_node(const ObPartTransCtx *tx_ctx, ObMemAttr attr(owner_id_, "MdsMemHash"); if (OB_TMP_FAIL(mem_stat_hash_.create(16, attr, attr))) { record_mem_ret_ = tmp_ret; - TRANS_LOG(WARN, "create mds mem stat failed", K(ret), K(tmp_ret), K(owner_id_), K(ls_id_), - K(tx_id_), K(record_mem_ret_)); + TRANS_LOG(WARN, "create mds mem stat failed", K(ret), K(tmp_ret), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } } if (OB_SUCCESS == tmp_ret) { if (OB_TMP_FAIL(mem_stat_hash_.get_refactored(cur_register_no, tmp_mem_stat))) { if (OB_HASH_NOT_EXIST != tmp_ret) { TRANS_LOG(ERROR, "get tmp_mem_stat from mem_stat_hash failed", K(ret), K(tmp_ret), - K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(ls_id_), K(tx_id_), + K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } tmp_mem_stat.reset(); @@ -129,7 +125,7 @@ int ObTxMDSCache::alloc_mds_node(const ObPartTransCtx *tx_ctx, if (OB_TMP_FAIL(mem_stat_hash_.set_refactored(cur_register_no, tmp_mem_stat, 1))) { record_mem_ret_ = tmp_ret; TRANS_LOG(WARN, "insert mem_stat_ into hash table failed", K(ret), K(tmp_ret), - K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(ls_id_), K(tx_id_), + K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } } @@ -157,12 +153,11 @@ void ObTxMDSCache::free_mds_node(common::ObString &data, uint64_t register_no) if (OB_TMP_FAIL(mem_stat_hash_.get_refactored(cur_register_no, tmp_mem_stat))) { if (record_mem_ret_ != OB_SUCCESS) { TRANS_LOG_RET(WARN, tmp_ret, "get tmp_mem_stat from mem_stat_hash failed", K(ret), K(tmp_ret), - K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(ls_id_), K(tx_id_), + K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } else { TRANS_LOG_RET(ERROR, tmp_ret, "get tmp_mem_stat from mem_stat_hash failed", K(ret), - K(tmp_ret), K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(ls_id_), - K(tx_id_), K(record_mem_ret_)); + K(tmp_ret), K(cur_register_no), K(tmp_mem_stat), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } tmp_mem_stat.reset(); } @@ -173,7 +168,7 @@ void ObTxMDSCache::free_mds_node(common::ObString &data, uint64_t register_no) if (OB_TMP_FAIL(mem_stat_hash_.erase_refactored(cur_register_no))) { TRANS_LOG_RET(ERROR, tmp_ret, "insert mem_stat_ into hash table failed", K(ret), K(tmp_ret), K(cur_register_no), K(tmp_mem_stat), K(tmp_mem_stat), K(owner_id_), - K(ls_id_), K(tx_id_), K(record_mem_ret_)); + K(tx_id_), K(record_mem_ret_)); } } } @@ -191,10 +186,10 @@ bool ObTxMDSCache::is_mem_leak() iter++) { if (record_mem_ret_ != OB_SUCCESS) { TRANS_LOG_RET(WARN, OB_ERR_UNEXPECTED, "mds node mem leak", K(iter->first), K(iter->second), - K(owner_id_), K(ls_id_), K(tx_id_), K(record_mem_ret_)); + K(owner_id_), K(tx_id_), K(record_mem_ret_)); } else { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "mds node mem leak", K(iter->first), - K(iter->second), K(owner_id_), K(ls_id_), K(tx_id_), K(record_mem_ret_)); + K(iter->second), K(owner_id_), K(tx_id_), K(record_mem_ret_)); } } } @@ -255,7 +250,7 @@ int ObTxMDSCache::rollback_last_mds_node() return ret; } -int ObTxMDSCache::fill_mds_log(ObPartTransCtx *ctx, +int ObTxMDSCache::fill_mds_log(ObTxCtx *ctx, ObTxMultiDataSourceLog &mds_log, ObTxMDSRange &mds_range, logservice::ObReplayBarrierType &barrier_flag, @@ -484,7 +479,7 @@ void ObTxMDSRange::reset() // start_iter_ = ObTxBufferNodeList::iterator(); // } -int ObTxMDSRange::init(ObPartTransCtx *tx_ctx) +int ObTxMDSRange::init(ObTxCtx *tx_ctx) { int ret = OB_SUCCESS; if (OB_NOT_NULL(tx_ctx_)) { diff --git a/src/storage/tx/ob_tx_ctx_mds.h b/src/storage/tx/ob_tx_ctx_mds.h index dcb05bc29..e055ce1c0 100644 --- a/src/storage/tx/ob_tx_ctx_mds.h +++ b/src/storage/tx/ob_tx_ctx_mds.h @@ -59,11 +59,11 @@ class ObTxMDSCache { reset(); } - int init(const share::ObLSID ls_id, const ObTransID tx_id); + int init(const ObTransID tx_id); void reset(); void destroy(); - int alloc_mds_node(const ObPartTransCtx *tx_ctx, + int alloc_mds_node(const ObTxCtx *tx_ctx, const char *buf, const int64_t buf_len, common::ObString &data, @@ -76,7 +76,7 @@ class ObTxMDSCache int try_recover_max_register_no(const ObTxBufferNodeArray & node_array); int insert_mds_node(ObTxBufferNode &buf_node); int rollback_last_mds_node(); - int fill_mds_log(ObPartTransCtx *ctx, + int fill_mds_log(ObTxCtx *ctx, ObTxMultiDataSourceLog &mds_log, ObTxMDSRange &mds_range, logservice::ObReplayBarrierType &barrier_flag, @@ -119,7 +119,6 @@ class ObTxMDSCache ObTxBufferNodeArray final_notify_array_; #ifdef ENABLE_DEBUG_LOG - share::ObLSID ls_id_; ObTransID tx_id_; int record_mem_ret_; @@ -134,7 +133,7 @@ class ObTxMDSRange void reset(); // void clear(); - int init(ObPartTransCtx *tx_ctx); + int init(ObTxCtx *tx_ctx); int update_range(ObTxBufferNodeList::iterator iter); int move_from_cache_to_arr(ObTxMDSCache &mds_cache, ObTxBufferNodeArray &mds_durable_arr); @@ -153,7 +152,7 @@ class ObTxMDSRange private: ObTxBufferNodeArray range_array_; - ObPartTransCtx *tx_ctx_; + ObTxCtx *tx_ctx_; // ObTxBufferNodeList *list_ptr_; // ObTxBufferNodeList::iterator start_iter_; diff --git a/src/storage/tx/ob_tx_data_functor.cpp b/src/storage/tx/ob_tx_data_functor.cpp index 542709c14..1914d175d 100644 --- a/src/storage/tx/ob_tx_data_functor.cpp +++ b/src/storage/tx/ob_tx_data_functor.cpp @@ -304,9 +304,9 @@ int LockForReadFunctor::inner_lock_for_read(const ObTxData &tx_data, ObTxCCCtx * // whether txn is prepared, because the state in exec_info will not be // updated as prepared until log is applied and the application is // asynchronous. So we need use version instead of state as judgement and - // mark it whenever we submit the commit/prepare log(using before_prepare) + // mark it whenever we submit the commit log(using before_prepare) if (tx_cc_ctx->prepare_version_.is_max()) { - // Case 2.2.1: data is not in 2pc state, so the prepare version and + // Case 2.2.1: data is not in prepared state, so the prepare version and // commit version of the data must be bigger than the read txn's // snapshot version, so we cannot read it and trans version is // unnecessary for the running txn @@ -391,7 +391,7 @@ int LockForReadFunctor::operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx // rollback_to) and the data for this txn is already dumped as sstable. In // such cases, reads will experience exceptions that the txn is in a RUNNING // state without tx_ctx. Therefore, we need to tolerate this exception. - ret = OB_REPLICA_NOT_READABLE; + ret = OB_EAGAIN; STORAGE_LOG(WARN, "lock for read meet stale data", KR(ret)); } else { for (int32_t i = 0; OB_ERR_SHARED_LOCK_CONFLICT == ret; i++) { @@ -448,19 +448,6 @@ int LockForReadFunctor::check_clog_disk_full_() return ret; } -int LockForReadFunctor::check_for_standby(const transaction::ObTransID &tx_id) -{ - int ret = OB_SUCCESS; - if (OB_SUCC(share::g_mp->trans_service()->check_for_standby(ls_id_, - tx_id, - lock_for_read_arg_.mvcc_acc_ctx_.snapshot_.version_, - can_read_, - trans_version_))) { - lock_for_read_arg_.mvcc_acc_ctx_.is_standby_read_ = true; - } - return ret; -} - int CleanoutTxStateFunctor::operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_data_functor.h b/src/storage/tx/ob_tx_data_functor.h index 1d7254319..591fde08a 100644 --- a/src/storage/tx/ob_tx_data_functor.h +++ b/src/storage/tx/ob_tx_data_functor.h @@ -186,21 +186,18 @@ class LockForReadFunctor : public ObITxDataCheckFunctor LockForReadFunctor(const transaction::ObLockForReadArg &lock_for_read_arg, bool &can_read, share::SCN &trans_version, - const share::ObLSID ls_id, ObCleanoutOp &cleanout_op, ObReCheckOp &recheck_op) : lock_for_read_arg_(lock_for_read_arg), can_read_(can_read), trans_version_(trans_version), - ls_id_(ls_id), cleanout_op_(cleanout_op), recheck_op_(recheck_op) {} virtual ~LockForReadFunctor() {} virtual int operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx = nullptr) override; virtual bool recheck() override; - int check_for_standby(const transaction::ObTransID &tx_id); INHERIT_TO_STRING_KV("ObITxDataCheckFunctor", ObITxDataCheckFunctor, K(lock_for_read_arg_), - K(can_read_), K(trans_version_), K(ls_id_)); + K(can_read_), K(trans_version_)); private: int inner_lock_for_read(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx); int check_clog_disk_full_(); @@ -208,7 +205,6 @@ class LockForReadFunctor : public ObITxDataCheckFunctor const transaction::ObLockForReadArg &lock_for_read_arg_; bool &can_read_; share::SCN &trans_version_; - share::ObLSID ls_id_; // Cleanout the tx node if necessary ObCleanoutOp &cleanout_op_; // ReCheck whether tx node is valid. diff --git a/src/storage/tx/ob_tx_data_op.cpp b/src/storage/tx/ob_tx_data_op.cpp index 0f1592837..4c8667071 100644 --- a/src/storage/tx/ob_tx_data_op.cpp +++ b/src/storage/tx/ob_tx_data_op.cpp @@ -222,7 +222,7 @@ int ObTxDataOp::reserve_tx_op_space(int64_t count) return ret; } -int ObTxDataOp::add_tx_op_batch(transaction::ObTransID tx_id, share::ObLSID ls_id, share::SCN op_scn, ObTxOpArray &tx_op_batch) +int ObTxDataOp::add_tx_op_batch(transaction::ObTransID tx_id, share::SCN op_scn, ObTxOpArray &tx_op_batch) { int ret = OB_SUCCESS; SpinWLockGuard lock_guard(lock_); @@ -237,11 +237,11 @@ int ObTxDataOp::add_tx_op_batch(transaction::ObTransID tx_id, share::ObLSID ls_i // !!! we must promise tx_op_batch atomic append into tx_op_list // otherwise tx_op replay filter with log_scn compare op_scn will cause serious problem if (OB_FAIL(ret)) { - STORAGE_LOG(ERROR, "tx_op_batch is not atomic append", K(tx_id), K(ls_id), K(tx_op_list_), K(tx_op_batch), K(op_scn)); + STORAGE_LOG(ERROR, "tx_op_batch is not atomic append", K(tx_id), K(tx_op_list_), K(tx_op_batch), K(op_scn)); ob_abort(); } } - STORAGE_LOG(INFO, "add_tx_op", K(ret), K(tx_id), K(ls_id), K(op_scn), K(tx_op_batch.count()), K(tx_op_list_.get_count()), K(tx_op_batch)); + STORAGE_LOG(INFO, "add_tx_op", K(ret), K(tx_id), K(op_scn), K(tx_op_batch.count()), K(tx_op_list_.get_count()), K(tx_op_batch)); return ret; } diff --git a/src/storage/tx/ob_tx_data_op.h b/src/storage/tx/ob_tx_data_op.h index 89f923313..7a4a6d0f9 100644 --- a/src/storage/tx/ob_tx_data_op.h +++ b/src/storage/tx/ob_tx_data_op.h @@ -151,7 +151,7 @@ class ObTxDataOp common::SpinRWLock &get_lock() { return lock_; } int64_t get_ref() { return ref_cnt_; } int add_tx_op(ObTxOp &tx_op); - int add_tx_op_batch(transaction::ObTransID tx_id, share::ObLSID ls_id, share::SCN op_scn, ObTxOpArray &tx_op_batch); + int add_tx_op_batch(transaction::ObTransID tx_id, share::SCN op_scn, ObTxOpArray &tx_op_batch); int reserve_tx_op_space(int64_t count); TO_STRING_KV(K_(ref_cnt), K_(undo_status_list), K_(tx_op_list)); diff --git a/src/storage/tx/ob_tx_elr_handler.cpp b/src/storage/tx/ob_tx_elr_handler.cpp index c432871fe..690505e62 100644 --- a/src/storage/tx/ob_tx_elr_handler.cpp +++ b/src/storage/tx/ob_tx_elr_handler.cpp @@ -15,7 +15,7 @@ */ #include "ob_tx_elr_handler.h" -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" #include "ob_trans_service.h" namespace oceanbase @@ -29,7 +29,7 @@ void ObTxELRHandler::reset() mt_ctx_ = NULL; } -int ObTxELRHandler::check_and_early_lock_release(bool has_row_updated, ObPartTransCtx *ctx) +int ObTxELRHandler::check_and_early_lock_release(bool has_row_updated, ObTxCtx *ctx) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_elr_handler.h b/src/storage/tx/ob_tx_elr_handler.h index 9c592e540..a1afb311a 100644 --- a/src/storage/tx/ob_tx_elr_handler.h +++ b/src/storage/tx/ob_tx_elr_handler.h @@ -28,7 +28,7 @@ class ObMemtableCtx; namespace transaction { -class ObPartTransCtx; +class ObTxCtx; enum TxELRState { @@ -46,7 +46,7 @@ class ObTxELRHandler ObTxELRHandler() : elr_prepared_state_(ELR_INIT), mt_ctx_(NULL) {} void reset(); - int check_and_early_lock_release(bool row_updated, ObPartTransCtx *ctx); + int check_and_early_lock_release(bool row_updated, ObTxCtx *ctx); void set_memtable_ctx(memtable::ObMemtableCtx *mt_ctx) { mt_ctx_ = mt_ctx; } memtable::ObMemtableCtx *get_memtable_ctx() const { return mt_ctx_; } diff --git a/src/storage/tx/ob_tx_log.cpp b/src/storage/tx/ob_tx_log.cpp index 494de2cb1..da6cb90a8 100644 --- a/src/storage/tx/ob_tx_log.cpp +++ b/src/storage/tx/ob_tx_log.cpp @@ -39,7 +39,6 @@ ObTxLogTypeChecker::need_replay_barrier(const ObTxLogType log_type, || data_source_type == ObTxDataSourceType::UNBIND_TABLET_NEW_MDS || data_source_type == ObTxDataSourceType::TABLET_SPLIT || data_source_type == ObTxDataSourceType::TABLET_BINDING - || data_source_type == ObTxDataSourceType::MV_NOTICE_SAFE || data_source_type == ObTxDataSourceType::UNBIND_LOB_TABLET || data_source_type == ObTxDataSourceType::DDL_COMPLETE_MDS) { barrier_flag = logservice::ObReplayBarrierType::PRE_BARRIER; @@ -90,8 +89,6 @@ ObTxLogType ObTxPrevLogType::convert_to_tx_log_type() ObTxLogType tx_log_type = ObTxLogType::UNKNOWN; if (TypeEnum::COMMIT_INFO == prev_log_type_) { tx_log_type = ObTxLogType::TX_COMMIT_INFO_LOG; - } else if (TypeEnum::PREPARE == prev_log_type_) { - tx_log_type = ObTxLogType::TX_PREPARE_LOG; } return tx_log_type; } @@ -166,28 +163,6 @@ DEFINE_GET_SERIALIZE_SIZE(ObTxLogHeader) // ============================== Tx Log serialization ============================= -int ObCtxRedoInfo::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } else { - // skip serialize cluster_version, since 4.2.4, cluster_version put in LogBlockHeader - TX_NO_NEED_SER(true, 1, compat_bytes_); - } - } else { - if (OB_FAIL(compat_bytes_.init(1))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - return ret; -} - -OB_TX_SERIALIZE_MEMBER(ObCtxRedoInfo, compat_bytes_, cluster_version_); - // RedoLogBody serialize mutator_buf in log block OB_DEF_SERIALIZE(ObTxRedoLog) { @@ -197,8 +172,6 @@ OB_DEF_SERIALIZE(ObTxRedoLog) if (mutator_size_ < 0) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "INVALID mutator_buf_"); - } else if (OB_FAIL(ctx_redo_info_.serialize(buf, buf_len, tmp_pos))) { - TRANS_LOG(WARN, "ctx_redo_info_ serialize failed", K(ret)); } else if ((tmp_size = static_cast(mutator_size_)) && OB_FAIL(serialization::encode_i32(buf, buf_len, tmp_pos, tmp_size))) { TRANS_LOG(WARN, "encode mutator_size_ error", K(ret)); @@ -214,10 +187,7 @@ OB_DEF_DESERIALIZE(ObTxRedoLog) int64_t org_pos = pos; int32_t tmp_size = 0; - if(OB_FAIL(ctx_redo_info_.deserialize(buf,data_len,pos))) - { - TRANS_LOG(WARN, "ctx_redo_info_ deserialize failed",K(ret)); - } else if (OB_FAIL(serialization::decode_i32(buf, data_len, pos, &tmp_size))) { + if (OB_FAIL(serialization::decode_i32(buf, data_len, pos, &tmp_size))) { TRANS_LOG(WARN, "decode mutator_size_ error", K(ret)); } else { // cluster_version_ = static_cast(tmp_cluster_version); @@ -238,382 +208,80 @@ OB_DEF_SERIALIZE_SIZE(ObTxRedoLog) len = mutator_size_; TRANS_LOG_RET(WARN, OB_ERR_UNEXPECTED, "mutator_buf_ has not set"); } else { - len = len + ctx_redo_info_.get_serialize_size(); len = len + MUTATOR_SIZE_NEED_BYTES; len = len + mutator_size_; } return len; } -// Other LogBody -OB_TX_SERIALIZE_MEMBER(ObTxActiveInfoLog, - compat_bytes_, - /* 1 */ scheduler_, - /* 2 */ trans_type_, - /* 3 */ session_id_, - /* 4 */ app_trace_id_str_, - /* 5 */ schema_version_, - /* 6 */ can_elr_, - /* 7 */ proposal_leader_, - /* 8 */ cur_query_start_time_, - /* 9 */ is_sub2pc_, - /* 10 */ is_dup_tx_, - /* 11 */ tx_expired_time_, - /* 12 */ epoch_, - /* 13 */ last_op_sn_, - /* 14 */ first_seq_no_, - /* 15 */ last_seq_no_, - /* 16 */ cluster_version_, - /* 17 */ max_submitted_seq_no_, - /* 18 */ xid_, - /* 19 */ serial_final_seq_no_, - /* 20 */ associated_session_id_, - /* 21 */ prio_op_array_); - -OB_TX_SERIALIZE_MEMBER(ObTxCommitInfoLog, - compat_bytes_, - /* 1 */ scheduler_, - /* 2 */ participants_, - /* 3 */ upstream_, - /* 4 */ is_sub2pc_, - /* 5 */ is_dup_tx_, - /* 6 */ can_elr_, - /* 7 */ incremental_participants_, - /* 8 */ cluster_version_, - /* 9 */ app_trace_id_str_, - /* 10 */ app_trace_info_, - /* 11 */ prev_record_lsn_, - /* 12 */ redo_lsns_, - /* 13 */ xid_, - /* 14 */ commit_parts_, - /* 15 */ epoch_); - -OB_TX_SERIALIZE_MEMBER(ObTxPrepareLog, - compat_bytes_, - /* 1 */ incremental_participants_, - /* 2 */ prev_lsn_, - /* 3 */ prev_log_type_); - -OB_TX_SERIALIZE_MEMBER(ObTxCommitLog, - compat_bytes_, +OB_SERIALIZE_MEMBER(ObTxCommitInfoLog, + /* 1 */ is_dup_tx_, + /* 2 */ can_elr_, + /* 3 */ app_trace_id_str_, + /* 4 */ app_trace_info_, + /* 5 */ prev_record_lsn_, + /* 6 */ redo_lsns_, + /* 7 */ xid_); + +OB_SERIALIZE_MEMBER(ObTxCommitLog, /* 1 */ commit_version_, /* 2 */ checksum_, - /* 3 */ incremental_participants_, - /* 4 */ multi_source_data_, - /* 5 */ trans_type_, - /* 6 */ tx_data_backup_, - /* 7 */ prev_lsn_, - /* 8 */ ls_log_info_arr_, - /* 9 */ checksum_sig_serde_, - /* 10 */ prev_log_type_); - -OB_TX_SERIALIZE_MEMBER(ObTxClearLog, compat_bytes_, /* 1 */ incremental_participants_); - -OB_TX_SERIALIZE_MEMBER(ObTxAbortLog, - compat_bytes_, - /* 1 */ multi_source_data_, - /* 2 */ tx_data_backup_); - -OB_TX_SERIALIZE_MEMBER(ObTxRecordLog, compat_bytes_, /* 1 */ prev_record_lsn_, /* 2 */ redo_lsns_); - -OB_TX_SERIALIZE_MEMBER(ObTxStartWorkingLog, compat_bytes_, /* 1 */ leader_epoch_); - -OB_TX_SERIALIZE_MEMBER(ObTxRollbackToLog, compat_bytes_, /* 1 */ from_, /* 2 */ to_); - -OB_TX_SERIALIZE_MEMBER(ObTxMultiDataSourceLog, compat_bytes_, /* 1 */ data_); - -OB_TX_SERIALIZE_MEMBER(ObTxDirectLoadIncLog, compat_bytes_, /* 1 */ ddl_log_type_, /* 2 */ log_buf_, /* 3 */ batch_key_); - -int ObTxActiveInfoLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(21))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(scheduler_.is_valid() == false, 1, compat_bytes_); - TX_NO_NEED_SER(trans_type_ == TransType::UNKNOWN_TRANS, 2, compat_bytes_); /*trans_type_*/ - TX_NO_NEED_SER(session_id_ == 0, 3, compat_bytes_); - TX_NO_NEED_SER(app_trace_id_str_.empty(), 4, compat_bytes_); - TX_NO_NEED_SER(schema_version_ == 0, 5, compat_bytes_); - TX_NO_NEED_SER(can_elr_ == false, 6, compat_bytes_); - TX_NO_NEED_SER(proposal_leader_.is_valid() == false, 7, compat_bytes_); - TX_NO_NEED_SER(cur_query_start_time_ == 0, 8, compat_bytes_); - TX_NO_NEED_SER(is_sub2pc_ == false, 9, compat_bytes_); - TX_NO_NEED_SER(is_dup_tx_ == false, 10, compat_bytes_); - TX_NO_NEED_SER(tx_expired_time_ == 0, 11, compat_bytes_); - TX_NO_NEED_SER(epoch_ == 0, 12, compat_bytes_); - TX_NO_NEED_SER(last_op_sn_ == 0, 13, compat_bytes_); - TX_NO_NEED_SER(!first_seq_no_.is_valid(), 14, compat_bytes_); - TX_NO_NEED_SER(!last_seq_no_.is_valid(), 15, compat_bytes_); - TX_NO_NEED_SER(true, 16, compat_bytes_); - TX_NO_NEED_SER(!max_submitted_seq_no_.is_valid(), 17, compat_bytes_); - TX_NO_NEED_SER(xid_.empty(), 18, compat_bytes_); - TX_NO_NEED_SER(!serial_final_seq_no_.is_valid(), 19, compat_bytes_); - TX_NO_NEED_SER(associated_session_id_ == 0, 20, compat_bytes_); - TX_NO_NEED_SER(prio_op_array_.empty(), 21, compat_bytes_); - } - - return ret; -} - -int ObTxCommitInfoLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(15))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(scheduler_.is_valid() == false, 1, compat_bytes_); - TX_NO_NEED_SER(participants_.empty(), 2, compat_bytes_); - TX_NO_NEED_SER(upstream_.is_valid() == false, 3, compat_bytes_); - TX_NO_NEED_SER(is_sub2pc_ == false, 4, compat_bytes_); - TX_NO_NEED_SER(is_dup_tx_ == false, 5, compat_bytes_); - TX_NO_NEED_SER(can_elr_ == false, 6, compat_bytes_); - TX_NO_NEED_SER(incremental_participants_.empty(), 7, compat_bytes_); - TX_NO_NEED_SER(true, 8, compat_bytes_); - TX_NO_NEED_SER(app_trace_id_str_.empty(), 9, compat_bytes_); - TX_NO_NEED_SER(app_trace_info_.empty(), 10, compat_bytes_); - TX_NO_NEED_SER(prev_record_lsn_.is_valid() == false, 11, compat_bytes_); - TX_NO_NEED_SER(redo_lsns_.empty(), 12, compat_bytes_); - TX_NO_NEED_SER(xid_.empty(), 13, compat_bytes_); - TX_NO_NEED_SER(commit_parts_.empty(), 14, compat_bytes_); - TX_NO_NEED_SER(epoch_ == 0, 15, compat_bytes_); - } - - return ret; -} + /* 3 */ multi_source_data_, + /* 4 */ tx_data_backup_, + /* 5 */ prev_lsn_, + /* 6 */ checksum_sig_serde_, + /* 7 */ prev_log_type_); -int ObTxPrepareLog::before_serialize() -{ - int ret = OB_SUCCESS; +OB_SERIALIZE_MEMBER(ObTxClearLog); - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(3))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(incremental_participants_.empty(), 1, compat_bytes_); - TX_NO_NEED_SER(prev_lsn_.is_valid() == false, 2, compat_bytes_); - TX_NO_NEED_SER(prev_log_type_.is_valid() == false, 3, compat_bytes_); - } - return ret; -} - -int ObTxCommitLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(10))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(!commit_version_.is_valid(), 1, compat_bytes_); - TX_NO_NEED_SER(checksum_ == 0, 2, compat_bytes_); - TX_NO_NEED_SER(incremental_participants_.empty(), 3, compat_bytes_); - TX_NO_NEED_SER(multi_source_data_.empty(), 4, compat_bytes_); - TX_NO_NEED_SER(trans_type_ == TransType::UNKNOWN_TRANS, 5, compat_bytes_); /*trans_type_*/ - TX_NO_NEED_SER(false, 6, compat_bytes_); // tx_data_backup_ - TX_NO_NEED_SER(prev_lsn_.is_valid() == false, 7, compat_bytes_); - TX_NO_NEED_SER(ls_log_info_arr_.empty(), 8, compat_bytes_); - TX_NO_NEED_SER(checksum_sig_.count() == 0, 9, compat_bytes_); - TX_NO_NEED_SER(prev_log_type_.is_valid() == false, 10, compat_bytes_); - } - return ret; -} - -int ObTxClearLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(1))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(incremental_participants_.empty(), 1, compat_bytes_); - } - - return ret; -} - -int ObTxAbortLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(multi_source_data_.empty(), 1, compat_bytes_); - TX_NO_NEED_SER(false, 2, compat_bytes_); // tx_data_backup_ - } - - return ret; -} - -int ObTxRecordLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(prev_record_lsn_.is_valid() == false, 1, compat_bytes_); - TX_NO_NEED_SER(redo_lsns_.empty(), 2, compat_bytes_); - } - return ret; -} - -int ObTxStartWorkingLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } +OB_SERIALIZE_MEMBER(ObTxAbortLog, + /* 1 */ multi_source_data_, + /* 2 */ tx_data_backup_); - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(leader_epoch_ == 0, 1, compat_bytes_); - } - return ret; -} +OB_SERIALIZE_MEMBER(ObTxRecordLog, /* 1 */ prev_record_lsn_, /* 2 */ redo_lsns_); -int ObTxRollbackToLog::before_serialize() -{ - int ret = OB_SUCCESS; +OB_SERIALIZE_MEMBER(ObTxRollbackToLog, /* 1 */ from_, /* 2 */ to_); - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } +OB_SERIALIZE_MEMBER(ObTxMultiDataSourceLog, /* 1 */ data_); - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(false, 1, compat_bytes_); - TX_NO_NEED_SER(false, 2, compat_bytes_); - } +OB_SERIALIZE_MEMBER(ObTxDirectLoadIncLog, /* 1 */ ddl_log_type_, /* 2 */ log_buf_, /* 3 */ batch_key_); - return ret; -} +ObTxCommitInfoLog::ObTxCommitInfoLog(bool is_dup_tx, + bool is_elr, + common::ObString &app_trace_id, + const common::ObString &app_trace_info, + const LogOffSet &prev_record_lsn, + ObRedoLSNArray &redo_lsns, + const ObXATransID &xid) + : is_dup_tx_(is_dup_tx), can_elr_(is_elr), + app_trace_id_str_(app_trace_id), app_trace_info_(app_trace_info), + prev_record_lsn_(prev_record_lsn), redo_lsns_(redo_lsns), xid_(xid) +{} -int ObTxMultiDataSourceLog::before_serialize() +ObTxCommitLog::ObTxCommitLog(share::SCN commit_version, + uint64_t checksum, + ObIArray &checksum_sig, + ObTxBufferNodeArray &multi_source_data, + LogOffSet prev_lsn, + ObTxPrevLogType prev_log_type) + : checksum_(checksum), checksum_sig_(checksum_sig), checksum_sig_serde_(checksum_sig_), + multi_source_data_(multi_source_data), prev_lsn_(prev_lsn), prev_log_type_(prev_log_type) { - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(1))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(false, 1, compat_bytes_); - } - - return ret; + commit_version_ = commit_version; } -int ObTxDirectLoadIncLog::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } - } else { - if (OB_FAIL(compat_bytes_.init(3))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - - if (OB_SUCC(ret)) { - TX_NO_NEED_SER(false, 1, compat_bytes_); - TX_NO_NEED_SER(false, 2, compat_bytes_); - TX_NO_NEED_SER(false, 3, compat_bytes_); - } - - return ret; -} +ObTxClearLog::ObTxClearLog() {} // ============================== Tx Log Body =========================== const ObTxLogType ObTxRedoLog::LOG_TYPE = ObTxLogType::TX_REDO_LOG; -const ObTxLogType ObTxActiveInfoLog::LOG_TYPE = ObTxLogType::TX_ACTIVE_INFO_LOG; const ObTxLogType ObTxCommitInfoLog::LOG_TYPE = ObTxLogType::TX_COMMIT_INFO_LOG; -const ObTxLogType ObTxPrepareLog::LOG_TYPE = ObTxLogType::TX_PREPARE_LOG; const ObTxLogType ObTxCommitLog::LOG_TYPE = ObTxLogType::TX_COMMIT_LOG; const ObTxLogType ObTxClearLog::LOG_TYPE = ObTxLogType::TX_CLEAR_LOG; const ObTxLogType ObTxAbortLog::LOG_TYPE = ObTxLogType::TX_ABORT_LOG; const ObTxLogType ObTxRecordLog::LOG_TYPE = ObTxLogType::TX_RECORD_LOG; // const ObTxLogType ObTxKeepAliveLog::LOG_TYPE = ObTxLogType::TX_KEEP_ALIVE_LOG; -const ObTxLogType ObTxStartWorkingLog::LOG_TYPE = ObTxLogType::TX_START_WORKING_LOG; const ObTxLogType ObTxRollbackToLog::LOG_TYPE = ObTxLogType::TX_ROLLBACK_TO_LOG; const ObTxLogType ObTxMultiDataSourceLog::LOG_TYPE = ObTxLogType::TX_MULTI_DATA_SOURCE_LOG; const ObTxLogType ObTxDirectLoadIncLog::LOG_TYPE = ObTxLogType::TX_DIRECT_LOAD_INC_LOG; @@ -641,8 +309,7 @@ int ObTxRedoLog::set_mutator_size(const int64_t size, const bool after_fill) } else if (!after_fill) { int len = 0; SERIALIZE_SIZE_HEADER(UNIS_VERSION); - ret = ctx_redo_info_.before_serialize(); - len = len + MUTATOR_SIZE_NEED_BYTES + ctx_redo_info_.get_serialize_size(); + len += MUTATOR_SIZE_NEED_BYTES; if (size <= len) { ret = OB_SIZE_OVERFLOW; TRANS_LOG(WARN, "mutator buf is not enough", K(len), K(size)); @@ -888,9 +555,7 @@ int ObTxRedoLog::format_row_data_(const memtable::ObRowData &row_data, ObAdminMu void ObTxMultiDataSourceLog::reset() { - compat_bytes_.reset(); data_.reset(); - before_serialize(); } int ObTxMultiDataSourceLog::fill_MDS_data(const ObTxBufferNode &node) @@ -983,23 +648,6 @@ int ObTxDLIncLogBuf::deserialize(const char *buf, const int64_t data_len, int64_ OB_SERIALIZE_MEMBER(ObTxDataBackup, start_log_ts_); -int ObTxActiveInfoLog::ob_admin_dump(ObAdminMutatorStringArg &arg) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(arg.writer_ptr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid arg writer is NULL", K(arg), K(ret)); - } else { - arg.writer_ptr_->dump_key("###"); - arg.writer_ptr_->start_object(); - arg.writer_ptr_->dump_key("Members"); - ObCStringHelper helper; - arg.writer_ptr_->dump_string(helper.convert(*this)); - arg.writer_ptr_->end_object(); - } - return ret; -} - int ObTxCommitInfoLog::ob_admin_dump(ObAdminMutatorStringArg &arg) { int ret = OB_SUCCESS; @@ -1019,23 +667,6 @@ int ObTxCommitInfoLog::ob_admin_dump(ObAdminMutatorStringArg &arg) return ret; } -int ObTxPrepareLog::ob_admin_dump(ObAdminMutatorStringArg &arg) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(arg.writer_ptr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid arg writer is NULL", K(arg), K(ret)); - } else { - arg.writer_ptr_->dump_key("###"); - arg.writer_ptr_->start_object(); - arg.writer_ptr_->dump_key("Members"); - ObCStringHelper helper; - arg.writer_ptr_->dump_string(helper.convert(*this)); - arg.writer_ptr_->end_object(); - } - return ret; -} - int ObTxCommitLog::ob_admin_dump(ObAdminMutatorStringArg &arg) { int ret = OB_SUCCESS; @@ -1106,23 +737,6 @@ int ObTxRecordLog::ob_admin_dump(ObAdminMutatorStringArg &arg) return ret; } -int ObTxStartWorkingLog::ob_admin_dump(ObAdminMutatorStringArg &arg) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(arg.writer_ptr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid arg writer is NULL", K(arg), K(ret)); - } else { - arg.writer_ptr_->dump_key("###"); - arg.writer_ptr_->start_object(); - arg.writer_ptr_->dump_key("Members"); - ObCStringHelper helper; - arg.writer_ptr_->dump_string(helper.convert(*this)); - arg.writer_ptr_->end_object(); - } - return ret; -} - int ObTxRollbackToLog::ob_admin_dump(ObAdminMutatorStringArg &arg) { int ret = OB_SUCCESS; @@ -1234,85 +848,13 @@ int ObTxAbortLog::init_tx_data_backup(const share::SCN &start_scn) return ret; } -// ============================== Tx Log Blcok ============================= -#define TX_LOG_BLOCK_SERIALIZE_MEMBERS \ - org_cluster_id_, \ - log_entry_no_, \ - tx_id_, \ - scheduler_, \ - __log_entry_no_, \ - cluster_version_, \ - flags_ - -OB_DEF_SERIALIZE(ObTxLogBlockHeader) -{ - int64_t pos_bk = pos; - int ret = OB_SUCCESS; - TX_SER_COMPAT_BYTES(compat_bytes_); - TX_LST_DO_CODE(OB_TX_UNIS_ENCODE, compat_bytes_, TX_LOG_BLOCK_SERIALIZE_MEMBERS); - // if the real serialized data size less than reserved - // must do pading - const int64_t pading_size = serialize_size_ - (pos - pos_bk); - MEMSET(buf + pos, 0 , pading_size); - pos += pading_size; - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObTxLogBlockHeader) -{ - if (serialize_size_ == 0) { - ob_abort(); - } - return serialize_size_; -} - -void ObTxLogBlockHeader::calc_serialize_size_() -{ - int64_t len = 0; - int64_t log_entry_no = log_entry_no_; - int64_t log_entry_no_ = INT64_MAX; - uint8_t flags = flags_; - uint8_t flags_ = UINT8_MAX; - TX_SER_SIZE_COMPAT_BYTES(compat_bytes_); - TX_LST_DO_CODE(OB_TX_UNIS_ADD_LEN, compat_bytes_, TX_LOG_BLOCK_SERIALIZE_MEMBERS); - flags_ = flags; - log_entry_no_ = log_entry_no; - serialize_size_ = len; -} - -OB_DEF_DESERIALIZE(ObTxLogBlockHeader) -{ - int ret = OB_SUCCESS; - UNF_UNUSED_DES; - TX_DSER_COMPAT_BYTES(compat_bytes_); - TX_LST_DO_CODE(OB_TX_UNIS_DECODE, compat_bytes_, TX_LOG_BLOCK_SERIALIZE_MEMBERS); - serialize_size_ = data_len; - return ret; -} - -int ObTxLogBlockHeader::before_serialize() -{ - int ret = OB_SUCCESS; - - if (compat_bytes_.is_inited()) { - if (OB_FAIL(compat_bytes_.set_all_member_need_ser())) { - TRANS_LOG(WARN, "reset all compat_bytes_ valid failed", K(ret)); - } else { - if (cluster_version_ == 0) { - ob_abort(); - } - TX_NO_NEED_SER(true, 2, compat_bytes_); - if (serialize_size_ == 0) { - calc_serialize_size_(); - } - } - } else { - if (OB_FAIL(compat_bytes_.init(7))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - return ret; -} +// ============================== Tx Log Block ============================= +OB_SERIALIZE_MEMBER(ObTxLogBlockHeader, + org_cluster_id_, + cluster_version_, + __log_entry_no_, + tx_id_, + flags_); const logservice::ObLogBaseType ObTxLogBlock::DEFAULT_LOG_BLOCK_TYPE = logservice::ObLogBaseType::TRANS_SERVICE_LOG_BASE_TYPE; // TRANS_LOG @@ -1348,7 +890,6 @@ int ObTxLogBlock::reuse_for_fill() pos_ = 0; // reserve place for headers, header will be filled back pos_ += log_base_header_.get_serialize_size(); // assume FIXED size - header_.before_serialize(); pos_ += header_.get_serialize_size(); } return ret; @@ -1377,7 +918,6 @@ int ObTxLogBlock::init_for_fill(const int64_t suggested_buf_size) pos_ = 0; // reserve place for headers, header will be filled back pos_ += log_base_header_.get_serialize_size(); // assume FIXED size - header_.before_serialize(); pos_ += header_.get_serialize_size(); inited_ = true; } @@ -1516,8 +1056,6 @@ int ObTxLogBlock::serialize_log_block_header_() TRANS_LOG(WARN, "unexpected empty serialize_buf", K(*this)); } else if (OB_FAIL(log_base_header_.serialize(serialize_buf, len_, pos_))) { TRANS_LOG(WARN, "serialize log base header error", K(ret)); - } else if (OB_FAIL(header_.before_serialize())) { - TRANS_LOG(WARN, "before serialize failed", K(ret), K(*this)); } else if (OB_FAIL(header_.serialize(serialize_buf, len_, pos_))) { TRANS_LOG(WARN, "serialize block header error", K(ret)); } @@ -1647,8 +1185,6 @@ int ObTxLogBlock::finish_mutator_buf(ObTxRedoLog &redo, const int64_t &mutator_s TRANS_LOG(WARN, "set mutator buf size error after fill", K(ret)); } else if (OB_FAIL(header.serialize(tmp_buf, len_, tmp_pos))) { TRANS_LOG(WARN, "serialize log header error", K(ret), K(header), K(*this)); - } else if (OB_FAIL(redo.before_serialize())) { - TRANS_LOG(WARN, "before serialize for redo failed", K(ret), K(redo)); } else if (OB_FAIL(redo.serialize(tmp_buf, len_, tmp_pos))) { TRANS_LOG(WARN, "serialize redo log body error", K(ret)); } else { diff --git a/src/storage/tx/ob_tx_log.h b/src/storage/tx/ob_tx_log.h index 3b2506675..f29976d30 100644 --- a/src/storage/tx/ob_tx_log.h +++ b/src/storage/tx/ob_tx_log.h @@ -18,11 +18,9 @@ #define OCEANBASE_TRANSACTION_OB_TX_LOG #include -#include "share/ob_ls_id.h" #include "storage/ddl/ob_ddl_clog.h" #include "storage/tx/ob_trans_define.h" #include "storage/tx/ob_tx_data_define.h" -#include "storage/tx/ob_tx_serialization.h" #include "storage/tx/ob_tx_big_segment_buf.h" #include "storage/tx/ob_trans_factory.h" #include "share/ob_admin_dump_helper.h" @@ -48,7 +46,6 @@ class ObRowData; namespace share { -class ObLSID; struct ObAdminMutatorStringArg; } @@ -72,10 +69,9 @@ namespace transaction { class ObTxLogCb; -class ObPartTransCtx; +class ObTxCtx; typedef int64_t TxID; -typedef share::ObLSID LSKey; typedef palf::LSN LogOffSet; // Trans Log @@ -86,16 +82,12 @@ enum class ObTxLogType : int64_t TX_ROLLBACK_TO_LOG = 0x2, TX_MULTI_DATA_SOURCE_LOG = 0x4, TX_DIRECT_LOAD_INC_LOG = 0x8, - TX_ACTIVE_INFO_LOG = 0x10, TX_RECORD_LOG = 0x20, TX_COMMIT_INFO_LOG = 0x40, - TX_PREPARE_LOG = 0x80, TX_COMMIT_LOG = 0x100, TX_ABORT_LOG = 0x200, TX_CLEAR_LOG = 0x400, - // logstream-level log - TX_START_WORKING_LOG = 0x100000, // BigSegment log TX_BIG_SEGMENT_LOG = 0x200000, TX_LOG_TYPE_LIMIT @@ -146,14 +138,11 @@ static const ObTxLogType TX_LOG_TYPE_MASK = (ObTxLogType::TX_REDO_LOG | ObTxLogType::TX_ROLLBACK_TO_LOG | ObTxLogType::TX_MULTI_DATA_SOURCE_LOG | ObTxLogType::TX_DIRECT_LOAD_INC_LOG | - ObTxLogType::TX_ACTIVE_INFO_LOG | ObTxLogType::TX_RECORD_LOG | ObTxLogType::TX_COMMIT_INFO_LOG | - ObTxLogType::TX_PREPARE_LOG | ObTxLogType::TX_COMMIT_LOG | ObTxLogType::TX_ABORT_LOG | - ObTxLogType::TX_CLEAR_LOG | - ObTxLogType::TX_START_WORKING_LOG); + ObTxLogType::TX_CLEAR_LOG); class ObTxLogTypeChecker { public: @@ -166,8 +155,7 @@ class ObTxLogTypeChecker { } static bool is_state_log(const ObTxLogType log_type) { - return ObTxLogType::TX_PREPARE_LOG == log_type || - ObTxLogType::TX_COMMIT_LOG == log_type || + return ObTxLogType::TX_COMMIT_LOG == log_type || ObTxLogType::TX_ABORT_LOG == log_type || ObTxLogType::TX_CLEAR_LOG == log_type; } @@ -175,10 +163,6 @@ class ObTxLogTypeChecker { { return (TX_LOG_TYPE_MASK & log_type) != 0; } - static bool is_ls_log(const ObTxLogType log_type) - { - return ObTxLogType::TX_START_WORKING_LOG == log_type; - } static bool is_mds_log(const ObTxLogType log_type) { return ObTxLogType::TX_MULTI_DATA_SOURCE_LOG == log_type; @@ -218,7 +202,6 @@ class ObTxPrevLogType UNKOWN = 0, SELF = 1, COMMIT_INFO = 2, - PREPARE = 3, }; public: @@ -230,11 +213,10 @@ class ObTxPrevLogType void set_self() { prev_log_type_ = TypeEnum::SELF; } bool is_self() const { return TypeEnum::SELF == prev_log_type_; } - void set_prepare() { prev_log_type_ = TypeEnum::PREPARE; } void set_commit_info() { prev_log_type_ = TypeEnum::COMMIT_INFO; } bool is_normal_log() const { - return TypeEnum::COMMIT_INFO == prev_log_type_ || TypeEnum::PREPARE == prev_log_type_; + return TypeEnum::COMMIT_INFO == prev_log_type_; } ObTxLogType convert_to_tx_log_type(); @@ -271,21 +253,6 @@ class ObTxRedoLogTempRef public: }; -struct ObCtxRedoInfo -{ - ObCtxRedoInfo(const int64_t cluster_version) - : cluster_version_(cluster_version) - { - }; - int before_serialize(); - - ObTxSerCompatByte compat_bytes_; - // serialize before mutator_buf by fixed length - uint64_t cluster_version_; - - OB_UNIS_VERSION(1); -}; - class ObTxRedoLog { OB_UNIS_VERSION(1); @@ -294,22 +261,15 @@ class ObTxRedoLog public: ObTxRedoLog(ObTxRedoLogTempRef &temp_ref) - : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1), - ctx_redo_info_(0) + : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1) { - before_serialize(); - } - ObTxRedoLog(const uint64_t &cluster_version) - : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1), - ctx_redo_info_(cluster_version) - { - before_serialize(); + UNUSED(temp_ref); } + ObTxRedoLog() : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1) {} char *get_mutator_buf() { return mutator_buf_; } const char *get_replay_mutator_buf() const { return replay_mutator_buf_; } const int64_t &get_mutator_size() const { return mutator_size_; } - const uint64_t &get_cluster_version() const { return ctx_redo_info_.cluster_version_; } //------------ Only invoke in ObTxLogBlock int set_mutator_buf(char *buf); @@ -332,12 +292,7 @@ class ObTxRedoLog TO_STRING_KV( K(LOG_TYPE), // KP(mutator_buf_), - K(mutator_size_), - // KP(replay_mutator_buf_), - K(ctx_redo_info_.cluster_version_)); - -public: - int before_serialize() { return ctx_redo_info_.before_serialize(); } + K(mutator_size_)); private: //------------ For ob_admin int format_mutator_row_(const memtable::ObMemtableMutatorRow &row, share::ObAdminMutatorStringArg &arg); @@ -349,7 +304,6 @@ class ObTxRedoLog const char *replay_mutator_buf_; // for replay, only set by deserialize int64_t mutator_size_; - ObCtxRedoInfo ctx_redo_info_; }; // for dist trans write it's multi source data, the same as redo, @@ -379,11 +333,7 @@ class ObTxMultiDataSourceLog static const ObTxLogType LOG_TYPE; TO_STRING_KV(K(LOG_TYPE), K_(data)); -public: - int before_serialize(); - private: - ObTxSerCompatByte compat_bytes_; ObTxBufferNodeArray data_; }; @@ -604,9 +554,7 @@ class ObTxDirectLoadIncLog ObTxDirectLoadIncLog(const ObTxDirectLoadIncLog::ConstructArg &arg) : ddl_log_type_(arg.ddl_log_type_), log_buf_(arg.dli_buf_) - { - before_serialize(); - } + {} // const ObDDLRedoLog &get_ddl_redo_log() { return ddl_redo_log_; } @@ -618,9 +566,6 @@ class ObTxDirectLoadIncLog static const ObTxLogType LOG_TYPE; TO_STRING_KV(K(LOG_TYPE), K(ddl_log_type_), K(log_buf_)); -public: - int before_serialize(); - #ifdef ENABLE_DEBUG_LOG // only for unittest static int64_t direct_load_inc_submit_log_cnt_; @@ -631,8 +576,6 @@ class ObTxDirectLoadIncLog #endif private: - ObTxSerCompatByte compat_bytes_; - DirectLoadIncLogType ddl_log_type_; ObTxDLIncLogBuf &log_buf_; @@ -640,162 +583,16 @@ class ObTxDirectLoadIncLog ObDDLIncLogBasic batch_key_; }; -class ObTxActiveInfoLogTempRef { -public: - ObTxActiveInfoLogTempRef() - : scheduler_(), app_trace_id_str_(), proposal_leader_(), xid_(), prio_op_array_() {} - -public: - common::ObAddr scheduler_; - common::ObString app_trace_id_str_; - common::ObAddr proposal_leader_; - ObXATransID xid_; - tablelock::ObTableLockPrioOpArray prio_op_array_; -}; - -class ObTxActiveInfoLog -{ - OB_UNIS_VERSION(1); - -public: - ObTxActiveInfoLog(ObTxActiveInfoLogTempRef &temp_ref) - : scheduler_(temp_ref.scheduler_), trans_type_(TransType::SP_TRANS), session_id_(0), associated_session_id_(0), - app_trace_id_str_(temp_ref.app_trace_id_str_), schema_version_(0), can_elr_(false), - proposal_leader_(temp_ref.proposal_leader_), cur_query_start_time_(0), is_sub2pc_(false), - is_dup_tx_(false), tx_expired_time_(0), epoch_(0), last_op_sn_(0), first_seq_no_(), - last_seq_no_(), max_submitted_seq_no_(), serial_final_seq_no_(), cluster_version_(0), - xid_(temp_ref.xid_), prio_op_array_(temp_ref.prio_op_array_) - { - before_serialize(); - } - ObTxActiveInfoLog(common::ObAddr &scheduler, - int trans_type, - int session_id, - uint32_t associated_session_id, - common::ObString &app_trace_id_str, - int64_t schema_version, - bool elr, - common::ObAddr &proposal_leader, - int64_t cur_query_start_time, - bool is_sub2pc, - bool is_dup_tx, - int64_t tx_expired_time, - int64_t epoch, - int64_t last_op_sn, - ObTxSEQ first_seq_no, - ObTxSEQ last_seq_no, - ObTxSEQ max_submitted_seq_no, - uint64_t cluster_version, - const ObXATransID &xid, - ObTxSEQ serial_final_seq_no, - tablelock::ObTableLockPrioOpArray &prio_op_array) - : scheduler_(scheduler), trans_type_(trans_type), session_id_(session_id), - associated_session_id_(associated_session_id), - app_trace_id_str_(app_trace_id_str), schema_version_(schema_version), can_elr_(elr), - proposal_leader_(proposal_leader), cur_query_start_time_(cur_query_start_time), - is_sub2pc_(is_sub2pc), is_dup_tx_(is_dup_tx), tx_expired_time_(tx_expired_time), - epoch_(epoch), last_op_sn_(last_op_sn), first_seq_no_(first_seq_no), last_seq_no_(last_seq_no), - max_submitted_seq_no_(max_submitted_seq_no), serial_final_seq_no_(serial_final_seq_no), - cluster_version_(cluster_version), xid_(xid), prio_op_array_(prio_op_array) - { - before_serialize(); - }; - - const common::ObAddr &get_scheduler() const { return scheduler_; } - int get_trans_type() const { return trans_type_; } - int get_session_id() const { return session_id_; } - int get_associated_session_id() const { return associated_session_id_; } - const common::ObString &get_app_trace_id() const { return app_trace_id_str_; } - const int64_t &get_schema_version() { return schema_version_; } - bool is_elr() const { return can_elr_; } - const common::ObAddr &get_proposal_leader() { return proposal_leader_; } - const int64_t &get_cur_query_start_time() { return cur_query_start_time_; } - bool is_sub2pc() const { return is_sub2pc_; } - bool is_dup_tx() const { return is_dup_tx_; } - int64_t get_tx_expired_time() const { return tx_expired_time_; } - int64_t get_last_op_sn() const { return last_op_sn_; } - int64_t get_epoch() const { return epoch_; } - ObTxSEQ get_first_seq_no() const { return first_seq_no_; } - ObTxSEQ get_last_seq_no() const { return last_seq_no_; } - ObTxSEQ get_max_submitted_seq_no() const { return max_submitted_seq_no_; } - ObTxSEQ get_serial_final_seq_no() const { return serial_final_seq_no_; } - uint64_t get_cluster_version() const { return cluster_version_; } - const ObXATransID &get_xid() const { return xid_; } - const tablelock::ObTableLockPrioOpArray &get_prio_op_array() const { return prio_op_array_; } - // for ob_admin - int ob_admin_dump(share::ObAdminMutatorStringArg &arg); - - static const ObTxLogType LOG_TYPE; - TO_STRING_KV(K(LOG_TYPE), - K(scheduler_), - K(trans_type_), - K(session_id_), - K(associated_session_id_), - K(app_trace_id_str_), - K(schema_version_), - K(can_elr_), - K(proposal_leader_), - K(cur_query_start_time_), - K(is_sub2pc_), - K(is_dup_tx_), - K(tx_expired_time_), - K(epoch_), - K(last_op_sn_), - K(first_seq_no_), - K(last_seq_no_), - K(max_submitted_seq_no_), - K(serial_final_seq_no_), - K(cluster_version_), - K(xid_), - K(prio_op_array_)); - -public: - int before_serialize(); - -private: - ObTxSerCompatByte compat_bytes_; - common::ObAddr &scheduler_; - int trans_type_; - int session_id_; - uint32_t associated_session_id_; - common::ObString &app_trace_id_str_; - int64_t schema_version_; - bool can_elr_; - common::ObAddr &proposal_leader_; - int64_t cur_query_start_time_; - bool is_sub2pc_; - bool is_dup_tx_; - int64_t tx_expired_time_; - - // sql execution relative - int64_t epoch_; - int64_t last_op_sn_; - ObTxSEQ first_seq_no_; - ObTxSEQ last_seq_no_; - - // ctrl savepoint written log - ObTxSEQ max_submitted_seq_no_; - ObTxSEQ serial_final_seq_no_; - uint64_t cluster_version_; - ObXATransID xid_; - tablelock::ObTableLockPrioOpArray &prio_op_array_; -}; - class ObTxCommitInfoLogTempRef { public: ObTxCommitInfoLogTempRef() - : scheduler_(), participants_(), commit_parts_(), app_trace_id_str_(), app_trace_info_(), - incremental_participants_(), prev_record_lsn_(), redo_lsns_(), xid_() + : app_trace_id_str_(), app_trace_info_(), prev_record_lsn_(), redo_lsns_(), xid_() {} public: - common::ObAddr scheduler_; - share::ObLSArray participants_; - ObTxCommitParts commit_parts_; common::ObString app_trace_id_str_; common::ObString app_trace_info_; - share::ObLSArray incremental_participants_; LogOffSet prev_record_lsn_; ObRedoLSNArray redo_lsns_; ObXATransID xid_; @@ -807,86 +604,41 @@ class ObTxCommitInfoLog public: ObTxCommitInfoLog(ObTxCommitInfoLogTempRef &temp_ref) - : scheduler_(temp_ref.scheduler_), participants_(temp_ref.participants_), upstream_(), - is_sub2pc_(false), is_dup_tx_(false), can_elr_(false), - incremental_participants_(temp_ref.incremental_participants_), cluster_version_(0), + : is_dup_tx_(false), can_elr_(false), app_trace_id_str_(temp_ref.app_trace_id_str_), app_trace_info_(temp_ref.app_trace_info_), prev_record_lsn_(temp_ref.prev_record_lsn_), redo_lsns_(temp_ref.redo_lsns_), - xid_(temp_ref.xid_), commit_parts_(temp_ref.commit_parts_), epoch_(0) - { - before_serialize(); - } - ObTxCommitInfoLog(common::ObAddr &scheduler, - share::ObLSArray &participants, - LSKey &upstream, - bool is_sub2pc, - bool is_dup_tx, + xid_(temp_ref.xid_) + {} + ObTxCommitInfoLog(bool is_dup_tx, bool is_elr, common::ObString &app_trace_id, const common::ObString &app_trace_info, const LogOffSet &prev_record_lsn, ObRedoLSNArray &redo_lsns, - share::ObLSArray &incremental_participants, - uint64_t cluster_version, - const ObXATransID &xid, - ObTxCommitParts &commit_parts, - int64_t epoch) - : scheduler_(scheduler), participants_(participants), upstream_(upstream), - is_sub2pc_(is_sub2pc), is_dup_tx_(is_dup_tx), can_elr_(is_elr), - incremental_participants_(incremental_participants), cluster_version_(cluster_version), - app_trace_id_str_(app_trace_id), app_trace_info_(app_trace_info), - prev_record_lsn_(prev_record_lsn), redo_lsns_(redo_lsns), xid_(xid), commit_parts_(commit_parts), epoch_(epoch) - { - before_serialize(); - }; + const ObXATransID &xid); - const common::ObAddr &get_scheduler() const { return scheduler_; } - const share::ObLSArray &get_participants() const { return participants_; } - const LSKey &get_upstream() const { return upstream_; } - bool is_sub2pc() const { return is_sub2pc_; } bool is_dup_tx() const { return is_dup_tx_; } bool is_elr() const { return can_elr_; } const common::ObString &get_app_trace_id() const { return app_trace_id_str_; } const common::ObString &get_app_trace_info() const { return app_trace_info_; } const ObRedoLSNArray &get_redo_lsns() const { return redo_lsns_; } const LogOffSet &get_prev_record_lsn() const { return prev_record_lsn_; } - const share::ObLSArray &get_incremental_participants() const { return incremental_participants_; } - uint64_t get_cluster_version() const { return cluster_version_; } const ObXATransID &get_xid() const { return xid_; } - int64_t get_epoch() const { return epoch_; } - const ObTxCommitParts &get_commit_parts() const { return commit_parts_; } int ob_admin_dump(share::ObAdminMutatorStringArg &arg); static const ObTxLogType LOG_TYPE; TO_STRING_KV(K(LOG_TYPE), - K(scheduler_), - K(participants_), - K(upstream_), - K(is_sub2pc_), K(is_dup_tx_), K(can_elr_), - K(incremental_participants_), - K(cluster_version_), K(app_trace_id_str_), K(app_trace_info_), K(prev_record_lsn_), K(redo_lsns_), - K(xid_), - K(commit_parts_), - K(epoch_)) -public: - int before_serialize(); + K(xid_)) private: - ObTxSerCompatByte compat_bytes_; - common::ObAddr &scheduler_; - share::ObLSArray &participants_; - LSKey upstream_; - bool is_sub2pc_; bool is_dup_tx_; bool can_elr_; - share::ObLSArray &incremental_participants_; - uint64_t cluster_version_; //--------- for liboblog ----------- common::ObString &app_trace_id_str_; @@ -895,54 +647,6 @@ class ObTxCommitInfoLog ObRedoLSNArray &redo_lsns_; // for xa ObXATransID xid_; - // for commit participants - ObTxCommitParts &commit_parts_; - int64_t epoch_; -}; - -class ObTxPrepareLogTempRef -{ -public: - ObTxPrepareLogTempRef() : incremental_participants_() {} -public: - share::ObLSArray incremental_participants_; -}; - -class ObTxPrepareLog -{ - OB_UNIS_VERSION(1); - -public: - ObTxPrepareLog(ObTxPrepareLogTempRef &temp_ref) - : incremental_participants_(temp_ref.incremental_participants_), prev_lsn_(), prev_log_type_() - { - before_serialize(); - }; - ObTxPrepareLog(share::ObLSArray &incremental_participants, LogOffSet &commit_info_lsn, ObTxPrevLogType prev_log_type) - : incremental_participants_(incremental_participants), prev_lsn_(commit_info_lsn), prev_log_type_(prev_log_type) - { - before_serialize(); - }; - - const share::ObLSArray &get_incremental_participants() const { return incremental_participants_; } - const LogOffSet &get_prev_lsn() { return prev_lsn_; } - const ObTxPrevLogType &get_prev_log_type() const { return prev_log_type_; } - void set_prev_lsn(const LogOffSet &lsn) { prev_lsn_ = lsn; } - int ob_admin_dump(share::ObAdminMutatorStringArg &arg); - - static const ObTxLogType LOG_TYPE; - TO_STRING_KV(K(LOG_TYPE), K(incremental_participants_), K(prev_lsn_), K(prev_log_type_)); - -public: - int before_serialize(); - -private: - ObTxSerCompatByte compat_bytes_; - share::ObLSArray &incremental_participants_; - - //--------- for liboblog ----------- - LogOffSet prev_lsn_; - ObTxPrevLogType prev_log_type_; }; class ObTxDataBackup @@ -965,14 +669,12 @@ class ObTxDataBackup class ObTxCommitLogTempRef { public: - ObTxCommitLogTempRef() : checksum_signature_(), incremental_participants_(), multi_source_data_(), ls_log_info_arr_() + ObTxCommitLogTempRef() : checksum_signature_(), multi_source_data_() { checksum_signature_.set_max_print_count(1024); } public: ObSEArray checksum_signature_; - share::ObLSArray incremental_participants_; ObTxBufferNodeArray multi_source_data_; - ObLSLogInfoArray ls_log_info_arr_; }; class ObTxCommitLog @@ -986,36 +688,19 @@ class ObTxCommitLog ObTxCommitLog(ObTxCommitLogTempRef &temp_ref) : commit_version_(), checksum_(0), checksum_sig_(temp_ref.checksum_signature_), checksum_sig_serde_(checksum_sig_), - incremental_participants_(temp_ref.incremental_participants_), - multi_source_data_(temp_ref.multi_source_data_), trans_type_(TransType::SP_TRANS), - tx_data_backup_(), prev_lsn_(), ls_log_info_arr_(temp_ref.ls_log_info_arr_), prev_log_type_() - { - before_serialize(); - } + multi_source_data_(temp_ref.multi_source_data_), tx_data_backup_(), prev_lsn_(), prev_log_type_() + {} ObTxCommitLog(share::SCN commit_version, uint64_t checksum, ObIArray &checksum_sig, - share::ObLSArray &incremental_participants, ObTxBufferNodeArray &multi_source_data, - int32_t trans_type, LogOffSet prev_lsn, - ObLSLogInfoArray &ls_log_info_arr, - ObTxPrevLogType prev_log_type) - : checksum_(checksum), checksum_sig_(checksum_sig), checksum_sig_serde_(checksum_sig_), - incremental_participants_(incremental_participants), multi_source_data_(multi_source_data), - trans_type_(trans_type), prev_lsn_(prev_lsn), ls_log_info_arr_(ls_log_info_arr), prev_log_type_(prev_log_type) - { - commit_version_ = commit_version; - before_serialize(); - } + ObTxPrevLogType prev_log_type); int init_tx_data_backup(const share::SCN &start_scn); const ObTxDataBackup &get_tx_data_backup() const { return tx_data_backup_; } share::SCN get_commit_version() const { return commit_version_; } uint64_t get_checksum() const { return checksum_; } - const share::ObLSArray &get_incremental_participants() const { return incremental_participants_; } const ObTxBufferNodeArray &get_multi_source_data() const { return multi_source_data_; } - const int32_t &get_trans_type() const { return trans_type_; } - const ObLSLogInfoArray &get_ls_log_info_arr() const { return ls_log_info_arr_; } const LogOffSet &get_prev_lsn() const { return prev_lsn_; } const ObTxPrevLogType &get_prev_log_type() const { return prev_log_type_; } void set_prev_lsn(const LogOffSet &lsn) { prev_lsn_ = lsn; } @@ -1029,44 +714,29 @@ class ObTxCommitLog K(commit_version_), K(checksum_), K_(checksum_sig), - K(incremental_participants_), K(multi_source_data_), - K(trans_type_), K(tx_data_backup_), K(prev_lsn_), - K(ls_log_info_arr_), K(prev_log_type_)); -public: - int before_serialize(); - private: - ObTxSerCompatByte compat_bytes_; - share::SCN commit_version_; // equal to INVALID_COMMIT_VERSION in Single LS Transaction + share::SCN commit_version_; uint64_t checksum_; ObIArray &checksum_sig_; ObIArraySerDeTrait checksum_sig_serde_; - share::ObLSArray &incremental_participants_; ObTxBufferNodeArray &multi_source_data_; - // need by a committed transaction which replay from commit log - int32_t trans_type_; - ObTxDataBackup tx_data_backup_; //--------- for liboblog ----------- LogOffSet prev_lsn_; - ObLSLogInfoArray &ls_log_info_arr_; ObTxPrevLogType prev_log_type_; }; class ObTxClearLogTempRef { public: - ObTxClearLogTempRef() : incremental_participants_() {} - -public: - share::ObLSArray incremental_participants_; + ObTxClearLogTempRef() {} }; class ObTxClearLog @@ -1075,28 +745,16 @@ class ObTxClearLog public: ObTxClearLog(ObTxClearLogTempRef &temp_ref) - : incremental_participants_(temp_ref.incremental_participants_) { - before_serialize(); - } - ObTxClearLog(share::ObLSArray &incremental_participants) - : incremental_participants_(incremental_participants) - { - before_serialize(); + UNUSED(temp_ref); } - const share::ObLSArray &get_incremental_participants() const { return incremental_participants_; } + ObTxClearLog(); int ob_admin_dump(share::ObAdminMutatorStringArg &arg); static const ObTxLogType LOG_TYPE; - TO_STRING_KV(K(LOG_TYPE), K(incremental_participants_)); - -public: - int before_serialize(); + TO_STRING_KV(K(LOG_TYPE)); -private: - ObTxSerCompatByte compat_bytes_; - share::ObLSArray &incremental_participants_; }; class ObTxAbortLogTempRef @@ -1115,14 +773,10 @@ class ObTxAbortLog ObTxAbortLog(ObTxAbortLogTempRef &temp_ref) : multi_source_data_(temp_ref.multi_source_data_), tx_data_backup_() - { - before_serialize(); - } + {} ObTxAbortLog(ObTxBufferNodeArray &multi_source_data) : multi_source_data_(multi_source_data), tx_data_backup_() - { - before_serialize(); - } + {} const ObTxBufferNodeArray &get_multi_source_data() const { return multi_source_data_; } int init_tx_data_backup(const share::SCN &start_scn); @@ -1136,11 +790,7 @@ class ObTxAbortLog static const ObTxLogType LOG_TYPE; TO_STRING_KV(K(LOG_TYPE), K(multi_source_data_)); -public: - int before_serialize(); - private: - ObTxSerCompatByte compat_bytes_; ObTxBufferNodeArray &multi_source_data_; ObTxDataBackup tx_data_backup_; }; @@ -1162,14 +812,10 @@ class ObTxRecordLog public: ObTxRecordLog(ObTxRecordLogTempRef &temp_ref) : prev_record_lsn_(temp_ref.prev_record_lsn_), redo_lsns_(temp_ref.redo_lsns_) - { - before_serialize(); - } + {} ObTxRecordLog(const LogOffSet &prev_record_lsn, ObRedoLSNArray &redo_lsns) : prev_record_lsn_(prev_record_lsn), redo_lsns_(redo_lsns) - { - before_serialize(); - } + {} const ObRedoLSNArray &get_redo_lsns() { return redo_lsns_; } const LogOffSet &get_prev_record_lsn() { return prev_record_lsn_; } @@ -1179,50 +825,12 @@ class ObTxRecordLog static const ObTxLogType LOG_TYPE; TO_STRING_KV(K(LOG_TYPE), K(prev_record_lsn_), K(redo_lsns_)); -public: - int before_serialize(); - private: - ObTxSerCompatByte compat_bytes_; LogOffSet prev_record_lsn_; ObRedoLSNArray &redo_lsns_; }; -class ObTxStartWorkingLogTempRef -{ -public: - ObTxStartWorkingLogTempRef() {} -}; - -class ObTxStartWorkingLog -{ -public: - OB_UNIS_VERSION(1); - -public: - ObTxStartWorkingLog(ObTxStartWorkingLogTempRef &temp_ref) : leader_epoch_(0) - { - UNUSED(temp_ref); - before_serialize(); - } - ObTxStartWorkingLog(int64_t leader_epoch) : leader_epoch_(leader_epoch) { before_serialize(); } - - const int64_t &get_leader_epoch() { return leader_epoch_; } - - int ob_admin_dump(share::ObAdminMutatorStringArg &arg); - - static const ObTxLogType LOG_TYPE; - TO_STRING_KV(K(LOG_TYPE), K(leader_epoch_)); - -public: - int before_serialize(); - -private: - ObTxSerCompatByte compat_bytes_; - int64_t leader_epoch_; -}; - class ObTxRollbackToLog { public: @@ -1230,7 +838,7 @@ class ObTxRollbackToLog public: ObTxRollbackToLog() = default; ObTxRollbackToLog(const ObTxSEQ from, const ObTxSEQ to) - : from_(from), to_(to) {before_serialize();} + : from_(from), to_(to) {} ObTxSEQ get_from() const { return from_; } @@ -1241,10 +849,7 @@ class ObTxRollbackToLog static const ObTxLogType LOG_TYPE; TO_STRING_KV(KP(this), K(LOG_TYPE), K_(from), K_(to)); -public: - int before_serialize(); private: - ObTxSerCompatByte compat_bytes_; ObTxSEQ from_; ObTxSEQ to_; }; @@ -1269,45 +874,35 @@ class ObTxLogBlockHeader cluster_version_ = 0; log_entry_no_ = 0; tx_id_ = 0; - scheduler_.reset(); flags_ = 0; - serialize_size_ = 0; } ObTxLogBlockHeader(): __log_entry_no_(log_entry_no_) { reset(); - before_serialize(); }; ObTxLogBlockHeader(const uint64_t org_cluster_id, const int64_t cluster_version, const int64_t log_entry_no, - const ObTransID &tx_id, - const common::ObAddr &scheduler) + const ObTransID &tx_id) : __log_entry_no_(log_entry_no_), flags_(0) { - init(org_cluster_id, cluster_version, log_entry_no, tx_id, scheduler); - before_serialize(); + init(org_cluster_id, cluster_version, log_entry_no, tx_id); } void init(const uint64_t org_cluster_id, const int64_t cluster_version, const int64_t log_entry_no, - const ObTransID &tx_id, - const common::ObAddr &scheduler) { + const ObTransID &tx_id) { org_cluster_id_ = org_cluster_id; cluster_version_ = cluster_version; log_entry_no_ = log_entry_no; tx_id_ = tx_id; - scheduler_ = scheduler; flags_ = 0; - serialize_size_ = 0; } - void calc_serialize_size_(); uint64_t get_org_cluster_id() const { return org_cluster_id_; } int64_t get_cluster_version() const { return cluster_version_; } int64_t get_log_entry_no() const { return log_entry_no_; } void set_log_entry_no(int64_t entry_no) { log_entry_no_ = entry_no; } const ObTransID &get_tx_id() const { return tx_id_; } - const common::ObAddr &get_scheduler() const { return scheduler_; } bool is_valid() const { return org_cluster_id_ >= 0; } void set_serial_final() { flags_ |= SERIAL_FINAL; } @@ -1315,23 +910,19 @@ class ObTxLogBlockHeader void set_has_async_index() { flags_ |= HAS_ASYNC_INDEX; } bool has_async_index() const { return (flags_ & HAS_ASYNC_INDEX) == HAS_ASYNC_INDEX; } uint8_t flags() const { return flags_; } - TO_STRING_KV(K_(compat_bytes), K_(org_cluster_id), K_(cluster_version), K_(log_entry_no), K_(tx_id), K_(scheduler), K_(flags)); + TO_STRING_KV(K_(org_cluster_id), K_(cluster_version), K_(log_entry_no), K_(tx_id), K_(flags)); public: - int before_serialize(); // the last serial log static const uint8_t SERIAL_FINAL = ((uint8_t)1) << 0; // tx contains redo that involves tables with async indexes (for Change Stream fast filtering) static const uint8_t HAS_ASYNC_INDEX = ((uint8_t)1) << 1; private: - int64_t serialize_size_; - ObTxSerCompatByte compat_bytes_; uint64_t org_cluster_id_; int64_t cluster_version_; int64_t log_entry_no_; FixSizeTrait_int64_t __log_entry_no_; // serialize helper member, hiden for others ObTransID tx_id_; - common::ObAddr scheduler_; uint8_t flags_; }; @@ -1574,8 +1165,6 @@ int ObTxLogBlock::add_new_log(T &tx_log_body, ObTxBigSegmentBuf *big_segment_buf } else if (T::LOG_TYPE == ObTxLogType::TX_REDO_LOG) { // ret = OB_EAGAIN; TRANS_LOG(DEBUG, "insert redo_log type into cb_arg_array_", K(tx_log_body), KPC(this)); - } else if (OB_FAIL(tx_log_body.before_serialize())) { - TRANS_LOG(WARN, "before serialize failed", K(ret), K(tx_log_body), K(*this)); } else { const int64_t serialize_size = header.get_serialize_size() + tx_log_body.get_serialize_size(); if (ObTxLogTypeChecker::can_be_spilt(T::LOG_TYPE) && BIG_SEGMENT_SPILT_SIZE < serialize_size + tmp_pos) { diff --git a/src/storage/tx/ob_tx_log_adapter.cpp b/src/storage/tx/ob_tx_log_adapter.cpp index 8a152e7a6..e65c7fc8b 100644 --- a/src/storage/tx/ob_tx_log_adapter.cpp +++ b/src/storage/tx/ob_tx_log_adapter.cpp @@ -118,30 +118,6 @@ int ObLSTxLogAdapter::submit_log(const char *buf, return ret; } -int ObLSTxLogAdapter::get_role(bool &is_leader, int64_t &epoch) -{ - int ret = OB_SUCCESS; - - ObRole role = INVALID_ROLE; - if (OB_ISNULL(log_handler_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), KP(log_handler_)); - } else if (OB_FAIL(log_handler_->get_role(role, epoch))) { - if (ret == OB_NOT_INIT || ret == OB_NOT_RUNNING) { - ret = OB_SUCCESS; - is_leader = false; - } else { - TRANS_LOG(WARN, "get role failed", K(ret)); - } - } else if (LEADER == role) { - is_leader = true; - } else { - is_leader = false; - } - - return ret; -} - int ObLSTxLogAdapter::get_max_decided_scn(SCN &scn) { int ret = OB_SUCCESS; @@ -154,21 +130,6 @@ int ObLSTxLogAdapter::get_max_decided_scn(SCN &scn) return ret; } -int ObLSTxLogAdapter::get_palf_committed_max_scn(share::SCN &scn) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(log_handler_) || !log_handler_->is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), KP(log_handler_)); - } else if (OB_FAIL(log_handler_->get_max_decided_scn_as_leader(scn))) { - TRANS_LOG(WARN, "get palf committed_max_scn fail", K(ret)); - } else if (!scn.is_valid_and_not_min()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "get an invalid scn", K(ret), K(scn)); - } - return ret; -} - int ObLSTxLogAdapter::get_append_mode_initial_scn(share::SCN &ref_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_log_adapter.h b/src/storage/tx/ob_tx_log_adapter.h index afde6cb4a..6a6b84471 100644 --- a/src/storage/tx/ob_tx_log_adapter.h +++ b/src/storage/tx/ob_tx_log_adapter.h @@ -67,9 +67,7 @@ class ObITxLogAdapter const bool need_nonblock, const int64_t retry_timeout_us = 1000) = 0; - virtual int get_role(bool &is_leader, int64_t &epoch) = 0; virtual int get_max_decided_scn(share::SCN &scn) = 0; - virtual int get_palf_committed_max_scn(share::SCN &scn) const = 0; virtual int get_append_mode_initial_scn(share::SCN &ref_scn) = 0; }; @@ -87,9 +85,7 @@ class ObLSTxLogAdapter : public ObITxLogAdapter ObTxBaseLogCb *cb, const bool need_nonblock, const int64_t retry_timeout_us = 1000); - int get_role(bool &is_leader, int64_t &epoch); int get_max_decided_scn(share::SCN &scn); - int get_palf_committed_max_scn(share::SCN &scn) const; int get_append_mode_initial_scn(share::SCN &ref_scn); diff --git a/src/storage/tx/ob_tx_log_cb_define.cpp b/src/storage/tx/ob_tx_log_cb_define.cpp index 0e55ae1ed..2e492e8a5 100644 --- a/src/storage/tx/ob_tx_log_cb_define.cpp +++ b/src/storage/tx/ob_tx_log_cb_define.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_tx_log_cb_define.h" namespace oceanbase @@ -48,7 +48,7 @@ int ObTxLogCbGroup::init(const int64_t group_no) return ret; } -int ObTxLogCbGroup::occupy_by_tx(ObPartTransCtx *tx_ctx) +int ObTxLogCbGroup::occupy_by_tx(ObTxCtx *tx_ctx) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_log_cb_define.h b/src/storage/tx/ob_tx_log_cb_define.h index 22d0a9f37..52f987b40 100644 --- a/src/storage/tx/ob_tx_log_cb_define.h +++ b/src/storage/tx/ob_tx_log_cb_define.h @@ -56,14 +56,14 @@ class ObTxLogCbGroup : public common::ObDLinkBase bool is_occupied() const { return tx_id_.get_id() > 0; } bool is_reserved() const { return get_group_no() == RESERVED_LOG_CB_GROUP_NO; } - int occupy_by_tx(ObPartTransCtx *tx_ctx); + int occupy_by_tx(ObTxCtx *tx_ctx); int check_and_reset_log_cbs(const bool skip_check); int64_t get_group_no() const { return ATOMIC_LOAD(&group_no_); } int64_t get_occupy_ts() const { return occupy_ts_; } - ObPartTransCtx *get_tx_ctx() const { return tx_ctx_; } + ObTxCtx *get_tx_ctx() const { return tx_ctx_; } const ObTransID get_trans_id() const { return tx_id_; } ObTxLogCb *get_log_cb_by_index(const int64_t index) @@ -99,7 +99,7 @@ class ObTxLogCbGroup : public common::ObDLinkBase private: int64_t group_no_; ObTransID tx_id_; - ObPartTransCtx *tx_ctx_; + ObTxCtx *tx_ctx_; ObTxLogCb log_cbs_[MAX_LOG_CB_COUNT_IN_GROUP]; int64_t occupy_ts_; diff --git a/src/storage/tx/ob_tx_log_cb_mgr.cpp b/src/storage/tx/ob_tx_log_cb_mgr.cpp index e99e9c5b5..2399a60cc 100644 --- a/src/storage/tx/ob_tx_log_cb_mgr.cpp +++ b/src/storage/tx/ob_tx_log_cb_mgr.cpp @@ -15,14 +15,14 @@ */ #include "storage/tx/ob_tx_log_cb_mgr.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { namespace transaction { -int ObTxLogCbPoolMgr::init(const ObLSID ls_id) +int ObTxLogCbPoolMgr::init() { int ret = OB_SUCCESS; @@ -32,7 +32,6 @@ int ObTxLogCbPoolMgr::init(const ObLSID ls_id) } else { clear_sync_size_history_(); - ls_id_ = ls_id; ATOMIC_STORE(&is_inited_, true); } @@ -43,7 +42,7 @@ void ObTxLogCbPoolMgr::reset() { ATOMIC_STORE(&is_inited_, false); ATOMIC_STORE(&idle_pool_ptr_, nullptr); - ATOMIC_STORE(&ls_occupying_cnt_, 0); + ATOMIC_STORE(&occupying_cnt_, 0); ATOMIC_STORE(&acquire_extra_log_cb_group_failed_cnt_, 0); clear_sync_size_history_(); { @@ -97,44 +96,6 @@ int ObTxLogCbPoolMgr::clear_log_cb_pool(const bool for_replay) return ret; } -int ObTxLogCbPoolMgr::switch_to_leader(const int64_t active_tx_cnt) -{ - int ret = OB_SUCCESS; - ATOMIC_STORE(&allow_expand_, true); - - const int64_t active_tx_default_log_pool_cnt = - ObTxLogCbGroup::ACTIVE_TX_DEFAULT_LOG_GROUP_COUNT * active_tx_cnt - % ObTxLogCbPool::MAX_LOG_CB_GROUP_COUNT_IN_POOL - == 0 - ? ObTxLogCbGroup::ACTIVE_TX_DEFAULT_LOG_GROUP_COUNT * active_tx_cnt - / ObTxLogCbPool::MAX_LOG_CB_GROUP_COUNT_IN_POOL - : ObTxLogCbGroup::ACTIVE_TX_DEFAULT_LOG_GROUP_COUNT * active_tx_cnt - / ObTxLogCbPool::MAX_LOG_CB_GROUP_COUNT_IN_POOL - + 1; - int64_t target_log_pool_cnt = - active_tx_default_log_pool_cnt <= 0 ? 1 : active_tx_default_log_pool_cnt; - - if (OB_SUCC(ret)) { - int64_t pool_list_size = 0; - { - SpinRLockGuard guard(pool_list_rw_lock_); - const int64_t pool_list_size = pool_list_.get_size(); - } - if (target_log_pool_cnt >= pool_list_size) { - // do nothing - } else { - for (int i = pool_list_size; OB_SUCC(ret) && i < target_log_pool_cnt; i++) { - if (OB_FAIL(append_new_log_cb_pool_())) { - TRANS_LOG(WARN, "append a new log cb pool failed", K(ret), K(i), K(target_log_pool_cnt), - K(active_tx_default_log_pool_cnt), K(pool_list_size), KPC(this)); - } - } - } - } - - return ret; -} - #ifdef ERRSIM ERRSIM_POINT_DEF(EN_ADJUST_TX_LOG_CB_POOL) #endif @@ -332,7 +293,7 @@ int ObTxLogCbPoolMgr::adjust_log_cb_pool(const int64_t active_tx_cnt) return ret; } -int ObTxLogCbPoolMgr::acquire_idle_log_cb_group(ObTxLogCbGroup *&group_ptr, ObPartTransCtx *tx_ctx) +int ObTxLogCbPoolMgr::acquire_idle_log_cb_group(ObTxLogCbGroup *&group_ptr, ObTxCtx *tx_ctx) { int ret = OB_SUCCESS; @@ -355,11 +316,11 @@ int ObTxLogCbPoolMgr::acquire_idle_log_cb_group(ObTxLogCbGroup *&group_ptr, ObPa KPC(this)); } else { TRANS_LOG(INFO, "[Log Cb Group Life] occupy a idle group by tx", K(ret), - K(tx_ctx->get_trans_id()), K(tx_ctx->get_ls_id()), K(ref_guard), KPC(group_ptr)); + K(tx_ctx->get_trans_id()), K(ref_guard), KPC(group_ptr)); } if (OB_SUCC(ret)) { - ATOMIC_INC(&ls_occupying_cnt_); + ATOMIC_INC(&occupying_cnt_); } else if (OB_TX_NOLOGCB == ret) { ATOMIC_INC(&acquire_extra_log_cb_group_failed_cnt_); } @@ -374,7 +335,7 @@ bool ObTxLogCbPoolMgr::is_all_busy() return pool_list_.get_size() == 0 || (pool_list_.get_size() > 0 - && ATOMIC_LOAD(&ls_occupying_cnt_) + && ATOMIC_LOAD(&occupying_cnt_) == pool_list_.get_size() * ObTxLogCbPool::MAX_LOG_CB_GROUP_COUNT_IN_POOL); } @@ -569,8 +530,7 @@ int ObTxLogCbPoolMgr::print_sync_size_history_() } } - _TRANS_LOG(INFO, "[LogCbPool Adjust] print sync size history : { %s }", ls_id_.id(), - sync_size_his_print_buf); + _TRANS_LOG(INFO, "[LogCbPool Adjust] print sync size history: { %s }", sync_size_his_print_buf); return ret; } diff --git a/src/storage/tx/ob_tx_log_cb_mgr.h b/src/storage/tx/ob_tx_log_cb_mgr.h index 18092f772..07e3c0597 100644 --- a/src/storage/tx/ob_tx_log_cb_mgr.h +++ b/src/storage/tx/ob_tx_log_cb_mgr.h @@ -28,7 +28,7 @@ namespace transaction #define MAX_SYNC_SIZE_HISTORY_RECORD_SIZE 10 // #define MAX_SYNC_SIZE_HISTORY_ARRAY_SIZE = MAX_SYNC_SIZE_HISTORY_RECORD_SIZE * 2; -class ObPartTransCtx; +class ObTxCtx; typedef common::ObDList TxLogCbPoolList; @@ -40,23 +40,20 @@ class ObTxLogCbPoolMgr public: ObTxLogCbPoolMgr() : allocator_("TxLogCbPool") { reset(); } - int init(const ObLSID ls_id); + int init(); void reset(); void destroy(); - // release in switch_to_follower or replay int clear_log_cb_pool(const bool for_replay); - int switch_to_leader(const int64_t active_tx_cnt); int adjust_log_cb_pool(const int64_t active_tx_cnt); - int acquire_idle_log_cb_group(ObTxLogCbGroup *&group_ptr, ObPartTransCtx *tx_ctx); + int acquire_idle_log_cb_group(ObTxLogCbGroup *&group_ptr, ObTxCtx *tx_ctx); - void dec_ls_occupying_cnt() {ATOMIC_DEC(&ls_occupying_cnt_);} + void dec_occupying_cnt() {ATOMIC_DEC(&occupying_cnt_);} bool is_all_busy(); - TO_STRING_KV(K(ls_id_), - K(is_inited_), + TO_STRING_KV(K(is_inited_), K(allow_expand_), K(pool_list_.get_size()), KP(idle_pool_ptr_)); @@ -104,7 +101,6 @@ class ObTxLogCbPoolMgr private: bool is_inited_; - ObLSID ls_id_; TransModulePageAllocator allocator_; common::SpinRWLock pool_list_rw_lock_; @@ -114,7 +110,7 @@ class ObTxLogCbPoolMgr common::SpinRWLock sync_size_his_lock_; int64_t sync_size_history_[MAX_SYNC_SIZE_HISTORY_RECORD_SIZE * 2]; - int64_t ls_occupying_cnt_; + int64_t occupying_cnt_; int64_t acquire_extra_log_cb_group_failed_cnt_; // modified by the read-only lock diff --git a/src/storage/tx/ob_tx_log_operator.h b/src/storage/tx/ob_tx_log_operator.h index 09c152307..b160f498f 100644 --- a/src/storage/tx/ob_tx_log_operator.h +++ b/src/storage/tx/ob_tx_log_operator.h @@ -19,7 +19,7 @@ #include "storage/ls/ob_ls_ddl_log_handler.h" #include "storage/tx/ob_trans_define.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_tx_log.h" namespace oceanbase @@ -104,13 +104,12 @@ class ObTxCtxLogOperator int tmp_ret = OB_SUCCESS; if (OB_FAIL(deserialize_log_())) { TRANS_LOG(WARN, "deserialize the log failed", K(ret), K(tx_ctx_->trans_id_), - K(tx_ctx_->ls_id_), KPC(this)); + KPC(this)); } else if (OB_FAIL(replay_out_ctx_())) { TRANS_LOG(WARN, "replay log out ctx failed", K(ret), K(tx_ctx_->trans_id_), - K(tx_ctx_->ls_id_), KPC(this)); - } else if (OB_FAIL(replay_in_ctx_())) { - TRANS_LOG(WARN, "replay log in ctx failed", K(ret), K(tx_ctx_->trans_id_), K(tx_ctx_->ls_id_), KPC(this)); + } else if (OB_FAIL(replay_in_ctx_())) { + TRANS_LOG(WARN, "replay log in ctx failed", K(ret), K(tx_ctx_->trans_id_), KPC(this)); } if (OB_FAIL(ret)) { @@ -124,7 +123,7 @@ class ObTxCtxLogOperator public: // submit - ObTxCtxLogOperator(ObPartTransCtx *tx_ctx_ptr, + ObTxCtxLogOperator(ObTxCtx *tx_ctx_ptr, ObTxLogBlock *log_block_ptr, typename T::ConstructArg *construct_arg, const typename T::SubmitArg &submit_arg) @@ -136,7 +135,7 @@ class ObTxCtxLogOperator }; // apply - ObTxCtxLogOperator(ObPartTransCtx *tx_ctx_ptr, ObTxLogCb *log_cb_ptr) + ObTxCtxLogOperator(ObTxCtx *tx_ctx_ptr, ObTxLogCb *log_cb_ptr) : tx_ctx_(tx_ctx_ptr), log_object_ptr_(nullptr), log_block_(nullptr), construct_arg_(nullptr), scn_(log_cb_ptr->get_log_ts()), lsn_(log_cb_ptr->get_lsn()) { @@ -145,7 +144,7 @@ class ObTxCtxLogOperator }; // replay - ObTxCtxLogOperator(ObPartTransCtx *tx_ctx_ptr, + ObTxCtxLogOperator(ObTxCtx *tx_ctx_ptr, ObTxLogBlock *log_block_ptr, typename T::ConstructArg *construct_arg, const typename T::ReplayArg &replay_arg, @@ -174,7 +173,7 @@ class ObTxCtxLogOperator const palf::LSN &get_lsn() { return lsn_; } private: - ObPartTransCtx *tx_ctx_; + ObTxCtx *tx_ctx_; char log_object_memory_[sizeof(T)]; T *log_object_ptr_; ObTxLogBlock *log_block_; @@ -214,8 +213,7 @@ OB_INLINE int ObTxCtxLogOperator::prepare_generic_resource_() // block. // the log_entry_no will be backfill before log-block to be submitted log_block_->get_header().init(tx_ctx_->cluster_id_, tx_ctx_->cluster_version_, - INT64_MAX /*log_entry_no*/, tx_ctx_->trans_id_, - tx_ctx_->exec_info_.scheduler_); + INT64_MAX /*log_entry_no*/, tx_ctx_->trans_id_); } log_op_arg_.submit_arg_.suggested_buf_size_ = log_op_arg_.submit_arg_.suggested_buf_size_ <= 0 ? ObTxAdaptiveLogBuf::NORMAL_LOG_BUF_SIZE @@ -232,8 +230,7 @@ OB_INLINE int ObTxCtxLogOperator::prepare_generic_resource_() TRANS_LOG(WARN, "get log cb failed", K(ret), KPC(this)); } } else if (OB_FAIL(tx_ctx_->acquire_ctx_ref_())) { - TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(tx_ctx_->trans_id_), K(tx_ctx_->ls_id_), - KPC(this)); + TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(tx_ctx_->trans_id_), KPC(this)); } else if (OB_FALSE_IT(log_op_arg_.submit_arg_.hold_tx_ctx_ref_ = true)) { // do nothing } @@ -285,7 +282,7 @@ OB_INLINE int ObTxCtxLogOperator::insert_into_log_block_() TRANS_LOG(WARN, "add new log failed", KR(ret), KPC(this)); } else { TRANS_LOG(DEBUG, "the buffer is not enough in log_block", K(ret), K(tx_ctx_->trans_id_), - K(tx_ctx_->ls_id_), KPC(this)); + KPC(this)); } } @@ -304,8 +301,6 @@ template OB_INLINE int ObTxCtxLogOperator::submit_log_block_out_() { int ret = OB_SUCCESS; - bool is_2pc_state_log = false; - if (tx_ctx_->is_exiting()) { ret = OB_TRANS_IS_EXITING; TRANS_LOG(WARN, "the tx ctx is exiting", K(ret), K(T::LOG_TYPE), KPC(tx_ctx_)); @@ -314,25 +309,15 @@ OB_INLINE int ObTxCtxLogOperator::submit_log_block_out_() ret = OB_TRANS_KILLED; TRANS_LOG(ERROR, "tx has been aborting, can not submit log", K(ret), K(T::LOG_TYPE), KPC(tx_ctx_)); - } else if (tx_ctx_->is_follower_()) { - ret = OB_NOT_MASTER; - TRANS_LOG(ERROR, "we can not submit a tx log on the follower", K(ret), K(T::LOG_TYPE), + } else if (tx_ctx_->is_for_replay()) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(ERROR, "a replay context cannot submit transaction logs", K(ret), K(T::LOG_TYPE), KPC(tx_ctx_)); - } else if (tx_ctx_->exec_info_.data_complete_ - && tx_ctx_->start_working_log_ts_ > tx_ctx_->exec_info_.max_applying_log_ts_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, - "There exists a data completed transaction whose start_working_log_ts_ is greater " - "than any of its log_ts", - K(ret), K(T::LOG_TYPE), KPC(tx_ctx_)); - tx_ctx_->print_trace_log_(); } else if (ObTxLogTypeChecker::is_data_log(T::LOG_TYPE) && tx_ctx_->get_downstream_state() >= ObTxState::REDO_COMPLETE) { ret = OB_STATE_NOT_MATCH; TRANS_LOG(ERROR, "the data log can not be submitted after the commit info log", K(ret), K(T::LOG_TYPE), KPC(tx_ctx_)); - } else if (is_contain_stat_log(log_block_->get_cb_arg_array()) - && FALSE_IT(is_2pc_state_log = true)) { } else { const int64_t real_replay_hint = log_op_arg_.submit_arg_.replay_hint_ > 0 ? log_op_arg_.submit_arg_.replay_hint_ @@ -402,13 +387,11 @@ OB_INLINE void ObTxCtxLogOperator::after_submit_log_fail_(const int submit_re if (log_op_arg_.submit_arg_.log_cb_->get_prev() != nullptr || log_op_arg_.submit_arg_.log_cb_->get_next() != nullptr) { TRANS_LOG(ERROR, "the log cb is not alone", K(submit_ret), K(tmp_ret), - K(tx_ctx_->get_trans_id()), K(tx_ctx_->get_ls_id()), - KPC(log_op_arg_.submit_arg_.log_cb_), K(T::LOG_TYPE), KPC(this)); + K(tx_ctx_->get_trans_id()), KPC(log_op_arg_.submit_arg_.log_cb_), K(T::LOG_TYPE), KPC(this)); } if (OB_TMP_FAIL(tx_ctx_->return_log_cb_(log_op_arg_.submit_arg_.log_cb_))) { TRANS_LOG(ERROR, "free the log cb failed", K(submit_ret), K(tmp_ret), - K(tx_ctx_->get_trans_id()), K(tx_ctx_->get_ls_id()), - KPC(log_op_arg_.submit_arg_.log_cb_), K(T::LOG_TYPE), KPC(this)); + K(tx_ctx_->get_trans_id()), KPC(log_op_arg_.submit_arg_.log_cb_), K(T::LOG_TYPE), KPC(this)); } } if (log_op_arg_.submit_arg_.hold_tx_ctx_ref_) { diff --git a/src/storage/tx/ob_tx_loop_worker.cpp b/src/storage/tx/ob_tx_loop_worker.cpp index ca32fc618..e3a8b1d13 100644 --- a/src/storage/tx/ob_tx_loop_worker.cpp +++ b/src/storage/tx/ob_tx_loop_worker.cpp @@ -19,6 +19,7 @@ #include "storage/tx/ob_trans_service.h" #include "storage/tx/ob_leak_checker.h" #include "storage/tx/ob_weak_read_util.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/ls/ob_ls.h" namespace oceanbase @@ -87,7 +88,6 @@ void ObTxLoopWorker::reset() { last_tx_gc_ts_ = 0; last_retain_ctx_gc_ts_ = 0; - last_check_start_working_retry_ts_ = 0; last_log_cb_pool_adjust_ts_ = 0; last_tenant_config_refresh_ts_ = 0; stop_flag_ = true; @@ -101,7 +101,6 @@ void ObTxLoopWorker::runTimerTask() lib::set_thread_name("TxLoopWorker"); bool can_gc_tx = false; bool can_gc_retain_ctx = false; - bool can_check_and_retry_start_working = false; bool can_adjust_log_cb_pool = false; start_time_us = ObTimeUtility::current_time(); @@ -123,13 +122,6 @@ void ObTxLoopWorker::runTimerTask() can_gc_retain_ctx = true; } - if (common::ObClockGenerator::getClock() - last_check_start_working_retry_ts_ - > TX_START_WORKING_RETRY_INTERVAL) { - TRANS_LOG(INFO, "try to retry start_working"); - last_check_start_working_retry_ts_ = common::ObClockGenerator::getClock(); - can_check_and_retry_start_working = true; - } - if (common::ObClockGenerator::getClock() - last_log_cb_pool_adjust_ts_ > TX_LOG_CB_POOL_ADJUST_INTERVAL) { TRANS_LOG(INFO, "try to adjust log cb pool"); @@ -144,7 +136,7 @@ void ObTxLoopWorker::runTimerTask() last_tenant_config_refresh_ts_ = common::ObClockGenerator::getClock(); } - (void)scan_all_ls_(can_gc_tx, can_gc_retain_ctx, can_check_and_retry_start_working, can_adjust_log_cb_pool); + (void)maintain_tx_state_(can_gc_tx, can_gc_retain_ctx, can_adjust_log_cb_pool); // TODO shanyan.g // 1) We use max(max_commit_ts, gts_cache) as read snapshot, @@ -156,106 +148,68 @@ void ObTxLoopWorker::runTimerTask() UNUSED(time_used); can_gc_tx = false; can_gc_retain_ctx = false; - can_check_and_retry_start_working = false; can_adjust_log_cb_pool = false; } -int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, - bool can_gc_retain_ctx, - bool can_check_and_retry_start_working, - bool can_adjust_log_cb_pool) +int ObTxLoopWorker::maintain_tx_state_(bool can_tx_gc, + bool can_gc_retain_ctx, + bool can_adjust_log_cb_pool) { int ret = OB_SUCCESS; - int iter_ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObSharedGuard ls_iter_guard; - ObLSIterator *iter_ptr = nullptr; + ObLS *tenant_ls = nullptr; ObLS *cur_ls_ptr = nullptr; - int64_t ls_cnt = 0; - - if (OB_ISNULL(share::g_mp->ls_service()) - || OB_FAIL(share::g_mp->ls_service()->get_ls_iter(ls_iter_guard, ObLSGetMod::TRANS_MOD)) - || !ls_iter_guard.is_valid()) { - if (OB_SUCCESS == ret) { - ret = OB_INVALID_ARGUMENT; - } - TRANS_LOG(WARN, "[Tx Loop Worker] get ls iter failed", K(ret), KP(share::g_mp->ls_service())); - } else if (OB_ISNULL(iter_ptr = ls_iter_guard.get_ptr())) { - TRANS_LOG(WARN, "[Tx Loop Worker] ls iter_ptr is nullptr", KP(iter_ptr)); + if (OB_ISNULL(share::g_mp->ls_service())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "[Tx Loop Worker] ls service is null", K(ret), KP(share::g_mp->ls_service())); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + TRANS_LOG(WARN, "[Tx Loop Worker] get transaction storage failed", K(ret), KP(share::g_mp->ls_service())); } else { - iter_ret = OB_SUCCESS; - cur_ls_ptr = nullptr; - while (OB_SUCCESS == (iter_ret = iter_ptr->get_next(cur_ls_ptr))) { - SCN min_start_scn = SCN::invalid_scn(); - SCN max_decided_scn = SCN::invalid_scn(); - MinStartScnStatus status = MinStartScnStatus::UNKOWN; - common::ObRole role = common::ObRole::INVALID_ROLE; - int64_t base_proposal_id, proposal_id; - - if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_role(role, base_proposal_id))) { - TRANS_LOG(WARN, "get role failed", K(tmp_ret), K(cur_ls_ptr->get_ls_id())); - status = MinStartScnStatus::UNKOWN; - } else if (role == common::ObRole::FOLLOWER) { - status = MinStartScnStatus::UNKOWN; - } - - // tx gc, interval = 15s - if (can_tx_gc) { - // TODO shanyan.g close ctx gc temporarily because of logical bug - // - - // ATTENTION : get_max_decided_scn must before iterating all trans ctx. - // set max_decided_scn as default value - if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_max_decided_scn(max_decided_scn))) { - TRANS_LOG(WARN, "get max decided scn failed", KR(tmp_ret), K(min_start_scn)); - max_decided_scn.set_invalid(); - } else { - (void)cur_ls_ptr->update_min_start_scn_info(max_decided_scn); - } - min_start_scn = max_decided_scn; - do_tx_gc_(cur_ls_ptr, min_start_scn, status); - } - - if (MinStartScnStatus::UNKOWN == status) { - // do nothing - } else if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_role(role, proposal_id))) { - TRANS_LOG(WARN, "get role failed", K(tmp_ret), K(cur_ls_ptr->get_ls_id())); - status = MinStartScnStatus::UNKOWN; - } else if (role == common::ObRole::FOLLOWER) { - status = MinStartScnStatus::UNKOWN; - } else if (base_proposal_id != proposal_id) { - status = MinStartScnStatus::UNKOWN; + cur_ls_ptr = tenant_ls; + SCN min_start_scn = SCN::invalid_scn(); + SCN max_decided_scn = SCN::invalid_scn(); + MinStartScnStatus status = MinStartScnStatus::UNKOWN; + // tx gc, interval = 15s + if (can_tx_gc) { + // TODO shanyan.g close ctx gc temporarily because of logical bug + // + + // ATTENTION : get_max_decided_scn must before iterating all trans ctx. + // set max_decided_scn as default value + if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_max_decided_scn(max_decided_scn))) { + TRANS_LOG(WARN, "get max decided scn failed", KR(tmp_ret), K(min_start_scn)); + max_decided_scn.set_invalid(); + } else { + (void)cur_ls_ptr->update_min_start_scn_info(max_decided_scn); } + min_start_scn = max_decided_scn; + do_tx_gc_(cur_ls_ptr, min_start_scn, status); + } - if (MinStartScnStatus::UNKOWN == status) { - min_start_scn.reset(); - } else if (MinStartScnStatus::NO_CTX == status) { - min_start_scn.set_min(); - } + if (MinStartScnStatus::UNKOWN == status) { + min_start_scn.reset(); + } else if (MinStartScnStatus::NO_CTX == status) { + min_start_scn.set_min(); + } - // keep alive, interval = 5s - do_keep_alive_(cur_ls_ptr, min_start_scn, status); + // keep alive, interval = 5s + do_keep_alive_(cur_ls_ptr, min_start_scn, status); - // Single-node lite: drive each LS's local weak-read timestamp here (the - // tenant WRS service that used to do this has been deleted). Keeps - // ls_weak_read_ts_ advancing for readability/compaction/replay. - do_update_ls_weak_read_ts_(cur_ls_ptr); + // Drive the local weak-read timestamp here. The tenant WRS service that + // used to do this has been deleted. + do_update_ls_weak_read_ts_(cur_ls_ptr); - if (can_gc_retain_ctx) { - do_retain_ctx_gc_(cur_ls_ptr); - } + if (can_gc_retain_ctx) { + do_retain_ctx_gc_(cur_ls_ptr); + } - if (can_check_and_retry_start_working) { - do_start_working_retry_(cur_ls_ptr); - } - // ignore ret - (void)cur_ls_ptr->get_tx_svr()->check_all_readonly_tx_clean_up(); + // ignore ret + (void)cur_ls_ptr->get_tx_svr()->check_all_readonly_tx_clean_up(); - if (can_adjust_log_cb_pool) { - do_log_cb_pool_adjust_(cur_ls_ptr, role); - } + if (can_adjust_log_cb_pool) { + do_log_cb_pool_adjust_(cur_ls_ptr); } } @@ -282,16 +236,12 @@ void ObTxLoopWorker::do_update_ls_weak_read_ts_(ObLS *ls_ptr) int ret = OB_SUCCESS; SCN version; bool need_skip = false; - bool is_user_ls = false; const int64_t max_stale_time = ObWeakReadUtil::max_stale_time_for_weak_consistency( ObWeakReadUtil::IGNORE_TENANT_EXIST_WARN); - if (OB_ISNULL(ls_ptr)) { - // do nothing - } else if (OB_FAIL(ls_ptr->get_ls_wrs_handler()->generate_ls_weak_read_snapshot_version( - *ls_ptr, need_skip, is_user_ls, version, max_stale_time))) { + if (OB_FAIL(ls_ptr->get_ls_wrs_handler()->generate_ls_weak_read_snapshot_version( + *ls_ptr, need_skip, version, max_stale_time))) { if (REACH_TIME_INTERVAL(5 * 1000 * 1000)) { - TRANS_LOG(WARN, "[Tx Loop Worker] generate ls weak read snapshot version fail", - K(ret), K(ls_ptr->get_ls_id())); + TRANS_LOG(WARN, "[Tx Loop Worker] generate ls weak read snapshot version fail", K(ret)); } } UNUSED(ret); @@ -301,10 +251,10 @@ void ObTxLoopWorker::do_tx_gc_(ObLS *ls_ptr, SCN &min_start_scn, MinStartScnStat { int ret = OB_SUCCESS; - if (OB_FAIL(ls_ptr->get_tx_svr()->check_scheduler_status(min_start_scn, status))) { - TRANS_LOG(WARN, "[Tx Loop Worker] check tx scheduler failed", K(ret), K(*ls_ptr)); + if (OB_FAIL(ls_ptr->get_tx_svr()->check_tx_status(min_start_scn, status))) { + TRANS_LOG(WARN, "[Tx Loop Worker] check transaction status failed", K(ret), K(*ls_ptr)); } else { - TRANS_LOG(INFO, "[Tx Loop Worker] check tx scheduler success", K(*ls_ptr)); + TRANS_LOG(INFO, "[Tx Loop Worker] check transaction status success", K(*ls_ptr)); } UNUSED(ret); @@ -333,7 +283,7 @@ void ObTxLoopWorker::update_max_commit_ts_() int64_t n = ObClockGenerator::getClock(); if (n >= expire_ts) { ret = OB_TIMEOUT; - } else if (OB_FAIL(OB_TS_MGR.get_gts(NULL, snapshot))) { + } else if (OB_FAIL(OB_TS_MGR.get_gts(snapshot))) { if (OB_EAGAIN == ret) { ob_usleep(500, true/*is_idle_sleep*/); } else { @@ -369,36 +319,19 @@ void ObTxLoopWorker::do_retain_ctx_gc_(ObLS *ls_ptr) K(*ls_ptr)); } - retain_ctx_mgr->print_retain_ctx_info(ls_ptr->get_ls_id()); - retain_ctx_mgr->try_advance_retain_ctx_gc(ls_ptr->get_ls_id()); + retain_ctx_mgr->print_retain_ctx_info(); + retain_ctx_mgr->try_advance_retain_ctx_gc(); UNUSED(ret); } -void ObTxLoopWorker::do_start_working_retry_(ObLS *ls_ptr) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(ls_ptr->retry_apply_start_working_log())) { - TRANS_LOG(WARN, "retry to apply start working log failed", K(ret), KPC(ls_ptr)); - } -} - -void ObTxLoopWorker::do_log_cb_pool_adjust_(ObLS *ls_ptr, const common::ObRole role) +void ObTxLoopWorker::do_log_cb_pool_adjust_(ObLS *ls_ptr) { int ret = OB_SUCCESS; int64_t active_tx_cnt = 0; (void)ls_ptr->get_tx_svr()->get_active_tx_count(active_tx_cnt); - if (common::is_strong_leader(role)) { - if (OB_FAIL(ls_ptr->get_tx_svr()->get_log_cb_pool_mgr()->adjust_log_cb_pool(active_tx_cnt))) { - TRANS_LOG(WARN, "adjust log cb pool failed", K(ret), K(role), KPC(ls_ptr)); - } - } else { - // log handler follower, not tx follower - if (OB_FAIL( - ls_ptr->get_tx_svr()->get_log_cb_pool_mgr()->clear_log_cb_pool(false /*for replay*/))) { - TRANS_LOG(WARN, "clear log cb pools on a follower failed", K(ret), K(role), KPC(ls_ptr)); - } + if (OB_FAIL(ls_ptr->get_tx_svr()->get_log_cb_pool_mgr()->adjust_log_cb_pool(active_tx_cnt))) { + TRANS_LOG(WARN, "adjust log cb pool failed", K(ret), KPC(ls_ptr)); } } diff --git a/src/storage/tx/ob_tx_loop_worker.h b/src/storage/tx/ob_tx_loop_worker.h index 7a18d7173..1b0d97607 100644 --- a/src/storage/tx/ob_tx_loop_worker.h +++ b/src/storage/tx/ob_tx_loop_worker.h @@ -23,11 +23,6 @@ namespace oceanbase { -namespace share -{ -class ObLSID; -}; - namespace storage { class ObLS; @@ -44,7 +39,6 @@ class ObTxLoopWorker : public common::ObTimerTask const static int64_t KEEP_ALIVE_PRINT_INFO_INTERVAL = 5 * 60 * 1000 * 1000; // 5min const static int64_t TX_GC_INTERVAL = 5 * 1000 * 1000; // 5s const static int64_t TX_RETAIN_CTX_GC_INTERVAL = 5 * 1000 * 1000; // 5s - const static int64_t TX_START_WORKING_RETRY_INTERVAL = 5 * 1000 * 1000; //5s const static int64_t TX_LOG_CB_POOL_ADJUST_INTERVAL = 1 * 60 * 1000 * 1000; // 1min public: ObTxLoopWorker() { reset(); } @@ -61,20 +55,18 @@ class ObTxLoopWorker : public common::ObTimerTask virtual void runTimerTask() override; private: - int scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx, bool can_check_and_retry_start_working, bool can_adjust_log_cb_pool); + int maintain_tx_state_(bool can_tx_gc, bool can_gc_retain_ctx, bool can_adjust_log_cb_pool); void do_keep_alive_(ObLS *ls, const share::SCN &min_start_scn, MinStartScnStatus status); // 100ms void do_update_ls_weak_read_ts_(ObLS *ls); void do_tx_gc_(ObLS *ls, share::SCN &min_start_scn, MinStartScnStatus &status); // 15s void update_max_commit_ts_(); void do_retain_ctx_gc_(ObLS * ls); // 15s - void do_start_working_retry_(ObLS * ls); - void do_log_cb_pool_adjust_(ObLS *ls, const common::ObRole role); + void do_log_cb_pool_adjust_(ObLS *ls); void refresh_tenant_config_(); private: int64_t last_tx_gc_ts_; int64_t last_retain_ctx_gc_ts_; - int64_t last_check_start_working_retry_ts_; int64_t last_log_cb_pool_adjust_ts_; int64_t last_tenant_config_refresh_ts_; common::ObTimer timer_; diff --git a/src/storage/tx/ob_tx_ls_log_writer.cpp b/src/storage/tx/ob_tx_ls_log_writer.cpp deleted file mode 100644 index 6369b644d..000000000 --- a/src/storage/tx/ob_tx_ls_log_writer.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_tx_ls_log_writer.h" -#include "storage/tx/ob_trans_ctx_mgr.h" - -namespace oceanbase -{ - -using namespace logservice; -using namespace palf; -using namespace share; - -namespace transaction -{ - -//****************** ObTxLsLogLimit *********************** - -int64_t ObTxLSLogLimit::LOG_BUF_SIZE = 0; - -uint16_t ObTxLSLogLimit::get_max_parallel(const ObTxLogType &type) -{ - uint16_t tmp_para = 0; - switch (type) { - case ObTxLogType::TX_START_WORKING_LOG: { - tmp_para = START_WORKING_MAX_PARALLEL; - break; - } - default: { - tmp_para = 0; - break; - } - } - return tmp_para; -} - - -void ObTxLSLogLimit::decide_log_buf_size() -{ - LOG_BUF_SIZE = 0; - ObLogBaseHeader base_header; - ObTxLogHeader tx_header; - ObTxLogBlockHeader block_header(UINT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, common::ObAddr()); - ObTxStartWorkingLog sw_log(INT_MAX64); - - block_header.before_serialize(); - // sw_log.before_serialize(); - int64_t max_body_size = 0; - LOG_BUF_SIZE = base_header.get_serialize_size() + tx_header.get_serialize_size() + block_header.get_serialize_size(); - max_body_size = OB_MAX(max_body_size, sw_log.get_serialize_size()); - - LOG_BUF_SIZE = LOG_BUF_SIZE + max_body_size; -} - -//******************************************************** - -//****************** ObTxLSLogCb ************************* -ObTxLSLogCb::ObTxLSLogCb(ObTxLSLogWriter *base) -{ - log_buf_ = nullptr; - reset(); - base_wr_ = base; -} - -ObTxLSLogCb::~ObTxLSLogCb() -{ - ObTxBaseLogCb::reset(); - reset(); -} - -void ObTxLSLogCb::reuse() -{ - ObTxBaseLogCb::reuse(); - type_ = ObTxLogType::UNKNOWN; - pos_ = 0; -} - -void ObTxLSLogCb::reset() -{ - reuse(); - if (nullptr != log_buf_) { - ob_free(log_buf_); - } - log_buf_ = nullptr; -} - -int ObTxLSLogCb::occupy_by(const ObTxLogType &log_type) -{ - int ret = OB_SUCCESS; - if (!ObTxLogTypeChecker::is_ls_log(log_type) || OB_ISNULL(base_wr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KR(ret), K(log_type), KP(base_wr_)); - } else if (ObTxLogType::UNKNOWN != type_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "[TxLsLogWriter] cb has been used", KR(ret)); - } else { - type_ = log_type; - } - return ret; -} - -int ObTxLSLogCb::on_success() -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(base_wr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KP(base_wr_)); - } else if (OB_FAIL(base_wr_->on_success(this))) { - TRANS_LOG(WARN, "[TxLsLogWriter] on_success failed", KR(ret), K(log_ts_), K(type_)); - } - return ret; -} - -int ObTxLSLogCb::on_failure() -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(base_wr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KP(base_wr_)); - } else if (OB_FAIL(base_wr_->on_failure(this))) { - TRANS_LOG(WARN, "[TxLsLogWriter] on_failure failed", KR(ret), K(log_ts_), K(type_)); - } - return ret; -} - -int ObTxLSLogCb::alloc_log_buf_() -{ - int ret = OB_SUCCESS; - - ObMemAttr attr("TxLSLogBuf"); - SET_USE_500(attr); - if (0 == ObTxLSLogLimit::LOG_BUF_SIZE || nullptr != log_buf_) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KR(ret), K(ObTxLSLogLimit::LOG_BUF_SIZE), KP(log_buf_)); - } else if (nullptr - == (log_buf_ = (char *)ob_malloc(ObTxLSLogLimit::LOG_BUF_SIZE, attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "[TxLsLogWriter] allocate memory failed", KR(ret), K(ObTxLSLogLimit::LOG_BUF_SIZE)); - } - return ret; -} -//******************************************************** - -//******************** ObTxLSLogWriter ******************* -ObTxLSLogWriter::ObTxLSLogWriter() : cbs_lock_(common::ObLatchIds::TX_LS_LOG_WRITER_LOCK) { reset(); } - -ObTxLSLogWriter::~ObTxLSLogWriter() { reset(); } - -int ObTxLSLogWriter::init(const ObLSID &ls_id, - ObITxLogAdapter * adapter, - ObLSTxCtxMgr *ctx_mgr) - { - int ret = OB_SUCCESS; - - if (OB_NOT_NULL(tx_log_adapter_) || OB_NOT_NULL(ctx_mgr_)) { - ret = OB_INIT_TWICE; - } else if (!ls_id.is_valid() || OB_ISNULL(adapter) || OB_ISNULL(ctx_mgr)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", K(ls_id), KP(adapter), KP(ctx_mgr)); - } else { - - ls_id_ = ls_id; - - ctx_mgr_ = ctx_mgr; - tx_log_adapter_ = adapter; - ObTxLSLogLimit::decide_log_buf_size(); - while (free_cbs_.get_size() < DEFAULT_LOG_CB_CNT && OB_SUCC(ret)) { - if (OB_FAIL(append_free_log_cb_())) { - TRANS_LOG(WARN, "init free log cb error", KR(ret)); - } - } - } - - if (OB_SUCC(ret)) { - ATOMIC_STORE(&is_stopped_, false); - TRANS_LOG(DEBUG, "[TxLsLogWriter] init success", K(ret), K(ls_id_)); - } else { - TRANS_LOG(WARN, "[TxLsLogWriter] init failed", K(ret), K(ls_id_)); - } - return ret; -} - -int ObTxLSLogWriter::stop() -{ - int ret = OB_SUCCESS; - - ATOMIC_STORE(&is_stopped_, true); - - return ret; -} - -int ObTxLSLogWriter::wait() -{ - int ret = OB_SUCCESS; - ObSpinLockGuard cbs_guard(cbs_lock_); - if (!keep_alive_cbs_.is_empty() || !start_working_cbs_.is_empty()) { - ret = OB_EAGAIN; - } - return ret; -} - - -void ObTxLSLogWriter::reset() -{ - destroy(); - ATOMIC_STORE(&is_stopped_, true); - keep_alive_cbs_.reset(); - start_working_cbs_.reset(); - free_cbs_.reset(); - - ls_id_.reset(); - ctx_mgr_ = nullptr; - tx_log_adapter_ = nullptr; -} - -void ObTxLSLogWriter::destroy() -{ - destroy_cbs_(keep_alive_cbs_); - destroy_cbs_(start_working_cbs_); - destroy_cbs_(free_cbs_); -} - -int ObTxLSLogWriter::submit_start_working_log(const int64_t &leader_epoch, SCN &log_ts) -{ - int ret = OB_SUCCESS; - - ObTxStartWorkingLog sw_log(leader_epoch); - if (OB_FAIL(submit_ls_log_(sw_log, logservice::ObReplayBarrierType::STRICT_BARRIER, false, log_ts))) { - TRANS_LOG(WARN, "[TxLsLogWriter] submit start working log failed", KR(ret)); - } - - return ret; -} - -int ObTxLSLogWriter::on_success(ObTxLSLogCb *cb) -{ - int ret = OB_SUCCESS; - - ObSpinLockGuard cb_guard(cbs_lock_); - - const ObTxLogType log_type = cb->get_log_type(); - - switch (log_type) { - case ObTxLogType::TX_START_WORKING_LOG: { - if (OB_FAIL(ctx_mgr_->on_start_working_log_cb_succ(cb->get_log_ts()))) { - TRANS_LOG(WARN, "start working log callback failed", KR(ret)); - } - break; - } - // TODO, other types - default: { - TRANS_LOG(WARN, "unknown log type", K(log_type)); - } - } - return_log_cb_(cb); - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "[TxLsLogWriter] on success", KR(ret), K(cb->get_log_type()), - K(cb->get_log_ts())); - } - return ret; -} - -int ObTxLSLogWriter::on_failure(ObTxLSLogCb *cb) -{ - int ret = OB_SUCCESS; - - ObSpinLockGuard cb_guard(cbs_lock_); - - const ObTxLogType log_type = cb->get_log_type(); - - switch (log_type) { - case ObTxLogType::TX_START_WORKING_LOG: { - if (OB_FAIL(ctx_mgr_->on_start_working_log_cb_fail())) { - TRANS_LOG(WARN, "start working log callback failed", KR(ret)); - } - break; - } - // TODO, other types - default: { - TRANS_LOG(WARN, "unknown log type", K(log_type)); - } - } - TRANS_LOG(INFO, "[TxLsLogWriter] on failure", KR(ret), K(cb->get_log_type()), - K(cb->get_log_ts())); - return_log_cb_(cb); - return ret; -} - -int ObTxLSLogWriter::get_log_cb_(const ObTxLogType &log_type, ObTxLSLogCb *&cb) -{ - int ret = OB_SUCCESS; - ObDList *tmp_cbs = nullptr; - - if (nullptr == (tmp_cbs = get_target_cbs_(log_type))) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] INVALID LOG TYPE", K(log_type)); - } else if (reach_parallel_cbs_limit_(log_type, tmp_cbs->get_size())) { - ret = OB_TX_NOLOGCB; - TRANS_LOG(INFO, "[TxLsLogWriter] reach max parallel limit, need retry", KR(ret), K(log_type), - K(tmp_cbs->get_size())); - } else if (!enough_log_cb_() && (OB_FAIL(append_free_log_cb_()))) { - TRANS_LOG(WARN, "[TxLsLogWriter] append free log_cb error", KR(ret)); - } else { - // move from free_cbs_ - cb = free_cbs_.remove_first(); - if (OB_FAIL(cb->occupy_by(log_type))) { - TRANS_LOG(ERROR, "[TxLsLogWriter] occupy by ls log_cb failed", KR(ret), K(log_type), K(*cb)); - free_cbs_.add_last(cb); - } else { - tmp_cbs->add_last(cb); - } - } - - return ret; -} - -int ObTxLSLogWriter::return_log_cb_(ObTxLSLogCb *cb) -{ - int ret = OB_SUCCESS; - - ObDList *tmp_cbs = nullptr; - - if (nullptr == cb || nullptr == (tmp_cbs = get_target_cbs_(cb->get_log_type()))) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] INVALID LOG TYPE", K(cb->get_log_type())); - } else { - tmp_cbs->remove(cb); - cb->reuse(); - free_cbs_.add_first(cb); - - TRANS_LOG(DEBUG, "[TxLsLogWriter] success return log_cb", K(cb->get_log_type()), - K(tmp_cbs->get_size()), K(free_cbs_.get_size())); - } - return ret; -} - -bool ObTxLSLogWriter::enough_log_cb_() { return free_cbs_.is_empty(); } - -int ObTxLSLogWriter::append_free_log_cb_() -{ - int ret = OB_SUCCESS; - char *cb_buf = nullptr; - ObTxLSLogCb *tmp_cb = nullptr; - if (nullptr - == (cb_buf = (char *)ob_malloc(sizeof(ObTxLSLogCb) * APPEND_LOG_CB_CNT, - ObMemAttr("ObTxLSLogCb")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - for (int i = 0; i < APPEND_LOG_CB_CNT; i++) { - TRANS_LOG(DEBUG, "[TxLsLogWriter] append_free_log_cb", K(ret), K(free_cbs_.get_size()), K(i)); - tmp_cb = new (cb_buf + i * sizeof(ObTxLSLogCb)) ObTxLSLogCb(this); - free_cbs_.add_last(tmp_cb); - } - } - - return ret; -} - -bool ObTxLSLogWriter::reach_parallel_cbs_limit_(const ObTxLogType &log_type, const int32_t &cbs_cnt) -{ - bool res = true; - if(cbs_cnt < ObTxLSLogLimit::get_max_parallel(log_type)) - { - res = false; - } - - return res; -} -ObDList *ObTxLSLogWriter::get_target_cbs_(const ObTxLogType &type) -{ - ObDList *ptr = nullptr; - switch (type) { - case ObTxLogType::TX_START_WORKING_LOG: { - ptr = &start_working_cbs_; - break; - } - default: { - ptr = nullptr; - break; - } - } - return ptr; -} - -int64_t ObTxLSLogWriter::generate_replay_hint_(const ObTxLogType &type) -{ - int64_t hint = 0; - const int64_t id = ls_id_.id(); - int64_t log_type = static_cast(type); - hint = murmurhash(&id, sizeof(id), hint); - hint = murmurhash(&log_type, sizeof(log_type), hint); - return std::abs(hint); -} - -void ObTxLSLogWriter::destroy_cbs_(common::ObDList &cbs) -{ - ObTxLSLogCb *cb_ptr = nullptr; - while (!cbs.is_empty()) { - if (OB_NOT_NULL(cb_ptr = cbs.remove_first())) { - cb_ptr->~ObTxLSLogCb(); - ob_free(cb_ptr); - } - } -} - -//******************************************************** -} // namespace transaction -} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_ls_log_writer.h b/src/storage/tx/ob_tx_ls_log_writer.h deleted file mode 100644 index 2b9f47213..000000000 --- a/src/storage/tx/ob_tx_ls_log_writer.h +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TX_LOGSTREAM_LOG_WRITER -#define OCEANBASE_TRANSACTION_OB_TX_LOGSTREAM_LOG_WRITER - -#include "lib/list/ob_dlink_node.h" -#include "lib/lock/ob_spin_lock.h" -#include "logservice/palf/palf_callback.h" -#include "storage/tx/ob_tx_log.h" -#include "storage/tx/ob_tx_log_adapter.h" -#include "logservice/ob_log_base_header.h" - -namespace oceanbase { - -namespace share -{ -class ObLSID; -} - -namespace logservice -{ -class ObLogBaseHeader; -class ObLogHandler; -class AppendCb; -} - -namespace palf -{ -class LSN; -} // namespace palf - -namespace transaction -{ -class ObLSTxCtxMgr; -class ObITxLogAdapter; - -class ObTxLSLogLimit -{ -public: - static const uint16_t START_WORKING_MAX_PARALLEL = 1; - - static uint16_t get_max_parallel(const ObTxLogType &type); - -public: - static int64_t LOG_BUF_SIZE; - static void decide_log_buf_size(); - -private: - ObTxLSLogLimit(); - DISALLOW_COPY_AND_ASSIGN(ObTxLSLogLimit); -}; - -class ObTxLSLogWriter; - -// used by one thread -class ObTxLSLogCb : public ObTxBaseLogCb, - public common::ObDLinkBase -{ -public: - ObTxLSLogCb(ObTxLSLogWriter *base); - ~ObTxLSLogCb(); - void reuse(); - void reset(); - - int occupy_by(const ObTxLogType & log_type); - template - int serialize_ls_log(T &ls_log, - int64_t replay_hint, - const enum logservice::ObReplayBarrierType barrier_type = logservice::ObReplayBarrierType::NO_NEED_BARRIER); - char *get_log_buf() { return log_buf_; } - int64_t get_log_pos() { return pos_; } - const ObTxLogType &get_log_type() { return type_; } -public: - int on_success(); - int on_failure(); - const char *get_cb_name() const override { return "TxLSLogCb"; } - -public: - INHERIT_TO_STRING_KV("ObTxBaseLogCb", ObTxBaseLogCb, K(type_), KP(log_buf_), K(pos_), KP(base_wr_)); - -private: - bool need_alloc_buf_() { return nullptr == log_buf_; } - OB_NOINLINE int alloc_log_buf_(); - -private: - ObTxLogType type_; // Unkown == unused, not init - char *log_buf_; - int64_t pos_; - ObTxLSLogWriter *base_wr_; -}; - -template -int ObTxLSLogCb::serialize_ls_log(T &ls_log, - int64_t replay_hint, - const enum logservice::ObReplayBarrierType barrier_type) -{ - int ret = OB_SUCCESS; - if (!ObTxLogTypeChecker::is_ls_log(T::LOG_TYPE) || replay_hint < 0 || T::LOG_TYPE != type_ - || OB_ISNULL(base_wr_)) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KR(ret), K(T::LOG_TYPE), K(replay_hint), - KP(base_wr_)); - } else if (need_alloc_buf_() && OB_FAIL(alloc_log_buf_())) { - TRANS_LOG(WARN, "alloc log buf failed", KR(ret)); - } else { - logservice::ObLogBaseHeader base_header(logservice::ObLogBaseType::TRANS_SERVICE_LOG_BASE_TYPE, - barrier_type, replay_hint); - ObTransID fake_tx_id(0); // fake a invalid tx_id - ObTxLogBlockHeader block_header(1, 1, 1, fake_tx_id, ObAddr()); - ObTxLogHeader tx_header(T::LOG_TYPE); - if (OB_FAIL(block_header.before_serialize())) { - TRANS_LOG(WARN, "[TxLsLogWriter] before serialize block header error", KR(ret), - K(block_header)); - } else if (OB_FAIL(base_header.serialize(log_buf_, ObTxLSLogLimit::LOG_BUF_SIZE, pos_))) { - TRANS_LOG(WARN, "[TxLsLogWriter] serialize base header error", KR(ret), KP(log_buf_), - K(pos_)); - } else if (OB_FAIL(block_header.serialize(log_buf_, ObTxLSLogLimit::LOG_BUF_SIZE, pos_))) { - TRANS_LOG(WARN, "[TxLsLogWriter] serialize block header error", KR(ret), KP(log_buf_), - K(pos_)); - } else if (OB_FAIL(tx_header.serialize(log_buf_, ObTxLSLogLimit::LOG_BUF_SIZE, pos_))) { - TRANS_LOG(WARN, "[TxLsLogWriter] serialize tx header error", KR(ret), KP(log_buf_), K(pos_)); - } else if (OB_FAIL(ls_log.serialize(log_buf_, ObTxLSLogLimit::LOG_BUF_SIZE, pos_))) { - TRANS_LOG(WARN, "[TxLsLogWriter] serialize ls_log error", KR(ret), KP(log_buf_), K(pos_)); - } - } - return ret; -} - -class ObTxLSLogWriter -{ -public: - const static uint8_t DEFAULT_LOG_CB_CNT = 1; - const static uint8_t APPEND_LOG_CB_CNT = 1; - - typedef common::ObSEArray - TxLsLogCbArray; - -public: - ObTxLSLogWriter(); - ~ObTxLSLogWriter(); - int init(const share::ObLSID &ls_id, - ObITxLogAdapter *adapter, - ObLSTxCtxMgr *ctx_mgr); - int stop(); - //TODO wait until log_cb finished - int wait(); - void reset(); - void destroy(); - -public: - int submit_start_working_log(const int64_t &leader_epoch, share::SCN &log_ts); - - -public: - int on_success(ObTxLSLogCb *cb); - int on_failure(ObTxLSLogCb *cb); - -private: - template - int submit_ls_log_(T &ls_log, - const enum logservice::ObReplayBarrierType barrier_type, - bool need_nonblock, - share::SCN &log_ts); - int get_log_cb_(const ObTxLogType &log_type, ObTxLSLogCb *&cb); - int return_log_cb_(ObTxLSLogCb *cb); - bool enough_log_cb_(); - int append_free_log_cb_(); - - bool reach_parallel_cbs_limit_(const ObTxLogType &log_type, const int32_t &cbs_cnt); - - common::ObDList *get_target_cbs_(const ObTxLogType &log_type); - int64_t generate_replay_hint_(const ObTxLogType &type); - - void destroy_cbs_(common::ObDList &cbs); - -private: - bool is_stopped_; - - mutable common::ObSpinLock cbs_lock_; - common::ObDList free_cbs_; - common::ObDList keep_alive_cbs_; - common::ObDList start_working_cbs_; - - - share::ObLSID ls_id_; - ObLSTxCtxMgr *ctx_mgr_; - ObITxLogAdapter *tx_log_adapter_; -}; - -template -int ObTxLSLogWriter::submit_ls_log_(T &ls_log, - const enum logservice::ObReplayBarrierType barrier_type, - bool need_nonblock, - share::SCN &log_ts) -{ - int ret = OB_SUCCESS; - ObSpinLockGuard cb_guard(cbs_lock_); - ObTxLSLogCb *cb = nullptr; - int64_t replay_hint = generate_replay_hint_(T::LOG_TYPE); - if (OB_FAIL(get_log_cb_(T::LOG_TYPE, cb))) { - if (OB_TX_NOLOGCB == ret) { - ret = OB_EAGAIN; - } else { - TRANS_LOG(WARN, "[TxLsLogWriter] can not get log cb", KR(ret), K(T::LOG_TYPE), KP(cb)); - } - } else if (nullptr == cb || OB_FAIL(cb->serialize_ls_log(ls_log, replay_hint, barrier_type))) { - TRANS_LOG(WARN, "[TxLsLogWriter] serialize ls log error", KR(ret), K(T::LOG_TYPE), KP(cb)); - } else if (OB_FAIL(tx_log_adapter_->submit_log(cb->get_log_buf(), cb->get_log_pos(), share::SCN::min_scn(), cb, need_nonblock))) { - TRANS_LOG(WARN, "[TxLsLogWriter] submit ls log failed", KR(ret), K(T::LOG_TYPE), K(ls_id_), KPC(cb)); - return_log_cb_(cb); - cb = nullptr; - } else { - log_ts = cb->get_log_ts(); - TRANS_LOG(INFO, "[TxLsLogWriter] submit ls log success", K(ret), K(T::LOG_TYPE), K(ls_id_), KPC(cb)); - } - return ret; -} - -} // namespace transaction -} // namespace oceanbase -#endif diff --git a/src/storage/tx/ob_tx_ls_state_mgr.cpp b/src/storage/tx/ob_tx_ls_state_mgr.cpp deleted file mode 100644 index fd0dc0074..000000000 --- a/src/storage/tx/ob_tx_ls_state_mgr.cpp +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_tx_ls_state_mgr.h" -#include "src/storage/tx_storage/ob_ls_map.h" - -#define LOAD_CUR_STATE_CONTAINER \ - TxLSStateContainer ls_state_container; \ - ls_state_container.state_container_ = ATOMIC_LOAD(&cur_state_.state_container_); - -namespace oceanbase -{ - -namespace transaction -{ - -int ObTxLSStateMgr::TxLSStateHashFunc::operator()(const TxStateActionPair &key, - TxLSStateContainer &res) const -{ - int ret = OB_SUCCESS; - - res = key.base_state_; - switch (key.exec_action_) { - case TxLSAction::START: { - if (key.base_state_.state_val_.state_ == TxLSState::INIT) { - clear_all_flag_(res); - res.state_val_.state_ = TxLSState::F_WORKING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::STOP: { - - if (key.base_state_.state_val_.state_ == TxLSState::INIT) { - // do nothing - } else { - clear_all_flag_(res); - res.state_val_.state_ = TxLSState::STOPPED; - } - break; - } - case TxLSAction::ONLINE: { - res.state_val_.offline_flag_ = false; - res.state_val_.block_start_tx_flag_ = false; - res.state_val_.block_start_readonly_flag_ = false; - break; - } - case TxLSAction::OFFLINE: { - res.state_val_.offline_flag_ = true; - break; - } - case TxLSAction::LEADER_REVOKE_GRACEFULLY: { - if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING - || key.base_state_.state_val_.state_ == TxLSState::T_SYNC_FAILED - || key.base_state_.state_val_.state_ == TxLSState::R_SYNC_FAILED) { - res.state_val_.state_ = TxLSState::F_WORKING; - res.state_val_.block_start_normal_tx_flag_ = false; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::LEADER_REVOKE_FORCEDLLY: { - if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING - || key.base_state_.state_val_.state_ == TxLSState::T_SYNC_FAILED - || key.base_state_.state_val_.state_ == TxLSState::R_SYNC_FAILED) { - res.state_val_.state_ = TxLSState::F_WORKING; - res.state_val_.block_start_normal_tx_flag_ = false; - } else if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING - || key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { - res.state_val_.state_ = TxLSState::F_SWL_PENDING; - res.state_val_.block_start_normal_tx_flag_ = false; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::SWL_CB_SUCC: { - if (key.base_state_.state_val_.state_ == TxLSState::T_SYNC_PENDING) { - res.state_val_.state_ = TxLSState::T_APPLY_PENDING; - } else if (key.base_state_.state_val_.state_ == TxLSState::R_SYNC_PENDING) { - res.state_val_.state_ = TxLSState::R_APPLY_PENDING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::SWL_CB_FAIL: { - - if (key.base_state_.state_val_.state_ == TxLSState::T_SYNC_PENDING) { - res.state_val_.state_ = TxLSState::T_SYNC_FAILED; - } else if (key.base_state_.state_val_.state_ == TxLSState::R_SYNC_PENDING) { - res.state_val_.state_ = TxLSState::R_SYNC_FAILED; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::APPLY_SWL_SUCC: { - if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING - || key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { - res.state_val_.state_ = TxLSState::L_WORKING; - } else if (key.base_state_.state_val_.state_ == TxLSState::F_SWL_PENDING) { - res.state_val_.state_ = TxLSState::F_WORKING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::APPLY_SWL_FAIL: { - if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING) { - res.state_val_.state_ = TxLSState::T_APPLY_PENDING; - } else if (key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { - res.state_val_.state_ = TxLSState::R_APPLY_PENDING; - } else if (key.base_state_.state_val_.state_ == TxLSState::F_SWL_PENDING) { - res.state_val_.state_ = TxLSState::F_SWL_PENDING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::LEADER_TAKEOVER: { - if (key.base_state_.state_val_.state_ == TxLSState::F_WORKING) { - res.state_val_.state_ = TxLSState::T_SYNC_PENDING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::RESUME_LEADER: { - if (key.base_state_.state_val_.state_ == TxLSState::F_WORKING) { - res.state_val_.state_ = TxLSState::R_SYNC_PENDING; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::BLOCK_START_TX: { - res.state_val_.block_start_tx_flag_ = true; - break; - } - case TxLSAction::BLOCK_START_READONLY: { - res.state_val_.block_start_readonly_flag_ = true; - break; - } - case TxLSAction::BLOCK_START_NORMAL_TX: { - if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING) { - res.state_val_.block_start_normal_tx_flag_ = true; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::UNBLOCK_NORMAL_TX: { - if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING) { - res.state_val_.block_start_normal_tx_flag_ = false; - } else { - ret = OB_STATE_NOT_MATCH; - } - break; - } - case TxLSAction::BLOCK_START_WR: { - res.state_val_.block_start_tx_flag_ = true; - res.state_val_.block_start_readonly_flag_ = true; - break; - } - default: { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unkonwn action for ls_tx_state", K(ret), K(key), K(res.state_val_)); - } - } - - return ret; -} - -int ObTxLSStateMgr::switch_tx_ls_state(const TxLSAction action, const share::SCN start_working_scn) -{ - int ret = OB_SUCCESS; - if (TxLSAction::SWL_CB_SUCC == action) { - share::SCN tmp_max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); - if (!max_applying_start_working_ts_.atomic_bcas(tmp_max_applied_swl_scn, - tmp_max_applied_swl_scn)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "prev start working log has not been applied", K(ret), "action", - action_str(action), K(start_working_scn), KPC(this)); - } else { - max_applying_start_working_ts_.inc_update(start_working_scn); - } - } else if (TxLSAction::APPLY_SWL_SUCC == action) { - if (!max_applying_start_working_ts_.atomic_bcas(start_working_scn, start_working_scn)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "Try to applied a unexpected start working scn", K(ret), "action", - action_str(action), K(start_working_scn), KPC(this)); - } else { - max_applied_start_working_ts_.inc_update(start_working_scn); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(execute_tx_ls_action_(action))) { - TRANS_LOG(WARN, "execute tx ls action failed", K(ret), K(ret), "action", action_str(action), - K(start_working_scn), KPC(this)); - } - return ret; -} - -void ObTxLSStateMgr::restore_tx_ls_state() -{ - int ret = OB_SUCCESS; - TxLSStateContainer restore_state_container; - restore_state_container.state_container_ = 0; - restore_state_container.state_container_ = ATOMIC_LOAD(&prev_state_.state_container_); - ATOMIC_STORE(&cur_state_.state_container_, restore_state_container.state_container_); - TRANS_LOG(INFO, "[Tx LS State] restore ls tx state successfully", K(ret), - K(restore_state_container.state_val_)); -} - -bool ObTxLSStateMgr::need_retry_apply_SWL(share::SCN &applying_swl_scn) -{ - bool need_retry = false; - LOAD_CUR_STATE_CONTAINER - if (ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING - || ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING - || ls_state_container.state_val_.state_ == TxLSState::F_SWL_PENDING) { - - share::SCN max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); - - if (max_applying_start_working_ts_.atomic_bcas(max_applied_swl_scn, max_applied_swl_scn)) { - need_retry = false; - } else { - applying_swl_scn = max_applying_start_working_ts_.atomic_load(); - need_retry = true; - } - } - - return need_retry; -} - -bool ObTxLSStateMgr::waiting_SWL_cb() -{ - bool is_waiting = false; - - LOAD_CUR_STATE_CONTAINER - - if (ls_state_container.state_val_.state_ == TxLSState::R_SYNC_PENDING - || ls_state_container.state_val_.state_ == TxLSState::T_SYNC_PENDING) { - is_waiting = true; - } - - return is_waiting; -} - -void ObTxLSStateMgr::replay_SWL_succ(const share::SCN &swl_scn) -{ - int ret = OB_SUCCESS; - share::SCN tmp_max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); - if (!max_applying_start_working_ts_.atomic_bcas(tmp_max_applied_swl_scn, - tmp_max_applied_swl_scn)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "prev start working log has not been applied", K(ret), KPC(this)); - } - max_applying_start_working_ts_.inc_update(swl_scn); - max_applied_start_working_ts_.inc_update(swl_scn); -} - -void ObTxLSStateMgr::iter_ctx_mgr_stat_info(uint64_t &state_container, - bool &is_master, - bool &is_stopped) const -{ - LOAD_CUR_STATE_CONTAINER - is_master = (ls_state_container.state_val_.state_ == TxLSState::L_WORKING); - is_stopped = (ls_state_container.state_val_.state_ == TxLSState::STOPPED); - state_container = ls_state_container.state_container_; -} - -bool ObTxLSStateMgr::is_master() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.state_ == TxLSState::L_WORKING; -} - -bool ObTxLSStateMgr::is_follower() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.state_ == TxLSState::F_WORKING - || ls_state_container.state_val_.state_ == TxLSState::F_SWL_PENDING; -} - -bool ObTxLSStateMgr::is_block_start_tx() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.block_start_tx_flag_; -} - - -bool ObTxLSStateMgr::is_block_start_normal_tx() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.block_start_normal_tx_flag_; -} - -bool ObTxLSStateMgr::is_block_WR() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.block_start_tx_flag_ - || ls_state_container.state_val_.block_start_readonly_flag_; -} - -bool ObTxLSStateMgr::is_leader_takeover_pending() const -{ - LOAD_CUR_STATE_CONTAINER - return is_switch_leader_pending_(ls_state_container) - || is_resume_leader_pending_(ls_state_container); -} - -bool ObTxLSStateMgr::is_start_working_apply_pending() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING - || ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING; -} - -bool ObTxLSStateMgr::is_switch_leader_pending() const -{ - LOAD_CUR_STATE_CONTAINER - return is_switch_leader_pending_(ls_state_container); -} - -bool ObTxLSStateMgr::is_resume_leader_pending() const -{ - LOAD_CUR_STATE_CONTAINER - return is_resume_leader_pending_(ls_state_container); -} - -bool ObTxLSStateMgr::is_follower_swl_pending() const -{ - LOAD_CUR_STATE_CONTAINER - return is_follower_swl_pending_(ls_state_container); -} - -bool ObTxLSStateMgr::is_stopped() const -{ - LOAD_CUR_STATE_CONTAINER - return ls_state_container.state_val_.state_ == TxLSState::STOPPED; -} - -int ObTxLSStateMgr::execute_tx_ls_action_(const TxLSAction action) -{ - int ret = OB_SUCCESS; - - TxLSStateContainer res_state; - res_state.state_container_ = 0; - TxStateActionPair pair; - pair.base_state_ = cur_state_; - pair.exec_action_ = action; - - TxLSStateHashFunc state_func; - - ret = state_func(pair, res_state); - - if (OB_SUCCESS != ret || UINT64_MAX == res_state.state_container_) { - TRANS_LOG(WARN, "[Tx LS State] switch ls tx state failed", K(ret), K(pair), - K(res_state.state_val_), KPC(this)); - } else { - ATOMIC_STORE(&prev_state_.state_container_, pair.base_state_.state_container_); - ATOMIC_STORE(&cur_state_.state_container_, res_state.state_container_); - TRANS_LOG(INFO, "[Tx LS State] switch ls tx state successfully", K(ret), K(pair), - K(res_state.state_val_), KPC(this)); - } - - return ret; -} - -void ObTxLSStateMgr::clear_all_flag_(TxLSStateContainer &cur_state) -{ - cur_state.state_val_.offline_flag_ = false; - cur_state.state_val_.block_start_tx_flag_ = false; - cur_state.state_val_.block_start_readonly_flag_ = false; - cur_state.state_val_.block_start_normal_tx_flag_ = false; -} - -bool ObTxLSStateMgr::is_switch_leader_pending_(const TxLSStateContainer &ls_state_container) -{ - return ls_state_container.state_val_.state_ == TxLSState::T_SYNC_PENDING - || ls_state_container.state_val_.state_ == TxLSState::T_SYNC_FAILED - || ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING; -} - -bool ObTxLSStateMgr::is_resume_leader_pending_(const TxLSStateContainer &ls_state_container) -{ - - return ls_state_container.state_val_.state_ == TxLSState::R_SYNC_PENDING - || ls_state_container.state_val_.state_ == TxLSState::R_SYNC_FAILED - || ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING; -} - -bool ObTxLSStateMgr::is_follower_swl_pending_(const TxLSStateContainer & ls_state_container) -{ - return ls_state_container.state_val_.state_ == TxLSState::F_SWL_PENDING; -} - -} // namespace transaction - -} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_ls_state_mgr.h b/src/storage/tx/ob_tx_ls_state_mgr.h deleted file mode 100644 index 6169cfa9f..000000000 --- a/src/storage/tx/ob_tx_ls_state_mgr.h +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TX_LS_STATE_MGR -#define OCEANBASE_TRANSACTION_OB_TX_LS_STATE_MGR - -#include "lib/hash/ob_hashmap.h" -#include "ob_trans_define.h" - -/*** - * The Binary Representation of the LS State - * - * | ... Reserved Bits (44) | - * | Block Start Tx Flag (1) | Block Start Normal Tx Flag (1) | Block Start ReadOnly Flag (1) | - * | Offline Flag (1) | - * | TxLSState (16) | - */ - -/*** - * State Machine - * - * @Init: - * +--------------------+------------+----------+---------------+--------------+ - * | BaseState | BaseFlag | Action | TargetState | TargetFlag | - * +====================+============+==========+===============+==============+ - * | INIT | NONE | START | F_WORKING | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | INIT | NONE | STOP | INIT | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | STOPPED | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | F_WORKING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | L_WORKING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | T_SYNC_PENDING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | R_SYNC_PENDING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | T_APPLYING_PENDING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * | R_APPLYING_PENDING | UNLIMITED | STOP | STOPPED | NONE | - * +--------------------+------------+----------+---------------+--------------+ - * - * @Leader Takeover: - * +-----------------+------------+-----------------+-----------------+--------------+ - * | BaseState | BaseFlag | Action | TargetState | TargetFlag | - * +=================+============+=================+=================+==============+ - * | F_WORKING | UNLIMITED | LEADER_TAKEOVER | T_SYNC_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | F_WORKING | UNLIMITED | RESUME_LEADER | R_SYNC_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | T_SYNC_PENDING | UNLIMITED | SWL_CB_SUCC | T_APPLY_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | R_SYNC_PENDING | UNLIMITED | SWL_CB_SUCC | R_APPLY_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | T_SYNC_PENDING | UNLIMITED | SWL_CB_FAIL | T_SYNC_FAILED | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | R_SYNC_PENDING | UNLIMITED | SWL_CB_FAIL | R_SYNC_FAILED | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | T_APPLY_PENDING | UNLIMITED | APPLY_SWL_SUCC | L_WORKING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | R_APPLY_PENDING | UNLIMITED | APPLY_SWL_SUCC | L_WORKING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | T_APPLY_PENDING | UNLIMITED | APPLY_SWL_FAIL | T_APPLY_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * | R_APPLY_PENDING | UNLIMITED | APPLY_SWL_FAIL | R_APPLY_PENDING | UNLIMITED | - * +-----------------+------------+-----------------+-----------------+--------------+ - * - * @Leader Revoke: - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | BaseState | BaseFlag | Action | TargetState | TargetFlag | - * +=================+============+==========================+===============+===============================+ - * | L_WORKING | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | L_WORKING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | T_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | R_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | T_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | R_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | T_APPLY_PENDING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | R_APPLY_PENDING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | F_SWL_PENDING | UNLIMITED | APPLY_SWL_FAIL | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * | F_SWL_PENDING | UNLIMITED | APPLY_SWL_SUCC | F_WORKING | BLOCK_STATR_NORMAL_TX = False | - * +-----------------+------------+--------------------------+---------------+-------------------------------+ - * - * @Offline: - * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ - * | BaseState | BaseFlag | Action | TargetState | TargetFlag | - * +=============+============+==========+===============+=======================================================================+ - * | UNLIMITED | UNLIMITED | OFFLINE | UNLIMITED | OFFLINE = True | - * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ - * | UNLIMITED | UNLIMITED | ONLINE | UNLIMITED | OFFLINE = False, BLOCK_START_TX = False, BLOCK_START_READONLY = False | - * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ - * Notion: Before 4.3, the ls may be started to working without online operation. - * - * @Block Tx - * +-------------+------------+-----------------------+---------------+-------------------------------+ - * | BaseState | BaseFlag | Action | TargetState | TargetFlag | - * +=============+============+=======================+===============+===============================+ - * | UNLIMITED | UNLIMITED | BLOCK_START_TX | UNLIMITED | BLOCK_START_TX = True | - * +-------------+------------+-----------------------+---------------+-------------------------------+ - * | UNLIMITED | UNLIMITED | BLOCK_START_READONLY | UNLIMITED | BLOCK_START_READONLY = True | - * +-------------+------------+-----------------------+---------------+-------------------------------+ - * | L_WORKING | UNLIMITED | BLOCK_START_NORMAL_TX | UNLIMITED | BLOCK_START_NORMAL_TX = True | - * +-------------+------------+-----------------------+---------------+-------------------------------+ - * | L_WORKING | UNLIMITED | UNBLOCK_NORMAL_TX | UNLIMITED | BLOCK_START_NORMAL_TX = False | - * +-------------+------------+-----------------------+---------------+-------------------------------+ - */ - -namespace oceanbase -{ - -namespace transaction -{ - -class ObTxLSStateMgr -{ - -public: - enum TxLSAction : uint64_t - { - UNKOWN = 0, - START = 1, - STOP = 2, - ONLINE = 3, - OFFLINE = 4, - LEADER_REVOKE_GRACEFULLY = 5, - LEADER_REVOKE_FORCEDLLY = 6, - SWL_CB_SUCC = 7, - SWL_CB_FAIL = 8, - APPLY_SWL_SUCC = 9, - APPLY_SWL_FAIL = 10, - LEADER_TAKEOVER = 12, - RESUME_LEADER = 13, - BLOCK_START_TX = 14, - BLOCK_START_READONLY = 15, - BLOCK_START_NORMAL_TX = 16, - UNBLOCK_NORMAL_TX = 17, - BLOCK_START_WR = 18 - }; - - enum TxLSState : uint16_t - { - INIT = 0, - F_WORKING = 1, - L_WORKING = 4, - STOPPED = 14, - T_SYNC_PENDING = 15, - R_SYNC_PENDING = 16, - T_SYNC_FAILED = 17, - R_SYNC_FAILED = 18, - T_APPLY_PENDING = 19, - R_APPLY_PENDING = 20, - F_SWL_PENDING = 21 - }; - -#define TCM_STATE_CASE_TO_STR(state) \ - case state: \ - str = #state; \ - break; - - static const char *state_str(uint16_t state) - { - const char *str = "INVALID"; - switch (state) { - TCM_STATE_CASE_TO_STR(INIT); - TCM_STATE_CASE_TO_STR(F_WORKING); - TCM_STATE_CASE_TO_STR(L_WORKING); - TCM_STATE_CASE_TO_STR(STOPPED); - TCM_STATE_CASE_TO_STR(T_SYNC_PENDING); - TCM_STATE_CASE_TO_STR(R_SYNC_PENDING); - TCM_STATE_CASE_TO_STR(T_SYNC_FAILED); - TCM_STATE_CASE_TO_STR(R_SYNC_FAILED); - TCM_STATE_CASE_TO_STR(T_APPLY_PENDING); - TCM_STATE_CASE_TO_STR(R_APPLY_PENDING); - TCM_STATE_CASE_TO_STR(F_SWL_PENDING); - - default: - break; - } - return str; - } - - static const char *action_str(TxLSAction action) - { - const char *str = "INVALID"; - switch (action) { - TCM_STATE_CASE_TO_STR(UNKOWN); - TCM_STATE_CASE_TO_STR(START); - TCM_STATE_CASE_TO_STR(STOP); - TCM_STATE_CASE_TO_STR(ONLINE); - TCM_STATE_CASE_TO_STR(OFFLINE); - TCM_STATE_CASE_TO_STR(LEADER_REVOKE_GRACEFULLY); - TCM_STATE_CASE_TO_STR(LEADER_REVOKE_FORCEDLLY); - TCM_STATE_CASE_TO_STR(SWL_CB_SUCC); - TCM_STATE_CASE_TO_STR(SWL_CB_FAIL); - TCM_STATE_CASE_TO_STR(APPLY_SWL_SUCC); - TCM_STATE_CASE_TO_STR(APPLY_SWL_FAIL); - TCM_STATE_CASE_TO_STR(LEADER_TAKEOVER); - TCM_STATE_CASE_TO_STR(RESUME_LEADER); - TCM_STATE_CASE_TO_STR(BLOCK_START_TX); - TCM_STATE_CASE_TO_STR(BLOCK_START_READONLY); - TCM_STATE_CASE_TO_STR(BLOCK_START_NORMAL_TX); - TCM_STATE_CASE_TO_STR(UNBLOCK_NORMAL_TX); - TCM_STATE_CASE_TO_STR(BLOCK_START_WR); - default: - break; - } - return str; - } -#undef TCM_STATE_CASE_TO_STR - - union TxLSStateContainer - { - struct StateVal - { - uint16_t state_ : 16; - uint8_t offline_flag_ : 1; - uint8_t block_start_tx_flag_ : 1; - uint8_t block_start_normal_tx_flag_ : 1; - uint8_t block_start_readonly_flag_ : 1; - uint64_t reserved_bit_ : 44; - - TO_STRING_KV("state_", - state_str(state_), - K(offline_flag_), - K(block_start_tx_flag_), - K(block_start_normal_tx_flag_), - K(block_start_readonly_flag_)); - } state_val_; - uint64_t state_container_; - }; - - struct TxStateActionPair - { - TxLSStateContainer base_state_; - TxLSAction exec_action_; - - TxStateActionPair() { reset(); } - void reset() - { - base_state_.state_container_ = 0; - exec_action_ = TxLSAction::UNKOWN; - } - - TO_STRING_KV(K(base_state_.state_val_), "exec_action_", action_str(exec_action_)); - }; - struct TxLSStateHashFunc - { - int operator()(const TxStateActionPair &key, TxLSStateContainer &res) const; - }; - - // typedef common::hash:: - // ObHashMap - // StateActionHashMap; - -public: - ObTxLSStateMgr() { reset(); } - - int init(const share::ObLSID ls_id) - { - ls_id_ = ls_id; - return OB_SUCCESS; - } - - void reset() - { - ls_id_.reset(); - cur_state_.state_container_ = 0; - prev_state_.state_container_ = 0; - } - int switch_tx_ls_state(const TxLSAction action, - const share::SCN start_working_scn = share::SCN::invalid_scn()); - void restore_tx_ls_state(); - - bool need_retry_apply_SWL(share::SCN &applying_swl_scn); - bool waiting_SWL_cb(); - void replay_SWL_succ(const share::SCN & swl_scn); - - void iter_ctx_mgr_stat_info(uint64_t &state_container, bool &is_master, bool &is_stopped) const; - // TxLSState get_state() const; - bool is_master() const; - bool is_follower() const; - bool is_block_start_tx() const; - bool is_block_start_normal_tx() const; - bool is_block_WR() const; - bool is_start_working_apply_pending() const; - bool is_leader_takeover_pending() const; - bool is_switch_leader_pending() const; - bool is_resume_leader_pending() const; - bool is_follower_swl_pending() const; - bool is_stopped() const; - - TO_STRING_KV(K(ls_id_), - K(cur_state_.state_val_), - K(prev_state_.state_val_), - K(max_applying_start_working_ts_), - K(max_applied_start_working_ts_)); - -private: - int execute_tx_ls_action_(const TxLSAction action); - static void clear_all_flag_(TxLSStateContainer &cur_state); - - static bool is_switch_leader_pending_(const TxLSStateContainer &ls_state_container); - static bool is_resume_leader_pending_(const TxLSStateContainer &ls_state_container); - static bool is_follower_swl_pending_(const TxLSStateContainer &ls_state_container); - -private: - share::ObLSID ls_id_; - TxLSStateContainer cur_state_; - TxLSStateContainer prev_state_; - - share::SCN max_applied_start_working_ts_; - share::SCN max_applying_start_working_ts_; -}; - -} // namespace transaction - -} // namespace oceanbase -#endif diff --git a/src/storage/tx/ob_tx_msg.cpp b/src/storage/tx/ob_tx_msg.cpp deleted file mode 100644 index 528cf3e91..000000000 --- a/src/storage/tx/ob_tx_msg.cpp +++ /dev/null @@ -1,506 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ob_tx_msg.h" - -namespace oceanbase -{ -namespace transaction -{ -OB_SERIALIZE_MEMBER(ObTxMsg, - type_, - cluster_version_, - tx_id_, - receiver_, - sender_addr_, - sender_, - request_id_, - timestamp_, - epoch_, - cluster_id_); -// NOTICE: DO NOT MODIFY FOLLOING MACRO DEFINES, IT IS RESERVED FOR COMPATIBLE WITH OLD <= 4.1.2 -#define ObTxSubPrepareMsg_V1_MEMBERS expire_ts_, xid_, parts_, app_trace_info_ -#define ObTxSubPrepareRespMsg_V1_MEMBERS ret_ -#define ObTxSubCommitMsg_V1_MEMBERS xid_ -#define ObTxSubCommitRespMsg_V1_MEMBERS ret_ -#define ObTxSubRollbackMsg_V1_MEMBERS xid_ -#define ObTxSubRollbackRespMsg_V1_MEMBERS ret_ -#define ObTxCommitMsg_V1_MEMBERS expire_ts_, parts_, app_trace_info_ -#define ObTxCommitRespMsg_V1_MEMBERS ret_, commit_version_ -#define ObTxAbortMsg_V1_MEMBERS reason_ -#define ObTxKeepaliveMsg_V1_MEMBERS status_ -#define ObTxKeepaliveRespMsg_V1_MEMBERS status_ -#define Ob2pcPrepareReqMsg_V1_MEMBERS upstream_, app_trace_info_ -#define Ob2pcPrepareRespMsg_V1_MEMBERS prepare_version_, prepare_info_array_ -#define Ob2pcPreCommitReqMsg_V1_MEMBERS commit_version_ -#define Ob2pcPreCommitRespMsg_V1_MEMBERS commit_version_ -#define Ob2pcCommitReqMsg_V1_MEMBERS commit_version_, prepare_info_array_ -#define Ob2pcCommitRespMsg_V1_MEMBERS commit_version_, commit_log_scn_ -#define Ob2pcAbortReqMsg_V1_MEMBERS upstream_ -#define Ob2pcAbortRespMsg_V1_MEMBERS -#define Ob2pcClearReqMsg_V1_MEMBERS max_commit_log_scn_ -#define Ob2pcClearRespMsg_V1_MEMBERS -#define Ob2pcPrepareRedoReqMsg_V1_MEMBERS xid_, upstream_, app_trace_info_ -#define Ob2pcPrepareRedoRespMsg_V1_MEMBERS -#define Ob2pcPrepareVersionReqMsg_V1_MEMBERS -#define Ob2pcPrepareVersionRespMsg_V1_MEMBERS prepare_version_, prepare_info_array_ -#define ObAskStateMsg_V1_MEMBERS snapshot_ -#define ObAskStateRespMsg_V1_MEMBERS state_info_array_ -#define ObCollectStateMsg_V1_MEMBERS snapshot_ -#define ObCollectStateRespMsg_V1_MEMBERS state_info_ - -#define CONCAT_REF(b) it_.b -#define APPLY_FUNC_(f, ...) f(__VA_ARGS__) -#define OB_TX_MSG_SERDE(CLZ, P_CLZ, ...) \ - struct CLZ##_box { CLZ##_box(CLZ &x) : it_(x) {} CLZ &it_; OB_UNIS_VERSION(1); }; \ - APPLY_FUNC_(OB_SERIALIZE_MEMBER, CLZ##_box, LST_DO(CONCAT_REF, (,), ##__VA_ARGS__)); \ - int CLZ::serialize(SERIAL_PARAMS) const \ - { \ - int ret = P_CLZ::serialize(buf, buf_len, pos); \ - if (OB_SUCC(ret)) { \ - CLZ##_box x(const_cast(*this)); \ - ret = x.serialize(buf, buf_len, pos); \ - } \ - return ret; \ - } \ - int CLZ::deserialize(DESERIAL_PARAMS) { \ - int ret = OB_SUCCESS; \ - ret = P_CLZ::deserialize(buf, data_len, pos); \ - if (OB_SUCC(ret)) { \ - CLZ##_box x(*this); \ - ret = x.deserialize(buf, data_len, pos); \ - } \ - return ret; \ - } \ - int64_t CLZ::get_serialize_size() const { \ - int64_t len = P_CLZ::get_serialize_size(); \ - CLZ##_box x(const_cast(*this)); \ - len += x.get_serialize_size(); \ - return len; \ - } - -OB_TX_MSG_SERDE(ObTxSubPrepareMsg, ObTxMsg, expire_ts_, xid_, parts_, app_trace_info_, commit_parts_); -OB_TX_MSG_SERDE(ObTxSubPrepareRespMsg, ObTxMsg, ret_); -OB_TX_MSG_SERDE(ObTxSubCommitMsg, ObTxMsg, xid_); -OB_TX_MSG_SERDE(ObTxSubCommitRespMsg, ObTxMsg, ret_); -OB_TX_MSG_SERDE(ObTxSubRollbackMsg, ObTxMsg, xid_); -OB_TX_MSG_SERDE(ObTxSubRollbackRespMsg, ObTxMsg, ret_); -OB_TX_MSG_SERDE(ObTxCommitMsg, ObTxMsg, expire_ts_, parts_, app_trace_info_, commit_start_scn_, commit_parts_); -OB_TX_MSG_SERDE(ObTxCommitRespMsg, ObTxMsg, ret_, commit_version_); -OB_TX_MSG_SERDE(ObTxAbortMsg, ObTxMsg, reason_); -OB_TX_MSG_SERDE(ObTxKeepaliveMsg, ObTxMsg, status_); -OB_TX_MSG_SERDE(ObTxKeepaliveRespMsg, ObTxMsg, status_); -OB_TX_MSG_SERDE(Ob2pcPrepareReqMsg, ObTxMsg, upstream_, app_trace_info_); -OB_TX_MSG_SERDE(Ob2pcPrepareRespMsg, ObTxMsg, prepare_version_, prepare_info_array_); -OB_TX_MSG_SERDE(Ob2pcPreCommitReqMsg, ObTxMsg, commit_version_); -OB_TX_MSG_SERDE(Ob2pcPreCommitRespMsg, ObTxMsg, commit_version_); -OB_TX_MSG_SERDE(Ob2pcCommitReqMsg, ObTxMsg, commit_version_, prepare_info_array_); -OB_TX_MSG_SERDE(Ob2pcCommitRespMsg, ObTxMsg, commit_version_, commit_log_scn_); -OB_TX_MSG_SERDE(Ob2pcAbortReqMsg, ObTxMsg, upstream_); -OB_TX_MSG_SERDE(Ob2pcAbortRespMsg, ObTxMsg); -OB_TX_MSG_SERDE(Ob2pcClearReqMsg, ObTxMsg, max_commit_log_scn_); -OB_TX_MSG_SERDE(Ob2pcClearRespMsg, ObTxMsg); -OB_TX_MSG_SERDE(Ob2pcPrepareRedoReqMsg, ObTxMsg, xid_, upstream_, app_trace_info_); -OB_TX_MSG_SERDE(Ob2pcPrepareRedoRespMsg, ObTxMsg); -OB_TX_MSG_SERDE(Ob2pcPrepareVersionReqMsg, ObTxMsg, upstream_); -OB_TX_MSG_SERDE(Ob2pcPrepareVersionRespMsg, ObTxMsg, prepare_version_, prepare_info_array_); -OB_TX_MSG_SERDE(ObAskStateMsg, ObTxMsg, snapshot_, ori_ls_id_, ori_addr_); -OB_TX_MSG_SERDE(ObAskStateRespMsg, ObTxMsg, state_info_array_); -OB_TX_MSG_SERDE(ObCollectStateMsg, ObTxMsg, snapshot_, check_info_); -OB_TX_MSG_SERDE(ObCollectStateRespMsg, ObTxMsg, state_info_); -OB_SERIALIZE_MEMBER((ObTxRollbackSPRespMsg, ObTxMsg), ret_, orig_epoch_); - -OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) -{ - int len = 0; - len += ObTxMsg::get_serialize_size(); - LST_DO_CODE(OB_UNIS_ADD_LEN, savepoint_, op_sn_, tx_seq_base_); - if (OB_NOT_NULL(tx_ptr_)) { - OB_UNIS_ADD_LEN(true); - OB_UNIS_ADD_LEN(*tx_ptr_); - } else { - OB_UNIS_ADD_LEN(false); - } - OB_UNIS_ADD_LEN(flag_); - OB_UNIS_ADD_LEN(specified_from_scn_); - return len; -} - -OB_DEF_SERIALIZE(ObTxRollbackSPMsg) -{ - int ret = ObTxMsg::serialize(buf, buf_len, pos); - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_ENCODE, savepoint_, op_sn_, tx_seq_base_); - if (OB_NOT_NULL(tx_ptr_)) { - OB_UNIS_ENCODE(true); - OB_UNIS_ENCODE(*tx_ptr_); - } else { - OB_UNIS_ENCODE(false); - } - OB_UNIS_ENCODE(flag_); - OB_UNIS_ENCODE(specified_from_scn_); - } - return ret; -} - -OB_DEF_DESERIALIZE(ObTxRollbackSPMsg) -{ - int ret = ObTxMsg::deserialize(buf, data_len, pos); - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_DECODE, savepoint_, op_sn_, tx_seq_base_); - bool has_tx_ptr = false; - OB_UNIS_DECODE(has_tx_ptr); - if (has_tx_ptr) { - void *buffer = ob_malloc(sizeof(ObTxDesc), "TxDesc"); - if (OB_ISNULL(buffer)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - ObTxDesc *tmp = new(buffer)ObTxDesc(); - OB_UNIS_DECODE(*tmp); - tx_ptr_ = tmp; - } - } - OB_UNIS_DECODE(flag_); - OB_UNIS_DECODE(specified_from_scn_); - } - return ret; -} - -bool ObTxMsg::is_valid() const -{ - bool ret = false; - if (type_ != TX_UNKNOWN && cluster_version_ > 0 && true - && tx_id_.is_valid() && timestamp_ > 0 && request_id_ > -1 - && sender_.is_valid() && receiver_.is_valid() && sender_addr_.is_valid() - && cluster_id_ > OB_INVALID_CLUSTER_ID) { - ret = true; - } - return ret; -} - -bool ObTxSubPrepareMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == SUBPREPARE && expire_ts_ > OB_INVALID_TIMESTAMP - && xid_.is_valid() && !xid_.empty() && parts_.count() > 0) { - ret = true; - } - return ret; -} - -bool ObTxSubPrepareRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == SUBPREPARE_RESP) { - ret = true; - } - return ret; -} - -bool ObTxSubCommitMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == SUBCOMMIT - && xid_.is_valid() && !xid_.empty()) { - ret = true; - } - return ret; -} - -bool ObTxSubCommitRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == SUBCOMMIT_RESP) { - ret = true; - } - return ret; -} - -bool ObTxSubRollbackMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid()&& type_ == SUBROLLBACK - && xid_.is_valid() && !xid_.empty()) { - ret = true; - } - return ret; -} - -bool ObTxSubRollbackRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == SUBROLLBACK_RESP) { - ret = true; - } - return ret; -} - -bool ObTxCommitMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_COMMIT - && expire_ts_ > OB_INVALID_TIMESTAMP - && parts_.count() > 0) { - ret = true; - } - return ret; -} - -bool ObTxCommitRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_COMMIT_RESP - && ((OB_SUCCESS == ret_ && commit_version_.is_valid()) - || (OB_SUCCESS != ret_ && !commit_version_.is_valid()))) { - ret = true; - } - return ret; -} - -bool ObTxAbortMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_ABORT) { - ret = true; - } - return ret; -} - -bool ObTxRollbackSPMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == ROLLBACK_SAVEPOINT - && savepoint_.is_valid() && op_sn_ > -1) { - ret = true; - } - return ret; -} - -bool ObTxKeepaliveMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == KEEPALIVE) { - ret = true; - } - return ret; -} - -bool ObTxKeepaliveRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == KEEPALIVE_RESP) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_REQ - && upstream_.is_valid()) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_RESP - && prepare_version_.is_valid() - && prepare_info_array_.count() > 0) { - ret = true; - } - return ret; -} - -bool Ob2pcPreCommitReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PRE_COMMIT_REQ) { - ret = true; - } - return ret; -} - -bool Ob2pcPreCommitRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PRE_COMMIT_RESP) { - ret = true; - } - return ret; -} - -bool Ob2pcCommitReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_COMMIT_REQ - && commit_version_.is_valid() - && prepare_info_array_.count() > 0) { - ret = true; - } - return ret; -} - -bool Ob2pcCommitRespMsg::is_valid() const { - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_COMMIT_RESP && - commit_version_.is_valid()) { - ret = true; - } - if (!commit_log_scn_.is_valid()) { - ret = false; - } - return ret; -} - -bool Ob2pcAbortReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_ABORT_REQ) { - ret = true; - } - return ret; -} - -bool Ob2pcAbortRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_ABORT_RESP) { - ret = true; - } - return ret; -} - -bool Ob2pcClearReqMsg::is_valid() const { - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_CLEAR_REQ) { - ret = true; - } - if (!max_commit_log_scn_.is_valid()) { - ret = false; - } - return ret; -} - -bool Ob2pcClearRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_CLEAR_RESP) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareRedoReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_REDO_REQ - && xid_.is_valid() && !xid_.empty() && upstream_.is_valid()) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareRedoRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_REDO_RESP) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareVersionReqMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_VERSION_REQ - // open after no version can upgrade from with no upstream - /*&& upstream_.is_valid()*/) { - ret = true; - } - return ret; -} - -bool Ob2pcPrepareVersionRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == TX_2PC_PREPARE_VERSION_RESP - && prepare_version_.is_valid() - && prepare_info_array_.count() > 0) { - ret = true; - } - return ret; -} - -bool ObAskStateMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == ASK_STATE - && snapshot_.is_valid()) { - ret = true; - } - return ret; -} - -bool ObAskStateRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == ASK_STATE_RESP - && state_info_array_.count() > 0) { - ret = true; - } - return ret; -} - -bool ObCollectStateMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == COLLECT_STATE - && snapshot_.is_valid()) { - ret = true; - } - return ret; -} - -bool ObCollectStateRespMsg::is_valid() const -{ - bool ret = false; - if (ObTxMsg::is_valid() && type_ == COLLECT_STATE_RESP - && state_info_.is_valid()) { - ret = true; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObAskTxStateFor4377Msg, tx_id_, ls_id_); -OB_SERIALIZE_MEMBER(ObAskTxStateFor4377RespMsg, is_alive_, ret_); - -bool ObAskTxStateFor4377Msg::is_valid() const -{ - bool ret = false; - if (tx_id_.is_valid() - && ls_id_.is_valid()) { - ret = true; - } - return ret; -} - - -} // transaction -} // oceanbase diff --git a/src/storage/tx/ob_tx_msg.h b/src/storage/tx/ob_tx_msg.h deleted file mode 100644 index 5eef57d2f..000000000 --- a/src/storage/tx/ob_tx_msg.h +++ /dev/null @@ -1,616 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TX_MSG_ -#define OCEANBASE_TRANSACTION_OB_TX_MSG_ - -#include "share/scn.h" -#include "share/ob_define.h" -#include "ob_trans_define.h" -#include "share/rpc/ob_batch_proxy.h" -namespace oceanbase -{ -namespace transaction -{ - enum TX_MSG_TYPE - { - TX_UNKNOWN = 0, - /* for xa */ - SUBPREPARE = 1, - SUBPREPARE_RESP = 2, - SUBCOMMIT = 3, - SUBCOMMIT_RESP = 4, - SUBROLLBACK = 5, - SUBROLLBACK_RESP = 6, - /* for trans */ - TX_COMMIT = 20, - TX_COMMIT_RESP = 21, - TX_ABORT = 22, - /* for 2PC */ - TX_2PC_PREPARE_REDO_REQ = 40, - TX_2PC_PREPARE_REDO_RESP = 41, - TX_2PC_PREPARE_VERSION_REQ = 42, - TX_2PC_PREPARE_VERSION_RESP = 43, - TX_2PC_PREPARE_REQ = 44, - TX_2PC_PREPARE_RESP = 45, - TX_2PC_PRE_COMMIT_REQ = 46, - TX_2PC_PRE_COMMIT_RESP = 47, - TX_2PC_COMMIT_REQ = 48, - TX_2PC_COMMIT_RESP = 49, - TX_2PC_ABORT_REQ = 50, - TX_2PC_ABORT_RESP = 51, - TX_2PC_CLEAR_REQ = 52, - TX_2PC_CLEAR_RESP = 53, - /* for others */ - ROLLBACK_SAVEPOINT = 60, - KEEPALIVE = 61, - KEEPALIVE_RESP = 62, - /* for standby read */ - ASK_STATE = 63, - ASK_STATE_RESP = 64, - COLLECT_STATE = 65, - COLLECT_STATE_RESP = 66, - // rollback savepoint resp - ROLLBACK_SAVEPOINT_RESP = 67, - /* for txn free route */ - TX_FREE_ROUTE_PUSH_STATE = 80, - TX_FREE_ROUTE_CHECK_ALIVE = 81, - TX_FREE_ROUTE_CHECK_ALIVE_RESP = 82, - }; - - struct ObTxMsg : public obcall::ObIFill - { - explicit ObTxMsg(const int16_t msg_type = TX_UNKNOWN) : - type_(msg_type), - cluster_version_(0), - tx_id_(), - receiver_(share::ObLSID::INVALID_LS_ID), - epoch_(-1), - sender_addr_(), - sender_(share::ObLSID::INVALID_LS_ID), - request_id_(-1), - timestamp_(ObTimeUtility::current_time()), - cluster_id_(OB_INVALID_CLUSTER_ID) - {} - ~ObTxMsg() {} - int16_t type_; - int64_t cluster_version_; - ObTransID tx_id_; - share::ObLSID receiver_; - /* the target participant's born epoch, used to verify its health */ - int64_t epoch_; - /* useful when send rsp to sender */ - ObAddr sender_addr_; - share::ObLSID sender_; - int64_t request_id_; - int64_t timestamp_; - int64_t cluster_id_; - VIRTUAL_TO_STRING_KV(K_(type), - K_(cluster_version), - K_(tx_id), - K_(receiver), - K_(sender), - K_(sender_addr), - K_(epoch), - K_(request_id), - K_(timestamp), - K_(cluster_id)); - OB_UNIS_VERSION_V(1); - public: - virtual bool is_valid() const; - share::ObLSID get_receiver() const { return receiver_; } - int64_t get_epoch() const { return epoch_; } - ObAddr get_sender_addr() const { return sender_addr_; } - share::ObLSID get_sender() const { return sender_; } - ObTransID get_trans_id() const { return tx_id_; } - int16_t get_msg_type() const { return type_; } - int64_t get_request_id() const { return request_id_; } - int64_t get_timestamp() const { return timestamp_; } - - int64_t get_cluster_id() const { return cluster_id_; } - int64_t get_cluster_version() const { return cluster_version_; } - virtual int fill_buffer(char* buf, int64_t size, int64_t &filled_size) const override - { - filled_size = 0; - return serialize(buf, size, filled_size); - } - virtual int64_t get_req_size() const override { return get_serialize_size(); } - }; - - // for XA - struct ObTxSubPrepareMsg : public ObTxMsg { - ObTxSubPrepareMsg() : - ObTxMsg(SUBPREPARE), - expire_ts_(OB_INVALID_TIMESTAMP), - xid_(), - parts_(), - commit_parts_() - {} - int64_t expire_ts_; - ObXATransID xid_; - share::ObLSArray parts_; - ObTxCommitParts commit_parts_; - common::ObString app_trace_info_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(expire_ts), K_(xid), K_(parts), - K_(app_trace_info)); - OB_UNIS_VERSION(1); - }; - - struct ObTxSubPrepareRespMsg : public ObTxMsg { - ObTxSubPrepareRespMsg() : - ObTxMsg(SUBPREPARE_RESP) - {} - int ret_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret)); - OB_UNIS_VERSION(1); - }; - - struct ObTxSubCommitMsg : public ObTxMsg { - ObTxSubCommitMsg() : - ObTxMsg(SUBCOMMIT), - xid_() - {} - ObXATransID xid_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K(xid_)); - OB_UNIS_VERSION(1); - }; - - struct ObTxSubCommitRespMsg : public ObTxMsg { - ObTxSubCommitRespMsg() : - ObTxMsg(SUBCOMMIT_RESP) - {} - int ret_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret)); - OB_UNIS_VERSION(1); - }; - - struct ObTxSubRollbackMsg : public ObTxMsg { - ObTxSubRollbackMsg() : - ObTxMsg(SUBROLLBACK), - xid_() - {} - ObXATransID xid_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K(xid_)); - OB_UNIS_VERSION(1); - }; - - struct ObTxSubRollbackRespMsg : public ObTxMsg { - ObTxSubRollbackRespMsg() : - ObTxMsg(SUBROLLBACK_RESP) - {} - int ret_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret)); - OB_UNIS_VERSION(1); - }; - - // for trans - struct ObTxCommitMsg : public ObTxMsg { - ObTxCommitMsg() : - ObTxMsg(TX_COMMIT), - expire_ts_(OB_INVALID_TIMESTAMP), - parts_() - { commit_start_scn_.set_max(); } - int64_t expire_ts_; - share::SCN commit_start_scn_; - share::ObLSArray parts_; - common::ObString app_trace_info_; - ObTxCommitParts commit_parts_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(expire_ts), K_(commit_start_scn), K_(parts), K_(app_trace_info), K_(commit_parts)); - OB_UNIS_VERSION(1); - }; - struct ObTxCommitRespMsg : public ObTxMsg { - ObTxCommitRespMsg() : - ObTxMsg(TX_COMMIT_RESP) - {} - share::SCN commit_version_; - int ret_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(commit_version)); - OB_UNIS_VERSION(1); - }; - - struct ObTxAbortMsg : public ObTxMsg { - ObTxAbortMsg() : - ObTxMsg(TX_ABORT) - {} - int reason_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(reason)); - OB_UNIS_VERSION(1); - }; - - struct ObTxRollbackSPMsg : public ObTxMsg { - ObTxRollbackSPMsg() : - ObTxMsg(ROLLBACK_SAVEPOINT), - savepoint_(), - op_sn_(-1), - tx_seq_base_(0), - tx_ptr_(NULL), - flag_(USE_ASYNC_RESP), - specified_from_scn_() - {} - ~ObTxRollbackSPMsg() { - if (OB_NOT_NULL(tx_ptr_)) { - tx_ptr_->~ObTxDesc(); - ob_free((void*)tx_ptr_); - tx_ptr_ = NULL; - } - specified_from_scn_.reset(); - } - ObTxSEQ savepoint_; - int64_t op_sn_; - int64_t tx_seq_base_; - const ObTxDesc *tx_ptr_; - uint8_t flag_; - ObTxSEQ specified_from_scn_; - bool use_async_resp() const { return (flag_ & USE_ASYNC_RESP) !=0; } - const static uint8_t USE_ASYNC_RESP = 0x01; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, - K_(savepoint), K_(op_sn), K_(tx_seq_base), K_(flag), - K_(specified_from_scn), KP_(tx_ptr)); - OB_UNIS_VERSION(1); - }; - - struct ObTxRollbackSPRespMsg : public ObTxMsg { - ObTxRollbackSPRespMsg() : - ObTxMsg(ROLLBACK_SAVEPOINT_RESP), - ret_(-1), - orig_epoch_(0) - {} - ~ObTxRollbackSPRespMsg() { - ret_ = -1; - orig_epoch_ = 0; - } - int ret_; - int64_t orig_epoch_; - - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(orig_epoch)); - OB_UNIS_VERSION(1); - }; - - struct ObTxKeepaliveMsg : public ObTxMsg { - ObTxKeepaliveMsg() : - ObTxMsg(KEEPALIVE) - {} - int64_t status_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(status)); - OB_UNIS_VERSION(1); - }; - - struct ObTxKeepaliveRespMsg : public ObTxMsg { - ObTxKeepaliveRespMsg() : - ObTxMsg(KEEPALIVE_RESP) - {} - int64_t status_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(status)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareReqMsg : public ObTxMsg - { - public: - Ob2pcPrepareReqMsg() : - ObTxMsg(TX_2PC_PREPARE_REQ), - upstream_(share::ObLSID::INVALID_LS_ID) - {} - public: - share::ObLSID upstream_; - ObString app_trace_info_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(upstream)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareRespMsg : public ObTxMsg - { - public: - Ob2pcPrepareRespMsg() : - ObTxMsg(TX_2PC_PREPARE_RESP), - prepare_info_array_() - {} - public: - share::SCN prepare_version_; - ObLSLogInfoArray prepare_info_array_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(prepare_version), K_(prepare_info_array)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPreCommitReqMsg : public ObTxMsg - { - public: - Ob2pcPreCommitReqMsg() : - ObTxMsg(TX_2PC_PRE_COMMIT_REQ) - {} - public: - share::SCN commit_version_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPreCommitRespMsg : public ObTxMsg - { - public: - Ob2pcPreCommitRespMsg() : - ObTxMsg(TX_2PC_PRE_COMMIT_RESP) - {} - public: - //set commit_version when the root participant - //which recover from prepare log receive a pre_commit response - //because the coord_state_ will be set as pre_commit - share::SCN commit_version_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcCommitReqMsg : public ObTxMsg - { - public: - Ob2pcCommitReqMsg() : - ObTxMsg(TX_2PC_COMMIT_REQ), - prepare_info_array_() - {} - public: - share::SCN commit_version_; - ObLSLogInfoArray prepare_info_array_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version), K_(prepare_info_array)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcCommitRespMsg : public ObTxMsg - { - public: - Ob2pcCommitRespMsg() : ObTxMsg(TX_2PC_COMMIT_RESP) {} - - public: - bool is_valid() const; - share::SCN commit_version_; - share::SCN commit_log_scn_; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version), K_(commit_log_scn)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcAbortReqMsg : public ObTxMsg - { - public: - Ob2pcAbortReqMsg() : - ObTxMsg(TX_2PC_ABORT_REQ), - upstream_(share::ObLSID::INVALID_LS_ID) - {} - public: - bool is_valid() const; - share::ObLSID upstream_; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(upstream)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcAbortRespMsg : public ObTxMsg - { - public: - Ob2pcAbortRespMsg() : - ObTxMsg(TX_2PC_ABORT_RESP) - {} - public: - bool is_valid() const; - OB_UNIS_VERSION(1); - }; - - struct Ob2pcClearReqMsg : public ObTxMsg - { - public: - Ob2pcClearReqMsg() : - ObTxMsg(TX_2PC_CLEAR_REQ) - {} - public: - bool is_valid() const; - share::SCN max_commit_log_scn_; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(max_commit_log_scn)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcClearRespMsg : public ObTxMsg - { - public: - Ob2pcClearRespMsg() : - ObTxMsg(TX_2PC_CLEAR_RESP) - {} - public: - bool is_valid() const; - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareRedoReqMsg : public ObTxMsg - { - public: - Ob2pcPrepareRedoReqMsg() : - ObTxMsg(TX_2PC_PREPARE_REDO_REQ), - xid_(), - upstream_(share::ObLSID::INVALID_LS_ID) - {} - public: - ObXATransID xid_; - share::ObLSID upstream_; - ObString app_trace_info_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(xid), K_(upstream)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareRedoRespMsg : public ObTxMsg - { - public: - Ob2pcPrepareRedoRespMsg() : - ObTxMsg(TX_2PC_PREPARE_REDO_RESP) - {} - public: - bool is_valid() const; - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareVersionReqMsg : public ObTxMsg - { - public: - Ob2pcPrepareVersionReqMsg() : - ObTxMsg(TX_2PC_PREPARE_VERSION_REQ), - upstream_(share::ObLSID::INVALID_LS_ID) - {} - public: - share::ObLSID upstream_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(upstream)); - OB_UNIS_VERSION(1); - }; - - struct Ob2pcPrepareVersionRespMsg : public ObTxMsg - { - public: - Ob2pcPrepareVersionRespMsg() : - ObTxMsg(TX_2PC_PREPARE_VERSION_RESP), - prepare_info_array_() - {} - public: - share::SCN prepare_version_; - ObLSLogInfoArray prepare_info_array_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(prepare_version), K_(prepare_info_array)); - OB_UNIS_VERSION(1); - }; - - struct ObAskStateMsg : public ObTxMsg - { - public: - ObAskStateMsg() : - ObTxMsg(ASK_STATE), - snapshot_(), - ori_ls_id_(), - ori_addr_() - {} - public: - share::SCN snapshot_; - share::ObLSID ori_ls_id_; - ObAddr ori_addr_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(snapshot), K_(ori_ls_id), K_(ori_addr)); - OB_UNIS_VERSION(1); - }; - - struct ObAskStateRespMsg : public ObTxMsg - { - public: - ObAskStateRespMsg() : - ObTxMsg(ASK_STATE_RESP), - state_info_array_() - {} - public: - ObStateInfoArray state_info_array_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(state_info_array)); - OB_UNIS_VERSION(1); - }; - - struct ObCollectStateMsg : public ObTxMsg - { - public: - ObCollectStateMsg() : - ObTxMsg(COLLECT_STATE), - snapshot_(), - check_info_() - {} - public: - share::SCN snapshot_; - ObStandbyCheckInfo check_info_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(snapshot), K_(check_info)); - OB_UNIS_VERSION(1); - }; - - struct ObCollectStateRespMsg : public ObTxMsg - { - public: - ObCollectStateRespMsg() : - ObTxMsg(COLLECT_STATE_RESP), - state_info_() - {} - public: - ObStateInfo state_info_; - bool is_valid() const; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(state_info)); - OB_UNIS_VERSION(1); - }; - - class ObTxMsgTypeChecker - { - public: - static bool is_valid_msg_type(const int16_t msg_type) - { - return ((0 <= msg_type && 6 >= msg_type) - || (20 <= msg_type && 22 >= msg_type) - || (40 <= msg_type && 49 >= msg_type) - || (50 <= msg_type && 53 >= msg_type) - || (60 <= msg_type && 67 >= msg_type)); - } - - static bool is_2pc_msg_type(const int16_t msg_type) - { - return (msg_type >= TX_2PC_PREPARE_REQ && msg_type <= TX_2PC_CLEAR_RESP) - || (TX_2PC_PREPARE_REDO_REQ <= msg_type && TX_2PC_PREPARE_VERSION_RESP >= msg_type); - } - }; - - - struct ObAskTxStateFor4377Msg - { - public: - ObAskTxStateFor4377Msg() : - tx_id_(), - ls_id_() {} - public: - ObTransID tx_id_; - share::ObLSID ls_id_; - bool is_valid() const; - TO_STRING_KV(K_(tx_id), K_(ls_id)); - OB_UNIS_VERSION(1); - }; - - struct ObAskTxStateFor4377RespMsg - { - public: - ObAskTxStateFor4377RespMsg() : - is_alive_(false), - ret_(OB_SUCCESS) {} - public: - bool is_alive_; - int ret_; - TO_STRING_KV(K_(is_alive), K_(ret)); - OB_UNIS_VERSION(1); - }; -} -} - -#endif // OCEANBASE_TRANSACTION_OB_TX_MSG_ diff --git a/src/storage/tx/ob_tx_on_demand_print.cpp b/src/storage/tx/ob_tx_on_demand_print.cpp index d68a5e14b..bb827cea8 100644 --- a/src/storage/tx/ob_tx_on_demand_print.cpp +++ b/src/storage/tx/ob_tx_on_demand_print.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -22,25 +22,23 @@ namespace oceanbase namespace transaction { -IMPL_ON_DEMAND_PRINT_FUNC(ObTxSubState) +IMPL_ON_DEMAND_PRINT_FUNC(ObTxRuntimeState) { int ret = OB_SUCCESS; int tmp_pos = 0; // if (flag_.is_valid()) { - ON_DEMAND_START_PRINT(SubState); + ON_DEMAND_START_PRINT(RuntimeState); TX_KV_PRINT_WITH_ERR(flag_.info_log_submitted_ > 0, info_log_submitted, flag_.info_log_submitted_, " "); - TX_KV_PRINT_WITH_ERR(flag_.gts_waiting_ > 0, gts_waiting, flag_.gts_waiting_, " "); TX_KV_PRINT_WITH_ERR(flag_.state_log_submitting_ > 0, state_log_submitting, flag_.state_log_submitting_, " "); TX_KV_PRINT_WITH_ERR(flag_.state_log_submitted_ > 0, state_log_submitted, flag_.state_log_submitted_, " "); - TX_KV_PRINT_WITH_ERR(flag_.prepare_notify_ > 0, prepare_notify, flag_.prepare_notify_, " "); TX_KV_PRINT_WITH_ERR(flag_.force_abort_ > 0, force_abort, flag_.force_abort_, " "); - ON_DEMAND_END_PRINT(SubState); + ON_DEMAND_END_PRINT(RuntimeState); // } return ret; } @@ -53,12 +51,9 @@ IMPL_ON_DEMAND_PRINT_FUNC(ObTxExecInfo) ON_DEMAND_START_PRINT(ExecInfo); TX_KV_PRINT_WITH_ERR(true, downstream_state, to_str_tx_state(state_), ", "); - TX_KV_PRINT_WITH_ERR(true, upstream, upstream_, ", "); - TX_KV_PRINT_WITH_ERR(true, participants,participants_, ", "); + TX_KV_PRINT_WITH_ERR(true, has_write_state, has_write_state_, ", "); TX_KV_PRINT_WITH_ERR(true, redo_log_no, redo_lsns_.count(), ", "); - TX_KV_PRINT_WITH_ERR(true, scheduler, scheduler_, ", "); TX_KV_PRINT_WITH_ERR(true, prepare_version, prepare_version_, ", "); - TX_KV_PRINT_WITH_ERR(true, trans_type, trans_type_, ", "); TX_KV_PRINT_WITH_ERR(true, next_log_entry_no, next_log_entry_no_, ", "); TX_KV_PRINT_WITH_ERR(true, max_applied_log_ts, max_applied_log_ts_, ", "); TX_KV_PRINT_WITH_ERR(true, max_appling_log_ts, max_applying_log_ts_, ", "); @@ -69,19 +64,13 @@ IMPL_ON_DEMAND_PRINT_FUNC(ObTxExecInfo) TX_KV_PRINT_WITH_ERR(true, need_checksum, need_checksum_, ", "); TX_KV_PRINT_WITH_ERR(true, data_complete, data_complete_, ", "); TX_KV_PRINT_WITH_ERR(true, is_dup_tx, is_dup_tx_, ", "); - TX_KV_PRINT_WITH_ERR(true, exec_epoch, exec_epoch_, ", "); - TX_KV_PRINT_WITH_ERR(!incremental_participants_.empty(),incremental_participants, incremental_participants_, ", "); - TX_KV_PRINT_WITH_ERR(!intermediate_participants_.empty(), intermediate_participants, intermediate_participants_, ", "); TX_KV_PRINT_WITH_ERR(prev_record_lsn_.is_valid(), prev_record_lsn, prev_record_lsn_, ", "); TX_KV_PRINT_WITH_ERR(!redo_lsns_.empty(), redo_lsns, redo_lsns_, ", "); TX_KV_PRINT_WITH_ERR(!multi_data_source_.empty(), multi_data_source, multi_data_source_, ", "); TX_KV_PRINT_WITH_ERR(max_durable_lsn_.is_valid(),max_durable_lsn , max_durable_lsn_, ", "); - TX_KV_PRINT_WITH_ERR(!prepare_log_info_arr_.empty(),prepare_log_info_arr , prepare_log_info_arr_, ", "); TX_KV_PRINT_WITH_ERR(!xid_.empty(), xid , xid_, ", "); - TX_KV_PRINT_WITH_ERR(is_sub2pc_, is_sub2pc , is_sub2pc_, ", "); - TX_KV_PRINT_WITH_ERR(!commit_parts_.empty(), commit_parts , commit_parts_, ", "); TX_KV_PRINT_WITH_ERR(serial_final_scn_.is_valid(), serial_final_scn, serial_final_scn_, ", "); TX_KV_PRINT_WITH_ERR(serial_final_seq_no_.is_valid(), serial_final_seq_no,serial_final_seq_no_, ", "); @@ -93,7 +82,7 @@ IMPL_ON_DEMAND_PRINT_FUNC(ObTxExecInfo) return ret; } -IMPL_ON_DEMAND_PRINT_FUNC(ObPartTransCtx) +IMPL_ON_DEMAND_PRINT_FUNC(ObTxCtx) { int ret = OB_SUCCESS; int tmp_pos = 0; @@ -103,11 +92,7 @@ IMPL_ON_DEMAND_PRINT_FUNC(ObPartTransCtx) TX_KV_PRINT_WITH_ERR(!busy_cbs_.is_empty(), busy_cbs_cnt, busy_cbs_.get_size(), ", "); TX_KV_PRINT_WITH_ERR(!busy_cbs_.is_empty(), oldest_busy_cb, busy_cbs_.get_first(), ", "); - TX_PRINT_FUNC_WITH_ERR(sub_state_.is_valid(), sub_state_.on_demand_print_, ", "); - - - TX_KV_PRINT_WITH_ERR(!coord_prepare_info_arr_.empty(), coord_prepare_info_arr_, - coord_prepare_info_arr_, ", "); + TX_PRINT_FUNC_WITH_ERR(runtime_state_.is_valid(), runtime_state_.on_demand_print_, ", "); TX_KV_PRINT_WITH_ERR(get_retain_cause() != RetainCause::UNKOWN, retain_cause, retain_cause_, ", "); diff --git a/src/storage/tx/ob_tx_print_time_guard.cpp b/src/storage/tx/ob_tx_print_time_guard.cpp new file mode 100644 index 000000000..4bc63ab1b --- /dev/null +++ b/src/storage/tx/ob_tx_print_time_guard.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "storage/tx/ob_tx_print_time_guard.h" +#include "lib/utility/ob_print_utils.h" + +namespace oceanbase +{ +namespace transaction +{ + +int64_t ObTxPrintTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int ret = common::OB_SUCCESS; + int64_t pos = 0; + const double total_diff = end_ts_ - start_ts_; + if (total_diff > 0) { + common::databuff_printf(buf, buf_len, pos, " [Total : %f ms] ", total_diff / 1000); + } + + for (int64_t i = 0; i < MAX_CLICK_COUNT; i++) { + const double tmp_diff = click_end_ts_[i] - click_start_ts_[i]; + if (tmp_diff > 0) { + common::databuff_printf(buf, buf_len, pos, " [%s : %f ms] ", click_str_[i], tmp_diff / 1000); + } + } + + if (0 == pos) { + ret = common::databuff_printf(buf, buf_len, pos, "invalid TxPrintTimeGuard"); + } + return common::OB_SUCCESS == ret ? pos : 0; +} + +} // namespace transaction +} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_print_time_guard.h b/src/storage/tx/ob_tx_print_time_guard.h new file mode 100644 index 000000000..a4ec15e5c --- /dev/null +++ b/src/storage/tx/ob_tx_print_time_guard.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_TRANSACTION_OB_TX_PRINT_TIME_GUARD_H_ +#define OCEANBASE_TRANSACTION_OB_TX_PRINT_TIME_GUARD_H_ + +#include +#include "lib/time/ob_time_utility.h" + +namespace oceanbase +{ +namespace transaction +{ + +class ObTxPrintTimeGuard +{ +public: + ObTxPrintTimeGuard() + : start_ts_(common::ObTimeUtility::fast_current_time()), end_ts_(0) + { + memset(click_str_, 0, sizeof(click_str_)); + memset(click_start_ts_, 0, sizeof(click_start_ts_)); + memset(click_end_ts_, 0, sizeof(click_end_ts_)); + } + + void click_start(const char *str, const int64_t click_index) + { + click_str_[click_index] = str; + click_start_ts_[click_index] = common::ObTimeUtility::fast_current_time(); + } + + void click_end(const int64_t click_index) + { + click_end_ts_[click_index] = common::ObTimeUtility::fast_current_time(); + } + + int64_t get_diff() + { + end_ts_ = common::ObTimeUtility::fast_current_time(); + return end_ts_ - start_ts_; + } + + ~ObTxPrintTimeGuard() + { + end_ts_ = common::ObTimeUtility::fast_current_time(); + } + + int64_t to_string(char *buf, const int64_t buf_len) const; + +private: + static const int64_t MAX_CLICK_COUNT = 16; + const char *click_str_[MAX_CLICK_COUNT]; + int64_t start_ts_; + int64_t end_ts_; + int64_t click_start_ts_[MAX_CLICK_COUNT]; + int64_t click_end_ts_[MAX_CLICK_COUNT]; +}; + +} // namespace transaction +} // namespace oceanbase + +#endif // OCEANBASE_TRANSACTION_OB_TX_PRINT_TIME_GUARD_H_ diff --git a/src/storage/tx/ob_tx_redo_submitter.h b/src/storage/tx/ob_tx_redo_submitter.h index 58c2e686b..9bc3da5f0 100644 --- a/src/storage/tx/ob_tx_redo_submitter.h +++ b/src/storage/tx/ob_tx_redo_submitter.h @@ -17,23 +17,22 @@ #ifndef OCEANBASE_TRANSACTION_OB_TRANS_REDO_SUBMITTER #define OCEANBASE_TRANSACTION_OB_TRANS_REDO_SUBMITTER -#include "ob_trans_part_ctx.h" +#include "ob_tx_ctx.h" namespace oceanbase { namespace transaction { -class ObPartTransCtx; +class ObTxCtx; class ObTxRedoSubmitter { public: - ObTxRedoSubmitter(ObPartTransCtx &tx_ctx, + ObTxRedoSubmitter(ObTxCtx &tx_ctx, memtable::ObMemtableCtx &mt_ctx) : tx_ctx_(tx_ctx), mt_ctx_(mt_ctx), tx_id_(tx_ctx.get_trans_id()), - ls_id_(tx_ctx.get_ls_id()), log_block_(NULL), log_cb_(NULL), helper_(NULL), @@ -84,7 +83,6 @@ class ObTxRedoSubmitter int after_submit_redo_out_(); public: TO_STRING_KV(K_(tx_id), - K_(ls_id), K_(from_all_list), K_(flush_all), K_(flush_freeze_clock), @@ -94,10 +92,9 @@ class ObTxRedoSubmitter K_(submit_out_cnt), K_(submit_cb_list_idx)); private: - ObPartTransCtx &tx_ctx_; + ObTxCtx &tx_ctx_; memtable::ObMemtableCtx &mt_ctx_; const ObTransID tx_id_; - const share::ObLSID ls_id_; ObTxLogBlock *log_block_; ObTxLogCb *log_cb_; memtable::ObRedoLogSubmitHelper *helper_; @@ -408,7 +405,7 @@ int ObTxRedoSubmitter::fill_log_block_(memtable::ObTxFillRedoCtx &ctx) bool need_retry = false; do { need_retry = false; - ObTxRedoLog log(tx_ctx_.get_cluster_version()); + ObTxRedoLog log; ret = log_block_->prepare_mutator_buf(log); ctx.buf_ = log.get_mutator_buf(); ctx.buf_len_ = log.get_mutator_size(); diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index 052297600..97620bda9 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -21,6 +21,9 @@ #include "storage/tx/ob_tx_replay_executor.h" #include "storage/tablelock/ob_lock_memtable.h" #include "logservice/replayservice/ob_tablet_replay_executor.h" +#include "storage/ls/ob_ls.h" +#include "storage/ls/ob_ls_tx_service.h" +#include "storage/ob_relative_table.h" namespace oceanbase { @@ -36,13 +39,11 @@ int ObTxReplayExecutor::execute(storage::ObLS *ls, const int skip_pos, const palf::LSN &lsn, const SCN &log_timestamp, - const logservice::ObLogBaseHeader &base_header, - const ObLSID &ls_id) + const logservice::ObLogBaseHeader &base_header) { int ret = OB_SUCCESS; ObTxReplayExecutor replay_executor(ls, - ls_id, ls_tx_srv, lsn, log_timestamp, @@ -53,7 +54,7 @@ int ObTxReplayExecutor::execute(storage::ObLS *ls, TRANS_LOG(ERROR, "invaild arguments", K(replay_executor), K(buf), K(size)); } else if (OB_FAIL(replay_executor.do_replay_(buf, size, skip_pos))) { TRANS_LOG(WARN, "replay_executor.do_replay failed", K(ret), - K(replay_executor), K(buf), K(size), K(skip_pos), K(ls_id)); + K(replay_executor), K(buf), K(size), K(skip_pos)); } else { if (log_timestamp <= ls->get_ls_wrs_handler()->get_ls_weak_read_ts()) { SCN min_log_service_scn; @@ -65,8 +66,7 @@ int ObTxReplayExecutor::execute(storage::ObLS *ls, K(buf), K(size), K(skip_pos), - K(min_log_service_scn), - K(ls_id)); + K(min_log_service_scn)); } } return ret; @@ -85,16 +85,6 @@ int ObTxReplayExecutor::do_replay_(const char *buf, const int64_t size, const in while (OB_SUCC(ret)) { if (OB_FAIL(try_get_tx_ctx_())) { TRANS_LOG(WARN, "try get tx ctx failed", K(ret), KPC(this)); - } else if (OB_ISNULL(ctx_)) { - // StartWorkingLog - if (OB_FAIL(log_block_.get_next_log(header))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - TRANS_LOG(ERROR, "[Replay Tx] get_next_log error in replay_buf", KPC(this)); - } - } } else if (OB_FAIL(iter_next_log_for_replay_(header))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -113,10 +103,6 @@ int ObTxReplayExecutor::do_replay_(const char *buf, const int64_t size, const in } else if (OB_FAIL(replay_redo_())) { TRANS_LOG(WARN, "[Replay Tx] replay redo log error", K(ret)); } - } else if (log_type == ObTxLogType::TX_START_WORKING_LOG) { - if (OB_FAIL(replay_start_working_())) { - TRANS_LOG(WARN, "[Replay Tx] replay clear log error", KR(ret)); - } } else { ret = replay_tx_log_(log_type); } @@ -174,24 +160,12 @@ int ObTxReplayExecutor::replay_tx_log_(const ObTxLogType log_type) } break; } - case ObTxLogType::TX_ACTIVE_INFO_LOG: { - if (OB_FAIL(replay_active_info_())) { - TRANS_LOG(WARN, "replay active_state error", K(ret)); - } - break; - } case ObTxLogType::TX_COMMIT_INFO_LOG: { if (OB_FAIL(replay_commit_info_())) { TRANS_LOG(WARN, "[Replay Tx] replay commit info log error", K(ret)); } break; } - case ObTxLogType::TX_PREPARE_LOG: { - if (OB_FAIL(replay_prepare_())) { - TRANS_LOG(WARN, "[Replay Tx] replay prepare log error", K(ret)); - } - break; - } case ObTxLogType::TX_COMMIT_LOG: { if (OB_FAIL(replay_commit_())) { TRANS_LOG(WARN, "[Replay Tx] replay commit log error", K(ret)); @@ -276,23 +250,19 @@ int ObTxReplayExecutor::try_get_tx_ctx_() } else if (OB_TRANS_CTX_NOT_EXIST == ret) { ret = OB_SUCCESS; bool tx_ctx_existed = false; - common::ObAddr scheduler = log_block_.get_header().get_scheduler(); // since 4.2.4, cluster version in log_block_header const uint64_t cluster_version = log_block_.get_header().get_cluster_version(); ObTxCreateArg arg(true, /* for_replay */ - PartCtxSource::REPLAY, + TxCtxSource::REPLAY, tx_id, - ls_id_, log_block_.get_header().get_org_cluster_id(), cluster_version, 0, /*session_id*/ 0, /*client_sid*/ 0, /*associated_session_id*/ - scheduler, INT64_MAX, /*trans_expired_time_*/ ls_tx_srv_->get_trans_service()); ObTxDataThrottleGuard tx_data_throttle_guard( - ls_id_, true /* for_replay_ */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); if (OB_FAIL(ls_tx_srv_->create_tx_ctx(arg, tx_ctx_existed, ctx_))) { @@ -414,7 +384,6 @@ int ObTxReplayExecutor::replay_rollback_to_() ObTxRollbackToLog log; const bool pre_barrier = base_header_.need_pre_replay_barrier(); ObTxDataThrottleGuard tx_data_throttle_guard( - ls_id_, true /* for_replay_ */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); if (OB_FAIL(log_block_.deserialize_log_body(log))) { @@ -427,44 +396,12 @@ int ObTxReplayExecutor::replay_rollback_to_() return ret; } -int ObTxReplayExecutor::replay_active_info_() -{ - int ret = OB_SUCCESS; - ObTxActiveInfoLogTempRef temp_ref; - ObTxActiveInfoLog active_info_log(temp_ref); - if (OB_FAIL(log_block_.deserialize_log_body(active_info_log))) { - TRANS_LOG(WARN, "[Replay Tx] deserialize log body error", K(ret), K(active_info_log), K(lsn_), - K(log_ts_ns_)); - } else if (OB_FAIL( - ctx_->replay_active_info(active_info_log, lsn_, log_ts_ns_, tx_part_log_no_))) { - TRANS_LOG(WARN, "[Replay Tx] replay active_info in tx_ctx error", K(ret), K(lsn_), - K(log_ts_ns_)); - } - return ret; -} - -int ObTxReplayExecutor::replay_start_working_() -{ - int ret = OB_SUCCESS; - ObTxStartWorkingLogTempRef temp_ref; - ObTxStartWorkingLog start_working_log(temp_ref); - if (OB_FAIL(log_block_.deserialize_log_body(start_working_log))) { - TRANS_LOG(WARN, "[Replay Tx] deserialize start_working log body error", KR(ret), K(start_working_log), - K(lsn_), K(log_ts_ns_)); - } else if (OB_FAIL(ls_tx_srv_->replay_start_working_log(start_working_log, log_ts_ns_))) { - TRANS_LOG(WARN, "[Replay Tx] replay start_working log in tx_ctx error", KR(ret), K(lsn_), - K(log_ts_ns_)); - } - return ret; -} - int ObTxReplayExecutor::replay_multi_source_data_() { int ret = OB_SUCCESS; ObTxMultiDataSourceLog log; - ObMdsThrottleGuard mds_throttle_guard(ls_id_, - true /* for_replay */, + ObMdsThrottleGuard mds_throttle_guard(true /* for_replay */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); @@ -508,22 +445,6 @@ int ObTxReplayExecutor::replay_commit_info_() return ret; } -int ObTxReplayExecutor::replay_prepare_() -{ - int ret = OB_SUCCESS; - ObTxPrepareLogTempRef temp_ref; - ObTxPrepareLog prepare_log(temp_ref); - - if (OB_FAIL(log_block_.deserialize_log_body(prepare_log))) { - TRANS_LOG(WARN, "[Replay Tx] deserialize log body error", K(ret), K(prepare_log), K(lsn_), - K(log_ts_ns_)); - } else if (OB_FAIL(ctx_->replay_prepare(prepare_log, lsn_, log_ts_ns_, tx_part_log_no_))) { - TRANS_LOG(WARN, "[Replay Tx] replay prepare in tx_ctx error", K(ret), K(lsn_), K(log_ts_ns_)); - } - - return ret; -} - int ObTxReplayExecutor::replay_commit_() { int ret = OB_SUCCESS; @@ -669,7 +590,6 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo, const bool s K(timeguard.get_diff()), K(log_ts_ns_), K(ctx_->get_trans_id()), - K(ctx_->get_ls_id()), K(mvcc_row_count_), K(table_lock_row_count_)); } @@ -714,7 +634,6 @@ int ObTxReplayExecutor::replay_one_row_in_memtable_(ObMutatorRowHeader &row_head } else { ObTablet *tablet = tablet_handle.get_obj(); storage::ObStoreCtx storeCtx; - storeCtx.ls_id_ = ctx_->get_ls_id(); (void)storeCtx.mvcc_acc_ctx_.init_replay( *ctx_, *mt_ctx_, @@ -785,7 +704,6 @@ int ObTxReplayExecutor::replay_row_(storage::ObStoreCtx &store_ctx, memtable::ObMemtableMutatorIterator *mmi_ptr) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet->get_ls_id(); const common::ObTabletID &tablet_id = tablet->get_tablet_id(); common::ObTimeGuard timeguard("replay_row_in_memtable", 10_ms); storage::ObIMemtable *mem_ptr = nullptr; @@ -798,22 +716,19 @@ int ObTxReplayExecutor::replay_row_(storage::ObStoreCtx &store_ctx, } else if (OB_FAIL(prepare_memtable_replay_(w_guard, mem_ptr))) { if (OB_NO_NEED_UPDATE == ret) { TRANS_LOG(DEBUG, "[Replay Tx] Not need replay row for tablet", - K(ret), K(ls_id), K(tablet_id), K(log_ts_ns_), + K(ret), K(tablet_id), K(log_ts_ns_), K(tx_part_log_no_), K(mmi_ptr->get_row_head())); } else if (OB_TABLET_NOT_EXIST == ret) { - if (OB_UNLIKELY(!true)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tenant config is invalid", K(ret)); - } else if (GCONF._allow_skip_replay_redo_after_detete_tablet) { + if (GCONF._allow_skip_replay_redo_after_detete_tablet) { ret = OB_NO_NEED_UPDATE; TRANS_LOG(WARN, "[Replay Tx] tablet does not exist while preparing memtable for replay, allow to skip this clog replaying for emergency", - K(ret), K(ls_id), K(tablet_id), K_(log_ts_ns)); + K(ret), K(tablet_id), K_(log_ts_ns)); } else { TRANS_LOG(ERROR, "[Replay Tx] tablet does not exist while preparing memtable for replay", - K(ret), K(ls_id), K(tablet_id), K_(log_ts_ns)); + K(ret), K(tablet_id), K_(log_ts_ns)); } } else { - TRANS_LOG(WARN, "[Replay Tx] prepare for replay failed", K(ret), K(ls_id), K(tablet_id), KP(mem_ptr), KP(mmi_ptr)); + TRANS_LOG(WARN, "[Replay Tx] prepare for replay failed", K(ret), K(tablet_id), KP(mem_ptr), KP(mmi_ptr)); } // dynamic_cast will check whether this is really a ObMemtable. } else if (OB_ISNULL(data_mem_ptr = static_cast(mem_ptr))) { @@ -866,11 +781,10 @@ void ObTxReplayExecutor::rewrite_replay_retry_code_(int &ret_code) if (ret_code == OB_MINOR_FREEZE_NOT_ALLOW || ret_code == OB_SCN_OUT_OF_BOUND || ret_code == OB_ALLOCATE_MEMORY_FAILED) { TRANS_LOG(INFO, "rewrite replay error_code as OB_EAGAIN for retry", K(ret_code), - K(ls_->get_ls_id()), K(log_ts_ns_)); + K(log_ts_ns_)); ret_code = OB_EAGAIN; } } } } // namespace oceanbase - diff --git a/src/storage/tx/ob_tx_replay_executor.h b/src/storage/tx/ob_tx_replay_executor.h index d3ad421a5..ed90c0d6a 100644 --- a/src/storage/tx/ob_tx_replay_executor.h +++ b/src/storage/tx/ob_tx_replay_executor.h @@ -46,10 +46,10 @@ class ObStorageTableGuard; namespace transaction { class ObTxLogBlock; -class ObPartTransCtx; +class ObTxCtx; class ObTransService; -typedef ObPartTransCtx ReplayTxCtx; +typedef ObTxCtx ReplayTxCtx; class ObTxReplayExecutor { public: @@ -60,13 +60,11 @@ class ObTxReplayExecutor { const int skip_pos, const palf::LSN &lsn, const share::SCN &log_timestamp, - const logservice::ObLogBaseHeader &base_header, - const share::ObLSID &ls_id); + const logservice::ObLogBaseHeader &base_header); public: TO_STRING_KV(KP(ctx_), KP(ls_), - K(ls_id_), KP(ls_tx_srv_), KP(mt_ctx_), K(log_block_), @@ -78,14 +76,12 @@ class ObTxReplayExecutor { private: ObTxReplayExecutor(storage::ObLS *ls, - const share::ObLSID &ls_id, storage::ObLSTxService *ls_tx_srv, const palf::LSN &lsn, const share::SCN &log_timestamp, const logservice::ObLogBaseHeader &base_header) : ctx_(nullptr), ls_(ls), - ls_id_(ls_id), ls_tx_srv_(ls_tx_srv), replay_queue_(0), replaying_log_entry_no_(0), @@ -118,13 +114,10 @@ class ObTxReplayExecutor { int replay_redo_(); int replay_tx_log_(const ObTxLogType log_type); int replay_rollback_to_(); - int replay_active_info_(); int replay_commit_info_(); - int replay_prepare_(); int replay_commit_(); int replay_abort_(); int replay_clear_(); - int replay_start_working_(); int replay_multi_source_data_(); int replay_record_(); bool is_tx_log_replay_queue() const { return replay_queue_ == 0; } @@ -148,7 +141,6 @@ class ObTxReplayExecutor { ReplayTxCtx *ctx_; storage::ObLS *ls_; - share::ObLSID ls_id_; storage::ObLSTxService *ls_tx_srv_; ObTxLogBlock log_block_; diff --git a/src/storage/tx/ob_tx_result_struct.cpp b/src/storage/tx/ob_tx_result_struct.cpp index 55a630a54..8ad33bc40 100644 --- a/src/storage/tx/ob_tx_result_struct.cpp +++ b/src/storage/tx/ob_tx_result_struct.cpp @@ -30,14 +30,5 @@ DEF_TO_STRING(ObCreateTabletBatchInTransRes) OB_SERIALIZE_MEMBER(ObCreateTabletBatchInTransRes, ret_, tx_result_); -void ObRegisterTxDataResult::reset() -{ - result_ = OB_SUCCESS; - tx_result_.reset(); - return; -} - -OB_SERIALIZE_MEMBER(ObRegisterTxDataResult, result_, tx_result_); - } // namespace obcall } // namespace oceanbase diff --git a/src/storage/tx/ob_tx_result_struct.h b/src/storage/tx/ob_tx_result_struct.h index 553e4428f..660ea06c6 100644 --- a/src/storage/tx/ob_tx_result_struct.h +++ b/src/storage/tx/ob_tx_result_struct.h @@ -43,21 +43,6 @@ struct ObCreateTabletBatchInTransRes using ObRemoveTabletsInTransRes = ObCreateTabletBatchInTransRes; -struct ObRegisterTxDataResult -{ - OB_UNIS_VERSION(1); -public: - ObRegisterTxDataResult() : result_(common::OB_SUCCESS), tx_result_() {} - ~ObRegisterTxDataResult() {} - void reset(); - TO_STRING_KV(K_(result), K_(tx_result)); -public: - int64_t result_; - transaction::ObTxExecResult tx_result_; -private: - DISALLOW_COPY_AND_ASSIGN(ObRegisterTxDataResult); -}; - } // namespace obcall } // namespace oceanbase #endif diff --git a/src/storage/tx/ob_tx_retain_ctx_mgr.cpp b/src/storage/tx/ob_tx_retain_ctx_mgr.cpp index 234703bae..7d92f6304 100644 --- a/src/storage/tx/ob_tx_retain_ctx_mgr.cpp +++ b/src/storage/tx/ob_tx_retain_ctx_mgr.cpp @@ -16,7 +16,7 @@ #include "ob_tx_retain_ctx_mgr.h" #include "share/rc/ob_module_provider.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_trans_service.h" #include "storage/tx_storage/ob_ls_service.h" @@ -27,17 +27,15 @@ using namespace share; namespace transaction { -ObAdvanceLSCkptTask::ObAdvanceLSCkptTask(share::ObLSID ls_id, SCN ckpt_ts) +ObAdvanceLSCkptTask::ObAdvanceLSCkptTask(SCN ckpt_ts) { ; task_type_ = ObTransRetryTaskType::ADVANCE_LS_CKPT_TASK; - ls_id_ = ls_id; target_ckpt_ts_ = ckpt_ts; } void ObAdvanceLSCkptTask::reset() { - ls_id_ = share::ObLSID::INVALID_LS_ID; target_ckpt_ts_.reset(); } @@ -45,29 +43,26 @@ int ObAdvanceLSCkptTask::try_advance_ls_ckpt_ts() { int ret = OB_SUCCESS; - storage::ObLSHandle ls_handle; + storage::ObLS *tenant_ls = nullptr; if (OB_ISNULL(share::g_mp->ls_service()) - || OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, storage::ObLSGetMod::TRANS_MOD)) - || !ls_handle.is_valid()) { + || OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { if (OB_SUCCESS == ret) { ret = OB_INVALID_ARGUMENT; } TRANS_LOG(WARN, "get ls faild", K(ret), K(share::g_mp->ls_service())); - } else if (OB_FAIL(ls_handle.get_ls()->advance_checkpoint_by_flush( + } else if (OB_FAIL(tenant_ls->advance_checkpoint_by_flush( target_ckpt_ts_, INT64_MAX, /*timeout*/ false, /*is_tenant_freeze*/ ObFreezeSourceFlag::GC_RETAIN_CTX))) { - TRANS_LOG(WARN, "advance checkpoint ts failed", K(ret), K(ls_id_), K(target_ckpt_ts_)); + TRANS_LOG(WARN, "advance checkpoint ts failed", K(ret), K(target_ckpt_ts_)); } if (OB_SUCC(ret)) { - TRANS_LOG(INFO, "[RetainCtxMgr] advance ls checkpoint ts success", K(ret), K(ls_id_), - K(target_ckpt_ts_)); + TRANS_LOG(INFO, "[RetainCtxMgr] advance checkpoint ts success", K(ret), K(target_ckpt_ts_)); } else { - TRANS_LOG(WARN, "[RetainCtxMgr] advance ls checkpoint ts failed", K(ret), K(ls_id_), - K(target_ckpt_ts_)); + TRANS_LOG(WARN, "[RetainCtxMgr] advance checkpoint ts failed", K(ret), K(target_ckpt_ts_)); } return ret; @@ -85,7 +80,7 @@ ObIRetainCtxCheckFunctor::~ObIRetainCtxCheckFunctor() } } -int ObIRetainCtxCheckFunctor::init(ObPartTransCtx *ctx, RetainCause cause) +int ObIRetainCtxCheckFunctor::init(ObTxCtx *ctx, RetainCause cause) { int ret = OB_SUCCESS; if (ctx == nullptr || cause == RetainCause::UNKOWN) { @@ -97,7 +92,6 @@ int ObIRetainCtxCheckFunctor::init(ObPartTransCtx *ctx, RetainCause cause) tx_ctx_ = ctx; tx_ctx_->set_retain_cause(cause); tx_id_ = tx_ctx_->get_trans_id(); - ls_id_ = tx_ctx_->get_ls_id(); // TRANS_LOG( INFO , "before inc retain ctx ref", K(ret),KPC(this),K(*tx_ctx_)); // tx_ctx_->acquire_ctx_ref(); // TRANS_LOG( INFO , "after inc retain ctx ref", K(ret),KPC(this),K(*tx_ctx_)); @@ -123,7 +117,7 @@ int ObIRetainCtxCheckFunctor::del_retain_ctx() return ret; } -int ObMDSRetainCtxFunctor::init(ObPartTransCtx *ctx, +int ObMDSRetainCtxFunctor::init(ObTxCtx *ctx, RetainCause cause, const SCN &final_log_ts) { @@ -264,7 +258,7 @@ int ObTxRetainCtxMgr::force_gc_retain_ctx() return ret; } -int ObTxRetainCtxMgr::print_retain_ctx_info(share::ObLSID ls_id) +int ObTxRetainCtxMgr::print_retain_ctx_info() { int ret = OB_SUCCESS; @@ -272,25 +266,20 @@ int ObTxRetainCtxMgr::print_retain_ctx_info(share::ObLSID ls_id) SpinRLockGuard guard(retain_ctx_lock_); tg.click(); if (!retain_ctx_list_.is_empty()) { - TRANS_LOG(INFO, "[RetainCtxMgr] print retain ctx", K(ls_id), KPC(this), + TRANS_LOG(INFO, "[RetainCtxMgr] print retain ctx", KPC(this), KPC(retain_ctx_list_.get_first()), KPC(retain_ctx_list_.get_last())); } return ret; } -void ObTxRetainCtxMgr::try_advance_retain_ctx_gc(share::ObLSID ls_id) +void ObTxRetainCtxMgr::try_advance_retain_ctx_gc() { int ret = OB_SUCCESS; - - int64_t CUR_LS_CNT = share::g_mp->ls_service()->get_ls_map()->get_ls_count(); - if (CUR_LS_CNT == 0) { - CUR_LS_CNT = 1; - } const int64_t IDLE_GC_INTERVAL = 30 * 60 * 1000 * 1000; // 30 min // const int64_t MIN_RETAIN_CTX_GC_THRESHOLD = 1000; const int64_t MIN_RETAIN_CTX_GC_THRESHOLD = ::oceanbase::lib::get_tenant_memory_limit() - / sizeof(ObPartTransCtx) / CUR_LS_CNT / 100; + / sizeof(ObTxCtx) / 100; ObTimeGuard tg(__func__, 1 * 1000 * 1000); SpinRLockGuard guard(retain_ctx_lock_); @@ -302,7 +291,7 @@ void ObTxRetainCtxMgr::try_advance_retain_ctx_gc(share::ObLSID ls_id) if (retain_ctx_list_.get_size() <= 0) { // do nothing } else if (retain_ctx_list_.get_size() - <= std::min(MAX_PART_CTX_COUNT / 10 / CUR_LS_CNT, MIN_RETAIN_CTX_GC_THRESHOLD) + <= std::min(MAX_TX_CTX_COUNT / 10, MIN_RETAIN_CTX_GC_THRESHOLD) && (OB_INVALID_TIMESTAMP == last_push_gc_task_ts_ || (OB_INVALID_TIMESTAMP != last_push_gc_task_ts_ && cur_time - last_push_gc_task_ts_ <= IDLE_GC_INTERVAL))) { @@ -311,14 +300,14 @@ void ObTxRetainCtxMgr::try_advance_retain_ctx_gc(share::ObLSID ls_id) share::mtl_malloc(sizeof(ObAdvanceLSCkptTask), "ad_ckpt_task")))) { ret = OB_ALLOCATE_MEMORY_FAILED; TRANS_LOG(WARN, "alloc ObAdvanceLSCkptTask failed", K(ret)); - } else if (OB_FALSE_IT(new (task) ObAdvanceLSCkptTask(ls_id, max_wait_ckpt_ts_))) { + } else if (OB_FALSE_IT(new (task) ObAdvanceLSCkptTask(max_wait_ckpt_ts_))) { } else if (share::g_mp->trans_service()->push(task)) { TRANS_LOG(INFO, "[RetainCtxMgr] push ObAdvanceLSCkptTask failed", K(ret), - K(retain_ctx_list_.get_size()), K(last_push_gc_task_ts_), K(ls_id), + K(retain_ctx_list_.get_size()), K(last_push_gc_task_ts_), K(MIN_RETAIN_CTX_GC_THRESHOLD), KPC(this)); } else { TRANS_LOG(INFO, "[RetainCtxMgr] push ObAdvanceLSCkptTask success", K(ret), - K(retain_ctx_list_.get_size()), K(last_push_gc_task_ts_), K(ls_id), + K(retain_ctx_list_.get_size()), K(last_push_gc_task_ts_), K(MIN_RETAIN_CTX_GC_THRESHOLD), KPC(this)); last_push_gc_task_ts_ = cur_time; } diff --git a/src/storage/tx/ob_tx_retain_ctx_mgr.h b/src/storage/tx/ob_tx_retain_ctx_mgr.h index 7e63adeb0..2ae878777 100644 --- a/src/storage/tx/ob_tx_retain_ctx_mgr.h +++ b/src/storage/tx/ob_tx_retain_ctx_mgr.h @@ -33,20 +33,19 @@ class ObLS; namespace transaction { -class ObPartTransCtx; +class ObTxCtx; class ObTxRetainCtxMgr; class ObAdvanceLSCkptTask : public ObTransTask { public: - ObAdvanceLSCkptTask(share::ObLSID ls_id, share::SCN target_ts); + explicit ObAdvanceLSCkptTask(share::SCN target_ts); ~ObAdvanceLSCkptTask() { reset(); } void reset(); int try_advance_ls_ckpt_ts(); private: - share::ObLSID ls_id_; share::SCN target_ckpt_ts_; }; @@ -60,7 +59,7 @@ class ObIRetainCtxCheckFunctor : public common::ObDLinkBase 0 && seq >> 62 == 0); + OB_ASSERT(seq > 0 && seq >> 48 == 0); OB_ASSERT(branch >= 0); } private: explicit ObTxSEQ(int64_t raw_v): raw_val_(raw_v) {} public: - // old version builder - static ObTxSEQ mk_v0(int64_t seq_v) - { - OB_ASSERT(seq_v > 0); - return ObTxSEQ(seq_v); - } static const ObTxSEQ &INVL() { static ObTxSEQ v; return v; } static const ObTxSEQ &MIN_VAL() { static ObTxSEQ v(1, 0); return v; } static const ObTxSEQ &MAX_VAL() { static ObTxSEQ v(INT64_MAX); return v; } @@ -60,20 +54,16 @@ class ObTxSEQ ObTxSEQ clone_with_seq(int64_t seq_abs, int64_t seq_base) const { ObTxSEQ n = *this; - if (n_format_) { - n.seq_ = seq_abs - seq_base; - } else { - n.seq_v0_ = seq_abs; - } + n.seq_ = seq_abs - seq_base; return n; } bool operator>(const ObTxSEQ &b) const { - return n_format_ ? seq_ > b.seq_ : seq_v0_ > b.seq_v0_; + return seq_ > b.seq_; } bool operator>=(const ObTxSEQ &b) const { - return *this > b || (n_format_ ? (seq_ == b.seq_) : (seq_v0_ == b.seq_v0_)); + return seq_ >= b.seq_; } bool operator<(const ObTxSEQ &b) const { @@ -92,26 +82,23 @@ class ObTxSEQ return !(*this == b); } ObTxSEQ &operator++() { - if (n_format_) { ++seq_; } else { ++seq_v0_; } + ++seq_; return *this; } ObTxSEQ operator+(int n) const { - int64_t s = n_format_ ? seq_ + n : seq_v0_ + n; - return n_format_ ? ObTxSEQ(s, branch_) : ObTxSEQ::mk_v0(s); + return ObTxSEQ(seq_ + n, branch_); } ObTxSEQ operator-(int n) const { - int64_t s = n_format_ ? seq_ - n : seq_v0_ - n; - return n_format_ ? ObTxSEQ(s, branch_) : ObTxSEQ::mk_v0(s); + return ObTxSEQ(seq_ - n, branch_); } uint64_t hash() const { return murmurhash(&raw_val_, sizeof(raw_val_), 0); } // atomic incremental update int64_t inc_update(const ObTxSEQ &b) { return common::inc_update(&raw_val_, b.raw_val_); } uint64_t cast_to_int() const { return raw_val_; } static ObTxSEQ cast_from_int(int64_t seq) { return ObTxSEQ(seq); } - bool support_branch() const { return n_format_; } // return sequence number - int64_t get_seq() const { return n_format_ ? seq_ : seq_v0_; } - int16_t get_branch() const { return n_format_ ? branch_ : 0; } + int64_t get_seq() const { return seq_; } + int16_t get_branch() const { return branch_; } ObTxSEQ &set_branch(int16_t branch) { branch_ = branch; return *this; } // atomic Load/Store void atomic_reset() { ATOMIC_SET(&raw_val_, 0); } @@ -122,16 +109,10 @@ class ObTxSEQ private: union { int64_t raw_val_; - union { - struct { // v0, old_version - uint64_t seq_v0_ :62; - }; - struct { // new_version - uint64_t branch_ :15; - uint64_t seq_ :47; - uint64_t n_format_ :1; - uint64_t _sign_ :1; - }; + struct { + uint64_t branch_ :15; + uint64_t seq_ :48; + uint64_t _sign_ :1; }; }; }; @@ -142,7 +123,7 @@ inline int64_t ObTxSEQ::to_string(char* buf, const int64_t buf_len) const int64_t pos = 0; if (raw_val_ == INT64_MAX) { BUF_PRINTF("MAX"); - } else if (_sign_ == 0 && n_format_) { + } else if (is_valid()) { J_OBJ_START(); J_KV(K_(branch), "seq", seq_); J_OBJ_END(); diff --git a/src/storage/tx/ob_tx_serialization.cpp b/src/storage/tx/ob_tx_serialization.cpp deleted file mode 100644 index 344df8fb6..000000000 --- a/src/storage/tx/ob_tx_serialization.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "storage/tx/ob_tx_serialization.h" - -namespace oceanbase -{ -namespace transaction -{ - - // const static uint8_t BASE_VALID_BIT = 1; - // const static uint8_t NEXT_BYTE_VALID = BASE_VALID_BIT << 7; - -const uint8_t ObTxSerCompatByte::BASE_VALID_BIT = 1; -const uint8_t ObTxSerCompatByte::NEXT_BYTE_VALID = BASE_VALID_BIT << 7; -const uint8_t ObTxSerCompatByte::BASE_VALID_BYTE = UINT8_MAX; - -const uint8_t ObTxSerCompatByte::MAX_OBJECT_COUNT_PER_BYTE = 7; -const uint8_t ObTxSerCompatByte::MAX_BIT_COUNT_PER_BYTE = 8; - - -void ObTxSerCompatByte::clear_compat_bytes_() -{ - memset(compat_bytes_,0 , MAX_COMPAT_BYTES_COUNT); -} - -void ObTxSerCompatByte::reset() -{ - total_obj_cnt_ = 0; - total_byte_cnt_ = 0; - clear_compat_bytes_(); -} - -int ObTxSerCompatByte::init(int64_t total_object_count) -{ - int ret = OB_SUCCESS; - - if (total_obj_cnt_ > 0) { - ret = OB_INIT_TWICE; - TRANS_LOG(WARN, "init compat_bytes_ twice", K(ret), KPC(this)); - } else if (total_object_count < 0 || total_obj_cnt_ > 0) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(total_object_count), KPC(this)); - } else { - const int64_t total_byte_cnt = total_object_count / MAX_OBJECT_COUNT_PER_BYTE - + (total_object_count % MAX_OBJECT_COUNT_PER_BYTE == 0 ? 0 : 1); - total_obj_cnt_ = total_object_count; - init_all_bytes_valid_(total_byte_cnt); - } - return ret; -} - -int ObTxSerCompatByte::set_all_member_need_ser() -{ - int ret = OB_SUCCESS; - - if (total_obj_cnt_ <= 0 || total_byte_cnt_ <= 0) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "compat bytes have not been inited", K(ret), KPC(this)); - } else { - init_all_bytes_valid_(total_byte_cnt_); - } - - return ret; -} - -int ObTxSerCompatByte::set_object_flag(int64_t object_index, bool is_valid) -{ - int ret = OB_SUCCESS; - - int64_t byte_index = -1; - uint8_t bit_index = 0; - - if (OB_FAIL(cal_object_location_(object_index, byte_index, bit_index))) { - TRANS_LOG(WARN, "cal object location failed", K(ret)); - } else if (OB_FALSE_IT(set_object_flag_(byte_index, bit_index, is_valid))) { - } - - // TRANS_LOG(INFO, "set object flag", K(object_index), K(is_valid), K(byte_index), K(bit_index), - // K(compat_bytes_[byte_index])); - return ret; -} - -bool ObTxSerCompatByte::is_object_valid(int64_t object_index) const -{ - int ret = OB_SUCCESS; - bool is_valid = false; - - int64_t byte_index = -1; - uint8_t bit_index = 0; - - if (OB_FAIL(cal_object_location_(object_index, byte_index, bit_index, true/*ignore_compat_warn*/))) { - if (OB_INVALID_ARGUMENT == ret) { - // ret = OB_SUCCESS; - is_valid = false; - } else { - TRANS_LOG(WARN, "cal object location failed", K(ret)); - } - } else if (OB_FALSE_IT(is_object_valid_(byte_index, bit_index, is_valid))) { - } - - return is_valid; -} - -int ObTxSerCompatByte::cal_object_location_(int64_t object_index, - int64_t &byte_index, - uint8_t &bit_index, - const bool ignore_compat_warn) const -{ - int ret = OB_SUCCESS; - - if (object_index < 1 || object_index > total_obj_cnt_) { - ret = OB_INVALID_ARGUMENT; - if (!ignore_compat_warn) { - TRANS_LOG(WARN, "object index may be invalid", K(ret), K(object_index), KPC(this)); - } - } else { - byte_index = object_index / MAX_OBJECT_COUNT_PER_BYTE - 1; - bit_index = object_index % MAX_OBJECT_COUNT_PER_BYTE; - if (bit_index == 0) { - bit_index = MAX_OBJECT_COUNT_PER_BYTE - 1; - } else { - bit_index -= 1; - byte_index += 1; - } - } - return ret; -} - -void ObTxSerCompatByte::set_object_flag_(int64_t byte_index, uint8_t bit_index, bool is_valid) -{ - uint8_t bit_flag = BASE_VALID_BIT << bit_index; - - if (byte_index > total_byte_cnt_) { - TRANS_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "invalid byte index", K(byte_index), K(bit_index), K(is_valid), KPC(this)); - } else if (is_valid) { - compat_bytes_[byte_index] |= bit_flag; - } else { - compat_bytes_[byte_index] &= ~bit_flag; - } -} - -void ObTxSerCompatByte::is_object_valid_(int64_t byte_index, - uint8_t bit_index, - bool &is_valid) const -{ - uint8_t bit_check_flag = BASE_VALID_BIT << bit_index; - - if (byte_index > total_byte_cnt_) { - TRANS_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "invalid byte index", K(byte_index), K(bit_index), K(is_valid), KPC(this)); - } else { - is_valid = (compat_bytes_[byte_index] & bit_check_flag) != 0; - } -} - -int ObTxSerCompatByte::init_all_bytes_valid_(const int64_t total_byte_cnt) -{ - int ret = OB_SUCCESS; - const int64_t remain_bits = total_obj_cnt_ % MAX_OBJECT_COUNT_PER_BYTE; - - if (OB_FALSE_IT(clear_compat_bytes_())) { - - } else { - - total_byte_cnt_ = total_byte_cnt; - - for (int64_t i = 0; i < total_byte_cnt && OB_SUCC(ret); i++) { - - compat_bytes_[i] = BASE_VALID_BYTE; - - if (i < total_byte_cnt - 1) { - // do nothing - } else if (remain_bits == 0) { - compat_bytes_[i] &= ~NEXT_BYTE_VALID; - } else { - for (uint8_t j = remain_bits; j < MAX_BIT_COUNT_PER_BYTE; j++) { - set_object_flag_(i, j, false); - } - } - } - - if (OB_FAIL(ret)) { - total_byte_cnt_ = 0; - } - } - return ret; -} - -int ObTxSerCompatByte::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - - for (int64_t i = 0; i < total_byte_cnt_; i++) { - if (OB_FAIL(serialization::encode_i8(buf, buf_len, pos, compat_bytes_[i]))) { - TRANS_LOG(WARN, "encode compat byte", K(ret)); - } - } - - return ret; -} - -int ObTxSerCompatByte::deserialize(const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - - uint8_t compat_byte = 0; - - int64_t loop_cnt = 0; - - reset(); - do { - if (OB_FAIL(serialization::decode_i8(buf, data_len, pos, (int8_t *)(&compat_byte)))) { - TRANS_LOG(WARN, "decode compat_byte failed", K(ret), K(data_len), K(pos), K(compat_byte), - K(loop_cnt)); - } else if (OB_FALSE_IT(compat_bytes_[loop_cnt] = compat_byte)) { - // do nothing - } - loop_cnt++; - } while ((compat_byte & NEXT_BYTE_VALID) != 0 && OB_SUCC(ret)); - - if (OB_SUCC(ret)) { - total_byte_cnt_ = loop_cnt; - total_obj_cnt_ = total_byte_cnt_ * MAX_OBJECT_COUNT_PER_BYTE; - } - - return ret; -} - -int64_t ObTxSerCompatByte::get_serialize_size(void) const { return total_byte_cnt_; } - - -int64_t ObTxPrintTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int ret = OB_SUCCESS; - int64_t pos = 0; - - double total_diff = end_ts_ - start_ts_; - if (total_diff > 0) { - databuff_printf(buf, buf_len, pos, " [Total : %f ms] ", total_diff / 1000); - } - - for (int i = 0; i < MAX_CLICK_COUNT; i++) { - - double tmp_diff = click_end_ts_[i] - click_start_ts_[i]; - if (tmp_diff > 0) { - - databuff_printf(buf, buf_len, pos, " [%s : %f ms] ", click_str_[i], tmp_diff / 1000); - } - } - - if (pos == 0 || OB_FAIL(ret)) { - pos = 0; - ret = databuff_printf(buf, buf_len, pos, "invalid TxPrintTimeGuard"); - } - - return pos; -} - -} // namespace transacation -} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_serialization.h b/src/storage/tx/ob_tx_serialization.h deleted file mode 100644 index b4fbb6533..000000000 --- a/src/storage/tx/ob_tx_serialization.h +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TX_SERIALIZATION -#define OCEANBASE_TRANSACTION_OB_TX_SERIALIZATION - -#include "lib/container/ob_se_array.h" -#include "lib/utility/ob_macro_utils.h" -#include "lib/utility/ob_unify_serialize.h" - -#define MAX_COMPAT_BYTES_COUNT 14 - -namespace oceanbase -{ -namespace transaction -{ -typedef common::ObSEArray CompatByteArray; - -class ObTxSerCompatByte -{ -public: - NEED_SERIALIZE_AND_DESERIALIZE; - - // object_index start from 1 - // byte_index start from 0 - // bit_index start from 0 -public: - ObTxSerCompatByte() { reset(); } - void reset(); - int init(int64_t total_object_count); - bool is_inited() { return total_byte_cnt_ > 0 && total_obj_cnt_ > 0; } - int set_all_member_need_ser(); - int set_object_flag(int64_t object_index, bool is_valid); - bool is_object_valid(int64_t object_index) const; - - const static uint8_t BASE_VALID_BIT; - const static uint8_t NEXT_BYTE_VALID; - const static uint8_t BASE_VALID_BYTE; - - const static uint8_t MAX_OBJECT_COUNT_PER_BYTE; - const static uint8_t MAX_BIT_COUNT_PER_BYTE; - - TO_STRING_KV(K(total_obj_cnt_), K(total_byte_cnt_), KP(this)); - -private: - void clear_compat_bytes_(); - int cal_object_location_(int64_t object_index, - int64_t &byte_index, - uint8_t &bit_index, - const bool ignore_compat_warn = false) const; - void set_object_flag_(int64_t byte_index, uint8_t bit_index, bool is_valid); - void is_object_valid_(int64_t byte_index, uint8_t bit_index, bool &is_valid) const; - int init_all_bytes_valid_(const int64_t total_byte_cnt); - -private: - int64_t total_obj_cnt_; - int64_t total_byte_cnt_; - uint8_t compat_bytes_[MAX_COMPAT_BYTES_COUNT]; - // CompatByteArray compat_bytes_; -}; - -#define TX_NO_NEED_SER(NO_NEED_SER_COND, IDX, COMPAT_BYTES) \ - if (OB_SUCC(ret)) { \ - if (NO_NEED_SER_COND) { \ - if (OB_FAIL(COMPAT_BYTES.set_object_flag(IDX, false))) \ - TRANS_LOG(WARN, "set object invalid failed", K(NO_NEED_SER_COND), K(IDX), \ - K(COMPAT_BYTES)); \ - } \ - } - -#define TX_IS_NEED_SER_OR_DE(COMPAT_BYTES, IDX) COMPAT_BYTES.is_object_valid(IDX) - -#define TX_SER_COMPAT_BYTES(COMPAT_BYTES) \ - if (OB_SUCC(ret)) { \ - if (OB_FAIL(COMPAT_BYTES.serialize(buf, buf_len, pos))) { \ - TRANS_LOG(WARN, "serialize compat_bytes_ failed",K(ret), K(buf_len),K(pos),K(COMPAT_BYTES)); \ - } \ - } - -#define TX_DSER_COMPAT_BYTES(COMPAT_BYTES) \ - if (OB_SUCC(ret)) { \ - if (OB_FAIL(COMPAT_BYTES.deserialize(buf, data_len, pos))) { \ - TRANS_LOG(WARN, "deserialize compat_bytes_ failed",K(ret), K(data_len),K(pos),K(COMPAT_BYTES)); \ - } \ - } - -#define TX_SER_SIZE_COMPAT_BYTES(COMPAT_BYTES) len += COMPAT_BYTES.get_serialize_size(); - - - - -#define TX_LST_DO_1(M, arg, s, P, ...) P(M, arg, 1, ##__VA_ARGS__) -#define TX_LST_DO_2(M, arg, s, P, ...) TX_LST_DO_1(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 2, ##__VA_ARGS__) -#define TX_LST_DO_3(M, arg, s, P, ...) TX_LST_DO_2(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 3, ##__VA_ARGS__) -#define TX_LST_DO_4(M, arg, s, P, ...) TX_LST_DO_3(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 4, ##__VA_ARGS__) -#define TX_LST_DO_5(M, arg, s, P, ...) TX_LST_DO_4(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 5, ##__VA_ARGS__) -#define TX_LST_DO_6(M, arg, s, P, ...) TX_LST_DO_5(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 6, ##__VA_ARGS__) -#define TX_LST_DO_7(M, arg, s, P, ...) TX_LST_DO_6(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 7, ##__VA_ARGS__) -#define TX_LST_DO_8(M, arg, s, P, ...) TX_LST_DO_7(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 8, ##__VA_ARGS__) -#define TX_LST_DO_9(M, arg, s, P, ...) TX_LST_DO_8(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 9, ##__VA_ARGS__) -#define TX_LST_DO_10(M, arg, s, P, ...) TX_LST_DO_9(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 10, ##__VA_ARGS__) -#define TX_LST_DO_11(M, arg, s, P, ...) TX_LST_DO_10(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 11, ##__VA_ARGS__) -#define TX_LST_DO_12(M, arg, s, P, ...) TX_LST_DO_11(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 12, ##__VA_ARGS__) -#define TX_LST_DO_13(M, arg, s, P, ...) TX_LST_DO_12(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 13, ##__VA_ARGS__) -#define TX_LST_DO_14(M, arg, s, P, ...) TX_LST_DO_13(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 14, ##__VA_ARGS__) -#define TX_LST_DO_15(M, arg, s, P, ...) TX_LST_DO_14(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 15, ##__VA_ARGS__) -#define TX_LST_DO_16(M, arg, s, P, ...) TX_LST_DO_15(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 16, ##__VA_ARGS__) -#define TX_LST_DO_17(M, arg, s, P, ...) TX_LST_DO_16(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 17, ##__VA_ARGS__) -#define TX_LST_DO_18(M, arg, s, P, ...) TX_LST_DO_17(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 18, ##__VA_ARGS__) -#define TX_LST_DO_19(M, arg, s, P, ...) TX_LST_DO_18(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 19, ##__VA_ARGS__) -#define TX_LST_DO_20(M, arg, s, P, ...) TX_LST_DO_19(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 20, ##__VA_ARGS__) -#define TX_LST_DO_21(M, arg, s, P, ...) TX_LST_DO_20(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 21, ##__VA_ARGS__) -#define TX_LST_DO_22(M, arg, s, P, ...) TX_LST_DO_21(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 22, ##__VA_ARGS__) -#define TX_LST_DO_23(M, arg, s, P, ...) TX_LST_DO_22(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 23, ##__VA_ARGS__) -#define TX_LST_DO_24(M, arg, s, P, ...) TX_LST_DO_23(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 24, ##__VA_ARGS__) -#define TX_LST_DO_25(M, arg, s, P, ...) TX_LST_DO_24(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 25, ##__VA_ARGS__) -#define TX_LST_DO_26(M, arg, s, P, ...) TX_LST_DO_25(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 26, ##__VA_ARGS__) -#define TX_LST_DO_27(M, arg, s, P, ...) TX_LST_DO_26(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 27, ##__VA_ARGS__) -#define TX_LST_DO_28(M, arg, s, P, ...) TX_LST_DO_27(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 28, ##__VA_ARGS__) -#define TX_LST_DO_29(M, arg, s, P, ...) TX_LST_DO_28(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 29, ##__VA_ARGS__) -#define TX_LST_DO_30(M, arg, s, P, ...) TX_LST_DO_29(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 30, ##__VA_ARGS__) -#define TX_LST_DO_31(M, arg, s, P, ...) TX_LST_DO_30(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 31, ##__VA_ARGS__) -#define TX_LST_DO_32(M, arg, s, P, ...) TX_LST_DO_31(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 32, ##__VA_ARGS__) -#define TX_LST_DO_33(M, arg, s, P, ...) TX_LST_DO_32(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 33, ##__VA_ARGS__) -#define TX_LST_DO_34(M, arg, s, P, ...) TX_LST_DO_33(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 34, ##__VA_ARGS__) -#define TX_LST_DO_35(M, arg, s, P, ...) TX_LST_DO_34(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 35, ##__VA_ARGS__) -#define TX_LST_DO_36(M, arg, s, P, ...) TX_LST_DO_35(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 36, ##__VA_ARGS__) -#define TX_LST_DO_37(M, arg, s, P, ...) TX_LST_DO_36(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 37, ##__VA_ARGS__) -#define TX_LST_DO_38(M, arg, s, P, ...) TX_LST_DO_37(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 38, ##__VA_ARGS__) -#define TX_LST_DO_39(M, arg, s, P, ...) TX_LST_DO_38(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 39, ##__VA_ARGS__) -#define TX_LST_DO_40(M, arg, s, P, ...) TX_LST_DO_39(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 40, ##__VA_ARGS__) -#define TX_LST_DO_41(M, arg, s, P, ...) TX_LST_DO_40(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 41, ##__VA_ARGS__) -#define TX_LST_DO_42(M, arg, s, P, ...) TX_LST_DO_41(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 42, ##__VA_ARGS__) -#define TX_LST_DO_43(M, arg, s, P, ...) TX_LST_DO_42(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 43, ##__VA_ARGS__) -#define TX_LST_DO_44(M, arg, s, P, ...) TX_LST_DO_43(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 44, ##__VA_ARGS__) -#define TX_LST_DO_45(M, arg, s, P, ...) TX_LST_DO_44(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 45, ##__VA_ARGS__) -#define TX_LST_DO_46(M, arg, s, P, ...) TX_LST_DO_45(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 46, ##__VA_ARGS__) -#define TX_LST_DO_47(M, arg, s, P, ...) TX_LST_DO_46(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 47, ##__VA_ARGS__) -#define TX_LST_DO_48(M, arg, s, P, ...) TX_LST_DO_47(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 48, ##__VA_ARGS__) -#define TX_LST_DO_49(M, arg, s, P, ...) TX_LST_DO_48(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 49, ##__VA_ARGS__) -#define TX_LST_DO_50(M, arg, s, P, ...) TX_LST_DO_49(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 50, ##__VA_ARGS__) -#define TX_LST_DO_51(M, arg, s, P, ...) TX_LST_DO_50(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 51, ##__VA_ARGS__) -#define TX_LST_DO_52(M, arg, s, P, ...) TX_LST_DO_51(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 52, ##__VA_ARGS__) -#define TX_LST_DO_53(M, arg, s, P, ...) TX_LST_DO_52(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 53, ##__VA_ARGS__) -#define TX_LST_DO_54(M, arg, s, P, ...) TX_LST_DO_53(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 54, ##__VA_ARGS__) -#define TX_LST_DO_55(M, arg, s, P, ...) TX_LST_DO_54(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 55, ##__VA_ARGS__) -#define TX_LST_DO_56(M, arg, s, P, ...) TX_LST_DO_55(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 56, ##__VA_ARGS__) -#define TX_LST_DO_57(M, arg, s, P, ...) TX_LST_DO_56(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 57, ##__VA_ARGS__) -#define TX_LST_DO_58(M, arg, s, P, ...) TX_LST_DO_57(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 58, ##__VA_ARGS__) -#define TX_LST_DO_59(M, arg, s, P, ...) TX_LST_DO_58(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 59, ##__VA_ARGS__) -#define TX_LST_DO_60(M, arg, s, P, ...) TX_LST_DO_59(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 60, ##__VA_ARGS__) -#define TX_LST_DO_61(M, arg, s, P, ...) TX_LST_DO_60(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 61, ##__VA_ARGS__) -#define TX_LST_DO_62(M, arg, s, P, ...) TX_LST_DO_61(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 62, ##__VA_ARGS__) -#define TX_LST_DO_63(M, arg, s, P, ...) TX_LST_DO_62(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 63, ##__VA_ARGS__) -#define TX_LST_DO_64(M, arg, s, P, ...) TX_LST_DO_63(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 64, ##__VA_ARGS__) -#define TX_LST_DO_65(M, arg, s, P, ...) TX_LST_DO_64(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 65, ##__VA_ARGS__) -#define TX_LST_DO_66(M, arg, s, P, ...) TX_LST_DO_65(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 66, ##__VA_ARGS__) -#define TX_LST_DO_67(M, arg, s, P, ...) TX_LST_DO_66(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 67, ##__VA_ARGS__) -#define TX_LST_DO_68(M, arg, s, P, ...) TX_LST_DO_67(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 68, ##__VA_ARGS__) -#define TX_LST_DO_69(M, arg, s, P, ...) TX_LST_DO_68(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 69, ##__VA_ARGS__) -#define TX_LST_DO_70(M, arg, s, P, ...) TX_LST_DO_69(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 70, ##__VA_ARGS__) -#define TX_LST_DO_71(M, arg, s, P, ...) TX_LST_DO_70(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 71, ##__VA_ARGS__) -#define TX_LST_DO_72(M, arg, s, P, ...) TX_LST_DO_71(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 72, ##__VA_ARGS__) -#define TX_LST_DO_73(M, arg, s, P, ...) TX_LST_DO_72(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 73, ##__VA_ARGS__) -#define TX_LST_DO_74(M, arg, s, P, ...) TX_LST_DO_73(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 74, ##__VA_ARGS__) -#define TX_LST_DO_75(M, arg, s, P, ...) TX_LST_DO_74(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 75, ##__VA_ARGS__) -#define TX_LST_DO_76(M, arg, s, P, ...) TX_LST_DO_75(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 76, ##__VA_ARGS__) -#define TX_LST_DO_77(M, arg, s, P, ...) TX_LST_DO_76(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 77, ##__VA_ARGS__) -#define TX_LST_DO_78(M, arg, s, P, ...) TX_LST_DO_77(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 78, ##__VA_ARGS__) -#define TX_LST_DO_79(M, arg, s, P, ...) TX_LST_DO_78(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 79, ##__VA_ARGS__) -#define TX_LST_DO_80(M, arg, s, P, ...) TX_LST_DO_79(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 80, ##__VA_ARGS__) -#define TX_LST_DO_81(M, arg, s, P, ...) TX_LST_DO_80(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 81, ##__VA_ARGS__) -#define TX_LST_DO_82(M, arg, s, P, ...) TX_LST_DO_81(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 82, ##__VA_ARGS__) -#define TX_LST_DO_83(M, arg, s, P, ...) TX_LST_DO_82(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 83, ##__VA_ARGS__) -#define TX_LST_DO_84(M, arg, s, P, ...) TX_LST_DO_83(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 84, ##__VA_ARGS__) -#define TX_LST_DO_85(M, arg, s, P, ...) TX_LST_DO_84(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 85, ##__VA_ARGS__) -#define TX_LST_DO_86(M, arg, s, P, ...) TX_LST_DO_85(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 86, ##__VA_ARGS__) -#define TX_LST_DO_87(M, arg, s, P, ...) TX_LST_DO_86(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 87, ##__VA_ARGS__) -#define TX_LST_DO_88(M, arg, s, P, ...) TX_LST_DO_87(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 88, ##__VA_ARGS__) -#define TX_LST_DO_89(M, arg, s, P, ...) TX_LST_DO_88(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 89, ##__VA_ARGS__) -#define TX_LST_DO_90(M, arg, s, P, ...) TX_LST_DO_89(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 90, ##__VA_ARGS__) -#define TX_LST_DO_91(M, arg, s, P, ...) TX_LST_DO_90(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 91, ##__VA_ARGS__) -#define TX_LST_DO_92(M, arg, s, P, ...) TX_LST_DO_91(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 92, ##__VA_ARGS__) -#define TX_LST_DO_93(M, arg, s, P, ...) TX_LST_DO_92(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 93, ##__VA_ARGS__) -#define TX_LST_DO_94(M, arg, s, P, ...) TX_LST_DO_93(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 94, ##__VA_ARGS__) -#define TX_LST_DO_95(M, arg, s, P, ...) TX_LST_DO_94(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 95, ##__VA_ARGS__) -#define TX_LST_DO_96(M, arg, s, P, ...) TX_LST_DO_95(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 96, ##__VA_ARGS__) -#define TX_LST_DO_97(M, arg, s, P, ...) TX_LST_DO_96(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 97, ##__VA_ARGS__) -#define TX_LST_DO_98(M, arg, s, P, ...) TX_LST_DO_97(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 98, ##__VA_ARGS__) -#define TX_LST_DO_99(M, arg, s, P, ...) TX_LST_DO_98(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 99, ##__VA_ARGS__) -#define TX_LST_DO_100(M, arg, s, P, ...) LST_DO_99(M, arg, s, P, ##__VA_ARGS__)SELF s P(M, arg, 100, ##__VA_ARGS__) - -#define TX_LST_DO__(N, M, arg, s, P, ...) TX_LST_DO_##N(M, arg, s, P, ##__VA_ARGS__) -#define TX_LST_DO_(...) TX_LST_DO__(__VA_ARGS__) -#define TX_LST_DO(M, arg, s, ...) \ - TX_LST_DO_(ARGS_NUM(__VA_ARGS__), M, arg, s, TX_PROC_ONE, ##__VA_ARGS__) - -#define TX_PROC_ONE(M, arg, IDX, ...) M(SELECT(IDX, __VA_ARGS__), IDX, arg) - -#define TX_LST_DO_CODE(M, arg, ...) TX_LST_DO(M, arg, (;), ##__VA_ARGS__) - -#define OB_TX_UNIS_ENCODE(obj, IDX, arg) \ - if (OB_SUCC(ret) && TX_IS_NEED_SER_OR_DE(arg, IDX)) { \ - if (OB_FAIL(NS_::encode(buf, buf_len, pos, obj))) { \ - RPC_WARN("encode object fail", "name", MSTR(obj), K(buf_len), K(pos), K(ret)); \ - } \ - } - -#define OB_TX_UNIS_DECODE(obj, IDX, arg) \ - if (OB_SUCC(ret) && TX_IS_NEED_SER_OR_DE(arg, IDX) && pos < data_len) { \ - if (OB_FAIL(NS_::decode(buf, data_len, pos, obj))) { \ - RPC_WARN("decode object fail", "name", MSTR(obj), K(data_len), K(pos), K(ret)); \ - } \ - } - -#define OB_TX_UNIS_ADD_LEN(obj, IDX, arg) \ - if (TX_IS_NEED_SER_OR_DE(arg, IDX)) { \ - len += NS_::encoded_length(obj); \ - } - -#define OB_TX_SERIALIZE_MEMBER(CLS, COMPAT_ARG, ...) \ - OB_DEF_SERIALIZE(CLS) \ - { \ - int ret = OK_; \ - UNF_UNUSED_SER; \ - TX_SER_COMPAT_BYTES(COMPAT_ARG); \ - TX_LST_DO_CODE(OB_TX_UNIS_ENCODE, COMPAT_ARG, ##__VA_ARGS__); \ - return ret; \ - } \ - OB_DEF_DESERIALIZE(CLS) \ - { \ - int ret = OK_; \ - UNF_UNUSED_DES; \ - TX_DSER_COMPAT_BYTES(COMPAT_ARG); \ - TX_LST_DO_CODE(OB_TX_UNIS_DECODE, COMPAT_ARG, ##__VA_ARGS__); \ - return ret; \ - } \ - OB_DEF_SERIALIZE_SIZE(CLS) \ - { \ - int64_t len = 0; \ - TX_SER_SIZE_COMPAT_BYTES(COMPAT_ARG); \ - TX_LST_DO_CODE(OB_TX_UNIS_ADD_LEN, COMPAT_ARG, ##__VA_ARGS__); \ - return len; \ - } - - - -class ObTxPrintTimeGuard -{ -public: - ObTxPrintTimeGuard() - { - start_ts_ = ObTimeUtility::fast_current_time(); - end_ts_ = 0; - memset(click_str_, 0, sizeof(const char *) * MAX_CLICK_COUNT); - memset(click_start_ts_, 0, sizeof(int64_t) * MAX_CLICK_COUNT); - memset(click_end_ts_, 0, sizeof(int64_t) * MAX_CLICK_COUNT); - } - - void click_start(const char * str, const int64_t click_index) - { - click_str_[click_index] = str; - click_start_ts_[click_index] = ObTimeUtility::fast_current_time(); - } - - void click_end(const int64_t click_index) - { - click_end_ts_[click_index] = ObTimeUtility::fast_current_time(); - } - - int64_t get_diff() - { - end_ts_ = ObTimeUtility::fast_current_time(); - return end_ts_ - start_ts_; - } - - ~ObTxPrintTimeGuard() - { - end_ts_ = ObTimeUtility::fast_current_time(); - } - - int64_t to_string(char *buf, const int64_t buf_len) const; - -private: - static const int64_t MAX_CLICK_COUNT = 16; -private: - const char * click_str_[MAX_CLICK_COUNT]; - int start_ts_; - int end_ts_; - int64_t click_start_ts_[MAX_CLICK_COUNT]; - int64_t click_end_ts_[MAX_CLICK_COUNT]; -}; - -} // namespace transaction -} // namespace oceanbase - -#endif - -// serialization diff --git a/src/storage/tx/ob_tx_standby_cleanup.h b/src/storage/tx/ob_tx_standby_cleanup.h deleted file mode 100644 index 4f1699ebd..000000000 --- a/src/storage/tx/ob_tx_standby_cleanup.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_TX_STANDBY_CLEANUP_ -#define OCEANBASE_TRANSACTION_OB_TX_STANDBY_CLEANUP_ - -#include "ob_trans_define.h" - -namespace oceanbase -{ - -namespace transaction -{ -class ObTxStandbyCleanupTask : public ObTransTask -{ -public: - ObTxStandbyCleanupTask() : ObTransTask(ObTransRetryTaskType::STANDBY_CLEANUP_TASK) - {} - ~ObTxStandbyCleanupTask() { destroy(); } -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_TRANSACTION_OB_TX_STANDBY_CLEANUP_ diff --git a/src/storage/tx/ob_tx_stat.cpp b/src/storage/tx/ob_tx_stat.cpp index 62f262b32..4ec34da8e 100644 --- a/src/storage/tx/ob_tx_stat.cpp +++ b/src/storage/tx/ob_tx_stat.cpp @@ -30,25 +30,20 @@ void ObTxStat::reset() addr_.reset(); tx_id_.reset(); has_decided_ = false; - ls_id_.reset(); - participants_.reset(); + has_write_state_ = false; tx_ctx_create_time_ = -1; tx_expired_time_ = -1; ref_cnt_ = -1; last_op_sn_ = 0; pending_write_ = 0; state_ = static_cast(ObTxState::UNKNOWN); - tx_type_ = TransType::UNKNOWN_TRANS; part_tx_action_ = ObPartTransAction::UNKNOWN; tx_ctx_addr_ = (void*)0; pending_log_size_ = 0; flushed_log_size_ = 0; - role_state_ = -1; // RoleState::INVALID session_id_ = 0; - scheduler_addr_.reset(); is_exiting_ = false; xid_.reset(); - coord_.reset(); last_request_ts_ = OB_INVALID_TIMESTAMP; busy_cbs_cnt_ = 0; replay_completeness_ = -1; @@ -56,18 +51,16 @@ void ObTxStat::reset() callback_list_stats_.reset(); } int ObTxStat::init(const common::ObAddr &addr, const ObTransID &tx_id, const bool has_decided, - const share::ObLSID &ls_id, const share::ObLSArray &participants, + const bool has_write_state, const int64_t tx_ctx_create_time, const int64_t tx_expired_time, const int64_t ref_cnt, const int64_t last_op_sn, const int64_t pending_write, const int64_t state, - const int tx_type, const int64_t part_tx_action, + const int64_t part_tx_action, const void* const tx_ctx_addr, const int64_t pending_log_size, const int64_t flushed_log_size, - const int64_t role_state, const int64_t session_id, const uint32_t client_sid, - const common::ObAddr &scheduler, const bool is_exiting, const ObXATransID &xid, - const share::ObLSID &coord, const int64_t last_request_ts, + const int64_t last_request_ts, SCN start_scn, SCN end_scn, SCN rec_scn, const int busy_cbs_cnt, int replay_completeness, @@ -77,36 +70,26 @@ int ObTxStat::init(const common::ObAddr &addr, const ObTransID &tx_id, const bo if (is_inited_) { TRANS_LOG(WARN, "ObTxStat init twice"); ret = OB_INIT_TWICE; - } else if (OB_SUCCESS != (ret = participants_.assign(participants))) { - TRANS_LOG(WARN, "ls array assign error", KR(ret), K(participants)); } else { is_inited_ = true; addr_ = addr; tx_id_ = tx_id; has_decided_ = has_decided; - ls_id_ = ls_id; + has_write_state_ = has_write_state; tx_ctx_create_time_ = tx_ctx_create_time; tx_expired_time_ = tx_expired_time; ref_cnt_ = ref_cnt; last_op_sn_ = last_op_sn; pending_write_ = pending_write; state_ = state; - tx_type_ = tx_type; part_tx_action_ = part_tx_action; tx_ctx_addr_ = tx_ctx_addr; pending_log_size_ = pending_log_size; flushed_log_size_ = flushed_log_size; - role_state_ = role_state; session_id_ = session_id; client_sid_ = client_sid; - scheduler_addr_ = scheduler; is_exiting_ = is_exiting; xid_ = xid; - if (part_tx_action == ObPartTransAction::COMMIT && !coord.is_valid()) { - coord_ = ls_id; - } else { - coord_ = coord; - } last_request_ts_ = last_request_ts; start_scn_ = start_scn; end_scn_ = end_scn; @@ -119,7 +102,6 @@ int ObTxStat::init(const common::ObAddr &addr, const ObTransID &tx_id, const bo } int ObTxLockStat::init(const common::ObAddr &addr, - const share::ObLSID &ls_id, const ObMemtableKeyInfo &memtable_key_info, uint32_t session_id, uint32_t client_sid, @@ -136,7 +118,6 @@ int ObTxLockStat::init(const common::ObAddr &addr, } else { is_inited_ = true; addr_ = addr; - ls_id_ = ls_id; memtable_key_info_ = memtable_key_info; session_id_ = session_id; proxy_session_id_ = proxy_session_id; @@ -153,7 +134,6 @@ void ObTxLockStat::reset() { is_inited_ = false; addr_.reset(); - ls_id_.reset(); memtable_key_info_.reset(); session_id_ = 0; proxy_session_id_ = 0; @@ -169,8 +149,8 @@ int ObTxSchedulerStat::init(const common::ObAddr &addr, const int64_t state, const int64_t cluster_id, const ObXATransID &xid, - const share::ObLSID &coord_id, - const ObTxPartList &parts, + const bool has_write_state, + const ObTxWriteState &write_state, const ObTxIsolationLevel &isolation, const share::SCN &snapshot_version, const ObTxAccessMode &access_mode, @@ -189,8 +169,6 @@ int ObTxSchedulerStat::init(const common::ObAddr &addr, if (is_inited_) { TRANS_LOG(WARN, "ObTxSchedulerStat init twice"); ret = OB_INIT_TWICE; - } else if (OB_FAIL(parts_.assign(parts))) { - TRANS_LOG(WARN, "parts assign error", KR(ret), K(parts)); } else if (OB_FAIL(get_valid_savepoints(savepoints))) { TRANS_LOG(WARN, "savepoints assign error", KR(ret), K(savepoints)); } else { @@ -202,7 +180,10 @@ int ObTxSchedulerStat::init(const common::ObAddr &addr, state_ = state; cluster_id_ = cluster_id; xid_ = xid; - coord_id_ = coord_id; + has_write_state_ = has_write_state; + if (has_write_state_) { + write_state_ = write_state; + } isolation_ = isolation; snapshot_version_ = snapshot_version; access_mode_ = access_mode; @@ -228,8 +209,8 @@ void ObTxSchedulerStat::reset() state_ = 0; cluster_id_ = -1; xid_.reset(); - coord_id_.reset(); - parts_.reset(); + has_write_state_ = false; + write_state_ = ObTxWriteState(); isolation_ = ObTxIsolationLevel::INVALID; snapshot_version_.reset(); access_mode_ = ObTxAccessMode::INVL; @@ -248,20 +229,9 @@ void ObTxSchedulerStat::reset() int64_t ObTxSchedulerStat::get_parts_str(char* buf, const int64_t buf_len) { int64_t pos = 0; - J_ARRAY_START(); - for (int i = 0; i < parts_.count(); i++) { - J_OBJ_START(); - J_KV("id", parts_.at(i).id_.id()); - J_COMMA(); - J_KV("addr", parts_.at(i).addr_); - J_COMMA(); - J_KV("epoch", parts_.at(i).epoch_); - J_OBJ_END(); - if (i < parts_.count() - 1) { - J_COMMA(); - } - } - J_ARRAY_END(); + J_OBJ_START(); + J_KV("write_state", write_state_); + J_OBJ_END(); return pos; } diff --git a/src/storage/tx/ob_tx_stat.h b/src/storage/tx/ob_tx_stat.h index bab4e8dd8..eaf83d2be 100644 --- a/src/storage/tx/ob_tx_stat.h +++ b/src/storage/tx/ob_tx_stat.h @@ -59,32 +59,29 @@ struct ObTxStat ~ObTxStat() { } void reset(); int init(const common::ObAddr &addr, const ObTransID &tx_id, const bool has_decided, - const share::ObLSID &ls_id, const share::ObLSArray &participants, + const bool has_write_state, const int64_t tx_ctx_create_time, const int64_t tx_expired_time, const int64_t ref_cnt, const int64_t last_op_sn, const int64_t pending_write, const int64_t state, - const int tx_type, const int64_t part_tx_action, + const int64_t part_tx_action, const void* const tx_ctx_addr, const int64_t pending_log_size, const int64_t flushed_log_size, - const int64_t role_state, const int64_t session_id, const uint32_t client_sid, - const common::ObAddr &scheduler, const bool is_exiting, const ObXATransID &xid, - const share::ObLSID &coord, const int64_t last_request_ts, + const int64_t last_request_ts, share::SCN start_scn, share::SCN end_scn, share::SCN rec_scn, const int busy_cbs_cnt, int replay_completeness, share::SCN serial_final_scn); TO_STRING_KV(K_(addr), K_(tx_id), - K_(has_decided), K_(ls_id), K_(participants), + K_(has_decided), K_(has_write_state), K_(tx_ctx_create_time), K_(tx_expired_time), K_(ref_cnt), - K_(last_op_sn), K_(pending_write), K_(state), K_(tx_type), + K_(last_op_sn), K_(pending_write), K_(state), KP_(tx_ctx_addr), K_(pending_log_size), K_(flushed_log_size), - K_(role_state), K_(session_id), K_(client_sid), - K_(scheduler_addr), K_(is_exiting), - K_(xid), K_(coord), K_(last_request_ts), - K_(xid), K_(coord), K_(last_request_ts), + K_(session_id), K_(client_sid), + K_(is_exiting), + K_(xid), K_(last_request_ts), K_(start_scn), K_(end_scn), K_(rec_scn), K_(busy_cbs_cnt), K_(serial_final_scn), @@ -95,26 +92,21 @@ struct ObTxStat common::ObAddr addr_; ObTransID tx_id_; bool has_decided_; - share::ObLSID ls_id_; - share::ObLSArray participants_; + bool has_write_state_; int64_t tx_ctx_create_time_; int64_t tx_expired_time_; int64_t ref_cnt_; int64_t last_op_sn_; int64_t pending_write_; int64_t state_; - int tx_type_; int64_t part_tx_action_; const void *tx_ctx_addr_; int64_t pending_log_size_; int64_t flushed_log_size_; - int64_t role_state_; int64_t session_id_; uint32_t client_sid_; - common::ObAddr scheduler_addr_; bool is_exiting_; ObXATransID xid_; - share::ObLSID coord_; int64_t last_request_ts_; share::SCN start_scn_; share::SCN end_scn_; @@ -155,7 +147,6 @@ class ObTxLockStat ObTxLockStat() { reset(); } ~ObTxLockStat() {} int init(const common::ObAddr &addr, - const share::ObLSID &ls_id, const ObMemtableKeyInfo &memtable_key_info, uint32_t session_id, uint32_t client_sid, @@ -166,7 +157,6 @@ class ObTxLockStat void reset(); const common::ObAddr &get_addr() const { return addr_; } - const share::ObLSID &get_ls_id() const { return ls_id_; } const ObMemtableKeyInfo &get_memtable_key_info() const { return memtable_key_info_; } uint32_t get_session_id() const { return session_id_; } uint64_t get_proxy_session_id() const { return proxy_session_id_; } @@ -176,7 +166,6 @@ class ObTxLockStat int64_t get_tx_expired_time() const { return tx_expired_time_; } TO_STRING_KV(K_(addr), - K_(ls_id), K_(memtable_key_info), K_(session_id), K_(proxy_session_id), @@ -187,7 +176,6 @@ class ObTxLockStat private: bool is_inited_; common::ObAddr addr_; - share::ObLSID ls_id_; ObMemtableKeyInfo memtable_key_info_; uint32_t session_id_; uint64_t proxy_session_id_; @@ -203,15 +191,15 @@ class ObTxSchedulerStat ObTxSchedulerStat() { reset(); } ~ObTxSchedulerStat() { } void reset(); - int init(const common::ObAddr &addr, - const uint32_t sess_id, - const uint32_t client_sid, - const ObTransID &tx_id, - const int64_t state, - const int64_t cluster_id, - const ObXATransID &xid, - const share::ObLSID &coord_id, - const ObTxPartList &parts, + int init(const common::ObAddr &addr, + const uint32_t sess_id, + const uint32_t client_sid, + const ObTransID &tx_id, + const int64_t state, + const int64_t cluster_id, + const ObXATransID &xid, + const bool has_write_state, + const ObTxWriteState &write_state, const ObTxIsolationLevel &isolation, const share::SCN &snapshot_version, const ObTxAccessMode &access_mode, @@ -225,9 +213,9 @@ class ObTxSchedulerStat const ObTxSavePointList &savepoints, const int16_t abort_cause, const bool can_elr); - TO_STRING_KV(K_(addr), K_(sess_id), K_(client_sid), - K_(tx_id), K_(state), K_(cluster_id), - K_(xid), K_(coord_id), K_(parts), + TO_STRING_KV(K_(addr), K_(sess_id), K_(client_sid), + K_(tx_id), K_(state), K_(cluster_id), + K_(xid), K_(has_write_state), K_(write_state), K_(isolation), K_(snapshot_version), K_(access_mode), K_(op_sn), K_(flag), K_(active_ts), K_(expire_ts), @@ -244,10 +232,10 @@ class ObTxSchedulerStat uint32_t client_sid_; ObTransID tx_id_; int64_t state_; - int64_t cluster_id_; - ObXATransID xid_; - share::ObLSID coord_id_; - ObTxPartList parts_; + int64_t cluster_id_; + ObXATransID xid_; + bool has_write_state_; + ObTxWriteState write_state_; ObTxIsolationLevel isolation_; share::SCN snapshot_version_; ObTxAccessMode access_mode_; diff --git a/src/storage/tx/ob_tx_timestamp_waiter.cpp b/src/storage/tx/ob_tx_timestamp_waiter.cpp new file mode 100644 index 000000000..25e983e62 --- /dev/null +++ b/src/storage/tx/ob_tx_timestamp_waiter.cpp @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ob_tx_timestamp_waiter.h" +#include "lib/ob_running_mode.h" +#include "share/rc/ob_tenant_base.h" +#include "ob_ts_mgr.h" +#include "ob_tx_ctx.h" + +namespace oceanbase +{ +using namespace common; +using namespace share; + +namespace transaction +{ + +int ObTxTimestampCallbackWorker::init(ObTxTimestampWaiter *waiter) +{ + int ret = OB_SUCCESS; + const int64_t thread_count = lib::is_mini_mode() ? 1 : MAX(common::get_cpu_count() / 12, 1L); + + waiter_ = waiter; + set_run_wrapper(MTL_CTX()); + if (OB_FAIL(common::ObSimpleThreadPool::init(thread_count, MAX_TASK_NUM, "TxTsCb"))) { + TRANS_LOG(WARN, "timestamp callback worker init failed", KR(ret), K(thread_count)); + waiter_ = NULL; + } + return ret; +} + +void ObTxTimestampCallbackWorker::handle(void *task) +{ + waiter_->handle_callback_(static_cast(task)); +} + +ObTxTimestampWaiter::ObTxTimestampWaiter() + : share::ObThreadPool(1), + is_inited_(false), + running_(false), + latest_gts_(0), + ts_mgr_(NULL), + wait_queue_(), + cond_(), + callback_worker_() +{ +} + +int ObTxTimestampWaiter::init(ObTsMgr *ts_mgr) +{ + int ret = OB_SUCCESS; + + if (is_inited_) { + ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "timestamp waiter init twice", KR(ret)); + } else if (OB_ISNULL(ts_mgr)) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid timestamp manager", KR(ret)); + } else if (OB_FAIL(cond_.init(ObWaitEventIds::DEFAULT_COND_WAIT))) { + TRANS_LOG(WARN, "timestamp waiter condition init failed", KR(ret)); + } else if (OB_FAIL(share::ObThreadPool::init())) { + TRANS_LOG(WARN, "timestamp waiter thread init failed", KR(ret)); + } else if (FALSE_IT(share::ObThreadPool::set_run_wrapper(MTL_CTX()))) { + } else if (OB_FAIL(callback_worker_.init(this))) { + TRANS_LOG(WARN, "timestamp callback worker init failed", KR(ret)); + } else { + ts_mgr_ = ts_mgr; + is_inited_ = true; + TRANS_LOG(INFO, "timestamp waiter init success"); + } + + if (OB_FAIL(ret)) { + callback_worker_.destroy(); + share::ObThreadPool::destroy(); + cond_.destroy(); + } + return ret; +} + +int ObTxTimestampWaiter::start() +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "timestamp waiter not init", KR(ret)); + } else { + ATOMIC_STORE(&running_, true); + if (OB_FAIL(share::ObThreadPool::start())) { + ATOMIC_STORE(&running_, false); + TRANS_LOG(WARN, "timestamp waiter start failed", KR(ret)); + } + } + return ret; +} + +void ObTxTimestampWaiter::stop() +{ + if (is_inited_ && is_running_()) { + { + ObThreadCondGuard guard(cond_); + ATOMIC_STORE(&running_, false); + cond_.broadcast(); + } + share::ObThreadPool::stop(); + callback_worker_.stop(); + share::ObThreadPool::wait(); + callback_worker_.wait(); + drain_wait_queue_(); + TRANS_LOG(INFO, "timestamp waiter stopped"); + } +} + +void ObTxTimestampWaiter::destroy() +{ + if (is_inited_) { + stop(); + drain_wait_queue_(); + callback_worker_.destroy(); + share::ObThreadPool::destroy(); + cond_.destroy(); + ts_mgr_ = NULL; + latest_gts_ = 0; + is_inited_ = false; + } +} + +int ObTxTimestampWaiter::wait_gts_elapse(const SCN &target_scn, + ObTxCtx *ctx, + bool &need_wait) +{ + int ret = OB_SUCCESS; + SCN current_gts; + bool gts_ready = true; + need_wait = false; + + if (OB_FAIL(ts_mgr_->get_gts(current_gts))) { + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; + gts_ready = false; + } else { + TRANS_LOG(WARN, "get gts for timestamp wait failed", KR(ret), K(target_scn)); + } + } + + if (OB_SUCC(ret) && (!gts_ready || current_gts < target_scn)) { + ObThreadCondGuard guard(cond_); + if (!is_running_()) { + ret = OB_NOT_RUNNING; + TRANS_LOG(WARN, "timestamp waiter is not running", KR(ret)); + } else if (OB_FAIL(wait_queue_.push(static_cast(ctx)))) { + TRANS_LOG(ERROR, "push timestamp wait task failed", KR(ret), KP(ctx)); + } else { + need_wait = true; + cond_.signal(); + } + } + return ret; +} + +void ObTxTimestampWaiter::run1() +{ + lib::set_thread_name("TxTsWaiter"); + + while (is_running_()) { + if (wait_queue_.size() == 0) { + ObThreadCondGuard guard(cond_); + if (is_running_() && wait_queue_.size() == 0) { + cond_.wait(); + } + } else { + int ret = OB_SUCCESS; + SCN gts; + if (OB_FAIL(ts_mgr_->get_gts(gts))) { + if (OB_EAGAIN != ret) { + TRANS_LOG(WARN, "get gts in timestamp waiter failed", KR(ret)); + } + } else { + ATOMIC_STORE(&latest_gts_, gts.get_val_for_gts()); + dispatch_ready_tasks_(gts); + } + + ObThreadCondGuard guard(cond_); + if (is_running_() && wait_queue_.size() > 0) { + cond_.wait_us(POLL_INTERVAL_US); + } + } + } +} + +void ObTxTimestampWaiter::dispatch_ready_tasks_(const SCN >s) +{ + int ret = OB_SUCCESS; + int64_t task_count = wait_queue_.size(); + + while (task_count-- > 0 && is_running_()) { + ObLink *link = NULL; + if (OB_FAIL(wait_queue_.pop(link))) { + break; + } + ObTxCtx *ctx = static_cast(link); + if (ctx->get_commit_version() > gts) { + wait_queue_.push(link); + break; + } else if (OB_FAIL(callback_worker_.push(ctx))) { + if (is_running_()) { + wait_queue_.push(link); + } else { + ctx->gts_callback_interrupted(); + } + if (OB_EAGAIN != ret && OB_IN_STOP_STATE != ret) { + TRANS_LOG(WARN, "push timestamp callback failed", KR(ret), KP(ctx)); + } + break; + } + } +} + +void ObTxTimestampWaiter::handle_callback_(ObTxCtx *ctx) +{ + if (!is_running_()) { + ctx->gts_callback_interrupted(); + } else { + int ret = OB_SUCCESS; + SCN gts; + (void)gts.convert_for_gts(ATOMIC_LOAD(&latest_gts_)); + if (OB_FAIL(ctx->gts_elapse_callback(gts))) { + if (OB_EAGAIN == ret) { + requeue_or_interrupt_(ctx); + } else { + TRANS_LOG(WARN, "timestamp elapse callback failed", KR(ret), KP(ctx), K(gts)); + } + } + } +} + +void ObTxTimestampWaiter::requeue_or_interrupt_(ObTxCtx *ctx) +{ + bool requeued = false; + { + ObThreadCondGuard guard(cond_); + if (is_running_()) { + wait_queue_.push(static_cast(ctx)); + cond_.signal(); + requeued = true; + } + } + if (!requeued) { + ctx->gts_callback_interrupted(); + } +} + +void ObTxTimestampWaiter::drain_wait_queue_() +{ + ObLink *link = NULL; + while (OB_SUCCESS == wait_queue_.pop(link)) { + ObTxCtx *ctx = static_cast(link); + ctx->gts_callback_interrupted(); + } +} + +} // namespace transaction +} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_timestamp_waiter.h b/src/storage/tx/ob_tx_timestamp_waiter.h new file mode 100644 index 000000000..f4f7a953d --- /dev/null +++ b/src/storage/tx/ob_tx_timestamp_waiter.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_TRANSACTION_OB_TX_TIMESTAMP_WAITER_ +#define OCEANBASE_TRANSACTION_OB_TX_TIMESTAMP_WAITER_ + +#include "lib/lock/ob_thread_cond.h" +#include "lib/queue/ob_link_queue.h" +#include "lib/thread/ob_simple_thread_pool.h" +#include "share/ob_thread_pool.h" + +namespace oceanbase +{ +namespace share +{ +class SCN; +} +namespace transaction +{ + +class ObTsMgr; +class ObTxCtx; + +class ObTxTimestampWaiter; + +class ObTxTimestampCallbackWorker final : public common::ObSimpleThreadPool +{ +public: + ObTxTimestampCallbackWorker() : waiter_(NULL) {} + int init(ObTxTimestampWaiter *waiter); +protected: + void handle(void *task) override; +private: + static const int64_t MAX_TASK_NUM = 10240; + ObTxTimestampWaiter *waiter_; +}; + +class ObTxTimestampWaiter final : public share::ObThreadPool +{ + friend class ObTxTimestampCallbackWorker; +public: + ObTxTimestampWaiter(); + ~ObTxTimestampWaiter() { destroy(); } + + int init(ObTsMgr *ts_mgr); + int start() override; + void stop() override; + void destroy(); + int wait_gts_elapse(const share::SCN &target_scn, ObTxCtx *ctx, bool &need_wait); + +private: + void run1() override; + void dispatch_ready_tasks_(const share::SCN >s); + void handle_callback_(ObTxCtx *ctx); + void requeue_or_interrupt_(ObTxCtx *ctx); + void drain_wait_queue_(); + bool is_running_() const { return ATOMIC_LOAD(&running_); } + +private: + static const int64_t POLL_INTERVAL_US = 500; + bool is_inited_; + bool running_; + uint64_t latest_gts_; + ObTsMgr *ts_mgr_; + common::ObLinkQueue wait_queue_; + common::ObThreadCond cond_; + ObTxTimestampCallbackWorker callback_worker_; +}; + +} // namespace transaction +} // namespace oceanbase + +#endif // OCEANBASE_TRANSACTION_OB_TX_TIMESTAMP_WAITER_ diff --git a/src/storage/tx/ob_weak_read_util.cpp b/src/storage/tx/ob_weak_read_util.cpp index da2096b55..4bfc29657 100644 --- a/src/storage/tx/ob_weak_read_util.cpp +++ b/src/storage/tx/ob_weak_read_util.cpp @@ -63,12 +63,14 @@ int ObWeakReadUtil::generate_min_weak_read_version(SCN &scn) max_stale_time = GCONF.max_stale_time_for_weak_consistency - buffer_time; } else { //standby, restore, invalid - max_stale_time = GCONF.max_stale_time_for_weak_consistency + transaction::ObTimestampService::PREALLOCATE_RANGE_FOR_SWITHOVER - buffer_time; + max_stale_time = GCONF.max_stale_time_for_weak_consistency + + transaction::ObTimestampService::TIMESTAMP_RECOVERY_SAFETY_RANGE + - buffer_time; } max_stale_time = std::max(max_stale_time, static_cast(DEFAULT_REPLICA_KEEPALIVE_INTERVAL)); SCN tmp_scn; - if (OB_FAIL(OB_TS_MGR.get_gts(NULL, tmp_scn))) { + if (OB_FAIL(OB_TS_MGR.get_gts(tmp_scn))) { TRANS_LOG(WARN, "get gts cache error", K(ret)); } else { // the unit of max_stale_time is us, we should change to ns diff --git a/src/storage/tx/ob_xa_define.cpp b/src/storage/tx/ob_xa_define.cpp index b917074ea..99dd06629 100644 --- a/src/storage/tx/ob_xa_define.cpp +++ b/src/storage/tx/ob_xa_define.cpp @@ -226,8 +226,8 @@ void ObXAStatistics::reset() ATOMIC_STORE(&total_failure_xa_prepare_, 0); ATOMIC_STORE(&total_success_xa_1pc_commit_, 0); ATOMIC_STORE(&total_failure_xa_1pc_commit_, 0); - ATOMIC_STORE(&total_success_xa_2pc_commit_, 0); - ATOMIC_STORE(&total_failure_xa_2pc_commit_, 0); + ATOMIC_STORE(&total_success_xa_prepared_commit_, 0); + ATOMIC_STORE(&total_failure_xa_prepared_commit_, 0); ATOMIC_STORE(&total_xa_rollback_, 0); } diff --git a/src/storage/tx/ob_xa_define.h b/src/storage/tx/ob_xa_define.h index 28f2a9f0c..f1b7918e7 100644 --- a/src/storage/tx/ob_xa_define.h +++ b/src/storage/tx/ob_xa_define.h @@ -271,14 +271,14 @@ class ObXAStatistics void inc_failure_xa_prepare() { ATOMIC_INC(&total_failure_xa_prepare_); } void inc_success_xa_1pc_commit() { ATOMIC_INC(&total_success_xa_1pc_commit_); } void inc_failure_xa_1pc_commit() { ATOMIC_INC(&total_failure_xa_1pc_commit_); } - void inc_success_xa_2pc_commit() { ATOMIC_INC(&total_success_xa_2pc_commit_); } - void inc_failure_xa_2pc_commit() { ATOMIC_INC(&total_failure_xa_2pc_commit_); } + void inc_success_xa_prepared_commit() { ATOMIC_INC(&total_success_xa_prepared_commit_); } + void inc_failure_xa_prepared_commit() { ATOMIC_INC(&total_failure_xa_prepared_commit_); } void inc_xa_rollback() { ATOMIC_INC(&total_xa_rollback_); } TO_STRING_KV(K_(total_active_xa_ctx_count), K_(total_standby_clearup_count), K_(total_success_xa_start), K_(total_failure_xa_start), K_(total_success_xa_prepare), K_(total_failure_xa_prepare), K_(total_success_xa_1pc_commit), K_(total_failure_xa_1pc_commit), - K_(total_success_xa_2pc_commit), K_(total_failure_xa_2pc_commit), + K_(total_success_xa_prepared_commit), K_(total_failure_xa_prepared_commit), K_(total_xa_rollback)); private: @@ -293,8 +293,8 @@ class ObXAStatistics int64_t total_failure_xa_prepare_; int64_t total_success_xa_1pc_commit_; int64_t total_failure_xa_1pc_commit_; - int64_t total_success_xa_2pc_commit_; - int64_t total_failure_xa_2pc_commit_; + int64_t total_success_xa_prepared_commit_; + int64_t total_failure_xa_prepared_commit_; int64_t total_xa_rollback_; }; diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index ffc4b330d..45d849838 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -39,35 +39,30 @@ void ObStoreCtxGuard::reset() int ret = OB_SUCCESS; static const int64_t WARN_TIME_US = 5 * 1000 * 1000; if (IS_INIT) { - if (OB_NOT_NULL(handle_.get_ls())) { - if (ctx_.is_valid() && OB_FAIL(handle_.get_ls()->revert_store_ctx(ctx_))) { - LOG_WARN("revert transaction context fail", K(ret), K_(ls_id)); + if (OB_NOT_NULL(ls_)) { + if (ctx_.is_valid() && OB_FAIL(ls_->revert_store_ctx(ctx_))) { + LOG_WARN("revert transaction context fail", K(ret)); } - handle_.reset(); + ls_ = nullptr; } const int64_t guard_used_us = ObClockGenerator::getClock() - init_ts_; if (guard_used_us >= WARN_TIME_US) { - LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "guard used too much time", K(guard_used_us), K_(ls_id), K(lbt())); + LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "guard used too much time", K(guard_used_us), K(lbt())); } ctx_.reset(); - ls_id_.reset(); is_inited_ = false; } } -int ObStoreCtxGuard::init(const share::ObLSID &ls_id) +int ObStoreCtxGuard::init(ObLS *ls) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument(s)", K(ret), K(ls_id)); } else { - ls_id_ = ls_id; ctx_.reset(); - ctx_.ls_id_ = ls_id; + ls_ = ls; is_inited_ = true; init_ts_ = ObClockGenerator::getClock(); } @@ -130,9 +125,7 @@ int ObAccessService::check_tenant_out_of_memstore_limit_(bool &is_out_of_mem) return ret; } -int ObAccessService::check_data_disk_full_( - const share::ObLSID &ls_id, - bool &is_full) +int ObAccessService::check_data_disk_full_(bool &is_full) { int ret = OB_SUCCESS; is_full = false; // lite: sys tenant -> this disk-full check path is user-tenant-only @@ -140,15 +133,11 @@ int ObAccessService::check_data_disk_full_( } int ObAccessService::pre_check_lock( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m) { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; - int64_t user_specified_snapshot = - transaction::ObTransVersion::INVALID_TRANS_VERSION; transaction::ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; @@ -156,37 +145,30 @@ int ObAccessService::pre_check_lock( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tx_desc.is_valid()) + } else if (OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tx_desc), K(param)); - } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, - param.expired_time_, /*timeout*/ + LOG_WARN("invalid argument", K(ret), K(tx_desc), K(param)); + } else if (OB_FAIL(get_write_store_ctx_guard_(param.expired_time_, /*timeout*/ tx_desc, snapshot, 0,/*branch_id*/ write_flag, ctx_guard))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); + LOG_WARN("fail to check query allowed", K(ret)); } else { - ret = ls->check_lock_conflict(ctx_guard.get_store_ctx(), param); + ret = ctx_guard.get_ls()->check_lock_conflict(ctx_guard.get_store_ctx(), param); } return ret; } ERRSIM_POINT_DEF(EN_OB_NOT_MASTER_IN_TABLELOCK) int ObAccessService::lock_obj( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m) { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; transaction::ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; @@ -197,37 +179,29 @@ int ObAccessService::lock_obj( } else if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tx_desc.is_valid()) + } else if (OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tx_desc), K(param)); - } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, - param.expired_time_, /*timeout*/ + LOG_WARN("invalid argument", K(ret), K(tx_desc), K(param)); + } else if (OB_FAIL(get_write_store_ctx_guard_(param.expired_time_, /*timeout*/ tx_desc, snapshot, 0, /*branch_id*/ write_flag, ctx_guard))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); + LOG_WARN("fail to check query allowed", K(ret)); } else { - ret = ls->lock(ctx_guard.get_store_ctx(), param); + ret = ctx_guard.get_ls()->lock(ctx_guard.get_store_ctx(), param); } return ret; } int ObAccessService::unlock_obj( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m) { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; - int64_t user_specified_snapshot = transaction::ObTransVersion::INVALID_TRANS_VERSION; transaction::ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; @@ -236,37 +210,29 @@ int ObAccessService::unlock_obj( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tx_desc.is_valid()) + } else if (OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tx_desc), K(param)); - } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, - param.expired_time_, /*timeout*/ + LOG_WARN("invalid argument", K(ret), K(tx_desc), K(param)); + } else if (OB_FAIL(get_write_store_ctx_guard_(param.expired_time_, /*timeout*/ tx_desc, snapshot, 0,/*branch_id*/ write_flag, ctx_guard))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); + LOG_WARN("fail to check query allowed", K(ret)); } else { - ret = ls->unlock(ctx_guard.get_store_ctx(), param); + ret = ctx_guard.get_ls()->unlock(ctx_guard.get_store_ctx(), param); } return ret; } int ObAccessService::replace_obj_lock( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObReplaceLockParam &lock_param) { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; - int64_t user_specified_snapshot = transaction::ObTransVersion::INVALID_TRANS_VERSION; transaction::ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; @@ -275,42 +241,33 @@ int ObAccessService::replace_obj_lock( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tx_desc.is_valid()) + } else if (OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!lock_param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), - K(ls_id), K(tx_desc), K(lock_param), - K(ls_id.is_valid()), K(tx_desc.is_valid()), K(lock_param.is_valid())); - } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, - lock_param.expired_time_, /*timeout*/ + } else if (OB_FAIL(get_write_store_ctx_guard_(lock_param.expired_time_, /*timeout*/ tx_desc, snapshot, 0,/*branch_id*/ write_flag, ctx_guard))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); + LOG_WARN("fail to check query allowed", K(ret)); } else { - ret = ls->replace_lock(ctx_guard.get_store_ctx(), lock_param); + ret = ctx_guard.get_ls()->replace_lock(ctx_guard.get_store_ctx(), lock_param); } return ret; } -int ObAccessService::add_lock_into_queue(const share::ObLSID &ls_id, - transaction::ObTxDesc &tx_desc, +int ObAccessService::add_lock_into_queue(transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m) { int ret = OB_SUCCESS; ObStoreCtxGuard ctx_guard; - ObLS *ls = nullptr; transaction::ObTxReadSnapshot snapshot; snapshot.init_none_read(); concurrent_control::ObWriteFlag write_flag; @@ -319,24 +276,19 @@ int ObAccessService::add_lock_into_queue(const share::ObLSID &ls_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tx_desc.is_valid()) + } else if (OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tx_desc), K(param)); - } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, - param.expired_time_, /*timeout*/ + LOG_WARN("invalid argument", K(ret), K(tx_desc), K(param)); + } else if (OB_FAIL(get_write_store_ctx_guard_(param.expired_time_, /*timeout*/ tx_desc, snapshot, 0, /*branch_id*/ write_flag, ctx_guard))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); + LOG_WARN("fail to check query allowed", K(ret)); } else { - ret = ls->add_lock_into_queue(ctx_guard.get_store_ctx(), param); + ret = ctx_guard.get_ls()->add_lock_into_queue(ctx_guard.get_store_ctx(), param); } return ret; } @@ -346,15 +298,11 @@ int ObAccessService::table_scan( ObNewRowIterator *&result) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = vparam.ls_id_; const common::ObTabletID &data_tablet_id = vparam.tablet_id_; ObASHTabletIdSetterGuard ash_tablet_id_guard(data_tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, data_tablet_id.id()); ObTableScanIterator *iter = nullptr; ObTabletHandle tablet_handle; - ObLS *ls = nullptr; - ObLSTabletService *tablet_service = nullptr; ObTableScanParam ¶m = static_cast(vparam); ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; @@ -379,26 +327,20 @@ int ObAccessService::table_scan( LOG_WARN("alloc table scan iterator fail", K(ret)); } else if (FALSE_IT(result = iter)) { // upper layer responsible for releasing iter object - } else if (OB_FAIL(check_read_allowed_(ls_id, - data_tablet_id, + } else if (OB_FAIL(check_read_allowed_(data_tablet_id, access_type, param, tablet_handle, iter->get_ctx_guard(), user_specified_snapshot_scn))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("fail to check query allowed", K(ret), K(data_tablet_id)); } // skip inner table, one key reason is to let tablet merge going - } else if (OB_ISNULL(ls = iter->get_ctx_guard().get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); - } else if (OB_FAIL(tablet_service->table_scan(tablet_handle, *iter, param))) { + } else if (OB_FAIL(iter->get_ctx_guard().get_ls()->get_tablet_svr()->table_scan( + tablet_handle, *iter, param))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("Fail to scan table, ", K(ret), K(ls_id), K(param)); + LOG_WARN("Fail to scan table, ", K(ret), K(param)); } } else { NG_TRACE(storage_table_scan_end); @@ -430,11 +372,7 @@ int ObAccessService::table_rescan( } } else { ObTableScanIterator *iter = static_cast(result); - const share::ObLSID &ls_id = vparam.ls_id_; const common::ObTabletID &data_tablet_id = vparam.tablet_id_; - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); - ObLS *ls = nullptr; - ObLSTabletService *tablet_service = nullptr; ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; SCN user_specified_snapshot_scn; @@ -443,26 +381,20 @@ int ObAccessService::table_rescan( user_specified_snapshot_scn = param.fb_snapshot_; } NG_TRACE(storage_table_scan_begin); - if (OB_FAIL(check_read_allowed_(ls_id, - data_tablet_id, + if (OB_FAIL(check_read_allowed_(data_tablet_id, access_type, param, /*scan_param*/ tablet_handle, iter->get_ctx_guard(), user_specified_snapshot_scn))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("fail to check query allowed", K(ret), K(result), K(ls_id), K(data_tablet_id)); + LOG_WARN("fail to check query allowed", K(ret), K(result), K(data_tablet_id)); } // skip inner table, one key reason is to let tablet merge going - } else if (OB_ISNULL(ls = iter->get_ctx_guard().get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(result), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); - } else if (OB_FAIL(tablet_service->table_rescan(tablet_handle, param, result))) { + } else if (OB_FAIL(iter->get_ctx_guard().get_ls()->get_tablet_svr()->table_rescan( + tablet_handle, param, result))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("Fail to scan table, ", K(ret), K(result), K(ls_id), K(param)); + LOG_WARN("Fail to scan table, ", K(ret), K(result), K(param)); } } else { NG_TRACE(storage_table_scan_end); @@ -493,7 +425,6 @@ int ObAccessService::table_advance_scan(ObVTableScanParam &vparam, ObNewRowItera } int ObAccessService::get_write_store_ctx_guard( - const share::ObLSID &ls_id, const int64_t timeout, transaction::ObTxDesc &tx_desc, const transaction::ObTxReadSnapshot &snapshot, @@ -503,22 +434,20 @@ int ObAccessService::get_write_store_ctx_guard( const transaction::ObTxSEQ &spec_seq_no) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tx_desc.is_valid() || !snapshot.is_valid())) { + } else if (OB_UNLIKELY(!tx_desc.is_valid() || !snapshot.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tx_desc), K(snapshot)); + LOG_WARN("invalid argument", K(ret), K(tx_desc), K(snapshot)); } else if (OB_FAIL(get_write_store_ctx_guard_( - ls_id, timeout, tx_desc, snapshot, branch_id, write_flag, ctx_guard, spec_seq_no))) { - LOG_WARN("fail to get write store ctx gurad", K(ret), K(ls_id), K(tx_desc)); + timeout, tx_desc, snapshot, branch_id, write_flag, ctx_guard, spec_seq_no))) { + LOG_WARN("fail to get write store ctx gurad", K(ret), K(tx_desc)); } return ret; } int ObAccessService::get_write_store_ctx_guard_( - const share::ObLSID &ls_id, const int64_t timeout, transaction::ObTxDesc &tx_desc, const transaction::ObTxReadSnapshot &snapshot, @@ -529,21 +458,18 @@ int ObAccessService::get_write_store_ctx_guard_( { int ret = OB_SUCCESS; ObLS *ls = nullptr; - if (OB_FAIL(ctx_guard.init(ls_id))) { - LOG_WARN("ctx_guard init fail", K(ret), K(ls_id), K(tx_desc)); + if (OB_FAIL(ls_svr_->get_ls(ls))) { + LOG_WARN("get log stream failed.", K(ret)); + } else if (OB_FAIL(ctx_guard.init(ls))) { + LOG_WARN("ctx_guard init fail", K(ret), K(tx_desc)); // DML statement will always use invalid snapshot - } else if (OB_FAIL(ls_svr_->get_ls(ls_id, ctx_guard.get_ls_handle(), ObLSGetMod::DAS_MOD))) { - LOG_WARN("get log stream failed.", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(ls_id)); } else { ObStoreCtx &ctx = ctx_guard.get_store_ctx(); ctx.ls_ = ls; ctx.timeout_ = timeout; ctx.branch_ = branch_id; if (OB_FAIL(ls->get_write_store_ctx(tx_desc, snapshot, write_flag, ctx, spec_seq_no))) { - LOG_WARN("can not get write store ctx", K(ret), K(ls_id), K(snapshot), K(tx_desc)); + LOG_WARN("can not get write store ctx", K(ret), K(snapshot), K(tx_desc)); } } if (OB_FAIL(ret)) { @@ -561,20 +487,10 @@ int ObAccessService::construct_store_ctx_other_variables_( ObStoreCtxGuard &ctx_guard) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls.get_ls_id(); ObLSTabletService *tablet_service = ls.get_tablet_svr(); - if (OB_ISNULL(tablet_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); - } else if (OB_FAIL(tablet_service->check_allow_to_read())) { - if (OB_REPLICA_NOT_READABLE == ret) { - LOG_WARN("replica unreadable", K(ret), K(ls_id), K(tablet_id)); - } else { - LOG_ERROR("failed to check allow to read", K(ret), K(ls_id), K(tablet_id)); - } - } else if (OB_FAIL(tablet_service->get_tablet_with_timeout( + if (OB_FAIL(tablet_service->get_tablet_with_timeout( tablet_id, tablet_handle, timeout, ObMDSGetTabletMode::READ_READABLE_COMMITED, snapshot))) { - LOG_WARN("failed to check and get tablet", K(ret), K(ls_id), K(tablet_id), K(timeout), K(snapshot)); + LOG_WARN("failed to check and get tablet", K(ret), K(tablet_id), K(timeout), K(snapshot)); } if (OB_TABLET_IS_SPLIT_SRC == ret) { ObArray tmp_tablet_ids; @@ -588,10 +504,10 @@ int ObAccessService::construct_store_ctx_other_variables_( LOG_WARN("split src tablet has active memtable, retry", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_service->get_tablet_with_timeout( tablet_id, tablet_handle, timeout, ObMDSGetTabletMode::READ_WITHOUT_CHECK, share::SCN::max_scn()))) { - LOG_WARN("failed to check and get tablet", K(ret), K(ls_id), K(tablet_id), K(timeout), K(snapshot)); + LOG_WARN("failed to check and get tablet", K(ret), K(tablet_id), K(timeout), K(snapshot)); } else if (OB_UNLIKELY(tablet_handle.get_obj()->is_empty_shell())) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("split src tablet is empty shell", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("split src tablet is empty shell", K(ret), K(tablet_id)); } } return ret; @@ -602,7 +518,6 @@ int ObAccessService::construct_store_ctx_other_variables_( * if replica can serve read, store_ctx will be prepared */ int ObAccessService::check_read_allowed_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObStoreAccessType access_type, const ObTableScanParam &scan_param, @@ -613,14 +528,11 @@ int ObAccessService::check_read_allowed_( int ret = OB_SUCCESS; ObLS *ls = nullptr; - LOG_TRACE("print check read allowed, scan param", K(ls_id), K(tablet_id)); - if (OB_FAIL(ctx_guard.init(ls_id))) { - LOG_WARN("ctx_guard init fail", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_svr_->get_ls(ls_id, ctx_guard.get_ls_handle(), ObLSGetMod::DAS_MOD))) { - LOG_WARN("get log stream failed.", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(ls_id)); + LOG_TRACE("print check read allowed, scan param", K(tablet_id)); + if (OB_FAIL(ls_svr_->get_ls(ls))) { + LOG_WARN("get log stream failed.", K(ret)); + } else if (OB_FAIL(ctx_guard.init(ls))) { + LOG_WARN("ctx_guard init fail", K(ret)); } else { ObStoreCtx &ctx = ctx_guard.get_store_ctx(); ctx.ls_ = ls; @@ -630,7 +542,7 @@ int ObAccessService::check_read_allowed_( if (OB_FAIL(ls->get_read_store_ctx(user_specified_snapshot, scan_param.tx_lock_timeout_, ctx))) { - LOG_WARN("get read store ctx fail", K(user_specified_snapshot), K(ls_id), K(ret)); + LOG_WARN("get read store ctx fail", K(user_specified_snapshot), K(ret)); } } else { bool read_latest = access_type == ObStoreAccessType::READ_LATEST; @@ -643,14 +555,14 @@ int ObAccessService::check_read_allowed_( read_latest, scan_param.tx_lock_timeout_, ctx))) { - LOG_WARN("get read store ctx fail", K(ret), K(read_latest), K(spec_snapshot), K(user_specified_snapshot), K(ls_id)); + LOG_WARN("get read store ctx fail", K(ret), K(read_latest), K(spec_snapshot), K(user_specified_snapshot)); } } else if (OB_FAIL(ls->get_read_store_ctx(scan_param.snapshot_, read_latest, scan_param.tx_lock_timeout_, ctx, scan_param.trans_desc_))) { - LOG_WARN("get read store ctx fail", K(ret), K(read_latest), K(scan_param.snapshot_), K(ls_id)); + LOG_WARN("get read store ctx fail", K(ret), K(read_latest), K(scan_param.snapshot_)); } if (OB_FAIL(ret)) { } else if (read_latest) { @@ -682,7 +594,6 @@ int ObAccessService::check_read_allowed_( } } if (OB_FAIL(ret)) { - // single tenant never drops tenant; no extra handling for OB_LS_NOT_EXIST } else if (OB_FAIL(construct_store_ctx_other_variables_(*ls, tablet_id, scan_param.timeout_, ctx.mvcc_acc_ctx_.get_snapshot_version(), tablet_handle, ctx_guard))) { if (OB_SNAPSHOT_DISCARDED == ret && scan_param.fb_snapshot_.is_valid()) { @@ -690,8 +601,6 @@ int ObAccessService::check_read_allowed_( } else { LOG_WARN("failed to check replica allow to read", K(ret), K(tablet_id), "timeout", scan_param.timeout_); } - } else if (OB_FAIL(check_mlog_safe_(*tablet_handle.get_obj(), scan_param))) { - LOG_WARN("failed to check_mlog_safe", KR(ret), K(tablet_id), K(scan_param)); } } return ret; @@ -704,7 +613,6 @@ int ObAccessService::check_read_allowed_( * if can serve write, store_ctx will be prepared */ int ObAccessService::check_write_allowed_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObStoreAccessType access_type, const ObDMLBaseParam &dml_param, @@ -716,7 +624,7 @@ int ObAccessService::check_write_allowed_( int ret = OB_SUCCESS; bool is_out_of_mem = false; bool is_disk_full = false; - ObLS *ls = nullptr; + ObLS *const ls = ctx_guard.get_ls(); ObLockID lock_id; ObLockParam lock_param; const ObTableLockMode lock_mode = ROW_EXCLUSIVE; @@ -737,14 +645,11 @@ int ObAccessService::check_write_allowed_( } else if (is_out_of_mem && !tablet_id.is_inner_tablet()) { ret = OB_TENANT_OUT_OF_MEM; LOG_WARN("this tenant is already out of memstore limit", K(ret)); - } else if (OB_FAIL(check_data_disk_full_(ls_id, is_disk_full))) { + } else if (OB_FAIL(check_data_disk_full_(is_disk_full))) { LOG_WARN("fail to check data disk full", K(ret)); } else if (is_disk_full) { ret = OB_USER_OUTOF_DATA_DISK_SPACE; LOG_WARN("data disk full, you should not do io now", K(ret)); - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), K(ls_id)); } else { ObStoreCtx &store_ctx = ctx_guard.get_store_ctx(); store_ctx.tablet_id_ = tablet_id; @@ -795,7 +700,6 @@ int ObAccessService::check_write_allowed_( } int ObAccessService::delete_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -804,40 +708,33 @@ int ObAccessService::delete_rows( int64_t &affected_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), KP(row_iter)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->delete_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -850,7 +747,6 @@ int ObAccessService::delete_rows( } int ObAccessService::put_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -859,40 +755,33 @@ int ObAccessService::put_rows( int64_t &affected_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), K(row_iter)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->put_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -905,7 +794,6 @@ int ObAccessService::put_rows( } int ObAccessService::insert_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -914,40 +802,33 @@ int ObAccessService::insert_rows( int64_t &affected_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), KP(row_iter)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->insert_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -959,8 +840,7 @@ int ObAccessService::insert_rows( return ret; } -int ObAccessService::insert_rows_with_fetch_dup(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, +int ObAccessService::insert_rows_with_fetch_dup(const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, @@ -971,40 +851,33 @@ int ObAccessService::insert_rows_with_fetch_dup(const share::ObLSID &ls_id, blocksstable::ObDatumRowIterator *&duplicated_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_UNLIKELY(duplicated_column_ids.count() <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), K(duplicated_column_ids)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->insert_rows_with_fetch_dup(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -1021,7 +894,6 @@ int ObAccessService::insert_rows_with_fetch_dup(const share::ObLSID &ls_id, int ObAccessService::update_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -1031,40 +903,33 @@ int ObAccessService::update_rows( int64_t &affected_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_UNLIKELY(updated_column_ids.count() <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), K(updated_column_ids)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->update_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -1078,7 +943,6 @@ int ObAccessService::update_rows( } int ObAccessService::lock_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -1088,10 +952,8 @@ int ObAccessService::lock_rows( int64_t &affected_rows) { ObASHTabletIdSetterGuard ash_tablet_id_guard(tablet_id.id()); - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); int ret = OB_SUCCESS; - ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; // Attention!!! This handle is only used for ObLSTabletService, will be reset inside ObLSTabletService. ObTabletHandle tablet_handle; @@ -1099,29 +961,24 @@ int ObAccessService::lock_rows( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(abs_lock_timeout), K(lock_flag), KP(row_iter)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::ROW_LOCK, dml_param, lock_wait_timeout_ts, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_ISNULL(ls = dml_param.store_ctx_guard_->get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); + } else if (OB_ISNULL(tablet_service = dml_param.store_ctx_guard_->get_ls()->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null.", K(ret)); } else { ret = tablet_service->lock_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), @@ -1144,20 +1001,16 @@ int ObAccessService::estimate_row_count( int64_t &physical_row_count) const { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; if (IS_NOT_INIT) { ret = OB_ERROR; LOG_WARN("ob access service is not running.", K(ret)); } else if (OB_UNLIKELY(!param.is_estimate_valid() || !scan_range.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(param), K(scan_range), K(ret)); - } else if (OB_FAIL(ls_svr_->get_ls(param.ls_id_, ls_handle, ObLSGetMod::DAS_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(param.ls_id_)); - } else if (nullptr == (ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected null", K(ret)); - } else if (OB_FAIL(ls->get_tablet_svr()->estimate_row_count( + } else if (OB_FAIL(ls_svr_->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->estimate_row_count( param, scan_range, timeout_us, est_records, logical_row_count, physical_row_count))) { if (OB_TABLET_NOT_EXIST != ret) { @@ -1168,7 +1021,6 @@ int ObAccessService::estimate_row_count( } int ObAccessService::estimate_block_count_and_row_count( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, int64_t ¯o_block_count, @@ -1177,57 +1029,47 @@ int ObAccessService::estimate_block_count_and_row_count( int64_t &memtable_row_count) const { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ls_id), K(tablet_id), K(ret)); - } else if (OB_FAIL(ls_svr_->get_ls(ls_id, ls_handle, ObLSGetMod::DAS_MOD))) { - LOG_WARN("failed to get log stream", K(ret), K(ls_id)); - } else if (nullptr == (ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected null", K(ret)); - } else if (OB_FAIL(ls->get_tablet_svr()->estimate_block_count_and_row_count( + LOG_WARN("invalid argument", K(tablet_id), K(ret)); + } else if (OB_FAIL(ls_svr_->get_ls(tenant_ls))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(tenant_ls->get_tablet_svr()->estimate_block_count_and_row_count( tablet_id, timeout_us, macro_block_count, micro_block_count, sstable_row_count, memtable_row_count))) { - LOG_WARN("failed to estimate block count and row count", K(ret), K(ls_id), K(tablet_id), K(timeout_us)); + LOG_WARN("failed to estimate block count and row count", K(ret), K(tablet_id), K(timeout_us)); } return ret; } int ObAccessService::get_multi_ranges_cost( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const common::ObIArray &ranges, int64_t &total_size) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; ObLSTabletService *tablet_service = nullptr; if (IS_NOT_INIT) { ret = OB_ERROR; LOG_WARN("ob access service is not running", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(ranges.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls_svr_->get_ls(ls_id, ls_handle, ObLSGetMod::DAS_MOD))) { - LOG_WARN("get log stream failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls hould not be null", K(ret), K(ls_id)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("invalid argument", K(ret), K(tablet_id)); + } else if (OB_FAIL(ls_svr_->get_ls(tenant_ls))) { + LOG_WARN("get log stream failed", K(ret)); + } else if (OB_ISNULL(tablet_service = tenant_ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null", K(ret)); } else if (OB_FAIL(tablet_service->get_multi_ranges_cost(tablet_id, timeout_us, ranges, total_size))) { - LOG_WARN("Fail to get multi ranges cost", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("Fail to get multi ranges cost", K(ret), K(tablet_id)); } return ret; } @@ -1266,9 +1108,6 @@ int ObAccessService::revert_scan_iter(ObNewRowIterator *iter) } else if (iter->get_type() == ObNewRowIterator::ObTableScanIterator) { ObTableScanIterator *table_scan_iter = nullptr; table_scan_iter = static_cast(iter); - if (OB_FAIL(table_scan_iter->check_ls_offline_after_read())) { - LOG_WARN("discover ls offline after table scan", K(ret), KPC(table_scan_iter)); - } mtl_sop_return_checked(ObTableScanIterator, table_scan_iter); } else { iter->~ObNewRowIterator(); @@ -1279,7 +1118,6 @@ int ObAccessService::revert_scan_iter(ObNewRowIterator *iter) } int ObAccessService::split_multi_ranges( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const ObIArray &ranges, @@ -1288,89 +1126,48 @@ int ObAccessService::split_multi_ranges( ObArrayArray &multi_range_split_array) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLS *ls = nullptr; + ObLS *tenant_ls = nullptr; ObLSTabletService *tablet_service = nullptr; if (IS_NOT_INIT) { ret = OB_ERROR; LOG_WARN("ob access service is not running", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(ranges.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls_svr_->get_ls(ls_id, ls_handle, ObLSGetMod::DAS_MOD))) { - LOG_WARN("get log stream failed", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls hould not be null", K(ret), K(ls_id)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { + LOG_WARN("invalid argument", K(ret), K(tablet_id)); + } else if (OB_FAIL(ls_svr_->get_ls(tenant_ls))) { + LOG_WARN("get log stream failed", K(ret)); + } else if (OB_ISNULL(tablet_service = tenant_ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null", K(ret), K(ls_id)); + LOG_ERROR("tablet service should not be null", K(ret)); } else if (OB_FAIL(tablet_service->split_multi_ranges( tablet_id, timeout_us, ranges, expected_task_count, allocator, multi_range_split_array))) { - LOG_WARN("Fail to split multi ranges", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("Fail to split multi ranges", K(ret), K(tablet_id)); } return ret; } -int ObAccessService::check_mlog_safe_( - const ObTablet &tablet, - const ObTableScanParam &scan_param) -{ - int ret = OB_SUCCESS; - int64_t begin_version = -1; - if (OB_ISNULL(scan_param.table_param_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("scan_param.table_param_ is NULL", KR(ret), K(scan_param), K(tablet)); - } else if (scan_param.table_param_->is_mlog_table()) { - if (OB_ISNULL(scan_param.op_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("scan_param.op_ is NULL", KR(ret), K(scan_param), K(tablet)); - } else if (OB_ISNULL(scan_param.op_filters_)) { - // query has no filter - } else if (OB_FAIL(get_query_begin_version_for_mlog(*scan_param.op_filters_, scan_param.op_->get_eval_ctx(), begin_version))) { - LOG_WARN("failed to get_query_begin_version_for_mlog", KR(ret), K(scan_param), K(tablet)); - } else if (-1 != begin_version) { - ObTabletCreateDeleteMdsUserData user_data; - if (OB_FAIL(tablet.ObITabletMdsInterface::get_tablet_status( - share::SCN::max_scn(), user_data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US))) { - LOG_WARN("failed to get tablet status", KR(ret), K(tablet)); - } else if (begin_version < user_data.create_commit_version_) { - ret = OB_INVALID_ARGUMENT; - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "mlog query begin version which is more than tablet create commit version, materialized view need complete fresh"); - LOG_WARN("mlog query begin version is more than create commit version", KR(ret), K(begin_version), K(user_data), K(tablet)); - } - } - } - return ret; -} - - int ObAccessService::inner_tablet_scan( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTableScanParam ¶m, ObNewRowIterator *&result) { int ret = OB_SUCCESS; - ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(ls_id_, ls_id.id()); ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(tablet_id_, tablet_id.id()); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (!ls_id.is_valid() || !tablet_id.is_valid()) { + } else if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(lbt())); - } else if (OB_FAIL(do_table_scan_(ls_id, tablet_id, param, result))) { - LOG_WARN("failed to do table scan", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(lbt())); + } else if (OB_FAIL(do_table_scan_(tablet_id, param, result))) { + LOG_WARN("failed to do table scan", K(ret), K(tablet_id)); } return ret; } int ObAccessService::do_table_scan_( - const share::ObLSID &ls_id, const common::ObTabletID &data_tablet_id, ObTableScanParam ¶m, ObNewRowIterator *&result) @@ -1378,8 +1175,6 @@ int ObAccessService::do_table_scan_( int ret = OB_SUCCESS; ObTableScanIterator *iter = nullptr; ObTabletHandle tablet_handle; - ObLS *ls = nullptr; - ObLSTabletService *tablet_service = nullptr; ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; SCN user_specified_snapshot_scn; @@ -1390,38 +1185,32 @@ int ObAccessService::do_table_scan_( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob access service is not running.", K(ret)); - } else if (!ls_id.is_valid() || !data_tablet_id.is_valid()) { + } else if (!data_tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(data_tablet_id), K(lbt())); + LOG_WARN("invalid argument", K(ret), K(data_tablet_id), K(lbt())); } else if (OB_NOT_NULL(result)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("The result_ is already pointed to an valid object", - K(ret), K(ls_id), K(data_tablet_id), KPC(result), K(lbt())); + K(ret), K(data_tablet_id), KPC(result), K(lbt())); } else if (OB_ISNULL(iter = mtl_sop_borrow(ObTableScanIterator))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("alloc table scan iterator fail", K(ret)); } else if (FALSE_IT(result = iter)) { // upper layer responsible for releasing iter object - } else if (OB_FAIL(check_read_allowed_(ls_id, - data_tablet_id, + } else if (OB_FAIL(check_read_allowed_(data_tablet_id, access_type, param, tablet_handle, iter->get_ctx_guard(), user_specified_snapshot_scn))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(data_tablet_id)); + LOG_WARN("fail to check query allowed", K(ret), K(data_tablet_id)); } // skip inner table, one key reason is to let tablet merge going - } else if (OB_ISNULL(ls = iter->get_ctx_guard().get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls should not be null", K(ret), KP(ls)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("tablet service should not be null.", K(ret), K(ls_id)); - } else if (OB_FAIL(tablet_service->table_scan(tablet_handle, *iter, param))) { + } else if (OB_FAIL(iter->get_ctx_guard().get_ls()->get_tablet_svr()->table_scan( + tablet_handle, *iter, param))) { if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("Fail to scan table, ", K(ret), K(ls_id), K(param)); + LOG_WARN("Fail to scan table, ", K(ret), K(param)); } } else { NG_TRACE(storage_table_scan_end); @@ -1441,10 +1230,7 @@ int ObAccessService::scan_block_stat(ObBlockStatScanParam &scan_param, ObBlockSt } else { ObTableScanParam &table_scan_param = *scan_param.get_scan_param(); ObStoreCtxGuard &ctx_guard = iter.get_ctx_guard(); - ObLSID ls_id = table_scan_param.ls_id_; ObTabletID tablet_id = table_scan_param.tablet_id_; - ObLS *ls = nullptr; - ObLSTabletService *tablet_service = nullptr; ObTabletHandle tablet_handle; ObStoreAccessType access_type = table_scan_param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; @@ -1453,7 +1239,6 @@ int ObAccessService::scan_block_stat(ObBlockStatScanParam &scan_param, ObBlockSt user_specified_snapshot_scn = table_scan_param.fb_snapshot_; } if (OB_FAIL(check_read_allowed_( - ls_id, tablet_id, access_type, table_scan_param, @@ -1461,17 +1246,12 @@ int ObAccessService::scan_block_stat(ObBlockStatScanParam &scan_param, ObBlockSt ctx_guard, user_specified_snapshot_scn))) { if (OB_UNLIKELY(OB_TABLET_NOT_EXIST != ret)) { - LOG_WARN("fail to check read allowed", K(ret), K(ls_id), K(tablet_id), K(access_type)); + LOG_WARN("fail to check read allowed", K(ret), K(tablet_id), K(access_type)); } - } else if (OB_ISNULL(ls = ctx_guard.get_ls_handle().get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("unexpected nullptr to ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(tablet_service = ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("unexpected nullptr to tablet service", K(ret), K(ls_id)); - } else if (OB_FAIL(tablet_service->scan_block_stat(tablet_handle, scan_param, iter))) { + } else if (OB_FAIL(ctx_guard.get_ls()->get_tablet_svr()->scan_block_stat( + tablet_handle, scan_param, iter))) { if (OB_UNLIKELY(OB_TABLET_NOT_EXIST != ret)) { - LOG_WARN("fail to scan block stat", K(ret), K(ls_id), K(tablet_id), K(scan_param)); + LOG_WARN("fail to scan block stat", K(ret), K(tablet_id), K(scan_param)); } } } diff --git a/src/storage/tx_storage/ob_access_service.h b/src/storage/tx_storage/ob_access_service.h index 34d63abeb..537f9646a 100644 --- a/src/storage/tx_storage/ob_access_service.h +++ b/src/storage/tx_storage/ob_access_service.h @@ -19,10 +19,8 @@ #include "share/ob_i_tablet_scan.h" #include "storage/ls/ob_ls_tablet_service.h" -#include "storage/tx_storage/ob_ls_map.h" #include "storage/access/ob_table_scan_range.h" #include "sql/optimizer/stat/ob_stat_define.h" -#include "storage/tx_storage/ob_ls_handle.h" namespace oceanbase { @@ -30,9 +28,6 @@ namespace common { class ObTabletID; class ObNewRowIterator; } -namespace share { -class ObLSID; -} namespace transaction { class ObTxDesc; @@ -46,22 +41,21 @@ class ObStoreCtx; class ObStoreCtxGuard { public: - ObStoreCtxGuard() : is_inited_(false), init_ts_(0) + ObStoreCtxGuard() : is_inited_(false), ls_(nullptr), init_ts_(0) { } ~ObStoreCtxGuard() { reset(); } - int init(const share::ObLSID &ls_id); + int init(ObLS *ls); void reset(); ObStoreCtx &get_store_ctx() { return ctx_; } - ObLSHandle &get_ls_handle() { return handle_; } + ObLS *get_ls() const { return ls_; } private: bool is_inited_; ObStoreCtx ctx_; - share::ObLSID ls_id_; - ObLSHandle handle_; + ObLS *ls_; int64_t init_ts_; DISALLOW_COPY_AND_ASSIGN(ObStoreCtxGuard); @@ -78,39 +72,30 @@ class ObAccessService : public common::ObITabletScan void destroy(); public: // pre_check_lock - // @param [in] ls_id, this check op will be processed at which logstream. // @param [in] tx_desc, the trans context. // @param [in] param, contain all the check parameters. int pre_check_lock( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m); // lock obj - // @param [in] ls_id, this lock op will be processed at which logstream. // @param [in] tx_desc, the trans context. // @param [in] param, contain all the lock parameters. int lock_obj( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m); // unlock obj - // @param [in] ls_id, this unlock op will be processed at which logstream. // @param [in] tx_desc, the trans context. // @param [in] param, contain all the unlock parameters. int unlock_obj( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m); // replace lock of obj - // @param [in] ls_id, this replace lock operation will be processed at which logstream. // @param [in] tx_desc, the trans context. // @param [in] lock_param, contain all the parameters of previous lock, and lock_owner / lock_mode of new lock int replace_obj_lock( - const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObReplaceLockParam &lock_param); - int add_lock_into_queue(const share::ObLSID &ls_id, - transaction::ObTxDesc &tx_desc, + int add_lock_into_queue(transaction::ObTxDesc &tx_desc, const transaction::tablelock::ObLockParam ¶m); // ObITabletScan interface virtual int table_scan( @@ -123,13 +108,11 @@ class ObAccessService : public common::ObITabletScan virtual int reuse_scan_iter(const bool switch_param, common::ObNewRowIterator *iter) override; virtual int revert_scan_iter(common::ObNewRowIterator *iter) override; virtual int get_multi_ranges_cost( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const common::ObIArray &ranges, int64_t &total_size) override; virtual int split_multi_ranges( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, const common::ObIArray &ranges, @@ -137,7 +120,6 @@ class ObAccessService : public common::ObITabletScan common::ObIAllocator &allocator, common::ObArrayArray &multi_range_split_array) override; int get_write_store_ctx_guard( - const share::ObLSID &ls_id, const int64_t timeout, transaction::ObTxDesc &tx_desc, const transaction::ObTxReadSnapshot &snapshot, @@ -148,7 +130,6 @@ class ObAccessService : public common::ObITabletScan // DML interface int delete_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -156,7 +137,6 @@ class ObAccessService : public common::ObITabletScan blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int put_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -164,7 +144,6 @@ class ObAccessService : public common::ObITabletScan blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -172,7 +151,6 @@ class ObAccessService : public common::ObITabletScan blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_rows_with_fetch_dup( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -183,7 +161,6 @@ class ObAccessService : public common::ObITabletScan int64_t &affected_rows, blocksstable::ObDatumRowIterator *&duplicated_rows); int update_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -192,7 +169,6 @@ class ObAccessService : public common::ObITabletScan blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -208,7 +184,6 @@ class ObAccessService : public common::ObITabletScan int64_t &logical_row_count, int64_t &physical_row_count) const; int estimate_block_count_and_row_count( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t timeout_us, int64_t ¯o_block_count, @@ -218,7 +193,6 @@ class ObAccessService : public common::ObITabletScan int inner_tablet_scan( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTableScanParam ¶m, ObNewRowIterator *&result); @@ -227,11 +201,8 @@ class ObAccessService : public common::ObITabletScan protected: int check_tenant_out_of_memstore_limit_(bool &is_out_of_mem); - int check_data_disk_full_( - const share::ObLSID &ls_id, - bool &is_full); + int check_data_disk_full_(bool &is_full); int get_write_store_ctx_guard_( - const share::ObLSID &ls_id, const int64_t timeout, transaction::ObTxDesc &tx_desc, const transaction::ObTxReadSnapshot &snapshot, @@ -240,7 +211,6 @@ class ObAccessService : public common::ObITabletScan ObStoreCtxGuard &ctx_guard, const transaction::ObTxSEQ &spec_seq_no = transaction::ObTxSEQ::INVL()); int check_read_allowed_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObStoreAccessType access_type, const ObTableScanParam &scan_param, @@ -248,7 +218,6 @@ class ObAccessService : public common::ObITabletScan ObStoreCtxGuard &ctx_guard, share::SCN user_specified_snapshot); int check_write_allowed_( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObStoreAccessType access_type, const ObDMLBaseParam &dml_param, @@ -263,10 +232,6 @@ class ObAccessService : public common::ObITabletScan const share::SCN &snapshot, ObTabletHandle &tablet_handle, ObStoreCtxGuard &ctx_guard); - static int check_mlog_safe_( - const ObTablet &tablet, - const ObTableScanParam &scan_param); - static OB_INLINE int64_t get_lock_wait_timeout_(const int64_t abs_lock_timeout, const int64_t stmt_timeout) { return (abs_lock_timeout < 0 ? stmt_timeout : (abs_lock_timeout > stmt_timeout ? stmt_timeout : abs_lock_timeout)); @@ -274,7 +239,6 @@ class ObAccessService : public common::ObITabletScan private: int do_table_scan_( - const share::ObLSID &ls_id, const common::ObTabletID &data_tablet_id, ObTableScanParam ¶m, ObNewRowIterator *&result); diff --git a/src/storage/tx_storage/ob_checkpoint_service.cpp b/src/storage/tx_storage/ob_checkpoint_service.cpp index fdd165fd9..1441961c3 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.cpp +++ b/src/storage/tx_storage/ob_checkpoint_service.cpp @@ -21,6 +21,7 @@ #include "logservice/ob_log_service.h" #include "share/ob_global_stat_proxy.h" #include "observer/ob_server_struct.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -137,70 +138,33 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask() { STORAGE_LOG(INFO, "====== checkpoint timer task ======"); int ret = OB_SUCCESS; - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - ObLSService *ls_svr = share::g_mp->ls_service(); int64_t cs_min_dep_lsn_val = 0; - - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", K(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", K(ret)); + DEBUG_SYNC(BEFORE_CHECKPOINT_TASK); + ObLS *tenant_ls = nullptr; + palf::LSN checkpoint_lsn; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(tenant_ls->get_data_checkpoint()->check_can_move_to_active_in_newcreate())) { + STORAGE_LOG(WARN, "check can move to active failed", K(ret)); + } else if (OB_FAIL(tenant_ls->get_checkpoint_executor()->update_clog_checkpoint())) { + STORAGE_LOG(WARN, "update_clog_checkpoint failed", K(ret)); + } else if (OB_FAIL(ObGlobalStatProxy::get_change_stream_min_dep_lsn( + *GCTX.sql_proxy_, false/*for_update*/, cs_min_dep_lsn_val))) { + STORAGE_LOG(WARN, "get_change_stream_min_dep_lsn failed, skip constraint", KR(ret)); } else { - DEBUG_SYNC(BEFORE_CHECKPOINT_TASK); - ObLS *ls = nullptr; - int ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - ObLSHandle ls_handle; - ObCheckpointExecutor *checkpoint_executor = nullptr; - ObDataCheckpoint *data_checkpoint = nullptr; - palf::LSN checkpoint_lsn; - if (OB_FAIL(ls_svr->get_ls(ls->get_ls_id(), ls_handle, ObLSGetMod::APPLY_MOD))) { - STORAGE_LOG(WARN, "get log stream failed", K(ret), K(ls->get_ls_id())); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "log stream not exist", K(ret), K(ls->get_ls_id())); - } else if (OB_ISNULL(data_checkpoint = ls->get_data_checkpoint())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "data_checkpoint should not be null", K(ret), K(ls->get_ls_id())); - } else if (OB_FAIL(data_checkpoint->check_can_move_to_active_in_newcreate())) { - STORAGE_LOG(WARN, "check can move to active failed", K(ret), K(ls->get_ls_id())); - } else if (OB_ISNULL(checkpoint_executor = ls->get_checkpoint_executor())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "checkpoint_executor should not be null", K(ls->get_ls_id())); - } else if (OB_FAIL(checkpoint_executor->update_clog_checkpoint())) { - STORAGE_LOG(WARN, "update_clog_checkpoint failed", K(ret), K(ls->get_ls_id())); - } else if (OB_FAIL(ObGlobalStatProxy::get_change_stream_min_dep_lsn( - *GCTX.sql_proxy_, false/*for_update*/, cs_min_dep_lsn_val))) { - STORAGE_LOG(WARN, "get_change_stream_min_dep_lsn failed, skip constraint", KR(ret)); - } else { - checkpoint_lsn = ls->get_clog_base_lsn(); - palf::LSN cs_min_dep_lsn = palf::LSN(cs_min_dep_lsn_val); - if (cs_min_dep_lsn < checkpoint_lsn) { - FLOG_INFO("[CHECKPOINT] constrain base_lsn by change_stream_min_dep_lsn", - K(checkpoint_lsn), K(cs_min_dep_lsn)); - checkpoint_lsn = cs_min_dep_lsn; - } - - if (OB_FAIL(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { - STORAGE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); - } else { - FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", - K(checkpoint_lsn), K(ls->get_ls_id())); - } - } + checkpoint_lsn = tenant_ls->get_clog_base_lsn(); + palf::LSN cs_min_dep_lsn = palf::LSN(cs_min_dep_lsn_val); + if (cs_min_dep_lsn < checkpoint_lsn) { + FLOG_INFO("[CHECKPOINT] constrain base_lsn by change_stream_min_dep_lsn", + K(checkpoint_lsn), K(cs_min_dep_lsn)); + checkpoint_lsn = cs_min_dep_lsn; } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (ls_cnt > 0) { - STORAGE_LOG(INFO, "succeed to update_clog_checkpoint", K(ret), K(ls_cnt)); - } else { - STORAGE_LOG(INFO, "no logstream", K(ret), K(ls_cnt)); - } + + if (OB_FAIL(tenant_ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { + STORAGE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); + } else { + FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", K(checkpoint_lsn)); + STORAGE_LOG(INFO, "succeed to update_clog_checkpoint"); } } } @@ -210,44 +174,20 @@ int ObCheckPointService::flush_to_recycle_clog_() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - ObLSService *ls_svr = share::g_mp->ls_service(); - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", K(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", K(ret)); + ObLS *tenant_ls = nullptr; + bool flushed = false; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "get log stream failed", K(ret)); + } else if (tenant_ls->get_data_checkpoint()->is_flushing()) { + STORAGE_LOG(TRACE, "data_checkpoint is flushing"); + } else if (OB_TMP_FAIL(tenant_ls->get_checkpoint_executor()->update_clog_checkpoint())) { + STORAGE_LOG(WARN, "update_clog_checkpoint failed", KR(tmp_ret)); + } else if (OB_TMP_FAIL(tenant_ls->flush_to_recycle_clog())) { + STORAGE_LOG(WARN, "flush ls to recycle clog failed", KR(tmp_ret)); } else { - ObLS *ls = nullptr; - int64_t ls_cnt = 0; - int64_t succ_ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - ObCheckpointExecutor *checkpoint_executor = ls->get_checkpoint_executor(); - ObDataCheckpoint *data_checkpoint = ls->get_data_checkpoint(); - if (OB_ISNULL(checkpoint_executor) || OB_ISNULL(data_checkpoint)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "checkpoint_executor or data_checkpoint should not be null", - KP(checkpoint_executor), KP(data_checkpoint)); - } else if (data_checkpoint->is_flushing()) { - STORAGE_LOG(TRACE, "data_checkpoint is flushing"); - } else if (OB_TMP_FAIL(checkpoint_executor->update_clog_checkpoint())) { - STORAGE_LOG(WARN, "update_clog_checkpoint failed", KR(tmp_ret), KP(checkpoint_executor), KP(data_checkpoint)); - } else if (OB_TMP_FAIL(ls->flush_to_recycle_clog())) { - STORAGE_LOG(WARN, "flush ls to recycle clog failed", KR(tmp_ret), KPC(ls)); - } else { - ++succ_ls_cnt; - } - } - STORAGE_LOG(DEBUG, "finish flush to recycle clog", KR(ret), K(ls_cnt), K(succ_ls_cnt)); - - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - } + flushed = true; } + STORAGE_LOG(DEBUG, "finish flush to recycle clog", KR(ret), K(flushed)); return ret; } @@ -256,44 +196,14 @@ void ObCheckPointService::ObTraversalFlushTask::runTimerTask() { STORAGE_LOG(INFO, "====== traversal_flush timer task ======"); int ret = OB_SUCCESS; - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - ObLSService *ls_svr = share::g_mp->ls_service(); ObCurTraceId::init(GCONF.self_addr_); - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", K(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", K(ret)); + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "get log stream failed", K(ret)); + } else if (OB_FAIL(tenant_ls->get_checkpoint_executor()->traversal_flush())) { + STORAGE_LOG(WARN, "traversal_flush failed", K(ret)); } else { - ObLS *ls = nullptr; - int ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - ObLSHandle ls_handle; - ObCheckpointExecutor *checkpoint_executor = nullptr; - if (OB_FAIL(ls_svr->get_ls(ls->get_ls_id(), ls_handle, ObLSGetMod::APPLY_MOD))) { - STORAGE_LOG(WARN, "get log stream failed", K(ret), K(ls->get_ls_id())); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "log stream not exist", K(ret), K(ls->get_ls_id())); - } else if (OB_ISNULL(checkpoint_executor = ls->get_checkpoint_executor())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "checkpoint_executor should not be null", K(ls->get_ls_id())); - } else if (OB_FAIL(checkpoint_executor->traversal_flush())) { - STORAGE_LOG(WARN, "traversal_flush failed", K(ret), K(ls->get_ls_id())); - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (ls_cnt > 0) { - STORAGE_LOG(INFO, "succeed to traversal_flush", K(ret), K(ls_cnt)); - } else { - STORAGE_LOG(INFO, "no logstream", K(ret), K(ls_cnt)); - } - } + STORAGE_LOG(INFO, "succeed to traversal_flush"); } ObCurTraceId::reset(); } @@ -314,21 +224,6 @@ void ObCheckPointService::ObCheckClogDiskUsageTask::runTimerTask() } } -struct AdvanceCkptFunctorForLS { -public: - int operator()(ObLS &ls) - { - int ret = OB_SUCCESS; - if (OB_FAIL(ls.advance_checkpoint_by_flush( - SCN::max_scn(), INT64_MAX /*timeout*/, false /*is_tenant_freeze*/, ObFreezeSourceFlag::CLOG_CHECKPOINT))) { - STORAGE_LOG(WARN, "flush ls to recycle clog failed", KR(ret), K(ls)); - } - - // return OB_SUCCESS to iterate all logstreams - return OB_SUCCESS; - } -}; - void ObCheckPointService::ObAdvanceCkptTask::runTimerTask() { int ret = OB_SUCCESS; @@ -345,10 +240,14 @@ void ObCheckPointService::ObAdvanceCkptTask::runTimerTask() const int64_t current_ts = ObClockGenerator::getClock(); const int64_t prev_advance_ckpt_task_ts = share::g_mp->check_point_service()->prev_advance_ckpt_task_ts(); if (current_ts - prev_advance_ckpt_task_ts > advance_checkpoint_interval) { - AdvanceCkptFunctorForLS advance_ckpt_func_for_ls; - if (OB_FAIL(share::g_mp->ls_service()->foreach_ls(advance_ckpt_func_for_ls))) { - STORAGE_LOG(WARN, "for each ls functor failed", KR(ret)); - } else { + ObLS *tenant_ls = nullptr; + if (OB_FAIL(share::g_mp->ls_service()->get_ls(tenant_ls))) { + STORAGE_LOG(WARN, "get log stream failed", KR(ret)); + } else if (OB_FAIL(tenant_ls->advance_checkpoint_by_flush( + SCN::max_scn(), INT64_MAX /*timeout*/, false /*is_tenant_freeze*/, ObFreezeSourceFlag::CLOG_CHECKPOINT))) { + STORAGE_LOG(WARN, "flush ls to recycle clog failed", KR(ret)); + } + if (OB_SUCC(ret)) { share::g_mp->check_point_service()->set_prev_advance_ckpt_task_ts(current_ts); } } else { diff --git a/src/storage/tx_storage/ob_empty_shell_object_checker.cpp b/src/storage/tx_storage/ob_empty_shell_object_checker.cpp index b4f74bcc8..f67cbaf49 100644 --- a/src/storage/tx_storage/ob_empty_shell_object_checker.cpp +++ b/src/storage/tx_storage/ob_empty_shell_object_checker.cpp @@ -193,7 +193,6 @@ int ObDDLEmptyShellChecker::check_split_src_deleted_tablet( can_become_empty_shell = false; SCN decided_scn; const ObTabletID tablet_id(tablet.get_tablet_meta().tablet_id_); - const share::ObLSID ls_id(tablet.get_tablet_meta().ls_id_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "not init", K(ret)); @@ -208,7 +207,7 @@ int ObDDLEmptyShellChecker::check_split_src_deleted_tablet( need_retry = true; if (REACH_THREAD_TIME_INTERVAL(1 * 1000 * 1000/*1s*/)) { STORAGE_LOG(INFO, "decided_scn is smaller than tablet delete commit scn", - K(ls_id), K(tablet_id), K(user_data), K(decided_scn)); + K(tablet_id), K(user_data), K(decided_scn)); } } else { int tmp_ret = OB_SUCCESS; diff --git a/src/storage/tx_storage/ob_empty_shell_task.cpp b/src/storage/tx_storage/ob_empty_shell_task.cpp index 43813064b..04a82e535 100644 --- a/src/storage/tx_storage/ob_empty_shell_task.cpp +++ b/src/storage/tx_storage/ob_empty_shell_task.cpp @@ -18,7 +18,7 @@ #include "lib/stat/ob_diagnostic_info_guard.h" #include "storage/tx_storage/ob_empty_shell_task.h" #include "share/rc/ob_module_provider.h" -#include "lib/literals/ob_literals.h" // ObLSIterator +#include "lib/literals/ob_literals.h" #include "storage/tx_storage/ob_ls_service.h" // ObLSService #include "storage/tablet/ob_tablet_iterator.h" #include "storage/meta_store/ob_server_storage_meta_service.h" @@ -41,9 +41,7 @@ void ObEmptyShellTask::runTimerTask() { STORAGE_LOG(INFO, "====== [emptytablet] empty shell timer task ======", K(GC_EMPTY_TABLET_SHELL_INTERVAL)); int ret = OB_SUCCESS; - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - ObLSService *ls_svr = share::g_mp->ls_service(); + ObLS *ls = nullptr; bool skip_empty_shell_task = false; RLOCAL_STATIC(int64_t, times) = 0; times = (times + 1) % GLOBAL_EMPTY_CHECK_INTERVAL_TIMES; @@ -53,63 +51,40 @@ void ObEmptyShellTask::runTimerTask() if (!SERVER_STORAGE_META_SERVICE.is_started()) { // do nothing STORAGE_LOG(DEBUG, "ob block manager has not started"); - } else if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "mtl ObLSService should not be null", KR(ret)); } else if (OB_UNLIKELY(skip_empty_shell_task)) { // do nothing - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", KR(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", KR(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + STORAGE_LOG(WARN, "get log stream failed", KR(ret)); } else { - ObLS *ls = NULL; - int ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - // one ls failed to handle empty shell task should not bother another ls - if (OB_ITER_END == ret) { - break; - } else { - ObTabletEmptyShellHandler *tablet_empty_shell_handler = NULL; - if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is NULL", KR(ret)); - } else if (FALSE_IT(tablet_empty_shell_handler = ls->get_tablet_empty_shell_handler())) { - } else if (tablet_empty_shell_handler->check_stop()) { - STORAGE_LOG(INFO, "[emptytablet] tablet_gc_handler is stop", K(ls->get_ls_id())); - } else if (0 == times || tablet_empty_shell_handler->get_empty_shell_trigger()) { - STORAGE_LOG(INFO, "[emptytablet] task check ls", "ls_id", ls->get_ls_id(), K(tablet_empty_shell_handler)); - tablet_empty_shell_handler->set_empty_shell_trigger(false); - obsys::ObRLockGuard lock(tablet_empty_shell_handler->wait_lock_); - bool need_retry = false; - common::ObTabletIDArray empty_shell_tablet_ids; - if (OB_FAIL(tablet_empty_shell_handler->get_empty_shell_tablet_ids(empty_shell_tablet_ids, need_retry))) { - need_retry = true; - STORAGE_LOG(WARN, "[emptytablet] tablet_empty_shell_handler get empty shell tablet ids failed", K(ret), "ls_id", ls->get_ls_id()); - } else if (empty_shell_tablet_ids.empty()) { - // do nothing - } else if (OB_FAIL(tablet_empty_shell_handler->update_tablets_to_empty_shell(ls, empty_shell_tablet_ids))) { - need_retry = true; - STORAGE_LOG(WARN, "update tablet to empty shell failed", KR(ret), "ls_id", ls->get_ls_id()); - } - if (need_retry) { - STORAGE_LOG(INFO, "[emptytablet] tablet become empty shell error, need retry", KR(ret), KPC(ls), K(empty_shell_tablet_ids)); - if (!tablet_empty_shell_handler->get_empty_shell_trigger()) { - tablet_empty_shell_handler->set_empty_shell_trigger(true); - } + ObTabletEmptyShellHandler *const tablet_empty_shell_handler = + ls->get_tablet_empty_shell_handler(); + if (tablet_empty_shell_handler->check_stop()) { + STORAGE_LOG(INFO, "[emptytablet] tablet_gc_handler is stop"); + } else if (0 == times || tablet_empty_shell_handler->get_empty_shell_trigger()) { + STORAGE_LOG(INFO, "[emptytablet] task check ls", K(tablet_empty_shell_handler)); + tablet_empty_shell_handler->set_empty_shell_trigger(false); + obsys::ObRLockGuard lock(tablet_empty_shell_handler->wait_lock_); + bool need_retry = false; + common::ObTabletIDArray empty_shell_tablet_ids; + if (OB_FAIL(tablet_empty_shell_handler->get_empty_shell_tablet_ids(empty_shell_tablet_ids, need_retry))) { + need_retry = true; + STORAGE_LOG(WARN, "[emptytablet] tablet_empty_shell_handler get empty shell tablet ids failed", K(ret)); + } else if (empty_shell_tablet_ids.empty()) { + // do nothing + } else if (OB_FAIL(tablet_empty_shell_handler->update_tablets_to_empty_shell(ls, empty_shell_tablet_ids))) { + need_retry = true; + STORAGE_LOG(WARN, "update tablet to empty shell failed", KR(ret)); + } + if (need_retry) { + STORAGE_LOG(INFO, "[emptytablet] tablet become empty shell error, need retry", KR(ret), KPC(ls), K(empty_shell_tablet_ids)); + if (!tablet_empty_shell_handler->get_empty_shell_trigger()) { + tablet_empty_shell_handler->set_empty_shell_trigger(true); } } } } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (ls_cnt > 0) { - STORAGE_LOG(INFO, "[emptytablet] succeed to change tablet to empty shell", KR(ret), K(ls_cnt), K(times)); - } else { - STORAGE_LOG(INFO, "[emptytablet] no logstream", KR(ret), K(ls_cnt), K(times)); - } - } + if (OB_SUCC(ret)) { + STORAGE_LOG(INFO, "[emptytablet] succeed to change tablet to empty shell", KR(ret), K(times)); } } @@ -217,12 +192,12 @@ int ObTabletEmptyShellHandler::update_tablets_to_empty_shell(ObLS *ls, const com for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { const ObTabletID &tablet_id = tablet_ids.at(i); if (OB_FAIL(ls->get_tablet_svr()->update_tablet_to_empty_shell(tablet_id))) { - STORAGE_LOG(WARN, "failed to update tablet to shell", K(ret), K(ls->get_ls_id()), K(tablet_id)); + STORAGE_LOG(WARN, "failed to update tablet to shell", K(ret), K(tablet_id)); } else if (OB_FAIL(ddl_empty_shell_checker_.erase_tablet_record(tablet_id))) { STORAGE_LOG(WARN, "erase ddl tablet record failed", K(ret)); } else { #ifdef ERRSIM - SERVER_EVENT_ADD("gc", "turn_into_empty_shell", "ls_id", ls->get_ls_id(), "tablet_id", tablet_id); + SERVER_EVENT_ADD("gc", "turn_into_empty_shell", "tablet_id", tablet_id); #endif } } @@ -234,7 +209,6 @@ int ObTabletEmptyShellHandler::check_candidate_tablet_(const ObTablet &tablet, b { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; can_become_shell = false; bool is_written = false; @@ -242,7 +216,7 @@ int ObTabletEmptyShellHandler::check_candidate_tablet_(const ObTablet &tablet, b if (tablet.is_empty_shell()) { // do nothing } else if (OB_FAIL(tablet.check_tablet_status_written(is_written))) { - STORAGE_LOG(WARN, "failed to check tablet status written", K(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(WARN, "failed to check tablet status written", K(ret), K(tablet_id)); } else { ObTabletCreateDeleteMdsUserData data; mds::MdsWriter writer; // will be removed later @@ -253,90 +227,32 @@ int ObTabletEmptyShellHandler::check_candidate_tablet_(const ObTablet &tablet, b if (OB_EMPTY_RESULT == ret) { ret = OB_SUCCESS; STORAGE_LOG(INFO, "tablet status is null, may be create tx is aborted or create user data has not been written", - K(ret), K(ls_id), K(tablet_id)); + K(ret), K(tablet_id)); if (!is_written) { // mds table has not been written, do nothing } else { can_become_shell = true; - STORAGE_LOG(INFO, "sys tenant tablet can become shell", KR(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(INFO, "sys tenant tablet can become shell", KR(ret), K(tablet_id)); } } else { - STORAGE_LOG(WARN, "failed to get latest tablet status", K(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(WARN, "failed to get latest tablet status", K(ret), K(tablet_id)); } } else if (mds::TwoPhaseCommitState::ON_COMMIT == trans_stat && data.tablet_status_.is_deleted_for_gc()) { - STORAGE_LOG(INFO, "delete tx is committed", K(ret), K(ls_id), K(tablet_id), K(trans_stat), K(data)); + STORAGE_LOG(INFO, "delete tx is committed", K(ret), K(tablet_id), K(trans_stat), K(data)); can_become_shell = true; - STORAGE_LOG(INFO, "sys tenant tablet can become shell", KR(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(INFO, "sys tenant tablet can become shell", KR(ret), K(tablet_id)); } } return ret; } -int ObTabletEmptyShellHandler::check_reserved_deleted_tablet_( - const ObTablet &tablet, - const ObTabletCreateDeleteMdsUserData &user_data, - bool &can, - bool &need_retry) -{ - int ret = OB_SUCCESS; - can = false; - SCN decided_scn; - ObMigrationStatus migration_status; - ObLSService *ls_service = NULL; - ObLSHandle ls_handle; - ObLS *ls = NULL; - const ObTabletID tablet_id(tablet.get_tablet_meta().tablet_id_); - if (!user_data.is_valid() || !user_data.reserved_ls_id_.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "arguments are invalid", K(ret), K(user_data)); - } else if (OB_ISNULL(ls_service = share::g_mp->ls_service())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls service should not be null", K(ret), KP(ls_service)); - } else if (OB_FAIL(ls_service->get_ls(user_data.reserved_ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { - if (OB_LS_NOT_EXIST == ret) { - can = true; - STORAGE_LOG(INFO, "reserved dest ls not exist, src tablet can become empty shell", K(ret), "ls_id", user_data.reserved_ls_id_, K(user_data)); - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "failed to get ls", K(ret), K(user_data)); - } - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls should not be NULL", K(ret), K(user_data)); - } else if (OB_FAIL(ls->get_migration_status(migration_status))) { - STORAGE_LOG(WARN, "ls is get migration status", K(ret),K(user_data)); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_ADD == migration_status - || ObMigrationStatus::OB_MIGRATION_STATUS_MIGRATE == migration_status - || ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD == migration_status) { - can = true; - STORAGE_LOG(INFO, "migration_status is OB_MIGRATION_STATUS_MIGRATE", K(ret), - "ls_id", ls_->get_ls_id(), K(tablet_id), K(can), K(user_data), K(migration_status)); - } else if (tablet.get_tablet_meta().ha_status_.is_expected_status_deleted()) { - can = true; - STORAGE_LOG(INFO, "tablet is expected status deleted", KR(ret), "tablet_meta", tablet.get_tablet_meta()); - } else if (OB_FAIL(ls->get_max_decided_scn(decided_scn))) { - STORAGE_LOG(WARN, "failed to get max decided scn", K(ret), K(user_data)); - } else if (decided_scn >= user_data.delete_commit_scn_) { - can = true; - STORAGE_LOG(INFO, "decided_scn is bigger than reserved finish scn", K(ret), - "ls_id", ls_->get_ls_id(), K(tablet_id), K(can), K(user_data), K(decided_scn)); - } else { - need_retry = true; - if (REACH_THREAD_TIME_INTERVAL(1_s)) { - STORAGE_LOG(INFO, "decided_scn is smaller than tablet delete commit scn", - "ls_id", ls_->get_ls_id(), K(tablet_id), K(user_data), K(decided_scn)); - } - } - return ret; -} - int ObTabletEmptyShellHandler::get_readable_scn(share::SCN &readable_scn) { int ret = OB_SUCCESS; - if (OB_FAIL(storage::get_sys_ls_readable_scn(readable_scn))) { + if (OB_FAIL(storage::get_ls_readable_scn(readable_scn))) { LOG_WARN("failed to get_max_decided_scn", KR(ret)); } return ret; @@ -350,7 +266,7 @@ bool ObTabletEmptyShellHandler::get_empty_shell_trigger() const void ObTabletEmptyShellHandler::set_empty_shell_trigger(bool is_trigger) { ATOMIC_STORE(&is_trigger_, is_trigger); - STORAGE_LOG(INFO, "[emptytablet] set empty shell trigger", "ls_id", ls_->get_ls_id(), K(is_trigger)); + STORAGE_LOG(INFO, "[emptytablet] set empty shell trigger", K(is_trigger)); } int ObTabletEmptyShellHandler::offline() diff --git a/src/storage/tx_storage/ob_empty_shell_task.h b/src/storage/tx_storage/ob_empty_shell_task.h index 1fd051687..1a411cd07 100644 --- a/src/storage/tx_storage/ob_empty_shell_task.h +++ b/src/storage/tx_storage/ob_empty_shell_task.h @@ -56,13 +56,6 @@ class ObTabletEmptyShellHandler int check_candidate_tablet_(const ObTablet &tablet, bool &can_become_shell, bool &need_retry); int get_empty_shell_tablet_ids(common::ObTabletIDArray &empty_shell_tablet_ids, bool &need_retry); int update_tablets_to_empty_shell(ObLS *ls, const common::ObIArray &tablet_ids); - // Conditions for a reserved deleted tablet status to become an empty shell in standby database(1 or 2 or 3 or 4): - // 1. Tenant-level replayable scn is greater than the finish_scn of reserved deleted tablet - // 2. The node dest_ls where the tablet resides does not exist; - // 3. The migration status of dest_ls is OB_MIGRATION_STATUS_MIGRATE; - // 4. The replay decided scn of dest_ls is greater than the finish_scn of reserved deleted tablet - int check_reserved_deleted_tablet_(const ObTablet &tablet, const ObTabletCreateDeleteMdsUserData &user_data, bool &can, bool &need_retry); - static int get_readable_scn(share::SCN &readable_scn); public: diff --git a/src/storage/tx_storage/ob_ls_handle.cpp b/src/storage/tx_storage/ob_ls_handle.cpp deleted file mode 100644 index 8fe569206..000000000 --- a/src/storage/tx_storage/ob_ls_handle.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX STORAGE - - -#include "ob_ls_handle.h" -#include "storage/tx_storage/ob_ls_map.h" - -namespace oceanbase -{ -namespace storage -{ -ObLSHandle::ObLSHandle() - : ls_map_(nullptr), - ls_(nullptr), - mod_(ObLSGetMod::INVALID_MOD) -{ - INIT_OBJ_LEAK_DEBUG_NODE(node_, this, share::LEAK_CHECK_OBJ_LS_HANDLE); -} - -ObLSHandle::ObLSHandle(const ObLSHandle &other) - : ls_map_(nullptr), - ls_(nullptr), - mod_(ObLSGetMod::INVALID_MOD) -{ - INIT_OBJ_LEAK_DEBUG_NODE(node_, this, share::LEAK_CHECK_OBJ_LS_HANDLE); - *this = other; -} - -ObLSHandle::~ObLSHandle() -{ - reset(); -} - -bool ObLSHandle::is_valid() const -{ - return (nullptr != ls_ && nullptr != ls_map_); -} - -ObLSHandle &ObLSHandle::operator=(const ObLSHandle &other) -{ - int ret = OB_SUCCESS; - if (&other != this) { - if (nullptr != ls_ && nullptr != ls_map_) { - reset(); - } - if (nullptr != other.ls_ && nullptr != other.ls_map_ && OB_SUCC(other.ls_->get_ref_mgr().inc(other.mod_))) { - ls_ = other.ls_; - ls_map_ = other.ls_map_; - mod_ = other.mod_; - } else { - LOG_WARN("ls assign fail", K(ret), K(other), K(ls_), K(ls_map_)); - } - } - return *this; -} - -int ObLSHandle::set_ls(const ObLSMap &ls_map, ObLS &ls, const ObLSGetMod &mod) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_SUCC(ls.get_ref_mgr().inc(mod))) { - ls_map_ = &ls_map; - ls_ = &ls; - mod_ = mod; - } - return ret; -} - -void ObLSHandle::reset() -{ - if (OB_NOT_NULL(ls_map_) && OB_NOT_NULL(ls_)) { - ls_map_->revert_ls(ls_, mod_); - ls_map_ = nullptr; - ls_ = nullptr; - mod_ = ObLSGetMod::INVALID_MOD; - } -} -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/tx_storage/ob_ls_handle.h b/src/storage/tx_storage/ob_ls_handle.h deleted file mode 100644 index 41e1b4f35..000000000 --- a/src/storage/tx_storage/ob_ls_handle.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_STORAGE_TX_STORAGE_OB_LS_HANDLE -#define OCEANBASE_STORAGE_TX_STORAGE_OB_LS_HANDLE - -#include "lib/utility/ob_print_utils.h" -#include "share/leak_checker/obj_leak_checker.h" -#include "storage/ls/ob_ls_get_mod.h" - -namespace oceanbase -{ -namespace storage -{ -class ObLSMap; -class ObLS; - -class ObLSHandle final -{ -public: - ObLSHandle(); - ObLSHandle(const ObLSHandle &other); - ObLSHandle &operator=(const ObLSHandle &other); - ~ObLSHandle(); - int set_ls(const ObLSMap &ls_map, ObLS &ls, const ObLSGetMod &mod); - void reset(); - bool is_valid() const; - ObLS *get_ls() { return ls_; } - ObLS *get_ls() const { return ls_; } - TO_STRING_KV(KP(ls_map_), KP(ls_), K(mod_)); -private: - const ObLSMap *ls_map_; - ObLS *ls_; - ObLSGetMod mod_; - DEFINE_OBJ_LEAK_DEBUG_NODE(node_); -}; -} // namespace storage -} // namespace oceanbase - -#endif // OCEANBASE_STORAGE_TX_STORAGE_OB_LS_HANDLE diff --git a/src/storage/tx_storage/ob_ls_map.cpp b/src/storage/tx_storage/ob_ls_map.cpp deleted file mode 100644 index e01ed1807..000000000 --- a/src/storage/tx_storage/ob_ls_map.cpp +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX STORAGE - - -#include "ob_ls_map.h" -#include "storage/tx_storage/ob_ls_service.h" - -using namespace oceanbase::share; -using namespace oceanbase::common; -using namespace oceanbase::common::hash; - -namespace oceanbase -{ -namespace storage -{ -// ------------------- ObLSIterator -------------------- // -ObLSIterator::ObLSIterator() - : lss_(), - bucket_pos_(0), - array_idx_(0), - ls_map_(NULL), - mod_(ObLSGetMod::INVALID_MOD) -{ -} - -ObLSIterator::~ObLSIterator() -{ - reset(); -} - -void ObLSIterator::reset() -{ - if (OB_NOT_NULL(ls_map_)) { - for (int64_t i = 0; i < lss_.count(); ++i) { - ls_map_->revert_ls(lss_.at(i), mod_); - } - lss_.reuse(); - bucket_pos_ = 0; - array_idx_ = 0; - mod_ = ObLSGetMod::INVALID_MOD; - } -} - -int ObLSIterator::get_next(ObLS *&ls) -{ - int ret = OB_SUCCESS; - - if (OB_ISNULL(ls_map_)) { - ret = OB_NOT_INIT; - LOG_WARN("The ls service is NULL, ", K(ret)); - } else { - while(OB_SUCC(ret)) { - if (array_idx_ < lss_.count()) { - ls = lss_.at(array_idx_); - array_idx_++; - break; - } else { - if (lss_.count() > 0) { - for (int64_t i = 0; i < lss_.count(); ++i) { - ls_map_->revert_ls(lss_.at(i), mod_); - } - lss_.reuse(); - } - array_idx_ = 0; - - if (OB_ISNULL(ls_map_->ls_buckets_) || bucket_pos_ >= ls_map_->BUCKETS_CNT) { - ret = OB_ITER_END; - } else { - if (OB_NOT_NULL(ls_map_->ls_buckets_[bucket_pos_])) { - ObQSyncLockReadGuard guard(ls_map_->buckets_lock_[bucket_pos_]); - ls = ls_map_->ls_buckets_[bucket_pos_]; - - while (OB_NOT_NULL(ls) && OB_SUCC(ret)) { - if (OB_FAIL(ls->get_ref_mgr().inc(mod_))) { - LOG_WARN("ls inc ref fail", K(ret)); - } else if (OB_FAIL(lss_.push_back(ls))) { - LOG_WARN("Fail to push ls to array, ", K(ret)); - ls->get_ref_mgr().dec(mod_); - } else { - ls = (ObLS *)ls->next_; - } - } - } - bucket_pos_++; - } - } - } - } - return ret; -} - -// ------------------- ObLSMap -------------------- // -void ObLSMap::reset() -{ - if (OB_NOT_NULL(ls_buckets_)) { - ObLS *ls = nullptr; - ObLS *next_ls = nullptr; - for (int64_t i = 0; i < BUCKETS_CNT; ++i) { - ls = ls_buckets_[i]; - while (OB_NOT_NULL(ls)) { - next_ls = (ObLS *)ls->next_; - ls->get_ref_mgr().set_delete(); - // here mod muse the same as add_ls - revert_ls(ls, ObLSGetMod::TXSTORAGE_MOD); - ls = next_ls; - } - } - ob_free(ls_buckets_); - ls_buckets_ = NULL; - } - if (OB_NOT_NULL(buckets_lock_)) { - for (int64_t i = 0; i < BUCKETS_CNT; ++i) { - if (OB_NOT_NULL(buckets_lock_ + i)) { - (buckets_lock_ + i)->~ObQSyncLock(); - } - } - ob_free(buckets_lock_); - buckets_lock_ = nullptr; - } - ls_cnt_ = 0; - - ls_allocator_ = nullptr; - is_inited_ = false; -} - -int ObLSMap::init(ObIAllocator *ls_allocator) -{ - int ret = OB_SUCCESS; - const char *OB_LS_MAP = "LSMap"; - ObMemAttr mem_attr(OB_LS_MAP); - void *buf = NULL; - - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("ObLSMap init twice", K(ret)); - } else if (OB_ISNULL(ls_allocator)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret)); - } else if (OB_ISNULL(buckets_lock_ = (common::ObQSyncLock*)ob_malloc(sizeof(common::ObQSyncLock) * BUCKETS_CNT, mem_attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_ISNULL(buf = ob_malloc(sizeof(ObLS*) * BUCKETS_CNT, mem_attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("Fail to allocate memory, ", K(ret), LITERAL_K(BUCKETS_CNT)); - } else { - for (int64_t i = 0 ; i < BUCKETS_CNT; ++i) { - new(buckets_lock_ + i) common::ObQSyncLock(); - if (OB_FAIL((buckets_lock_ + i)->init(mem_attr))) { - LOG_WARN("buckets_lock_ init fail", K(ret)); - for (int64_t j = 0 ; j <= i; ++j) { - (buckets_lock_ + j)->destroy(); - } - ob_free(buf); - buf = NULL; - ob_free(buckets_lock_); - buckets_lock_ = NULL; - break; - } - } - if (OB_LIKELY(ret == common::OB_SUCCESS)) { - MEMSET(buf, 0, sizeof(ObLS*) * BUCKETS_CNT); - ls_buckets_ = new (buf) ObLS*[BUCKETS_CNT]; - - ls_allocator_ = ls_allocator; - is_inited_ = true; - } - } - - return ret; -} - -void ObLSMap::destroy() -{ - reset(); -} - -int ObLSMap::add_ls( - ObLS &ls) -{ - int ret = OB_SUCCESS; - ObLS *prev = NULL; - ObLS *curr = NULL; - const ObLSID &ls_id = ls.get_ls_id(); - LOG_INFO("ls map add ls", - K(ls_id), KP(&ls), "ref", ls.get_ref_mgr().get_total_ref_cnt()); - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLSMap not init", K(ret), K(ls_id)); - } else { - int64_t pos = ls_id.hash() % BUCKETS_CNT; - ObQSyncLockWriteGuard guard(buckets_lock_[pos]); - curr = ls_buckets_[pos]; - while (OB_NOT_NULL(curr)) { - if (curr->get_ls_id() == ls_id) { - break; - } else { - prev = curr; - curr = static_cast(curr->next_); - } - } - - if (OB_ISNULL(curr)) { - int64_t cnt = ATOMIC_AAF(&ls_cnt_, 1); - if (OB_FAIL(ls.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD))) { - ATOMIC_DEC(&ls_cnt_); - LOG_WARN("ls inc ref fail", K(ret), K(ls_id)); - } else { - ls.next_ = ls_buckets_[pos]; - ls_buckets_[pos] = &ls; - } - } else { - ret = OB_ENTRY_EXIST; - } - } - - LOG_INFO("ls map finish add ls", - K(ls_id), KP(&ls), "ref", ls.get_ref_mgr().get_total_ref_cnt(), K(ret)); - return ret; -} - -int ObLSMap::del_ls(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLS *prev = NULL; - ObLS *ls = NULL; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLSMap not init", K(ret), K(ls_id)); - } else { - int64_t pos = ls_id.hash() % BUCKETS_CNT; - // deal with the deadlock problem: - // - { - //remove ls from map - ObQSyncLockWriteGuard guard(buckets_lock_[pos]); - ls = ls_buckets_[pos]; - while (OB_NOT_NULL(ls)) { - if (ls->get_ls_id() == ls_id) { - break; - } else { - prev = ls; - ls = (ObLS *)ls->next_; - } - } - - if (OB_ISNULL(ls)) { - ret = OB_LS_NOT_EXIST; - } else { - LOG_INFO("ls service del ls", K(ls_id), - KP(ls), "ref", ls->get_ref_mgr().get_total_ref_cnt()); - if (OB_ISNULL(prev)) { - ls_buckets_[pos] = (ObLS *)ls->next_; - } else { - prev->next_ = ls->next_; - } - ls->next_ = NULL; - } - } - del_ls_impl(ls); - } - - return ret; -} - -void ObLSMap::del_ls_impl(ObLS *ls) -{ - int ret = OB_SUCCESS; - if (nullptr != ls) { - const ObLSID &ls_id = ls->get_ls_id(); - ATOMIC_DEC(&ls_cnt_); - ls->get_ref_mgr().set_delete(); - // here mod must the same with add_ls - revert_ls(ls, ObLSGetMod::TXSTORAGE_MOD); - } -} - -int ObLSMap::get_ls(const share::ObLSID &ls_id, - ObLSHandle &handle, - ObLSGetMod mod) const -{ - int ret = OB_SUCCESS; - ObLS *ls = NULL; - int64_t pos = 0; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLSMap not init", K(ret), K(ls_id)); - } else { - pos = ls_id.hash() % BUCKETS_CNT; - ObQSyncLockReadGuard bucket_guard(buckets_lock_[pos]); - ls = ls_buckets_[pos]; - while (OB_NOT_NULL(ls)) { - if (ls->get_ls_id() == ls_id) { - break; - } else { - ls = static_cast(ls->next_); - } - } - - if (OB_ISNULL(ls)) { - ret = OB_LS_NOT_EXIST; - } else if (OB_FAIL(handle.set_ls(*this, *ls, mod))) { - LOG_WARN("get_ls fail", K(ret), K(ls_id)); - } - } - return ret; -} - -int ObLSMap::get_all_ls_id(ObIArray &ls_id_array) -{ - int ret = OB_SUCCESS; - ObLS *ls = NULL; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObLSMap not init", K(ret)); - } else { - for (int64_t bucket_idx = 0 ; OB_SUCC(ret) && bucket_idx < BUCKETS_CNT; ++bucket_idx) { - ObQSyncLockReadGuard bucket_guard(buckets_lock_[bucket_idx]); - ls = ls_buckets_[bucket_idx]; - while (OB_SUCC(ret) && OB_NOT_NULL(ls)) { - if (OB_FAIL(ls_id_array.push_back(ls->get_ls_id()))) { - LOG_WARN("failed to push back ls id", K(ret), KP(ls)); - } else { - ls = static_cast(ls->next_); - } - } // end of while - } // end of for - } - return ret; -} - -} // end namespace storage -}; // end namespace oceanbase diff --git a/src/storage/tx_storage/ob_ls_map.h b/src/storage/tx_storage/ob_ls_map.h deleted file mode 100644 index 0c2e3d018..000000000 --- a/src/storage/tx_storage/ob_ls_map.h +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_STORAGE_OB_LS_MAP_ -#define OCEANBASE_STORAGE_OB_LS_MAP_ - -#include "lib/oblog/ob_log_module.h" -#include "lib/allocator/ob_concurrent_fifo_allocator.h" -#include "lib/container/ob_iarray.h" -#include "lib/lock/ob_qsync_lock.h" -#include "storage/ls/ob_ls.h" -#include "share/leak_checker/obj_leak_checker.h" - -namespace oceanbase -{ -namespace storage -{ -class ObLS; -class ObLSHandle; - -class ObLSMap -{ -public: - friend class ObLSIterator; - ObLSMap() - : is_inited_(false), - ls_allocator_(nullptr), - ls_cnt_(0), - ls_buckets_(nullptr), - buckets_lock_(nullptr) - { - reset(); - } - ~ObLSMap() { destroy(); } - void reset(); - int init(common::ObIAllocator *ls_allocator); - void destroy(); - // allow_multi_true is used during replay - int add_ls(ObLS &ls); - int del_ls(const share::ObLSID &ls_id); - int get_all_ls_id(ObIArray &ls_id_array); - int get_ls(const share::ObLSID &ls_id, - ObLSHandle &handle, - ObLSGetMod mod) const; - OB_INLINE void revert_ls(ObLS *ls, ObLSGetMod mod) const; - template - int operate_ls(const share::ObLSID &ls_id, Function &fn); - bool is_empty() const { return 0 == ATOMIC_LOAD(&ls_cnt_); } - int64_t get_ls_count() const { return ATOMIC_LOAD(&ls_cnt_); } - static TCRef &get_tcref() - { - static TCRef tcref(16); - return tcref; - } -private: - OB_INLINE void free_ls(ObLS *ls) const; - void del_ls_impl(ObLS *ls); - int choose_preserve_ls(ObLS *left_ls, ObLS *right_ls, ObLS *&result_ls); - int remove_duplicate_ls_in_linklist(ObLS *&head); -private: - static const bool ENABLE_RECOVER_ALL_ZONE = false; - bool is_inited_; - - common::ObIAllocator *ls_allocator_; - // total ls in current server of current tenant - int64_t ls_cnt_; - ObLS **ls_buckets_; - const static int64_t BUCKETS_CNT = 1 << 8; - common::ObQSyncLock *buckets_lock_; -}; - -//iterate all lss -class ObLSIterator -{ -public: - ObLSIterator(); - virtual ~ObLSIterator(); - virtual int get_next(ObLS *&ls); - void reset(); - void set_ls_map(ObLSMap &ls_map, ObLSGetMod mod) { - ls_map_ = &ls_map; - mod_ = mod; - } - TO_STRING_KV("ls_count", lss_.count(), K_(bucket_pos), K_(array_idx)); -private: - common::ObArray lss_; - int64_t bucket_pos_; - int64_t array_idx_; - ObLSMap *ls_map_; - ObLSGetMod mod_; -}; - -OB_INLINE void ObLSMap::free_ls(ObLS *ls) const -{ - int ret = OB_SUCCESS; - ls->~ObLS(); - ls_allocator_->free(ls); -} - -OB_INLINE void ObLSMap::revert_ls(ObLS *ls, ObLSGetMod mod) const -{ - if (OB_NOT_NULL(ls)) { - if (ls->get_ref_mgr().dec(mod)) { - STORAGE_LOG(INFO, "ObLSMap free ls", KP(ls), K(mod), K(ls->get_ls_id())); - free_ls(ls); - } - } -} - -template -int ObLSMap::operate_ls(const share::ObLSID &ls_id, - Function &fn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObLSMap not init", K(ret), K(ls_id)); - } else { - const int64_t pos = ls_id.hash() % BUCKETS_CNT; - ObLS *ls = ls_buckets_[pos]; - common::ObQSyncLockReadGuard bucket_guard(buckets_lock_[pos]); - while (OB_NOT_NULL(ls)) { - if (ls->get_ls_id() == ls_id) { - break; - } else { - ls = static_cast(ls->next_); - } - } - if (OB_ISNULL(ls)) { - ret = OB_LS_NOT_EXIST; - } else { - ret = fn(ls_id, ls); - } - } - return ret; -} - -} -} -#endif // OCEANBASE_STORAGE_OB_LS_MAP_ diff --git a/src/storage/tx_storage/ob_ls_safe_destroy_task.cpp b/src/storage/tx_storage/ob_ls_safe_destroy_task.cpp deleted file mode 100644 index f4ef11ab6..000000000 --- a/src/storage/tx_storage/ob_ls_safe_destroy_task.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX STORAGE -#include "storage/tx_storage/ob_ls_safe_destroy_task.h" -#include "storage/tx_storage/ob_ls_service.h" - -namespace oceanbase -{ -using namespace share; -namespace storage -{ - -ObLSSafeDestroyTask::ObLSSafeDestroyTask() - : is_inited_(false), - ls_handle_(), - ls_service_(nullptr) -{ - type_ = ObSafeDestroyTask::ObSafeDestroyTaskType::LS; -} - -ObLSSafeDestroyTask::~ObLSSafeDestroyTask() -{ - destroy(); -} - - -bool ObLSSafeDestroyTask::safe_to_destroy() -{ - int ret = OB_SUCCESS; - bool is_safe = false; - ObLS *ls = nullptr; - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - // there is no ls protected. safe to destroy - if (IS_NOT_INIT) { - is_safe = true; - } else if (OB_FAIL(guard.switch_to(false))) { - LOG_WARN("switch tenant failed", K(ret)); - } else if (OB_UNLIKELY(!ls_handle_.is_valid())) { - is_safe = true; - } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret)); - // the ls is not safe to destroy. - } else if (OB_LIKELY(!ls->safe_to_destroy())) { - // do nothing - } else { - is_safe = true; - } - return is_safe; -} - -void ObLSSafeDestroyTask::destroy() -{ - int ret = OB_SUCCESS; - // 1. reset ls_handle. - // 2. tell the ls service, a ls is released. - if (is_inited_) { - // the ls tablet reset may use some MTL component we must switch to this tenant. - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - if (OB_FAIL(guard.switch_to(false))) { - LOG_WARN("switch to tenant failed, we cannot release the ls, because it is may core", - K(ret), KPC(this)); - } else { - ls_handle_.reset(); - ls_service_->dec_ls_safe_destroy_task_cnt(); - ls_service_ = nullptr; - is_inited_ = false; - } - } -} - -int ObLSSafeDestroyTask::get_ls_id(ObLSID &ls_id) const -{ - int ret = OB_SUCCESS; - ObLS *ls = nullptr; - MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); - // there is no ls protected. safe to destroy - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("task is not inited", K(ret)); - } else if (OB_FAIL(guard.switch_to(false))) { - LOG_WARN("switch tenant failed", K(ret)); - } else if (OB_UNLIKELY(!ls_handle_.is_valid())) { - LOG_WARN("ls handle invalid", K(ret), K_(ls_handle)); - } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret)); - // the ls is not safe to destroy. - } else { - ls_id = ls->get_ls_id(); - } - return ret; -} - - -} // storage -} // oceanbase diff --git a/src/storage/tx_storage/ob_ls_safe_destroy_task.h b/src/storage/tx_storage/ob_ls_safe_destroy_task.h deleted file mode 100644 index 35ea4d340..000000000 --- a/src/storage/tx_storage/ob_ls_safe_destroy_task.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEABASE_STORAGE_OB_LS_SAFE_DESTROY_TASK_ -#define OCEABASE_STORAGE_OB_LS_SAFE_DESTROY_TASK_ - -#include "storage/tx_storage/ob_ls_map.h" -#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle -#include "storage/tx_storage/ob_safe_destroy_handler.h" - -namespace oceanbase -{ -namespace storage -{ -class ObLSSafeDestroyTask : public ObSafeDestroyTask -{ -public: - ObLSSafeDestroyTask(); - ~ObLSSafeDestroyTask(); - virtual bool safe_to_destroy() override; - virtual void destroy() override; - int get_ls_id(share::ObLSID &ls_id) const; - INHERIT_TO_STRING_KV("ObSafeDestroyTask", ObSafeDestroyTask, K_(is_inited), - K_(ls_handle), KP_(ls_service)); -private: - bool is_inited_; - // used to switch to the tenant to make sure destroy process is right. - // contain the ls need to check. - ObLSHandle ls_handle_; - // the ls service, if a ls is not safe to destroy the ls service - // should not safe to destroy too. - ObLSService *ls_service_; -}; - -class ObSafeDestroyCheckLSExist -{ -public: - explicit ObSafeDestroyCheckLSExist(const share::ObLSID &ls_id) - : err_code_(common::OB_SUCCESS), - exist_(false), - ls_id_(ls_id) - {} - int is_exist() const { return exist_; } - int get_ret_code() const { return err_code_; } -private: - int err_code_; - bool exist_; - share::ObLSID ls_id_; -}; - -} // storage -} // oceanbase -#endif diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index 654c267cf..49215e5c0 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -22,7 +22,6 @@ #include "share/ls/ob_restore_status.h" #include "logservice/ob_log_service.h" #include "observer/ob_srv_network_frame.h" -#include "storage/tx_storage/ob_ls_safe_destroy_task.h" #include "storage/tx/ob_trans_service.h" #include "storage/meta_store/ob_server_storage_meta_service.h" #include "storage/meta_store/ob_tenant_storage_meta_service.h" @@ -48,15 +47,11 @@ ObLSService::ObLSService() : is_inited_(false), is_running_(false), is_stopped_(false), - ls_map_(), + ls_(nullptr), ls_allocator_(), - iter_allocator_(), change_lock_(common::ObLatchIds::LS_CHANGE_LOCK), storage_svr_rpc_proxy_(), - storage_rpc_(), - safe_ls_destroy_task_cnt_(0), - iter_cnt_(0), - max_ls_cnt_(0) + storage_rpc_() {} ObLSService::~ObLSService() @@ -68,56 +63,26 @@ void ObLSService::destroy() { int ret = OB_SUCCESS; LOG_INFO("destroy ls service", KP(this)); - if (is_running_ || !is_stopped_) { + if (!is_inited_) { + return; + } else if (is_running_ || !is_stopped_) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("should has been stopped before destroy", K(ret), K_(is_running), K_(is_stopped), KP(this)); } - if (ATOMIC_LOAD(&iter_cnt_) != 0) { + if (OB_NOT_NULL(ls_)) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls iter cnt is not 0", K(ret), K_(iter_cnt), KP(this)); + LOG_ERROR("ls service still owns log stream while destroying", K(ret), KP_(ls)); + } + OB_ASSERT(OB_SUCCESS == ret); + if (OB_SUCCESS != ret) { + return; } - ls_map_.reset(); ls_allocator_.destroy(); - iter_allocator_.destroy(); storage_svr_rpc_proxy_.destroy(); storage_rpc_.destroy(); - max_ls_cnt_ = 0; is_inited_ = false; } -bool ObLSService::is_empty() -{ - bool is_safe = (ls_map_.is_empty() && - ATOMIC_LOAD(&safe_ls_destroy_task_cnt_) == 0); - if (!is_safe && REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - bool is_t3m_meta_released = false; - share::g_mp->tenant_meta_mem_mgr()->check_all_meta_mem_released(is_t3m_meta_released, "ObLSService"); //just for debug - LOG_INFO("ls service is not empty and not safe to destroy", K(ls_map_.is_empty()), - K_(safe_ls_destroy_task_cnt), K(is_t3m_meta_released)); - } - return is_safe; -} - -void ObLSService::inc_ls_safe_destroy_task_cnt() -{ - ATOMIC_INC(&safe_ls_destroy_task_cnt_); -} - -void ObLSService::dec_ls_safe_destroy_task_cnt() -{ - ATOMIC_DEC(&safe_ls_destroy_task_cnt_); -} - -void ObLSService::inc_iter_cnt() -{ - ATOMIC_INC(&iter_cnt_); -} - -void ObLSService::dec_iter_cnt() -{ - ATOMIC_DEC(&iter_cnt_); -} - int ObLSService::get_resource_constraint_value(ObResoureConstraintValue &constraint_value) { int ret = OB_SUCCESS; @@ -137,30 +102,10 @@ int ObLSService::get_resource_constraint_value(ObResoureConstraintValue &constra int ObLSService::get_resource_constraint_value_(ObResoureConstraintValue &constraint_value) { int ret = OB_SUCCESS; - int64_t config_value = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER; - int64_t memory_value = INT64_MAX; - int64_t clog_disk_value = INT64_MAX; - // 1. configuration - if (OB_LIKELY(true)) { - config_value = (GCONF._max_ls_cnt_per_server != 0 - ? GCONF._max_ls_cnt_per_server : config_value); - } - - // 2. memory - const int64_t tenant_memory = lib::get_tenant_memory_limit(); - memory_value = OB_MAX(tenant_memory - SMALL_TENANT_MEMORY_LIMIT, 0) / TENANT_MEMORY_PER_LS_NEED + - OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; - - // 3. clog disk - palf::PalfOptions palf_opts; - if (OB_FAIL(share::g_mp->log_service()->get_palf_options(palf_opts))) { - LOG_WARN("get palf options failed", K(ret)); - } else { - const palf::PalfDiskOptions &disk_opts = palf_opts.disk_options_; - clog_disk_value = disk_opts.log_disk_usage_limit_size_ / MIN_DISK_SIZE_PER_PALF_INSTANCE; - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(constraint_value.set_type_value(CONFIGURATION_CONSTRAINT, config_value))) { + const int64_t config_value = 1; + const int64_t memory_value = 1; + const int64_t clog_disk_value = 1; + if (OB_FAIL(constraint_value.set_type_value(CONFIGURATION_CONSTRAINT, config_value))) { LOG_WARN("set_type_value failed", K(ret), K(CONFIGURATION_CONSTRAINT), K(config_value)); } else if (OB_FAIL(constraint_value.set_type_value(MEMORY_CONSTRAINT, memory_value))) { LOG_WARN("set_type_value failed", K(ret), K(MEMORY_CONSTRAINT), K(memory_value)); @@ -184,8 +129,8 @@ int ObLSService::get_current_info(share::ObResourceInfo &info) } else if (OB_FAIL(get_resource_constraint_value_(constraint_value))) { LOG_WARN("get resource constraint value failed", K(ret)); } else { - info.curr_utilization_ = ls_map_.get_ls_count() + ATOMIC_LOAD(&safe_ls_destroy_task_cnt_); - info.max_utilization_ = ATOMIC_LOAD(&max_ls_cnt_); + info.curr_utilization_ = 1; + info.max_utilization_ = info.curr_utilization_; info.reserved_value_ = 0; // reserve value will be used later constraint_value.get_min_constraint(info.min_constraint_type_, info.min_constraint_value_); } @@ -202,8 +147,7 @@ int ObLSService::cal_min_phy_resource_needed(share::ObMinPhyResourceResult &min_ ret = OB_NOT_RUNNING; LOG_WARN("the ls service is not running ,the resource info may not right.", K(ret)); } else { - int64_t ls_cnt = ls_map_.get_ls_count() + ATOMIC_LOAD(&safe_ls_destroy_task_cnt_); - ret = cal_min_phy_resource_needed_(ls_cnt, min_phy_res); + ret = cal_min_phy_resource_needed_(min_phy_res); } return ret; } @@ -211,27 +155,15 @@ int ObLSService::cal_min_phy_resource_needed(share::ObMinPhyResourceResult &min_ int ObLSService::cal_min_phy_resource_needed(const int64_t num, ObMinPhyResourceResult &min_phy_res) { - int ret = OB_SUCCESS; - ret = cal_min_phy_resource_needed_(num, min_phy_res); - return ret; + UNUSED(num); + return cal_min_phy_resource_needed_(min_phy_res); } -int ObLSService::cal_min_phy_resource_needed_(const int64_t num, - ObMinPhyResourceResult &min_phy_res) +int ObLSService::cal_min_phy_resource_needed_(ObMinPhyResourceResult &min_phy_res) { int ret = OB_SUCCESS; - int64_t ls_cnt = num; - int64_t clog_disk_bytes = 0; - int64_t memory_bytes = 0; - // 1. memory - // if the ls num is smaller than OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT, - // just return SMALL_TENANT_MEMORY_LIMIT. - memory_bytes = (SMALL_TENANT_MEMORY_LIMIT - + OB_MAX(ls_cnt - OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT, 0) - * TENANT_MEMORY_PER_LS_NEED); - memory_bytes = ls_cnt > 0 ? memory_bytes : 0; - // 2. clog disk - clog_disk_bytes = MIN_DISK_SIZE_PER_PALF_INSTANCE * OB_MAX(0, ls_cnt); + const int64_t memory_bytes = SMALL_TENANT_MEMORY_LIMIT; + const int64_t clog_disk_bytes = MIN_DISK_SIZE_PER_PALF_INSTANCE; if (OB_FAIL(min_phy_res.set_type_value(PHY_RESOURCE_MEMORY, memory_bytes))) { LOG_WARN("set type value failed", K(PHY_RESOURCE_MEMORY), K(memory_bytes)); @@ -250,31 +182,18 @@ int ObLSService::stop() } else if (!is_running_ || is_stopped_) { // do nothing } else { - // remove all the ls from ls map and push it into the - // safe to destroy thread. - common::ObSharedGuard ls_iter; - ObLSHandle handle; ObLS *ls = nullptr; const bool remove_from_disk = false; lib::ObMutexGuard change_guard(change_lock_); - if (OB_FAIL(get_ls_iter(ls_iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_ERROR("fail to get next ls", K(ret)); - } - } else if (OB_FAIL(handle.set_ls(ls_map_, *ls, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("get ls handle failed", K(ret), KPC(ls)); - } else if (OB_FAIL(safe_remove_ls_(handle, remove_from_disk))) { - LOG_WARN("safe remove ls failed", K(ret), KPC(ls)); - } - } - if (OB_ITER_END == ret) { + if (OB_FAIL(get_ls(ls))) { + if (OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls", K(ret)); } + } else if (OB_FAIL(stop_and_remove_ls_(ls, remove_from_disk))) { + LOG_WARN("safe remove ls failed", K(ret), KPC(ls)); } is_running_ = false; is_stopped_ = true; @@ -285,25 +204,8 @@ int ObLSService::stop() int ObLSService::wait() { - int ret = OB_SUCCESS; - int64_t retry_times = 0; - int64_t begin_time = ObTimeUtility::current_time(); - while(!is_empty()) { - ++retry_times; - if (retry_times % 100 == 0) { - LOG_WARN("ls service wait empty for too much time", K(retry_times), K(begin_time)); - } - ob_usleep(100 * 1000); // 100 ms - } - retry_times = 0; - while(ATOMIC_LOAD(&iter_cnt_) != 0) { - ++retry_times; - if (retry_times % 100 == 0) { - LOG_WARN("ls service wait ls iter for too much time", K(retry_times), K_(iter_cnt), K(begin_time)); - } - ob_usleep(100 * 1000); // 100 ms - } - return ret; + free_ls_(ls_); + return OB_SUCCESS; } int ObLSService::mtl_init(ObLSService* &ls_service) @@ -317,26 +219,15 @@ int ObLSService::init() { int ret = OB_SUCCESS; const char *OB_LS_SERVICE = "LSSvr"; - const char *OB_LS_ITER = "LSIter"; const int64_t LS_ALLOC_TOTAL_LIMIT = 1024 * 1024 * 1024; - const int64_t ITER_ALLOC_TOTAL_LIMIT = 1024 * 1024 * 1024; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ls service is inited.", K_(is_inited), K(ret)); - } else if (!true) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret)); } else if (OB_FAIL(ls_allocator_.init(common::OB_MALLOC_NORMAL_BLOCK_SIZE, OB_LS_SERVICE, LS_ALLOC_TOTAL_LIMIT))) { LOG_WARN("fail to init ls allocator, ", K(ret)); - } else if (OB_FAIL(iter_allocator_.init(common::OB_MALLOC_NORMAL_BLOCK_SIZE, - OB_LS_ITER, - ITER_ALLOC_TOTAL_LIMIT))) { - LOG_WARN("fail to init iter allocator, ", K(ret)); - } else if (OB_FAIL(ls_map_.init(&ls_allocator_))) { - LOG_WARN("fail to init ls map", K(ret)); } else if (OB_FAIL(storage_svr_rpc_proxy_.init(GCTX.self_addr()))) { LOG_WARN("failed to init storage svr rpc proxy", K(ret)); } else if (OB_FAIL(storage_rpc_.init(&storage_svr_rpc_proxy_, GCTX.self_addr()))) { @@ -344,6 +235,11 @@ int ObLSService::init() } else { is_inited_ = true; } + if (OB_FAIL(ret)) { + ls_allocator_.destroy(); + storage_svr_rpc_proxy_.destroy(); + storage_rpc_.destroy(); + } return ret; } @@ -363,15 +259,10 @@ int ObLSService::start() return ret; } -int ObLSService::inner_create_ls_(const share::ObLSID &lsid, - const ObMigrationStatus &migration_status, - const ObRestoreStatus &restore_status, - const SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info, +int ObLSService::inner_create_ls_(const SCN &create_scn, ObLS *&ls) { int ret = OB_SUCCESS; - const char *OB_LS_MODE = "ObLS"; ObMemAttr memattr(OB_LS_MODE); void *buf = NULL; @@ -380,12 +271,9 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, LOG_WARN("failed to alloc ls", K(ret)); } else if (FALSE_IT(ls = new (buf) ObLS())) { - } else if (OB_FAIL(ls->init(lsid, - migration_status, - restore_status, - create_scn, - major_mv_merge_info))) { - LOG_WARN("fail to init ls", K(ret), K(lsid)); + } else if (OB_FAIL(ls->init(ObRestoreStatus(ObRestoreStatus::Status::NONE), + create_scn))) { + LOG_WARN("fail to init ls", K(ret)); } if (OB_FAIL(ret) && NULL != ls) { ls->~ObLS(); @@ -395,140 +283,64 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, return ret; } -int ObLSService::inner_del_ls_(ObLS *&ls) +void ObLSService::free_ls_(ObLS *ls) { - int ret = OB_SUCCESS; - if (OB_ISNULL(ls)) { - // do nothing - } else { + if (OB_NOT_NULL(ls)) { + if (ls_ == ls) { + ls_ = nullptr; + } ls->~ObLS(); ls_allocator_.free(ls); - ls = NULL; } - - return ret; } -int ObLSService::add_ls_to_map_(ObLS *ls) +int ObLSService::publish_ls_(ObLS *ls) { int ret = OB_SUCCESS; - if (OB_FAIL(ls_map_.add_ls(*ls))) { - LOG_WARN("add ls failed.", K(ret), K(ls->get_ls_id())); + if (OB_ISNULL(ls)) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_NOT_NULL(ls_)) { + ret = OB_INIT_TWICE; + LOG_WARN("the tenant log stream already exists", K(ret), KP(ls), KP_(ls)); } else { - // update the max ls cnt - const int64_t normal_ls_count = ls_map_.get_ls_count(); - const int64_t removeing_ls_count = ATOMIC_LOAD(&safe_ls_destroy_task_cnt_); - inc_update(&max_ls_cnt_, normal_ls_count + removeing_ls_count); - } - return ret; -} - -int ObLSService::remove_ls_from_map_(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ls_map_.del_ls(ls_id)) && - OB_LS_NOT_EXIST != ret) { - LOG_ERROR("delete ls from map failed", K(ret), K(ls_id)); - } - if (OB_LS_NOT_EXIST == ret) { - ret = OB_SUCCESS; + ls_ = ls; + LOG_INFO("publish tenant log stream", KP(ls)); } return ret; } -int ObLSService::create_ls(const ObTenantRole &tenant_role) +int ObLSService::create_ls() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; LOG_INFO("create_ls begin"); DEBUG_SYNC(BEFORE_CREATE_USER_LS); - ObCreateLSCommonArg arg; - arg.tenant_role_ = tenant_role; - arg.restore_status_ = ObRestoreStatus(ObRestoreStatus::Status::NONE); - arg.create_scn_ = SCN::base_scn(); - arg.create_type_ = ObLSCreateType::NORMAL; - arg.need_create_inner_tablet_ = true; - if (OB_FAIL(create_ls_(arg))) { + if (OB_FAIL(create_ls_())) { LOG_WARN("create ls failed", K(ret)); } - if (OB_LS_EXIST == ret) { - // new create ls can not be gc because it does not has memberlist. - // we need reuse the ls if it exist. - ret = OB_SUCCESS; - } FLOG_INFO("create_ls finish", K(ret)); return ret; } -int ObLSService::create_ls_for_ha() +int ObLSService::post_create_ls_(ObLS *ls) { int ret = OB_SUCCESS; - ObCreateLSCommonArg arg; - arg.tenant_role_ = RESTORE_TENANT_ROLE; - arg.restore_status_ = ObRestoreStatus(ObRestoreStatus::Status::RESTORE_DOING); - arg.create_scn_ = SCN::min_scn(); - arg.create_type_ = ObLSCreateType::RESTORE; - arg.need_create_inner_tablet_ = false; - if (OB_FAIL(create_ls_(arg))) { - LOG_WARN("create sys ls for ha failed", KR(ret), K(arg.restore_status_), K(arg.create_scn_)); - } - FLOG_INFO("create_ls_for_ha finish", K(ret)); - return ret; -} - -int ObLSService::post_create_ls_(const int64_t create_type, - ObLS *&ls) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; bool need_online = false; if (OB_FAIL(ls->check_ls_need_online(need_online))) { LOG_WARN("check ls need online failed", K(ret)); } else if (need_online && OB_FAIL(ls->online_without_lock())) { LOG_ERROR("ls start failed", K(ret)); - } else { - switch(create_type) { - case ObLSCreateType::NORMAL: { - if (OB_FAIL(ls->set_start_work_state())) { - LOG_ERROR("ls set start work state failed", KR(ret), KPC(ls)); - } else { - ls->enable_to_read(); - } - break; - } - case ObLSCreateType::RESTORE: { - if (OB_FAIL(ls->set_start_ha_state())) { - LOG_ERROR("ls set start ha state failed", KR(ret), KPC(ls)); - } else if (OB_FAIL(ls->set_ls_allow_to_read())) { - LOG_ERROR("failed to set ls allow to read", KR(ret), KPC(ls)); - } - break; - } - case ObLSCreateType::MIGRATE: { - if (OB_FAIL(ls->set_start_ha_state())) { - LOG_ERROR("ls set start ha state failed", KR(ret), KPC(ls)); - } - break; - } - default: { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("should not be here.", KR(ret)); - } // default - } // switch + } else if (OB_FAIL(ls->set_start_work_state())) { + LOG_ERROR("ls set start work state failed", KR(ret), KPC(ls)); } - if (OB_SUCCESS != (tmp_ret = ls->report_replica_info())) { - LOG_WARN("fail to report ls", KR(tmp_ret), KPC(ls)); - } return ret; } int ObLSService::replay_create_ls(const int64_t ls_epoch, const ObLSMeta &ls_meta) { int ret = OB_SUCCESS; - bool ls_is_existed = false; lib::ObMutexGuard change_guard(change_lock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -536,16 +348,8 @@ int ObLSService::replay_create_ls(const int64_t ls_epoch, const ObLSMeta &ls_met } else if (OB_UNLIKELY(!ls_meta.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_meta)); - } else if (OB_FAIL(check_ls_exist(ls_meta.ls_id_, ls_is_existed))) { - LOG_WARN("fail to check log stream existence", K(ret), K(ls_meta)); - } else if (!ls_is_existed) { - if (OB_FAIL(replay_create_ls_(ls_epoch, ls_meta))) { - LOG_WARN("fail to create ls for replay", K(ret), K(ls_meta)); - } - } else if (OB_FAIL(replay_update_ls_(ls_meta))) { - LOG_WARN("fail to update ls for replay", K(ret), K(ls_meta)); - } else { - // do nothing + } else if (OB_FAIL(replay_create_ls_(ls_epoch, ls_meta))) { + LOG_WARN("fail to create ls for replay", K(ret), K(ls_meta)); } return ret; @@ -555,20 +359,12 @@ int ObLSService::replay_update_ls(const ObLSMeta &ls_meta) { int ret = OB_SUCCESS; - bool ls_is_existed = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("the ls service has not been inited", K(ret)); } else if (OB_UNLIKELY(!ls_meta.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_meta)); - } else if (OB_FAIL(check_ls_exist(ls_meta.ls_id_, ls_is_existed))) { - LOG_WARN("fail to check log stream existence", K(ret), K(ls_meta)); - } else if (!ls_is_existed) { - LOG_WARN("ls not exit, update will create a new one", K(ls_meta)); - if (OB_FAIL(replay_create_ls_(0/*ls_epoch*/, ls_meta))) { - LOG_WARN("fail to create ls for replay", K(ret), K(ls_meta)); - } } else if (OB_FAIL(replay_update_ls_(ls_meta))) { LOG_WARN("fail to update ls for replay", K(ret), K(ls_meta)); } @@ -576,90 +372,34 @@ int ObLSService::replay_update_ls(const ObLSMeta &ls_meta) return ret; } -int ObLSService::restore_update_ls(const ObLSMetaPackage &meta_package) -{ - int ret = OB_SUCCESS; - - bool ls_is_existed = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("the ls service has not been inited", K(ret)); - } else if (OB_UNLIKELY(!meta_package.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(meta_package)); - } else if (OB_FAIL(restore_update_ls_(meta_package))) { - LOG_WARN("fail to update ls for replay", K(ret), K(meta_package)); - } - - return ret; -} - -int ObLSService::replay_remove_ls(const share::ObLSID &ls_id) +int ObLSService::replay_remove_ls() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("the ls service has not been inited", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(replay_remove_ls_(ls_id))) { - LOG_WARN("fail to remove ls for replay", K(ret), K(ls_id)); + } else if (OB_FAIL(replay_remove_ls_())) { + LOG_WARN("fail to remove ls for replay", K(ret)); } return ret; } -int ObLSService::replay_create_ls_commit(const share::ObLSID &ls_id) +int ObLSService::replay_create_ls_commit() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; ObLS *ls = nullptr; - ObLSHandle ls_handle; - int64_t create_type; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("the ls service has not been inited", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(get_ls(ls_id, ls_handle, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("fail to get ls", K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ls_id)); + } else if (OB_FAIL(get_ls(ls))) { + LOG_WARN("fail to get ls", K(ret)); } else { ObLSLockGuard lock_ls(ls); - if (OB_FAIL(ls->get_create_type(create_type))) { - LOG_WARN("get ls create type failed", K(ret)); - } else { - switch(create_type) { - case ObLSCreateType::NORMAL: { - if (OB_FAIL(ls->set_start_work_state())) { - LOG_ERROR("ls set start work state failed", KR(ret), K(ls_id)); - } - break; - } - case ObLSCreateType::RESTORE: { - if (OB_FAIL(ls->set_start_ha_state())) { - LOG_ERROR("ls set start ha state failed", KR(ret), K(ls_id)); - } - break; - } - case ObLSCreateType::MIGRATE: { - if (OB_FAIL(ls->set_start_ha_state())) { - LOG_ERROR("ls set start ha state failed", KR(ret), K(ls_id)); - } - break; - } - default: { - if (OB_FAIL(ls->set_start_work_state())) { - LOG_ERROR("ls set start work state failed", KR(ret), K(ls_id)); - } - } // default - } // switch + if (OB_FAIL(ls->set_start_work_state())) { + LOG_ERROR("ls set start work state failed", KR(ret), KPC(ls)); } - FLOG_INFO("replay create ls", KR(ret), K(ls_id)); + FLOG_INFO("replay create ls", KR(ret), KPC(ls)); } return ret; } @@ -672,92 +412,73 @@ int ObLSService::gc_ls_after_replay_slog() int tmp_ret = OB_SUCCESS; static const int64_t SLEEP_TS = 100_ms; ObLSPersistentState ls_status; - common::ObSharedGuard ls_iter; ObLS *ls = nullptr; + bool need_free = false; lib::ObMutexGuard change_guard(change_lock_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("the ls service has not been inited", K(ret)); - } else if (OB_FAIL(get_ls_iter(ls_iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); + } else if (OB_FAIL(get_ls(ls))) { + if (OB_LS_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls", K(ret)); + } + // this must be succeed } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_ERROR("fail to get next ls", K(ret)); + ls_status = ls->get_persistent_state(); + if (ls_status.is_need_gc()) { + do { + if (OB_TMP_FAIL(ls->stop())) { + LOG_WARN("ls stop failed", K(tmp_ret), KP(ls)); + } else { + ls->wait(); } - } else if (nullptr == ls) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls is null", K(ret)); - // this must be succeed - } else { - ls_status = ls->get_persistent_state(); - if (ls_status.is_need_gc()) { - do { - if (OB_TMP_FAIL(ls->stop())) { - LOG_WARN("ls stop failed", K(tmp_ret), K(ls->get_ls_id()), KP(ls)); - } else { - ls->wait(); - } - if (OB_SUCCESS != tmp_ret) { - ob_usleep(SLEEP_TS); - } - } while(tmp_ret != OB_SUCCESS); - } - ObLSLockGuard lock_ls(ls); - if (ls_status.is_init_state()) { - do { - if (OB_TMP_FAIL(TENANT_STORAGE_META_PERSISTER.abort_create_ls(ls->get_ls_id(), ls->get_ls_epoch()))) { - LOG_ERROR("fail to write create ls abort slog", K(tmp_ret), KPC(ls)); - } - if (OB_TMP_FAIL(tmp_ret)) { - ob_usleep(SLEEP_TS); - } - } while (tmp_ret != OB_SUCCESS); - remove_ls_(ls, true/*remove_from_disk*/, false/*write_slog*/); - } else if (ls_status.is_zombie_state()) { - remove_ls_(ls, true/*remove_from_disk*/, false/*write_slog*/); + if (OB_SUCCESS != tmp_ret) { + ob_usleep(SLEEP_TS); } + } while(tmp_ret != OB_SUCCESS); + } + { + ObLSLockGuard lock_ls(ls); + if (ls_status.is_init_state()) { + do { + if (OB_TMP_FAIL(TENANT_STORAGE_META_PERSISTER.abort_create_ls())) { + LOG_ERROR("fail to write create ls abort slog", K(tmp_ret), KPC(ls)); + } + if (OB_TMP_FAIL(tmp_ret)) { + ob_usleep(SLEEP_TS); + } + } while (tmp_ret != OB_SUCCESS); + remove_ls_(ls, true/*remove_from_disk*/, false/*write_slog*/); + need_free = true; + } else if (ls_status.is_zombie_state()) { + remove_ls_(ls, true/*remove_from_disk*/, false/*write_slog*/); + need_free = true; } } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; + if (need_free) { + free_ls_(ls); } } return ret; } -// online all the ls int ObLSService::online_ls() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - common::ObSharedGuard ls_iter; ObLS *ls = nullptr; - int64_t create_type = ObLSCreateType::NORMAL; - if (OB_FAIL(get_ls_iter(ls_iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_ERROR("fail to get next ls", K(ret)); - } - } else if (nullptr == ls) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls is null", K(ret)); - } else { - ObLSLockGuard lock_ls(ls); - if (OB_FAIL(ls->get_create_type(create_type))) { - LOG_WARN("get ls create type failed", K(ret)); - } else if (OB_FAIL(post_create_ls_(create_type, ls))) { - LOG_WARN("post create ls failed", K(ret)); - } - } - } - if (OB_ITER_END == ret) { + if (OB_FAIL(get_ls(ls))) { + if (OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls", K(ret)); + } + } else { + ObLSLockGuard lock_ls(ls); + if (OB_FAIL(post_create_ls_(ls))) { + LOG_WARN("post create ls failed", K(ret)); } } @@ -768,104 +489,61 @@ int ObLSService::replay_update_ls_(const ObLSMeta &ls_meta) { int ret = OB_SUCCESS; ObLS *ls = nullptr; - ObLSHandle ls_handle; - if (OB_FAIL(get_ls(ls_meta.ls_id_, ls_handle, ObLSGetMod::TXSTORAGE_MOD))) { + if (OB_FAIL(get_ls(ls))) { LOG_WARN("fail to get ls", K(ls_meta)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ls_meta)); } else if (OB_FAIL(ls->set_ls_meta(ls_meta))) { LOG_WARN("fail to set ls's meta for replay", K(ls_meta)); } return ret; } -int ObLSService::restore_update_ls_(const ObLSMetaPackage &meta_package) +int ObLSService::replay_remove_ls_() { int ret = OB_SUCCESS; ObLS *ls = nullptr; - ObLSHandle ls_handle; - ObLSMeta ls_meta = meta_package.ls_meta_; - const bool is_rebuild = false; - if (OB_FAIL(get_ls(ls_meta.ls_id_, ls_handle, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("fail to get ls", K(meta_package)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(meta_package)); - } else if (OB_FAIL(ls->set_clog_checkpoint(ls_meta.get_clog_base_lsn(), ls_meta.get_clog_checkpoint_scn(), true/*write_slog*/))) { - LOG_WARN("failed to set clog checkpoint", K(meta_package)); - } else if (OB_FAIL(ls->advance_base_info(meta_package.palf_meta_, is_rebuild))) { - LOG_WARN("failed to advance base lsn", K(meta_package)); - } - return ret; -} - -int ObLSService::replay_remove_ls_(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - ObLS *ls = nullptr; - ObLSHandle ls_handle; - if (OB_FAIL(get_ls(ls_id, ls_handle, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("fail to get ls", K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ls_id)); + if (OB_FAIL(get_ls(ls))) { + LOG_WARN("fail to get ls", K(ret)); } else if (OB_FAIL(ls->set_remove_state())) { - LOG_ERROR("ls set remove state failed", KR(ret), K(ls_id)); + LOG_ERROR("ls set remove state failed", KR(ret), KPC(ls)); } else { } - if (OB_LS_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } return ret; } int ObLSService::replay_create_ls_(const int64_t ls_epoch, const ObLSMeta &ls_meta) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; ObLS *ls = nullptr; - bool need_retry = true; ObLSCreateState state = ObLSCreateState::CREATE_STATE_INIT; - ObMigrationStatus migration_status; - ObRestoreStatus restore_status; - - if (OB_FAIL(ls_meta.get_migration_status(migration_status))) { - LOG_WARN("failed to get migration status", K(ret), K(ls_meta)); - } else if (OB_FAIL(ls_meta.get_restore_status(restore_status))) { - LOG_WARN("failed to get restore status", K(ret), K(ls_meta)); - } else if (OB_FAIL(inner_create_ls_(ls_meta.ls_id_, - migration_status, - restore_status, - ls_meta.get_clog_checkpoint_scn(), - ls_meta.get_major_mv_merge_info(), + + if (OB_SUCCESS == (ret = get_ls(ls))) { + ObLSLockGuard lock_ls(ls); + if (OB_FAIL(ls->set_ls_meta(ls_meta))) { + LOG_WARN("fail to update ls meta for replay", K(ret), K(ls_meta)); + } else if (OB_FAIL(ls->set_ls_epoch(ls_epoch))) { + LOG_WARN("fail to update ls epoch for replay", K(ret), K(ls_epoch)); + } else { + LOG_INFO("updated existing ls for replay", K(ls_epoch), K(ls_meta)); + } + } else if (OB_LS_NOT_EXIST != ret) { + LOG_WARN("fail to get ls before replay create", K(ret), K(ls_meta)); + } else if (FALSE_IT(ret = OB_SUCCESS)) { + } else if (OB_FAIL(inner_create_ls_(ls_meta.get_clog_checkpoint_scn(), ls))) { - LOG_WARN("fail to inner create ls", K(ret), K(ls_meta.ls_id_)); - } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_INNER_CREATED)) { - // do nothing + LOG_WARN("fail to inner create ls", K(ret)); } else { - // Note: if the ls is created successfully, the two argument is not used at palf. - // if the ls is not created successfully, the ls will be GC later, and the argument - // just ensure the palf will not return an error code. - palf::PalfBaseInfo palf_base_info; - palf_base_info.generate_by_default(); - ObTenantRole unused_tenant_role = share::PRIMARY_TENANT_ROLE; - const bool allow_log_sync = false; + state = ObLSCreateState::CREATE_STATE_LS_ALLOCATED; ObLSLockGuard lock_ls(ls); if (OB_FAIL(ls->set_ls_meta(ls_meta))) { LOG_WARN("set ls meta failed", K(ret), K(ls_meta)); } else if (OB_FAIL(ls->set_ls_epoch(ls_epoch))) { LOG_WARN("fail to set ls epoch", K(ret), K(ls_epoch)); - } else if (OB_FAIL(add_ls_to_map_(ls))) { - LOG_WARN("fail to add ls to the map", K(ret), K(ls_meta.ls_id_)); - } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_ADDED_TO_MAP)) { - // do nothing - } else if (OB_FAIL(ls->load_ls(unused_tenant_role, - palf_base_info, - allow_log_sync))) { + } else if (OB_FAIL(publish_ls_(ls))) { + LOG_WARN("fail to publish replayed ls", K(ret), K(ls_meta)); + } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_PUBLISHED)) { + } else if (OB_FAIL(ls->load_ls())) { LOG_WARN("enable ls palf failed", K(ret), K(ls_meta)); } else { - ls->disable_to_read(); LOG_INFO("success replay create ls", K(ret), K(ls_meta)); } } @@ -875,51 +553,43 @@ int ObLSService::replay_create_ls_(const int64_t ls_epoch, const ObLSMeta &ls_me return ret; } -int ObLSService::get_ls( - const share::ObLSID &ls_id, - ObLSHandle &handle, - ObLSGetMod mod) +int ObLSService::get_ls(ObLS *&ls) { int ret = OB_SUCCESS; - + ls = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ls_id)); - } else if (OB_FAIL(ls_map_.get_ls(ls_id, handle, mod))) { - if (OB_LS_NOT_EXIST != ret || REACH_TIME_INTERVAL(1000 * 1000)) { - LOG_WARN("get log stream fail", K(ret), K(ls_id)); - } + } else if (OB_ISNULL(ls_)) { + ret = OB_LS_NOT_EXIST; + } else { + ls = ls_; } return ret; } -int ObLSService::safe_remove_ls_(ObLSHandle handle, const bool remove_from_disk) +int ObLSService::stop_and_remove_ls_(ObLS *ls, const bool remove_from_disk) { int ret = OB_SUCCESS; - ObLS *ls = NULL; int64_t process_point = 0; // for test - if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("log stream is null, unexpected error"); + if (OB_ISNULL(ls)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid ls", K(ret)); } else if (OB_BREAK_FAIL(ls->offline())) { LOG_WARN("ls offline failed", K(ret), KP(ls)); } else if (OB_BREAK_FAIL(ls->stop())) { LOG_WARN("stop ls failed", K(ret), KP(ls)); } else if (FALSE_IT(ls->wait())) { } else { - ObLSSafeDestroyTask *task = nullptr; - const ObLSID &ls_id = ls->get_ls_id(); - static const int64_t SLEEP_TS = 100_ms; - ObLSLockGuard lock_ls(ls); - const bool write_slog = remove_from_disk; - if (remove_from_disk && OB_BREAK_FAIL(ls->set_remove_state())) { - LOG_WARN("ls set remove state failed", KR(ret), K(ls_id)); - } else { - remove_ls_(ls, remove_from_disk, write_slog); + { + ObLSLockGuard lock_ls(ls); + const bool write_slog = remove_from_disk; + if (remove_from_disk && OB_BREAK_FAIL(ls->set_remove_state())) { + LOG_WARN("ls set remove state failed", KR(ret)); + } else { + remove_ls_(ls, remove_from_disk, write_slog); + } } } return ret; @@ -928,11 +598,9 @@ int ObLSService::safe_remove_ls_(ObLSHandle handle, const bool remove_from_disk) void ObLSService::remove_ls_(ObLS *ls, const bool remove_from_disk, const bool write_slog) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls->get_ls_id(); static const int64_t SLEEP_TS = 100_ms; int64_t retry_cnt = 0; int64_t success_step = 0; - transaction::ObTransService *tx_svr = share::g_mp->trans_service(); do { // We must do prepare_for_safe_destroy to remove tablets from ObLSTabletService before writing the remove_ls_slog, @@ -950,7 +618,7 @@ void ObLSService::remove_ls_(ObLS *ls, const bool remove_from_disk, const bool w } if (success_step < 2 && OB_SUCC(ret)) { // todo zk250686_ copy tablet_id_set to tablet_free_pending_array - if(write_slog && OB_FAIL(TENANT_STORAGE_META_PERSISTER.delete_ls(ls_id, ls->get_ls_epoch()))) { + if(write_slog && OB_FAIL(TENANT_STORAGE_META_PERSISTER.delete_ls())) { LOG_ERROR("fail to write remove ls slog", K(ret)); } else { success_step = 2; @@ -958,53 +626,33 @@ void ObLSService::remove_ls_(ObLS *ls, const bool remove_from_disk, const bool w } if (success_step < 3 && OB_SUCC(ret)) { if (remove_from_disk && OB_FAIL(ls->remove_ls())) { - LOG_WARN("remove ls from disk failed", K(ret), K(remove_from_disk), K(ls_id)); + LOG_WARN("remove ls from disk failed", K(ret), K(remove_from_disk)); } else { success_step = 3; } } - if (success_step < 4 && OB_SUCC(ret)) { - if (OB_FAIL(remove_ls_from_map_(ls_id))) { - LOG_WARN("remove log stream from map fail", K(ret), K(ls_id)); - } else { - success_step = 4; - } - } - if (success_step < 5 && OB_SUCC(ret)) { - if (OB_FAIL(tx_svr->remove_tablet(ls_id))) { - LOG_WARN("remove tablet cache fail", K(ret), K(ls_id)); - } else { - success_step = 5; - } - } if (OB_FAIL(ret)) { retry_cnt++; ob_usleep(SLEEP_TS); if (retry_cnt % 100 == 0) { - LOG_ERROR("remove_ls_ cost too much time", K(ret), KP(ls), K(ls_id), K(success_step)); + LOG_ERROR("remove_ls_ cost too much time", K(ret), KP(ls), K(success_step)); } } } while (OB_FAIL(ret)); } -int ObLSService::create_ls_(const ObCreateLSCommonArg &arg) +int ObLSService::create_ls_() { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; int64_t abs_timeout_ts = INT64_MAX; ObLSCreateState state = ObLSCreateState::CREATE_STATE_INIT; ObLS *ls = NULL; - bool ls_exist = false; - bool waiting_destroy = false; int64_t process_point = 0; + const SCN create_scn = SCN::base_scn(); palf::PalfBaseInfo palf_base_info; palf_base_info.generate_by_default(); - palf_base_info.prev_log_info_.scn_ = arg.create_scn_; + palf_base_info.prev_log_info_.scn_ = create_scn; - const storage::ObMajorMVMergeInfo major_mv_merge_info; - common::ObMemberList member_list; - common::GlobalLearnerList learner_list; - const ObMigrationStatus migration_status(ObMigrationStatus::OB_MIGRATION_STATUS_NONE); const lib::Worker::CompatMode compat_mode(lib::Worker::CompatMode::MYSQL); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -1023,60 +671,40 @@ int ObLSService::create_ls_(const ObCreateLSCommonArg &arg) } else if (OB_BREAK_FAIL(change_guard.get_ret())) { LOG_WARN("lock failed, try again later", K(ret)); ret = OB_EAGAIN; - } else if (OB_BREAK_FAIL(check_ls_exist(share::SYS_LS, ls_exist))) { - LOG_WARN("check ls exist failed", K(ret)); - } else if (ls_exist) { - ret = OB_LS_EXIST; - LOG_ERROR("ls exist, cannot create ls now", K(ret)); - } else if (OB_BREAK_FAIL(check_ls_waiting_safe_destroy(share::SYS_LS, - waiting_destroy))) { - LOG_WARN("check ls waiting safe destroy failed", K(ret)); - } else if (waiting_destroy) { - ret = OB_LS_WAITING_SAFE_DESTROY; - LOG_WARN("ls waiting for destroy, need retry later", K(ret)); - } else if (OB_BREAK_FAIL(inner_create_ls_(share::SYS_LS, - migration_status, - arg.restore_status_, - arg.create_scn_, - major_mv_merge_info/*not init is ok*/, + } else if (OB_NOT_NULL(ls_)) { + ret = OB_INIT_TWICE; + LOG_WARN("the tenant log stream already exists", K(ret)); + } else if (OB_BREAK_FAIL(inner_create_ls_(create_scn, ls))) { LOG_WARN("create ls failed", K(ret)); } else { - state = ObLSCreateState::CREATE_STATE_INNER_CREATED; + state = ObLSCreateState::CREATE_STATE_LS_ALLOCATED; int64_t ls_epoch = 0; ObLSLockGuard lock_ls(ls); const ObLSMeta &ls_meta = ls->get_ls_meta(); - if (OB_BREAK_FAIL(add_ls_to_map_(ls))) { - LOG_WARN("add log stream to map failed.", K(ret)); - } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_ADDED_TO_MAP)) { - // do nothing + if (OB_BREAK_FAIL(publish_ls_(ls))) { + LOG_WARN("publish log stream failed", K(ret), K(ls_meta)); + } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_PUBLISHED)) { } else if (OB_BREAK_FAIL(TENANT_STORAGE_META_PERSISTER.prepare_create_ls(ls_meta, ls_epoch))) { LOG_ERROR("fail to write create log stream slog", K(ls_meta)); } else if (OB_FAIL(ls->set_ls_epoch(ls_epoch))) { LOG_WARN("fail to set ls epoch", K(ret)); } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_WRITE_PREPARE_SLOG)) { - } else if (OB_BREAK_FAIL(ls->create_ls(arg.tenant_role_/*tenant_role*/, - palf_base_info, - REPLICA_TYPE_FULL/*replica_type*/))) { + } else if (OB_BREAK_FAIL(ls->create_ls(palf_base_info))) { LOG_WARN("enable ls palf failed", K(ret), K(ls_meta)); } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_PALF_ENABLED)) { - } else if (arg.need_create_inner_tablet_ && OB_FAIL(ls->create_ls_inner_tablet(compat_mode, arg.create_scn_))) { + } else if (OB_FAIL(ls->create_ls_inner_tablet(compat_mode, create_scn))) { LOG_WARN("create ls inner tablet failed", K(ret), K(ls_meta)); } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_INNER_TABLET_CREATED)) { - } else if (OB_BREAK_FAIL(TENANT_STORAGE_META_PERSISTER.commit_create_ls( - ls->get_ls_id(), ls->get_ls_epoch()))) { + } else if (OB_BREAK_FAIL(TENANT_STORAGE_META_PERSISTER.commit_create_ls())) { LOG_ERROR("fail to write create log stream commit slog", K(ret), K(ls_meta)); } else if (OB_BREAK_FAIL(ls->finish_create_ls())) { LOG_WARN("finish create ls failed", KR(ret)); } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_FINISH)) { - } else if (OB_BREAK_FAIL(post_create_ls_(arg.create_type_, ls))) { - LOG_WARN("post create ls failed", K(ret), K(ls_meta), K(arg.create_type_)); - } else if (OB_FAIL(member_list.add_member(ObMember(GCTX.self_addr(), 1/*timestamp*/)))) { - LOG_WARN("failed to add member", KR(ret)); - } else if (OB_FAIL(ls->set_initial_member_list(member_list, - 1/*paxos_replica_num*/, - learner_list))) { - LOG_WARN("failed to set member list", KR(ret)); + } else if (OB_BREAK_FAIL(post_create_ls_(ls))) { + LOG_WARN("post create ls failed", K(ret), K(ls_meta)); + } else if (OB_FAIL(ls->bootstrap())) { + LOG_WARN("failed to bootstrap local log", KR(ret)); } } if (OB_BREAK_FAIL(ret)) { @@ -1092,19 +720,18 @@ void ObLSService::del_ls_after_create_ls_failed_(ObLSCreateState& in_ls_create_s int tmp_ret = OB_SUCCESS; bool need_retry = false; const bool remove_from_disk = true; - ObLSHandle handle; ObLSCreateState ls_create_state = in_ls_create_state; if (OB_NOT_NULL(ls)) { do { need_retry = false; tmp_ret = OB_SUCCESS; if (ls_create_state >= ObLSCreateState::CREATE_STATE_FINISH) { - if (OB_TMP_FAIL(handle.set_ls(ls_map_, *ls, ObLSGetMod::TXSTORAGE_MOD))) { - need_retry = true; - LOG_WARN("get ls handle failed", K(tmp_ret), KPC(ls)); - } else if (OB_TMP_FAIL(safe_remove_ls_(handle, remove_from_disk))) { + if (OB_TMP_FAIL(stop_and_remove_ls_(ls, remove_from_disk))) { need_retry = true; LOG_WARN("safe remove ls failed", K(tmp_ret)); + } else { + free_ls_(ls); + ls_create_state = ObLSCreateState::CREATE_STATE_INIT; } } else { if (ls_create_state >= ObLSCreateState::CREATE_STATE_INNER_TABLET_CREATED) { @@ -1129,224 +756,42 @@ void ObLSService::del_ls_after_create_ls_failed_(ObLSCreateState& in_ls_create_s if (OB_TMP_FAIL(ls->set_remove_state())) { need_retry = true; LOG_ERROR("fail to set ls remove state", K(tmp_ret), KPC(ls)); - } else if (OB_TMP_FAIL(TENANT_STORAGE_META_PERSISTER.abort_create_ls(ls->get_ls_id(), ls->get_ls_epoch()))) { + } else if (OB_TMP_FAIL(TENANT_STORAGE_META_PERSISTER.abort_create_ls())) { need_retry = true; LOG_ERROR("fail to write create log stream abort slog", K(tmp_ret), KPC(ls)); } else { - ls_create_state = ObLSCreateState::CREATE_STATE_ADDED_TO_MAP; + ls_create_state = ObLSCreateState::CREATE_STATE_PUBLISHED; } } if (OB_TMP_FAIL(tmp_ret)) { - } else if (ls_create_state >= ObLSCreateState::CREATE_STATE_ADDED_TO_MAP) { - if (OB_TMP_FAIL(remove_ls_from_map_(ls->get_ls_id()))) { - need_retry = true; - LOG_ERROR_RET(tmp_ret, "remove ls from map failed", K(tmp_ret)); - } else { - ls_create_state = ObLSCreateState::CREATE_STATE_INIT; - } + } else if (ls_create_state >= ObLSCreateState::CREATE_STATE_PUBLISHED) { + ls_ = nullptr; + ls_create_state = ObLSCreateState::CREATE_STATE_LS_ALLOCATED; } if (OB_TMP_FAIL(tmp_ret)) { - } else if (ls_create_state >= ObLSCreateState::CREATE_STATE_INNER_CREATED) { - if (OB_TMP_FAIL(inner_del_ls_(ls))) { + } else if (ls_create_state >= ObLSCreateState::CREATE_STATE_LS_ALLOCATED) { + if (OB_TMP_FAIL(ls->prepare_for_safe_destroy())) { need_retry = true; - LOG_ERROR_RET(tmp_ret, "inner del ls failed.", K(tmp_ret)); + LOG_WARN("prepare failed ls for destroy failed", K(tmp_ret), KPC(ls)); } else { + free_ls_(ls); ls_create_state = ObLSCreateState::CREATE_STATE_INIT; } } } } while (need_retry); } + in_ls_create_state = ls_create_state; } -int ObLSService::check_ls_exist(const share::ObLSID &ls_id, bool &exist) -{ - int ret = OB_SUCCESS; - ObLSHandle handle; - exist = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id)); - } else if (OB_FAIL(get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { - if (OB_LS_NOT_EXIST != ret) { - LOG_DEBUG("get log stream failed", K(ls_id), K(ret)); - } - } else if (OB_ISNULL(handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get log stream failed", K(ls_id), K(ret)); - } else { - exist = true; - } - if (OB_LS_NOT_EXIST == ret) { - exist = false; - ret = OB_SUCCESS; - } - return ret; -} - -int ObLSService::check_ls_waiting_safe_destroy(const share::ObLSID &ls_id, bool &waiting) -{ - int ret = OB_SUCCESS; - waiting = false; - return ret; -} - -ERRSIM_POINT_DEF(ALLOC_LS_ITER_GUARD_FAIL) -int ObLSService::get_ls_iter(common::ObSharedGuard &guard, ObLSGetMod mod) -{ - int ret = OB_SUCCESS; - ObLSIterator *ls_iter = NULL; - void *buf = NULL; - const char* LS = "ObLSIter"; - ObMemAttr attr(LS); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (is_stopped_) { - ret = OB_NOT_RUNNING; - LOG_WARN("ls service is stopped.", K(ret), KP(this)); - } else if (NULL == (buf = iter_allocator_.alloc(sizeof(ObLSIterator), attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("Fail to allocate memory for log stream iterator.", K(ret)); - } else { - ls_iter = new (buf) ObLSIterator(); - ls_iter->set_ls_map(ls_map_, mod); - inc_iter_cnt(); - if (OB_FAIL(ALLOC_LS_ITER_GUARD_FAIL)) { - LOG_WARN("ALLOC_LS_ITER_GUARD_FAIL"); - } else if (OB_FAIL(guard.assign(ls_iter, [&](ObLSIterator *iter) mutable { - iter->~ObLSIterator(); - iter_allocator_.free(iter); - dec_iter_cnt(); - }))) { - LOG_WARN("create guard failed.", K(ret)); - } - // if assign failed, we need free the memory we have allocated. - if (OB_FAIL(ret)) { - ls_iter->~ObLSIterator(); - iter_allocator_.free(ls_iter); - dec_iter_cnt(); - } - } - return ret; -} - -int ObLSService::get_ls_ids(common::ObIArray &ls_id_array) +int ObLSService::diagnose(DiagnoseInfo &info) { int ret = OB_SUCCESS; - ls_id_array.reuse(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(ls_map_.get_all_ls_id(ls_id_array))) { - LOG_WARN("failed to get all ls id", K(ret)); - } - return ret; -} - -int ObLSService::iterate_diagnose(const ObFunction &func) -{ - int ret = OB_SUCCESS; - common::ObSharedGuard ls_iter; ObLS *ls = nullptr; - - if (OB_FAIL(get_ls_iter(ls_iter, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_ERROR("fail to get next ls", K(ret)); - } - } else if (nullptr == ls) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ls is null", K(ret)); - } else if (OB_FAIL(func(*ls))) { - LOG_WARN("iter ls diagnose failed", K(ret)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - } - return ret; -} - -int ObLSService::get_restore_status_( - ObRestoreStatus &restore_status) -{ - int ret = OB_SUCCESS; - - restore_status = ObRestoreStatus::Status::NONE; - bool is_primary = true; - - - if (OB_FAIL(ObShareUtil::is_primary_cluster(is_primary))) { - LOG_WARN("fail to check whether is primary cluster", KR(ret)); - } else if (is_primary) { - restore_status = ObRestoreStatus::Status::NONE; - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("only supported primary cluster now", KR(ret)); - } - - return ret; -} - -ObRestoreStatus ObLSService::get_restore_status_by_tenant_role_(const ObTenantRole& tenant_role) -{ - ObRestoreStatus restore_status(ObRestoreStatus::Status::NONE); - - if (tenant_role.is_restore()) { - restore_status = ObRestoreStatus::Status::RESTORE_DOING; - } else { - restore_status = ObRestoreStatus::Status::NONE; - } - - return restore_status; -} - -int64_t ObLSService::get_create_type_by_tenant_role_(const ObTenantRole& tenant_role) -{ - int64_t create_type = ObLSCreateType::NORMAL; - - if (tenant_role.is_restore()) { - create_type = ObLSCreateType::RESTORE; - } else { - create_type = ObLSCreateType::NORMAL; - } - - return create_type; -} - -int ObLSService::dump_ls_info() -{ - int ret = OB_SUCCESS; - common::ObSharedGuard ls_iter; - ObLS *ls = nullptr; - ObLSMeta ls_meta; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(get_ls_iter(ls_iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("failed to get ls iter", K(ret)); - } - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next ls", K(ret)); - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is null", K(ret)); - } else if (OB_FAIL(ls->get_ls_meta(ls_meta))) { - LOG_WARN("fail to get ls meta", K(ret)); - } else { - FLOG_INFO("dump ls info", K(ls_meta)); - } + if (OB_FAIL(get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_FAIL(ls->diagnose(info))) { + LOG_WARN("diagnose ls failed", K(ret)); } return ret; } @@ -1362,7 +807,7 @@ namespace oceanbase namespace share { -// get_sys_ls_readable_scn has been demoted to storage::free function(see end of file) +// get_ls_readable_scn is a storage helper backed by the tenant LS service. } // namespace share @@ -1377,7 +822,7 @@ namespace share ObMdsThrottleGuard::~ObMdsThrottleGuard() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + storage::ObLS *ls = nullptr; ObThrottleInfoGuard share_ti_guard; ObThrottleInfoGuard module_ti_guard; @@ -1385,17 +830,14 @@ ObMdsThrottleGuard::~ObMdsThrottleGuard() MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "get ls handle failed", KR(ret), K(ls_id_)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + STORAGE_LOG(WARN, "get ls failed", KR(ret)); } else { (void)TxShareMemThrottleUtil::do_throttle(for_replay_, abs_expire_time_, share::mds_throttled_alloc(), share::g_mp->tenant_freezer()->exist_ls_throttle_is_skipping(), - ls_handle.get_ls()->is_offline(), + ls->is_offline(), *throttle_tool_, share_ti_guard, module_ti_guard); @@ -1420,24 +862,21 @@ ObMdsThrottleGuard::~ObMdsThrottleGuard() ObTxDataThrottleGuard::~ObTxDataThrottleGuard() { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + storage::ObLS *ls = nullptr; ObThrottleInfoGuard share_ti_guard; ObThrottleInfoGuard module_ti_guard; if (OB_ISNULL(throttle_tool_)) { MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id_)); - } else if (OB_ISNULL(ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "get ls handle failed", KR(ret), K(ls_id_)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + STORAGE_LOG(WARN, "get ls failed", KR(ret)); } else { (void)TxShareMemThrottleUtil::do_throttle(for_replay_, abs_expire_time_, share::tx_data_throttled_alloc(), share::g_mp->tenant_freezer()->exist_ls_throttle_is_skipping(), - ls_handle.get_ls()->is_offline(), + ls->is_offline(), *throttle_tool_, share_ti_guard, module_ti_guard); @@ -1462,45 +901,6 @@ ObTxDataThrottleGuard::~ObTxDataThrottleGuard() } // namespace share } // namespace oceanbase -// ===== definition moved from share/ob_global_autoinc_service.cpp(ObLS real user) ===== -namespace oceanbase -{ -namespace share -{ - -int ObGlobalAutoIncService::check_leader_(bool &is_leader) -{ - int ret = OB_SUCCESS; - is_leader = ATOMIC_LOAD(&is_leader_); - if (OB_LIKELY(is_leader)) { - } else if (ATOMIC_LOAD(&is_switching_)) { - ret = OB_NOT_MASTER; - LOG_WARN("service is switching to leader", K(ret), KP(this), K(*this)); - } else { - // try to get role from logstream - ObRole role = ObRole::INVALID_ROLE; - int64_t proposal_id = 0; - ObSpinLockGuard lock(cache_ls_lock_); - if (OB_ISNULL(cache_ls_)) { - ret = OB_NOT_MASTER; - LOG_WARN("cache ls is null", K(ret)); - } else if (OB_FAIL(cache_ls_->get_log_handler()->get_role(role, proposal_id))) { - int tmp_ret = ret; - ret = OB_NOT_MASTER; - LOG_WARN("get ls role fail", K(ret), K(tmp_ret)); - } else if (common::ObRole::LEADER == role) { - is_leader = true; - } else { - is_leader = false; - } - } - - return ret; -} - -} // namespace share -} // namespace oceanbase - // ===== definition moved from share resource_limit_calculator(X-macro inventory 2fn) ===== namespace oceanbase { @@ -1689,21 +1089,12 @@ namespace oceanbase namespace storage { using namespace oceanbase::share; -int get_sys_ls_readable_scn(SCN &readable_scn) +int get_ls_readable_scn(SCN &readable_scn) { int ret = OB_SUCCESS; - ObLSService *ls_svr = share::g_mp->ls_service(); - ObLSHandle ls_handle; ObLS *ls = nullptr; - share::SCN offline_scn; - if (OB_ISNULL(ls_svr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("pointer is null", KR(ret), KP(ls_svr)); - } else if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, ObLSGetMod::RS_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { LOG_WARN("get log stream failed", KR(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log stream is null", KR(ret), K(ls_handle)); } else if (OB_FAIL(ls->get_max_decided_scn(readable_scn))) { LOG_WARN("failed to get_max_decided_scn", KR(ret), KPC(ls)); } diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index dbd268747..fe515ed1e 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -18,21 +18,14 @@ #define OCEABASE_STORAGE_LS_SERVICE_ #include "storage/tablet/ob_batch_create_tablet_arg.h" -#include "lib/guard/ob_shared_guard.h" #include "lib/allocator/ob_concurrent_fifo_allocator.h" // ObConcurrentFIFOAllocator -#include "storage/tx_storage/ob_ls_map.h" +#include "storage/ls/ob_ls.h" #include "storage/ob_storage_rpc.h" -#include "storage/ls/ob_ls_meta_package.h" // ObLSMetaPackage #include "share/resource_limit_calculator/ob_resource_limit_calculator.h" -#include "storage/mview/ob_major_mv_merge_info.h" #include "storage/tx/ob_tx_result_struct.h" namespace oceanbase { -namespace share -{ -class ObLSID; -} namespace blocksstable { class ObBaseStorageLogger; @@ -42,17 +35,15 @@ class ObSuperBlockMetaEntry; namespace storage { class ObLS; -class ObLSIterator; -class ObLSHandle; +struct DiagnoseInfo; struct ObLSMeta; -// Maintain the tenant <-> log streams mapping relationship +// Maintain the tenant's single log stream. // Support log stream meta persistent and checkpoint class ObLSService : public ObIResourceLimitCalculatorHandler { - static const int64_t DEFAULT_LOCK_TIMEOUT = 60_s; - static const int64_t SMALL_TENANT_MEMORY_LIMIT = 4LL * 1024 * 1024 * 1024; // 4G - static const int64_t TENANT_MEMORY_PER_LS_NEED = 200 * 1024 * 1024L; // 200MB + static const int64_t DEFAULT_LOCK_TIMEOUT = 60_s; + static const int64_t SMALL_TENANT_MEMORY_LIMIT = 4LL * 1024 * 1024 * 1024; // 4G public: int64_t break_point = -1; // just for test public: @@ -65,11 +56,6 @@ class ObLSService : public ObIResourceLimitCalculatorHandler int stop(); int wait(); void destroy(); - bool is_empty(); - void inc_ls_safe_destroy_task_cnt(); - void dec_ls_safe_destroy_task_cnt(); - void inc_iter_cnt(); - void dec_iter_cnt(); public: // for limit calculator virtual int get_current_info(share::ObResourceInfo &info) override; @@ -78,62 +64,29 @@ class ObLSService : public ObIResourceLimitCalculatorHandler virtual int cal_min_phy_resource_needed(const int64_t num, share::ObMinPhyResourceResult &min_phy_res) override; public: // create a LS - int create_ls(const ObTenantRole &tenant_role = PRIMARY_TENANT_ROLE); - // create a LS for HA (standby bootstrap/restore use case) - int create_ls_for_ha(); + int create_ls(); // create a LS for replay or update LS's meta // @param [in] ls_epoch, the epoch increases monotonically in tenant scope when an ls is created // @param [in] ls_meta, all the parameters that is needed to create a LS for replay int replay_create_ls(const int64_t ls_epoch, const ObLSMeta &ls_meta); // replay create ls commit slog. - // @param [in] ls_id, the create process of which is committed. - int replay_create_ls_commit(const share::ObLSID &ls_id); + int replay_create_ls_commit(); // create a LS for replay or update LS's meta // @param [in] ls_meta, all the parameters that is needed to create a LS for replay int replay_update_ls(const ObLSMeta &ls_meta); - // update LS's meta for restore - // @param [in] meta_package, all the parameters that is needed to for restore - int restore_update_ls(const ObLSMetaPackage &meta_package); - // set a LS to REMOVED state and gc it later. - // for replay create ls abort or remove - // @param [in] ls_id, which ls need to be set REMOVED. - int replay_remove_ls(const share::ObLSID &ls_id); + // Set the LS to REMOVED state for replay create abort or remove. + int replay_remove_ls(); - // @param [in] ls_id, which ls does we need, mod is the module to get ls - // @param [out] handle, a guard of the specified logsream. - int get_ls(const share::ObLSID &ls_id, - ObLSHandle &handle, - ObLSGetMod mod); - // @param [in] func, iterate all ls diagnose info - int iterate_diagnose(const ObFunction &func); + // get the only log stream in seekdb. + int get_ls(ObLS *&ls); + int diagnose(DiagnoseInfo &info); // remove the ls that is creating and write abort slog. int gc_ls_after_replay_slog(); - // online all ls + // online the log stream int online_ls(); - // check whether a ls exist or not. - // @param [in] ls_id, the ls we will check. - // @param [out] exist, true if the ls exist, else false. - int check_ls_exist(const share::ObLSID &ls_id, - bool &exist); - // check whether a ls waiting for destroy or not. - // @param [in] ls_id, the ls we will check. - // @param [out] waiting, true if the ls waiting for destroy, else false. - int check_ls_waiting_safe_destroy(const share::ObLSID &ls_id, - bool &waiting); - // get a log stream iterator. - // @param [out] guard, the iterator created. - // use guard just like a pointer of ObLSIterator - int get_ls_iter(common::ObSharedGuard &guard, ObLSGetMod mod); - - template - int foreach_ls(FUNC &func); - - // get all ls ids - int get_ls_ids(common::ObIArray &ls_id_array); - // tablet operation in transactions // Create tablets for a ls // @param [in] tx_desc, trans descriptor @@ -153,72 +106,41 @@ class ObLSService : public ObIResourceLimitCalculatorHandler obcall::ObStorageRpcProxy *get_storage_rpc_proxy() { return &storage_svr_rpc_proxy_; } storage::ObStorageRpc *get_storage_rpc() { return &storage_rpc_; } - ObLSMap *get_ls_map() { return &ls_map_; } - int64_t get_ls_count() const { return ls_map_.get_ls_count(); } - int dump_ls_info(); - - TO_STRING_KV(K_(is_inited)); private: enum class ObLSCreateState { CREATE_STATE_INIT = 0, // begin - CREATE_STATE_INNER_CREATED = 1, // inner_create_ls_ succ - CREATE_STATE_ADDED_TO_MAP = 2, // add_ls_to_map_ succ - CREATE_STATE_WRITE_PREPARE_SLOG = 3, // write_prepare_create_ls_slog_ succ - CREATE_STATE_PALF_ENABLED = 4, // enable_palf succ - CREATE_STATE_INNER_TABLET_CREATED = 5, // have created inner tablet - CREATE_STATE_FINISH - }; - - struct ObCreateLSCommonArg - { - ObTenantRole tenant_role_; - ObRestoreStatus restore_status_; - share::SCN create_scn_; - int64_t create_type_; // ObLSCreateType::* - bool need_create_inner_tablet_; + CREATE_STATE_LS_ALLOCATED = 1, + CREATE_STATE_PUBLISHED = 2, + CREATE_STATE_WRITE_PREPARE_SLOG = 3, + CREATE_STATE_PALF_ENABLED = 4, + CREATE_STATE_INNER_TABLET_CREATED = 5, + CREATE_STATE_FINISH = 6 }; - int create_ls_(const ObCreateLSCommonArg &arg); - int inner_create_ls_(const share::ObLSID &lsid, - const ObMigrationStatus &migration_status, - const ObRestoreStatus &restore_status, - const share::SCN &create_scn, - const ObMajorMVMergeInfo &major_mv_merge_info, - ObLS *&ls); - int inner_del_ls_(ObLS *&ls); - int add_ls_to_map_(ObLS *ls); - int remove_ls_from_map_(const share::ObLSID &ls_id); + int create_ls_(); + int inner_create_ls_(const share::SCN &create_scn, ObLS *&ls); + int publish_ls_(ObLS *ls); + void free_ls_(ObLS *ls); void remove_ls_(ObLS *ls, const bool remove_from_disk, const bool write_slog); - int safe_remove_ls_(ObLSHandle handle, const bool remove_from_disk); - int restore_update_ls_(const ObLSMetaPackage &meta_package); - int replay_remove_ls_(const share::ObLSID &ls_id); + int stop_and_remove_ls_(ObLS *ls, const bool remove_from_disk); + int replay_remove_ls_(); int replay_create_ls_(const int64_t ls_epoch, const ObLSMeta &ls_meta); int replay_update_ls_(const ObLSMeta &ls_meta); - int post_create_ls_(const int64_t create_type, - ObLS *&ls); + int post_create_ls_(ObLS *ls); void del_ls_after_create_ls_failed_(ObLSCreateState& ls_create_state, ObLS *ls); - int alloc_ls_(ObLS *&ls); - int get_restore_status_( - ObRestoreStatus &restore_status); - ObRestoreStatus get_restore_status_by_tenant_role_(const ObTenantRole& tenant_role); - int64_t get_create_type_by_tenant_role_(const ObTenantRole& tenant_role); - // for resource limit calculator - int cal_min_phy_resource_needed_(const int64_t ls_cnt, - ObMinPhyResourceResult &min_phy_res); + int cal_min_phy_resource_needed_(ObMinPhyResourceResult &min_phy_res); int get_resource_constraint_value_(ObResoureConstraintValue &constraint_value); private: bool is_inited_; bool is_running_; // used by create/remove, only can be used after start and before stop. - bool is_stopped_; // only for ls iter, get ls iter will cause OB_NOT_RUNNING after stop. - // a map from ls id to ls - ObLSMap ls_map_; + bool is_stopped_; + ObLS *ls_; common::ObConcurrentFIFOAllocator ls_allocator_; - common::ObConcurrentFIFOAllocator iter_allocator_; // protect the create and remove process lib::ObMutex change_lock_; @@ -226,57 +148,16 @@ class ObLSService : public ObIResourceLimitCalculatorHandler obcall::ObStorageRpcProxy storage_svr_rpc_proxy_; storage::ObStorageRpc storage_rpc_; - // for safe destroy - // store the ls is removing - int64_t safe_ls_destroy_task_cnt_; - - // record the count of ls iter - int64_t iter_cnt_; - - // for limit calculator - // the max ls cnt after observer start - int64_t max_ls_cnt_; DISALLOW_COPY_AND_ASSIGN(ObLSService); }; -template -int ObLSService::foreach_ls(FUNC &func) -{ - int ret = OB_SUCCESS; - - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - if (OB_FAIL(get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", K(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", K(ret)); - } else { - ObLS *ls = nullptr; - while (OB_SUCC(ret)) { - if (OB_FAIL(iter->get_next(ls))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - STORAGE_LOG(WARN, "iter next ls failed", KR(ret), KP(this)); - } - } else if (OB_FAIL(func(*ls))) { - STORAGE_LOG(WARN, "do function on ls failed", K(ret)); - } - } - } - return ret; -} - } } namespace oceanbase { namespace storage { -// demoted from share::ObShareUtil(tenant-level query, gets the sys LS readable SCN through MTL ObLSService) -int get_sys_ls_readable_scn(share::SCN &readable_scn); +int get_ls_readable_scn(share::SCN &readable_scn); } } diff --git a/src/storage/tx_storage/ob_safe_destroy_handler.cpp b/src/storage/tx_storage/ob_safe_destroy_handler.cpp deleted file mode 100644 index 2aec30529..000000000 --- a/src/storage/tx_storage/ob_safe_destroy_handler.cpp +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define USING_LOG_PREFIX SERVER -#include "storage/tx_storage/ob_safe_destroy_handler.h" - -namespace oceanbase -{ -namespace storage -{ - -void ObSafeDestroyTask::set_receive_timestamp(const int64_t receive_timestamp) -{ - // only set the first time. - if (0 == recv_timestamp_) { - recv_timestamp_ = receive_timestamp; - } -} - -void ObSafeDestroyTask::update_retry_info() -{ - int64_t current_time = ObTimeUtility::current_time(); - - retry_times_++; - last_execute_timestamp_ = current_time; - if ((current_time - recv_timestamp_) >= PRINT_INTERVAL_TIME && - retry_times_ % PRINT_TIMES_INTERVAL == 0) { - LOG_WARN_RET(OB_ERR_UNEXPECTED, "safe destroy task alive too long", KPC(this)); - } -} - -void ObSafeDestroyTask::update_execute_timestamp() -{ - last_execute_timestamp_ = ObTimeUtility::current_time(); -} - -bool ObSafeDestroyProcessFunctor::operator()( - ObSafeDestroyTask *task, - bool &need_requeue) const -{ - int ret = OB_SUCCESS; - need_requeue = false; - if (OB_ISNULL(task)) { - ret = OB_INVALID_ARGUMENT; - LOG_ERROR("queue pop NULL task", K(task), K(ret)); - } else { - if (task->safe_to_destroy()) { - task->destroy(); - task->~ObSafeDestroyTask(); - ob_free(task); - } else { - task->update_retry_info(); - need_requeue = true; - } - } - return true; // always continue to loop -} - -ObSafeDestroyTaskQueue::ObSafeDestroyTaskQueue() - : looping_(false), - stop_(false), - queue_() -{ -} - -ObSafeDestroyTaskQueue::~ObSafeDestroyTaskQueue() -{ - destroy(); -} - -int ObSafeDestroyTaskQueue::init(int queue_capacity) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(queue_.init(queue_capacity))) { - LOG_WARN("init queue failed", K(ret)); - } - return ret; -} - -int ObSafeDestroyTaskQueue::push(ObSafeDestroyTask *task) -{ - int ret = OB_SUCCESS; - // allow mutil thread push into the queue. - RDLockGuard guard(rwlock_); - if (stop_) { - ret = OB_NOT_RUNNING; - LOG_WARN("thread is stopped", K(stop_), K(ret)); - } else if (OB_ISNULL(task)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("task should not be null", K(ret), KP(task)); - } else { - task->set_receive_timestamp(ObTimeUtility::current_time()); - if (OB_FAIL(queue_.push(task))) { - LOG_WARN("push into queue failed", K(ret)); - } - } - return ret; -} - -bool ObSafeDestroyTaskQueue::stop_finished() -{ - return (stop_ && - !looping_ && - 0 == queue_.size()); -} - -void ObSafeDestroyTaskQueue::push_into_queue_(void *task) -{ - int ret = OB_SUCCESS; - static const int64_t SLEEP_TS = 100 * 1000; // 100_ms - // retry until success. - while (OB_FAIL(queue_.push(task))) { - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - LOG_WARN("push into the queue failed, retry again", K(ret), K(task)); - } - ob_usleep(SLEEP_TS); - } -} - -void ObSafeDestroyTaskQueue::loop() -{ - int ret = OB_SUCCESS; - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - LOG_INFO("ObSafeDestroyTaskQueue::loop begin", K(queue_.size())); - } - ObSafeDestroyProcessFunctor fn; - if (OB_FAIL(for_each(fn))) { - LOG_WARN("loop failed", K(ret)); - } - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - LOG_INFO("ObSafeDestroyTaskQueue::loop finish", K(ret), K(queue_.size())); - } -} - -void ObSafeDestroyTaskQueue::destroy() -{ - queue_.destroy(); - stop_ = true; - looping_ = false; -} - -ObSafeDestroyHandler::ObSafeDestroyHandler() - : is_inited_(false), - queue_(), - last_process_timestamp_(0) -{} - -ObSafeDestroyHandler::~ObSafeDestroyHandler() -{ - destroy(); -} - - - -int ObSafeDestroyHandler::stop() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("safe destroy thread not inited", K(ret)); - } else { - queue_.stop(); - // we only stop the queue to prevent push task into the queue. - // the timer will be stop at the wait function. - LOG_INFO("ObSafeDestroyHandler stopped"); - } - return ret; -} - -void ObSafeDestroyHandler::wait() -{ - int ret = OB_SUCCESS; - static const int64_t SLEEP_TS = 100 * 1000; // 100_ms - int64_t start_ts = ObTimeUtility::current_time(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("safe destroy thread not inited", K(ret)); - } else { - while (!queue_.stop_finished()) { - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute - LOG_WARN_RET(OB_ERR_TOO_MUCH_TIME, "the safe destroy thread wait cost too much time", - K(ObTimeUtility::current_time() - start_ts)); - } - ob_usleep(SLEEP_TS); - } - } -} - -void ObSafeDestroyHandler::destroy() -{ - LOG_INFO("ObSafeDestroyHandler::destroy"); - is_inited_ = false; - last_process_timestamp_ = 0; - queue_.destroy(); -} - - - -} // observer -} // oceanbase diff --git a/src/storage/tx_storage/ob_safe_destroy_handler.h b/src/storage/tx_storage/ob_safe_destroy_handler.h deleted file mode 100644 index 2fe4b620a..000000000 --- a/src/storage/tx_storage/ob_safe_destroy_handler.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEABASE_SERVER_OB_SAFE_DESTROY_THREAD_ -#define OCEABASE_SERVER_OB_SAFE_DESTROY_THREAD_ - -#include "lib/lock/ob_tc_rwlock.h" -#include "lib/ob_define.h" -#include "lib/queue/ob_lighty_queue.h" -#include "lib/task/ob_timer.h" - -namespace oceanbase -{ -namespace storage -{ -class ObSafeDestroyTask -{ -public: - enum class ObSafeDestroyTaskType - { - INVALID, - LS, - }; - const static int64_t PRINT_INTERVAL_TIME = 10 * 60 * 1000 * 1000; // a task only print if it - // is not complete after 10 min - const static int64_t PRINT_TIMES_INTERVAL = 10; // every 2 second loop once. -public: - ObSafeDestroyTask() - : type_(ObSafeDestroyTaskType::INVALID), - recv_timestamp_(0), - last_execute_timestamp_(0), - retry_times_(0) - {} - virtual ~ObSafeDestroyTask() {} - virtual bool safe_to_destroy() = 0; - virtual void destroy() = 0; - ObSafeDestroyTaskType get_type() const { return type_; } - void set_receive_timestamp(const int64_t receive_timestamp); - void update_retry_info(); - void update_execute_timestamp(); - int64_t get_last_execute_timestamp() const - { return last_execute_timestamp_; } - VIRTUAL_TO_STRING_KV(K_(recv_timestamp), K_(retry_times), - K_(last_execute_timestamp)); -protected: - ObSafeDestroyTaskType type_; - // the time the task is received - int64_t recv_timestamp_; - // the last execute timestamp - int64_t last_execute_timestamp_; - // how many times the task has been processed. - int64_t retry_times_; -}; // end of class ObSafeDestroyTask - -class ObSafeDestroyProcessFunctor -{ -public: - bool operator()(ObSafeDestroyTask *task, bool &need_requeue) const; -}; - -class ObSafeDestroyTaskQueue -{ - using RWLock = common::TCRWLock; - using RDLockGuard = common::TCRLockGuard; - using WRLockGuard = common::TCWLockGuard; - -public: - static const int LIGHTY_QUEUE_SIZE = (1 << 10); - ObSafeDestroyTaskQueue(); - virtual ~ObSafeDestroyTaskQueue(); - - int init(int queue_capacity = LIGHTY_QUEUE_SIZE); - void stop() { stop_ = true; } - bool stop_finished(); - int push(ObSafeDestroyTask *task); - void loop(); - void destroy(); - // For each. - // Call fn on every element of this task queue. - // fn: bool operator()(const Task &task); - // If operator() returns false, for_each() would stop immediately. - template - int for_each(Function &fn); -private: - void push_into_queue_(void *task); - DISALLOW_COPY_AND_ASSIGN(ObSafeDestroyTaskQueue); -protected: - bool looping_; - // stop push into new task but not stop process exist task. - bool stop_; - - // protect queue - RWLock rwlock_; - // the queue of task - common::ObLightyQueue queue_; -}; - -template -int ObSafeDestroyTaskQueue::for_each(Function &fn) -{ - int ret = OB_SUCCESS; - int64_t timeout = 0; // no wait - void *task = nullptr; - ObSafeDestroyTask *safe_destroy_task = nullptr; - bool loop_finish = false; - // make sure no one can push into the queue - WRLockGuard guard(rwlock_); - int64_t begin_loop_timestamp = ObTimeUtility::current_time(); - looping_ = true; - while (!loop_finish) { - // push a task to break if we loop all the task one time. - if (OB_FAIL(queue_.pop(task, timeout))) { - // there is no task now. just stop. - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - loop_finish = true; - } else { - SERVER_LOG(WARN, "queue pop task fail", K(ret), K(&queue_)); - } - } else if (OB_ISNULL(task)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(ERROR, "queue pop successfully but task is NULL", K(ret), K(task)); - } else { - bool need_requeue = true; - safe_destroy_task = reinterpret_cast(task); - if (safe_destroy_task->get_last_execute_timestamp() >= begin_loop_timestamp) { - // loop all the task in the queue one time, break. - loop_finish = true; - } else if (!fn(safe_destroy_task, need_requeue)) { - // no need process again. - loop_finish = true; - } else { - // do nothing - } - // push the task into the queue again. - if (need_requeue) { - push_into_queue_(task); - safe_destroy_task->update_execute_timestamp(); - } - } - } // loop one time - looping_ = false; - return ret; -} - -class ObSafeDestroyHandler -{ - const static int64_t TASK_SCHEDULER_INTERVAL = 2 * 1000 * 1000; // 2_s -public: - ObSafeDestroyHandler(); - ~ObSafeDestroyHandler(); - int stop(); - void wait(); - void destroy(); - // add a task into the thread - // task will be execute every TASK_SCHEDULER_INTERVAL - static OB_INLINE ObSafeDestroyHandler &get_instance(); - // For each. - // Call fn on every element of this task queue. - // fn: bool operator()(const Task &task); - // If operator() returns false, for_each() would stop immediately. - template - int for_each(Function &fn); -private: - bool is_inited_; - // push the safe destroy task into the queue. - ObSafeDestroyTaskQueue queue_; - int64_t last_process_timestamp_; -}; - -template -int ObSafeDestroyHandler::for_each(Function &fn) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - SERVER_LOG(WARN, "safe destroy thread not inited", K(ret)); - } else if (OB_FAIL(queue_.for_each(fn))) { - SERVER_LOG(WARN, "push safe destroy task failed", K(ret)); - } else { - // do nothing - } - return ret; -} - - -} // observer -} // oceanbase -#endif diff --git a/src/storage/tx_storage/ob_tablet_gc_service.cpp b/src/storage/tx_storage/ob_tablet_gc_service.cpp index 3b94965bd..1614ef871 100755 --- a/src/storage/tx_storage/ob_tablet_gc_service.cpp +++ b/src/storage/tx_storage/ob_tablet_gc_service.cpp @@ -18,6 +18,7 @@ #include "ob_tablet_gc_service.h" #include "share/rc/ob_module_provider.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/tablet/ob_tablet_iterator.h" #include "logservice/ob_log_service.h" #include "storage/meta_store/ob_tenant_storage_meta_service.h" @@ -114,37 +115,22 @@ void ObTabletGCService::ObTabletChangeTask::runTimerTask() RLOCAL_STATIC(int64_t, times) = 0; times = (times + 1) % GLOBAL_GC_CHECK_INTERVAL_TIMES; int ret = OB_SUCCESS; - ObLSIterator *iter = NULL; - common::ObSharedGuard guard; - ObLSService *ls_svr = share::g_mp->ls_service(); + ObLS *ls = nullptr; bool skip_gc_task = false; skip_gc_task = (OB_SUCCESS != (OB_E(EventTable::EN_TABLET_GC_TASK_FAILED) OB_SUCCESS)); - if (OB_ISNULL(ls_svr)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "mtl ObLSService should not be null", KR(ret)); - } else if (OB_UNLIKELY(skip_gc_task)) { + if (OB_UNLIKELY(skip_gc_task)) { // do nothing - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - STORAGE_LOG(WARN, "get log stream iter failed", KR(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "iter is NULL", KR(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + STORAGE_LOG(WARN, "get log stream failed", KR(ret)); } else { - ObLS *ls = NULL; - int ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - ObTabletGCHandler *tablet_gc_handler = NULL; - if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is NULL", KR(ret)); - } else if (FALSE_IT(tablet_gc_handler = ls->get_tablet_gc_handler())) { - } else if (tablet_gc_handler->check_stop()) { - STORAGE_LOG(INFO, "[tabletchange] tablet_gc_handler is offline", K(ls->get_ls_id())); + ObTabletGCHandler *const tablet_gc_handler = ls->get_tablet_gc_handler(); + if (tablet_gc_handler->check_stop()) { + STORAGE_LOG(INFO, "[tabletchange] tablet_gc_handler is offline"); } else { uint8_t tablet_persist_trigger = tablet_gc_handler->get_tablet_persist_trigger_and_reset(); - STORAGE_LOG(INFO, "[tabletchange] task check ls", K(ls->get_ls_id()), K(tablet_persist_trigger)); + STORAGE_LOG(INFO, "[tabletchange] task check ls", K(tablet_persist_trigger)); if (times == 0 || ObTabletGCHandler::is_set_tablet_persist_trigger(tablet_persist_trigger) || ObTabletGCHandler::is_tablet_gc_trigger(tablet_persist_trigger)) { @@ -162,18 +148,18 @@ void ObTabletGCService::ObTabletChangeTask::runTimerTask() if (OB_ISNULL(freezer)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "freezer should not null", K(ls->get_ls_id()), KR(ret)); + STORAGE_LOG(WARN, "freezer should not null", KR(ret)); } // 1. get minor merge point else if (OB_FAIL(ls->get_max_decided_scn(decided_scn))) { need_retry = true; - STORAGE_LOG(WARN, "decide_max_decided_scn failed", KR(ret), K(freezer->get_ls_id())); + STORAGE_LOG(WARN, "decide_max_decided_scn failed", KR(ret)); } else if (!decided_scn.is_valid() || SCN::min_scn() == decided_scn || decided_scn < ls->get_tablet_change_checkpoint_scn()) { need_retry = true; STORAGE_LOG(INFO, "no any log callback and no need to update clog checkpoint", - K(freezer->get_ls_id()), K(decided_scn), KPC(ls), K(ls->get_ls_meta())); + K(decided_scn), KPC(ls), K(ls->get_ls_meta())); } // 2. get gc tablet and get unpersist_tablet_ids else if (OB_FAIL(tablet_gc_handler->get_unpersist_tablet_ids(deleted_tablets, @@ -212,12 +198,7 @@ void ObTabletGCService::ObTabletChangeTask::runTimerTask() need_retry = true; STORAGE_LOG(WARN, "failed to set_tablet_change_checkpoint_scn", KPC(ls), KR(ret), K(decided_scn)); } - // 8. set ls reserved scn - else if (!only_persist && OB_FAIL(tablet_gc_handler->set_ls_reserved_scn(deleted_tablets))) { - need_retry = true; - STORAGE_LOG(ERROR, "failed to set ls reserved scn", KPC(ls), KR(ret), K(decided_scn)); - } - // 9. check and gc deleted_tablets + // 8. check and gc deleted_tablets else if (!only_persist && !deleted_tablets.empty() && OB_FAIL(tablet_gc_handler->gc_tablets(deleted_tablets))) { need_retry = true; @@ -236,14 +217,8 @@ void ObTabletGCService::ObTabletChangeTask::runTimerTask() } } } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (ls_cnt > 0) { - STORAGE_LOG(INFO, "[tabletchange] succeed to gc_tablet", KR(ret), K(ls_cnt), K(times)); - } else { - STORAGE_LOG(INFO, "[tabletchange] no logstream", KR(ret), K(ls_cnt), K(times)); - } - } + if (OB_SUCC(ret)) { + STORAGE_LOG(INFO, "[tabletchange] succeed to gc_tablet", KR(ret), K(times)); } } @@ -414,7 +389,6 @@ int ObTabletGCHandler::check_tablet_need_gc_( } } else { // for create tablet abort - const share::ObLSID &ls_id = tablet->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; ObTabletCreateDeleteMdsUserData data; mds::MdsWriter writer;// will be removed later @@ -427,9 +401,9 @@ int ObTabletGCHandler::check_tablet_need_gc_( if (OB_EMPTY_RESULT == ret) { ret = OB_SUCCESS; if (!tablet_status_is_written) { - STORAGE_LOG(INFO, "tablet status has not been written", K(ret), K(ls_id), K(tablet_id), KP(tablet), K(need_gc)); + STORAGE_LOG(INFO, "tablet status has not been written", K(ret), K(tablet_id), KP(tablet), K(need_gc)); } else if (OB_FAIL(check_tablet_from_aborted_tx(*tablet, need_gc))) { - STORAGE_LOG(WARN, "failed to check tablet from aborted tx", K(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(WARN, "failed to check tablet from aborted tx", K(ret), K(tablet_id)); } } else { STORAGE_LOG(WARN, "failed to get CreateDeleteMdsUserData", KR(ret), KPC(tablet)); @@ -444,20 +418,19 @@ int ObTabletGCHandler::check_tablet_need_gc_( int ObTabletGCHandler::check_tablet_from_aborted_tx(const ObTablet &tablet, TabletGCStatus &gc_status) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; share::SCN rec_scn; if (OB_FAIL(tablet.get_mds_table_rec_scn(rec_scn))) { - STORAGE_LOG(WARN, "failed to get mds table rec scn", K(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(WARN, "failed to get mds table rec scn", K(ret), K(tablet_id)); } else if (rec_scn.is_max()) { gc_status = TabletGCStatus::NEED_GC_IMMEDIATELY; STORAGE_LOG(INFO, "mds table rec scn is MAX, redo log has NOT been written, should delete tablet instantly", - K(ret), K(ls_id), K(tablet_id), K(rec_scn), K(gc_status)); + K(ret), K(tablet_id), K(rec_scn), K(gc_status)); } else { gc_status = TabletGCStatus::NOT_NEED_GC; STORAGE_LOG(INFO, "mds table rec scn is NOT MAX, redo log has been written, try to convert to empty shell", - K(ret), K(ls_id), K(tablet_id), K(rec_scn), K(gc_status)); + K(ret), K(tablet_id), K(rec_scn), K(gc_status)); } return ret; @@ -613,11 +586,11 @@ int ObTabletGCHandler::wait_unpersist_tablet_ids_flushed(const common::ObTabletI if (OB_FAIL(ret)) { } else if (rec_scn > decided_scn) { STORAGE_LOG(INFO, "[tabletgc] finish tablet flush", K(rec_scn), K(decided_scn), - K(retry_times), K(i), K(ls_->get_ls_id()), KPC(tablet)); + K(retry_times), K(i), KPC(tablet)); i++; } else { STORAGE_LOG(INFO, "[tabletgc] wait tablet flush", K(rec_scn), K(decided_scn), - K(retry_times), K(i), K(ls_->get_ls_id()), KPC(tablet)); + K(retry_times), K(i), KPC(tablet)); break; } } @@ -662,12 +635,10 @@ int ObTabletGCHandler::gc_tablets(const common::ObIArray &delete } else { #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("tablet_gc", "gc_tablet_finish", - "ls_id", tablet_handle.get_obj()->get_tablet_meta().ls_id_.id(), "tablet_id", tablet_handle.get_obj()->get_tablet_meta().tablet_id_.id(), "path_id", 0); #endif STORAGE_LOG(INFO, "gc tablet finish", K(ret), - "ls_id", tablet_handle.get_obj()->get_tablet_meta().ls_id_, "tablet_id", tablet_handle.get_obj()->get_tablet_meta().tablet_id_); } } @@ -680,67 +651,6 @@ int ObTabletGCHandler::gc_tablets(const common::ObIArray &delete return ret; } -int ObTabletGCHandler::get_max_tablet_reserved_scn( - const common::ObIArray &deleted_tablets, - share::SCN &reserved_scn) -{ - int ret = OB_SUCCESS; - const bool need_initial_state = false; - const bool need_sorted_tablet_id = false; - ObHALSTabletIDIterator iter(ls_->get_ls_id(), need_initial_state, need_sorted_tablet_id); - share::SCN max_reserved_scn = share::SCN::min_scn(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "tablet gc handler is not inited", KR(ret)); - } else { - ObTabletCreateDeleteMdsUserData mds_data; - ObTablet *tablet = nullptr; - for (int i = 0; OB_SUCC(ret) && i < deleted_tablets.count(); i++) { - mds_data.reset(); - tablet = deleted_tablets.at(i).get_obj(); - if (OB_ISNULL(tablet)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet is null", K(ret), K(i), "tablet_handle", deleted_tablets.at(i)); - } else if (OB_FAIL(tablet->get_latest_committed(mds_data))) { - if (OB_EMPTY_RESULT == ret) { - ret = OB_SUCCESS; - LOG_INFO("create tablet abort, need gc", K(ret), - "ls_id", tablet->get_tablet_meta().ls_id_, - "tablet_id", tablet->get_tablet_meta().tablet_id_); - continue; - } else { - LOG_WARN("failed to get mds table", KR(ret), - "ls_id", tablet->get_tablet_meta().ls_id_, - "tablet_id", tablet->get_tablet_meta().tablet_id_); - } - } else if (share::SCN::invalid_scn() == mds_data.reserved_scn_) { - // do nothing - } else { - reserved_scn = mds_data.reserved_scn_; - max_reserved_scn = MAX(reserved_scn, max_reserved_scn); - } - } - if (OB_SUCC(ret)) { - reserved_scn = max_reserved_scn; - } - } - return ret; -} - -int ObTabletGCHandler::set_ls_reserved_scn(const common::ObIArray &deleted_tablets) -{ - int ret = OB_SUCCESS; - share::SCN tablet_max_reserved_scn; - if (0 == deleted_tablets.count()) { - // do nothing - } else if (OB_FAIL(get_max_tablet_reserved_scn(deleted_tablets, tablet_max_reserved_scn))) { - STORAGE_LOG(WARN, "failed to get max tablet reserved scn", K(ret)); - } else if (OB_FAIL(ls_->inc_update_reserved_scn(tablet_max_reserved_scn))) { - LOG_WARN("failed to set reserved scn", K(ret)); - } - return ret; -} - int ObTabletGCHandler::offline() { int ret = OB_SUCCESS; diff --git a/src/storage/tx_storage/ob_tablet_gc_service.h b/src/storage/tx_storage/ob_tablet_gc_service.h index 096cb4c34..8ed9e3291 100755 --- a/src/storage/tx_storage/ob_tablet_gc_service.h +++ b/src/storage/tx_storage/ob_tablet_gc_service.h @@ -96,8 +96,6 @@ class ObTabletGCHandler const bool only_persist, bool &need_retry, bool &no_need_wait_persist); - int get_max_tablet_reserved_scn(const common::ObIArray &deleted_tablets, share::SCN &reserved_scn); - int set_ls_reserved_scn(const common::ObIArray &deleted_tablets); int gc_tablets(const common::ObIArray &deleted_tablets); bool check_stop() { return ATOMIC_LOAD(&update_enabled_) == false; } int disable_gc(); diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index a81f99d13..03477cb8e 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -65,6 +65,7 @@ ObTenantFreezer::~ObTenantFreezer() void ObTenantFreezer::destroy() { freeze_trigger_timer_.destroy(); + freeze_thread_pool_.thread_pool_.reset(); is_freezing_tx_data_ = false; self_.reset(); freezer_stat_.reset(); @@ -135,6 +136,9 @@ int ObTenantFreezer::stop() LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); } else { freeze_trigger_timer_.stop(); + if (freeze_thread_pool_.thread_pool_.is_valid()) { + freeze_thread_pool_.thread_pool_->stop(); + } // task_list_.stop_all(); LOG_INFO("[TenantFreezer] ObTenantFreezer stoped done", K_(tenant_info)); } @@ -144,6 +148,9 @@ int ObTenantFreezer::stop() void ObTenantFreezer::wait() { freeze_trigger_timer_.wait(); + if (freeze_thread_pool_.thread_pool_.is_valid()) { + freeze_thread_pool_.thread_pool_->wait(); + } // task_list_.wait_all(); LOG_INFO("[TenantFreezer] ObTenantFreezer wait done", K_(tenant_info)); } @@ -152,28 +159,22 @@ bool ObTenantFreezer::exist_ls_freezing() { int ret = OB_SUCCESS; bool exist_ls_freezing = false; - common::ObSharedGuard iter; + ObLS *ls = nullptr; ObLSService *ls_srv = share::g_mp->ls_service(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); + } else if (OB_ISNULL(ls_srv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] ls service is null", KR(ret)); + } else if (OB_FAIL(ls_srv->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", KR(ret)); } else { - ObLS *ls = nullptr; - while (OB_SUCC(iter->get_next(ls))) { - if (ls->get_freezer()->is_ls_freeze_running()) { - exist_ls_freezing = true; - break; - } - } - - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - } - - if (OB_FAIL(ret)) { - LOG_WARN("[TenantFreezer] iter ls failed", K(ret)); + if (OB_ISNULL(ls->get_freezer())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] freezer is null", KR(ret)); + } else { + exist_ls_freezing = ls->get_freezer()->is_ls_freeze_running(); } } @@ -190,27 +191,22 @@ bool ObTenantFreezer::exist_ls_throttle_is_skipping() ATOMIC_BCAS(&throttle_is_skipping_cache_.update_ts_, last_update_ts, cur_ts)) { bool exist_ls_throttle_is_skipping = false; - common::ObSharedGuard iter; + ObLS *ls = nullptr; ObLSService *ls_srv = share::g_mp->ls_service(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); + } else if (OB_ISNULL(ls_srv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] ls service is null", KR(ret)); + } else if (OB_FAIL(ls_srv->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", KR(ret)); } else { - ObLS *ls = nullptr; - while (OB_SUCC(iter->get_next(ls))) { - if (ls->get_freezer()->throttle_is_skipping()) { - exist_ls_throttle_is_skipping = true; - break; - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - } - - if (OB_FAIL(ret)) { - LOG_WARN("[TenantFreezer] iter ls failed", K(ret)); + if (OB_ISNULL(ls->get_freezer())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] freezer is null", KR(ret)); + } else { + exist_ls_throttle_is_skipping = ls->get_freezer()->throttle_is_skipping(); } } @@ -342,38 +338,27 @@ int ObTenantFreezer::tenant_freeze_data_() { int ret = OB_SUCCESS; int first_fail_ret = OB_SUCCESS; - common::ObSharedGuard iter; - ObLSService *ls_srv = share::g_mp->ls_service(); + ObLS *ls = nullptr; FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret)); ObTenantFreezeGuard freeze_guard(ret, tenant_info_); - if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", KR(ret)); } else { - ObLS *ls = nullptr; - int ls_cnt = 0; - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - // wait until this ls freeze finished to make sure not freeze frequently because - // of this ls freeze stuck. - if (OB_FAIL(ls_freeze_data_(ls))) { - if (OB_SUCCESS == first_fail_ret) { - first_fail_ret = ret; - } - LOG_ERROR("[TenantFreezer] fail to freeze logstream", KR(ret), K(ls->get_ls_id())); - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (ls_cnt > 0) { - LOG_INFO("[TenantFreezer] succeed to freeze tenant", KR(ret), K(ls_cnt)); - } else { - LOG_WARN("[TenantFreezer] no logstream", KR(ret), K(ls_cnt)); + // Wait for the freeze to finish to avoid repeatedly freezing a stuck LS. + if (OB_FAIL(ls_freeze_data_(ls))) { + if (OB_SUCCESS == first_fail_ret) { + first_fail_ret = ret; } + LOG_ERROR("[TenantFreezer] fail to freeze logstream", KR(ret)); } if (first_fail_ret != OB_SUCCESS && first_fail_ret != OB_ITER_END) { ret = first_fail_ret; } + if (OB_SUCC(ret)) { + LOG_INFO("[TenantFreezer] succeed to freeze tenant", KR(ret)); + } } return ret; @@ -384,11 +369,7 @@ int ObTenantFreezer::tenant_freeze(const ObFreezeSourceFlag source) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSService *ls_svr = share::g_mp->ls_service(); - common::ObSharedGuard guard; - ObLSIterator *iter = NULL; ObLS *ls = nullptr; - int ls_cnt = 0; int64_t abs_timeout_ts = INT64_MAX; if (IS_NOT_INIT) { @@ -397,21 +378,13 @@ int ObTenantFreezer::tenant_freeze(const ObFreezeSourceFlag source) } else if (OB_FAIL(ObShareUtil::get_abs_timeout(MAX_FREEZE_TIMEOUT_US /* default timeout */, abs_timeout_ts))) { LOG_WARN("get timeout ts failed", KR(ret)); - } else if (OB_FAIL(ls_svr->get_ls_iter(guard, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("get log stream iter failed", K(ret)); - } else if (OB_ISNULL(iter = guard.get_ptr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("iter is NULL", K(ret)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", KR(ret)); } else if (OB_FAIL(set_tenant_freezing_())) { LOG_WARN("set tenant freeze failed", K(ret)); } else { - for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - if (OB_TMP_FAIL(ls_freeze_all_unit_(ls, abs_timeout_ts, source))) { - LOG_WARN("ls freeze all unit failed", K(tmp_ret), K(ls->get_ls_id())); - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; + if (OB_TMP_FAIL(ls_freeze_all_unit_(ls, abs_timeout_ts, source))) { + LOG_WARN("ls freeze all unit failed", K(tmp_ret)); } } @@ -427,36 +400,7 @@ int ObTenantFreezer::tenant_freeze(const ObFreezeSourceFlag source) return ret; } -int ObTenantFreezer::ls_freeze_all_unit(const share::ObLSID &ls_id, - const ObFreezeSourceFlag source) -{ - int ret = OB_SUCCESS; - ObLSService *ls_srv = share::g_mp->ls_service(); - ObLSHandle handle; - ObLS *ls = nullptr; - const bool need_rewrite_tablet_meta = false; - int64_t abs_timeout_ts = 0; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (OB_FAIL(ObShareUtil::get_abs_timeout(MAX_FREEZE_TIMEOUT_US /* default timeout */, - abs_timeout_ts))) { - LOG_WARN("get timeout ts failed", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[TenantFreezer] ls is null", KR(ret), K(ls_id)); - } else if (OB_FAIL(ls_freeze_all_unit_(ls, abs_timeout_ts, source))) { - LOG_WARN("[TenantFreezer] logstream freeze failed", KR(ret), K(ls_id)); - } - - return ret; -} - -int ObTenantFreezer::tablet_freeze(share::ObLSID ls_id, - const common::ObTabletID &tablet_id, +int ObTenantFreezer::tablet_freeze(const common::ObTabletID &tablet_id, const bool is_sync, const int64_t max_retry_time_us, const bool need_rewrite_tablet_meta, @@ -464,15 +408,12 @@ int ObTenantFreezer::tablet_freeze(share::ObLSID ls_id, { int ret = OB_SUCCESS; bool is_cache_hit = false; - ObLSService *ls_srv = share::g_mp->ls_service(); - ObLSHandle handle; ObLS *ls = nullptr; // 0 as default timeout ts const int64_t abs_timeout_ts = (0 == max_retry_time_us) ? 0 : ObClockGenerator::getClock() + max_retry_time_us; FLOG_INFO("[TenantFreezer] tablet_freeze start", KR(ret), - K(ls_id), K(is_sync), K(need_rewrite_tablet_meta), K(tablet_id), @@ -481,19 +422,14 @@ int ObTenantFreezer::tablet_freeze(share::ObLSID ls_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (!ls_id.is_valid()) { - LOG_ERROR("[TenantFreezer] ls id can not be invalid", KR(ret), K(tablet_id)); - } else if (OB_FAIL(ls_srv->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[TenantFreezer] ls is null", KR(ret), K(ls_id)); + } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get ls", K(ret)); } else if (OB_FAIL(ls->tablet_freeze(tablet_id, is_sync, abs_timeout_ts, need_rewrite_tablet_meta, source))) { - LOG_ERROR("[TenantFreezer] fail to freeze tablet", KR(ret), K(ls_id), K(tablet_id)); + LOG_ERROR("[TenantFreezer] fail to freeze tablet", KR(ret), K(tablet_id)); if (OB_NOT_RUNNING == ret) { ret = OB_SUCCESS; } @@ -600,40 +536,23 @@ int ObTenantFreezer::get_tx_data_info_for_freeze_(int64_t &tenant_tx_data_frozen int ret = OB_SUCCESS; tenant_tx_data_frozen_mem_used = 0; tenant_tx_data_active_mem_used = 0; - common::ObSharedGuard iter; + ObLS *ls = nullptr; ObLSService *ls_srv = share::g_mp->ls_service(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); + } else if (OB_ISNULL(ls_srv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] ls service is null", KR(ret)); + } else if (OB_FAIL(ls_srv->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", KR(ret)); } else { - ObLS *ls = nullptr; - int ls_cnt = 0; - for (; OB_SUCC(ret) && OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - int tmp_ret = OB_SUCCESS; - int64_t ls_tx_data_frozen_mem_used = 0; - int64_t ls_tx_data_active_mem_used = 0; - if (!for_statistic_print && OB_NOT_NULL(ls) && ls->tx_table_need_re_freeze()) { - need_re_freeze = true; - break; - } - - if (OB_TMP_FAIL(get_ls_tx_data_memory_info_( - ls, ls_tx_data_frozen_mem_used, ls_tx_data_active_mem_used, for_statistic_print))) { - LOG_WARN("[TenantFreezer] fail to get tx data mem used in one ls", KR(ret), K(ls->get_ls_id())); - } else { - tenant_tx_data_frozen_mem_used += ls_tx_data_frozen_mem_used; - tenant_tx_data_active_mem_used += ls_tx_data_active_mem_used; - } - } - - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (0 == ls_cnt) { - LOG_WARN("[TenantFreezer] no logstream", KR(ret), K(ls_cnt), K(tenant_info_)); - } + if (!for_statistic_print && ls->tx_table_need_re_freeze()) { + need_re_freeze = true; + } else if (OB_FAIL(get_ls_tx_data_memory_info_( + ls, tenant_tx_data_frozen_mem_used, tenant_tx_data_active_mem_used, for_statistic_print))) { + LOG_WARN("[TenantFreezer] fail to get tx data memory usage", KR(ret)); } } return ret; @@ -650,10 +569,7 @@ int ObTenantFreezer::get_ls_tx_data_memory_info_(ObLS *ls, ObTxDataMemtableMgr *memtable_mgr = nullptr; ObSEArray memtable_handles; ObTxDataMemtable *memtable = nullptr; - if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[TenantFreezer] get ls tx data mem used failed.", KR(ret)); - } else if (OB_FAIL(ls->get_tablet_svr()->get_tx_data_memtable_mgr(mgr_handle))) { + if (OB_FAIL(ls->get_tablet_svr()->get_tx_data_memtable_mgr(mgr_handle))) { LOG_WARN("[TenantFreezer] get tx data memtable mgr failed.", KR(ret)); } else if (OB_ISNULL(memtable_mgr = static_cast(mgr_handle.get_memtable_mgr()))) { @@ -684,8 +600,7 @@ int ObTenantFreezer::get_ls_tx_data_memory_info_(ObLS *ls, if (for_statistic_print) { LOG_INFO("TxData Memory Statistic(logstream info): ", - "ls_id", ls->get_ls_id(), - "Frozen TxData Memory(MB)", ls_tx_data_frozen_mem_used/ONE_MB, + "Frozen TxData Memory(MB)", ls_tx_data_frozen_mem_used/ONE_MB, "Active TxData Memory(MB)", ls_tx_data_active_mem_used/ONE_MB); } @@ -757,12 +672,11 @@ int ObTenantFreezer::do_freeze_diagnose() return ret; } -void ObTenantFreezer::record_freezer_source_event(const ObLSID &ls_id, - const ObFreezeSourceFlag source) +void ObTenantFreezer::record_freezer_source_event(const ObFreezeSourceFlag source) { if (is_valid_freeze_source((source))) { ATOMIC_AAF(&freezer_stat_.captured_source_times_[static_cast(source)], 1); - STORAGE_LOG(INFO, "[Freezer] freeze from source", K(ls_id), "freeze_source", obj_to_cstring(source)); + STORAGE_LOG(INFO, "[Freezer] freeze from source", "freeze_source", obj_to_cstring(source)); } } diff --git a/src/storage/tx_storage/ob_tenant_freezer.h b/src/storage/tx_storage/ob_tenant_freezer.h index a7a8fc544..0d757c05d 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.h +++ b/src/storage/tx_storage/ob_tenant_freezer.h @@ -28,7 +28,6 @@ #include "storage/multi_data_source/runtime_utility/mds_factory.h" #include "storage/compaction/ob_compaction_util.h" #include "storage/ls/ob_freezer_define.h" -#include "share/ob_ls_id.h" namespace oceanbase { @@ -146,14 +145,8 @@ struct PeriodicalUpdateValueCache { // freeze all the checkpoint unit of this tenant. int tenant_freeze(const ObFreezeSourceFlag source); - // freeze a ls, if the ls is freezing, do nothing and return OB_ENTRY_EXIST. - // if there is some process hold the ls lock or a OB_EAGAIN occur, we will retry - // until timeout. - int ls_freeze_all_unit(const share::ObLSID &ls_id, - const ObFreezeSourceFlag source); // freeze a tablet - int tablet_freeze(share::ObLSID ls_id, - const common::ObTabletID &tablet_id, + int tablet_freeze(const common::ObTabletID &tablet_id, const bool is_sync, const int64_t max_retry_time, const bool need_rewrite_tablet_meta, @@ -165,8 +158,7 @@ struct PeriodicalUpdateValueCache { int do_freeze_diagnose(); // record freeze source history - void record_freezer_source_event(const share::ObLSID &ls_id, - const ObFreezeSourceFlag source); + void record_freezer_source_event(const ObFreezeSourceFlag source); // report freeze source history void report_freezer_source_events(); @@ -277,7 +269,7 @@ struct PeriodicalUpdateValueCache { ObLS *ls, const int64_t abs_timeout_ts = INT64_MAX, const ObFreezeSourceFlag source = ObFreezeSourceFlag::INVALID_SOURCE); - // freeze all the ls of this tenant. + // Freeze the tenant's log stream. // return the first failed code. int tenant_freeze_data_(); // we can only deal with freeze one by one. @@ -387,8 +379,6 @@ class ObTenantTxDataFreezeGuard ObTenantFreezer *tenant_freezer_; }; -// Validates ls_id and allocates a tenant-wide monotonic checkpoint batch trace id (via MTL). - } // namespace storage } // namespace oceanbase diff --git a/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp b/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp index 2325b8685..136d0225a 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp @@ -69,43 +69,26 @@ static int do_tx_data_table_freeze_(const ObTenantFreezeArg &arg) LOG_INFO("start tx data table self freeze task in rpc handle thread", K(arg)); - common::ObSharedGuard iter_guard; + ObLS *ls = nullptr; ObTenantTxDataFreezeGuard tenant_freeze_guard; ObLSService *ls_srv = share::g_mp->ls_service(); ObTenantFreezer *freezer = share::g_mp->tenant_freezer(); if (OB_FAIL(tenant_freeze_guard.init(freezer))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", K(ret)); + LOG_WARN("[TenantFreezer] fail to init tenant freeze guard", K(ret)); } else if (!tenant_freeze_guard.can_freeze()) { // skip tx data self freeze due to another freeze task is running - } else if (OB_FAIL(ls_srv->get_ls_iter(iter_guard, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", K(ret)); + } else if (OB_ISNULL(ls_srv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] ls service is null", K(ret)); + } else if (OB_FAIL(ls_srv->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", K(ret)); } else { - int ls_cnt = 0; - while (OB_SUCC(ret)) - { - ObTxTableGuard tx_table_guard; - ObLS *ls = nullptr; - if (OB_FAIL(iter_guard->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next ls failed.", KR(ret), K(arg)); - } - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected nullptr", KR(ret), K(arg)); - } else if (OB_FAIL(ls->get_tx_table_guard(tx_table_guard))) { - LOG_WARN("get tx table guard failed.", KR(ret), K(arg)); - } else if (OB_FAIL(tx_table_guard.self_freeze_task())) { - LOG_WARN("freeze tx data table failed.", KR(ret), K(arg)); - } - ++ls_cnt; - } - - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (0 == ls_cnt) { - LOG_WARN("[TenantFreezer] no logstream", K(ret), K(ls_cnt)); - } + ObTxTableGuard tx_table_guard; + if (OB_FAIL(ls->get_tx_table_guard(tx_table_guard))) { + LOG_WARN("get tx table guard failed.", KR(ret), K(arg)); + } else if (OB_FAIL(tx_table_guard.self_freeze_task())) { + LOG_WARN("freeze tx data table failed.", KR(ret), K(arg)); } } @@ -167,39 +150,20 @@ static int do_mds_table_freeze_(const ObTenantFreezeArg &arg) int ret = OB_SUCCESS; LOG_INFO("start mds table self freeze task in rpc handle thread", K(arg)); - common::ObSharedGuard iter_guard; + ObLS *ls = nullptr; ObLSService *ls_srv = share::g_mp->ls_service(); - if (OB_FAIL(ls_srv->get_ls_iter(iter_guard, ObLSGetMod::TXSTORAGE_MOD))) { - LOG_WARN("[TenantFreezer] fail to get log stream iterator", K(ret)); + if (OB_ISNULL(ls_srv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[TenantFreezer] ls service is null", K(ret)); + } else if (OB_FAIL(ls_srv->get_ls(ls))) { + LOG_WARN("[TenantFreezer] fail to get log stream", K(ret)); } else { - int ls_cnt = 0; - while (OB_SUCC(ret)) { - ObLS *ls = nullptr; - MdsTableMgrHandle mgr_handle; - ObMdsTableMgr *mds_table_mgr = nullptr; - - if (OB_FAIL(iter_guard->get_next(ls))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next ls failed.", KR(ret), K(arg)); - } - } else if (OB_ISNULL(ls) || OB_ISNULL(ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls is unexpected nullptr", KR(ret), K(arg)); - } else { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ls->flush_mds_table(INT64_MAX))) { - LOG_WARN("flush mds table failed", KR(tmp_ret), KPC(ls)); - } - } - ++ls_cnt; - } - - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - if (0 == ls_cnt) { - LOG_WARN("[TenantFreezer] no logstream", K(ret), K(ls_cnt)); - } + if (OB_ISNULL(ls->get_tablet_svr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet service is null", KR(ret), K(arg)); + } else if (OB_FAIL(ls->flush_mds_table(INT64_MAX))) { + LOG_WARN("flush mds table failed", KR(ret), KPC(ls)); } } diff --git a/src/storage/tx_storage/ob_tx_leak_checker.h b/src/storage/tx_storage/ob_tx_leak_checker.h index 31da3597b..7259c5de5 100644 --- a/src/storage/tx_storage/ob_tx_leak_checker.h +++ b/src/storage/tx_storage/ob_tx_leak_checker.h @@ -18,7 +18,6 @@ #include "share/leak_checker/ob_leak_checker.h" #include "share/rc/ob_module_provider.h" #include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" #include "lib/profile/ob_trace_id.h" #include "storage/ob_common_id_utils.h" namespace oceanbase @@ -108,18 +107,15 @@ struct ObReadOnlyTxCheckerValue public: ObReadOnlyTxCheckerValue() : timestamp_(0), - ls_id_(), tablet_id_(), extra_(nullptr) { } ~ObReadOnlyTxCheckerValue() = default; - TO_STRING_KV(K_(timestamp), K_(ls_id), - K_(tablet_id), KPC_(extra)); + TO_STRING_KV(K_(timestamp), K_(tablet_id), KPC_(extra)); public: int64_t timestamp_; - share::ObLSID ls_id_; common::ObTabletID tablet_id_; ObReadExInfo *extra_; }; @@ -129,90 +125,76 @@ struct ObReadOnlyTxPrinter bool operator()(const ObReadOnlyTxCheckerKey &k, const ObReadOnlyTxCheckerValue &v) { bool ret = true; - if (true && v.ls_id_ == ls_id_) { - if (OB_ISNULL(v.extra_)) { - COMMON_LOG(INFO, "LEAK_CHECKER ", - "key", k, - "value", v); - } else if (OB_NOT_NULL(v.extra_)) { - if (v.extra_->type_ >= ObReadExInfo::WITH_LBT) { - ObReadExInfoBT *extra_info = static_cast(v.extra_); - COMMON_LOG(INFO, "LEAK_CHECKER ", - "key", k, - "value", v, - KPC(extra_info)); - } else if (v.extra_->type_ >= ObReadExInfo::WITH_TRACE) { - ObReadExInfoTrace *extra_info = static_cast(v.extra_); - COMMON_LOG(INFO, "LEAK_CHECKER ", - "key", k, - "value", v, - KPC(extra_info)); - } else if (v.extra_->type_ >= ObReadExInfo::WITH_PLAN) { - ObReadExInfoPlan *extra_info = static_cast(v.extra_); - COMMON_LOG(INFO, "LEAK_CHECKER ", + if (OB_ISNULL(v.extra_)) { + COMMON_LOG(INFO, "LEAK_CHECKER ", "key", k, - "value", v, - KPC(extra_info)); - } + "value", v); + } else if (OB_NOT_NULL(v.extra_)) { + if (v.extra_->type_ >= ObReadExInfo::WITH_LBT) { + ObReadExInfoBT *extra_info = static_cast(v.extra_); + COMMON_LOG(INFO, "LEAK_CHECKER ", + "key", k, + "value", v, + KPC(extra_info)); + } else if (v.extra_->type_ >= ObReadExInfo::WITH_TRACE) { + ObReadExInfoTrace *extra_info = static_cast(v.extra_); + COMMON_LOG(INFO, "LEAK_CHECKER ", + "key", k, + "value", v, + KPC(extra_info)); + } else if (v.extra_->type_ >= ObReadExInfo::WITH_PLAN) { + ObReadExInfoPlan *extra_info = static_cast(v.extra_); + COMMON_LOG(INFO, "LEAK_CHECKER ", + "key", k, + "value", v, + KPC(extra_info)); } } return ret; } -public: - - share::ObLSID ls_id_; }; -struct DiagnoseFunctor +struct ObReadOnlyTxDiagnoseFunctor { - DiagnoseFunctor(const share::ObLSID ls_id, - char *buf, - const int64_t pos, - const int64_t len) - : ls_id_(ls_id), - buf_(buf), pos_(pos), len_(len) + ObReadOnlyTxDiagnoseFunctor(char *buf, + const int64_t pos, + const int64_t len) + : buf_(buf), pos_(pos), len_(len) {} - ~DiagnoseFunctor() {} + ~ObReadOnlyTxDiagnoseFunctor() {} bool operator()(const ObReadOnlyTxCheckerKey &k, const ObReadOnlyTxCheckerValue &v) { + UNUSED(k); bool bool_ret = true; int ret = OB_SUCCESS; - if (true && v.ls_id_ == ls_id_) { - if (OB_FAIL(databuff_printf(buf_, len_, pos_, "{tablet_id:%ld", - v.tablet_id_.id()))){ - // break the print - bool_ret = false; - } - if (OB_SUCC(ret) && OB_NOT_NULL(v.extra_)) { - if (OB_SUCC(ret) && v.extra_->type_ >= ObReadExInfo::WITH_PLAN) { - ObReadExInfoPlan *info = static_cast(v.extra_); - if (OB_FAIL(databuff_printf(buf_, len_, pos_, ", plan_id:%ld", - info->plan_id_))){ - // break the print - bool_ret = false; - } - } - if (OB_SUCC(ret) && v.extra_->type_ >= ObReadExInfo::WITH_TRACE) { - ObReadExInfoTrace *info = static_cast(v.extra_); - if (OB_FAIL(databuff_print_multi_objs(buf_, len_, pos_, ", trace:", - info->trace_id_))){ - // break the print - bool_ret = false; - } + if (OB_FAIL(databuff_printf(buf_, len_, pos_, "{tablet_id:%ld", + v.tablet_id_.id()))){ + bool_ret = false; + } + if (OB_SUCC(ret) && OB_NOT_NULL(v.extra_)) { + if (OB_SUCC(ret) && v.extra_->type_ >= ObReadExInfo::WITH_PLAN) { + ObReadExInfoPlan *info = static_cast(v.extra_); + if (OB_FAIL(databuff_printf(buf_, len_, pos_, ", plan_id:%ld", + info->plan_id_))){ + bool_ret = false; } } - if (OB_SUCC(ret)) { - if (OB_FAIL(databuff_printf(buf_, len_, pos_, "}, "))){ - // break the print + if (OB_SUCC(ret) && v.extra_->type_ >= ObReadExInfo::WITH_TRACE) { + ObReadExInfoTrace *info = static_cast(v.extra_); + if (OB_FAIL(databuff_print_multi_objs(buf_, len_, pos_, ", trace:", + info->trace_id_))){ bool_ret = false; } } } + if (OB_SUCC(ret)) { + if (OB_FAIL(databuff_printf(buf_, len_, pos_, "}, "))){ + bool_ret = false; + } + } return bool_ret; } public: - - share::ObLSID ls_id_; char *buf_; int64_t pos_; int64_t len_; @@ -247,7 +229,6 @@ typedef share::ObBaseLeakCheckertrans_service()->get_unique_seq(); \ value.timestamp_ = ObClockGenerator::getClock(); \ - value.ls_id_ = ctx.ls_id_; \ value.tablet_id_ = ctx.tablet_id_; \ ctx.check_seq_ = key.seq_; \ if (OB_UNLIKELY(tx_debug_level >= 4)) { \ @@ -290,10 +271,9 @@ typedef share::ObBaseLeakCheckertrans_service()->get_read_tx_checker().for_each(fn); \ } while(0) diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.cpp b/src/storage/tx_table/ob_tx_ctx_memtable.cpp index 822183454..c23282c5e 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable.cpp @@ -44,15 +44,13 @@ ObTxCtxMemtable::~ObTxCtxMemtable() void ObTxCtxMemtable::reset() { ls_ctx_mgr_guard_.reset(); - ls_id_.reset(); ObITable::reset(); is_frozen_ = false; max_end_scn_.set_min(); is_inited_ = false; } -int ObTxCtxMemtable::init(const ObITable::TableKey &table_key, - const ObLSID &ls_id) +int ObTxCtxMemtable::init(const ObITable::TableKey &table_key) { int ret = OB_SUCCESS; @@ -61,13 +59,12 @@ int ObTxCtxMemtable::init(const ObITable::TableKey &table_key, STORAGE_LOG(WARN, "init tx ctx memtable twice", KR(ret)); } else if (OB_FAIL(ObITable::init(table_key))) { STORAGE_LOG(WARN, "ObITable::init fail"); - } else if (OB_FAIL(ls_ctx_mgr_guard_.init(ls_id))) { - STORAGE_LOG(WARN, "ls ctx mgr guard acquire ref failed", K(ret), K(ls_id)); + } else if (OB_FAIL(ls_ctx_mgr_guard_.init())) { + STORAGE_LOG(WARN, "ls ctx mgr guard acquire ref failed", K(ret)); } else { - ls_id_ = ls_id; max_end_scn_.set_min(); is_inited_ = true; - TRANS_LOG(INFO, "ob tx ctx memtable init successfully", K(ls_id), K(table_key)); + TRANS_LOG(INFO, "ob tx ctx memtable init successfully", K(table_key)); } return ret; @@ -248,7 +245,6 @@ int ObTxCtxMemtable::flush(SCN recycle_scn, bool need_freeze) if (OB_SUCC(ret) && is_frozen_memtable()) { compaction::ObTabletMergeDagParam param; - param.ls_id_ = ls_id_; param.tablet_id_ = LS_TX_CTX_TABLET; param.merge_type_ = compaction::MINI_MERGE; param.merge_version_ = ObVersionRange::MIN_VERSION; @@ -257,7 +253,7 @@ int ObTxCtxMemtable::flush(SCN recycle_scn, bool need_freeze) TRANS_LOG(ERROR, "failed to schedule tablet merge dag", K(ret)); } } else { - TRANS_LOG(INFO, "tx ctx memtable flush successfully", KPC(this), K(ls_id_)); + TRANS_LOG(INFO, "tx ctx memtable flush successfully", KPC(this)); } } diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.h b/src/storage/tx_table/ob_tx_ctx_memtable.h index 7d49e9c6e..0da6de932 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.h +++ b/src/storage/tx_table/ob_tx_ctx_memtable.h @@ -47,7 +47,7 @@ class ObTxCtxMemtable : public ObIMemtable, public checkpoint::ObCommonCheckpoin ObTxCtxMemtable(); ~ObTxCtxMemtable(); - int init(const ObITable::TableKey &table_key, const share::ObLSID &ls_id); + int init(const ObITable::TableKey &table_key); void reset(); int on_memtable_flushed() override; @@ -102,7 +102,7 @@ class ObTxCtxMemtable : public ObIMemtable, public checkpoint::ObCommonCheckpoin void set_max_end_scn(const share::SCN end_scn); INHERIT_TO_STRING_KV("ObITable", ObITable, KP(this), K_(snapshot_version), - K_(ls_id), K_(is_frozen), K_(max_end_scn)); + K_(is_frozen), K_(max_end_scn)); private: bool is_inited_; diff --git a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp index 6a12c365e..9dce7c28e 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp @@ -30,8 +30,7 @@ namespace storage { ObTxCtxMemtableMgr::ObTxCtxMemtableMgr() - : ls_id_(), - lock_def_() + : lock_def_() { lock_.lock_type_ = LockType::OB_SPIN_RWLOCK; lock_.lock_ = &lock_def_; @@ -47,18 +46,16 @@ void ObTxCtxMemtableMgr::destroy() int ret = OB_SUCCESS; const int64_t ref_cnt = get_ref(); if (OB_UNLIKELY(0 != ref_cnt)) { - LOG_ERROR("ref cnt is NOT 0", K(ret), K(ref_cnt), K_(ls_id), KPC(this)); + LOG_ERROR("ref cnt is NOT 0", K(ret), K(ref_cnt), KPC(this)); } MemMgrWLockGuard lock_guard(lock_); reset_tables(); - ls_id_.reset(); freezer_ = NULL; is_inited_ = false; } int ObTxCtxMemtableMgr::init(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) { @@ -67,11 +64,10 @@ int ObTxCtxMemtableMgr::init(const common::ObTabletID &tablet_id, int ret = OB_SUCCESS; - ls_id_ = ls_id; freezer_ = freezer; t3m_ = t3m; is_inited_ = true; - LOG_INFO("tx ctx memtable mgr init successfully", K(ls_id), K(tablet_id), K(this)); + LOG_INFO("tx ctx memtable mgr init successfully", K(tablet_id), K(this)); return ret; } @@ -95,7 +91,7 @@ int ObTxCtxMemtableMgr::create_memtable(const CreateMemtableArg &arg) // TODO: Donot use pool to create the only memtable if (get_memtable_count_() > 0) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tx ctx memtable already exists, should not create again", K(ret), K_(ls_id)); + LOG_WARN("tx ctx memtable already exists, should not create again", K(ret)); } else if (OB_FAIL(t3m_->acquire_tx_ctx_memtable(handle))) { LOG_WARN("failed to create memtable", K(ret)); } else if (OB_ISNULL(table = handle.get_table())) { @@ -106,7 +102,7 @@ int ObTxCtxMemtableMgr::create_memtable(const CreateMemtableArg &arg) if (NULL == tx_ctx_memtable) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid tx_ctx_memtable", K(ret), KPC(table)); - } else if (OB_FAIL(tx_ctx_memtable->init(table_key, ls_id_))) { + } else if (OB_FAIL(tx_ctx_memtable->init(table_key))) { LOG_WARN("memtable init fail.", KR(ret)); } else if (OB_FAIL(add_memtable_(handle))) { LOG_WARN("add memtable fail.", KR(ret)); @@ -114,10 +110,9 @@ int ObTxCtxMemtableMgr::create_memtable(const CreateMemtableArg &arg) ret = OB_ERR_UNEXPECTED; LOG_WARN("ls_tx_svr is null", K(ret)); } else if (OB_FAIL(ls_tx_svr->register_common_checkpoint(checkpoint::TX_CTX_MEMTABLE_TYPE, tx_ctx_memtable))) { - LOG_WARN("tx ctx memtable register_common_checkpoint failed", K(ret), K(ls_id_)); + LOG_WARN("tx ctx memtable register_common_checkpoint failed", K(ret)); } else { - LOG_INFO("tx ctx memtable mgr create memtable successfully", - K(ls_id_), KPC(tx_ctx_memtable)); + LOG_INFO("tx ctx memtable mgr create memtable successfully", KPC(tx_ctx_memtable)); } } @@ -174,9 +169,9 @@ int ObTxCtxMemtableMgr::unregister_from_common_checkpoint_(const ObTxCtxMemtable LOG_WARN("ls_tx_svr is null", K(ret)); } else if (OB_FAIL(ls_tx_svr->unregister_common_checkpoint(checkpoint::TX_CTX_MEMTABLE_TYPE, memtable))) { - LOG_WARN("tx ctx unregister_common_checkpoint failed", K(ret), K(ls_id_), K(memtable)); + LOG_WARN("tx ctx unregister_common_checkpoint failed", K(ret), K(memtable)); } else { - LOG_INFO("unregister from common checkpoint successfully", K_(ls_id), K(memtable)); + LOG_INFO("unregister from common checkpoint successfully", K(memtable)); } return ret; } @@ -193,12 +188,12 @@ int ObTxCtxMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, // for force const int64_t idx = get_memtable_idx(memtable_head_); if (nullptr != tables_[idx] && memtable == tables_[idx]) { - LOG_INFO("release head memtable", K(ret), K_(ls_id), KP(memtable)); + LOG_INFO("release head memtable", K(ret), KP(memtable)); if (OB_TMP_FAIL(unregister_from_common_checkpoint_(memtable))) { - LOG_WARN("unregister from common checkpoint failed", K(tmp_ret), K_(ls_id), K(memtable)); + LOG_WARN("unregister from common checkpoint failed", K(tmp_ret), K(memtable)); } release_head_memtable(); - FLOG_INFO("succeed to release tx ctx memtable", K(ret), K_(ls_id)); + FLOG_INFO("succeed to release tx ctx memtable", K(ret)); } } else if (!force) { // just for flush @@ -209,4 +204,3 @@ int ObTxCtxMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, } // namespace storage } // namespace oceanbase - diff --git a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h index 750ffb40c..333730c3a 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h +++ b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h @@ -54,7 +54,6 @@ class ObTxCtxMemtableMgr : public ObIMemtableMgr // Init the memtable mgr, we use logstream id to fetch the ls_ctx_mgr and t3m // to alloc the memtable. virtual int init(const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) override; virtual void destroy() override; @@ -71,7 +70,6 @@ class ObTxCtxMemtableMgr : public ObIMemtableMgr int unregister_from_common_checkpoint_(const ObTxCtxMemtable *memtable); private: - share::ObLSID ls_id_; common::SpinRWLock lock_def_; }; @@ -79,4 +77,3 @@ class ObTxCtxMemtableMgr : public ObIMemtableMgr } // namespace oceanbase #endif // OCEANBASE_STORAGE_OB_TX_CTX_MEMTABLE_MGR - diff --git a/src/storage/tx_table/ob_tx_ctx_table.cpp b/src/storage/tx_table/ob_tx_ctx_table.cpp index 55e071604..7112bcf7d 100644 --- a/src/storage/tx_table/ob_tx_ctx_table.cpp +++ b/src/storage/tx_table/ob_tx_ctx_table.cpp @@ -17,7 +17,7 @@ #include "ob_tx_ctx_table.h" #include "share/rc/ob_module_provider.h" #include "storage/tx/ob_trans_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { @@ -103,21 +103,18 @@ int ObTxCtxTableRecoverHelper::recover_one_tx_ctx_(transaction::ObLSTxCtxMgr* ls ObTxCtxTableInfo& ctx_info) { int ret = OB_SUCCESS; - transaction::ObPartTransCtx *tx_ctx = NULL; + transaction::ObTxCtx *tx_ctx = NULL; bool tx_ctx_existed = true; - common::ObAddr scheduler; // since 4.3 cluster_version in ctx_info uint64_t cluster_version = ctx_info.cluster_version_; transaction::ObTxCreateArg arg(true, /* for_replay */ - PartCtxSource::RECOVER, + TxCtxSource::RECOVER, ctx_info.tx_id_, - ctx_info.ls_id_, ctx_info.cluster_id_, /* cluster_id */ cluster_version, 0, /*session_id*/ 0, /*client_sid*/ 0, /*associated_session_id*/ - scheduler, INT64_MAX, share::g_mp->trans_service()); if (OB_FAIL(ls_tx_ctx_mgr->create_tx_ctx(arg, @@ -215,10 +212,9 @@ int ObTxCtxTableRecoverHelper::recover(const blocksstable::ObDatumRow &row, if (OB_SUCC(ret) && buf_completed) { ctx_info_.reset(); - transaction::ObPartTransCtx *tx_ctx = NULL; + transaction::ObTxCtx *tx_ctx = NULL; int64_t pos = 0; bool tx_ctx_existed = true; - ctx_info_.set_compatible_version(curr_meta.get_version()); if (FALSE_IT(TLOCAL_P_TX_BUFFER_NODE_ARRAY = &ctx_info_.exec_info_.multi_data_source_)) {// FIXME: for compat issue, should be removed after barrier version } else if (OB_FAIL(ctx_info_.deserialize(deserialize_buf, deserialize_buf_length, pos, tx_data_table))) { STORAGE_LOG(WARN, "failed to deserialize status_info", K(ret), K_(ctx_info)); @@ -244,10 +240,8 @@ int ObTxCtxTableRecoverHelper::recover(const blocksstable::ObDatumRow &row, } ObTxCtxTable::ObTxCtxTable() - : ls_id_(), - ls_tx_ctx_mgr_(NULL) + : ls_tx_ctx_mgr_(NULL) { - ls_id_.reset(); } ObTxCtxTable::~ObTxCtxTable() @@ -255,28 +249,22 @@ ObTxCtxTable::~ObTxCtxTable() reset(); } -int ObTxCtxTable::init(const ObLSID& ls_id) { - return acquire_ref_(ls_id); +int ObTxCtxTable::init() { + return acquire_ref_(); } -OB_WEAK_SYMBOL int ObTxCtxTable::acquire_ref_(const ObLSID& ls_id) +OB_WEAK_SYMBOL int ObTxCtxTable::acquire_ref_() { reset(); int ret = OB_SUCCESS; transaction::ObTransService *txs = NULL; - ls_id_ = ls_id; if (NULL == ls_tx_ctx_mgr_) { if (OB_ISNULL(txs = share::g_mp->trans_service())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "trans_service get fail", K(ret)); - } else if (OB_FAIL(txs->get_tx_ctx_mgr().get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr_))) { - TRANS_LOG(ERROR, "get ls tx ctx mgr with ref failed", KP(txs)); - } else if (NULL == ls_tx_ctx_mgr_) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ls tx ctx mgr is null", KP(txs)); } else { - TRANS_LOG(INFO, "get ls tx ctx mgr successfully", KP(txs)); + ls_tx_ctx_mgr_ = &txs->get_tx_ctx_mgr().get_tx_ctx_manager(); } } @@ -286,19 +274,7 @@ OB_WEAK_SYMBOL int ObTxCtxTable::acquire_ref_(const ObLSID& ls_id) OB_WEAK_SYMBOL int ObTxCtxTable::release_ref_() { int ret = OB_SUCCESS; - transaction::ObTransService *txs = NULL; - - if (NULL != ls_tx_ctx_mgr_) { - if (OB_ISNULL(txs = share::g_mp->trans_service())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "trans_service get fail", K(ret)); - } else if (OB_FAIL(txs->get_tx_ctx_mgr().revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr_))) { - TRANS_LOG(ERROR, "revert ls tx ctx mgr with ref failed", KP(txs)); - } else { - TRANS_LOG(INFO, "revert ls tx ctx mgr successfully", K(ls_id_), K(this)); - ls_tx_ctx_mgr_ = NULL; - } - } + ls_tx_ctx_mgr_ = NULL; return ret; } @@ -313,7 +289,6 @@ void ObTxCtxTable::reset() { recover_helper_.reset(); release_ref_(); - ls_id_.reset(); ls_tx_ctx_mgr_ = NULL; } diff --git a/src/storage/tx_table/ob_tx_ctx_table.h b/src/storage/tx_table/ob_tx_ctx_table.h index d64c016a0..b2fc3f859 100644 --- a/src/storage/tx_table/ob_tx_ctx_table.h +++ b/src/storage/tx_table/ob_tx_ctx_table.h @@ -17,7 +17,6 @@ #ifndef OCEANBASE_STORAGE_OB_TX_CTX_TABLE #define OCEANBASE_STORAGE_OB_TX_CTX_TABLE -#include "share/ob_ls_id.h" #include "storage/ob_i_store.h" #include "storage/tx_table/ob_tx_table_define.h" #include "storage/tx_table/tx_table_local_buffer.h" @@ -61,7 +60,6 @@ class ObTxCtxTableRecoverHelper struct ObTxCtxTableMeta prev_meta_; ObTxLocalBuffer buf_; int64_t prev_end_pos_; - share::ObLSID ls_id_; ObTxCtxTableInfo ctx_info_; }; @@ -97,7 +95,7 @@ class ObTxCtxTable ObTxCtxTable(); ~ObTxCtxTable(); - int init(const share::ObLSID& ls_id); + int init(); void reset(); @@ -108,7 +106,7 @@ class ObTxCtxTable // We need the reference count to keep the life cycle of the ls_tx_ctx_mgr. // And the ObTxTable is a guard for automatical management of it. - int acquire_ref_(const share::ObLSID& ls_id); + int acquire_ref_(); int release_ref_(); @@ -123,7 +121,6 @@ class ObTxCtxTable transaction::ObLSTxCtxMgr *get_ls_tx_ctx_mgr(); private: - share::ObLSID ls_id_; transaction::ObLSTxCtxMgr *ls_tx_ctx_mgr_; private: diff --git a/src/storage/tx_table/ob_tx_data_cache.h b/src/storage/tx_table/ob_tx_data_cache.h index acb82a4a3..1bfbbb9d7 100644 --- a/src/storage/tx_table/ob_tx_data_cache.h +++ b/src/storage/tx_table/ob_tx_data_cache.h @@ -31,35 +31,31 @@ namespace storage { class ObTxDataCacheKey : public common::ObIKVCacheKey { public: - ObTxDataCacheKey() : ls_id_(0), tx_id_(0) {} - ObTxDataCacheKey(share::ObLSID ls_id, transaction::ObTransID tx_id) - : ls_id_(ls_id), tx_id_(tx_id) {} + ObTxDataCacheKey() : tx_id_(0) {} + explicit ObTxDataCacheKey(transaction::ObTransID tx_id) : tx_id_(tx_id) {} ~ObTxDataCacheKey() {} bool is_valid() const { - bool is_valid = true && ls_id_.is_valid() && tx_id_.is_valid(); + bool is_valid = tx_id_.is_valid(); return is_valid; } - share::ObLSID get_ls_id() const { return ls_id_; } - transaction::ObTransID get_tx_id() const { return tx_id_; } - TO_STRING_KV(K_(ls_id), K_(tx_id)); + TO_STRING_KV(K_(tx_id)); public: // derived from ObIKVCacheKey virtual bool operator==(const ObIKVCacheKey &other) const { const ObTxDataCacheKey &rhs = static_cast(other); - bool equal = (true && ls_id_ == rhs.get_ls_id() && tx_id_ == rhs.get_tx_id()); + bool equal = tx_id_ == rhs.get_tx_id(); return equal; } virtual uint64_t hash() const { uint64_t hash_code = 0; - hash_code = murmurhash(&ls_id_, sizeof(ls_id_), hash_code); hash_code = murmurhash(&tx_id_, sizeof(tx_id_), hash_code); return hash_code; } @@ -75,7 +71,7 @@ class ObTxDataCacheKey : public common::ObIKVCacheKey { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid argument", KR(ret), K(buf_len), K(size())); } else { - ObTxDataCacheKey *new_key = new (buf) ObTxDataCacheKey(ls_id_, tx_id_); + ObTxDataCacheKey *new_key = new (buf) ObTxDataCacheKey(tx_id_); if (OB_ISNULL(new_key)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "new key ptr is null", KR(ret), KPC(this)); @@ -88,7 +84,6 @@ class ObTxDataCacheKey : public common::ObIKVCacheKey { private: - share::ObLSID ls_id_; transaction::ObTransID tx_id_; }; diff --git a/src/storage/tx_table/ob_tx_data_memtable.cpp b/src/storage/tx_table/ob_tx_data_memtable.cpp index 8b94722b5..74cd6aa4d 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable.cpp @@ -61,7 +61,6 @@ int ObTxDataMemtable::init(const ObITable::TableKey &table_key, occupied_size_[i] = 0; total_undo_node_cnt_[i] = 0; } - ls_id_ = freezer_->get_ls_id(); construct_list_done_ = false; pre_process_done_ = false; do_recycle_ = false; @@ -185,7 +184,6 @@ int ObTxDataMemtable::insert(ObTxData *tx_data) if (tx_data->op_guard_->get_undo_status_list().undo_node_cnt_ == 10 || tx_data->op_guard_->get_undo_status_list().undo_node_cnt_ % 100 == 0) { STORAGE_LOG(INFO, "attention! this tx write too many rollback to savepoint log", - "ls_id", get_ls_id(), "tx_id", tx_data->tx_id_, "state", ObTxData::get_state_string(tx_data->state_), "undo_node_cnt", tx_data->op_guard_->get_undo_status_list().undo_node_cnt_, @@ -271,7 +269,6 @@ int ObTxDataMemtable::pre_process_for_merge() STORAGE_LOG(INFO, "[TX DATA MERGE]pre process for merge done", KR(ret), - K(get_ls_id()), KP(this), K(tg), K(fake_tx_data_guard), @@ -349,8 +346,7 @@ int ObTxDataMemtable::pre_process_commit_version_row_(ObTxData *fake_tx_data) } else { do_recycle_ = true; } - STORAGE_LOG( - INFO, "pre-process commit versions row", K(get_ls_id()), K(do_recycle_), K(current_time), K(prev_recycle_time)); + STORAGE_LOG(INFO, "pre-process commit versions row", K(do_recycle_), K(current_time), K(prev_recycle_time)); if (OB_FAIL(fill_in_cur_commit_versions_(cur_commit_versions)/*step 1*/)) { STORAGE_LOG(WARN, "periodical select commit version failed.", KR(ret)); @@ -919,10 +915,10 @@ bool ObTxDataMemtable::ready_for_flush() bool_ret = true; STORAGE_LOG(INFO, "memtable is frozen yet.", KP(this)); } else if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { - STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(get_ls_id())); + STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret)); } else if (max_consequent_callbacked_scn >= key_.scn_range_.end_scn_) { state_ = ObTxDataMemtable::State::FROZEN; - STORAGE_LOG(INFO, "[TX DATA MERGE]tx data memtable is frozen", K(get_ls_id()), KP(this)); + STORAGE_LOG(INFO, "[TX DATA MERGE]tx data memtable is frozen", KP(this)); set_snapshot_version(get_min_tx_scn()); bool_ret = true; stat_change_ts_.ready_for_flush_time_ = ObTimeUtil::fast_current_time(); @@ -950,7 +946,6 @@ int ObTxDataMemtable::flush() { int ret = OB_SUCCESS; compaction::ObTabletMergeDagParam param; - param.ls_id_ = freezer_->get_ls_id(); param.tablet_id_ = key_.tablet_id_; param.merge_type_ = compaction::MINI_MERGE; param.merge_version_ = ObVersionRange::MIN_VERSION; @@ -963,7 +958,6 @@ int ObTxDataMemtable::flush() STORAGE_LOG(INFO, "[TX DATA MERGE]schedule flush tx data memtable task done", KR(ret), - K(get_ls_id()), KP(this), K(param), KPC(this)); @@ -1088,11 +1082,6 @@ int ObTxDataMemtable::cmp_key_(const int64_t &lhs, const int64_t &rhs) return int_ret; } -share::ObLSID ObTxDataMemtable::get_ls_id() const -{ - return OB_ISNULL(freezer_) ? share::ObLSID(ObLSID::INVALID_LS_ID) : freezer_->get_ls_id(); -} - int ObTxDataMemtable::dump2text(const char *fname) { int ret = OB_SUCCESS; @@ -1112,9 +1101,6 @@ int ObTxDataMemtable::dump2text(const char *fname) ret = OB_IO_ERROR; STORAGE_LOG(WARN, "open file fail:", K(fname)); } else { - int64_t ls_id = freezer_->get_ls_id().id(); - - fprintf(fd, "ls_id=%ld\n", ls_id); ObCStringHelper helper; const char *key_ptr = NULL; const char *min_tx_scn_ptr = NULL; diff --git a/src/storage/tx_table/ob_tx_data_memtable.h b/src/storage/tx_table/ob_tx_data_memtable.h index 083d8090f..83915c20f 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.h +++ b/src/storage/tx_table/ob_tx_data_memtable.h @@ -224,7 +224,6 @@ class ObTxDataMemtable : public ObIMemtable int get_iter_start_and_count(const transaction::ObTransID &tx_id, ObTxDataLinkNode *&start_node, int64_t &iterate_row_cnt); - share::ObLSID get_ls_id() const; int64_t get_total_undo_node_cnt() const; /** diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp index cceafb758..dcaa4a748 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp @@ -32,7 +32,6 @@ namespace checkpoint ObTxDataMemtableMgr::ObTxDataMemtableMgr() : is_freezing_(false), - ls_id_(), mini_merge_recycle_commit_versions_ts_(0), tx_data_table_(nullptr), ls_tablet_svr_(nullptr) @@ -51,12 +50,11 @@ void ObTxDataMemtableMgr::destroy() int ret = OB_SUCCESS; const int64_t ref_cnt = get_ref(); if (OB_UNLIKELY(0 != ref_cnt)) { - STORAGE_LOG(ERROR, "ref cnt is NOT 0", K(ret), K(ref_cnt), K_(ls_id), KPC(this)); + STORAGE_LOG(ERROR, "ref cnt is NOT 0", K(ret), K(ref_cnt), KPC(this)); } MemMgrWLockGuard guard(lock_); reset_tables(); - ls_id_.reset(); tablet_id_.reset(); tx_data_table_ = nullptr; ls_tablet_svr_ = nullptr; @@ -66,12 +64,11 @@ void ObTxDataMemtableMgr::destroy() } int ObTxDataMemtableMgr::init(const common::ObTabletID &tablet_id, - const ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObTxTable *tx_table = nullptr; if (IS_INIT) { ret = OB_INIT_TWICE; @@ -79,19 +76,19 @@ int ObTxDataMemtableMgr::init(const common::ObTabletID &tablet_id, } else if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_ISNULL(freezer) || OB_ISNULL(t3m)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid arguments", K(ret), K(tablet_id), KP(freezer), KP(t3m)); - } else if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::TRANS_MOD))){ - STORAGE_LOG(WARN, "Get ls from ls service failed.", KR(ret)); - } else if (OB_ISNULL(tx_table = ls_handle.get_ls()->get_tx_table())) { + } else if (OB_ISNULL(ls = freezer->get_ls())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Get ls from freezer failed.", KR(ret)); + } else if (OB_ISNULL(tx_table = ls->get_tx_table())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "Get tx table from ls failed.", KR(ret)); } else { reset_tables(); - ls_id_ = ls_id; tablet_id_ = tablet_id; t3m_ = t3m; freezer_ = freezer; tx_data_table_ = tx_table->get_tx_data_table(); - ls_tablet_svr_ = ls_handle.get_ls()->get_tablet_svr(); + ls_tablet_svr_ = ls->get_tablet_svr(); mini_merge_recycle_commit_versions_ts_ = 0; ObLSTxService *ls_tx_svr = nullptr; if (OB_ISNULL(ls_tx_svr = freezer_->get_ls_tx_svr())) { @@ -99,7 +96,7 @@ int ObTxDataMemtableMgr::init(const common::ObTabletID &tablet_id, STORAGE_LOG(WARN, "ls_tx_svr is null", K(ret), KP(freezer_)); } else if (OB_FAIL(ls_tx_svr->register_common_checkpoint( checkpoint::TX_DATA_MEMTABLE_TYPE, this))) { - STORAGE_LOG(WARN, "tx_data register_common_checkpoint failed", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "tx_data register_common_checkpoint failed", K(ret)); } else if (OB_ISNULL(tx_data_table_) || OB_ISNULL(ls_tablet_svr_)) { ret = OB_ERR_NULL_VALUE; STORAGE_LOG(WARN, "Init tx data memtable mgr failed.", KR(ret)); @@ -143,7 +140,7 @@ int ObTxDataMemtableMgr::release_head_memtable_(ObIMemtable *imemtable, if (true == memtable->do_recycled()) { mini_merge_recycle_commit_versions_ts_ = ObClockGenerator::getClock(); } - STORAGE_LOG(INFO, "[TX DATA MERGE]tx data memtable mgr release head memtable", K(ls_id_), KP(memtable), KPC(memtable)); + STORAGE_LOG(INFO, "[TX DATA MERGE]tx data memtable mgr release head memtable", KP(memtable), KPC(memtable)); release_head_memtable(); } else { ret = OB_INVALID_ARGUMENT; @@ -215,7 +212,7 @@ int ObTxDataMemtableMgr::create_memtable_(const SCN clog_checkpoint_scn, int ObTxDataMemtableMgr::freeze() { int ret = OB_SUCCESS; - STORAGE_LOG(INFO, "start freeze tx data memtable", K(ls_id_)); + STORAGE_LOG(INFO, "start freeze tx data memtable"); if (IS_NOT_INIT) { ret = OB_ERR_UNEXPECTED; @@ -303,7 +300,6 @@ int ObTxDataMemtableMgr::freeze_() STORAGE_LOG(INFO, "[TX DATA MERGE]tx data memtable freeze success", KR(ret), - K(ls_id_), KP(freeze_memtable), K(get_memtable_count_()), KPC(freeze_memtable), @@ -349,7 +345,6 @@ int ObTxDataMemtableMgr::calc_new_memtable_buckets_cnt_(const double load_factor new_buckets_cnt = expect_buckets_cnt; STORAGE_LOG(INFO, "finish calculate new tx data memtable buckets cnt", - K(ls_id_), K(load_factory), K(old_buckets_cnt), K(new_buckets_cnt), @@ -363,7 +358,7 @@ int ObTxDataMemtableMgr::get_active_memtable(ObTableHandleV2 &handle) const MemMgrRLockGuard lock_guard(lock_); if (0 == memtable_tail_) { ret = OB_EAGAIN; - STORAGE_LOG(INFO, "tx data memtable is not created yet. try agagin.", K(ret), K(ls_id_), K(memtable_tail_)); + STORAGE_LOG(INFO, "tx data memtable is not created yet. try agagin.", K(ret), K(memtable_tail_)); } else if (0 == get_memtable_count_()) { ret = OB_ENTRY_NOT_EXIST; STORAGE_LOG(WARN, "the tx data memtable manager is empty. may be offline", KR(ret), K(get_memtable_count_())); @@ -427,7 +422,7 @@ int ObTxDataMemtableMgr::get_all_memtables_for_write(ObTxDataMemtableWriteGuard MemMgrRLockGuard lock_guard(lock_); for (int64_t i = memtable_head_; OB_SUCC(ret) && i < memtable_tail_; ++i) { if (OB_FAIL(write_guard.push_back_table(tables_[get_memtable_idx(i)], t3m_))) { - STORAGE_LOG(WARN, "push back table to write guard failed", KR(ret), K(ls_id_)); + STORAGE_LOG(WARN, "push back table to write guard failed", KR(ret)); } } return ret; diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.h b/src/storage/tx_table/ob_tx_data_memtable_mgr.h index eb5d43ead..8e248ccfb 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.h +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.h @@ -96,7 +96,6 @@ using SliceAllocator = ObSliceAlloc; ObTxDataMemtableMgr(); virtual ~ObTxDataMemtableMgr(); int init(const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, ObFreezer *freezer, ObTenantMetaMemMgr *t3m) override; virtual void destroy() override; @@ -155,7 +154,6 @@ using SliceAllocator = ObSliceAlloc; INHERIT_TO_STRING_KV("ObIMemtableMgr", ObIMemtableMgr, K_(is_freezing), - K_(ls_id), K_(mini_merge_recycle_commit_versions_ts), KP_(tx_data_table), KP_(ls_tablet_svr)); @@ -187,7 +185,6 @@ using SliceAllocator = ObSliceAlloc; private: // ObTxDataMemtableMgr bool is_freezing_; - share::ObLSID ls_id_; int64_t mini_merge_recycle_commit_versions_ts_; ObTxDataTable *tx_data_table_; ObLSTabletService *ls_tablet_svr_; diff --git a/src/storage/tx_table/ob_tx_data_table.cpp b/src/storage/tx_table/ob_tx_data_table.cpp index 1a62b7cd6..fb45be222 100644 --- a/src/storage/tx_table/ob_tx_data_table.cpp +++ b/src/storage/tx_table/ob_tx_data_table.cpp @@ -64,11 +64,9 @@ int ObTxDataTable::init(ObLS *ls, ObTxCtxTable *tx_ctx_table) ObMemAttr("CommitVersions")); ls_ = ls; - ls_id_ = ls->get_ls_id(); memtable_mgr_ = static_cast(memtable_mgr_handle.get_memtable_mgr()); tx_ctx_table_ = tx_ctx_table; tablet_id_ = LS_TX_DATA_TABLET; - latest_reserved_scn_.reset(); is_inited_ = true; FLOG_INFO("tx data table init success", K(sizeof(ObTxData)), K(sizeof(ObTxDataLinkNode)), KPC(this)); @@ -179,7 +177,6 @@ void ObTxDataTable::reset() tx_ctx_table_ = nullptr; calc_upper_trans_version_cache_.reset(); memtables_cache_.reuse(); - latest_reserved_scn_.reset(); is_started_ = false; is_inited_ = false; } @@ -187,7 +184,7 @@ void ObTxDataTable::reset() int ObTxDataTable::prepare_for_safe_destroy() { int ret = clean_memtables_cache_(); - LOG_INFO("tx data table prepare for safe destroy", KR(ret), K(ls_id_)); + LOG_INFO("tx data table prepare for safe destroy", KR(ret)); return ret; } @@ -232,7 +229,6 @@ int ObTxDataTable::online() TCWLockGuard lock_guard(calc_upper_trans_version_cache_.lock_); calc_upper_trans_version_cache_.reset(); } - latest_reserved_scn_.reset(); is_started_ = true; } @@ -359,7 +355,6 @@ int ObTxDataTable::insert_(ObTxData *&tx_data, ObTxDataMemtableWriteGuard &write STORAGE_LOG(WARN, "insert tx data into tx data memtable failed", KR(ret), - K(ls_id_), KPC(tx_data), KPC(tx_data_memtable)); } else { @@ -672,7 +667,6 @@ int ObTxDataTable::get_recycle_scn(SCN &recycle_scn) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; - ObMigrationStatus migration_status; ObRestoreStatus restore_status; ObTimeGuard tg("get recycle scn", 10L * 1000L * 1000L /* 10 seconds */); SCN min_end_scn = SCN::max_scn(); @@ -689,22 +683,17 @@ int ObTxDataTable::get_recycle_scn(SCN &recycle_scn) } else if (OB_UNLIKELY(EN_COMPACTION_TX_DATA_GET_MIN_SCN)) { ret = OB_EAGAIN; STORAGE_LOG(INFO, "EN_COMPACTION_TX_DATA_GET_MIN_SCN", KR(ret)); - } else if (OB_FAIL(ls_->get_migration_status(migration_status))) { - STORAGE_LOG(WARN, "get migration status failed", KR(ret), "ls_id", ls_->get_ls_id()); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) { - recycle_scn.set_min(); - STORAGE_LOG(INFO, "logstream is in migration state. skip recycle tx data", "ls_id", ls_->get_ls_id()); } else if (OB_FAIL(ls_->get_restore_status(restore_status))) { - STORAGE_LOG(WARN, "get restore status failed", KR(ret), "ls_id", ls_->get_ls_id()); + STORAGE_LOG(WARN, "get restore status failed", KR(ret)); } else if (ObRestoreStatus::Status::NONE != restore_status.get_status()) { recycle_scn.set_min(); - STORAGE_LOG(INFO, "logstream is in restore state. skip recycle tx data", "ls_id", ls_->get_ls_id()); + STORAGE_LOG(INFO, "logstream is in restore state. skip recycle tx data"); } else if (FALSE_IT(tg.click("iterate tablets start"))) { } else if (OB_FAIL(ls_tablet_svr_->get_ls_min_end_scn(min_end_scn_from_latest_tablets, min_end_scn_from_old_tablets))) { STORAGE_LOG(WARN, "fail to get ls min end log ts", KR(ret)); - } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->get_min_end_scn_from_major_dag(ls_->get_ls_id(), min_end_scn_from_major_dag))) { - STORAGE_LOG(WARN, "fail to get ls min end log ts from major dag", KR(ret), "ls_id", ls_->get_ls_id()); + } else if (OB_FAIL(share::g_mp->tenant_dag_scheduler()->get_min_end_scn_from_major_dag(min_end_scn_from_major_dag))) { + STORAGE_LOG(WARN, "fail to get min end log ts from major dag", KR(ret)); } else if (FALSE_IT(tg.click("iterate tablets finish"))) { } else { min_end_scn = std::min(min_end_scn_from_old_tablets, min_end_scn_from_latest_tablets); @@ -717,7 +706,7 @@ int ObTxDataTable::get_recycle_scn(SCN &recycle_scn) SCN snapshot_version; MonotonicTs unused_ts(0); int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(OB_TS_MGR.get_gts(MonotonicTs(1), NULL, snapshot_version, unused_ts))) { + if (OB_TMP_FAIL(OB_TS_MGR.get_gts(MonotonicTs(1), snapshot_version, unused_ts))) { LOG_WARN("failed to get snapshot version", K(tmp_ret)); // recycle nothing this time recycle_scn.set_min(); @@ -729,8 +718,7 @@ int ObTxDataTable::get_recycle_scn(SCN &recycle_scn) FLOG_INFO("get tx data recycle scn finish.", KR(ret), - "ls_id", ls_->get_ls_id(), - K(recycle_scn), + K(recycle_scn), K(min_end_scn_from_old_tablets), K(min_end_scn_from_latest_tablets), K(min_end_scn_from_major_dag)); @@ -749,26 +737,24 @@ int ObTxDataTable::get_recycle_scn(SCN &recycle_scn) int ObTxDataTable::self_freeze_task() { int ret = OB_SUCCESS; - const ObLSID ls_id = get_ls_id(); - - STORAGE_LOG(DEBUG, "start tx data table self freeze task", K(ls_id)); + STORAGE_LOG(DEBUG, "start tx data table self freeze task"); const int64_t current_time = ObClockGenerator::getClock(); int64_t last_freeze_ts = 0; if (freeze_freq_controller_.can_freeze(current_time, last_freeze_ts)) { if (OB_FAIL(memtable_mgr_->flush(SCN::max_scn(), true))) { if (OB_NO_NEED_MERGE == ret) { - STORAGE_LOG(INFO, "tx data table no need merge", KR(ret), K(ls_id)); + STORAGE_LOG(INFO, "tx data table no need merge", KR(ret)); } else { (void)freeze_freq_controller_.rollback_freeze_ts(current_time, last_freeze_ts); - STORAGE_LOG(WARN, "self freeze of tx data memtable failed.", KR(ret), K(ls_id), KPC(memtable_mgr_)); + STORAGE_LOG(WARN, "self freeze of tx data memtable failed.", KR(ret), KPC(memtable_mgr_)); } } } else { // skip freeze tx data this time } - STORAGE_LOG(DEBUG, "finish tx data table self freeze task", KR(ret), K(get_ls_id())); + STORAGE_LOG(DEBUG, "finish tx data table self freeze task", KR(ret)); return ret; } @@ -780,7 +766,7 @@ int ObTxDataTable::get_upper_trans_version_before_given_scn(const SCN sstable_en bool skip_calc = false; upper_trans_version.set_max(); - STORAGE_LOG(DEBUG, "start get upper trans version", K(get_ls_id())); + STORAGE_LOG(DEBUG, "start get upper trans version"); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -805,12 +791,11 @@ int ObTxDataTable::get_upper_trans_version_before_given_scn(const SCN sstable_en ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "invalid cache for upper trans version calculation. ", KR(ret)); } else if (OB_FAIL(calc_upper_trans_scn_(sstable_end_scn, upper_trans_version))) { - STORAGE_LOG(WARN, "calc upper trans version failed", KR(ret), "ls_id", get_ls_id()); + STORAGE_LOG(WARN, "calc upper trans version failed", KR(ret)); } else { FLOG_INFO("get upper trans version finish.", KR(ret), K(skip_calc), - K(ls_id_), K(sstable_end_scn), K(upper_trans_version)); } @@ -919,7 +904,7 @@ bool ObTxDataTable::skip_this_sstable_end_scn_(const SCN &sstable_end_scn) // make sure the max decided log ts is greater than sstable_end_scn if (OB_FAIL(ls_->get_max_decided_scn(max_decided_scn))) { need_skip = true; - STORAGE_LOG(WARN, "get max decided log ts failed", KR(ret), K(ls_id_).id()); + STORAGE_LOG(WARN, "get max decided log ts failed", KR(ret)); } // check if the min_start_scn_in_ctx is larger than sstable_end_scn @@ -972,8 +957,7 @@ int ObTxDataTable::check_min_start_in_ctx_(const SCN &sstable_end_scn, if (OB_FAIL(ls_->get_uncommitted_tx_min_start_scn(min_start_scn, effective_scn))) { need_skip = true; STORAGE_LOG(DEBUG, "get uncommited tx min_start_scn failed", KR(ret), K(sstable_end_scn), K(max_decided_scn)); - } else if (min_start_scn <= sstable_end_scn || max_decided_scn <= effective_scn || - (latest_reserved_scn_.is_valid() && effective_scn < latest_reserved_scn_)) { + } else if (min_start_scn <= sstable_end_scn || max_decided_scn <= effective_scn) { need_skip = true; STORAGE_LOG(DEBUG, "skip calculate upper_trans_version", @@ -981,7 +965,6 @@ int ObTxDataTable::check_min_start_in_ctx_(const SCN &sstable_end_scn, K(max_decided_scn), K(min_start_scn), K(effective_scn), - K(latest_reserved_scn_), K(need_skip)); } else { // there is no ctx whose start_scn less than sstable_end_scn @@ -1137,7 +1120,6 @@ int ObTxDataTable::calc_upper_trans_scn_(const SCN sstable_end_scn, SCN &upper_t K(sstable_end_scn), K(upper_trans_version), K(calc_upper_trans_version_cache_), - K(ls_id_), "array_count", array.count(), "chose_idx", l); @@ -1196,7 +1178,7 @@ int ObTxDataTable::get_start_tx_scn(SCN &start_tx_scn) } else if (OB_ISNULL(oldest_minor_sstable = static_cast( table_store_wrapper.get_member()->get_minor_sstables().get_boundary_table(false /*is_last*/)))) { start_tx_scn.set_max(); - STORAGE_LOG(INFO, "this logstream do not have tx data sstable", K(start_tx_scn), K(get_ls_id()), KPC(tablet)); + STORAGE_LOG(INFO, "this logstream do not have tx data sstable", K(start_tx_scn), KPC(tablet)); } else if (OB_FAIL(oldest_minor_sstable->get_meta(meta_handle))) { LOG_WARN("fail to get sstable meta", K(ret)); } else if (FALSE_IT(start_tx_scn = meta_handle.get_sstable_meta().get_filled_tx_scn())) { @@ -1217,19 +1199,6 @@ void ObTxDataTable::disable_upper_trans_calculation() } } -void ObTxDataTable::enable_upper_trans_calculation(const share::SCN latest_reserved_scn) -{ - { - TCWLockGuard lock_guard(calc_upper_trans_version_cache_.lock_); - calc_upper_trans_version_cache_.reset(); - } - if (latest_reserved_scn_.is_valid()) { - latest_reserved_scn_ = SCN::max(latest_reserved_scn, latest_reserved_scn_); - } else { - latest_reserved_scn_ = latest_reserved_scn; - } -} - int ObTxDataTable::dump_single_tx_data_2_text(const int64_t tx_id_int, FILE *fd) { int ret = OB_SUCCESS; @@ -1294,7 +1263,7 @@ int ObTxDataTable::dump_tx_data_in_sstable_2_text_(const ObTransID tx_id, FILE * } ERRSIM_POINT_DEF(EN_TX_DATA_MAX_FREEZE_INTERVAL_SECOND) -bool ObTxDataTable::FreezeFrequencyController::need_re_freeze(const share::ObLSID ls_id) +bool ObTxDataTable::FreezeFrequencyController::need_re_freeze() { // Inject logic to modify default freeze interval int tmp_ret = OB_SUCCESS; @@ -1316,7 +1285,6 @@ bool ObTxDataTable::FreezeFrequencyController::need_re_freeze(const share::ObLSI need_re_freeze = true; STORAGE_LOG(INFO, "tx data table need refreeze", - K(ls_id), K(max_freeze_interval), KTIME(last_request_ts), KTIME(last_freeze_ts), @@ -1326,8 +1294,6 @@ bool ObTxDataTable::FreezeFrequencyController::need_re_freeze(const share::ObLSI return need_re_freeze; } -share::ObLSID ObTxDataTable::get_ls_id() { return ls_id_; } - int ObTxDataTable::get_sstable_recycle_scn(share::SCN &recycle_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/tx_table/ob_tx_data_table.h b/src/storage/tx_table/ob_tx_data_table.h index 4756a5c53..be549f1c8 100644 --- a/src/storage/tx_table/ob_tx_data_table.h +++ b/src/storage/tx_table/ob_tx_data_table.h @@ -107,7 +107,7 @@ class ObTxDataTable return false; } - bool need_re_freeze(const share::ObLSID ls_id); + bool need_re_freeze(); void rollback_freeze_ts(const int64_t expected_val, const int64_t rollback_val) { @@ -137,10 +137,8 @@ class ObTxDataTable public: // ObTxDataTable ObTxDataTable() - : is_inited_(false), + : is_inited_(false), is_started_(false), - latest_reserved_scn_(), - ls_id_(), tablet_id_(0), arena_allocator_(), tx_data_allocator_(nullptr), @@ -159,7 +157,7 @@ class ObTxDataTable virtual void stop(); virtual void reset(); virtual void destroy(); - bool need_re_freeze() { return freeze_freq_controller_.need_re_freeze(ls_id_); } + bool need_re_freeze() { return freeze_freq_controller_.need_re_freeze(); } int offline(); int online(); @@ -249,7 +247,6 @@ class ObTxDataTable TO_STRING_KV(KP(this), K_(is_inited), K_(is_started), - K_(ls_id), K_(tablet_id), K_(memtables_cache), KP_(ls), @@ -262,9 +259,7 @@ class ObTxDataTable share::ObTenantTxDataAllocator *get_tx_data_allocator() { return tx_data_allocator_; } TxDataReadSchema &get_read_schema() { return read_schema_; }; - share::ObLSID get_ls_id(); void disable_upper_trans_calculation(); - void enable_upper_trans_calculation(const share::SCN latest_reserved_scn); private: virtual ObTxDataMemtableMgr *get_memtable_mgr_() { return memtable_mgr_; } @@ -347,8 +342,6 @@ class ObTxDataTable static const int64_t LS_TX_DATA_SCHEMA_COLUMN_CNT = 5; bool is_inited_; bool is_started_; - share::SCN latest_reserved_scn_; - share::ObLSID ls_id_; ObTabletID tablet_id_; // Allocator to allocate ObTxData and ObUndoStatus ObArenaAllocator arena_allocator_; diff --git a/src/storage/tx_table/ob_tx_table.cpp b/src/storage/tx_table/ob_tx_table.cpp index 6e30991a6..08906f960 100644 --- a/src/storage/tx_table/ob_tx_table.cpp +++ b/src/storage/tx_table/ob_tx_table.cpp @@ -42,18 +42,17 @@ int ObTxTable::init(ObLS *ls) LOG_WARN("invalid argument", K(ret), KP(ls)); } else if (OB_FAIL(tx_data_table_.init(ls, &tx_ctx_table_))) { LOG_WARN("tx data table init failed", K(ret)); - } else if (OB_FAIL(tx_ctx_table_.init(ls->get_ls_id()))) { + } else if (OB_FAIL(tx_ctx_table_.init())) { LOG_WARN("tx ctx table init failed", K(ret)); } else { ls_ = ls; - ls_id_ = ls->get_ls_id(); epoch_ = 0; state_ = TxTableState::OFFLINE; mini_cache_hit_cnt_ = 0; kv_cache_hit_cnt_ = 0; read_tx_data_table_cnt_ = 0; recycle_record_.reset(); - LOG_INFO("init tx table successfully", K(ret), K(ls->get_ls_id())); + LOG_INFO("init tx table successfully", K(ret)); calc_upper_trans_is_disabled_ = false; is_inited_ = true; } @@ -114,7 +113,7 @@ int ObTxTable::offline_tx_data_table_() ret = OB_NOT_INIT; STORAGE_LOG(WARN, "tx table is not init", KR(ret)); } else if (OB_FAIL(tx_data_table_.offline())) { - STORAGE_LOG(WARN, "tx data table offline failed", KR(ret), K(ls_id_)); + STORAGE_LOG(WARN, "tx data table offline failed", KR(ret)); } return ret; } @@ -137,7 +136,7 @@ int ObTxTable::offline() recycle_record_.reset(); (void)disable_upper_trans_calculation(); ATOMIC_STORE(&state_, TxTableState::OFFLINE); - LOG_INFO("tx table offline succeed", K(ls_id_), KPC(this)); + LOG_INFO("tx table offline succeed", KPC(this)); } return ret; @@ -162,7 +161,7 @@ int ObTxTable::online() (void)reset_ctx_min_start_scn_info_(); ATOMIC_STORE(&state_, ObTxTable::ONLINE); ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); - LOG_INFO("tx table online succeed", K(ls_id_), KPC(this)); + LOG_INFO("tx table online succeed", KPC(this)); } return ret; @@ -185,10 +184,9 @@ int ObTxTable::create_tablet(const lib::Worker::CompatMode compat_mode, const SC LOG_WARN("not inited", K(ret)); } else { - const share::ObLSID &ls_id = ls_->get_ls_id(); - if (OB_FAIL(create_data_tablet_(ls_id, compat_mode, create_scn))) { + if (OB_FAIL(create_data_tablet_(compat_mode, create_scn))) { LOG_WARN("create data tablet failed", K(ret)); - } else if (OB_FAIL(create_ctx_tablet_(ls_id, compat_mode, create_scn))) { + } else if (OB_FAIL(create_ctx_tablet_(compat_mode, create_scn))) { LOG_WARN("create ctx tablet failed", K(ret)); } if (OB_FAIL(ret)) { @@ -267,9 +265,8 @@ int ObTxTable::get_ctx_table_schema_(share::schema::ObTableSchema &schema) return ret; } -int ObTxTable::create_ctx_tablet_(const ObLSID ls_id, - const lib::Worker::CompatMode compat_mode, - const share::SCN &create_scn) +int ObTxTable::create_ctx_tablet_(const lib::Worker::CompatMode compat_mode, + const share::SCN &create_scn) { int ret = OB_SUCCESS; share::schema::ObTableSchema table_schema; @@ -280,12 +277,11 @@ int ObTxTable::create_ctx_tablet_(const ObLSID ls_id, } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, false/*skip_column_info*/, DATA_CURRENT_VERSION))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); - } else if (OB_FAIL(ls_->create_ls_inner_tablet(ls_id, - LS_TX_CTX_TABLET, + } else if (OB_FAIL(ls_->create_ls_inner_tablet(LS_TX_CTX_TABLET, ObLS::LS_INNER_TABLET_FROZEN_SCN, create_tablet_schema, create_scn))) { - LOG_WARN("create tx ctx tablet failed", K(ret), K(ls_id), + LOG_WARN("create tx ctx tablet failed", K(ret), K(LS_TX_CTX_TABLET), K(ObLS::LS_INNER_TABLET_FROZEN_SCN), K(table_schema), K(compat_mode), K(create_scn)); } @@ -393,8 +389,7 @@ int ObTxTable::get_data_table_schema_(share::schema::ObTableSchema &schema) return ret; } -int ObTxTable::create_data_tablet_(const ObLSID ls_id, - const lib::Worker::CompatMode compat_mode, +int ObTxTable::create_data_tablet_(const lib::Worker::CompatMode compat_mode, const share::SCN &create_scn) { int ret = OB_SUCCESS; @@ -406,12 +401,11 @@ int ObTxTable::create_data_tablet_(const ObLSID ls_id, } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, false/*skip_column_info*/, DATA_CURRENT_VERSION))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); - } else if (OB_FAIL(ls_->create_ls_inner_tablet(ls_id, - LS_TX_DATA_TABLET, + } else if (OB_FAIL(ls_->create_ls_inner_tablet(LS_TX_DATA_TABLET, ObLS::LS_INNER_TABLET_FROZEN_SCN, create_tablet_schema, create_scn))) { - LOG_WARN("create tx data tablet failed", K(ret), K(ls_id), + LOG_WARN("create tx data tablet failed", K(ret), K(LS_TX_DATA_TABLET), K(ObLS::LS_INNER_TABLET_FROZEN_SCN), K(table_schema), K(compat_mode), K(create_scn)); } @@ -421,9 +415,8 @@ int ObTxTable::create_data_tablet_(const ObLSID ls_id, int ObTxTable::remove_tablet_(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - const share::ObLSID &ls_id = ls_->get_ls_id(); - if (OB_FAIL(ls_->remove_ls_inner_tablet(ls_id, tablet_id))) { - LOG_WARN("remove ls inner tablet failed", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(ls_->remove_ls_inner_tablet(tablet_id))) { + LOG_WARN("remove ls inner tablet failed", K(ret), K(tablet_id)); } return ret; } @@ -593,7 +586,6 @@ void ObTxTable::destroy() { tx_data_table_.destroy(); tx_ctx_table_.reset(); - ls_id_.reset(); ls_ = nullptr; epoch_ = 0; ctx_min_start_scn_info_.reset(); @@ -677,7 +669,7 @@ int ObTxTable::check_with_tx_data(ObReadTxDataArg &read_tx_data_arg, ObITxDataCh // already find tx data and do function with cache } else if (OB_FAIL(check_tx_data_in_tables_(read_tx_data_arg, fn))) { if (OB_TRANS_CTX_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "check tx data in tables failed", KR(ret), K(ls_id_), K(read_tx_data_arg)); + STORAGE_LOG(WARN, "check tx data in tables failed", KR(ret), K(read_tx_data_arg)); } } @@ -710,7 +702,7 @@ int ObTxTable::check_tx_data_in_mini_cache_(ObReadTxDataArg &read_tx_data_arg, O int ObTxTable::check_tx_data_in_kv_cache_(ObReadTxDataArg &read_tx_data_arg, ObITxDataCheckFunctor &fn) { int ret = OB_SUCCESS; - ObTxDataCacheKey key(ls_id_, read_tx_data_arg.tx_id_); + ObTxDataCacheKey key(read_tx_data_arg.tx_id_); ObTxDataValueHandle val_handle; if (OB_FAIL(OB_TX_DATA_KV_CACHE.get_row(key, val_handle))) { @@ -725,10 +717,10 @@ int ObTxTable::check_tx_data_in_kv_cache_(ObReadTxDataArg &read_tx_data_arg, ObI const ObTxData *tx_data = nullptr; if (OB_ISNULL(cache_val)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "cache value is nullptr", KR(ret), K(read_tx_data_arg), K(ls_id_), K(val_handle)); + STORAGE_LOG(ERROR, "cache value is nullptr", KR(ret), K(read_tx_data_arg), K(val_handle)); } else if (OB_ISNULL(tx_data = cache_val->get_tx_data())) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "tx data in cache value is nullptr", KR(ret), K(read_tx_data_arg), K(ls_id_), KPC(cache_val)); + STORAGE_LOG(ERROR, "tx data in cache value is nullptr", KR(ret), K(read_tx_data_arg), KPC(cache_val)); } else { EVENT_INC(ObStatEventIds::TX_DATA_HIT_KV_CACHE_COUNT); ret = fn(*tx_data); @@ -766,7 +758,6 @@ int ObTxTable::check_tx_data_in_tables_(ObReadTxDataArg &read_tx_data_arg, ObITx STORAGE_LOG(WARN, "check with tx data in tx data table failed", KR(ret), - K(ls_id_), K(read_tx_data_arg), K(tx_data_guard), K(recycled_scn), @@ -793,7 +784,7 @@ int ObTxTable::check_tx_data_in_tables_(ObReadTxDataArg &read_tx_data_arg, ObITx int ObTxTable::put_tx_data_into_kv_cache_(const ObTxData &tx_data) { int ret = OB_SUCCESS; - const ObTxDataCacheKey key(ls_id_, tx_data.tx_id_); + const ObTxDataCacheKey key(tx_data.tx_id_); ObTxDataCacheValue cache_value; if (OB_FAIL(cache_value.init(tx_data))) { @@ -823,17 +814,17 @@ void ObTxTable::check_state_and_epoch_(const transaction::ObTransID tx_id, if (OB_UNLIKELY(read_epoch != epoch)) { // offline or online has been executed on this tx table, return a specific error code to retry ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("tx table epoch changed", KR(ret), K(ls_id_), "state", get_state_string(state), K(read_epoch), K(epoch)); + LOG_WARN("tx table epoch changed", KR(ret), "state", get_state_string(state), K(read_epoch), K(epoch)); } else if (OB_UNLIKELY(TxTableState::ONLINE != state)) { ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("tx table is not online", KR(ret), K(ls_id_), "state", get_state_string(state), K(read_epoch), K(epoch)); + LOG_WARN("tx table is not online", KR(ret), "state", get_state_string(state), K(read_epoch), K(epoch)); } else if (OB_FAIL(ret)) { SCN max_decided_scn = SCN::invalid_scn(); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(ls_->get_max_decided_scn(max_decided_scn))) { LOG_WARN("get max decided scn failed", KR(tmp_ret)); } - LOG_WARN("check with tx data failed.", KR(ret), K(tx_id), K(read_epoch), K(max_decided_scn), K(ls_id_), KPC(this)); + LOG_WARN("check with tx data failed.", KR(ret), K(tx_id), K(read_epoch), K(max_decided_scn), KPC(this)); } } @@ -909,7 +900,6 @@ int ObTxTable::lock_for_read(ObReadTxDataArg &read_tx_data_arg, LockForReadFunctor fn(lock_for_read_arg, can_read, trans_version, - ls_id_, cleanout_op, recheck_op); int ret = check_with_tx_data(read_tx_data_arg, fn); @@ -957,7 +947,6 @@ int ObTxTable::get_recycle_scn_(const int64_t tx_result_retention_s, share::SCN STORAGE_LOG(WARN, "this tx table is migrating or has migrated", KR(ret), - K(ls_id_), K(state), K(prev_epoch), K(after_epoch)); @@ -974,7 +963,6 @@ int ObTxTable::get_recycle_scn_(const int64_t tx_result_retention_s, share::SCN STORAGE_LOG(INFO, "current recycle scn is close to last recycle scn, skip recycle once", KR(ret), - K(ls_id_), KTIME(real_recycle_scn.convert_to_ts()), KTIME(recycle_record_.last_recycle_scn_.convert_to_ts())); } @@ -1063,7 +1051,6 @@ void ObTxTable::recycle_tx_data_finish(const share::SCN current_recycle_scn) recycle_record_.last_recycle_ts_ = current_ts; FLOG_INFO("finish recycle tx data once", - K(ls_id_), K(last_recycle_scn), K(current_recycle_scn), KTIME(last_recycle_ts), @@ -1092,15 +1079,6 @@ void ObTxTable::disable_upper_trans_calculation() FLOG_INFO("disable upper trans version calculation", KPC(this)); } -void ObTxTable::enable_upper_trans_calculation(const share::SCN latest_reserved_scn) -{ - reset_ctx_min_start_scn_info_(); - (void)tx_data_table_.enable_upper_trans_calculation(latest_reserved_scn); - ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); - FLOG_INFO("enable upper trans version calculation", KPC(this)); -} - - int ObTxTable::cleanout_tx_node(ObReadTxDataArg &read_tx_data_arg, memtable::ObMvccRow &value, memtable::ObMvccTransNode &tnode, @@ -1162,10 +1140,6 @@ int ObTxTable::dump_single_tx_data_2_text(const int64_t tx_id_int, const char *f ret = OB_IO_ERROR; STORAGE_LOG(WARN, "open file fail:", K(fname)); } else { - int64_t ls_id = ls_->get_ls_id().id(); - - fprintf(fd, "ls_id=%ld\n", ls_id); - if (OB_SUCC(tx_ctx_table_.dump_single_tx_data_2_text(tx_id_int, fd))) { } else if (OB_TRANS_CTX_NOT_EXIST == ret) { ret = OB_SUCCESS; diff --git a/src/storage/tx_table/ob_tx_table.h b/src/storage/tx_table/ob_tx_table.h index d81e5ec0a..944bb3b4c 100644 --- a/src/storage/tx_table/ob_tx_table.h +++ b/src/storage/tx_table/ob_tx_table.h @@ -124,7 +124,6 @@ class ObTxTable calc_upper_trans_is_disabled_(false), epoch_(INVALID_READ_EPOCH), state_(OFFLINE), - ls_id_(), ls_(nullptr), tx_data_table_(default_tx_data_table_), mini_cache_hit_cnt_(0), @@ -138,7 +137,6 @@ class ObTxTable calc_upper_trans_is_disabled_(false), epoch_(INVALID_READ_EPOCH), state_(OFFLINE), - ls_id_(), ls_(nullptr), tx_data_table_(tx_data_table), mini_cache_hit_cnt_(0), @@ -341,12 +339,10 @@ class ObTxTable const char* get_state_string(const int64_t state) const; void disable_upper_trans_calculation(); - void enable_upper_trans_calculation(const share::SCN latest_reserved_scn); int get_tx_data_sstable_recycle_scn(share::SCN &recycle_scn); TO_STRING_KV(KP(this), - K_(ls_id), K_(is_inited), K_(calc_upper_trans_is_disabled), K_(epoch), @@ -364,17 +360,13 @@ class ObTxTable int get_tx_table_guard(ObTxTableGuard &guard); int64_t get_epoch() const { return ATOMIC_LOAD(&epoch_); } TxTableState get_state() const { return ATOMIC_LOAD(&state_); } - share::ObLSID get_ls_id() const { return ls_id_; } - static int64_t get_filter_col_idx(); private: - int create_data_tablet_(const share::ObLSID ls_id, - const lib::Worker::CompatMode compat_mode, - const share::SCN &create_scn); - int create_ctx_tablet_(const share::ObLSID ls_id, - const lib::Worker::CompatMode compat_mode, - const share::SCN &create_scn); + int create_data_tablet_(const lib::Worker::CompatMode compat_mode, + const share::SCN &create_scn); + int create_ctx_tablet_(const lib::Worker::CompatMode compat_mode, + const share::SCN &create_scn); int remove_tablet_(const common::ObTabletID &tablet_id); int get_data_table_schema_(share::schema::ObTableSchema &schema); int get_ctx_table_schema_(share::schema::ObTableSchema &schema); @@ -402,7 +394,6 @@ class ObTxTable bool calc_upper_trans_is_disabled_; int64_t epoch_ CACHE_ALIGNED; TxTableState state_ CACHE_ALIGNED; - share::ObLSID ls_id_; ObLS *ls_; ObTxCtxTable tx_ctx_table_; // The Tx Data will be inserted into tx_data_table_ after transaction commit or abort diff --git a/src/storage/tx_table/ob_tx_table_define.cpp b/src/storage/tx_table/ob_tx_table_define.cpp index 2f74e533e..38fd08c1d 100644 --- a/src/storage/tx_table/ob_tx_table_define.cpp +++ b/src/storage/tx_table/ob_tx_table_define.cpp @@ -106,8 +106,6 @@ int ObTxCtxTableInfo::serialize_(char *buf, if (OB_FAIL(tx_id_.serialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "serialize tx_id fail.", KR(ret), K(pos), K(buf_len)); - } else if (OB_FAIL(ls_id_.serialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "serialize ls_id fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, cluster_id_))) { TRANS_LOG(WARN, "encode cluster id failed", K(cluster_id_), K(buf_len), K(pos), K(ret)); } else if (OB_FAIL(tx_data_guard_.tx_data()->serialize(buf, buf_len, pos))) { @@ -150,8 +148,6 @@ int ObTxCtxTableInfo::deserialize_(const char *buf, if (OB_FAIL(tx_id_.deserialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "deserialize tx_id fail.", KR(ret), K(pos), K(buf_len)); - } else if (OB_FAIL(ls_id_.deserialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "deserialize ls_id fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, &cluster_id_))) { TRANS_LOG(WARN, "encode cluster_id fail", K(cluster_id_), K(buf_len), K(pos), K(ret)); } else if (OB_FAIL(tx_data_guard_.tx_data()->deserialize(buf, buf_len, pos, *tx_data_table.get_tx_data_allocator()))) { @@ -160,19 +156,8 @@ int ObTxCtxTableInfo::deserialize_(const char *buf, TRANS_LOG(WARN, "deserialize exec_info fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(table_lock_info_.deserialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "deserialize exec_info fail.", KR(ret), K(pos), K(buf_len)); - } - // _NOTE_ - // before 4.2.1.1, the serialize size of table_lock_info_ - // is not accurate(which larger than real serialize size), - // this caused the size of ObTxCtxTableInfo is also inaccurate - // - // when deserialize use `compatible_version_` to decide whether - // guess extra members by examine remain buf size - if (OB_SUCC(ret) && compatible_version_ >= ObTxCtxTableMeta::VERSION_1 && buf_len > pos) { - // has remains, continue to deserialize new members - if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, (int64_t*)&cluster_version_))) { - TRANS_LOG(WARN, "dencode cluster_version fail", K(buf_len), K(pos), K(ret)); - } + } else if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, (int64_t*)&cluster_version_))) { + TRANS_LOG(WARN, "decode cluster_version fail", K(buf_len), K(pos), K(ret)); } return ret; @@ -194,7 +179,6 @@ int64_t ObTxCtxTableInfo::get_serialize_size_(void) const { int64_t len = 0; len += tx_id_.get_serialize_size(); - len += ls_id_.get_serialize_size(); len += serialization::encoded_length_vi64(cluster_id_); len += serialization::encoded_length_vi64((int64_t)cluster_version_); len += (OB_NOT_NULL(tx_data_guard_.tx_data()) ? tx_data_guard_.tx_data()->get_serialize_size() : 0); @@ -209,13 +193,6 @@ bool ObTxCtxTableInfo::is_valid() const if (OB_UNLIKELY(!tx_id_.is_valid())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "tx id not valid", K(ret), K(tx_id_)); - } else if (OB_UNLIKELY(!ls_id_.is_valid())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "ls id not valid", K(ret), K(ls_id_)); - // TODO: gengli this is invalid if it a uncommited tx - // } else if (OB_UNLIKELY(!state_info_.is_valid())) { - // ret = OB_ERR_UNEXPECTED; - // TRANS_LOG(ERROR, "state info not valid", K(ret), K(state_info_)); } else { // do nothing } @@ -241,16 +218,12 @@ int ObTxCtxTableMeta::serialize_(char* buf, const int64_t buf_len, int64_t &pos) if (OB_FAIL(tx_id_.serialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "serialize tx_id fail.", KR(ret), K(pos), K(buf_len)); - } else if (OB_FAIL(ls_id_.serialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "serialize ls_id fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, tx_ctx_serialize_size_))) { TRANS_LOG(WARN, "encode serizlize_size_ fail", K(buf_len), K(pos), K(ret)); } else if (OB_FAIL(serialization::encode_vi32(buf, buf_len, pos, row_num_))) { TRANS_LOG(WARN, "encode row_num fail", K(buf_len), K(pos), K(ret)); } else if (OB_FAIL(serialization::encode_vi32(buf, buf_len, pos, row_idx_))) { TRANS_LOG(WARN, "encode row_idx fail", K(buf_len), K(pos), K(ret)); - } else if (OB_FAIL(serialization::encode_vi32(buf, buf_len, pos, version_))) { - TRANS_LOG(WARN, "encode version fail", K(buf_len), K(pos), K(ret)); } else { TRANS_LOG(DEBUG, "ObTxCtxTableMeta encode succ", K(buf_len), K(pos)); } @@ -276,20 +249,12 @@ int ObTxCtxTableMeta::deserialize_(const char* buf, const int64_t buf_len, int64 if (OB_FAIL(tx_id_.deserialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "deserialize tx_id fail.", KR(ret), K(pos), K(buf_len)); - } else if (OB_FAIL(ls_id_.deserialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "deserialize ls_id fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, &tx_ctx_serialize_size_))) { TRANS_LOG(WARN, "deserialize serizlize_size fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::decode_vi32(buf, buf_len, pos, &row_num_))) { TRANS_LOG(WARN, "deserialize row_num fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(serialization::decode_vi32(buf, buf_len, pos, &row_idx_))) { TRANS_LOG(WARN, "deserialize row_idx fail.", KR(ret), K(pos), K(buf_len)); - } else if (pos < buf_len) { // decode the version field - if (OB_FAIL(serialization::decode_vi32(buf, buf_len, pos, &version_))) { - TRANS_LOG(WARN, "deserialize version fail.", KR(ret), K(pos), K(buf_len)); - } - } else { - version_ = VERSION_0; // VERSION_0 without version field serialized } return ret; } @@ -310,11 +275,9 @@ int64_t ObTxCtxTableMeta::get_serialize_size_() const { int64_t len = 0; len += tx_id_.get_serialize_size(); - len += ls_id_.get_serialize_size(); len += serialization::encoded_length_vi64(tx_ctx_serialize_size_); len += serialization::encoded_length_vi32(row_num_); len += serialization::encoded_length_vi32(row_idx_); - len += serialization::encoded_length_vi32(version_); return len; } @@ -493,4 +456,3 @@ void ObITxDataCheckFunctor::set_may_exist_undecided_state_in_tx_data_table() } // end namespace transaction } // end namespace oceanbase - diff --git a/src/storage/tx_table/ob_tx_table_define.h b/src/storage/tx_table/ob_tx_table_define.h index 5c94a3de8..f66545471 100644 --- a/src/storage/tx_table/ob_tx_table_define.h +++ b/src/storage/tx_table/ob_tx_table_define.h @@ -78,9 +78,6 @@ struct ObTxCtxTableInfo ObTxDataTable &tx_data_table); int64_t get_serialize_size() const; - void set_compatible_version(const int32_t version) { - compatible_version_ = version; - } private: int serialize_(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize_(const char *buf, @@ -96,27 +93,21 @@ struct ObTxCtxTableInfo void reset() { tx_id_.reset(); - ls_id_.reset(); cluster_id_ = OB_INVALID_CLUSTER_ID; cluster_version_ = 0; tx_data_guard_.reset(); exec_info_.reset(); table_lock_info_.reset(); - compatible_version_ = -1; } void destroy() { reset(); } - TO_STRING_KV(K_(tx_id), K_(ls_id), K_(cluster_id), K_(tx_data_guard), + TO_STRING_KV(K_(tx_id), K_(cluster_id), K_(tx_data_guard), K_(exec_info), K_(table_lock_info), K_(cluster_version)); transaction::ObTransID tx_id_; - share::ObLSID ls_id_; int64_t cluster_id_; uint64_t cluster_version_; ObTxDataGuard tx_data_guard_; transaction::ObTxExecInfo exec_info_; transaction::tablelock::ObTableLockInfo table_lock_info_; - // used to handle compatible issue when deserialize, - // not serialized, set from ObTxCtxTableMeta - int32_t compatible_version_; }; struct ObTxCtxTableMeta @@ -131,13 +122,11 @@ struct ObTxCtxTableMeta ~ObTxCtxTableMeta() { destroy(); } bool is_valid() const - { return tx_id_.is_valid() && ls_id_.is_valid(); } + { return tx_id_.is_valid(); } void reset() { - version_ = VERSION_1; tx_id_.reset(); - ls_id_.reset(); tx_ctx_serialize_size_ = 0; row_num_ = 0; row_idx_ = 0; @@ -145,9 +134,7 @@ struct ObTxCtxTableMeta void destroy() { reset(); } ObTxCtxTableMeta &operator=(const ObTxCtxTableMeta &r) { - version_ = r.version_; tx_id_ = r.tx_id_; - ls_id_ = r.ls_id_; tx_ctx_serialize_size_ = r.tx_ctx_serialize_size_; row_num_ = r.row_num_; row_idx_ = r.row_idx_; @@ -155,12 +142,10 @@ struct ObTxCtxTableMeta } void init(transaction::ObTransID tx_id, - share::ObLSID ls_id, int64_t row_value_serialize_size, int32_t row_num, int32_t row_idx) { tx_id_ = tx_id; - ls_id_ = ls_id; tx_ctx_serialize_size_ = row_value_serialize_size; row_num_ = row_num; row_idx_ = row_idx; @@ -192,7 +177,6 @@ struct ObTxCtxTableMeta bool is_multi_row_next_extent(const ObTxCtxTableMeta& next) const { return next.tx_id_ == tx_id_ && - next.ls_id_ == ls_id_ && next.tx_ctx_serialize_size_ == tx_ctx_serialize_size_ && next.row_num_ == row_num_ && next.row_idx_ == row_idx_ + 1 && @@ -203,26 +187,13 @@ struct ObTxCtxTableMeta { return tx_ctx_serialize_size_; } - int32_t get_version() const - { - return version_; - } - TO_STRING_KV(K_(tx_id), K_(ls_id), K_(tx_ctx_serialize_size), K_(row_num), K_(row_idx), K_(version)); + TO_STRING_KV(K_(tx_id), K_(tx_ctx_serialize_size), K_(row_num), K_(row_idx)); private: int serialize_(char* buf, const int64_t buf_len, int64_t &pos) const; int deserialize_(const char* buf, const int64_t buf_len, int64_t &pos); int64_t get_serialize_size_() const; -public: - static constexpr int VERSION_0 = 0; - // V1, fix bug: - // ctx serialized size record in header not equals to real serialized size - // it is because of the CommonID's get serialize_size always return 8 - // but it use variant encoding - static constexpr int VERSION_1 = 1; private: - int32_t version_; transaction::ObTransID tx_id_; - share::ObLSID ls_id_; int64_t tx_ctx_serialize_size_; int32_t row_num_; int32_t row_idx_; @@ -270,40 +241,6 @@ class ObITxDataCheckFunctor VIRTUAL_TO_STRING_KV(K_(tx_data_check_data)); public: - // Reserved compatibility data needs to rely both on - // the tx table state before the reserved_scn, as well as - // the tx table state after the reserved_scn. - // Otherwise: - // 1. If the tx table state before the reserved_scn is - // not relied upon, there could be a loss of rollbacks in the - // previous undo_status. - // 2. If the tx table state after the reserved_scn is - // not relied upon, there could be a loss of the most recent decided txn - // state in the later side. - // - // When uncommitted data exists around the reserved SCN, it is necessary to - // fuse tx data states from both sides of the boundary. - // - // Therefore, it's necessary to fuse the txn states. While we should note that - // txn is composed of txn states(state), commit versions(commit_version) and - // rollback sequences(undo_status). Among these: - // 1. From the previous side, what's needed are the txn states of the already - // committed transactions before the reserved_scn, along with their - // commit versions and rollback sequences, as well as the rollback - // sequences of txns that are not yet committed. - // 2. From the later side, what's required are the transaction states, - // commit versions, and rollback sequences of transactions after the - // reserved_scn. - // - // Hence, the later side of the txn state for uncommitted data on the previous side - // should follow these steps: - // - From the previous side, if a txn has been committed(meaning there - // is a transaction state, commit version, or contained in the rollback - // sequence), it can be directly obtained from the previous side. - // - If a txn is uncommitted on the previous side (no decided txn state, commit - // version, or rollback sequence exists), then its details need to be - // determined from the txn state, commit version, or rollback sequence on - // the later side. ObTxDataCheckData tx_data_check_data_; // In the tx_data_table, defensive error reporting strategies are implemented, // which means that potential errors are proactively handled and reported diff --git a/src/storage/tx_table/ob_tx_table_guards.cpp b/src/storage/tx_table/ob_tx_table_guards.cpp index 0e900c46d..f9a429399 100644 --- a/src/storage/tx_table/ob_tx_table_guards.cpp +++ b/src/storage/tx_table/ob_tx_table_guards.cpp @@ -85,7 +85,6 @@ int ObTxTableGuards::lock_for_read( LockForReadFunctor fn(lock_for_read_arg, can_read, trans_version, - tx_table_guard_.get_ls_id(), cleanout_op, recheck_op); @@ -110,7 +109,6 @@ int ObTxTableGuards::lock_for_read( LockForReadFunctor fn(lock_for_read_arg, can_read, trans_version, - tx_table_guard_.get_ls_id(), clean_nothing_op, recheck_nothing_op); @@ -141,11 +139,6 @@ int ObTxTableGuards::cleanout_tx_node( return ret; } -bool ObTxTableGuards::check_ls_offline() -{ - return tx_table_guard_.check_ls_offline(); -} - int ObTxTableGuards::check_with_tx_data( const transaction::ObTransID &data_tx_id, ObITxDataCheckFunctor &functor) diff --git a/src/storage/tx_table/ob_tx_table_guards.h b/src/storage/tx_table/ob_tx_table_guards.h index ee4e1818b..cbd277f77 100644 --- a/src/storage/tx_table/ob_tx_table_guards.h +++ b/src/storage/tx_table/ob_tx_table_guards.h @@ -137,8 +137,6 @@ class ObTxTableGuards const transaction::ObTransID &data_tx_id, ObITxDataCheckFunctor &functor); - bool check_ls_offline(); - TO_STRING_KV(K_(tx_table_guard)); DISABLE_COPY_ASSIGN(ObTxTableGuards); diff --git a/src/storage/tx_table/ob_tx_table_interface.cpp b/src/storage/tx_table/ob_tx_table_interface.cpp index 5eeffec25..1fac487a5 100644 --- a/src/storage/tx_table/ob_tx_table_interface.cpp +++ b/src/storage/tx_table/ob_tx_table_interface.cpp @@ -137,34 +137,5 @@ int ObTxTableGuard::self_freeze_task() } } -bool ObTxTableGuard::check_ls_offline() -{ - bool discover_ls_offline = false; - int ret = OB_SUCCESS; - - if (OB_ISNULL(tx_table_)) { - ret = OB_NOT_INIT; - discover_ls_offline = false; - STORAGE_LOG(WARN, "tx table is nullptr", K(ret), K(discover_ls_offline)); - } else { - int64_t cur_epoch = tx_table_->get_epoch(); - ObTxTable::TxTableState tx_table_state = tx_table_->get_state(); - if (cur_epoch != epoch_ || tx_table_state == ObTxTable::TxTableState::PREPARE_OFFLINE - || tx_table_state == ObTxTable::TxTableState::OFFLINE) { - discover_ls_offline = true; - - STORAGE_LOG(INFO, "discover ls offline", K(discover_ls_offline), K(cur_epoch), K(epoch_), - K(tx_table_state)); - } - } - - return discover_ls_offline; -} - -share::ObLSID ObTxTableGuard::get_ls_id() const -{ - return tx_table_->get_ls_id(); -} - } // namespace storage } // end namespace oceanbase diff --git a/src/storage/tx_table/ob_tx_table_interface.h b/src/storage/tx_table/ob_tx_table_interface.h index 11bca8973..268836468 100644 --- a/src/storage/tx_table/ob_tx_table_interface.h +++ b/src/storage/tx_table/ob_tx_table_interface.h @@ -59,8 +59,6 @@ class ObTxTableGuard ObTxTable *get_tx_table() const { return tx_table_; } - share::ObLSID get_ls_id() const; - int64_t get_epoch() const { return epoch_; } ObTxDataMiniCache &get_mini_cache() { return mini_cache_; } @@ -99,8 +97,6 @@ class ObTxTableGuard int self_freeze_task(); - bool check_ls_offline(); - void reuse() { mini_cache_.reset(); } TO_STRING_KV(KP_(tx_table), K_(epoch), K(mini_cache_)); diff --git a/src/storage/tx_table/ob_tx_table_iterator.cpp b/src/storage/tx_table/ob_tx_table_iterator.cpp index 3881b9b71..9832d1ca4 100644 --- a/src/storage/tx_table/ob_tx_table_iterator.cpp +++ b/src/storage/tx_table/ob_tx_table_iterator.cpp @@ -16,7 +16,7 @@ #include "ob_tx_table_iterator.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "src/storage/ls/ob_ls.h" namespace oceanbase @@ -703,7 +703,7 @@ int ObTxCtxMemtableScanIterator::init(ObTxCtxMemtable *tx_ctx_memtable) int ObTxCtxMemtableScanIterator::serialize_next_tx_ctx_(ObTxLocalBuffer &buffer, int64_t &serialize_size, - transaction::ObPartTransCtx *&tx_ctx) + transaction::ObTxCtx *&tx_ctx) { int ret = OB_SUCCESS; bool need_retry = true; @@ -733,7 +733,7 @@ int ObTxCtxMemtableScanIterator::inner_get_next_row(const ObDatumRow *&row) int ret = OB_SUCCESS; ObTxCtxTableMeta curr_meta; - transaction::ObPartTransCtx *tx_ctx = NULL; + transaction::ObTxCtx *tx_ctx = NULL; char *row_buf = NULL; int64_t need_merge_length = 0; int64_t cur_merge_length = 0; @@ -758,7 +758,6 @@ int ObTxCtxMemtableScanIterator::inner_get_next_row(const ObDatumRow *&row) } } else { (void)curr_meta.init(tx_ctx->get_trans_id(), - tx_ctx->get_ls_id(), serialize_size, (serialize_size + MAX_VALUE_LENGTH_ - 1) / MAX_VALUE_LENGTH_ /* row_num */, 0 /* row_idx */); diff --git a/src/storage/tx_table/ob_tx_table_iterator.h b/src/storage/tx_table/ob_tx_table_iterator.h index c2e310293..99e6ab358 100644 --- a/src/storage/tx_table/ob_tx_table_iterator.h +++ b/src/storage/tx_table/ob_tx_table_iterator.h @@ -272,7 +272,7 @@ class ObTxCtxMemtableScanIterator : public memtable::ObIMemtableIterator return ret; } private: - int serialize_next_tx_ctx_(ObTxLocalBuffer &buffer, int64_t &serialize_size, transaction::ObPartTransCtx *&tx_ctx); + int serialize_next_tx_ctx_(ObTxLocalBuffer &buffer, int64_t &serialize_size, transaction::ObTxCtx *&tx_ctx); private: const static int64_t TX_CTX_META_BUF_LENGTH = 256; diff --git a/src/storage/vector_index/ob_vector_index_refresh.cpp b/src/storage/vector_index/ob_vector_index_refresh.cpp index a7cfbd2ec..7263e23c4 100644 --- a/src/storage/vector_index/ob_vector_index_refresh.cpp +++ b/src/storage/vector_index/ob_vector_index_refresh.cpp @@ -18,10 +18,12 @@ #define USING_LOG_PREFIX STORAGE #include "storage/vector_index/ob_vector_index_refresh.h" +#include "share/inner_table/ob_inner_table_schema_constants.h" #include "share/rc/ob_module_provider.h" #include "rootserver/ob_rs_serial_call.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" #include "sql/engine/cmd/ob_ddl_executor_util.h" +#include "observer/vector_index/ob_vector_index_async_task.h" namespace oceanbase { namespace storage { @@ -483,7 +485,8 @@ int ObVectorIndexRefresher::do_refresh() { common::sqlclient::ObMySQLResult *result = nullptr; ObSqlString select_sql; if (OB_FAIL(select_sql.append_fmt( - "select tablet_id from oceanbase.__all_tablet_to_ls where table_id = %lu", + "select tablet_id from oceanbase.%s where table_id = %lu", + OB_ALL_TABLET_TO_TABLE_TNAME, domain_table_schema->get_table_id()))) { LOG_WARN("fail to assign sql", KR(ret)); } else if (OB_FAIL(refresh_ctx_->trans_->read( @@ -712,7 +715,6 @@ int ObVectorIndexRefresher::do_rebuild() { rebuild_index_arg.index_action_type_ = obcall::ObIndexArg::ADD_INDEX; rebuild_index_arg.parallelism_ = refresh_ctx_->idx_parallel_creation_; rebuild_index_arg.vidx_refresh_info_.index_params_ = idx_parameters; - rebuild_index_arg.rebuild_index_type_ = obcall::ObRebuildIndexArg::RebuildIndexType::REBUILD_INDEX_TYPE_VEC; if (OB_FAIL(rebuild_index_arg.based_schema_object_infos_.push_back( ObBasedSchemaObjectInfo(domain_table_schema->get_table_id(), TABLE_SCHEMA, diff --git a/tools/deploy/mysql_test/include/backup_spm_var.inc b/tools/deploy/mysql_test/include/backup_spm_var.inc index b1080c97a..eb65346e3 100644 --- a/tools/deploy/mysql_test/include/backup_spm_var.inc +++ b/tools/deploy/mysql_test/include/backup_spm_var.inc @@ -1,7 +1,7 @@ --disable_query_log --disable_result_log -let $use_spb_bk = query_get_value(select value from oceanbase.__tenant_virtual_session_variable where VARIABLE_NAME = 'optimizer_use_sql_plan_baselines',value, 1); -let $cap_spb_bk = query_get_value(select value from oceanbase.__tenant_virtual_session_variable where VARIABLE_NAME = 'optimizer_capture_sql_plan_baselines',value, 1); +let $use_spb_bk = query_get_value(select variable_value from information_schema.session_variables where variable_name = 'optimizer_use_sql_plan_baselines', variable_value, 1); +let $cap_spb_bk = query_get_value(select variable_value from information_schema.session_variables where variable_name = 'optimizer_capture_sql_plan_baselines', variable_value, 1); set global optimizer_use_sql_plan_baselines = 'OFF'; set global optimizer_capture_sql_plan_baselines = 'OFF'; set session optimizer_use_sql_plan_baselines = 'OFF'; diff --git a/tools/deploy/mysql_test/include/minor_freeze_tablet_and_wait_done.inc b/tools/deploy/mysql_test/include/minor_freeze_tablet_and_wait_done.inc index 3921ccabd..83a974cba 100644 --- a/tools/deploy/mysql_test/include/minor_freeze_tablet_and_wait_done.inc +++ b/tools/deploy/mysql_test/include/minor_freeze_tablet_and_wait_done.inc @@ -6,7 +6,7 @@ connect (obsys_minor,$OBMYSQL_MS0,admin,$OBMYSQL_PWD,test,$OBMYSQL_PORT); connection obsys_minor; -let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_ls where table_id = (select /*+QUERY_TIMEOUT(10000000)*/ table_id from oceanbase.__all_table where table_name = $__freeze_table_name__), tablet_id, 1); +let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_table where table_id = (select /*+QUERY_TIMEOUT(10000000)*/ table_id from oceanbase.__all_table where table_name = $__freeze_table_name__), tablet_id, 1); eval alter system minor freeze tenant sys tablet_id = $__tablet_id__; sleep 5; diff --git a/tools/deploy/mysql_test/include/wait_mini_minor_merge_for_table.inc b/tools/deploy/mysql_test/include/wait_mini_minor_merge_for_table.inc index f6e3c50c5..442198dab 100644 --- a/tools/deploy/mysql_test/include/wait_mini_minor_merge_for_table.inc +++ b/tools/deploy/mysql_test/include/wait_mini_minor_merge_for_table.inc @@ -12,7 +12,7 @@ while ($i > 0) { sleep 1; dec $i; - let $cur_minor_count = query_get_value(select count(*) > 0 as c from oceanbase.__all_virtual_tablet_compaction_history where type = 'MINOR_MERGE' and tablet_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $__table_id__) and finish_time >= '$__now_time__', c, 1); + let $cur_minor_count = query_get_value(select count(*) > 0 as c from oceanbase.__all_virtual_tablet_compaction_history where type = 'MINOR_MERGE' and tablet_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $__table_id__) and finish_time >= '$__now_time__', c, 1); if ($cur_minor_count > 0) { let $i = -5; diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 1586a6052..0182fddbf 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -279,16 +279,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_DEPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_RUNNING_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_AI_MODELS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_AI_MODEL_ENDPOINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -319,7 +309,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_SPACE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -358,7 +347,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | V$OB_GROUP_IO_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | V$OB_LOG_TRANSPORT_DEST_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -375,8 +363,8 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | +| def | oceanbase | V$OB_TRANSACTION_WRITE_STATE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_VECTOR_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -448,20 +436,11 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_location | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_location_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_ncomp_dll_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -523,7 +502,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_reorganize_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | +| def | oceanbase | __all_tablet_to_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_task_opt_stat_gather_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_temp_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -590,7 +569,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_load_data_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_virtual_log_transport_dest_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_long_ops_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_macro_block_marker_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -604,7 +582,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_minor_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_virtual_mview_running_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -905,7 +882,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_location | 1 | oceanbase | idx_location_name | 1 | location_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 1 | location_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mlog | 0 | oceanbase | PRIMARY | 1 | mlog_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 2 | parent_column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -916,27 +892,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_mock_fk_parent_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 2 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 4 | detail_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 1 | oceanbase | idx_mview_refresh_run_stats_num_mvs_current | 1 | num_mvs_current | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_end_time | 1 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 2 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_params | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | 0 | oceanbase | PRIMARY | 1 | id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 4 | step | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 1 | database_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 2 | key_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 3 | compile_db_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -1126,8 +1081,8 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_tablet_reorganize_history | 0 | oceanbase | PRIMARY | 2 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_dest | 1 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_src | 1 | src_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_task_opt_stat_gather_history | 0 | oceanbase | PRIMARY | 1 | task_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -1288,16 +1243,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_DEPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVIEW_RUNNING_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_AI_MODELS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_AI_MODEL_ENDPOINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1328,7 +1273,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_SPACE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1367,7 +1311,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | V$OB_GROUP_IO_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | V$OB_LOG_TRANSPORT_DEST_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1384,8 +1327,8 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | +| def | oceanbase | V$OB_TRANSACTION_WRITE_STATE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$OB_VECTOR_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1456,20 +1399,11 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_location | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_location_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_ncomp_dll_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1525,7 +1459,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_reorganize_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | +| def | oceanbase | __all_tablet_to_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_table_opt_stat_gather_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1599,7 +1533,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_load_data_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_virtual_log_transport_dest_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_long_ops_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_macro_block_marker_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1613,7 +1546,6 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_mem_leak_checker_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_minor_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | -| def | oceanbase | __all_virtual_mview_running_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | | def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | FALSE | 0 | INDEX | @@ -1949,7 +1881,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_location | 1 | oceanbase | idx_location_name | 1 | location_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 1 | location_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mlog | 0 | oceanbase | PRIMARY | 1 | mlog_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 2 | parent_column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -1960,27 +1891,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_mock_fk_parent_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 2 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 4 | detail_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 1 | oceanbase | idx_mview_refresh_run_stats_num_mvs_current | 1 | num_mvs_current | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_end_time | 1 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 2 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_params | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | 0 | oceanbase | PRIMARY | 1 | id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 4 | step | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 1 | database_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 2 | key_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 3 | compile_db_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -2170,8 +2080,8 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_tablet_reorganize_history | 0 | oceanbase | PRIMARY | 2 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_dest | 1 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_src | 1 | src_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_task_opt_stat_gather_history | 0 | oceanbase | PRIMARY | 1 | task_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -2419,7 +2329,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_location | 1 | oceanbase | idx_location_name | 1 | location_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 1 | location_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_location_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mlog | 0 | oceanbase | PRIMARY | 1 | mlog_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 1 | mock_fk_parent_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mock_fk_parent_table_column | 0 | oceanbase | PRIMARY | 2 | parent_column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -2430,27 +2339,6 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_mock_fk_parent_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 2 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 4 | detail_table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_run_stats | 1 | oceanbase | idx_mview_refresh_run_stats_num_mvs_current | 1 | num_mvs_current | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_end_time | 1 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats | 1 | oceanbase | idx_mview_refresh_stats_mview_end_time | 2 | end_time | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_params | 0 | oceanbase | PRIMARY | 1 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | 0 | oceanbase | PRIMARY | 1 | id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 1 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 3 | retry_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_mview_refresh_stmt_stats | 0 | oceanbase | PRIMARY | 4 | step | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 1 | database_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 2 | key_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ncomp_dll | 0 | oceanbase | PRIMARY | 3 | compile_db_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -2640,8 +2528,8 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_tablet_reorganize_history | 0 | oceanbase | PRIMARY | 2 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_dest | 1 | dest_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_reorganize_history | 1 | oceanbase | idx_tablet_his_table_id_src | 1 | src_tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | -| def | oceanbase | __all_tablet_to_ls | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_tablet_to_table | 1 | oceanbase | idx_tablet_to_table_id | 1 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 1 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_tablet_to_table_history | 0 | oceanbase | PRIMARY | 2 | schema_version | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_task_opt_stat_gather_history | 0 | oceanbase | PRIMARY | 1 | task_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | diff --git a/tools/deploy/mysql_test/test_suite/direct_load_data/include/wait_direct_load_memtable_dump.inc b/tools/deploy/mysql_test/test_suite/direct_load_data/include/wait_direct_load_memtable_dump.inc index 4cebf2a04..8c013f3e7 100644 --- a/tools/deploy/mysql_test/test_suite/direct_load_data/include/wait_direct_load_memtable_dump.inc +++ b/tools/deploy/mysql_test/test_suite/direct_load_data/include/wait_direct_load_memtable_dump.inc @@ -13,7 +13,7 @@ while($__i__ > 0) { sleep 1; dec $__i__; - let $__memstore_cnt__ = query_get_value(select count(*) as cnt from oceanbase.__all_virtual_table_mgr where table_type=4 and tablet_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $__table_id__), cnt, 1); + let $__memstore_cnt__ = query_get_value(select count(*) as cnt from oceanbase.__all_virtual_table_mgr where table_type=4 and tablet_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $__table_id__), cnt, 1); if ($__memstore_cnt__ == 0) { let $__i__ = -5; diff --git a/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_lock.test b/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_lock.test index 8bfa24852..a4a7e2d3d 100644 --- a/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_lock.test +++ b/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_lock.test @@ -74,12 +74,12 @@ connection obsys; --echo # 检查__all_virtual_obj_lock中Fork相关的锁（源表A） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a0_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_a0_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a0_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_a0_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表B） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b0_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_b0_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b0_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_b0_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 释放 fork A to B，让它继续执行 @@ -175,17 +175,17 @@ connection obsys; --echo # 检查__all_virtual_obj_lock中Fork相关的锁（源表A） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_a1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_a1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表B） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_b1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_b1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表C） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_c1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_c1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 释放 fork A to C，让它先完成 @@ -207,12 +207,12 @@ connection obsys; --echo # 检查__all_virtual_obj_lock中Fork相关的锁（源表A） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_a1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_a1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_a1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表B） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_b1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b1_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_b1_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 释放 fork A to B，让它继续执行 @@ -315,12 +315,12 @@ connection obsys; --echo # 检查__all_virtual_obj_lock中Fork相关的锁（表B） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_b2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_b2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表C） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_c2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_c2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 释放 fork B to C，让它先完成 @@ -342,12 +342,12 @@ connection obsys; --echo # 检查__all_virtual_obj_lock中Fork相关的锁（表B） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_b2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_b2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_b2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 检查__all_virtual_obj_lock中Fork相关的锁（fork表C） --disable_query_log ---eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_c2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 +--eval select count(*), lock_mode, owner_id, obj_type, owner_type from oceanbase.__all_virtual_obj_lock where obj_id = $table_c2_id or obj_id in (select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_c2_id) group by lock_mode, owner_id, obj_type, owner_type order by 1, 2, 3, 4, 5 --enable_query_log --echo # 释放 fork A to B，让它完成 diff --git a/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_merge.test b/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_merge.test index 56cf3559f..531569743 100644 --- a/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_merge.test +++ b/tools/deploy/mysql_test/test_suite/fork_table/t/fork_table_merge.test @@ -75,13 +75,13 @@ connection obsys; --disable_query_log --disable_result_log let $table_id = `select table_id from oceanbase.__all_table where table_name = 't1' limit 1`; -let $tablet_cnt = `select count(*) as c from oceanbase.__all_tablet_to_ls where table_id = $table_id`; +let $tablet_cnt = `select count(*) as c from oceanbase.__all_tablet_to_table where table_id = $table_id`; let $wait_timeout= 30; let $wait_condition= select /*+query_timeout(100000000)*/ count(distinct tablet_id) = $tablet_cnt from oceanbase.__all_virtual_tablet_compaction_history where tablet_id in ( - select tablet_id from oceanbase.__all_tablet_to_ls where table_id = $table_id + select tablet_id from oceanbase.__all_tablet_to_table where table_id = $table_id ) and type = 'MAJOR_MERGE'; --source mysql_test/include/wait_condition.inc diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index fa5112f3a..661b46b86 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -248,7 +248,6 @@ system_memory sys_bkgd_migration_change_member_list_timeout sys_bkgd_migration_retry_num sys_bkgd_net_percentage -tablet_ls_cache_priority tablet_meta_table_check_interval tablet_meta_table_scan_batch_count tablet_size @@ -260,7 +259,7 @@ tenant_sql_login_thread_count tenant_sql_net_thread_count tenant_task_queue_size trace_log_slow_query_watermark -trx_2pc_retry_interval +trx_commit_retry_interval ttl_thread_score undo_retention unit_gc_wait_time @@ -338,7 +337,6 @@ _enable_hgby_llc_ndv_adaptive _enable_hgby_skew_detection _enable_index_merge _enable_inner_session_mgr -_enable_insertup_replace_gts_opt _enable_in_range_optimization _enable_kvcache_hazard_pointer _enable_log_cache @@ -423,7 +421,6 @@ _log_writer_parallelism _ls_gc_wait_readonly_tx_time _ls_migration_wait_completing_timeout _max_elr_dependent_trx_count -_max_ls_cnt_per_server _max_malloc_sample_interval _max_partition_count_to_collect_statistic _max_px_workers_per_cpu @@ -466,7 +463,6 @@ _ob_pl_compile_max_concurrency _ob_query_rate_limit _ob_sqlstat_enable _ob_ssl_invited_nodes -_ob_trans_rpc_timeout _optimizer_ads_time_limit _optimizer_better_inlist_costing _optimizer_group_by_placement @@ -527,7 +523,6 @@ _ss_micro_cache_memory_percentage _ss_new_leader_overwrite_delay _ss_old_ver_retention_time _stall_threshold_for_dynamic_worker -_standby_max_replay_gap_time _storage_meta_memory_limit_percentage _storage_stream_rpc_buffer_size _stream_rpc_max_wait_timeout diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index ac23b35e5..ca6794546 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -770,12 +770,6 @@ EXTRA_INFO varchar(256) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_ROOTSERVICE_EVENT_HISTORY limit 1); cnt 1 -desc oceanbase.DBA_OB_TABLET_TO_LS; -Field Type Null Key Default Extra -TABLET_ID bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLET_TO_LS limit 1); -cnt -1 desc oceanbase.DBA_OB_TABLET_REPLICAS; Field Type Null Key Default Extra CREATE_TIME bigint(20) YES NULL @@ -925,12 +919,12 @@ DEF_FREELISTS bigint(0) NO DEF_FREELIST_GROUPS bigint(0) NO DEF_LOGGING varchar(7) NO DEF_COMPRESSION varchar(8) NO -DEF_COMPRESS_FOR varchar(12) NO +DEF_COMPRESS_FOR varchar(12) NO NULL DEF_BUFFER_POOL varchar(7) NO DEF_FLASH_CACHE varchar(7) NO DEF_CELL_FLASH_CACHE varchar(7) NO REF_PTN_CONSTRAINT_NAME varchar(30) NO -INTERVAL text NO +INTERVAL text YES NULL AUTOLIST varchar(3) NO INTERVAL_SUBPARTITION text NO AUTOLIST_SUBPARTITION varchar(3) NO @@ -1340,26 +1334,22 @@ MEMORY_USAGE bigint(20) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt 1 -desc oceanbase.V$OB_TRANSACTION_PARTICIPANTS; +desc oceanbase.V$OB_TRANSACTION_WRITE_STATE; Field Type Null Key Default Extra SESSION_ID bigint(20) NO NULL -SCHEDULER_ADDR varchar(64) NO NULL -TX_TYPE varchar(11) NO TX_ID bigint(20) NO NULL -PARTICIPANTS varchar(1024) NO NULL +WRITE_STATE varchar(1024) NO NULL CTX_CREATE_TIME timestamp(6) YES NULL TX_EXPIRED_TIME timestamp(6) YES NULL STATE varchar(13) NO ACTION varchar(10) NO PENDING_LOG_SIZE bigint(20) NO NULL FLUSHED_LOG_SIZE bigint(20) NO NULL -ROLE varchar(8) NO -COORD bigint(20) NO NULL LAST_REQUEST_TIME timestamp(6) YES NULL FORMATID bigint(20) NO GLOBALID varchar(257) NO NULL BRANCHID varchar(257) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_PARTICIPANTS limit 1); +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_WRITE_STATE limit 1); cnt 1 desc oceanbase.V$OB_TABLET_COMPACTION_HISTORY; @@ -1765,13 +1755,7 @@ cnt 1 desc oceanbase.V$OB_LOG_STAT; Field Type Null Key Default Extra -ROLE varchar(32) NO NULL -PROPOSAL_ID bigint(20) NO NULL -CONFIG_VERSION varchar(128) NO NULL ACCESS_MODE varchar(32) NO NULL -PAXOS_MEMBER_LIST varchar(1024) NO NULL -PAXOS_REPLICA_NUM bigint(20) NO NULL -IN_SYNC varchar(3) NO NULL BASE_LSN bigint(20) unsigned NO NULL BEGIN_LSN bigint(20) unsigned NO NULL BEGIN_SCN bigint(20) unsigned NO NULL @@ -1779,9 +1763,6 @@ END_LSN bigint(20) unsigned NO NULL END_SCN bigint(20) unsigned NO NULL MAX_LSN bigint(20) unsigned NO NULL MAX_SCN bigint(20) unsigned NO NULL -ARBITRATION_MEMBER varchar(128) NO NULL -DEGRADED_LIST varchar(1024) NO NULL -LEARNER_LIST longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_STAT limit 1); cnt 1 @@ -2111,8 +2092,7 @@ SESSION_ID bigint(20) NO NULL TX_ID bigint(20) NO NULL STATE varchar(18) NO CLUSTER_ID bigint(20) NO NULL -COORDINATOR bigint(20) NO NULL -PARTICIPANTS varchar(1024) YES NULL +WRITE_STATE varchar(1024) YES NULL ISOLATION_LEVEL varchar(16) NO SNAPSHOT_VERSION bigint(20) unsigned YES NULL ACCESS_MODE varchar(10) NO @@ -2489,170 +2469,6 @@ Timestamp datetime YES select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.columns_priv limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_LOGS; -Field Type Null Key Default Extra -LOG_OWNER varchar(128) NO -MASTER varchar(128) NO -LOG_TABLE varchar(128) NO -LOG_TRIGGER varchar(128) NO -ROWIDS varchar(3) NO -PRIMARY_KEY varchar(3) NO -OBJECT_ID varchar(3) NO -FILTER_COLUMNS varchar(3) NO -SEQUENCE varchar(3) NO -INCLUDE_NEW_VALUES varchar(3) NO -PURGE_ASYNCHRONOUS varchar(3) NO -PURGE_DEFERRED varchar(3) NO -PURGE_START datetime YES NULL -PURGE_INTERVAL varchar(200) YES NULL -LAST_PURGE_DATE datetime YES NULL -LAST_PURGE_STATUS bigint(1) NO -NUM_ROWS_PURGED bigint(20) YES NULL -COMMIT_SCN_BASED varchar(3) NO -STAGING_LOG varchar(3) NO -PURGE_DOP bigint(20) NO -LAST_PURGE_TIME bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_LOGS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEWS; -Field Type Null Key Default Extra -OWNER varchar(128) NO -MVIEW_NAME varchar(128) NO -CONTAINER_NAME varchar(128) NO -QUERY longtext NO NULL -QUERY_LEN bigint(20) NO -UPDATABLE varchar(1) NO -UPDATE_LOG varchar(128) NO -MASTER_ROLLBACK_SEG varchar(128) NO -MASTER_LINK varchar(128) NO -REWRITE_ENABLED varchar(1) NO -REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(32) NO -REFRESH_METHOD varchar(8) NO -BUILD_MODE varchar(9) NO -FAST_REFRESHABLE varchar(18) NO -LAST_REFRESH_TYPE varchar(8) NO -LAST_REFRESH_DATE datetime YES NULL -LAST_REFRESH_END_TIME datetime NO -STALENESS varchar(19) NO -AFTER_FAST_REFRESH varchar(19) NO -UNKNOWN_PREBUILT varchar(1) NO -UNKNOWN_PLSQL_FUNC varchar(1) NO -UNKNOWN_EXTERNAL_TABLE varchar(1) NO -UNKNOWN_CONSIDER_FRESH varchar(1) NO -UNKNOWN_IMPORT varchar(1) NO -UNKNOWN_TRUSTED_FD varchar(1) NO -COMPILE_STATE varchar(19) NO -USE_NO_INDEX varchar(1) NO -STALE_SINCE datetime NO -NUM_PCT_TABLES bigint(0) NO -NUM_FRESH_PCT_REGIONS bigint(0) NO -NUM_STALE_PCT_REGIONS bigint(0) NO -SEGMENT_CREATED varchar(3) NO -EVALUATION_EDITION varchar(128) NO -UNUSABLE_BEFORE varchar(128) NO -UNUSABLE_BEGINNING varchar(128) NO -DEFAULT_COLLATION varchar(100) NO -ON_QUERY_COMPUTATION varchar(1) NO -REFRESH_DOP bigint(20) NO -DATA_SYNC_SCN bigint(20) unsigned NO -DATA_SYNC_DELAY varchar(128) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEWS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS; -Field Type Null Key Default Extra -PARAMETER_NAME varchar(16) NO -VALUE varchar(40) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_PARAMS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -COLLECTION_LEVEL varchar(8) NO -RETENTION_PERIOD bigint(20) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_PARAMS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_RUN_STATS; -Field Type Null Key Default Extra -RUN_OWNER varchar(128) NO NULL -REFRESH_ID bigint(20) NO NULL -NUM_MVS bigint(20) NO NULL -MVIEWS text NO NULL -BASE_TABLES text YES NULL -METHOD text YES NULL -ROLLBACK_SEG text YES NULL -PUSH_DEFERRED_RPC varchar(1) NO -REFRESH_AFTER_ERRORS varchar(1) NO -PURGE_OPTION bigint(20) NO NULL -PARALLELISM bigint(20) NO NULL -HEAP_SIZE bigint(20) NO NULL -ATOMIC_REFRESH varchar(1) NO -NESTED varchar(1) NO -OUT_OF_PLACE varchar(1) NO -NUMBER_OF_FAILURES bigint(20) NO NULL -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -COMPLETE_STATS_AVAILABLE varchar(1) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_RUN_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -REFRESH_METHOD varchar(30) NO -REFRESH_OPTIMIZATIONS text NO -ADDITIONAL_EXECUTIONS text NO -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -INITIAL_NUM_ROWS bigint(20) NO NULL -FINAL_NUM_ROWS bigint(20) NO NULL -RESULT bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_CHANGE_STATS; -Field Type Null Key Default Extra -TBL_OWNER varchar(128) NO -TBL_NAME varchar(128) NO -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -NUM_ROWS_INS bigint(20) YES NULL -NUM_ROWS_UPD bigint(20) YES NULL -NUM_ROWS_DEL bigint(20) YES NULL -NUM_ROWS_DL_INS bigint(1) NO -PMOPS_OCCURRED varchar(1) NO -PMOP_DETAILS text NO -NUM_ROWS bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_CHANGE_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STMT_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -STEP bigint(20) NO NULL -SQLID varchar(32) YES NULL -STMT longtext NO NULL -EXECUTION_TIME bigint(20) NO NULL -EXECUTION_PLAN longtext YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STMT_STATS limit 1); -cnt -1 desc oceanbase.V$OB_COMPATIBILITY_CONTROL; Field Type Null Key Default Extra NAME varchar(128) NO NULL @@ -3147,26 +2963,6 @@ REQUIRED_SIZE decimal(42,0) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLE_SPACE_USAGE limit 1); cnt 1 -desc oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT; -Field Type Null Key Default Extra -CLIENT_IP varchar(46) NO NULL -CLIENT_PID bigint(20) NO NULL -CLIENT_TYPE varchar(7) NO NULL -START_SERVE_TIME timestamp(6) NO NULL -LAST_SERVE_TIME timestamp(6) NO NULL -LAST_READ_SOURCE varchar(7) NO NULL -LAST_REQUEST_TYPE varchar(24) NO NULL -LAST_REQUEST_LOG_LSN bigint(20) unsigned NO NULL -LAST_REQUEST_LOG_SCN bigint(20) unsigned NO NULL -LAST_FAILED_REQUEST longtext NO NULL -AVG_REQUEST_PROCESS_TIME bigint(20) NO NULL -AVG_REQUEST_QUEUE_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_SIZE bigint(20) NO NULL -AVG_LOG_TRANSPORT_BANDWIDTH varchar(29) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT limit 1); -cnt -1 desc information_schema.INNODB_SYS_FIELDS; Field Type Null Key Default Extra INDEX_ID bigint(0) unsigned NO @@ -3296,27 +3092,6 @@ MAX_CONCURRENCY bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_SQL_CCL_STATUS limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_RUNNING_JOBS; -Field Type Null Key Default Extra -TABLE_NAME varchar(256) NO -JOB_TYPE varchar(64) NO -SESSION_ID bigint(20) unsigned NO NULL -READ_SNAPSHOT bigint(20) NO NULL -PARALLEL bigint(20) NO NULL -JOB_START_TIME timestamp(6) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_RUNNING_JOBS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEW_DEPS; -Field Type Null Key Default Extra -MVIEW_OWNER varchar(128) NO -MVIEW_NAME varchar(256) NO -DEP_OWNER varchar(128) NO -DEP_NAME varchar(256) NO -DEP_TYPE varchar(64) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_DEPS limit 1); -cnt -1 desc oceanbase.DBA_OB_DYNAMIC_PARTITION_TABLES; Field Type Null Key Default Extra DATABASE_NAME varchar(128) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql_when_compare_sensitive.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql_when_compare_sensitive.result index 5e2fc48d6..6a55c4cd9 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql_when_compare_sensitive.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql_when_compare_sensitive.result @@ -770,12 +770,6 @@ EXTRA_INFO varchar(256) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_ROOTSERVICE_EVENT_HISTORY limit 1); cnt 1 -desc oceanbase.DBA_OB_TABLET_TO_LS; -Field Type Null Key Default Extra -TABLET_ID bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLET_TO_LS limit 1); -cnt -1 desc oceanbase.DBA_OB_TABLET_REPLICAS; Field Type Null Key Default Extra CREATE_TIME bigint(20) YES NULL @@ -925,12 +919,12 @@ DEF_FREELISTS bigint(0) NO DEF_FREELIST_GROUPS bigint(0) NO DEF_LOGGING varchar(7) NO DEF_COMPRESSION varchar(8) NO -DEF_COMPRESS_FOR varchar(12) NO +DEF_COMPRESS_FOR varchar(12) NO NULL DEF_BUFFER_POOL varchar(7) NO DEF_FLASH_CACHE varchar(7) NO DEF_CELL_FLASH_CACHE varchar(7) NO REF_PTN_CONSTRAINT_NAME varchar(30) NO -INTERVAL text NO +INTERVAL text YES NULL AUTOLIST varchar(3) NO INTERVAL_SUBPARTITION text NO AUTOLIST_SUBPARTITION varchar(3) NO @@ -1340,26 +1334,22 @@ MEMORY_USAGE bigint(20) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt 1 -desc oceanbase.V$OB_TRANSACTION_PARTICIPANTS; +desc oceanbase.V$OB_TRANSACTION_WRITE_STATE; Field Type Null Key Default Extra SESSION_ID bigint(20) NO NULL -SCHEDULER_ADDR varchar(64) NO NULL -TX_TYPE varchar(11) NO TX_ID bigint(20) NO NULL -PARTICIPANTS varchar(1024) NO NULL +WRITE_STATE varchar(1024) NO NULL CTX_CREATE_TIME timestamp(6) YES NULL TX_EXPIRED_TIME timestamp(6) YES NULL STATE varchar(13) NO ACTION varchar(10) NO PENDING_LOG_SIZE bigint(20) NO NULL FLUSHED_LOG_SIZE bigint(20) NO NULL -ROLE varchar(8) NO -COORD bigint(20) NO NULL LAST_REQUEST_TIME timestamp(6) YES NULL FORMATID bigint(20) NO GLOBALID varchar(257) NO NULL BRANCHID varchar(257) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_PARTICIPANTS limit 1); +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_WRITE_STATE limit 1); cnt 1 desc oceanbase.V$OB_TABLET_COMPACTION_HISTORY; @@ -1765,13 +1755,7 @@ cnt 1 desc oceanbase.V$OB_LOG_STAT; Field Type Null Key Default Extra -ROLE varchar(32) NO NULL -PROPOSAL_ID bigint(20) NO NULL -CONFIG_VERSION varchar(128) NO NULL ACCESS_MODE varchar(32) NO NULL -PAXOS_MEMBER_LIST varchar(1024) NO NULL -PAXOS_REPLICA_NUM bigint(20) NO NULL -IN_SYNC varchar(3) NO NULL BASE_LSN bigint(20) unsigned NO NULL BEGIN_LSN bigint(20) unsigned NO NULL BEGIN_SCN bigint(20) unsigned NO NULL @@ -1779,9 +1763,6 @@ END_LSN bigint(20) unsigned NO NULL END_SCN bigint(20) unsigned NO NULL MAX_LSN bigint(20) unsigned NO NULL MAX_SCN bigint(20) unsigned NO NULL -ARBITRATION_MEMBER varchar(128) NO NULL -DEGRADED_LIST varchar(1024) NO NULL -LEARNER_LIST longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_STAT limit 1); cnt 1 @@ -2111,8 +2092,7 @@ SESSION_ID bigint(20) NO NULL TX_ID bigint(20) NO NULL STATE varchar(18) NO CLUSTER_ID bigint(20) NO NULL -COORDINATOR bigint(20) NO NULL -PARTICIPANTS varchar(1024) YES NULL +WRITE_STATE varchar(1024) YES NULL ISOLATION_LEVEL varchar(16) NO SNAPSHOT_VERSION bigint(20) unsigned YES NULL ACCESS_MODE varchar(10) NO @@ -2489,170 +2469,6 @@ Timestamp datetime YES select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.columns_priv limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_LOGS; -Field Type Null Key Default Extra -LOG_OWNER varchar(128) NO -MASTER varchar(128) NO -LOG_TABLE varchar(128) NO -LOG_TRIGGER varchar(128) NO -ROWIDS varchar(3) NO -PRIMARY_KEY varchar(3) NO -OBJECT_ID varchar(3) NO -FILTER_COLUMNS varchar(3) NO -SEQUENCE varchar(3) NO -INCLUDE_NEW_VALUES varchar(3) NO -PURGE_ASYNCHRONOUS varchar(3) NO -PURGE_DEFERRED varchar(3) NO -PURGE_START datetime YES NULL -PURGE_INTERVAL varchar(200) YES NULL -LAST_PURGE_DATE datetime YES NULL -LAST_PURGE_STATUS bigint(1) NO -NUM_ROWS_PURGED bigint(20) YES NULL -COMMIT_SCN_BASED varchar(3) NO -STAGING_LOG varchar(3) NO -PURGE_DOP bigint(20) NO -LAST_PURGE_TIME bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_LOGS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEWS; -Field Type Null Key Default Extra -OWNER varchar(128) NO -MVIEW_NAME varchar(128) NO -CONTAINER_NAME varchar(128) NO -QUERY longtext NO NULL -QUERY_LEN bigint(20) NO -UPDATABLE varchar(1) NO -UPDATE_LOG varchar(128) NO -MASTER_ROLLBACK_SEG varchar(128) NO -MASTER_LINK varchar(128) NO -REWRITE_ENABLED varchar(1) NO -REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(32) NO -REFRESH_METHOD varchar(8) NO -BUILD_MODE varchar(9) NO -FAST_REFRESHABLE varchar(18) NO -LAST_REFRESH_TYPE varchar(8) NO -LAST_REFRESH_DATE datetime YES NULL -LAST_REFRESH_END_TIME datetime NO -STALENESS varchar(19) NO -AFTER_FAST_REFRESH varchar(19) NO -UNKNOWN_PREBUILT varchar(1) NO -UNKNOWN_PLSQL_FUNC varchar(1) NO -UNKNOWN_EXTERNAL_TABLE varchar(1) NO -UNKNOWN_CONSIDER_FRESH varchar(1) NO -UNKNOWN_IMPORT varchar(1) NO -UNKNOWN_TRUSTED_FD varchar(1) NO -COMPILE_STATE varchar(19) NO -USE_NO_INDEX varchar(1) NO -STALE_SINCE datetime NO -NUM_PCT_TABLES bigint(0) NO -NUM_FRESH_PCT_REGIONS bigint(0) NO -NUM_STALE_PCT_REGIONS bigint(0) NO -SEGMENT_CREATED varchar(3) NO -EVALUATION_EDITION varchar(128) NO -UNUSABLE_BEFORE varchar(128) NO -UNUSABLE_BEGINNING varchar(128) NO -DEFAULT_COLLATION varchar(100) NO -ON_QUERY_COMPUTATION varchar(1) NO -REFRESH_DOP bigint(20) NO -DATA_SYNC_SCN bigint(20) unsigned NO -DATA_SYNC_DELAY varchar(128) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEWS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS; -Field Type Null Key Default Extra -PARAMETER_NAME varchar(16) NO -VALUE varchar(40) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_PARAMS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -COLLECTION_LEVEL varchar(8) NO -RETENTION_PERIOD bigint(20) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_PARAMS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_RUN_STATS; -Field Type Null Key Default Extra -RUN_OWNER varchar(128) NO NULL -REFRESH_ID bigint(20) NO NULL -NUM_MVS bigint(20) NO NULL -MVIEWS text NO NULL -BASE_TABLES text YES NULL -METHOD text YES NULL -ROLLBACK_SEG text YES NULL -PUSH_DEFERRED_RPC varchar(1) NO -REFRESH_AFTER_ERRORS varchar(1) NO -PURGE_OPTION bigint(20) NO NULL -PARALLELISM bigint(20) NO NULL -HEAP_SIZE bigint(20) NO NULL -ATOMIC_REFRESH varchar(1) NO -NESTED varchar(1) NO -OUT_OF_PLACE varchar(1) NO -NUMBER_OF_FAILURES bigint(20) NO NULL -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -COMPLETE_STATS_AVAILABLE varchar(1) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_RUN_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -REFRESH_METHOD varchar(30) NO -REFRESH_OPTIMIZATIONS text NO -ADDITIONAL_EXECUTIONS text NO -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -INITIAL_NUM_ROWS bigint(20) NO NULL -FINAL_NUM_ROWS bigint(20) NO NULL -RESULT bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_CHANGE_STATS; -Field Type Null Key Default Extra -TBL_OWNER varchar(128) NO -TBL_NAME varchar(128) NO -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -NUM_ROWS_INS bigint(20) YES NULL -NUM_ROWS_UPD bigint(20) YES NULL -NUM_ROWS_DEL bigint(20) YES NULL -NUM_ROWS_DL_INS bigint(1) NO -PMOPS_OCCURRED varchar(1) NO -PMOP_DETAILS text NO -NUM_ROWS bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_CHANGE_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STMT_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -STEP bigint(20) NO NULL -SQLID varchar(32) YES NULL -STMT longtext NO NULL -EXECUTION_TIME bigint(20) NO NULL -EXECUTION_PLAN longtext YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STMT_STATS limit 1); -cnt -1 desc oceanbase.V$OB_COMPATIBILITY_CONTROL; Field Type Null Key Default Extra NAME varchar(128) NO NULL @@ -3147,26 +2963,6 @@ REQUIRED_SIZE decimal(42,0) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLE_SPACE_USAGE limit 1); cnt 1 -desc oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT; -Field Type Null Key Default Extra -CLIENT_IP varchar(46) NO NULL -CLIENT_PID bigint(20) NO NULL -CLIENT_TYPE varchar(7) NO NULL -START_SERVE_TIME timestamp(6) NO NULL -LAST_SERVE_TIME timestamp(6) NO NULL -LAST_READ_SOURCE varchar(7) NO NULL -LAST_REQUEST_TYPE varchar(24) NO NULL -LAST_REQUEST_LOG_LSN bigint(20) unsigned NO NULL -LAST_REQUEST_LOG_SCN bigint(20) unsigned NO NULL -LAST_FAILED_REQUEST longtext NO NULL -AVG_REQUEST_PROCESS_TIME bigint(20) NO NULL -AVG_REQUEST_QUEUE_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_SIZE bigint(20) NO NULL -AVG_LOG_TRANSPORT_BANDWIDTH varchar(29) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT limit 1); -cnt -1 desc information_schema.INNODB_SYS_FIELDS; Field Type Null Key Default Extra INDEX_ID bigint(0) unsigned NO @@ -3296,27 +3092,6 @@ MAX_CONCURRENCY bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_SQL_CCL_STATUS limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_RUNNING_JOBS; -Field Type Null Key Default Extra -TABLE_NAME varchar(256) NO -JOB_TYPE varchar(64) NO -SESSION_ID bigint(20) unsigned NO NULL -READ_SNAPSHOT bigint(20) NO NULL -PARALLEL bigint(20) NO NULL -JOB_START_TIME timestamp(6) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_RUNNING_JOBS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEW_DEPS; -Field Type Null Key Default Extra -MVIEW_OWNER varchar(128) NO -MVIEW_NAME varchar(256) NO -DEP_OWNER varchar(128) NO -DEP_NAME varchar(256) NO -DEP_TYPE varchar(64) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_DEPS limit 1); -cnt -1 desc oceanbase.DBA_OB_DYNAMIC_PARTITION_TABLES; Field Type Null Key Default Extra DATABASE_NAME varchar(128) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index e8c65f119..47b9cabbc 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -771,12 +771,6 @@ EXTRA_INFO varchar(256) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_ROOTSERVICE_EVENT_HISTORY limit 1); cnt 1 -desc oceanbase.DBA_OB_TABLET_TO_LS; -Field Type Null Key Default Extra -TABLET_ID bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLET_TO_LS limit 1); -cnt -1 desc oceanbase.DBA_OB_TABLET_REPLICAS; Field Type Null Key Default Extra CREATE_TIME bigint(20) YES NULL @@ -926,12 +920,12 @@ DEF_FREELISTS bigint(0) NO DEF_FREELIST_GROUPS bigint(0) NO DEF_LOGGING varchar(7) NO DEF_COMPRESSION varchar(8) NO -DEF_COMPRESS_FOR varchar(12) NO +DEF_COMPRESS_FOR varchar(12) NO NULL DEF_BUFFER_POOL varchar(7) NO DEF_FLASH_CACHE varchar(7) NO DEF_CELL_FLASH_CACHE varchar(7) NO REF_PTN_CONSTRAINT_NAME varchar(30) NO -INTERVAL text NO +INTERVAL text YES NULL AUTOLIST varchar(3) NO INTERVAL_SUBPARTITION text NO AUTOLIST_SUBPARTITION varchar(3) NO @@ -1341,26 +1335,22 @@ MEMORY_USAGE bigint(20) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt 1 -desc oceanbase.V$OB_TRANSACTION_PARTICIPANTS; +desc oceanbase.V$OB_TRANSACTION_WRITE_STATE; Field Type Null Key Default Extra SESSION_ID bigint(20) NO NULL -SCHEDULER_ADDR varchar(64) NO NULL -TX_TYPE varchar(11) NO TX_ID bigint(20) NO NULL -PARTICIPANTS varchar(1024) NO NULL +WRITE_STATE varchar(1024) NO NULL CTX_CREATE_TIME timestamp(6) YES NULL TX_EXPIRED_TIME timestamp(6) YES NULL STATE varchar(13) NO ACTION varchar(10) NO PENDING_LOG_SIZE bigint(20) NO NULL FLUSHED_LOG_SIZE bigint(20) NO NULL -ROLE varchar(8) NO -COORD bigint(20) NO NULL LAST_REQUEST_TIME timestamp(6) YES NULL FORMATID bigint(20) NO GLOBALID varchar(257) NO NULL BRANCHID varchar(257) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_PARTICIPANTS limit 1); +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TRANSACTION_WRITE_STATE limit 1); cnt 1 desc oceanbase.V$OB_TABLET_COMPACTION_HISTORY; @@ -1766,13 +1756,7 @@ cnt 1 desc oceanbase.V$OB_LOG_STAT; Field Type Null Key Default Extra -ROLE varchar(32) NO NULL -PROPOSAL_ID bigint(20) NO NULL -CONFIG_VERSION varchar(128) NO NULL ACCESS_MODE varchar(32) NO NULL -PAXOS_MEMBER_LIST varchar(1024) NO NULL -PAXOS_REPLICA_NUM bigint(20) NO NULL -IN_SYNC varchar(3) NO NULL BASE_LSN bigint(20) unsigned NO NULL BEGIN_LSN bigint(20) unsigned NO NULL BEGIN_SCN bigint(20) unsigned NO NULL @@ -1780,9 +1764,6 @@ END_LSN bigint(20) unsigned NO NULL END_SCN bigint(20) unsigned NO NULL MAX_LSN bigint(20) unsigned NO NULL MAX_SCN bigint(20) unsigned NO NULL -ARBITRATION_MEMBER varchar(128) NO NULL -DEGRADED_LIST varchar(1024) NO NULL -LEARNER_LIST longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_STAT limit 1); cnt 1 @@ -2112,8 +2093,7 @@ SESSION_ID bigint(20) NO NULL TX_ID bigint(20) NO NULL STATE varchar(18) NO CLUSTER_ID bigint(20) NO NULL -COORDINATOR bigint(20) NO NULL -PARTICIPANTS varchar(1024) YES NULL +WRITE_STATE varchar(1024) YES NULL ISOLATION_LEVEL varchar(16) NO SNAPSHOT_VERSION bigint(20) unsigned YES NULL ACCESS_MODE varchar(10) NO @@ -2490,170 +2470,6 @@ Timestamp datetime YES select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.columns_priv limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_LOGS; -Field Type Null Key Default Extra -LOG_OWNER varchar(128) NO -MASTER varchar(128) NO -LOG_TABLE varchar(128) NO -LOG_TRIGGER varchar(128) NO -ROWIDS varchar(3) NO -PRIMARY_KEY varchar(3) NO -OBJECT_ID varchar(3) NO -FILTER_COLUMNS varchar(3) NO -SEQUENCE varchar(3) NO -INCLUDE_NEW_VALUES varchar(3) NO -PURGE_ASYNCHRONOUS varchar(3) NO -PURGE_DEFERRED varchar(3) NO -PURGE_START datetime YES NULL -PURGE_INTERVAL varchar(200) YES NULL -LAST_PURGE_DATE datetime YES NULL -LAST_PURGE_STATUS bigint(1) NO -NUM_ROWS_PURGED bigint(20) YES NULL -COMMIT_SCN_BASED varchar(3) NO -STAGING_LOG varchar(3) NO -PURGE_DOP bigint(20) NO -LAST_PURGE_TIME bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_LOGS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEWS; -Field Type Null Key Default Extra -OWNER varchar(128) NO -MVIEW_NAME varchar(128) NO -CONTAINER_NAME varchar(128) NO -QUERY longtext NO NULL -QUERY_LEN bigint(20) NO -UPDATABLE varchar(1) NO -UPDATE_LOG varchar(128) NO -MASTER_ROLLBACK_SEG varchar(128) NO -MASTER_LINK varchar(128) NO -REWRITE_ENABLED varchar(1) NO -REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(32) NO -REFRESH_METHOD varchar(8) NO -BUILD_MODE varchar(9) NO -FAST_REFRESHABLE varchar(18) NO -LAST_REFRESH_TYPE varchar(8) NO -LAST_REFRESH_DATE datetime YES NULL -LAST_REFRESH_END_TIME datetime NO -STALENESS varchar(19) NO -AFTER_FAST_REFRESH varchar(19) NO -UNKNOWN_PREBUILT varchar(1) NO -UNKNOWN_PLSQL_FUNC varchar(1) NO -UNKNOWN_EXTERNAL_TABLE varchar(1) NO -UNKNOWN_CONSIDER_FRESH varchar(1) NO -UNKNOWN_IMPORT varchar(1) NO -UNKNOWN_TRUSTED_FD varchar(1) NO -COMPILE_STATE varchar(19) NO -USE_NO_INDEX varchar(1) NO -STALE_SINCE datetime NO -NUM_PCT_TABLES bigint(0) NO -NUM_FRESH_PCT_REGIONS bigint(0) NO -NUM_STALE_PCT_REGIONS bigint(0) NO -SEGMENT_CREATED varchar(3) NO -EVALUATION_EDITION varchar(128) NO -UNUSABLE_BEFORE varchar(128) NO -UNUSABLE_BEGINNING varchar(128) NO -DEFAULT_COLLATION varchar(100) NO -ON_QUERY_COMPUTATION varchar(1) NO -REFRESH_DOP bigint(20) NO -DATA_SYNC_SCN bigint(20) unsigned NO -DATA_SYNC_DELAY varchar(128) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEWS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS; -Field Type Null Key Default Extra -PARAMETER_NAME varchar(16) NO -VALUE varchar(40) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS_PARAMS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -COLLECTION_LEVEL varchar(8) NO -RETENTION_PERIOD bigint(20) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_PARAMS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_RUN_STATS; -Field Type Null Key Default Extra -RUN_OWNER varchar(128) NO NULL -REFRESH_ID bigint(20) NO NULL -NUM_MVS bigint(20) NO NULL -MVIEWS text NO NULL -BASE_TABLES text YES NULL -METHOD text YES NULL -ROLLBACK_SEG text YES NULL -PUSH_DEFERRED_RPC varchar(1) NO -REFRESH_AFTER_ERRORS varchar(1) NO -PURGE_OPTION bigint(20) NO NULL -PARALLELISM bigint(20) NO NULL -HEAP_SIZE bigint(20) NO NULL -ATOMIC_REFRESH varchar(1) NO -NESTED varchar(1) NO -OUT_OF_PLACE varchar(1) NO -NUMBER_OF_FAILURES bigint(20) NO NULL -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -COMPLETE_STATS_AVAILABLE varchar(1) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_RUN_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -REFRESH_METHOD varchar(30) NO -REFRESH_OPTIMIZATIONS text NO -ADDITIONAL_EXECUTIONS text NO -START_TIME datetime NO NULL -END_TIME datetime NO NULL -ELAPSED_TIME bigint(20) NO NULL -LOG_SETUP_TIME bigint(1) NO -LOG_PURGE_TIME bigint(20) NO NULL -INITIAL_NUM_ROWS bigint(20) NO NULL -FINAL_NUM_ROWS bigint(20) NO NULL -RESULT bigint(20) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_CHANGE_STATS; -Field Type Null Key Default Extra -TBL_OWNER varchar(128) NO -TBL_NAME varchar(128) NO -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -NUM_ROWS_INS bigint(20) YES NULL -NUM_ROWS_UPD bigint(20) YES NULL -NUM_ROWS_DEL bigint(20) YES NULL -NUM_ROWS_DL_INS bigint(1) NO -PMOPS_OCCURRED varchar(1) NO -PMOP_DETAILS text NO -NUM_ROWS bigint(20) YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_CHANGE_STATS limit 1); -cnt -1 -desc oceanbase.DBA_MVREF_STMT_STATS; -Field Type Null Key Default Extra -MV_OWNER varchar(128) NO -MV_NAME varchar(128) NO -REFRESH_ID bigint(20) NO NULL -STEP bigint(20) NO NULL -SQLID varchar(32) YES NULL -STMT longtext NO NULL -EXECUTION_TIME bigint(20) NO NULL -EXECUTION_PLAN longtext YES NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STMT_STATS limit 1); -cnt -1 desc oceanbase.V$OB_COMPATIBILITY_CONTROL; Field Type Null Key Default Extra NAME varchar(128) NO NULL @@ -3148,26 +2964,6 @@ REQUIRED_SIZE decimal(42,0) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TABLE_SPACE_USAGE limit 1); cnt 1 -desc oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT; -Field Type Null Key Default Extra -CLIENT_IP varchar(46) NO NULL -CLIENT_PID bigint(20) NO NULL -CLIENT_TYPE varchar(7) NO NULL -START_SERVE_TIME timestamp(6) NO NULL -LAST_SERVE_TIME timestamp(6) NO NULL -LAST_READ_SOURCE varchar(7) NO NULL -LAST_REQUEST_TYPE varchar(24) NO NULL -LAST_REQUEST_LOG_LSN bigint(20) unsigned NO NULL -LAST_REQUEST_LOG_SCN bigint(20) unsigned NO NULL -LAST_FAILED_REQUEST longtext NO NULL -AVG_REQUEST_PROCESS_TIME bigint(20) NO NULL -AVG_REQUEST_QUEUE_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_TIME bigint(20) NO NULL -AVG_REQUEST_READ_LOG_SIZE bigint(20) NO NULL -AVG_LOG_TRANSPORT_BANDWIDTH varchar(29) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LOG_TRANSPORT_DEST_STAT limit 1); -cnt -1 desc information_schema.INNODB_SYS_FIELDS; Field Type Null Key Default Extra INDEX_ID bigint(0) unsigned NO @@ -3297,27 +3093,6 @@ MAX_CONCURRENCY bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_SQL_CCL_STATUS limit 1); cnt 1 -desc oceanbase.DBA_MVIEW_RUNNING_JOBS; -Field Type Null Key Default Extra -TABLE_NAME varchar(256) NO -JOB_TYPE varchar(64) NO -SESSION_ID bigint(20) unsigned NO NULL -READ_SNAPSHOT bigint(20) NO NULL -PARALLEL bigint(20) NO NULL -JOB_START_TIME timestamp(6) NO NULL -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_RUNNING_JOBS limit 1); -cnt -1 -desc oceanbase.DBA_MVIEW_DEPS; -Field Type Null Key Default Extra -MVIEW_OWNER varchar(128) NO -MVIEW_NAME varchar(256) NO -DEP_OWNER varchar(128) NO -DEP_NAME varchar(256) NO -DEP_TYPE varchar(64) NO -select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_DEPS limit 1); -cnt -1 desc oceanbase.DBA_OB_DYNAMIC_PARTITION_TABLES; Field Type Null Key Default Extra DATABASE_NAME varchar(128) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/show_sys_tables_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/show_sys_tables_in_sys.result index ea8f735d6..9d8eed942 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/show_sys_tables_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/show_sys_tables_in_sys.result @@ -7,7 +7,7 @@ id field type length precision nullable 19 row_id 5 0 20 0 20 column_name 22 128 -1 0 21 column_value 22 65536 -1 1 -table oceanbase.__all_table column_count 95 table_id 3 +table oceanbase.__all_table column_count 93 table_id 3 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_virtual_core_column_table where table_id=3 order by column_id; id field type length precision nullable 16 gmt_create 18 0 25 1 @@ -92,19 +92,17 @@ id field type length precision nullable 95 lob_inrow_threshold 5 0 20 0 96 auto_increment_cache_size 5 0 20 0 97 external_properties 22 1048576 -1 1 -98 local_session_vars 30 536870911 -1 1 -99 duplicate_read_consistency 5 0 20 0 -100 index_params 22 256 -1 0 -101 micro_index_clustered 1 0 4 0 -102 mv_mode 5 0 20 0 -103 parser_properties 30 536870911 -1 0 -104 enable_macro_block_bloom_filter 1 0 4 0 -105 storage_cache_policy 22 1048576 -1 0 -106 merge_engine_type 5 0 20 0 -107 semistruct_encoding_type 5 0 20 0 -108 dynamic_partition_policy 22 4096 -1 0 -109 external_location_id 5 0 20 0 -110 external_sub_path 22 1048576 -1 1 +98 duplicate_read_consistency 5 0 20 0 +99 index_params 22 256 -1 0 +100 micro_index_clustered 1 0 4 0 +101 parser_properties 30 536870911 -1 0 +102 enable_macro_block_bloom_filter 1 0 4 0 +103 storage_cache_policy 22 1048576 -1 0 +104 merge_engine_type 5 0 20 0 +105 semistruct_encoding_type 5 0 20 0 +106 dynamic_partition_policy 22 4096 -1 0 +107 external_location_id 5 0 20 0 +108 external_sub_path 22 1048576 -1 1 table oceanbase.__all_column column_count 34 table_id 4 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_virtual_core_column_table where table_id=4 order by column_id; id field type length precision nullable @@ -403,7 +401,7 @@ id field type length precision nullable 31 priv_create_view 5 0 20 1 32 priv_show_view 5 0 20 1 33 priv_others 5 0 20 1 -table oceanbase.__all_table_history column_count 96 table_id 114 +table oceanbase.__all_table_history column_count 94 table_id 114 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=114 order by column_id; id field type length precision nullable 16 gmt_create 18 0 25 1 @@ -489,19 +487,17 @@ id field type length precision nullable 96 lob_inrow_threshold 5 0 20 1 97 auto_increment_cache_size 5 0 20 1 98 external_properties 22 1048576 -1 1 -99 local_session_vars 30 536870911 -1 1 -100 duplicate_read_consistency 5 0 20 1 -101 index_params 22 256 -1 1 -102 micro_index_clustered 1 0 4 1 -103 mv_mode 5 0 20 1 -104 parser_properties 30 536870911 -1 1 -105 enable_macro_block_bloom_filter 1 0 4 1 -106 storage_cache_policy 22 1048576 -1 1 -107 merge_engine_type 5 0 20 1 -108 semistruct_encoding_type 5 0 20 1 -109 dynamic_partition_policy 22 4096 -1 1 -110 external_location_id 5 0 20 1 -111 external_sub_path 22 1048576 -1 1 +99 duplicate_read_consistency 5 0 20 1 +100 index_params 22 256 -1 1 +101 micro_index_clustered 1 0 4 1 +102 parser_properties 30 536870911 -1 1 +103 enable_macro_block_bloom_filter 1 0 4 1 +104 storage_cache_policy 22 1048576 -1 1 +105 merge_engine_type 5 0 20 1 +106 semistruct_encoding_type 5 0 20 1 +107 dynamic_partition_policy 22 4096 -1 1 +108 external_location_id 5 0 20 1 +109 external_sub_path 22 1048576 -1 1 table oceanbase.__all_column_history column_count 35 table_id 115 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=115 order by column_id; id field type length precision nullable @@ -1775,7 +1771,7 @@ id field type length precision nullable 21 valchar 22 4000 -1 1 22 chgtime 18 0 25 1 23 spare1 5 0 20 1 -table oceanbase.__all_tablet_to_ls column_count 4 table_id 343 +table oceanbase.__all_tablet_to_table column_count 4 table_id 343 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=343 order by column_id; id field type length precision nullable 16 gmt_create 18 0 25 1 @@ -1801,7 +1797,7 @@ id field type length precision nullable 20 data_checksum 5 0 20 0 21 row_count 5 0 20 0 22 column_checksums 22 65536 -1 1 -table oceanbase.__all_pending_transaction column_count 14 table_id 375 +table oceanbase.__all_pending_transaction column_count 13 table_id 375 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=375 order by column_id; id field type length precision nullable 16 gmt_create 18 0 25 1 @@ -1810,14 +1806,13 @@ id field type length precision nullable 19 gtrid 22 128 -1 0 20 bqual 22 128 -1 0 21 format_id 5 0 20 0 -22 coordinator 5 0 20 0 -23 scheduler_ip 22 128 -1 0 -24 scheduler_port 5 0 20 0 -25 state 5 0 20 0 -26 spare1 5 0 20 1 -27 spare2 5 0 20 1 -28 spare3 22 128 -1 1 -29 spare4 22 128 -1 1 +22 scheduler_ip 22 128 -1 0 +23 scheduler_port 5 0 20 0 +24 state 5 0 20 0 +25 spare1 5 0 20 1 +26 spare2 5 0 20 1 +27 spare3 22 128 -1 1 +28 spare4 22 128 -1 1 table oceanbase.__all_scheduler_job column_count 48 table_id 377 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=377 order by column_id; id field type length precision nullable @@ -2063,132 +2058,6 @@ id field type length precision nullable 27 spare2 5 0 20 1 28 spare3 22 4096 -1 1 29 spare4 22 4096 -1 1 -table oceanbase.__all_mlog column_count 13 table_id 463 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=463 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 mlog_id 5 0 20 0 -19 purge_mode 5 0 20 0 -20 purge_start 18 0 25 1 -21 purge_next 22 128 -1 1 -22 purge_job 22 128 -1 1 -23 last_purge_scn 10 0 20 1 -24 last_purge_date 18 0 25 1 -25 last_purge_time 5 0 20 1 -26 last_purge_rows 5 0 20 1 -27 last_purge_trace_id 22 64 -1 1 -28 schema_version 5 0 20 0 -table oceanbase.__all_mview column_count 19 table_id 464 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=464 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 mview_id 5 0 20 0 -19 build_mode 5 0 20 0 -20 refresh_mode 5 0 20 0 -21 refresh_method 5 0 20 0 -22 refresh_start 18 0 25 1 -23 refresh_next 22 128 -1 1 -24 refresh_job 22 128 -1 1 -25 last_refresh_scn 10 0 20 1 -26 last_refresh_type 5 0 20 1 -27 last_refresh_date 18 0 25 1 -28 last_refresh_time 5 0 20 1 -29 last_refresh_trace_id 22 64 -1 1 -30 schema_version 5 0 20 0 -31 refresh_dop 5 0 20 0 -32 data_sync_scn 10 0 20 0 -33 is_synced 1 0 4 0 -34 nested_refresh_mode 5 0 20 0 -table oceanbase.__all_mview_refresh_stats_sys_defaults column_count 5 table_id 465 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=465 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 id 5 0 20 0 -19 collection_level 5 0 20 0 -20 retention_period 5 0 20 0 -table oceanbase.__all_mview_refresh_stats_params column_count 5 table_id 466 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=466 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 mview_id 5 0 20 0 -19 collection_level 5 0 20 0 -20 retention_period 5 0 20 0 -table oceanbase.__all_mview_refresh_run_stats column_count 25 table_id 467 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=467 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 refresh_id 5 0 20 0 -19 run_user_id 5 0 20 0 -20 num_mvs_total 5 0 20 0 -21 num_mvs_current 5 0 20 0 -22 mviews 22 4000 -1 0 -23 base_tables 22 4000 -1 1 -24 method 22 4000 -1 1 -25 rollback_seg 22 4000 -1 1 -26 push_deferred_rpc 1 0 4 0 -27 refresh_after_errors 1 0 4 0 -28 purge_option 5 0 20 0 -29 parallelism 5 0 20 0 -30 heap_size 5 0 20 0 -31 atomic_refresh 1 0 4 0 -32 nested 1 0 4 0 -33 out_of_place 1 0 4 0 -34 number_of_failures 5 0 20 0 -35 start_time 18 0 25 0 -36 end_time 18 0 25 0 -37 elapsed_time 5 0 20 0 -38 log_purge_time 5 0 20 0 -39 complete_stats_avaliable 1 0 4 0 -40 trace_id 22 64 -1 1 -table oceanbase.__all_mview_refresh_stats column_count 14 table_id 468 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=468 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 refresh_id 5 0 20 0 -19 mview_id 5 0 20 0 -20 retry_id 5 0 20 0 -21 refresh_type 5 0 20 0 -22 start_time 18 0 25 0 -23 end_time 18 0 25 0 -24 elapsed_time 5 0 20 0 -25 log_purge_time 5 0 20 0 -26 initial_num_rows 5 0 20 0 -27 final_num_rows 5 0 20 0 -28 num_steps 5 0 20 0 -29 result 5 0 20 0 -table oceanbase.__all_mview_refresh_change_stats column_count 10 table_id 469 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=469 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 refresh_id 5 0 20 0 -19 mview_id 5 0 20 0 -20 retry_id 5 0 20 0 -21 detail_table_id 5 0 20 0 -22 num_rows_ins 5 0 20 1 -23 num_rows_upd 5 0 20 1 -24 num_rows_del 5 0 20 1 -25 num_rows 5 0 20 1 -table oceanbase.__all_mview_refresh_stmt_stats column_count 11 table_id 470 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=470 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 refresh_id 5 0 20 0 -19 mview_id 5 0 20 0 -20 retry_id 5 0 20 0 -21 step 5 0 20 0 -22 sqlid 22 32 -1 1 -23 stmt 30 536870911 -1 0 -24 execution_time 5 0 20 0 -25 execution_plan 30 536870911 -1 1 -26 result 5 0 20 0 table oceanbase.__all_dbms_lock_allocated column_count 6 table_id 471 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=471 order by column_id; id field type length precision nullable @@ -2332,17 +2201,6 @@ id field type length precision nullable 23 table_name 22 1024 -1 1 24 column_name 22 1024 -1 1 25 all_priv 5 0 20 1 -table oceanbase.__all_mview_dep column_count 8 table_id 518 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=518 order by column_id; -id field type length precision nullable -16 gmt_create 18 0 25 1 -17 gmt_modified 18 0 25 1 -18 mview_id 5 0 20 0 -19 p_order 5 0 20 0 -20 p_obj 5 0 20 0 -21 p_type 5 0 20 0 -22 qbcid 5 0 20 0 -23 flags 5 0 20 0 table oceanbase.__all_scheduler_job_run_detail_v2 column_count 33 table_id 519 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=519 order by column_id; id field type length precision nullable @@ -3050,76 +2908,70 @@ id field type length precision nullable 32 freeze_state 22 256 -1 0 33 freeze_time_dist 22 256 -1 0 34 compaction_info_list 22 128 -1 0 -table oceanbase.__all_virtual_trans_stat column_count 31 table_id 11027 +table oceanbase.__all_virtual_trans_stat column_count 26 table_id 11027 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11027 order by column_id; id field type length precision nullable -16 trans_type 5 0 20 0 -17 trans_id 5 0 20 0 -18 session_id 5 0 20 0 -19 scheduler_addr 22 64 -1 0 -20 is_decided 1 0 4 0 -21 participants 22 1024 -1 0 -22 ctx_create_time 18 0 25 1 -23 expired_time 18 0 25 1 -24 ref_cnt 5 0 20 0 -25 last_op_sn 5 0 20 0 -26 pending_write 5 0 20 0 -27 state 5 0 20 0 -28 part_trans_action 5 0 20 0 -29 trans_ctx_addr 22 20 -1 0 -30 mem_ctx_id 5 0 20 0 -31 pending_log_size 5 0 20 0 -32 flushed_log_size 5 0 20 0 -33 role 5 0 20 0 -34 is_exiting 5 0 20 0 -35 coordinator 5 0 20 0 -36 last_request_time 18 0 25 1 -37 gtrid 22 128 -1 0 -38 bqual 22 128 -1 0 -39 format_id 5 0 20 0 -40 start_scn 10 0 20 0 -41 end_scn 10 0 20 0 -42 rec_scn 10 0 20 0 -43 busy_cbs 5 0 20 0 -44 replay_complete 5 0 20 0 -45 serial_log_final_scn 5 0 20 0 -46 callback_list_stats 30 536870911 -1 0 +16 trans_id 5 0 20 0 +17 session_id 5 0 20 0 +18 is_decided 1 0 4 0 +19 write_state 22 1024 -1 0 +20 ctx_create_time 18 0 25 1 +21 expired_time 18 0 25 1 +22 ref_cnt 5 0 20 0 +23 last_op_sn 5 0 20 0 +24 pending_write 5 0 20 0 +25 state 5 0 20 0 +26 part_trans_action 5 0 20 0 +27 trans_ctx_addr 22 20 -1 0 +28 pending_log_size 5 0 20 0 +29 flushed_log_size 5 0 20 0 +30 is_exiting 5 0 20 0 +31 last_request_time 18 0 25 1 +32 gtrid 22 128 -1 0 +33 bqual 22 128 -1 0 +34 format_id 5 0 20 0 +35 start_scn 10 0 20 0 +36 end_scn 10 0 20 0 +37 rec_scn 10 0 20 0 +38 busy_cbs 5 0 20 0 +39 replay_complete 5 0 20 0 +40 serial_log_final_scn 5 0 20 0 +41 callback_list_stats 30 536870911 -1 0 table oceanbase.__all_virtual_trans_ctx_mgr_stat column_count 6 table_id 11028 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11028 order by column_id; id field type length precision nullable -16 is_master 5 0 20 0 -17 is_stopped 5 0 20 0 -18 state 5 0 20 0 -19 state_str 22 64 -1 0 +16 is_stopped 5 0 20 0 +17 block_tx 5 0 20 0 +18 block_normal_tx 5 0 20 0 +19 block_all 5 0 20 0 20 total_trans_ctx_count 5 0 20 0 21 mgr_addr 5 0 20 0 -table oceanbase.__all_virtual_trans_scheduler column_count 23 table_id 11029 +table oceanbase.__all_virtual_trans_scheduler column_count 22 table_id 11029 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11029 order by column_id; id field type length precision nullable 16 session_id 5 0 20 0 17 trans_id 5 0 20 0 18 state 5 0 20 0 19 cluster_id 5 0 20 0 -20 coordinator 5 0 20 0 -21 participants 22 1024 -1 1 -22 isolation_level 5 0 20 0 -23 snapshot_version 10 0 20 1 -24 access_mode 5 0 20 0 -25 tx_op_sn 5 0 20 0 -26 flag 5 0 20 0 -27 active_time 18 0 25 1 -28 expire_time 18 0 25 1 -29 timeout_us 5 0 20 0 -30 ref_cnt 5 0 20 0 -31 tx_desc_addr 22 20 -1 0 -32 savepoints 22 1024 -1 1 -33 savepoints_total_cnt 5 0 20 0 -34 internal_abort_cause 5 0 20 0 -35 can_early_lock_release 1 0 4 0 -36 gtrid 22 128 -1 1 -37 bqual 22 128 -1 1 -38 format_id 5 0 20 0 -table oceanbase.__all_virtual_core_all_table column_count 93 table_id 11035 +20 write_state 22 1024 -1 1 +21 isolation_level 5 0 20 0 +22 snapshot_version 10 0 20 1 +23 access_mode 5 0 20 0 +24 tx_op_sn 5 0 20 0 +25 flag 5 0 20 0 +26 active_time 18 0 25 1 +27 expire_time 18 0 25 1 +28 timeout_us 5 0 20 0 +29 ref_cnt 5 0 20 0 +30 tx_desc_addr 22 20 -1 0 +31 savepoints 22 1024 -1 1 +32 savepoints_total_cnt 5 0 20 0 +33 internal_abort_cause 5 0 20 0 +34 can_early_lock_release 1 0 4 0 +35 gtrid 22 128 -1 1 +36 bqual 22 128 -1 1 +37 format_id 5 0 20 0 +table oceanbase.__all_virtual_core_all_table column_count 91 table_id 11035 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11035 order by column_id; id field type length precision nullable 16 table_id 5 0 20 0 @@ -3202,19 +3054,17 @@ id field type length precision nullable 93 lob_inrow_threshold 5 0 20 0 94 auto_increment_cache_size 5 0 20 0 95 external_properties 22 1048576 -1 1 -96 local_session_vars 30 536870911 -1 1 -97 duplicate_read_consistency 5 0 20 0 -98 index_params 22 256 -1 0 -99 micro_index_clustered 1 0 4 0 -100 mv_mode 5 0 20 0 -101 parser_properties 30 536870911 -1 0 -102 enable_macro_block_bloom_filter 1 0 4 0 -103 storage_cache_policy 22 1048576 -1 0 -104 merge_engine_type 5 0 20 0 -105 semistruct_encoding_type 5 0 20 0 -106 dynamic_partition_policy 22 4096 -1 0 -107 external_location_id 5 0 20 0 -108 external_sub_path 22 1048576 -1 1 +96 duplicate_read_consistency 5 0 20 0 +97 index_params 22 256 -1 0 +98 micro_index_clustered 1 0 4 0 +99 parser_properties 30 536870911 -1 0 +100 enable_macro_block_bloom_filter 1 0 4 0 +101 storage_cache_policy 22 1048576 -1 0 +102 merge_engine_type 5 0 20 0 +103 semistruct_encoding_type 5 0 20 0 +104 dynamic_partition_policy 22 4096 -1 0 +105 external_location_id 5 0 20 0 +106 external_sub_path 22 1048576 -1 1 table oceanbase.__all_virtual_core_column_table column_count 32 table_id 11036 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11036 order by column_id; id field type length precision nullable @@ -3435,19 +3285,17 @@ id field type length precision nullable 26 last_miss_ts 5 0 20 0 27 last_miss_return_ts 5 0 20 0 28 next 5 0 20 0 -table oceanbase.__all_virtual_trans_lock_stat column_count 10 table_id 11085 +table oceanbase.__all_virtual_trans_lock_stat column_count 8 table_id 11085 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11085 order by column_id; id field type length precision nullable 16 trans_id 5 0 20 0 -17 table_id 5 0 20 0 -18 tablet_id 5 0 20 0 -19 rowkey 22 512 -1 1 -20 session_id 5 0 20 0 -21 proxy_session_id 22 512 -1 0 -22 ctx_create_time 18 0 25 1 -23 expired_time 18 0 25 1 -24 time_after_recv 5 0 20 0 -25 row_lock_addr 10 0 20 1 +17 tablet_id 5 0 20 0 +18 rowkey 22 512 -1 1 +19 session_id 5 0 20 0 +20 ctx_create_time 18 0 25 1 +21 expired_time 18 0 25 1 +22 time_after_recv 5 0 20 0 +23 row_lock_addr 10 0 20 1 table oceanbase.__all_virtual_show_create_tablegroup column_count 3 table_id 11087 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=11087 order by column_id; id field type length precision nullable @@ -3816,7 +3664,7 @@ id field type length precision nullable 35 execute_priv 22 1 -1 0 36 event_priv 22 1 -1 0 37 trigger_priv 22 1 -1 0 -table oceanbase.__all_virtual_lock_wait_stat column_count 25 table_id 12013 +table oceanbase.__all_virtual_lock_wait_stat column_count 18 table_id 12013 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12013 order by column_id; id field type length precision nullable 16 tablet_id 5 0 20 0 @@ -3837,13 +3685,6 @@ id field type length precision nullable 31 trans_id 5 0 20 0 32 holder_trans_id 5 0 20 0 33 holder_session_id 5 0 20 0 -34 assoc_session_id 5 0 20 0 -35 wait_timeout 5 0 20 0 -36 tx_active_ts 5 0 20 0 -37 node_id 5 0 20 0 -38 node_type 5 0 20 0 -39 remote_addr 22 64 -1 0 -40 is_placeholder 5 0 20 0 table oceanbase.__all_virtual_collation column_count 7 table_id 12031 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12031 order by column_id; id field type length precision nullable @@ -4116,7 +3957,7 @@ id field type length precision nullable 22 drift_size 5 0 20 0 23 workarea_count 5 0 20 0 24 manual_calc_count 5 0 20 0 -table oceanbase.__all_virtual_id_service column_count 8 table_id 12176 +table oceanbase.__all_virtual_id_service column_count 7 table_id 12176 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12176 order by column_id; id field type length precision nullable 16 id_service_type 5 0 20 0 @@ -4126,7 +3967,6 @@ id field type length precision nullable 20 latest_log_scn 10 0 20 0 21 pre_allocated_range 5 0 20 0 22 submit_log_ts 5 0 20 0 -23 is_master 1 0 4 0 table oceanbase.__all_virtual_sql_plan_monitor column_count 42 table_id 12185 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12185 order by column_id; id field type length precision nullable @@ -4348,26 +4188,17 @@ id field type length precision nullable 44 store_status 22 128 -1 0 45 store_trans_status 22 128 -1 0 46 message 22 4096 -1 0 -table oceanbase.__all_virtual_log_stat column_count 17 table_id 12254 +table oceanbase.__all_virtual_log_stat column_count 8 table_id 12254 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12254 order by column_id; id field type length precision nullable -16 role 22 32 -1 0 -17 proposal_id 5 0 20 0 -18 config_version 22 128 -1 0 -19 access_mode 22 32 -1 0 -20 paxos_member_list 22 1024 -1 0 -21 paxos_replica_num 5 0 20 0 -22 in_sync 1 0 4 0 -23 base_lsn 10 0 20 0 -24 begin_lsn 10 0 20 0 -25 begin_scn 10 0 20 0 -26 end_lsn 10 0 20 0 -27 end_scn 10 0 20 0 -28 max_lsn 10 0 20 0 -29 max_scn 10 0 20 0 -30 arbitration_member 22 128 -1 0 -31 degraded_list 22 1024 -1 0 -32 learner_list 30 536870911 -1 0 +16 access_mode 22 32 -1 0 +17 base_lsn 10 0 20 0 +18 begin_lsn 10 0 20 0 +19 begin_scn 10 0 20 0 +20 end_lsn 10 0 20 0 +21 end_scn 10 0 20 0 +22 max_lsn 10 0 20 0 +23 max_scn 10 0 20 0 table oceanbase.__all_virtual_tablet_replica_checksum column_count 7 table_id 12258 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12258 order by column_id; id field type length precision nullable @@ -4446,41 +4277,28 @@ id field type length precision nullable 29 sqc_id 5 0 20 0 30 batch_id 5 0 20 0 31 max_hold_memory 5 0 20 0 -table oceanbase.__all_virtual_apply_stat column_count 4 table_id 12280 +table oceanbase.__all_virtual_apply_stat column_count 2 table_id 12280 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12280 order by column_id; id field type length precision nullable -16 role 22 32 -1 0 -17 end_lsn 10 0 20 0 -18 proposal_id 5 0 20 0 -19 pending_cnt 5 0 20 0 -table oceanbase.__all_virtual_replay_stat column_count 6 table_id 12281 +16 end_lsn 10 0 20 0 +17 pending_cnt 5 0 20 0 +table oceanbase.__all_virtual_replay_stat column_count 5 table_id 12281 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12281 order by column_id; id field type length precision nullable -16 role 22 32 -1 0 -17 end_lsn 10 0 20 0 -18 enabled 1 0 4 0 -19 unsubmitted_lsn 10 0 20 0 -20 unsubmitted_log_scn 10 0 20 0 -21 pending_cnt 5 0 20 0 -table oceanbase.__all_virtual_ls_info column_count 16 table_id 12287 +16 end_lsn 10 0 20 0 +17 enabled 1 0 4 0 +18 unsubmitted_lsn 10 0 20 0 +19 unsubmitted_log_scn 10 0 20 0 +20 pending_cnt 5 0 20 0 +table oceanbase.__all_virtual_ls_info column_count 6 table_id 12287 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12287 order by column_id; id field type length precision nullable -16 replica_type 22 16 -1 0 -17 ls_state 22 16 -1 0 -18 tablet_count 5 0 20 0 -19 weak_read_scn 10 0 20 0 -20 need_rebuild 22 3 -1 0 -21 checkpoint_scn 10 0 20 0 -22 checkpoint_lsn 10 0 20 0 -23 migrate_status 5 0 20 0 -24 rebuild_seq 5 0 20 0 -25 tablet_change_checkpoint_scn 10 0 20 0 -26 reserved_scn 10 0 20 0 -27 tx_blocked 5 0 20 0 -28 required_data_disk_size 5 0 20 0 -29 mv_major_merge_scn 10 0 20 0 -30 mv_publish_scn 10 0 20 0 -31 mv_safe_scn 10 0 20 0 +16 tablet_count 5 0 20 0 +17 weak_read_scn 10 0 20 0 +18 checkpoint_scn 10 0 20 0 +19 checkpoint_lsn 10 0 20 0 +20 tablet_change_checkpoint_scn 10 0 20 0 +21 tx_blocked 5 0 20 0 table oceanbase.__all_virtual_tablet_info column_count 10 table_id 12288 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12288 order by column_id; id field type length precision nullable @@ -4697,37 +4515,18 @@ id field type length precision nullable 24 elapse 5 0 20 0 25 tags 30 536870911 -1 0 26 logs 30 536870911 -1 0 -table oceanbase.__all_virtual_ha_diagnose column_count 28 table_id 12340 +table oceanbase.__all_virtual_ha_diagnose column_count 9 table_id 12340 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12340 order by column_id; id field type length precision nullable -16 election_role 22 32 -1 0 -17 election_epoch 5 0 20 0 -18 palf_role 22 32 -1 0 -19 palf_state 22 32 -1 0 -20 palf_proposal_id 5 0 20 0 -21 log_handler_role 22 32 -1 0 -22 log_handler_proposal_id 5 0 20 0 -23 log_handler_takeover_state 22 32 -1 0 -24 log_handler_takeover_log_type 22 32 -1 0 -25 max_applied_scn 10 0 20 0 -26 max_replayed_lsn 10 0 20 0 -27 max_replayed_scn 10 0 20 0 -28 replay_diagnose_info 22 1024 -1 0 -29 gc_state 22 32 -1 0 -30 gc_start_ts 5 0 20 0 -31 archive_scn 10 0 20 0 -32 checkpoint_scn 10 0 20 0 -33 min_rec_scn 10 0 20 0 -34 min_rec_scn_log_type 22 32 -1 0 -35 restore_handler_role 22 32 -1 0 -36 restore_proposal_id 5 0 20 0 -37 restore_context_info 22 1024 -1 0 -38 restore_err_context_info 22 1024 -1 0 -39 enable_sync 1 0 4 0 -40 enable_vote 1 0 4 0 -41 arb_srv_info 22 1024 -1 0 -42 parent 22 1024 -1 0 -43 readonly_tx 22 1024 -1 0 +16 palf_state 22 32 -1 0 +17 max_applied_scn 10 0 20 0 +18 max_replayed_lsn 10 0 20 0 +19 max_replayed_scn 10 0 20 0 +20 replay_diagnose_info 22 1024 -1 0 +21 checkpoint_scn 10 0 20 0 +22 min_rec_scn 10 0 20 0 +23 min_rec_scn_log_type 22 32 -1 0 +24 readonly_tx 22 1024 -1 0 table oceanbase.__all_virtual_sql_plan column_count 44 table_id 12359 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12359 order by column_id; id field type length precision nullable @@ -4872,14 +4671,11 @@ id field type length precision nullable 28 state_info 22 1024 -1 0 29 last_check_timestamp 18 0 25 0 30 extension 22 4096 -1 0 -table oceanbase.__all_virtual_timestamp_service column_count 5 table_id 12395 +table oceanbase.__all_virtual_timestamp_service column_count 2 table_id 12395 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12395 order by column_id; id field type length precision nullable 16 ts_value 5 0 20 0 17 ts_type 22 100 -1 0 -18 service_role 22 100 -1 0 -19 role 22 100 -1 0 -20 service_epoch 5 0 20 0 table oceanbase.__all_virtual_px_p2p_datahub column_count 6 table_id 12397 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12397 order by column_id; id field type length precision nullable @@ -5065,24 +4861,6 @@ select column_id id, column_name field, data_type type, data_length length, data id field type length precision nullable 16 devname 22 128 -1 0 17 speed_Mbps 5 0 20 0 -table oceanbase.__all_virtual_log_transport_dest_stat column_count 15 table_id 12491 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12491 order by column_id; -id field type length precision nullable -16 client_ip 22 46 -1 0 -17 client_pid 5 0 20 0 -18 client_type 5 0 20 0 -19 start_serve_time 18 0 25 0 -20 last_serve_time 18 0 25 0 -21 last_read_source 5 0 20 0 -22 last_request_type 5 0 20 0 -23 last_request_log_lsn 10 0 20 0 -24 last_request_log_scn 10 0 20 0 -25 last_failed_request 30 536870911 -1 0 -26 avg_request_process_time 5 0 20 0 -27 avg_request_queue_time 5 0 20 0 -28 avg_request_read_log_time 5 0 20 0 -29 avg_request_read_log_size 5 0 20 0 -30 avg_log_transport_bandwidth 5 0 20 0 table oceanbase.__all_virtual_vector_index_info column_count 14 table_id 12496 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12496 order by column_id; id field type length precision nullable @@ -5182,16 +4960,6 @@ id field type length precision nullable 17 format_sqlid 22 32 -1 0 18 current_concurrency 5 0 20 0 19 max_concurrency 5 0 20 0 -table oceanbase.__all_virtual_mview_running_job column_count 7 table_id 12535 -select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12535 order by column_id; -id field type length precision nullable -16 table_id 5 0 20 0 -17 job_type 10 0 20 0 -18 session_id 10 0 20 0 -19 read_snapshot 5 0 20 0 -20 parallel 5 0 20 0 -21 job_start_time 18 0 25 0 -22 target_data_sync_scn 10 0 20 0 table oceanbase.__all_virtual_dynamic_partition_table column_count 11 table_id 12536 select column_id id, column_name field, data_type type, data_length length, data_precision `precision`, nullable from __all_column where table_id=12536 order by column_id; id field type length precision nullable diff --git a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/commit_after_minor_merge.test b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/commit_after_minor_merge.test index d272b5726..1fa923423 100644 --- a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/commit_after_minor_merge.test +++ b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/commit_after_minor_merge.test @@ -41,7 +41,7 @@ let $__table_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ table_id f # prepare data, minor merge with dirty trx but committed in sstable connection conn1; -let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_ls where table_id = $__table_id__, tablet_id, 1); +let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_table where table_id = $__table_id__, tablet_id, 1); insert into test_commit_after_minor_merge_t values (0,0); begin; diff --git a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/dirty_trx_minor_merge.test b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/dirty_trx_minor_merge.test index 2cb77247d..07a695bfe 100644 --- a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/dirty_trx_minor_merge.test +++ b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/dirty_trx_minor_merge.test @@ -39,7 +39,7 @@ let $__table_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ table_id f connection conn1; -let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_ls where table_id = $__table_id__, tablet_id, 1); +let $__tablet_id__=query_get_value(select /*+QUERY_TIMEOUT(10000000)*/ tablet_id from oceanbase.__all_tablet_to_table where table_id = $__table_id__, tablet_id, 1); --source mysql_test/include/minor_freeze_tablet_and_wait_done.inc diff --git a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/query_uncommitted_data2.test b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/query_uncommitted_data2.test index 28091a9e5..006357ff1 100644 --- a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/query_uncommitted_data2.test +++ b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/query_uncommitted_data2.test @@ -52,7 +52,7 @@ while ($i > 0) { sleep 1; dec $i; - let $memtable_count = query_get_value(select count(1) as c from oceanbase.__all_virtual_tablet_memstore_info where tablet_id in (select tablet_id from __all_tablet_to_ls where table_id = $__table_id__) and is_active = 'NO', c, 1); + let $memtable_count = query_get_value(select count(1) as c from oceanbase.__all_virtual_tablet_memstore_info where tablet_id in (select tablet_id from __all_tablet_to_table where table_id = $__table_id__) and is_active = 'NO', c, 1); if ($memtable_count == 0) { let $i = -5; @@ -90,7 +90,7 @@ while ($i > 0) { sleep 1; dec $i; - let $memtable_count = query_get_value(select count(1) as c from oceanbase.__all_virtual_tablet_memstore_info where tablet_id in (select tablet_id from __all_tablet_to_ls where table_id = $__table_id__) and is_active = 'NO', c, 1); + let $memtable_count = query_get_value(select count(1) as c from oceanbase.__all_virtual_tablet_memstore_info where tablet_id in (select tablet_id from __all_tablet_to_table where table_id = $__table_id__) and is_active = 'NO', c, 1); if ($memtable_count == 0) { let $i = -5; diff --git a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/reuse_dirty_macro.test b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/reuse_dirty_macro.test index dd4a5a758..9685462f8 100644 --- a/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/reuse_dirty_macro.test +++ b/tools/deploy/mysql_test/test_suite/merge_uncommitted/t/reuse_dirty_macro.test @@ -72,7 +72,7 @@ connection conn1; select count(1) from test_uncommitted_trx_minor_merge; connection conn0; -let $use_old=query_get_value(select multiplexed_macro_block_count from oceanbase.__all_virtual_tablet_compaction_history where tablet_id in (select tablet_id from __all_tablet_to_ls where table_id = $__table_id__) order by finish_time desc limit 1, multiplexed_macro_block_count, 1); +let $use_old=query_get_value(select multiplexed_macro_block_count from oceanbase.__all_virtual_tablet_compaction_history where tablet_id in (select tablet_id from __all_tablet_to_table where table_id = $__table_id__) order by finish_time desc limit 1, multiplexed_macro_block_count, 1); if ($use_old > 0) { --echo should not reuse macro diff --git a/tools/deploy/mysql_test/test_suite/vector_index/t/drop_vector_index.test b/tools/deploy/mysql_test/test_suite/vector_index/t/drop_vector_index.test index 571ab1bb2..83abe6f66 100644 --- a/tools/deploy/mysql_test/test_suite/vector_index/t/drop_vector_index.test +++ b/tools/deploy/mysql_test/test_suite/vector_index/t/drop_vector_index.test @@ -237,10 +237,6 @@ call DBMS_AI_SERVICE.DROP_AI_MODEL_ENDPOINT('ob_embed_endpoint'); #show index from t_vec; #connection conn_sys; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%vid_rowkey%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%rowkey_vid%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%idx1%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%idx2%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; #connection conn_user; #use yjl_vec; @@ -252,10 +248,6 @@ call DBMS_AI_SERVICE.DROP_AI_MODEL_ENDPOINT('ob_embed_endpoint'); #show index from t_vec; #select count(table_name) from oceanbase.__all_table where table_name like '%vid_rowkey%' and database_id = (select database_id from oceanbase.__all_database where database_name='yjl_vec'); #connection conn_sys; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%vid_rowkey%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%rowkey_vid%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%idx1%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; -#select count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS where table_name like '%idx2%' and DATABASE_NAME = 'yjl_vec' and ROLE='LEADER'; --echo connection conn_user; diff --git a/unittest/libobcdc/log_generator.h b/unittest/libobcdc/log_generator.h index f6654b4cc..c2a99f1c2 100644 --- a/unittest/libobcdc/log_generator.h +++ b/unittest/libobcdc/log_generator.h @@ -67,16 +67,15 @@ class ObTxLogBlockBuilder // usage example: generate a log_entry with redo+commit_info // LogEntry log_entry; // palf::LSN lsn; -// ObTxLogGenerator log_generator(xxx); +// ObTxLogGenerator log_generator(tx_id, cluster_id); // log_generator.gen_redo_log() // log_generator.gen_commit_info_log() // log_generator.gen_log_entry(log_entry, lsn) class ObTxLogGenerator { public: - ObTxLogGenerator(const uint64_t tenant_id, const int64_t ls_id, const int64_t tx_id, const uint64_t cluster_id) - : tls_id_(tenant_id, share::ObLSID(ls_id)), - block_builder_(tx_id, cluster_id), + ObTxLogGenerator(const int64_t tx_id, const uint64_t cluster_id) + : block_builder_(tx_id, cluster_id), lsn_arr_(), last_record_lsn_(), last_normal_lsn_(), @@ -86,7 +85,6 @@ class ObTxLogGenerator trace_id_str_("obcdc_unittest_trace"), can_elr_(false), is_dup_(false), - is_sub2pc_(false), epoch_(1024), last_op_scn_(2048), checksum_(29890209), @@ -193,7 +191,6 @@ class ObTxLogGenerator // lsn of last log_entry, may be invalid if haven't generate any log_entry. palf::LSN last_lsn_(); private: - logservice::TenantLSID tls_id_; ObTxLogBlockBuilder block_builder_; ObLogLSNArray lsn_arr_; LSN last_record_lsn_; @@ -204,7 +201,6 @@ class ObTxLogGenerator common::ObString trace_id_str_; bool can_elr_; bool is_dup_; - bool is_sub2pc_; int64_t epoch_; int64_t last_op_scn_; int64_t checksum_; @@ -300,13 +296,12 @@ void ObTxLogGenerator::gen_commit_info_log() void ObTxLogGenerator::gen_prepare_log() { - share::ObLSArray inc_ls_arr; transaction::LogOffSet prev_lsn = last_lsn_(); ObTxPrevLogType prev_log_type(ObTxPrevLogType::TypeEnum::COMMIT_INFO); - ObTxPrepareLog prepare_log(inc_ls_arr, prev_lsn, prev_log_type); + ObTxPrepareLog prepare_log(prev_lsn, prev_log_type); LOG_DEBUG("gen prepare_log", K(prepare_log)); EXPECT_EQ(OB_SUCCESS, block_builder_.fill_tx_log_except_redo(prepare_log)); - trans_type_ = transaction::TransType::DIST_TRANS; // dist trans. + trans_type_ = transaction::TransType::SP_TRANS; } void ObTxLogGenerator::gen_commit_log() @@ -315,27 +310,17 @@ void ObTxLogGenerator::gen_commit_log() share::SCN commit_version; commit_version.convert_for_logservice(commit_ts); uint64_t checksum = 0; - share::ObLSArray inc_ls_arr; ObTxBufferNodeArray mds_arr; - transaction::ObLSLogInfoArray ls_info_arr; - if (transaction::TransType::DIST_TRANS == trans_type_) { - ObLSLogInfo ls_info1(share::ObLSID(1), LSN(190)); - ObLSLogInfo ls_info2(share::ObLSID(1001), last_lsn_()); - EXPECT_EQ(OB_SUCCESS, ls_info_arr.push_back(ls_info1)); - EXPECT_EQ(OB_SUCCESS, ls_info_arr.push_back(ls_info2)); - } ObArray checksum_signature; ObTxPrevLogType prev_log_type(ObTxPrevLogType::TypeEnum::PREPARE); ObTxCommitLog commit_log( commit_version, checksum, checksum_signature, - inc_ls_arr, mds_arr, trans_type_, last_lsn_(), - ls_info_arr, prev_log_type); LOG_DEBUG("gen commit_log", K(commit_log)); EXPECT_EQ(OB_SUCCESS, block_builder_.fill_tx_log_except_redo(commit_log)); diff --git a/unittest/logservice/CMakeLists.txt b/unittest/logservice/CMakeLists.txt index b30d98390..815c2ff95 100644 --- a/unittest/logservice/CMakeLists.txt +++ b/unittest/logservice/CMakeLists.txt @@ -12,7 +12,6 @@ log_unittest(test_log_meta_entry) log_unittest(test_log_meta) # log_unittest(test_palf_handle_impl) ob_unittest(test_log_state_mgr) -ob_unittest(test_log_reconfirm) ob_unittest(test_log_sliding_window) # ob_unittest(test_log_submit_log) ob_unittest(test_log_group_buffer) @@ -23,7 +22,6 @@ ob_unittest(test_fixed_sliding_window) # ob_unittest(palf_performance_unittest) ob_unittest(test_ob_tuple) #ob_unittest(test_ob_role_change_service) -ob_unittest(test_log_config_mgr) ob_unittest(test_clear_up_tmp_files) ob_unittest(test_log_dir_match) ob_unittest(test_server_log_block_mgr) @@ -34,4 +32,3 @@ ob_unittest(test_palf_throttling) ob_unittest(test_log_external_storage_handler) #ob_unittest(test_log_external_storage_io_task) ob_unittest(test_log_io_utils) -ob_unittest(test_fetch_log_engine) diff --git a/unittest/logservice/mock_logservice_container/mock_election.h b/unittest/logservice/mock_logservice_container/mock_election.h deleted file mode 100644 index 3d7215035..000000000 --- a/unittest/logservice/mock_logservice_container/mock_election.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_ELECTION_ -#define OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_ELECTION_ - -#include "logservice/palf/election_self.h" - -namespace oceanbase -{ -namespace palf -{ -using namespace election; - -namespace mockelection -{ -class MockElection : public palf::election::Election -{ -public: - MockElection() : leader_epoch_(0), role_(common::ObRole::FOLLOWER) {} - ~MockElection() {} - - void stop() override final - {} - int can_set_memberlist(const palf::LogConfigVersion &new_config_version) const override final - { - UNUSED(new_config_version); - return OB_SUCCESS; - } - // Set member list - int set_memberlist(const MemberList &new_member_list) override final - { - UNUSED(new_member_list); - return OB_SUCCESS; - } - int set_priority(ElectionPriority *) override final { return OB_SUCCESS; } - int reset_priority() override final { return OB_SUCCESS; } - // Get the current role of the election - int get_role(common::ObRole &role, int64_t &epoch) const override final - { - role = role_; - epoch = leader_epoch_; - return OB_SUCCESS; - } - // If you are the leader, then you get the accurate leader, if you are not the leader, then you get the lease owner - int get_current_leader_likely(common::ObAddr &p_addr, - int64_t &p_cur_leader_epoch) const override final - { - p_addr = leader_; - p_cur_leader_epoch = leader_epoch_; - return OB_SUCCESS; - } - // For internal testing to switch the main use - virtual int change_leader_to(const common::ObAddr &dest_addr) override final - { - UNUSED(dest_addr); - return OB_SUCCESS; - } - virtual int temporarily_downgrade_protocol_priority(const int64_t time_us, const char *reason) override final - { - UNUSED(time_us); - UNUSED(reason); - return OB_SUCCESS; - } - // Get local address - const common::ObAddr &get_self_addr() const override final - { - return self_; - } - // print log - virtual int64_t to_string(char *buf, const int64_t buf_len) const override final - { - UNUSED(buf); - UNUSED(buf_len); - return 0; - } -public: - common::ObAddr self_; - common::ObAddr leader_; - int64_t leader_epoch_; - common::ObRole role_; -}; -} // end of mockelection -} // end of palf -} // end of oceanbase - -#endif diff --git a/unittest/logservice/mock_logservice_container/mock_log_config_mgr.h b/unittest/logservice/mock_logservice_container/mock_log_config_mgr.h deleted file mode 100755 index 5ccfb53b5..000000000 --- a/unittest/logservice/mock_logservice_container/mock_log_config_mgr.h +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_LOG_CONFIG_MGR_ -#define OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_LOG_CONFIG_MGR_ - -#define private public -#include "logservice/palf/log_config_mgr.h" -#include "logservice/palf/log_sliding_window.h" -#undef private - -namespace oceanbase -{ -namespace palf -{ -namespace election -{ -class Election; -} - -class MockLogConfigMgr : public LogConfigMgr -{ -public: - MockLogConfigMgr() {} - virtual ~MockLogConfigMgr() {} - - int init(const int64_t palf_id, - const common::ObAddr &self, - const LogConfigMeta &log_ms_meta, - LogEngine *log_engine, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - election::Election *election, - LogModeMgr *mode_mgr) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(self); - log_ms_meta_ = log_ms_meta; - UNUSED(log_engine); - UNUSED(sw); - UNUSED(state_mgr); - UNUSED(election); - UNUSED(mode_mgr); - return ret; - } - void destroy() {} - - // require caller holds WLock in PalfHandleImpl - int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t replica_num, - const int64_t proposal_id, - LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(replica_num); - UNUSED(proposal_id); - UNUSED(config_version); - return ret; - } - int set_initial_member_list(const common::ObMemberList &member_list, - const common::ObMember &arb_replica, - const int64_t replica_num, - const int64_t proposal_id, - LogConfigVersion &config_version) - { - UNUSEDx(member_list, arb_replica, replica_num, proposal_id, config_version); - return OB_SUCCESS; - } - // following get ops need caller holds RLock in PalfHandleImpl - int64_t get_accept_proposal_id() const - { - return log_ms_meta_.proposal_id_; - } - int get_global_learner_list(common::GlobalLearnerList &learner_list) const - { - int ret = OB_SUCCESS; - UNUSED(learner_list); - return ret; - } - int get_degraded_learner_list(common::GlobalLearnerList &learner_list) const - { - int ret = OB_SUCCESS; - UNUSED(learner_list); - return ret; - } - int get_curr_member_list(common::ObMemberList &member_list, int64_t &replica_num) const - { - int ret = OB_SUCCESS; - if (OB_FAIL(log_ms_meta_.curr_.config_.get_expected_paxos_memberlist(member_list, replica_num))) { - PALF_LOG(WARN, "get_expected_paxos_memberlist failed", KR(ret), K_(palf_id), K_(self)); - } - return ret; - } - int get_log_sync_member_list(common::ObMemberList &member_list, int64_t &replica_num) const - { - int ret = OB_SUCCESS; - if (OB_FAIL(member_list.deep_copy(log_ms_meta_.curr_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), KPC(this)); - } else { - replica_num = log_ms_meta_.curr_.config_.log_sync_replica_num_; - } - return ret; - } - int get_log_sync_member_list_for_generate_committed_lsn( - ObMemberList &prev_member_list, - int64_t &prev_replica_num, - ObMemberList &curr_member_list, - int64_t &curr_replica_num, - bool &is_before_barrier, - LSN &barrier_lsn) const - { - int ret = OB_SUCCESS; - LSN prev_committed_end_lsn; - sw_->get_committed_end_lsn(prev_committed_end_lsn); - is_before_barrier = false; - barrier_lsn = LSN(PALF_INITIAL_LSN_VAL); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); - } else if (OB_FAIL(curr_member_list.deep_copy(log_ms_meta_.curr_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else if (FALSE_IT(curr_replica_num = log_ms_meta_.curr_.config_.log_sync_replica_num_)) { - } else if (OB_UNLIKELY(prev_committed_end_lsn < reconfig_barrier_.prev_end_lsn_ && - reconfig_barrier_.prev_end_lsn_.is_valid())) { - is_before_barrier = true; - barrier_lsn = reconfig_barrier_.prev_end_lsn_; - if (OB_FAIL(prev_member_list.deep_copy(log_ms_meta_.prev_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), K_(palf_id), K_(self)); - } else { - prev_replica_num = log_ms_meta_.prev_.config_.log_sync_replica_num_; - } - } else { } - return ret; - } - int get_alive_member_list_with_arb(common::ObMemberList &member_list, int64_t &replica_num) const - { - int ret = OB_SUCCESS; - GlobalLearnerList all_learners; - if (OB_FAIL(log_ms_meta_.curr_.config_.convert_to_complete_config(member_list, replica_num, all_learners))) { - PALF_LOG(WARN, "convert_to_complete_config failed", K(ret), KPC(this)); - } - return ret; - } - int get_prev_member_list(common::ObMemberList &member_list) const - { - int ret = OB_SUCCESS; - if (OB_FAIL(member_list.deep_copy(log_ms_meta_.prev_.config_.log_sync_memberlist_))) { - PALF_LOG(WARN, "deep_copy member_list failed", KR(ret), KPC(this)); - } - return ret; - } - int get_children_list(LogLearnerList &children) const - { - int ret = OB_SUCCESS; - return ret; - } - int get_config_version(LogConfigVersion &config_version) const - { - int ret = OB_SUCCESS; - config_version = log_ms_meta_.curr_.config_.config_version_; - return ret; - } - int get_replica_num(int64_t &replica_num) const - { - int ret = OB_SUCCESS; - common::ObMemberList member_list; - if (OB_FAIL(log_ms_meta_.curr_.config_.get_expected_paxos_memberlist(member_list, replica_num))) { - PALF_LOG(WARN, "get_expected_paxos_memberlist failed", KR(ret), KPC(this)); - } - return ret; - } - int leader_do_loop_work() - { - int ret = OB_SUCCESS; - return ret; - } - - // for PalfHandleImpl::one_stage_config_change_ - int check_config_change_args(const LogConfigChangeArgs &args, bool &is_already_finished) const - { - int ret = OB_SUCCESS; - UNUSED(args); - UNUSED(is_already_finished); - return ret; - } - // require caller holds WLock in PalfHandleImpl - int apply_config_meta(const int64_t curr_proposal_id, - const int64_t prev_log_proposal_id, - const LSN prev_lsn, - const LogConfigChangeArgs &args, - bool &is_already_finished, - LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(curr_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(args); - UNUSED(is_already_finished); - UNUSED(config_version); - return ret; - } - int submit_config_log(const int64_t proposal_id, const LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(proposal_id); - UNUSED(config_version); - return ret; - } - int check_ms_log_committed(const int64_t proposal_id, const LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(proposal_id); - UNUSED(config_version); - return ret; - } - - // for reconfirm - int submit_start_working_log(const int64_t &log_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(log_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(config_version); - return ret; - } - - // for PalfHandleImpl::receive_config_log - bool can_receive_config_log(const LogConfigVersion &config_version) const - { - int ret = OB_SUCCESS; - UNUSED(config_version); - return ret; - } - int after_flush_config_log(const LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(config_version); - return ret; - } - int submit_change_access_mode_log(const int64_t &log_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const AccessMode &access_mode, - LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(log_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(access_mode); - UNUSED(config_version); - return ret; - } - // follower receives member change log and needs to perform forward validation - int receive_config_log(const common::ObAddr &leader, const LogConfigMeta &meta) - { - int ret = OB_SUCCESS; - UNUSED(leader); - UNUSED(meta); - return ret; - } - - // for PalfHandleImpl::ack_config_log - int ack_config_log(const common::ObAddr &sender, - const int64_t proposal_id, - const LogConfigVersion &config_version) - { - int ret = OB_SUCCESS; - UNUSED(sender); - UNUSED(proposal_id); - UNUSED(config_version); - return ret; - } - int wait_config_log_persistence(const LogConfigVersion &config_version) const - { - int ret = OB_SUCCESS; - UNUSED(config_version); - return ret; - } - // broadcast leader info to global learners, only called in leader active - int submit_broadcast_leader_info(const int64_t proposal_id) const - { - int ret = OB_SUCCESS; - UNUSED(proposal_id); - return ret; - } - void reset_status() - {} -}; - -} // end of palf -} // end of oceanbase - -#endif diff --git a/unittest/logservice/mock_logservice_container/mock_log_engine.h b/unittest/logservice/mock_logservice_container/mock_log_engine.h index 35bf25550..61a162c0d 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_engine.h +++ b/unittest/logservice/mock_logservice_container/mock_log_engine.h @@ -33,52 +33,11 @@ namespace palf class MockLogEngine : public LogEngine { public: - MockLogEngine() - : parent_itself_(), - candidate_list_(), - reg_ret_(RegisterReturn::INVALID_REG_RET) - {} + MockLogEngine() = default; virtual ~MockLogEngine() {} - int init(const int64_t palf_id, - const char *base_dir, - const LogMeta &log_meta, - const LSN &base_lsn, - const int64_t base_lsn_ts_ns, - common::ObILogAllocator *alloc_mgr, - LogRpc *log_rpc, - LogIOWorker *log_io_worker) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(base_dir); - UNUSED(log_meta); - UNUSED(base_lsn); - UNUSED(base_lsn_ts_ns); - UNUSED(alloc_mgr); - UNUSED(log_rpc); - UNUSED(log_io_worker); - return ret; - } void destroy() {} - int load(const int64_t palf_id, - const char *base_dir, - common::ObILogAllocator *alloc_mgr, - LogRpc *log_rpc, - LogIOWorker *log_io_worker, - LogGroupEntryHeader &entry_header) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(base_dir); - UNUSED(alloc_mgr); - UNUSED(log_rpc); - UNUSED(log_io_worker); - UNUSED(entry_header); - return ret; - } - int submit_flush_log_task( const FlushLogCbCtx &flush_log_cb_ctx, const char *buf, @@ -101,42 +60,6 @@ class MockLogEngine : public LogEngine return ret; } - int submit_flush_prepare_meta_task( - const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogPrepareMeta &prepare_meta) - { - int ret = OB_SUCCESS; - UNUSED(flush_meta_cb_ctx); - UNUSED(prepare_meta); - return ret; - } - - int submit_flush_change_config_meta_task( - const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogConfigMeta &config_meta) - { - int ret = OB_SUCCESS; - UNUSED(flush_meta_cb_ctx); - UNUSED(config_meta); - return ret; - } - int submit_flush_mode_meta_task( - const FlushMetaCbCtx &flush_meta_cb_ctx, - const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - UNUSED(flush_meta_cb_ctx); - UNUSED(mode_meta); - return ret; - } - - int update_config_meta_sync( - const LogConfigMeta &config_meta) - { - int ret = OB_SUCCESS; - UNUSED(config_meta); - return ret; - } int submit_flush_snapshot_meta_task( const FlushMetaCbCtx &flush_meta_cb_ctx, @@ -148,13 +71,6 @@ class MockLogEngine : public LogEngine return ret; } - int submit_truncate_log_task( - const TruncateLogCbCtx &truncate_log_cb_ctx) - { - int ret = OB_SUCCESS; - UNUSED(truncate_log_cb_ctx); - return ret; - } int submit_truncate_prefix_blocks_task( const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx) @@ -180,13 +96,6 @@ class MockLogEngine : public LogEngine return ret; } - int after_truncate_log( - LogIOTruncateLogTask *log_io_task) - { - int ret = OB_SUCCESS; - UNUSED(log_io_task); - return ret; - } int after_truncate_prefix_blocks( LogIOTruncatePrefixBlocksTask *log_io_task) @@ -279,270 +188,6 @@ class MockLogEngine : public LogEngine return ret; } - int submit_push_log_req( - const common::ObMemberList &member_list, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(push_log_type); - UNUSED(msg_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(curr_lsn); - UNUSED(write_buf); - return ret; - } - - int submit_push_log_req( - const common::ObAddr &addr, - const PushLogType &push_log_type, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const LSN &curr_lsn, - const LogWriteBuf &write_buf) override - { - int ret = OB_SUCCESS; - UNUSED(addr); - UNUSED(push_log_type); - UNUSED(msg_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(curr_lsn); - UNUSED(write_buf); - return ret; - } - - int submit_push_log_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const LSN &lsn, - const bool is_batch) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(msg_proposal_id); - UNUSED(lsn); - UNUSED(is_batch); - return ret; - } - - int submit_prepare_meta_req(const ObMemberList &member_list, const int64_t &log_proposal_id) override - { - UNUSEDx(member_list, log_proposal_id); - return OB_SUCCESS; - } - - int submit_prepare_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id, - const bool vote_granted, - const int64_t &log_proposal_id, - const LSN &lsn, - const LSN &committed_end_lsn, - const LogModeMeta &mode_meta) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(msg_proposal_id); - UNUSED(vote_granted); - UNUSED(log_proposal_id); - UNUSED(lsn); - UNUSED(committed_end_lsn); - UNUSED(mode_meta); - return ret; - } - - int submit_change_config_meta_req( - const common::ObMemberList &member_list, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &config_meta) override - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(msg_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(prev_mode_pid); - UNUSED(config_meta); - return ret; - } - - int submit_change_config_meta_req( - const common::GlobalLearnerList &member_list, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &config_meta) override - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(msg_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(prev_mode_pid); - UNUSED(config_meta); - return ret; - } - - int submit_change_config_meta_req( - const common::ResendConfigLogList &member_list, - const int64_t &msg_proposal_id, - const int64_t &prev_log_proposal_id, - const LSN &prev_lsn, - const int64_t &prev_mode_pid, - const LogConfigMeta &config_meta) override - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(msg_proposal_id); - UNUSED(prev_log_proposal_id); - UNUSED(prev_lsn); - UNUSED(prev_mode_pid); - UNUSED(config_meta); - return ret; - } - - int submit_change_config_meta_resp( - const common::ObAddr &server, - const int64_t msg_proposal_id, - const LogConfigVersion &config_version) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(msg_proposal_id); - UNUSED(config_version); - return ret; - } - - int submit_change_mode_meta_req( - const common::ObMemberList &member_list, - const int64_t &msg_proposal_id, - const bool is_applied_mode_meta, - const LogModeMeta &mode_meta) override - { - int ret = OB_SUCCESS; - UNUSED(member_list); - UNUSED(msg_proposal_id); - UNUSED(is_applied_mode_meta); - UNUSED(mode_meta); - return ret; - } - - int submit_change_mode_meta_resp( - const common::ObAddr &server, - const int64_t &msg_proposal_id) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(msg_proposal_id); - return ret; - } - - int submit_config_change_pre_check_req( - const common::ObAddr &server, - const LogConfigVersion &config_version, - const bool need_purge_throttling, - const int64_t timeout_ns, - LogGetMCStResp &resp) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(config_version); - UNUSED(timeout_ns); - UNUSED(need_purge_throttling); - resp.is_normal_replica_ = true; - resp.need_update_config_meta_ = false; - resp.max_flushed_end_lsn_ = LSN(PALF_INITIAL_LSN_VAL); - return ret; - } - - int submit_fetch_log_req( - const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t msg_proposal_id, - const LSN &prev_lsn, - const LSN &lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count, - const int64_t accepted_mode_pid) - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(fetch_type); - UNUSED(msg_proposal_id); - UNUSED(prev_lsn); - UNUSED(lsn); - UNUSED(fetch_log_size); - UNUSED(fetch_log_count); - UNUSED(accepted_mode_pid); - return ret; - } - - int submit_notify_rebuild_req( - const ObAddr &server, - const LSN &base_lsn, - const LogInfo &base_prev_log_info) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(base_lsn); - return ret; - } - - int submit_register_parent_req(const common::ObAddr &server, - const LogLearner &child_itself, - const bool is_to_leader) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(child_itself); - UNUSED(is_to_leader); - return ret; - } - int submit_register_parent_resp(const common::ObAddr &server, - const LogLearner &parent_itself, - const LogCandidateList &candidate_list, - const RegisterReturn reg_ret) override - { - int ret = OB_SUCCESS; - UNUSED(server); - parent_itself_ = parent_itself; - candidate_list_ = candidate_list; - reg_ret_ = reg_ret; - return ret; - } - int submit_retire_parent_req(const common::ObAddr &server, const LogLearner &child_itself) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(child_itself); - return ret; - } - int submit_retire_child_req(const common::ObAddr &server, const LogLearner &parent_itself) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(parent_itself); - return ret; - } - int submit_learner_keepalive_req(const common::ObAddr &server, const LogLearner &sender_itself) override - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(sender_itself); - return ret; - } LogMeta get_log_meta() const { @@ -563,36 +208,7 @@ class MockLogEngine : public LogEngine LogStorage *get_log_storage() { return &log_storage_; } LogStorage *get_log_meta_storage() { return &log_meta_storage_; } - void reset_register_parent_resp_ret() - { - parent_itself_.reset(); - candidate_list_.reset(); - reg_ret_ = RegisterReturn::INVALID_REG_RET; - } - int check_config_meta_size(const LogConfigMeta &config_meta) const - { - int ret = OB_SUCCESS; - LogMeta log_meta; - if (OB_FAIL(log_meta.update_log_config_meta(config_meta))) { - PALF_LOG(ERROR, "LogMeta update_log_config_meta failed", K(ret), K_(palf_id), K_(is_inited)); - } else { - LogMetaEntryHeader log_meta_entry_header; - const int64_t log_meta_entry_header_len = log_meta_entry_header.get_serialize_size(); - const int64_t log_meta_body_len = log_meta.get_serialize_size(); - if (log_meta_entry_header_len + log_meta_body_len > MAX_META_ENTRY_SIZE) { - ret = OB_INVALID_ARGUMENT; - PALF_LOG(WARN, "check_config_meta_size failed", K(ret), K_(palf_id), - K(log_meta_entry_header_len), K(log_meta_body_len), K(config_meta)); - } - } - return ret; - } -public: - // register_parent_resp ret - LogLearner parent_itself_; - LogCandidateList candidate_list_; - RegisterReturn reg_ret_; }; } // end of palf diff --git a/unittest/logservice/mock_logservice_container/mock_log_mode_mgr.h b/unittest/logservice/mock_logservice_container/mock_log_mode_mgr.h index e05dda592..77509d7ff 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_mode_mgr.h +++ b/unittest/logservice/mock_logservice_container/mock_log_mode_mgr.h @@ -31,109 +31,9 @@ class MockLogModeMgr : public LogModeMgr public: MockLogModeMgr() {} virtual ~MockLogModeMgr() {} - - int init(const int64_t palf_id, - const common::ObAddr &self, - const LogModeMeta &log_mode_meta, - LogStateMgr *state_mgr, - LogEngine *log_engine, - LogConfigMgr *config_mgr, - LogSlidingWindow *sw) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(self); - UNUSED(log_mode_meta); - UNUSED(state_mgr); - UNUSED(log_engine); - UNUSED(config_mgr); - UNUSED(sw); - return ret; - } - void destroy() {} - void reset_status() - {} - int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const - { - int ret = OB_SUCCESS; - access_mode = applied_mode_meta_.access_mode_; - mode_version = applied_mode_meta_.mode_version_; - return ret; - } - int get_ref_ts_ns(int64_t &mode_version, int64_t &ref_ts_ns) const - { - int ret = OB_SUCCESS; - UNUSEDx(ref_ts_ns, mode_version); - return ret; - } - bool is_state_changed() const - { - return false; - } - LogModeMeta get_accepted_mode_meta() const - { - return mock_accepted_mode_meta_; - } - LogModeMeta get_last_submit_mode_meta() const - { - return mock_last_submit_mode_meta_; - } - int reconfirm_mode_meta() - { - int ret = OB_SUCCESS; - return ret; - } - int change_access_mode(const int64_t proposal_id, - const AccessMode &access_mode, - const int64_t ref_ts_ns, - bool &need_role_change) - { - int ret = OB_SUCCESS; - UNUSEDx(proposal_id, access_mode, ref_ts_ns, need_role_change); - return ret; - } - int handle_prepare_response(const common::ObAddr &server, - const int64_t proposal_id, - const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - UNUSEDx(server, proposal_id, mode_meta); - return ret; - } - bool can_receive_mode_meta(const int64_t proposal_id, - const LogModeMeta &mode_meta, - bool &has_accepted) - { - UNUSEDx(proposal_id, mode_meta, has_accepted); - return false; - } - int receive_mode_meta(const common::ObAddr &server, - const int64_t proposal_id, - const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - UNUSEDx(server, proposal_id, mode_meta); - return ret; - } - int after_flush_mode_meta(const LogModeMeta &mode_meta) - { - int ret = OB_SUCCESS; - UNUSEDx(mode_meta); - return ret; - } - int ack_mode_meta(const common::ObAddr &server, const int64_t proposal_id) - { - int ret = OB_SUCCESS; - UNUSEDx(server, proposal_id); - return ret; - } -public: - LogModeMeta mock_applied_mode_meta_; - LogModeMeta mock_accepted_mode_meta_; - LogModeMeta mock_last_submit_mode_meta_; }; -} // end of palf -} // end of oceanbase +} // namespace palf +} // namespace oceanbase #endif diff --git a/unittest/logservice/mock_logservice_container/mock_log_reconfirm.h b/unittest/logservice/mock_logservice_container/mock_log_reconfirm.h deleted file mode 100644 index cea099090..000000000 --- a/unittest/logservice/mock_logservice_container/mock_log_reconfirm.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_LOG_RECONFIRM_ -#define OCEANBASE_UNITTEST_LOGSERVICE_MOCK_CONTAINER_LOG_RECONFIRM_ -#include "logservice/palf/log_reconfirm.h" - -namespace oceanbase -{ -namespace palf -{ - -class MockLogReconfirm : public LogReconfirm -{ -public: - MockLogReconfirm() : mock_ret_(OB_SUCCESS) {} - ~MockLogReconfirm() {} - - int init(const int64_t palf_id, - const ObAddr &self, - LogSlidingWindow *sw, - LogStateMgr *state_mgr, - LogConfigMgr *mm, - LogEngine *log_engine) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(self); - UNUSED(sw); - UNUSED(state_mgr); - UNUSED(mm); - UNUSED(log_engine); - return ret; - } - void destroy() {} - void reset_state() {} - bool need_start_up() - { - return false; - } - int reconfirm() - { - return mock_ret_; - } - int handle_prepare_response(const common::ObAddr &server, - const int64_t &src_proposal_id, - const int64_t &accept_proposal_id, - const LSN &last_lsn) - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(src_proposal_id); - UNUSED(accept_proposal_id); - UNUSED(last_lsn); - return ret; - } - int receive_log(const common::ObAddr &server, - const LSN &prev_lsn, - const int64_t &prev_proposal_id, - const LSN &lsn, - const char *buf, - const int64_t buf_len) - { - int ret = OB_SUCCESS; - UNUSED(server); - UNUSED(prev_lsn); - UNUSED(prev_proposal_id); - UNUSED(lsn); - UNUSED(buf); - UNUSED(buf_len); - return ret; - } -public: - int mock_ret_; -}; - -} // end of palf -} // end of oceanbase - -#endif diff --git a/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h b/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h index c4c8192cd..513a68d3f 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h +++ b/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h @@ -33,33 +33,15 @@ class MockLogSlidingWindow : public LogSlidingWindow { public: MockLogSlidingWindow() - : pending_end_lsn_(0), mock_start_id_(1) + : all_log_flushed_(true), + all_committed_slided_out_(true), + activated_(false), + pending_end_lsn_(0), + mock_start_id_(1) {} virtual ~MockLogSlidingWindow() {} public: void destroy() {} - int init(const int64_t palf_id, - const common::ObAddr &self, - LogStateMgr *state_mgr, - LogConfigMgr *mm, - LogEngine *log_engine, - palf::PalfFSCbWrapper *palf_fs_cb, - common::ObILogAllocator *alloc_mgr, - const PalfBaseInfo &palf_base_info, - const bool is_normal_replica) - { - int ret = OB_SUCCESS; - UNUSED(palf_id); - UNUSED(self); - UNUSED(state_mgr); - UNUSED(mm); - UNUSED(log_engine); - UNUSED(palf_fs_cb); - UNUSED(alloc_mgr); - UNUSED(palf_base_info); - UNUSED(is_normal_replica); - return ret; - } int sliding_cb(const int64_t sn, const FixedSlidingWindowSlot *data) { int ret = OB_SUCCESS; @@ -91,37 +73,13 @@ class MockLogSlidingWindow : public LogSlidingWindow UNUSED(committed_end_lsn); return ret; } - int get_server_ack_info(const common::ObAddr &server, LsnTsInfo &lsn_ts_info) - { - UNUSEDx(server, lsn_ts_info); - return OB_SUCCESS; - } - int try_fetch_log(const FetchTriggerType &fetch_log_type, - const LSN prev_lsn = LSN(), - const LSN fetch_start_lsn = LSN(), - const int64_t fetch_start_log_id = OB_INVALID_LOG_ID) - { - int ret = OB_SUCCESS; - UNUSED(prev_lsn); - UNUSED(fetch_start_lsn); - UNUSED(fetch_start_log_id); - return ret; - } - int try_fetch_log_for_reconfirm(const common::ObAddr &dest, const LSN &fetch_end_lsn, bool &is_fetched) - { - int ret = OB_SUCCESS; - UNUSED(dest); - UNUSED(fetch_end_lsn); - is_fetched = true; - return ret; - } bool is_empty() const { return true; } bool check_all_log_has_flushed() { - return true; + return all_log_flushed_; } int get_majority_match_lsn(LSN &majority_match_lsn) { @@ -143,50 +101,12 @@ class MockLogSlidingWindow : public LogSlidingWindow UNUSED(log_timestamp); return ret; } - int submit_group_log(const LSN &lsn, - const char *buf, - const int64_t buf_len) - { - int ret = OB_SUCCESS; - UNUSED(lsn); - UNUSED(buf); - UNUSED(buf_len); - return ret; - } - int receive_log(const common::ObAddr &src_server, - const PushLogType push_log_type, - const LSN &prev_lsn, - const int64_t &prev_log_pid, - const LSN &lsn, - const char *buf, - const int64_t buf_len, - const bool need_check_clean_log, - TruncateLogInfo &truncate_log_info) override - { - int ret = OB_SUCCESS; - UNUSED(src_server); - UNUSED(push_log_type); - UNUSED(prev_lsn); - UNUSED(prev_log_pid); - UNUSED(lsn); - UNUSED(buf); - UNUSED(buf_len); - UNUSED(need_check_clean_log); - UNUSED(truncate_log_info); - return ret; - } int after_flush_log(const FlushLogCbCtx &flush_cb_ctx) { int ret = OB_SUCCESS; UNUSED(flush_cb_ctx); return ret; } - int after_truncate(const TruncateLogCbCtx &truncate_cb_ctx) - { - int ret = OB_SUCCESS; - UNUSED(truncate_cb_ctx); - return ret; - } int ack_log(const common::ObAddr &src_server, const LSN &end_lsn) { int ret = OB_SUCCESS; @@ -194,31 +114,12 @@ class MockLogSlidingWindow : public LogSlidingWindow UNUSED(end_lsn); return ret; } - int truncate(const TruncateLogInfo &truncate_log_info, const LSN &expected_prev_lsn, const int64_t expected_prev_log_pid) override - { - int ret = OB_SUCCESS; - UNUSED(truncate_log_info); - UNUSED(expected_prev_lsn); - UNUSED(expected_prev_log_pid); - return ret; - } int truncate_for_rebuild(const PalfBaseInfo &palf_base_info) { int ret = OB_SUCCESS; UNUSED(palf_base_info); return ret; } - bool pre_check_for_config_log(const int64_t &msg_proposal_id, - const LSN &lsn, - const int64_t &log_proposal_id, - TruncateLogInfo &truncate_log_info) override - { - UNUSED(msg_proposal_id); - UNUSED(lsn); - UNUSED(log_proposal_id); - UNUSED(truncate_log_info); - return true; - } // ================= log sync part end int append_disk_log(const LSN &lsn, const LogGroupEntry &group_entry) { @@ -237,80 +138,26 @@ class MockLogSlidingWindow : public LogSlidingWindow { end_lsn = max_flushed_end_lsn_; } - int get_max_flushed_log_info(LSN &lsn, - LSN &end_lsn, - int64_t &log_proposal_id) const - { - int ret = OB_SUCCESS; - lsn = mock_max_flushed_lsn_; - end_lsn = mock_max_flushed_end_lsn_; - log_proposal_id = mock_max_flushed_log_pid_; - return ret; - } int clean_log(const bool need_clear_log_exist_flag) { int ret = OB_SUCCESS; UNUSED(need_clear_log_exist_flag); return ret; } - int clean_cached_log(const int64_t begin_log_id, - const LSN &lsn, - const LSN &prev_lsn, - const int64_t prev_log_pid) override - { - int ret = OB_SUCCESS; - UNUSED(begin_log_id); - UNUSED(lsn); - UNUSED(prev_lsn); - UNUSED(prev_log_pid); - return ret; - } - int to_follower_pending(LSN &last_lsn) - { - int ret = OB_SUCCESS; - last_lsn = pending_end_lsn_; - return ret; - } - int to_leader_reconfirm() override - { - int ret = OB_SUCCESS; - return ret; - } - int to_leader_active() + int activate() override { - int ret = OB_SUCCESS; - return ret; + activated_ = true; + return OB_SUCCESS; } int try_advance_committed_end_lsn(const LSN &end_lsn) { - int ret = OB_SUCCESS; - UNUSED(end_lsn); - return ret; + pending_end_lsn_ = end_lsn; + return OB_SUCCESS; } int64_t get_last_submit_log_id_() const { return 1; } - int get_last_submit_log_info(LSN &last_submit_lsn, int64_t &log_id, int64_t &log_proposal_id) const - { - int ret = OB_SUCCESS; - last_submit_lsn = mock_last_submit_lsn_; - log_id = mock_last_submit_log_id_; - log_proposal_id = mock_last_submit_pid_; - return ret; - } - int get_last_submit_log_info(LSN &last_submit_lsn, - LSN &last_submit_end_lsn, - int64_t &log_id, - int64_t &log_proposal_id) const - { - int ret = OB_SUCCESS; - last_submit_lsn = mock_last_submit_lsn_; - last_submit_end_lsn = mock_last_submit_end_lsn_; - log_id = mock_last_submit_log_id_; - log_proposal_id = mock_last_submit_pid_; - return ret; - } int get_last_slide_end_lsn(LSN &out_end_lsn) const { int ret = OB_SUCCESS; @@ -336,54 +183,20 @@ class MockLogSlidingWindow : public LogSlidingWindow prev_lsn = LSN(PALF_INITIAL_LSN_VAL); prev_log_id = OB_INVALID_LOG_ID; committed_end_lsn = LSN(PALF_INITIAL_LSN_VAL + 1000); - return true; - } - // location cache will be removed TODO by yunlong - int set_location_cache_cb(PalfLocationCacheCb *lc_cb) - { - int ret = OB_SUCCESS; - UNUSED(lc_cb); - return ret; - } - int reset_location_cache_cb() - { - int ret = OB_SUCCESS; - return ret; - } - int config_change_update_match_lsn_map(const ObMemberList &added_memberlist, - const ObMemberList &removed_memberlist, - const ObMemberList &new_log_sync_memberlist, - const int64_t new_replica_num) - { - UNUSED(added_memberlist); - UNUSED(removed_memberlist); - UNUSED(new_log_sync_memberlist); - UNUSED(new_replica_num); - return OB_SUCCESS; - } - int get_server_ack_info(const common::ObAddr &server, LsnTsInfo &ack_info) const - { - UNUSED(server); - ack_info.last_ack_time_us_ = common::ObTimeUtility::current_time(); - ack_info.lsn_ = LSN(PALF_INITIAL_LSN_VAL); - return OB_SUCCESS; - } - int get_leader_from_cache(common::ObAddr &leader) const - { - leader = state_mgr_->get_leader(); - return OB_SUCCESS; + return all_committed_slided_out_; } public: + bool all_log_flushed_; + bool all_committed_slided_out_; + bool activated_; palf::MockLogStateMgr *state_mgr_; LSN pending_end_lsn_; int64_t mock_start_id_; int64_t mock_last_submit_log_id_; LSN mock_last_submit_lsn_; LSN mock_last_submit_end_lsn_; - int64_t mock_last_submit_pid_; LSN mock_max_flushed_lsn_; LSN mock_max_flushed_end_lsn_; - int64_t mock_max_flushed_log_pid_; }; } // end of palf diff --git a/unittest/logservice/mock_logservice_container/mock_log_state_mgr.h b/unittest/logservice/mock_logservice_container/mock_log_state_mgr.h index 02379b856..ec12f32e9 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_state_mgr.h +++ b/unittest/logservice/mock_logservice_container/mock_log_state_mgr.h @@ -30,167 +30,39 @@ class MockLogStateMgr : public LogStateMgr { public: MockLogStateMgr() - : mock_proposal_id_(1) - {} - ~MockLogStateMgr() {} - - virtual void destroy() {} - virtual int init(const int64_t key, - const common::ObAddr &self, - const LogPrepareMeta &log_prepare_meta, - election::Election* election, - LogSlidingWindow *sw, - LogReconfirm *reconfirm, - LogEngine *log_engine, - LogConfigMgr *mm, - palf::PalfRoleChangeCbWrapper *palf_role_change_cb) - { - int ret = OB_SUCCESS; - UNUSED(key); - UNUSED(self); - UNUSED(log_prepare_meta); - UNUSED(election); - UNUSED(sw); - UNUSED(reconfirm); - UNUSED(log_engine); - UNUSED(mm); - UNUSED(palf_role_change_cb); - return ret; - } - virtual bool is_state_changed() { return false; } - virtual int switch_state() { return OB_SUCCESS; } - virtual int handle_prepare_request(const common::ObAddr &new_leader, - const int64_t &proposal_id) - { - int ret = OB_SUCCESS; - UNUSED(new_leader); - UNUSED(proposal_id); - return ret; - } - virtual int set_scan_disk_log_finished() - { - return OB_SUCCESS; - } - virtual bool can_append(const int64_t proposal_id, const bool need_check_proposal_id) const - { - UNUSED(proposal_id); - UNUSED(need_check_proposal_id); - return true; - } - virtual bool can_raw_write() const - { - return true; - } - virtual bool can_slide_sw() const - { - return true; - } - virtual bool can_revoke(const int64_t proposal_id) const - { - return true; - } - virtual int check_role_leader_and_state() const - { - return OB_SUCCESS; - } - virtual int64_t get_proposal_id() const - { - return mock_proposal_id_; - } - virtual common::ObRole get_role() const { return role_; } - virtual int16_t get_state() const { return state_; } - virtual void get_role_and_state(common::ObRole &role, ObReplicaState &state) const - { - } - virtual const common::ObAddr &get_leader() const { return leader_; } - virtual int64_t get_leader_epoch() const { return leader_epoch_; } - virtual bool check_epoch_is_same_with_election(const int64_t expected_epoch) const - { - UNUSED(expected_epoch); - return true; - } - virtual bool can_handle_prepare_response(const int64_t &proposal_id) const - { - UNUSED(proposal_id); - return true; - } - virtual bool can_handle_prepare_request(const int64_t &proposal_id) const + : is_changing_config_with_arb_(false) { - UNUSED(proposal_id); - return true; + state_ = ACTIVE; } - virtual bool can_handle_committed_info(const int64_t &proposal_id) const - { - UNUSED(proposal_id); - return true; - } - virtual bool can_receive_log(const int64_t &proposal_id) const - { - bool bool_ret = false; - if (proposal_id == get_proposal_id() && true == is_sync_enabled()) { - bool_ret = (is_follower_active() || is_follower_pending() || is_leader_reconfirm()); - } - return bool_ret; - } - virtual bool can_send_log_ack(const int64_t &proposal_id) const - { - UNUSED(proposal_id); - return true; - } - virtual bool can_receive_log_ack(const int64_t &proposal_id) const - { - UNUSED(proposal_id); - return true; - } - virtual bool can_truncate_log() const - { - return true; - } - virtual bool need_freeze_group_buffer() const - { - return false; - } - virtual bool is_leader_active() const { return role_ == LEADER && state_ == ACTIVE; } - virtual bool is_follower_pending() const { return role_ == FOLLOWER && state_ == PENDING; } - virtual bool is_follower_active() const { return role_ == FOLLOWER && state_ == ACTIVE; } - virtual bool is_leader_reconfirm() const { return role_ == LEADER && state_ == RECONFIRM; } - virtual int truncate(const LSN &truncate_begin_lsn) - { - UNUSED(truncate_begin_lsn); - return OB_SUCCESS; - } - virtual int enable_sync() - { - return OB_SUCCESS; - } - virtual int disable_sync() - { - return OB_SUCCESS; - } - virtual bool is_sync_enabled() const - { - return true; - } - virtual int set_changing_config_with_arb() + ~MockLogStateMgr() {} + + void destroy() override {} + bool is_state_changed() override { return false; } + int switch_state() override { return OB_SUCCESS; } + int set_scan_disk_log_finished() override { return OB_SUCCESS; } + bool can_append() const override { return true; } + bool can_slide_sw() const override { return true; } + LogState get_state() const override { return static_cast(state_); } + bool need_freeze_group_buffer() const override { return false; } + bool is_active() const override { return ACTIVE == state_; } + bool is_recovering() const override { return RECOVERING == state_; } + int set_changing_config_with_arb() { is_changing_config_with_arb_ = true; return OB_SUCCESS; } - virtual int reset_changing_config_with_arb() + int reset_changing_config_with_arb() { is_changing_config_with_arb_ = false; return OB_SUCCESS; } - virtual bool is_changing_config_with_arb() const - { - return is_changing_config_with_arb_; - } + bool is_changing_config_with_arb() const { return is_changing_config_with_arb_; } + public: - int64_t mock_proposal_id_; bool is_changing_config_with_arb_; }; -} // end of palf -} // end of oceanbase +} // namespace palf +} // namespace oceanbase #endif diff --git a/unittest/logservice/mock_logservice_container/mock_ls_service.h b/unittest/logservice/mock_logservice_container/mock_ls_service.h deleted file mode 100644 index b4cb1a7c7..000000000 --- a/unittest/logservice/mock_logservice_container/mock_ls_service.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_LOGSERVICE_MOCK_LOG_STREAM_SERVICE_ -#define OCEANBASE_UNITTEST_LOGSERVICE_MOCK_LOG_STREAM_SERVICE_ -#include "storage/tx_storage/ob_ls_service.h" -namespace oceanbase -{ -namespace logservice -{ -class MockLS : public storage::ObLS -{ -public: - int replay(const palf::LSN &lsn, - const int64_t &log_timestamp, - const int64_t &log_size, - const char *log_buf) - { - REPLAY_LOG(INFO, "replay log", K(lsn), K(log_timestamp), K(log_size), K(log_size)); - return OB_SUCCESS; - } -}; - -class MockLSMap : public storage::ObLSMap -{ -public: - void revert_ls(ObLS *ls) - { - // do nothing - } -}; - -class MockLSService : public storage::ObLSService -{ -public: - int get_ls(const share::ObLSID &ls_id, - ObLSHandle &handle) - { - handle.set_ls(map_, ls_); - return OB_SUCCESS; - } -private: - MockLS ls_; - MockLSMap map_; -}; -} -} -#endif //OCEANBASE_UNITTEST_LOGSERVICE_MOCK_LOG_STREAM_SERVICE_ diff --git a/unittest/logservice/mock_logservice_container/mock_palf_fs_cb_wrapper.h b/unittest/logservice/mock_logservice_container/mock_palf_fs_cb_wrapper.h index af30e423f..7f245be20 100644 --- a/unittest/logservice/mock_logservice_container/mock_palf_fs_cb_wrapper.h +++ b/unittest/logservice/mock_logservice_container/mock_palf_fs_cb_wrapper.h @@ -39,12 +39,11 @@ class MockPalfFSCbWrapper : public PalfFSCbWrapper { { UNUSED(cb_impl); } - int update_end_lsn(int64_t id, const LSN &end_lsn, const int64_t proposal_id) + int update_end_lsn(const LSN &end_lsn, const share::SCN &end_scn) { int ret = OB_SUCCESS; - UNUSED(id); UNUSED(end_lsn); - UNUSED(proposal_id); + UNUSED(end_scn); return ret; } }; diff --git a/unittest/logservice/mock_logservice_container/mock_palf_role_change_cb_wrapper.h b/unittest/logservice/mock_logservice_container/mock_palf_role_change_cb_wrapper.h index 700f275f0..271353b15 100644 --- a/unittest/logservice/mock_logservice_container/mock_palf_role_change_cb_wrapper.h +++ b/unittest/logservice/mock_logservice_container/mock_palf_role_change_cb_wrapper.h @@ -39,11 +39,9 @@ class MockPalfRoleChangeCbWrapper : public PalfRoleChangeCbWrapper { { UNUSED(cb_impl); } - int on_role_change(int64_t id) + int on_role_change() { - int ret = OB_SUCCESS; - UNUSED(id); - return ret; + return OB_SUCCESS; } }; diff --git a/unittest/logservice/test_fetch_log_engine.cpp b/unittest/logservice/test_fetch_log_engine.cpp deleted file mode 100644 index 817c3e652..000000000 --- a/unittest/logservice/test_fetch_log_engine.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "lib/oblog/ob_log.h" -#include "logservice/ob_tenant_mutil_allocator.h" - -#define private public -#include "logservice/palf/fetch_log_engine.h" -#include "logservice/palf/palf_env_impl.h" -#include "logservice/palf/palf_options.h" -#undef private - -namespace oceanbase -{ -namespace palf -{ - -TEST(TestFetchLogEngine, disabled_lifecycle) -{ - PalfEnvImpl palf_env; - common::ObTenantMutilAllocator allocator; - FetchLogEngine engine; - - ASSERT_EQ(OB_SUCCESS, engine.init(&palf_env, &allocator, false)); - EXPECT_FALSE(engine.is_enabled()); - EXPECT_EQ(0, engine.get_thread_count()); - EXPECT_EQ(OB_SUCCESS, engine.start()); - EXPECT_EQ(OB_SUCCESS, engine.stop()); - EXPECT_EQ(OB_SUCCESS, engine.wait()); - EXPECT_EQ(nullptr, engine.alloc_fetch_log_task()); - EXPECT_EQ(OB_NOT_SUPPORTED, engine.submit_fetch_log_task(nullptr)); - EXPECT_EQ(OB_SUCCESS, engine.update_replayable_point(share::SCN::base_scn())); - - engine.destroy(); - engine.destroy(); -} - -TEST(TestPalfEnvImpl, fetch_log_engine_option_is_read_only) -{ - PalfEnvImpl palf_env; - palf_env.is_inited_ = true; - palf_env.enable_fetch_log_engine_ = false; - - PalfOptions current_options; - ASSERT_EQ(OB_SUCCESS, palf_env.get_options(current_options)); - EXPECT_FALSE(current_options.enable_fetch_log_engine_); - - current_options.disk_options_.log_disk_usage_limit_size_ = 1024 * 1024 * 1024L; - current_options.disk_options_.log_disk_utilization_threshold_ = 80; - current_options.disk_options_.log_disk_utilization_limit_threshold_ = 95; - current_options.disk_options_.log_disk_throttling_percentage_ = 60; - current_options.disk_options_.log_disk_throttling_maximum_duration_ = 7200LL * 1000 * 1000; - current_options.disk_options_.log_writer_parallelism_ = 1; - current_options.enable_fetch_log_engine_ = true; - ASSERT_TRUE(current_options.is_valid()); - EXPECT_EQ(OB_NOT_SUPPORTED, palf_env.update_options(current_options)); - - palf_env.is_inited_ = false; -} - -TEST(TestPalfHandleImpl, get_log_is_not_supported_when_fetch_is_disabled) -{ - PalfEnvImpl palf_env; - common::ObTenantMutilAllocator allocator; - FetchLogEngine engine; - PalfHandleImpl palf_handle; - common::ObAddr server; - - ASSERT_TRUE(server.set_ip_addr("127.0.0.1", 1000)); - ASSERT_EQ(OB_SUCCESS, engine.init(&palf_env, &allocator, false)); - palf_handle.is_inited_ = true; - palf_handle.palf_id_ = 1; - palf_handle.fetch_log_engine_ = &engine; - EXPECT_EQ(OB_NOT_SUPPORTED, - palf_handle.get_log(server, FETCH_LOG_FOLLOWER, 1, LSN(0), LSN(0), 1, 1, 1)); - - palf_handle.is_inited_ = false; - palf_handle.fetch_log_engine_ = nullptr; -} - -TEST(TestPalfOptions, fetch_log_engine_default) -{ - PalfOptions options; - EXPECT_TRUE(options.enable_fetch_log_engine_); - options.enable_fetch_log_engine_ = false; - options.reset(); - EXPECT_TRUE(options.enable_fetch_log_engine_); -} - -} // namespace palf -} // namespace oceanbase - -int main(int argc, char **argv) -{ - OB_LOGGER.set_file_name("test_fetch_log_engine.log", true); - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/unittest/logservice/test_log_config_mgr.cpp b/unittest/logservice/test_log_config_mgr.cpp deleted file mode 100755 index b71ac29a0..000000000 --- a/unittest/logservice/test_log_config_mgr.cpp +++ /dev/null @@ -1,821 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#define private public -#include "share/rc/ob_tenant_base.h" -#include "logservice/palf/log_config_mgr.h" -#include "mock_logservice_container/mock_election.h" -#include "mock_logservice_container/mock_log_sliding_window.h" -#include "mock_logservice_container/mock_log_engine.h" -#include "mock_logservice_container/mock_log_mode_mgr.h" -#include "mock_logservice_container/mock_log_reconfirm.h" -#include "share/ob_cluster_version.h" -#undef private -namespace oceanbase -{ -namespace unittest -{ -using namespace common; -using namespace share; -using namespace palf; - -const ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 1000); -const ObAddr addr2(ObAddr::IPV4, "127.0.0.2", 1000); -const ObAddr addr3(ObAddr::IPV4, "127.0.0.3", 1000); -const ObAddr addr4(ObAddr::IPV4, "127.0.0.4", 1000); -const ObAddr addr5(ObAddr::IPV4, "127.0.0.5", 1000); -const ObAddr addr6(ObAddr::IPV4, "127.0.0.6", 1000); -const ObAddr addr7(ObAddr::IPV4, "127.0.0.7", 1000); -const ObAddr addr8(ObAddr::IPV4, "127.0.0.8", 1000); -const ObAddr addr9(ObAddr::IPV4, "127.0.0.9", 1000); -const int64_t INIT_ELE_EPOCH = 1; -const int64_t INIT_PROPOSAL_ID = 1; - -int build_log_sync_member_list(ObMemberList &member_list) -{ - member_list.reset(); - return member_list.add_server(addr1); -} - -int64_t get_default_log_sync_replica_num() -{ - return 1; -} - -int build_default_log_config_info(LogConfigInfoV2 &config_info, - GlobalLearnerList &learner_list, - LogConfigVersion &config_version) -{ - ObMemberList member_list; - int ret = build_log_sync_member_list(member_list); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(config_version.generate(1, 1))) { - } else { - ret = config_info.generate(member_list, get_default_log_sync_replica_num(), - learner_list, config_version); - } - return ret; -} - -class TestLogConfigMgr : public ::testing::Test -{ -public: - TestLogConfigMgr() - { - mock_election_ = OB_NEW(mockelection::MockElection, "TestLog"); - mock_state_mgr_ = OB_NEW(MockLogStateMgr, "TestLog"); - mock_sw_ = OB_NEW(MockLogSlidingWindow, "TestLog"); - mock_log_engine_ = OB_NEW(MockLogEngine, "TestLog"); - mock_mode_mgr_ = OB_NEW(MockLogModeMgr, "TestLog"); - mock_reconfirm_ = OB_NEW(MockLogReconfirm, "TestLog"); - mock_plugins_ = OB_NEW(LogPlugins, "TestLog"); - } - ~TestLogConfigMgr() - { - OB_DELETE(MockElection, "TestLog", mock_election_); - OB_DELETE(MockLogStateMgr, "TestLog", mock_state_mgr_); - OB_DELETE(MockLogSlidingWindow, "TestLog", mock_sw_); - OB_DELETE(MockLogEngine, "TestLog", mock_log_engine_); - OB_DELETE(MockLogModeMgr, "TestLog", mock_mode_mgr_); - OB_DELETE(MockLogReconfirm, "TestLog", mock_reconfirm_); - OB_DELETE(LogPlugins, "TestLog", mock_plugins_); - } - void init_test_log_config_env(const common::ObAddr &self, - const LogConfigInfoV2 &config_info, - LogConfigMgr &cm, - common::ObRole role = LEADER, - ObReplicaState state = ACTIVE) - { - mock_state_mgr_->role_ = role; - mock_state_mgr_->state_ = state; - mock_state_mgr_->leader_ = (LEADER == role)? self: ObAddr(); - mock_election_->role_ = role; - mock_election_->leader_epoch_ = INIT_ELE_EPOCH; - mock_sw_->mock_last_submit_lsn_ = LSN(PALF_INITIAL_LSN_VAL); - mock_sw_->mock_last_submit_end_lsn_ = LSN(PALF_INITIAL_LSN_VAL); - mock_sw_->mock_last_submit_pid_ = INIT_PROPOSAL_ID; - mock_sw_->state_mgr_ = mock_state_mgr_; - ASSERT_TRUE(config_info.is_valid()); - PALF_LOG(INFO, "init_test_log_config_env", K(role), K(state), K(mock_state_mgr_->leader_), K(mock_state_mgr_->role_)); - mock_log_engine_->reset_register_parent_resp_ret(); - common::GlobalLearnerList learner_list; - const int64_t init_pid = INIT_PROPOSAL_ID; - LogConfigMeta config_meta; - ASSERT_EQ(OB_SUCCESS, config_meta.generate(init_pid, config_info, config_info, 1, LSN(0), 1)); - ASSERT_EQ(OB_SUCCESS, cm.init(1, self, config_meta, mock_log_engine_, mock_sw_, mock_state_mgr_, mock_election_, - mock_mode_mgr_, mock_reconfirm_, mock_plugins_)); - } -public: - mockelection::MockElection *mock_election_; - palf::MockLogStateMgr *mock_state_mgr_; - palf::MockLogSlidingWindow *mock_sw_; - palf::MockLogEngine *mock_log_engine_; - palf::MockLogModeMgr *mock_mode_mgr_; - palf::MockLogReconfirm *mock_reconfirm_; - palf::LogPlugins *mock_plugins_; -}; - -TEST_F(TestLogConfigMgr, test_remove_child_is_not_learner) -{ - LogConfigMgr cm; - LogLearnerList removed_children; - LogLearnerList retire_children; - LogLearnerList target_children; - EXPECT_EQ(OB_SUCCESS, cm.all_learnerlist_.add_learner(ObMember(addr1, -1))); - EXPECT_EQ(OB_SUCCESS, cm.all_learnerlist_.add_learner(ObMember(addr4, -1))); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(LogLearner(addr1, 1))); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(LogLearner(addr2, 1))); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(LogLearner(addr3, 1))); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(LogLearner(addr4, 1))); - EXPECT_EQ(OB_SUCCESS, target_children.add_learner(LogLearner(addr1, 1))); - EXPECT_EQ(OB_SUCCESS, target_children.add_learner(LogLearner(addr4, 1))); - EXPECT_EQ(OB_SUCCESS, retire_children.add_learner(LogLearner(addr2, 1))); - EXPECT_EQ(OB_SUCCESS, retire_children.add_learner(LogLearner(addr3, 1))); - EXPECT_EQ(OB_SUCCESS, cm.remove_child_is_not_learner_(removed_children)); - EXPECT_TRUE(cm.children_.learner_addr_equal(target_children)); - EXPECT_TRUE(retire_children.learner_addr_equal(removed_children)); - PALF_LOG(INFO, "children", K(cm.children_)); -} - -TEST_F(TestLogConfigMgr, test_handle_learner_keepalive) -{ - LogConfigMgr cm; - cm.is_inited_ = true; - LogLearner child(addr1, 1); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(child)); - EXPECT_EQ(OB_SUCCESS, cm.handle_learner_keepalive_resp(child)); - LogLearner child_in_list; - EXPECT_EQ(OB_SUCCESS, cm.children_.get_learner_by_addr(addr1, child_in_list)); - EXPECT_GT(child_in_list.keepalive_ts_, 0); -} - -TEST_F(TestLogConfigMgr, test_config_change_lock) -{ - ObMemberList init_member_list; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - GlobalLearnerList learner_list; - LogConfigInfoV2 default_config_info; - ASSERT_EQ(OB_SUCCESS, default_config_info.generate(init_member_list, get_default_log_sync_replica_num(), - learner_list, init_config_version)); - - - const int64_t lock_type_try_lock = ConfigChangeLockType::LOCK_PAXOS_MEMBER_CHANGE; - const int64_t lock_type_unlock = ConfigChangeLockType::LOCK_NOTHING; - LogLockMeta lock_meta; - EXPECT_EQ(OB_SUCCESS, lock_meta.generate(2, lock_type_try_lock)); - LogConfigInfoV2 locked_config_info; - ASSERT_EQ(OB_SUCCESS, locked_config_info.generate(init_member_list, get_default_log_sync_replica_num(), - learner_list, init_config_version, lock_meta)); - - LogLockMeta unlock_meta = lock_meta; - unlock_meta.unlock(); - LogConfigInfoV2 unlock_config_info; - ASSERT_EQ(OB_SUCCESS, unlock_config_info.generate(init_member_list, get_default_log_sync_replica_num(), - learner_list, init_config_version, unlock_meta)); - - std::vector config_info_list; - std::vector arg_list; - std::vector expect_ret_list; - std::vector expect_finished_list; - std::vector expect_whether_lock_list; - std::vector expect_lock_owner_list; - std::vector get_config_change_stat_lock_owner_list; - std::vector get_config_change_stat_lock_stat_list; - - //[try_lock] invalid argument - config_info_list.push_back(default_config_info); - arg_list.push_back(LogConfigChangeArgs(-1, lock_type_try_lock, TRY_LOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_INVALID_ARGUMENT); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(false); - expect_lock_owner_list.push_back(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER); - get_config_change_stat_lock_owner_list.push_back(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER); - get_config_change_stat_lock_stat_list.push_back(false); - - //[try_lock] OB_SUCCESS is_finish =false - config_info_list.push_back(default_config_info); - arg_list.push_back(LogConfigChangeArgs(2, lock_type_try_lock, TRY_LOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_SUCCESS); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(true); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(true); - - //[try_lock] OB_SUCCESS is_finish = true - config_info_list.push_back(locked_config_info); - arg_list.push_back(LogConfigChangeArgs(2, lock_type_try_lock, TRY_LOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_SUCCESS); - expect_finished_list.push_back(true); - expect_whether_lock_list.push_back(true); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(true); - - //[try_lock] OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT - config_info_list.push_back(locked_config_info); - arg_list.push_back(LogConfigChangeArgs(3, lock_type_try_lock, TRY_LOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(true); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(true); - - - //[try_unlock]: owner is someone else, return OB_SUCCESS - config_info_list.push_back(locked_config_info); - arg_list.push_back(LogConfigChangeArgs(1, lock_type_unlock, UNLOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_STATE_NOT_MATCH); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(true); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(true); - - //[try_unlock]: need unlock and unlocked successfully - config_info_list.push_back(locked_config_info); - arg_list.push_back(LogConfigChangeArgs(2, lock_type_unlock, UNLOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_SUCCESS); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(false); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(false); - - //[try_unlock]: is already unlock - config_info_list.push_back(unlock_config_info); - arg_list.push_back(LogConfigChangeArgs(2, lock_type_unlock, UNLOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_SUCCESS); - expect_finished_list.push_back(true); - expect_whether_lock_list.push_back(false); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(false); - - //[try_unlock]: bigger lock owner - config_info_list.push_back(unlock_config_info); - arg_list.push_back(LogConfigChangeArgs(3, lock_type_unlock, UNLOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_STATE_NOT_MATCH); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(false); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(false); - -//[try_unlock]: smaller lock owner - config_info_list.push_back(unlock_config_info); - arg_list.push_back(LogConfigChangeArgs(1, lock_type_unlock, UNLOCK_CONFIG_CHANGE)); - expect_ret_list.push_back(OB_STATE_NOT_MATCH); - expect_finished_list.push_back(false); - expect_whether_lock_list.push_back(false); - expect_lock_owner_list.push_back(2); - get_config_change_stat_lock_owner_list.push_back(2); - get_config_change_stat_lock_stat_list.push_back(false); - - - - for (int i = 0; i < arg_list.size(); ++i) { - PALF_LOG(INFO, "test_check_config_change_args for lock begin case", K(i+1)); - LogConfigMgr cm; - LogConfigVersion config_version, expect_config_version; - init_test_log_config_env(addr1, config_info_list[i], cm); - init_config_version.generate(cm.log_ms_meta_.proposal_id_, 1); - expect_config_version = init_config_version; - expect_config_version.inc_update_version(cm.log_ms_meta_.proposal_id_); - bool already_finished = false; - int tmp_ret = OB_SUCCESS; - LSN prev_lsn; - prev_lsn.val_ = PALF_INITIAL_LSN_VAL; - tmp_ret = cm.append_config_meta_(1, arg_list[i], already_finished); - config_version = cm.log_ms_meta_.curr_.config_.config_version_; - ASSERT_EQ(tmp_ret, expect_ret_list[i]) << "ret failed case: " << (i+1); - ASSERT_EQ(already_finished, expect_finished_list[i]) << "finished failed case:" << (i+1); - ASSERT_EQ(cm.log_ms_meta_.curr_.lock_meta_.is_locked(), expect_whether_lock_list[i])<< "lock_stat failed case:" << (i+1); - ASSERT_EQ(cm.log_ms_meta_.curr_.lock_meta_.lock_owner_, expect_lock_owner_list[i])<< "lock_owner failed case:" << (i+1); - - int64_t get_lock_owner = -1; - bool is_locked = false; - ASSERT_EQ(OB_SUCCESS, cm.get_config_change_lock_stat(get_lock_owner, is_locked)); - ASSERT_EQ(get_lock_owner, get_config_change_stat_lock_owner_list[i])<< "get_lock_owner failed case:" << (i+1); - ASSERT_EQ(is_locked, get_config_change_stat_lock_stat_list[i])<< "get_lock_stat failed case:" << (i+1); - if (tmp_ret == OB_SUCCESS) { - if (already_finished) { - ASSERT_EQ(config_version, init_config_version) << i; - } else { - ASSERT_EQ(config_version, expect_config_version) << i; - } - } - PALF_LOG(INFO, "test_check_config_change_args for lock end case", K(i+1)); - } -} - -TEST_F(TestLogConfigMgr, test_apply_config_meta) -{ - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - LogConfigInfoV2 default_config_info; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - ASSERT_EQ(OB_SUCCESS, default_config_info.generate(init_member_list, 1, learner_list, init_config_version)); - - struct ConfigCase { - LogConfigChangeArgs args; - int expect_ret; - bool expect_finished; - }; - const ConfigCase cases[] = { - { LogConfigChangeArgs(), OB_INVALID_ARGUMENT, false }, - { LogConfigChangeArgs(ObMember(addr1, -1), 1, ADD_MEMBER), OB_SUCCESS, true }, - { LogConfigChangeArgs(ObMember(addr4, -1), 0, ADD_LEARNER), OB_SUCCESS, false }, - { LogConfigChangeArgs(ObMember(addr1, -1), 0, ADD_LEARNER), OB_INVALID_ARGUMENT, false }, - { LogConfigChangeArgs(ObMember(addr1, -1), 1, REMOVE_MEMBER), OB_NOT_ALLOW_REMOVING_LEADER, false }, - }; - for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) { - LogConfigMgr cm; - bool already_finished = false; - init_test_log_config_env(addr1, default_config_info, cm); - LogConfigChangeArgs args = cases[i].args; - args.config_version_ = init_config_version; - const int tmp_ret = cm.append_config_meta_(1, args, already_finished); - EXPECT_EQ(tmp_ret, cases[i].expect_ret) << "case: " << (i + 1); - EXPECT_EQ(already_finished, cases[i].expect_finished) << "case: " << (i + 1); - } -} - -TEST_F(TestLogConfigMgr, test_submit_start_working_log) -{ - PALF_LOG(INFO, "test_submit_start_working_log begin case"); - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - LogConfigInfoV2 config_info; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - ASSERT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), - learner_list, init_config_version)); - { - LogConfigMgr cm; - LogConfigVersion sw_config_version; - init_test_log_config_env(addr1, config_info, cm, FOLLOWER); - EXPECT_EQ(OB_NOT_MASTER, cm.confirm_start_working_log(INIT_PROPOSAL_ID, INIT_ELE_EPOCH, sw_config_version)); - } - { - LogConfigMgr cm; - LogConfigVersion sw_config_version; - init_test_log_config_env(addr1, config_info, cm, LEADER, RECONFIRM); - EXPECT_EQ(OB_SUCCESS, cm.log_ms_meta_.curr_.config_.learnerlist_.add_learner(ObMember(addr4, -1))); - LSN prev_lsn; - prev_lsn.val_ = PALF_INITIAL_LSN_VAL; - mock_sw_->mock_last_submit_lsn_ = prev_lsn; - mock_sw_->mock_last_submit_end_lsn_ = prev_lsn; - mock_sw_->mock_last_submit_pid_ = INVALID_PROPOSAL_ID; - mock_mode_mgr_->mock_last_submit_mode_meta_.proposal_id_ = 1; - mock_sw_->mock_max_flushed_lsn_ = prev_lsn; - mock_sw_->mock_max_flushed_log_pid_ = INVALID_PROPOSAL_ID; - mock_mode_mgr_->mock_accepted_mode_meta_.proposal_id_ = 1; - LogConfigVersion expect_config_version; - ASSERT_EQ(OB_SUCCESS, expect_config_version.generate(1, 2)); - bool unused_bool = false; - EXPECT_EQ(OB_EAGAIN, cm.confirm_start_working_log(INIT_PROPOSAL_ID, INIT_ELE_EPOCH, sw_config_version)); - EXPECT_EQ(OB_SUCCESS, cm.ack_config_log(addr4, 1, expect_config_version, unused_bool)); - EXPECT_EQ(OB_SUCCESS, cm.ack_config_log(addr1, 1, expect_config_version, unused_bool)); - EXPECT_EQ(OB_SUCCESS, cm.confirm_start_working_log(INIT_PROPOSAL_ID, INIT_ELE_EPOCH, sw_config_version)); - EXPECT_EQ(0, cm.state_); - EXPECT_EQ(0, cm.resend_log_list_.get_member_number()); - } - PALF_LOG(INFO, "test_submit_start_working_log end case"); -} - -TEST_F(TestLogConfigMgr, test_submit_config_log) -{ - PALF_LOG(INFO, "test_submit_config_log begin case"); - LogConfigInfoV2 config_info; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_default_log_config_info(config_info, learner_list, init_config_version)); - { - LogConfigMgr cm; - init_test_log_config_env(addr1, config_info, cm, LEADER); - int64_t proposal_id = INVALID_PROPOSAL_ID; - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, proposal_id, 1, LSN(0), 1, cm.log_ms_meta_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, 1, 1, LSN(), 1, cm.log_ms_meta_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, 1, 1, LSN(0), INVALID_PROPOSAL_ID, cm.log_ms_meta_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, 1, 1, LSN(0), 1, LogConfigMeta())); - proposal_id = 1; - mock_sw_->mock_max_flushed_lsn_.val_ = PALF_INITIAL_LSN_VAL; - mock_sw_->mock_max_flushed_log_pid_ = 0; - mock_mode_mgr_->mock_accepted_mode_meta_.proposal_id_ = 1; - EXPECT_EQ(OB_SUCCESS, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, proposal_id, INVALID_PROPOSAL_ID, LSN(0), 1, cm.log_ms_meta_)); - EXPECT_GT(cm.last_submit_config_log_time_us_, 0); - } - { - LogConfigMgr cm; - init_test_log_config_env(addr1, config_info, cm); - init_config_version.generate(cm.log_ms_meta_.proposal_id_, 1); - const int64_t proposal_id = 1; - LogConfigVersion config_version; - LSN prev_lsn; - prev_lsn.val_ = PALF_INITIAL_LSN_VAL; - LogConfigChangeArgs args; - args.type_ = palf::ADD_LEARNER; - args.server_ = common::ObMember(addr4, -1); - args.config_version_ = init_config_version; - mock_sw_->mock_last_submit_lsn_ = prev_lsn; - mock_sw_->mock_last_submit_end_lsn_ = prev_lsn; - mock_sw_->mock_last_submit_pid_ = INVALID_PROPOSAL_ID; - mock_mode_mgr_->mock_last_submit_mode_meta_.proposal_id_ = 0; - mock_sw_->mock_max_flushed_lsn_.val_ = PALF_INITIAL_LSN_VAL; - mock_sw_->mock_max_flushed_log_pid_ = 0; - mock_mode_mgr_->mock_accepted_mode_meta_.proposal_id_ = 0; - EXPECT_EQ(OB_EAGAIN, cm.change_config_(args, proposal_id, INIT_ELE_EPOCH, config_version)); - EXPECT_EQ(OB_SUCCESS, cm.submit_config_log_(cm.log_ms_meta_.curr_.config_.log_sync_memberlist_, - proposal_id, 0, prev_lsn, 0, cm.log_ms_meta_)); - EXPECT_GT(cm.last_submit_config_log_time_us_, 0); - EXPECT_EQ(cm.state_, 1); - } - PALF_LOG(INFO, "test_submit_config_log end case"); -} - - -TEST_F(TestLogConfigMgr, test_after_flush_config_log) -{ - PALF_LOG(INFO, "test_after_flush_config_log begin case"); - LogConfigInfoV2 config_info; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_default_log_config_info(config_info, learner_list, init_config_version)); - // self is paxos member, retire parent - // child is not learner, retire child - LogConfigMgr cm; - init_test_log_config_env(addr1, config_info, cm); - cm.parent_ = addr2; - cm.register_time_us_ = 555; - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(LogLearner(addr3, 1))); - EXPECT_EQ(OB_SUCCESS, cm.after_flush_config_log(cm.log_ms_meta_.curr_.config_.config_version_)); - EXPECT_EQ(OB_INVALID_TIMESTAMP, cm.register_time_us_); - EXPECT_EQ(OB_INVALID_TIMESTAMP, cm.parent_keepalive_time_us_); - EXPECT_EQ(OB_INVALID_TIMESTAMP, cm.last_submit_register_req_time_us_); - EXPECT_FALSE(cm.parent_.is_valid()); - EXPECT_EQ(0, cm.children_.get_member_number()); - PALF_LOG(INFO, "test_after_flush_config_log end case"); -} - -TEST_F(TestLogConfigMgr, test_handle_register_parent_req) -{ - PALF_LOG(INFO, "test_handle_register_parent_req begin case"); - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - { - // non-learner (paxos member) register to leader is rejected (not in learner list) - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm); - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.handle_register_parent_req(LogLearner(addr3, 1), true)); - EXPECT_EQ(0, cm.children_.get_member_number()); - EXPECT_EQ(RegisterReturn::INVALID_REG_RET, mock_log_engine_->reg_ret_); - } - { - // duplicate register req - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm); - LogLearner child(addr4, 1); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(child)); - cm.all_learnerlist_.add_learner(ObMember(addr4, -1)); - child.register_time_us_ = 1; - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_req(child, true)); - EXPECT_EQ(RegisterReturn::REGISTER_DONE, mock_log_engine_->reg_ret_); - } - { - // register to leader - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm); - LogLearner child1(addr4, 1); - LogLearner exist_child(addr5, 1); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(child1)); - EXPECT_EQ(OB_SUCCESS, cm.children_.add_learner(exist_child)); - cm.all_learnerlist_.add_learner(ObMember(child1.get_server(), -1)); - cm.all_learnerlist_.add_learner(ObMember(exist_child.get_server(), -1)); - child1.register_time_us_ = 1; - // child1 already in children_ -> REGISTER_DONE (refresh) - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_req(child1, true)); - EXPECT_EQ(RegisterReturn::REGISTER_DONE, mock_log_engine_->reg_ret_); - mock_log_engine_->reset_register_parent_resp_ret(); - LogLearner child2(addr6, 1); - child2.register_time_us_ = 1; - cm.all_learnerlist_.add_learner(ObMember(child2.get_server(), -1)); - // new child with existing children -> redirected (REGISTER_CONTINUE), not added to leader's children - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_req(child2, true)); - EXPECT_EQ(RegisterReturn::REGISTER_CONTINUE, mock_log_engine_->reg_ret_); - EXPECT_FALSE(cm.children_.contains(child2)); - EXPECT_EQ(cm.self_, mock_log_engine_->parent_itself_.server_); - EXPECT_EQ(child2.register_time_us_, mock_log_engine_->parent_itself_.register_time_us_); - } - { - // register to follower - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm); - LogLearner child2(addr4, 1); - child2.register_time_us_ = 1; - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_req(child2, false)); - EXPECT_EQ(REGISTER_DONE, mock_log_engine_->reg_ret_); - EXPECT_TRUE(cm.children_.contains(addr4)); - mock_log_engine_->reset_register_parent_resp_ret(); - cm.children_.add_learner(LogLearner(addr5, 1)); - cm.children_.add_learner(LogLearner(addr6, 1)); - cm.children_.add_learner(LogLearner(addr7, 1)); - cm.children_.add_learner(LogLearner(addr8, 1)); - LogLearner child9(addr9, 1); - child9.register_time_us_ = 1; - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_req(child9, false)); - EXPECT_EQ(REGISTER_CONTINUE, mock_log_engine_->reg_ret_); - PALF_LOG(INFO, "trace", K(cm.children_), K(mock_log_engine_->candidate_list_)); - } - PALF_LOG(INFO, "test_handle_register_parent_req end case"); -} - -TEST_F(TestLogConfigMgr, test_handle_register_parent_resp) -{ - PALF_LOG(INFO, "test_handle_register_parent_resp begin case"); - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - { - // state not match - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm, FOLLOWER); - EXPECT_EQ(OB_STATE_NOT_MATCH, cm.handle_register_parent_resp(LogLearner(addr4, 1), LogCandidateList(), REGISTER_DONE)); - // register done - cm.register_time_us_ = 1; - cm.last_submit_register_req_time_us_ = 1; - LogLearner parent(addr4, 1); - parent.register_time_us_ = 1; - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_resp(parent, LogCandidateList(), REGISTER_DONE)); - EXPECT_EQ(OB_INVALID_TIMESTAMP, cm.last_submit_register_req_time_us_); - } - { - // register continue - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm, FOLLOWER); - cm.register_time_us_ = 1; - cm.last_submit_register_req_time_us_ = 1; - LogLearner parent(addr4, 1); - parent.register_time_us_ = 1; - // cnadidate list is empty - EXPECT_EQ(OB_ERR_UNEXPECTED, cm.handle_register_parent_resp(parent, LogCandidateList(), REGISTER_CONTINUE)); - LogCandidateList candidate_list; - candidate_list.add_learner(common::ObMember(addr2, -1)); - // register continue - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_resp(parent, candidate_list, REGISTER_CONTINUE)); - EXPECT_GT(cm.last_submit_register_req_time_us_, 1); - EXPECT_EQ(1, cm.register_time_us_); - } - { - // register not master - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr4, config_info, cm, FOLLOWER); - mock_state_mgr_->leader_ = addr1; - cm.register_time_us_ = 1; - cm.last_submit_register_req_time_us_ = 1; - LogLearner parent(addr4, 1); - parent.register_time_us_ = 1; - LogCandidateList candidate_list; - candidate_list.add_learner(common::ObMember(addr2, -1)); - // register continue - EXPECT_EQ(OB_SUCCESS, cm.handle_register_parent_resp(parent, candidate_list, REGISTER_NOT_MASTER)); - EXPECT_EQ(cm.last_submit_register_req_time_us_, 1); - EXPECT_EQ(cm.register_time_us_, 1); - } - PALF_LOG(INFO, "test_handle_register_parent_resp end case"); -} - -TEST_F(TestLogConfigMgr, test_check_children_health) -{ - PALF_LOG(INFO, "test_check_children_health begin case"); - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - { - // active leader - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr1, config_info, cm); - LogLearner timeout_child(addr4, 1); - LogLearner child_a(addr5, 1); - LogLearner normal_child(addr6, 1); - child_a.keepalive_ts_ = ObTimeUtility::current_time_ns(); - normal_child.keepalive_ts_ = ObTimeUtility::current_time_ns(); - cm.children_.add_learner(timeout_child); - cm.children_.add_learner(child_a); - cm.children_.add_learner(normal_child); - (void) cm.check_children_health(); - EXPECT_EQ(2, cm.children_.get_member_number()); - EXPECT_TRUE(cm.children_.contains(normal_child.server_)); - EXPECT_TRUE(cm.children_.contains(child_a.server_)); - } - { - // active follower - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - LogLearner timeout_child(addr4, 1); - LogLearner normal_child(addr6, 1); - normal_child.keepalive_ts_ = ObTimeUtility::current_time_ns(); - cm.children_.add_learner(timeout_child); - cm.children_.add_learner(normal_child); - (void) cm.check_children_health(); - LogLearnerList children; - EXPECT_EQ(OB_SUCCESS, cm.get_children_list(children)); - EXPECT_EQ(1, children.get_member_number()); - EXPECT_TRUE(children.contains(normal_child.server_)); - } - PALF_LOG(INFO, "test_check_children_health end case"); -} - -TEST_F(TestLogConfigMgr, test_check_parent_health) -{ - PALF_LOG(INFO, "test_check_parent_health begin case"); - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_log_sync_member_list(init_member_list)); - ASSERT_EQ(OB_SUCCESS, init_config_version.generate(1, 1)); - { - // registering timeout - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - cm.last_submit_register_req_time_us_ = 1; - mock_state_mgr_->leader_ = addr1; - EXPECT_EQ(OB_SUCCESS, cm.check_parent_health()); - EXPECT_GT(cm.register_time_us_, 1); - EXPECT_GT(cm.last_submit_register_req_time_us_, 1); - } - { - // first registration - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - mock_state_mgr_->leader_ = addr1; - EXPECT_EQ(OB_SUCCESS, cm.check_parent_health()); - EXPECT_GT(cm.register_time_us_, 1); - EXPECT_GT(cm.last_submit_register_req_time_us_, 1); - } - { - // parent timeout - LogConfigMgr cm; - LogConfigInfoV2 config_info; - EXPECT_EQ(OB_SUCCESS, config_info.generate(init_member_list, get_default_log_sync_replica_num(), learner_list, init_config_version)); - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - mock_state_mgr_->leader_ = addr1; - cm.parent_ = addr4; - EXPECT_EQ(OB_SUCCESS, cm.check_parent_health()); - EXPECT_GT(cm.register_time_us_, 1); - EXPECT_GT(cm.last_submit_register_req_time_us_, 1); - EXPECT_FALSE(cm.parent_.is_valid()); - } - PALF_LOG(INFO, "test_check_parent_health end case"); -} - -TEST_F(TestLogConfigMgr, test_handle_retire_msg) -{ - PALF_LOG(INFO, "test_handle_retire_msg end case"); - LogConfigInfoV2 config_info; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_default_log_config_info(config_info, learner_list, init_config_version)); - { - LogConfigMgr cm; - init_test_log_config_env(addr4, config_info, cm, FOLLOWER); - mock_state_mgr_->leader_ = addr1; - cm.parent_ = addr2; - cm.register_time_us_ = 100; - LogLearner parent(addr2, 100); - EXPECT_EQ(OB_SUCCESS, cm.handle_retire_child(parent)); - EXPECT_GT(cm.last_submit_register_req_time_us_, 0); - EXPECT_FALSE(cm.parent_.is_valid()); - } - { - LogConfigMgr cm; - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - const LogLearner child = LogLearner(addr4, 100); - cm.children_.add_learner(child); - EXPECT_EQ(OB_SUCCESS, cm.handle_retire_parent(child)); - EXPECT_EQ(0, cm.children_.get_member_number()); - } - { - // different register_time_us - LogConfigMgr cm; - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - LogLearner child = LogLearner(addr4, 100); - cm.children_.add_learner(child); - child.register_time_us_ = 200; - EXPECT_EQ(OB_SUCCESS, cm.handle_learner_keepalive_resp(child)); - EXPECT_EQ(child.keepalive_ts_, OB_INVALID_TIMESTAMP); - } - { - LogConfigMgr cm; - init_test_log_config_env(addr2, config_info, cm, FOLLOWER); - const LogLearner child = LogLearner(addr4, 100); - cm.children_.add_learner(child); - EXPECT_EQ(OB_INVALID_TIMESTAMP, child.keepalive_ts_); - EXPECT_EQ(OB_SUCCESS, cm.handle_learner_keepalive_resp(child)); - EXPECT_GT(cm.children_.get_learner(0).keepalive_ts_, 0); - } - PALF_LOG(INFO, "test_handle_retire_msg end case"); -} - -TEST_F(TestLogConfigMgr, test_init_by_default_config_meta) -{ - const int64_t init_log_proposal_id = 0; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - init_config_version.generate(init_log_proposal_id, 0); - init_config_info.generate(init_config_version); - LogConfigMeta log_config_meta; - LogConfigMgr cm; - EXPECT_EQ(OB_SUCCESS, log_config_meta.generate_for_default(init_log_proposal_id, - init_config_info, init_config_info)); - EXPECT_EQ(OB_SUCCESS, cm.init(1, addr1, log_config_meta, mock_log_engine_, - mock_sw_, mock_state_mgr_, mock_election_, - mock_mode_mgr_, mock_reconfirm_, mock_plugins_)); -} - -TEST_F(TestLogConfigMgr, test_too_many_learners) -{ - LogConfigInfoV2 config_info; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - ASSERT_EQ(OB_SUCCESS, build_default_log_config_info(config_info, learner_list, init_config_version)); - LogConfigMgr cm; - init_test_log_config_env(addr1, config_info, cm, LEADER); - - ObAddr::VER ip_type = ObAddr::IPV4; - const char* addr_str = "255.255.255.255"; - const int32_t port = INT32_MAX; - for (int i = 0; i < 200; i++) { - EXPECT_EQ(OB_SUCCESS, cm.log_ms_meta_.curr_.config_.learnerlist_.add_server(ObAddr(ip_type, addr_str, port - i))); - } - - // test add_learner - LogConfigChangeArgs args(ObMember(addr4, -1), 0, ADD_LEARNER); - bool unused_bool = false; - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.check_config_change_args_(args, unused_bool)); -} - -} // end namespace unittest -} // end namespace oceanbase - -using namespace oceanbase; -int main(int argc, char **argv) -{ - const std::string rm_base_dir_cmd = "rm -f test_log_config_mgr.log"; - system(rm_base_dir_cmd.c_str()); - OB_LOGGER.set_file_name("test_log_config_mgr.log", true); - OB_LOGGER.set_log_level("INFO"); - PALF_LOG(INFO, "begin unittest::test_log_config_mgr"); - ::testing::InitGoogleTest(&argc, argv); - ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); - return RUN_ALL_TESTS(); -} diff --git a/unittest/logservice/test_log_entry_and_group_entry.cpp b/unittest/logservice/test_log_entry_and_group_entry.cpp index 6c24f2cc2..5d4f46d47 100644 --- a/unittest/logservice/test_log_entry_and_group_entry.cpp +++ b/unittest/logservice/test_log_entry_and_group_entry.cpp @@ -58,7 +58,6 @@ TEST(TestLogGroupEntryHeader, test_group_entry_header_wrap_checksum) int64_t log_id = 1; LSN committed_lsn; committed_lsn.val_ = 1; - int64_t proposal_id = 1; int64_t log_checksum = 0; // test LogEntry and LogEntryHeader @@ -82,8 +81,8 @@ TEST(TestLogGroupEntryHeader, test_group_entry_header_wrap_checksum) group_log_data_len = group_log_len - group_entry_header_size; PALF_LOG(INFO, "before group_header generate", K(group_log_data_len), K(write_buf), K(sub_val)); EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, group_log_data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, group_log_data_len, + max_scn, log_id, committed_lsn, log_checksum)); } is_padding_log = true; @@ -94,8 +93,8 @@ TEST(TestLogGroupEntryHeader, test_group_entry_header_wrap_checksum) group_log_data_len = group_log_len - group_entry_header_size; PALF_LOG(INFO, "before group_header generate", K(group_log_data_len), K(write_buf), K(sub_val)); EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, group_log_data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, group_log_data_len, + max_scn, log_id, committed_lsn, log_checksum)); } group_log_len = group_entry_header_size + 2 * (log_entry_header_size + data_len); @@ -106,8 +105,8 @@ TEST(TestLogGroupEntryHeader, test_group_entry_header_wrap_checksum) group_log_data_len = group_log_len - group_entry_header_size; PALF_LOG(INFO, "before group_header generate", K(group_log_data_len), K(write_buf), K(sub_val)); EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, group_log_data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, group_log_data_len, + max_scn, log_id, committed_lsn, log_checksum)); } is_padding_log = true; @@ -118,14 +117,14 @@ TEST(TestLogGroupEntryHeader, test_group_entry_header_wrap_checksum) group_log_data_len = group_log_len - group_entry_header_size; PALF_LOG(INFO, "before group_header generate", K(group_log_data_len), K(write_buf), K(sub_val)); EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, group_log_data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, group_log_data_len, + max_scn, log_id, committed_lsn, log_checksum)); } is_padding_log = true; EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, group_log_data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, group_log_data_len, + max_scn, log_id, committed_lsn, log_checksum)); } TEST(TestLogGroupEntryHeader, test_log_group_entry_header) @@ -150,7 +149,6 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header) int64_t log_id = 1; LSN committed_lsn; committed_lsn.val_ = 1; - int64_t proposal_id = 1; int64_t log_checksum = 0; // test LogEntry and LogEntryHeader @@ -170,14 +168,14 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header) K(write_buf.get_total_size())); max_scn.reset(); EXPECT_EQ(OB_INVALID_ARGUMENT, - header.generate(false, is_padding_log, write_buf, data_len + log_entry_header_size, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, data_len + log_entry_header_size, + max_scn, log_id, committed_lsn, log_checksum)); max_scn.set_base(); int64_t defalut_acc = 10; min_timestamp = 1; EXPECT_EQ(OB_SUCCESS, - header.generate(false, is_padding_log, write_buf, data_len + log_entry_header_size, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(is_padding_log, write_buf, data_len + log_entry_header_size, + max_scn, log_id, committed_lsn, log_checksum)); header.update_accumulated_checksum(defalut_acc); header.update_header_checksum(); EXPECT_TRUE( @@ -195,13 +193,6 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header) EXPECT_TRUE( header1.check_integrity(buf + log_group_entry_header_size, data_len + log_entry_header_size)); EXPECT_TRUE(header1 == header); - int64_t new_proposal_id = 1000; - LSN new_lsn(10000); - EXPECT_EQ(OB_SUCCESS, header1.update_log_proposal_id(new_proposal_id)); - EXPECT_EQ(OB_SUCCESS, header1.update_committed_end_lsn(new_lsn)); - EXPECT_EQ(new_proposal_id, header1.get_log_proposal_id()); - EXPECT_EQ(new_lsn, header1.get_committed_end_lsn()); - header1.update_header_checksum(); EXPECT_TRUE(header1.check_header_integrity()); EXPECT_TRUE( header1.check_integrity(buf + log_group_entry_header_size, data_len + log_entry_header_size)); @@ -303,10 +294,9 @@ TEST(TestPaddingLogEntry, test_padding_log_entry) LogWriteBuf write_buf; EXPECT_EQ(OB_SUCCESS, write_buf.push_back(padding_buffer, padding_buffer_len)); - bool is_raw_write = false; bool is_padding_log = true; int64_t data_checksum = 0; - EXPECT_EQ(OB_SUCCESS, padding_group_entry_header.generate(is_raw_write, is_padding_log, write_buf, padding_data_len, padding_group_scn, 1, LSN(0), 1, data_checksum)); + EXPECT_EQ(OB_SUCCESS, padding_group_entry_header.generate(is_padding_log, write_buf, padding_data_len, padding_group_scn, 1, LSN(0), data_checksum)); padding_group_entry_header.update_accumulated_checksum(0); padding_group_entry_header.update_header_checksum(); padding_group_entry.header_ = padding_group_entry_header; @@ -406,220 +396,6 @@ TEST(TestPaddingLogEntry, test_generate_padding_log_entry) out_buf = nullptr; } -int16_t get_log_entry_header_parity_checksum(const LogEntryHeader &header) -{ - bool bool_ret = parity_check(reinterpret_cast(header.magic_)); - bool_ret ^= parity_check(reinterpret_cast(header.version_)); - bool_ret ^= parity_check(reinterpret_cast(header.log_size_)); - bool_ret ^= parity_check((header.scn_.get_val_for_logservice())); - bool_ret ^= parity_check(reinterpret_cast(header.data_checksum_)); - int64_t tmp_flag = (header.flag_ & ~(0x0001)); - bool_ret ^= parity_check(reinterpret_cast(tmp_flag)); - PALF_LOG(INFO, "get_log_entry_header_parity_checksum", K(header), K(tmp_flag)); - return bool_ret ? 1 : 0; -} - -int16_t get_log_group_entry_header_parity_checksum(const LogGroupEntryHeader &header) -{ - bool bool_ret = parity_check(reinterpret_cast(header.magic_)); - bool_ret ^= parity_check(reinterpret_cast(header.version_)); - bool_ret ^= parity_check(reinterpret_cast(header.group_size_)); - bool_ret ^= parity_check(reinterpret_cast(header.proposal_id_)); - bool_ret ^= parity_check(header.committed_end_lsn_.val_); - bool_ret ^= parity_check(header.max_scn_.get_val_for_logservice()); - bool_ret ^= parity_check(reinterpret_cast(header.accumulated_checksum_)); - bool_ret ^= parity_check(reinterpret_cast(header.log_id_)); - int64_t tmp_flag = (header.flag_ & ~(0x1)); - bool_ret ^= parity_check(reinterpret_cast(tmp_flag)); - return bool_ret ? 1 : 00; -} - -TEST(TestUpgraedCompatibility, test_log_entry_header) -{ - PALF_LOG(INFO, "TestUpgraedCompatibility"); - { - PALF_LOG(INFO, "case1. test_log_entry"); - { - // New version parsing old version data - LogEntryHeader ori_header; - constexpr int64_t data_len = 100; - char log_data[data_len] = "helloworld"; - EXPECT_EQ(OB_SUCCESS, ori_header.generate_header(log_data, data_len, share::SCN::min_scn())); - EXPECT_EQ(true, ori_header.check_header_integrity()); - EXPECT_EQ(true, ori_header.check_integrity(log_data, data_len)); - ori_header.version_ = LogEntryHeader::LOG_ENTRY_HEADER_VERSION; - ori_header.flag_ &= ~LogEntryHeader::CRC16_MASK; - ori_header.flag_ |= get_log_entry_header_parity_checksum(ori_header); - constexpr int64_t serialize_buf_len = data_len + sizeof(LogEntryHeader); - char serialize_buf[serialize_buf_len]; - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, ori_header.serialize(serialize_buf, serialize_buf_len, pos)); - pos = 0; - LogEntryHeader new_header; - EXPECT_EQ(OB_SUCCESS, new_header.deserialize(serialize_buf, serialize_buf_len, pos)); - EXPECT_EQ(true, new_header.check_header_integrity()); - EXPECT_EQ(true, new_header.check_integrity(log_data, data_len)); - EXPECT_EQ(new_header.flag_, ori_header.flag_); - EXPECT_EQ(new_header.data_checksum_, ori_header.data_checksum_); - EXPECT_EQ(new_header.version_, ori_header.version_); - // Validate padding log - ori_header.version_ = LogEntryHeader::LOG_ENTRY_HEADER_VERSION2; - ori_header.flag_ &= ~LogEntryHeader::CRC16_MASK; - ori_header.flag_ |= LogEntryHeader::PADDING_TYPE_MASK_VERSION2; - ori_header.update_header_checksum_(); - ori_header.update_header_checksum_(); - EXPECT_EQ(true, ori_header.is_padding_log_()); - EXPECT_EQ(true, ori_header.check_header_integrity()); - pos = 0; - EXPECT_EQ(OB_SUCCESS, ori_header.serialize(serialize_buf, serialize_buf_len, pos)); - pos = 0; - new_header.reset(); - EXPECT_EQ(OB_SUCCESS, new_header.deserialize(serialize_buf, serialize_buf_len, pos)); - EXPECT_EQ(true, new_header.check_header_integrity()); - EXPECT_EQ(true, new_header.is_padding_log_()); - - ori_header.version_ = LogEntryHeader::LOG_ENTRY_HEADER_VERSION; - ori_header.flag_ &= ~LogEntryHeader::CRC16_MASK; - ori_header.flag_ |= LogEntryHeader::PADDING_TYPE_MASK; - ori_header.flag_ |= get_log_entry_header_parity_checksum(ori_header); - - pos = 0; - EXPECT_EQ(OB_SUCCESS, ori_header.serialize(serialize_buf, serialize_buf_len, pos)); - new_header.reset(); - pos = 0; - EXPECT_EQ(OB_SUCCESS, new_header.deserialize(serialize_buf, serialize_buf_len, pos)); - EXPECT_EQ(true, new_header.check_header_integrity()); - EXPECT_EQ(true, new_header.is_padding_log_()); - PALF_LOG(INFO, "print new_header", K(new_header), "is_padding", new_header.is_padding_log_()); - EXPECT_EQ(new_header.flag_, ori_header.flag_); - EXPECT_EQ(new_header.data_checksum_, ori_header.data_checksum_); - EXPECT_EQ(new_header.version_, ori_header.version_); - } - PALF_LOG(INFO, "case2. test_log_group_entry"); - { - // Construct three LogEntry, their version are respectively 2 1 2 - // | GroupHeader version x | EntryHeader version 2 | EntryHeader version1 | EntryHeader version2 | - constexpr int64_t log_group_buf_len = 4096; - char log_group_buf[log_group_buf_len] = {0}; - memset(log_group_buf, log_group_buf_len, 'c'); - LogEntryHeader log_entry_header1; - constexpr int64_t data_len = 100; - EXPECT_EQ(OB_SUCCESS, log_entry_header1.generate_header(log_group_buf + sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader), data_len, share::SCN::min_scn())); - EXPECT_EQ(true, log_entry_header1.check_header_integrity()); - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, log_entry_header1.serialize(log_group_buf + sizeof(LogGroupEntryHeader), log_group_buf_len - sizeof(LogGroupEntryHeader), pos)); - - PALF_LOG(INFO, "print log_entry_header1", K(log_entry_header1)); - LogEntryHeader log_entry_header2; - pos = 0; - EXPECT_EQ(OB_SUCCESS, log_entry_header2.deserialize(log_group_buf + sizeof(LogGroupEntryHeader), log_group_buf_len - sizeof(LogGroupEntryHeader), pos)); - EXPECT_EQ(true, log_entry_header2.check_header_integrity()); - EXPECT_EQ(true, log_entry_header2.check_integrity(log_group_buf + sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader), data_len)); - EXPECT_EQ(log_entry_header2.flag_, log_entry_header1.flag_); - EXPECT_EQ(log_entry_header2.data_checksum_, log_entry_header1.data_checksum_); - EXPECT_EQ(log_entry_header2.version_, log_entry_header1.version_); - log_entry_header2.version_ = LogEntryHeader::LOG_ENTRY_HEADER_VERSION; - log_entry_header2.flag_ = 0; - log_entry_header2.flag_ |= get_log_entry_header_parity_checksum(log_entry_header2); - PALF_LOG(INFO, "print log_entry_header2", K(log_entry_header2)); - EXPECT_EQ(true, log_entry_header2.check_header_integrity()); - EXPECT_EQ(true, log_entry_header2.check_integrity(log_group_buf + sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader), data_len)); - pos = sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader) + data_len; - EXPECT_EQ(OB_SUCCESS, log_entry_header2.serialize(log_group_buf, log_group_buf_len, pos)); - - LogEntryHeader log_entry_header3; - pos = 0; - EXPECT_EQ(OB_SUCCESS, log_entry_header3.deserialize(log_group_buf + sizeof(LogGroupEntryHeader), log_group_buf_len - sizeof(LogGroupEntryHeader), pos)); - PALF_LOG(INFO, "print log_entry_header3", K(log_entry_header3)); - EXPECT_EQ(true, log_entry_header3.check_header_integrity()); - EXPECT_EQ(true, log_entry_header3.check_integrity(log_group_buf + sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader), data_len)); - EXPECT_EQ(log_entry_header3.flag_, log_entry_header1.flag_); - EXPECT_EQ(log_entry_header3.data_checksum_, log_entry_header1.data_checksum_); - EXPECT_EQ(log_entry_header3.version_, log_entry_header1.version_); - EXPECT_EQ(true, log_entry_header3.check_header_integrity()); - EXPECT_EQ(true, log_entry_header3.check_integrity(log_group_buf + sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader), data_len)); - EXPECT_EQ(true, log_entry_header3.check_header_integrity()); - pos = sizeof(LogGroupEntryHeader) + sizeof(LogEntryHeader) + data_len + sizeof(LogEntryHeader) + data_len; - EXPECT_EQ(OB_SUCCESS, log_entry_header3.serialize(log_group_buf, log_group_buf_len, pos)); - - pos = 0; - LogGroupEntryHeader group_header; - constexpr int64_t serialize_buf_len = 3*(data_len + sizeof(LogEntryHeader)); - LogWriteBuf write_buf; - write_buf.push_back(log_group_buf, serialize_buf_len + sizeof(LogGroupEntryHeader)); - int64_t log_checksum = 0; - EXPECT_EQ(OB_SUCCESS, - group_header.generate(false, false, write_buf, serialize_buf_len, - share::SCN::min_scn(), 1, LSN(0), 1, log_checksum)); - group_header.update_accumulated_checksum(1024); - group_header.update_header_checksum(); - group_header.update_header_checksum(); - EXPECT_EQ(true, group_header.check_header_integrity()); - EXPECT_EQ(true, group_header.check_integrity(log_group_buf+sizeof(LogGroupEntryHeader), serialize_buf_len)); - EXPECT_EQ(false, group_header.is_raw_write()); - EXPECT_EQ(false, group_header.is_padding_log()); - char log_group_serialize_buf[log_group_buf_len] = {0}; - pos = 0; - EXPECT_EQ(OB_SUCCESS, group_header.serialize(log_group_serialize_buf, log_group_buf_len, pos)); - LogGroupEntryHeader new_group_header; - pos = 0; - EXPECT_EQ(OB_SUCCESS, new_group_header.deserialize(log_group_serialize_buf, log_group_buf_len, pos)); - EXPECT_EQ(true, new_group_header.check_header_integrity()); - EXPECT_EQ(true, new_group_header.check_integrity(log_group_buf+sizeof(LogGroupEntryHeader), serialize_buf_len)); - EXPECT_EQ(false, new_group_header.is_raw_write()); - EXPECT_EQ(false, new_group_header.is_padding_log()); - - // pair - std::vector> input{{true, false}, {false, true}, {true, true}}; - - for (auto elem : input) { - LogGroupEntryHeader tmp_header = group_header; - tmp_header.reset(); - EXPECT_EQ(OB_SUCCESS, - tmp_header.generate(elem.first, elem.second, write_buf, serialize_buf_len, - share::SCN::min_scn(), 1, LSN(0), 1, log_checksum)); - tmp_header.update_accumulated_checksum(1023); - tmp_header.update_header_checksum(); - tmp_header.update_header_checksum(); - EXPECT_EQ(true, tmp_header.check_header_integrity()); - EXPECT_EQ(true, tmp_header.check_integrity(log_group_buf+sizeof(LogGroupEntryHeader), serialize_buf_len)); - EXPECT_EQ(elem.first, tmp_header.is_raw_write()); - EXPECT_EQ(LogGroupEntryHeader::PADDING_TYPE_MASK_VERSION2, tmp_header.get_padding_mask_()); - PALF_LOG(INFO, "runlin print tmp_header", K(tmp_header)); - EXPECT_EQ(LogGroupEntryHeader::RAW_WRITE_MASK_VERSION2, tmp_header.get_raw_write_mask_()); - // Construct the LogGroupEntryHeader of the old version - tmp_header.version_ = LogGroupEntryHeader::LOG_GROUP_ENTRY_HEADER_VERSION; - tmp_header.flag_ &= ~LogGroupEntryHeader::CRC16_MASK; - tmp_header.update_write_mode(elem.first); - tmp_header.flag_ |= (elem.second ? LogGroupEntryHeader::PADDING_TYPE_MASK : 0); - tmp_header.flag_ |= get_log_group_entry_header_parity_checksum(tmp_header); - EXPECT_EQ(elem.second, tmp_header.is_padding_log()); - EXPECT_EQ(elem.first, tmp_header.is_raw_write()); - EXPECT_EQ(true, tmp_header.check_header_integrity()); - EXPECT_EQ(true, tmp_header.check_integrity(log_group_buf+sizeof(LogGroupEntryHeader), serialize_buf_len)); - pos = 0; - EXPECT_EQ(OB_SUCCESS, tmp_header.serialize(log_group_serialize_buf, log_group_buf_len, pos)); - EXPECT_EQ(LogGroupEntryHeader::PADDING_TYPE_MASK, tmp_header.get_padding_mask_()); - EXPECT_EQ(LogGroupEntryHeader::RAW_WRITE_MASK, tmp_header.get_raw_write_mask_()); - - PALF_LOG(INFO, "test new binary parse old binary", K(tmp_header), K(elem.first), K(elem.second)); - // New version parsing old version data - LogGroupEntryHeader new_group_header; - pos = 0; - EXPECT_EQ(OB_SUCCESS, new_group_header.deserialize(log_group_serialize_buf, log_group_buf_len, pos)); - EXPECT_EQ(true, new_group_header.check_header_integrity()); - EXPECT_EQ(true, new_group_header.check_integrity(log_group_buf+sizeof(LogGroupEntryHeader), serialize_buf_len)); - EXPECT_EQ(elem.first, new_group_header.is_raw_write()); - EXPECT_EQ(elem.second, new_group_header.is_padding_log()); - // Parse the old version data - EXPECT_EQ(LogGroupEntryHeader::PADDING_TYPE_MASK, new_group_header.get_padding_mask_()); - EXPECT_EQ(LogGroupEntryHeader::RAW_WRITE_MASK, new_group_header.get_raw_write_mask_()); - } - - } - } -} - void bit_flip(uint8_t *ptr, int len, int bit_count) { // Ensure magic and version are not flipped @@ -694,11 +470,10 @@ TEST(TestBitFlip, test_log_group_entry_header) int64_t log_id = 1; LSN committed_lsn; committed_lsn.val_ = 1; - int64_t proposal_id = 1; int64_t log_checksum = 0; EXPECT_EQ(OB_SUCCESS, - header.generate(false, true, write_buf, data_len, - max_scn, log_id, committed_lsn, proposal_id, log_checksum)); + header.generate(true, write_buf, data_len, + max_scn, log_id, committed_lsn, log_checksum)); const int header_len = sizeof(LogGroupEntryHeader); header.update_header_checksum(); PALF_LOG(INFO, "origin header", K(header)); diff --git a/unittest/logservice/test_log_group_buffer.cpp b/unittest/logservice/test_log_group_buffer.cpp index 34bc48aad..73164115e 100644 --- a/unittest/logservice/test_log_group_buffer.cpp +++ b/unittest/logservice/test_log_group_buffer.cpp @@ -40,14 +40,10 @@ class TestLogGroupBuffer : public ::testing::Test virtual void SetUp(); virtual void TearDown(); protected: - int64_t palf_id_; LogGroupBuffer log_group_buffer_; }; -TestLogGroupBuffer::TestLogGroupBuffer() - : palf_id_(1) -{ -} +TestLogGroupBuffer::TestLogGroupBuffer() {} TestLogGroupBuffer::~TestLogGroupBuffer() { @@ -293,24 +289,12 @@ TEST_F(TestLogGroupBuffer, test_check_log_buf_wrapped) EXPECT_TRUE(is_wrapped); } -TEST_F(TestLogGroupBuffer, test_to_leader) -{ - EXPECT_EQ(OB_NOT_INIT, log_group_buffer_.to_leader()); - LSN start_lsn(100); - EXPECT_EQ(OB_SUCCESS, log_group_buffer_.init(start_lsn)); - EXPECT_EQ(OB_SUCCESS, log_group_buffer_.to_leader()); -#if LEADER_DEFAULT_GROUP_BUFFER_SIZE != FOLLOWER_DEFAULT_GROUP_BUFFER_SIZE - EXPECT_EQ(OB_STATE_NOT_MATCH, log_group_buffer_.to_leader()); -#endif -} - -TEST_F(TestLogGroupBuffer, test_to_follower) +TEST_F(TestLogGroupBuffer, test_activate) { - EXPECT_EQ(OB_NOT_INIT, log_group_buffer_.to_follower()); + EXPECT_EQ(OB_NOT_INIT, log_group_buffer_.activate()); LSN start_lsn(100); EXPECT_EQ(OB_SUCCESS, log_group_buffer_.init(start_lsn)); - EXPECT_EQ(OB_SUCCESS, log_group_buffer_.to_follower()); - EXPECT_EQ(OB_SUCCESS, log_group_buffer_.to_follower()); + EXPECT_EQ(OB_SUCCESS, log_group_buffer_.activate()); } TEST_F(TestLogGroupBuffer, test_read_data) diff --git a/unittest/logservice/test_log_meta.cpp b/unittest/logservice/test_log_meta.cpp index 4e598d677..de1afc008 100644 --- a/unittest/logservice/test_log_meta.cpp +++ b/unittest/logservice/test_log_meta.cpp @@ -20,183 +20,55 @@ #include #include "share/ob_cluster_version.h" -char **global_argv = nullptr; namespace oceanbase { namespace unittest { -using namespace common; using namespace palf; -TEST(TestLogMeta, test_log_meta) { int64_t proposal_id = INVALID_PROPOSAL_ID; proposal_id = 1; ObAddr addr(ObAddr::IPV4, "127.0.0.1", 4096); - - // Prepare meta - LogPrepareMeta log_prepare_meta1; - EXPECT_EQ(OB_SUCCESS, log_prepare_meta1.generate(LogVotedFor(), proposal_id)); - - // Membership meta - LogConfigMeta log_config_meta1; - ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 4096); - ObAddr addr2(ObAddr::IPV4, "127.0.0.1", 4097); - ObMember member1(addr1, 1); - ObMember member2(addr2, 1); - ObAddr addr3(ObAddr::IPV4, "127.0.0.1", 4098); - ObAddr addr4(ObAddr::IPV4, "127.0.0.1", 4099); - ObMember learner1(addr3, 1); - ObMember learner2(addr4, 1); - LSN prev_lsn; prev_lsn.val_ = 1; - int64_t prev_log_proposal_id = INVALID_PROPOSAL_ID; prev_log_proposal_id = 1; - int64_t prev_config_seq = 1; - int64_t prev_replica_num = 1; - ObMemberList prev_member_list; - prev_member_list.add_member(member1); - int64_t curr_config_seq = 1; - int64_t curr_replica_num = 1; - LSN curr_lsn; curr_lsn.val_ = 1; - int64_t curr_log_proposal_id = INVALID_PROPOSAL_ID; curr_log_proposal_id = 1; - ObMemberList curr_member_list; - curr_member_list.add_member(member2); - common::GlobalLearnerList prev_learner_list; - prev_learner_list.add_learner(learner1); - common::GlobalLearnerList curr_learner_list; - curr_learner_list.add_learner(learner2); - - LogConfigVersion prev_config_version; - LogConfigVersion curr_config_version; - EXPECT_EQ(OB_INVALID_ARGUMENT, prev_config_version.generate(INVALID_PROPOSAL_ID, -1)); - EXPECT_EQ(OB_SUCCESS, prev_config_version.generate(prev_log_proposal_id, prev_config_seq)); - EXPECT_EQ(OB_SUCCESS, curr_config_version.generate(curr_log_proposal_id, curr_config_seq)); - EXPECT_TRUE(curr_config_version.is_valid()); - - LogConfigInfoV2 prev_config_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, prev_config_info.generate(prev_member_list, -1, prev_learner_list, prev_config_version)); - EXPECT_EQ(OB_SUCCESS, prev_config_info.generate(prev_member_list, prev_replica_num, prev_learner_list, prev_config_version)); - LogConfigInfoV2 curr_config_info; - EXPECT_EQ(OB_SUCCESS, curr_config_info.generate(curr_member_list, curr_replica_num, curr_learner_list, curr_config_version)); - EXPECT_TRUE(curr_config_info.is_valid()); - EXPECT_EQ(OB_SUCCESS, log_config_meta1.generate(curr_log_proposal_id, prev_config_info, curr_config_info, - curr_log_proposal_id, LSN(0), curr_log_proposal_id)); - EXPECT_TRUE(log_config_meta1.is_valid()); - - // Snapshot meta - LogInfo log_info; log_info.generate_by_default(); - LogSnapshotMeta log_snapshot_meta1; - LSN lsn; lsn.val_ = 1; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_snapshot_meta1.generate(LSN(), log_info, LSN())); - EXPECT_EQ(OB_SUCCESS, log_snapshot_meta1.generate(lsn, log_info, lsn)); - EXPECT_TRUE(log_snapshot_meta1.is_valid()); - - // replica property meta - LogReplicaPropertyMeta replica_meta1; - EXPECT_EQ(OB_SUCCESS, replica_meta1.generate(true, LogReplicaType::NORMAL_REPLICA)); - - const int64_t init_log_proposal_id(0); - LogConfigMeta log_config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - EXPECT_EQ(OB_SUCCESS, init_config_version.generate(init_log_proposal_id, 0)); - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(init_config_version)); - log_config_meta.version_ = LogConfigMeta::LOG_CONFIG_META_VERSION_INC; - log_config_meta.proposal_id_ = init_log_proposal_id; - log_config_meta.curr_ = init_config_info; - log_config_meta.prev_ = init_config_info; - LogMeta log_meta1; - log_meta1.update_log_prepare_meta(log_prepare_meta1); - EXPECT_EQ(OB_SUCCESS, log_meta1.update_log_config_meta(log_config_meta)); - // Test invalid - EXPECT_FALSE(log_meta1.is_valid()); - - const int64_t BUFSIZE = 1 << 21; - char buf[BUFSIZE]; - int64_t pos = 0; - // Test serialize and deserialize - EXPECT_EQ(OB_SUCCESS, log_meta1.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, log_meta1.get_serialize_size()); - pos = 0; - LogMeta log_meta2; - EXPECT_EQ(OB_SUCCESS, log_meta2.deserialize(buf, BUFSIZE, pos)); - EXPECT_EQ(log_prepare_meta1.log_proposal_id_, - log_meta2.get_log_prepare_meta().log_proposal_id_); -} - -TEST(TestLogMeta, test_log_meta_generate) +PalfBaseInfo build_base_info() { - LogMeta meta1, meta2; - LSN prev_lsn(10000), lsn(20000); - int64_t init_pid(2); - share::SCN init_scn; - init_scn.convert_for_logservice(10); - int64_t init_cksum(10); PalfBaseInfo base_info; - LogInfo log_info; - log_info.log_id_ = 1; - log_info.scn_ = init_scn; - log_info.log_proposal_id_ = init_pid; - log_info.accum_checksum_ = init_cksum; - // invalid lsn - log_info.lsn_ = lsn; - base_info.curr_lsn_ = prev_lsn; - base_info.prev_log_info_ = log_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, meta1.generate_by_palf_base_info(base_info, AccessMode::APPEND, palf::NORMAL_REPLICA)); - // valid lsn - log_info.lsn_ = prev_lsn; - base_info.curr_lsn_ = lsn; - base_info.prev_log_info_ = log_info; - EXPECT_EQ(OB_SUCCESS, meta1.generate_by_palf_base_info(base_info, AccessMode::APPEND, palf::NORMAL_REPLICA)); - EXPECT_EQ(meta1.log_prepare_meta_.log_proposal_id_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_config_meta_.proposal_id_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_config_meta_.curr_.config_.config_version_.proposal_id_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_config_meta_.prev_.config_.config_version_.proposal_id_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_mode_meta_.proposal_id_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_mode_meta_.mode_version_, base_info.prev_log_info_.log_proposal_id_); - EXPECT_EQ(meta1.log_snapshot_meta_.base_lsn_, base_info.curr_lsn_); - EXPECT_EQ(meta1.log_snapshot_meta_.prev_log_info_, base_info.prev_log_info_); + base_info.prev_log_info_.log_id_ = 1; + base_info.prev_log_info_.lsn_ = LSN(10000); + base_info.prev_log_info_.scn_.convert_for_logservice(10); + base_info.prev_log_info_.accum_checksum_ = 10; + base_info.curr_lsn_ = LSN(20000); + return base_info; } -TEST(TestLogMeta, test_max_length_learner_list) +TEST(TestLogMeta, generate_single_log_meta) { - const int64_t init_log_proposal_id(0); - LogConfigMeta log_config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - EXPECT_EQ(OB_SUCCESS, init_config_version.generate(init_log_proposal_id, 0)); - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(init_config_version)); - log_config_meta.version_ = LogConfigMeta::LOG_CONFIG_META_VERSION_INC; - log_config_meta.proposal_id_ = init_log_proposal_id; - log_config_meta.curr_ = init_config_info; - log_config_meta.prev_ = init_config_info; - LogMeta log_meta1; - EXPECT_EQ(OB_SUCCESS, log_meta1.update_log_config_meta(log_config_meta)); - - ObAddr::VER ip_type = ObAddr::IPV4; - const char* addr_str = "255.255.255.255"; - const int32_t port = INT32_MAX; + const PalfBaseInfo base_info = build_base_info(); + LogMeta meta; + ASSERT_EQ(OB_SUCCESS, meta.generate_by_palf_base_info(base_info, AccessMode::APPEND)); + EXPECT_TRUE(meta.is_valid()); + EXPECT_EQ(base_info.curr_lsn_, meta.log_snapshot_meta_.base_lsn_); +} - for (int i = 0; i < OB_MAX_MEMBER_NUMBER; i++) { - log_meta1.log_config_meta_.curr_.config_.log_sync_memberlist_.add_member(ObMember(ObAddr(ip_type, addr_str, port - i), INT64_MAX)); - log_meta1.log_config_meta_.prev_.config_.log_sync_memberlist_.add_member(ObMember(ObAddr(ip_type, addr_str, port - i), INT64_MAX)); - log_meta1.log_config_meta_.curr_.config_.arbitration_member_ = ObMember(ObAddr(ip_type, addr_str, port - i), INT64_MAX); - log_meta1.log_config_meta_.prev_.config_.arbitration_member_ = ObMember(ObAddr(ip_type, addr_str, port - i), INT64_MAX); - } - PALF_LOG(INFO, "IPV4 base serialize size", "size", log_meta1.get_serialize_size()); +TEST(TestLogMeta, serialize_single_log_meta) +{ + LogMeta meta; + ASSERT_EQ(OB_SUCCESS, meta.generate_by_palf_base_info(build_base_info(), AccessMode::APPEND)); + char buf[4096]; + int64_t pos = 0; + ASSERT_EQ(OB_SUCCESS, meta.serialize(buf, sizeof(buf), pos)); + EXPECT_EQ(meta.get_serialize_size(), pos); - for (int i = 0; i < 60; i++) { - log_meta1.log_config_meta_.curr_.config_.learnerlist_.add_server(ObAddr(ip_type, addr_str, port - i)); - } - const int64_t member_serialize_size = ObMember(ObAddr(ObAddr::IPV4, addr_str, port), INT64_MAX).get_serialize_size(); - const int64_t addr_serialize_size = ObAddr(ObAddr::IPV4, addr_str, port).get_serialize_size(); - PALF_LOG(INFO, "60 IPV4 serialize size", "size", log_meta1.get_serialize_size(), K(member_serialize_size), K(addr_serialize_size)); -} -} + LogMeta restored; + pos = 0; + ASSERT_EQ(OB_SUCCESS, restored.deserialize(buf, sizeof(buf), pos)); + EXPECT_TRUE(restored.is_valid()); + EXPECT_EQ(meta.log_mode_meta_.access_mode_, restored.log_mode_meta_.access_mode_); } +} // namespace unittest +} // namespace oceanbase + int main(int argc, char **argv) { OB_LOGGER.set_file_name("test_log_meta.log", true); - OB_LOGGER.set_log_level("INFO"); - PALF_LOG(INFO, "begin unittest::test_log_meta"); ::testing::InitGoogleTest(&argc, argv); oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); - oceanbase::ObClusterVersion::get_instance().update_cluster_version(CLUSTER_CURRENT_VERSION); return RUN_ALL_TESTS(); } diff --git a/unittest/logservice/test_log_meta_entry.cpp b/unittest/logservice/test_log_meta_entry.cpp index 46f44df78..5739055ea 100644 --- a/unittest/logservice/test_log_meta_entry.cpp +++ b/unittest/logservice/test_log_meta_entry.cpp @@ -15,123 +15,54 @@ */ #include -#include "logservice/palf/log_meta_entry.h" // LogMetaEntry -#include "logservice/palf/log_meta.h" // LogMeta - -#include +#include "logservice/palf/log_meta_entry.h" +#include "logservice/palf/log_meta.h" #include "share/ob_cluster_version.h" namespace oceanbase { namespace unittest { - -using namespace common; using namespace palf; -TEST(TestLogMetaEntry, test_log_meta_entry) +TEST(TestLogMetaEntry, serialize_single_log_meta_entry) { - int64_t proposal_id = INVALID_PROPOSAL_ID; - proposal_id = 1; - ObAddr addr(ObAddr::IPV4, "127.0.0.1", 4096); - // Stream meta - - // Prepare meta - LogPrepareMeta log_prepare_meta1; - EXPECT_EQ(OB_SUCCESS, log_prepare_meta1.generate(LogVotedFor(), proposal_id)); - - // Membership meta - LogConfigMeta log_config_meta1; - ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 4096); - ObAddr addr2(ObAddr::IPV4, "127.0.0.1", 4097); - ObAddr addr3(ObAddr::IPV4, "127.0.0.1", 4098); - ObAddr addr4(ObAddr::IPV4, "127.0.0.1", 4099); - ObMember learner1(addr3, 1); - ObMember learner2(addr4, 1); - ObMember member1(addr1, 1); - ObMember member2(addr2, 1); - LSN prev_lsn; prev_lsn.val_ = 1; - int64_t prev_log_proposal_id = INVALID_PROPOSAL_ID; prev_log_proposal_id = 1; - int64_t prev_config_seq = 1; - int64_t prev_replica_num = 1; - ObMemberList prev_member_list; - prev_member_list.add_member(member1); - int64_t curr_config_seq = 1; - int64_t curr_replica_num = 1; - LSN curr_lsn; curr_lsn.val_ = 1; - int64_t curr_log_proposal_id = INVALID_PROPOSAL_ID; curr_log_proposal_id = 1; - ObMemberList curr_member_list; - curr_member_list.add_member(member2); - common::GlobalLearnerList prev_learner_list; - prev_learner_list.add_learner(learner1); - common::GlobalLearnerList curr_learner_list; - curr_learner_list.add_learner(learner2); - - LogConfigInfoV2 prev_config_info; - LogConfigInfoV2 curr_config_info; - LogConfigVersion prev_config_version; - LogConfigVersion curr_config_version; - - EXPECT_EQ(OB_SUCCESS, prev_config_version.generate(prev_log_proposal_id, prev_config_seq)); - EXPECT_EQ(OB_SUCCESS, curr_config_version.generate(curr_log_proposal_id, curr_config_seq)); - EXPECT_EQ(OB_SUCCESS, prev_config_info.generate(prev_member_list, prev_replica_num, prev_learner_list, prev_config_version)); - EXPECT_EQ(OB_SUCCESS, curr_config_info.generate(curr_member_list, curr_replica_num, prev_learner_list, curr_config_version)); - EXPECT_EQ(OB_SUCCESS, log_config_meta1.generate(curr_log_proposal_id, prev_config_info, curr_config_info, - curr_log_proposal_id, LSN(0), curr_log_proposal_id)); - - // Snapshot meta - LogSnapshotMeta log_snapshot_meta1; - LSN lsn; lsn.val_ = 1; - LogInfo prev_log_info; prev_log_info.generate_by_default(); - EXPECT_EQ(OB_SUCCESS, log_snapshot_meta1.generate(lsn, prev_log_info, lsn)); - - LogReplicaPropertyMeta replica_meta1; - replica_meta1.generate(true, LogReplicaType::NORMAL_REPLICA); - - LogMeta log_meta1; - // Test invalid - EXPECT_FALSE(log_meta1.is_valid()); - - const int64_t BUFSIZE = 1 << 21; - char buf[BUFSIZE]; - int64_t pos = 0; - // Test serialize and deserialize - EXPECT_EQ(OB_SUCCESS, log_meta1.update_log_config_meta(log_config_meta1)); - EXPECT_EQ(OB_SUCCESS, log_meta1.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, log_meta1.get_serialize_size()); - - LogMetaEntryHeader log_meta_entry_header; - EXPECT_EQ(OB_SUCCESS, log_meta_entry_header.generate(buf, pos)); - EXPECT_TRUE(log_meta_entry_header.check_integrity(buf, pos)); - - // Test invalid argument - LogMetaEntry log_meta_entry1; - EXPECT_FALSE(log_meta_entry1.is_valid()); - EXPECT_EQ(OB_SUCCESS, log_meta_entry1.generate(log_meta_entry_header, buf)); - EXPECT_TRUE(log_meta_entry1.is_valid()); - - // Test serialize and deserialize - char buf1[BUFSIZE]; - pos = 0; - EXPECT_EQ(OB_SUCCESS, log_meta_entry1.serialize(buf1, BUFSIZE, pos)); - EXPECT_EQ(pos, log_meta_entry1.get_serialize_size()); - EXPECT_TRUE(log_meta_entry1.check_integrity()); - LogMetaEntry log_meta_entry2; - pos = 0; - EXPECT_EQ(OB_SUCCESS, log_meta_entry2.deserialize(buf1, BUFSIZE, pos)); - EXPECT_TRUE(log_meta_entry2.check_integrity()); + PalfBaseInfo base_info; + base_info.prev_log_info_.log_id_ = 1; + base_info.prev_log_info_.lsn_ = LSN(10000); + base_info.prev_log_info_.scn_.convert_for_logservice(10); + base_info.prev_log_info_.accum_checksum_ = 10; + base_info.curr_lsn_ = LSN(20000); + + LogMeta meta; + ASSERT_EQ(OB_SUCCESS, meta.generate_by_palf_base_info(base_info, AccessMode::APPEND)); + char meta_buf[4096]; + int64_t meta_pos = 0; + ASSERT_EQ(OB_SUCCESS, meta.serialize(meta_buf, sizeof(meta_buf), meta_pos)); + + LogMetaEntryHeader header; + ASSERT_EQ(OB_SUCCESS, header.generate(meta_buf, meta_pos)); + ASSERT_TRUE(header.check_integrity(meta_buf, meta_pos)); + + LogMetaEntry entry; + ASSERT_EQ(OB_SUCCESS, entry.generate(header, meta_buf)); + char entry_buf[8192]; + int64_t entry_pos = 0; + ASSERT_EQ(OB_SUCCESS, entry.serialize(entry_buf, sizeof(entry_buf), entry_pos)); + + LogMetaEntry restored; + entry_pos = 0; + ASSERT_EQ(OB_SUCCESS, restored.deserialize(entry_buf, sizeof(entry_buf), entry_pos)); + EXPECT_TRUE(restored.check_integrity()); } -} // end of unittest -} // end of oceanbase +} // namespace unittest +} // namespace oceanbase int main(int argc, char **argv) { OB_LOGGER.set_file_name("test_log_meta_entry.log", true); - OB_LOGGER.set_log_level("INFO"); - PALF_LOG(INFO, "begin unittest::test_log_meta_entry"); ::testing::InitGoogleTest(&argc, argv); oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); - oceanbase::ObClusterVersion::get_instance().update_cluster_version(CLUSTER_CURRENT_VERSION); return RUN_ALL_TESTS(); } diff --git a/unittest/logservice/test_log_meta_info.cpp b/unittest/logservice/test_log_meta_info.cpp index f6256464d..5bcdc5441 100755 --- a/unittest/logservice/test_log_meta_info.cpp +++ b/unittest/logservice/test_log_meta_info.cpp @@ -15,7 +15,7 @@ */ #define private public -#include "logservice/palf/log_meta_info.h" // LogPrepareMeta... +#include "logservice/palf/log_meta_info.h" #undef private #include #include "share/ob_cluster_version.h" @@ -24,320 +24,9 @@ namespace oceanbase { using namespace common; using namespace palf; -constexpr int64_t palf::LogConfigMeta::LOG_CONFIG_META_VERSION; -constexpr int64_t palf::LogConfigMeta::LOG_CONFIG_META_VERSION_INC; namespace unittest { -TEST(TestLogMetaInfos, test_log_voted_for) -{ - common::ObAddr addr(ObAddr::IPV4, "127.0.0.1", 1234); - LogVotedFor voted_for(addr); - PALF_LOG(INFO, "voted_for" , K(voted_for)); - char buf[1024]; - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, voted_for.serialize(buf, 1024, pos)); - LogVotedFor voted_for1; - pos = 0; - EXPECT_EQ(OB_SUCCESS, voted_for1.deserialize(buf, 1024, pos)); - ObAddr addr2; - addr2.version_ = ObAddr::IPV4; - addr2.ip_.v4_ = voted_for1.voted_for_[0]; - addr2.port_ = voted_for1.voted_for_[1]; - EXPECT_EQ(addr, addr2); - ObAddr addr_v6; - addr_v6.version_ = ObAddr::IPV6; - addr_v6.set_ipv6_addr("fe80::754d:f834:4606:13", 1234); - LogVotedFor voted_for_v6(addr_v6); - pos = 0; - voted_for_v6.serialize(buf, 1024, pos); - LogVotedFor voted_for_v6_1; - pos = 0; - voted_for_v6_1.deserialize(buf, 1024, pos); - ObAddr addr_v6_1; - addr_v6_1.version_ = ObAddr::IPV6; - memcpy(addr_v6_1.ip_.v6_, voted_for_v6.voted_for_, 2*sizeof(int64_t)); - addr_v6_1.port_ = voted_for_v6.voted_for_[2]; - EXPECT_EQ(addr_v6_1, addr_v6); - PALF_LOG(INFO, "voted_for_v6", K(voted_for_v6), K(addr_v6)); -} - -TEST(TestLogMetaInfos, test_log_prepare_meta) -{ - static const int64_t BUFSIZE = 1 << 21; - char buf[BUFSIZE]; - int64_t proposal_id = INVALID_PROPOSAL_ID; - proposal_id = 1; - LogPrepareMeta log_prepare_meta1; - // Test invalid argument - EXPECT_FALSE(log_prepare_meta1.is_valid()); - common::ObAddr addr(ObAddr::IPV4, "127.0.0.1", 1234); - LogVotedFor voted_for(addr); - EXPECT_EQ(OB_SUCCESS, log_prepare_meta1.generate(voted_for, proposal_id)); - EXPECT_TRUE(log_prepare_meta1.is_valid()); - - // Test serialize and deserialize - EXPECT_TRUE(log_prepare_meta1.is_valid()); - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, log_prepare_meta1.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, log_prepare_meta1.get_serialize_size()); - pos = 0; - LogPrepareMeta log_prepare_meta2; - EXPECT_EQ(OB_SUCCESS, log_prepare_meta2.deserialize(buf, BUFSIZE, pos)); - EXPECT_EQ(log_prepare_meta1.log_proposal_id_, log_prepare_meta1.log_proposal_id_); -} - -TEST(TestLogMetaInfos, test_log_config_meta) -{ - //test LogLockMeta - LogLockMeta lock_meta; - EXPECT_EQ(false, lock_meta.is_valid()); - lock_meta.version_ = LogLockMeta::LOG_LOCK_META_VERSION; - EXPECT_EQ(true, lock_meta.is_valid()); - EXPECT_EQ(false, lock_meta.is_lock_owner_valid()); - - lock_meta.lock_owner_ = 1; - lock_meta.lock_type_ = LOCK_NOTHING; - EXPECT_EQ(false, lock_meta.is_valid()); - EXPECT_EQ(OB_INVALID_ARGUMENT, lock_meta.generate(1, LOCK_NOTHING)); - EXPECT_EQ(OB_INVALID_ARGUMENT, lock_meta.generate(-1, LOCK_PAXOS_MEMBER_CHANGE)); - EXPECT_EQ(OB_SUCCESS, lock_meta.generate(1, LOCK_PAXOS_MEMBER_CHANGE)); - EXPECT_EQ(true, lock_meta.is_locked()); - EXPECT_EQ(true, lock_meta.is_lock_owner_valid()); - - lock_meta.unlock(); - EXPECT_EQ(false, lock_meta.is_locked()); - EXPECT_EQ(LOCK_NOTHING, lock_meta.lock_type_); - EXPECT_EQ(1, lock_meta.lock_owner_); - EXPECT_EQ(true, lock_meta.version_ == LogLockMeta::LOG_LOCK_META_VERSION); - EXPECT_EQ(true, lock_meta.is_lock_owner_valid()); - - lock_meta.reset_as_unlocked(); - EXPECT_EQ(false, lock_meta.is_locked()); - EXPECT_EQ(LOCK_NOTHING, lock_meta.lock_type_); - EXPECT_EQ(-1, lock_meta.lock_owner_); - EXPECT_EQ(true, lock_meta.version_ == LogLockMeta::LOG_LOCK_META_VERSION); - EXPECT_EQ(false, lock_meta.is_lock_owner_valid()); - - LogLockMeta new_lock_meta; - new_lock_meta = lock_meta; - EXPECT_EQ(true, new_lock_meta == lock_meta); - - const int64_t SER_BUF_SIZE = 512; - char ser_buf[SER_BUF_SIZE] = {0}; - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, lock_meta.serialize(ser_buf, SER_BUF_SIZE, pos)); - int64_t new_pos = 0; - EXPECT_EQ(OB_SUCCESS, new_lock_meta.deserialize(ser_buf, pos, new_pos)); - EXPECT_EQ(pos, new_pos); - EXPECT_EQ(new_lock_meta, lock_meta); - //end of test LogLockMeta - - static const int64_t BUFSIZE = 1 << 21; - ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 4096); - ObAddr addr2(ObAddr::IPV4, "127.0.0.1", 4097); - ObAddr addr3(ObAddr::IPV4, "127.0.0.1", 4098); - ObAddr addr4(ObAddr::IPV4, "127.0.0.1", 4099); - ObMember member1(addr1, 1); - ObMember member2(addr2, 1); - ObMember learner1(addr3, 1); - ObMember learner2(addr4, 1); - LSN prev_lsn; prev_lsn.val_ = 1; - int64_t prev_log_proposal_id = 1; - int64_t prev_config_seq = 1; - int64_t prev_replica_num = 1; - ObMemberList prev_member_list; - prev_member_list.add_member(member1); - int64_t curr_config_seq = 1; - int64_t curr_replica_num = 1; - LSN curr_lsn; curr_lsn.val_ = 1; - int64_t curr_log_proposal_id = 1; - ObMemberList curr_member_list; - curr_member_list.add_member(member2); - common::GlobalLearnerList prev_learner_list; - prev_learner_list.add_learner(learner1); - common::GlobalLearnerList curr_learner_list; - curr_learner_list.add_learner(learner2); - - LogConfigVersion prev_config_version; - LogConfigVersion curr_config_version; - LogConfigInfoV2 prev_config_info; - LogConfigInfoV2 curr_config_info; - LogConfigInfo old_config_info; - LogConfigInfoV2 new_config_info; - - // log barrier - const int64_t barrier_log_proposal_id = 3; - const LSN barrier_lsn = LSN(300); - const int64_t barrier_mode_pid = 4; - - // Test default version - LogConfigMeta default_config_meta; - EXPECT_EQ(OB_SUCCESS, default_config_meta.generate_for_default(curr_log_proposal_id, prev_config_info, curr_config_info)); - EXPECT_TRUE(default_config_meta.is_valid()); - EXPECT_EQ(palf::LogConfigMeta::LOG_CONFIG_META_VERSION_INC, default_config_meta.version_); - - // Test invalid argument - LogConfigMeta log_config_meta; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_config_meta.generate(curr_log_proposal_id, prev_config_info, curr_config_info, - barrier_log_proposal_id, barrier_lsn, barrier_mode_pid)); - EXPECT_FALSE(log_config_meta.is_valid()); - - EXPECT_EQ(OB_SUCCESS, prev_config_version.generate(prev_log_proposal_id, prev_config_seq)); - EXPECT_EQ(OB_SUCCESS, curr_config_version.generate(curr_log_proposal_id, curr_config_seq)); - lock_meta.reset(); - EXPECT_EQ(OB_INVALID_ARGUMENT, prev_config_info.generate(prev_member_list, prev_replica_num, prev_learner_list, prev_config_version, lock_meta)); - lock_meta.reset_as_unlocked(); - ASSERT_EQ(OB_SUCCESS, prev_config_info.generate(prev_member_list, prev_replica_num, prev_learner_list, prev_config_version, lock_meta)); - EXPECT_EQ(OB_SUCCESS, prev_config_info.generate(prev_member_list, prev_replica_num, prev_learner_list, prev_config_version)); - EXPECT_EQ(OB_SUCCESS, curr_config_info.generate(curr_member_list, curr_replica_num, curr_learner_list, curr_config_version)); - EXPECT_TRUE(curr_config_info.is_valid()); - EXPECT_TRUE(prev_config_info.is_valid()); - - // test lists overlap -// { -// LogConfigInfo invalid_info = curr_config_info; //TODO(yaoying): fix it -// invalid_info.learnerlist_.add_learner(member2); -// EXPECT_FALSE(invalid_info.is_valid()); -// } - - EXPECT_EQ(OB_SUCCESS, old_config_info.generate(curr_member_list, curr_replica_num, curr_learner_list, curr_config_version)); - EXPECT_EQ(OB_SUCCESS, new_config_info.generate(old_config_info)); - EXPECT_EQ(true, new_config_info.lock_meta_.is_valid()); - EXPECT_EQ(false, new_config_info.lock_meta_.is_locked()); - - // test basic serialization - { - char buf[BUFSIZE]; - LogConfigMeta log_config_meta1; - EXPECT_EQ(OB_SUCCESS, log_config_meta1.generate(curr_log_proposal_id, prev_config_info, curr_config_info, - barrier_log_proposal_id, barrier_lsn, barrier_mode_pid)); - EXPECT_TRUE(log_config_meta1.is_valid()); - - // Test serialzie and deserialize - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, log_config_meta1.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, log_config_meta1.get_serialize_size()); - pos = 0; - LogConfigMeta log_config_meta2; - EXPECT_EQ(OB_SUCCESS, log_config_meta2.deserialize(buf, BUFSIZE, pos)); - EXPECT_TRUE(log_config_meta1.proposal_id_ == log_config_meta2.proposal_id_); - EXPECT_TRUE(log_config_meta1.prev_ == - log_config_meta2.prev_); - EXPECT_TRUE(log_config_meta1.curr_ == - log_config_meta2.curr_); - EXPECT_TRUE(log_config_meta1.prev_log_proposal_id_ == log_config_meta2.prev_log_proposal_id_); - EXPECT_TRUE(log_config_meta1.prev_lsn_ == log_config_meta2.prev_lsn_); - EXPECT_TRUE(log_config_meta1.prev_mode_pid_ == log_config_meta2.prev_mode_pid_); - PALF_LOG(INFO, "trace", K(log_config_meta1), K(log_config_meta2)); - } - // test compatibility (new code deserializes old data) - { - char buf[BUFSIZE]; - LogConfigMeta log_config_meta1; - EXPECT_EQ(OB_SUCCESS, log_config_meta1.generate(curr_log_proposal_id, prev_config_info, curr_config_info, - barrier_log_proposal_id, barrier_lsn, barrier_mode_pid)); - EXPECT_TRUE(log_config_meta1.is_valid()); - // assign old version - log_config_meta1.version_ = LogConfigMeta::LOG_CONFIG_META_VERSION; - - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, log_config_meta1.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, log_config_meta1.get_serialize_size()); - pos = 0; - LogConfigMeta log_config_meta2; - EXPECT_EQ(OB_SUCCESS, log_config_meta2.deserialize(buf, BUFSIZE, pos)); - EXPECT_TRUE(log_config_meta1.proposal_id_ == log_config_meta2.proposal_id_); - EXPECT_TRUE(log_config_meta1.prev_ == - log_config_meta2.prev_); - EXPECT_TRUE(log_config_meta1.curr_ == - log_config_meta2.curr_); - EXPECT_EQ(log_config_meta2.prev_log_proposal_id_, INVALID_PROPOSAL_ID); - EXPECT_FALSE(log_config_meta2.prev_lsn_.is_valid()); - EXPECT_EQ(log_config_meta2.prev_mode_pid_, INVALID_PROPOSAL_ID); - PALF_LOG(INFO, "trace", K(log_config_meta1), K(log_config_meta2)); - } -} - -TEST(TestLogMetaInfos, test_log_config_info_convert) -{ - ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 4096); - ObAddr addr2(ObAddr::IPV4, "127.0.0.1", 4097); - ObAddr addr3(ObAddr::IPV4, "127.0.0.1", 4098); - ObAddr addr4(ObAddr::IPV4, "127.0.0.1", 4099); - ObAddr addr5(ObAddr::IPV4, "127.0.0.1", 4100); - ObAddr addr6(ObAddr::IPV4, "127.0.0.1", 4101); - int64_t curr_config_seq = 1; - LogConfigVersion curr_config_version; - int64_t curr_log_proposal_id = INVALID_PROPOSAL_ID; curr_log_proposal_id = 1; - EXPECT_EQ(OB_SUCCESS, curr_config_version.generate(curr_log_proposal_id, curr_config_seq)); -#if OB_MAX_MEMBER_NUMBER > 1 - // 2F1A, 2 degraded learners - { - LogConfigInfoV2 curr_config_info_v2; - - LogConfigInfo &curr_config_info = curr_config_info_v2.config_; - int64_t log_sync_replica_num = 2; - ObMemberList log_sync_member_list; - log_sync_member_list.add_member(ObMember(addr1, 1)); - log_sync_member_list.add_member(ObMember(addr2, 1)); - common::ObMember arb_replica = ObMember(addr3, 1); - ObMemberList expected_paxos_memberlist = log_sync_member_list; - expected_paxos_memberlist.add_member(arb_replica); - const int64_t expected_paxos_replica_num = log_sync_replica_num + 1; - - common::GlobalLearnerList curr_learner_list; - curr_learner_list.add_learner(ObMember(addr4, 1)); - curr_config_info.degraded_learnerlist_.add_learner(ObMember(addr5, 1)); - curr_config_info.degraded_learnerlist_.add_learner(ObMember(addr6, 1)); - GlobalLearnerList expected_learner_list; - expected_learner_list.append(curr_learner_list); - expected_learner_list.append(curr_config_info.degraded_learnerlist_); - - EXPECT_EQ(OB_SUCCESS, curr_config_info.generate(log_sync_member_list, log_sync_replica_num, curr_learner_list, curr_config_version)); - curr_config_info.arbitration_member_ = arb_replica; - - common::ObMemberList result_memberlist; - int64_t result_replica_num; - GlobalLearnerList result_learners; - EXPECT_EQ(OB_SUCCESS, curr_config_info_v2.convert_to_complete_config(result_memberlist, result_replica_num, result_learners)); - EXPECT_EQ(result_replica_num, expected_paxos_replica_num); - EXPECT_TRUE(result_memberlist.member_addr_equal(expected_paxos_memberlist)); - EXPECT_EQ(3, result_memberlist.get_member_number()); - EXPECT_TRUE(result_learners.learner_addr_equal(expected_learner_list)); - EXPECT_EQ(3, result_learners.get_member_number()); - } -#else - // single-member mode: verify convert_to_complete_config with sync member and learners - { - LogConfigInfoV2 curr_config_info_v2; - LogConfigInfo &curr_config_info = curr_config_info_v2.config_; - const int64_t log_sync_replica_num = 1; - ObMemberList log_sync_member_list; - log_sync_member_list.add_member(ObMember(addr1, 1)); - common::GlobalLearnerList curr_learner_list; - curr_learner_list.add_learner(ObMember(addr4, 1)); - curr_config_info.degraded_learnerlist_.add_learner(ObMember(addr5, 1)); - GlobalLearnerList expected_learner_list; - expected_learner_list.append(curr_learner_list); - expected_learner_list.append(curr_config_info.degraded_learnerlist_); - - EXPECT_EQ(OB_SUCCESS, curr_config_info.generate(log_sync_member_list, log_sync_replica_num, - curr_learner_list, curr_config_version)); - - common::ObMemberList result_memberlist; - int64_t result_replica_num = 0; - GlobalLearnerList result_learners; - EXPECT_EQ(OB_SUCCESS, curr_config_info_v2.convert_to_complete_config(result_memberlist, result_replica_num, result_learners)); - EXPECT_EQ(log_sync_replica_num, result_replica_num); - EXPECT_TRUE(result_memberlist.member_addr_equal(log_sync_member_list)); - EXPECT_EQ(1, result_memberlist.get_member_number()); - EXPECT_TRUE(result_learners.learner_addr_equal(expected_learner_list)); - EXPECT_EQ(2, result_learners.get_member_number()); - } -#endif -} - TEST(TestLogMetaInfos, test_log_mode_meta) { static const int64_t BUFSIZE = 1 << 21; @@ -349,11 +38,9 @@ TEST(TestLogMetaInfos, test_log_mode_meta) share::SCN invalid_scn; // Test invalid argument EXPECT_FALSE(log_mode_meta1.is_valid()); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, 1, AccessMode::INVALID_ACCESS_MODE, share::SCN::min_scn())); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, 1, AccessMode::APPEND, invalid_scn)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, INVALID_PROPOSAL_ID, AccessMode::APPEND, share::SCN::min_scn())); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(INVALID_PROPOSAL_ID, 1, AccessMode::APPEND, share::SCN::min_scn())); - EXPECT_EQ(OB_SUCCESS, log_mode_meta1.generate(1, 1, AccessMode::APPEND, share::SCN::min_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(AccessMode::INVALID_ACCESS_MODE, share::SCN::min_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(AccessMode::APPEND, invalid_scn)); + EXPECT_EQ(OB_SUCCESS, log_mode_meta1.generate(AccessMode::APPEND, share::SCN::min_scn())); EXPECT_TRUE(log_mode_meta1.is_valid()); // Test serialize and deserialize @@ -363,9 +50,7 @@ TEST(TestLogMetaInfos, test_log_mode_meta) pos = 0; LogModeMeta log_mode_meta2; EXPECT_EQ(OB_SUCCESS, log_mode_meta2.deserialize(buf, BUFSIZE, pos)); - const bool equal = (log_mode_meta1.mode_version_ == log_mode_meta2.mode_version_ && - log_mode_meta1.proposal_id_ == log_mode_meta2.proposal_id_ && - log_mode_meta1.access_mode_ == log_mode_meta2.access_mode_ && + const bool equal = (log_mode_meta1.access_mode_ == log_mode_meta2.access_mode_ && log_mode_meta1.ref_scn_ == log_mode_meta2.ref_scn_); EXPECT_TRUE(equal); } @@ -381,13 +66,11 @@ TEST(TestLogMetaInfos, test_log_snapshot_meta) LogInfo prev_log_info; prev_log_info.generate_by_default(); // Test invalid argument EXPECT_FALSE(log_snapshot_meta1.is_valid()); - PALF_LOG(INFO, "compatibility test case 1", K(log_snapshot_meta1)); LSN base_lsn(2*PALF_BLOCK_SIZE), prev_tail_lsn(PALF_BLOCK_SIZE); EXPECT_EQ(OB_SUCCESS, log_snapshot_meta1.generate(base_lsn, prev_log_info, prev_tail_lsn)); - PALF_LOG(INFO, "compatibility test case 1, trace", K(log_snapshot_meta1)); EXPECT_EQ(true, log_snapshot_meta1.prev_log_info_.is_valid()); EXPECT_EQ(true, log_snapshot_meta1.prev_log_tail_lsn_.is_valid()); - EXPECT_EQ(LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION_V2, log_snapshot_meta1.version_); + EXPECT_EQ(LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION, log_snapshot_meta1.version_); LogInfo result_log_info; LSN input_curr_lsn = prev_tail_lsn; LSN output_prev_tail_lsn; @@ -397,25 +80,6 @@ TEST(TestLogMetaInfos, test_log_snapshot_meta) // return OB_ENTRY_NOT_EXIST when base_lsn is not same as prev_tail_lsn of prev_log_info EXPECT_EQ(OB_ENTRY_NOT_EXIST, log_snapshot_meta1.get_prev_log_info(base_lsn, result_log_info, output_prev_tail_lsn)); EXPECT_TRUE(log_snapshot_meta1.is_valid()); - // Validate SN version - { - // for creating a new log stream, prev_tail_lsn must be valid, here we verify the upgrade scenario, prev_tail_lsn_sn is invalid - LogSnapshotMeta log_snapshot_meta_sn; - LSN base_lsn_sn(2*PALF_BLOCK_SIZE), prev_tail_lsn_sn; - EXPECT_EQ(OB_SUCCESS, log_snapshot_meta_sn.generate(base_lsn_sn, prev_log_info, prev_tail_lsn_sn)); - PALF_LOG(INFO, "compatibility test case sn, trace", K(log_snapshot_meta_sn)); - EXPECT_EQ(true, log_snapshot_meta_sn.prev_log_info_.is_valid()); - EXPECT_EQ(false, log_snapshot_meta_sn.prev_log_tail_lsn_.is_valid()); - EXPECT_EQ(LogSnapshotMeta::LOG_SNAPSHOT_META_VERSION_V2, log_snapshot_meta_sn.version_); - LogInfo result_log_info; - LSN input_curr_lsn = base_lsn_sn; - LSN output_prev_tail_lsn; - // In the SN upgrade scenario, after advancing the snapshot meta, prev_log_info is expected to no longer be relied upon, get_prev_log_info returns OB_ENTRY_NOT_EXIST - EXPECT_EQ(OB_ENTRY_NOT_EXIST, log_snapshot_meta_sn.get_prev_log_info(input_curr_lsn, result_log_info, output_prev_tail_lsn)); - EXPECT_EQ(result_log_info.is_valid(), false); - EXPECT_EQ(output_prev_tail_lsn.is_valid(), false); - } - // Test serialize and deserialize int64_t pos = 0; EXPECT_EQ(OB_SUCCESS, log_snapshot_meta1.serialize(buf, BUFSIZE, pos)); @@ -427,72 +91,6 @@ TEST(TestLogMetaInfos, test_log_snapshot_meta) log_snapshot_meta2.base_lsn_); } -TEST(TestLogReplicaPropertyMeta, test_log_replica_property_meta) -{ - static const int64_t BUFSIZE = 1 << 21; - char buf[BUFSIZE]; - LogReplicaPropertyMeta replica_meta; - LogReplicaPropertyMeta replica_meta1; - EXPECT_FALSE(replica_meta.is_valid()); - EXPECT_EQ(OB_SUCCESS, replica_meta.generate(true, LogReplicaType::NORMAL_REPLICA)); - replica_meta.reset(); - EXPECT_FALSE(replica_meta.is_valid()); - EXPECT_EQ(OB_SUCCESS, replica_meta.generate(false, LogReplicaType::NORMAL_REPLICA)); - replica_meta1 = replica_meta; - EXPECT_EQ(replica_meta.version_, replica_meta1.version_); - EXPECT_EQ(replica_meta.allow_vote_, replica_meta1.allow_vote_); - - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, replica_meta.generate(true, LogReplicaType::NORMAL_REPLICA)); - EXPECT_EQ(OB_SUCCESS, replica_meta.serialize(buf, BUFSIZE, pos)); - EXPECT_EQ(pos, replica_meta.get_serialize_size()); - pos = 0; - EXPECT_EQ(OB_SUCCESS, replica_meta1.deserialize(buf, BUFSIZE, pos)); - EXPECT_EQ(replica_meta.version_, replica_meta1.version_); - EXPECT_EQ(replica_meta.allow_vote_, replica_meta1.allow_vote_); -} - -TEST(TestLogMetaInfos, test_log_config_version) -{ - LogConfigVersion cv1; - EXPECT_FALSE(cv1.is_valid()); - EXPECT_EQ(OB_INVALID_ARGUMENT, cv1.generate(0, -1)); - EXPECT_EQ(OB_NOT_INIT, cv1.inc_update_version(1)); - EXPECT_EQ(OB_SUCCESS, cv1.generate(1, 1)); - EXPECT_EQ(OB_INVALID_ARGUMENT, cv1.inc_update_version(0)); - EXPECT_EQ(OB_SUCCESS, cv1.inc_update_version(1)); - EXPECT_EQ(2, cv1.config_seq_); - cv1.reset(); - EXPECT_FALSE(cv1.is_valid()); - { - LogConfigVersion cv2; - LogConfigVersion cv3; - EXPECT_FALSE(cv2 < cv3); - EXPECT_FALSE(cv2 > cv3); - EXPECT_TRUE(cv2 <= cv3); - EXPECT_TRUE(cv2 >= cv3); - } - { - LogConfigVersion cv2; - LogConfigVersion cv3; - EXPECT_EQ(OB_SUCCESS, cv2.generate(1, 1)); - EXPECT_TRUE(cv2 > cv3); - EXPECT_TRUE(cv2 >= cv3); - EXPECT_FALSE(cv2 == cv3); - } - { - LogConfigVersion cv2; - LogConfigVersion cv3; - EXPECT_EQ(OB_SUCCESS, cv2.generate(1, 1)); - EXPECT_EQ(OB_SUCCESS, cv3.generate(1, 1)); - EXPECT_TRUE(cv2 == cv3); - EXPECT_EQ(OB_SUCCESS, cv2.inc_update_version(1)); - EXPECT_TRUE(cv2 > cv3); - EXPECT_EQ(OB_SUCCESS, cv3.inc_update_version(2)); - EXPECT_TRUE(cv2 < cv3); - } -} - } // end of unittest } // end of oceanbase @@ -506,4 +104,3 @@ int main(int args, char **argv) oceanbase::ObClusterVersion::get_instance().update_cluster_version(CLUSTER_CURRENT_VERSION); return RUN_ALL_TESTS(); } - diff --git a/unittest/logservice/test_log_mode_mgr.cpp b/unittest/logservice/test_log_mode_mgr.cpp index 6eef5871d..d81f2f791 100644 --- a/unittest/logservice/test_log_mode_mgr.cpp +++ b/unittest/logservice/test_log_mode_mgr.cpp @@ -15,12 +15,11 @@ */ #include +#include "share/ob_cluster_version.h" #define private public #include "logservice/palf/log_mode_mgr.h" -#include "mock_logservice_container/mock_election.h" #include "mock_logservice_container/mock_log_sliding_window.h" #include "mock_logservice_container/mock_log_engine.h" -#include "mock_logservice_container/mock_log_config_mgr.h" #undef private namespace oceanbase { @@ -31,69 +30,26 @@ using namespace share; using namespace palf; const ObAddr addr1(ObAddr::IPV4, "127.0.0.1", 1000); -const ObAddr addr2(ObAddr::IPV4, "127.0.0.2", 1000); -const ObAddr addr3(ObAddr::IPV4, "127.0.0.3", 1000); -const ObAddr addr4(ObAddr::IPV4, "127.0.0.4", 1000); -const ObAddr addr5(ObAddr::IPV4, "127.0.0.5", 1000); -const ObAddr addr6(ObAddr::IPV4, "127.0.0.6", 1000); -const ObAddr addr7(ObAddr::IPV4, "127.0.0.7", 1000); -const ObAddr addr8(ObAddr::IPV4, "127.0.0.8", 1000); -const ObAddr addr9(ObAddr::IPV4, "127.0.0.9", 1000); class TestLogModeMgr : public ::testing::Test { public: TestLogModeMgr() { - mock_election_ = OB_NEW(mockelection::MockElection, "TestLog"); mock_state_mgr_ = OB_NEW(MockLogStateMgr, "TestLog"); mock_sw_ = OB_NEW(MockLogSlidingWindow, "TestLog"); mock_log_engine_ = OB_NEW(MockLogEngine, "TestLog"); - mock_config_mgr_ = OB_NEW(MockLogConfigMgr, "TestLog"); } ~TestLogModeMgr() { - OB_DELETE(MockElection, "TestLog", mock_election_); OB_DELETE(MockLogStateMgr, "TestLog", mock_state_mgr_); OB_DELETE(MockLogSlidingWindow, "TestLog", mock_sw_); OB_DELETE(MockLogEngine, "TestLog", mock_log_engine_); - OB_DELETE(MockLogConfigMgr, "TestLog", mock_config_mgr_); - } - void init_test_mode_mgr_env(const common::ObAddr &self, - const LogModeMeta &mode_meta, - LogModeMgr &mode_mgr, - common::ObRole role = LEADER, - ObReplicaState state = ACTIVE) - { - mode_mgr.destroy(); - mock_state_mgr_->role_ = role; - mock_state_mgr_->state_ = state; - mock_state_mgr_->leader_ = (LEADER == role)? self: ObAddr(); - const int64_t init_pid = 1; - const int64_t init_replica_num = 1; - LogConfigInfo config_info; - LogConfigMeta config_meta; - LogConfigInfoV2 default_config_info; - ObMemberList init_member_list; - GlobalLearnerList learner_list; - LogConfigVersion init_config_version; - EXPECT_EQ(OB_SUCCESS, init_config_version.generate(init_pid, 1)); - EXPECT_EQ(OB_SUCCESS, init_member_list.add_server(self)); - EXPECT_EQ(OB_SUCCESS, default_config_info.generate(init_member_list, init_replica_num, learner_list, init_config_version)); - EXPECT_EQ(OB_SUCCESS, config_meta.generate(init_pid, default_config_info, default_config_info, init_pid, LSN(0), init_pid)); - mock_config_mgr_->destroy(); - mock_config_mgr_->log_ms_meta_ = config_meta; - EXPECT_TRUE(mode_meta.is_valid()); - PALF_LOG(INFO, "init_test_mode_mgr_env", K(role), K(state), K(mock_state_mgr_->leader_), K(mock_state_mgr_->role_)); - mock_log_engine_->reset_register_parent_resp_ret(); - EXPECT_EQ(OB_SUCCESS, mode_mgr.init(1, self, mode_meta, mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); } public: - mockelection::MockElection *mock_election_; palf::MockLogStateMgr *mock_state_mgr_; palf::MockLogSlidingWindow *mock_sw_; palf::MockLogEngine *mock_log_engine_; - palf::MockLogConfigMgr *mock_config_mgr_; }; TEST_F(TestLogModeMgr, test_init) @@ -102,242 +58,23 @@ TEST_F(TestLogModeMgr, test_init) LogModeMgr mode_mgr; LogModeMeta valid_meta, invalid_meta; ObAddr invalid_addr; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, share::SCN::base_scn())); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(-1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, invalid_addr, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, addr1, invalid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, addr1, valid_meta, - NULL, mock_log_engine_, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, NULL, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, NULL, mock_sw_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, NULL)); - EXPECT_EQ(OB_SUCCESS, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); - EXPECT_EQ(OB_INIT_TWICE, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(AccessMode::APPEND, share::SCN::base_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(invalid_addr, valid_meta)); + EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(addr1, invalid_meta)); + EXPECT_EQ(OB_SUCCESS, mode_mgr.init(addr1, valid_meta)); + EXPECT_EQ(OB_INIT_TWICE, mode_mgr.init(addr1, valid_meta)); PALF_LOG(INFO, "test_init case"); } TEST_F(TestLogModeMgr, test_can_interface) { PALF_LOG(INFO, "test_can_interface case"); - std::vector access_mode_list; - std::vector state_list; - std::vector can_append_list; - std::vector can_raw_list; - - access_mode_list.push_back(AccessMode::APPEND); - state_list.push_back(ModeChangeState::MODE_INIT); - can_append_list.push_back(true); - can_raw_list.push_back(false); - - access_mode_list.push_back(AccessMode::APPEND); - state_list.push_back(ModeChangeState::MODE_PREPARE); - can_append_list.push_back(false); - can_raw_list.push_back(false); - - access_mode_list.push_back(AccessMode::APPEND); - state_list.push_back(ModeChangeState::MODE_ACCEPT); - can_append_list.push_back(true); - can_raw_list.push_back(false); - - access_mode_list.push_back(AccessMode::RAW_WRITE); - state_list.push_back(ModeChangeState::MODE_INIT); - can_append_list.push_back(false); - can_raw_list.push_back(true); - - access_mode_list.push_back(AccessMode::RAW_WRITE); - state_list.push_back(ModeChangeState::MODE_PREPARE); - can_append_list.push_back(false); - can_raw_list.push_back(false); - - access_mode_list.push_back(AccessMode::RAW_WRITE); - state_list.push_back(ModeChangeState::MODE_ACCEPT); - can_append_list.push_back(false); - can_raw_list.push_back(true); - - for (int i = 0; i < access_mode_list.size(); i++) { - LogModeMgr mode_mgr; - mode_mgr.applied_mode_meta_.access_mode_ = access_mode_list[i]; - mode_mgr.state_ = state_list[i]; - EXPECT_EQ(can_append_list[i], mode_mgr.can_append()); - EXPECT_EQ(can_raw_list[i], mode_mgr.can_raw_write()); - } + LogModeMgr mode_mgr; + mode_mgr.applied_mode_meta_.access_mode_ = AccessMode::APPEND; + EXPECT_TRUE(mode_mgr.can_append()); PALF_LOG(INFO, "test_can_interface case"); } -TEST_F(TestLogModeMgr, test_receive_mode_meta) -{ - PALF_LOG(INFO, "test_receive_mode_meta case"); - int64_t pid = 1; - LogModeMeta valid_meta, invalid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, share::SCN::base_scn())); - bool has_accepted = false; - - { - // not init, invalid arg - LogModeMgr mode_mgr; - EXPECT_FALSE(mode_mgr.can_receive_mode_meta(pid, valid_meta, has_accepted)); - - EXPECT_EQ(OB_SUCCESS, mode_mgr.init(1, addr1, valid_meta, - mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); - - EXPECT_FALSE(mode_mgr.can_receive_mode_meta(INVALID_PROPOSAL_ID, valid_meta, has_accepted)); - EXPECT_FALSE(mode_mgr.can_receive_mode_meta(pid, invalid_meta, has_accepted)); - } - { - // check role - LogModeMgr mode_mgr; - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - EXPECT_FALSE(mode_mgr.can_receive_mode_meta(pid, valid_meta, has_accepted)); - } - { - // has accepted - LogModeMgr mode_mgr; - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, FOLLOWER, ACTIVE); - EXPECT_TRUE(mode_mgr.can_receive_mode_meta(pid, valid_meta, has_accepted)); - EXPECT_TRUE(has_accepted); - } - { - // can accpet - LogModeMgr mode_mgr; - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, FOLLOWER, ACTIVE); - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(2, 2, AccessMode::APPEND, share::SCN::base_scn())); - EXPECT_TRUE(mode_mgr.can_receive_mode_meta(pid, valid_meta, has_accepted)); - EXPECT_FALSE(has_accepted); - EXPECT_EQ(OB_SUCCESS, mode_mgr.receive_mode_meta(addr2, 2, false, valid_meta)); - EXPECT_EQ(valid_meta.mode_version_, mode_mgr.last_submit_mode_meta_.mode_version_); - EXPECT_GT(valid_meta.mode_version_, mode_mgr.accepted_mode_meta_.mode_version_); - EXPECT_GT(valid_meta.mode_version_, mode_mgr.applied_mode_meta_.mode_version_); - EXPECT_EQ(OB_SUCCESS, mode_mgr.after_flush_mode_meta(false, valid_meta)); - EXPECT_EQ(valid_meta.mode_version_, mode_mgr.accepted_mode_meta_.mode_version_); - // receive applied mode_meta - EXPECT_EQ(OB_SUCCESS, mode_mgr.after_flush_mode_meta(true, valid_meta)); - EXPECT_EQ(valid_meta.mode_version_, mode_mgr.applied_mode_meta_.mode_version_); - } -} - -TEST_F(TestLogModeMgr, test_can_change_access_mode) -{ - PALF_LOG(INFO, "test_can_change_access_mode case"); - const int64_t mode_version = 1; - const AccessMode access_mode = AccessMode::APPEND; - const share::SCN ref_scn = share::SCN::base_scn(); - LogModeMeta valid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, share::SCN::base_scn())); - { - LogModeMgr mode_mgr; - // role check - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, FOLLOWER, ACTIVE); - EXPECT_EQ(OB_NOT_MASTER, mode_mgr.can_change_access_mode_(mode_version)); - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, FOLLOWER, PENDING); - EXPECT_EQ(OB_NOT_MASTER, mode_mgr.can_change_access_mode_(mode_version)); - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, RECONFIRM); - EXPECT_EQ(OB_NOT_MASTER, mode_mgr.change_access_mode(mode_version, access_mode, ref_scn)); - // mode_version check - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - EXPECT_EQ(OB_STATE_NOT_MATCH, mode_mgr.can_change_access_mode_(2)); - // normal - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - EXPECT_EQ(OB_SUCCESS, mode_mgr.can_change_access_mode_(mode_version)); - } - { - // test see newer log - LogModeMgr mode_mgr; - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - mode_mgr.state_ = MODE_PREPARE; - mode_mgr.new_proposal_id_ = 2; - mode_mgr.leader_epoch_ = mock_state_mgr_->get_leader_epoch(); - mode_mgr.local_max_log_pid_ = 1; - mode_mgr.local_max_lsn_ = LSN(PALF_INITIAL_LSN_VAL); - mode_mgr.majority_cnt_ = 2; - mode_mgr.max_majority_accepted_pid_ = 1; - mode_mgr.max_majority_lsn_ = LSN(PALF_INITIAL_LSN_VAL); - mode_mgr.ack_list_.reset(); - EXPECT_EQ(OB_SUCCESS, mode_mgr.follower_list_.add_server(addr2)); - EXPECT_EQ(OB_SUCCESS, mode_mgr.handle_prepare_response(addr2, 2, 2, LSN(PALF_INITIAL_LSN_VAL + 1), valid_meta)); - EXPECT_EQ(OB_NOT_MASTER, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); - EXPECT_EQ(MODE_PREPARE, mode_mgr.state_); - } - PALF_LOG(INFO, "test_can_change_access_mode case"); -} - -TEST_F(TestLogModeMgr, test_can_switch_access_mode) -{ - PALF_LOG(INFO, "test_can_switch_access_mode case"); - std::vector src_access_mode_list; - std::vector dst_access_mode_list; - std::vector ret_list; - - src_access_mode_list.push_back(AccessMode::APPEND); - dst_access_mode_list.push_back(AccessMode::APPEND); - ret_list.push_back(false); - - src_access_mode_list.push_back(AccessMode::APPEND); - dst_access_mode_list.push_back(AccessMode::RAW_WRITE); - ret_list.push_back(true); - - src_access_mode_list.push_back(AccessMode::RAW_WRITE); - dst_access_mode_list.push_back(AccessMode::RAW_WRITE); - ret_list.push_back(false); - - src_access_mode_list.push_back(AccessMode::RAW_WRITE); - dst_access_mode_list.push_back(AccessMode::APPEND); - ret_list.push_back(true); - - for (int i = 0; i < src_access_mode_list.size(); i++) { - const bool expected = (ret_list[i] == can_switch_access_mode_(src_access_mode_list[i], dst_access_mode_list[i])); - EXPECT_TRUE(expected); - } - PALF_LOG(INFO, "test_can_switch_access_mode case"); -} - -TEST_F(TestLogModeMgr, test_change_access_mode) -{ - PALF_LOG(INFO, "test_change_access_mode case"); - const int64_t mode_version = 1; - const AccessMode access_mode = AccessMode::APPEND; - const share::SCN ref_scn = share::SCN::base_scn(); - LogModeMeta valid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, share::SCN::base_scn())); - { - // not init, invalid args - LogModeMgr mode_mgr; - share::SCN invalid_scn; - EXPECT_EQ(OB_NOT_INIT, mode_mgr.change_access_mode(mode_version, access_mode, ref_scn)); - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.change_access_mode(-1, access_mode, ref_scn)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.change_access_mode(mode_version, AccessMode::INVALID_ACCESS_MODE, ref_scn)); - EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.change_access_mode(mode_version, access_mode, invalid_scn)); - } - { - LogModeMgr mode_mgr; - init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); - // single replica reaches prepare majority immediately and switches to accept - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); - EXPECT_EQ(MODE_ACCEPT, mode_mgr.state_); - // switch to accept - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); - EXPECT_EQ(MODE_ACCEPT, mode_mgr.state_); - EXPECT_EQ(OB_SUCCESS, mode_mgr.ack_mode_meta(addr1, 2)); - // should not reach majority before leader's AccessMode is flushed - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); - // self is flushed - EXPECT_EQ(OB_SUCCESS, mode_mgr.after_flush_mode_meta(false, mode_mgr.last_submit_mode_meta_)); - EXPECT_EQ(OB_SUCCESS, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); - EXPECT_EQ(MODE_INIT, mode_mgr.state_); - EXPECT_EQ(2, mode_mgr.applied_mode_meta_.mode_version_); - EXPECT_EQ(AccessMode::RAW_WRITE, mode_mgr.applied_mode_meta_.access_mode_); - } - PALF_LOG(INFO, "test_change_access_mode case"); -} - } // end namespace unittest } // end namespace oceanbase @@ -347,7 +84,7 @@ int main(int argc, char **argv) system(rm_base_dir_cmd.c_str()); OB_LOGGER.set_file_name("test_log_mode_mgr.log", true); OB_LOGGER.set_log_level("INFO"); - PALF_LOG(INFO, "begin unittest::test_log_config_mgr"); + PALF_LOG(INFO, "begin unittest::test_log_mode_mgr"); ::testing::InitGoogleTest(&argc, argv); oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); return RUN_ALL_TESTS(); diff --git a/unittest/logservice/test_log_reconfirm.cpp b/unittest/logservice/test_log_reconfirm.cpp deleted file mode 100644 index 261493a3b..000000000 --- a/unittest/logservice/test_log_reconfirm.cpp +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#define private public -#include "logservice/palf/log_reconfirm.h" -#include "mock_logservice_container/mock_log_config_mgr.h" -#include "mock_logservice_container/mock_log_engine.h" -#include "mock_logservice_container/mock_log_sliding_window.h" -#include "mock_logservice_container/mock_log_mode_mgr.h" -#undef private - -namespace oceanbase -{ -using namespace common; -using namespace palf; -using namespace share; - -namespace unittest -{ - -class TestLogReconfirm : public ::testing::Test -{ -public: - TestLogReconfirm(); - virtual ~TestLogReconfirm(); -public: - virtual void SetUp(); - virtual void TearDown(); -public: - class MockLogReconfirm : public LogReconfirm - { - public: - MockLogReconfirm() {} - ~MockLogReconfirm() {} - }; - void set_default_config_meta(); -public: - int64_t palf_id_; - common::ObAddr self_; - MockLogSlidingWindow mock_sw_; - MockLogStateMgr mock_state_mgr_; - MockLogConfigMgr mock_mm_; - MockLogModeMgr mock_mode_mgr_; - MockLogEngine mock_log_engine_; - MockLogReconfirm log_reconfirm_; -}; - -TestLogReconfirm::TestLogReconfirm() {} -TestLogReconfirm::~TestLogReconfirm() {} - -void TestLogReconfirm::SetUp() -{ - palf_id_ = 1001; - self_.set_ip_addr("127.0.0.1", 12345); -} - -void TestLogReconfirm::TearDown() -{} - -TEST_F(TestLogReconfirm, test_init) -{ - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.init(palf_id_, self_, NULL, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - NULL, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, NULL, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, NULL, &mock_log_engine_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, NULL)); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_INIT_TWICE, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); -} - -TEST_F(TestLogReconfirm, test_reset_state) -{ - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - log_reconfirm_.reset_state(); - log_reconfirm_.state_ = MockLogReconfirm::WAITING_LOG_FLUSHED; - log_reconfirm_.reset_state(); -} - -void TestLogReconfirm::set_default_config_meta() -{ - int64_t curr_proposal_id = 1; - ObMemberList default_mlist; - default_mlist.add_server(self_); - GlobalLearnerList learners; - LogConfigMeta config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - init_config_version.generate(curr_proposal_id, 0); - init_config_info.config_.config_version_ = init_config_version; - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(default_mlist, 1, learners, init_config_version)); - config_meta.curr_ = init_config_info; - mock_mm_.log_ms_meta_ = config_meta; -} - -TEST_F(TestLogReconfirm, test_init_reconfirm) -{ - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_NOT_INIT, log_reconfirm_.init_reconfirm_()); - set_default_config_meta(); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init_reconfirm_()); -} - -TEST_F(TestLogReconfirm, test_handle_prepare_response) -{ - int64_t src_proposal_id = 2; - int64_t src_accept_proposal_id = 1; - ObAddr src_server; - LSN last_lsn; - LSN committed_end_lsn; - EXPECT_EQ(OB_NOT_INIT, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); - src_server = self_; - log_reconfirm_.curr_paxos_follower_list_.add_server(self_); - last_lsn.val_ = 0; - committed_end_lsn.val_ = 0; - log_reconfirm_.new_proposal_id_ = 2; - EXPECT_EQ(OB_STATE_NOT_MATCH, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); - // src_proposal_id < majority_max_accept_pid_, not receive - log_reconfirm_.majority_max_accept_pid_ = 2; - log_reconfirm_.majority_max_lsn_.val_ = 100; - last_lsn.val_ = 50; - log_reconfirm_.state_ = MockLogReconfirm::FETCH_MAX_LOG_LSN; - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); - // src_proposal_id == majority_max_accept_pid_, receive - src_accept_proposal_id = 2; - last_lsn.val_ = 120; - committed_end_lsn.val_ = 100; - EXPECT_EQ(0, log_reconfirm_.follower_end_lsn_list_.count()); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); - EXPECT_EQ(1, log_reconfirm_.follower_end_lsn_list_.count()); - EXPECT_EQ(committed_end_lsn, log_reconfirm_.follower_end_lsn_list_[0].last_flushed_end_lsn_); - EXPECT_EQ(last_lsn, log_reconfirm_.majority_max_lsn_); - log_reconfirm_.majority_max_accept_pid_ = 1; - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.handle_prepare_response(src_server, src_proposal_id, - src_accept_proposal_id, last_lsn, committed_end_lsn)); -} - -TEST_F(TestLogReconfirm, test_try_fetch_log) -{ - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.init(palf_id_, self_, &mock_sw_, - &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_)); - mock_sw_.max_flushed_end_lsn_.val_ = 100; - log_reconfirm_.majority_max_lsn_.val_ = 100; - EXPECT_EQ(true, log_reconfirm_.is_fetch_log_finished_()); - mock_sw_.mock_start_id_ = 1; - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.try_fetch_log_()); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.try_fetch_log_()); - mock_sw_.mock_start_id_ = 5; - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.try_fetch_log_()); - EXPECT_EQ(OB_SUCCESS, log_reconfirm_.try_fetch_log_()); -} - -} // END of unittest -} // end of oceanbase - -int main(int argc, char **argv) -{ - system("rm -f ./test_log_reconfirm.log"); - OB_LOGGER.set_file_name("test_log_reconfirm.log", true); - OB_LOGGER.set_log_level("TRACE"); - PALF_LOG(INFO, "begin unittest::test_log_reconfirm"); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/unittest/logservice/test_log_sliding_window.cpp b/unittest/logservice/test_log_sliding_window.cpp index 1a84a9c9e..8ddcf7423 100644 --- a/unittest/logservice/test_log_sliding_window.cpp +++ b/unittest/logservice/test_log_sliding_window.cpp @@ -15,9 +15,10 @@ */ #include "logservice/palf/log_io_task_cb_utils.h" +#include "logservice/ob_tenant_mutil_allocator.h" #include #define private public -#include "mock_logservice_container/mock_log_config_mgr.h" +#include "logservice/palf/log_sliding_window.h" #include "mock_logservice_container/mock_log_mode_mgr.h" #include "mock_logservice_container/mock_log_engine.h" #include "mock_logservice_container/mock_log_state_mgr.h" @@ -53,36 +54,13 @@ class TestLogSlidingWindow : public ::testing::Test return false; } }; - class MockLocCb : public PalfLocationCacheCb - { - virtual int get_leader(const int64_t id, common::ObAddr &leader) - { - UNUSED(id); - UNUSED(leader); - return OB_SUCCESS; - } - virtual int nonblock_get_leader(const int64_t id, common::ObAddr &leader) - { - UNUSED(id); - UNUSED(leader); - return OB_SUCCESS; - } - virtual int nonblock_get_leader(const uint64_t tenant_id, int64_t id, common::ObAddr &leader) - { - UNUSEDx(tenant_id, id, leader); - return OB_SUCCESS; - } - }; public: - int64_t palf_id_; common::ObAddr self_; MockLogStateMgr mock_state_mgr_; - MockLogConfigMgr mock_mm_; MockLogModeMgr mock_mode_mgr_; MockLogEngine mock_log_engine_; MockPalfFSCbWrapper palf_fs_cb_; ObTenantMutilAllocator *alloc_mgr_; - LogPlugins *plugins_; MockPublicLogSlidingWindow log_sw_; char *data_buf_; }; @@ -93,7 +71,6 @@ TestLogSlidingWindow::~TestLogSlidingWindow() {} const static uint64_t tenant_id = 1001; void TestLogSlidingWindow::SetUp() { - palf_id_ = 1001; self_.set_ip_addr("127.0.0.1", 12345); int ret = ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); @@ -105,7 +82,6 @@ void TestLogSlidingWindow::SetUp() OB_ASSERT(FALSE); } alloc_mgr_ = new (buf) common::ObTenantMutilAllocator(); - plugins_ = new LogPlugins(); data_buf_ = (char*)ob_malloc(64 * 1024 * 1024, attr); // init MTL ObTenantBase tbase(tenant_id); @@ -127,7 +103,6 @@ void gen_default_palf_base_info_(PalfBaseInfo &palf_base_info) prev_log_info.log_id_ = 0; prev_log_info.scn_.set_min(); prev_log_info.lsn_ = default_prev_lsn; - prev_log_info.log_proposal_id_ = INVALID_PROPOSAL_ID; prev_log_info.accum_checksum_ = -1; palf_base_info.prev_log_info_ = prev_log_info; palf_base_info.curr_lsn_ = default_prev_lsn; @@ -140,7 +115,7 @@ TEST_F(TestLogSlidingWindow, test_log_checksum) int64_t cal_checksum = -1; int64_t expected_acc_checksum = 389839115; LogChecksum checksum_obj; - EXPECT_EQ(OB_SUCCESS, checksum_obj.init(1, last_log_acc_checksum)); + EXPECT_EQ(OB_SUCCESS, checksum_obj.init(last_log_acc_checksum)); EXPECT_EQ(OB_SUCCESS, checksum_obj.acquire_accum_checksum(data_checksum, cal_checksum)); EXPECT_EQ(OB_SUCCESS, checksum_obj.verify_accum_checksum(data_checksum, expected_acc_checksum)); PALF_LOG(INFO, "finish test checksum", K(last_log_acc_checksum), K(data_checksum), K(cal_checksum), K(expected_acc_checksum)); @@ -151,106 +126,41 @@ TEST_F(TestLogSlidingWindow, test_init) PalfBaseInfo base_info; gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(palf_id_, self_, NULL, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, NULL, &palf_fs_cb_, NULL, plugins_, base_info, true)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - NULL, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, NULL, NULL, &palf_fs_cb_, NULL, plugins_, base_info, true)); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(self_, NULL, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, NULL, &palf_fs_cb_, NULL, base_info)); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(self_, &mock_state_mgr_, + NULL, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.init(self_, &mock_state_mgr_, NULL, NULL, &palf_fs_cb_, NULL, base_info)); // init succ - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); // init twice - EXPECT_EQ(OB_INIT_TWICE, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); + EXPECT_EQ(OB_INIT_TWICE, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); } TEST_F(TestLogSlidingWindow, test_private_func_batch_01) { - LSN lsn, end_lsn; + LSN end_lsn; int64_t log_id = OB_INVALID_LOG_ID; - int64_t log_pid = INVALID_PROPOSAL_ID; - EXPECT_EQ(OB_NOT_INIT, log_sw_.get_last_submit_log_info(lsn, log_id, log_pid)); - EXPECT_EQ(OB_NOT_INIT, log_sw_.get_max_flushed_log_info(lsn, end_lsn, log_pid)); EXPECT_EQ(OB_NOT_INIT, log_sw_.get_last_slide_end_lsn(end_lsn)); PalfBaseInfo base_info; gen_default_palf_base_info_(base_info); // init succ - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); log_id = 10 + PALF_SLIDING_WINDOW_SIZE; EXPECT_EQ(false, log_sw_.can_receive_larger_log_(log_id)); - EXPECT_EQ(false, log_sw_.leader_can_submit_larger_log_(log_id)); - EXPECT_EQ(false, log_sw_.leader_can_submit_larger_log_(PALF_SLIDING_WINDOW_SIZE + 1)); - EXPECT_EQ(OB_SUCCESS, log_sw_.get_last_submit_log_info(lsn, log_id, log_pid)); - EXPECT_EQ(OB_SUCCESS, log_sw_.get_max_flushed_log_info(lsn, end_lsn, log_pid)); + EXPECT_EQ(false, log_sw_.can_submit_larger_log_(log_id)); + EXPECT_EQ(false, log_sw_.can_submit_larger_log_(PALF_SLIDING_WINDOW_SIZE + 1)); EXPECT_EQ(OB_SUCCESS, log_sw_.get_last_slide_end_lsn(end_lsn)); share::SCN scn = log_sw_.get_last_slide_scn(); } -TEST_F(TestLogSlidingWindow, test_to_follower_pending) -{ - LSN last_lsn; - EXPECT_EQ(OB_NOT_INIT, log_sw_.to_follower_pending(last_lsn)); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - // init succ - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - char *buf = data_buf_; - int64_t buf_len = 1 * 1024 * 1024; - share::SCN ref_scn; - ref_scn.convert_for_logservice(99); - LSN lsn; - share::SCN scn; - buf_len = 2 * 1024 * 1024; - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - EXPECT_EQ(OB_SUCCESS, log_sw_.to_follower_pending(last_lsn)); -} - -TEST_F(TestLogSlidingWindow, test_fetch_log) -{ - FetchTriggerType fetch_log_type = NOTIFY_REBUILD; - LSN prev_lsn; - LSN fetch_start_lsn; - int64_t fetch_start_log_id = OB_INVALID_LOG_ID; - EXPECT_EQ(OB_NOT_INIT, log_sw_.try_fetch_log(fetch_log_type, prev_lsn, fetch_start_lsn, fetch_start_log_id)); - common::ObAddr dest; - LSN fetch_end_lsn; - bool is_fetched = false; - EXPECT_EQ(OB_NOT_INIT, log_sw_.try_fetch_log_for_reconfirm(dest, fetch_end_lsn, is_fetched)); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - // init succ - MockLocCb cb; - EXPECT_EQ(OB_SUCCESS, plugins_->add_plugin(static_cast(&cb))); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - prev_lsn.val_ = 1; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.try_fetch_log(fetch_log_type, prev_lsn, fetch_start_lsn, fetch_start_log_id)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.try_fetch_log_for_reconfirm(dest, fetch_end_lsn, is_fetched)); - dest = self_; - fetch_end_lsn.val_ = 100; - EXPECT_EQ(OB_SUCCESS, log_sw_.try_fetch_log_for_reconfirm(dest, fetch_end_lsn, is_fetched)); - EXPECT_EQ(true, is_fetched); - fetch_start_lsn.val_ = PALF_INITIAL_LSN_VAL; - prev_lsn = fetch_start_lsn; - fetch_start_log_id = 1; - EXPECT_EQ(OB_SUCCESS, log_sw_.try_fetch_log(fetch_log_type, prev_lsn, fetch_start_lsn, fetch_start_log_id)); - mock_state_mgr_.disable_sync(); - EXPECT_EQ(OB_SUCCESS, log_sw_.try_fetch_log(fetch_log_type, prev_lsn, fetch_start_lsn, fetch_start_log_id)); -} - TEST_F(TestLogSlidingWindow, test_report_log_task_trace) { EXPECT_EQ(OB_NOT_INIT, log_sw_.report_log_task_trace(1)); PalfBaseInfo base_info; gen_default_palf_base_info_(base_info); // init succ - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); EXPECT_EQ(OB_SUCCESS, log_sw_.report_log_task_trace(1)); char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; @@ -263,15 +173,6 @@ TEST_F(TestLogSlidingWindow, test_report_log_task_trace) EXPECT_EQ(OB_SUCCESS, log_sw_.report_log_task_trace(1)); } -TEST_F(TestLogSlidingWindow, test_set_location_cache_cb) -{ - MockLocCb cb; - MockLocCb *null_cb = NULL; - EXPECT_EQ(OB_INVALID_ARGUMENT, plugins_->add_plugin(static_cast(null_cb))); - EXPECT_EQ(OB_SUCCESS, plugins_->add_plugin(static_cast(&cb))); - EXPECT_EQ(OB_OP_NOT_ALLOW, plugins_->add_plugin(static_cast(&cb))); -} - TEST_F(TestLogSlidingWindow, test_submit_log) { PALF_LOG(INFO, "begin test_submit_log"); @@ -284,8 +185,7 @@ TEST_F(TestLogSlidingWindow, test_submit_log) LSN lsn; share::SCN scn; EXPECT_EQ(OB_NOT_INIT, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.submit_log(NULL, buf_len, ref_scn, lsn, scn)); buf_len = 0; EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); @@ -308,226 +208,6 @@ TEST_F(TestLogSlidingWindow, test_submit_log) EXPECT_EQ(OB_EAGAIN, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); } -TEST_F(TestLogSlidingWindow, test_submit_group_log) -{ - PALF_LOG(INFO, "begin test_submit_group_log"); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - mock_state_mgr_.mock_proposal_id_ = 100; - LSN lsn(10); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.submit_group_log(lsn, NULL, 1024)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.submit_group_log(lsn, data_buf_, -1)); - // generate log entry and group entry - LogEntryHeader log_entry_header; - LogGroupEntryHeader group_header; - share::SCN max_scn; - max_scn.convert_for_logservice(111111); - int64_t log_id = 1; - LSN committed_end_lsn(0); - int64_t log_proposal_id = 10; - char log_data[2048]; - int64_t log_data_len = 2048; - int64_t group_data_checksum = -1; - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - static const int64_t DATA_BUF_LEN = 64 * 1024 * 1024; - int64_t group_header_size = LogGroupEntryHeader::HEADER_SER_SIZE; - int64_t pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); - LogWriteBuf write_buf; - int64_t log_entry_size = pos + log_data_len; - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - int64_t accum_checksum = 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - int64_t group_entry_size = pos + log_entry_size; - // submit group_log - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_group_log(lsn, data_buf_, group_entry_size)); -} - -TEST_F(TestLogSlidingWindow, test_receive_log) -{ - PALF_LOG(INFO, "begin test_receive_log======================="); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; - ObAddr src_server = self_; - PushLogType push_log_type = PUSH_LOG; - LSN prev_lsn(0); - int64_t prev_log_proposal_id = curr_proposal_id; - LSN lsn; - int64_t log_data_len = 2048; - int64_t log_entry_size = log_data_len; - int64_t group_entry_size = log_entry_size; - TruncateLogInfo truncate_log_info; - EXPECT_EQ(OB_NOT_INIT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, \ - group_entry_size, true, truncate_log_info)); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - - char *buf = data_buf_; - int64_t buf_len = 2 * 1024 * 1024; - share::SCN ref_scn; - ref_scn.convert_for_logservice(999); - share::SCN scn; - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - // update lsn for next group entry - lsn.val_ = lsn.val_ + LogEntryHeader::HEADER_SER_SIZE + buf_len; - // generate new group entry - LogEntryHeader log_entry_header; - LogGroupEntryHeader group_header; - share::SCN max_scn; - max_scn.convert_for_logservice(111111); - int64_t log_id = 2; - LSN committed_end_lsn(0); - int64_t log_proposal_id = 20; - char log_data[2048]; - log_data_len = 2048; - int64_t group_data_checksum = -1; - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - static const int64_t DATA_BUF_LEN = 64 * 1024 * 1024; - int64_t group_header_size = LogGroupEntryHeader::HEADER_SER_SIZE; - int64_t pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); - log_entry_size = pos + log_data_len; - // gen 2nd log entry - max_scn.convert_for_logservice(222222); - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + log_entry_size + pos, log_data, log_data_len); - log_entry_size += (pos + log_data_len); - // gen group log - LogWriteBuf write_buf; - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, log_entry_size + group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - int64_t accum_checksum = 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - // receive group_log - src_server.reset(); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - src_server = self_; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, NULL, group_entry_size, true, truncate_log_info)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, 0, true, truncate_log_info)); - LSN tmp_lsn; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, tmp_lsn, data_buf_, 0, true, truncate_log_info)); - // prev_log_proposal_id not match - prev_log_proposal_id = curr_proposal_id - 1; - EXPECT_EQ(OB_STATE_NOT_MATCH, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // use correct prev_log_proposal_id - prev_log_proposal_id = curr_proposal_id; - // handle submit log - EXPECT_EQ(OB_SUCCESS, log_sw_.period_freeze_last_log()); - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - LSN old_lsn = lsn; - // test lsn > group_buffer capacity case, will return -4023 - lsn.val_ = log_sw_.group_buffer_.get_available_buffer_size(); - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // reset correct lsn - lsn = old_lsn; - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // ignore clean check, receive log_id=2 success - PALF_LOG(INFO, "begin receive log with log_id=2, and ignore clean check"); - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); - // update lsn to next log - lsn.val_ = lsn.val_ + group_entry_size; - // test log_id exceeds range case - log_id = 999999; - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // test cache log case - log_id = 102; - const int64_t cache_log_lsn_gap = 1024 * 1024; - lsn.val_ += cache_log_lsn_gap; - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // test need truncate cached log_task case - log_id = 3; - lsn.val_ -= cache_log_lsn_gap; - prev_lsn = old_lsn; - prev_log_proposal_id = log_proposal_id; - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - // inc log_proposal_id to trigger clean cached log_task - log_proposal_id += 1; - group_header.update_log_proposal_id(log_proposal_id); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - PALF_LOG(INFO, "begin receive log with log_id=3, and proposal_id 21"); - // handle submit log - EXPECT_EQ(OB_SUCCESS, log_sw_.period_freeze_last_log()); - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - EXPECT_TRUE(TRUNCATE_CACHED_LOG_TASK == truncate_log_info.truncate_type_); - EXPECT_EQ(log_id, truncate_log_info.truncate_log_id_); - EXPECT_EQ(OB_SUCCESS, log_sw_.clean_cached_log(truncate_log_info.truncate_log_id_, lsn, prev_lsn, prev_log_proposal_id)); - // do not check need clean log - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); - log_sw_.max_flushed_end_lsn_ = lsn + group_entry_size; - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - // test not match case - PALF_LOG(INFO, "begin tese not match case"); - // Increase flush lsn - log_sw_.max_flushed_end_lsn_.val_ += 100; - // Increase log_id, construct prev log hole - log_id += 100; - uint64_t new_val = max_scn.get_val_for_logservice() - 10; - max_scn.convert_for_logservice(new_val); - LogWriteBuf write_buf1; - EXPECT_EQ(OB_SUCCESS, write_buf1.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf1, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - // return -4109 because lsn < max_flushed_end_lsn - EXPECT_EQ(OB_STATE_NOT_MATCH, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - PALF_LOG(INFO, "finish test_receive_log======================="); -} - TEST_F(TestLogSlidingWindow, test_after_flush_log) { PALF_LOG(INFO, "begin test_after_flush_log"); @@ -536,24 +216,8 @@ TEST_F(TestLogSlidingWindow, test_after_flush_log) PalfBaseInfo base_info; gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - - // set default config meta - ObMemberList default_mlist; - default_mlist.add_server(self_); - GlobalLearnerList learners; - LogConfigMeta config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - init_config_version.generate(curr_proposal_id, 0); - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(default_mlist, 1, learners, init_config_version)); - config_meta.curr_ = init_config_info; - mock_mm_.log_ms_meta_ = config_meta; - mock_mm_.sw_ = &log_sw_; + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; @@ -569,306 +233,15 @@ TEST_F(TestLogSlidingWindow, test_after_flush_log) LSN group_log_lsn; group_log_lsn.val_ = lsn.val_ - LogGroupEntryHeader::HEADER_SER_SIZE; flush_log_ctx.lsn_ = group_log_lsn; - flush_log_ctx.log_proposal_id_ = curr_proposal_id; flush_log_ctx.total_len_ = LogGroupEntryHeader::HEADER_SER_SIZE + LogEntryHeader::HEADER_SER_SIZE + buf_len;; - flush_log_ctx.curr_proposal_id_ = curr_proposal_id + 1; flush_log_ctx.begin_ts_ = ObTimeUtility::current_time(); EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_sw_.after_flush_log(flush_log_ctx)); flush_log_ctx.log_id_ = 2; EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - log_sw_.is_truncating_ = true; - log_sw_.last_truncate_lsn_ = lsn; flush_log_ctx.log_id_ = 1; - flush_log_ctx.curr_proposal_id_ = curr_proposal_id; - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - log_sw_.is_truncating_ = false; - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id + 1; - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - flush_log_ctx.log_proposal_id_ = curr_proposal_id - 1; - EXPECT_EQ(OB_STATE_NOT_MATCH, log_sw_.after_flush_log(flush_log_ctx)); - mock_state_mgr_.role_ = FOLLOWER; - mock_state_mgr_.state_ = ACTIVE; - flush_log_ctx.log_proposal_id_ = curr_proposal_id; - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - mock_state_mgr_.role_ = LEADER; - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); + EXPECT_EQ(OB_ERR_OUT_OF_LOWER_BOUND, log_sw_.after_flush_log(flush_log_ctx)); } -TEST_F(TestLogSlidingWindow, test_truncate_log) -{ - PALF_LOG(INFO, "begin test_truncate_log"); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; - - char *buf = data_buf_; - int64_t buf_len = 2 * 1024 * 1024; - share::SCN ref_scn; - ref_scn.convert_for_logservice(999); - LSN lsn; - share::SCN scn; - // submit first log - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - EXPECT_EQ(OB_SUCCESS, log_sw_.period_freeze_last_log()); - // generate new group entry - LogEntryHeader log_entry_header; - LogGroupEntryHeader group_header; - share::SCN max_scn; - max_scn.convert_for_logservice(111111); - int64_t log_id = 2; - LSN committed_end_lsn(0); - int64_t log_proposal_id = 20; - char log_data[2048]; - int64_t log_data_len = 2048; - int64_t group_data_checksum = -1; - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - static const int64_t DATA_BUF_LEN = 64 * 1024 * 1024; - int64_t group_header_size = LogGroupEntryHeader::HEADER_SER_SIZE; - int64_t pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); - int64_t log_entry_size = pos + log_data_len; - // gen 2nd log entry - max_scn.convert_for_logservice(222222); - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + log_entry_size + pos, log_data, log_data_len); - log_entry_size += (pos + log_data_len); - // gen group log - LogWriteBuf write_buf; - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - int64_t accum_checksum = 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - int64_t group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - // receive group_log - ObAddr src_server = self_; - PushLogType push_log_type = PUSH_LOG; - LSN prev_lsn(0); - int64_t prev_log_proposal_id = curr_proposal_id; - lsn = log_sw_.get_max_lsn(); - src_server.reset(); - TruncateLogInfo truncate_log_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - src_server = self_; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, NULL, group_entry_size, true, truncate_log_info)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, 0, true, truncate_log_info)); - LSN tmp_lsn; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, tmp_lsn, data_buf_, 0, true, truncate_log_info)); - prev_log_proposal_id = curr_proposal_id; - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); - // test need truncate case - PALF_LOG(INFO, "begin test need truncate case"); - group_header.update_log_proposal_id(log_proposal_id + 1); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - EXPECT_EQ(OB_EAGAIN, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, true, truncate_log_info)); - EXPECT_TRUE(TRUNCATE_LOG == truncate_log_info.truncate_type_); - EXPECT_EQ(log_id, truncate_log_info.truncate_log_id_); - EXPECT_EQ(lsn, truncate_log_info.truncate_begin_lsn_); - EXPECT_EQ(log_proposal_id, truncate_log_info.truncate_log_proposal_id_); - // update flush end lsn - log_sw_.max_flushed_end_lsn_ = lsn + group_entry_size; - LSN test_lsn(0); - int64_t saved_truncate_log_id = truncate_log_info.truncate_log_id_; - truncate_log_info.truncate_log_id_ = 0; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.truncate(truncate_log_info, test_lsn, prev_log_proposal_id)); - truncate_log_info.truncate_log_id_ = saved_truncate_log_id; - EXPECT_EQ(0, prev_lsn.val_); - EXPECT_EQ(OB_SUCCESS, log_sw_.truncate(truncate_log_info, prev_lsn, prev_log_proposal_id)); - - TruncateLogCbCtx test_truncate_ctx(test_lsn); - EXPECT_EQ(OB_STATE_NOT_MATCH, log_sw_.after_truncate(test_truncate_ctx)); - TruncateLogCbCtx invalid_ctx; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.after_truncate(invalid_ctx)); - TruncateLogCbCtx truncate_log_cb_ctx(truncate_log_info.truncate_begin_lsn_); - EXPECT_EQ(OB_SUCCESS, log_sw_.after_truncate(truncate_log_cb_ctx)); -} - -TEST_F(TestLogSlidingWindow, test_ack_log) -{ - PALF_LOG(INFO, "begin test_ack_log"); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; - log_sw_.self_ = self_; - - // set default config meta - ObMemberList default_mlist; - default_mlist.add_server(self_); - GlobalLearnerList learners; - LogConfigMeta config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - init_config_version.generate(curr_proposal_id, 0); - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(default_mlist, 1, learners, init_config_version)); - config_meta.curr_ = init_config_info; - mock_mm_.log_ms_meta_ = config_meta; - mock_mm_.sw_ = &log_sw_; - - char *buf = data_buf_; - int64_t buf_len = 2 * 1024 * 1024; - share::SCN ref_scn; - ref_scn.convert_for_logservice(999); - LSN lsn; - share::SCN scn; - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - LSN end_lsn = lsn + LogEntryHeader::HEADER_SER_SIZE + buf_len; - ObAddr server; - server.set_ip_addr("127.0.0.1", 12346); - default_mlist.add_server(server); - LSN invalid_lsn; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.ack_log(server, invalid_lsn)); - LogConfigInfoV2 new_config_info = init_config_info; - new_config_info.config_.log_sync_memberlist_ = default_mlist; - mock_mm_.log_ms_meta_.curr_ = new_config_info; - - mock_state_mgr_.role_ = LEADER; - mock_state_mgr_.state_ = ACTIVE; - - FlushLogCbCtx flush_log_ctx; - flush_log_ctx.log_id_ = 1; - flush_log_ctx.scn_ = scn; - LSN group_log_lsn; - group_log_lsn.val_ = lsn.val_ - LogGroupEntryHeader::HEADER_SER_SIZE; - flush_log_ctx.lsn_ = group_log_lsn; - flush_log_ctx.log_proposal_id_ = curr_proposal_id; - flush_log_ctx.total_len_ = LogGroupEntryHeader::HEADER_SER_SIZE + LogEntryHeader::HEADER_SER_SIZE + buf_len;; - flush_log_ctx.curr_proposal_id_ = curr_proposal_id; - flush_log_ctx.begin_ts_ = ObTimeUtility::current_time(); - EXPECT_EQ(OB_SUCCESS, log_sw_.after_flush_log(flush_log_ctx)); - - EXPECT_EQ(OB_SUCCESS, log_sw_.ack_log(server, end_lsn)); -} - -TEST_F(TestLogSlidingWindow, test_truncate_for_rebuild) -{ - PALF_LOG(INFO, "begin test_truncate_for_rebuild"); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; - - char *buf = data_buf_; - int64_t buf_len = 2 * 1024 * 1024; - share::SCN ref_scn; - ref_scn.convert_for_logservice(999); - LSN lsn; - share::SCN scn; - EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - // generate new group entry - LogEntryHeader log_entry_header; - LogGroupEntryHeader group_header; - share::SCN max_scn; - max_scn.convert_for_logservice(111111); - int64_t log_id = 2; - LSN committed_end_lsn(0); - int64_t log_proposal_id = 20; - char log_data[2048]; - int64_t log_data_len = 2048; - int64_t group_data_checksum = -1; - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - static const int64_t DATA_BUF_LEN = 64 * 1024 * 1024; - int64_t group_header_size = LogGroupEntryHeader::HEADER_SER_SIZE; - int64_t pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); - int64_t log_entry_size = pos + log_data_len; - // gen 2nd log entry - max_scn.convert_for_logservice(222222); - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + log_entry_size + pos, log_data, log_data_len); - log_entry_size += (pos + log_data_len); - // gen group log - LogWriteBuf write_buf; - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - int64_t accum_checksum = 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - int64_t group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - // receive group_log - ObAddr src_server = self_; - PushLogType push_log_type = PUSH_LOG; - LSN prev_lsn(0); - int64_t prev_log_proposal_id = curr_proposal_id; - lsn = log_sw_.get_max_lsn(); - src_server.reset(); - src_server = self_; - TruncateLogInfo truncate_log_info; - // handle submit log - EXPECT_EQ(OB_SUCCESS, log_sw_.period_freeze_last_log()); - PALF_LOG(INFO, "begin receive log with log_id=2"); - EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); - // gen next group log - log_id = 10; - uint64_t new_val = max_scn.get_val_for_logservice() + 100; - max_scn.convert_for_logservice(new_val); - LogWriteBuf write_buf1; - EXPECT_EQ(OB_SUCCESS, write_buf1.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf1, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - accum_checksum += 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - group_header.serialize(data_buf_, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - // lsn < last_submit_end_lsn, return -4016 - EXPECT_EQ(OB_ERR_UNEXPECTED, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); - - // update flush end lsn - log_sw_.max_flushed_end_lsn_ = lsn + group_entry_size; - - PalfBaseInfo new_base_info; - new_base_info.prev_log_info_.log_id_ = 0; - - EXPECT_EQ(OB_SUCCESS, log_sw_.truncate_for_rebuild(new_base_info)); - new_base_info.prev_log_info_.log_id_ = 6; - new_val = max_scn.get_val_for_logservice() - 50; - new_base_info.prev_log_info_.scn_.convert_for_logservice(new_val); - new_base_info.prev_log_info_.log_proposal_id_ = curr_proposal_id; - new_base_info.prev_log_info_.lsn_ = lsn - 200; - new_base_info.curr_lsn_ = lsn - 100; - // test prev_log not exist case - EXPECT_EQ(OB_SUCCESS, log_sw_.truncate_for_rebuild(new_base_info)); -} TEST_F(TestLogSlidingWindow, test_append_disk_log) { @@ -878,10 +251,7 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) EXPECT_EQ(OB_NOT_INIT, log_sw_.append_disk_log(lsn, group_entry)); PalfBaseInfo base_info; gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; + EXPECT_EQ(OB_SUCCESS, log_sw_.init(self_, &mock_state_mgr_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); // generate new group entry LogEntry log_entry; LogEntryHeader log_entry_header; @@ -890,7 +260,6 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) max_scn.convert_for_logservice(111111); int64_t log_id = 1; LSN committed_end_lsn(0); - int64_t log_proposal_id = 20; char log_data[2048]; int64_t log_data_len = 2048; int64_t group_data_checksum = -1; @@ -925,8 +294,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) log_entry_size += (pos + log_data_len); // gen group log LogWriteBuf write_buf; - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); + EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.generate(false, write_buf, log_entry_size, max_scn, log_id, + committed_end_lsn, group_data_checksum)); const int64_t total_group_log_size = group_header_size + log_entry_size; const int64_t first_part_len = total_group_log_size / 2; EXPECT_TRUE(first_part_len > 0); @@ -934,8 +303,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) // continous buf EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, first_part_len)); EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_ + first_part_len, second_part_len)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); + EXPECT_EQ(OB_SUCCESS, group_header.generate(false, write_buf, log_entry_size, max_scn, log_id, + committed_end_lsn, group_data_checksum)); // non-continous buf group_header.reset(); write_buf.reset(); @@ -944,8 +313,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) memcpy(second_buf, data_buf_ + first_part_len, second_part_len); EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, first_part_len)); EXPECT_EQ(OB_SUCCESS, write_buf.push_back(second_buf, second_part_len)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); + EXPECT_EQ(OB_SUCCESS, group_header.generate(false, write_buf, log_entry_size, max_scn, log_id, + committed_end_lsn, group_data_checksum)); int64_t accum_checksum = 100; (void) group_header.update_accumulated_checksum(accum_checksum); // calculate header parity flag @@ -972,8 +341,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) // gen group log LogWriteBuf write_buf1; EXPECT_EQ(OB_SUCCESS, write_buf1.push_back(data_buf_, log_entry_size+group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); + EXPECT_EQ(OB_SUCCESS, group_header.generate(false, write_buf, log_entry_size, max_scn, log_id, + committed_end_lsn, group_data_checksum)); accum_checksum += 100; (void) group_header.update_accumulated_checksum(accum_checksum); // calculate header parity flag @@ -987,80 +356,6 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) EXPECT_EQ(OB_SUCCESS, log_sw_.append_disk_log(lsn, group_entry)); } -TEST_F(TestLogSlidingWindow, test_group_entry_truncate) -{ - PALF_LOG(INFO, "begin test_group_entry_truncate"); - LSN lsn(0); - LogGroupEntry group_entry; - LogGroupEntryHeader group_header; - share::SCN truncate_scn; - truncate_scn.convert_for_logservice(111113); - int64_t pre_accum_checksum = 123456; - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.truncate(NULL, 1024, truncate_scn, pre_accum_checksum)); - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.truncate(data_buf_, 0, truncate_scn, pre_accum_checksum)); - PalfBaseInfo base_info; - gen_default_palf_base_info_(base_info); - EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, - &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, plugins_, base_info, true)); - int64_t curr_proposal_id = 10; - mock_state_mgr_.mock_proposal_id_ = curr_proposal_id; - // generate new group entry - LogEntry log_entry; - LogEntryHeader log_entry_header; - share::SCN max_scn; - max_scn.convert_for_logservice(111111); - int64_t log_id = 1; - LSN committed_end_lsn(0); - int64_t log_proposal_id = 20; - char log_data[2048]; - int64_t log_data_len = 2048; - int64_t group_data_checksum = -1; - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - static const int64_t DATA_BUF_LEN = 64 * 1024 * 1024; - int64_t group_header_size = LogGroupEntryHeader::HEADER_SER_SIZE; - int64_t pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); - int64_t dser_pos = 0; - int64_t log_entry_size = pos + log_data_len; - // gen 2nd log entry - max_scn.convert_for_logservice(222222); - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); - pos = 0; - log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); - EXPECT_TRUE(pos > 0); - memcpy(data_buf_ + group_header_size + log_entry_size + pos, log_data, log_data_len); - log_entry_size += (pos + log_data_len); - // gen group log - LogWriteBuf write_buf; - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - const int64_t total_group_log_size = group_header_size + log_entry_size; - const int64_t first_part_len = total_group_log_size / 2; - EXPECT_TRUE(first_part_len > 0); - const int64_t second_part_len = total_group_log_size - first_part_len; - // continous buf - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_, first_part_len)); - EXPECT_EQ(OB_SUCCESS, write_buf.push_back(data_buf_ + first_part_len, second_part_len)); - EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf, log_entry_size, max_scn, log_id, - committed_end_lsn, log_proposal_id, group_data_checksum)); - int64_t accum_checksum = 100; - (void) group_header.update_accumulated_checksum(accum_checksum); - // calculate header parity flag - (void) group_header.update_header_checksum(); - pos = 0; - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.serialize(NULL, DATA_BUF_LEN, pos)); - EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.serialize(data_buf_, 0, pos)); - EXPECT_EQ(OB_SUCCESS, group_header.serialize(data_buf_, DATA_BUF_LEN, pos)); - EXPECT_TRUE(pos == group_header_size); - int64_t group_entry_size = pos + log_entry_size; - EXPECT_TRUE(group_header.check_integrity(data_buf_ + group_header_size, group_entry_size - group_header_size)); - EXPECT_EQ(OB_SUCCESS, group_entry.generate(group_header, data_buf_ + group_header_size)); - EXPECT_TRUE(group_entry.check_integrity()); - EXPECT_EQ(OB_SUCCESS, group_header.truncate(data_buf_ + group_header_size, log_entry_size, truncate_scn, pre_accum_checksum)); -} - } // END of unittest } // end of oceanbase diff --git a/unittest/logservice/test_log_state_mgr.cpp b/unittest/logservice/test_log_state_mgr.cpp index 26f1406a9..d2783ca93 100644 --- a/unittest/logservice/test_log_state_mgr.cpp +++ b/unittest/logservice/test_log_state_mgr.cpp @@ -17,12 +17,8 @@ #include #define private public #include "mock_logservice_container/mock_log_engine.h" -#include "mock_logservice_container/mock_election.h" #include "mock_logservice_container/mock_log_sliding_window.h" -#include "mock_logservice_container/mock_log_reconfirm.h" -#include "mock_logservice_container/mock_log_config_mgr.h" #include "mock_logservice_container/mock_log_mode_mgr.h" -#include "mock_logservice_container/mock_palf_role_change_cb_wrapper.h" #undef private namespace oceanbase @@ -43,18 +39,10 @@ class TestLogStateMgr: public ::testing::Test virtual void SetUp(); virtual void TearDown(); public: - int64_t palf_id_; common::ObAddr self_; - LogPrepareMeta log_prepare_meta_; - LogReplicaPropertyMeta log_replica_property_meta_; - ::oceanbase::palf::mockelection::MockElection mock_election_; MockLogSlidingWindow mock_sw_; - MockLogReconfirm mock_reconfirm_; MockLogEngine mock_log_engine_; - MockLogConfigMgr mock_config_mgr_; MockLogModeMgr mock_mode_mgr_; - MockPalfRoleChangeCbWrapper mock_role_change_cb_; - LogPlugins plugins_; LogStateMgr state_mgr_; }; @@ -63,11 +51,7 @@ TestLogStateMgr::~TestLogStateMgr() {} void TestLogStateMgr::SetUp() { - palf_id_ = 1001; self_.set_ip_addr("127.0.0.1", 12345); - LogVotedFor voted_for; - log_prepare_meta_.generate(voted_for, 1); - log_replica_property_meta_.generate(true, LogReplicaType::NORMAL_REPLICA); } void TestLogStateMgr::TearDown() @@ -75,89 +59,34 @@ void TestLogStateMgr::TearDown() TEST_F(TestLogStateMgr, test_init) { - EXPECT_EQ(OB_INVALID_ARGUMENT, state_mgr_.init(palf_id_, self_, log_prepare_meta_, log_replica_property_meta_, NULL, &mock_sw_, - &mock_reconfirm_, &mock_log_engine_, &mock_config_mgr_, &mock_mode_mgr_, &mock_role_change_cb_, &plugins_)); - EXPECT_EQ(OB_INVALID_ARGUMENT, state_mgr_.init(palf_id_, self_, log_prepare_meta_, log_replica_property_meta_, &mock_election_, NULL, - &mock_reconfirm_, &mock_log_engine_, &mock_config_mgr_, &mock_mode_mgr_, &mock_role_change_cb_, &plugins_)); - EXPECT_EQ(OB_SUCCESS, state_mgr_.init(palf_id_, self_, log_prepare_meta_, log_replica_property_meta_, &mock_election_, &mock_sw_, - &mock_reconfirm_, &mock_log_engine_, &mock_config_mgr_, &mock_mode_mgr_, &mock_role_change_cb_, &plugins_)); - EXPECT_EQ(OB_INIT_TWICE, state_mgr_.init(palf_id_, self_, log_prepare_meta_, log_replica_property_meta_, &mock_election_, &mock_sw_, - &mock_reconfirm_, &mock_log_engine_, &mock_config_mgr_, &mock_mode_mgr_, &mock_role_change_cb_, &plugins_)); + EXPECT_EQ(OB_INVALID_ARGUMENT, state_mgr_.init(self_, NULL, &mock_mode_mgr_)); + EXPECT_EQ(OB_SUCCESS, state_mgr_.init(self_, &mock_sw_, &mock_mode_mgr_)); + EXPECT_EQ(OB_INIT_TWICE, state_mgr_.init(self_, &mock_sw_, &mock_mode_mgr_)); } -TEST_F(TestLogStateMgr, replay_to_leader_active) +TEST_F(TestLogStateMgr, recover_to_active) { - EXPECT_EQ(OB_SUCCESS, state_mgr_.init(palf_id_, self_, log_prepare_meta_, log_replica_property_meta_, &mock_election_, &mock_sw_, - &mock_reconfirm_, &mock_log_engine_, &mock_config_mgr_, &mock_mode_mgr_, &mock_role_change_cb_, &plugins_)); - // set default config meta - ObMemberList default_mlist; - default_mlist.add_server(self_); - GlobalLearnerList learners; - LogConfigMeta config_meta; - LogConfigInfoV2 init_config_info; - LogConfigVersion init_config_version; - EXPECT_EQ(OB_SUCCESS, init_config_version.generate(1, 0)); - EXPECT_EQ(OB_SUCCESS, init_config_info.generate(default_mlist, 1, learners, init_config_version)); - config_meta.curr_ = init_config_info; - mock_config_mgr_.log_ms_meta_ = config_meta; - + EXPECT_EQ(OB_SUCCESS, state_mgr_.init(self_, &mock_sw_, &mock_mode_mgr_)); + EXPECT_EQ(INIT, state_mgr_.get_state()); EXPECT_EQ(OB_SUCCESS, state_mgr_.set_scan_disk_log_finished()); - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); // replay to follower active - EXPECT_EQ(true, state_mgr_.is_follower_active()); - mock_config_mgr_.self_ = self_; - mock_election_.self_ = self_; - - // test follower to leader - mock_election_.leader_ = self_; - mock_election_.leader_epoch_ = 10; EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_leader_active()); + EXPECT_TRUE(state_mgr_.is_recovering()); - // test leader to follower - mock_election_.leader_.reset(); - mock_election_.leader_epoch_ = 0; - mock_sw_.pending_end_lsn_.val_ = 100; - mock_sw_.last_slide_end_lsn_.val_ = 90; + mock_sw_.all_log_flushed_ = false; EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_follower_pending()); - mock_sw_.last_slide_end_lsn_.val_ = 100; - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_follower_active()); + EXPECT_TRUE(state_mgr_.is_recovering()); + EXPECT_FALSE(mock_sw_.activated_); - // test follower pending to reconfirm to leader active - mock_election_.leader_ = self_; - mock_election_.leader_epoch_ = 11; - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_leader_active()); - mock_election_.leader_.reset(); - mock_election_.leader_epoch_ = 0; - mock_sw_.pending_end_lsn_.val_ = 200; - mock_sw_.last_slide_end_lsn_.val_ = 190; + mock_sw_.all_log_flushed_ = true; + mock_sw_.all_committed_slided_out_ = false; EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_follower_pending()); - // pending to reconfirm - mock_election_.leader_ = self_; - mock_election_.leader_epoch_ = 12; - mock_reconfirm_.mock_ret_ = OB_EAGAIN; - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_leader_reconfirm()); - // reconfirm to pending - mock_election_.leader_.reset(); - mock_election_.leader_epoch_ = 0; - mock_sw_.pending_end_lsn_.val_ = 300; - mock_sw_.last_slide_end_lsn_.val_ = 290; - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_follower_pending()); - // pending to reconfirm - mock_election_.leader_ = self_; - mock_election_.leader_epoch_ = 13; - EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_leader_reconfirm()); - // reconfirm to pending - // reconfirm to leader active - mock_reconfirm_.mock_ret_ = OB_SUCCESS; + EXPECT_TRUE(state_mgr_.is_recovering()); + EXPECT_EQ(mock_sw_.mock_max_flushed_end_lsn_, mock_sw_.pending_end_lsn_); + + mock_sw_.all_committed_slided_out_ = true; EXPECT_EQ(OB_SUCCESS, state_mgr_.switch_state()); - EXPECT_EQ(true, state_mgr_.is_leader_active()); + EXPECT_TRUE(state_mgr_.is_active()); + EXPECT_TRUE(mock_sw_.activated_); } } // END of unittest diff --git a/unittest/logservice/test_lsn_allocator.cpp b/unittest/logservice/test_lsn_allocator.cpp index 2168cc910..1c423e48d 100644 --- a/unittest/logservice/test_lsn_allocator.cpp +++ b/unittest/logservice/test_lsn_allocator.cpp @@ -36,14 +36,10 @@ class TestLSNAllocator : public ::testing::Test virtual void SetUp(); virtual void TearDown(); protected: - int64_t palf_id_; LSNAllocator lsn_allocator_; }; -TestLSNAllocator::TestLSNAllocator() - : palf_id_(1) -{ -} +TestLSNAllocator::TestLSNAllocator() {} TestLSNAllocator::~TestLSNAllocator() { diff --git a/unittest/logservice/test_palf_throttling.cpp b/unittest/logservice/test_palf_throttling.cpp index 7f6edf8eb..0b939d438 100644 --- a/unittest/logservice/test_palf_throttling.cpp +++ b/unittest/logservice/test_palf_throttling.cpp @@ -81,7 +81,6 @@ TEST_F(TestPalfThrottling, test_palf_options) wrapper.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_ = total_disk_size; wrapper.disk_opts_for_stopping_writing_.log_disk_utilization_threshold_ = 80; wrapper.disk_opts_for_stopping_writing_.log_disk_utilization_limit_threshold_ = utilization_limit_threshold; - wrapper.disk_opts_for_stopping_writing_.log_writer_parallelism_ = 1; int64_t unrecyclable_size = 0; wrapper.set_cur_unrecyclable_log_disk_size(unrecyclable_size); ASSERT_EQ(false, wrapper.need_throttling()); @@ -410,4 +409,3 @@ int main(int argc, char **argv) ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/unittest/logservice/test_role_change_handler.cpp b/unittest/logservice/test_role_change_handler.cpp index f18a841c1..84c3327f2 100644 --- a/unittest/logservice/test_role_change_handler.cpp +++ b/unittest/logservice/test_role_change_handler.cpp @@ -16,7 +16,7 @@ #include #define private public -#include "logservice/rcservice/ob_role_change_handler.h" +#include "logservice/localservice/ob_local_log_handler_set.h" #undef private namespace oceanbase @@ -24,34 +24,24 @@ namespace oceanbase using namespace logservice; namespace unittest { -class MockRoleChangeHandler : public ObIRoleChangeSubHandler +class MockRoleChangeHandler : public ObILocalLogHandler { public: - void switch_to_follower_forcedly() override final + void deactivate() override final {} - int switch_to_leader() override final - { - return OB_SUCCESS; - } - int switch_to_follower_gracefully() override final - { - return OB_SUCCESS; - } - int resume_leader() override final + int activate() override final { return OB_SUCCESS; } }; TEST(TestRoleChangeHander, test_basic_func) { - ObRoleChangeHandler handler; + ObLocalLogHandlerSet handler; ObLogBaseType type = ObLogBaseType::TRANS_SERVICE_LOG_BASE_TYPE; MockRoleChangeHandler mock_handler; - RCDiagnoseInfo unused_diagnose_info; - handler.register_handler(type, &mock_handler); - handler.switch_to_leader(unused_diagnose_info); - handler.switch_to_follower_forcedly(); - handler.switch_to_follower_gracefully(); + ASSERT_EQ(OB_SUCCESS, handler.register_handler(type, &mock_handler)); + ASSERT_EQ(OB_SUCCESS, handler.activate()); + handler.deactivate(); } } // end namespace unittest } // end namespace oceanbase diff --git a/unittest/logservice/test_server_log_block_mgr.cpp b/unittest/logservice/test_server_log_block_mgr.cpp index 9d5ff2ffa..354203bee 100644 --- a/unittest/logservice/test_server_log_block_mgr.cpp +++ b/unittest/logservice/test_server_log_block_mgr.cpp @@ -30,9 +30,8 @@ using namespace common; using namespace palf; namespace unittest { -static int ls_id_array_[] = {1}; -static const int64_t DEFAULT_LS_COUNT = sizeof(ls_id_array_) / sizeof(ls_id_array_[0]); -static const char *log_or_meta_[2] = {"log", "meta"}; +static const char *LOG_STREAM_DIR = "log_stream"; +static const char *LOG_META_DIR = "meta"; class TestServerLogBlockMgr : public ::testing::Test { public: @@ -44,11 +43,11 @@ class TestServerLogBlockMgr : public ::testing::Test virtual ~TestServerLogBlockMgr() {} - static int create_ls_in_tenant(const int ls_count, const char *tenant_dir); + static int create_log_stream_dir(const char *tenant_dir); - static int remove_ls_in_tenant(const char *tenant_dir); + static int remove_log_stream_dir(const char *tenant_dir); - int create_new_blocks_at(const int64_t ls_id, const palf::FileDesc &fd, + int create_new_blocks_at(const palf::FileDesc &fd, const palf::block_id_t start_block_id, const int64_t block_cnt) { int ret = OB_SUCCESS; @@ -58,13 +57,13 @@ class TestServerLogBlockMgr : public ::testing::Test int64_t pos = 0; bool result = false; const palf::block_id_t block_id = start_block_id + i; - databuff_printf(block_path, OB_MAX_FILE_NAME_LENGTH, pos, "%s/%lu/%s/%lu", - tenant_string_, ls_id, log_or_meta_[ls_id % 2], block_id); + databuff_printf(block_path, OB_MAX_FILE_NAME_LENGTH, pos, "%s/%s/%s/%lu", + tenant_string_, LOG_STREAM_DIR, LOG_META_DIR, block_id); if (OB_FAIL(block_id_to_string(block_id, create_block_path, OB_MAX_FILE_NAME_LENGTH))) { CLOG_LOG(ERROR, "block_id_to_string failed", K(ret)); } else if (OB_FAIL(log_block_mgr_.create_block_at(fd, create_block_path, ObServerLogBlockMgr::BLOCK_SIZE))) { - CLOG_LOG(ERROR, "create_block_at failed", K(ret), K(ls_id), K(fd), + CLOG_LOG(ERROR, "create_block_at failed", K(ret), K(fd), K(start_block_id), K(block_id)); } else if (OB_FAIL(FileDirectoryUtils::is_exists(block_path, result))) { CLOG_LOG(ERROR, "is_exists failed", K(ret), K(block_path), K(result), K(i), @@ -79,7 +78,7 @@ class TestServerLogBlockMgr : public ::testing::Test return ret; } - int delete_blocks_at(const int64_t ls_id, const palf::FileDesc &fd, + int delete_blocks_at(const palf::FileDesc &fd, const palf::block_id_t start_block_id, const int64_t block_cnt) { int ret = OB_SUCCESS; @@ -89,12 +88,12 @@ class TestServerLogBlockMgr : public ::testing::Test int64_t pos = 0; bool result = false; const palf::block_id_t block_id = start_block_id + i; - databuff_printf(block_path, OB_MAX_FILE_NAME_LENGTH, pos, "%s/%lu/%s/%lu", - tenant_string_, ls_id, log_or_meta_[ls_id % 2], block_id); + databuff_printf(block_path, OB_MAX_FILE_NAME_LENGTH, pos, "%s/%s/%s/%lu", + tenant_string_, LOG_STREAM_DIR, LOG_META_DIR, block_id); if (OB_FAIL(block_id_to_string(block_id, remove_block_path, OB_MAX_FILE_NAME_LENGTH))) { CLOG_LOG(ERROR, "block_id_to_string failed", K(ret)); } else if (OB_FAIL(log_block_mgr_.remove_block_at(fd, remove_block_path))) { - CLOG_LOG(ERROR, "delete_new_block_at failed", K(ret), K(ls_id), K(fd), + CLOG_LOG(ERROR, "delete_new_block_at failed", K(ret), K(fd), K(start_block_id), K(i)); } else if (OB_FAIL(FileDirectoryUtils::is_exists(block_path, result))) { CLOG_LOG(ERROR, "is_exists failed", K(ret), K(block_path), K(result), K(i), @@ -112,20 +111,15 @@ class TestServerLogBlockMgr : public ::testing::Test public: virtual void SetUp(); virtual void TearDown(); - static const char *log_pool_base_path_; - static const char *log_disk_path_; + static const char *log_disk_base_path_; static const char *tenant_string_; - static std::map tenant_ls_fd_map_; - static std::vector tenant_1_ls_fd_; + static int log_stream_fd_; ObServerLogBlockMgr log_block_mgr_; - ObSpinLock lock_; - std::map map_; }; -const char *TestServerLogBlockMgr::log_pool_base_path_ = "clog_disk/clog"; +const char *TestServerLogBlockMgr::log_disk_base_path_ = "clog_disk/clog"; const char *TestServerLogBlockMgr::tenant_string_ = "clog_disk/clog/sys"; -std::map TestServerLogBlockMgr::tenant_ls_fd_map_; -std::vector TestServerLogBlockMgr::tenant_1_ls_fd_; +int TestServerLogBlockMgr::log_stream_fd_ = -1; void TestServerLogBlockMgr::SetUpTestCase() { @@ -134,73 +128,63 @@ void TestServerLogBlockMgr::SetUpTestCase() if (OB_FAIL(FileDirectoryUtils::create_directory(tenant_string_))) { CLOG_LOG(ERROR, "FileDirectoryUtils create_directory failed", K(ret), K(tenant_string_)); - } else if (OB_FAIL(create_ls_in_tenant(DEFAULT_LS_COUNT, tenant_string_))) { - CLOG_LOG(ERROR, "create_ls_in_tenant failed", K(ret)); + } else if (OB_FAIL(create_log_stream_dir(tenant_string_))) { + CLOG_LOG(ERROR, "create log stream directory failed", K(ret)); } else { - CLOG_LOG(INFO, "SetUpTestSuite success", K(log_pool_base_path_)); + CLOG_LOG(INFO, "SetUpTestSuite success", K(log_disk_base_path_)); } } void TestServerLogBlockMgr::TearDownTestCase() { - remove_ls_in_tenant(tenant_string_); + remove_log_stream_dir(tenant_string_); system("rm -rf clog_disk"); } -int TestServerLogBlockMgr::create_ls_in_tenant(const int ls_count, const char *tenant_dir) +int TestServerLogBlockMgr::create_log_stream_dir(const char *tenant_dir) { int ret = OB_SUCCESS; - for (int i = 0; i < ls_count && OB_SUCC(ret); i++) { - char ls_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; - int fd = -1; - snprintf(ls_path, OB_MAX_FILE_NAME_LENGTH, "%s/%d", tenant_dir, ls_id_array_[i]); - if (-1 == ::mkdir(ls_path, ObServerLogBlockMgr::CREATE_DIR_MODE)) { - ret = palf::convert_sys_errno(); - CLOG_LOG(ERROR, "::mkdir failed", K(ret)); - } else if (FALSE_IT(snprintf(ls_path, OB_MAX_FILE_NAME_LENGTH, "%s/%d/%s", tenant_dir, - ls_id_array_[i], log_or_meta_[ls_id_array_[i] % 2])) - || -1 == ::mkdir(ls_path, ObServerLogBlockMgr::CREATE_DIR_MODE)) { - ret = palf::convert_sys_errno(); - CLOG_LOG(ERROR, "::mkdir failed", K(ret)); - } else if (-1 == (fd = ::open(ls_path, ObServerLogBlockMgr::OPEN_DIR_FLAG))) { - ret = palf::convert_sys_errno(); - CLOG_LOG(ERROR, "::open failed", K(ret)); - } else { - tenant_ls_fd_map_.insert(std::make_pair(ls_id_array_[i], fd)); - } + char ls_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; + int fd = -1; + snprintf(ls_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", tenant_dir, LOG_STREAM_DIR); + if (-1 == ::mkdir(ls_path, ObServerLogBlockMgr::CREATE_DIR_MODE)) { + ret = palf::convert_sys_errno(); + CLOG_LOG(ERROR, "::mkdir failed", K(ret)); + } else if (FALSE_IT(snprintf(ls_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s/%s", tenant_dir, + LOG_STREAM_DIR, LOG_META_DIR)) + || -1 == ::mkdir(ls_path, ObServerLogBlockMgr::CREATE_DIR_MODE)) { + ret = palf::convert_sys_errno(); + CLOG_LOG(ERROR, "::mkdir failed", K(ret)); + } else if (-1 == (fd = ::open(ls_path, ObServerLogBlockMgr::OPEN_DIR_FLAG))) { + ret = palf::convert_sys_errno(); + CLOG_LOG(ERROR, "::open failed", K(ret)); + } else { + log_stream_fd_ = fd; } return ret; } -int TestServerLogBlockMgr::remove_ls_in_tenant(const char *tenant_dir) +int TestServerLogBlockMgr::remove_log_stream_dir(const char *tenant_dir) { int ret = OB_SUCCESS; - for (auto iter = tenant_ls_fd_map_.begin(); iter != tenant_ls_fd_map_.end(); iter++) { - char ls_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; - snprintf(ls_path, OB_MAX_FILE_NAME_LENGTH, "%s/%lu", tenant_dir, iter->first); - if (-1 == FileDirectoryUtils::delete_directory_rec(ls_path)) { - ret = palf::convert_sys_errno(); - CLOG_LOG(ERROR, "::rmdir failed", K(ret), K(ls_path)); - } else if (-1 == ::close(iter->second)) { - ret = palf::convert_sys_errno(); - CLOG_LOG(ERROR, "::close failed", K(ret), K(ls_path)); - } else { - } + char stream_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'}; + snprintf(stream_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", tenant_dir, LOG_STREAM_DIR); + if (-1 == FileDirectoryUtils::delete_directory_rec(stream_path)) { + ret = palf::convert_sys_errno(); + CLOG_LOG(ERROR, "::rmdir failed", K(ret), K(stream_path)); + } else if (-1 == ::close(log_stream_fd_)) { + ret = palf::convert_sys_errno(); + CLOG_LOG(ERROR, "::close failed", K(ret), K(stream_path)); } return ret; } void TestServerLogBlockMgr::SetUp() { - OB_ASSERT(OB_SUCCESS == log_block_mgr_.init(log_pool_base_path_)); - log_block_mgr_.get_tenants_log_disk_size_func_ = [this](int64_t &out) -> int + OB_ASSERT(OB_SUCCESS == log_block_mgr_.init(log_disk_base_path_)); + log_block_mgr_.get_tenants_log_disk_size_func_ = [](int64_t &out) -> int { - for(auto pair : map_) - { - out += pair.second; - CLOG_LOG(INFO, "current pair", K(pair.first), K(pair.second)); - } - CLOG_LOG(INFO, "get_tenants_log_disk_size_func_ success", K(out), K(log_block_mgr_)); + out = 0; return OB_SUCCESS; }; } @@ -213,12 +197,11 @@ void TestServerLogBlockMgr::TearDown() using namespace palf; TEST_F(TestServerLogBlockMgr, basic_func) { - const int64_t ls_id = 1; - EXPECT_EQ(OB_SUCCESS, create_new_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 10)); + EXPECT_EQ(OB_SUCCESS, create_new_blocks_at(log_stream_fd_, 0, 10)); int64_t in_use_size_byte; EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte)); EXPECT_EQ(10*ObServerLogBlockMgr::BLOCK_SIZE, in_use_size_byte); - EXPECT_EQ(OB_SUCCESS, delete_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 10)); + EXPECT_EQ(OB_SUCCESS, delete_blocks_at(log_stream_fd_, 0, 10)); EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte)); EXPECT_EQ(0, in_use_size_byte); } @@ -227,81 +210,46 @@ TEST_F(TestServerLogBlockMgr, restart_for_empty_log_disk) { log_block_mgr_.destroy(); int64_t in_use_size_byte; - EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_)); + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_disk_base_path_)); EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte)); EXPECT_EQ(0, in_use_size_byte); } -TEST_F(TestServerLogBlockMgr, allocate_blocks_in_tenant) +TEST_F(TestServerLogBlockMgr, allocate_blocks_in_log_stream) { - const int64_t ls_id = 1; - EXPECT_EQ(OB_SUCCESS, create_new_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 10)); - EXPECT_EQ(OB_SUCCESS, delete_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 3)); + EXPECT_EQ(OB_SUCCESS, create_new_blocks_at(log_stream_fd_, 0, 10)); + EXPECT_EQ(OB_SUCCESS, delete_blocks_at(log_stream_fd_, 0, 3)); } TEST_F(TestServerLogBlockMgr, restart_for_non_empty_log_disk) { log_block_mgr_.destroy(); - EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_)); - const int64_t ls_id = 1; - EXPECT_EQ(OB_SUCCESS, delete_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 3, 7)); + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_disk_base_path_)); + EXPECT_EQ(OB_SUCCESS, delete_blocks_at(log_stream_fd_, 3, 7)); } -TEST_F(TestServerLogBlockMgr, dirty_ls_dir_and_log_pool_file) +TEST_F(TestServerLogBlockMgr, unexpected_root_directory_and_tmp_file) { system("mkdir clog_disk/clog/tenant_0111"); system("mkdir clog_disk/clog/tenant_0111/log"); system("touch clog_disk/clog/tenant_0111/log/0"); system("touch clog_disk/clog/tenant_0111/log/1"); - system("touch clog_disk/clog/sys/1/meta/10000.tmp"); + system("touch clog_disk/clog/sys/log_stream/meta/10000.tmp"); #ifdef __APPLE__ // macOS doesn't have fallocate, use dd instead - system("dd if=/dev/zero of=clog_disk/clog/sys/1/meta/10000.tmp bs=67108863 count=1 2>/dev/null"); + system("dd if=/dev/zero of=clog_disk/clog/sys/log_stream/meta/10000.tmp bs=67108863 count=1 2>/dev/null"); #else - system("fallocate -l 67108863 clog_disk/clog/sys/1/meta/10000.tmp "); + system("fallocate -l 67108863 clog_disk/clog/sys/log_stream/meta/10000.tmp "); #endif log_block_mgr_.destroy(); bool result = false; - EXPECT_EQ(OB_ERR_UNEXPECTED, log_block_mgr_.init(log_pool_base_path_)); - EXPECT_EQ(OB_SUCCESS, FileDirectoryUtils::is_exists("clog_disk/clog/sys/1/meta/10000.tmp",result)); + EXPECT_EQ(OB_ERR_UNEXPECTED, log_block_mgr_.init(log_disk_base_path_)); + EXPECT_EQ(OB_SUCCESS, FileDirectoryUtils::is_exists("clog_disk/clog/sys/log_stream/meta/10000.tmp",result)); EXPECT_EQ(false, result); system("rm -rf clog_disk/clog/tenant_0111"); - EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_)); + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_disk_base_path_)); } - -// This case will cause disk space not enough, ignore it. -//TEST_F(TestServerLogBlockMgr, expand_when_disk_space_not_enough) -//{ -// const int64_t reserved_size = 100 * 1024 * ObServerLogBlockMgr::GB; -// const int64_t aligned_reserved_size = log_block_mgr_.lower_align_(reserved_size); -// ObServerLogBlockMgr::LogPoolMeta origin_meta = log_block_mgr_.get_log_pool_meta_guarded_by_lock_(); -// EXPECT_EQ(OB_ALLOCATE_DISK_SPACE_FAILED, log_block_mgr_.resize_(aligned_reserved_size)); -// EXPECT_EQ(origin_meta, log_block_mgr_.get_log_pool_meta_guarded_by_lock_()); -//} - -// TEST_F(TestServerLogBlockMgr, performance) -// { -// const int64_t reserved_size = 2 * ObServerLogBlockMgr::GB; -// const int64_t free_size = log_block_mgr_.get_free_size_guarded_by_lock_(); -// const int64_t aligned_reserved_size = log_block_mgr_.lower_align_(reserved_size); -// EXPECT_EQ(OB_NOT_SUPPORTED, log_block_mgr_.resize_(aligned_reserved_size)); -// EXPECT_EQ(OB_SUCCESS, log_block_mgr_.resize_(log_block_mgr_.get_total_size_guarded_by_lock_() - free_size)); -// EXPECT_EQ(OB_ENTRY_NOT_EXIST, create_new_blocks_at(1, tenant_ls_fd_map_[1], 100000, 10)); -// -// system("mkdir clog_disk/tmp.dir"); -// const int64_t reserved_size1 = 1024 * ObServerLogBlockMgr::GB; -// const int64_t aligned_reserved_size1 = log_block_mgr_.lower_align_(reserved_size1); -// int64_t start_ts = ObTimeUtility::current_time(); -// int fd = ::open("clog_disk/tmp.dir", ObServerLogBlockMgr::OPEN_DIR_FLAG); -// EXPECT_EQ(OB_SUCCESS, log_block_mgr_.allocate_blocks_at_tmp_dir_(fd, 0, aligned_reserved_size1/ObServerLogBlockMgr::BLOCK_SIZE)); -// int64_t cost_ts = ObTimeUtility::current_time() - start_ts; -// ::close(fd); -// EXPECT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec("clog_disk/tmp.dir")); -// CLOG_LOG(INFO, "allocate cost", K(cost_ts)); -// EXPECT_EQ(OB_SUCCESS, log_block_mgr_.resize_(aligned_reserved_size1)); -// } - TEST_F(TestServerLogBlockMgr, check_dir_is_empty) { system("mkdir clog_disk/test"); diff --git a/unittest/logservice/test_tablet_replay_executor.cpp b/unittest/logservice/test_tablet_replay_executor.cpp index c4bc94bfa..f96ae4b04 100644 --- a/unittest/logservice/test_tablet_replay_executor.cpp +++ b/unittest/logservice/test_tablet_replay_executor.cpp @@ -78,8 +78,7 @@ int ObTabletReplayExecutor::replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletCreateDeleteMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = 123; UNUSEDx(tablet_handle, mds, ctx, scn); @@ -90,8 +89,7 @@ int ObTabletReplayExecutor::replay_to_mds_table_( storage::ObTabletHandle &tablet_handle, const ObTabletBindingMdsUserData &mds, storage::mds::MdsCtx &ctx, - const share::SCN &scn, - const bool for_old_mds) + const share::SCN &scn) { int ret = 456; UNUSEDx(tablet_handle, mds, ctx, scn); diff --git a/unittest/mtlenv/mock_gts_source.h b/unittest/mtlenv/mock_gts_source.h deleted file mode 100644 index 6ad98a213..000000000 --- a/unittest/mtlenv/mock_gts_source.h +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_OB_LTS_SOURCE_ -#define OCEANBASE_TRANSACTION_OB_LTS_SOURCE_ - -#include -#include "lib/utility/ob_print_utils.h" -#include "storage/tx/ob_trans_version_mgr.h" - -#define OB_TRANS_VERSION (ObTransVersionMgr::get_instance()) - -namespace oceanbase -{ -namespace transaction -{ -class MockObGtsSource -{ -public: - MockObGtsSource() {} - ~MockObGtsSource() {} -public: - int update_gts(const int64_t gts, bool &update) - { - int ret = OB_SUCCESS; - if (0 >= gts) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(gts)); - } else if (OB_FAIL(OB_TRANS_VERSION.update_publish_version(gts))) { - TRANS_LOG(WARN, "update publish version failed", K(ret), K(gts)); - } else { - update = false; - } - return ret; - } - int update_local_trans_version(const int64_t version, bool &update) - { - return update_gts(version, update); - } - int get_gts(const MonotonicTs stc, ObTsCbTask *task, share::SCN >s_scn, MonotonicTs &receive_gts_ts) - { - UNUSED(receive_gts_ts); - int ret = OB_SUCCESS; - int64_t gts = 0; - if (!stc.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(stc), KP(task)); - } else if (OB_FAIL(OB_TRANS_VERSION.get_and_update_local_trans_version(gts))) { - TRANS_LOG(WARN, "get local trans version failed", KR(ret)); - } else if (OB_FAIL(gts_scn.convert_for_logservice(gts))) { - TRANS_LOG(WARN, "failed to convert", K(gts), KR(ret)); - } else { - // do nothing - } - return ret; - } - int get_gts(ObTsCbTask *task, share::SCN >s_scn) - { - UNUSED(task); - int ret = OB_SUCCESS; - int64_t gts = 0; - if (OB_FAIL(OB_TRANS_VERSION.get_and_update_local_trans_version(gts))) { - TRANS_LOG(WARN, "get local trans version failed", KR(ret)); - } else if (OB_FAIL(gts_scn.convert_for_logservice(gts))) { - TRANS_LOG(WARN, "failed to convert", K(gts), KR(ret)); - } - return ret; - } - /* int get_local_trans_version(const MonotonicTs stc, ObTsCbTask *task, int64_t &version, MonotonicTs &receive_gts_ts) - { - return get_gts(stc, task, version, receive_gts_ts); - } - int get_local_trans_version(ObTsCbTask *task, int64_t &version) - { - return get_gts(task, version); - } */ - int wait_gts_elapse(const share::SCN &scn, ObTsCbTask *task, bool &need_wait) - { - int ret = OB_SUCCESS; - if (!scn.is_valid() || NULL == task) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(scn), KP(task)); - } else if (OB_FAIL(OB_TRANS_VERSION.update_publish_version(scn.get_val_for_logservice()))) { - TRANS_LOG(WARN, "update publish version failed", KR(ret), K(scn)); - } else { - need_wait = false; - } - return ret; - } - - int wait_gts_elapse(const share::SCN >s) - { - int ret = OB_SUCCESS; - if (!gts.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(gts)); - } else if (OB_FAIL(OB_TRANS_VERSION.update_publish_version(gts.get_val_for_logservice()))) { - TRANS_LOG(WARN, "update publish version failed", KR(ret), K(gts)); - } else { - // do nothing - } - return ret; - } - int refresh_gts(const bool need_refresh) - { - // do nothing - UNUSED(need_refresh); - return OB_SUCCESS; - } - int update_base_ts(const int64_t base_ts) - { - int ret = OB_SUCCESS; - if (0 > base_ts) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(base_ts)); - } else if (OB_FAIL(OB_TRANS_VERSION.update_local_trans_version(base_ts))) { - TRANS_LOG(WARN, "update local trans version failed", KR(ret), K(base_ts)); - } else { - TRANS_LOG(INFO, "update base ts success", K(base_ts)); - } - return ret; - } - int get_base_ts(int64_t &base_ts) - { - int ret = OB_SUCCESS; - if (OB_FAIL(OB_TRANS_VERSION.get_local_trans_version(base_ts))) { - TRANS_LOG(WARN, "get local trans version failed", KR(ret)); - } else { - // do nothing - } - return ret; - } - bool is_external_consistent() { return false; } -public: - TO_STRING_KV("ts_source", "Mock_GTS"); -}; -} -}//end of namespace oceanbase - -#endif //OCEANBASE_TRANSACTION_OB_LTS_SOURCE_ diff --git a/unittest/mtlenv/mock_tenant_module_env.h b/unittest/mtlenv/mock_tenant_module_env.h index afe40a569..ffc462281 100644 --- a/unittest/mtlenv/mock_tenant_module_env.h +++ b/unittest/mtlenv/mock_tenant_module_env.h @@ -58,7 +58,6 @@ #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/compaction/ob_tenant_medium_checker.h" -#include "storage/slog/ob_storage_logger_manager.h" #include "storage/slog/ob_storage_logger.h" #include "storage/compaction/ob_sstable_merge_info_mgr.h" #include "storage/compaction/ob_tenant_freeze_info_mgr.h" @@ -74,7 +73,7 @@ #include "storage/tx/ob_timestamp_service.h" #include "storage/tx/ob_trans_id_service.h" #include "storage/ob_tenant_tablet_stat_mgr.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/ob_file_system_router.h" #include "storage/access/ob_table_scan_iterator.h" #include "mtlenv/ob_mittest_utils.h" @@ -243,7 +242,6 @@ class MockTenantModuleEnv observer::ObSrvNetworkFrame net_frame_; omt::ObMultiTenant multi_tenant_; MockObService ob_service_; - share::ObLocationService location_service_; share::schema::ObMultiVersionSchemaService &schema_service_; sql::ObSql sql_engine_; ObSQLSessionMgr session_mgr_; @@ -459,7 +457,6 @@ void MockTenantModuleEnv::init_gctx_gconf() GCONF.observer_id = 1; GCONF.cluster_id = 1; GCTX.self_addr_seq_.set_addr(self_addr_); - GCTX.location_service_ = &location_service_; GCTX.schema_service_ = &schema_service_; GCTX.net_frame_ = &net_frame_; GCTX.ob_service_ = &ob_service_; @@ -513,11 +510,6 @@ int MockTenantModuleEnv::init_before_start_mtl() STORAGE_LOG(ERROR, "init server checkpoint slog handler fail", K(ret)); } else if (OB_FAIL(multi_tenant_.init(self_addr_, &sql_proxy_, false))) { STORAGE_LOG(WARN, "fail to init env", K(ret)); - } else if (OB_FAIL(ObTsMgr::get_instance().init(self_addr_, - schema_service_, location_service_))) { - STORAGE_LOG(WARN, "fail to init env", K(ret)); - } else if (OB_FAIL(ObTsMgr::get_instance().start())) { - STORAGE_LOG(WARN, "fail to init env", K(ret)); } else if (OB_FAIL(tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache"))) { STORAGE_LOG(WARN, "init tmp block cache failed", KR(ret)); } else if (OB_FAIL(tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache"))) { @@ -605,28 +597,6 @@ int MockTenantModuleEnv::start_() } else if (OB_FAIL(tenant->acquire_more_worker(TENANT_WORKER_COUNT, succ_num))) { } else if (OB_FAIL(guard_.switch_to(tenant))) { // make module set ready in this thread STORAGE_LOG(ERROR, "fail to switch to sys tenant", K(ret)); - } else { - ObLogService *log_service = share::g_mp->log_service(); - if (OB_ISNULL(log_service) || OB_ISNULL(log_service->palf_env_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "fail to switch to sys tenant", KP(log_service)); - } else { - palf::PalfEnvImpl *palf_env_impl = &log_service->palf_env_->palf_env_impl_; - palf::LogIOWorkerWrapper &log_iow_wrapper = palf_env_impl->log_io_worker_wrapper_; - palf::LogIOWorkerConfig new_config; - palf_env_impl->init_log_io_worker_config_(1, new_config); - new_config.io_worker_num_ = 4; - log_iow_wrapper.destory_and_free_log_io_workers_(); - if (OB_FAIL(log_iow_wrapper.create_and_init_log_io_workers_( - new_config, &palf_env_impl->cb_thread_pool_, palf_env_impl->log_alloc_mgr_, palf_env_impl))) { - STORAGE_LOG(WARN, "failed to create_and_init_log_io_workers_", K(new_config)); - } else if (FALSE_IT(log_iow_wrapper.log_writer_parallelism_ = new_config.io_worker_num_)) { - } else if (FALSE_IT(log_iow_wrapper.is_user_tenant_ = true)) { - } else if (OB_FAIL(log_iow_wrapper.start_())) { - STORAGE_LOG(WARN, "failed to start_ log_iow_wrapper", K(new_config)); - } else { - } - } } return ret; } @@ -656,10 +626,6 @@ void MockTenantModuleEnv::destroy() OB_STORAGE_OBJECT_MGR.wait(); OB_STORAGE_OBJECT_MGR.destroy(); - ObTsMgr::get_instance().stop(); - ObTsMgr::get_instance().wait(); - ObTsMgr::get_instance().destroy(); - net_frame_.sql_nio_stop(); net_frame_.stop(); net_frame_.wait(); diff --git a/unittest/share/cache/test_vtable_event_recycle_buffer.cpp b/unittest/share/cache/test_vtable_event_recycle_buffer.cpp index 7428131b9..3f7379b31 100644 --- a/unittest/share/cache/test_vtable_event_recycle_buffer.cpp +++ b/unittest/share/cache/test_vtable_event_recycle_buffer.cpp @@ -178,14 +178,6 @@ TEST_F(TestObVtableEventRecycleBuffer, basic_test) { ASSERT_EQ(OB_SUCCESS, vtable_event_buffer.buffer_bkt_[0].cache_.for_each({0}, for_each_dummy_op)); } -// TEST_F(TestObVtableEventRecycleBuffer, test_global_event_buffer) { -// ASSERT_EQ(OB_SUCCESS, observer::ObMdsEventBuffer::init()); -// observer::MdsEvent e; -// ASSERT_EQ(OB_SUCCESS, e.set_event(DefaultAllocator::get_instance(), "TEST EVENT", ObString("test info str"))); -// observer::ObMdsEventBuffer::append({1, share::ObLSID(1), ObTabletID(1)}, e); -// observer::ObMdsEventBuffer::destroy(); -// } - } } diff --git a/unittest/share/compaction/test_checksum_operator.cpp b/unittest/share/compaction/test_checksum_operator.cpp index 138efa0bd..272afa8bd 100644 --- a/unittest/share/compaction/test_checksum_operator.cpp +++ b/unittest/share/compaction/test_checksum_operator.cpp @@ -29,10 +29,10 @@ TEST(ObPTSqlSplicer, batch) { int ret = OB_SUCCESS; share::ObDMLSqlSplicer splicer; ObSqlString sql; - ObString result("INSERT INTO tname (gmt_modified, gmt_create, tenant_id, tablet_id, svr_ip, svr_port, ls_id, compaction_scn, row_count, data_checksum, column_checksums, b_column_checksums) VALUES " - "(now(6), now(6), 1004, 200001, '127.0.0.1', 1001, 1001, 1234567890, 100, 1234567, 'column_checksums-01', 'b_column_checksums-01')," - "(now(6), now(6), 1004, 200001, '127.0.0.2', 1002, 1001, 1234567890, 100, 1234567, 'column_checksums-02', 'b_column_checksums-02')," - "(now(6), now(6), 1004, 200001, '127.0.0.3', 1003, 1001, 1234567890, 100, 1234567, 'column_checksums-03', 'b_column_checksums-03') " + ObString result("INSERT INTO tname (gmt_modified, gmt_create, tenant_id, tablet_id, svr_ip, svr_port, compaction_scn, row_count, data_checksum, column_checksums, b_column_checksums) VALUES " + "(now(6), now(6), 1004, 200001, '127.0.0.1', 1001, 1234567890, 100, 1234567, 'column_checksums-01', 'b_column_checksums-01')," + "(now(6), now(6), 1004, 200001, '127.0.0.2', 1002, 1234567890, 100, 1234567, 'column_checksums-02', 'b_column_checksums-02')," + "(now(6), now(6), 1004, 200001, '127.0.0.3', 1003, 1234567890, 100, 1234567, 'column_checksums-03', 'b_column_checksums-03') " "ON DUPLICATE KEY UPDATE " "gmt_modified=VALUES(gmt_modified)," "gmt_create=VALUES(gmt_create)," @@ -40,7 +40,6 @@ TEST(ObPTSqlSplicer, batch) { "tablet_id=VALUES(tablet_id)," "svr_ip=VALUES(svr_ip)," "svr_port=VALUES(svr_port)," - "ls_id=VALUES(ls_id)," "compaction_scn=VALUES(compaction_scn)," "row_count=VALUES(row_count)," "data_checksum=VALUES(data_checksum)," @@ -63,7 +62,6 @@ TEST(ObPTSqlSplicer, batch) { ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("tablet_id", 200001)); ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("svr_ip", ip)); ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("svr_port", ips[i].get_port())); - ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("ls_id", 1001)); ASSERT_EQ(OB_SUCCESS, splicer.add_column("compaction_scn", 1234567890)); ASSERT_EQ(OB_SUCCESS, splicer.add_column("row_count", 100)); ASSERT_EQ(OB_SUCCESS, splicer.add_column("data_checksum", 1234567)); diff --git a/unittest/share/scheduler/test_dag_net_in_dag_scheduler.cpp b/unittest/share/scheduler/test_dag_net_in_dag_scheduler.cpp index 16d169f18..12c6a5a7d 100644 --- a/unittest/share/scheduler/test_dag_net_in_dag_scheduler.cpp +++ b/unittest/share/scheduler/test_dag_net_in_dag_scheduler.cpp @@ -794,7 +794,7 @@ class ObFatherDagNet : public ObIDagNet { public: ObFatherDagNet() : - ObIDagNet(ObDagNetType::DAG_NET_TYPE_MIGRATION), + ObIDagNet(ObDagNetType::DAG_NET_TYPE_RESERVED_7), id_(ObTimeUtility::current_time() + random()), op_() {} @@ -1808,7 +1808,7 @@ class ObLoopDagNet : public ObIDagNet { public: ObLoopDagNet() : - ObIDagNet(ObDagNetType::DAG_NET_TYPE_MIGRATION), + ObIDagNet(ObDagNetType::DAG_NET_TYPE_RESERVED_7), id_(ObTimeUtility::current_time() + random()), op_() {} diff --git a/unittest/share/scheduler/test_dag_scheduler.cpp b/unittest/share/scheduler/test_dag_scheduler.cpp index 1461bc37a..59ecee1de 100644 --- a/unittest/share/scheduler/test_dag_scheduler.cpp +++ b/unittest/share/scheduler/test_dag_scheduler.cpp @@ -1049,78 +1049,17 @@ TEST_F(TestDagScheduler, test_emergency_task) EXPECT_EQ(3, op.value()); } -class TestCompMidCancelDag : public compaction::ObTabletMergeDag +class TestTabletMergeDag : public compaction::ObTabletMergeDag { public: - TestCompMidCancelDag() + TestTabletMergeDag() : compaction::ObTabletMergeDag(ObDagType::DAG_TYPE_MERGE_EXECUTE){} - virtual const share::ObLSID & get_ls_id() const override { return ls_id_; } virtual lib::Worker::CompatMode get_compat_mode() const override { return lib::Worker::CompatMode::MYSQL; } private: - DISALLOW_COPY_AND_ASSIGN(TestCompMidCancelDag); + DISALLOW_COPY_AND_ASSIGN(TestTabletMergeDag); }; -TEST_F(TestDagScheduler, test_check_ls_compaction_dag_exist_with_cancel) -{ - ObTenantDagScheduler *scheduler = MTL(ObTenantDagScheduler*); - ASSERT_TRUE(nullptr != scheduler); - ASSERT_EQ(OB_SUCCESS, scheduler->init( time_slice, 64)); - EXPECT_EQ(OB_SUCCESS, scheduler->set_thread_score(ObDagPrio::DAG_PRIO_COMPACTION_MID, 1)); - EXPECT_EQ(OB_SUCCESS, scheduler->set_thread_score(ObDagPrio::DAG_PRIO_COMPACTION_LOW, 1)); - EXPECT_EQ(1, scheduler->prio_sche_[ObDagPrio::DAG_PRIO_COMPACTION_MID].limits_); - EXPECT_EQ(1, scheduler->prio_sche_[ObDagPrio::DAG_PRIO_COMPACTION_LOW].limits_); - - LoopWaitTask *wait_task = nullptr; - LoopWaitTask *wait_task2 = nullptr; - const int64_t dag_cnt = 6; - // add 6 dag at prio = DAG_PRIO_COMPACTION_MID - const int64_t ls_cnt = 2; - ObLSID ls_ids[ls_cnt] = {ObLSID(1), ObLSID(2)}; - bool finish_flag[ls_cnt] = {false, false}; - for (int64_t i = 0; i < dag_cnt; ++i) { - const int64_t idx = i % ls_cnt; - TestCompMidCancelDag *dag = NULL; - EXPECT_EQ(OB_SUCCESS, scheduler->alloc_dag(dag)); - dag->ls_id_ = ls_ids[idx]; - dag->tablet_id_ = ObTabletID(i); - EXPECT_EQ(OB_SUCCESS, alloc_task(*dag, wait_task)); - EXPECT_EQ(OB_SUCCESS, wait_task->init(1, 2, finish_flag[idx])); - EXPECT_EQ(OB_SUCCESS, dag->add_task(*wait_task)); - EXPECT_EQ(OB_SUCCESS, scheduler->add_dag(dag)); - } - // add 2 dag at prio = DAG_PRIO_COMPACTION_LOW - for (int64_t i = 0; i < ls_cnt; ++i) { - ObBatchFreezeTabletsDag *batch_freeze_dag = NULL; - EXPECT_EQ(OB_SUCCESS, scheduler->alloc_dag(batch_freeze_dag)); - batch_freeze_dag->param_.ls_id_ = ls_ids[i]; - EXPECT_EQ(OB_SUCCESS, alloc_task(*batch_freeze_dag, wait_task2)); - EXPECT_EQ(OB_SUCCESS, wait_task2->init(1, 2, finish_flag[i])); - EXPECT_EQ(OB_SUCCESS, batch_freeze_dag->add_task(*wait_task2)); - EXPECT_EQ(OB_SUCCESS, scheduler->add_dag(batch_freeze_dag)); - } - EXPECT_EQ(dag_cnt, scheduler->dag_cnts_[ObDagType::DAG_TYPE_MERGE_EXECUTE]); - CHECK_EQ_UTIL_TIMEOUT(1, scheduler->get_running_task_cnt(ObDagPrio::DAG_PRIO_COMPACTION_MID)); - CHECK_EQ_UTIL_TIMEOUT(1, scheduler->get_running_task_cnt(ObDagPrio::DAG_PRIO_COMPACTION_LOW)); - - // cancel waiting dag of ls_ids[0], all dag of ls_ids[1] will be destroyed when check_cancel - bool exist = false; - EXPECT_EQ(OB_SUCCESS, scheduler->check_ls_compaction_dag_exist_with_cancel(ls_ids[0], exist)); - EXPECT_EQ(exist, true); - EXPECT_EQ(4, scheduler->dag_cnts_[ObDagType::DAG_TYPE_MERGE_EXECUTE]); - - EXPECT_EQ(OB_SUCCESS, scheduler->check_ls_compaction_dag_exist_with_cancel(ls_ids[1], exist)); - EXPECT_EQ(exist, false); - - finish_flag[0] = true; - wait_scheduler(); - - EXPECT_EQ(OB_SUCCESS, scheduler->check_ls_compaction_dag_exist_with_cancel(ls_ids[0], exist)); - EXPECT_EQ(exist, false); - EXPECT_EQ(OB_SUCCESS, scheduler->check_ls_compaction_dag_exist_with_cancel(ls_ids[0], exist)); - EXPECT_EQ(exist, false); -} - TEST_F(TestDagScheduler, test_cancel_running_dag) { ObTenantDagScheduler *scheduler = MTL(ObTenantDagScheduler*); @@ -1279,15 +1218,13 @@ TEST_F(TestDagScheduler, DISABLED_test_max_concurrent_task) EXPECT_EQ(7, scheduler->prio_sche_[ObDagPrio::DAG_PRIO_COMPACTION_MID].limits_); const int64_t dag_cnt = 3; - ObLSID ls_id(1001); bool finish_flag[dag_cnt] = {false, false, false}; for (int64_t idx = 0; idx < dag_cnt; ++idx) { - TestCompMidCancelDag *dag = nullptr; + TestTabletMergeDag *dag = nullptr; LoopWaitTask *wait_task = nullptr; ObTabletID tablet_id(200001 + idx); EXPECT_EQ(OB_SUCCESS, scheduler->alloc_dag(dag)); dag->max_concurrent_task_cnt_ = 2; - dag->ls_id_ = ls_id; dag->tablet_id_ = tablet_id; EXPECT_EQ(OB_SUCCESS, alloc_task(*dag, wait_task)); EXPECT_EQ(OB_SUCCESS, wait_task->init(1, 10 /*cnt*/, finish_flag[idx])); diff --git a/unittest/sql/engine/test_op_engine.cpp b/unittest/sql/engine/test_op_engine.cpp index f43225bf5..91edebf19 100644 --- a/unittest/sql/engine/test_op_engine.cpp +++ b/unittest/sql/engine/test_op_engine.cpp @@ -57,7 +57,6 @@ void TestOpEngine::SetUp() ASSERT_EQ(prepare_io(ObTestOpConfig::get_instance().test_filename_prefix_), OB_SUCCESS); // init mock location service, used in optimizer compute table property - GCTX.location_service_ = &mock_location_service_; // Single-tenant seekdb: low-layer modules are reached through share::g_mp // (ObIModuleProvider). Publish a test provider exposing the ObDataAccessService // (used in ObTableScanOp ctor) and the ObTenantIOManager. @@ -189,7 +188,8 @@ int TestOpEngine::prepare_io(const std::string & test_data_name_suffix) LOG_WARN("add device channel failed", K(ret)); } else if (OB_FAIL(SERVER_STORAGE_META_SERVICE.init())) { LOG_WARN("fail to init storage meta service", K(ret)); - } else if (FALSE_IT(SERVER_STORAGE_META_SERVICE.get_slogger_manager().need_reserved_ = false)) { + } else if (OB_FAIL(SERVER_STORAGE_META_SERVICE.set_need_reserved_for_test(false))) { + LOG_WARN("fail to set need reserved for test", K(ret)); } else if (OB_FAIL(OB_STORAGE_OBJECT_MGR.init(storage_env_.default_block_size_))) { LOG_WARN("init block manager fail", K(ret)); } else if (OB_FAIL(ObIOManager::get_instance().start())) { diff --git a/unittest/sql/engine/test_op_engine.h b/unittest/sql/engine/test_op_engine.h index 429fc3643..1cf572648 100644 --- a/unittest/sql/engine/test_op_engine.h +++ b/unittest/sql/engine/test_op_engine.h @@ -22,39 +22,6 @@ #include "storage/blocksstable/ob_block_sstable_struct.h" namespace test { -class MockLocationService : public share::ObLocationService -{ -public: - MockLocationService() - {} - virtual ~MockLocationService() - {} - - virtual int nonblock_get(const ObTabletID &tablet_id, ObLSID &ls_id) - { - ls_id = ObLSID::SYS_LS_ID; - return OB_SUCCESS; - } - - virtual int nonblock_get(const int64_t cluster_id, const ObLSID &ls_id, - ObLSLocation &location) - { - int ret = OB_SUCCESS; - - ObAddr add; - ObReplicaProperty relica_pro; - ObLSRestoreStatus ls_restore_sta; - add.set_ip_addr("1.1.1.1", 8888); - ObLSReplicaLocation ls_replica_loc; - - if (OB_FAIL(location.init(1, ls_id, 1))) { - } else if (OB_FAIL(ls_replica_loc.init(add, LEADER, 100, REPLICA_TYPE_FULL, relica_pro, ls_restore_sta, 100))) { - } else { - location.add_replica_location(ls_replica_loc); - } - return ret; - } -}; class TestOpEngine : public TestOptimizerUtils { public: @@ -102,7 +69,6 @@ class TestOpEngine : public TestOptimizerUtils std::vector> temp_cmp_data_; ObTenantBase tbase_; - MockLocationService mock_location_service_; // std::string test_config_file_; std::string env_dir_; diff --git a/unittest/sql/parser/print_parser_tree.result b/unittest/sql/parser/print_parser_tree.result index 6517fcab0..9428bed4c 100644 --- a/unittest/sql/parser/print_parser_tree.result +++ b/unittest/sql/parser/print_parser_tree.result @@ -1191,74 +1191,6 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[id], value=[9223372036854775807] ************** Case 60 *************** -alter system report replica; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REPORT_REPLICA], str_value_=[], value=[0] - -************** Case 61 *************** -alter system report replica server = '127.0.0.1:80'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REPORT_REPLICA], str_value_=[], value=[0] - |--[0],[T_IP_PORT], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=["127.0.0.1":80], value=[9223372036854775807] - -************** Case 62 *************** -alter system report replica zone = 'z1'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REPORT_REPLICA], str_value_=[], value=[0] - |--[0],[T_ZONE], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[z1], value=[9223372036854775807] - -************** Case 63 *************** -alter system recycle replica; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_RECYCLE_REPLICA], str_value_=[], value=[0] - -************** Case 64 *************** -alter system recycle replica server = '127.0.0.1:80'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_RECYCLE_REPLICA], str_value_=[], value=[0] - |--[0],[T_IP_PORT], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=["127.0.0.1":80], value=[9223372036854775807] - -************** Case 65 *************** -alter system recycle replica server '127.0.0.1:80'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_RECYCLE_REPLICA], str_value_=[], value=[0] - |--[0],[T_IP_PORT], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=["127.0.0.1":80], value=[9223372036854775807] - -************** Case 66 *************** -alter system recycle replica zone = 'z1'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_RECYCLE_REPLICA], str_value_=[], value=[0] - |--[0],[T_ZONE], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[z1], value=[9223372036854775807] - -************** Case 67 *************** -alter system recycle replica zone 'z1'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_RECYCLE_REPLICA], str_value_=[], value=[0] - |--[0],[T_ZONE], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[z1], value=[9223372036854775807] - -************** Case 68 *************** alter system major freeze; question_mask_size: 0 @@ -1266,7 +1198,7 @@ question_mask_size: 0 |--[0],[T_FREEZE], str_value_=[], value=[0] |--[0],[T_INT], str_value_=[], value=[1] -************** Case 69 *************** +************** Case 61 *************** alter system start merge zone = 'z1'; question_mask_size: 0 @@ -1276,7 +1208,7 @@ question_mask_size: 0 |--[1],[T_ZONE], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[z1], value=[9223372036854775807] -************** Case 70 *************** +************** Case 62 *************** alter system suspend merge; question_mask_size: 0 @@ -1284,7 +1216,7 @@ question_mask_size: 0 |--[0],[T_MERGE_CONTROL], str_value_=[], value=[0] |--[0],[T_INT], str_value_=[], value=[2] -************** Case 71 *************** +************** Case 63 *************** alter system suspend merge tenant = sys; question_mask_size: 0 @@ -1294,7 +1226,7 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[sys], value=[9223372036854775807] -************** Case 72 *************** +************** Case 64 *************** alter system suspend merge tenant = all_user; question_mask_size: 0 @@ -1304,7 +1236,7 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[all_user], value=[9223372036854775807] -************** Case 73 *************** +************** Case 65 *************** alter system suspend merge tenant = all_meta; question_mask_size: 0 @@ -1314,7 +1246,7 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[all_meta], value=[9223372036854775807] -************** Case 74 *************** +************** Case 66 *************** alter system resume merge; question_mask_size: 0 @@ -1322,7 +1254,7 @@ question_mask_size: 0 |--[0],[T_MERGE_CONTROL], str_value_=[], value=[0] |--[0],[T_INT], str_value_=[], value=[3] -************** Case 75 *************** +************** Case 67 *************** alter system resume merge tenant = sys; question_mask_size: 0 @@ -1332,7 +1264,7 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[sys], value=[9223372036854775807] -************** Case 76 *************** +************** Case 68 *************** alter system resume merge tenant = all_user; question_mask_size: 0 @@ -1342,7 +1274,7 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[all_user], value=[9223372036854775807] -************** Case 77 *************** +************** Case 69 *************** alter system resume merge tenant = all_meta; question_mask_size: 0 @@ -1352,14 +1284,14 @@ question_mask_size: 0 |--[1],[T_TENANT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[all_meta], value=[9223372036854775807] -************** Case 78 *************** +************** Case 70 *************** alter system clear roottable; question_mask_size: 0 |--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] |--[0],[T_CLEAR_ROOT_TABLE], str_value_=[], value=[0] -************** Case 79 *************** +************** Case 71 *************** alter system clear roottable tenant = 'xxx'; question_mask_size: 0 @@ -1368,7 +1300,7 @@ question_mask_size: 0 |--[0],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[xxx], value=[9223372036854775807] -************** Case 80 *************** +************** Case 72 *************** alter system modify zone 'z1' set region 'r1' question_mask_size: 0 @@ -1379,9 +1311,9 @@ question_mask_size: 0 |--[2],[T_LINK_NODE], str_value_=[], value=[9223372036854775807] |--[0],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 81 *************** +************** Case 73 *************** alter system start zone 'z1' region 'r1' -************** Case 82 *************** +************** Case 74 *************** alter system modify zone 'z1' IDC 'idc1' question_mask_size: 0 @@ -1392,7 +1324,7 @@ question_mask_size: 0 |--[2],[T_LINK_NODE], str_value_=[], value=[9223372036854775807] |--[0],[T_IDC], str_value_=[idc1], value=[9223372036854775807] -************** Case 83 *************** +************** Case 75 *************** alter system modify zone 'z1' set IDC 'idc1', region 'r1' question_mask_size: 0 @@ -1404,7 +1336,7 @@ question_mask_size: 0 |--[0],[T_IDC], str_value_=[idc1], value=[9223372036854775807] |--[1],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 84 *************** +************** Case 76 *************** alter system add zone 'z1' idc 'idc1', region 'r1' question_mask_size: 0 @@ -1416,7 +1348,7 @@ question_mask_size: 0 |--[0],[T_IDC], str_value_=[idc1], value=[9223372036854775807] |--[1],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 85 *************** +************** Case 77 *************** alter system add zone 'z1' idc 'idc1' question_mask_size: 0 @@ -1427,7 +1359,7 @@ question_mask_size: 0 |--[2],[T_LINK_NODE], str_value_=[], value=[9223372036854775807] |--[0],[T_IDC], str_value_=[idc1], value=[9223372036854775807] -************** Case 86 *************** +************** Case 78 *************** alter system modify zone 'z1' question_mask_size: 0 @@ -1436,11 +1368,11 @@ question_mask_size: 0 |--[0],[T_INT], str_value_=[], value=[5] |--[1],[T_VARCHAR], str_value_=[z1], value=[9223372036854775807] -************** Case 87 *************** +************** Case 79 *************** alter system start zone 'z1' IDC 'idc1' -************** Case 88 *************** +************** Case 80 *************** alter system start zone 'z1' zone_type 'ReadWrite' -************** Case 89 *************** +************** Case 81 *************** alter system add zone 'z1' zone_type 'ReadWrite', idc 'idc1', region 'r1' question_mask_size: 0 @@ -1453,7 +1385,7 @@ question_mask_size: 0 |--[1],[T_IDC], str_value_=[idc1], value=[9223372036854775807] |--[2],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 90 *************** +************** Case 82 *************** alter system add zone 'z1' zone_type 'ReadWrite', idc 'idc1' question_mask_size: 0 @@ -1465,7 +1397,7 @@ question_mask_size: 0 |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] |--[1],[T_IDC], str_value_=[idc1], value=[9223372036854775807] -************** Case 91 *************** +************** Case 83 *************** alter system add zone 'z1' zone_type 'ReadWrite', region 'r1' question_mask_size: 0 @@ -1477,7 +1409,7 @@ question_mask_size: 0 |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] |--[1],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 92 *************** +************** Case 84 *************** alter system add zone 'z1' zone_type 'ReadWrite' question_mask_size: 0 @@ -1488,7 +1420,7 @@ question_mask_size: 0 |--[2],[T_LINK_NODE], str_value_=[], value=[9223372036854775807] |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] -************** Case 93 *************** +************** Case 85 *************** alter system modify zone 'z1' zone_type 'ReadWrite', idc 'idc1', region 'r1' question_mask_size: 0 @@ -1501,7 +1433,7 @@ question_mask_size: 0 |--[1],[T_IDC], str_value_=[idc1], value=[9223372036854775807] |--[2],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 94 *************** +************** Case 86 *************** alter system modify zone 'z1' zone_type 'ReadWrite', idc 'idc1' question_mask_size: 0 @@ -1513,7 +1445,7 @@ question_mask_size: 0 |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] |--[1],[T_IDC], str_value_=[idc1], value=[9223372036854775807] -************** Case 95 *************** +************** Case 87 *************** alter system modify zone 'z1' zone_type 'ReadWrite', region 'r1' question_mask_size: 0 @@ -1525,7 +1457,7 @@ question_mask_size: 0 |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] |--[1],[T_REGION], str_value_=[r1], value=[9223372036854775807] -************** Case 96 *************** +************** Case 88 *************** alter system modify zone 'z1' zone_type 'ReadWrite' question_mask_size: 0 @@ -1536,7 +1468,7 @@ question_mask_size: 0 |--[2],[T_LINK_NODE], str_value_=[], value=[9223372036854775807] |--[0],[T_ZONE_TYPE], str_value_=[ReadWrite], value=[9223372036854775807] -************** Case 97 *************** +************** Case 89 *************** select * from t1 where c1>ANY(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1569,7 +1501,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 98 *************** +************** Case 90 *************** select * from t1 where c1>SOME(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1602,7 +1534,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 99 *************** +************** Case 91 *************** select * from t1 where c1>ALL(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1635,7 +1567,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 100 *************** +************** Case 92 *************** select * from t1 where c1>(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1667,7 +1599,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 101 *************** +************** Case 93 *************** select * from t1 where c11); question_mask_size: 0 @@ -1700,7 +1632,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 102 *************** +************** Case 94 *************** select * from t1 where c11); question_mask_size: 0 @@ -1733,7 +1665,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 103 *************** +************** Case 95 *************** select * from t1 where c11); question_mask_size: 0 @@ -1766,7 +1698,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 104 *************** +************** Case 96 *************** select * from t1 where c1<(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1798,7 +1730,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 105 *************** +************** Case 97 *************** select * from t1 where c1>=ANY(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1831,7 +1763,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 106 *************** +************** Case 98 *************** select * from t1 where c1>=SOME(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1864,7 +1796,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 107 *************** +************** Case 99 *************** select * from t1 where c1>=ALL(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1897,7 +1829,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 108 *************** +************** Case 100 *************** select * from t1 where c1>=(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1929,7 +1861,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 109 *************** +************** Case 101 *************** select * from t1 where c1<=ANY(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1962,7 +1894,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 110 *************** +************** Case 102 *************** select * from t1 where c1<=SOME(select c1 from t2 where c2>1); question_mask_size: 0 @@ -1995,7 +1927,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 111 *************** +************** Case 103 *************** select * from t1 where c1<=ALL(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2028,7 +1960,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 112 *************** +************** Case 104 *************** select * from t1 where c1<=(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2060,7 +1992,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 113 *************** +************** Case 105 *************** select * from t1 where c1=ANY(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2093,7 +2025,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 114 *************** +************** Case 106 *************** select * from t1 where c1=SOME(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2126,7 +2058,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 115 *************** +************** Case 107 *************** select * from t1 where c1=ALL(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2159,7 +2091,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 116 *************** +************** Case 108 *************** select * from t1 where c1=(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2191,7 +2123,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 117 *************** +************** Case 109 *************** select * from t1 where c1!=ANY(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2224,7 +2156,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 118 *************** +************** Case 110 *************** select * from t1 where c1!=SOME(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2257,7 +2189,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 119 *************** +************** Case 111 *************** select * from t1 where c1!=ALL(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2290,7 +2222,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 120 *************** +************** Case 112 *************** select * from t1 where c1!=(select c1 from t2 where c2>1); question_mask_size: 0 @@ -2322,7 +2254,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 121 *************** +************** Case 113 *************** select * from t1 where c1 in (select c1 from t2 where c2>1); question_mask_size: 0 @@ -2354,7 +2286,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 122 *************** +************** Case 114 *************** select * from t1 where c1 not in (select c1 from t2 where c2>1); question_mask_size: 0 @@ -2386,7 +2318,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 123 *************** +************** Case 115 *************** select * from t1 where exists (select c1 from t2 where c2>1); question_mask_size: 0 @@ -2416,7 +2348,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 124 *************** +************** Case 116 *************** select * from t1 where not exists (select c1 from t2 where c2>1); question_mask_size: 0 @@ -2447,7 +2379,7 @@ question_mask_size: 0 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_INT], str_value_=[1], value=[1] -************** Case 125 *************** +************** Case 117 *************** select * from t1 where (select c1 from t1) like (select c2 from t2); question_mask_size: 0 @@ -2481,7 +2413,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 126 *************** +************** Case 118 *************** select * from t1 where (select c1 from t1) not like (select c2 from t2); question_mask_size: 0 @@ -2515,7 +2447,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 127 *************** +************** Case 119 *************** select * from t1 where (c1) in (select c1 from t2); question_mask_size: 0 @@ -2542,7 +2474,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 128 *************** +************** Case 120 *************** select * from t1 where (c1, c2) in (select c1, c2 from t2); question_mask_size: 0 @@ -2575,7 +2507,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 129 *************** +************** Case 121 *************** select * from t1 where ROW(c1, c2) in (select c1, c2 from t2); question_mask_size: 0 @@ -2608,7 +2540,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 130 *************** +************** Case 122 *************** set names latin1; question_mask_size: 0 @@ -2617,7 +2549,7 @@ question_mask_size: 0 |--[0],[T_SET_NAMES], str_value_=[], value=[9223372036854775807] |--[0],[T_CHAR_CHARSET], str_value_=[latin1], value=[9223372036854775807] -************** Case 131 *************** +************** Case 123 *************** set names 'latin1'; question_mask_size: 0 @@ -2626,7 +2558,7 @@ question_mask_size: 0 |--[0],[T_SET_NAMES], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[latin1], value=[9223372036854775807] -************** Case 132 *************** +************** Case 124 *************** set names utf8 collate 'utf8_general_ci'; question_mask_size: 0 @@ -2636,7 +2568,7 @@ question_mask_size: 0 |--[0],[T_CHAR_CHARSET], str_value_=[utf8], value=[9223372036854775807] |--[1],[T_COLLATION], str_value_=[utf8_general_ci], value=[9223372036854775807] -************** Case 133 *************** +************** Case 125 *************** set names utf8 collate utf8_general_ci; question_mask_size: 0 @@ -2646,7 +2578,7 @@ question_mask_size: 0 |--[0],[T_CHAR_CHARSET], str_value_=[utf8], value=[9223372036854775807] |--[1],[T_COLLATION], str_value_=[utf8_general_ci], value=[9223372036854775807] -************** Case 134 *************** +************** Case 126 *************** set character set utf8; question_mask_size: 0 @@ -2655,7 +2587,7 @@ question_mask_size: 0 |--[0],[T_SET_CHARSET], str_value_=[], value=[9223372036854775807] |--[0],[T_CHAR_CHARSET], str_value_=[utf8], value=[9223372036854775807] -************** Case 135 *************** +************** Case 127 *************** set character set 'utf8'; question_mask_size: 0 @@ -2664,7 +2596,7 @@ question_mask_size: 0 |--[0],[T_SET_CHARSET], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[utf8], value=[9223372036854775807] -************** Case 136 *************** +************** Case 128 *************** set charset utf8; question_mask_size: 0 @@ -2673,7 +2605,7 @@ question_mask_size: 0 |--[0],[T_SET_CHARSET], str_value_=[], value=[9223372036854775807] |--[0],[T_CHAR_CHARSET], str_value_=[utf8], value=[9223372036854775807] -************** Case 137 *************** +************** Case 129 *************** select _utf8 'abc', _utf8mb4 'def' collate utf8mb4_general_ci from t1 where c1 collate utf8_bin = 'xyz' collate utf8_bin; question_mask_size: 0 @@ -2711,7 +2643,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[xyz], value=[9223372036854775807] |--[1],[T_COLLATION], str_value_=[utf8_bin], value=[9223372036854775807] -************** Case 138 *************** +************** Case 130 *************** select * from t1 where c1=?; question_mask_size: 1 @@ -2730,7 +2662,7 @@ question_mask_size: 1 |--[2],[T_IDENT], str_value_=[c1], value=[9223372036854775807] |--[1],[T_QUESTIONMARK], str_value_=[], value=[0] -************** Case 139 *************** +************** Case 131 *************** select * from t1 where c1>?; question_mask_size: 1 @@ -2749,7 +2681,7 @@ question_mask_size: 1 |--[2],[T_IDENT], str_value_=[c1], value=[9223372036854775807] |--[1],[T_QUESTIONMARK], str_value_=[], value=[0] -************** Case 140 *************** +************** Case 132 *************** select * from t1 where (select c1 from t1 where c1 = ?) not like (select c2 from t2 where c2=?); question_mask_size: 2 @@ -2793,7 +2725,7 @@ question_mask_size: 2 |--[2],[T_IDENT], str_value_=[c2], value=[9223372036854775807] |--[1],[T_QUESTIONMARK], str_value_=[], value=[1] -************** Case 141 *************** +************** Case 133 *************** select * from t1 join t2; question_mask_size: 0 @@ -2812,7 +2744,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 142 *************** +************** Case 134 *************** select * from t1 inner join t2; question_mask_size: 0 @@ -2831,7 +2763,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 143 *************** +************** Case 135 *************** select * from t1 cross join t2; question_mask_size: 0 @@ -2850,7 +2782,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 144 *************** +************** Case 136 *************** select * from t1 cross join t2 join t3; question_mask_size: 0 @@ -2874,7 +2806,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t3], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t3], value=[9223372036854775807] -************** Case 145 *************** +************** Case 137 *************** select "1234"; question_mask_size: 0 @@ -2885,7 +2817,7 @@ question_mask_size: 0 |--[0],[T_VARCHAR], str_value_=[1234], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[1234], value=[9223372036854775807] -************** Case 146 *************** +************** Case 138 *************** select '1234'; question_mask_size: 0 @@ -2896,7 +2828,7 @@ question_mask_size: 0 |--[0],[T_VARCHAR], str_value_=[1234], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[1234], value=[9223372036854775807] -************** Case 147 *************** +************** Case 139 *************** create table test(c1 varchar(3) binary charset utf8mb4); question_mask_size: 0 @@ -2912,7 +2844,7 @@ question_mask_size: 0 |--[0],[T_CHARSET], str_value_=[utf8mb4], value=[9223372036854775807] |--[2],[T_BINARY], str_value_=[], value=[1] -************** Case 148 *************** +************** Case 140 *************** replace into test values(1,2); question_mask_size: 0 @@ -2929,7 +2861,7 @@ question_mask_size: 0 |--[1],[T_INT], str_value_=[2], value=[2] |--[1],[T_REPLACE], str_value_=[], value=[9223372036854775807] -************** Case 149 *************** +************** Case 141 *************** replace ignore into test values(1,2); question_mask_size: 0 @@ -2947,7 +2879,7 @@ question_mask_size: 0 |--[1],[T_REPLACE], str_value_=[], value=[9223372036854775807] |--[3],[T_IGNORE], str_value_=[], value=[9223372036854775807] -************** Case 150 *************** +************** Case 142 *************** insert ignore into test values(1,2); question_mask_size: 0 @@ -2965,7 +2897,7 @@ question_mask_size: 0 |--[1],[T_INSERT], str_value_=[], value=[9223372036854775807] |--[3],[T_IGNORE], str_value_=[], value=[9223372036854775807] -************** Case 151 *************** +************** Case 143 *************** insert ignore into test values(1,2) on duplicate key update c2 = c1 + 1; question_mask_size: 0 @@ -2991,7 +2923,7 @@ question_mask_size: 0 |--[1],[T_INSERT], str_value_=[], value=[9223372036854775807] |--[3],[T_IGNORE], str_value_=[], value=[9223372036854775807] -************** Case 152 *************** +************** Case 144 *************** create outline out_name on select * from t1; question_mask_size: 0 @@ -3010,7 +2942,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 153 *************** +************** Case 145 *************** create or replace outline out_name on select * from t1; question_mask_size: 0 @@ -3030,7 +2962,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 154 *************** +************** Case 146 *************** create outline out_name on select * from t1; question_mask_size: 0 @@ -3049,7 +2981,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 155 *************** +************** Case 147 *************** create outline out_name on select /*+ssssss*/* from t1; question_mask_size: 0 @@ -3068,7 +3000,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 156 *************** +************** Case 148 *************** create outline out_name on select /*+ssssss*/* from t1 to select /*+ssssss*/* from t1; question_mask_size: 0 @@ -3095,7 +3027,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 157 *************** +************** Case 149 *************** create outline out_name on select /*+ssssss*/* from t1 to select * from t1; question_mask_size: 0 @@ -3122,7 +3054,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 158 *************** +************** Case 150 *************** create outline out_name on insert /*+ssssss*/ into t1 values(1, 2) to insert /*+ssssss*/ into t1 values(1, 2); question_mask_size: 0 @@ -3155,7 +3087,7 @@ question_mask_size: 0 |--[1],[T_INSERT], str_value_=[], value=[9223372036854775807] |--[5],[T_INT], str_value_=[], value=[0] -************** Case 159 *************** +************** Case 151 *************** drop outline out_name; question_mask_size: 0 @@ -3165,7 +3097,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[out_name], value=[9223372036854775807] |--[1],[T_INT], str_value_=[], value=[0] -************** Case 160 *************** +************** Case 152 *************** drop outline db_name.out_name; question_mask_size: 0 @@ -3176,7 +3108,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[out_name], value=[9223372036854775807] |--[1],[T_INT], str_value_=[], value=[0] -************** Case 161 *************** +************** Case 153 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1') set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3196,7 +3128,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 162 *************** +************** Case 154 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1') set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3217,7 +3149,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 163 *************** +************** Case 155 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), locality = 'auto_locality_strategy' set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3239,7 +3171,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 164 *************** +************** Case 156 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), locality = 'auto_locality_strategy' set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3262,7 +3194,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 165 *************** +************** Case 157 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), logonly_replica_num = 1 set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3284,7 +3216,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 166 *************** +************** Case 158 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), logonly_replica_num = 1 set ob_tcp_invited_nodes='%'; question_mask_size: 0 @@ -3307,7 +3239,7 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] |--[1],[T_VARCHAR], str_value_=[%], value=[9223372036854775807] -************** Case 167 *************** +************** Case 159 *************** select /*+max_concurrent(-10)*/* from t1; question_mask_size: 0 @@ -3322,7 +3254,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[19],[T_HINT_OPTION_LIST], str_value_=[], value=[9223372036854775807] -************** Case 168 *************** +************** Case 160 *************** select /*+max_concurrent(10)*/* from t1; question_mask_size: 0 @@ -3339,25 +3271,25 @@ question_mask_size: 0 |--[0],[T_MAX_CONCURRENT], str_value_=[], value=[9223372036854775807] |--[0],[T_INT], str_value_=[10], value=[10] -************** Case 169 *************** +************** Case 161 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 170 *************** +************** Case 162 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales where t2.id!= t1.id when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 171 *************** +************** Case 163 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales where t1.id < 3 delete where t1.id < 5 when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 172 *************** +************** Case 164 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales delete where t1.sales =2 when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 173 *************** +************** Case 165 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales) where t2.id > 0; -************** Case 174 *************** +************** Case 166 *************** merge into targetTable t1 using sourceTable t2 on (1 = 2) when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales) where t2.id > 0; -************** Case 175 *************** +************** Case 167 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales; -************** Case 176 *************** +************** Case 168 *************** merge into targetTable using sourceTable on (targetTable.id = sourceTable.id) when matched then update set targetTable.sales = sourceTable.sales; -************** Case 177 *************** +************** Case 169 *************** merge into targetTable using (select * from t1 ) sourceTable on (targetTable.id = sourceTable.id) when matched then update set targetTable.sales = sourceTable.sales; -************** Case 178 *************** +************** Case 170 *************** select unique(c1), c2 from t1; question_mask_size: 0 @@ -3377,7 +3309,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] -************** Case 179 *************** +************** Case 171 *************** select unique(c1+1), c2 from t1; question_mask_size: 0 @@ -3399,7 +3331,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] -************** Case 180 *************** +************** Case 172 *************** select distinct unique( max(c1)), c2 from t1; question_mask_size: 0 @@ -3421,7 +3353,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] -************** Case 181 *************** +************** Case 173 *************** select unique distinct( max(c1)), c2 from t1; question_mask_size: 0 @@ -3443,9 +3375,9 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] -************** Case 182 *************** +************** Case 174 *************** select unique(*) from t1; -************** Case 183 *************** +************** Case 175 *************** alter tablegroup tg2 add table t1,t2; question_mask_size: 0 @@ -3458,7 +3390,7 @@ question_mask_size: 0 |--[1],[T_RELATION_FACTOR], str_value_=[t2], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t2], value=[9223372036854775807] -************** Case 184 *************** +************** Case 176 *************** insert into t1 values(X''); question_mask_size: 0 @@ -3474,7 +3406,7 @@ question_mask_size: 0 |--[0],[T_HEX_STRING], str_value_=[], value=[9223372036854775807] |--[1],[T_INSERT], str_value_=[], value=[9223372036854775807] -************** Case 185 *************** +************** Case 177 *************** create table t1(`thedate` date NOT NULL COMMENT '日期'); question_mask_size: 0 @@ -3492,7 +3424,7 @@ question_mask_size: 0 |--[1],[T_COMMENT], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[日期], value=[9223372036854775807] -************** Case 186 *************** +************** Case 178 *************** alter system bootstrap REGION 'sys_region' ZONE 'zone1' SERVER '10.101.74.122:41425'; question_mask_size: 0 @@ -3504,218 +3436,3 @@ question_mask_size: 0 |--[1],[T_VARCHAR], str_value_=[zone1], value=[9223372036854775807] |--[2],[T_VARCHAR], str_value_=["10.101.74.122":41425], value=[9223372036854775807] -************** Case 187 *************** -alter system cancel balance job; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_CANCEL_BALANCE_JOB], str_value_=[], value=[0] - -************** Case 188 *************** -alter system cancel balance job tenant 'mysql'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_CANCEL_BALANCE_JOB], str_value_=[], value=[0] - |--[0],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql], value=[9223372036854775807] - -************** Case 189 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] - |--[4],[T_INT], str_value_=[3], value=[3] - |--[5],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 190 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] - |--[4],[T_INT], str_value_=[3], value=[3] - -************** Case 191 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] - -************** Case 192 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - -************** Case 193 *************** -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_INT], str_value_=[3], value=[3] - |--[3],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 194 *************** -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_INT], str_value_=[3], value=[3] - -************** Case 195 *************** -alter system remove replica ls=100 server='100.88.107.212:5000'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - -************** Case 196 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - |--[3],[T_INT], str_value_=[3], value=[3] - |--[4],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 197 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - |--[3],[T_INT], str_value_=[3], value=[3] - -************** Case 198 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] - -************** Case 199 *************** -alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MODIFY_LS_PAXOS_REPLICA_NUM], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_INT], str_value_=[3], value=[3] - |--[2],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 200 *************** -alter system modify ls=100 paxos_replica_num=3; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MODIFY_LS_PAXOS_REPLICA_NUM], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_INT], str_value_=[3], value=[3] - -************** Case 201 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] - |--[4],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 202 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] - -************** Case 203 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] - |--[0],[T_LS], str_value_=[], value=[9223372036854775807] - |--[0],[T_INT], str_value_=[100], value=[100] - |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] - -************** Case 204 *************** -alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_CANCEL_LS_REPLICA_TASK], str_value_=[], value=[0] - |--[0],[T_VARCHAR], str_value_=[xxx], value=[9223372036854775807] - |--[1],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] - |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] - -************** Case 205 *************** -alter system cancel replica task task_id = 'xxx'; -question_mask_size: 0 - -|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] - |--[0],[T_CANCEL_LS_REPLICA_TASK], str_value_=[], value=[0] - |--[0],[T_VARCHAR], str_value_=[xxx], value=[9223372036854775807] - diff --git a/unittest/sql/parser/test_parser.result b/unittest/sql/parser/test_parser.result index 9c2fa9e72..e4caf5213 100644 --- a/unittest/sql/parser/test_parser.result +++ b/unittest/sql/parser/test_parser.result @@ -7249,244 +7249,6 @@ question_mask_size: 0 ] } ************** Case 60 *************** -alter system report replica; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REPORT_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { } - ] - } - ] -} -************** Case 61 *************** -alter system report replica server = '127.0.0.1:80'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REPORT_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_IP_PORT", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"127.0.0.1:80" - } - ] - } - ] - } - ] -} -************** Case 62 *************** -alter system report replica zone = 'z1'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REPORT_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ZONE", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":2, - "str_val":"z1" - } - ] - } - ] - } - ] -} -************** Case 63 *************** -alter system recycle replica; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_RECYCLE_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { } - ] - } - ] -} -************** Case 64 *************** -alter system recycle replica server = '127.0.0.1:80'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_RECYCLE_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_IP_PORT", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"127.0.0.1:80" - } - ] - } - ] - } - ] -} -************** Case 65 *************** -alter system recycle replica server '127.0.0.1:80'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_RECYCLE_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_IP_PORT", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"127.0.0.1:80" - } - ] - } - ] - } - ] -} -************** Case 66 *************** -alter system recycle replica zone = 'z1'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_RECYCLE_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ZONE", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":2, - "str_val":"z1" - } - ] - } - ] - } - ] -} -************** Case 67 *************** -alter system recycle replica zone 'z1'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_RECYCLE_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ZONE", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":2, - "str_val":"z1" - } - ] - } - ] - } - ] -} -************** Case 68 *************** alter system major freeze; question_mask_size: 0 { @@ -7513,7 +7275,7 @@ question_mask_size: 0 } ] } -************** Case 69 *************** +************** Case 61 *************** alter system start merge zone = 'z1'; question_mask_size: 0 { @@ -7552,7 +7314,7 @@ question_mask_size: 0 } ] } -************** Case 70 *************** +************** Case 62 *************** alter system suspend merge; question_mask_size: 0 { @@ -7578,7 +7340,7 @@ question_mask_size: 0 } ] } -************** Case 71 *************** +************** Case 63 *************** alter system suspend merge tenant = sys; question_mask_size: 0 { @@ -7617,7 +7379,7 @@ question_mask_size: 0 } ] } -************** Case 72 *************** +************** Case 64 *************** alter system suspend merge tenant = all_user; question_mask_size: 0 { @@ -7656,7 +7418,7 @@ question_mask_size: 0 } ] } -************** Case 73 *************** +************** Case 65 *************** alter system suspend merge tenant = all_meta; question_mask_size: 0 { @@ -7695,7 +7457,7 @@ question_mask_size: 0 } ] } -************** Case 74 *************** +************** Case 66 *************** alter system resume merge; question_mask_size: 0 { @@ -7721,7 +7483,7 @@ question_mask_size: 0 } ] } -************** Case 75 *************** +************** Case 67 *************** alter system resume merge tenant = sys; question_mask_size: 0 { @@ -7760,7 +7522,7 @@ question_mask_size: 0 } ] } -************** Case 76 *************** +************** Case 68 *************** alter system resume merge tenant = all_user; question_mask_size: 0 { @@ -7799,7 +7561,7 @@ question_mask_size: 0 } ] } -************** Case 77 *************** +************** Case 69 *************** alter system resume merge tenant = all_meta; question_mask_size: 0 { @@ -7838,7 +7600,7 @@ question_mask_size: 0 } ] } -************** Case 78 *************** +************** Case 70 *************** alter system clear roottable; question_mask_size: 0 { @@ -7858,7 +7620,7 @@ question_mask_size: 0 } ] } -************** Case 79 *************** +************** Case 71 *************** alter system clear roottable tenant = 'xxx'; question_mask_size: 0 { @@ -7891,7 +7653,7 @@ question_mask_size: 0 } ] } -************** Case 80 *************** +************** Case 72 *************** alter system modify zone 'z1' set region 'r1' question_mask_size: 0 { @@ -7936,9 +7698,9 @@ question_mask_size: 0 } ] } -************** Case 81 *************** +************** Case 73 *************** alter system start zone 'z1' region 'r1' -************** Case 82 *************** +************** Case 74 *************** alter system modify zone 'z1' IDC 'idc1' question_mask_size: 0 { @@ -7983,7 +7745,7 @@ question_mask_size: 0 } ] } -************** Case 83 *************** +************** Case 75 *************** alter system modify zone 'z1' set IDC 'idc1', region 'r1' question_mask_size: 0 { @@ -8034,7 +7796,7 @@ question_mask_size: 0 } ] } -************** Case 84 *************** +************** Case 76 *************** alter system add zone 'z1' idc 'idc1', region 'r1' question_mask_size: 0 { @@ -8085,7 +7847,7 @@ question_mask_size: 0 } ] } -************** Case 85 *************** +************** Case 77 *************** alter system add zone 'z1' idc 'idc1' question_mask_size: 0 { @@ -8130,7 +7892,7 @@ question_mask_size: 0 } ] } -************** Case 86 *************** +************** Case 78 *************** alter system modify zone 'z1' question_mask_size: 0 { @@ -8162,11 +7924,11 @@ question_mask_size: 0 } ] } -************** Case 87 *************** +************** Case 79 *************** alter system start zone 'z1' IDC 'idc1' -************** Case 88 *************** +************** Case 80 *************** alter system start zone 'z1' zone_type 'ReadWrite' -************** Case 89 *************** +************** Case 81 *************** alter system add zone 'z1' zone_type 'ReadWrite', idc 'idc1', region 'r1' question_mask_size: 0 { @@ -8223,7 +7985,7 @@ question_mask_size: 0 } ] } -************** Case 90 *************** +************** Case 82 *************** alter system add zone 'z1' zone_type 'ReadWrite', idc 'idc1' question_mask_size: 0 { @@ -8274,7 +8036,7 @@ question_mask_size: 0 } ] } -************** Case 91 *************** +************** Case 83 *************** alter system add zone 'z1' zone_type 'ReadWrite', region 'r1' question_mask_size: 0 { @@ -8325,7 +8087,7 @@ question_mask_size: 0 } ] } -************** Case 92 *************** +************** Case 84 *************** alter system add zone 'z1' zone_type 'ReadWrite' question_mask_size: 0 { @@ -8370,7 +8132,7 @@ question_mask_size: 0 } ] } -************** Case 93 *************** +************** Case 85 *************** alter system modify zone 'z1' zone_type 'ReadWrite', idc 'idc1', region 'r1' question_mask_size: 0 { @@ -8427,7 +8189,7 @@ question_mask_size: 0 } ] } -************** Case 94 *************** +************** Case 86 *************** alter system modify zone 'z1' zone_type 'ReadWrite', idc 'idc1' question_mask_size: 0 { @@ -8478,7 +8240,7 @@ question_mask_size: 0 } ] } -************** Case 95 *************** +************** Case 87 *************** alter system modify zone 'z1' zone_type 'ReadWrite', region 'r1' question_mask_size: 0 { @@ -8529,7 +8291,7 @@ question_mask_size: 0 } ] } -************** Case 96 *************** +************** Case 88 *************** alter system modify zone 'z1' zone_type 'ReadWrite' question_mask_size: 0 { @@ -8574,7 +8336,7 @@ question_mask_size: 0 } ] } -************** Case 97 *************** +************** Case 89 *************** select * from t1 where c1>ANY(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -8849,7 +8611,7 @@ question_mask_size: 0 } ] } -************** Case 98 *************** +************** Case 90 *************** select * from t1 where c1>SOME(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -9124,7 +8886,7 @@ question_mask_size: 0 } ] } -************** Case 99 *************** +************** Case 91 *************** select * from t1 where c1>ALL(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -9399,7 +9161,7 @@ question_mask_size: 0 } ] } -************** Case 100 *************** +************** Case 92 *************** select * from t1 where c1>(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -9666,7 +9428,7 @@ question_mask_size: 0 } ] } -************** Case 101 *************** +************** Case 93 *************** select * from t1 where c11); question_mask_size: 0 { @@ -9941,7 +9703,7 @@ question_mask_size: 0 } ] } -************** Case 102 *************** +************** Case 94 *************** select * from t1 where c11); question_mask_size: 0 { @@ -10216,7 +9978,7 @@ question_mask_size: 0 } ] } -************** Case 103 *************** +************** Case 95 *************** select * from t1 where c11); question_mask_size: 0 { @@ -10491,7 +10253,7 @@ question_mask_size: 0 } ] } -************** Case 104 *************** +************** Case 96 *************** select * from t1 where c1<(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -10758,7 +10520,7 @@ question_mask_size: 0 } ] } -************** Case 105 *************** +************** Case 97 *************** select * from t1 where c1>=ANY(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -11033,7 +10795,7 @@ question_mask_size: 0 } ] } -************** Case 106 *************** +************** Case 98 *************** select * from t1 where c1>=SOME(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -11308,7 +11070,7 @@ question_mask_size: 0 } ] } -************** Case 107 *************** +************** Case 99 *************** select * from t1 where c1>=ALL(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -11583,7 +11345,7 @@ question_mask_size: 0 } ] } -************** Case 108 *************** +************** Case 100 *************** select * from t1 where c1>=(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -11850,7 +11612,7 @@ question_mask_size: 0 } ] } -************** Case 109 *************** +************** Case 101 *************** select * from t1 where c1<=ANY(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -12125,7 +11887,7 @@ question_mask_size: 0 } ] } -************** Case 110 *************** +************** Case 102 *************** select * from t1 where c1<=SOME(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -12400,7 +12162,7 @@ question_mask_size: 0 } ] } -************** Case 111 *************** +************** Case 103 *************** select * from t1 where c1<=ALL(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -12675,7 +12437,7 @@ question_mask_size: 0 } ] } -************** Case 112 *************** +************** Case 104 *************** select * from t1 where c1<=(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -12942,7 +12704,7 @@ question_mask_size: 0 } ] } -************** Case 113 *************** +************** Case 105 *************** select * from t1 where c1=ANY(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -13217,7 +12979,7 @@ question_mask_size: 0 } ] } -************** Case 114 *************** +************** Case 106 *************** select * from t1 where c1=SOME(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -13492,7 +13254,7 @@ question_mask_size: 0 } ] } -************** Case 115 *************** +************** Case 107 *************** select * from t1 where c1=ALL(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -13767,7 +13529,7 @@ question_mask_size: 0 } ] } -************** Case 116 *************** +************** Case 108 *************** select * from t1 where c1=(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -14034,7 +13796,7 @@ question_mask_size: 0 } ] } -************** Case 117 *************** +************** Case 109 *************** select * from t1 where c1!=ANY(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -14309,7 +14071,7 @@ question_mask_size: 0 } ] } -************** Case 118 *************** +************** Case 110 *************** select * from t1 where c1!=SOME(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -14584,7 +14346,7 @@ question_mask_size: 0 } ] } -************** Case 119 *************** +************** Case 111 *************** select * from t1 where c1!=ALL(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -14859,7 +14621,7 @@ question_mask_size: 0 } ] } -************** Case 120 *************** +************** Case 112 *************** select * from t1 where c1!=(select c1 from t2 where c2>1); question_mask_size: 0 { @@ -15126,7 +14888,7 @@ question_mask_size: 0 } ] } -************** Case 121 *************** +************** Case 113 *************** select * from t1 where c1 in (select c1 from t2 where c2>1); question_mask_size: 0 { @@ -15393,7 +15155,7 @@ question_mask_size: 0 } ] } -************** Case 122 *************** +************** Case 114 *************** select * from t1 where c1 not in (select c1 from t2 where c2>1); question_mask_size: 0 { @@ -15660,7 +15422,7 @@ question_mask_size: 0 } ] } -************** Case 123 *************** +************** Case 115 *************** select * from t1 where exists (select c1 from t2 where c2>1); question_mask_size: 0 { @@ -15911,7 +15673,7 @@ question_mask_size: 0 } ] } -************** Case 124 *************** +************** Case 116 *************** select * from t1 where not exists (select c1 from t2 where c2>1); question_mask_size: 0 { @@ -16170,7 +15932,7 @@ question_mask_size: 0 } ] } -************** Case 125 *************** +************** Case 117 *************** select * from t1 where (select c1 from t1) like (select c2 from t2); question_mask_size: 0 { @@ -16481,7 +16243,7 @@ question_mask_size: 0 } ] } -************** Case 126 *************** +************** Case 118 *************** select * from t1 where (select c1 from t1) not like (select c2 from t2); question_mask_size: 0 { @@ -16792,7 +16554,7 @@ question_mask_size: 0 } ] } -************** Case 127 *************** +************** Case 119 *************** select * from t1 where (c1) in (select c1 from t2); question_mask_size: 0 { @@ -17021,7 +16783,7 @@ question_mask_size: 0 } ] } -************** Case 128 *************** +************** Case 120 *************** select * from t1 where (c1, c2) in (select c1, c2 from t2); question_mask_size: 0 { @@ -17298,7 +17060,7 @@ question_mask_size: 0 } ] } -************** Case 129 *************** +************** Case 121 *************** select * from t1 where ROW(c1, c2) in (select c1, c2 from t2); question_mask_size: 0 { @@ -17575,7 +17337,7 @@ question_mask_size: 0 } ] } -************** Case 130 *************** +************** Case 122 *************** set names latin1; question_mask_size: 0 { @@ -17609,7 +17371,7 @@ question_mask_size: 0 } ] } -************** Case 131 *************** +************** Case 123 *************** set names 'latin1'; question_mask_size: 0 { @@ -17643,7 +17405,7 @@ question_mask_size: 0 } ] } -************** Case 132 *************** +************** Case 124 *************** set names utf8 collate 'utf8_general_ci'; question_mask_size: 0 { @@ -17682,7 +17444,7 @@ question_mask_size: 0 } ] } -************** Case 133 *************** +************** Case 125 *************** set names utf8 collate utf8_general_ci; question_mask_size: 0 { @@ -17721,7 +17483,7 @@ question_mask_size: 0 } ] } -************** Case 134 *************** +************** Case 126 *************** set character set utf8; question_mask_size: 0 { @@ -17754,7 +17516,7 @@ question_mask_size: 0 } ] } -************** Case 135 *************** +************** Case 127 *************** set character set 'utf8'; question_mask_size: 0 { @@ -17787,7 +17549,7 @@ question_mask_size: 0 } ] } -************** Case 136 *************** +************** Case 128 *************** set charset utf8; question_mask_size: 0 { @@ -17820,7 +17582,7 @@ question_mask_size: 0 } ] } -************** Case 137 *************** +************** Case 129 *************** select _utf8 'abc', _utf8mb4 'def' collate utf8mb4_general_ci from t1 where c1 collate utf8_bin = 'xyz' collate utf8_bin; question_mask_size: 0 { @@ -18092,7 +17854,7 @@ question_mask_size: 0 } ] } -************** Case 138 *************** +************** Case 130 *************** select * from t1 where c1=?; question_mask_size: 1 { @@ -18229,7 +17991,7 @@ question_mask_size: 1 } ] } -************** Case 139 *************** +************** Case 131 *************** select * from t1 where c1>?; question_mask_size: 1 { @@ -18366,7 +18128,7 @@ question_mask_size: 1 } ] } -************** Case 140 *************** +************** Case 132 *************** select * from t1 where (select c1 from t1 where c1 = ?) not like (select c2 from t2 where c2=?); question_mask_size: 2 { @@ -18753,7 +18515,7 @@ question_mask_size: 2 } ] } -************** Case 141 *************** +************** Case 133 *************** select * from t1 join t2; question_mask_size: 0 { @@ -18895,7 +18657,7 @@ question_mask_size: 0 } ] } -************** Case 142 *************** +************** Case 134 *************** select * from t1 inner join t2; question_mask_size: 0 { @@ -19037,7 +18799,7 @@ question_mask_size: 0 } ] } -************** Case 143 *************** +************** Case 135 *************** select * from t1 cross join t2; question_mask_size: 0 { @@ -19179,7 +18941,7 @@ question_mask_size: 0 } ] } -************** Case 144 *************** +************** Case 136 *************** select * from t1 cross join t2 join t3; question_mask_size: 0 { @@ -19364,7 +19126,7 @@ question_mask_size: 0 } ] } -************** Case 145 *************** +************** Case 137 *************** select "1234"; question_mask_size: 0 { @@ -19438,7 +19200,7 @@ question_mask_size: 0 } ] } -************** Case 146 *************** +************** Case 138 *************** select '1234'; question_mask_size: 0 { @@ -19512,7 +19274,7 @@ question_mask_size: 0 } ] } -************** Case 147 *************** +************** Case 139 *************** create table test(c1 varchar(3) binary charset utf8mb4); question_mask_size: 0 { @@ -19608,7 +19370,7 @@ question_mask_size: 0 } ] } -************** Case 148 *************** +************** Case 140 *************** replace into test values(1,2); question_mask_size: 0 { @@ -19708,7 +19470,7 @@ question_mask_size: 0 } ] } -************** Case 149 *************** +************** Case 141 *************** replace ignore into test values(1,2); question_mask_size: 0 { @@ -19813,7 +19575,7 @@ question_mask_size: 0 } ] } -************** Case 150 *************** +************** Case 142 *************** insert ignore into test values(1,2); question_mask_size: 0 { @@ -19918,7 +19680,7 @@ question_mask_size: 0 } ] } -************** Case 151 *************** +************** Case 143 *************** insert ignore into test values(1,2) on duplicate key update c2 = c1 + 1; question_mask_size: 0 { @@ -20084,7 +19846,7 @@ question_mask_size: 0 } ] } -************** Case 152 *************** +************** Case 144 *************** create outline out_name on select * from t1; question_mask_size: 0 { @@ -20220,7 +19982,7 @@ question_mask_size: 0 } ] } -************** Case 153 *************** +************** Case 145 *************** create or replace outline out_name on select * from t1; question_mask_size: 0 { @@ -20361,7 +20123,7 @@ question_mask_size: 0 } ] } -************** Case 154 *************** +************** Case 146 *************** create outline out_name on select * from t1; question_mask_size: 0 { @@ -20497,7 +20259,7 @@ question_mask_size: 0 } ] } -************** Case 155 *************** +************** Case 147 *************** create outline out_name on select /*+ssssss*/* from t1; question_mask_size: 0 { @@ -20633,7 +20395,7 @@ question_mask_size: 0 } ] } -************** Case 156 *************** +************** Case 148 *************** create outline out_name on select /*+ssssss*/* from t1 to select /*+ssssss*/* from t1; question_mask_size: 0 { @@ -20856,7 +20618,7 @@ question_mask_size: 0 } ] } -************** Case 157 *************** +************** Case 149 *************** create outline out_name on select /*+ssssss*/* from t1 to select * from t1; question_mask_size: 0 { @@ -21079,7 +20841,7 @@ question_mask_size: 0 } ] } -************** Case 158 *************** +************** Case 150 *************** create outline out_name on insert /*+ssssss*/ into t1 values(1, 2) to insert /*+ssssss*/ into t1 values(1, 2); question_mask_size: 0 { @@ -21304,7 +21066,7 @@ question_mask_size: 0 } ] } -************** Case 159 *************** +************** Case 151 *************** drop outline out_name; question_mask_size: 0 { @@ -21345,7 +21107,7 @@ question_mask_size: 0 } ] } -************** Case 160 *************** +************** Case 152 *************** drop outline db_name.out_name; question_mask_size: 0 { @@ -21391,7 +21153,7 @@ question_mask_size: 0 } ] } -************** Case 161 *************** +************** Case 153 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1') set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -21504,7 +21266,7 @@ question_mask_size: 0 } ] } -************** Case 162 *************** +************** Case 154 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1') set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -21622,7 +21384,7 @@ question_mask_size: 0 } ] } -************** Case 163 *************** +************** Case 155 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), locality = 'auto_locality_strategy' set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -21750,7 +21512,7 @@ question_mask_size: 0 } ] } -************** Case 164 *************** +************** Case 156 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), locality = 'auto_locality_strategy' set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -21883,7 +21645,7 @@ question_mask_size: 0 } ] } -************** Case 165 *************** +************** Case 157 *************** create tenant tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), logonly_replica_num = 1 set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -22010,7 +21772,7 @@ question_mask_size: 0 } ] } -************** Case 166 *************** +************** Case 158 *************** create tenant if not exists tt1 replica_num = 1, primary_zone = 'zone1', resource_pool_list = ('zone1'), logonly_replica_num = 1 set ob_tcp_invited_nodes='%'; question_mask_size: 0 { @@ -22142,7 +21904,7 @@ question_mask_size: 0 } ] } -************** Case 167 *************** +************** Case 159 *************** select /*+max_concurrent(-10)*/* from t1; question_mask_size: 0 { @@ -22246,7 +22008,7 @@ question_mask_size: 0 } ] } -************** Case 168 *************** +************** Case 160 *************** select /*+max_concurrent(10)*/* from t1; question_mask_size: 0 { @@ -22366,25 +22128,25 @@ question_mask_size: 0 } ] } -************** Case 169 *************** +************** Case 161 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 170 *************** +************** Case 162 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales where t2.id!= t1.id when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 171 *************** +************** Case 163 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales where t1.id < 3 delete where t1.id < 5 when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 172 *************** +************** Case 164 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales delete where t1.sales =2 when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales); -************** Case 173 *************** +************** Case 165 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales) where t2.id > 0; -************** Case 174 *************** +************** Case 166 *************** merge into targetTable t1 using sourceTable t2 on (1 = 2) when not matched then insert(t1.id, t1.sales) values(t2.id, t2.sales) where t2.id > 0; -************** Case 175 *************** +************** Case 167 *************** merge into targetTable t1 using sourceTable t2 on (t1.id = t2.id) when matched then update set t1.sales = t2.sales; -************** Case 176 *************** +************** Case 168 *************** merge into targetTable using sourceTable on (targetTable.id = sourceTable.id) when matched then update set targetTable.sales = sourceTable.sales; -************** Case 177 *************** +************** Case 169 *************** merge into targetTable using (select * from t1 ) sourceTable on (targetTable.id = sourceTable.id) when matched then update set targetTable.sales = sourceTable.sales; -************** Case 178 *************** +************** Case 170 *************** select unique(c1), c2 from t1; question_mask_size: 0 { @@ -22530,7 +22292,7 @@ question_mask_size: 0 } ] } -************** Case 179 *************** +************** Case 171 *************** select unique(c1+1), c2 from t1; question_mask_size: 0 { @@ -22690,7 +22452,7 @@ question_mask_size: 0 } ] } -************** Case 180 *************** +************** Case 172 *************** select distinct unique( max(c1)), c2 from t1; question_mask_size: 0 { @@ -22851,7 +22613,7 @@ question_mask_size: 0 } ] } -************** Case 181 *************** +************** Case 173 *************** select unique distinct( max(c1)), c2 from t1; question_mask_size: 0 { @@ -23012,9 +22774,9 @@ question_mask_size: 0 } ] } -************** Case 182 *************** +************** Case 174 *************** select unique(*) from t1; -************** Case 183 *************** +************** Case 175 *************** alter tablegroup tg2 add table t1,t2; question_mask_size: 0 { @@ -23079,7 +22841,7 @@ question_mask_size: 0 } ] } -************** Case 184 *************** +************** Case 176 *************** insert into t1 values(X''); question_mask_size: 0 { @@ -23173,7 +22935,7 @@ question_mask_size: 0 } ] } -************** Case 185 *************** +************** Case 177 *************** create table t1(`thedate` date NOT NULL COMMENT '日期'); question_mask_size: 0 { @@ -23281,7 +23043,7 @@ question_mask_size: 0 } ] } -************** Case 186 *************** +************** Case 178 *************** alter system bootstrap REGION 'sys_region' ZONE 'zone1' SERVER '10.101.74.122:41425'; question_mask_size: 0 { @@ -23335,918 +23097,3 @@ question_mask_size: 0 } ] } -************** Case 187 *************** -alter system cancel balance job; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_CANCEL_BALANCE_JOB", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { } - ] - } - ] -} -************** Case 188 *************** -alter system cancel balance job tenant 'mysql'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_CANCEL_BALANCE_JOB", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":5, - "str_val":"mysql" - } - ] - } - ] - } - ] -} -************** Case 189 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ADD_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5001" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 190 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ADD_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5001" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { } - ] - } - ] -} -************** Case 191 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ADD_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5001" - }, - { }, - { } - ] - } - ] -} -************** Case 192 *************** -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_ADD_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { }, - { }, - { } - ] - } - ] -} -************** Case 193 *************** -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REMOVE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 194 *************** -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REMOVE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { } - ] - } - ] -} -************** Case 195 *************** -alter system remove replica ls=100 server='100.88.107.212:5000'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_REMOVE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { }, - { } - ] - } - ] -} -************** Case 196 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MODIFY_LS_REPLICA_TYPE", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 197 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MODIFY_LS_REPLICA_TYPE", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { } - ] - } - ] -} -************** Case 198 *************** -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MODIFY_LS_REPLICA_TYPE", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":1, - "str_val":"F" - }, - { }, - { } - ] - } - ] -} -************** Case 199 *************** -alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MODIFY_LS_PAXOS_REPLICA_NUM", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 200 *************** -alter system modify ls=100 paxos_replica_num=3; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MODIFY_LS_PAXOS_REPLICA_NUM", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_INT", - "int_val":3, - "str_len":1, - "str_val":"3" - }, - { } - ] - } - ] -} -************** Case 201 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MIGRATE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5001" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 202 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MIGRATE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5001" - }, - { } - ] - } - ] -} -************** Case 203 *************** -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_MIGRATE_LS_REPLICA", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_LS", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_INT", - "int_val":100, - "str_len":3, - "str_val":"100" - } - ] - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":19, - "str_val":"100.88.107.212:5000" - }, - { }, - { } - ] - } - ] -} -************** Case 204 *************** -alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_CANCEL_LS_REPLICA_TASK", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":3, - "str_val":"xxx" - }, - { - "type":"T_TENANT_NAME", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":12, - "str_val":"mysql_tenant" - } - ] - } - ] - } - ] -} -************** Case 205 *************** -alter system cancel replica task task_id = 'xxx'; -question_mask_size: 0 -{ - "type":"T_STMT_LIST", - "int_val":9223372036854775807, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_CANCEL_LS_REPLICA_TASK", - "int_val":0, - "str_len":0, - "str_val":"", - "children": [ - { - "type":"T_VARCHAR", - "int_val":9223372036854775807, - "str_len":3, - "str_val":"xxx" - }, - { } - ] - } - ] -} diff --git a/unittest/sql/parser/test_parser.sql b/unittest/sql/parser/test_parser.sql index 3e0915f9e..1e18cac89 100644 --- a/unittest/sql/parser/test_parser.sql +++ b/unittest/sql/parser/test_parser.sql @@ -48,18 +48,6 @@ update d.t1 PARTITION (p2) SET id = 2 WHERE name = 'Jill'; INSERT INTO d.t1 PARTITION (p3, p4) VALUES (24, 'Tim', 'Greene', 3, 1), (26, 'Linda', 'Mills', 2, 1); REPLACE INTO d.t1 PARTITION (p0) VALUES (20, 'Jan', 'Jones', 3, 2); SELECT e.id, s.city, d.name FROM e JOIN stores PARTITION (p1) ON e.id=s.id JOIN departments PARTITION (p0) ON e.id=d.id; -alter system switch replica leader zone = 'z1'; -alter system switch replica leader server = '127.0.0.1:80'; -alter system switch replica leader partition_id = '1%8@3001' server = '127.0.0.1:80'; -alter system switch replica leader partition_id '1%8@3001' server '127.0.0.1:80'; -alter system report replica; -alter system report replica server = '127.0.0.1:80'; -alter system report replica zone = 'z1'; -alter system recycle replica; -alter system recycle replica server = '127.0.0.1:80'; -alter system recycle replica server '127.0.0.1:80'; -alter system recycle replica zone = 'z1'; -alter system recycle replica zone 'z1'; alter system major freeze; alter system start merge zone = 'z1'; alter system suspend merge; diff --git a/unittest/sql/parser/test_parser.test b/unittest/sql/parser/test_parser.test index 0e7c124e0..432cd1645 100644 --- a/unittest/sql/parser/test_parser.test +++ b/unittest/sql/parser/test_parser.test @@ -61,18 +61,6 @@ update d.t1 PARTITION (p2) SET id = 2 WHERE name = 'Jill'; INSERT INTO d.t1 PARTITION (p3, p4) VALUES (24, 'Tim', 'Greene', 3, 1), (26, 'Linda', 'Mills', 2, 1); REPLACE INTO d.t1 PARTITION (p0) VALUES (20, 'Jan', 'Jones', 3, 2); SELECT e.id, s.city, d.name FROM e JOIN stores PARTITION (p1) ON e.id=s.id JOIN departments PARTITION (p0) ON e.id=d.id; -#alter system switch replica leader zone = 'z1'; -#alter system switch replica leader server = '127.0.0.1:80'; -#alter system switch replica leader partition_id = '1%8@3001' server = '127.0.0.1:80'; -#alter system switch replica leader partition_id '1%8@3001' server '127.0.0.1:80'; -alter system report replica; -alter system report replica server = '127.0.0.1:80'; -alter system report replica zone = 'z1'; -alter system recycle replica; -alter system recycle replica server = '127.0.0.1:80'; -alter system recycle replica server '127.0.0.1:80'; -alter system recycle replica zone = 'z1'; -alter system recycle replica zone 'z1'; alter system major freeze; alter system start merge zone = 'z1'; alter system suspend merge; @@ -287,30 +275,3 @@ alter tablegroup tg2 add table t1,t2; insert into t1 values(X''); create table t1(`thedate` date NOT NULL COMMENT '日期'); alter system bootstrap REGION 'sys_region' ZONE 'zone1' SERVER '10.101.74.122:41425'; -alter system cancel balance job; -alter system cancel balance job tenant 'mysql'; - - - -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; -alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; - -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; -alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; -alter system remove replica ls=100 server='100.88.107.212:5000'; - -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; -alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; - -alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; -alter system modify ls=100 paxos_replica_num=3; - -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; -alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; - -alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; -alter system cancel replica task task_id = 'xxx'; diff --git a/unittest/storage/CMakeLists.txt b/unittest/storage/CMakeLists.txt index 6658cedc8..7878d3d27 100644 --- a/unittest/storage/CMakeLists.txt +++ b/unittest/storage/CMakeLists.txt @@ -82,7 +82,6 @@ storage_unittest(test_mds_dump_kv multi_data_source/test_mds_dump_kv.cpp) #storage_unittest(test_memtable_multi_version_row_iterator memtable/test_memtable_multi_version_row_iterator.cpp) #storage_unittest(test_new_table_store) storage_unittest(test_mds_table_handle multi_data_source/test_mds_table_handle.cpp) -storage_unittest(test_is_old_mds multi_data_source/test_is_old_mds.cpp) storage_unittest(test_fixed_size_block_allocator) storage_unittest(test_dag_warning_history) storage_unittest(test_storage_schema) @@ -98,7 +97,6 @@ storage_unittest(test_storage_log_replay slog/test_storage_log_replay.cpp) storage_unittest(test_linked_macro_block slog_ckpt/test_linked_macro_block.cpp) storage_unittest(test_tablet_dumped_medium_info test_tablet_dumped_medium_info.cpp) #storage_unittest(test_create_tablet_clog tx_storage/test_create_tablet_clog.cpp) -storage_unittest(test_safe_destroy_handler tx_storage/test_safe_destroy_handler.cpp) storage_unittest(test_simple_rows_merger) storage_unittest(test_global_iterator_pool) storage_unittest(test_partition_incremental_range_spliter) @@ -112,5 +110,4 @@ storage_unittest(test_medium_list_checker compaction/test_medium_list_checker.cp storage_unittest(test_diagnose_info_mgr compaction/test_diagnose_info_mgr.cpp) storage_unittest(test_tablet_create_mds_ctx test_tablet_create_mds_ctx.cpp) -storage_unittest(test_sstable_log_ts_range_cut test_sstable_log_ts_range_cut.cpp) storage_unittest(test_compaction_iter compaction/test_compaction_iter.cpp) diff --git a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h index b2170b16d..1de715291 100644 --- a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h +++ b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h @@ -196,7 +196,6 @@ class ObMultiVersionSSTableTest : public ::testing::Test static const uint64_t tenant_id_ = 1001; static const uint64_t tablet_id_ = 300000; static const uint64_t table_id_ = 300000; - static const uint64_t ls_id_ = 1001; ObMergeType merge_type_; ObTenantFreezeInfoMgr *mgr_; @@ -240,9 +239,8 @@ void ObMultiVersionSSTableTest::SetUpTestCase() SERVER_STORAGE_META_SERVICE.is_started_ = true; //OK(init_io_device("multi_version_test")); - // create ls - ObLSHandle ls_handle; - ret = TestDmlCommon::create_ls(tenant_id_, ObLSID(ls_id_), ls_handle); + ObLS *ls = nullptr; + ret = TestDmlCommon::create_ls(ls); ASSERT_EQ(OB_SUCCESS, ret); } @@ -391,15 +389,14 @@ void ObMultiVersionSSTableTest::prepare_table_schema( void ObMultiVersionSSTableTest::init_tablet() { - ObLSID ls_id(ls_id_); ObTabletID tablet_id(tablet_id_); - ObLSHandle ls_handle; + ObLS *ls = nullptr; ObLSService *ls_svr = MTL(ObLSService*); - ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD)); + ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls)); ObTabletHandle tablet_handle; void *ptr = nullptr; - ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle)); + ASSERT_EQ(OB_SUCCESS, ls->get_tablet(tablet_id, tablet_handle)); ObTablet *tablet = tablet_handle.get_obj(); ASSERT_EQ(OB_SUCCESS, ObStorageSchemaUtil::alloc_storage_schema(allocator_, tablet->storage_schema_addr_.ptr_)); tablet->storage_schema_addr_.get_ptr()->init(allocator_, table_schema_, lib::Worker::CompatMode::MYSQL); @@ -423,11 +420,10 @@ void ObMultiVersionSSTableTest::reset_writer( root_index_builder_ = nullptr; } - ObLSID ls_id(ls_id_); ObTabletID tablet_id(tablet_id_); SCN scn; scn.convert_for_tx(snapshot_version); - ASSERT_EQ(OB_SUCCESS, data_desc_.init(false/*is_ddl*/, table_schema_, ls_id, tablet_id, merge_type, snapshot_version, DATA_VERSION_1_0_0_0, + ASSERT_EQ(OB_SUCCESS, data_desc_.init(false/*is_ddl*/, table_schema_, tablet_id, merge_type, snapshot_version, DATA_VERSION_1_0_0_0, table_schema_.get_micro_index_clustered(), 0/*concurrent_cnt*/, scn)); void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder)); root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need writer buffer*/); diff --git a/unittest/storage/blocksstable/test_data_store_desc.cpp b/unittest/storage/blocksstable/test_data_store_desc.cpp index 0dc715b06..ddd6f4851 100644 --- a/unittest/storage/blocksstable/test_data_store_desc.cpp +++ b/unittest/storage/blocksstable/test_data_store_desc.cpp @@ -38,7 +38,6 @@ class TestObDataStoreDesc : public blocksstable::TestDataFilePrepare TestObDataStoreDesc() : blocksstable::TestDataFilePrepare(&getter, "test_data_store_desc"), tenant_base_(1), - mock_ls_id_(1), mock_tablet_id_(1) {} ~TestObDataStoreDesc() = default; @@ -56,7 +55,6 @@ class TestObDataStoreDesc : public blocksstable::TestDataFilePrepare } ObTenantBase tenant_base_; - share::ObLSID mock_ls_id_; ObTabletID mock_tablet_id_; }; @@ -82,18 +80,17 @@ TEST_F(TestObDataStoreDesc, test_static_desc) share::SCN scn; scn.convert_for_tx(100); ASSERT_EQ(OB_INVALID_ARGUMENT, - static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + static_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MINI_MERGE, snapshot, share::SCN::invalid_scn(), 1 /*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */, 0/*concurrent_cnt*/)); ASSERT_EQ(OB_SUCCESS, - static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + static_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MINI_MERGE, snapshot, scn, 1 /*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */, 0/*concurrent_cnt*/)); ASSERT_TRUE(static_desc.is_valid()); ASSERT_EQ(static_desc.is_ddl_, false); ASSERT_EQ(static_desc.merge_type_, MINI_MERGE); - ASSERT_EQ(static_desc.ls_id_, mock_ls_id_); ASSERT_EQ(static_desc.tablet_id_, mock_tablet_id_); ASSERT_EQ(static_desc.compressor_type_, ObStaticDataStoreDesc::DEFAULT_MINOR_COMPRESSOR_TYPE); ASSERT_EQ(static_desc.schema_version_, table_schema.schema_version_); @@ -105,13 +102,12 @@ TEST_F(TestObDataStoreDesc, test_static_desc) ObStaticDataStoreDesc static_desc2; ASSERT_EQ(OB_SUCCESS, - static_desc2.init(true/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + static_desc2.init(true/*is_ddl*/, table_schema, mock_tablet_id_, MAJOR_MERGE, snapshot, scn, DATA_VERSION_1_0_0_0, EXEC_MODE_LOCAL, false /* micro_index_clustered */, 0/*concurrent_cnt*/)); ASSERT_TRUE(static_desc2.is_valid()); ASSERT_EQ(static_desc2.is_ddl_, true); ASSERT_EQ(static_desc2.merge_type_, MAJOR_MERGE); - ASSERT_EQ(static_desc2.ls_id_, mock_ls_id_); ASSERT_EQ(static_desc2.tablet_id_, mock_tablet_id_); ASSERT_EQ(static_desc2.compressor_type_, ObCompressorType::ZSTD_1_3_8_COMPRESSOR); ASSERT_EQ(static_desc2.schema_version_, table_schema.schema_version_); @@ -160,7 +156,7 @@ TEST_F(TestObDataStoreDesc, test_whole_data_desc) ObTableSchema table_schema; TestSchemaPrepare::prepare_schema(table_schema, 5); ASSERT_EQ(OB_SUCCESS, - whole_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + whole_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MAJOR_MERGE, snapshot, DATA_VERSION_1_0_0_0, table_schema.get_micro_index_clustered(), 0/*concurrent_cnt*/, @@ -170,11 +166,11 @@ TEST_F(TestObDataStoreDesc, test_whole_data_desc) // point to other static desc member ObStaticDataStoreDesc static_desc; ASSERT_EQ(OB_INVALID_ARGUMENT, - static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + static_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MINI_MERGE, snapshot, share::SCN::invalid_scn(), 0/*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */, 0 /*concurrent_cnt*/)); ASSERT_EQ(OB_SUCCESS, - static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + static_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MAJOR_MERGE, snapshot, share::SCN::invalid_scn(), DATA_VERSION_1_0_0_0, EXEC_MODE_LOCAL, false /* micro_index_clustered */, 0 /*concurrent_cnt*/)); whole_desc.desc_.static_desc_ = &static_desc; @@ -192,7 +188,7 @@ TEST_F(TestObDataStoreDesc, gen_index_desc) share::SCN scn; scn.convert_for_tx(100); ASSERT_EQ(OB_SUCCESS, - data_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, + data_desc.init(false/*is_ddl*/, table_schema, mock_tablet_id_, MAJOR_MERGE, snapshot, 1/*clsuter_version*/, table_schema.get_micro_index_clustered(), 0/*concurrent_cnt*/)); ASSERT_TRUE(data_desc.is_valid()); diff --git a/unittest/storage/blocksstable/test_index_block_aggregator.cpp b/unittest/storage/blocksstable/test_index_block_aggregator.cpp index 9d9d3e80c..457da2446 100644 --- a/unittest/storage/blocksstable/test_index_block_aggregator.cpp +++ b/unittest/storage/blocksstable/test_index_block_aggregator.cpp @@ -1030,9 +1030,9 @@ TEST_F(TestIndexBlockAggregator, test_loose_min_max_data_desc) ASSERT_EQ(OB_SUCCESS, storage_schema.init(arena, table_schema, lib::Worker::CompatMode::MYSQL)); - ASSERT_EQ(OB_SUCCESS, major_static_desc.init(false, table_schema, ObLSID(1002), ObTabletID(200000), compaction::MAJOR_MERGE, 10000, share::SCN::invalid_scn(), + ASSERT_EQ(OB_SUCCESS, major_static_desc.init(false, table_schema, ObTabletID(200000), compaction::MAJOR_MERGE, 10000, share::SCN::invalid_scn(), DATA_VERSION_1_0_0_0, compaction::EXEC_MODE_LOCAL, false, 0)); - ASSERT_EQ(OB_SUCCESS, minor_static_desc.init(false, table_schema, ObLSID(1002), ObTabletID(200000), compaction::MINI_MERGE, 1, share::SCN::base_scn(), + ASSERT_EQ(OB_SUCCESS, minor_static_desc.init(false, table_schema, ObTabletID(200000), compaction::MINI_MERGE, 1, share::SCN::base_scn(), DATA_VERSION_1_0_0_0, compaction::EXEC_MODE_LOCAL, false, 0)); ASSERT_EQ(OB_SUCCESS, major_data_desc.init(major_static_desc, storage_schema)); diff --git a/unittest/storage/blocksstable/test_macro_block_id.cpp b/unittest/storage/blocksstable/test_macro_block_id.cpp index abab2ecb5..4a4bf3975 100644 --- a/unittest/storage/blocksstable/test_macro_block_id.cpp +++ b/unittest/storage/blocksstable/test_macro_block_id.cpp @@ -103,12 +103,9 @@ TEST_F(TestMacroBlockId, verification) TEST_F(TestMacroBlockId, test_path_id) { int ret = OB_SUCCESS; - blocksstable::ObStorageObjectOpt curr_opt; MacroBlockId test_block_id; - uint64_t test_ls_id = 1001; uint64_t test_tablet_id = 200001; uint64_t test_tablet_version = ObStorageObjectOpt::INVALID_TABLET_VERSION; - curr_opt.set_ss_private_tablet_meta_object_opt(test_ls_id, test_tablet_id, test_tablet_version); OB_LOG(INFO, "before set"); hex_dump(&test_block_id.fourth_id_, sizeof(int64_t), @@ -118,11 +115,11 @@ TEST_F(TestMacroBlockId, test_path_id) // in ss mode test_block_id.set_version_v2(); test_block_id.set_id_mode((uint64_t)ObMacroBlockIdMode::ID_MODE_SHARE); - test_block_id.set_storage_object_type(static_cast(curr_opt.object_type_)); + test_block_id.set_storage_object_type(static_cast(ObStorageObjectType::PRIVATE_META_MACRO)); test_block_id.set_incarnation_id(0); - test_block_id.set_second_id(curr_opt.ss_private_tablet_opt_.ls_id_); - test_block_id.set_third_id(curr_opt.ss_private_tablet_opt_.tablet_id_); - test_block_id.set_meta_version_id(curr_opt.ss_private_tablet_opt_.version_); + test_block_id.set_second_id(test_tablet_id); + test_block_id.set_third_id(1/*server_id*/); + test_block_id.set_meta_version_id(test_tablet_version); test_block_id.set_meta_path_id(-1); OB_LOG(INFO, "after set"); diff --git a/unittest/storage/blocksstable/test_row_reader.cpp b/unittest/storage/blocksstable/test_row_reader.cpp index fa375faa4..cc2df34b3 100644 --- a/unittest/storage/blocksstable/test_row_reader.cpp +++ b/unittest/storage/blocksstable/test_row_reader.cpp @@ -22,6 +22,7 @@ #include "ob_data_file_prepare.h" +#include "storage/memtable/ob_nop_bitmap.h" #include "unittest/storage/mock_ob_table_read_info.h" #ifndef INT24_MIN diff --git a/unittest/storage/blocksstable/test_sstable_meta.cpp b/unittest/storage/blocksstable/test_sstable_meta.cpp index d3021514f..28d452706 100644 --- a/unittest/storage/blocksstable/test_sstable_meta.cpp +++ b/unittest/storage/blocksstable/test_sstable_meta.cpp @@ -376,53 +376,6 @@ void TestSSTableMeta::prepare_create_sstable_param() ASSERT_EQ(OB_SUCCESS, ObSSTableMergeRes::fill_column_checksum_for_empty_major(param_.column_cnt_, param_.column_checksums_)); } -class TestMigrationSSTableParam : public TestSSTableMeta -{ -public: - TestMigrationSSTableParam(); - virtual ~TestMigrationSSTableParam() = default; - virtual void SetUp() override; - virtual void TearDown() override; -private: - storage::ObStorageSchema storage_schema_; - ObSSTableMeta sstable_meta_; - storage::ObITable::TableKey table_key_; -}; - -TestMigrationSSTableParam::TestMigrationSSTableParam() - : storage_schema_(), - sstable_meta_(), - table_key_() -{ -} - -void TestMigrationSSTableParam::SetUp() -{ - TestSSTableMeta::SetUp(); - ASSERT_EQ(OB_SUCCESS, storage_schema_.init(allocator_, table_schema_, lib::get_compat_mode())); - ASSERT_TRUE(!sstable_meta_.is_valid()); - sstable_meta_.reset(); - ASSERT_TRUE(!sstable_meta_.is_valid()); - ASSERT_EQ(OB_SUCCESS, sstable_meta_.init(param_, allocator_)); - ASSERT_TRUE(sstable_meta_.is_valid()); - ASSERT_TRUE(sstable_meta_.data_root_info_.is_valid()); - ASSERT_TRUE(sstable_meta_.macro_info_.is_valid()); - ASSERT_TRUE(sstable_meta_.get_col_checksum_cnt() > 0); - table_key_.table_type_ = ObITable::TableType::MAJOR_SSTABLE; - table_key_.tablet_id_ = 1101; - table_key_.version_range_.base_version_ = 0; - table_key_.version_range_.snapshot_version_ = 11; - ASSERT_TRUE(table_key_.is_valid()); -} - -void TestMigrationSSTableParam::TearDown() -{ - table_key_.reset(); - sstable_meta_.reset(); - storage_schema_.reset(); - TestSSTableMeta::TearDown(); -} - TEST_F(TestRootBlockInfo, test_load_and_transform_root_block) { ASSERT_TRUE(root_info_.get_addr().is_block()); @@ -930,122 +883,6 @@ TEST_F(TestSSTableMeta, test_sstable_meta_deep_copy) ASSERT_EQ(0, MEMCMP(flat_meta_2->tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.get_variable_size())); } -TEST_F(TestMigrationSSTableParam, test_empty_sstable_serialize_and_deserialize) -{ - ObMigrationSSTableParam mig_param; - ASSERT_TRUE(!mig_param.is_valid()); - mig_param.reset(); - ASSERT_TRUE(!mig_param.is_valid()); - mig_param.basic_meta_ = sstable_meta_.get_basic_meta(); - for (int64_t i = 0; i < sstable_meta_.get_col_checksum_cnt(); ++i) { - ASSERT_EQ(OB_SUCCESS, mig_param.column_checksums_.push_back(sstable_meta_.get_col_checksum()[i])); - } - mig_param.table_key_ = table_key_; - char str[15] ="test serialize"; - mig_param.root_block_addr_.set_mem_addr(0, 15); - mig_param.root_block_buf_ = str; - MacroBlockId block_id(4096,0,0); - mig_param.data_block_macro_meta_addr_.set_block_addr(block_id, 1024, 2048, ObMetaDiskAddr::DiskType::BLOCK); - ASSERT_EQ(nullptr, mig_param.data_block_macro_meta_buf_); - mig_param.is_meta_root_ = true; - - ASSERT_TRUE(mig_param.is_valid()); - ASSERT_TRUE(mig_param.basic_meta_.is_valid()); - ASSERT_TRUE(mig_param.column_checksums_.count() > 0); - ASSERT_TRUE(mig_param.table_key_.is_valid()); - ASSERT_EQ(sstable_meta_.get_basic_meta(), mig_param.basic_meta_); - ASSERT_EQ(table_key_, mig_param.table_key_); - ASSERT_EQ(sstable_meta_.get_col_checksum_cnt(), mig_param.column_checksums_.count()); - - int64_t pos = 0; - const int64_t buf_len = mig_param.get_serialize_size(); - char *buf = new char [buf_len]; - ASSERT_EQ(OB_SUCCESS, mig_param.serialize(buf, buf_len, pos)); - ASSERT_TRUE(mig_param.is_valid()); - ASSERT_TRUE(mig_param.basic_meta_.is_valid()); - ASSERT_TRUE(mig_param.column_checksums_.count() > 0); - ASSERT_TRUE(mig_param.table_key_.is_valid()); - - ObMigrationSSTableParam tmp_param; - pos = 0; - ASSERT_EQ(OB_SUCCESS, tmp_param.deserialize(buf, buf_len, pos)); - ASSERT_TRUE(tmp_param.is_valid()); - ASSERT_TRUE(tmp_param.basic_meta_.is_valid()); - ASSERT_TRUE(tmp_param.column_checksums_.count() > 0); - ASSERT_TRUE(tmp_param.table_key_.is_valid()); - ASSERT_EQ(tmp_param.basic_meta_, mig_param.basic_meta_); - ASSERT_EQ(tmp_param.table_key_, mig_param.table_key_); - ASSERT_EQ(tmp_param.column_checksums_.count(), mig_param.column_checksums_.count()); - ASSERT_EQ(tmp_param.root_block_addr_, mig_param.root_block_addr_); - ASSERT_EQ(0, strcmp(tmp_param.root_block_buf_, mig_param.root_block_buf_)); - ASSERT_EQ(tmp_param.data_block_macro_meta_addr_, mig_param.data_block_macro_meta_addr_); - ASSERT_EQ(nullptr, tmp_param.data_block_macro_meta_buf_); - ASSERT_EQ(tmp_param.is_meta_root_, mig_param.is_meta_root_); -} - -TEST_F(TestMigrationSSTableParam, test_migrate_sstable) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTabletCreateSSTableParam src_sstable_param; - src_sstable_param.table_key_.table_type_ = ObITable::TableType::MAJOR_SSTABLE; - src_sstable_param.table_key_.tablet_id_ = tablet_id_; - src_sstable_param.table_key_.version_range_.base_version_ = ObVersionRange::MIN_VERSION; - src_sstable_param.table_key_.version_range_.snapshot_version_ = snapshot_version_; - src_sstable_param.schema_version_ = 3; - src_sstable_param.create_snapshot_version_ = 0; - src_sstable_param.progressive_merge_round_ = 0; - src_sstable_param.progressive_merge_step_ = 0; - src_sstable_param.table_mode_ = table_schema_.get_table_mode_struct(); - src_sstable_param.index_type_ = table_schema_.get_index_type(); - src_sstable_param.rowkey_column_cnt_ = 1; - src_sstable_param.root_block_addr_.set_none_addr(); - src_sstable_param.data_block_macro_meta_addr_.set_none_addr(); - src_sstable_param.root_row_store_type_ = ObRowStoreType::FLAT_ROW_STORE; - src_sstable_param.latest_row_store_type_ = ObRowStoreType::FLAT_ROW_STORE; - src_sstable_param.data_index_tree_height_ = 0; - src_sstable_param.index_blocks_cnt_ = 0; - src_sstable_param.data_blocks_cnt_ = 0; - src_sstable_param.micro_block_cnt_ = 0; - src_sstable_param.use_old_macro_block_count_ = 0; - src_sstable_param.column_cnt_ = 1; - src_sstable_param.data_checksum_ = 0; - src_sstable_param.occupy_size_ = 0; - src_sstable_param.ddl_scn_.set_min(); - src_sstable_param.filled_tx_scn_.set_min(); - src_sstable_param.tx_data_recycle_scn_.set_min(); - src_sstable_param.original_size_ = 0; - src_sstable_param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - src_sstable_param.encrypt_id_ = 1234; - src_sstable_param.master_key_id_ = 5678; - src_sstable_param.table_shared_flag_.set_shared_sstable(); - src_sstable_param.recycle_version_ = 0; - src_sstable_param.root_macro_seq_ = 0; - src_sstable_param.row_count_ = 0; - src_sstable_param.sstable_logic_seq_ = 0; - src_sstable_param.nested_offset_ = 0; - src_sstable_param.nested_size_ = 0; - ret = src_sstable_param.column_checksums_.push_back(2022); - ASSERT_EQ(OB_SUCCESS, ret); - - ObSSTableMeta src_meta; - ret = src_meta.init(src_sstable_param, allocator); - ASSERT_EQ(OB_SUCCESS, ret); - - ObMigrationSSTableParam mig_param; - mig_param.basic_meta_ = src_meta.get_basic_meta(); - mig_param.table_key_ = src_sstable_param.table_key_; - for (int64_t i = 0; i < sstable_meta_.get_col_checksum_cnt(); ++i) { - ASSERT_EQ(OB_SUCCESS, mig_param.column_checksums_.push_back(src_meta.get_col_checksum()[i])); - } - - ObTabletCreateSSTableParam dest_sstable_param; - ret = dest_sstable_param.init_for_ha(mig_param); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_TRUE(dest_sstable_param.encrypt_id_ == src_sstable_param.encrypt_id_); - ASSERT_TRUE(dest_sstable_param.table_shared_flag_ == src_sstable_param.table_shared_flag_); -} } // end namespace unittest } // end namespace oceanbase diff --git a/unittest/storage/compaction/test_compaction_iter.cpp b/unittest/storage/compaction/test_compaction_iter.cpp index 1886544ab..20d09d4fe 100644 --- a/unittest/storage/compaction/test_compaction_iter.cpp +++ b/unittest/storage/compaction/test_compaction_iter.cpp @@ -19,444 +19,207 @@ #define protected public #define private public +#include "src/storage/compaction/ob_compaction_schedule_iterator.h" #include "src/storage/ob_i_store.h" -#include "mtlenv/mock_tenant_module_env.h" + namespace oceanbase { using namespace share; using namespace common; -namespace storage -{ -bool ObLSHandle::is_valid() const -{ - return true; -} - -ObLSHandle& ObLSHandle::operator=(const ObLSHandle &other) { - return *this; -} - -} - namespace unittest { class MockObCompactionScheduleIterator : public compaction::ObCompactionScheduleIterator { public: - MockObCompactionScheduleIterator(const int64_t batch_tablet_cnt) + MockObCompactionScheduleIterator() : ObCompactionScheduleIterator(true/*is_major*/), - mock_tablet_id_cnt_(0), - tablet_cnt_in_ls_array_(NULL), + mock_tablet_cnt_(0), error_tablet_id_(), - errno_(OB_SUCCESS) - { - max_batch_tablet_cnt_ = batch_tablet_cnt; - } - int init( - const int64_t ls_cnt, - const int64_t max_batch_tablet_cnt, - const int64_t tablet_cnt_per_ls, - const int64_t error_tablet_idx, - const int input_errno); + errno_(OB_SUCCESS), + touch_counts_() + {} + int init( - const ObIArray &tablet_cnt_in_ls_array, - const int64_t max_batch_tablet_cnt, - const int64_t error_tablet_idx, - const int input_errno); - int init_map(); - virtual int get_cur_ls_handle(ObLSHandle &ls_handle) override - { - return OB_SUCCESS; - } - virtual int get_tablet_ids() override + const int64_t max_batch_tablet_cnt, + const int64_t tablet_cnt, + const int64_t error_tablet_idx = -1, + const int input_errno = OB_SUCCESS) { int ret = OB_SUCCESS; - int64_t touch_cnt = 0; - const ObLSID &ls_id = ls_ids_.at(ls_idx_); - int64_t tablet_cnt = 0; - if (OB_NOT_NULL(tablet_cnt_in_ls_array_)) { - tablet_cnt = tablet_cnt_in_ls_array_->at(ls_idx_); + if (OB_UNLIKELY(max_batch_tablet_cnt <= 0 || tablet_cnt < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(max_batch_tablet_cnt), K(tablet_cnt)); } else { - tablet_cnt = mock_tablet_id_cnt_; - } - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_cnt; ++i) { - const ObTabletID &cur_tablet_id = ObTabletID(i + 1); - ObTabletLSPair pair(cur_tablet_id, ls_id); - if (OB_SUCC(tablet_ids_.array_.push_back(cur_tablet_id))) { - if (OB_SUCCESS != tablet_map_.get_refactored(pair, touch_cnt)) { - touch_cnt = 0; - if (OB_FAIL(tablet_map_.set_refactored(pair, touch_cnt))) { - LOG_WARN("failed to set refactor", KR(ret), K(cur_tablet_id), K(touch_cnt)); - } + max_batch_tablet_cnt_ = max_batch_tablet_cnt; + mock_tablet_cnt_ = tablet_cnt; + error_tablet_id_ = ObTabletID(error_tablet_idx + 1); + errno_ = input_errno; + touch_counts_.reset(); + for (int64_t i = 0; OB_SUCC(ret) && i < mock_tablet_cnt_; ++i) { + if (OB_FAIL(touch_counts_.push_back(0))) { + LOG_WARN("failed to init touch count", KR(ret), K(i)); } } + ObLS *ls = nullptr; + if (OB_SUCC(ret) + && OB_FAIL(compaction::ObBasicMergeScheduleIterator::init(max_batch_tablet_cnt, ls))) { + LOG_WARN("failed to init basic iterator", KR(ret)); + } } - if (OB_SUCC(ret)) { - LOG_INFO("success to get tablet ids", KR(ret), K(tablet_cnt)); + return ret; + } + + virtual int get_tablet_ids() override + { + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < mock_tablet_cnt_; ++i) { + if (OB_FAIL(tablet_ids_.array_.push_back(ObTabletID(i + 1)))) { + LOG_WARN("failed to push tablet id", KR(ret), K(i)); + } } return ret; } + virtual int get_tablet_handle(const ObTabletID &tablet_id, ObTabletHandle &tablet_handle) override { int ret = OB_SUCCESS; - int64_t touch_cnt = 0; - ObTabletLSPair pair(tablet_id, ls_ids_.at(ls_idx_)); + UNUSED(tablet_handle); + const int64_t idx = tablet_id.id() - 1; if (tablet_id == error_tablet_id_) { ret = errno_; - } else if (OB_FAIL(tablet_map_.get_refactored(pair, touch_cnt))) { - LOG_WARN("failed to get refactor", KR(ret), K(tablet_id), K(touch_cnt)); - } else if (FALSE_IT(++touch_cnt)) { - } else if (OB_FAIL(tablet_map_.set_refactored(pair, touch_cnt, 1/*overwrite*/))) { - LOG_WARN("failed to set refactor", KR(ret), K(tablet_id), K(touch_cnt)); + } else if (OB_UNLIKELY(idx < 0 || idx >= touch_counts_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet id is out of range", KR(ret), K(tablet_id), K_(mock_tablet_cnt)); } else { - LOG_INFO("success to set refactor", KR(ret), K(tablet_id), K(pair), K(touch_cnt)); + touch_counts_.at(idx)++; } return ret; } - void prepare_ls_id_array(const int64_t ls_cnt) + + int check_touch_counts(const int64_t error_tablet_idx, const int input_errno) { - for (int64_t i = 0; i < ls_cnt; ++i) { - ASSERT_EQ(OB_SUCCESS, ls_ids_.push_back(ObLSID(i + 1))); + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < touch_counts_.count(); ++i) { + const bool should_touch = OB_SUCCESS == input_errno + || (OB_TABLET_NOT_EXIST == input_errno && i != error_tablet_idx) + || (OB_TABLET_NOT_EXIST != input_errno && i < error_tablet_idx); + const int64_t expected = should_touch ? 1 : 0; + if (touch_counts_.at(i) != expected) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet touch count is unexpected", KR(ret), K(i), + "actual", touch_counts_.at(i), K(expected), + K(error_tablet_idx), K(input_errno)); + } } + return ret; } - int check_valid(const int64_t expect_touch_cnt); - static const int64_t DEFAULT_BUCKET_NUM = 1024; - typedef hash::ObHashMap TabletID2CntMap; - TabletID2CntMap tablet_map_; - int64_t mock_tablet_id_cnt_; - const ObIArray *tablet_cnt_in_ls_array_; + + int64_t mock_tablet_cnt_; ObTabletID error_tablet_id_; int errno_; + ObArray touch_counts_; }; -int MockObCompactionScheduleIterator::init_map() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(tablet_map_.created())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet map is created", KR(ret)); - } else if (OB_FAIL(tablet_map_.create(DEFAULT_BUCKET_NUM, "MockMap", "MockMap"))) { - LOG_WARN("failed to create tablet map", KR(ret)); - } - return ret; -} - -int MockObCompactionScheduleIterator::init( - const int64_t ls_cnt, - const int64_t max_batch_tablet_cnt, - const int64_t tablet_cnt_per_ls, - const int64_t error_tablet_idx, - const int input_errno) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(init_map())) { - LOG_WARN("failed to create tablet map", KR(ret)); - } else { - prepare_ls_id_array(ls_cnt); - max_batch_tablet_cnt_ = max_batch_tablet_cnt; - mock_tablet_id_cnt_ = tablet_cnt_per_ls; - error_tablet_id_ = ObTabletID(error_tablet_idx + 1); - errno_ = input_errno; - } - return ret; -} - -int MockObCompactionScheduleIterator::init( - const ObIArray &tablet_cnt_in_ls_array, - const int64_t max_batch_tablet_cnt, - const int64_t error_tablet_idx, - const int input_errno) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(tablet_cnt_in_ls_array.empty() || max_batch_tablet_cnt <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_cnt_in_ls_array), K(max_batch_tablet_cnt)); - } else if (OB_FAIL(init_map())) { - LOG_WARN("failed to create tablet map", KR(ret)); - } else { - prepare_ls_id_array(tablet_cnt_in_ls_array.count()); - max_batch_tablet_cnt_ = max_batch_tablet_cnt; - tablet_cnt_in_ls_array_ = &tablet_cnt_in_ls_array; - error_tablet_id_ = ObTabletID(error_tablet_idx + 1); - errno_ = input_errno; - } - return ret; -} - -int MockObCompactionScheduleIterator::check_valid(const int64_t expect_touch_cnt) -{ - int ret = OB_SUCCESS; - int64_t touch_cnt = 0; - for (int64_t ls_idx = 0; ls_idx < ls_ids_.count(); ++ls_idx) { - const ObLSID &ls_id = ls_ids_.at(ls_idx); - for (int64_t i = 0; OB_SUCC(ret) && i < mock_tablet_id_cnt_; ++i) { - const ObTabletID &cur_tablet_id = ObTabletID(i + 1); - ObTabletLSPair pair(cur_tablet_id, ls_id); - if (!error_tablet_id_.is_valid() - || (cur_tablet_id < error_tablet_id_ || (OB_TABLET_NOT_EXIST == errno_ && cur_tablet_id > error_tablet_id_))) { - if (OB_FAIL(tablet_map_.get_refactored(pair, touch_cnt))) { - LOG_WARN("failed to get refactor", KR(ret), K(cur_tablet_id), K(pair), K(touch_cnt)); - } else if (touch_cnt != expect_touch_cnt) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet touch cnt is unexpected", KR(ret), K(pair), K(touch_cnt), K(expect_touch_cnt), K(error_tablet_id_)); - } - } - } // for - } // for - return ret; -} - class TestCompactionIter : public ::testing::Test { public: void test_iter( - const int64_t ls_cnt, - const int64_t max_batch_tablet_cnt, - const int64_t tablet_cnt_per_ls, - const ObIArray *tablet_cnt_in_ls_array = NULL, - const int64_t error_tablet_idx = -1, - const int input_errno = OB_SUCCESS); + const int64_t max_batch_tablet_cnt, + const int64_t tablet_cnt, + const int64_t error_tablet_idx = -1, + const int input_errno = OB_SUCCESS); }; void TestCompactionIter::test_iter( - const int64_t ls_cnt, - const int64_t max_batch_tablet_cnt, - const int64_t tablet_cnt_per_ls, - const ObIArray *tablet_cnt_in_ls_array, - const int64_t error_tablet_idx, - const int input_errno) + const int64_t max_batch_tablet_cnt, + const int64_t tablet_cnt, + const int64_t error_tablet_idx, + const int input_errno) { - LOG_INFO("test_iter", K(ls_cnt), K(max_batch_tablet_cnt), K(tablet_cnt_per_ls), K(error_tablet_idx), K(input_errno)); - MockObCompactionScheduleIterator iter(max_batch_tablet_cnt); - if (OB_ISNULL(tablet_cnt_in_ls_array)) { - ASSERT_TRUE(ls_cnt > 0 && tablet_cnt_per_ls > 0); - ASSERT_EQ(OB_SUCCESS, iter.init(ls_cnt, max_batch_tablet_cnt, tablet_cnt_per_ls, error_tablet_idx, input_errno)); - } else { - ASSERT_TRUE(ls_cnt == 0 && tablet_cnt_per_ls == 0); - ASSERT_EQ(OB_SUCCESS, iter.init(*tablet_cnt_in_ls_array, max_batch_tablet_cnt, error_tablet_idx, input_errno)); - } + LOG_INFO("test_iter", K(max_batch_tablet_cnt), K(tablet_cnt), K(error_tablet_idx), K(input_errno)); + MockObCompactionScheduleIterator iter; + ASSERT_EQ(OB_SUCCESS, iter.init(max_batch_tablet_cnt, tablet_cnt, error_tablet_idx, input_errno)); int ret = OB_SUCCESS; - ObLSHandle ls_handle; ObTabletHandle tablet_handle; int64_t iter_batch_cnt = 0; - int iter_cnt = 0; + int64_t iter_cnt = 0; while (OB_SUCC(ret)) { - while (OB_SUCC(iter.get_next_ls(ls_handle))) { - while (OB_SUCC(ret)) { - if (OB_SUCC(iter.get_next_tablet(tablet_handle))) { - iter_cnt++; - } else { - if (OB_ITER_END != ret) { - iter.skip_cur_ls(); - } - ret = OB_SUCCESS; - break; + while (OB_SUCC(ret)) { + if (OB_SUCC(iter.get_next_tablet(tablet_handle))) { + iter_cnt++; + } else { + if (OB_ITER_END != ret) { + iter.finish_scan(); } - } // iter tablet - } // iter ls - ASSERT_EQ(OB_ITER_END, ret); - ret = OB_SUCCESS; + ret = OB_SUCCESS; + break; + } + } ++iter_batch_cnt; if (iter.is_valid()) { - ASSERT_EQ(iter.schedule_tablet_cnt_ >= max_batch_tablet_cnt, true); + ASSERT_GE(iter.schedule_tablet_cnt_, max_batch_tablet_cnt); iter.start_cur_batch(); } else { break; } - } // while - ASSERT_EQ(OB_SUCCESS, iter.check_valid(1)); - if (OB_ISNULL(tablet_cnt_in_ls_array)) { - if (input_errno == OB_SUCCESS) { - ASSERT_EQ(iter_cnt, ls_cnt * tablet_cnt_per_ls); - } else if (OB_TABLET_NOT_EXIST == input_errno) { - // for this errno, just skip this tablet - ASSERT_EQ(iter_cnt, ls_cnt * (tablet_cnt_per_ls - 1)); - } else { - ASSERT_EQ(iter_cnt, ls_cnt * error_tablet_idx); - } - } else { - int64_t expect_iter_cnt = 0; - for (int64_t idx = 0; idx < tablet_cnt_in_ls_array->count(); ++idx) { - if (input_errno == OB_SUCCESS) { - expect_iter_cnt += tablet_cnt_in_ls_array->at(idx); - } else if (OB_TABLET_NOT_EXIST == input_errno) { - // for this errno, just skip this tablet - expect_iter_cnt += (tablet_cnt_in_ls_array->at(idx) - (tablet_cnt_in_ls_array->at(idx) > error_tablet_idx ? 1 : 0)); - } else { - expect_iter_cnt += MIN(tablet_cnt_in_ls_array->at(idx), error_tablet_idx); - } - } // for - ASSERT_EQ(iter_cnt, expect_iter_cnt); } - ASSERT_EQ(iter_batch_cnt, MAX(1, iter_cnt / max_batch_tablet_cnt + (iter_cnt % max_batch_tablet_cnt != 0))); -} -TEST_F(TestCompactionIter, test_normal_loop) -{ - test_iter( - 3,/*ls_cnt*/ - 10000,/*max_batch_tablet_cnt*/ - 10000/*tablet_cnt_per_ls*/ - ); - test_iter( - 5,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 10000/*tablet_cnt_per_ls*/ - ); - test_iter( - 5,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 100/*tablet_cnt_per_ls*/ - ); - test_iter( - 5,/*ls_cnt*/ - 100,/*max_batch_tablet_cnt*/ - 100/*tablet_cnt_per_ls*/ - ); - test_iter( - 5,/*ls_cnt*/ - 200,/*max_batch_tablet_cnt*/ - 100/*tablet_cnt_per_ls*/ - ); + int64_t expect_iter_cnt = tablet_cnt; + if (OB_TABLET_NOT_EXIST == input_errno) { + expect_iter_cnt = tablet_cnt - (tablet_cnt > error_tablet_idx ? 1 : 0); + } else if (OB_SUCCESS != input_errno) { + expect_iter_cnt = MIN(tablet_cnt, error_tablet_idx); + } + ASSERT_EQ(iter_cnt, expect_iter_cnt); + ASSERT_EQ(OB_SUCCESS, iter.check_touch_counts(error_tablet_idx, input_errno)); + ASSERT_EQ(iter_batch_cnt, MAX(1, iter_cnt / max_batch_tablet_cnt + (iter_cnt % max_batch_tablet_cnt != 0))); } -TEST_F(TestCompactionIter, test_single_ls) +TEST_F(TestCompactionIter, test_tablet_iteration_normal_loop) { - test_iter( - 1,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 100/*tablet_cnt_per_ls*/ - ); - test_iter( - 1,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 1000/*tablet_cnt_per_ls*/ - ); - test_iter( - 1,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 10000/*tablet_cnt_per_ls*/ - ); + test_iter(10000, 0); + test_iter(10000, 100); + test_iter(1000, 1000); + test_iter(1000, 10000); + test_iter(100, 10000); } -TEST_F(TestCompactionIter, test_loop_with_not_exist_tablet) +TEST_F(TestCompactionIter, test_tablet_iteration_skips_missing_tablet) { - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 10000,/*tablet_cnt_per_ls*/ - NULL, - 50,/*error_tablet_idx*/ - OB_TABLET_NOT_EXIST/*errno*/ - ); - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 10000,/*tablet_cnt_per_ls*/ - NULL, - 50,/*error_tablet_idx*/ - OB_TABLET_NOT_EXIST/*errno*/ - ); - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 1001,/*tablet_cnt_per_ls*/ - NULL, - 999,/*error_tablet_idx*/ - OB_TABLET_NOT_EXIST/*errno*/ - ); + test_iter(1000, 10000, 50, OB_TABLET_NOT_EXIST); + test_iter(1000, 1001, 999, OB_TABLET_NOT_EXIST); } -TEST_F(TestCompactionIter, test_loop_with_errno) +TEST_F(TestCompactionIter, test_tablet_iteration_stops_on_error) { - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 10000,/*tablet_cnt_per_ls*/ - NULL, - 50,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 1000,/*tablet_cnt_per_ls*/ - NULL, - 999,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); - test_iter( - 3,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 999,/*tablet_cnt_per_ls*/ - NULL, - 999,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); - test_iter( - 3,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 999,/*tablet_cnt_per_ls*/ - NULL, - 0,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); - test_iter( - 2,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 50,/*tablet_cnt_per_ls*/ - NULL, - 1,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); + test_iter(1000, 10000, 50, OB_ERR_UNEXPECTED); + test_iter(1000, 1000, 999, OB_ERR_UNEXPECTED); + test_iter(1000, 999, 0, OB_ERR_UNEXPECTED); + test_iter(1000, 50, 1, OB_ERR_UNEXPECTED); } -TEST_F(TestCompactionIter, test_iter_with_tablet_cnt_list) +TEST_F(TestCompactionIter, test_tablet_iteration_can_restart) { - ObSEArray tablet_cnt_in_ls; - tablet_cnt_in_ls.push_back(100); - tablet_cnt_in_ls.push_back(0); - tablet_cnt_in_ls.push_back(2); - tablet_cnt_in_ls.push_back(0); - tablet_cnt_in_ls.push_back(300); - - test_iter( - 0,/*ls_cnt*/ - 1000,/*max_batch_tablet_cnt*/ - 0,/*tablet_cnt_per_ls*/ - &tablet_cnt_in_ls - ); - test_iter( - 0,/*ls_cnt*/ - 100,/*max_batch_tablet_cnt*/ - 0,/*tablet_cnt_per_ls*/ - &tablet_cnt_in_ls, - 50,/*error_tablet_idx*/ - OB_TABLET_NOT_EXIST/*errno*/ - ); - test_iter( - 0,/*ls_cnt*/ - 200,/*max_batch_tablet_cnt*/ - 0,/*tablet_cnt_per_ls*/ - &tablet_cnt_in_ls, - 50,/*error_tablet_idx*/ - OB_TABLET_NOT_EXIST/*errno*/ - ); - test_iter( - 0,/*ls_cnt*/ - 250,/*max_batch_tablet_cnt*/ - 0,/*tablet_cnt_per_ls*/ - &tablet_cnt_in_ls, - 50,/*error_tablet_idx*/ - OB_ERR_UNEXPECTED/*errno*/ - ); + MockObCompactionScheduleIterator iter; + ObTabletHandle tablet_handle; + ASSERT_EQ(OB_SUCCESS, iter.init(2, 3)); + ASSERT_EQ(OB_SUCCESS, iter.get_next_tablet(tablet_handle)); + ASSERT_EQ(OB_SUCCESS, iter.get_next_tablet(tablet_handle)); + iter.start_cur_batch(); + ASSERT_EQ(OB_SUCCESS, iter.get_next_tablet(tablet_handle)); + ASSERT_EQ(OB_ITER_END, iter.get_next_tablet(tablet_handle)); + ASSERT_TRUE(iter.is_scan_finish()); + + ASSERT_EQ(OB_SUCCESS, iter.init(2, 3)); + ASSERT_FALSE(iter.is_scan_finish()); + ASSERT_EQ(OB_SUCCESS, iter.get_next_tablet(tablet_handle)); } } // namespace unittest -} //namespace oceanbase +} // namespace oceanbase int main(int argc, char **argv) { diff --git a/unittest/storage/compaction/test_diagnose_info_mgr.cpp b/unittest/storage/compaction/test_diagnose_info_mgr.cpp index a1e17fc80..0a1c04d05 100644 --- a/unittest/storage/compaction/test_diagnose_info_mgr.cpp +++ b/unittest/storage/compaction/test_diagnose_info_mgr.cpp @@ -32,8 +32,7 @@ class TestDiagnoseInfoMgr : public ::testing::Test { public: TestDiagnoseInfoMgr() - : ls_id_(TEST_LS_ID), - suspect_info_mgr_(NULL), + : suspect_info_mgr_(NULL), diagnose_tablet_mgr_(NULL), is_inited_(false) {} @@ -46,8 +45,6 @@ class TestDiagnoseInfoMgr : public ::testing::Test const ObTabletID &tablet_id, ObScheduleSuspectInfo &info); static constexpr int64_t TEST_TENANT_ID = 1001; - static constexpr int64_t TEST_LS_ID = 9001; - share::ObLSID ls_id_; ObScheduleSuspectInfoMgr *suspect_info_mgr_; ObDiagnoseTabletMgr *diagnose_tablet_mgr_; ObDiagnoseInfoParam<2, 0> param_; @@ -93,7 +90,6 @@ int TestDiagnoseInfoMgr::gene_suspect_info( int ret = OB_SUCCESS; info.priority_ = static_cast(prio); info.merge_type_ = merge_type; - info.ls_id_ = ls_id_; info.tablet_id_ = tablet_id; info.info_param_ = ¶m_; return ret; @@ -102,7 +98,6 @@ int TestDiagnoseInfoMgr::gene_suspect_info( bool judge_equal(const ObScheduleSuspectInfo &a, const ObScheduleSuspectInfo &b) { return a.priority_ == b.priority_ - && a.ls_id_ == b.ls_id_ && a.tablet_id_ == b.tablet_id_ && a.merge_type_ == b.merge_type_; } @@ -147,27 +142,27 @@ TEST_F(TestDiagnoseInfoMgr, test_diagnose_tablet_mgr) const ObTabletID tablet_id(1); ASSERT_EQ(OB_SUCCESS, ObDiagnoseTabletMgr::mtl_init(diagnose_tablet_mgr_)); - ret = diagnose_tablet_mgr_->add_diagnose_tablet(ls_id_, tablet_id, TYPE_DIAGNOSE_TABLET_MAX); + ret = diagnose_tablet_mgr_->add_diagnose_tablet(tablet_id, TYPE_DIAGNOSE_TABLET_MAX); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); - ret = diagnose_tablet_mgr_->add_diagnose_tablet(ls_id_, tablet_id, TYPE_MINOR_MERGE); + ret = diagnose_tablet_mgr_->add_diagnose_tablet(tablet_id, TYPE_MINOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); - ret = diagnose_tablet_mgr_->add_diagnose_tablet(ls_id_, tablet_id, TYPE_MINOR_MERGE); + ret = diagnose_tablet_mgr_->add_diagnose_tablet(tablet_id, TYPE_MINOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); // same diagnose type is registed, return success anyway - ret = diagnose_tablet_mgr_->add_diagnose_tablet(ls_id_, tablet_id, TYPE_MINOR_MERGE); + ret = diagnose_tablet_mgr_->add_diagnose_tablet(tablet_id, TYPE_MINOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); // same diagnose type is registed, return success anyway - ret = diagnose_tablet_mgr_->add_diagnose_tablet(ls_id_, tablet_id, TYPE_RS_MAJOR_MERGE); + ret = diagnose_tablet_mgr_->add_diagnose_tablet(tablet_id, TYPE_RS_MAJOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); - ret = diagnose_tablet_mgr_->delete_diagnose_tablet(ls_id_, tablet_id, TYPE_MINOR_MERGE); + ret = diagnose_tablet_mgr_->delete_diagnose_tablet(tablet_id, TYPE_MINOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(1, diagnose_tablet_mgr_->diagnose_tablet_map_.size()); // after remove all flag, registed tablet is deleted - ret = diagnose_tablet_mgr_->delete_diagnose_tablet(ls_id_, tablet_id, TYPE_RS_MAJOR_MERGE); + ret = diagnose_tablet_mgr_->delete_diagnose_tablet(tablet_id, TYPE_RS_MAJOR_MERGE); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(0, diagnose_tablet_mgr_->diagnose_tablet_map_.size()); } diff --git a/unittest/storage/ddl/test_auto_split_polling_mgr.cpp b/unittest/storage/ddl/test_auto_split_polling_mgr.cpp index 98eda813c..1cd020450 100644 --- a/unittest/storage/ddl/test_auto_split_polling_mgr.cpp +++ b/unittest/storage/ddl/test_auto_split_polling_mgr.cpp @@ -44,7 +44,6 @@ using namespace compaction; static const uint64_t TEST_TENANT_A_ID = 1002; static const uint64_t TEST_TENANT_B_ID = 1003; static const uint64_t TEST_TENANT_C_ID = 1004; -static const ObLSID ls_id{1}; class TestSplitTaskScheduler : public ::testing::Test { @@ -105,9 +104,9 @@ TEST_F(TestSplitTaskScheduler, simple_push_and_pop) { ObArray task_array; ObArray ids; - ObAutoSplitTask task_a(ls_id, 1/*tablet_id*/, 1/*auto_split_size*/, 2/*used_disk_size*/, 0); - ObAutoSplitTask task_b(ls_id, 2/*tablet_id*/, 1/*auto_split_size*/, 3/*used_disk_size*/, 0); - ObAutoSplitTask task_c(ls_id, 3/*tablet_id*/, 1/*auto_split_size*/, 4/*used_disk_size*/, 0); + ObAutoSplitTask task_a(1/*tablet_id*/, 1/*auto_split_size*/, 2/*used_disk_size*/, 0); + ObAutoSplitTask task_b(2/*tablet_id*/, 1/*auto_split_size*/, 3/*used_disk_size*/, 0); + ObAutoSplitTask task_c(3/*tablet_id*/, 1/*auto_split_size*/, 4/*used_disk_size*/, 0); ASSERT_OK(task_array.push_back(task_a)); ASSERT_OK(polling_mgr_.push_tasks(task_array)); task_array.reuse(); @@ -154,8 +153,8 @@ TEST_F(TestSplitTaskScheduler, simple_push_and_pop) TEST_F(TestSplitTaskScheduler, test_reset) { ObArray task_array; - ObAutoSplitTask task_a(ls_id, 1/*tablet_id*/, 1/*auto_split_size*/, 2/*used_disk_size*/, 0); - ObAutoSplitTask task_b(ls_id, 2/*tablet_id*/, 1/*auto_split_size*/, 3/*used_disk_size*/, 0); + ObAutoSplitTask task_a(1/*tablet_id*/, 1/*auto_split_size*/, 2/*used_disk_size*/, 0); + ObAutoSplitTask task_b(2/*tablet_id*/, 1/*auto_split_size*/, 3/*used_disk_size*/, 0); ASSERT_OK(task_array.push_back(task_a)); ASSERT_OK(polling_mgr_.push_tasks(task_array)); task_array.reuse(); @@ -182,12 +181,12 @@ TEST_F(TestSplitTaskScheduler, test_reset) TEST_F(TestSplitTaskScheduler, single_tenant_hard_push_and_pop) { ObArray task_array; - ObAutoSplitTask task_high_prio(ls_id, 1000/*tablet_id*/, 1/*auto_split_size*/, 1000/*used_disk_size*/, 0); + ObAutoSplitTask task_high_prio(1000/*tablet_id*/, 1/*auto_split_size*/, 1000/*used_disk_size*/, 0); ASSERT_OK(task_array.push_back(task_high_prio)); ASSERT_OK(polling_mgr_.push_tasks(task_array)); task_array.reuse(); for (int64_t i = 2; i < ObAutoSplitTaskCache::CACHE_MAX_CAPACITY + 10; ++i ) { - ASSERT_OK(task_array.push_back(ObAutoSplitTask(ls_id, i/*tablet_id*/, 1/*auto_split_size*/, i/*used_disk_size*/, 0))); + ASSERT_OK(task_array.push_back(ObAutoSplitTask(i/*tablet_id*/, 1/*auto_split_size*/, i/*used_disk_size*/, 0))); ASSERT_OK(polling_mgr_.push_tasks(task_array)); task_array.reuse(); } @@ -245,11 +244,11 @@ TEST_F(TestSplitTaskScheduler, parallel_push_and_pop) for (int64_t i = 0; i < nums_1.count(); ++i ) { tmp_array.reuse(); int64_t num_1 = nums_1.at(i); - ASSERT_OK(tmp_array.push_back(ObAutoSplitTask(ls_id, num_1/*tablet_id*/, 1/*auto_split_size*/, num_1/*used_disk_size*/, 0))); + ASSERT_OK(tmp_array.push_back(ObAutoSplitTask(num_1/*tablet_id*/, 1/*auto_split_size*/, num_1/*used_disk_size*/, 0))); ASSERT_OK(task_array_1.push_back(tmp_array)); tmp_array.reuse(); int64_t num_2 = nums_2.at(i); - ASSERT_OK(tmp_array.push_back(ObAutoSplitTask(ls_id, num_2/*tablet_id*/, 1/*auto_split_size*/, num_2/*used_disk_size*/, 0))); + ASSERT_OK(tmp_array.push_back(ObAutoSplitTask(num_2/*tablet_id*/, 1/*auto_split_size*/, num_2/*used_disk_size*/, 0))); ASSERT_OK(task_array_2.push_back(tmp_array)); } diff --git a/unittest/storage/ddl/test_tablet_data_split.cpp b/unittest/storage/ddl/test_tablet_data_split.cpp index 9752c071f..5eaa507f8 100644 --- a/unittest/storage/ddl/test_tablet_data_split.cpp +++ b/unittest/storage/ddl/test_tablet_data_split.cpp @@ -36,7 +36,6 @@ using namespace compaction; static const int64_t TEST_TENANT_ID = 1; static const int64_t TEST_TABLE_ID = 8000; -static const ObLSID TEST_LS_ID = ObLSID(9001); static const ObTabletID TEST_TABLET_ID = ObTabletID(2323233); static const int64_t TEST_LOB_COL_CNT = 3; static const int64_t TEST_SPLIT_DST_CNT = 2; @@ -74,7 +73,6 @@ int TestDataSplit::prepare_mock_start_arg( obcall::ObDDLBuildSingleReplicaRequestArg &arg) { int ret = OB_SUCCESS; - arg.ls_id_ = TEST_LS_ID; arg.source_tablet_id_ = TEST_TABLET_ID; arg.dest_tablet_id_ = TEST_TABLET_ID; arg.source_table_id_ = TEST_TABLE_ID; @@ -87,7 +85,6 @@ int TestDataSplit::prepare_mock_start_arg( arg.execution_id_ = 1; arg.tablet_task_id_ = 1; arg.data_format_version_ = 1; - arg.dest_ls_id_ = TEST_LS_ID; arg.dest_schema_version_ = 1; if (is_split) { arg.ddl_type_ = 100; @@ -135,7 +132,6 @@ int TestDataSplit::prepare_mock_start_arg( int TestDataSplit::prepare_mock_finish_arg(obcall::ObTabletSplitArg &arg) { int ret = OB_SUCCESS; - arg.ls_id_ = TEST_LS_ID; arg.table_id_ = TEST_TABLE_ID; arg.lob_table_id_ = TEST_TABLE_ID; arg.schema_version_ = 1; diff --git a/unittest/storage/init_basic_struct.h b/unittest/storage/init_basic_struct.h index f2619768e..9bca8efcb 100644 --- a/unittest/storage/init_basic_struct.h +++ b/unittest/storage/init_basic_struct.h @@ -58,8 +58,7 @@ int __attribute__((weak)) build_test_schema(share::schema::ObTableSchema &table return build_test_schema(table_schema, table_id, "test_merge"); } -int __attribute__((weak)) gen_create_tablet_arg(const share::ObLSID &ls_id, - const ObTabletID &tablet_id, +int __attribute__((weak)) gen_create_tablet_arg(const ObTabletID &tablet_id, obcall::ObBatchCreateTabletArg &arg, const int64_t count = 1, share::schema::ObTableSchema *out_table_schema = nullptr) @@ -102,8 +101,8 @@ int __attribute__((weak)) gen_create_tablet_arg(const share::ObLSID &ls_id, ))) { STORAGE_LOG(WARN, "failed to init tablet info", KR(ret), K(index_tablet_ids), K(tablet_id), K(index_tablet_schema_idxs)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn(), false/*need_check_tablet_cnt*/))) { - STORAGE_LOG(WARN, "failed to init create tablet", KR(ret), K(OB_SERVER_TENANT_ID), K(ls_id)); + } else if (OB_FAIL(arg.init_create_tablet(share::SCN::min_scn(), false/*need_check_tablet_cnt*/))) { + STORAGE_LOG(WARN, "failed to init create tablet", KR(ret), K(OB_SERVER_TENANT_ID)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { STORAGE_LOG(WARN, "failed to push back table schema", KR(ret), K(table_schema)); } else if (OB_FAIL(arg.tablets_.push_back(tablet_info))) { diff --git a/unittest/storage/memtable/test_memtable_basic.cpp b/unittest/storage/memtable/test_memtable_basic.cpp index b994dc815..d1c1903bd 100644 --- a/unittest/storage/memtable/test_memtable_basic.cpp +++ b/unittest/storage/memtable/test_memtable_basic.cpp @@ -28,13 +28,12 @@ #include "storage/tx_storage/ob_tenant_freezer.h" #include "storage/blocksstable/ob_datum_row.h" #include "storage/memtable/ob_memtable_context.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "storage/tx/ob_multi_data_source.h" #include "storage/tx/ob_trans_define_v4.h" #include "storage/memtable/mvcc/ob_mvcc_row.h" #include "share/scn.h" #include "storage/ls/ob_ls.h" -#include "storage/tx_storage/ob_ls_map.h" #include "share/schema/ob_column_schema.h" #include "storage/ob_storage_schema.h" #include "storage/tablet/ob_tablet_multi_source_data.h" @@ -161,10 +160,7 @@ class TestMemtable : public testing::Test table_key.scn_range_.end_scn_ = share::SCN::max_scn(); int64_t schema_version = 1; uint32_t freeze_clock = 0; - ObLSHandle ls_handle; - ls_handle.set_ls(ls_map_, ls_, ObLSGetMod::DATA_MEMTABLE_MOD); - - return mt_table.init(table_key, ls_handle, &freezer_, &memtable_mgr_, schema_version, freeze_clock); + return mt_table.init(table_key, &ls_, &freezer_, &memtable_mgr_, schema_version, freeze_clock); } int mock_col_desc() { @@ -283,7 +279,6 @@ class TestMemtable : public testing::Test ObTableReadInfo read_info_; ObArenaAllocator allocator_; MemtableIDMap ctx_map_; - ObLSMap ls_map_; }; class RunCtxGuard @@ -403,7 +398,7 @@ class RunCtxGuard } TestMemtable *tm_; - ObPartTransCtx trans_ctx_; + ObTxCtx trans_ctx_; ObMemtableCtx mem_ctx_; ObTxDesc tx_desc_; }; diff --git a/unittest/storage/mock_access_service.cpp b/unittest/storage/mock_access_service.cpp index 0fcd4d258..c84db6492 100644 --- a/unittest/storage/mock_access_service.cpp +++ b/unittest/storage/mock_access_service.cpp @@ -27,7 +27,6 @@ MockObAccessService::MockObAccessService(ObLSTabletService *tablet_service) } int MockObAccessService::insert_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, @@ -38,27 +37,25 @@ int MockObAccessService::insert_rows( int ret = OB_SUCCESS; ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!ls_id.is_valid()) - || OB_UNLIKELY(!tablet_id.is_valid()) + if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_UNLIKELY(!tx_desc.is_valid()) || OB_UNLIKELY(!dml_param.is_valid()) || OB_UNLIKELY(column_ids.count() <= 0) || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(tx_desc), + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(tx_desc), K(dml_param), K(column_ids), K(row_iter)); } else if (OB_ISNULL(tablet_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet service is null", K(ret)); - } else if (OB_FAIL(check_write_allowed_(ls_id, - tablet_id, + } else if (OB_FAIL(check_write_allowed_(tablet_id, ObStoreAccessType::MODIFY, dml_param, dml_param.timeout_, tx_desc, tablet_handle, *dml_param.store_ctx_guard_))) { - LOG_WARN("fail to check query allowed", K(ret), K(ls_id), K(tablet_id)); + LOG_WARN("fail to check query allowed", K(ret), K(tablet_id)); } else { ret = tablet_service_->insert_rows(tablet_handle, dml_param.store_ctx_guard_->get_store_ctx(), diff --git a/unittest/storage/mock_access_service.h b/unittest/storage/mock_access_service.h index e1789b0c1..b04448dfc 100644 --- a/unittest/storage/mock_access_service.h +++ b/unittest/storage/mock_access_service.h @@ -32,7 +32,6 @@ class MockObAccessService : public ObAccessService virtual ~MockObAccessService() = default; public: int insert_rows( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, diff --git a/unittest/storage/mock_ob_log_handler.h b/unittest/storage/mock_ob_log_handler.h index fc5f6e378..3f2c21b9b 100644 --- a/unittest/storage/mock_ob_log_handler.h +++ b/unittest/storage/mock_ob_log_handler.h @@ -35,6 +35,7 @@ class MockObLogHandler : public logservice::ObILogHandler { public: MockObLogHandler(){}; + int bootstrap() override { return OB_SUCCESS; } virtual bool is_valid() const { return true; } virtual int append(const void *buffer, const int64_t nbytes, @@ -151,24 +152,6 @@ class MockObLogHandler : public logservice::ObILogHandler UNUSED(iter); return OB_SUCCESS; }; - int set_initial_member_list(const common::ObMemberList &member_list, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) - { - UNUSED(member_list); - UNUSED(paxos_replica_num); - UNUSED(learner_list); - return OB_SUCCESS; - } - int set_initial_member_list(const common::ObMemberList &member_list, - const common::ObMember &arb_replica, - const int64_t paxos_replica_num, - const common::GlobalLearnerList &learner_list) - { - UNUSEDx(member_list, arb_replica, paxos_replica_num); - UNUSED(learner_list); - return OB_SUCCESS; - } int get_end_scn(share::SCN &scn) const { UNUSED(scn); @@ -281,22 +264,10 @@ class MockObLogHandler : public logservice::ObILogHandler UNUSED(is_rebuild); return OB_SUCCESS; } - bool is_sync_enabled() const - { - return true; - } bool is_replay_enabled() const { return true; } - int enable_sync() - { - return OB_SUCCESS; - } - int disable_sync() - { - return OB_SUCCESS; - } int get_leader_config_version(palf::LogConfigVersion &config_version) const { UNUSED(config_version); @@ -434,25 +405,6 @@ class MockObLogHandler : public logservice::ObILogHandler return OB_SUCCESS; } - int try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us) - { - UNUSED(lock_owner); - UNUSED(timeout_us); - return OB_SUCCESS; - } - int unlock_config_change(const int64_t lock_owner, const int64_t timeout_us) - { - UNUSED(lock_owner); - UNUSED(timeout_us); - return OB_SUCCESS; - } - int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) - { - lock_owner = palf::OB_INVALID_CONFIG_CHANGE_LOCK_OWNER; - is_locked = false; - return OB_SUCCESS; - } - LSN base_lsn_; int64_t result_ts_ns_; share::SCN result_scn_; @@ -505,12 +457,6 @@ class MockObLogHandler : public logservice::ObILogHandler UNUSED(parent); return OB_SUCCESS; } - int register_rebuild_cb(palf::PalfRebuildCb *rebuild_cb) - { - UNUSED(rebuild_cb); - return OB_SUCCESS; - } - int unregister_rebuild_cb() { return OB_SUCCESS; } bool is_offline() const {return false;}; int offline() {return OB_SUCCESS;}; int online(const LSN &lsn, const share::SCN &scn) { UNUSED(lsn); UNUSED(scn); return OB_SUCCESS;}; @@ -522,4 +468,3 @@ class MockObLogHandler : public logservice::ObILogHandler #endif - diff --git a/unittest/storage/mockcontainer/mock_ob_partition_service.h b/unittest/storage/mockcontainer/mock_ob_partition_service.h index 036a5fb2f..70b1834cc 100644 --- a/unittest/storage/mockcontainer/mock_ob_partition_service.h +++ b/unittest/storage/mockcontainer/mock_ob_partition_service.h @@ -103,14 +103,6 @@ class MockObIPartitionService : public ObPartitionService // int(bool is_rollback, const ObPartitionArray &cur_stmt_all_participants, // const transaction::ObPartitionEpochArray &epoch_arr, // const ObPartitionArray &discard_participants, transaction::ObTransDesc &trans_desc)); - // MOCK_METHOD3(start_participant, - // int(transaction::ObTransDesc &trans_desc, - // const common::ObPartitionArray &participants, - // transaction::ObPartitionEpochArray &partition_epoch_arr)); - // MOCK_METHOD4(end_participant, int(bool is_rollback, - // transaction::ObTransDesc &trans_desc, - // const common::ObPartitionArray &participants, - // transaction::ObEndParticipantsRes &res)); MOCK_METHOD2(table_scan, int(ObVTableScanParam ¶m, common::ObNewRowIterator *&result)); MOCK_METHOD3(join_mv_scan, int(storage::ObTableScanParam &left_param, diff --git a/unittest/storage/mockcontainer/mock_ob_server.cpp b/unittest/storage/mockcontainer/mock_ob_server.cpp index e81ebb607..ca8d4cbb4 100644 --- a/unittest/storage/mockcontainer/mock_ob_server.cpp +++ b/unittest/storage/mockcontainer/mock_ob_server.cpp @@ -187,17 +187,6 @@ int MockObServer::init(const char *schema_file, } } - //init gts local cache mgr - if (OB_SUCC(ret)) { - if (OB_SUCCESS != (ret = OB_TS_MGR.init(self_addr_, - *GCTX.schema_service_, - *GCTX.location_service_))) { - STORAGE_LOG(WARN, "init gts local cache mgr failed", K(ret)); - } else { - STORAGE_LOG(INFO, "init gts local cache mgr success"); - } - } - //init multi tenant if (OB_SUCC(ret)) { if (OB_SUCCESS != (ret = init_multi_tenant())) { @@ -249,8 +238,6 @@ int MockObServer::start() ret = OB_NOT_INIT; } else if (OB_SUCCESS != (ret = net_frame_.start())) { STORAGE_LOG(WARN, "net frame start error", K(ret)); - } else if (OB_FAIL(OB_TS_MGR.start())) { - STORAGE_LOG(WARN, "start gts cache mgr error", K(ret)); } else { STORAGE_LOG(INFO, "net frame start success"); } @@ -273,7 +260,6 @@ int MockObServer::stop() } else if (OB_SUCCESS != (ret = net_frame_.stop())) { STORAGE_LOG(WARN, "net frame stop error", K(ret)); } else { - OB_TS_MGR.stop(); STORAGE_LOG(INFO, "net frame stop success"); } multi_tenant_.stop(); @@ -290,7 +276,6 @@ int MockObServer::wait() int ret = OB_SUCCESS; multi_tenant_.wait(); net_frame_.wait(); - (void)OB_TS_MGR.wait(); if (OB_SUCC(ret)) { STORAGE_LOG(INFO, "ob server wait success"); } diff --git a/unittest/storage/mockcontainer/mock_ob_server.h b/unittest/storage/mockcontainer/mock_ob_server.h index 1cd6b59ab..9a3f726d4 100644 --- a/unittest/storage/mockcontainer/mock_ob_server.h +++ b/unittest/storage/mockcontainer/mock_ob_server.h @@ -20,7 +20,6 @@ #include "storage/tx/ob_trans_service.h" #include "lib/string/ob_sql_string.h" #include "storage/blocksstable/ob_block_sstable_struct.h" -#include "storage/tx/ob_gts_rpc.h" #include "sql/session/ob_sql_session_mgr.h" #include "observer/ob_srv_network_frame.h" #include "share/config/ob_server_config.h" diff --git a/unittest/storage/multi_data_source/test_is_old_mds.cpp b/unittest/storage/multi_data_source/test_is_old_mds.cpp deleted file mode 100644 index 101f25882..000000000 --- a/unittest/storage/multi_data_source/test_is_old_mds.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "storage/schema_utils.h" -#include "storage/test_tablet_helper.h" -#include "storage/tablet/ob_mds_schema_helper.h" -namespace oceanbase { -namespace storage { -namespace mds { -void *DefaultAllocator::alloc(const int64_t size) { - void *ptr = std::malloc(size);// ob_malloc(size, "MDS"); - ATOMIC_INC(&alloc_times_); - MDS_LOG(DEBUG, "alloc obj", KP(ptr), K(size), K(lbt())); - return ptr; -} -void DefaultAllocator::free(void *ptr) { - ATOMIC_INC(&free_times_); - MDS_LOG(DEBUG, "free obj", KP(ptr), K(lbt())); - std::free(ptr);// ob_free(ptr); -} -void *MdsAllocator::alloc(const int64_t size) { - void *ptr = std::malloc(size);// ob_malloc(size, "MDS"); - ATOMIC_INC(&alloc_times_); - MDS_LOG(DEBUG, "alloc obj", KP(ptr), K(size), K(lbt())); - return ptr; -} -void MdsAllocator::free(void *ptr) { - ATOMIC_INC(&free_times_); - MDS_LOG(DEBUG, "free obj", KP(ptr), K(lbt())); - std::free(ptr);// ob_free(ptr); -} -}}} - -#define USING_LOG_PREFIX STORAGE - -#define UNITTEST - -namespace oceanbase -{ -namespace storage -{ -namespace checkpoint -{ -class TestIsOldMds : public ::testing::Test -{ -public: - TestIsOldMds() {} - virtual ~TestIsOldMds() = default; - - static void SetUpTestCase() - { - ObMdsSchemaHelper::get_instance().init(); - } - - static void TearDownTestCase() - { - } -}; - -TEST_F(TestIsOldMds, test_is_old_mds_for_unbind) { - int ret = OB_SUCCESS; - ObBatchUnbindTabletArg arg; - for (int64_t i =0; OB_SUCC(ret) && i < 5; i++) { - arg.orig_tablet_ids_.push_back(ObTabletID(1)); - ASSERT_EQ(OB_SUCCESS, ret); - } - - for (int64_t i =0; OB_SUCC(ret) && i < 5; i++) { - arg.hidden_tablet_ids_.push_back(ObTabletID(1)); - ASSERT_EQ(OB_SUCCESS, ret); - } - - int64_t len = 5000; - char buf[5000] = {0}; - int64_t pos = 0; - bool is_old_mds; - - arg.is_old_mds_ = false; - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, is_old_mds); - - pos = 0; - arg.is_old_mds_ = true; - memset(buf, 0, 5000); - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(true, is_old_mds); -} - -TEST_F(TestIsOldMds, test_is_old_mds_for_delete) { - int ret = OB_SUCCESS; - ObBatchRemoveTabletArg arg; - for (int64_t i =0; OB_SUCC(ret) && i < 5; i++) { - arg.tablet_ids_.push_back(ObTabletID(1)); - ASSERT_EQ(OB_SUCCESS, ret); - } - - int64_t len = 5000; - char buf[5000] = {0}; - int64_t pos = 0; - bool is_old_mds; - - arg.is_old_mds_ = false; - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, is_old_mds); - - pos = 0; - arg.is_old_mds_ = true; - memset(buf, 0, 5000); - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(true, is_old_mds); -} - -TEST_F(TestIsOldMds, test_is_old_mds_for_create) { - int ret = OB_SUCCESS; - ObBatchCreateTabletArg arg; - ObTableSchema table_schema; - TestSchemaUtils::prepare_data_schema(table_schema); - ret = arg.table_schemas_.push_back(table_schema); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.table_schemas_.push_back(table_schema); - ASSERT_EQ(OB_SUCCESS, ret); - for (int64_t i =0; OB_SUCC(ret) && i < 5; i++) { - obcall::ObCreateTabletInfo create_tablet_info; - create_tablet_info.compat_mode_ = lib::Worker::CompatMode::MYSQL; - create_tablet_info.data_tablet_id_ = ObTabletID(1); - ASSERT_EQ(OB_SUCCESS, ret); - for (int64_t j = 0; OB_SUCC(ret) && j < 5; ++j) { - create_tablet_info.tablet_ids_.push_back(ObTabletID(1)); - ASSERT_EQ(OB_SUCCESS, ret); - create_tablet_info.table_schema_index_.push_back(0); - ASSERT_EQ(OB_SUCCESS, ret); - } - arg.tablets_.push_back(create_tablet_info); - ASSERT_EQ(OB_SUCCESS, ret); - } - - int64_t len = 5000; - char buf[5000] = {0}; - int64_t pos = 0; - bool is_old_mds; - - arg.is_old_mds_ = false; - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, is_old_mds); - - pos = 0; - arg.is_old_mds_ = true; - memset(buf, 0, 5000); - ret = arg.serialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = arg.is_old_mds(buf, len, is_old_mds); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(true, is_old_mds); -} - -} -} // end namespace storage -} // end namespace oceanbase - -int main(int argc, char **argv) -{ - int ret = 0; - system("rm -f test_is_old_mds.log*"); - OB_LOGGER.set_file_name("test_is_old_mds.log", true); - OB_LOGGER.set_log_level("INFO"); - signal(49, SIG_IGN); - testing::InitGoogleTest(&argc, argv); - ret = RUN_ALL_TESTS(); - return ret; -} diff --git a/unittest/storage/multi_data_source/test_mds_dump_kv.cpp b/unittest/storage/multi_data_source/test_mds_dump_kv.cpp index 27e3a3c9b..d342f76bb 100644 --- a/unittest/storage/multi_data_source/test_mds_dump_kv.cpp +++ b/unittest/storage/multi_data_source/test_mds_dump_kv.cpp @@ -55,104 +55,6 @@ using namespace std; using namespace storage; using namespace mds; -struct OldMdsDumpNode -{ -public: - int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; - int deserialize(common::ObIAllocator &allocator, const char *buf, const int64_t data_len, int64_t &pos); - int64_t get_serialize_size() const; -public: - static constexpr int64_t UNIS_VERSION = 1; - - uint8_t mds_table_id_; - uint8_t mds_unit_id_; - uint32_t crc_check_number_; - MdsNodeStatus status_; - ObIAllocator *allocator_;// to release serialized buffer - int64_t writer_id_; // mostly is tx id - int64_t seq_no_;// if one writer write multi nodes in one same row, seq_no is used to order those writes - share::SCN redo_scn_; // log scn of redo - share::SCN end_scn_; // log scn of commit/abort - share::SCN trans_version_; // read as prepare version if phase is not COMMIT, or read as commit version - common::ObString user_data_;// different user data type serialized result(may be very large, but should be less than 1.5MB) -}; - -int OldMdsDumpNode::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - - LST_DO_CODE(OB_UNIS_ENCODE, - UNIS_VERSION, - mds_table_id_, - mds_unit_id_, - writer_id_, - seq_no_, - redo_scn_, - end_scn_, - trans_version_, - status_, - crc_check_number_, - user_data_); - - return ret; -} - -int OldMdsDumpNode::deserialize(common::ObIAllocator &allocator, const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - int64_t version = 0; - ObString user_data; - - LST_DO_CODE(OB_UNIS_DECODE, - version, - mds_table_id_, - mds_unit_id_, - writer_id_, - seq_no_, - redo_scn_, - end_scn_, - trans_version_, - status_, - crc_check_number_, - user_data); - - if (OB_SUCC(ret)) { - allocator_ = &allocator; - const int64_t len = user_data.length(); - char *buffer = nullptr; - if (0 == len) { - } else if (OB_ISNULL(buffer = static_cast(allocator_->alloc(len)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - MDS_LOG(WARN, "failed to allocate memory", K(ret), K(len)); - } else { - MEMCPY(buffer, user_data.ptr(), len); - user_data_.assign(buffer, len); - } - } - - return ret; -} - -int64_t OldMdsDumpNode::get_serialize_size() const -{ - int64_t len = 0; - - LST_DO_CODE(OB_UNIS_ADD_LEN, - UNIS_VERSION, - mds_table_id_, - mds_unit_id_, - writer_id_, - seq_no_, - redo_scn_, - end_scn_, - trans_version_, - status_, - crc_check_number_, - user_data_); - - return len; -} - class TestMdsDumpKV: public ::testing::Test { public: @@ -165,7 +67,6 @@ class TestMdsDumpKV: public ::testing::Test static void test_dump_node_convert_and_serialize_and_compare(); static void test_dump_dummy_key(); static void test_dump_kv_wrapper(); - static void test_dump_node_deserialize(); private: // disallow copy DISALLOW_COPY_AND_ASSIGN(TestMdsDumpKV); @@ -175,7 +76,7 @@ void TestMdsDumpKV::test_dump_node_convert_and_serialize_and_compare() { ExampleUserData2 first_data; ASSERT_EQ(OB_SUCCESS, first_data.assign(MdsAllocator::get_instance(), "123")); - UserMdsNode user_mds_node(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ::mk_v0(100)); + UserMdsNode user_mds_node(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ(100, 0)); ASSERT_EQ(OB_SUCCESS, meta::move_or_copy_or_assign(first_data, user_mds_node.user_data_, MdsAllocator::get_instance())); ASSERT_EQ(true, first_data.data_.ptr() != user_mds_node.user_data_.data_.ptr()); ASSERT_EQ(0, memcmp(first_data.data_.ptr(), user_mds_node.user_data_.data_.ptr(), first_data.data_.length())); @@ -198,7 +99,7 @@ void TestMdsDumpKV::test_dump_node_convert_and_serialize_and_compare() ASSERT_EQ(dump_node2.generate_hash(), dump_node2.crc_check_number_); ASSERT_EQ(dump_node2.crc_check_number_, dump_node.crc_check_number_); UserMdsNode user_mds_node2; - ASSERT_EQ(OB_SUCCESS, dump_node2.convert_to_user_mds_node(user_mds_node2, share::ObLSID(1), ObTabletID(1))); + ASSERT_EQ(OB_SUCCESS, dump_node2.convert_to_user_mds_node(user_mds_node2)); ASSERT_EQ(0, memcmp(user_mds_node.user_data_.data_.ptr(), user_mds_node2.user_data_.data_.ptr(), user_mds_node.user_data_.data_.length())); ASSERT_EQ(user_mds_node.redo_scn_, user_mds_node2.redo_scn_); ASSERT_EQ(user_mds_node.end_scn_, user_mds_node2.end_scn_); @@ -233,7 +134,7 @@ void TestMdsDumpKV::test_dump_kv_wrapper() // construct dump node ExampleUserData2 first_data; ASSERT_EQ(OB_SUCCESS, first_data.assign(MdsAllocator::get_instance(), "123")); - UserMdsNode user_mds_node(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ::mk_v0(1)); + UserMdsNode user_mds_node(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ(1, 0)); ASSERT_EQ(OB_SUCCESS, meta::move_or_copy_or_assign(first_data, user_mds_node.user_data_, MdsAllocator::get_instance())); ASSERT_EQ(true, first_data.data_.ptr() != user_mds_node.user_data_.data_.ptr()); ASSERT_EQ(0, memcmp(first_data.data_.ptr(), user_mds_node.user_data_.data_.ptr(), first_data.data_.length())); @@ -279,58 +180,9 @@ void TestMdsDumpKV::test_dump_kv_wrapper() */ } -void TestMdsDumpKV::test_dump_node_deserialize() -{ - int ret = OB_SUCCESS; - - ObTabletCreateDeleteMdsUserData user_data; - user_data.tablet_status_ = ObTabletStatus::NORMAL; - user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET; - user_data.create_commit_scn_ = share::SCN::plus(share::SCN::min_scn(), 10); - user_data.create_commit_version_ = 10; - - ObArenaAllocator allocator; - int64_t size = user_data.get_serialize_size(); - char *buf = static_cast(allocator.alloc(size)); - ASSERT_NE(nullptr, buf); - - int64_t pos = 0; - ret = user_data.serialize(buf, size, pos); - ASSERT_EQ(OB_SUCCESS, ret); - - OldMdsDumpNode old_dump_node; - old_dump_node.mds_table_id_ = 1; - old_dump_node.mds_unit_id_ = 1; - old_dump_node.crc_check_number_ = 0xabcd; - old_dump_node.status_.union_.field_.node_type_ = MdsNodeType::SET; - old_dump_node.status_.union_.field_.writer_type_ = WriterType::TRANSACTION; - old_dump_node.status_.union_.field_.state_ = TwoPhaseCommitState::ON_COMMIT; - old_dump_node.writer_id_ = 123; - old_dump_node.seq_no_ = 0; - old_dump_node.redo_scn_ = share::SCN::plus(share::SCN::min_scn(), 5); - old_dump_node.end_scn_ = share::SCN::plus(share::SCN::min_scn(), 10); - old_dump_node.trans_version_ = share::SCN::plus(share::SCN::min_scn(), 10); - old_dump_node.user_data_.assign(buf, size); - - size = old_dump_node.get_serialize_size(); - pos = 0; - buf = static_cast(allocator.alloc(size)); - ASSERT_NE(nullptr, buf); - ret = old_dump_node.serialize(buf, size, pos); - ASSERT_EQ(OB_SUCCESS, ret); - - // compat - MdsDumpNode dump_node; - pos = 0; - ret = dump_node.deserialize(allocator, buf, size, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(transaction::ObTxSEQ::MIN_VAL(), dump_node.seq_no_); -} - TEST_F(TestMdsDumpKV, test_convert_and_serialize_and_compare) { TestMdsDumpKV::test_dump_node_convert_and_serialize_and_compare(); } TEST_F(TestMdsDumpKV, test_dump_dummy_key) { TestMdsDumpKV::test_dump_dummy_key(); } TEST_F(TestMdsDumpKV, test_dump_kv_wrapper) { TestMdsDumpKV::test_dump_kv_wrapper(); } -TEST_F(TestMdsDumpKV, test_dump_node_deserialize) { TestMdsDumpKV::test_dump_node_deserialize(); } } } diff --git a/unittest/storage/multi_data_source/test_mds_node.cpp b/unittest/storage/multi_data_source/test_mds_node.cpp index 7b0fc72ae..e6b887de2 100644 --- a/unittest/storage/multi_data_source/test_mds_node.cpp +++ b/unittest/storage/multi_data_source/test_mds_node.cpp @@ -102,8 +102,8 @@ struct UserDataWithCallBack TEST_F(TestMdsNode, call_user_method) { MdsRow row; - MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ::mk_v0(1));// commit finally - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {share::ObLSID(0), 0})); + MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ(1, 0));// commit finally + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {0})); ctx.on_redo(mock_scn(1)); ctx.before_prepare(); ctx.on_prepare(mock_scn(2)); @@ -116,8 +116,8 @@ TEST_F(TestMdsNode, call_user_method) { TEST_F(TestMdsNode, release_node_while_node_in_ctx) { MdsRow row; - MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ::mk_v0(1));// commit finally - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {share::ObLSID(0), 0})); + MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ(1, 0));// commit finally + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {0})); ctx.on_redo(mock_scn(1)); ctx.before_prepare(); row.~MdsRow(); @@ -132,17 +132,17 @@ TEST_F(TestMdsNode, release_node_while_node_in_ctx_concurrent) { call_try_on_commit = 0; call_try_on_abort = 0; MdsRow row; - MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ::mk_v0(1));// commit finally + MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(100)), transaction::ObTxSEQ(1, 0));// commit finally // Submitting these nodes will take 50ms - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {share::ObLSID(0), 0})); + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(1), ctx, {0})); ASSERT_EQ(OB_SUCCESS, ctx.inc_seq_no()); - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(2), ctx, {share::ObLSID(0), 0})); + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(2), ctx, {0})); ASSERT_EQ(OB_SUCCESS, ctx.inc_seq_no()); - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(3), ctx, {share::ObLSID(0), 0})); + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(3), ctx, {0})); ASSERT_EQ(OB_SUCCESS, ctx.inc_seq_no()); - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(4), ctx, {share::ObLSID(0), 0})); + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(4), ctx, {0})); ASSERT_EQ(OB_SUCCESS, ctx.inc_seq_no()); - ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(5), ctx, {share::ObLSID(0), 0})); + ASSERT_EQ(OB_SUCCESS, row.set(UserDataWithCallBack(5), ctx, {0})); ASSERT_EQ(OB_SUCCESS, ctx.inc_seq_no()); std::thread t1([&ctx]() { @@ -217,8 +217,8 @@ TEST_F(TestMdsNode, release_node_while_node_in_ctx_concurrent) { // } TEST_F(TestMdsNode, test_node_print) { - UserMdsNode node0(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ::mk_v0(100)); - UserMdsNode node1(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ::mk_v0(100)); + UserMdsNode node0(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ(100, 0)); + UserMdsNode node1(nullptr, MdsNodeType::SET, WriterType::TRANSACTION, 1, transaction::ObTxSEQ(100, 0)); } diff --git a/unittest/storage/multi_data_source/test_mds_table.cpp b/unittest/storage/multi_data_source/test_mds_table.cpp index 01646f95c..644938893 100644 --- a/unittest/storage/multi_data_source/test_mds_table.cpp +++ b/unittest/storage/multi_data_source/test_mds_table.cpp @@ -158,7 +158,7 @@ void TestMdsTable::compare_binary_key() { } void TestMdsTable::set() { - ASSERT_EQ(OB_SUCCESS, mds_table_.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, mds_table_.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MDS_LOG(INFO, "test sizeof", K(sizeof(MdsTableImpl)), K(sizeof(B)), K(mds_table_.p_mds_table_base_.ctrl_ptr_->ref_)); ExampleUserData1 data1(1); ExampleUserData2 data2; @@ -569,7 +569,6 @@ TEST_F(TestMdsTable, basic_trans_example) { // 1. Initialize to UnitTestMdsTable ASSERT_EQ(OB_SUCCESS, mth.init(mds::DefaultAllocator::get_instance(), ObTabletID(1), - share::ObLSID(1), share::SCN::min_scn(), nullptr)); MdsTableHandle mth2 = mth;// two reference counts @@ -593,7 +592,6 @@ TEST_F(TestMdsTable, basic_non_trans_example) { // 1. Initialize to UnitTestMdsTable ASSERT_EQ(OB_SUCCESS, mth.init(mds::DefaultAllocator::get_instance(), ObTabletID(1), - share::ObLSID(1), share::SCN::min_scn(), nullptr)); MdsTableHandle mth2 = mth;// two reference counts @@ -633,7 +631,7 @@ TEST_F(TestMdsTable, test_recycle) { TEST_F(TestMdsTable, test_recalculate_flush_scn_op) { MdsTableHandle mds_table; - ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MdsCtx ctx1(mds::MdsWriter(ObTransID(1))); MdsCtx ctx2(mds::MdsWriter(ObTransID(2))); MdsCtx ctx3(mds::MdsWriter(ObTransID(3))); diff --git a/unittest/storage/multi_data_source/test_mds_table_flush.cpp b/unittest/storage/multi_data_source/test_mds_table_flush.cpp index 81894f4a6..2af648677 100644 --- a/unittest/storage/multi_data_source/test_mds_table_flush.cpp +++ b/unittest/storage/multi_data_source/test_mds_table_flush.cpp @@ -116,7 +116,7 @@ int construct_tested_mds_table(MdsTableHandle &handle) { for (int i = 0; i < 7; ++i) { v_ctx.push_back(new MdsCtx(MdsWriter(transaction::ObTransID(i)))); } - if (OB_FAIL(handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111))) { + if (OB_FAIL(handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111))) { } else if (OB_FAIL(handle.set(1, *v_ctx[0]))) { } else if (FALSE_IT(v_ctx[0]->on_redo(mock_scn(50)))) { } else if (FALSE_IT(v_ctx[0]->on_commit(mock_scn(100), mock_scn(100)))) { @@ -260,7 +260,7 @@ TEST_F(TestMdsTableFlush, advance_rec_scn_consider_about_max_aborted_scn_on_mds_ TEST_F(TestMdsTableFlush, flush_scn_decline1) { MdsTableHandle handle; - ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MdsCtx ctx1(MdsWriter(transaction::ObTransID(1))); handle.set(ExampleUserKey(1), ExampleUserData1(1), ctx1); ctx1.on_redo(mock_scn(10)); @@ -281,7 +281,7 @@ TEST_F(TestMdsTableFlush, flush_scn_decline1) { TEST_F(TestMdsTableFlush, flush_scn_decline2) { MdsTableHandle handle; - ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MdsCtx ctx1(MdsWriter(transaction::ObTransID(1))); handle.set(ExampleUserKey(1), ExampleUserData1(1), ctx1); ctx1.on_redo(mock_scn(10)); diff --git a/unittest/storage/multi_data_source/test_mds_table_flusher.cpp b/unittest/storage/multi_data_source/test_mds_table_flusher.cpp index e4aa12bfa..763898e81 100644 --- a/unittest/storage/multi_data_source/test_mds_table_flusher.cpp +++ b/unittest/storage/multi_data_source/test_mds_table_flusher.cpp @@ -167,7 +167,6 @@ TEST_F(TestMdsTableFlush, flusher_for_all_order_with_enough_memory) { MdsTableHandle mds_table; ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), v_key[i].tablet_id_, - share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111, &mgr)); @@ -208,7 +207,6 @@ TEST_F(TestMdsTableFlush, flusher_for_all_order_with_limitted_memory_reserve_fai MdsTableHandle mds_table; ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), v_key[i].tablet_id_, - share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111, &mgr)); @@ -253,7 +251,6 @@ TEST_F(TestMdsTableFlush, flusher_for_one) { // MdsTableHandle mds_table; // ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), // v_key[i].tablet_id_, -// share::ObLSID(1), // share::SCN::min_scn(), // (ObTabletPointer*)0x111, // &mgr)); @@ -280,7 +277,6 @@ TEST_F(TestMdsTableFlush, flusher_for_one) { // MdsTableHandle mds_table; // ASSERT_EQ(OB_SUCCESS, mds_table.init(MdsAllocator::get_instance(), // v_key[i].tablet_id_, -// share::ObLSID(1), // share::SCN::min_scn(), // (ObTabletPointer*)0x111, // &mgr)); diff --git a/unittest/storage/multi_data_source/test_mds_table_forcelly_remove.cpp b/unittest/storage/multi_data_source/test_mds_table_forcelly_remove.cpp index 16dd75b72..b68683c72 100644 --- a/unittest/storage/multi_data_source/test_mds_table_forcelly_remove.cpp +++ b/unittest/storage/multi_data_source/test_mds_table_forcelly_remove.cpp @@ -86,7 +86,7 @@ class TestMdsTableForcellyRemove: public ::testing::Test TEST_F(TestMdsTableForcellyRemove, reset) { MdsTableHandle handle; - ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MdsCtx ctx1(MdsWriter(transaction::ObTransID(1))); handle.set(ExampleUserKey(1), ExampleUserData1(1), ctx1); ctx1.on_redo(mock_scn(10)); @@ -117,7 +117,7 @@ TEST_F(TestMdsTableForcellyRemove, reset) { TEST_F(TestMdsTableForcellyRemove, remove) { MdsTableHandle handle; - ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::ObLSID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); + ASSERT_EQ(OB_SUCCESS, handle.init(MdsAllocator::get_instance(), ObTabletID(1), share::SCN::min_scn(), (ObTabletPointer*)0x111)); MdsCtx ctx1(MdsWriter(transaction::ObTransID(1))); handle.set(ExampleUserKey(1), ExampleUserData1(1), ctx1); ctx1.on_redo(mock_scn(10)); diff --git a/unittest/storage/ob_truncate_info_helper.cpp b/unittest/storage/ob_truncate_info_helper.cpp index 0e5a5f825..e93fccf79 100644 --- a/unittest/storage/ob_truncate_info_helper.cpp +++ b/unittest/storage/ob_truncate_info_helper.cpp @@ -120,17 +120,15 @@ int TruncateInfoHelper::mock_truncate_partition( } int TruncateInfoHelper::get_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - ObLSHandle ls_handle; ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { COMMON_LOG(WARN, "failed to get ls", K(ret)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + } else if (OB_ISNULL(ls)) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(WARN, "ls is null", K(ret), KP(ls)); } else if (OB_FAIL(ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle))) { @@ -160,7 +158,6 @@ int TruncateInfoHelper::mock_part_key_idxs( int TruncateInfoHelper::read_distinct_truncate_info_array( common::ObArenaAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const ObVersionRange &read_version_range, ObTruncateInfoArray &truncate_info_array) @@ -169,8 +166,8 @@ int TruncateInfoHelper::read_distinct_truncate_info_array( ObTabletHandle tablet_handle; ObMdsInfoDistinctMgr distinct_mgr; truncate_info_array.reset(); - if (OB_FAIL(TruncateInfoHelper::get_tablet(ls_id, tablet_id, tablet_handle))) { - COMMON_LOG(WARN, "failed to get tablet", KR(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(TruncateInfoHelper::get_tablet(tablet_id, tablet_handle))) { + COMMON_LOG(WARN, "failed to get tablet", KR(ret), K(tablet_id)); } else if (OB_FAIL(distinct_mgr.init(allocator, *tablet_handle.get_obj(), nullptr, read_version_range, false/*access*/))) { COMMON_LOG(WARN, "failed to init distinct mgr", KR(ret), K(read_version_range)); } else if (OB_FAIL(truncate_info_array.init_for_first_creation(allocator))) { diff --git a/unittest/storage/ob_truncate_info_helper.h b/unittest/storage/ob_truncate_info_helper.h index 2175b2917..b9051eeb6 100644 --- a/unittest/storage/ob_truncate_info_helper.h +++ b/unittest/storage/ob_truncate_info_helper.h @@ -31,10 +31,6 @@ class ObRowkey; class ObTabletID; struct ObVersionRange; } -namespace share -{ -class ObLSID; -} namespace storage { struct ObTruncateInfo; @@ -81,7 +77,6 @@ class TruncateInfoHelper const int64_t list_val_cnt, ObTruncatePartition &part); static int get_tablet( - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle); static int mock_part_key_idxs( @@ -98,7 +93,6 @@ class TruncateInfoHelper } static int read_distinct_truncate_info_array( common::ObArenaAllocator &allocator, - const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const common::ObVersionRange &read_version_range, storage::ObTruncateInfoArray &truncate_info_array); diff --git a/unittest/storage/ob_uncommitted_trans_test.h b/unittest/storage/ob_uncommitted_trans_test.h index 4e5cf8617..021ff29e1 100644 --- a/unittest/storage/ob_uncommitted_trans_test.h +++ b/unittest/storage/ob_uncommitted_trans_test.h @@ -19,7 +19,7 @@ #include "lib/hash/ob_hashmap.h" #include "storage/tx/ob_trans_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "mockcontainer/mock_ob_iterator.h" namespace oceanbase diff --git a/unittest/storage/run_all_test.py b/unittest/storage/run_all_test.py index cfbb17bec..a22171d53 100755 --- a/unittest/storage/run_all_test.py +++ b/unittest/storage/run_all_test.py @@ -40,7 +40,6 @@ 'test_range_iterator', ), '../clog/' : ( - # 'test_fetch_log_engine', 'test_log_allocator', 'test_log_callback_engine', 'test_log_checksum_V2', diff --git a/unittest/storage/slog/simple_ob_storage_redo_module.h b/unittest/storage/slog/simple_ob_storage_redo_module.h index fcce9d7b9..3abe7deb1 100644 --- a/unittest/storage/slog/simple_ob_storage_redo_module.h +++ b/unittest/storage/slog/simple_ob_storage_redo_module.h @@ -29,7 +29,7 @@ #include "src/storage/slog/ob_storage_log_reader.h" #include "src/storage/slog/ob_storage_logger.h" #include "src/storage/ls/ob_ls_tablet_service.h" -#include "src/storage/tx_storage/ob_ls_handle.h" +#include "src/storage/ls/ob_ls.h" #include "src/storage/tx_storage/ob_ls_service.h" #include "src/storage/tablet/ob_tablet.h" #include "src/storage/meta_store/ob_tenant_storage_meta_service.h" @@ -155,7 +155,7 @@ class SimpleObStorageModule : public ObIRedoModule int read_from_disk_addr(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, char *&r_buf, int64_t &r_len); int read_from_share_blk(const ObMetaDiskAddr &addr, common::ObArenaAllocator &allocator, char *&buf, int64_t &buf_len); int read_from_slog(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, int64_t &pos); - int get_tablet_svr(const share::ObLSID &ls_id, ObLSTabletService *&ls_tablet_svr, ObLSHandle &ls_handle); + int get_tablet_svr(ObLSTabletService *&ls_tablet_svr, ObLS *&ls); bool operator ==(SimpleObStorageModule &redo_module); void reset(); @@ -200,11 +200,11 @@ int SimpleObStorageModule::replay(const ObRedoModuleReplayParam ¶m) if (ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE != main_type) { ret = OB_ERR_UNEXPECTED; STORAGE_REDO_LOG(WARN, "The main type is wrong.", K(main_type)); - } else if (ObRedoLogSubType::OB_REDO_LOG_CREATE_LS == sub_type) { + } else if (ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET == sub_type) { if (OB_FAIL(slogs_[index_].deserialize(buf, buf_len, pos))) { STORAGE_REDO_LOG(WARN, "Fail to recover slog.", K(sub_type), K(buf), K(buf_len)); } - } else if (ObRedoLogSubType::OB_REDO_LOG_DELETE_LS == sub_type) { + } else if (ObRedoLogSubType::OB_REDO_LOG_DELETE_TABLET == sub_type) { // do nothing } else if (ObRedoLogSubType::OB_REDO_LOG_EMPTY_SHELL_TABLET == sub_type) { if (OB_FAIL(inner_replay_empty_shell_tablet(param))) { @@ -221,20 +221,18 @@ int SimpleObStorageModule::replay(const ObRedoModuleReplayParam ¶m) } int SimpleObStorageModule::get_tablet_svr( - const ObLSID &ls_id, ObLSTabletService *&ls_tablet_svr, - ObLSHandle &ls_handle) + ObLS *&ls) { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "fail to get ls handle", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + if (OB_FAIL(share::g_mp->ls_service()->get_ls(ls))) { + STORAGE_LOG(WARN, "fail to get ls", K(ret)); + } else if (OB_ISNULL(ls)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "ls is null", K(ret)); } else if (OB_ISNULL(ls_tablet_svr = ls->get_tablet_svr())) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "tablet service is null", K(ret), K(ls_id)); + STORAGE_LOG(WARN, "tablet service is null", K(ret)); } return ret; } @@ -245,7 +243,7 @@ int SimpleObStorageModule::inner_replay_empty_shell_tablet(const ObRedoModuleRep int64_t pos = 0; ObArenaAllocator allocator; ObLSTabletService *ls_tablet_svr = nullptr; - ObLSHandle ls_handle; + ObLS *ls = nullptr; char *buf = nullptr; int64_t buf_len = 0; ObEmptyShellTabletLog slog; @@ -253,8 +251,8 @@ int SimpleObStorageModule::inner_replay_empty_shell_tablet(const ObRedoModuleRep STORAGE_LOG(WARN, "failed to serialize tablet_id_", K(ret), K(param.disk_addr_.size()), K(pos)); } else if (OB_FAIL(read_from_disk(param.disk_addr_, allocator, buf, buf_len))) { STORAGE_LOG(WARN, "read from disk failed", K(ret), K(param.disk_addr_), K(buf_len)); - } else if (OB_FAIL(get_tablet_svr(slog.ls_id_, ls_tablet_svr, ls_handle))) { - STORAGE_LOG(WARN, "get tablet svr failed", K(ret), K(slog.ls_id_)); + } else if (OB_FAIL(get_tablet_svr(ls_tablet_svr, ls))) { + STORAGE_LOG(WARN, "get tablet svr failed", K(ret)); } else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(param.disk_addr_, buf, buf_len, slog.tablet_id_))) { STORAGE_LOG(WARN, "replay empty shell tablet failed", K(ret), K(param.disk_addr_), K(slog.tablet_id_)); } diff --git a/unittest/storage/slog/test_storage_log_replay.cpp b/unittest/storage/slog/test_storage_log_replay.cpp index 378dc9528..fb502663c 100644 --- a/unittest/storage/slog/test_storage_log_replay.cpp +++ b/unittest/storage/slog/test_storage_log_replay.cpp @@ -142,9 +142,12 @@ TEST_F(TestStorageLogReplay, test_basic) // test normal replay (single write) build_storage(ObRandom::rand(1, 127)); - ObStorageLoggerManager &slogger_mgr = SERVER_STORAGE_META_SERVICE.get_slogger_manager(); ObStorageLogger *slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; @@ -152,7 +155,7 @@ TEST_F(TestStorageLogReplay, test_basic) ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, - ObRedoLogSubType::OB_REDO_LOG_CREATE_LS); + ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET); for (int i = 0; i < tenant_storage_.slog_cnt_; i++) { log_param.data_ = &tenant_storage_.slogs_[i]; ret = slogger->write_log(log_param); @@ -181,7 +184,11 @@ TEST_F(TestStorageLogReplay, test_basic) OB_DELETE(ObStorageLogger, ObModIds::TEST, slogger); slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; @@ -189,7 +196,7 @@ TEST_F(TestStorageLogReplay, test_basic) ObSEArray param_arr; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, - ObRedoLogSubType::OB_REDO_LOG_CREATE_LS); + ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET); for (int i = 0; i < tenant_storage_.slog_cnt_; i++) { log_param.data_ = &(tenant_storage_.slogs_[i]); param_arr.push_back(log_param); @@ -228,7 +235,7 @@ TEST_F(TestStorageLogReplay, test_basic) tenant_storage_.slogs_[tmp_cnt].block_cnt_ = 1; log_param.data_ = &(tenant_storage_.slogs_[tmp_cnt]); log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, - ObRedoLogSubType::OB_REDO_LOG_DELETE_LS); + ObRedoLogSubType::OB_REDO_LOG_DELETE_TABLET); ret = slogger->write_log(log_param); ASSERT_EQ(OB_SUCCESS, ret); @@ -256,9 +263,12 @@ TEST_F(TestStorageLogReplay, test_switch_file_replay) write_start_cursor.log_id_ = 1; SimpleObStorageModule redo_module; - ObStorageLoggerManager &slogger_mgr = SERVER_STORAGE_META_SERVICE.get_slogger_manager(); ObStorageLogger *slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; @@ -266,7 +276,7 @@ TEST_F(TestStorageLogReplay, test_switch_file_replay) ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, - ObRedoLogSubType::OB_REDO_LOG_CREATE_LS); + ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET); build_storage(ObRandom::rand(1, 127)); for (int i = 0; i < tenant_storage_.slog_cnt_; i++) { @@ -286,7 +296,11 @@ TEST_F(TestStorageLogReplay, test_switch_file_replay) OB_DELETE(ObStorageLogger, ObModIds::TEST, slogger); slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; @@ -315,9 +329,12 @@ TEST_F(TestStorageLogReplay, test_mock_restart) write_start_cursor.log_id_ = 1; SimpleObStorageModule redo_module; - ObStorageLoggerManager &slogger_mgr = SERVER_STORAGE_META_SERVICE.get_slogger_manager(); ObStorageLogger *slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; @@ -325,7 +342,7 @@ TEST_F(TestStorageLogReplay, test_mock_restart) ObStorageLogParam log_param; log_param.cmd_ = ObIRedoModule::gen_cmd(ObRedoLogMainType::OB_REDO_LOG_TENANT_STORAGE, - ObRedoLogSubType::OB_REDO_LOG_CREATE_LS); + ObRedoLogSubType::OB_REDO_LOG_UPDATE_TABLET); build_storage(40); // first time to write slog @@ -342,7 +359,11 @@ TEST_F(TestStorageLogReplay, test_mock_restart) // reset slogger and set its start cursor as replay_finish_cursor slogger->destroy(); - ASSERT_EQ(OB_SUCCESS, slogger->init(slogger_mgr, TEST_TENANT_ID)); + ASSERT_EQ(OB_SUCCESS, slogger->init( + OB_FILE_SYSTEM_ROUTER.get_slog_dir(), + ObLogConstants::MAX_LOG_FILE_SIZE, + OB_FILE_SYSTEM_ROUTER.get_slog_file_spec(), + true)); ASSERT_EQ(OB_SUCCESS, slogger->start()); slogger->is_start_ = false; ASSERT_EQ(OB_SUCCESS, slogger->start_log(replay_finish_cursor_)); diff --git a/unittest/storage/tablet/test_tablet.cpp b/unittest/storage/tablet/test_tablet.cpp index f0a8b1ce9..4f9e99783 100644 --- a/unittest/storage/tablet/test_tablet.cpp +++ b/unittest/storage/tablet/test_tablet.cpp @@ -45,340 +45,6 @@ using namespace share; } \ } while (false) \ -class ObTabletMetaV1 final -{ -public: - int64_t get_serialize_size() const; - int serialize(char *buf, const int64_t len, int64_t &pos) const; - -public: - static const int32_t TABLET_META_VERSION_V1 = 1; - - int32_t version_; - int32_t length_; - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObTabletID data_tablet_id_; - common::ObTabletID ref_tablet_id_; - bool has_next_tablet_; - share::SCN create_scn_; - share::SCN start_scn_; - share::SCN clog_checkpoint_scn_; - share::SCN ddl_checkpoint_scn_; - int64_t snapshot_version_; - int64_t multi_version_start_; - lib::Worker::CompatMode compat_mode_; - share::ObTabletAutoincSeq autoinc_seq_; - ObTabletHAStatus ha_status_; - ObTabletReportStatus report_status_; - ObTabletTxMultiSourceDataUnit tx_data_; - ObTabletBindingInfo ddl_data_; - ObTabletTableStoreFlag table_store_flag_; - share::SCN ddl_start_scn_; - int64_t ddl_snapshot_version_; - int64_t max_sync_storage_schema_version_; - int64_t ddl_execution_id_; - int64_t ddl_cluster_version_; - int64_t max_serialized_medium_scn_; - // keep member same to 4.1 !!! -}; - -int ObTabletMetaV1::serialize(char *buf, const int64_t len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - const int64_t length = get_serialize_size(); - - if (OB_UNLIKELY(length > len - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer's length is not enough", K(ret), K(length), K(len - new_pos)); - } else if (OB_FAIL(serialization::encode_i32(buf, len, new_pos, version_))) { - LOG_WARN("failed to serialize tablet meta's version", K(ret), K(len), K(new_pos), K_(version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i32(buf, len, new_pos, length))) { - LOG_WARN("failed to serialize tablet meta's length", K(ret), K(len), K(new_pos), K(length)); - } else if (new_pos - pos < length && OB_FAIL(ls_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ls id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize data tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ref_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ref tablet id", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_bool(buf, len, new_pos, has_next_tablet_))) { - LOG_WARN("failed to serialize has next tablet", K(ret), K(len), K(new_pos), K_(has_next_tablet)); - } else if (new_pos - pos < length && OB_FAIL(create_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize create scn", K(ret), K(len), K(new_pos), K_(create_scn)); - } else if (new_pos - pos < length && OB_FAIL(start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize start scn", K(ret), K(len), K(new_pos), K_(start_scn)); - } else if (new_pos - pos < length && OB_FAIL(clog_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(clog_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(ddl_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl checkpoint ts", K(ret), K(len), K(new_pos), K_(ddl_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, snapshot_version_))) { - LOG_WARN("failed to serialize snapshot version", K(ret), K(len), K(new_pos), K_(snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, multi_version_start_))) { - LOG_WARN("failed to serialize multi version start", K(ret), K(len), K(new_pos), K_(multi_version_start)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i8(buf, len, new_pos, static_cast(compat_mode_)))) { - LOG_WARN("failed to serialize compat mode", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(autoinc_seq_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize auto inc seq", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ha status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(report_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize report status", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(tx_data_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize multi source data", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_data_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl data", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(table_store_flag_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize table store flag", K(ret), K(len), K(new_pos)); - } else if (new_pos - pos < length && OB_FAIL(ddl_start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl start log ts", K(ret), K(len), K(new_pos), K_(ddl_start_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_snapshot_version_))) { - LOG_WARN("failed to serialize ddl snapshot version", K(ret), K(len), K(new_pos), K_(ddl_snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_sync_storage_schema_version_))) { - LOG_WARN("failed to serialize max_sync_storage_schema_version", K(ret), K(len), K(new_pos), K_(max_sync_storage_schema_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_execution_id_))) { - LOG_WARN("failed to serialize ddl execution id", K(ret), K(len), K(new_pos), K_(ddl_execution_id)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_cluster_version_))) { - LOG_WARN("failed to serialize ddl cluster version", K(ret), K(len), K(new_pos), K_(ddl_cluster_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_serialized_medium_scn_))) { - LOG_WARN("failed to serialize max_serialized_medium_scn", K(ret), K(len), K(new_pos), K_(max_serialized_medium_scn)); - } else if (OB_UNLIKELY(length != new_pos - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet meta's length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length)); - } else { - pos = new_pos; - } - - return ret; -} - -int64_t ObTabletMetaV1::get_serialize_size() const -{ - int64_t size = 0; - size += serialization::encoded_length_i32(version_); - size += serialization::encoded_length_i32(length_); - size += ls_id_.get_serialize_size(); - size += tablet_id_.get_serialize_size(); - size += data_tablet_id_.get_serialize_size(); - size += ref_tablet_id_.get_serialize_size(); - size += serialization::encoded_length_bool(has_next_tablet_); - size += create_scn_.get_fixed_serialize_size(); - size += start_scn_.get_fixed_serialize_size(); - size += clog_checkpoint_scn_.get_fixed_serialize_size(); - size += ddl_checkpoint_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(snapshot_version_); - size += serialization::encoded_length_i64(multi_version_start_); - size += serialization::encoded_length_i8(static_cast(compat_mode_)); - size += autoinc_seq_.get_serialize_size(); - size += ha_status_.get_serialize_size(); - size += report_status_.get_serialize_size(); - size += tx_data_.get_serialize_size(); - size += ddl_data_.get_serialize_size(); - size += table_store_flag_.get_serialize_size(); - size += ddl_start_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(ddl_snapshot_version_); - size += serialization::encoded_length_i64(max_sync_storage_schema_version_); - size += serialization::encoded_length_i64(ddl_execution_id_); - size += serialization::encoded_length_i64(ddl_cluster_version_); - size += serialization::encoded_length_i64(max_serialized_medium_scn_); - return size; -} - -struct ObMigrationTabletParamV1 final -{ -public: - int serialize(char *buf, const int64_t len, int64_t &pos) const; - int deserialize(const char *buf, const int64_t len, int64_t &pos); - int64_t get_serialize_size() const; - - // magic_number_ is added to support upgrade from old format(without version and length compatibility) - // The old format first member is ls_id_(also 8 bytes long), which is not possible be a negative number. - const static int64_t MAGIC_NUM = -20230111; - const static int64_t PARAM_VERSION = 1; - - TO_STRING_KV(K_(magic_number), - K_(version), - K_(ls_id), - K_(tablet_id), - K_(data_tablet_id), - K_(ref_tablet_id), - K_(create_scn), - K_(start_scn), - K_(clog_checkpoint_scn), - K_(ddl_checkpoint_scn), - K_(ddl_snapshot_version), - K_(ddl_start_scn), - K_(snapshot_version), - K_(multi_version_start), - K_(autoinc_seq), - K_(compat_mode), - K_(ha_status), - K_(report_status), - K_(tx_data), - K_(ddl_data), - K_(storage_schema), - K_(medium_info_list), - K_(table_store_flag), - K_(max_sync_storage_schema_version), - K_(max_serialized_medium_scn), - K_(ddl_commit_scn)); - -public: - int64_t magic_number_; - int64_t version_; - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - common::ObTabletID data_tablet_id_; - common::ObTabletID ref_tablet_id_; - share::SCN create_scn_; - share::SCN start_scn_; // for migration - share::SCN clog_checkpoint_scn_; - share::SCN ddl_checkpoint_scn_; - int64_t snapshot_version_; - int64_t multi_version_start_; - lib::Worker::CompatMode compat_mode_; - share::ObTabletAutoincSeq autoinc_seq_; - ObTabletHAStatus ha_status_; - ObTabletReportStatus report_status_; - ObTabletTxMultiSourceDataUnit tx_data_; - ObTabletBindingInfo ddl_data_; - ObStorageSchema storage_schema_; - compaction::ObMediumCompactionInfoList medium_info_list_; - ObTabletTableStoreFlag table_store_flag_; - share::SCN ddl_start_scn_; - int64_t ddl_snapshot_version_; - // max_sync_version may less than storage_schema.schema_version_ when major update schema - int64_t max_sync_storage_schema_version_; - int64_t ddl_execution_id_; - int64_t ddl_data_format_version_; - int64_t max_serialized_medium_scn_; - share::SCN ddl_commit_scn_; - - // Add new serialization member before this line, below members won't serialize - common::ObArenaAllocator allocator_; // for storage schema -}; - -int ObMigrationTabletParamV1::serialize(char *buf, const int64_t len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - int64_t new_pos = pos; - int64_t length = 0; - - if (OB_ISNULL(buf) - || OB_UNLIKELY(len <= 0) - || OB_UNLIKELY(pos < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(buf), K(len), K(pos)); - } else if (FALSE_IT(length = get_serialize_size())) { - } else if (OB_UNLIKELY(length > len - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("buffer's length is not enough", K(ret), K(length), K(len - new_pos)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, magic_number_))) { - LOG_WARN("failed to serialize magic number", K(ret), K(len), K(new_pos), K_(magic_number)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, version_))) { - LOG_WARN("failed to serialize version", K(ret), K(len), K(new_pos), K_(version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, length))) { - LOG_WARN("failed to serialize length", K(ret), K(len), K(new_pos), K(length)); - } else if (new_pos - pos < length && OB_FAIL(ls_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ls id", K(ret), K(len), K(new_pos), K_(ls_id)); - } else if (new_pos - pos < length && OB_FAIL(tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize tablet id", K(ret), K(len), K(new_pos), K_(tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(data_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize data tablet id", K(ret), K(len), K(new_pos), K_(data_tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(ref_tablet_id_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ref tablet id", K(ret), K(len), K(new_pos), K_(ref_tablet_id)); - } else if (new_pos - pos < length && OB_FAIL(create_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(create_scn)); - } else if (new_pos - pos < length && OB_FAIL(start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize start scn", K(ret), K(len), K(new_pos), K_(start_scn)); - } else if (new_pos - pos < length && OB_FAIL(clog_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(clog_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(ddl_checkpoint_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl checkpoint ts", K(ret), K(len), K(new_pos), K_(ddl_checkpoint_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, snapshot_version_))) { - LOG_WARN("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, multi_version_start_))) { - LOG_WARN("failed to serialize clog checkpoint ts", K(ret), K(len), K(new_pos), K_(multi_version_start)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i8(buf, len, new_pos, static_cast(compat_mode_)))) { - LOG_WARN("failed to serialize compat mode", K(ret), K(len), K(new_pos), K_(compat_mode)); - } else if (new_pos - pos < length && OB_FAIL(autoinc_seq_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize auto inc seq", K(ret), K(len), K(new_pos), K_(autoinc_seq)); - } else if (new_pos - pos < length && OB_FAIL(ha_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ha status", K(ret), K(len), K(new_pos), K_(ha_status)); - } else if (new_pos - pos < length && OB_FAIL(report_status_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize report status", K(ret), K(len), K(new_pos), K_(report_status)); - } else if (new_pos - pos < length && OB_FAIL(tx_data_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize multi source data", K(ret), K(len), K(new_pos), K_(tx_data)); - } else if (new_pos - pos < length && OB_FAIL(ddl_data_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl data", K(ret), K(len), K(new_pos), K_(ddl_data)); - } else if (new_pos - pos < length && OB_FAIL(storage_schema_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize storage schema", K(ret), K(len), K(new_pos), K_(storage_schema)); - } else if (new_pos - pos < length && OB_FAIL(medium_info_list_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize medium compaction list", K(ret), K(len), K(new_pos), K_(medium_info_list)); - } else if (new_pos - pos < length && OB_FAIL(table_store_flag_.serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize table store flag", K(ret), K(len), K(new_pos), K_(table_store_flag)); - } else if (new_pos - pos < length && OB_FAIL(ddl_start_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl start log ts", K(ret), K(len), K(new_pos), K_(ddl_start_scn)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_snapshot_version_))) { - LOG_WARN("failed to serialize ddl snapshot version", K(ret), K(len), K(new_pos), K_(ddl_snapshot_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_sync_storage_schema_version_))) { - LOG_WARN("failed to serialize max_sync_storage_schema_version", K(ret), K(len), K(new_pos), K_(max_sync_storage_schema_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_execution_id_))) { - LOG_WARN("failed to serialize ddl execution id", K(ret), K(len), K(new_pos), K_(ddl_execution_id)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, ddl_data_format_version_))) { - LOG_WARN("failed to serialize ddl data format version", K(ret), K(len), K(new_pos), K_(ddl_data_format_version)); - } else if (new_pos - pos < length && OB_FAIL(serialization::encode_i64(buf, len, new_pos, max_serialized_medium_scn_))) { - LOG_WARN("failed to serialize max_serialized_medium_scn", K(ret), K(len), K(new_pos), K_(max_serialized_medium_scn)); - } else if (new_pos - pos < length && OB_FAIL(ddl_commit_scn_.fixed_serialize(buf, len, new_pos))) { - LOG_WARN("failed to serialize ddl commit scn", K(ret), K(len), K(new_pos), K_(ddl_commit_scn)); - } else if (OB_UNLIKELY(length != new_pos - pos)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length)); - } else { - pos = new_pos; - } - - return ret; -} - -int64_t ObMigrationTabletParamV1::get_serialize_size() const -{ - int64_t size = 0; - int64_t length = 0; - size += serialization::encoded_length_i64(magic_number_); - size += serialization::encoded_length_i64(version_); - size += serialization::encoded_length_i64(length); - size += ls_id_.get_serialize_size(); - size += tablet_id_.get_serialize_size(); - size += data_tablet_id_.get_serialize_size(); - size += ref_tablet_id_.get_serialize_size(); - size += create_scn_.get_fixed_serialize_size(); - size += start_scn_.get_fixed_serialize_size(); - size += clog_checkpoint_scn_.get_fixed_serialize_size(); - size += ddl_checkpoint_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(snapshot_version_); - size += serialization::encoded_length_i64(multi_version_start_); - size += serialization::encoded_length_i8(static_cast(compat_mode_)); - size += autoinc_seq_.get_serialize_size(); - size += ha_status_.get_serialize_size(); - size += report_status_.get_serialize_size(); - size += tx_data_.get_serialize_size(); - size += ddl_data_.get_serialize_size(); - size += storage_schema_.get_serialize_size(); - size += medium_info_list_.get_serialize_size(); - size += table_store_flag_.get_serialize_size(); - size += ddl_start_scn_.get_fixed_serialize_size(); - size += serialization::encoded_length_i64(ddl_snapshot_version_); - size += serialization::encoded_length_i64(max_sync_storage_schema_version_); - size += serialization::encoded_length_i64(ddl_execution_id_); - size += serialization::encoded_length_i64(ddl_data_format_version_); - size += serialization::encoded_length_i64(max_serialized_medium_scn_); - size += ddl_commit_scn_.get_fixed_serialize_size(); - return size; -} - class TestTablet : public ::testing::Test { public: @@ -429,7 +95,7 @@ void TestTablet::TearDownTestCase() void TestTablet::SetUp() { OB_LOG(INFO, "ObTabletMeta", K(sizeof(ObTabletMeta)), - K(sizeof(ObTabletHAStatus)), K(sizeof(ObTabletReportStatus)), K(sizeof(ObTabletTableStoreFlag)), + K(sizeof(ObTabletRestoreState)), K(sizeof(ObTabletReportStatus)), K(sizeof(ObTabletTableStoreFlag)), K(sizeof(ObTabletSpaceUsage)), K(sizeof(lib::Worker::CompatMode))); const int64_t tablet_size = sizeof(ObTablet); @@ -447,122 +113,6 @@ void TestTablet::TearDown() { } -TEST_F(TestTablet, test_serialize_meta_compat) -{ - int ret = OB_SUCCESS; - ObTabletMetaV1 tablet_meta; - tablet_meta.version_ = ObTabletMetaV1::TABLET_META_VERSION_V1; - tablet_meta.ls_id_ = ObLSID(3); - tablet_meta.tablet_id_ = ObTabletID(1); - tablet_meta.data_tablet_id_ = ObTabletID(4); - tablet_meta.ref_tablet_id_ = ObTabletID(1); - bool has_next_tablet_ = true; - tablet_meta.create_scn_ = share::SCN::base_scn(); - tablet_meta.start_scn_ = share::SCN::base_scn(); - tablet_meta.clog_checkpoint_scn_ = share::SCN::base_scn(); - tablet_meta.ddl_checkpoint_scn_ = share::SCN::base_scn(); - tablet_meta.snapshot_version_ = 5; - tablet_meta.multi_version_start_ = 9; - tablet_meta.compat_mode_ = lib::Worker::CompatMode::MYSQL; - tablet_meta.autoinc_seq_.set_autoinc_seq_value(allocator_, 1); - ASSERT_EQ(OB_SUCCESS, tablet_meta.ha_status_.init_status()); - tablet_meta.ddl_start_scn_ = share::SCN::base_scn(); - tablet_meta.ddl_snapshot_version_ = 3; - tablet_meta.max_sync_storage_schema_version_ = 1; - tablet_meta.ddl_execution_id_ = 4; - tablet_meta.ddl_cluster_version_ = 1; - tablet_meta.max_serialized_medium_scn_ = 5; - tablet_meta.length_ = tablet_meta.get_serialize_size(); - tablet_meta.tx_data_.tablet_status_ = ObTabletStatus::NORMAL; - - char *buf = nullptr; - int64_t len = tablet_meta.get_serialize_size(); - buf = reinterpret_cast(allocator_.alloc(len)); - int64_t pos = 0; - ASSERT_EQ(OB_SUCCESS, tablet_meta.serialize(buf, len, pos)); - - pos = 0; - ObTablet tablet; - ObTabletAutoincSeq autoinc_seq; - ObTabletTxMultiSourceDataUnit tx_data; - ObTabletBindingInfo ddl_data; - ASSERT_EQ(OB_SUCCESS, - tablet.deserialize_meta_v1(allocator_, buf, len, pos, autoinc_seq, tx_data, ddl_data)); - uint64_t autoinc_seq_v1; - ASSERT_EQ(OB_SUCCESS, tablet_meta.autoinc_seq_.get_autoinc_seq_value(autoinc_seq_v1)); - uint64_t autoinc_seq_v2; - ASSERT_EQ(OB_SUCCESS, autoinc_seq.get_autoinc_seq_value(autoinc_seq_v2)); - ASSERT_EQ(autoinc_seq_v1, autoinc_seq_v2); - ASSERT_EQ(tx_data.tablet_status_, ObTabletStatus::NORMAL); -} - -TEST_F(TestTablet, test_serialize_mig_param_compat) -{ - int ret = OB_SUCCESS; - ObMigrationTabletParamV1 mig_param; - mig_param.magic_number_ = ObMigrationTabletParamV1::MAGIC_NUM; - mig_param.version_ = ObMigrationTabletParamV1::PARAM_VERSION; - mig_param.ls_id_ = ObLSID(3); - mig_param.tablet_id_ = ObTabletID(1); - mig_param.data_tablet_id_ = ObTabletID(4); - mig_param.ref_tablet_id_ = ObTabletID(1); - mig_param.create_scn_ = share::SCN::base_scn(); - mig_param.start_scn_ = share::SCN::base_scn(); - mig_param.clog_checkpoint_scn_ = share::SCN::base_scn(); - mig_param.ddl_checkpoint_scn_ = share::SCN::base_scn(); - mig_param.snapshot_version_ = 9; - mig_param.multi_version_start_ = 5; - mig_param.compat_mode_ = lib::Worker::CompatMode::MYSQL; - mig_param.autoinc_seq_.set_autoinc_seq_value(allocator_, 1); - ASSERT_EQ(OB_SUCCESS, mig_param.ha_status_.init_status()); - mig_param.tx_data_.tablet_status_ = ObTabletStatus::NORMAL; - mig_param.ddl_start_scn_ = share::SCN::base_scn(); - mig_param.ddl_snapshot_version_ = 3; - mig_param.max_sync_storage_schema_version_ = 1; - mig_param.ddl_execution_id_ = 4; - mig_param.ddl_data_format_version_ = 6; - mig_param.max_serialized_medium_scn_ = 5; - mig_param.ddl_commit_scn_ = share::SCN::base_scn(); - mig_param.storage_schema_.storage_schema_version_ = ObStorageSchema::STORAGE_SCHEMA_VERSION_LATEST; - mig_param.storage_schema_.schema_version_ = 1; - mig_param.storage_schema_.column_cnt_ = 1; - mig_param.storage_schema_.table_type_ = SYSTEM_TABLE; - const ObStorageColumnSchema column_schema; - mig_param.storage_schema_.column_array_.set_allocator(&mig_param.allocator_); - ASSERT_EQ(OB_SUCCESS, mig_param.storage_schema_.column_array_.reserve(mig_param.storage_schema_.column_cnt_)); - ASSERT_EQ(OB_SUCCESS, mig_param.storage_schema_.column_array_.push_back(column_schema)); - ASSERT_EQ(mig_param.storage_schema_.column_array_.count(), 1); - mig_param.storage_schema_.is_inited_ = true; - - int8_t head_val = 3; - int8_t tail_val = 4; - char *buf = nullptr; - int64_t len = serialization::encoded_length_i8(head_val) + mig_param.get_serialize_size() + serialization::encoded_length_i8(tail_val); - buf = reinterpret_cast(allocator_.alloc(len)); - int64_t pos = 0; - serialization::encode_i8(buf, len, pos, head_val); - ASSERT_EQ(OB_SUCCESS, mig_param.serialize(buf, len, pos)); - serialization::encode_i8(buf, len, pos, tail_val); - - ObMigrationTabletParam new_mig_param; - pos = 0; - serialization::decode_i8(buf, len, pos, &head_val); - ASSERT_EQ(head_val, 3); - ret = new_mig_param.deserialize(buf, len, pos); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(new_mig_param.storage_schema_.column_array_.count(), 1); - ASSERT_TRUE(new_mig_param.is_valid()); - serialization::decode_i8(buf, len, pos, &tail_val); - ASSERT_EQ(tail_val, 4); - ObTabletCreateDeleteMdsUserData user_data; - mds::MdsDumpNode *v = &new_mig_param.mds_data_.tablet_status_committed_kv_.v_; - pos = 0; - ret = user_data.deserialize(v->user_data_.ptr(), v->user_data_.length(), pos); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(user_data.tablet_status_, ObTabletStatus::NORMAL); - -} - class TestTableStore { public: diff --git a/unittest/storage/test_dag_warning_history.cpp b/unittest/storage/test_dag_warning_history.cpp index 0b5ba41e5..a9066868f 100644 --- a/unittest/storage/test_dag_warning_history.cpp +++ b/unittest/storage/test_dag_warning_history.cpp @@ -81,10 +81,18 @@ void TestDagWarningHistory::calc_info_cnt_per_page(ObIDag &dag, int64_t &info_me if (OB_FAIL(dag.gene_warning_info(tmp_info, allocator))) { COMMON_LOG(WARN, "failed to gene dag warning info", K(ret)); } - // every time will contain a header(16B) info_mem_size = tmp_info.get_deep_copy_size(); - info_cnt_per_page = (INFO_PAGE_SIZE - sizeof(ObFIFOAllocator::NormalPageHeader)) - / (info_mem_size + sizeof(ObFIFOAllocator::AllocHeader)); + info_cnt_per_page = 0; + int64_t offset = sizeof(ObFIFOAllocator::NormalPageHeader); + const int64_t align = alignof(ObFIFOAllocator::AllocHeader); + while (true) { + offset = upper_align(offset + sizeof(ObFIFOAllocator::AllocHeader), align); + if (offset + info_mem_size > INFO_PAGE_SIZE) { + break; + } + offset += info_mem_size; + ++info_cnt_per_page; + } STORAGE_LOG(INFO, "size", K(info_mem_size), K(info_cnt_per_page)); } @@ -113,7 +121,7 @@ class ObBasicDag : public ObIDag int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; - } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), 1, 2, "table_id", 10))) { + } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), 2, "table_id", 10))) { COMMON_LOG(WARN, "fail to add dag warning info param", K(ret)); } return ret; diff --git a/unittest/storage/test_dml_common.h b/unittest/storage/test_dml_common.h index 4572f702f..75d0f20b7 100644 --- a/unittest/storage/test_dml_common.h +++ b/unittest/storage/test_dml_common.h @@ -28,7 +28,6 @@ #include "common/ob_tablet_id.h" #include "lib/time/ob_clock_generator.h" #include "common/object/ob_obj_type.h" -#include "share/ob_ls_id.h" #include "share/ob_rpc_struct.h" #include "share/ob_storage_format.h" #include "storage/access/ob_table_param.h" @@ -37,9 +36,6 @@ #include "storage/ls/ob_ls_tablet_service.h" #include "storage/tx/ob_trans_define.h" #include "storage/tx/ob_trans_service.h" -#include "mock_ls_tablet_service.h" -#include "mock_access_service.h" -#include "init_basic_struct.h" #undef private #undef protected @@ -62,27 +58,7 @@ class TestTxCallback : public transaction::ObITxCallback class TestDmlCommon { public: - static int create_ls( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - ObLSHandle &ls_handle); - static int create_data_tablet( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id); - static int create_data_and_index_tablets( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - const common::ObIArray &index_tablet_id_array); - static int mock_ls_tablet_service( - const share::ObLSID &ls_id, - ObLSTabletService *&tablet_service); - static void delete_mocked_ls_tablet_service(ObLSTabletService *tablet_service); - static int mock_access_service( - ObLSTabletService *tablet_service, - MockObAccessService *&access_service); - static void delete_mocked_access_service(ObAccessService *access_service); + static int create_ls(ObLS *&ls); static int build_table_param( const ObTableSchema &table_schema, @@ -112,22 +88,8 @@ class TestDmlCommon static int build_tx_desc(const uint64_t tenant_id, ObTxDesc *&tx_desc); static void build_tx_param(ObTxParam &tx_param); static void release_tx_desc(ObTxDesc &tx_desc); - static int insert_data_to_tablet(); -protected: - static int build_pure_data_tablet_arg( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - obcall::ObBatchCreateTabletArg &arg); - static int build_mixed_tablets_arg( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - const common::ObIArray &index_tablet_id_array, - obcall::ObBatchCreateTabletArg &arg); public: static const uint64_t TX_EXPIRE_TIME_US = 120 * 1000 * 1000; // 120s - static const int64_t TEST_LS_ID = 2; static const uint64_t TEST_DATA_TABLE_ID = 50; static const uint64_t TEST_INDEX_TABLE_ID = 51; static constexpr const char *data_row_str = @@ -146,29 +108,17 @@ class TestDmlCommon "12 50 32 Boston Celtics T_DML_INSERT \n"; }; -int TestDmlCommon::create_ls( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - ObLSHandle &ls_handle) +int TestDmlCommon::create_ls(ObLS *&ls) { int ret = OB_SUCCESS; + ls = nullptr; ObLSService *ls_svr = MTL(ObLSService*); - bool b_exist = false; - ObLS *ls = nullptr; if (OB_FAIL(ls_svr->create_ls())) { STORAGE_LOG(WARN, "failed to create ls"); - } else if (OB_FAIL(ls_svr->check_ls_exist(ls_id, b_exist))) { - STORAGE_LOG(WARN, "failed to check ls exist", K(ls_id)); - } else if (!b_exist) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, ls does not exist", K(ret), K(ls_id)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "failed to get ls", K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_handle)); + } else if (OB_FAIL(ls_svr->get_ls(ls))) { + STORAGE_LOG(WARN, "failed to get ls"); } // check leader @@ -184,8 +134,7 @@ int TestDmlCommon::create_ls( ::sleep(1); } - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(ObRole::LEADER != role)) { + if (OB_SUCC(ret) && OB_UNLIKELY(ObRole::LEADER != role)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "unexpected error, role is not leader", K(ret), K(role)); } @@ -193,136 +142,6 @@ int TestDmlCommon::create_ls( return ret; } -int TestDmlCommon::create_data_tablet( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - ObLSHandle ls_handle; - obcall::ObBatchCreateTabletArg arg; - - if (OB_FAIL(create_ls(tenant_id, ls_id, ls_handle))) { - STORAGE_LOG(WARN, "failed to create ls", K(ret), K(tenant_id), K(ls_id)); - } else if (OB_FAIL(build_pure_data_tablet_arg(tenant_id, ls_id, tablet_id, arg))) { - STORAGE_LOG(WARN, "failed to build pure data tablet arg", K(ret), - K(tenant_id), K(ls_id), K(tablet_id)); - } else { - ret = OB_NOT_SUPPORTED; - } - - return ret; -} - -int TestDmlCommon::create_data_and_index_tablets( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - const common::ObIArray &index_tablet_id_array) -{ - int ret = OB_SUCCESS; - ObLSHandle ls_handle; - obcall::ObBatchCreateTabletArg arg; - - if (OB_FAIL(create_ls(tenant_id, ls_id, ls_handle))) { - STORAGE_LOG(WARN, "failed to create ls", K(ret), K(tenant_id), K(ls_id)); - } else if (OB_FAIL(build_mixed_tablets_arg(tenant_id, ls_id, - data_tablet_id, index_tablet_id_array, arg))) { - STORAGE_LOG(WARN, "failed to build pure data tablet arg", K(ret), - K(tenant_id), K(ls_id), K(data_tablet_id), K(index_tablet_id_array)); - } else { - ret = OB_NOT_SUPPORTED; - } - - return ret; -} - -int TestDmlCommon::mock_ls_tablet_service( - const share::ObLSID &ls_id, - ObLSTabletService *&tablet_service) -{ - int ret = OB_SUCCESS; - ObLSHandle ls_handle; - ObLSService *ls_svr = MTL(ObLSService*); - ObLS *ls = nullptr; - - if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - STORAGE_LOG(WARN, "failed to get ls", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ls is null", K(ret), K(ls)); - } else { - ObLSTabletService &svc = ls->ls_tablet_svr_; - MockInsertRowsLSTabletService *mock_svc = OB_NEW(MockInsertRowsLSTabletService, ObModIds::TEST); - ObArray tablet_ids; - ObLSTabletService::GetAllTabletIDOperator get_all_tablet_id_op(tablet_ids); - if (OB_ISNULL(mock_svc)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc memory", K(ret)); - } else if (OB_FAIL(svc.tablet_id_set_.foreach(get_all_tablet_id_op))) { - STORAGE_LOG(WARN, "fail to get all tablet ids from set", K(ret)); - } else { - mock_svc->ls_ = svc.ls_; - // ignore ObTxDataMemtableMgr and ObTxCtxMemtableMgr - - // copy hash set - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - const common::ObTabletID &tablet_id = tablet_ids.at(i); - if (mock_svc->tablet_id_set_.set(tablet_id)) { - STORAGE_LOG(WARN, "failed to insert tablet id", K(tablet_id)); - } - } - - // ignore ObBucketLock - mock_svc->is_inited_ = svc.is_inited_; - } - - if (OB_SUCC(ret)) { - tablet_service = mock_svc; - } - } - - return ret; -} - -void TestDmlCommon::delete_mocked_ls_tablet_service(ObLSTabletService *tablet_service) -{ - MockInsertRowsLSTabletService *svc = static_cast(tablet_service); - OB_DELETE(MockInsertRowsLSTabletService, ObModIds::TEST, svc); -} - -int TestDmlCommon::mock_access_service( - ObLSTabletService *tablet_service, - MockObAccessService *&access_service) -{ - int ret = OB_SUCCESS; - MockObAccessService *mock_svc = OB_NEW(MockObAccessService, ObModIds::TEST); - - if (OB_ISNULL(tablet_service)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid args", K(ret), K(tablet_service)); - } else if (OB_ISNULL(mock_svc)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc memory", K(ret)); - } else { - ObAccessService *svc = MTL(ObAccessService*); - // do copy - mock_svc->is_inited_ = svc->is_inited_; - mock_svc->ls_svr_ = svc->ls_svr_; - mock_svc->tablet_service_ = tablet_service; - - access_service = mock_svc; - } - - return ret; -} - -void TestDmlCommon::delete_mocked_access_service(ObAccessService *access_service) -{ - MockObAccessService *svc = static_cast(access_service); - OB_DELETE(MockObAccessService, ObModIds::TEST, svc); -} - int TestDmlCommon::build_table_param( const ObTableSchema &table_schema, const ObIArray &output_column_ids, @@ -386,7 +205,6 @@ int TestDmlCommon::build_table_scan_param_base_( scan_param.column_ids_.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3 scan_param.column_ids_.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4 - scan_param.ls_id_ = TestDmlCommon::TEST_LS_ID; scan_param.tablet_id_ = TEST_DATA_TABLE_ID; scan_param.table_param_ = &table_param; @@ -539,98 +357,6 @@ void TestDmlCommon::release_tx_desc(ObTxDesc &tx_desc) tx_service->release_tx(tx_desc); } -int TestDmlCommon::build_pure_data_tablet_arg( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - obcall::ObBatchCreateTabletArg &arg) -{ - int ret = OB_SUCCESS; - - ObCreateTabletInfo tablet_info; - ObArray tablet_id_array; - ObArray tablet_schema_index_array; - ObArray create_commit_versions; - share::schema::ObTableSchema table_schema; - build_data_table_schema(tenant_id, table_schema); - - arg.reset(); - if (OB_FAIL(tablet_id_array.push_back(data_tablet_id))) { - STORAGE_LOG(WARN, "failed to push tablet id into array", K(ret), K(data_tablet_id)); - } else if (OB_FAIL(tablet_schema_index_array.push_back(0))) { - STORAGE_LOG(WARN, "failed to push index into array", K(ret)); - } else if (OB_FAIL(tablet_info.init(tablet_id_array, data_tablet_id, tablet_schema_index_array, lib::Worker::CompatMode::MYSQL, false, create_commit_versions))) { - STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array), - K(data_tablet_id), K(tablet_schema_index_array)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn(), false/*need_check_tablet_cnt*/))) { - STORAGE_LOG(WARN, "failed to init create tablet", K(ret), K(tenant_id), K(ls_id)); - } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { - STORAGE_LOG(WARN, "failed to push back table schema", K(ret), K(table_schema)); - } else if (OB_FAIL(arg.tablets_.push_back(tablet_info))) { - STORAGE_LOG(WARN, "failed to push back tablet info", K(ret), K(tablet_info)); - } - - if (OB_FAIL(ret)) { - arg.reset(); - } - - return ret; -} - -int TestDmlCommon::build_mixed_tablets_arg( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &data_tablet_id, - const common::ObIArray &index_tablet_id_array, - obcall::ObBatchCreateTabletArg &arg) -{ - int ret = OB_SUCCESS; - - ObCreateTabletInfo tablet_info; - ObArray tablet_id_array; - ObArray tablet_schema_index_array; - ObArray create_commit_versions; - share::schema::ObTableSchema data_table_schema; - share::schema::ObTableSchema index_table_schema; - build_data_table_schema(tenant_id, data_table_schema); - build_index_table_schema(tenant_id, index_table_schema); - - arg.reset(); - if (OB_FAIL(tablet_id_array.push_back(data_tablet_id))) { - STORAGE_LOG(WARN, "failed to push tablet id into array", K(ret), K(data_tablet_id)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < index_tablet_id_array.count(); ++i) { - const common::ObTabletID &index_tablet_id = index_tablet_id_array.at(i); - if (OB_FAIL(tablet_id_array.push_back(index_tablet_id))) { - STORAGE_LOG(WARN, "failed to push back index tablet id", K(ret), K(index_tablet_id)); - } - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tablet_schema_index_array.push_back(0))) { - STORAGE_LOG(WARN, "failed to push index into array", K(ret)); - } else if (OB_FAIL(tablet_schema_index_array.push_back(1))) { - STORAGE_LOG(WARN, "failed to push index into array", K(ret)); - } else if (OB_FAIL(tablet_info.init(tablet_id_array, data_tablet_id, tablet_schema_index_array, lib::Worker::CompatMode::MYSQL, false, create_commit_versions))) { - STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array), - K(data_tablet_id), K(tablet_schema_index_array)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn(), false/*need_check_tablet_cnt*/))) { - STORAGE_LOG(WARN, "failed to init create tablet", K(ret), K(tenant_id), K(ls_id)); - } else if (OB_FAIL(arg.table_schemas_.push_back(data_table_schema))) { - STORAGE_LOG(WARN, "failed to push back data table schema", K(ret), K(data_table_schema)); - } else if (OB_FAIL(arg.table_schemas_.push_back(index_table_schema))) { - STORAGE_LOG(WARN, "failed to push back index table schema", K(ret), K(index_table_schema)); - } else if (OB_FAIL(arg.tablets_.push_back(tablet_info))) { - STORAGE_LOG(WARN, "failed to push back tablet info", K(ret), K(tablet_info)); - } - - if (OB_FAIL(ret)) { - arg.reset(); - } - - return ret; -} } // namespace storage } // namespace oceanbase diff --git a/unittest/storage/test_partition_incremental_range_spliter.cpp b/unittest/storage/test_partition_incremental_range_spliter.cpp index b10c07a23..4faec5699 100644 --- a/unittest/storage/test_partition_incremental_range_spliter.cpp +++ b/unittest/storage/test_partition_incremental_range_spliter.cpp @@ -21,6 +21,7 @@ #define protected public #include "storage/ob_partition_range_spliter.h" +#include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h" #include "storage/compaction/ob_tablet_merge_ctx.h" #include "share/rc/ob_module_provider.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" diff --git a/unittest/storage/test_partition_major_sstable_range_spliter.cpp b/unittest/storage/test_partition_major_sstable_range_spliter.cpp index a6fb3a279..e68a5e004 100644 --- a/unittest/storage/test_partition_major_sstable_range_spliter.cpp +++ b/unittest/storage/test_partition_major_sstable_range_spliter.cpp @@ -21,6 +21,7 @@ #define protected public #include "storage/ob_partition_range_spliter.h" +#include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h" #include "storage/compaction/ob_tablet_merge_ctx.h" namespace oceanbase diff --git a/unittest/storage/test_sstable_log_ts_range_cut.cpp b/unittest/storage/test_sstable_log_ts_range_cut.cpp deleted file mode 100644 index a7b0228b5..000000000 --- a/unittest/storage/test_sstable_log_ts_range_cut.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#define protected public -#define private public -#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" - -namespace oceanbase -{ -using namespace common; -using namespace blocksstable; -using namespace storage; -using namespace omt; -using namespace share; -namespace unittest -{ - -class TestMockSSTable -{ -public: - static void generate_mock_sstable( - const int64_t start_log_ts, - const int64_t end_log_ts, - ObSSTable &sstable, - blocksstable::ObSSTableMeta &meta) - { - meta.basic_meta_.root_row_store_type_ = ObRowStoreType::FLAT_ROW_STORE; - meta.basic_meta_.latest_row_store_type_ = ObRowStoreType::FLAT_ROW_STORE; - meta.basic_meta_.status_ = SSTABLE_READY_FOR_READ; - meta.data_root_info_.addr_.set_none_addr(); - meta.macro_info_.macro_meta_info_.addr_.set_none_addr(); - meta.basic_meta_.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - meta.is_inited_ = true; - - sstable.key_.table_type_ = ObITable::MINOR_SSTABLE; - sstable.key_.tablet_id_ = 1; - sstable.key_.scn_range_.start_scn_.convert_for_gts(start_log_ts); - sstable.key_.scn_range_.end_scn_.convert_for_gts(end_log_ts); - sstable.meta_ = &meta; - sstable.valid_for_reading_ = true; - } -}; - - -class TestSSTableScnRangeCut : public ::testing::Test -{ -public: - TestSSTableScnRangeCut() - { } - ~TestSSTableScnRangeCut() {} - void SetUp() - { - } - void TearDown() - { - } -private: - DISALLOW_COPY_AND_ASSIGN(TestSSTableScnRangeCut); -}; - -//normal condition -//sstable1 log_ts: [0,100) -//sstable2 log_ts: [100, 200) -//sstable3 log_ts: [200,300) - -TEST_F(TestSSTableScnRangeCut, sstable_scn_range_no_cross_and_continue) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTabletTableStore tablet_table_store; - - ObSSTable sstable1; - blocksstable::ObSSTableMeta meta1; - TestMockSSTable::generate_mock_sstable(0, 100, sstable1, meta1); - - ObSSTable sstable2; - blocksstable::ObSSTableMeta meta2; - TestMockSSTable::generate_mock_sstable(100, 200, sstable2, meta2); - - ObSSTable sstable3; - blocksstable::ObSSTableMeta meta3; - TestMockSSTable::generate_mock_sstable(200, 300, sstable3, meta3); - ObArray minor_sstables; - ObArray cut_minor_sstables; - - ret = minor_sstables.push_back(&sstable1); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable2); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable3); - ASSERT_EQ(OB_SUCCESS, ret); - ret = tablet_table_store.cut_ha_sstable_scn_range_(allocator, minor_sstables, cut_minor_sstables); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_EQ(0, cut_minor_sstables.at(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(100, cut_minor_sstables.at(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(100, cut_minor_sstables.at(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(200, cut_minor_sstables.at(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(200, cut_minor_sstables.at(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(300, cut_minor_sstables.at(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); -} - - -//sstable log ts is not continue with other sstable log ts -//sstable1 log_ts: [0,100) -//sstable2 log_ts: [200, 300) -//sstable3 log_ts: [300,500) - -TEST_F(TestSSTableScnRangeCut, sstable_scn_range_is_not_continue) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTabletTableStore tablet_table_store; - - ObSSTable sstable1; - blocksstable::ObSSTableMeta meta1; - TestMockSSTable::generate_mock_sstable(0, 100, sstable1, meta1); - - ObSSTable sstable2; - blocksstable::ObSSTableMeta meta2; - TestMockSSTable::generate_mock_sstable(200, 300, sstable2, meta2); - - ObSSTable sstable3; - blocksstable::ObSSTableMeta meta3; - TestMockSSTable::generate_mock_sstable(300, 500, sstable3, meta3); - - ObArray minor_sstables; - ObArray cut_minor_sstables; - - ret = minor_sstables.push_back(&sstable1); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable2); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable3); - ASSERT_EQ(OB_SUCCESS, ret); - ret = tablet_table_store.cut_ha_sstable_scn_range_(allocator, minor_sstables, cut_minor_sstables); - ASSERT_EQ(OB_ERR_UNEXPECTED, ret); -} - - -//sstable log ts contain other sstable log ts -//sstable1 log_ts: [0,100) -//sstable2 log_ts: [0, 200) -//sstable3 log_ts: [200,500) - -TEST_F(TestSSTableScnRangeCut, sstable_scn_range_contain) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTabletTableStore tablet_table_store; - - ObSSTable sstable1; - blocksstable::ObSSTableMeta meta1; - TestMockSSTable::generate_mock_sstable(0, 100, sstable1, meta1); - - ObSSTable sstable2; - blocksstable::ObSSTableMeta meta2; - TestMockSSTable::generate_mock_sstable(0, 200, sstable2, meta2); - - ObSSTable sstable3; - blocksstable::ObSSTableMeta meta3; - TestMockSSTable::generate_mock_sstable(200, 500, sstable3, meta3); - - ObArray minor_sstables; - ObArray cut_minor_sstables; - - ret = minor_sstables.push_back(&sstable1); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable2); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable3); - ASSERT_EQ(OB_SUCCESS, ret); - ret = tablet_table_store.cut_ha_sstable_scn_range_(allocator, minor_sstables, cut_minor_sstables); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_EQ(0, cut_minor_sstables.at(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(100, cut_minor_sstables.at(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(100, cut_minor_sstables.at(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(200, cut_minor_sstables.at(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(200, cut_minor_sstables.at(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(500, cut_minor_sstables.at(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - -} - -//sstable log ts contain other sstable log ts -//sstable1 log_ts: [0,100) -//sstable2 log_ts: [50, 200) -//sstable3 log_ts: [200,500) - -TEST_F(TestSSTableScnRangeCut, sstable_scn_range_has_overlap) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTabletTableStore tablet_table_store; - - ObSSTable sstable1; - blocksstable::ObSSTableMeta meta1; - TestMockSSTable::generate_mock_sstable(0, 100, sstable1, meta1); - - ObSSTable sstable2; - blocksstable::ObSSTableMeta meta2; - TestMockSSTable::generate_mock_sstable(50, 200, sstable2, meta2); - - ObSSTable sstable3; - blocksstable::ObSSTableMeta meta3; - TestMockSSTable::generate_mock_sstable(200, 500, sstable3, meta3); - - ObArray minor_sstables; - ObArray cut_minor_sstables; - - ret = minor_sstables.push_back(&sstable1); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable2); - ASSERT_EQ(OB_SUCCESS, ret); - ret = minor_sstables.push_back(&sstable3); - ASSERT_EQ(OB_SUCCESS, ret); - ret = tablet_table_store.cut_ha_sstable_scn_range_(allocator, minor_sstables, cut_minor_sstables); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_EQ(0, cut_minor_sstables.at(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(100, cut_minor_sstables.at(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(100, cut_minor_sstables.at(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(200, cut_minor_sstables.at(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); - - ASSERT_EQ(200, cut_minor_sstables.at(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field()); - ASSERT_EQ(500, cut_minor_sstables.at(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field()); -} - - -}//blocksstable -}//oceanbase - -int main(int argc, char** argv) -{ - system("rm -f test_sstable_log_ts_range_cut.log*"); - OB_LOGGER.set_file_name("test_sstable_log_ts_range_cut.log"); - OB_LOGGER.set_log_level("INFO"); - testing::InitGoogleTest(&argc, argv); - oceanbase::lib::set_memory_limit(40L << 30); - return RUN_ALL_TESTS(); -} diff --git a/unittest/storage/test_sstable_merge_info_mgr.cpp b/unittest/storage/test_sstable_merge_info_mgr.cpp index d02ef4e1b..6b77a6869 100644 --- a/unittest/storage/test_sstable_merge_info_mgr.cpp +++ b/unittest/storage/test_sstable_merge_info_mgr.cpp @@ -67,7 +67,6 @@ class TestSSTableMergeInfoMgr : public ::testing::Test void init_merge_history(ObSSTableMergeHistory &merge_info) { - merge_info.static_info_.ls_id_ = 1; merge_info.static_info_.tablet_id_ = 3; merge_info.static_info_.compaction_scn_ = 100; merge_info.static_info_.merge_type_ = ObMergeType::MINOR_MERGE; diff --git a/unittest/storage/test_tablet_create_mds_ctx.cpp b/unittest/storage/test_tablet_create_mds_ctx.cpp index 00cf42ea8..0143a4f0f 100644 --- a/unittest/storage/test_tablet_create_mds_ctx.cpp +++ b/unittest/storage/test_tablet_create_mds_ctx.cpp @@ -44,7 +44,6 @@ TEST_F(TestTabletCreateMdsCtx, start_mds_ctx) int ret = OB_SUCCESS; mds::ObTabletCreateMdsCtx mds_ctx{mds::MdsWriter{transaction::ObTransID{123}}}; - mds_ctx.set_ls_id(share::ObLSID(1001)); // serialize const int64_t serialize_size = mds_ctx.get_serialize_size(); @@ -59,7 +58,6 @@ TEST_F(TestTabletCreateMdsCtx, start_mds_ctx) ret = ctx.deserialize(buffer, serialize_size, pos); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(pos, serialize_size); - ASSERT_EQ(ctx.ls_id_, mds_ctx.ls_id_); ASSERT_EQ(ctx.writer_.writer_id_, mds_ctx.writer_.writer_id_); delete [] buffer; diff --git a/unittest/storage/test_tablet_helper.h b/unittest/storage/test_tablet_helper.h index 7cb46dc75..fd662761d 100644 --- a/unittest/storage/test_tablet_helper.h +++ b/unittest/storage/test_tablet_helper.h @@ -45,7 +45,7 @@ class TestTabletHelper const share::schema::ObTableSchema &table_schema, ObTabletCreateSSTableParam ¶m); static int create_tablet( - ObLSHandle &ls_handle, + ObLS *ls, const common::ObTabletID &tablet_id, const share::schema::ObTableSchema &table_schema, common::ObArenaAllocator &allocator, @@ -53,14 +53,14 @@ class TestTabletHelper const share::SCN &create_commit_scn, ObTabletHandle &handle); static int create_tablet( - ObLSHandle &ls_handle, + ObLS *ls, const common::ObTabletID &tablet_id, const share::schema::ObTableSchema &table_schema, common::ObArenaAllocator &allocator, const ObTabletStatus::Status tablet_status = ObTabletStatus::NORMAL, const share::SCN &create_commit_scn = share::SCN::min_scn()); static int remove_tablet( - const ObLSHandle &ls_handle, + ObLS *ls, const common::ObTabletID &tablet_id); }; @@ -112,7 +112,7 @@ inline void TestTabletHelper::prepare_sstable_param( } inline int TestTabletHelper::create_tablet( - ObLSHandle &ls_handle, + ObLS *ls, const common::ObTabletID &tablet_id, const share::schema::ObTableSchema &table_schema, common::ObArenaAllocator &allocator, @@ -122,7 +122,7 @@ inline int TestTabletHelper::create_tablet( { int ret = OB_SUCCESS; ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); - ObLSTabletService *ls_tablet_svr = ls_handle.get_ls()->get_tablet_svr(); + ObLSTabletService *ls_tablet_svr = ls->get_tablet_svr(); const lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL; ObArenaAllocator schema_allocator; ObCreateTabletSchema create_tablet_schema; @@ -137,27 +137,26 @@ inline int TestTabletHelper::create_tablet( STORAGE_LOG(WARN, "fill column checksum failed", K(ret), K(param)); } else { const int64_t snapshot_version = 1; - const share::ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - ObFreezer *freezer = ls_handle.get_ls()->get_freezer(); + ObFreezer *freezer = ls->get_freezer(); ObTabletHandle tablet_handle; - const ObTabletMapKey key(ls_id, tablet_id); + const ObTabletMapKey key(tablet_id); const bool need_create_empty_major_sstable = !(create_tablet_schema.is_user_hidden_table() || (create_tablet_schema.is_index_table() && !create_tablet_schema.can_read_index())); - if (OB_FAIL(t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, tablet_handle))) { - STORAGE_LOG(WARN, "t3m acquire tablet failed", K(ret), K(ls_id), K(tablet_id)); + if (OB_FAIL(t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls, tablet_handle))) { + STORAGE_LOG(WARN, "t3m acquire tablet failed", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->init_for_first_time_creation( *tablet_handle.get_allocator(), - ls_id, tablet_id, tablet_id, share::SCN::base_scn(), + tablet_id, tablet_id, share::SCN::base_scn(), snapshot_version, create_tablet_schema, need_create_empty_major_sstable, share::SCN::invalid_scn()/*clog_checkpoint_scn*/, share::SCN::invalid_scn()/*mds_checkpoint_scn*/, false/*is_split_dest_tablet*/, ObTabletID()/*split_src_tablet_id*/, false/*micro_index_clustered*/, freezer))){ - STORAGE_LOG(WARN, "failed to init tablet", K(ret), K(ls_id), K(tablet_id)); + STORAGE_LOG(WARN, "failed to init tablet", K(ret), K(tablet_id)); } else if (ObTabletStatus::Status::MAX != tablet_status) { ObTabletCreateDeleteMdsUserData data; data.tablet_status_ = tablet_status; data.create_commit_scn_ = create_commit_scn; data.create_commit_version_ = create_commit_scn.get_val_for_tx(); - if (tablet_status == ObTabletStatus::Status::DELETED || tablet_status == ObTabletStatus::Status::RESERVED_6) { + if (tablet_status == ObTabletStatus::Status::DELETED) { data.delete_commit_scn_ = share::SCN::plus(create_commit_scn, 100); data.delete_commit_version_ = data.delete_commit_scn_.get_val_for_tx(); } @@ -172,7 +171,7 @@ inline int TestTabletHelper::create_tablet( if (FAILEDx(tablet_handle.get_obj()->get_updating_tablet_pointer_param(param))) { STORAGE_LOG(WARN, "fail to get updating tablet pointer parameters", K(ret), K(tablet_handle)); } else if (OB_FAIL(t3m->compare_and_swap_tablet(key, tablet_handle, tablet_handle, param))) { - STORAGE_LOG(WARN, "failed to compare and swap tablet", K(ret), K(ls_id), K(tablet_id), K(param)); + STORAGE_LOG(WARN, "failed to compare and swap tablet", K(ret), K(tablet_id), K(param)); } else if (OB_FAIL(ls_tablet_svr->tablet_id_set_.set(tablet_id))){ STORAGE_LOG(WARN, "set tablet id failed", K(ret), K(tablet_id)); } else { @@ -183,7 +182,7 @@ inline int TestTabletHelper::create_tablet( } inline int TestTabletHelper::create_tablet( - ObLSHandle &ls_handle, + ObLS *ls, const common::ObTabletID &tablet_id, const share::schema::ObTableSchema &table_schema, common::ObArenaAllocator &allocator, @@ -193,21 +192,19 @@ inline int TestTabletHelper::create_tablet( int ret = OB_SUCCESS; ObTabletHandle tablet_handle; - if (OB_FAIL(create_tablet(ls_handle, tablet_id, table_schema, allocator, tablet_status, create_commit_scn, tablet_handle))) { + if (OB_FAIL(create_tablet(ls, tablet_id, table_schema, allocator, tablet_status, create_commit_scn, tablet_handle))) { STORAGE_LOG(WARN, "failed to create tablet", K(ret), K(tablet_id)); } return ret; } -inline int TestTabletHelper::remove_tablet(const ObLSHandle &ls_handle, const ObTabletID &tablet_id) +inline int TestTabletHelper::remove_tablet(ObLS *ls, const ObTabletID &tablet_id) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; - ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle); - const share::ObLSID &ls_id = ls_handle.get_ls()->get_ls_id(); - + ls->get_tablet(tablet_id, tablet_handle); ObTenantMetaMemMgr *t3m = share::g_mp->tenant_meta_mem_mgr(); ObTabletCreateDeleteMdsUserData data; ObTabletStatus status(ObTabletStatus::DELETING); @@ -221,8 +218,8 @@ inline int TestTabletHelper::remove_tablet(const ObLSHandle &ls_handle, const Ob if (OB_FAIL(tablet_handle.get_obj()->get_updating_tablet_pointer_param(param))) { STORAGE_LOG(WARN, "fail to get updating tablet pointer parameters", K(ret), K(tablet_handle)); } else if(OB_FAIL(t3m->compare_and_swap_tablet( - ObTabletMapKey(ls_id, tablet_id), tablet_handle, tablet_handle, param))) { - STORAGE_LOG(WARN, "failed to compare and swap tablet", K(ret), K(ls_id), K(tablet_id), K(disk_addr)); + ObTabletMapKey(tablet_id), tablet_handle, tablet_handle, param))) { + STORAGE_LOG(WARN, "failed to compare and swap tablet", K(ret), K(tablet_id), K(disk_addr)); } } diff --git a/unittest/storage/test_tablet_pointer_map.cpp b/unittest/storage/test_tablet_pointer_map.cpp index 6ab260fc0..2026137e9 100644 --- a/unittest/storage/test_tablet_pointer_map.cpp +++ b/unittest/storage/test_tablet_pointer_map.cpp @@ -60,12 +60,14 @@ class TestMetaPointerMap : public ::testing::Test private: static constexpr uint64_t TEST_TENANT_ID = OB_SERVER_TENANT_ID; + ObLS fake_ls_; ObTabletPointerMap tablet_map_; common::ObArenaAllocator allocator_; }; TestMetaPointerMap::TestMetaPointerMap() - : tablet_map_() + : fake_ls_(), + tablet_map_() { } @@ -98,10 +100,7 @@ void TestMetaPointerMap::TearDown() void TestMetaPointerMap::FakeLs(ObLS &ls) { - ls.ls_meta_.ls_id_.id_ = 1001; - ls.ls_meta_.migration_status_ = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; ls.ls_meta_.restore_status_ = ObRestoreStatus::Status::NONE; - ls.ls_meta_.rebuild_seq_ = 0; } class CalculateSize final @@ -135,11 +134,10 @@ int CalculateSize::operator()(common::hash::HashMapPairinit(&fake_ls); + ObLSTabletService *tablet_svr = fake_ls_.get_tablet_svr(); + int ret = tablet_svr->init(&fake_ls_); ASSERT_EQ(common::OB_SUCCESS, ret); ObDDLKvMgrHandle ddl_kv_mgr_hdl; @@ -151,13 +149,11 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_handle) ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl); ASSERT_EQ(common::OB_SUCCESS, ret); - ObLSHandle ls_handle; - ls_handle.ls_ = &fake_ls; - ObTabletPointer tablet_ptr(ls_handle, memtable_mgr_hdl); + ObTabletPointer tablet_ptr(&fake_ls_, memtable_mgr_hdl); ObMetaDiskAddr phy_addr; phy_addr.set_none_addr(); tablet_ptr.set_addr_with_reset_obj(phy_addr); - const ObTabletMapKey key(ObLSID(1001), ObTabletID(101)); + const ObTabletMapKey key(ObTabletID(101)); ret = tablet_map_.set(key, tablet_ptr); ASSERT_EQ(common::OB_SUCCESS, ret); @@ -199,10 +195,9 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_handle) TEST_F(TestMetaPointerMap, test_meta_pointer_map) { - ObLS fake_ls; - FakeLs(fake_ls); - ObLSTabletService *tablet_svr = fake_ls.get_tablet_svr(); - int ret = tablet_svr->init(&fake_ls); + FakeLs(fake_ls_); + ObLSTabletService *tablet_svr = fake_ls_.get_tablet_svr(); + int ret = tablet_svr->init(&fake_ls_); ASSERT_EQ(common::OB_SUCCESS, ret); ObDDLKvMgrHandle ddl_kv_mgr_hdl; @@ -214,13 +209,11 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_map) ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl); ASSERT_EQ(common::OB_SUCCESS, ret); - ObLSHandle ls_handle; - ls_handle.ls_ = &fake_ls; - ObTabletPointer tablet_ptr(ls_handle, memtable_mgr_hdl); + ObTabletPointer tablet_ptr(&fake_ls_, memtable_mgr_hdl); ObMetaDiskAddr phy_addr; phy_addr.set_none_addr(); tablet_ptr.set_addr_with_reset_obj(phy_addr); - const ObTabletMapKey key(ObLSID(1001), ObTabletID(101)); + const ObTabletMapKey key(ObTabletID(101)); ret = tablet_map_.set(key, tablet_ptr); ASSERT_EQ(common::OB_SUCCESS, ret); @@ -297,11 +290,10 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_map) TEST_F(TestMetaPointerMap, test_erase_and_load_concurrency) { - ObLS fake_ls; - FakeLs(fake_ls); + FakeLs(fake_ls_); - ObLSTabletService *tablet_svr = fake_ls.get_tablet_svr(); - int ret = tablet_svr->init(&fake_ls); + ObLSTabletService *tablet_svr = fake_ls_.get_tablet_svr(); + int ret = tablet_svr->init(&fake_ls_); ASSERT_EQ(common::OB_SUCCESS, ret); ObDDLKvMgrHandle ddl_kv_mgr_hdl; @@ -313,13 +305,11 @@ TEST_F(TestMetaPointerMap, test_erase_and_load_concurrency) ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl); ASSERT_EQ(common::OB_SUCCESS, ret); - ObLSHandle ls_handle; - ls_handle.ls_ = &fake_ls; - ObTabletPointer tablet_ptr(ls_handle, memtable_mgr_hdl); + ObTabletPointer tablet_ptr(&fake_ls_, memtable_mgr_hdl); ObMetaDiskAddr phy_addr; phy_addr.set_none_addr(); tablet_ptr.set_addr_with_reset_obj(phy_addr); - const ObTabletMapKey key(ObLSID(1001), ObTabletID(101)); + const ObTabletMapKey key(ObTabletID(101)); ret = tablet_map_.set(key, tablet_ptr); ASSERT_EQ(common::OB_SUCCESS, ret); diff --git a/unittest/storage/test_tenant_tablet_stat_mgr.cpp b/unittest/storage/test_tenant_tablet_stat_mgr.cpp index 19ccdadcd..a299a3283 100644 --- a/unittest/storage/test_tenant_tablet_stat_mgr.cpp +++ b/unittest/storage/test_tenant_tablet_stat_mgr.cpp @@ -100,7 +100,6 @@ void TestTenantTabletStatMgr::batch_report_stat(int64_t report_num) std::thread *threads = new std::thread[report_num]; for (int64_t i = 0; i < report_num; ++i) { ObTabletStat curr_stat; - curr_stat.ls_id_ = 1; curr_stat.tablet_id_ = 300001 + i; curr_stat.query_cnt_ = 100 * (i + 1); curr_stat.scan_physical_row_cnt_ = 10000 + i; @@ -120,7 +119,6 @@ namespace unittest TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_bucket) { ObTabletStat tablet_stat; - tablet_stat.ls_id_ = 1; tablet_stat.tablet_id_ = 1; tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; @@ -184,7 +182,6 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_bucket) TEST_F(TestTenantTabletStatMgr, basic_tablet_stream) { ObTabletStat tablet_stat; - tablet_stat.ls_id_ = 1; tablet_stat.tablet_id_ = 200123; tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 1000000; @@ -226,7 +223,6 @@ TEST_F(TestTenantTabletStatMgr, get_all_tablet_stat) int ret = OB_SUCCESS; ObTabletStat tablet_stat; - tablet_stat.ls_id_ = 1; tablet_stat.tablet_id_ = 1; tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; @@ -362,7 +358,6 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_mgr) ASSERT_TRUE(NULL != stat_mgr); ObTabletStat tablet_stat; - tablet_stat.ls_id_ = 1; tablet_stat.tablet_id_ = 200123; tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100000; @@ -374,17 +369,16 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_mgr) stat_mgr_->process_stats(); ObTabletStat res; - share::ObLSID ls_id(1); common::ObTabletID tablet_id(200123); storage::ObTabletStat unused_tablet_stat; share::schema::ObTableModeFlag unused_mode; - ret = stat_mgr_->get_latest_tablet_stat(ls_id, tablet_id, res, unused_tablet_stat, unused_mode); + ret = stat_mgr_->get_latest_tablet_stat(tablet_id, res, unused_tablet_stat, unused_mode); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(100, res.query_cnt_); ASSERT_EQ(1, stat_mgr_->stream_map_.size()); - ASSERT_EQ(OB_SUCCESS, stat_mgr_->clear_tablet_stat(ls_id, tablet_id)); - const ObTabletStatKey key(ls_id, tablet_id); + ASSERT_EQ(OB_SUCCESS, stat_mgr_->clear_tablet_stat(tablet_id)); + const ObTabletStatKey key(tablet_id); ObTabletStreamNode *stream_node = nullptr; ASSERT_TRUE(key.is_valid()); ASSERT_EQ(OB_SUCCESS, stat_mgr_->stream_map_.get_refactored(key, stream_node)); @@ -398,7 +392,7 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_mgr) } stat_mgr_->process_stats(); storage::ObTabletStat total_tablet_stat; - ret = stat_mgr_->get_latest_tablet_stat(ls_id, tablet_id, res, total_tablet_stat, unused_mode); + ret = stat_mgr_->get_latest_tablet_stat(tablet_id, res, total_tablet_stat, unused_mode); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(100 * 1000, total_tablet_stat.query_cnt_); ASSERT_EQ(100000 * 1000, total_tablet_stat.scan_logical_row_cnt_); @@ -434,13 +428,12 @@ TEST_F(TestTenantTabletStatMgr, bacth_clear_tablet_stat) batch_report_stat(report_num); stat_mgr_->process_stats(); - ObLSID ls_id(1); ObSEArray tablet_ids; for (int64_t i = 0; i < report_num; i++) { ASSERT_EQ(OB_SUCCESS, tablet_ids.push_back(ObTabletID(300001 + i))); } ASSERT_EQ(100, stat_mgr_->stream_map_.size()); - ASSERT_EQ(OB_SUCCESS, stat_mgr->batch_clear_tablet_stat(ls_id, tablet_ids)); + ASSERT_EQ(OB_SUCCESS, stat_mgr->batch_clear_tablet_stat(tablet_ids)); ObTenantTabletStatMgr::TabletStreamMap::iterator iter = stat_mgr_->stream_map_.begin(); for ( ; iter != stat_mgr_->stream_map_.end(); ++iter) { ASSERT_TRUE(iter->second->stream_.key_.is_valid()); diff --git a/unittest/storage/transaction/tablelock/test_ob_replace_lock_request.cpp b/unittest/storage/transaction/tablelock/test_ob_replace_lock_request.cpp index d4e79c308..6ffef0a07 100644 --- a/unittest/storage/transaction/tablelock/test_ob_replace_lock_request.cpp +++ b/unittest/storage/transaction/tablelock/test_ob_replace_lock_request.cpp @@ -135,7 +135,7 @@ bool unlock_req_is_equal(const ObLockRequest &req1, const ObLockRequest &req2) case transaction::tablelock::ObLockRequest::ObLockMsgType::UNLOCK_ALONE_TABLET_REQ: { const ObUnLockAloneTabletRequest &lock_req1 = static_cast(req1); const ObUnLockAloneTabletRequest &lock_req2 = static_cast(req2); - is_equal = lock_req1.table_id_ == lock_req2.table_id_ && lock_req1.ls_id_ == lock_req2.ls_id_ + is_equal = lock_req1.table_id_ == lock_req2.table_id_ && list_is_equal(lock_req1.tablet_ids_, lock_req1.tablet_ids_); TABLELOCK_LOG(INFO, "compare unlock request", K(lock_req1), K(lock_req2), K(is_equal)); break; @@ -194,7 +194,6 @@ TEST(ObReplaceLockRequest, test_replace_alone_tablet) unlock_req.op_type_ = OUT_TRANS_UNLOCK; unlock_req.timeout_us_ = 1000; unlock_req.table_id_ = 998; - unlock_req.ls_id_ = 1; unlock_req.tablet_ids_.push_back(ObTabletID(123)); unlock_req.tablet_ids_.push_back(ObTabletID(456)); unlock_req.tablet_ids_.push_back(ObTabletID(789)); diff --git a/unittest/storage/transaction/test_ob_trans_factory.cpp b/unittest/storage/transaction/test_ob_trans_factory.cpp index 70849514c..b09f58de0 100644 --- a/unittest/storage/transaction/test_ob_trans_factory.cpp +++ b/unittest/storage/transaction/test_ob_trans_factory.cpp @@ -18,7 +18,6 @@ #include "storage/tx/ob_trans_log.h" #include "storage/tx/ob_trans_ctx.h" #include "storage/tx/ob_tx_log_cb_define.h" -#include "src/share/location_cache/ob_location_service.h" #include namespace oceanbase diff --git a/unittest/storage/tx/CMakeLists.txt b/unittest/storage/tx/CMakeLists.txt index 0a597a8f4..63196861f 100644 --- a/unittest/storage/tx/CMakeLists.txt +++ b/unittest/storage/tx/CMakeLists.txt @@ -1,42 +1,17 @@ -# add_library(mock_2pc_ctx SHARED -# ob_mailbox.h -# ob_mailbox.cpp -# ob_mock_2pc_ctx.h -# ob_mock_2pc_ctx.cpp) -# -# target_link_libraries(mock_2pc_ctx PUBLIC oceanbase mockcontainer) - -# add_library(mock_tx_ctx SHARED -# ob_mailbox.h -# ob_mailbox.cpp -# ob_mock_tx_ctx.h -# ob_mock_tx_ctx.cpp) -# add_library(mock_tx_log_adapter SHARED ob_mock_tx_log_adapter.cpp ob_mock_tx_log_adapter.h) -# target_link_libraries(mock_tx_ctx PUBLIC oceanbase mockcontainer) target_link_libraries(mock_tx_log_adapter PUBLIC oceanbase mockcontainer) -# function(c2pc_unittest case) -# ob_unittest(${ARGV}) -# target_link_libraries(${case} PUBLIC mock_2pc_ctx) -# endfunction() - function(tx_unittest case) ob_unittest(${ARGV}) target_link_libraries(${case} PUBLIC mock_tx_log_adapter) endfunction() -# c2pc_unittest(test_simple_tx_commit) -# c2pc_unittest(test_dup_msg_tx_commit) -# c2pc_unittest(test_cycle_commit) # tx_unittest(test_simple_tx_ctx) -tx_unittest(test_ls_log_writer) tx_unittest(test_ob_trans_hashmap) storage_unittest(test_ob_tx_log) storage_unittest(test_ob_timestamp_service) -storage_unittest(test_ob_tx_msg) storage_unittest(test_undo_action) storage_unittest(test_ob_id_meta) # storage_unittest(test_ob_standby_read) diff --git a/unittest/storage/tx/it/test_register_mds.cpp b/unittest/storage/tx/it/test_register_mds.cpp index acc8bc51e..138385e2f 100644 --- a/unittest/storage/tx/it/test_register_mds.cpp +++ b/unittest/storage/tx/it/test_register_mds.cpp @@ -97,20 +97,16 @@ class ReplayLogEntryFunctor }; -OB_NOINLINE int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, - SCN &snapshot, - const bool is_read_only, - ObRole &role) +OB_NOINLINE int ObTransService::acquire_local_snapshot_(SCN &snapshot) { int ret = OB_SUCCESS; snapshot = tx_version_mgr_.get_max_commit_ts(false); - role = LEADER; return ret; } bool NOTIFY_MDS_ERRSIM = false; -OB_NOINLINE int ObPartTransCtx::errsim_notify_mds_() +OB_NOINLINE int ObTxCtx::errsim_notify_mds_() { int ret = OB_SUCCESS; @@ -133,7 +129,6 @@ class ObTestRegisterMDS : public ::testing::Test ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); const uint64_t tv = ObTimeUtility::current_time(); ObCurTraceId::set(&tv); - GCONF._ob_trans_rpc_timeout = 500; ObClockGenerator::init(); const testing::TestInfo *const test_info = testing::UnitTest::GetInstance()->current_test_info(); @@ -158,26 +153,23 @@ class ObTestRegisterMDS : public ::testing::Test #define GC_MDS_RETAIN_CTX(node) \ { \ - ObLSTxCtxMgr *ls_tx_ctx_mgr1 = nullptr; \ - ASSERT_EQ(OB_SUCCESS, node->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(node->ls_id_, ls_tx_ctx_mgr1)); \ - ls_tx_ctx_mgr1->get_retain_ctx_mgr().try_gc_retain_ctx(&node->fake_ls_); \ - ASSERT_EQ(OB_SUCCESS, node->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); \ + ObLSTxCtxMgr &tx_ctx_mgr = node->txs_.tx_ctx_mgr_.get_tx_ctx_manager(); \ + tx_ctx_mgr.get_retain_ctx_mgr().try_gc_retain_ctx(&node->fake_ls_); \ } TEST_F(ObTestRegisterMDS, basic) { - START_TWO_TX_NODE_WITH_LSID(n1, n2, 2001); + START_TX_REPLAY_PAIR(n1, n2); PREPARE_TX(n1, tx); PREPARE_TX_PARAM(tx_param); const char *mds_str = "register mds basic"; ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - mds_str, strlen(mds_str), 0)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + mds_str, strlen(mds_str))); n2->wait_all_redolog_applied(); ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - n2->set_as_follower_replica(*n1); ReplayLogEntryFunctor functor(n2); ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); @@ -190,7 +182,7 @@ TEST_F(ObTestRegisterMDS, basic) TEST_F(ObTestRegisterMDS, basic_big_mds) { - START_TWO_TX_NODE_WITH_LSID(n1, n2, 2003); + START_TX_REPLAY_PAIR(n1, n2); PREPARE_TX(n1, tx); PREPARE_TX_PARAM(tx_param); tx_param.timeout_us_ = 1000 * 1000 * 1000; @@ -200,13 +192,12 @@ TEST_F(ObTestRegisterMDS, basic_big_mds) memset(mds_str, 'M', sizeof(char) * char_count); ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - mds_str, char_count, 0)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + mds_str, char_count)); n1->wait_all_redolog_applied(); ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(100 * 1000))); - n2->set_as_follower_replica(*n1); ReplayLogEntryFunctor functor(n2); ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); @@ -219,7 +210,7 @@ TEST_F(ObTestRegisterMDS, basic_big_mds) TEST_F(ObTestRegisterMDS, merge_mds_log) { - START_TWO_TX_NODE_WITH_LSID(n1, n2, 2010); + START_TX_REPLAY_PAIR(n1, n2); PREPARE_TX(n1, tx); PREPARE_TX_PARAM(tx_param); tx_param.timeout_us_ = 1000 * 1000 * 1000; @@ -232,30 +223,30 @@ TEST_F(ObTestRegisterMDS, merge_mds_log) memset(large_mds_str, 'M', sizeof(char) * LARGE_CHAR_CNT); ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); n1->wait_all_redolog_applied(); - ObPartTransCtx *ctx = nullptr; - ASSERT_EQ(OB_SUCCESS, n1->get_tx_ctx(n1->ls_id_, tx.tx_id_, ctx)); + ObTxCtx *ctx = nullptr; + ASSERT_EQ(OB_SUCCESS, n1->get_tx_ctx(tx.tx_id_, ctx)); ASSERT_EQ(OB_SUCCESS, ctx->exec_info_.redo_lsns_.count() == 2); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - large_mds_str, LARGE_CHAR_CNT, 0)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + large_mds_str, LARGE_CHAR_CNT)); n1->wait_all_redolog_applied(); ASSERT_EQ(OB_SUCCESS, ctx->exec_info_.redo_lsns_.count() == (2 + 1 + LARGE_CHAR_CNT / common::OB_MAX_LOG_ALLOWED_SIZE)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - normal_mds_str, NORMAL_CHAR_CNT, 0)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); + ASSERT_EQ(OB_SUCCESS, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + normal_mds_str, NORMAL_CHAR_CNT)); n1->wait_all_redolog_applied(); ASSERT_EQ(OB_SUCCESS, ctx->exec_info_.redo_lsns_.count() == (2 + 1 + LARGE_CHAR_CNT / common::OB_MAX_LOG_ALLOWED_SIZE + 1)); @@ -264,7 +255,6 @@ TEST_F(ObTestRegisterMDS, merge_mds_log) ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(100 * 1000))); - n2->set_as_follower_replica(*n1); ReplayLogEntryFunctor functor(n2); ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); @@ -278,7 +268,7 @@ TEST_F(ObTestRegisterMDS, merge_mds_log) TEST_F(ObTestRegisterMDS, notify_mds_error) { - START_TWO_TX_NODE_WITH_LSID(n1, n2, 2005); + START_TX_REPLAY_PAIR(n1, n2); PREPARE_TX(n1, tx); PREPARE_TX_PARAM(tx_param); const char *mds_str = "register mds basic"; @@ -286,14 +276,13 @@ TEST_F(ObTestRegisterMDS, notify_mds_error) ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); NOTIFY_MDS_ERRSIM = true; - ASSERT_EQ(OB_ERR_UNEXPECTED, n1->txs_.register_mds_into_tx(tx, n1->ls_id_, ObTxDataSourceType::DDL_TRANS, - mds_str, strlen(mds_str), 0)); + ASSERT_EQ(OB_ERR_UNEXPECTED, n1->txs_.register_mds_into_tx(tx, ObTxDataSourceType::DDL_TRANS, + mds_str, strlen(mds_str))); NOTIFY_MDS_ERRSIM = false; n2->wait_all_redolog_applied(); ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - n2->set_as_follower_replica(*n1); ReplayLogEntryFunctor functor(n2); ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); diff --git a/unittest/storage/tx/it/test_tx.cpp b/unittest/storage/tx/it/test_tx.cpp index 51467fc70..d24d63bd3 100644 --- a/unittest/storage/tx/it/test_tx.cpp +++ b/unittest/storage/tx/it/test_tx.cpp @@ -90,15 +90,11 @@ class ObTestTx : public ::testing::Test ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); ObAddr ip_port(ObAddr::VER::IPV4, "119.119.0.1",2023); ObCurTraceId::init(ip_port); - GCONF._ob_trans_rpc_timeout = 500; ObClockGenerator::init(); const testing::TestInfo* const test_info = testing::UnitTest::GetInstance()->current_test_info(); // publish the fake module set as the process-global provider (see tx_node.h). publish_test_module_provider(G_TEST_MODULE_PROVIDER, MTL_MEM_ALLOC_MGR); - // reset the static fake location adapter so cross-test ls_id residue can't - // leak into this case (many cases reset themselves, some don't -> OB_HASH_EXIST). - ObTxNode::reset_localtion_adapter(); auto test_name = test_info->name(); _TRANS_LOG(INFO, ">>>> starting test : %s", test_name); } @@ -114,90 +110,13 @@ class ObTestTx : public ::testing::Test MsgBus bus_; }; -TEST_F(ObTestTx, basic) -{ - GCONF._ob_trans_rpc_timeout = 50; - ObTxNode::reset_localtion_adapter(); - - START_TWO_TX_NODE(n1, n2); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - GET_READ_SNAPSHOT(n1, tx, tx_param, snapshot); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp0); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp1); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 101, 113)); - int64_t val1 = 0, val2 = 0; - ASSERT_EQ(OB_SUCCESS, n1->read(tx, 100, val1)); - ASSERT_EQ(OB_SUCCESS, n2->read(tx, 101, val2)); - ASSERT_EQ(112, val1); - ASSERT_EQ(113, val2); - // rollback to savepoint - ROLLBACK_TO_IMPLICIT_SAVEPOINT(n1, tx, sp1, 1000 * 1000); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, n1->read(tx, 100, val1)); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, n2->read(tx, 101, val2)); - // write after rollback - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 114)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 101, 115)); - // read the write after rollback - ASSERT_EQ(OB_SUCCESS, n1->read(tx, 100, val1)); - ASSERT_EQ(OB_SUCCESS, n2->read(tx, 101, val2)); - ASSERT_EQ(114, val1); - ASSERT_EQ(115, val2); - COMMIT_TX(n1, tx, 500 * 1000); -} - -// TEST_F(ObTestTx, tx_2pc_blocking_and_get_gts_callback_concurrent_problem) -// { -// GCONF._ob_trans_rpc_timeout = 50; -// ObTxNode::reset_localtion_adapter(); -// -// START_ONE_TX_NODE(n1); -// PREPARE_TX(n1, tx); -// PREPARE_TX_PARAM(tx_param); -// GET_READ_SNAPSHOT(n1, tx, tx_param, snapshot); -// ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// -// ObPartTransCtx *part_ctx = NULL; -// ObLSID ls_id(1); -// ASSERT_EQ(OB_SUCCESS, n1->get_tx_ctx(ls_id, tx.tx_id_, part_ctx)); -// -// // mock gts waiting -// part_ctx->sub_state_.set_gts_waiting(); -// -// -// ObMonotonicTs stc(99); -// ObMonotonicTs srr(100); -// ObMonotonicTs rgt(100); -// share::SCN scn; -// scn.convert_for_gts(100); -// part_ctx->stc_ = stc; -// part_ctx->part_trans_action_ = ObPartTransAction::COMMIT; -// EXPECT_EQ(OB_SUCCESS, part_ctx->get_gts_callback(srr, scn, rgt)); -// EXPECT_EQ(true, part_ctx->ctx_tx_data_.get_commit_version() >= scn); -// ObLSID dst_ls_id(2); -// share::SCN start_scn; -// share::SCN end_scn; -// start_scn.convert_for_gts(888); -// end_scn.convert_for_gts(1000); -// part_ctx->ctx_tx_data_.set_start_log_ts(start_scn); -// n1->get_ts_mgr_().repair_get_gts_error(); -// } - TEST_F(ObTestTx, start_trans_expired) { - GCONF._ob_trans_rpc_timeout = 50; - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); + auto n1 = new ObTxNode(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); DEFER(delete(n1)); - DEFER(delete(n2)); ASSERT_EQ(OB_SUCCESS, n1->start()); - ASSERT_EQ(OB_SUCCESS, n2->start()); auto guard = n1->get_tx_guard(); ObTxDesc &tx = guard.get_tx_desc(); ObTxParam tx_param; @@ -213,2088 +132,45 @@ TEST_F(ObTestTx, start_trans_expired) ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); } -TEST_F(ObTestTx, rollback_savepoint_with_msg_lost) -{ - ObTxNode::reset_localtion_adapter(); - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - ASSERT_EQ(OB_SUCCESS, n2->start()); - ObTxDescGuard guard = n1->get_tx_guard(); - ObTxDesc &tx = guard.get_tx_desc(); - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, tx_param.isolation_, n1->ts_after_ms(5), snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - ObTxSEQ sp2; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp2)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 101, 113)); - // inject link failure between scheduler to participant 2 - ASSERT_EQ(OB_SUCCESS, bus_.inject_link_failure(n1->addr_, n2->addr_)); - // rollback to savepoint should hang, because of rollback msg can't be delivered - auto rollback_sp = [&] { - return n1->rollback_to_implicit_savepoint(tx, sp2, n1->ts_after_ms(5000), nullptr); - }; - auto async = test::make_async(rollback_sp); - async.wait_started(); - ASSERT_FALSE(async.is_evaled()); - // interrupt - ASSERT_EQ(OB_SUCCESS, n1->interrupt(tx, 101)); - {int i = 2000; - while(!async.is_evaled() && i-- > 0) { - usleep(1000); - } - ASSERT_TRUE(i > 0) << "interrupt savepoint rollback"; - } - ASSERT_EQ(OB_ERR_INTERRUPTED, async.get()); - ASSERT_EQ(OB_SUCCESS, bus_.repair_link_failure(n1->addr_, n2->addr_)); - ASSERT_EQ(OB_SUCCESS, n1->rollback_tx(tx)); - ASSERT_EQ(ObTxDesc::State::ROLLED_BACK, tx.state_); - // wait part_ctx gc - ObPartTransCtx *part_ctx = NULL; - ObLSID ls_id(2); - ASSERT_EQ(OB_SUCCESS, n2->get_tx_ctx(ls_id, tx.tx_id_, part_ctx)); - // release tx, then part_ctx can detect txn was terminated - ASSERT_EQ(OB_SUCCESS, guard.release()); - part_ctx->last_ask_scheduler_status_ts_ = 0; // ensure check_scheduler_status will not be skipped - ASSERT_EQ(OB_SUCCESS, part_ctx->check_scheduler_status()); - int i = 0; - while(!part_ctx->is_exiting_ && i++ < 100) { - usleep(500); - } - ASSERT_EQ(part_ctx->is_exiting_, true); - ASSERT_EQ(OB_SUCCESS, n2->revert_tx_ctx(part_ctx)); -} - -TEST_F(ObTestTx, rollback_savepoint_timeout) -{ - START_TWO_TX_NODE(n1, n2); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp1); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, 100, 111)); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp2); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, 101, 112)); - INJECT_LINK_FAILURE(n1, n2); - ASYNC_DO(async_op, ROLLBACK_TO_IMPLICIT_SAVEPOINT(n1, tx, sp2, 2000 * 1000)); - ASYNC_WAIT(async_op, 3000 * 1000, wait_ret); - REPAIR_LINK_FAILURE(n1, n2); - ROLLBACK_TX(n1, tx); -} - -TEST_F(ObTestTx, rollback_savepoint_with_uncertain_participants) -{ - START_ONE_TX_NODE(n1); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp); - ASSERT_TRUE(sp.is_valid()); - share::ObLSArray uncertain_parts; - ASSERT_EQ(OB_SUCCESS, uncertain_parts.push_back(share::ObLSID(1001))); - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_us(100000), &uncertain_parts)); - ASSERT_EQ(ObTxDesc::State::IDLE, tx.state_); - ASSERT_EQ(0, tx.parts_.count()); -} - -TEST_F(ObTestTx, rollback_savepoint_with_keep_tx) -{ - START_ONE_TX_NODE(n1); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - GET_READ_SNAPSHOT(n1, tx, tx_param, snapshot); - - { - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp); - ASSERT_TRUE(sp.is_valid()); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 200)); - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_ms(5), nullptr, ObTxCleanPolicy::KEEP)); - ASSERT_EQ(ObTxDesc::State::IMPLICIT_ACTIVE, tx.state_); - ASSERT_EQ(ObTxSEQ::INVL(), tx.active_scn_); - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_ms(5), nullptr)); - } - - { - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp); - ASSERT_TRUE(sp.is_valid()); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 200)); - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_ms(5), nullptr, ObTxCleanPolicy::KEEP)); - ASSERT_EQ(ObTxDesc::State::IMPLICIT_ACTIVE, tx.state_); - ASSERT_EQ(ObTxSEQ::INVL(), tx.active_scn_); - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_ms(5), nullptr)); - } -} - -TEST_F(ObTestTx, switch_to_follower_gracefully) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - DEFER(delete(n1)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - -// tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - ObTxSEQ sp1, sp2; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_gracefully()); - - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_NOT_MASTER, n1->read(snapshot, 100, val1)); - - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_leader()); - n1->wait_all_redolog_applied(); - - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 100, val1)); - ASSERT_EQ(112, val1); - - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp2)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 101, 113)); - } - - { - int64_t val2 = 0; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 101, val2)); - ASSERT_EQ(113, val2); - } - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, switch_to_follower_gracefully_then_forcedly) -{ - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - DEFER(delete(n1)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - ObTxSEQ sp1; - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_gracefully()); - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_NOT_MASTER, n1->read(snapshot, 100, val1)); - n1->wait_all_redolog_applied(); - ls_tx_ctx_mgr->switch_to_follower_forcedly(); - ASSERT_EQ(OB_NOT_MASTER, n1->read(snapshot, 100, val1)); - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_leader()); - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 100, val1)); - ASSERT_EQ(112, val1); - - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 101, 113)); - } - - { - int64_t val2 = 0; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 101, val2)); - ASSERT_EQ(113, val2); - } - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, switch_to_follower_forcedly) -{ - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - DEFER(delete(n1)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxSEQ sp1; - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - // disable keepalive msg, because switch to follower forcedly will send keepalive msg to notify - // scheduler abort tx - n1->add_drop_msg_type(KEEPALIVE); - { - ls_tx_ctx_mgr->switch_to_follower_forcedly(); - - int64_t val1 = 0, val2 = 0; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_NOT_MASTER, n1->read(snapshot, 100, val1)); - - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_leader()); - n1->wait_all_redolog_applied(); - - ASSERT_EQ(OB_TRANS_CTX_NOT_EXIST, n1->read(snapshot, 100, val1)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_TRANS_CTX_NOT_EXIST, n1->write(tx, snapshot, 101, 113)); - } - - { - ASSERT_EQ(OB_SUCCESS, n1->abort_tx(tx, OB_TRANS_CTX_NOT_EXIST)); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, resume_leader) -{ - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - DEFER(delete(n1)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - ObTxSEQ sp1; - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_gracefully()); - - int64_t val1 = 0; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_NOT_MASTER, n1->read(snapshot, 100, val1)); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->resume_leader()); - usleep(100 * 1000); - - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 100, val1)); - ASSERT_EQ(112, val1); - - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 101, 113)); - } - { - int64_t val2 = 0; - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 101, val2)); - ASSERT_EQ(113, val2); - } - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, switch_to_follower_gracefully_in_stmt_then_resume_leader) -{ - ObTxNode::reset_localtion_adapter(); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - DEFER(delete(n1)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxSEQ sp1; - - ObStoreCtx write_store_ctx; - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write_begin(tx, snapshot, write_store_ctx)); - ASSERT_EQ(OB_SUCCESS, n1->write_one_row(write_store_ctx, 1000, 1112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ls_tx_ctx_mgr->switch_to_follower_gracefully(); - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_NOT_MASTER, n1->write_one_row(write_store_ctx, 1001, 1113)); - - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_leader()); - n1->wait_all_redolog_applied(); - - //ASSERT_EQ(OB_SUCCESS, n1->read(tx, 100, val1)); - ASSERT_EQ(OB_SUCCESS, n1->write_one_row(write_store_ctx, 1001, 1113)); - ASSERT_EQ(OB_SUCCESS, n1->write_end(write_store_ctx)); - } - - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - int64_t val2 = 0, val3 = 0; - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 1000, val2)); - ASSERT_EQ(OB_SUCCESS, n1->read(snapshot, 1001, val3)); - - //ASSERT_EQ(112, val1); - ASSERT_EQ(1112, val2); - ASSERT_EQ(1113, val3); - } - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -class ReplayLogEntryFunctor -{ -public: - ReplayLogEntryFunctor(ObTxNode* n) : n_(n) {} - - int operator()(const void *buffer, - const int64_t nbytes, - const palf::LSN &lsn, - const int64_t ts_ns) { - return n_->replay(buffer, nbytes, lsn, ts_ns); - } -private: - ObTxNode* n_; -}; - TEST_F(ObTestTx, replay_basic) { - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 500 * 1000 * 1000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - { - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - } - n1->wait_all_redolog_applied(); - int64_t retry_count = 0; - while(ls_tx_ctx_mgr->get_tx_ctx_count() > 0) - { - TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected tx ctx counts", K(ls_tx_ctx_mgr->get_ls_id()), K(ls_tx_ctx_mgr->get_tx_ctx_count())); - ls_tx_ctx_mgr->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, true); - ls_tx_ctx_mgr->get_retain_ctx_mgr().print_retain_ctx_info(ls_tx_ctx_mgr->get_ls_id()); - retry_count++; - usleep(100*1000); - if(retry_count > 10) - { - ob_abort(); - } - } - ASSERT_EQ(0, ls_tx_ctx_mgr->get_tx_ctx_count()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); - } - - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - { - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n2->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 100, val1)); - ASSERT_EQ(112, val1); - } - - { - ASSERT_EQ(OB_SUCCESS, n2->commit_tx(tx, n2->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n2->release_tx(tx)); - } - } - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -} - -TEST_F(ObTestTx, replay_then_commit) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 50 * 1000 * 1000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - -// tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr1->switch_to_follower_gracefully()); - n1->wait_all_redolog_applied(); - - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - ObTxNode::get_location_adapter_().update_localtion(n2->ls_id_, n2->addr_); - - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 100, val1)); - ASSERT_EQ(112, val1); - - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 100, 113)); - } + START_TX_REPLAY_PAIR(n1, n2); + ObLSTxCtxMgr &n1_ctx_mgr = n1->txs_.tx_ctx_mgr_.get_tx_ctx_manager(); { + PREPARE_TX(n1, tx); + PREPARE_TX_PARAM(tx_param); + ASSERT_EQ(OB_SUCCESS, n1->start_tx(tx, tx_param)); + ASSERT_EQ(OB_SUCCESS, n1->write(tx, 100, 112)); ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - n2->wait_all_redolog_applied(); - } - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -} - -void do_async_read(ObTxNode* n, ObTxReadSnapshot& snapshot, int64_t key, int64_t& val) -{ - LOG_INFO("do sync commit begin"); - ASSERT_EQ(OB_SUCCESS, n->read(snapshot, key, val)); - LOG_INFO("do sync commit end"); -} - - -void do_async_commit(ObTxNode* n, ObTxDesc& tx, int& commit_ret) -{ - LOG_INFO("do async commit begin", K(tx)); - commit_ret = n->commit_tx(tx, n->ts_after_ms(50 * 1000)); - LOG_INFO("do async commit end", K(tx), K(commit_ret)); -} - -TEST_F(ObTestTx, wait_commit_version_elapse_block) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxNode::get_ts_mgr_().set_elapse_waiting_mode(); -// tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 113)); - } - - int commit_ret = OB_SUCCESS; - std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); - usleep(100 * 1000); - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_gracefully()); - - n2->wait_all_redolog_applied(); - - ObTxNode::get_ts_mgr_().clear_elapse_waiting_mode(); - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ObTxNode::get_ts_mgr_().update_fake_gts(1); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - ObTxDesc *tx_ptr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->acquire_tx(tx_ptr2)); - ObTxDesc &tx2 = *tx_ptr2; - - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx2, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_ENTRY_NOT_EXIST, n2->read(snapshot, 100, val1)); - } - - ObTxNode::get_ts_mgr_().update_fake_gts(ls_tx_ctx_mgr2->max_replay_commit_version_.get_val_for_gts()); - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx2, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 100, val1)); - ASSERT_EQ(113, val1); - } - - { - ASSERT_EQ(OB_SUCCESS, n2->commit_tx(tx2, n2->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n2->release_tx(tx2)); - } - - ObTxNode::get_ts_mgr_().elapse_callback(); - t.join(); - - ASSERT_EQ(OB_SUCCESS, commit_ret); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, wait_commit_version_elapse_block_and_switch_to_follower_forcedly) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 1000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxNode::get_ts_mgr_().set_elapse_waiting_mode(); -// tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 113)); - } - - int commit_ret = OB_SUCCESS; - std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); - usleep(100 * 1000); - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_forcedly()); - - n2->wait_all_redolog_applied(); - - ObTxNode::get_ts_mgr_().clear_elapse_waiting_mode(); - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ObTxNode::get_ts_mgr_().update_fake_gts(1); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - ObTxDesc *tx_ptr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->acquire_tx(tx_ptr2)); - ObTxDesc &tx2 = *tx_ptr2; - - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx2, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_ENTRY_NOT_EXIST, n2->read(snapshot, 100, val1)); - } - - ObTxNode::get_ts_mgr_().update_fake_gts(ls_tx_ctx_mgr2->max_replay_commit_version_.get_val_for_gts()); - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx2, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 100, val1)); - ASSERT_EQ(113, val1); - } - - { - ASSERT_EQ(OB_SUCCESS, n2->commit_tx(tx2, n2->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n2->release_tx(tx2)); } - - ObTxNode::get_ts_mgr_().elapse_callback(); - t.join(); - - ASSERT_EQ(OB_SUCCESS, commit_ret); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, get_gts_block_and_switch_to_follower_gracefully) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 10 * 1000 * 1000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxNode::get_ts_mgr_().set_get_gts_waiting_mode(); - // tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - int commit_ret = OB_SUCCESS; - std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); - usleep(100 * 1000); - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_gracefully()); n1->wait_all_redolog_applied(); + ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); + ASSERT_EQ(0, n1_ctx_mgr.get_tx_ctx_count()); - ObTxNode::get_ts_mgr_().clear_get_gts_waiting_mode(); - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - - n2->wait_all_redolog_applied(); - ObTxNode::get_location_adapter_().update_localtion(n2->ls_id_, n2->addr_); - - ObTxNode::get_ts_mgr_().get_gts_callback(); - t.join(); - ASSERT_EQ(OB_SUCCESS, commit_ret); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - n2->wait_all_redolog_applied(); - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, get_gts_block_and_switch_to_follower_forcedly) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - - DEFER(delete(n1)); - DEFER(delete(n2)); - ASSERT_EQ(OB_SUCCESS, n1->start()); - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxParam tx_param; - tx_param.timeout_us_ = 10 * 1000 * 1000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxNode::get_ts_mgr_().set_get_gts_waiting_mode(); -// tx - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ObTxSEQ sp1; - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - - int commit_ret = OB_SUCCESS; - std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); - usleep(100 * 1000); - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ObTransID fail_tx_id; - ls_tx_ctx_mgr->traverse_tx_to_submit_redo_log(fail_tx_id); - n1->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_follower_forcedly()); - n1->wait_all_redolog_applied(); - t.join(); - ASSERT_EQ(OB_TRANS_KILLED, commit_ret); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - - ObTxNode::get_ts_mgr_().clear_get_gts_waiting_mode(); - ObTxNode::get_ts_mgr_().get_gts_callback(); - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} - -TEST_F(ObTestTx, switch_to_follower_gracefully_in_stmt_rollback_to_last_savepoint_then_commit) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - LOG_INFO("n1 tx_ctx_mgr", K(&n1->txs_.tx_ctx_mgr_)); - - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - LOG_INFO("n2 tx_ctx_mgr", K(&n2->txs_.tx_ctx_mgr_)); - - DEFER(delete(n1)); - DEFER(delete(n2)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - ObTxParam tx_param; - tx_param.timeout_us_ = 100000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxSEQ sp1; - ObStoreCtx write_store_ctx; - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write_begin(tx, snapshot, write_store_ctx)); - ASSERT_EQ(OB_SUCCESS, n1->write_one_row(write_store_ctx, 1000, 1112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ls_tx_ctx_mgr->switch_to_follower_gracefully(); - n1->wait_all_redolog_applied(); - - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ObTxNode::get_location_adapter_().update_localtion(n2->ls_id_, n2->addr_); - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - ASSERT_EQ(OB_NOT_MASTER, n1->write_one_row(write_store_ctx, 1001, 1113)); - ASSERT_EQ(OB_SUCCESS, n1->write_end(write_store_ctx)); - { - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp1, n1->ts_after_ms(100), nullptr)); - } - - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 1001, 1113)); - } - - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - int64_t val1 = 0, val2 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 1001, val2)); - ASSERT_EQ(1113, val2); - } - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - } - - n2->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -} - -TEST_F(ObTestTx, switch_to_follower_gracefully_in_stmt_then_commit) -{ - int ret = OB_SUCCESS; - ObTxNode::reset_localtion_adapter(); - - oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); - LOG_INFO("n1 tx_ctx_mgr", K(&n1->txs_.tx_ctx_mgr_)); - - auto n2 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); - LOG_INFO("n2 tx_ctx_mgr", K(&n2->txs_.tx_ctx_mgr_)); - - DEFER(delete(n1)); - DEFER(delete(n2)); - - ASSERT_EQ(OB_SUCCESS, n1->start()); - - n2->set_as_follower_replica(*n1); - ASSERT_EQ(OB_SUCCESS, n2->start()); - - ObTxDesc *tx_ptr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); - ObTxDesc &tx = *tx_ptr; - ObTxParam tx_param; - tx_param.timeout_us_ = 100000000; - tx_param.access_mode_ = ObTxAccessMode::RW; - tx_param.isolation_ = ObTxIsolationLevel::RC; - tx_param.cluster_id_ = 100; - - ObTxSEQ sp1; - ObStoreCtx write_store_ctx; - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 1000, 1111)); - } - - ObTxSEQ sp2; - { - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, - tx_param.isolation_, - n1->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp2)); - ASSERT_EQ(OB_SUCCESS, n1->write_begin(tx, snapshot, write_store_ctx)); - ASSERT_EQ(OB_SUCCESS, n1->write_one_row(write_store_ctx, 1000, 1112)); - } - - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - ls_tx_ctx_mgr->switch_to_follower_gracefully(); - n1->wait_all_redolog_applied(); - - ReplayLogEntryFunctor functor(n2); - ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor)); - - ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); - - ObTxNode::get_location_adapter_().update_localtion(n2->ls_id_, n2->addr_); - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_leader()); - n2->wait_all_redolog_applied(); - - ASSERT_EQ(OB_NOT_MASTER, n1->write_one_row(write_store_ctx, 1001, 1113)); - ASSERT_EQ(OB_SUCCESS, n1->write_end(write_store_ctx)); - { - ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp2, n1->ts_after_ms(100), nullptr)); - } - - ObTxSEQ sp3; - { // prepare snapshot for read - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp3)); - ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 1001, 1113)); - } + auto replay_to_n2 = [n2](const void *buffer, + const int64_t nbytes, + const palf::LSN &lsn, + const int64_t ts_ns) { + return n2->replay(buffer, nbytes, lsn, ts_ns); + }; + ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(replay_to_n2)); + ObLSTxCtxMgr &n2_ctx_mgr = n2->txs_.tx_ctx_mgr_.get_tx_ctx_manager(); { + PREPARE_TX(n2, tx); ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx, - tx_param.isolation_, - n2->ts_after_ms(100), - snapshot)); - int64_t val1 = 0, val2 = 0; - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 1000, val1)); - ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 1001, val2)); - ASSERT_EQ(1111, val1); - ASSERT_EQ(1113, val2); + ASSERT_EQ(OB_SUCCESS, + n2->get_read_snapshot(tx, + ObTxIsolationLevel::RC, + n2->ts_after_ms(100), + snapshot)); + int64_t value = 0; + ASSERT_EQ(OB_SUCCESS, n2->read(snapshot, 100, value)); + ASSERT_EQ(112, value); } - - { - ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); - } - - n2->wait_all_redolog_applied(); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); - ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -} -// distributed tx -// TEST_F(ObTestTx, distributed_tx_participant_switch_to_follower_gracefully_in_stmt_then_commit) -// { -// int ret = OB_SUCCESS; -// GCONF._ob_trans_rpc_timeout = 5000 * 1000; -// ObTxNode::reset_localtion_adapter(); -// -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// LOG_INFO("n1 tx_ctx_mgr", K(&n1->txs_.tx_ctx_mgr_)); -// -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// LOG_INFO("n2 tx_ctx_mgr", K(&n2->txs_.tx_ctx_mgr_)); -// -// auto n3 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// LOG_INFO("n3 tx_ctx_mgr", K(&n3->txs_.tx_ctx_mgr_)); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// -// n3->set_as_follower_replica(*n2); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// ObTxParam tx_param; -// tx_param.timeout_us_ = 100000000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// ObTxSEQ sp1; -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 1000, 1111)); -// } -// -// ObTxSEQ sp2; -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp2)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 2000, 1111)); -// } -// -// ObStoreCtx write_store_ctx; -// ObTxSEQ sp3; -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp3)); -// ASSERT_EQ(OB_SUCCESS, n2->write_begin(tx, snapshot, write_store_ctx)); -// ASSERT_EQ(OB_SUCCESS, n2->write_one_row(write_store_ctx, 2000, 1112)); -// } -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); -// ls_tx_ctx_mgr2->switch_to_follower_gracefully(); -// n2->wait_all_redolog_applied(); -// -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// ASSERT_EQ(OB_NOT_MASTER, n2->write_one_row(write_store_ctx, 2000, 1113)); -// ASSERT_EQ(OB_SUCCESS, n2->write_end(write_store_ctx)); -// { -// ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp3, n1->ts_after_ms(100), nullptr)); -// } -// -// ObTxSEQ sp4; -// { // prepare snapshot for read -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp4)); -// ASSERT_EQ(OB_SUCCESS, n3->write(tx, snapshot, 2001, 1113)); -// } -// -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// int64_t val1 = 0, val2 = 0; -// ASSERT_EQ(OB_SUCCESS, n3->read(snapshot, 2000, val1)); -// ASSERT_EQ(OB_SUCCESS, n3->read(snapshot, 2001, val2)); -// ASSERT_EQ(1111, val1); -// ASSERT_EQ(1113, val2); -// } -// -// n3->add_drop_msg_type(TX_2PC_CLEAR_REQ); -// { -// ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); -// } -// n3->del_drop_msg_type(TX_2PC_CLEAR_REQ); -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// -// ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// } -// -// TEST_F(ObTestTx, distributed_tx_coordinator_switch_to_follower_gracefully_in_stmt_then_commit) -// { -// int ret = OB_SUCCESS; -// GCONF._ob_trans_rpc_timeout = 5000 * 1000; -// ObTxNode::reset_localtion_adapter(); -// -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// LOG_INFO("n1 tx_ctx_mgr", K(&n1->txs_.tx_ctx_mgr_)); -// -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// LOG_INFO("n2 tx_ctx_mgr", K(&n2->txs_.tx_ctx_mgr_)); -// -// auto n3 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// LOG_INFO("n3 tx_ctx_mgr", K(&n3->txs_.tx_ctx_mgr_)); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// -// n3->set_as_follower_replica(*n1); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// ObTxParam tx_param; -// tx_param.timeout_us_ = 100000000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// ObTxSEQ sp1; -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 1000, 1111)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 2000, 1111)); -// } -// -// ObStoreCtx write_store_ctx; -// ObTxSEQ sp3; -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp3)); -// ASSERT_EQ(OB_SUCCESS, n1->write_begin(tx, snapshot, write_store_ctx)); -// ASSERT_EQ(OB_SUCCESS, n1->write_one_row(write_store_ctx, 1000, 1112)); -// } -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// ls_tx_ctx_mgr1->switch_to_follower_gracefully(); -// n1->wait_all_redolog_applied(); -// -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// ASSERT_EQ(OB_NOT_MASTER, n1->write_one_row(write_store_ctx, 1000, 1113)); -// ASSERT_EQ(OB_SUCCESS, n1->write_end(write_store_ctx)); -// { -// ASSERT_EQ(OB_SUCCESS, n1->rollback_to_implicit_savepoint(tx, sp3, n1->ts_after_ms(100), nullptr)); -// } -// -// ObTxSEQ sp4; -// { // prepare snapshot for read -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp4)); -// ASSERT_EQ(OB_SUCCESS, n3->write(tx, snapshot, 1001, 1113)); -// } -// -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// int64_t val1 = 0, val2 = 0; -// ASSERT_EQ(OB_SUCCESS, n3->read(snapshot, 1000, val1)); -// ASSERT_EQ(OB_SUCCESS, n3->read(snapshot, 1001, val2)); -// ASSERT_EQ(1111, val1); -// ASSERT_EQ(1113, val2); -// } -// -// n3->add_drop_msg_type(TX_2PC_CLEAR_REQ); -// { -// ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); -// } -// n3->del_drop_msg_type(TX_2PC_CLEAR_REQ); -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); -// -// ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// } -// -// TEST_F(ObTestTx, distributed_tx_switch_to_follower_forcedly_in_prepare_state) -// { -// int ret = OB_SUCCESS; -// ObTxNode::reset_localtion_adapter(); -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// auto n3 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// n3->set_as_follower_replica(*n2); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxParam tx_param; -// tx_param.timeout_us_ = 500 * 1000 * 1000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// // prepare snapshot for write -// ObTxReadSnapshot snapshot; -// { -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// } -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 101, 113)); -// -// ObTxNode::get_ts_mgr_().set_get_gts_waiting_mode(); -// -// int commit_ret = OB_SUCCESS; -// std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); -// usleep(100 * 1000); -// -// ObPartTransCtx *n2_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->get_tx_ctx(n2->ls_id_, tx.tx_id_, n2_ctx)); -// int i = 0; -// while(!(n2_ctx->exec_info_.state_ == ObTxState::PREPARE) && i++ < 100) { -// usleep(5000); -// } -// ASSERT_NE(i, 100); -// ASSERT_EQ(OB_SUCCESS, n2->revert_tx_ctx(n2_ctx)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_follower_forcedly()); -// n2->wait_all_redolog_applied(); -// -// ObTxNode::get_ts_mgr_().clear_get_gts_waiting_mode(); -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// n3->add_drop_msg_type(TX_2PC_CLEAR_REQ); -// -// ObTxNode::get_ts_mgr_().get_gts_callback(); -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); -// -// t.join(); -// ASSERT_EQ(OB_SUCCESS, commit_ret); -// -// n3->del_drop_msg_type(TX_2PC_CLEAR_REQ); -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// -// ReplayLogEntryFunctor functor_n2(n2); -// ASSERT_EQ(OB_SUCCESS, n2->fake_tx_log_adapter_->replay_all(functor_n2)); -// ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); -// -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// } -// -// TEST_F(ObTestTx, distributed_tx_coordinator_switch_to_follower_forcedly_in_prepare_state) -// { -// int ret = OB_SUCCESS; -// ObTxNode::reset_localtion_adapter(); -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// auto n3 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// n3->set_as_follower_replica(*n1); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxParam tx_param; -// tx_param.timeout_us_ = 500 * 1000 * 1000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// // prepare snapshot for write -// ObTxReadSnapshot snapshot; -// { -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// } -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 101, 113)); -// -// n1->add_drop_msg_type(TX_2PC_PREPARE_RESP); -// -// int commit_ret = OB_SUCCESS; -// // async start commit -// std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); -// usleep(100 * 1000); -// -// // wait coordinator into prepare state -// ObPartTransCtx *n1_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->get_tx_ctx(n1->ls_id_, tx.tx_id_, n1_ctx)); -// int i = 0; -// while(!(n1_ctx->exec_info_.state_ == ObTxState::PREPARE) && i++ < 1000) { -// usleep(5000); -// } -// ASSERT_NE(i, 1001); -// ASSERT_EQ(OB_SUCCESS, n1->revert_tx_ctx(n1_ctx)); -// -// // switch coordinator to follower forcedly -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr1->switch_to_follower_forcedly()); -// n1->wait_all_redolog_applied(); -// -// // n3 takeover as leader -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); -// -// // wait commit complete on scheduler -// t.join(); -// ASSERT_EQ(OB_SUCCESS, commit_ret); -// -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// -// ReplayLogEntryFunctor functor_n1(n1); -// ASSERT_EQ(OB_SUCCESS, n1->fake_tx_log_adapter_->replay_all(functor_n1)); -// ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); -// -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// } -// -// TEST_F(ObTestTx, distributed_tx_participant_switch_to_follower_forcedly_in_pre_commit_state) -// { -// int ret = OB_SUCCESS; -// ObTxNode::reset_localtion_adapter(); -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// auto n3 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// n3->set_as_follower_replica(*n2); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxParam tx_param; -// tx_param.timeout_us_ = 500 * 1000 * 1000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// { -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n2->get_read_snapshot(tx, -// tx_param.isolation_, -// n2->ts_after_ms(100), -// snapshot)); -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n2->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 200, 112)); -// } -// { -// ASSERT_EQ(OB_SUCCESS, n2->commit_tx(tx, n2->ts_after_ms(500))); -// ASSERT_EQ(OB_SUCCESS, n2->release_tx(tx)); -// } -// n2->wait_all_redolog_applied(); -// } -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// // prepare snapshot for write -// ObTxReadSnapshot snapshot; -// { -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// } -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 101, 112)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 200, 113)); -// -// n2->add_drop_msg_type(TX_2PC_COMMIT_REQ); -// int commit_ret = OB_SUCCESS; -// std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); -// usleep(100 * 1000); -// -// ObPartTransCtx *n2_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->get_tx_ctx(n2->ls_id_, tx.tx_id_, n2_ctx)); -// int i = 0; -// while(!(n2_ctx->exec_info_.state_ == ObTxState::PRE_COMMIT) && i++ < 100) { -// usleep(5000); -// } -// ASSERT_NE(101, i); -// ASSERT_EQ(OB_SUCCESS, n2->revert_tx_ctx(n2_ctx)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr2 = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n2->ls_id_, ls_tx_ctx_mgr2)); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2->switch_to_follower_forcedly()); -// n2->wait_all_redolog_applied(); -// -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// LOG_INFO("max_commit_ts after switch_to_leader", -// K(n3->txs_.tx_version_mgr_.get_max_commit_ts(false))); -// -// n2->del_drop_msg_type(TX_2PC_COMMIT_REQ); -// n3->add_drop_msg_type(TX_2PC_COMMIT_REQ); -// t.join(); -// ASSERT_EQ(OB_SUCCESS, commit_ret); -// -// ObTxDesc *tx_ptr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->acquire_tx(tx_ptr3)); -// ObTxDesc &tx3 = *tx_ptr3; -// int64_t val1 = 0; -// ObTxReadSnapshot snapshot3; -// { -// ASSERT_EQ(OB_SUCCESS, n3->get_read_snapshot(tx3, -// tx_param.isolation_, -// n3->ts_after_ms(100), -// snapshot3)); -// } -// std::thread t_read(do_async_read, n3, std::ref(snapshot3), 200, std::ref(val1)); -// usleep(100 * 1000); -// n3->add_drop_msg_type(TX_2PC_CLEAR_REQ); -// n3->del_drop_msg_type(TX_2PC_COMMIT_REQ); -// n3->del_drop_msg_type(TX_2PC_CLEAR_REQ); -// -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// t_read.join(); -// ASSERT_EQ(113, val1); -// ASSERT_EQ(OB_SUCCESS, n3->release_tx(tx3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// ASSERT_EQ(0, ls_tx_ctx_mgr3->get_tx_ctx_count()); -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// ASSERT_EQ(OB_SUCCESS, n2->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr2)); -// } -// -// TEST_F(ObTestTx, distributed_tx_coordinator_switch_to_follower_forcedly_in_pre_commit_state) -// { -// int ret = OB_SUCCESS; -// ObTxNode::reset_localtion_adapter(); -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// auto n3 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// -// n3->set_as_follower_replica(*n1); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxParam tx_param; -// tx_param.timeout_us_ = 500ll * 1000 * 1000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// { -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// } -// { -// ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// } -// n1->wait_all_redolog_applied(); -// } -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// // prepare snapshot for write -// ObTxReadSnapshot snapshot; -// { -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// } -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 113)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 200, 113)); -// -// n1->add_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// n3->add_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// int commit_ret = OB_SUCCESS; -// std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); -// usleep(100 * 1000); -// -// ObPartTransCtx *n2_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->get_tx_ctx(n2->ls_id_, tx.tx_id_, n2_ctx)); -// int i = 0; -// while(!(n2_ctx->exec_info_.state_ == ObTxState::PRE_COMMIT) && i++ < 100) { -// usleep(5000); -// } -// ASSERT_NE(i, 101); -// ASSERT_EQ(OB_SUCCESS, n2->revert_tx_ctx(n2_ctx)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr1->switch_to_follower_forcedly()); -// n1->wait_all_redolog_applied(); -// -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// LOG_INFO("max_commit_ts after switch_to_leader", -// K(n3->txs_.tx_version_mgr_.get_max_commit_ts(false))); -// -// -// ObTxDesc *tx_ptr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->acquire_tx(tx_ptr3)); -// ObTxDesc &tx3 = *tx_ptr3; -// int64_t val1 = 0; -// ObTxReadSnapshot snapshot3; -// { -// ASSERT_EQ(OB_SUCCESS, n3->get_read_snapshot(tx3, -// tx_param.isolation_, -// n3->ts_after_ms(100), -// snapshot3)); -// } -// std::thread t_read(do_async_read, n3, std::ref(snapshot3), 100, std::ref(val1)); -// usleep(100 * 1000); -// n3->add_drop_msg_type(TX_2PC_COMMIT_RESP); -// n3->del_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// t.join(); -// ASSERT_EQ(OB_SUCCESS, commit_ret); -// n3->del_drop_msg_type(TX_2PC_COMMIT_RESP); -// -// t_read.join(); -// ASSERT_EQ(113, val1); -// ASSERT_EQ(OB_SUCCESS, n3->release_tx(tx3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// } -// -// TEST_F(ObTestTx, distributed_tx_coordinator_switch_to_follower_forcedly_in_participant_commit_state) -// { -// int ret = OB_SUCCESS; -// ObTxNode::reset_localtion_adapter(); -// oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_1_0_0_0); -// -// auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); -// auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); -// auto n3 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.3", 8888), bus_); -// -// DEFER(delete(n1)); -// DEFER(delete(n2)); -// DEFER(delete(n3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->start()); -// ASSERT_EQ(OB_SUCCESS, n2->start()); -// -// n3->set_as_follower_replica(*n1); -// ASSERT_EQ(OB_SUCCESS, n3->start()); -// -// ObTxParam tx_param; -// tx_param.timeout_us_ = 500ll * 1000 * 1000; -// tx_param.access_mode_ = ObTxAccessMode::RW; -// tx_param.isolation_ = ObTxIsolationLevel::RC; -// tx_param.cluster_id_ = 100; -// -// { -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// -// { -// ObTxReadSnapshot snapshot; -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); -// } -// { -// ASSERT_EQ(OB_SUCCESS, n1->commit_tx(tx, n1->ts_after_ms(500))); -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// } -// n1->wait_all_redolog_applied(); -// } -// -// ObTxDesc *tx_ptr = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->acquire_tx(tx_ptr)); -// ObTxDesc &tx = *tx_ptr; -// // prepare snapshot for write -// ObTxReadSnapshot snapshot; -// { -// ASSERT_EQ(OB_SUCCESS, n1->get_read_snapshot(tx, -// tx_param.isolation_, -// n1->ts_after_ms(100), -// snapshot)); -// } -// ObTxSEQ sp1; -// ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); -// ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 113)); -// ASSERT_EQ(OB_SUCCESS, n2->write(tx, snapshot, 200, 113)); -// -// n1->add_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// n3->add_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// int commit_ret = OB_SUCCESS; -// std::thread t(do_async_commit, n1, std::ref(tx), std::ref(commit_ret)); -// usleep(100 * 1000); -// -// ObPartTransCtx *n1_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->get_tx_ctx(n1->ls_id_, tx.tx_id_, n1_ctx)); -// int i = 0; -// while(!(n1_ctx->exec_info_.state_ == ObTxState::PRE_COMMIT) && i++ < 100) { -// usleep(5000); -// } -// ASSERT_NE(i, 101); -// ASSERT_EQ(OB_SUCCESS, n1->revert_tx_ctx(n1_ctx)); -// -// n1->fake_tx_log_adapter_->set_log_drop(); -// n1->del_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// -// ObPartTransCtx *n2_ctx = NULL; -// ASSERT_EQ(OB_SUCCESS, n2->get_tx_ctx(n2->ls_id_, tx.tx_id_, n2_ctx)); -// i = 0; -// while(!(n2_ctx->exec_info_.state_ == ObTxState::COMMIT) && i++ < 100) { -// usleep(50000); -// } -// ASSERT_NE(i, 101); -// ASSERT_EQ(OB_SUCCESS, n2->revert_tx_ctx(n2_ctx)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr1 = NULL; -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr1)); -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr1->switch_to_follower_forcedly()); -// n1->fake_tx_log_adapter_->clear_log_drop(); -// -// ReplayLogEntryFunctor functor(n3); -// ASSERT_EQ(OB_SUCCESS, n3->fake_tx_log_adapter_->replay_all(functor)); -// -// ObLSTxCtxMgr *ls_tx_ctx_mgr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n3->ls_id_, ls_tx_ctx_mgr3)); -// -// ObTxNode::get_location_adapter_().update_localtion(n3->ls_id_, n3->addr_); -// -// ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr3->switch_to_leader()); -// n3->wait_all_redolog_applied(); -// -// LOG_INFO("max_commit_ts after switch_to_leader", -// K(n3->txs_.tx_version_mgr_.get_max_commit_ts(false))); -// -// ObTxDesc *tx_ptr3 = NULL; -// ASSERT_EQ(OB_SUCCESS, n3->acquire_tx(tx_ptr3)); -// ObTxDesc &tx3 = *tx_ptr3; -// int64_t val1 = 0; -// ObTxReadSnapshot snapshot3; -// { -// ASSERT_EQ(OB_SUCCESS, n3->get_read_snapshot(tx3, -// tx_param.isolation_, -// n3->ts_after_ms(100), -// snapshot3)); -// } -// std::thread t_read(do_async_read, n3, std::ref(snapshot3), 100, std::ref(val1)); -// usleep(100 * 1000); -// n3->add_drop_msg_type(TX_2PC_COMMIT_RESP); -// n3->del_drop_msg_type(TX_2PC_PRE_COMMIT_RESP); -// t.join(); -// ASSERT_EQ(OB_SUCCESS, commit_ret); -// -// n3->del_drop_msg_type(TX_2PC_COMMIT_RESP); -// -// t_read.join(); -// ASSERT_EQ(113, val1); -// ASSERT_EQ(OB_SUCCESS, n3->release_tx(tx3)); -// -// ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); -// ASSERT_EQ(OB_SUCCESS, n3->wait_all_tx_ctx_is_destoryed()); -// ASSERT_EQ(OB_SUCCESS, n2->wait_all_tx_ctx_is_destoryed()); -// ASSERT_EQ(OB_SUCCESS, n3->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr3)); -// ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr1)); -// } - -TEST_F(ObTestTx, interrupt_get_read_snapshot) -{ - START_ONE_TX_NODE(n1); - PREPARE_TX(n1, tx); - ObTxReadSnapshot snapshot; - n1->get_ts_mgr_().inject_get_gts_error(OB_EAGAIN); - int ret = OB_SUCCESS; - do { - ASYNC_DO(acq_snapshot, n1->get_read_snapshot(tx, ObTxIsolationLevel::RC, n1->ts_after_ms(20 * 1000), snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->interrupt(tx, OB_TRANS_KILLED)); - ASYNC_WAIT(acq_snapshot, 2000 * 1000, wait_ret); - ret = wait_ret; - } while (OB_GTS_NOT_READY == ret); - ASSERT_EQ(OB_ERR_INTERRUPTED, ret); - ROLLBACK_TX(n1, tx); + ASSERT_EQ(0, n2_ctx_mgr.get_tx_ctx_count()); } TEST_F(ObTestTx, rollback_with_branch_savepoint) @@ -2357,44 +233,10 @@ TEST_F(ObTestTx, rollback_with_branch_savepoint) const int commit_ret = COMMIT_TX(n1, tx, 50000); \ EXPECT_EQ(commit_ret, OB_TRANS_ROLLBACKED); \ } -TEST_MARK_ABORT_AND_COMMIT(PART_ABORTED_) -TEST_MARK_ABORT_AND_COMMIT(PART_EPOCH_MISMATCH_) -TEST_MARK_ABORT_AND_COMMIT(PARTS_INCOMPLETE_) +TEST_MARK_ABORT_AND_COMMIT(WRITE_STATE_ABORTED_) +TEST_MARK_ABORT_AND_COMMIT(WRITE_STATE_INCOMPLETE_) #undef _MARK_ABORT_AND_COMMIT -TEST_F(ObTestTx, participant_abort_asynchronously) -{ - START_TWO_TX_NODE(n1, n2); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, global_sp1); - ASSERT_EQ(n1->write(tx, 1, 1), OB_SUCCESS); - ASSERT_EQ(n2->write(tx, 2, 2), OB_SUCCESS); - // n1 switch to follower forcedly, then switch to leader - FLUSH_REDO(n1); - n1->wait_tx_log_synced(); - SWITCH_TO_FOLLOWER_FORCEDLY(n1); - SWITCH_TO_LEADER(n1); - // check received participant aborted notify from n1 - n1->wait_all_msg_consumed(); - ASSERT_TRUE(tx.flags_.PART_ABORTED_); - ASSERT_EQ(tx.abort_cause_, ObTxAbortCause::PARTICIPANT_SWITCH_LEADER_DATA_INCOMPLETE); - share::ObLSArray extra_touched_ls; - extra_touched_ls.push_back(ObLSID(111)); - ASSERT_EQ(ROLLBACK_TO_IMPLICIT_SAVEPOINT_X(n1, tx, global_sp1, 2000, &extra_touched_ls), - OB_TRANS_NEED_ROLLBACK); - ASSERT_EQ(tx.state_, ObTxDesc::State::ABORTED); - bool found_touched_ls_id_in_participant_set = false; - for (int i = 0; i< tx.parts_.count(); i++) { - if (tx.parts_[i].id_.id() == 111) { - found_touched_ls_id_in_participant_set = true; - } - } - ASSERT_TRUE(found_touched_ls_id_in_participant_set); - const int commit_ret = COMMIT_TX(n1, tx, 5000); - EXPECT_EQ(commit_ret, OB_TRANS_ROLLBACKED); -} - //// /// APPEND NEW TEST HERE, USE PRE DEFINED MACRO IN FILE `test_tx.dsl` /// SEE EXAMPLE: TEST_F(ObTestTx, rollback_savepoint_timeout) diff --git a/unittest/storage/tx/it/test_tx_ctx.cpp b/unittest/storage/tx/it/test_tx_ctx.cpp index 1a3a4b988..036316c7b 100644 --- a/unittest/storage/tx/it/test_tx_ctx.cpp +++ b/unittest/storage/tx/it/test_tx_ctx.cpp @@ -84,7 +84,6 @@ class ObTestTxCtx : public ::testing::Test ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); const uint64_t tv = ObTimeUtility::current_time(); ObCurTraceId::set(&tv); - GCONF._ob_trans_rpc_timeout = 500; ObClockGenerator::init(); const testing::TestInfo *const test_info = testing::UnitTest::GetInstance()->current_test_info(); @@ -105,91 +104,6 @@ class ObTestTxCtx : public ::testing::Test MsgBus bus_; }; -TEST_F(ObTestTxCtx, DelayAbort) -{ - ObTxPartList backup_parts; - START_ONE_TX_NODE(n1); - PREPARE_TX_PARAM(tx_param); - PREPARE_TX(n1, tx); - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, - n1->get_read_snapshot(tx, tx_param.isolation_, n1->ts_after_ms(100), snapshot)); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp1); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp1)); - ASSERT_EQ(OB_SUCCESS, n1->write(tx, snapshot, 100, 112)); - } - ASSERT_EQ(OB_SUCCESS, backup_parts.assign(tx.parts_)); - tx.parts_.reset(); - - ObLSTxCtxMgr *ls_tx_ctx_mgr = nullptr; - ObPartTransCtx *tx_ctx = nullptr; - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr)); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->get_tx_ctx(tx.tx_id_, false /*for_replay*/, tx_ctx)); - GCONF._private_buffer_size = 1; - // ASSERT_EQ(OB_SUCCESS, tx_ctx->submit_log_impl_(ObTxLogType::TX_REDO_LOG)); - ASSERT_EQ(OB_SUCCESS, tx_ctx->submit_redo_after_write(false, ObTxSEQ())); - TRANS_LOG(INFO, "[TEST] after submit redo", K(tx_ctx->trans_id_), - K(tx_ctx->exec_info_.max_applied_log_ts_)); - n1->wait_all_redolog_applied(); - TRANS_LOG(INFO, "[TEST] after on_success redo", K(tx_ctx->trans_id_), - K(tx_ctx->exec_info_.max_applied_log_ts_)); - ASSERT_EQ(true, tx_ctx->exec_info_.redo_lsns_.count() > 0); - ASSERT_EQ(true, tx_ctx->exec_info_.max_applied_log_ts_.is_valid()); - ASSERT_EQ(ObTxState::INIT, tx_ctx->exec_info_.state_); - ASSERT_EQ(false, tx_ctx->is_follower_()); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->revert_tx_ctx(tx_ctx)); - } - // disable keepalive msg, because switch to follower forcedly will send keepalive msg to notify - // scheduler abort tx - TRANS_LOG(INFO, "add drop KEEPALIVE msg"); - n1->add_drop_msg_type(KEEPALIVE); - - ls_tx_ctx_mgr->switch_to_follower_forcedly(); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->get_tx_ctx(tx.tx_id_, true /*for_replay*/, tx_ctx)); - ASSERT_EQ(true, tx_ctx->exec_info_.redo_lsns_.count() > 0); - ASSERT_EQ(true, tx_ctx->exec_info_.max_applied_log_ts_.is_valid()); - ASSERT_EQ(ObTxState::INIT, tx_ctx->exec_info_.state_); - ASSERT_EQ(true, tx_ctx->is_follower_()); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->revert_tx_ctx(tx_ctx)); - } - - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->switch_to_leader()); - n1->wait_all_redolog_applied(); - - { - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->get_tx_ctx(tx.tx_id_, false /*for_replay*/, tx_ctx)); - TRANS_LOG(INFO, "[TEST] after leader takeover", K(tx_ctx->trans_id_), - K(tx_ctx->exec_info_.state_), K(tx_ctx->exec_info_.max_applied_log_ts_)); - ASSERT_EQ(true, tx_ctx->exec_info_.redo_lsns_.count() > 0); - ASSERT_EQ(true, tx_ctx->exec_info_.max_applied_log_ts_.is_valid()); - ASSERT_EQ(true, tx_ctx->sub_state_.is_force_abort()); - ASSERT_EQ(false, tx_ctx->sub_state_.is_state_log_submitted()); - ASSERT_EQ(ObTxState::INIT, tx_ctx->exec_info_.state_); - ASSERT_EQ(false, tx_ctx->is_follower_()); - ASSERT_EQ(OB_SUCCESS, ls_tx_ctx_mgr->revert_tx_ctx(tx_ctx)); - } - - { // prepare snapshot for write - ObTxReadSnapshot snapshot; - ASSERT_EQ(OB_SUCCESS, - n1->get_read_snapshot(tx, tx_param.isolation_, n1->ts_after_ms(100), snapshot)); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp2); - ASSERT_EQ(OB_SUCCESS, n1->create_implicit_savepoint(tx, tx_param, sp2)); - ASSERT_EQ(OB_TRANS_KILLED, n1->write(tx, snapshot, 1110, 1120)); - } - - ASSERT_EQ(OB_SUCCESS, tx.parts_.assign(backup_parts)); - ASSERT_EQ(OB_TRANS_KILLED, n1->commit_tx(tx, n1->ts_after_ms(500))); - ASSERT_EQ(OB_SUCCESS, n1->release_tx(tx)); - - ASSERT_EQ(OB_SUCCESS, n1->wait_all_tx_ctx_is_destoryed()); - ASSERT_EQ(OB_SUCCESS, n1->txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); -} } // namespace oceanbase int main(int argc, char **argv) diff --git a/unittest/storage/tx/it/test_tx_dsl.h b/unittest/storage/tx/it/test_tx_dsl.h index dbe9ab1bb..46d9c6db2 100644 --- a/unittest/storage/tx/it/test_tx_dsl.h +++ b/unittest/storage/tx/it/test_tx_dsl.h @@ -15,28 +15,20 @@ */ #define START_ONE_TX_NODE(n1) \ - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); \ + auto n1 = new ObTxNode(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); \ NAMED_DEFER(defer_n1, delete(n1)); \ ASSERT_EQ(OB_SUCCESS, n1->start()); \ LOG_INFO("##START_ONE_TX_NODE##", "node.addr_", n1->addr_); -#define START_TWO_TX_NODE(n1, n2) \ - auto n1 = new ObTxNode(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); \ - auto n2 = new ObTxNode(2, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); \ +#define START_TX_REPLAY_PAIR(n1, n2) \ + auto n1 = new ObTxNode(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); \ + auto n2 = new ObTxNode(ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); \ NAMED_DEFER(defer_n1, delete(n1)); \ NAMED_DEFER(defer_n2, delete(n2)); \ ASSERT_EQ(OB_SUCCESS, n1->start()); \ + n2->set_replay_source(*n1); \ ASSERT_EQ(OB_SUCCESS, n2->start()); \ - LOG_INFO("##START_TWO_TX_NODE##", K(n1->addr_), K(n2->addr_)); - -#define START_TWO_TX_NODE_WITH_LSID(n1, n2, ls_id) \ - auto n1 = new ObTxNode(ls_id, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 8888), bus_); \ - auto n2 = new ObTxNode(ls_id + 1, ObAddr(ObAddr::VER::IPV4, "127.0.0.2", 8888), bus_); \ - NAMED_DEFER(defer_n1, delete (n1)); \ - NAMED_DEFER(defer_n2, delete (n2)); \ - ASSERT_EQ(OB_SUCCESS, n1->start()); \ - ASSERT_EQ(OB_SUCCESS, n2->start()); \ - LOG_INFO("##START_TWO_TX_NODE##", K(n1->addr_), K(n2->addr_)); + LOG_INFO("##START_TX_REPLAY_PAIR##", K(n1->addr_), K(n2->addr_)); #define PREPARE_TX(n1, tx) \ ObTxDescGuard guard = n1->get_tx_guard(); \ @@ -62,10 +54,7 @@ ASSERT_EQ(OB_SUCCESS, n1->create_branch_savepoint(tx, branch, sp)); #define ROLLBACK_TO_IMPLICIT_SAVEPOINT(n1, tx, sp, timeout_us) \ - n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_us(timeout_us), nullptr) - -#define ROLLBACK_TO_IMPLICIT_SAVEPOINT_X(n1, tx, sp, timeout_us, extra_touched_ls) \ - n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_us(timeout_us), extra_touched_ls) + n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_us(timeout_us), false) #define INJECT_LINK_FAILURE(n1, n2) \ ASSERT_EQ(OB_SUCCESS, bus_.inject_link_failure(n1->addr_, n2->addr_)); \ @@ -102,36 +91,7 @@ #define FLUSH_REDO(n1) \ do { \ - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; \ - ASSERT_EQ(n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr), OB_SUCCESS); \ + ObLSTxCtxMgr &tx_ctx_mgr = n1->txs_.tx_ctx_mgr_.get_tx_ctx_manager(); \ ObTransID fail_tx_id; \ - ASSERT_EQ(ls_tx_ctx_mgr->traverse_tx_to_submit_redo_log(fail_tx_id, UINT32_MAX), OB_SUCCESS); \ - } while(0) - -#define SWITCH_TO_FOLLOWER_FORCEDLY(n1) \ - do { \ - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; \ - ASSERT_EQ(n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr), OB_SUCCESS); \ - ASSERT_EQ(ls_tx_ctx_mgr->switch_to_follower_forcedly(), OB_SUCCESS); \ - } while(0) - -#define SWITCH_TO_FOLLOWER_GRACEFULLY(n1) \ - do { \ - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; \ - ASSERT_EQ(n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr), OB_SUCCESS); \ - ASSERT_EQ(ls_tx_ctx_mgr->switch_to_follower_gracefully(), OB_SUCCESS); \ - } while(0) - -#define SWITCH_TO_LEADER(n1) \ - do { \ - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; \ - ASSERT_EQ(n1->txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(n1->ls_id_, ls_tx_ctx_mgr), OB_SUCCESS); \ - ASSERT_EQ(ls_tx_ctx_mgr->switch_to_leader(), OB_SUCCESS); \ - /* wait state to LEADER */ \ - while (!ls_tx_ctx_mgr->is_master()) { \ - if (REACH_TIME_INTERVAL(500_ms)) { \ - TRANS_LOG(INFO, "wait LS TxCtxMgr to be leader", KPC(ls_tx_ctx_mgr)); \ - } \ - usleep(10_ms); \ - } \ + ASSERT_EQ(tx_ctx_mgr.traverse_tx_to_submit_redo_log(fail_tx_id, UINT32_MAX), OB_SUCCESS); \ } while(0) diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index 9315ae81f..a7dbd0921 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -82,30 +82,13 @@ int ObTxDescGuard::release() { ObString ObTxNode::get_identifer_str() { - struct ID { - ObAddr addr; - int64_t ls_id; - DECLARE_TO_STRING { - int64_t pos = 0; - int32_t pos0 = 0; - addr.addr_to_buffer(buf, buf_len, pos0); - pos += pos0; - BUF_PRINTF("_%ld", ls_id); - return pos; - } - } identifer = { - .addr = addr_, - .ls_id = ls_id_.id() - }; - int64_t pos = identifer.to_string(buf_, sizeof(buf_)); + const int64_t pos = addr_.to_string(buf_, sizeof(buf_)); return ObString(pos, buf_); } -ObTxNode::ObTxNode(const int64_t ls_id, - const ObAddr &addr, +ObTxNode::ObTxNode(const ObAddr &addr, MsgBus &msg_bus) : name_(name_buf_), addr_(addr), - ls_id_(ls_id), tenant_id_(1001), tenant_(0, 10), fake_part_trans_ctx_pool_(false, false, 4), @@ -115,14 +98,13 @@ ObTxNode::ObTxNode(const int64_t ls_id, std::bind(&ObTxNode::handle_msg_, this, std::placeholders::_1)), t3m_(), - fake_rpc_(&msg_bus, addr, &get_location_adapter_()), lock_memtable_(), - fake_tx_log_adapter_(nullptr) + fake_tx_log_adapter_(nullptr), + owns_log_adapter_(false) { fake_part_trans_ctx_pool_.init(); addr.to_string(name_buf_, sizeof(name_buf_)); msg_consumer_.set_name(name_); - role_ = Leader; provider_.tenant_freezer_ = &fake_tenant_freezer_; provider_.part_trans_ctx_obj_pool_ = &fake_part_trans_ctx_pool_; fake_shared_mem_alloc_mgr_.init(); @@ -147,8 +129,6 @@ ObTxNode::ObTxNode(const int64_t ls_id, // txn service int ret = OB_SUCCESS; OZ(txs_.init(addr, - &fake_rpc_, - &get_location_adapter_(), &get_gti_source_(), &get_ts_mgr_(), &schema_service_)); @@ -158,9 +138,8 @@ ObTxNode::ObTxNode(const int64_t ls_id, OZ(fake_lock_wait_mgr_.init()); provider_.lock_wait_mgr_ = &fake_lock_wait_mgr_; ls_service_.is_inited_ = true; - OZ(ls_service_.ls_map_.init(lib::ObMallocAllocator::get_instance())); + ls_service_.is_stopped_ = true; provider_.ls_service_ = &ls_service_; - OZ (create_memtable_(100000, memtable_)); { ObColDesc col_desc; col_desc.col_id_ = 1; @@ -171,7 +150,6 @@ ObTxNode::ObTxNode(const int64_t ls_id, columns_.push_back(col_desc); } OZ(msg_bus.regist(addr, *this)); - OZ(drop_msg_type_set_.create(16)); FAST_FAIL(); } ObTxDescGuard ObTxNode::get_tx_guard() { @@ -186,23 +164,24 @@ int ObTxNode::start() { ObTenantEnv::set_tenant(&tenant_); share::g_mp = &provider_; - if (ObTxNodeRole::Leader == role_) { + if (nullptr == fake_tx_log_adapter_) { fake_tx_log_adapter_ = new ObFakeTxLogAdapter(); + owns_log_adapter_ = true; OZ(fake_tx_log_adapter_->start()); } get_ts_mgr_().reset(); OZ(msg_consumer_.start()); OZ(txs_.start()); - OZ(create_ls_(ls_id_)); - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - OZ(txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id_, ls_tx_ctx_mgr)); - if (ObTxNodeRole::Leader == role_) { - OZ(ls_tx_ctx_mgr->switch_to_leader()); - wait_all_redolog_applied(); - CK(ls_tx_ctx_mgr->is_master_()); - } - if (ls_tx_ctx_mgr) { - fake_tx_table_.tx_ctx_table_.ls_tx_ctx_mgr_ = ls_tx_ctx_mgr; + OZ(txs_.tx_ctx_mgr_.create_context_manager(&fake_tx_table_, + &fake_lock_table_, + *fake_ls_.get_tx_svr(), + (ObITxLogParam*)0x01, + fake_tx_log_adapter_)); + fake_ls_.get_tx_svr()->online(); + OZ(publish_fake_ls_()); + if (OB_SUCC(ret)) { + ObLSTxCtxMgr &tx_ctx_mgr = txs_.tx_ctx_mgr_.get_tx_ctx_manager(); + fake_tx_table_.tx_ctx_table_.ls_tx_ctx_mgr_ = &tx_ctx_mgr; fake_tx_table_.is_inited_ = true; fake_tx_table_.ls_ = &fake_ls_; fake_tx_table_.online(); @@ -212,57 +191,11 @@ int ObTxNode::start() { fake_ls_.tx_table_.is_inited_ = true; fake_ls_.tx_table_.online(); fake_ls_.ls_meta_.clog_checkpoint_scn_ = share::SCN::max_scn(); - } else { - abort(); + OZ(create_memtable_(100000, memtable_)); } return ret; } -struct MsgInfo { - void *msg_ptr_ = NULL; - int64_t recv_time_ = 0; - ObAddr sender_; - bool is_callback_msg_ = false; - bool is_sync_msg_ = false; - TxMsgCallbackMsg callback_msg_; - int16_t msg_type_; - int64_t size_; - const char* buf_; - TO_STRING_KV(K_(msg_ptr), - K_(recv_time), - K_(sender), - K_(is_callback_msg), - K_(is_sync_msg), - K_(msg_type), - K_(callback_msg), - K_(size)); -}; - -int get_msg_info(ObTxNode::MsgPack *pkt, MsgInfo& msg_info) -{ - int ret = OB_SUCCESS; - const char* buf = pkt->body_.ptr(); - int64_t size = pkt->body_.length(); - int64_t pos = 0; - char cat = 0; int16_t pcode = 0; - OZ (serialization::decode(buf, size, pos, cat)); - if (cat == 1) { - msg_info.is_callback_msg_ = true; - OZ (msg_info.callback_msg_.deserialize(buf, size, pos)); - msg_info.msg_type_ = msg_info.callback_msg_.type_; - } else { - OZ (serialization::decode(buf, size, pos, pcode)); - msg_info.msg_type_ = pcode; - msg_info.buf_ = buf + pos; - msg_info.size_ = size - pos; - } - msg_info.is_sync_msg_ = pkt->is_sync_msg_; - msg_info.msg_ptr_ = (void*)pkt->body_.ptr(); - msg_info.recv_time_ = pkt->recv_time_; - msg_info.sender_ = pkt->sender_; - return ret; -} - void ObTxNode::dump_msg_queue_() { int ret = OB_SUCCESS; @@ -270,9 +203,9 @@ void ObTxNode::dump_msg_queue_() int i = 0; while(OB_SUCC(msg_queue_.pop((ObLink*&)msg))) { ++i; - MsgInfo msg_info; - OZ (get_msg_info(msg, msg_info)); - TRANS_LOG(INFO,"[dump_msg]", K(i), K(msg_info), K(ret), KPC(this)); + TRANS_LOG(INFO,"[dump_msg]", K(i), K(ret), KPC(this)); + ob_free((void*)const_cast(msg->body_.ptr())); + delete msg; } } @@ -301,18 +234,18 @@ ObTxNode::~ObTxNode() __attribute__((optnone)) { TRANS_LOG(INFO, "destroy TxNode", KPC(this)); ObTenantEnv::set_tenant(&tenant_); share::g_mp = &provider_; - OZ(txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(fake_tx_table_.tx_ctx_table_.ls_tx_ctx_mgr_)); fake_tx_table_.tx_ctx_table_.ls_tx_ctx_mgr_ = nullptr; bool is_tx_clean = false; int retry_cnt = 0; do { usleep(2000); - txs_.block_tx(ls_id_, is_tx_clean); + txs_.block_tx(is_tx_clean); } while(!is_tx_clean && ++retry_cnt < 1000); OX(txs_.stop()); OZ(txs_.wait_()); - OZ(drop_ls_(ls_id_)); - if (role_ == Leader && fake_tx_log_adapter_) { + OZ(txs_.tx_ctx_mgr_.remove_context_manager(true)); + provider_.trans_service_ = nullptr; + if (owns_log_adapter_ && fake_tx_log_adapter_) { fake_tx_log_adapter_->stop(); fake_tx_log_adapter_->wait(); fake_tx_log_adapter_->destroy(); @@ -320,24 +253,15 @@ ObTxNode::~ObTxNode() __attribute__((optnone)) { msg_consumer_.stop(); msg_consumer_.wait(); dump_msg_queue_(); - //fake_tx_table_.is_inited_ = false; if (memtable_) { delete memtable_; + memtable_ = nullptr; } - if (role_ == Leader && fake_tx_log_adapter_) { + OZ(retire_fake_ls_()); + ls_service_.destroy(); + if (owns_log_adapter_ && fake_tx_log_adapter_) { delete fake_tx_log_adapter_; } - fake_ls_.ls_meta_.ls_id_ = ObLSID(1001); - // seekdb runs a single sys tenant that is never deleted. Under the global - // tenant-context model (g_tenant_ptr), this test's multiple ObTxNodes share - // one global tenant pointer, so a node's background threads may switch the - // pointer concurrently and a sibling node may already have cleaned up the - // shared LS/tx state. These teardown steps are therefore best-effort: - // tolerate benign cleanup errors (e.g. OB_LS_NOT_EXIST) instead of aborting. - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "ignore best-effort TxNode teardown error", K(ret)); - ret = OB_SUCCESS; - } FAST_FAIL(); ObTenantEnv::set_tenant(NULL); } @@ -350,65 +274,36 @@ int ObTxNode::create_memtable_(const int64_t tablet_id, memtable::ObMemtable *&m table_key.tablet_id_ = tablet_id; table_key.scn_range_.start_scn_.convert_for_gts(100); table_key.scn_range_.end_scn_.set_max(); - ObLSHandle ls_handle; - ls_handle.set_ls(ls_service_.ls_map_, fake_ls_, ObLSGetMod::DATA_MEMTABLE_MOD); - OZ (t->init(table_key, ls_handle, &fake_freezer_, &fake_memtable_mgr_, 0, 0)); + ObLS *ls = nullptr; + OZ(ls_service_.get_ls(ls)); + OZ(t->init(table_key, ls, &fake_freezer_, &fake_memtable_mgr_, 0, 0)); if (OB_SUCC(ret)) { mt = t; } else { delete t; } return ret; } -int ObTxNode::create_ls_(const ObLSID ls_id) { +int ObTxNode::publish_fake_ls_() +{ int ret = OB_SUCCESS; - OZ(txs_.tx_ctx_mgr_.create_ls(ls_id, - &fake_tx_table_, - &fake_lock_table_, - *fake_ls_.get_tx_svr(), - (ObITxLogParam*)0x01, - fake_tx_log_adapter_)); - if (Leader == role_) { - OZ(get_location_adapter_().fill(ls_id, addr_)); + if (OB_NOT_NULL(ls_service_.ls_)) { + ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "fake ls is already published", K(ret), KP(ls_service_.ls_)); + } else { + ls_service_.ls_ = &fake_ls_; } - fake_ls_.ls_meta_.ls_id_ = ls_id; - fake_ls_.get_tx_svr()->online(); - fake_ls_.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD); - share::g_mp->ls_service()->ls_map_.add_ls(fake_ls_); - return ret; -} - -int ObTxNode::drop_ls_(const ObLSID ls_id) { - int ret = OB_SUCCESS; - OZ(txs_.tx_ctx_mgr_.remove_ls(ls_id, true)); - get_location_adapter_().remove(ls_id); - OZ(share::g_mp->ls_service()->ls_map_.del_ls(ls_id)); return ret; } -int ObTxNode::recv_msg_callback_(TxMsgCallbackMsg &msg) +int ObTxNode::retire_fake_ls_() { - TRANS_LOG(INFO, "recv msg callback", K(msg), KPC(this)); - ObTenantEnv::set_tenant(&tenant_); - share::g_mp = &provider_; int ret = OB_SUCCESS; - switch(msg.type_) { - case TxMsgCallbackMsg::SAVEPOINT_ROLLBACK: - // ignore, has changed to use async resp msg - break; - case TxMsgCallbackMsg::NORMAL: - OZ(txs_.handle_trans_msg_callback(msg.sender_ls_id_, - msg.receiver_ls_id_, - msg.tx_id_, - msg.orig_msg_type_, - msg.tx_rpc_result_.status_, - msg.receiver_addr_, - msg.request_id_, - msg.tx_rpc_result_.private_data_)); - break; - default: + if (OB_NOT_NULL(ls_service_.ls_) && ls_service_.ls_ != &fake_ls_) { ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "unexpected ls in fake ls service", K(ret), KP(ls_service_.ls_)); + } else { + ls_service_.ls_ = nullptr; } - TRANS_LOG(INFO, "handle msg callback done", K(ret), K(msg), KPC(this)); return ret; } @@ -446,66 +341,8 @@ int ObTxNode::handle_msg_(MsgPack *pkt) ObTenantEnv::set_tenant(&tenant_); share::g_mp = &provider_; TRANS_LOG(INFO, "begin to handle_msg", "msg_ptr", OB_P(pkt->body_.ptr()), KPC(this)); - int ret = OB_SUCCESS; - MsgInfo msg_info; - OZ (get_msg_info(pkt, msg_info)); - auto sender = msg_info.sender_; - const char* buf = msg_info.buf_; - int64_t size = msg_info.size_; - int64_t pos = 0; - if (OB_SUCC(ret) && msg_info.is_callback_msg_) { - return recv_msg_callback_(msg_info.callback_msg_); - } - int16_t msg_type = msg_info.msg_type_; - if (OB_HASH_EXIST == drop_msg_type_set_.exist_refactored(msg_type)) { - TRANS_LOG(WARN, "drop msg", K(msg_type), KPC(this)); - return OB_SUCCESS; - } - switch (msg_type) { -#define TX_MSG_HANDLER__(t, clz, func) \ - case t: \ - { \ - clz msg; \ - ObTransRpcResult rslt; \ - OZ(msg.deserialize(buf, size, pos)); \ - TRANS_LOG(TRACE, "handle_msg::", K(msg), KPC(this)); \ - auto status = txs_.func(msg, rslt); \ - rslt.status_ = status; \ - OZ(fake_rpc_.send_msg_callback(sender, msg, rslt)); \ - break; \ - } - TX_MSG_HANDLER__(TX_COMMIT, ObTxCommitMsg, handle_trans_commit_request); - TX_MSG_HANDLER__(TX_COMMIT_RESP, ObTxCommitRespMsg, handle_trans_commit_response); - TX_MSG_HANDLER__(TX_ABORT, ObTxAbortMsg, handle_trans_abort_request); - TX_MSG_HANDLER__(KEEPALIVE, ObTxKeepaliveMsg, handle_trans_keepalive); - TX_MSG_HANDLER__(KEEPALIVE_RESP, ObTxKeepaliveRespMsg, handle_trans_keepalive_response); - TX_MSG_HANDLER__(ROLLBACK_SAVEPOINT_RESP, ObTxRollbackSPRespMsg, handle_sp_rollback_response); -#undef TX_MSG_HANDLER__ - case ROLLBACK_SAVEPOINT: - { - ObTxRollbackSPMsg msg; - obcall::ObTxRpcRollbackSPResult rslt; - OZ(msg.deserialize(buf, size, pos)); - TRANS_LOG(TRACE, "handle_msg", K(msg), KPC(this)); - OZ(txs_.handle_sp_rollback_request(msg, rslt), msg); - break; - } - case TX_2PC_PREPARE_REQ: - case TX_2PC_PREPARE_RESP: - case TX_2PC_PRE_COMMIT_REQ: - case TX_2PC_PRE_COMMIT_RESP: - case TX_2PC_COMMIT_REQ: - case TX_2PC_COMMIT_RESP: - case TX_2PC_ABORT_REQ: - case TX_2PC_ABORT_RESP: - case TX_2PC_CLEAR_REQ: - case TX_2PC_CLEAR_RESP: - OZ(txs_.handle_tx_batch_req(msg_type, buf + pos, size - pos, false)); - break; - default: - ret = OB_NOT_SUPPORTED; - } - if (msg_info.is_sync_msg_) { + int ret = OB_NOT_SUPPORTED; + if (pkt->is_sync_msg_) { pkt->resp_ready_ = true; pkt->cond_.signal(); } else { @@ -537,7 +374,6 @@ int ObTxNode::read(const ObTxReadSnapshot &snapshot, share::g_mp = &provider_; ObStoreCtx read_store_ctx; read_store_ctx.ls_ = &fake_ls_; - read_store_ctx.ls_id_ = ls_id_; OZ(txs_.get_read_store_ctx(snapshot, false, 5000ll * 1000, read_store_ctx)); // HACK, refine: mock LS's each member in some way read_store_ctx.mvcc_acc_ctx_.tx_table_guards_.tx_table_guard_.init(&fake_tx_table_); @@ -606,7 +442,7 @@ int ObTxNode::atomic_write(ObTxDesc &tx, const int64_t key, const int64_t value, OZ(create_implicit_savepoint(tx, tx_param, sp, true)); OZ(write(tx, key, value)); if (sp.is_valid() && OB_FAIL(ret)) { - OZ(rollback_to_implicit_savepoint(tx, sp, expire_ts, nullptr)); + OZ(rollback_to_implicit_savepoint(tx, sp, expire_ts, false)); } return ret; } @@ -637,7 +473,6 @@ int ObTxNode::write(ObTxDesc &tx, ObITable *mtb = memtable_; iter->add_table(mtb); write_store_ctx.ls_ = &fake_ls_; - write_store_ctx.ls_id_ = ls_id_; write_store_ctx.table_iter_ = iter; write_store_ctx.branch_ = branch; concurrent_control::ObWriteFlag write_flag; @@ -703,7 +538,6 @@ int ObTxNode::write_begin(ObTxDesc &tx, iter->reset(); ObITable *mtb = memtable_; iter->add_table(mtb); - write_store_ctx.ls_id_ = ls_id_; write_store_ctx.ls_ = &fake_ls_; write_store_ctx.table_iter_ = iter; concurrent_control::ObWriteFlag write_flag; @@ -793,7 +627,6 @@ int ObTxNode::replay(const void *buffer, share::SCN log_scn; log_scn.convert_for_tx(ts_ns); ObFakeTxReplayExecutor executor(&fake_ls_, - ls_id_, fake_ls_.get_tx_svr(), lsn, log_scn, diff --git a/unittest/storage/tx/it/tx_node.h b/unittest/storage/tx/it/tx_node.h index 62c37da51..bc49dca49 100644 --- a/unittest/storage/tx/it/tx_node.h +++ b/unittest/storage/tx/it/tx_node.h @@ -22,7 +22,7 @@ #include "lib/objectpool/ob_server_object_pool.h" #include "storage/tx/ob_trans_define.h" #include "storage/tx/ob_trans_service.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" #include "share/rc/ob_tenant_base.h" #include "observer/omt/ob_tenant.h" #include "storage/tablelock/ob_lock_memtable.h" @@ -38,7 +38,6 @@ #include "../mock_utils/msg_bus.h" #include "../mock_utils/basic_fake_define.h" -#include "../mock_utils/ob_fake_tx_rpc.h" #include "storage/allocator/ob_shared_memory_allocator_mgr.h" #include "sql/optimizer/stat/ob_opt_stat_monitor_manager.h" #include "storage/memtable/ob_lock_wait_mgr.h" @@ -128,11 +127,6 @@ class ObTxDescGuard { ObTxDesc *tx_desc_; }; -enum ObTxNodeRole { - Leader = 1, - Follower = 2, -}; - // Single-tenant seekdb dissolves ObTenantBase::set/get module slots; // modules now route through share::g_mp (ObIModuleProvider). This test injects // its fakes by overriding the relevant getters and pointing g_mp at this node. @@ -141,7 +135,7 @@ class FakeModuleProvider : public share::ObIModuleProvider { public: storage::ObTenantFreezer *tenant_freezer() override { return tenant_freezer_; } - ObPartTransCtxObjPool *part_trans_ctx_obj_pool() override { return part_trans_ctx_obj_pool_; } + ObTxCtxObjPool *part_trans_ctx_obj_pool() override { return part_trans_ctx_obj_pool_; } share::ObSharedMemAllocMgr *shared_mem_alloc_mgr() override { return shared_mem_alloc_mgr_; } transaction::ObTransService *trans_service() override { return trans_service_; } common::ObOptStatMonitorManager *opt_stat_monitor_manager() override { return opt_stat_monitor_manager_; } @@ -149,7 +143,7 @@ class FakeModuleProvider : public share::ObIModuleProvider storage::ObLSService *ls_service() override { return ls_service_; } storage::ObTenantFreezer *tenant_freezer_ = nullptr; - ObPartTransCtxObjPool *part_trans_ctx_obj_pool_ = nullptr; + ObTxCtxObjPool *part_trans_ctx_obj_pool_ = nullptr; share::ObSharedMemAllocMgr *shared_mem_alloc_mgr_ = nullptr; transaction::ObTransService *trans_service_ = nullptr; common::ObOptStatMonitorManager *opt_stat_monitor_manager_ = nullptr; @@ -161,7 +155,7 @@ class FakeModuleProvider : public share::ObIModuleProvider // Single-tenant seekdb routes module access through share::g_mp; a fake IT harness // must: (1) wire the provider's shared_mem_alloc_mgr so alloc/throttle resolve, // (2) point g_mp at it, (3) set g_modules_ready so MOD_SCOPE-gated paths (e.g. -// ObPartTransCtx::submit_log_impl_) run instead of being silently skipped -- the +// ObTxCtx::submit_log_impl_) run instead of being silently skipped -- the // real observer sets this true during tenant start (omt/ob_tenant.cpp), and // (4) init the alloc mgr. Intentionally never clears g_mp: the providers are static // and outlive every fixture, and clearing it in TearDown would crash members that @@ -177,18 +171,17 @@ inline void publish_test_module_provider(FakeModuleProvider &provider, class ObTxNode final : public MsgEndPoint { public: - ObTxNode(const int64_t ls_id, - const ObAddr &addr, + ObTxNode(const ObAddr &addr, MsgBus &msg_bus); ~ObTxNode(); int start(); - void set_as_follower_replica(ObTxNode& leader_replica) { - role_ = ObTxNodeRole::Follower; - fake_tx_log_adapter_ = leader_replica.fake_tx_log_adapter_; + void set_replay_source(ObTxNode &source) { + fake_tx_log_adapter_ = source.fake_tx_log_adapter_; + owns_log_adapter_ = false; } public: - TO_STRING_KV(KP(this), K(addr_), K_(ls_id), K(msg_queue_.size())); + TO_STRING_KV(KP(this), K(addr_), K(msg_queue_.size())); ObString get_identifer_str(); ObTxDescGuard get_tx_guard(); // the simple r/w interface @@ -234,10 +227,10 @@ class ObTxNode final : public MsgEndPoint { DELEGATE_TENANT_WITH_RET(txs_, release_explicit_savepoint, int); DELEGATE_TENANT_WITH_RET(txs_, rollback_to_implicit_savepoint, int); DELEGATE_TENANT_WITH_RET(txs_, interrupt, int); - int get_tx_ctx(const share::ObLSID &ls_id, const ObTransID &tx_id, ObPartTransCtx *&ctx) { - return txs_.tx_ctx_mgr_.get_tx_ctx(ls_id, tx_id, false, ctx); + int get_tx_ctx(const ObTransID &tx_id, ObTxCtx *&ctx) { + return txs_.tx_ctx_mgr_.get_tx_ctx(tx_id, false, ctx); } - int revert_tx_ctx(ObPartTransCtx *ctx) { return txs_.tx_ctx_mgr_.revert_tx_ctx(ctx); } + int revert_tx_ctx(ObTxCtx *ctx) { return txs_.tx_ctx_mgr_.revert_tx_ctx(ctx); } public: struct MsgPack : ObLink { MsgPack(const ObAddr &addr, ObString &body, bool is_sync_msg = false) @@ -256,39 +249,32 @@ class ObTxNode final : public MsgEndPoint { int handle_msg_(MsgPack *pkt); private: int create_memtable_(const int64_t tablet_id, memtable::ObMemtable *& mt); - int create_ls_(const ObLSID ls_id); - int drop_ls_(const ObLSID ls_id); - int recv_msg_callback_(TxMsgCallbackMsg &msg); + int publish_fake_ls_(); + int retire_fake_ls_(); void wait_all_redolog_applied() { while (fake_tx_log_adapter_->get_inflight_cnt() != 0) usleep(1000); } int wait_all_tx_ctx_is_destoryed() { int ret = OB_SUCCESS; - ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; - OZ(txs_.tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id_, ls_tx_ctx_mgr)); + ObLSTxCtxMgr &tx_ctx_mgr = txs_.tx_ctx_mgr_.get_tx_ctx_manager(); int i = 0; - int tx_count = ls_tx_ctx_mgr->get_tx_ctx_count(); + int tx_count = tx_ctx_mgr.get_tx_ctx_count(); for (i = 0; tx_count > 0 && i < 2000; ++i) { - tx_count = ls_tx_ctx_mgr->get_tx_ctx_count(); + tx_count = tx_ctx_mgr.get_tx_ctx_count(); usleep(500); } if (2000 == i) { ret = OB_ERR_UNEXPECTED; LOG_INFO("wait all tx ctx destoryed fail, print all tx:", K(tx_count)); const bool verbose = true; - ls_tx_ctx_mgr->print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); + tx_ctx_mgr.print_all_tx_ctx(ObLSTxCtxMgr::MAX_HASH_ITEM_PRINT, verbose); LOG_INFO("print all tx end", K(ret)); } - OZ(txs_.tx_ctx_mgr_.revert_ls_tx_ctx_mgr(ls_tx_ctx_mgr)); return ret; } void dump_msg_queue_(); public: - static ObFakeLocationAdapter &get_location_adapter_() { - static ObFakeLocationAdapter l; - return l; - } static ObFakeGtiSource &get_gti_source_() { static ObFakeGtiSource txIdGenerator; return txIdGenerator; @@ -302,26 +288,15 @@ class ObTxNode final : public MsgEndPoint { int64_t ts_after_us(int64_t d) const { return ObTimeUtility::current_time() + d; } int64_t ts_after_ms(int64_t d) const { return ObTimeUtility::current_time() + d * 1000; } -private: - static void reset_localtion_adapter() { - get_location_adapter_().reset(); - } public: - void add_drop_msg_type(TX_MSG_TYPE type) { - drop_msg_type_set_.set_refactored(type); - } - void del_drop_msg_type(TX_MSG_TYPE type) { - drop_msg_type_set_.erase_refactored(type); - } void wait_all_msg_consumed(); void wait_tx_log_synced(); public: ObString name_; char name_buf_[MAX_IP_PORT_LENGTH]; ObAddr addr_; - ObLSID ls_id_; int64_t tenant_id_; omt::ObTenant tenant_; - common::ObServerObjectPool fake_part_trans_ctx_pool_; + common::ObServerObjectPool fake_part_trans_ctx_pool_; ObTransService txs_; memtable::ObMemtable *memtable_; ObSEArray columns_; @@ -330,7 +305,6 @@ class ObTxNode final : public MsgEndPoint { QueueConsumer msg_consumer_; // fake objects storage::ObTenantMetaMemMgr t3m_; - ObFakeTransRpc fake_rpc_; ObFakeGtiSource fake_gti_source_; ObFakeTsMgr fake_ts_mgr_; share::schema::ObMultiVersionSchemaService schema_service_; @@ -341,14 +315,13 @@ class ObTxNode final : public MsgEndPoint { ObSharedMemAllocMgr fake_shared_mem_alloc_mgr_; ObLS fake_ls_; ObFreezer fake_freezer_; - ObTxNodeRole role_; ObFakeTxLogAdapter* fake_tx_log_adapter_; + bool owns_log_adapter_; ObTabletMemtableMgr fake_memtable_mgr_; ObOptStatMonitorManager fake_opt_stat_mgr_; ObLockWaitMgr fake_lock_wait_mgr_; ObLSService ls_service_; FakeModuleProvider provider_; - common::hash::ObHashSet drop_msg_type_set_; std::function extra_msg_handler_; char buf_[256]; }; diff --git a/unittest/storage/tx/mock_utils/basic_fake_define.h b/unittest/storage/tx/mock_utils/basic_fake_define.h index c7e30474d..0ae06fb0c 100644 --- a/unittest/storage/tx/mock_utils/basic_fake_define.h +++ b/unittest/storage/tx/mock_utils/basic_fake_define.h @@ -23,12 +23,11 @@ #include "storage/tx/ob_trans_define.h" #include "storage/tx_table/ob_tx_table.h" #include "lib/utility/ob_defer.h" -#include "storage/tx/ob_location_adapter.h" #include "storage/tx/ob_tx_log_adapter.h" #include "storage/tx/ob_ts_mgr.h" #include "storage/tx/ob_gti_source.h" #include "storage/tx/ob_tx_replay_executor.h" -#include "storage/tx/ob_trans_part_ctx.h" +#include "storage/tx/ob_tx_ctx.h" namespace oceanbase { using namespace share; @@ -135,64 +134,6 @@ class ObFakeTxTable : public ObTxTable { ObFakeTxDataTable tx_data_table_; }; -class ObFakeLocationAdapter : public ObILocationAdapter -{ -public: - ObFakeLocationAdapter() { - ls_addr_table_.create(16, ObModIds::TEST); - } - int init(share::schema::ObMultiVersionSchemaService *schema_service, - share::ObLocationService *location_service) { return OB_SUCCESS; } - void destroy() { } - void reset() { ls_addr_table_.reuse(); } - int get_leader(const int64_t cluster_id, - const share::ObLSID &ls_id, - common::ObAddr &leader) - { - int ret = OB_SUCCESS; - const common::ObAddr *a = ls_addr_table_.get(ls_id); - if (a == NULL) { ret = OB_ENTRY_NOT_EXIST; } - else { leader = *a; } - return ret; - } - int nonblock_get_leader(const int64_t cluster_id, - const share::ObLSID &ls_id, - common::ObAddr &leader) - { return get_leader(cluster_id, ls_id, leader); } - int nonblock_get(const int64_t cluster_id, - const share::ObLSID &ls_id, - share::ObLSLocation &location) - { - int ret = OB_SUCCESS; - common::ObAddr leader; - OZ(get_leader(cluster_id, ls_id, leader)); - ObLSReplicaLocation rep_loc; - ObLSRestoreStatus restore_status(ObLSRestoreStatus::Status::NONE); - auto p = ObReplicaProperty::create_property(100); - OZ(rep_loc.init(leader, ObRole::LEADER, 10000, ObReplicaType::REPLICA_TYPE_FULL, p, restore_status, 1)); - OZ(location.add_replica_location(rep_loc)); - return ret; - } -public: - // maintains functions - int fill(const share::ObLSID ls_id, const common::ObAddr &addr) - { - return ls_addr_table_.set_refactored(ls_id, addr); - } - int remove(const share::ObLSID ls_id) - { - return ls_addr_table_.erase_refactored(ls_id); - } - int update_localtion(const share::ObLSID ls_id, const common::ObAddr &addr) - { - ls_addr_table_.erase_refactored(ls_id); - return ls_addr_table_.set_refactored(ls_id, addr); - } - -private: - common::hash::ObHashMap ls_addr_table_; -}; - class ObFakeGtiSource : public ObIGtiSource { int get_trans_id(int64_t &trans_id) { @@ -213,163 +154,57 @@ class ObFakeTsMgr : public ObTsMgr void repair_get_gts_error() { get_gts_error_ = OB_SUCCESS; } -public: void reset() { get_gts_error_ = OB_SUCCESS; - elapse_waiting_mode_ = false; - get_gts_waiting_mode_ = false; } - int update_gts(const uint64_t tenant_id, const int64_t gts, bool &update) { return OB_SUCCESS; } + int get_gts(const MonotonicTs stc, - ObTsCbTask *task, share::SCN &scn, - MonotonicTs &receive_gts_ts) - { - int ret = OB_SUCCESS; - int gts = 0; - TRANS_LOG(INFO, "get gts begin", K(gts_), K(>s_)); - if (get_gts_error_) { - ret = get_gts_error_; - } else { - gts = ATOMIC_AAF(>s_, 1); - scn.convert_for_gts(gts); - if (task != nullptr && ATOMIC_LOAD(&get_gts_waiting_mode_)) { - get_gts_waiting_queue_.push(task); - ret = OB_EAGAIN; - } - } - TRANS_LOG(INFO, "get gts end", K(ret), K(gts_), K(gts), K(get_gts_waiting_mode_)); - return ret; - } - - int get_gts_sync(const MonotonicTs stc, - const int64_t timeout_us, - share::SCN &scn, - MonotonicTs &receive_gts_ts) - { - int ret = OB_SUCCESS; - const int64_t expire_ts = ObClockGenerator::getClock() + timeout_us; - - do { - int64_t n = ObClockGenerator::getClock(); - if (n >= expire_ts) { - ret = OB_TIMEOUT; - } else if (OB_FAIL(get_gts(stc, NULL, scn, receive_gts_ts))) { - if (OB_EAGAIN == ret) { - ob_usleep(500); - } - } - } while (OB_EAGAIN == ret); - - return ret; - return get_gts(stc, NULL, scn, receive_gts_ts); - } - - int get_gts(ObTsCbTask *task, share::SCN &scn) { - if (get_gts_error_) { return get_gts_error_; } - return OB_SUCCESS; - } - int get_ts_sync(const int64_t timeout_ts, - share::SCN &scn, bool &is_external_consistent) { return OB_SUCCESS; } - int get_ts_sync(const int64_t timeout_ts, - share::SCN &scn) { return OB_SUCCESS; } - int wait_gts_elapse(const share::SCN &scn, ObTsCbTask *task, - bool &need_wait) + MonotonicTs &receive_gts_ts) override { - TRANS_LOG(INFO, "wait_gts_elapse begin", K(gts_), K(scn)); - int ret = OB_SUCCESS; - if (task != nullptr && ATOMIC_LOAD(&elapse_waiting_mode_)) { - elapse_queue_.push(task); - callback_gts_ = scn.get_val_for_gts()+1; - need_wait = true; - } else { - update_fake_gts(scn.get_val_for_gts()); + UNUSED(stc); + int ret = get_gts_error_; + if (OB_SUCC(ret)) { + const int64_t gts = ATOMIC_AAF(>s_, 1); + ret = scn.convert_for_gts(gts); + receive_gts_ts = MonotonicTs::current_time(); } - TRANS_LOG(INFO, "wait_gts_elapse end", K(gts_), K(scn)); return ret; } - int wait_gts_elapse(const share::SCN &scn) + int get_gts(share::SCN &scn) override { - int ret = OB_SUCCESS; - if (scn.get_val_for_gts() > gts_) { - ret = OB_EAGAIN; + int ret = get_gts_error_; + if (OB_SUCC(ret)) { + ret = scn.convert_for_gts(ATOMIC_AAF(>s_, 1)); } return ret; } - int remove_dropped_tenant() { - return OB_SUCCESS; - } - - int interrupt_gts_callback_for_ls_offline(const share::ObLSID ls_id) { - UNUSED(ls_id); - return OB_SUCCESS; - } - - int update_base_ts(const int64_t base_ts) { return OB_SUCCESS; } - int get_base_ts(int64_t &base_ts) { return OB_SUCCESS; } - bool is_external_consistent() { return true; } - int get_gts_and_type(const MonotonicTs stc, int64_t >s, - int64_t &ts_type) { return OB_SUCCESS; } - int64_t gts_ = 100; - int64_t callback_gts_ = 100; - -public: - void update_fake_gts(int64_t gts) + int get_gts_sync(const int64_t timeout_us, share::SCN &scn) override { - TRANS_LOG(INFO, "update fake gts", K(gts_), K(gts), K(>s_)); - gts_ = gts; + UNUSED(timeout_us); + return get_gts(scn); } - void set_elapse_waiting_mode() { ATOMIC_SET(&elapse_waiting_mode_, true); } - void clear_elapse_waiting_mode() { ATOMIC_SET(&elapse_waiting_mode_, false); } - - void set_get_gts_waiting_mode() { ATOMIC_SET(&get_gts_waiting_mode_, true); } - void clear_get_gts_waiting_mode() { ATOMIC_SET(&get_gts_waiting_mode_, false); } - - void elapse_callback() + int wait_gts_elapse(const share::SCN &scn) override { - if (callback_gts_ > gts_) { - update_fake_gts(callback_gts_); - } - while(true) { - ObLink *task = elapse_queue_.pop(); - if (task) { - const MonotonicTs srr(MonotonicTs::current_time()); - share::SCN ts; - ts.convert_for_gts(gts_); - const MonotonicTs receive_gts_ts(gts_); - const ObGTSCacheTaskType task_type = WAIT_GTS_ELAPSING; - static_cast(task)->gts_elapse_callback(srr, ts); - } else { - break; - } + int ret = get_gts_error_; + if (OB_SUCC(ret)) { + update_fake_gts(scn.get_val_for_gts() + 1); } + return ret; } - void get_gts_callback() + void update_fake_gts(const int64_t gts) { - while(true) { - ObLink *task = get_gts_waiting_queue_.pop(); - if (task) { - const MonotonicTs srr(MonotonicTs::current_time()); - share::SCN ts; - ts.convert_for_gts(gts_); - const MonotonicTs receive_gts_ts(gts_); - const ObGTSCacheTaskType task_type = GET_GTS; - static_cast(task)->get_gts_callback(srr, ts, receive_gts_ts); - } else { - break; - } + int64_t old_gts = ATOMIC_LOAD(>s_); + while (old_gts < gts && !ATOMIC_BCAS(>s_, old_gts, gts)) { + old_gts = ATOMIC_LOAD(>s_); } } -public: - ObSpScLinkQueue elapse_queue_; - ObSpScLinkQueue get_gts_waiting_queue_; - bool elapse_waiting_mode_ = false; - bool get_gts_waiting_mode_ = false; + int64_t gts_ = 100; }; class ObFakeTxLogAdapter : public ObITxLogAdapter, public share::ObThreadPool @@ -518,12 +353,6 @@ class ObFakeTxLogAdapter : public ObITxLogAdapter, public share::ObThreadPool return ret; } - int get_role(bool &is_leader, int64_t &epoch) { - is_leader = true; - epoch = 1; - return OB_SUCCESS; - } - int get_max_decided_scn(share::SCN &scn) { int ret = OB_SUCCESS; share::SCN min_unreplayed_scn; @@ -569,10 +398,6 @@ class ObFakeTxLogAdapter : public ObITxLogAdapter, public share::ObThreadPool } return OB_SUCCESS; } - int get_palf_committed_max_scn(share::SCN &scn) const { - scn = max_committed_scn_; - return OB_SUCCESS; - } int get_append_mode_initial_scn(share::SCN &ref_scn) { int ret = OB_SUCCESS; ref_scn = share::SCN::invalid_scn(); @@ -617,12 +442,11 @@ class ObFakeTxReplayExecutor : public ObTxReplayExecutor { public: ObFakeTxReplayExecutor(storage::ObLS *ls, - const share::ObLSID &ls_id, storage::ObLSTxService *ls_tx_srv, const palf::LSN &lsn, const share::SCN &log_timestamp, const logservice::ObLogBaseHeader &base_header) - : ObTxReplayExecutor(ls, ls_id, ls_tx_srv, lsn, log_timestamp, base_header) + : ObTxReplayExecutor(ls, ls_tx_srv, lsn, log_timestamp, base_header) { memtable_ = nullptr; } ~ObFakeTxReplayExecutor() { } int set_memtable(memtable::ObMemtable* memtable) @@ -641,7 +465,7 @@ class ObFakeTxReplayExecutor : public ObTxReplayExecutor { int ret = OB_SUCCESS; storage::ObStoreCtx storeCtx; - storeCtx.ls_id_ = ctx_->get_ls_id(); + storeCtx.ls_ = ls_; storeCtx.mvcc_acc_ctx_.tx_ctx_ = ctx_; storeCtx.mvcc_acc_ctx_.mem_ctx_ = mt_ctx_; storeCtx.tablet_id_ = row_head.tablet_id_; diff --git a/unittest/storage/tx/mock_utils/ob_fake_tx_rpc.h b/unittest/storage/tx/mock_utils/ob_fake_tx_rpc.h deleted file mode 100644 index f527bfd38..000000000 --- a/unittest/storage/tx/mock_utils/ob_fake_tx_rpc.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_TRANSACTION_TEST_OB_FAKE_TX_RPC_ -#define OCEANBASE_TRANSACTION_TEST_OB_FAKE_TX_RPC_ -#include "msg_bus.h" -#include "storage/tx/ob_trans_define.h" -#include "storage/tx/ob_trans_rpc.h" -#include "storage/tx/ob_location_adapter.h" -#include "storage/tx/ob_tx_log_adapter.h" -namespace oceanbase { -using namespace share; -namespace transaction { -struct TxMsgCallbackMsg { -private: - TxMsgCallbackMsg(const ObTxMsg &msg, - const ObAddr &addr) { - orig_msg_type_ = msg.type_; - sender_ls_id_ = msg.sender_; - receiver_ls_id_ = msg.receiver_; - epoch_ = msg.epoch_; - tx_id_ = msg.tx_id_; - receiver_addr_ = addr; - request_id_ = msg.request_id_; - } -public: - TO_STRING_KV(K_(orig_msg_type), - K_(sender_ls_id), - K_(receiver_ls_id), - K_(epoch), - K_(tx_id), - K_(receiver_addr), - K_(request_id)); - TxMsgCallbackMsg() {} - TxMsgCallbackMsg(const ObTxMsg &msg, - const ObAddr &addr, - const obcall::ObTxRpcRollbackSPResult &r) - :TxMsgCallbackMsg(msg, addr) - { - type_ = MSG_TYPE::SAVEPOINT_ROLLBACK; - sp_rollback_result_ = r; - } - TxMsgCallbackMsg(const ObTxMsg &msg, - const ObAddr &addr, - const obcall::ObTransRpcResult &r) - :TxMsgCallbackMsg(msg, addr) - { - type_ = MSG_TYPE::NORMAL; - tx_rpc_result_ = r; - } -#define SERIALIZE_TX_TEST_MSG_CB_(ACT) \ - LST_DO_CODE(ACT, type_, orig_msg_type_, \ - sender_ls_id_, receiver_ls_id_, \ - epoch_, tx_id_, receiver_addr_, request_id_, \ - sp_rollback_result_, \ - tx_rpc_result_); - int serialize(SERIAL_PARAMS) const { - int ret = OB_SUCCESS; - SERIALIZE_TX_TEST_MSG_CB_(OB_UNIS_ENCODE); - return ret; - } - int deserialize(DESERIAL_PARAMS) { - int ret = OB_SUCCESS; - SERIALIZE_TX_TEST_MSG_CB_(OB_UNIS_DECODE); - return ret; - } - int64_t get_serialize_size() const { - int64_t len = 0; - SERIALIZE_TX_TEST_MSG_CB_(OB_UNIS_ADD_LEN); - return len; - } - - enum MSG_TYPE { SAVEPOINT_ROLLBACK = 10, - NORMAL = 100 }; - int type_; - int16_t orig_msg_type_; - ObLSID sender_ls_id_; - ObLSID receiver_ls_id_; - int64_t epoch_; - ObTransID tx_id_; - ObAddr receiver_addr_; - int64_t request_id_; - obcall::ObTxRpcRollbackSPResult sp_rollback_result_; - ObTransRpcResult tx_rpc_result_; -}; - -class ObFakeTransRpc : public ObITransRpc { -public: - ObFakeTransRpc(MsgBus *msg_bus, - const ObAddr &my_addr, - ObILocationAdapter *location_adapter) - : addr_(my_addr), msg_bus_(msg_bus), location_adapter_(location_adapter) {} - int start() { return OB_SUCCESS; } - void stop() {} - void wait() {} - void destroy() {} - int post_msg(const ObAddr &server, ObTxMsg &msg) - { - int ret = OB_SUCCESS; - int64_t size = msg.get_serialize_size() + 1 /*for msg category*/ + sizeof(int16_t) /* for tx_msg.type_ */; - char *buf = (char*)ob_malloc(size, ObNewModIds::TEST); - buf[0] = 0; // 0 not callback msg - int64_t pos = 1; - int16_t msg_type = msg.type_; - OZ(serialization::encode(buf, size, pos, msg_type)); - OZ(msg.serialize(buf, size, pos)); - ObString msg_str(size, buf); - TRANS_LOG(INFO, "post_msg", "msg_ptr", OB_P(buf), K(msg), "receiver", server); - OZ(msg_bus_->send_msg(msg_str, addr_, server)); - return ret; - } - int post_msg(const share::ObLSID &p, ObTxMsg &msg) - { - int ret = OB_SUCCESS; - common::ObAddr leader; - OZ(location_adapter_->nonblock_get_leader(0, p, leader)); - OZ(post_msg(leader, msg)); - return ret; - } - int ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &msg, - ObAskTxStateFor4377RespMsg &resp) - { - return OB_SUCCESS; - } - template - int send_msg_callback(const ObAddr &recv, - const ObTxMsg &msg, - MSG_RESULT_T &rslt); -private: - ObAddr addr_; - MsgBus *msg_bus_; - ObILocationAdapter *location_adapter_; -}; - -template -int ObFakeTransRpc::send_msg_callback(const ObAddr &recv, - const ObTxMsg &msg, - MSG_RESULT_T &rslt) -{ - int ret = OB_SUCCESS; - TxMsgCallbackMsg rmsg(msg, addr_, rslt); - int64_t size = rmsg.get_serialize_size() + 1 /* for msg_category */; - char *buf = (char*)ob_malloc(size, ObNewModIds::TEST); - buf[0] = 1;// callback - int64_t pos = 1; - OZ(rmsg.serialize(buf, size, pos)); - ObString str(size, buf); - TRANS_LOG(INFO, "send_msg_callback", K(recv), "msg_ptr", OB_P(buf), "orig_msg", msg); - OZ(msg_bus_->send_msg(str, addr_, recv)); - return ret; -} - -} // transaction -} // oceanbase - -#endif //OCEANBASE_TRANSACTION_TEST_OB_FAKE_TX_RPC_ diff --git a/unittest/storage/tx/ob_mailbox.h b/unittest/storage/tx/ob_mailbox.h deleted file mode 100644 index adf0a1a97..000000000 --- a/unittest/storage/tx/ob_mailbox.h +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_STORAGE_TX_OB_MAILBOX -#define OCEANBASE_UNITTEST_STORAGE_TX_OB_MAILBOX - -#include -#include -#include -#include -#include "lib/ob_errno.h" -#include "lib/utility/ob_macro_utils.h" -#include "lib/utility/ob_print_utils.h" -#include "storage/tx/ob_committer_define.h" -#include "storage/tx/ob_tx_msg.h" - -namespace oceanbase -{ -namespace transaction -{ -template -class ObMailBoxMgr; - -template -class ObMail -{ -public: - int init(int64_t from, - int64_t to, - uint64_t size, - const MailType &mail) - { - if (NULL != mail_) { - std::free(mail_); - } - - from_ = from; - to_ = to; - size_ = size; - mail_ = (MailType*)std::malloc(size); - std::memcpy((void*)mail_, (void*)(&mail), size); - return OB_SUCCESS; - } - ObMail() - { - mail_ = NULL; - } - ObMail(const ObMail &other) - { - from_ = other.from_; - to_ = other.to_; - size_ = other.size_; - mail_ = (MailType*)std::malloc(size_); - std::memcpy((void*)mail_, (void*)(other.mail_), size_); - } - ~ObMail() - { - if (NULL != mail_) { - std::free(mail_); - } - } - ObMail& operator=(const ObMail& other) - { - if (NULL != mail_) { - std::free(mail_); - } - - from_ = other.from_; - to_ = other.to_; - size_ = other.size_; - mail_ = (MailType*)std::malloc(size_); - std::memcpy((void*)mail_, (void*)(other.mail_), size_); - return *this; - } - /* ObMail operator=(const ObMail& other) */ - /* { */ - /* if (NULL != mail_) { */ - /* std::free(mail_); */ - /* } */ - - /* from_ = other.from_; */ - /* to_ = other.to_; */ - /* size_ = other.size_; */ - /* mail_ = (MailType*)std::malloc(size_); */ - /* std::memcpy((void*)mail_, (void*)(other.mail_), size_); */ - /* return *this; */ - /* } */ - bool operator<(const ObMail& other) const - { - return from_ < other.from_ - || to_ < other.to_ - || size_ < other.size_ - || (size_ == other.size_ && memcmp((void*)mail_, (void*)other.mail_, size_) < 0); - } - /* ObMail& operator=(const ObMail &other) */ - /* { */ - /* from_ = other.from_; */ - /* to_ = other.to_; */ - /* size_ = other.size_; */ - /* mail_ = (MailType*)std::malloc(size_); */ - /* std::memcpy((void*)mail_, (void*)(other.mail_), size_); */ - /* } */ - int64_t from_; - int64_t to_; - uint64_t size_; - MailType* mail_; - - TO_STRING_KV(K_(from), K_(to), K_(*mail)); -}; - -template -class ObMailHandler -{ -public: - virtual int handle(const ObMail& mail) = 0; - int64_t sign_ = 39; - - virtual TO_STRING_KV(K_(sign)); -}; - -template -class ObMailBox -{ -public: - int64_t addr_; - std::deque> mailbox_; - ObMailHandler *ctx_; - ObMailBoxMgr *mailbox_mgr_; - ~ObMailBox() - { - mailbox_.clear(); - } - bool empty() { return mailbox_.empty(); } - int init(int64_t addr, - ObMailBoxMgr *mailbox_mgr, - ObMailHandler *ctx); - int handle(const bool must_have = true); - int handle_all(); - int send(const ObMail& mail, - const int64_t receiver); - int send_to_head(const ObMail& mail, - const int64_t receiver); - int fetch_mail(ObMail& mail); - int64_t to_string(char *buffer, const int64_t size) const; -}; - -template -class ObMailBoxMgr -{ -public: - int64_t counter_ = 0; - std::map*> mgr_; - std::set> cache_msg_; - int register_mailbox(int64_t &addr, - ObMailBox &mailbox, - ObMailHandler *ctx); - int send(const ObMail& mail, - const int64_t receive); - int send_to_head(const ObMail& mail, - const int64_t receive); - bool random_dup_and_send(); - void reset(); -}; - -template -int ObMailBox::init(int64_t addr, - ObMailBoxMgr *mailbox_mgr, - ObMailHandler *ctx) -{ - int ret = OB_SUCCESS; - - mailbox_.clear(); - addr_ = addr; - ctx_ = ctx; - mailbox_mgr_ = mailbox_mgr; - - return ret; -} - -template -int ObMailBox::fetch_mail(ObMail &mail) -{ - int ret = OB_SUCCESS; - - if (mailbox_.empty()) { - TRANS_LOG(ERROR, "mailbox is empty, but must handle", K(*this)); - ob_abort(); - } else { - mail = mailbox_.front(); - mailbox_.pop_front(); - } - - return ret; -} - -template -int ObMailBox::handle(const bool must_have) -{ - int ret = OB_SUCCESS; - - if (must_have && mailbox_.empty()) { - TRANS_LOG(ERROR, "mailbox is empty, but must handle", K(*this)); - ob_abort(); - } else if (mailbox_.empty()) { - ret = OB_SUCCESS; - } else { - ObMail mail = mailbox_.front(); - mailbox_.pop_front(); - ret = ctx_->handle(mail); - } - - return ret; -} - -template -int ObMailBox::handle_all() -{ - int ret = OB_SUCCESS; - - while (OB_SUCC(ret) && !mailbox_.empty()) { - ObMail mail = mailbox_.front(); - mailbox_.pop_front(); - ret = ctx_->handle(mail); - } - - return ret; -} - -template -int ObMailBox::send(const ObMail& mail, - const int64_t receiver) -{ - int ret = OB_SUCCESS; - - ret = mailbox_mgr_->send(mail, receiver); - - return ret; -} - -template -int ObMailBox::send_to_head(const ObMail& mail, - const int64_t receiver) -{ - int ret = OB_SUCCESS; - - ret = mailbox_mgr_->send_to_head(mail, receiver); - - return ret; -} - -template -int64_t ObMailBox::to_string(char *buffer, const int64_t size) const -{ - int64_t pos = 0; - - if (nullptr != buffer && size > 0) { - databuff_printf(buffer, size, pos, "{addr: %ld, DEQUE: [", addr_); - for (auto it = mailbox_.begin(); it != mailbox_.end(); ++it) { - databuff_printf(buffer, size, pos, "("); - databuff_printf(buffer, size, pos, *it); - databuff_printf(buffer, size, pos, "), "); - } - - databuff_printf(buffer, size, pos, "]}"); - } - - return pos; -} - -template -int ObMailBoxMgr::register_mailbox(int64_t &addr, - ObMailBox &mailbox, - ObMailHandler *ctx) -{ - int ret = OB_SUCCESS; - - addr = ++counter_; - ret = mailbox.init(addr, this, ctx); - mgr_[addr] = &mailbox; - - TRANS_LOG(INFO, "register mailbox", K(mailbox), KP(ctx)); - return ret; -} - -template -void ObMailBoxMgr::reset() -{ - counter_ = 0; - mgr_.clear(); - cache_msg_.clear(); - TRANS_LOG(INFO, "reset mailbox",K(this)); -} - -template -int ObMailBoxMgr::send(const ObMail& mail, - const int64_t receiver) -{ - int ret = OB_SUCCESS; - - if (mgr_.count(mail.to_) != 0) { - cache_msg_.insert(mail); - mgr_[receiver]->mailbox_.push_back(mail); - TRANS_LOG(INFO, "send mailbox success", K(ret), K(mail), - K(*mgr_[receiver])); - } - - return ret; -} - -template -int ObMailBoxMgr::send_to_head(const ObMail& mail, - const int64_t receiver) -{ - int ret = OB_SUCCESS; - - if (mgr_.count(mail.to_) != 0) { - cache_msg_.insert(mail); - mgr_[receiver]->mailbox_.push_front(mail); - TRANS_LOG(INFO, "send to mailbox front success", K(ret), K(mail), - K(*mgr_[receiver])); - } - - return ret; -} - -template -bool ObMailBoxMgr::random_dup_and_send() -{ - int64_t idx = ObRandom::rand(0, cache_msg_.size() - 1); - - if (idx >= 0 && cache_msg_.size() >= 0) { - int i = 0; - bool found = false; - ObMail mail; - for (auto iter = cache_msg_.begin(); - iter != cache_msg_.end(); - iter++) { - if (idx == i) { - mail = *iter; - found = true; - break; - } - i++; - } - if (!found) { - ob_abort(); - } - mgr_[mail.to_]->mailbox_.push_front(mail); - TRANS_LOG(INFO, "random_dup_and_send success", K(idx), K(cache_msg_.size()), - K(mail)); - return true; - } else { - return false; - } -} - -} // namespace transaction -} // namespace oceanbase - -#endif // OCEANBASE_UNITTEST_STORAGE_TX_OB_MAILBOX diff --git a/unittest/storage/tx/ob_mock_2pc_ctx.h b/unittest/storage/tx/ob_mock_2pc_ctx.h deleted file mode 100644 index 4fec0bd13..000000000 --- a/unittest/storage/tx/ob_mock_2pc_ctx.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_2PC_CTX -#define OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_2PC_CTX -#include -#include -#include - -#include "ob_mailbox.h" -#include "lib/ob_errno.h" -#include "lib/utility/ob_macro_utils.h" -#include "lib/utility/ob_print_utils.h" -#include "lib/lock/ob_spin_lock.h" -#include "storage/tx/ob_committer_define.h" -#include "storage/tx/ob_two_phase_committer.h" - -namespace oceanbase -{ - -namespace common -{ -template <> -int64_t to_string>(const std::vector &obj, char *buffer, const int64_t buffer_size); -} - -namespace transaction -{ - -class MockObParticipants : public std::vector -{ -public: - int64_t to_string(char *buffer, const int64_t size) const; -}; - -class MockObLogQueue : public std::deque -{ -public: - int64_t to_string(char *buffer, const int64_t size) const; -}; - -// ObTxCtx inherit ObTxCycleTwoPhaseCommitter, ObTxOnePhaseCommitter and ObITxCommitter. -// It organize user interface based on ObITxCommitter. and implements it using both two -// transaction committer, ObTxCycleTwoPhaseCommitter and ObTxOnePhaseCommitter based on -// participants number -class MockOb2pcCtx : public ObTxCycleTwoPhaseCommitter, - public ObMailHandler -{ -public: - int init(ObMailBoxMgr *mgr); - virtual int handle(const ObMail& mail) override; - int64_t get_addr() { return addr_; } - int handle(const bool must_have = true); - int handle_all(); - int apply(); - // participant abnormally abort itself - int abort(); - bool check_status_valid(const bool should_commit); - - // transaction commit implementation. The transaction can be committed in one phase - // (with one consensus round and zero transport round latency) if it involves only - // one participant. Otherwise the transaction will invoke the circular two phase - // commit(with one consensus round and 2*H transport round latency) - int commit(const MockObParticipants& participants); - - int64_t get_coordinator() { return coordinator_; } - - bool is_real_downstream() { return true; } - - INHERIT_TO_STRING_KV("ObTxCycleTwoPhaseCommitter", - ObTxCycleTwoPhaseCommitter, - K_(addr), - K_(mailbox), - K_(upstream_state), - K_(downstream_state), - K_(tx_state), - K_(log_queue), - K_(participants), - K_(intermediate_participants), - K_(coordinator), - K_(sender)); -protected: - // Oceanbase's optimized transaction commit hook for its user. - // We promise the failure atomicity of the method and we mainly use these method - // to solve the concurrency control problem. - // - // For example, we use do/on_prepare to build the prepare version and mvcc_write - // to maintain read write conflict. We also use on_commit/abort to release lock and - // decide final transaction state. In Oceanbase's optimized, do_pre_commit is used to - // optimize single machine read latency and do/on_clear is used to maintain the state - // to recovery - virtual int do_prepare(bool &no_need_submit_log) override; - virtual int on_prepare() override; - virtual int do_pre_commit(bool& need_wait) override; - virtual int do_commit() override; - virtual int on_commit() override; - virtual int do_abort() override; - virtual int on_abort() override; - virtual int do_clear() override; - virtual int on_clear() override; - // for xa - virtual int reply_to_scheduler_for_sub2pc(int64_t msg_type) override; - - - // We implements the above concurrency control between transaction commit and transfer - // It's based on that if 2pc log is ahead transfer log, the transfer procedure need bring - // the 2pc state to the dst. Otherwise, the 2pc will invoke a cycle style commit - // which the parent node waits for the node's 2pc state response before responsing - // its 2pc state to his parent node. - virtual int64_t get_downstream_size() const override; - virtual int64_t get_self_id() override; - virtual Ob2PCRole get_2pc_role() const override; - virtual ObTxState get_downstream_state() const override; - virtual int set_downstream_state(const ObTxState state) override; - virtual ObTxState get_upstream_state() const override; - virtual int set_upstream_state(const ObTxState state) override; - virtual bool is_2pc_logging() const override; - virtual bool is_2pc_blocking() const { return false; }; - // for xa - virtual bool is_sub2pc() const override; - - virtual bool is_dup_tx() const override; - - virtual int merge_intermediate_participants() override; - - void add_intermediate_participants(const int64_t ls_id); - - void print_downstream(); - - virtual bool is_real_upstream() override; - - bool need_to_advance(); - - // Oceanbase's optimized log handler, if it returns success, the log is definitely proposed - // to the consensus layer and we can rely on its sequential commitment to submit the log - // without waiting consensus commit. While the log may be lost under consensus, so we need - // handle it with exception handler. - virtual int submit_log(const ObTwoPhaseCommitLogType& log_type) override; - // The msg poster is best effort, so we need rely on the timeout handler to retry the last - // message should be sent - // virtual int post_msg(const ObTwoPhaseCommitMsgType& msg_type) override; - virtual int post_msg(const ObTwoPhaseCommitMsgType& msg_type, - const int64_t participant_id) override; -private: - int64_t find_participant_id(int64_t participant_key); - virtual int apply_2pc_msg_(const ObTwoPhaseCommitMsgType msg_type) override; - -public: - common::ObSpinLock latch_; - int64_t addr_; - ObMailBox mailbox_; - ObTxState upstream_state_; - ObTxState downstream_state_; - // final state for tx commit - ObTxState tx_state_; - MockObLogQueue log_queue_; - int64_t coordinator_; - int64_t sender_; - MockObParticipants participants_; - MockObParticipants intermediate_participants_; - - ObMailBoxMgr* mailbox_mgr_; -}; - -} // transaction -} // oceanbase - -#endif // OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_2PC_CTX diff --git a/unittest/storage/tx/ob_mock_tx_ctx.h b/unittest/storage/tx/ob_mock_tx_ctx.h deleted file mode 100644 index a7d63b3ee..000000000 --- a/unittest/storage/tx/ob_mock_tx_ctx.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_TX_CTX -#define OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_TX_CTX - -#include "storage/tx/ob_trans_part_ctx.h" -#include "ob_mailbox.h" -// #include "ob_mock_2pc_ctx.h" - -namespace oceanbase -{ -namespace transaction -{ - -class MockObTxCtx : public ObMailHandler, - public ObPartTransCtx -{ -public: - ~MockObTxCtx() { destroy(); } - int init(const share::ObLSID &ls_id, - const ObTransID &trans_id, - ObLSTxCtxMgr *ctx_mgr, - ObTxData *tx_data, - ObMailBoxMgr *mgr); - virtual int submit_log(const ObTwoPhaseCommitLogType& log_type) override; - int handle(const bool must_have = true); - int handle_all(); - int apply(); - void change_to_leader() { role_state_ = TxCtxRoleState::LEADER; } - int handle(const ObMail& mail); - int64_t get_mailbox_addr() { return addr_; } - static int mock_tx_commit_msg(const ObTransID &trans_id, - const share::ObLSID &ls_id, - const std::vector &participants, - ObTxCommitMsg &msg); - static int build_scheduler_mailbox(ObMailBoxMgr* mailbox_mgr); - static int check_mail(ObMailBox mailbox, - int64_t from, - int64_t to, - int64_t type); - bool check_status_valid(const bool should_commit); - void destroy(); - void set_exiting_(); - virtual int register_timeout_task_(const int64_t interval_us); - virtual int unregister_timeout_task_(); - INHERIT_TO_STRING_KV("ObPartTransCtx", ObPartTransCtx, - K_(addr), K_(mailbox), K_(log_queue), K_(collected)); -public: - int64_t scheduler_addr_ = 0; - static ObMailBox scheduler_mailbox_; -protected: - virtual int post_msg_(const share::ObLSID &receiver, ObTxMsg &msg) override; - virtual int post_msg_(const ObAddr &receiver, ObTxMsg &msg) override; - virtual int get_gts_(share::SCN >s) override; - virtual int wait_gts_elapse_commit_version_(bool &need_wait) override; - virtual int get_local_max_read_version_(share::SCN &local_max_read_version) override; - virtual int update_local_max_commit_version_(const share::SCN &) override; -private: - MockObLogQueue log_queue_; - int64_t addr_; - ObMailBox mailbox_; - ObMailBoxMgr* mailbox_mgr_; - std::map addr_memo_; - std::map ls_memo_; -}; - -const static int64_t MOCK_SCHEDULER_ADDR = INT64_MAX; - - -} // transaction -} // oceanbase - -#endif // OCEANBASE_UNITTEST_STORAGE_TX_OB_MOCK_TX_CTX - diff --git a/unittest/storage/tx/ob_mock_tx_log_adapter.cpp b/unittest/storage/tx/ob_mock_tx_log_adapter.cpp index 7dd0f9167..6b4c3ff70 100644 --- a/unittest/storage/tx/ob_mock_tx_log_adapter.cpp +++ b/unittest/storage/tx/ob_mock_tx_log_adapter.cpp @@ -139,20 +139,7 @@ int MockTxLogAdapter::submit_log(const char *buf, waiting_cbs_.push_back(cb); } } else { - ret = OB_NOT_MASTER; - } - return ret; -} - -int MockTxLogAdapter::get_role(bool &is_leader, int64_t &epoch) -{ - int ret = OB_SUCCESS; - - if (ATOMIC_LOAD(&is_running_)) { - is_leader = true; - epoch = 1; - } else { - is_leader = false; + ret = OB_NOT_RUNNING; } return ret; } diff --git a/unittest/storage/tx/ob_mock_tx_log_adapter.h b/unittest/storage/tx/ob_mock_tx_log_adapter.h index 17cbd21df..36b8fe4cb 100644 --- a/unittest/storage/tx/ob_mock_tx_log_adapter.h +++ b/unittest/storage/tx/ob_mock_tx_log_adapter.h @@ -74,17 +74,11 @@ class MockTxLogAdapter : public ObITxLogAdapter, public ObSimpleThreadPool ObTxBaseLogCb *cb, const bool need_block, const int64_t retry_timeout_us = 1000); - int get_role(bool &is_leader, int64_t &epoch); int get_max_decided_scn(share::SCN &scn) { UNUSED(scn); return OB_SUCCESS; } - int get_palf_committed_max_scn(share::SCN &scn) const - { - UNUSED(scn); - return OB_SUCCESS; - } int get_append_mode_initial_scn(share::SCN &ref_scn) { int ret = OB_SUCCESS; ref_scn = share::SCN::invalid_scn(); diff --git a/unittest/storage/tx/test_ls_log_writer.cpp b/unittest/storage/tx/test_ls_log_writer.cpp deleted file mode 100644 index 1bc098545..000000000 --- a/unittest/storage/tx/test_ls_log_writer.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "lib/ob_name_id_def.h" -#include "lib/json/ob_yson.h" -#include "ob_mock_tx_log_adapter.h" -#include "storage/tx/ob_trans_ctx_mgr.h" -#include - -namespace oceanbase -{ - -using namespace transaction; -using namespace storage; -using namespace share; - -namespace transaction -{ -int ObTxLSLogCb::alloc_log_buf_() -{ - int ret = OB_SUCCESS; - - ObMemAttr attr("TxLSLogBuf"); - SET_USE_500(attr); - if (0 == ObTxLSLogLimit::LOG_BUF_SIZE || nullptr != log_buf_) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "[TxLsLogWriter] invalid arguments", KR(ret), K(ObTxLSLogLimit::LOG_BUF_SIZE), - KP(log_buf_)); - } else if (nullptr == (log_buf_ = (char *)ob_malloc(ObTxLSLogLimit::LOG_BUF_SIZE, attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "[TxLsLogWriter] allocate memory failed", KR(ret), - K(ObTxLSLogLimit::LOG_BUF_SIZE)); - } - - return ret; -} -} // namespace transaction - -namespace unittest -{ -MockTxLogAdapter tx_log_adapter; -MockTxLogParam param; -ObTxLSLogWriter ls_log_writer; - -class TestLSLogWriter : public ::testing::Test -{ -public: - virtual void SetUp() - { - tx_log_adapter.init(¶m); - tx_log_adapter.start(); - } - virtual void TearDown() - { - tx_log_adapter.stop(); - tx_log_adapter.wait(); - tx_log_adapter.destroy(); - } - -public: -}; - -const ObLSID TEST_LS_ID(735); - -TEST_F(TestLSLogWriter, submit_start_working_log) -{ - ObLSTxCtxMgr tmp_mgr; - common::ObConcurrentFIFOAllocator tmp_allocator; - - ObTxLogBlock replay_block; - int64_t replay_hint = 0; - share::SCN log_ts; - std::string log_string; - ObTxLogHeader log_header; - ObTxStartWorkingLogTempRef tmp_ref; - ObTxStartWorkingLog sw_log(tmp_ref); - int64_t test_leader_epoch = 1308; - - ASSERT_EQ(OB_SUCCESS, ls_log_writer.init(TEST_LS_ID, &tx_log_adapter, - (ObLSTxCtxMgr *)&tmp_mgr)); - ASSERT_EQ(OB_SUCCESS, ls_log_writer.submit_start_working_log(test_leader_epoch, log_ts)); - - ASSERT_EQ(true, tx_log_adapter.get_log(log_ts.get_val_for_gts(), log_string)); - ASSERT_EQ(OB_SUCCESS, replay_block.init_for_replay(log_string.c_str(), log_string.size())); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(log_header)); - EXPECT_EQ(ObTxLogType::TX_START_WORKING_LOG, log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(sw_log)); - EXPECT_EQ(test_leader_epoch, sw_log.get_leader_epoch()); -} - -} // namespace unittest -} // namespace oceanbase - -using namespace oceanbase; - -int main(int argc, char **argv) -{ - int ret = 1; - ObLogger &logger = ObLogger::get_logger(); - logger.set_file_name("test_ls_log_writer.log", true); - logger.set_log_level(OB_LOG_LEVEL_INFO); - testing::InitGoogleTest(&argc, argv); - ret = RUN_ALL_TESTS(); - return ret; -} diff --git a/unittest/storage/tx/test_ob_timestamp_service.cpp b/unittest/storage/tx/test_ob_timestamp_service.cpp index 9ea402ae1..df323082a 100644 --- a/unittest/storage/tx/test_ob_timestamp_service.cpp +++ b/unittest/storage/tx/test_ob_timestamp_service.cpp @@ -21,229 +21,41 @@ namespace oceanbase { using namespace common; using namespace transaction; -using namespace obcall; namespace unittest { -class MyResponseRpc +class TestTimestampService : public ObTimestampService { public: - MyResponseRpc() {} - ~MyResponseRpc() {} -public: - void set_valid_arg(const int status, - const ObAddr &sender, - const ObAddr self) - { - status_ = status; - sender_ = sender; - self_ = self; - } - int post(const ObAddr &server, const ObGtsErrResponse &msg) + int init_for_test() { - int ret = OB_SUCCESS; - if (!msg.is_valid() || - msg.get_status() != status_ || - msg.get_sender() != sender_ || - server != self_) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(server), K(msg), K(*this)); - } - return ret; - } - TO_STRING_KV(K_(status), K_(sender), K_(self)); -private: - int status_; - ObAddr sender_; - ObAddr self_; -}; - -class MyTimestampService : public ObTimestampService -{ -public: - MyTimestampService() {} - ~MyTimestampService() {} - void init(const ObAddr &self) - { - self_ = self; + int ret = ObTimestampService::init(); service_type_ = ServiceType::TimestampService; pre_allocated_range_ = TIMESTAMP_PREALLOCATED_RANGE; - last_id_ = 0; - limited_id_ = ObTimeUtility::current_time_ns() + TIMESTAMP_PREALLOCATED_RANGE; - } -public: - int handle_request(const ObGtsRequest &request, ObGtsRpcResult &result) - { - static int64_t total_cnt = 0; - static int64_t total_rt = 0; - static const int64_t STATISTICS_INTERVAL_US = 10000000; - const MonotonicTs start = MonotonicTs::current_time(); - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!request.is_valid())) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", KR(ret), K(request)); - } else { - TRANS_LOG(DEBUG, "handle gts request", K(request)); - int64_t gts = 0; - const MonotonicTs srr = request.get_srr(); - const ObAddr &requester = request.get_sender(); - int64_t end_id; - if (requester == self_) { - // Go local call to get gts - TRANS_LOG(DEBUG, "handle local gts request", K(requester)); - ret = handle_local_request_(request, result); - } else if (OB_FAIL(get_number(1, ObTimeUtility::current_time_ns(), gts, end_id))) { - TRANS_LOG(WARN, "get timestamp failed", KR(ret)); - int tmp_ret = OB_SUCCESS; - ObGtsErrResponse response; - if (OB_SUCCESS != (tmp_ret = result.init(ret, srr, 0, 0))) { - TRANS_LOG(WARN, "gts result init failed", K(tmp_ret), K(request)); - } else if (OB_SUCCESS != (tmp_ret = response.init(srr, ret, self_))) { - TRANS_LOG(WARN, "gts err response init failed", K(tmp_ret), K(request)); - } else if (OB_SUCCESS != (tmp_ret = rpc_.post(requester, response))) { - TRANS_LOG(WARN, "post gts err response failed", K(tmp_ret), K(response)); - } else { - TRANS_LOG(DEBUG, "post gts err response success", K(response)); - } - } else { - if (OB_FAIL(result.init(ret, srr, gts, gts))) { - TRANS_LOG(WARN, "gts result init failed", KR(ret), K(request)); - } - } - } - return ret; - } - int handle_local_request_(const ObGtsRequest &request, obcall::ObGtsRpcResult &result) - { - int ret = OB_SUCCESS; - int64_t gts = 0; - const MonotonicTs srr = request.get_srr(); - int64_t end_id; - if (OB_FAIL(get_number(1, ObTimeUtility::current_time_ns(), gts, end_id))) { - TRANS_LOG(WARN, "get timestamp failed", KR(ret)); - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = result.init(ret, srr, 0, 0))) { - TRANS_LOG(WARN, "gts result init failed", K(tmp_ret), K(request)); - } - } else { - if (OB_FAIL(result.init(ret, srr, gts, gts))) { - TRANS_LOG(WARN, "local gts result init failed", KR(ret), K(request)); - } - } + last_id_ = ObTimeUtility::current_time_ns(); + limited_id_ = last_id_ + TIMESTAMP_PREALLOCATED_RANGE; return ret; } -private: - int get_number(const int64_t range, const int64_t base_id, int64_t &start_id, int64_t &end_id) - { - int ret = OB_SUCCESS; - if (range != 1) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(range), K(base_id)); - } else if (base_id < ObTimeUtility::current_time()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(range), K(base_id)); - } else { - const int64_t last_id = ATOMIC_LOAD(&last_id_); - int64_t tmp_id = 0; - if (base_id > last_id) { - if (ATOMIC_BCAS(&last_id_, last_id, base_id + range)) { - tmp_id = base_id; - } else { - tmp_id = ATOMIC_FAA(&last_id_, range); - } - } else { - tmp_id = ATOMIC_FAA(&last_id_, range); - } - start_id = tmp_id; - } - return ret; - } - MyResponseRpc rpc_; -}; - -class TestObGtsMgr : public ::testing::Test -{ -public : - virtual void SetUp() {} - virtual void TearDown() {} }; -//////////////////////basic function test////////////////////////////////////////// - -TEST_F(TestObGtsMgr, handle_gts_request_by_leader) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - const ObAddr client(ObAddr::IPV4, "10.0.0.1", 10000); - const ObAddr server(ObAddr::IPV4, "10.0.0.1", 20000); - MyTimestampService ts_service; - MyResponseRpc response_rpc; - ts_service.init(server); - - const MonotonicTs srr = MonotonicTs::current_time(); - const int64_t ts_range = 1; - response_rpc.set_valid_arg(OB_SUCCESS, server, client); - ObGtsRequest request; - ObGtsRpcResult result; - EXPECT_EQ(OB_SUCCESS, request.init(srr, ts_range, client)); - EXPECT_EQ(OB_SUCCESS, ts_service.handle_request(request, result)); - EXPECT_EQ(OB_SUCCESS, result.get_status()); - EXPECT_EQ(srr, result.get_srr()); - EXPECT_EQ(ts_range - 1, result.get_gts_end() - result.get_gts_start()); - //EXPECT_TRUE(result.get_gts_start() >= srr); -} - -TEST_F(TestObGtsMgr, handle_local_gts_request) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - const ObAddr server(ObAddr::IPV4, "10.0.0.1", 20000); - MyTimestampService ts_service; - MyResponseRpc response_rpc; - ts_service.init(server); - - const MonotonicTs srr = MonotonicTs::current_time(); - const int64_t ts_range = 1; - response_rpc.set_valid_arg(OB_SUCCESS, server, server); - ObGtsRequest request; - ObGtsRpcResult result; - EXPECT_EQ(OB_SUCCESS, request.init(srr, ts_range, server)); - EXPECT_EQ(OB_SUCCESS, ts_service.handle_request(request, result)); - EXPECT_EQ(OB_SUCCESS, result.get_status()); - EXPECT_EQ(srr, result.get_srr()); - EXPECT_EQ(ts_range - 1, result.get_gts_end() - result.get_gts_start()); - // EXPECT_TRUE(result.get_gts_start() >= srr); -} - -TEST_F(TestObGtsMgr, invalid_argument) +TEST(TestObTimestampService, local_timestamp_is_monotonic) { - TRANS_LOG(INFO, "called", "func", test_info_->name()); - const ObAddr client(ObAddr::IPV4, "10.0.0.1", 10000); - const ObAddr server(ObAddr::IPV4, "10.0.0.1", 20000); - MyTimestampService ts_service; - MyResponseRpc response_rpc; + TestTimestampService service; + ASSERT_EQ(OB_SUCCESS, service.init_for_test()); + int64_t first = 0; + int64_t second = 0; - const MonotonicTs srr = MonotonicTs::current_time(); - const MonotonicTs stc; - const int64_t ts_range = 1; - ObGtsRequest request; - EXPECT_EQ(OB_INVALID_ARGUMENT, request.init(stc, ts_range, client)); - EXPECT_EQ(OB_INVALID_ARGUMENT, request.init(srr, 0, client)); - EXPECT_EQ(OB_INVALID_ARGUMENT, request.init(srr, ts_range, ObAddr())); + ASSERT_EQ(OB_SUCCESS, service.get_timestamp(first)); + ASSERT_EQ(OB_SUCCESS, service.get_timestamp(second)); + EXPECT_GT(first, 0); + EXPECT_GT(second, first); } -}//end of unittest -}//end of oceanbase - -using namespace oceanbase; -using namespace oceanbase::common; +} // namespace unittest +} // namespace oceanbase int main(int argc, char **argv) { - int ret = 1; - ObLogger &logger = ObLogger::get_logger(); - logger.set_file_name("test_ob_gts_mgr.log", true); - logger.set_log_level(OB_LOG_LEVEL_INFO); testing::InitGoogleTest(&argc, argv); - ret = RUN_ALL_TESTS(); - return ret; + return RUN_ALL_TESTS(); } diff --git a/unittest/storage/tx/test_ob_tx_log.cpp b/unittest/storage/tx/test_ob_tx_log.cpp index 93f9fe18e..c76300d79 100644 --- a/unittest/storage/tx/test_ob_tx_log.cpp +++ b/unittest/storage/tx/test_ob_tx_log.cpp @@ -39,77 +39,21 @@ class TestObTxLog : public ::testing::Test //const TEST TxID TEST_TX_ID = 1024; int64_t TEST_CLUSTER_VERSION = DATA_VERSION_1_0_0_0; -ObAddr TEST_ADDR(ObAddr::VER::IPV4,"1.0.0.1",606); -int TEST_TRANS_TYPE = 1; -int TEST_SESSION_ID = 56831; common::ObString TEST_TRACE_ID_STR("trace_id_test"); bool TEST_CAN_ELR = false; -int64_t TEST_QUERY_TIME = 90000; bool TEST_IS_DUP = true; -bool TEST_IS_SUB2PC = false; -int64_t TEST_EPOCH = 1315; -int64_t TEST_LAST_OP_SN = 1315; -auto TEST_FIRST_SCN = ObTxSEQ(1315, 0); -auto TEST_LAST_SCN = ObTxSEQ(1315, 0); uint64_t TEST_ORG_CLUSTER_ID = 1208; common::ObString TEST_TRCE_INFO("trace_info_test"); LogOffSet TEST_LOG_OFFSET(10); int64_t TEST_COMMIT_VERSION = 190878; int64_t TEST_CHECKSUM = 29890209; ObArray TEST_CHECKSUM_SIGNATURE_ARRAY; -int64_t TEST_SCHEMA_VERSION = 372837; -int64_t TEST_TX_EXPIRED_TIME = 12099087; int64_t TEST_LOG_ENTRY_NO = 1233; -ObTxSEQ TEST_MAX_SUBMITTED_SEQ_NO = ObTxSEQ(12345, 0); -ObTxSEQ TEST_SERIAL_FINAL_SEQ_NO = ObTxSEQ(12346, 0); -LSKey TEST_LS_KEY; ObXATransID TEST_XID; -ObTxPrevLogType TEST_PREV_LOG_TYPE(ObTxPrevLogType::TypeEnum::PREPARE); +ObTxPrevLogType TEST_PREV_LOG_TYPE(ObTxPrevLogType::TypeEnum::COMMIT_INFO); tablelock::ObTableLockPrioOpArray TEST_PRIO_OP_ARRAY; -struct OldTestLog -{ - ObTxSerCompatByte compat_bytes_; - int64_t tx_id_1; - int64_t tx_id_2; - - OldTestLog() - { - tx_id_1 = 0; - tx_id_2 = 0; - int ret = OB_SUCCESS; - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - OB_UNIS_VERSION(1); -}; - -OB_TX_SERIALIZE_MEMBER(OldTestLog, compat_bytes_, tx_id_1, tx_id_2); - -struct NewTestLog -{ - ObTxSerCompatByte compat_bytes_; - int64_t tx_id_1; - int64_t tx_id_2; - int64_t tx_id_3; - - NewTestLog() - { - tx_id_1 = 0; - tx_id_2 = 0; - tx_id_3 = 0; - int ret = OB_SUCCESS; - if (OB_FAIL(compat_bytes_.init(2))) { - TRANS_LOG(WARN, "init compat_bytes_ failed", K(ret)); - } - } - OB_UNIS_VERSION(1); -}; - -OB_TX_SERIALIZE_MEMBER(NewTestLog, compat_bytes_, tx_id_1, tx_id_2, tx_id_3); - // test ObTxLogBlockHeader TEST_F(TestObTxLog, tx_log_block_header) { @@ -118,7 +62,7 @@ TEST_F(TestObTxLog, tx_log_block_header) ObTxLogBlock fill_block, replay_block; ObTxLogBlockHeader &fill_block_header = fill_block.get_header(); - fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID), TEST_ADDR); + fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID)); fill_block_header.set_serial_final(); ASSERT_TRUE(fill_block_header.is_serial_final()); ASSERT_EQ(OB_SUCCESS, fill_block.init_for_fill()); @@ -144,7 +88,8 @@ TEST_F(TestObTxLog, tx_log_block_header) EXPECT_TRUE(replay_block_header.is_serial_final()); // reuse - fill_block.get_header().init(TEST_ORG_CLUSTER_ID + 1, TEST_CLUSTER_VERSION + 1, TEST_LOG_ENTRY_NO + 1, ObTransID(TEST_TX_ID + 1), TEST_ADDR); + fill_block.get_header().init(TEST_ORG_CLUSTER_ID + 1, TEST_CLUSTER_VERSION + 1, + TEST_LOG_ENTRY_NO + 1, ObTransID(TEST_TX_ID + 1)); fill_block.reuse_for_fill(); fill_block.seal(TEST_TX_ID + 1); buf = fill_block.get_buf(); @@ -170,77 +115,35 @@ TEST_F(TestObTxLog, tx_log_body_except_redo) ObTxLogBlock fill_block; ObTxLogBlock replay_block; - ObLSArray TEST_LS_ARRAY; - TEST_LS_ARRAY.push_back(LSKey()); - ObTxCommitParts TEST_COMMIT_PARTS; - TEST_COMMIT_PARTS.push_back(ObTxExecPart(TEST_LS_KEY, 0)); ObRedoLSNArray TEST_LOG_OFFSET_ARRY; TEST_LOG_OFFSET_ARRY.push_back(TEST_LOG_OFFSET); - ObLSLogInfoArray TEST_INFO_ARRAY; - TEST_INFO_ARRAY.push_back(ObLSLogInfo()); ObTxBufferNodeArray TEST_TX_BUFFER_NODE_ARRAY; ObString str("TEST CASE"); ObTxBufferNode node; - node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ::mk_v0(100), nullptr); + node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ(100, 0), nullptr); TEST_TX_BUFFER_NODE_ARRAY.push_back(node); - ObTxCommitInfoLog fill_commit_state(TEST_ADDR, - TEST_LS_ARRAY, - TEST_LS_KEY, - TEST_IS_SUB2PC, - TEST_IS_DUP, + ObTxCommitInfoLog fill_commit_state(TEST_IS_DUP, TEST_CAN_ELR, TEST_TRACE_ID_STR, TEST_TRCE_INFO, TEST_LOG_OFFSET, TEST_LOG_OFFSET_ARRY, - TEST_LS_ARRAY, - TEST_CLUSTER_VERSION, - TEST_XID, - TEST_COMMIT_PARTS, - TEST_EPOCH); - // ASSERT_EQ(OB_SUCCESS, fill_commit_state.before_serialize()); - ObTxActiveInfoLog fill_active_state(TEST_ADDR, - TEST_TRANS_TYPE, - TEST_SESSION_ID, - 0, - TEST_TRACE_ID_STR, - TEST_SCHEMA_VERSION, - TEST_CAN_ELR, - TEST_ADDR, - TEST_QUERY_TIME, - TEST_IS_SUB2PC, - TEST_IS_DUP, - TEST_TX_EXPIRED_TIME, - TEST_EPOCH, - TEST_LAST_OP_SN, - TEST_FIRST_SCN, - TEST_LAST_SCN, - TEST_MAX_SUBMITTED_SEQ_NO, - TEST_CLUSTER_VERSION, - TEST_XID, - TEST_SERIAL_FINAL_SEQ_NO, - TEST_PRIO_OP_ARRAY); - ObTxPrepareLog filll_prepare(TEST_LS_ARRAY, TEST_LOG_OFFSET, TEST_PREV_LOG_TYPE); + TEST_XID); ObTxCommitLog fill_commit(share::SCN::base_scn(), TEST_CHECKSUM, TEST_CHECKSUM_SIGNATURE_ARRAY, - TEST_LS_ARRAY, TEST_TX_BUFFER_NODE_ARRAY, - TEST_TRANS_TYPE, TEST_LOG_OFFSET, - TEST_INFO_ARRAY, TEST_PREV_LOG_TYPE); - ObTxClearLog fill_clear(TEST_LS_ARRAY); + ObTxClearLog fill_clear; ObTxAbortLog fill_abort(TEST_TX_BUFFER_NODE_ARRAY); ObTxRecordLog fill_record(TEST_LOG_OFFSET, TEST_LOG_OFFSET_ARRY); ObTxLogBlockHeader &header = fill_block.get_header(); - header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID), TEST_ADDR); + header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID)); ASSERT_EQ(OB_SUCCESS, fill_block.init_for_fill()); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_active_state)); ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_commit_state)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(filll_prepare)); ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_commit)); ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_clear)); ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_abort)); @@ -252,45 +155,12 @@ TEST_F(TestObTxLog, tx_log_body_except_redo) uint64_t tmp_cluster_id = replay_block.get_header().get_org_cluster_id(); EXPECT_EQ(TEST_ORG_CLUSTER_ID, tmp_cluster_id); - ObTxActiveInfoLogTempRef active_temp_ref; - ObTxActiveInfoLog replay_active_state(active_temp_ref); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_ACTIVE_INFO_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_active_state)); - EXPECT_EQ(TEST_ADDR, replay_active_state.get_scheduler()); - EXPECT_EQ(TEST_TRANS_TYPE, replay_active_state.get_trans_type()); - EXPECT_EQ(TEST_SESSION_ID, replay_active_state.get_session_id()); - EXPECT_EQ(TEST_TRACE_ID_STR, replay_active_state.get_app_trace_id()); - EXPECT_EQ(TEST_SCHEMA_VERSION, replay_active_state.get_schema_version()); - EXPECT_EQ(TEST_CAN_ELR, replay_active_state.is_elr()); - EXPECT_EQ(TEST_ADDR, replay_active_state.get_proposal_leader()); - EXPECT_EQ(TEST_QUERY_TIME, replay_active_state.get_cur_query_start_time()); - EXPECT_EQ(TEST_IS_SUB2PC, replay_active_state.is_sub2pc()); - EXPECT_EQ(TEST_IS_DUP, replay_active_state.is_dup_tx()); - EXPECT_EQ(TEST_TX_EXPIRED_TIME, replay_active_state.get_tx_expired_time()); - EXPECT_EQ(TEST_EPOCH, replay_active_state.get_epoch()); - EXPECT_EQ(TEST_LAST_OP_SN, replay_active_state.get_last_op_sn()); - EXPECT_EQ(TEST_FIRST_SCN, replay_active_state.get_first_seq_no()); - EXPECT_EQ(TEST_LAST_SCN, replay_active_state.get_last_seq_no()); - EXPECT_EQ(0, replay_active_state.get_cluster_version()); - EXPECT_EQ(TEST_XID, replay_active_state.get_xid()); - EXPECT_EQ(TEST_SERIAL_FINAL_SEQ_NO, replay_active_state.get_serial_final_seq_no()); - ObTxCommitInfoLogTempRef commit_state_temp_ref; ObTxCommitInfoLog replay_commit_state(commit_state_temp_ref); ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); EXPECT_EQ(ObTxLogType::TX_COMMIT_INFO_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_INVALID_ARGUMENT, - replay_block.deserialize_log_body(replay_active_state)); // error log type ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_commit_state)); - ObTxPrepareLogTempRef prepare_temp_ref; - ObTxPrepareLog replay_prepare(prepare_temp_ref); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_PREPARE_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_prepare)); - EXPECT_EQ(TEST_PREV_LOG_TYPE.prev_log_type_, replay_prepare.get_prev_log_type().prev_log_type_); - ObTxCommitLogTempRef commit_temp_ref; ObTxCommitLog replay_commit(commit_temp_ref); ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); @@ -320,52 +190,35 @@ TEST_F(TestObTxLog, tx_log_body_redo) ObTxLogBlock replay_block; ObTxLogBlock replay_block_2; - ObLSArray TEST_LS_ARRAY; - TEST_LS_ARRAY.push_back(LSKey()); - ObTxCommitParts TEST_COMMIT_PARTS; - TEST_COMMIT_PARTS.push_back(ObTxExecPart(TEST_LS_KEY, 0)); ObRedoLSNArray TEST_LOG_OFFSET_ARRY; TEST_LOG_OFFSET_ARRY.push_back(TEST_LOG_OFFSET); - ObLSLogInfoArray TEST_INFO_ARRAY; - TEST_INFO_ARRAY.push_back(ObLSLogInfo()); ObTxBufferNodeArray TEST_TX_BUFFER_NODE_ARRAY; ObString str("TEST CASE"); ObTxBufferNode node; - node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ::mk_v0(100), nullptr); + node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ(100, 0), nullptr); TEST_TX_BUFFER_NODE_ARRAY.push_back(node); - ObTxCommitInfoLog fill_commit_state(TEST_ADDR, - TEST_LS_ARRAY, - TEST_LS_KEY, - TEST_IS_SUB2PC, - TEST_IS_DUP, + ObTxCommitInfoLog fill_commit_state(TEST_IS_DUP, TEST_CAN_ELR, TEST_TRACE_ID_STR, TEST_TRCE_INFO, TEST_LOG_OFFSET, TEST_LOG_OFFSET_ARRY, - TEST_LS_ARRAY, - TEST_CLUSTER_VERSION, - TEST_XID, - TEST_COMMIT_PARTS, - TEST_EPOCH); + TEST_XID); ObTxCommitLog fill_commit(share::SCN::base_scn(), TEST_CHECKSUM, TEST_CHECKSUM_SIGNATURE_ARRAY, - TEST_LS_ARRAY, TEST_TX_BUFFER_NODE_ARRAY, - TEST_TRANS_TYPE, TEST_LOG_OFFSET, - TEST_INFO_ARRAY, TEST_PREV_LOG_TYPE); ObTxLogBlockHeader &fill_block_header = fill_block.get_header(); - fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID), TEST_ADDR); + fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID)); ASSERT_EQ(OB_SUCCESS, fill_block.init_for_fill()); ObString TEST_MUTATOR_BUF("FFF"); int64_t mutator_pos = 0; - ObTxRedoLog fill_redo(TEST_CLUSTER_VERSION); + ObTxRedoLog fill_redo; ASSERT_EQ(OB_SUCCESS, fill_block.prepare_mutator_buf(fill_redo)); ASSERT_EQ(OB_SUCCESS, serialization::encode(fill_redo.get_mutator_buf(), fill_redo.get_mutator_size(), @@ -404,7 +257,6 @@ TEST_F(TestObTxLog, tx_log_body_redo) replay_mutator_buf)); EXPECT_EQ(TEST_MUTATOR_BUF, replay_mutator_buf); // EXPECT_EQ(TEST_CLOG_ENCRYPT_INFO,replay_redo.get_clog_encrypt_info()); - EXPECT_EQ(replay_redo.get_cluster_version(), 0); ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(log_header)); EXPECT_EQ(ObTxLogType::TX_COMMIT_INFO_LOG, log_header.get_tx_log_type()); ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(log_header)); @@ -428,7 +280,6 @@ TEST_F(TestObTxLog, tx_log_body_redo) // replay_mutator_buf)); // EXPECT_EQ(TEST_MUTATOR_BUF, replay_mutator_buf); // EXPECT_EQ(TEST_CLOG_ENCRYPT_INFO,replay_redo.get_clog_encrypt_info()); - // EXPECT_EQ(TEST_CLUSTER_VERSION,replay_redo.get_cluster_version()); ASSERT_EQ(OB_SUCCESS, replay_block_2.get_next_log(log_header)); EXPECT_EQ(ObTxLogType::TX_COMMIT_INFO_LOG, log_header.get_tx_log_type()); ASSERT_EQ(OB_SUCCESS, replay_block_2.get_next_log(log_header)); @@ -440,290 +291,6 @@ TEST_F(TestObTxLog, tx_log_body_redo) } -TEST_F(TestObTxLog, test_compat_bytes) -{ - ObLSArray TEST_LS_ARRAY; - TEST_LS_ARRAY.push_back(LSKey()); - ObTxCommitParts TEST_COMMIT_PARTS; - TEST_COMMIT_PARTS.push_back(ObTxExecPart(TEST_LS_KEY, 0)); - ObRedoLSNArray TEST_LOG_OFFSET_ARRY; - TEST_LOG_OFFSET_ARRY.push_back(TEST_LOG_OFFSET); - ObLSLogInfoArray TEST_INFO_ARRAY; - TEST_INFO_ARRAY.push_back(ObLSLogInfo()); - ObTxBufferNodeArray TEST_TX_BUFFER_NODE_ARRAY; - ObString str("TEST CASE"); - ObTxBufferNode node; - node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ::mk_v0(100), nullptr); - TEST_TX_BUFFER_NODE_ARRAY.push_back(node); - - ObTxCommitInfoLog fill_commit_info(TEST_ADDR, - TEST_LS_ARRAY, - TEST_LS_KEY, - TEST_IS_SUB2PC, - TEST_IS_DUP, - TEST_CAN_ELR, - TEST_TRACE_ID_STR, - TEST_TRCE_INFO, - TEST_LOG_OFFSET, - TEST_LOG_OFFSET_ARRY, - TEST_LS_ARRAY, - TEST_CLUSTER_VERSION, - TEST_XID, - TEST_COMMIT_PARTS, - TEST_EPOCH); - ObTxCommitInfoLogTempRef commit_info_temp_ref; - ObTxCommitInfoLog replay_commit_info(commit_info_temp_ref); - - // ASSERT_EQ(OB_SUCCESS, fill_commit_info.before_serialize()); - TRANS_LOG(INFO, - "the size of commit info with all compat_bytes", - K(fill_commit_info.get_serialize_size())); - ASSERT_EQ(true, fill_commit_info.is_dup_tx()); - ASSERT_EQ(false, fill_commit_info.get_participants().empty()); - void *tmp_buf = ob_malloc(1 * 1024 * 1024, ObNewModIds::TEST); - int64_t pos = 0; - fill_commit_info.compat_bytes_.set_object_flag(2, false); - fill_commit_info.compat_bytes_.set_object_flag(5, false); - ASSERT_EQ(OB_SUCCESS, fill_commit_info.serialize((char *)tmp_buf, 1 * 1024 * 1024, pos)); - TRANS_LOG(INFO, - "the size of commit info with compat_bytes", - K(fill_commit_info.get_serialize_size()), - K(pos)); - - pos = 0; - ASSERT_EQ(OB_SUCCESS, - replay_commit_info.deserialize((const char *)tmp_buf, 1 * 1024 * 1024, pos)); - EXPECT_EQ(replay_commit_info.is_dup_tx(), false); - EXPECT_EQ(replay_commit_info.get_participants().empty(), true); - -} - -TEST_F(TestObTxLog, test_default_log_deserialize) -{ - ObTxLogBlock fill_block; - ObTxLogBlock replay_block; - - ObTxCommitInfoLogTempRef fill_commit_state_ref; - ObTxActiveInfoLogTempRef fill_active_state_ref; - ObTxPrepareLogTempRef fill_prepare_ref; - ObTxCommitLogTempRef fill_commit_ref; - ObTxClearLogTempRef fill_clear_ref; - ObTxAbortLogTempRef fill_abort_ref; - ObTxRecordLogTempRef fill_record_ref; - - ObTxCommitInfoLog fill_commit_state(fill_commit_state_ref); - ObTxActiveInfoLog fill_active_state(fill_active_state_ref); - ObTxPrepareLog fill_prepare(fill_prepare_ref); - ObTxCommitLog fill_commit(fill_commit_ref); - ObTxClearLog fill_clear(fill_clear_ref); - ObTxAbortLog fill_abort(fill_abort_ref); - ObTxRecordLog fill_record(fill_record_ref); - - // ObTxLogBlockHeader fill_block_header(TEST_ORG_CLUSTER_ID, TEST_LOG_ENTRY_NO, - // ObTransID(TEST_TX_ID)); - ObTxLogBlockHeader &fill_block_header = fill_block.get_header(); - fill_block_header.init(1, TEST_CLUSTER_VERSION, 2, ObTransID(3), TEST_ADDR); - ASSERT_EQ(OB_SUCCESS, fill_block.init_for_fill()); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_active_state)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_commit_state)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_prepare)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_commit)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_clear)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_abort)); - ASSERT_EQ(OB_SUCCESS, fill_block.add_new_log(fill_record)); - fill_block.seal(TEST_TX_ID); - - TxID id = 0; - int64_t fill_member_cnt = 0; - int64_t replay_member_cnt = 0; - ObTxLogHeader tx_log_header; - ObTxLogBlockHeader &replay_block_header = replay_block.get_header(); - ASSERT_EQ(OB_SUCCESS, replay_block.init_for_replay(fill_block.get_buf(), fill_block.get_size())); - fill_member_cnt = fill_block_header.compat_bytes_.total_obj_cnt_; - EXPECT_EQ(fill_block_header.get_org_cluster_id(), replay_block_header.get_org_cluster_id()); - replay_member_cnt++; - EXPECT_EQ(fill_block_header.get_log_entry_no(), replay_block_header.get_log_entry_no()); - replay_member_cnt++; - EXPECT_EQ(fill_block_header.get_tx_id().get_id(), replay_block_header.get_tx_id().get_id()); - replay_member_cnt++; - EXPECT_EQ(fill_block_header.get_scheduler(), replay_block_header.get_scheduler()); - replay_member_cnt++; - EXPECT_EQ(fill_block_header.get_cluster_version(), replay_block_header.get_cluster_version()); - replay_member_cnt++; - EXPECT_EQ(fill_block_header.flags(), replay_block_header.flags()); - replay_member_cnt++; - EXPECT_EQ(replay_member_cnt, fill_member_cnt - 1/*1 skipped*/); - - ObTxActiveInfoLogTempRef active_temp_ref; - ObTxActiveInfoLog replay_active_state(active_temp_ref); - fill_member_cnt = replay_member_cnt = 0; - fill_member_cnt = fill_active_state.compat_bytes_.total_obj_cnt_; - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_ACTIVE_INFO_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_active_state)); - EXPECT_EQ(fill_active_state.get_scheduler(), replay_active_state.get_scheduler()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_trans_type(), replay_active_state.get_trans_type()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_session_id(), replay_active_state.get_session_id()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_app_trace_id(), replay_active_state.get_app_trace_id()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_schema_version(), replay_active_state.get_schema_version()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.is_elr(), replay_active_state.is_elr()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_proposal_leader(), replay_active_state.get_proposal_leader()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_cur_query_start_time(), - replay_active_state.get_cur_query_start_time()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.is_sub2pc(), replay_active_state.is_sub2pc()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.is_dup_tx(), replay_active_state.is_dup_tx()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_tx_expired_time(), replay_active_state.get_tx_expired_time()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_epoch(), replay_active_state.get_epoch()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_last_op_sn(), replay_active_state.get_last_op_sn()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_first_seq_no(), replay_active_state.get_first_seq_no()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_last_seq_no(), replay_active_state.get_last_seq_no()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_cluster_version(), replay_active_state.get_cluster_version()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_max_submitted_seq_no(), replay_active_state.get_max_submitted_seq_no()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_xid(), replay_active_state.get_xid()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_serial_final_seq_no(), replay_active_state.get_serial_final_seq_no()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_associated_session_id(), replay_active_state.get_associated_session_id()); - replay_member_cnt++; - EXPECT_EQ(fill_active_state.get_prio_op_array().count(), replay_active_state.get_prio_op_array().count()); - replay_member_cnt++; - EXPECT_EQ(replay_member_cnt, fill_member_cnt); - - ObTxCommitInfoLogTempRef commit_state_temp_ref; - ObTxCommitInfoLog replay_commit_state(commit_state_temp_ref); - fill_member_cnt = replay_member_cnt = 0; - fill_member_cnt = fill_commit_state.compat_bytes_.total_obj_cnt_; - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_COMMIT_INFO_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_commit_state)); - EXPECT_EQ(fill_commit_state.get_scheduler(), replay_commit_state.get_scheduler()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_participants().count(), - replay_commit_state.get_participants().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_upstream(), replay_commit_state.get_upstream()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.is_sub2pc(), replay_commit_state.is_sub2pc()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.is_dup_tx(), replay_commit_state.is_dup_tx()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.is_elr(), replay_commit_state.is_elr()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_incremental_participants().count(), - replay_commit_state.get_incremental_participants().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_cluster_version(), replay_commit_state.get_cluster_version()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_app_trace_id(), replay_commit_state.get_app_trace_id()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_app_trace_info(), replay_commit_state.get_app_trace_info()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_prev_record_lsn(), replay_commit_state.get_prev_record_lsn()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_redo_lsns().count(), replay_commit_state.get_redo_lsns().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_xid(), replay_commit_state.get_xid()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_commit_parts().count(), replay_commit_state.get_commit_parts().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_epoch(), replay_commit_state.get_epoch()); - replay_member_cnt++; - EXPECT_EQ(replay_member_cnt, fill_member_cnt); - - ObTxPrepareLogTempRef prepare_temp_ref; - ObTxPrepareLog replay_prepare(prepare_temp_ref); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_PREPARE_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_prepare)); - fill_member_cnt = replay_member_cnt = 0; - fill_member_cnt = fill_prepare.compat_bytes_.total_obj_cnt_; - EXPECT_EQ(fill_prepare.get_incremental_participants().count(), - replay_prepare.get_incremental_participants().count()); - replay_member_cnt++; - EXPECT_EQ(fill_prepare.get_prev_lsn(), replay_prepare.get_prev_lsn()); - replay_member_cnt++; - EXPECT_EQ(fill_prepare.get_prev_log_type().prev_log_type_, replay_prepare.get_prev_log_type().prev_log_type_); - replay_member_cnt++; - EXPECT_EQ(replay_member_cnt, fill_member_cnt); - - ObTxCommitLogTempRef commit_temp_ref; - ObTxCommitLog replay_commit(commit_temp_ref); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_COMMIT_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_commit)); - - ObTxClearLogTempRef clear_temp_ref; - ObTxClearLog replay_clear(clear_temp_ref); - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_CLEAR_LOG, tx_log_header.get_tx_log_type()); - ASSERT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_clear)); - - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_ABORT_LOG, tx_log_header.get_tx_log_type()); - - ASSERT_EQ(OB_SUCCESS, replay_block.get_next_log(tx_log_header)); - EXPECT_EQ(ObTxLogType::TX_RECORD_LOG, tx_log_header.get_tx_log_type()); - - ASSERT_EQ(OB_ITER_END, replay_block.get_next_log(tx_log_header)); // ITER_END -} - -TEST_F(TestObTxLog, TestComapt) -{ - OldTestLog old_fill; - NewTestLog new_fill; - old_fill.tx_id_1 = TEST_TX_ID; - old_fill.tx_id_2 = TEST_TX_ID; - new_fill.tx_id_1 = TEST_TX_ID; - new_fill.tx_id_2 = TEST_TX_ID; - new_fill.tx_id_3 = TEST_TX_ID; - - char old_fill_buf[1024]; - char new_fill_buf[1024]; - memset(old_fill_buf, 0, sizeof(char) * 1024); - memset(new_fill_buf, 0, sizeof(char) * 1024); - - int64_t old_pos,new_pos; - int64_t old_size,new_size; - - old_pos = new_pos = 0; - EXPECT_EQ(OB_SUCCESS, old_fill.serialize(old_fill_buf, 1024, old_pos)); - EXPECT_EQ(OB_SUCCESS, new_fill.serialize(new_fill_buf, 1024, new_pos)); - old_size = old_pos; - new_size = new_pos; - TRANS_LOG(INFO, " serialize test fill log", K(old_size), K(new_size)); - - OldTestLog old_replay; - NewTestLog new_replay; - old_pos = new_pos = 0; - EXPECT_EQ(OB_SUCCESS, new_replay.deserialize(old_fill_buf, old_size, old_pos)); - EXPECT_EQ(OB_SUCCESS, old_replay.deserialize(new_fill_buf, new_size, new_pos)); - - EXPECT_EQ(TEST_TX_ID, old_replay.tx_id_1); - EXPECT_EQ(TEST_TX_ID, old_replay.tx_id_2); - - EXPECT_EQ(TEST_TX_ID, new_replay.tx_id_1); - EXPECT_EQ(TEST_TX_ID, new_replay.tx_id_2); - EXPECT_EQ(0, new_replay.tx_id_3); - -} - void test_big_commit_info_log(int64_t log_size) { @@ -732,23 +299,16 @@ void test_big_commit_info_log(int64_t log_size) ObTxBigSegmentBuf fill_big_segment; ObTxBigSegmentBuf replay_big_segment; - ObLSArray TEST_LS_ARRAY; - TEST_LS_ARRAY.push_back(LSKey()); - ObTxCommitParts TEST_COMMIT_PARTS; - TEST_COMMIT_PARTS.push_back(ObTxExecPart(TEST_LS_KEY, 0)); ObRedoLSNArray TEST_BIG_REDO_LSN_ARRAY; for (int i = 0; i < log_size / sizeof(palf::LSN); i++) { TEST_BIG_REDO_LSN_ARRAY.push_back(palf::LSN(i)); } EXPECT_EQ(TEST_BIG_REDO_LSN_ARRAY.count(), log_size / sizeof(palf::LSN)); - ObTxCommitInfoLog fill_commit_state(TEST_ADDR, TEST_LS_ARRAY, TEST_LS_KEY, TEST_IS_SUB2PC, - TEST_IS_DUP, TEST_CAN_ELR, TEST_TRACE_ID_STR, TEST_TRCE_INFO, - TEST_LOG_OFFSET, TEST_BIG_REDO_LSN_ARRAY, TEST_LS_ARRAY, - TEST_CLUSTER_VERSION, TEST_XID, - TEST_COMMIT_PARTS, TEST_EPOCH); + ObTxCommitInfoLog fill_commit_state(TEST_IS_DUP, TEST_CAN_ELR, TEST_TRACE_ID_STR, TEST_TRCE_INFO, + TEST_LOG_OFFSET, TEST_BIG_REDO_LSN_ARRAY, TEST_XID); ObTxLogBlockHeader &fill_block_header = fill_block.get_header(); - fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID), TEST_ADDR); + fill_block_header.init(TEST_ORG_CLUSTER_ID, TEST_CLUSTER_VERSION, TEST_LOG_ENTRY_NO, ObTransID(TEST_TX_ID)); ASSERT_EQ(OB_SUCCESS, fill_block.init_for_fill()); ASSERT_EQ(OB_LOG_TOO_LARGE, fill_block.add_new_log(fill_commit_state, &fill_big_segment)); @@ -810,45 +370,15 @@ void test_big_commit_info_log(int64_t log_size) ObTxCommitInfoLog replay_commit_state(commit_state_temp_ref); EXPECT_EQ(OB_SUCCESS, replay_block.deserialize_log_body(replay_commit_state)); - int64_t fill_member_cnt = 0; - int64_t replay_member_cnt = 0; - fill_member_cnt = replay_member_cnt = 0; - fill_member_cnt = fill_commit_state.compat_bytes_.total_obj_cnt_; - EXPECT_EQ(fill_commit_state.get_scheduler(), replay_commit_state.get_scheduler()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_participants().count(), - replay_commit_state.get_participants().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_upstream(), replay_commit_state.get_upstream()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.is_sub2pc(), replay_commit_state.is_sub2pc()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.is_dup_tx(), replay_commit_state.is_dup_tx()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.is_elr(), replay_commit_state.is_elr()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_incremental_participants().count(), - replay_commit_state.get_incremental_participants().count()); - replay_member_cnt++; - EXPECT_EQ(0, replay_commit_state.get_cluster_version()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.get_app_trace_id().length(), replay_commit_state.get_app_trace_id().length()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.get_app_trace_info().length(), replay_commit_state.get_app_trace_info().length()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.get_prev_record_lsn(), replay_commit_state.get_prev_record_lsn()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.get_redo_lsns().count(), replay_commit_state.get_redo_lsns().count()); - replay_member_cnt++; EXPECT_EQ(fill_commit_state.get_xid(), replay_commit_state.get_xid()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_commit_parts().count(), replay_commit_state.get_commit_parts().count()); - replay_member_cnt++; - EXPECT_EQ(fill_commit_state.get_epoch(), replay_commit_state.get_epoch()); - replay_member_cnt++; - EXPECT_EQ(replay_member_cnt, fill_member_cnt); } TEST_F(TestObTxLog, test_big_segment_log) @@ -868,25 +398,18 @@ TEST_F(TestObTxLog, test_commit_log_with_checksum_signature) checksum = ob_crc64(checksum, &checksum_i, sizeof(checksum_i)); checksum_signatures.push_back((uint8_t)(checksum_i & 0xFF)); } - ObLSArray ls_array; - ls_array.push_back(ObLSID(1001)); ObTxBufferNodeArray tx_buffer_node_array; ObTxBufferNode node; ObString str("hello,world"); - node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ::mk_v0(100), nullptr); + node.init(ObTxDataSourceType::LS_TABLE, str, share::SCN(), transaction::ObTxSEQ(100, 0), nullptr); tx_buffer_node_array.push_back(node); - ObLSLogInfoArray ls_info_array; - ls_info_array.push_back(ObLSLogInfo()); share::SCN scn; scn.convert_for_tx(101010101010101); ObTxCommitLog log0(scn, checksum, checksum_signatures, - ls_array, tx_buffer_node_array, - 1, LogOffSet(100), - ls_info_array, TEST_PREV_LOG_TYPE); int64_t size = log0.get_serialize_size(); char *buf = new char[size]; @@ -904,41 +427,12 @@ TEST_F(TestObTxLog, test_commit_log_with_checksum_signature) } } -TEST_F(TestObTxLog, test_start_working_log) -{ - ObTransID fake_tx_id(0); - ObTxLogBlockHeader header(1, 1, 1, fake_tx_id, ObAddr()); - EXPECT_EQ(0, header.get_serialize_size_()); - EXPECT_EQ(OB_SUCCESS, header.before_serialize()); - int64_t ser_size_ = header.get_serialize_size_(); - int64_t ser_size = header.get_serialize_size(); - EXPECT_NE(0, ser_size_); - char buf[256]; - MEMSET(buf, 0, 256); - int64_t pos = 0; - EXPECT_EQ(OB_SUCCESS, header.serialize(buf, 256, pos)); - EXPECT_EQ(pos, ser_size); - ObTxLogBlockHeader header2; - int64_t pos0 = 0; - EXPECT_EQ(OB_SUCCESS, header2.deserialize(buf, 256, pos0)); - EXPECT_LE(pos0, pos); - EXPECT_EQ(header2.tx_id_, fake_tx_id); - EXPECT_EQ(header2.cluster_version_, header.cluster_version_); - EXPECT_EQ(header2.log_entry_no_, header.log_entry_no_); -} - TEST_F(TestObTxLog, test_tx_block_header_serialize) { - // 1. user must call before_serialize, before get_serialize_size(), serialize() ObTransID tx_id(1024); - ObAddr addr(ObAddr::VER::IPV4, "127.2.3.4", 2048); - ObTxLogBlockHeader header(101, 102, 103, tx_id, addr); - EXPECT_EQ(0, header.serialize_size_); - EXPECT_EQ(OB_SUCCESS, header.before_serialize()); - int64_t ser_size_ = header.get_serialize_size_(); - EXPECT_EQ(ser_size_, header.get_serialize_size_()); - int64_t ser_size = header.get_serialize_size(); - EXPECT_GT(ser_size_, 0); + ObTxLogBlockHeader header(101, 102, 103, tx_id); + const int64_t ser_size = header.get_serialize_size(); + EXPECT_GT(ser_size, 0); char buf[256]; MEMSET(buf, 0, 256); int64_t pos = 0; @@ -948,23 +442,21 @@ TEST_F(TestObTxLog, test_tx_block_header_serialize) // test deserialize ok ObTxLogBlockHeader header2; int64_t pos0 = 0; - EXPECT_EQ(OB_SUCCESS, header2.deserialize(buf, 256, pos0)); - EXPECT_LE(pos0, pos); + EXPECT_EQ(OB_SUCCESS, header2.deserialize(buf, pos, pos0)); + EXPECT_EQ(pos0, pos); EXPECT_EQ(header2.tx_id_, tx_id); EXPECT_EQ(header2.org_cluster_id_, 101); EXPECT_EQ(header2.cluster_version_, 102); EXPECT_EQ(header2.log_entry_no_, 103); - EXPECT_EQ(header2.scheduler_, addr); - // the serilize size is always equals to header.serialize_size_ - header.serialize_size_ = 240; - int64_t ser_size2 = header.get_serialize_size(); - EXPECT_GT(ser_size2, 240); - EXPECT_EQ(240, header.get_serialize_size_()); + // The log entry number uses fixed-width encoding because the header is reserved + // before the final entry number is known. + header.set_log_entry_no(INT64_MAX); + EXPECT_EQ(ser_size, header.get_serialize_size()); MEMSET(buf, 0, 256); pos = 0; EXPECT_EQ(OB_SUCCESS, header.serialize(buf, 256, pos)); - EXPECT_EQ(pos, ser_size2); + EXPECT_EQ(pos, ser_size); } } // namespace unittest diff --git a/unittest/storage/tx/test_ob_tx_msg.cpp b/unittest/storage/tx/test_ob_tx_msg.cpp deleted file mode 100644 index 912d80922..000000000 --- a/unittest/storage/tx/test_ob_tx_msg.cpp +++ /dev/null @@ -1,760 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define protected public -#include "storage/tx/ob_tx_msg.h" -#include - -namespace oceanbase -{ -using namespace common; -using namespace transaction; -namespace unittest -{ - -class TestObTxMsg : public ::testing::Test -{ -public : - virtual void SetUp() {} - virtual void TearDown() {} - - // define port - static const int32_t PORT = 8080; - // ObAddr using ipv4 for test - static const ObAddr::VER IP_TYPE = ObAddr::IPV4; - // ip address for test - static const char *LOCAL_IP; - - static const share::ObLSID VALID_LS_ID; - static const share::ObLSID INVALID_LS_ID; -}; -const char *TestObTxMsg::LOCAL_IP = "127.0.0.1"; -const share::ObLSID TestObTxMsg::VALID_LS_ID = share::ObLSID(1); -const share::ObLSID TestObTxMsg::INVALID_LS_ID = share::ObLSID(-1); - -class MockObTxDesc -{ -public: - MockObTxDesc() - { - cluster_version_ = oceanbase::common::cal_version(4, 0, 0, 0); - tenant_id_ = 1001; - tx_id_ = 1; - receiver_ = TestObTxMsg::VALID_LS_ID; - epoch_ = 1; - self_ = ObAddr(TestObTxMsg::IP_TYPE, TestObTxMsg::LOCAL_IP, TestObTxMsg::PORT); - cluster_id_ = 1; - op_sn_ = 1; - } - ~MockObTxDesc() {} - int build_tx_commit_msg(ObTxCommitMsg &msg) - { - int ret = OB_SUCCESS; - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = epoch_; - msg.request_id_ = op_sn_; - msg.expire_ts_ = INT_MAX64; - if (OB_FAIL(msg.parts_.push_back(TestObTxMsg::VALID_LS_ID))) { - TRANS_LOG(WARN, "push_back parts fail", K(ret)); - } - return ret; - } - void build_tx_commit_resp_msg(ObTxCommitRespMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = epoch_; - msg.request_id_ = op_sn_; - msg.commit_version_ = share::SCN::max_scn(); - msg.ret_ = OB_SUCCESS; - } - void build_tx_abort_msg(ObTxAbortMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = epoch_; - msg.request_id_ = op_sn_; - msg.reason_ = 0; - } - void build_tx_rollback_sp_msg(ObTxRollbackSPMsg &msg, ObTxDesc *tx) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.savepoint_ = ObTxSEQ(1, 0); - msg.op_sn_ = op_sn_; - msg.tx_seq_base_ = 10000000001; - msg.tx_ptr_ = tx; - } - void build_tx_keepalive_msg(ObTxKeepaliveMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.cluster_id_ = cluster_id_; - msg.request_id_ = op_sn_; - msg.tx_id_ = tx_id_; - msg.sender_addr_ = self_; - msg.sender_ = share::ObLSID(1000001); - msg.receiver_ = share::SCHEDULER_LS; - msg.epoch_ = 1000; - msg.status_ = OB_SUCCESS; - } - int build_tx_2pc_prepare_msg(Ob2pcPrepareReqMsg &msg) - { - int ret = OB_SUCCESS; - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.upstream_ = receiver_; - msg.app_trace_info_ = ObString("test"); - return ret; - } - int build_tx_2pc_prepare_resp_msg(Ob2pcPrepareRespMsg &msg) - { - int ret = OB_SUCCESS; - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.prepare_version_ = share::SCN::max_scn(); - if (OB_FAIL(msg.prepare_info_array_.push_back(ObLSLogInfo()))) { - TRANS_LOG(WARN, "push_back info fail", K(ret)); - } - return ret; - } - void build_tx_2pc_pre_commit_msg(Ob2pcPreCommitReqMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.commit_version_ = share::SCN::max_scn(); - } - void build_tx_2pc_pre_commit_resp_msg(Ob2pcPreCommitRespMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.commit_version_ = share::SCN::max_scn(); - } - int build_tx_2pc_commit_msg(Ob2pcCommitReqMsg &msg) - { - int ret = OB_SUCCESS; - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.commit_version_ = share::SCN::max_scn(); - if (OB_FAIL(msg.prepare_info_array_.push_back(ObLSLogInfo()))) { - TRANS_LOG(WARN, "push_back info fail", K(ret)); - } - return ret; - } - void build_tx_2pc_abort_msg(Ob2pcAbortReqMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - } - int build_tx_sub_prepare_msg(ObTxSubPrepareMsg &msg) - { - int ret = OB_SUCCESS; - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.expire_ts_ = INT_MAX64; - msg.request_id_ = op_sn_; - msg.xid_ = ObXATransID(); - msg.xid_.set(ObString("aaa"), ObString("bbb"), 1); - if (OB_FAIL(msg.parts_.push_back(TestObTxMsg::VALID_LS_ID))) { - TRANS_LOG(WARN, "push_back parts fail", K(ret)); - } - return ret; - } - void build_tx_sub_commit_msg(ObTxSubCommitMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.xid_ = ObXATransID(); - msg.xid_.set(ObString("aaa"), ObString("bbb"), 1); - } - void build_tx_sub_rollback_msg(ObTxSubRollbackMsg &msg) - { - msg.cluster_version_ = cluster_version_; - msg.tx_id_ = tx_id_; - msg.receiver_ = receiver_; - msg.sender_addr_ = self_; - msg.sender_ = share::SCHEDULER_LS; - msg.cluster_id_ = cluster_id_; - msg.timestamp_ = op_sn_; - msg.epoch_ = -1; - msg.request_id_ = op_sn_; - msg.xid_ = ObXATransID(); - msg.xid_.set(ObString("aaa"), ObString("bbb"), 1); - } -public: - int64_t cluster_version_; - uint64_t tenant_id_; - ObTransID tx_id_; - share::ObLSID receiver_; - int64_t epoch_; - int64_t cluster_id_; - common::ObAddr self_; - uint64_t op_sn_; -}; - -TEST_F(TestObTxMsg, trans_commit_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxCommitMsg msg; - ASSERT_EQ(OB_SUCCESS, tx.build_tx_commit_msg(msg)); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxCommitMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.expire_ts_, msg1.expire_ts_); - EXPECT_EQ(msg.parts_.count(), msg1.parts_.count()); -} - -TEST_F(TestObTxMsg, trans_commit_resp_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxCommitRespMsg msg; - tx.build_tx_commit_resp_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxCommitRespMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.ret_, msg1.ret_); - EXPECT_EQ(msg.commit_version_, msg1.commit_version_); -} - -TEST_F(TestObTxMsg, trans_abort_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxAbortMsg msg; - tx.build_tx_abort_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxAbortMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.reason_, msg1.reason_); -} - -TEST_F(TestObTxMsg, trans_rollback_sp_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxRollbackSPMsg msg; - ObTxDesc tx1; - ObTxPart p; - p.id_ = TestObTxMsg::VALID_LS_ID; - tx1.parts_.push_back(p); - tx.build_tx_rollback_sp_msg(msg, &tx1); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxRollbackSPMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.savepoint_, msg1.savepoint_); - EXPECT_EQ(msg.op_sn_, msg1.op_sn_); - EXPECT_EQ(msg.tx_seq_base_, msg1.tx_seq_base_); - EXPECT_EQ(msg.tx_ptr_->parts_[0].id_, msg1.tx_ptr_->parts_[0].id_); - if (OB_NOT_NULL(msg.tx_ptr_)) { - msg.tx_ptr_ = NULL; - } -} - -TEST_F(TestObTxMsg, trans_keepalive_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxKeepaliveMsg msg; - tx.build_tx_keepalive_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxKeepaliveMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.status_, msg1.status_); -} - -TEST_F(TestObTxMsg, trans_2pc_prepare_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcPrepareReqMsg msg; - ASSERT_EQ(OB_SUCCESS, tx.build_tx_2pc_prepare_msg(msg)); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - Ob2pcPrepareReqMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.upstream_, msg1.upstream_); - EXPECT_EQ(msg.app_trace_info_, msg1.app_trace_info_); -} - -TEST_F(TestObTxMsg, trans_2pc_prepare_resp_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcPrepareRespMsg msg; - ASSERT_EQ(OB_SUCCESS, tx.build_tx_2pc_prepare_resp_msg(msg)); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - Ob2pcPrepareRespMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.prepare_version_, msg1.prepare_version_); -} - -TEST_F(TestObTxMsg, trans_2pc_pre_commit_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcPreCommitReqMsg msg; - tx.build_tx_2pc_pre_commit_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - // test the deserialization of ObTransMsg - Ob2pcPreCommitReqMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.commit_version_, msg1.commit_version_); -} - -TEST_F(TestObTxMsg, trans_2pc_pre_commit_resp_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcPreCommitRespMsg msg; - tx.build_tx_2pc_pre_commit_resp_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - Ob2pcPreCommitRespMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.commit_version_, msg1.commit_version_); -} - -TEST_F(TestObTxMsg, trans_2pc_commit_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcCommitReqMsg msg; - ASSERT_EQ(OB_SUCCESS, tx.build_tx_2pc_commit_msg(msg)); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - Ob2pcCommitReqMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.commit_version_, msg1.commit_version_); -} - -TEST_F(TestObTxMsg, trans_2pc_abort_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - Ob2pcAbortReqMsg msg; - tx.build_tx_2pc_abort_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - Ob2pcAbortReqMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); -} - -TEST_F(TestObTxMsg, trans_sub_prepare_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxSubPrepareMsg msg; - ASSERT_EQ(OB_SUCCESS, tx.build_tx_sub_prepare_msg(msg)); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxSubPrepareMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.xid_, msg1.xid_); - EXPECT_EQ(msg.parts_.count(), msg1.parts_.count()); -} - -TEST_F(TestObTxMsg, trans_sub_commit_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxSubCommitMsg msg; - tx.build_tx_sub_commit_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxSubCommitMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.xid_, msg1.xid_); -} - -TEST_F(TestObTxMsg, trans_sub_rollback_msg) -{ - TRANS_LOG(INFO, "called", "func", test_info_->name()); - - MockObTxDesc tx; - ObTxSubRollbackMsg msg; - tx.build_tx_sub_rollback_msg(msg); - ASSERT_TRUE(msg.is_valid()); - - // test the serialization of ObTransMsg - int64_t pos = 0; - const int64_t BUFFER_SIZE = 10240; - char buffer[BUFFER_SIZE]; - ASSERT_EQ(OB_SUCCESS, msg.serialize(buffer, BUFFER_SIZE, pos)); - - // test the deserialization of ObTransMsg - ObTxSubRollbackMsg msg1; - int64_t start_index = 0; - ASSERT_EQ(OB_SUCCESS, msg1.deserialize(buffer, pos, start_index)); - EXPECT_EQ(msg.get_msg_type(), msg1.get_msg_type()); - EXPECT_EQ(msg.cluster_version_, msg1.cluster_version_); - EXPECT_EQ(msg.get_trans_id(), msg1.get_trans_id()); - EXPECT_EQ(msg.get_receiver(), msg1.get_receiver()); - EXPECT_EQ(msg.get_epoch(), msg1.get_epoch()); - EXPECT_EQ(msg.get_sender_addr(), msg1.get_sender_addr()); - EXPECT_EQ(msg.get_sender(), msg1.get_sender()); - EXPECT_EQ(msg.get_request_id(), msg1.get_request_id()); - EXPECT_EQ(msg.get_timestamp(), msg1.get_timestamp()); - EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); - EXPECT_EQ(msg.xid_, msg1.xid_); -} - -}//end of unittest -}//end of oceanbase - -using namespace oceanbase; -using namespace oceanbase::common; - -int main(int argc, char **argv) -{ - int ret = 1; - ObLogger &logger = ObLogger::get_logger(); - logger.set_file_name("test_ob_tx_msg.log", true); - logger.set_log_level(OB_LOG_LEVEL_INFO); - testing::InitGoogleTest(&argc, argv); - ret = RUN_ALL_TESTS(); - return ret; -} diff --git a/unittest/storage/tx/test_redo_submitter.cpp b/unittest/storage/tx/test_redo_submitter.cpp index a3b50bd04..7da65db03 100644 --- a/unittest/storage/tx/test_redo_submitter.cpp +++ b/unittest/storage/tx/test_redo_submitter.cpp @@ -73,7 +73,6 @@ class ObTestRedoSubmitter : public ::testing::Test ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); ObAddr ip_port(ObAddr::VER::IPV4, "119.119.0.1",2023); ObCurTraceId::init(ip_port); - GCONF._ob_trans_rpc_timeout = 500; ObClockGenerator::init(); const testing::TestInfo* const test_info = testing::UnitTest::GetInstance()->current_test_info(); @@ -82,11 +81,10 @@ class ObTestRedoSubmitter : public ::testing::Test // prepare for test tx_ctx.exec_info_.state_ = ObTxState::INIT; - tx_ctx.exec_info_.scheduler_ = common::ObAddr(common::ObAddr::VER::IPV4, "127.0.0.1", 8888); tx_ctx.exec_info_.next_log_entry_no_ = 0; tx_ctx.cluster_version_ = DATA_CURRENT_VERSION; - ObLSID ls_id(1001); ObTransID tx_id(777); - EXPECT_EQ(OB_SUCCESS,tx_ctx.init_log_cbs_(ls_id, tx_id)); + ObTransID tx_id(777); + EXPECT_EQ(OB_SUCCESS,tx_ctx.init_log_cbs_(tx_id)); mock_ptr = &mdo_; } virtual void TearDown() override @@ -99,7 +97,7 @@ class ObTestRedoSubmitter : public ::testing::Test ObMallocAllocator::get_instance()->recycle_tenant_allocator(); } MockImpl mdo_; - ObPartTransCtx tx_ctx; + ObTxCtx tx_ctx; ObMemtableCtx mt_ctx; }; @@ -112,18 +110,18 @@ int succ_submit_redo_log_out(ObTxLogBlock & b, { submitted_scn.convert_for_tx(123123123); if (log_cb) { - ((ObPartTransCtx*)(log_cb->get_group_ptr()->get_tx_ctx()))->return_log_cb_(log_cb); + ((ObTxCtx*)(log_cb->get_group_ptr()->get_tx_ctx()))->return_log_cb_(log_cb); log_cb = NULL; } return OB_SUCCESS; } -bool ObPartTransCtx::is_parallel_logging() const +bool ObTxCtx::is_parallel_logging() const { return mock_ptr->is_parallel_logging(); } -int ObPartTransCtx::submit_redo_log_out(ObTxLogBlock &log_block, +int ObTxCtx::submit_redo_log_out(ObTxLogBlock &log_block, ObTxLogCb *&log_cb, memtable::ObRedoLogSubmitHelper &helper, const int64_t replay_hint, @@ -539,7 +537,7 @@ TEST_F(ObTestRedoSubmitter, submit_by_switch_leader_or_on_commit_serial_logging) ObTxRedoSubmitter submitter(tx_ctx, mt_ctx); ObTxLogBlock log_block; ObTransID tx_id(101); - log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id, ObAddr()); + log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id); log_block.init_for_fill(); memtable::ObRedoLogSubmitHelper helper; EXPECT_EQ(OB_BLOCK_FROZEN, submitter.fill(log_block, helper)); @@ -584,7 +582,7 @@ TEST_F(ObTestRedoSubmitter, submit_by_switch_leader_or_on_commit_parallel_loggin ObTxRedoSubmitter submitter(tx_ctx, mt_ctx); ObTxLogBlock log_block; ObTransID tx_id(101); - log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id, ObAddr()); + log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id); log_block.init_for_fill(); memtable::ObRedoLogSubmitHelper helper; EXPECT_EQ(OB_SUCCESS, submitter.fill(log_block, helper)); @@ -627,7 +625,7 @@ TEST_F(ObTestRedoSubmitter, submit_by_switch_leader_or_on_commit_parallel_loggin ObTxRedoSubmitter submitter(tx_ctx, mt_ctx); ObTxLogBlock log_block; ObTransID tx_id(101); - log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id, ObAddr()); + log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id); log_block.init_for_fill(); memtable::ObRedoLogSubmitHelper helper; EXPECT_EQ(OB_BLOCK_FROZEN, submitter.fill(log_block, helper)); @@ -663,7 +661,7 @@ TEST_F(ObTestRedoSubmitter, submit_ROW_SIZE_TOO_LARGE) ObTxRedoSubmitter submitter(tx_ctx, mt_ctx); ObTxLogBlock log_block; ObTransID tx_id(101); - log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id, ObAddr()); + log_block.get_header().init(1, DATA_CURRENT_VERSION, 101, tx_id); log_block.init_for_fill(); memtable::ObRedoLogSubmitHelper helper; EXPECT_EQ(OB_ERR_TOO_BIG_ROWSIZE, submitter.fill(log_block, helper)); diff --git a/unittest/storage/tx/test_trans_callback_mgr_fill_redo.cpp b/unittest/storage/tx/test_trans_callback_mgr_fill_redo.cpp index 369db5a86..8d3a66abc 100644 --- a/unittest/storage/tx/test_trans_callback_mgr_fill_redo.cpp +++ b/unittest/storage/tx/test_trans_callback_mgr_fill_redo.cpp @@ -104,7 +104,6 @@ class ObTestRedoFill : public ::testing::Test ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(); ObAddr ip_port(ObAddr::VER::IPV4, "119.119.0.1",2023); ObCurTraceId::init(ip_port); - GCONF._ob_trans_rpc_timeout = 500; ObClockGenerator::init(); const testing::TestInfo* const test_info = testing::UnitTest::GetInstance()->current_test_info(); @@ -755,4 +754,3 @@ int main(int argc, char **argv) return RUN_ALL_TESTS(); } - diff --git a/unittest/storage/tx_storage/test_safe_destroy_handler.cpp b/unittest/storage/tx_storage/test_safe_destroy_handler.cpp deleted file mode 100644 index c396b106e..000000000 --- a/unittest/storage/tx_storage/test_safe_destroy_handler.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2025 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define USING_LOG_PREFIX STORAGETEST - -#define protected public -#define private public - -#include "storage/tx_storage/ob_safe_destroy_handler.h" - -namespace oceanbase -{ -using namespace common; -namespace storage -{ -class TestSafeDestroyHandler : public ::testing::Test -{ -public: - TestSafeDestroyHandler() = default; - virtual ~TestSafeDestroyHandler() = default; -}; - - -TEST_F(TestSafeDestroyHandler, destroy_before_init) -{ - int ret = OB_SUCCESS; - ObSafeDestroyHandler handler; - ret = handler.stop(); - ASSERT_EQ(OB_NOT_INIT, ret); - handler.wait(); - handler.destroy(); -} - - -} // namespace storage -} // namespace oceanbase - -int main(int argc, char **argv) -{ - system("rm -f ./test_safe_destroy_handler.log*"); - - OB_LOGGER.set_file_name("test_safe_destroy_handler.log", true); - OB_LOGGER.set_log_level("INFO"); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/unittest/storage/tx_table/test_tx_ctx_table.cpp b/unittest/storage/tx_table/test_tx_ctx_table.cpp index bf7274116..55920aa81 100644 --- a/unittest/storage/tx_table/test_tx_ctx_table.cpp +++ b/unittest/storage/tx_table/test_tx_ctx_table.cpp @@ -20,9 +20,12 @@ #define private public #define UNITTEST -#include "storage/tx/ob_trans_part_ctx.h" -#include "storage/mock_ob_log_handler.h" -#include "logservice/ob_log_handler.h" +#include "storage/ls/ob_freezer.h" +#include "storage/ls/ob_ls.h" +#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" +#include "storage/tx/ob_tx_ctx.h" +#include "storage/tx_table/ob_tx_ctx_memtable_mgr.h" +#include "storage/tx_table/ob_tx_ctx_table.h" #include "share/rc/ob_module_provider.h" namespace oceanbase @@ -97,11 +100,7 @@ class TestTxCtxTable : public ::testing::Test public: TestTxCtxTable(): ls_(), - ls_tx_service_(&ls_), - ls_data_checkpoint_(), - log_handler_(), tablet_id_(LS_TX_DATA_TABLET), - ls_id_(1), tenant_id_(1), freezer_(&ls_), t3m_(), @@ -117,11 +116,6 @@ class TestTxCtxTable : public ::testing::Test static ObLSTxCtxMgr ls_tx_ctx_mgr_; static ObLSTxCtxMgr ls_tx_ctx_mgr2_; ObLS ls_; - ObLSWRSHandler ls_loop_worker_; - ObLSTxService ls_tx_service_; - ObLSTabletService ls_tablet_service_; - checkpoint::ObDataCheckpoint ls_data_checkpoint_; - MockObLogHandler log_handler_; static int64_t ref_count_; protected: virtual void SetUp() override @@ -132,7 +126,6 @@ class TestTxCtxTable : public ::testing::Test EXPECT_EQ(OB_SUCCESS, t3m_.init()); EXPECT_EQ(OB_SUCCESS, ls_tx_ctx_mgr_.init( - ls_id_, &ls_.tx_table_, ls_.get_lock_table(), (ObTsMgr *)(0x01), @@ -141,7 +134,6 @@ class TestTxCtxTable : public ::testing::Test nullptr)); EXPECT_EQ(OB_SUCCESS, ls_tx_ctx_mgr2_.init( - ls_id_, &ls_.tx_table_, ls_.get_lock_table(), (ObTsMgr *)(0x01), @@ -151,7 +143,6 @@ class TestTxCtxTable : public ::testing::Test ref_count_ = 0; ctx_mt_mgr_ = new ObTxCtxMemtableMgr(); EXPECT_EQ(OB_SUCCESS, ctx_mt_mgr_->init(tablet_id_, - ls_id_, &freezer_, &t3m_)); mt_mgr_ = ctx_mt_mgr_; @@ -185,7 +176,6 @@ class TestTxCtxTable : public ::testing::Test } public: ObTabletID tablet_id_; - ObLSID ls_id_; int64_t tenant_id_; ObFreezer freezer_; ObTenantMetaMemMgr t3m_; @@ -270,11 +260,9 @@ TEST_F(TestTxCtxTable, test_tx_ctx_memtable_mgr) row = NULL; ObTransID id1(1); - ObLSID ls_id(1); - static ObPartTransCtx ctx1; + static ObTxCtx ctx1; ctx1.trans_id_ = id1; ctx1.is_inited_ = true; - ctx1.ls_id_ = ls_id; ctx1.exec_info_.max_applying_log_ts_.convert_from_ts(1); ctx1.replay_completeness_.set(true); ctx1.rec_log_ts_.convert_from_ts(996); @@ -283,10 +271,9 @@ TEST_F(TestTxCtxTable, test_tx_ctx_memtable_mgr) ctx1.ctx_tx_data_.test_init(data1, &ls_tx_ctx_mgr_); ObTransID id2(2); - static ObPartTransCtx ctx2; + static ObTxCtx ctx2; ctx2.trans_id_ = id2; ctx2.is_inited_ = true; - ctx2.ls_id_ = ls_id; ctx2.exec_info_.max_applying_log_ts_.convert_from_ts(2); ctx2.replay_completeness_.set(true); ctx2.rec_log_ts_.convert_from_ts(996); @@ -322,7 +309,6 @@ TEST_F(TestTxCtxTable, test_tx_ctx_memtable_mgr) ls_tx_ctx_mgr_recover->reset(); EXPECT_EQ(OB_SUCCESS, ls_tx_ctx_mgr_recover->init( - TestTxCtxTable::ls_id_, &ls_.tx_table_, &ls_.lock_table_, (ObTsMgr *)(0x01), @@ -375,13 +361,13 @@ TEST_F(TestTxCtxTable, test_tx_ctx_memtable_mgr) namespace storage { -int ObTxCtxTable::acquire_ref_(const ObLSID& ls_id) +int ObTxCtxTable::acquire_ref_() { int ret = OB_SUCCESS; ls_tx_ctx_mgr_ = &unittest::TestTxCtxTable::ls_tx_ctx_mgr_; unittest::TestTxCtxTable::ref_count_++; - TRANS_LOG(INFO, "[TX_CTX_TABLE] tx ctx table acquire ref", K(ls_id), K(unittest::TestTxCtxTable::ref_count_), K(this)); + TRANS_LOG(INFO, "[TX_CTX_TABLE] tx ctx table acquire ref", K(unittest::TestTxCtxTable::ref_count_), K(this)); return ret; } @@ -406,8 +392,7 @@ void ObTxData::dec_ref() namespace transaction { -int ObLSTxCtxMgr::init(const ObLSID &ls_id, - ObTxTable *tx_table, +int ObLSTxCtxMgr::init(ObTxTable *tx_table, ObLockTable *lock_table, ObTsMgr *ts_mgr, ObTransService *txs, @@ -421,41 +406,19 @@ int ObLSTxCtxMgr::init(const ObLSID &ls_id, TRANS_LOG(WARN, "ObLSTxCtxMgr inited twice"); ret = OB_INIT_TWICE; } else { - //if (ObTransCtxType::PARTICIPANT == ctx_type) { - // bool is_dup_table = false; - // // FIXME. xiaoshi.xjl - // //} else if (OB_FAIL(txs->check_duplicated_partition(ls_id_, is_dup_table))) { - // // TRANS_LOG(WARN, "check duplicated partition serving error", KR(ret), K(ls_id_)); - // if (is_dup_table && OB_FAIL(init_dup_table_mgr())) { - // TRANS_LOG(WARN, "failed to init dup table", K(ret), K(ls_id)); - // } else { - // // do nothing - // } - //} - //if (OB_SUCC(ret) && (ObTransCtxType::PARTICIPANT == ctx_type)) { - // if (OB_ISNULL(core_local_partition_audit_info_ = ObCoreLocalPartitionAuditInfoFactory::alloc())) { - // ret = OB_ALLOCATE_MEMORY_FAILED; - // TRANS_LOG(WARN, "alloc partition audit info error", KR(ret), K(ls_id)); - // } else if (OB_FAIL(core_local_partition_audit_info_->init(OB_PARTITION_AUDIT_LOCAL_STORAGE_COUNT))) { - // TRANS_LOG(WARN, "ls_id audit info init error", KR(ret), K(ls_id)); - // } else { - // // do nothing - // } - //} if (OB_FAIL(ls_tx_ctx_map_.init(lib::ObMemAttr("LSTxCtxMgr")))) { TRANS_LOG(WARN, "ls_tx_ctx_map_ init fail", KR(ret)); - } else if (OB_FAIL(tx_ls_state_mgr_.init(ls_id))) { - TRANS_LOG(WARN, "init tx_ls_state_mgr_ failed", KR(ret)); - } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::START))) { - TRANS_LOG(WARN, "start ls_tx_ctx_mgr failed",K(ret),K(tx_ls_state_mgr_)); } else { is_inited_ = true; - ls_id_ = ls_id; + stopped_ = false; + block_tx_ = false; + block_normal_tx_ = false; + block_all_ = false; tx_table_ = tx_table; lock_table_ = lock_table, txs_ = txs; ts_mgr_ = ts_mgr; - TRANS_LOG(INFO, "ObLSTxCtxMgr inited success", KP(this), K(ls_id)); + TRANS_LOG(INFO, "ObLSTxCtxMgr inited success", KP(this)); } }